From ffe2c8d0accb5c33c98408dfb1039d7387ddffd1 Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Tue, 30 Sep 2025 15:04:14 +0300 Subject: Improve backup --- templates/etc_crontab.j2 | 6 +- templates/usr_local_bin_backup.sh.j2 | 283 +++++++++++++++++++++++++++++------ 2 files changed, 245 insertions(+), 44 deletions(-) (limited to 'templates') diff --git a/templates/etc_crontab.j2 b/templates/etc_crontab.j2 index 6632294..62b1b04 100644 --- a/templates/etc_crontab.j2 +++ b/templates/etc_crontab.j2 @@ -24,5 +24,7 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin # Update dynamic DNS */1 * * * * root logto -e /var/log/do_dyndns.err /usr/local/bin/do_dyndns.sh -# Take backups every day at 3:05 AM -5 3 * * * root logto /var/log/backup /usr/local/bin/backup +# Take snapshots every day at 3:05 AM +5 3 * * * root logto /var/log/backup /usr/local/bin/backup snapshot prune-local +# Send backup snapshots to remote every third day at 4:05 AM +5 4 */3 * * root logto /var/log/backup /usr/local/bin/backup send-to-remote prune-remote notify diff --git a/templates/usr_local_bin_backup.sh.j2 b/templates/usr_local_bin_backup.sh.j2 index 1286b93..1fd099d 100644 --- a/templates/usr_local_bin_backup.sh.j2 +++ b/templates/usr_local_bin_backup.sh.j2 @@ -1,48 +1,247 @@ -#!/usr/bin/env bash +{% raw %}#!/usr/bin/env bash set -euo pipefail +# ===== CONFIG (filled by Ansible) ===== +KEEP_LOCAL=30 +KEEP_REMOTE=10 KEYFILE="/root/.ssh/backup" -DATASET="{{ backup_zfs_dataset }}" -USER="{{ backup_ssh_user }}" -HOST="{{ backup_ssh_host }}" -KEEP=10 - -# END OF CONFIG - -HOST_DIR=backup -DATE="$(date +%Y-%m-%d-%H-%M-%S%z | tr '+' '-')" -SNAPSHOT="${DATASET}@${DATE}" -TARGET="${HOST}/${HOST_DIR}/${SNAPSHOT}.enc" - -echo "Taking snapshot \"$SNAPSHOT\"" -(set -x; zfs snapshot "$SNAPSHOT") - -echo "Snapshot created" -echo "Encrypting snapshot and backing up to target \"${TARGET}\"" - -(set -x; zfs send "$SNAPSHOT" | age -e -i "$KEYFILE" | curl --key "$KEYFILE" -u ${USER}: -T - "sftp://${TARGET}") - -echo "Getting listing of existing backups" -# Note: sorting only produces a time-ordered list inside one timezone -# This is fine because we're only taking a snapshot every day, so the timezone issue is not relevant -BACKUPS=$(set -x; echo "ls ${HOST_DIR}" | sftp -i "${KEYFILE}" "${USER}@${HOST}" 2>/dev/null | tail -n +2 | sort) -COUNT=$(set -x; echo $BACKUPS | wc -w | awk '{print $1}') - -if [ ! "$COUNT" -gt "$KEEP" ]; then - echo "Acceptable number of backups on remote, no need to prune" - echo "Done" - exit 0 -fi - -echo "Pruning old backups, keeping latest ${KEEP}" -for BACKUP in $BACKUPS; do - if [ ! "$COUNT" -gt "$KEEP" ]; then - echo "Acceptable number of backups on remote, no need to prune" - break +HOST_DIR="backup" +DATASET="{% endraw %}{{ backup_zfs_dataset }}{% raw %}" +USER="{% endraw %}{{ backup_ssh_user }}{% raw %}" +HOST="{% endraw %}{{ backup_ssh_host }}{% raw %}" +EMAIL_TO=root +# ===================================== + +# ----- Helpers ----- +die() { echo "Error: $*" >&2; exit 1; } + +require_cmds() { + local cmds=("$@") + for c in "${cmds[@]}"; do command -v "$c" >/dev/null 2>&1 || die "Missing command: $c"; done +} + +timestamp() { + # Replace '+' with '-' so timezone is filename-safe and lexicographically sortable within TZ. + date +%Y-%m-%d-%H-%M-%S%z | tr '+' '-' +} + +latest_snapshot_for_dataset() { + # Latest snapshot on the TOP dataset only (newest first). Returns e.g. zroot@2025-09-30-... + zfs list -t snapshot -o name -S creation "$DATASET" 2>/dev/null \ + | awk 'NR==2{print; exit}' +} + +list_top_snapshots_newest_first() { + # Only list snapshots on the top dataset (not children), newest first. + zfs list -t snapshot -o name -S creation "$DATASET" 2>/dev/null \ + | awk 'NR>1{print $1}' +} + +list_remote_backups_sorted() { + # Note: sorting strictly orders only within the same timezone in the name, which is fine for our daily cadence. + # We store files under ${HOST_DIR}/@.enc + echo "ls ${HOST_DIR}" \ + | sftp -q -i "${KEYFILE}" "${USER}@${HOST}" 2>/dev/null \ + | tail -n +2 | sort +} + +# Upload a local file to the storage box path "${HOST_DIR}/" +sftp_put() { + local local_file="$1" + local remote_name="$2" # just the filename, no directory + sftp -i "${KEYFILE}" "${USER}@${HOST}" </dev/null 2>&1; then + echo "Remote directory already exists: ${HOST_DIR}" + return 0 + fi + + echo "Creating remote directory: ${HOST_DIR}" + set +e + sftp -i "${KEYFILE}" "${USER}@${HOST}" </dev/null 2>&1; then + echo "Remote directory created: ${HOST_DIR}" + else + die "Remote directory ${HOST_DIR} not found after creation" + fi +} + +cmd_snapshot() { + echo "" + echo "[snapshot]" + require_cmds zfs date tr + local ts snap + ts="$(timestamp)" + snap="${DATASET}@${ts}" + echo "Taking recursive snapshot \"${snap}\"" + (set -x; zfs snapshot -r "${snap}") + echo "Recursive snapshot \"${snap}\" created" +} + +cmd_send_to_remote() { + echo "" + echo "[send-to-remote]" + require_cmds zfs age sftp mktemp stat + + local snap base target tmp size_bytes + snap="$(latest_snapshot_for_dataset)" + [[ -n "${snap}" ]] || die "No snapshot found to send. Run 'snapshot' first or ensure the dataset has snapshots." + + base="$(basename "${snap}")" # e.g., zroot@2025-09-30-12-00-00-0300 + target="${base}.enc" + tmp="$(mktemp -t backup_send.XXXXXX)" + trap 'rm -f "${tmp}"' EXIT + + echo "Creating encrypted replication stream to temp file: ${tmp}" + # -R: recursive hierarchy, -v: progress to stderr, -c: send compressed (keeps on-disk compression) + (set -x; zfs send -Rvc "${snap}" | age -e -i "${KEYFILE}" > "${tmp}") + + size_bytes="$(stat -f %z "${tmp}" 2>/dev/null || stat -c %s "${tmp}" 2>/dev/null || echo "unknown")" + echo "Local stream size: ${size_bytes} bytes" + + echo "Uploading via SFTP to ${HOST}:${HOST_DIR}/${target}" + if sftp_put "${tmp}" "${target}"; then + echo "Upload complete" + else + die "SFTP upload failed (does the remote directory '${HOST_DIR}' exist? Run 'init-remote')" + fi + + rm -f "${tmp}" + trap - EXIT +} + +cmd_prune_remote() { + echo "" + echo "[prune-remote]" + require_cmds sftp awk sort wc + + echo "Fetching remote backup listing from sftp://${HOST}/${HOST_DIR}" + BACKUPS="$(list_remote_backups_sorted || true)" + mapfile -t BACKUP_ARR < <(printf "%s\n" "${BACKUPS}") + local count="${#BACKUP_ARR[@]}" + + if [[ "${count}" -le "${KEEP_REMOTE}" ]]; then + echo "Remote backups (${count}) <= KEEP_REMOTE (${KEEP_REMOTE}); nothing to prune." + return 0 + fi + + echo "Pruning remote backups, keeping latest ${KEEP_REMOTE} (will delete $(("${count}" - "${KEEP_REMOTE}")))" + local to_delete_count=$((count - KEEP_REMOTE)) + local deleted=0 + for ((i=0; i