Implement daily backups

4 files changed, 61 insertions, 2 deletions

diff --git a/playbook.yml b/playbook.yml
index 5608ef7..d8de148 100644
--- a/playbook.yml
+++ b/playbook.yml
@@ -24,6 +24,7 @@
     backup_ssh_user: "{{ lookup('env', 'BACKUP_SSH_USER') }}"
     backup_ssh_privkey_file: "{{ lookup('env', 'BACKUP_SSH_PRIVKEY_FILE') }}"
     backup_ssh_pubkey_file: "{{ lookup('env', 'BACKUP_SSH_PUBKEY_FILE') }}"
+    backup_zfs_dataset: zroot
     lan_ipv4_cidr: 192.168.0.10/16
     lan_ipv4_gateway: 192.168.0.1
     lan_search_domain: local.jan.systems
@@ -32,7 +33,7 @@
     jails:
       - { name: ingress, num: 1 }
       - { name: postgres, num: 2 }
-      - { name: backup, num: 3 }
+      - { name: irc_thelounge, num: 3 }
       - { name: taulubot, num: 4 }
       - { name: veetibot, num: 5 }
       - { name: hommabot, num: 6 }
@@ -43,7 +44,6 @@
       - { name: spliit, num: 11 }
       - { name: stirling-pdf, num: 12 }
       - { name: ente, num: 13 }
-      - { name: irc_thelounge, num: 14 }
     static_sites:
       - { site: "homepage", host: "jan.systems", tls: false }
       - { site: "homepage", host: "pursotin.jan.systems", tls: true }
diff --git a/tasks/general.yml b/tasks/general.yml
index ec748bf..df24be2 100644
--- a/tasks/general.yml
+++ b/tasks/general.yml
@@ -90,3 +90,11 @@
     owner: root
     group: wheel
     mode: "0644"
+
+- name: Copy backup script
+  template:
+    src: usr_local_bin_backup.sh.j2
+    dest: /usr/local/bin/backup
+    owner: root
+    group: wheel
+    mode: "0755"
diff --git a/templates/etc_crontab.j2 b/templates/etc_crontab.j2
index a6c48e4..6632294 100644
--- a/templates/etc_crontab.j2
+++ b/templates/etc_crontab.j2
@@ -23,3 +23,6 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
 
 # Update dynamic DNS
 */1     *       *       *       *       root    logto -e /var/log/do_dyndns.err /usr/local/bin/do_dyndns.sh
+
+# Take backups every day at 3:05 AM
+5       3       *       *       *       root    logto /var/log/backup /usr/local/bin/backup
diff --git a/templates/usr_local_bin_backup.sh.j2 b/templates/usr_local_bin_backup.sh.j2
new file mode 100644
index 0000000..1286b93
--- /dev/null
+++ b/templates/usr_local_bin_backup.sh.j2
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+KEYFILE="/root/.ssh/backup"
+DATASET="{{ backup_zfs_dataset }}"
+USER="{{ backup_ssh_user }}"
+HOST="{{ backup_ssh_host }}"
+KEEP=10
+
+# END OF CONFIG
+
+HOST_DIR=backup
+DATE="$(date +%Y-%m-%d-%H-%M-%S%z | tr '+' '-')"
+SNAPSHOT="${DATASET}@${DATE}"
+TARGET="${HOST}/${HOST_DIR}/${SNAPSHOT}.enc"
+
+echo "Taking snapshot \"$SNAPSHOT\""
+(set -x; zfs snapshot "$SNAPSHOT")
+
+echo "Snapshot created"
+echo "Encrypting snapshot and backing up to target \"${TARGET}\""
+
+(set -x; zfs send "$SNAPSHOT" | age -e -i "$KEYFILE" | curl --key "$KEYFILE" -u ${USER}: -T - "sftp://${TARGET}")
+
+echo "Getting listing of existing backups"
+# Note: sorting only produces a time-ordered list inside one timezone
+# This is fine because we're only taking a snapshot every day, so the timezone issue is not relevant
+BACKUPS=$(set -x; echo "ls ${HOST_DIR}" | sftp -i "${KEYFILE}" "${USER}@${HOST}" 2>/dev/null | tail -n +2 | sort)
+COUNT=$(set -x; echo $BACKUPS | wc -w | awk '{print $1}')
+
+if [ ! "$COUNT" -gt "$KEEP" ]; then
+    echo "Acceptable number of backups on remote, no need to prune"
+    echo "Done"
+    exit 0
+fi
+
+echo "Pruning old backups, keeping latest ${KEEP}"
+for BACKUP in $BACKUPS; do
+    if [ ! "$COUNT" -gt "$KEEP" ]; then
+        echo "Acceptable number of backups on remote, no need to prune"
+        break
+    fi
+
+    (set -x; echo "rm ${BACKUP}" | sftp -i "${KEYFILE}" "${USER}@${HOST}")
+    COUNT=$((COUNT - 1))
+done
+
+echo "Done"