Rework

9 files changed, 588 insertions, 0 deletions

diff --git a/roles/host_prod/handlers/main.yml b/roles/host_prod/handlers/main.yml
new file mode 100644
index 0000000..76ec23e
--- /dev/null
+++ b/roles/host_prod/handlers/main.yml
@@ -0,0 +1,13 @@
+- name: Apply rc.conf
+  shell: service kld start; service netif restart && service routing restart
+
+- name: Mount filesystems
+  shell: mount -a
+
+- name: Restart smartd
+  service:
+    name: smartd
+    state: restarted
+
+- name: Run newaliases
+  shell: newaliases
diff --git a/roles/host_prod/tasks/main.yml b/roles/host_prod/tasks/main.yml
new file mode 100644
index 0000000..ec635ce
--- /dev/null
+++ b/roles/host_prod/tasks/main.yml
@@ -0,0 +1,153 @@
+- name: Set up /etc/rc.conf
+  template:
+    src: "{{ playbook_dir }}/roles/host/templates/etc_rc.conf.j2"
+    dest: /etc/rc.conf
+    owner: root
+    group: wheel
+    mode: "0644"
+  vars:
+    is_prod: true
+  notify: Apply rc.conf
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- name: Install hardware packages
+  package:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - vm-bhyve
+    - drm-kmod
+
+- name: Set up fstab
+  template:
+    src: etc_fstab.j2
+    dest: /etc/fstab
+    owner: root
+    group: wheel
+    mode: "0644"
+  notify: Mount filesystems
+
+- name: Set up bhyve
+  block:
+    - name: Create vm dataset
+      community.general.zfs:
+        name: zroot/vm
+        state: present
+
+    - name: Check if vm-bhyve is initialized
+      stat:
+        path: /zroot/vm/.config
+      register: vm_init_check
+
+    - name: Run vm init
+      shell: vm init
+      when: not vm_init_check.stat.exists
+
+- name: Copy backup SSH private key
+  copy:
+    src: "{{ backup_ssh_privkey_file }}"
+    dest: /root/.ssh/backup
+    owner: root
+    group: wheel
+    mode: "0600"
+
+- name: Copy backup SSH public key
+  copy:
+    src: "{{ backup_ssh_pubkey_file }}"
+    dest: /root/.ssh/backup.pub
+    owner: root
+    group: wheel
+    mode: "0644"
+
+- name: Copy SSH config
+  template:
+    src: root_ssh_config.j2
+    dest: /root/.ssh/config
+    owner: root
+    group: wheel
+    mode: "0644"
+
+- name: Copy backup script
+  template:
+    src: usr_local_bin_backup.sh.j2
+    dest: /usr/local/bin/backup
+    owner: root
+    group: wheel
+    mode: "0755"
+
+- name: Create dma config directory
+  file:
+    path: /etc/dma
+    state: directory
+    owner: root
+    group: wheel
+    mode: "0755"
+
+- name: Configure dma.conf
+  template:
+    src: etc_dma_dma.conf.j2
+    dest: /etc/dma/dma.conf
+    owner: root
+    group: wheel
+    mode: "0644"
+
+- name: Configure dma auth.conf
+  template:
+    src: etc_dma_auth.conf.j2
+    dest: /etc/dma/auth.conf
+    owner: root
+    group: mail
+    mode: "0640"
+
+- name: Configure mail aliases
+  template:
+    src: etc_aliases.j2
+    dest: /etc/aliases
+    owner: root
+    group: wheel
+    mode: "0644"
+  notify: Run newaliases
+
+- name: Install smartmontools
+  package:
+    name: smartmontools
+    state: present
+
+- name: Deploy smartd.conf
+  template:
+    src: "{{ playbook_dir }}/roles/host/templates/usr_local_etc_smartd.conf.j2"
+    dest: /usr/local/etc/smartd.conf
+    owner: root
+    group: wheel
+    mode: "0644"
+  notify: Restart smartd
+
+- name: Start smartd
+  service:
+    name: smartd
+    state: started
+
+- name: Deploy zpool_gpt_labels rc.d script
+  template:
+    src: "{{ playbook_dir }}/roles/host/templates/usr_local_etc_rc.d_zpool_gpt_labels.j2"
+    dest: /usr/local/etc/rc.d/zpool_gpt_labels
+    owner: root
+    group: wheel
+    mode: "0755"
+  register: zpool_gpt_labels_script
+
+- name: Run zpool_gpt_labels
+  service:
+    name: zpool_gpt_labels
+    state: started
+  when: zpool_gpt_labels_script is changed
+
+- name: Deploy safepf script
+  copy:
+    src: "{{ playbook_dir }}/roles/host_prod/templates/usr_local_bin_safepf.sh"
+    dest: /usr/local/bin/safepf
+    owner: root
+    group: wheel
+    mode: "0755"
diff --git a/roles/host_prod/templates/etc_aliases.j2 b/roles/host_prod/templates/etc_aliases.j2
new file mode 100644
index 0000000..4fe3e7f
--- /dev/null
+++ b/roles/host_prod/templates/etc_aliases.j2
@@ -0,0 +1 @@
+*: {{ dma_to_address }}
diff --git a/roles/host_prod/templates/etc_dma_auth.conf.j2 b/roles/host_prod/templates/etc_dma_auth.conf.j2
new file mode 100644
index 0000000..a1000ea
--- /dev/null
+++ b/roles/host_prod/templates/etc_dma_auth.conf.j2
@@ -0,0 +1 @@
+{{ smtp_user }}|{{ smtp_host }}:{{ smtp_password }}
diff --git a/roles/host_prod/templates/etc_dma_dma.conf.j2 b/roles/host_prod/templates/etc_dma_dma.conf.j2
new file mode 100644
index 0000000..dfaa04f
--- /dev/null
+++ b/roles/host_prod/templates/etc_dma_dma.conf.j2
@@ -0,0 +1,8 @@
+SMARTHOST {{ smtp_host }}
+PORT {{ smtp_port }}
+AUTHPATH /etc/dma/auth.conf
+SECURETRANSFER
+STARTTLS
+MAILNAME {{ dma_mail_hostname }}
+MASQUERADE {{ ansible_hostname }}@{{ dma_mail_hostname }}
+#NULLCLIENT
diff --git a/roles/host_prod/templates/etc_fstab.j2 b/roles/host_prod/templates/etc_fstab.j2
new file mode 100644
index 0000000..95d3fa0
--- /dev/null
+++ b/roles/host_prod/templates/etc_fstab.j2
@@ -0,0 +1,7 @@
+# Device                Mountpoint      FStype  Options                 Dump    Pass#
+/dev/gpt/efiboot0       /boot/efi       msdosfs rw                      2       2
+/dev/nda0p3.eli         none            swap    sw                      0       0
+/dev/nda1p3.eli         none            swap    sw                      0       0
+/dev/nda2p3.eli         none            swap    sw                      0       0
+/dev/nda3p3.eli         none            swap    sw                      0       0
+tmpfs                   /tmp            tmpfs   rw,mode=777,size=2g     0       0
diff --git a/roles/host_prod/templates/root_ssh_config.j2 b/roles/host_prod/templates/root_ssh_config.j2
new file mode 100644
index 0000000..96f78f2
--- /dev/null
+++ b/roles/host_prod/templates/root_ssh_config.j2
@@ -0,0 +1,4 @@
+Host backup
+    HostName {{ backup_ssh_host }}
+    User {{ backup_ssh_user }}
+    IdentityFile /root/.ssh/backup
diff --git a/roles/host_prod/templates/usr_local_bin_backup.sh.j2 b/roles/host_prod/templates/usr_local_bin_backup.sh.j2
new file mode 100644
index 0000000..f6a0e2c
--- /dev/null
+++ b/roles/host_prod/templates/usr_local_bin_backup.sh.j2
@@ -0,0 +1,365 @@
+{% raw %}#!/usr/bin/env bash
+set -euo pipefail
+
+# ===== CONFIG (filled by Ansible) =====
+KEEP_LOCAL=30
+KEEP_REMOTE=10
+KEYFILE="/root/.ssh/backup"
+HOST_DIR="backup"
+DATASET="{% endraw %}{{ backup_zfs_dataset }}{% raw %}"
+BACKUP_EXCLUDE_PROP="{% endraw %}{{ backup_zfs_exclude_property | default('com.pursotin:backup') }}{% raw %}"
+USER="{% endraw %}{{ backup_ssh_user }}{% raw %}"
+HOST="{% endraw %}{{ backup_ssh_host }}{% raw %}"
+EMAIL_TO=root
+# =====================================
+
+# ----- Globals for notification -----
+STARTED_AT="$(date '+%Y-%m-%dT%H:%M:%S%z')"
+FINISHED_AT=""
+MESSAGE_LOG=""
+BACKUP_NAME=""          # e.g. zroot@2025-09-30-12-00-00-0300.enc
+BACKUP_SIZE_BYTES=""    # numeric bytes
+
+# ----- Helpers -----
+die() { echo "Error: $*" >&2; exit 1; }
+
+require_cmds() {
+  local cmds=("$@")
+  for c in "${cmds[@]}"; do command -v "$c" >/dev/null 2>&1 || die "Missing command: $c"; done
+}
+
+log_note() {
+  # Echo to console and append to message buffer
+  local msg="$1"
+  echo "$msg"
+  MESSAGE_LOG+="$msg"$'\n'
+}
+
+timestamp() {
+  # Replace '+' with '-' so timezone is filename-safe and lexicographically sortable within TZ.
+  date +%Y-%m-%d-%H-%M-%S%z | tr '+' '-'
+}
+
+humanize_bytes() {
+  local bytes="$1"
+  if [[ "${bytes}" =~ ^[0-9]+$ ]]; then
+    awk -v b="${bytes}" '
+      BEGIN {
+        unit_count = split("B KiB MiB GiB TiB PiB EiB", units, " ")
+        i = 1
+        while (b >= 1024 && i < unit_count) {
+          b = b / 1024
+          i++
+        }
+        if (i == 1) {
+          printf "%.0f %s\n", b, units[i]
+        } else {
+          printf "%.2f %s\n", b, units[i]
+        }
+      }
+    '
+  else
+    echo "unknown"
+  fi
+}
+
+latest_snapshot_for_dataset() {
+  # Latest snapshot on the TOP dataset only (newest first). Returns e.g. zroot@2025-09-30-...
+  zfs list -t snapshot -o name -S creation "$DATASET" 2>/dev/null \
+    | awk 'NR==2{print; exit}'
+}
+
+list_top_snapshots_newest_first() {
+  # Only list snapshots on the top dataset (not children), newest first.
+  zfs list -t snapshot -o name -S creation "$DATASET" 2>/dev/null \
+    | awk 'NR>1{print $1}'
+}
+
+list_excluded_datasets() {
+  # List datasets (including inherited values) where backup property is explicitly false.
+  zfs get -r -H -o name,value "${BACKUP_EXCLUDE_PROP}" "${DATASET}" 2>/dev/null \
+    | awk 'tolower($2)=="false"{print $1}'
+}
+
+list_remote_backups_sorted() {
+  # We store files under ${HOST_DIR}/<dataset>@<timestamp>.enc
+  echo "ls ${HOST_DIR}" \
+    | sftp -q -i "${KEYFILE}" "${USER}@${HOST}" 2>/dev/null \
+    | tail -n +2 | sort
+}
+
+# Upload a local file to the storage box path "${HOST_DIR}/<remote_name>"
+sftp_put() {
+  local local_file="$1"
+  local remote_name="$2"  # just the filename, no directory
+  sftp -i "${KEYFILE}" "${USER}@${HOST}" <<EOF
+put ${local_file} ${HOST_DIR}/${remote_name}
+EOF
+}
+
+# ----- Subcommands -----
+cmd_init_remote() {
+  echo ""
+  echo "[init-remote]"
+  require_cmds sftp
+
+  echo "Checking for remote directory: ${HOST_DIR}"
+  if echo "ls ${HOST_DIR}" | sftp -i "${KEYFILE}" "${USER}@${HOST}" >/dev/null 2>&1; then
+    echo "Remote directory already exists: ${HOST_DIR}"
+    log_note "init-remote: verified remote directory '${HOST_DIR}'"
+    return 0
+  fi
+
+  echo "Creating remote directory: ${HOST_DIR}"
+  set +e
+  sftp -i "${KEYFILE}" "${USER}@${HOST}" <<EOF
+mkdir ${HOST_DIR}
+EOF
+  rc=$?
+  set -e
+  if [[ $rc -ne 0 ]]; then
+    die "Failed to create remote directory ${HOST_DIR}"
+  fi
+
+  # Re-check
+  if echo "ls ${HOST_DIR}" | sftp -q -i "${KEYFILE}" "${USER}@${HOST}" >/dev/null 2>&1; then
+    echo "Remote directory created: ${HOST_DIR}"
+    log_note "init-remote: created remote directory '${HOST_DIR}'"
+  else
+    die "Remote directory ${HOST_DIR} not found after creation"
+  fi
+}
+
+cmd_snapshot() {
+  echo ""
+  echo "[snapshot]"
+  require_cmds zfs date tr
+  local ts snap
+  ts="$(timestamp)"
+  snap="${DATASET}@${ts}"
+  echo "Taking recursive snapshot \"${snap}\""
+  (set -x; zfs snapshot -r "${snap}")
+  echo "Recursive snapshot \"${snap}\" created"
+  log_note "snapshot: created recursive snapshot '${snap}'"
+}
+
+cmd_send_to_remote() {
+  echo ""
+  echo "[send-to-remote]"
+  require_cmds zfs age sftp mktemp stat awk sort
+
+  local snap base target tmp size_bytes ds parent skip
+  local -a send_args excluded_raw excluded
+  snap="$(latest_snapshot_for_dataset)"
+  [[ -n "${snap}" ]] || die "No snapshot found to send. Run 'snapshot' first or ensure the dataset has snapshots."
+
+  base="$(basename "${snap}")"       # e.g., zroot@2025-09-30-12-00-00-0300
+  target="${base}.enc"
+  tmp="$(mktemp -t backup_send.XXXXXX)"
+  trap 'rm -f "${tmp}"' EXIT
+
+  send_args=(-Rvc)
+  mapfile -t excluded_raw < <(list_excluded_datasets | sort)
+  excluded=()
+  for ds in "${excluded_raw[@]}"; do
+    skip=0
+    [[ -n "${ds}" ]] || continue
+    for parent in "${excluded[@]}"; do
+      if [[ "${ds}" == "${parent}" || "${ds}" == "${parent}/"* ]]; then
+        skip=1
+        break
+      fi
+    done
+    [[ "${skip}" -eq 1 ]] || excluded+=("${ds}")
+  done
+
+  if [[ "${#excluded[@]}" -gt 0 ]]; then
+    echo "Excluding datasets where ${BACKUP_EXCLUDE_PROP}=false:"
+    for ds in "${excluded[@]}"; do
+      if [[ "${ds}" == "${DATASET}" ]]; then
+        die "Top dataset ${DATASET} is excluded via ${BACKUP_EXCLUDE_PROP}=false; refusing to create an empty backup stream."
+      fi
+      echo "  - ${ds}"
+      send_args+=(-X "${ds}")
+    done
+    log_note "send-to-remote: excluded ${#excluded[@]} dataset(s) using ${BACKUP_EXCLUDE_PROP}=false"
+  fi
+
+  echo "Creating encrypted replication stream to temp file: ${tmp}"
+  # -R: recursive hierarchy, -v: progress to stderr, -c: send compressed (keeps on-disk compression)
+  (set -x; zfs send "${send_args[@]}" "${snap}" | age -e -i "${KEYFILE}" > "${tmp}")
+
+  size_bytes="$(stat -f %z "${tmp}" 2>/dev/null || stat -c %s "${tmp}" 2>/dev/null || echo "unknown")"
+  echo "Local stream size: ${size_bytes} bytes"
+
+  echo "Uploading via SFTP to ${HOST}:${HOST_DIR}/${target}"
+  if sftp_put "${tmp}" "${target}"; then
+    echo "Upload complete"
+  else
+    die "SFTP upload failed (does the remote directory '${HOST_DIR}' exist? Run 'init-remote')"
+  fi
+
+  # Set globals for notify()
+  BACKUP_NAME="${target}"
+  BACKUP_SIZE_BYTES="${size_bytes}"
+
+  log_note "send-to-remote: uploaded '${snap}' as '${target}' (${size_bytes} bytes) to ${HOST}:${HOST_DIR}"
+  rm -f "${tmp}"
+  trap - EXIT
+}
+
+cmd_prune_remote() {
+  echo ""
+  echo "[prune-remote]"
+  require_cmds sftp awk sort wc
+
+  echo "Fetching remote backup listing from sftp://${HOST}/${HOST_DIR}"
+  BACKUPS="$(list_remote_backups_sorted || true)"
+  mapfile -t BACKUP_ARR < <(printf "%s\n" "${BACKUPS}")
+  local count="${#BACKUP_ARR[@]}"
+
+  if [[ "${count}" -le "${KEEP_REMOTE}" ]]; then
+    echo "Remote backups (${count}) <= KEEP_REMOTE (${KEEP_REMOTE}); nothing to prune."
+    log_note "prune-remote: kept ${count} (<= ${KEEP_REMOTE}); no deletions"
+    return 0
+  fi
+
+  echo "Pruning remote backups, keeping latest ${KEEP_REMOTE} (will delete $(("${count}" - "${KEEP_REMOTE}")))"
+  local to_delete_count=$((count - KEEP_REMOTE))
+  local deleted=0
+  for ((i=0; i<to_delete_count; i++)); do
+    b="${BACKUP_ARR[$i]}"
+    [[ -n "${b}" ]] || continue
+    echo "Deleting remote: ${b}"
+    set +e
+    echo "rm ${b}" | sftp -i "${KEYFILE}" "${USER}@${HOST}"
+    rc=$?
+    set -e
+    if [[ $rc -ne 0 ]]; then
+      die "Failed to delete remote file: ${b}"
+    fi
+    deleted=$((deleted+1))
+  done
+  echo "Remote prune done. Deleted: ${deleted}"
+  log_note "prune-remote: deleted ${deleted}, kept ${KEEP_REMOTE}"
+}
+
+cmd_prune_local() {
+  echo ""
+  echo "[prune-local]"
+  require_cmds zfs awk
+
+  echo "Pruning local snapshots on dataset: ${DATASET}; keeping latest ${KEEP_LOCAL}"
+  mapfile -t snaps < <(list_top_snapshots_newest_first)
+  local total="${#snaps[@]}"
+  if [[ "${total}" -le "${KEEP_LOCAL}" ]]; then
+    echo "Local snapshots (${total}) <= KEEP_LOCAL (${KEEP_LOCAL}); nothing to prune."
+    log_note "prune-local: kept ${total} (<= ${KEEP_LOCAL}); no deletions"
+    return 0
+  fi
+
+  local deleted=0
+  # List is newest-first; skip first KEEP_LOCAL and destroy the rest RECURSIVELY across the tree
+  for ((i=KEEP_LOCAL; i<total; i++)); do
+    s="${snaps[$i]}"   # e.g., zroot@2025-09-30-...
+    echo "Destroying recursive snapshot: ${s}"
+    (set -x; zfs destroy -r "${s}")
+    deleted=$((deleted+1))
+  done
+  echo "Local prune done. Deleted: ${deleted}"
+  log_note "prune-local: deleted ${deleted}, kept ${KEEP_LOCAL}"
+}
+
+cmd_notify() {
+  echo ""
+  echo "[notify]"
+  require_cmds mail date awk
+  local backup_size_human
+  FINISHED_AT="$(date '+%Y-%m-%dT%H:%M:%S%z')"
+
+  # Derive snapshot name if not set yet (best-effort)
+  if [[ -z "${BACKUP_NAME}" ]]; then
+    # Try to infer from latest snapshot
+    latest="$(latest_snapshot_for_dataset || true)"
+    if [[ -n "${latest}" ]]; then
+      BACKUP_NAME="$(basename "${latest}").enc"
+    else
+      BACKUP_NAME="unknown"
+    fi
+  fi
+
+  backup_size_human="$(humanize_bytes "${BACKUP_SIZE_BYTES:-unknown}")"
+
+  # Compose subject and body
+  local subject="Backup completed: ${BACKUP_NAME}"
+  local body=""
+  body+="Backup run summary"$'\n'
+  body+="Started:  ${STARTED_AT}"$'\n'
+  body+="Finished: ${FINISHED_AT}"$'\n'
+  body+="Dataset:  ${DATASET}"$'\n'
+  body+="Remote:   ${USER}@${HOST}:${HOST_DIR}"$'\n'
+  body+="Name:     ${BACKUP_NAME}"$'\n'
+  body+="Size:     ${backup_size_human} (${BACKUP_SIZE_BYTES:-unknown} bytes)"$'\n'
+  body+=$'\n'
+  body+="Steps:"$'\n'
+  body+="${MESSAGE_LOG:-<no steps recorded>}"$'\n'
+
+  echo "Sending notification email..."
+  printf "%s\n" "$body" | mail -s "$subject" "${EMAIL_TO}"
+  log_note "notify: email sent to ${EMAIL_TO}"
+}
+
+usage() {
+  cat <<EOF
+Usage: $(basename "$0") [subcommand [subcommand ...]]
+
+Subcommands (executed in order):
+  init-remote      Create the remote backup directory (\$HOST_DIR). Succeeds if it already exists.
+  snapshot         Create a new **recursive** ZFS snapshot for \$DATASET
+  send-to-remote   Create encrypted replication stream to a temp file, then upload via SFTP
+  prune-remote     Keep latest \$KEEP_REMOTE backups on remote (default ${KEEP_REMOTE})
+  prune-local      Keep latest \$KEEP_LOCAL local snapshots on the top dataset (default ${KEEP_LOCAL})
+  notify           Send notification email (includes steps, timestamps, backup name & size)
+
+Notes:
+  - Run 'init-remote' once before the first upload, or anytime after changing \$HOST_DIR.
+  - Local pruning destroys older snapshots **recursively** to maintain consistency across descendants.
+  - Set \${BACKUP_EXCLUDE_PROP}=false on a dataset to exclude it (and inherited descendants) from send-to-remote.
+    Example: zfs set com.pursotin:backup=false zroot/some/heavy-dataset
+
+Examples:
+  $(basename "$0") init-remote snapshot send-to-remote prune-remote prune-local notify
+EOF
+}
+
+# ----- Main -----
+main() {
+  if [[ $# -eq 0 ]]; then
+    usage
+    exit 1
+  fi
+
+  # sanity
+  [[ -n "${DATASET}" ]] || die "DATASET not set"
+  [[ -n "${USER}" && -n "${HOST}" ]] || die "USER/HOST not set"
+  [[ -r "${KEYFILE}" ]] || die "KEYFILE not readable: ${KEYFILE}"
+
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      init-remote)    cmd_init_remote ;;
+      snapshot)        cmd_snapshot ;;
+      send-to-remote)  cmd_send_to_remote ;;
+      prune-remote)    cmd_prune_remote ;;
+      prune-local)     cmd_prune_local ;;
+      notify)          cmd_notify ;;
+      -h|--help|help)  usage; exit 0 ;;
+      *) die "Unknown subcommand: $1" ;;
+    esac
+    shift
+  done
+
+  echo "Done."
+}
+
+main "$@"
+{% endraw %}
diff --git a/roles/host_prod/templates/usr_local_bin_safepf.sh b/roles/host_prod/templates/usr_local_bin_safepf.sh
new file mode 100644
index 0000000..1b6efee
--- /dev/null
+++ b/roles/host_prod/templates/usr_local_bin_safepf.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -eu
+
+cmd="pfctl -f /etc/pf.conf"
+read -p "Run this command (y/n)? $cmd " answer
+if [ "$answer" = "y" ]; then
+    (set -x; $cmd)
+else
+    exit 0
+fi
+
+cmd="service pf restart"
+read -p "Run this command (y/n)? $cmd " answer
+if [ "$answer" = "y" ]; then
+    (set -x; $cmd)
+else
+    exit 0
+fi
+
+timeout=60
+echo "Running safety timeout ($timeout seconds). Press CTRL-C if everything is working."
+while [ $timeout -gt 0 ]
+do
+    sleep 1
+    timeout=$((timeout - 1))
+    echo -n "."
+done
+
+echo "Timeout reached. Enabling empty pf rules"
+
+set -x
+mv /etc/pf.conf /etc/pf.conf.locked_out
+echo "" > /etc/pf.conf
+pfctl -f /etc/pf.conf
+service pf restart