Improvements

9 files changed, 242 insertions, 17 deletions

diff --git a/tasks/jail_ingress.yml b/tasks/jail_ingress.yml
index 71d87fa..87dff41 100644
--- a/tasks/jail_ingress.yml
+++ b/tasks/jail_ingress.yml
@@ -6,6 +6,29 @@
     - { jail: ingress, package: goaccess }
   include_tasks: pkg_jail_install.yml
 
+- name: Configure pf.conf
+  template:
+    src: ingress/etc_pf.conf.j2
+    dest: /usr/local/jails/containers/ingress/etc/pf.conf
+    owner: root
+    group: wheel
+    mode: "0644"
+
+- name: Enable gateway mode
+  shell: jexec ingress sysrc gateway_enable=YES && jexec ingress sysctl net.inet.ip.forwarding=1
+
+- name: Enable pf
+  shell: jexec ingress sysrc pf_enable=YES
+
+- name: Check if pf is running
+  shell: service -j ingress pf status
+  register: pf_status
+  changed_when: pf_status.rc != 0
+
+- name: Start pf
+  shell: service -j ingress pf start
+  when: pf_status.rc != 0
+
 - name: Generate LetsEncrypt certs
   shell: jexec ingress certbot certonly --nginx -n -m jan@jantuomi.fi --agree-tos -d {{ item }}
   loop: "{{ ingress_routes | map(attribute='host') | unique }}"
@@ -59,6 +82,14 @@
     group: wheel
     mode: "0600"
 
+- name: Set up gen_goaccess.sh
+  template:
+    src: ingress/usr_local_bin_gen_goaccess.sh.j2
+    dest: /usr/local/jails/containers/ingress/usr/local/bin/gen_goaccess.sh
+    owner: root
+    group: wheel
+    mode: "0755"
+
 - name: Set up crontab
   template:
     src: ingress/etc_crontab.j2
diff --git a/tasks/jails.yml b/tasks/jails.yml
index 99eb938..fe94126 100644
--- a/tasks/jails.yml
+++ b/tasks/jails.yml
@@ -67,6 +67,14 @@
         group: wheel
         mode: "0644"
 
+    - name: Configure devfs.rules
+      template:
+        src: etc_devfs.rules.j2
+        dest: /etc/devfs.rules
+        owner: root
+        group: wheel
+        mode: "0644"
+
     - name: Configure individual jails
       loop: "{{ jails }}"
       loop_control:
diff --git a/templates/etc_devfs.rules.j2 b/templates/etc_devfs.rules.j2
new file mode 100644
index 0000000..7145400
--- /dev/null
+++ b/templates/etc_devfs.rules.j2
@@ -0,0 +1,82 @@
+# "as-is" to the devfs(8) command with the exception that
+# any references to other rulesets will be expanded first. These
+# references must include a dollar sign '$' in front of the
+# name to be expanded properly.
+#
+#
+
+# Very basic and secure ruleset: Hide everything.
+# Used as a basis for other rules.
+#
+[devfsrules_hide_all=1]
+add hide
+
+# Basic devices typically necessary.
+# Requires: devfsrules_hide_all
+#
+[devfsrules_unhide_basic=2]
+add path null unhide
+add path zero unhide
+add path crypto unhide
+add path random unhide
+add path urandom unhide
+
+# Devices typically needed to support logged-in users.
+# Requires: devfsrules_hide_all
+#
+[devfsrules_unhide_login=3]
+add path 'ptyp*' unhide
+add path 'ptyq*' unhide
+add path 'ptyr*' unhide
+add path 'ptys*' unhide
+add path 'ptyP*' unhide
+add path 'ptyQ*' unhide
+add path 'ptyR*' unhide
+add path 'ptyS*' unhide
+add path 'ptyl*' unhide
+add path 'ptym*' unhide
+add path 'ptyn*' unhide
+add path 'ptyo*' unhide
+add path 'ptyL*' unhide
+add path 'ptyM*' unhide
+add path 'ptyN*' unhide
+add path 'ptyO*' unhide
+add path 'ttyp*' unhide
+add path 'ttyq*' unhide
+add path 'ttyr*' unhide
+add path 'ttys*' unhide
+add path 'ttyP*' unhide
+add path 'ttyQ*' unhide
+add path 'ttyR*' unhide
+add path 'ttyS*' unhide
+add path 'ttyl*' unhide
+add path 'ttym*' unhide
+add path 'ttyn*' unhide
+add path 'ttyo*' unhide
+add path 'ttyL*' unhide
+add path 'ttyM*' unhide
+add path 'ttyN*' unhide
+add path 'ttyO*' unhide
+add path ptmx unhide
+add path pts unhide
+add path 'pts/*' unhide
+add path fd unhide
+add path 'fd/*' unhide
+add path stdin unhide
+add path stdout unhide
+add path stderr unhide
+
+[devfsrules_jail=4]
+add include $devfsrules_hide_all
+add include $devfsrules_unhide_basic
+add include $devfsrules_unhide_login
+add path pf unhide
+#add path fuse unhide
+#add path zfs unhide
+
+[devfsrules_jail_postgres=5]
+add include $devfsrules_hide_all
+add include $devfsrules_unhide_basic
+add include $devfsrules_unhide_login
+add include $devfsrules_jail
+add path 'bpf*' unhide
diff --git a/templates/etc_jail.conf.d_[jailname].conf.j2 b/templates/etc_jail.conf.d_[jailname].conf.j2
index 7f7a049..ce0e7ba 100644
--- a/templates/etc_jail.conf.d_[jailname].conf.j2
+++ b/templates/etc_jail.conf.d_[jailname].conf.j2
@@ -3,26 +3,44 @@
 {{ jail.name }} {
   # STARTUP/LOGGING/VNET
   vnet;
-  vnet.interface    = "epl{{ jail.num }}b";
   persist;
-  exec.prestart    += "ifconfig epair{{ jail.num }} create || echo 'Failed to create epair{{ jail.num }}'";
-  exec.prestart    += "ifconfig epair{{ jail.num }}a name epl{{ jail.num }}a";
-  exec.prestart    += "ifconfig epair{{ jail.num }}b name epl{{ jail.num }}b";
-  exec.prestart    += "ifconfig epl{{ jail.num }}a up";
-  exec.prestart    += "ifconfig brlan0 addm epl{{ jail.num }}a";
   exec.clean;
+  exec.prestart     = "ifconfig epair{{ jail.num }}000 create || echo 'Failed to create epair{{ jail.num }}000'";
+  exec.prestart    += "ifconfig epair{{ jail.num }}000a name epl{{ jail.num }}a";
+  exec.prestart    += "ifconfig epair{{ jail.num }}000b name epl{{ jail.num }}b";
+  exec.prestart    += "ifconfig brlan0 addm epl{{ jail.num }}a";
+  {% if jail.name == "ingress" -%}
+  exec.prestart    += "ifconfig epair{{ jail.num }}001 create || echo 'Failed to create epair{{ jail.num }}001'";
+  exec.prestart    += "ifconfig epair{{ jail.num }}001a name epw1a";
+  exec.prestart    += "ifconfig epair{{ jail.num }}001b name epw1b";
+  exec.prestart    += "ifconfig brwan0 addm epw{{ jail.num }}a";
+  {% endif %}
+
   exec.start        = "/bin/sh /etc/rc";
-  exec.poststart    = "jexec ${name} ifconfig epl{{ jail.num }}b 192.168.2.{{ jail.num }}/16 up";
+
+  exec.poststart    = "ifconfig epl{{ jail.num }}b vnet ${name}";
+  exec.poststart   += "jexec ${name} ifconfig epl{{ jail.num }}b up";
   exec.poststart   += "ifconfig epl{{ jail.num }}a up";
-  exec.poststart   += "jexec ${name} route add default 192.168.0.1 || echo 'Failed to add default route'";
+  exec.poststart   += "jexec ${name} ifconfig epl{{ jail.num }}b 192.168.2.{{ jail.num }}/16";
+  {% for ing in jails -%}
+  {% if ing.name == "ingress" -%}
+  exec.poststart   += "jexec ${name} route add default 192.168.2.{{ ing.num }} || echo 'Failed to add default route'";
+  {% endif %}
+  {% endfor %}
   {% if jail.name == "ingress" -%}
+  exec.poststart   += "ifconfig epw{{ jail.num }}b vnet ${name}";
   exec.poststart   += "jexec ${name} ifconfig epw{{ jail.num }}b up";
+  exec.poststart   += "ifconfig epw{{ jail.num }}a up";
   exec.poststart   += "jexec ${name} service dhclient restart epw{{ jail.num }}b";
   {% endif %}
 
   exec.stop         = "/bin/sh /etc/rc.shutdown";
-  exec.poststop    += "ifconfig epl{{ jail.num }}b -vnet $name"; # workaround to bug 238326: move epl{{ jail.num }}b from the jail to the host when stopping jail services
-  exec.poststop    += "ifconfig epl{{ jail.num }}b destroy";     # and then destroy the pair by destroying one end of it
+
+  exec.poststop    += "ifconfig epl{{ jail.num }}a destroy";
+  {% if jail.name == "ingress" -%}
+  exec.poststop    += "ifconfig epw{{ jail.num }}a destroy";
+  {% endif %}
+
   exec.consolelog   = "/var/log/jail_console_${name}.log";
 
   # PERMISSIONS
@@ -31,6 +49,9 @@
   mount.devfs;
   {%if jail.name == "postgres" -%}
   allow.sysvipc;
+  devfs_ruleset = 5;
+  {% else %}
+  devfs_ruleset = 4;
   {% endif %}
 
   # HOSTNAME/PATH
diff --git a/templates/ingress/etc_crontab.j2 b/templates/ingress/etc_crontab.j2
index e0c3d4e..d78e769 100644
--- a/templates/ingress/etc_crontab.j2
+++ b/templates/ingress/etc_crontab.j2
@@ -17,4 +17,4 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
 {% endfor %}
 
 # Update goaccess report HTML
-0       *       *       *       *       www    (cat /var/log/nginx/access.log | awk '$8=$1$8' | /usr/local/bin/goaccess --log-format=VCOMBINED -j "$(($(nproc) + 1))" --keep-last=30 -a -o /var/www/goaccess/index.html --persist --restore)
+0       *       *       *       *       root    /usr/local/bin/gen_goaccess.sh
diff --git a/templates/ingress/etc_pf.conf.j2 b/templates/ingress/etc_pf.conf.j2
new file mode 100644
index 0000000..c8045fd
--- /dev/null
+++ b/templates/ingress/etc_pf.conf.j2
@@ -0,0 +1,19 @@
+{% for jail in jails -%}
+{% if jail.name == 'ingress' -%}
+# Interfaces & nets
+lan = "epl{{ jail.num }}b"
+wan = "epw{{ jail.num }}b"
+lan_net = "{{ lan_ipv4_cidr }}"
+
+# Keep PF out of loopback, drop by default if you add blocks later
+set skip on lo0
+set block-policy drop
+
+# NAT: translate LAN traffic to the WAN interface address, and log it
+nat log on $wan from $lan_net to any -> ($wan)
+
+# Allow and log the LAN traffic so NAT can happen
+pass in  on $lan inet from $lan_net to any keep state
+pass out on $wan inet from $lan_net to any keep state
+{% endif %}
+{% endfor %}
diff --git a/templates/ingress/usr_local_bin_gen_goaccess.sh.j2 b/templates/ingress/usr_local_bin_gen_goaccess.sh.j2
new file mode 100644
index 0000000..fb7a7bc
--- /dev/null
+++ b/templates/ingress/usr_local_bin_gen_goaccess.sh.j2
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+cat /var/log/nginx/access.log | awk '$8=$1$8' | /usr/local/bin/goaccess --log-format=VCOMBINED -j "$(($(nproc) + 1))" --keep-last=30 -a -o /var/www/goaccess/index.html --persist --restore
+chmod 644 /var/www/goaccess/index.html
+chown www /var/www/goaccess/index.html
+chgrp www /var/www/goaccess/index.html
diff --git a/templates/ingress/usr_local_etc_nginx_nginx.conf.j2 b/templates/ingress/usr_local_etc_nginx_nginx.conf.j2
index 0ae1e56..037a724 100644
--- a/templates/ingress/usr_local_etc_nginx_nginx.conf.j2
+++ b/templates/ingress/usr_local_etc_nginx_nginx.conf.j2
@@ -101,6 +101,9 @@ http {
         include /usr/local/etc/letsencrypt/options-ssl-nginx.conf;
         ssl_dhparam /usr/local/etc/letsencrypt/ssl-dhparams.pem;
 
+        # No error logging for static files
+        error_log off;
+
         location / {
             root {{ route.static }};
             index index.html;
diff --git a/templates/usr_local_bin_backup.sh.j2 b/templates/usr_local_bin_backup.sh.j2
index 1fd099d..3c4d977 100644
--- a/templates/usr_local_bin_backup.sh.j2
+++ b/templates/usr_local_bin_backup.sh.j2
@@ -12,6 +12,13 @@ HOST="{% endraw %}{{ backup_ssh_host }}{% raw %}"
 EMAIL_TO=root
 # =====================================
 
+# ----- Globals for notification -----
+STARTED_AT="$(date '+%Y-%m-%dT%H:%M:%S%z')"
+FINISHED_AT=""
+MESSAGE_LOG=""
+BACKUP_NAME=""          # e.g. zroot@2025-09-30-12-00-00-0300.enc
+BACKUP_SIZE_BYTES=""    # numeric bytes
+
 # ----- Helpers -----
 die() { echo "Error: $*" >&2; exit 1; }
 
@@ -20,6 +27,13 @@ require_cmds() {
   for c in "${cmds[@]}"; do command -v "$c" >/dev/null 2>&1 || die "Missing command: $c"; done
 }
 
+log_note() {
+  # Echo to console and append to message buffer
+  local msg="$1"
+  echo "$msg"
+  MESSAGE_LOG+="$msg"$'\n'
+}
+
 timestamp() {
   # Replace '+' with '-' so timezone is filename-safe and lexicographically sortable within TZ.
   date +%Y-%m-%d-%H-%M-%S%z | tr '+' '-'
@@ -38,7 +52,6 @@ list_top_snapshots_newest_first() {
 }
 
 list_remote_backups_sorted() {
-  # Note: sorting strictly orders only within the same timezone in the name, which is fine for our daily cadence.
   # We store files under ${HOST_DIR}/<dataset>@<timestamp>.enc
   echo "ls ${HOST_DIR}" \
     | sftp -q -i "${KEYFILE}" "${USER}@${HOST}" 2>/dev/null \
@@ -63,6 +76,7 @@ cmd_init_remote() {
   echo "Checking for remote directory: ${HOST_DIR}"
   if echo "ls ${HOST_DIR}" | sftp -i "${KEYFILE}" "${USER}@${HOST}" >/dev/null 2>&1; then
     echo "Remote directory already exists: ${HOST_DIR}"
+    log_note "init-remote: verified remote directory '${HOST_DIR}'"
     return 0
   fi
 
@@ -80,6 +94,7 @@ EOF
   # Re-check
   if echo "ls ${HOST_DIR}" | sftp -q -i "${KEYFILE}" "${USER}@${HOST}" >/dev/null 2>&1; then
     echo "Remote directory created: ${HOST_DIR}"
+    log_note "init-remote: created remote directory '${HOST_DIR}'"
   else
     die "Remote directory ${HOST_DIR} not found after creation"
   fi
@@ -95,6 +110,7 @@ cmd_snapshot() {
   echo "Taking recursive snapshot \"${snap}\""
   (set -x; zfs snapshot -r "${snap}")
   echo "Recursive snapshot \"${snap}\" created"
+  log_note "snapshot: created recursive snapshot '${snap}'"
 }
 
 cmd_send_to_remote() {
@@ -125,6 +141,11 @@ cmd_send_to_remote() {
     die "SFTP upload failed (does the remote directory '${HOST_DIR}' exist? Run 'init-remote')"
   fi
 
+  # Set globals for notify()
+  BACKUP_NAME="${target}"
+  BACKUP_SIZE_BYTES="${size_bytes}"
+
+  log_note "send-to-remote: uploaded '${snap}' as '${target}' (${size_bytes} bytes) to ${HOST}:${HOST_DIR}"
   rm -f "${tmp}"
   trap - EXIT
 }
@@ -141,6 +162,7 @@ cmd_prune_remote() {
 
   if [[ "${count}" -le "${KEEP_REMOTE}" ]]; then
     echo "Remote backups (${count}) <= KEEP_REMOTE (${KEEP_REMOTE}); nothing to prune."
+    log_note "prune-remote: kept ${count} (<= ${KEEP_REMOTE}); no deletions"
     return 0
   fi
 
@@ -161,6 +183,7 @@ cmd_prune_remote() {
     deleted=$((deleted+1))
   done
   echo "Remote prune done. Deleted: ${deleted}"
+  log_note "prune-remote: deleted ${deleted}, kept ${KEEP_REMOTE}"
 }
 
 cmd_prune_local() {
@@ -173,6 +196,7 @@ cmd_prune_local() {
   local total="${#snaps[@]}"
   if [[ "${total}" -le "${KEEP_LOCAL}" ]]; then
     echo "Local snapshots (${total}) <= KEEP_LOCAL (${KEEP_LOCAL}); nothing to prune."
+    log_note "prune-local: kept ${total} (<= ${KEEP_LOCAL}); no deletions"
     return 0
   fi
 
@@ -185,12 +209,43 @@ cmd_prune_local() {
     deleted=$((deleted+1))
   done
   echo "Local prune done. Deleted: ${deleted}"
+  log_note "prune-local: deleted ${deleted}, kept ${KEEP_LOCAL}"
 }
 
 cmd_notify() {
-    echo "Sending notification email..."
-    local latest=$(latest_snapshot_for_dataset "${DATASET}")
-    echo "Backup of snapshot ${latest} completed at $(date)" | mail -s "Backup completed" "${EMAIL_TO}"
+  echo ""
+  echo "[notify]"
+  require_cmds mail date
+  FINISHED_AT="$(date '+%Y-%m-%dT%H:%M:%S%z')"
+
+  # Derive snapshot name if not set yet (best-effort)
+  if [[ -z "${BACKUP_NAME}" ]]; then
+    # Try to infer from latest snapshot
+    latest="$(latest_snapshot_for_dataset || true)"
+    if [[ -n "${latest}" ]]; then
+      BACKUP_NAME="$(basename "${latest}").enc"
+    else
+      BACKUP_NAME="unknown"
+    fi
+  fi
+
+  # Compose subject and body
+  local subject="Backup completed: ${BACKUP_NAME}"
+  local body=""
+  body+="Backup run summary"$'\n'
+  body+="Started:  ${STARTED_AT}"$'\n'
+  body+="Finished: ${FINISHED_AT}"$'\n'
+  body+="Dataset:  ${DATASET}"$'\n'
+  body+="Remote:   ${USER}@${HOST}:${HOST_DIR}"$'\n'
+  body+="Name:     ${BACKUP_NAME}"$'\n'
+  body+="Size:     ${BACKUP_SIZE_BYTES:-unknown} bytes"$'\n'
+  body+=$'\n'
+  body+="Steps:"$'\n'
+  body+="${MESSAGE_LOG:-<no steps recorded>}"$'\n'
+
+  echo "Sending notification email..."
+  printf "%s\n" "$body" | mail -s "$subject" "${EMAIL_TO}"
+  log_note "notify: email sent to ${EMAIL_TO}"
 }
 
 usage() {
@@ -203,14 +258,14 @@ Subcommands (executed in order):
   send-to-remote   Create encrypted replication stream to a temp file, then upload via SFTP
   prune-remote     Keep latest \$KEEP_REMOTE backups on remote (default ${KEEP_REMOTE})
   prune-local      Keep latest \$KEEP_LOCAL local snapshots on the top dataset (default ${KEEP_LOCAL})
-  notify           Send notification email
+  notify           Send notification email (includes steps, timestamps, backup name & size)
 
 Notes:
   - Run 'init-remote' once before the first upload, or anytime after changing \$HOST_DIR.
   - Local pruning destroys older snapshots **recursively** to maintain consistency across descendants.
 
 Examples:
-  $(basename "$0") init-remote snapshot send-to-remote prune-remote prune-local
+  $(basename "$0") init-remote snapshot send-to-remote prune-remote prune-local notify
 EOF
 }