diff options
32 files changed, 361 insertions, 279 deletions
diff --git a/filter_plugins/ipv4.py b/filter_plugins/ipv4.py index 0bf8c67..d0043b6 100644 --- a/filter_plugins/ipv4.py +++ b/filter_plugins/ipv4.py @@ -40,6 +40,29 @@ def ipv4_prefixlen(cidr): return cidr.split("/")[1] +def normalize_zfs(entries): + """Expand short ZFS dataset names to full zroot/jails/volumes/ paths.""" + result = [] + for entry in entries: + e = dict(entry) + if '/' not in e.get('name', ''): + e['name'] = 'zroot/jails/volumes/' + e['name'] + result.append(e) + return result + + +def normalize_nullfs(entries): + """Expand short nullfs src names to full /usr/local/jails/volumes/ paths.""" + result = [] + for entry in entries: + e = dict(entry) + src = e.get('src', '') + if not src.startswith('/'): + e['src'] = '/usr/local/jails/volumes/' + src + result.append(e) + return result + + class FilterModule(object): def filters(self): return { @@ -48,4 +71,6 @@ class FilterModule(object): "ipv4_nth_cidr": ipv4_nth_cidr, "ipv4_host": ipv4_host, "ipv4_prefixlen": ipv4_prefixlen, + "normalize_zfs": normalize_zfs, + "normalize_nullfs": normalize_nullfs, } diff --git a/inventory/prod/hosts.yml b/inventory/prod/hosts.yml index c6f5fa9..ab2f80f 100644 --- a/inventory/prod/hosts.yml +++ b/inventory/prod/hosts.yml @@ -11,6 +11,8 @@ all: lan_ipv4_gateway: "192.168.0.1" lan_ipv4_cidr: "192.168.0.10/16" poudriere_repo_ip: "192.168.2.21" + jail_lan_cidr: "192.168.2.0/16" + jail_lan_offset: 0 is_prod: true jails: vars: diff --git a/inventory/test/hosts.yml b/inventory/test/hosts.yml index d9f5bfb..2624b52 100644 --- a/inventory/test/hosts.yml +++ b/inventory/test/hosts.yml @@ -16,6 +16,9 @@ all: lan_ipv4_gateway: "10.0.20.1" lan_ipv4_cidr: "10.0.20.2/24" poudriere_repo_ip: "10.0.20.121" + jail_lan_cidr: "10.0.20.0/24" + jail_lan_offset: 100 + ingress_wan_static: "10.0.20.3/24" is_prod: false jails: vars: diff --git a/roles/host/tasks/main.yml b/roles/host/tasks/main.yml index 2a18e7a..78be7c6 100644 --- a/roles/host/tasks/main.yml +++ b/roles/host/tasks/main.yml @@ -105,6 +105,14 @@ group: wheel mode: "0755" +- name: Deploy jail_net script + template: + src: "{{ role_path }}/../jail/templates/usr_local_bin_jail_net.j2" + dest: /usr/local/bin/jail_net + owner: root + group: wheel + mode: "0755" + - name: Check if quarterly ports tree exists stat: path: /usr/ports/quarterly/Mk @@ -157,7 +165,7 @@ - { name: "zroot/jails/media" } - { name: "zroot/jails/templates" } - { name: "zroot/jails/containers" } - - { name: "zroot/jails/volumes", mountpoint: "none" } + - { name: "zroot/jails/volumes", mountpoint: "/usr/local/jails/volumes" } - { name: "zroot/storage", mountpoint: "/usr/local/jails/volumes/storage" } - { name: "zroot/netdata_db", mountpoint: "/var/db/netdata" } - { name: "zroot/ports", mountpoint: "/usr/ports" } diff --git a/roles/jail/tasks/jail_launch.yml b/roles/jail/tasks/jail_launch.yml index 31ccdf2..803290c 100644 --- a/roles/jail/tasks/jail_launch.yml +++ b/roles/jail/tasks/jail_launch.yml @@ -1,5 +1,6 @@ - name: Start services - service: - name: "{{ item }}" - state: started + shell: "service {{ item }} start || service {{ item }} status" loop: "{{ services | default([]) }}" + register: _svc_start + changed_when: "'already running' not in _svc_start.stdout and 'is running' not in _svc_start.stdout" + failed_when: "_svc_start.rc != 0 and 'already running' not in _svc_start.stdout and 'is running' not in _svc_start.stdout" diff --git a/roles/jail/tasks/jail_setup.yml b/roles/jail/tasks/jail_setup.yml index f50562a..2c8581d 100644 --- a/roles/jail/tasks/jail_setup.yml +++ b/roles/jail/tasks/jail_setup.yml @@ -1,4 +1,11 @@ # Host-side setup (runs on the jail host via SSH) + +# Normalize zfs and nullfs entries: expand short names to full paths +- name: "Normalize zfs and nullfs entries for {{ jail_name }}" + set_fact: + _zfs: "{{ zfs | default([]) | normalize_zfs }}" + _nullfs: "{{ nullfs | default([]) | normalize_nullfs }}" + - name: "Check if {{ jail_name }} container exists" shell: "zfs list -o name | grep -Fxq 'zroot/jails/containers/{{ jail_name }}'" failed_when: false @@ -13,102 +20,6 @@ when: jail_exists.rc == 0 delegate_to: "{{ jail_delegate_host }}" -# ZFS volume migration -- name: "Check ZFS volume state for {{ jail_name }}" - shell: | - if zfs list {{ item.name }} >/dev/null 2>&1; then - echo "exists" - elif [ -d "/usr/local/jails/containers/{{ jail_name }}{{ item.mountpoint }}" ] && \ - [ "$(ls -A /usr/local/jails/containers/{{ jail_name }}{{ item.mountpoint }})" ]; then - echo "migrate" - else - echo "empty" - fi - loop: "{{ zfs | default([]) }}" - loop_control: - label: "{{ item.name }}" - register: _zfs_volume_state - changed_when: false - delegate_to: "{{ jail_delegate_host }}" - -- name: "*** VOLUME MIGRATION REQUIRED: {{ jail_name }} ***" - pause: - prompt: | - - ════════════════════════════════════════════════════════════════ - VOLUME MIGRATION: {{ jail_name }} - ════════════════════════════════════════════════════════════════ - The following paths contain data that will be migrated - to new ZFS volumes: - {% for result in _zfs_volume_state.results | default([]) if result.stdout == 'migrate' %} - - {{ result.item.mountpoint }} → {{ result.item.name }} - {% endfor %} - - This will: - 1. Create ZFS volumes (staged) - 2. Stop the jail - 3. Copy existing data into the volumes - ════════════════════════════════════════════════════════════════ - - Press Enter to continue or Ctrl+C to abort - when: (_zfs_volume_state.results | default([]) | selectattr('stdout', 'equalto', 'migrate') | list | length) > 0 - -- name: "Create new ZFS volumes (staged) for {{ jail_name }}" - community.general.zfs: - name: "{{ item.item.name }}" - state: present - extra_zfs_properties: - mountpoint: "/usr/local/jails/volumes/.staging/{{ jail_name }}{{ item.item.mountpoint }}" - loop: "{{ _zfs_volume_state.results | default([]) }}" - loop_control: - label: "{{ item.item.name }}" - when: item.stdout == "migrate" - delegate_to: "{{ jail_delegate_host }}" - -- name: "Create staging mount points for {{ jail_name }}" - file: - path: "/usr/local/jails/volumes/.staging/{{ jail_name }}{{ item.item.mountpoint }}" - state: directory - loop: "{{ _zfs_volume_state.results | default([]) }}" - loop_control: - label: "{{ item.item.name }}" - when: item.stdout == "migrate" - delegate_to: "{{ jail_delegate_host }}" - -- name: "Mount staged volumes for {{ jail_name }}" - shell: "zfs mount {{ item.item.name }} || true" - loop: "{{ _zfs_volume_state.results | default([]) }}" - loop_control: - label: "{{ item.item.name }}" - when: item.stdout == "migrate" - changed_when: false - delegate_to: "{{ jail_delegate_host }}" - -- name: "Stop {{ jail_name }} jail for volume migration" - shell: "service jail stop {{ jail_name }} || true" - when: (_zfs_volume_state.results | default([]) | selectattr('stdout', 'equalto', 'migrate') | list | length) > 0 - delegate_to: "{{ jail_delegate_host }}" - -- name: "Copy data into staged volumes for {{ jail_name }}" - shell: | - cp -a "/usr/local/jails/containers/{{ jail_name }}{{ item.item.mountpoint }}/." \ - "/usr/local/jails/volumes/.staging/{{ jail_name }}{{ item.item.mountpoint }}/" - loop: "{{ _zfs_volume_state.results | default([]) }}" - loop_control: - label: "{{ item.item.name }}" - when: item.stdout == "migrate" - delegate_to: "{{ jail_delegate_host }}" - -- name: "Set final mountpoint on migrated volumes for {{ jail_name }}" - shell: | - zfs unmount {{ item.item.name }} || true - zfs set mountpoint="/usr/local/jails/containers/{{ jail_name }}{{ item.item.mountpoint }}" {{ item.item.name }} - loop: "{{ _zfs_volume_state.results | default([]) }}" - loop_control: - label: "{{ item.item.name }}" - when: item.stdout == "migrate" - delegate_to: "{{ jail_delegate_host }}" - # Userland migration - name: "*** USERLAND MIGRATION REQUIRED: {{ jail_name }} ***" pause: @@ -168,28 +79,41 @@ community.general.zfs: name: "{{ item.name }}" state: present - extra_zfs_properties: - mountpoint: "/usr/local/jails/containers/{{ jail_name }}{{ item.mountpoint }}" - loop: "{{ zfs | default([]) }}" + loop: "{{ _zfs }}" loop_control: label: "{{ item.name }}" delegate_to: "{{ jail_delegate_host }}" -- name: "Create mount points for ZFS volumes in {{ jail_name }}" +- name: "Inherit mountpoint on ZFS volumes for {{ jail_name }}" + shell: | + source=$(zfs get -H -o source mountpoint {{ item.name }}) + if [ "$source" != "inherited" ] && [ "$source" != "received" ]; then + zfs unmount {{ item.name }} 2>/dev/null + zfs inherit mountpoint {{ item.name }} + echo "changed" + fi + loop: "{{ _zfs }}" + loop_control: + label: "{{ item.name }}" + register: _zfs_inherit + changed_when: "'changed' in _zfs_inherit.stdout" + delegate_to: "{{ jail_delegate_host }}" + +- name: "Set ownership on ZFS volume host mountpoints for {{ jail_name }}" file: - path: "/usr/local/jails/containers/{{ jail_name }}{{ item.mountpoint }}" + path: "/usr/local/jails/volumes/{{ item.name | basename }}" state: directory owner: "{{ item.owner | default('root') }}" group: "{{ item.group | default('wheel') }}" mode: "{{ item.mode | default('0755') }}" - loop: "{{ zfs | default([]) }}" + loop: "{{ _zfs }}" loop_control: - label: "{{ item.mountpoint }}" + label: "{{ item.name }}" delegate_to: "{{ jail_delegate_host }}" - name: "Mount ZFS volumes for {{ jail_name }}" shell: "zfs mount {{ item.name }} || true" - loop: "{{ zfs | default([]) }}" + loop: "{{ _zfs }}" loop_control: label: "{{ item.name }}" changed_when: false @@ -210,7 +134,7 @@ file: path: "{{ item.src }}" state: directory - loop: "{{ nullfs | default([]) }}" + loop: "{{ _nullfs }}" loop_control: label: "{{ item.src }}" delegate_to: "{{ jail_delegate_host }}" @@ -222,7 +146,7 @@ owner: root group: wheel mode: "0755" - loop: "{{ nullfs | default([]) }}" + loop: "{{ _nullfs }}" loop_control: label: "{{ item.dst }}" delegate_to: "{{ jail_delegate_host }}" @@ -238,17 +162,15 @@ jail: name: "{{ jail_name }}" num: "{{ jail_num }}" - ip: "{{ jail_lan_cidr | ipv4_nth_cidr(jail_num | int + jail_lan_offset | int) }}" devfs_ruleset: "{{ devfs_ruleset | default(4) }}" options: "{{ jail_conf_options | default([]) }}" - default_route: "{{ not no_default_route | default(false) }}" exec_prestart: "{{ exec_prestart | default([]) }}" exec_start: "{{ exec_start | default([]) }}" exec_poststart: "{{ exec_poststart | default([]) }}" exec_prestop: "{{ exec_prestop | default([]) }}" exec_stop: "{{ exec_stop | default([]) }}" exec_poststop: "{{ exec_poststop | default([]) }}" - mounts: "{{ nullfs | default([]) }}" + nullfs: "{{ _nullfs }}" delegate_to: "{{ jail_delegate_host }}" - name: "Start {{ jail_name }} jail" @@ -289,7 +211,7 @@ changed_when: "'Number of packages to be installed' in pkg_result.stdout" - name: Install packages from poudriere - shell: "pkg install -y -r poudriere {{ pkg_custom | join(' ') }}" + shell: "timeout 120 pkg install -y -r poudriere {{ pkg_custom | join(' ') }}" when: pkg_custom is defined and pkg_custom | length > 0 register: pkg_custom_result changed_when: "'Installing' in pkg_custom_result.stdout" diff --git a/roles/jail/templates/jail_conf.j2 b/roles/jail/templates/jail_conf.j2 index 2a0ea27..aa88049 100644 --- a/roles/jail/templates/jail_conf.j2 +++ b/roles/jail/templates/jail_conf.j2 @@ -9,28 +9,19 @@ {% endfor %} devfs_ruleset = {{ jail.devfs_ruleset }}; - host.hostname = "{{ jail.name }}"; + host.hostname = "${name}"; path = "/usr/local/jails/containers/${name}"; exec.start = "/bin/sh /etc/rc"; exec.stop = "/bin/sh /etc/rc.shutdown"; - # LAN epair - exec.prestart += "ifconfig epl{{ jail.num }}a destroy 2>/dev/null || true"; - exec.prestart += "ifconfig epair{{ jail.num }}000 create"; - exec.prestart += "ifconfig epair{{ jail.num }}000a name epl{{ jail.num }}a"; - exec.prestart += "ifconfig epair{{ jail.num }}000b name epl{{ jail.num }}b"; - exec.prestart += "ifconfig epl{{ jail.num }}b ether random"; - exec.prestart += "ifconfig brlan0 addm epl{{ jail.num }}a"; - exec.poststart += "ifconfig epl{{ jail.num }}b vnet ${name}"; - exec.poststart += "ifconfig epl{{ jail.num }}a up"; - exec.poststart += "jexec ${name} ifconfig epl{{ jail.num }}b up"; - exec.poststart += "jexec ${name} ifconfig epl{{ jail.num }}b {{ jail.ip }}"; - exec.poststart += "jexec ${name} route delete default || true"; -{% if jail.default_route %} - exec.poststart += "jexec ${name} route add default {{ ingress_ip }} || true"; -{% endif %} - exec.poststop += "ifconfig epl{{ jail.num }}a destroy 2>/dev/null || true"; + # Networking + exec.prestart += "jail_net prestart {{ jail.num }} ${name}"; + exec.poststart += "jail_net poststart {{ jail.num }} ${name}"; + exec.prestop += "jail_net prestop {{ jail.num }} ${name}"; + exec.poststop += "jail_net poststop {{ jail.num }} ${name}"; + + # Custom hooks {% for cmd in jail.exec_prestart %} exec.prestart += "{{ cmd }}"; {% endfor %} @@ -49,8 +40,9 @@ {% for cmd in jail.exec_poststop %} exec.poststop += "{{ cmd }}"; {% endfor %} -{% for mount in jail.mounts %} - exec.prestart += "mount -t nullfs {{ mount.src }} /usr/local/jails/containers/{{ jail.name }}{{ mount.dst }} || true"; - exec.poststop += "umount /usr/local/jails/containers/{{ jail.name }}{{ mount.dst }} || true"; + + # nullfs mounts +{% for mount in jail.nullfs %} + mount += "{{ mount.src }} ${path}{{ mount.dst }} nullfs {{ mount.mode | default('rw') }} 0 0"; {% endfor %} } diff --git a/roles/jail/templates/usr_local_bin_jail_net.j2 b/roles/jail/templates/usr_local_bin_jail_net.j2 new file mode 100644 index 0000000..e9873a3 --- /dev/null +++ b/roles/jail/templates/usr_local_bin_jail_net.j2 @@ -0,0 +1,127 @@ +#!/bin/sh +# jail_net - manage jail LAN networking lifecycle +# Usage: jail_net <prestart|poststart|prestop|poststop> <jail_num> <jail_name> + +set -e + +STAGE="$1" +NUM="$2" +NAME="$3" + +BRIDGE="brlan0" +LAN_BASE="{{ jail_lan_cidr | ipv4_host }}" +LAN_PREFIX="{{ jail_lan_cidr | ipv4_prefixlen }}" +LAN_OFFSET="{{ jail_lan_offset }}" +DEFAULT_ROUTE="{{ jail_lan_cidr | ipv4_nth(1 + jail_lan_offset | int) }}" + +# Compute jail IP from jail_num + offset +_nth=$(( LAN_OFFSET + NUM )) +_a=$(echo "$LAN_BASE" | cut -d. -f1) +_b=$(echo "$LAN_BASE" | cut -d. -f2) +_c=$(echo "$LAN_BASE" | cut -d. -f3) +_d=$(echo "$LAN_BASE" | cut -d. -f4) +_total=$(( (_a << 24) + (_b << 16) + (_c << 8) + _d + _nth )) +IP="$(( (_total >> 24) & 255 )).$(( (_total >> 16) & 255 )).$(( (_total >> 8) & 255 )).$(( _total & 255 ))/${LAN_PREFIX}" + +# --- Cleanup functions --- + +_cleanup_lan_prestart() { + echo "jail_net: prestart failed for ${NAME}, cleaning up LAN interfaces" >&2 + ifconfig epair${NUM}000a destroy 2>/dev/null || true + ifconfig epl${NUM}a destroy 2>/dev/null || true +} + +_cleanup_wan_prestart() { + echo "jail_net: prestart failed for ${NAME}, cleaning up WAN interfaces" >&2 + ifconfig epair${NUM}001a destroy 2>/dev/null || true + ifconfig epw${NUM}a destroy 2>/dev/null || true +} + +_cleanup_poststart() { + echo "jail_net: poststart failed for ${NAME}" >&2 + ifconfig epl${NUM}a destroy 2>/dev/null || true + if [ "$NUM" -eq 1 ]; then + ifconfig epw${NUM}a destroy 2>/dev/null || true + fi +} + +# --- Stage functions --- + +_prestart() { + trap '_cleanup_lan_prestart' EXIT + + ifconfig epl${NUM}a destroy 2>/dev/null || true + ifconfig epair${NUM}000 create + ifconfig epair${NUM}000a name epl${NUM}a + ifconfig epair${NUM}000b name epl${NUM}b + ifconfig epl${NUM}b ether random + ifconfig ${BRIDGE} addm epl${NUM}a + + if [ "$NUM" -eq 1 ]; then + trap '_cleanup_lan_prestart; _cleanup_wan_prestart' EXIT + ifconfig epw${NUM}a destroy 2>/dev/null || true + ifconfig epair${NUM}001 create + ifconfig epair${NUM}001a name epw${NUM}a + ifconfig epair${NUM}001b name epw${NUM}b +{% if is_prod %} + ifconfig brwan0 addm epw${NUM}a +{% else %} + ifconfig brlan0 addm epw${NUM}a +{% endif %} + fi + + trap - EXIT +} + +_poststart() { + trap '_cleanup_poststart' EXIT + + ifconfig epl${NUM}b vnet ${NAME} + ifconfig epl${NUM}a up + jexec ${NAME} ifconfig epl${NUM}b up + jexec ${NAME} ifconfig epl${NUM}b ${IP} + jexec ${NAME} route delete default || true + if [ "$NUM" -ne 1 ]; then + jexec ${NAME} route add default ${DEFAULT_ROUTE} || true + fi + + if [ "$NUM" -eq 1 ]; then + ifconfig epw${NUM}b vnet ${NAME} + ifconfig epw${NUM}a up + jexec ${NAME} ifconfig epw${NUM}b up +{% if is_prod %} + jexec ${NAME} dhclient epw${NUM}b + jexec ${NAME} route add 10.6.210.0/24 {{ lan_ipv4_gateway }} || true +{% else %} + jexec ${NAME} ifconfig epw${NUM}b inet {{ ingress_wan_static }} + jexec ${NAME} route add default {{ lan_ipv4_gateway }} +{% endif %} + fi + + trap - EXIT +} + +_prestop() { + : +} + +_poststop() { + set +e + ifconfig epl${NUM}a destroy 2>/dev/null + if [ "$NUM" -eq 1 ]; then + ifconfig epw${NUM}a destroy 2>/dev/null + fi +} + +# --- Dispatch --- + +case "$STAGE" in + prestart) _prestart ;; + poststart) _poststart ;; + prestop) _prestop ;; + poststop) _poststop ;; + *) + echo "Usage: jail_net <prestart|poststart|prestop|poststop> <num> <name>" >&2 + exit 1 + ;; +esac diff --git a/roles/jails/01_ingress/defaults/main.yml b/roles/jails/01_ingress/defaults/main.yml index f5e848b..37840eb 100644 --- a/roles/jails/01_ingress/defaults/main.yml +++ b/roles/jails/01_ingress/defaults/main.yml @@ -1,12 +1,15 @@ userland: "15.1-RELEASE" devfs_ruleset: 6 -no_default_route: true zfs: - - name: zroot/jails/volumes/ingress_letsencrypt - mountpoint: /usr/local/etc/letsencrypt - - name: zroot/jails/volumes/ingress_logs - mountpoint: /var/log/nginx + - name: ingress_letsencrypt + - name: ingress_logs + +nullfs: + - src: ingress_letsencrypt + dst: /usr/local/etc/letsencrypt + - src: ingress_logs + dst: /var/log/nginx ingress_routes: - { host: jan.systems, jail: homepage } @@ -37,8 +40,6 @@ contact_email: jan@jantuomi.fi jail_conf_options: - "allow.raw_sockets" -nullfs: [] - pkg: - pcre2 - nginx diff --git a/roles/jails/01_ingress/tasks/main.yml b/roles/jails/01_ingress/tasks/main.yml index 6c8a30b..98b8d13 100644 --- a/roles/jails/01_ingress/tasks/main.yml +++ b/roles/jails/01_ingress/tasks/main.yml @@ -1,39 +1,3 @@ -- name: Set WAN hooks (test) - set_fact: - exec_prestart: - - "ifconfig epw{{ jail_num }}a destroy 2>/dev/null || true" - - "ifconfig epair{{ jail_num }}001 create" - - "ifconfig epair{{ jail_num }}001a name epw{{ jail_num }}a" - - "ifconfig epair{{ jail_num }}001b name epw{{ jail_num }}b" - - "ifconfig brlan0 addm epw{{ jail_num }}a" - exec_poststart: - - "ifconfig epw{{ jail_num }}b vnet {{ jail_name }}" - - "ifconfig epw{{ jail_num }}a up" - - "jexec {{ jail_name }} ifconfig epw{{ jail_num }}b up" - - "jexec {{ jail_name }} ifconfig epw{{ jail_num }}b inet {{ ingress_wan_static }}" - - "jexec {{ jail_name }} route add default {{ lan_ipv4_gateway }}" - exec_poststop: - - "ifconfig epw{{ jail_num }}a destroy 2>/dev/null || true" - when: not is_prod - -- name: Set WAN hooks (prod) - set_fact: - exec_prestart: - - "ifconfig epw{{ jail_num }}a destroy 2>/dev/null || true" - - "ifconfig epair{{ jail_num }}001 create" - - "ifconfig epair{{ jail_num }}001a name epw{{ jail_num }}a" - - "ifconfig epair{{ jail_num }}001b name epw{{ jail_num }}b" - - "ifconfig brwan0 addm epw{{ jail_num }}a" - exec_poststart: - - "ifconfig epw{{ jail_num }}b vnet {{ jail_name }}" - - "ifconfig epw{{ jail_num }}a up" - - "jexec {{ jail_name }} ifconfig epw{{ jail_num }}b up" - - "jexec {{ jail_name }} dhclient epw{{ jail_num }}b" - - "jexec {{ jail_name }} route add 10.6.210.0/24 {{ lan_ipv4_gateway }} || true" - exec_poststop: - - "ifconfig epw{{ jail_num }}a destroy 2>/dev/null || true" - when: is_prod - - import_role: name: jail tasks_from: jail_setup diff --git a/roles/jails/02_postgres/defaults/main.yml b/roles/jails/02_postgres/defaults/main.yml index 77097b0..47c0e5d 100644 --- a/roles/jails/02_postgres/defaults/main.yml +++ b/roles/jails/02_postgres/defaults/main.yml @@ -2,12 +2,15 @@ userland: "15.1-RELEASE" devfs_ruleset: 5 zfs: - - name: zroot/jails/volumes/postgres_data - mountpoint: /var/db/postgres + - name: postgres_data owner: "770" group: "770" mode: "0755" +nullfs: + - src: postgres_data + dst: /var/db/postgres + jail_conf_options: - "allow.raw_sockets" - "allow.sysvipc" diff --git a/roles/jails/03_irc_thelounge/defaults/main.yml b/roles/jails/03_irc_thelounge/defaults/main.yml index cc29940..e9bfef2 100644 --- a/roles/jails/03_irc_thelounge/defaults/main.yml +++ b/roles/jails/03_irc_thelounge/defaults/main.yml @@ -1,8 +1,11 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/irc_thelounge - mountpoint: /var/db/thelounge + - name: irc_thelounge + +nullfs: + - src: irc_thelounge + dst: /var/db/thelounge pkg_custom: - thelounge diff --git a/roles/jails/04_taulubot/defaults/main.yml b/roles/jails/04_taulubot/defaults/main.yml index 1c3f946..7bbb32d 100644 --- a/roles/jails/04_taulubot/defaults/main.yml +++ b/roles/jails/04_taulubot/defaults/main.yml @@ -1,8 +1,11 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/taulubot - mountpoint: /usr/local/taulubot/photos + - name: taulubot + +nullfs: + - src: taulubot + dst: /usr/local/taulubot/photos pkg_custom: - taulubot diff --git a/roles/jails/05_homepage/defaults/main.yml b/roles/jails/05_homepage/defaults/main.yml index 99c74e5..cb53bd7 100644 --- a/roles/jails/05_homepage/defaults/main.yml +++ b/roles/jails/05_homepage/defaults/main.yml @@ -1,17 +1,20 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/homepage_www - mountpoint: /var/www - - name: zroot/jails/volumes/homepage_agate_tls - mountpoint: /usr/local/etc/agate/tls - owner: "80" + - name: homepage_www + - name: homepage_agate_tls + owner: "80" # www group: "80" - - name: zroot/jails/volumes/homepage_gemini - mountpoint: /var/gemini + - name: homepage_gemini nullfs: - - src: /usr/local/jails/containers/ingress/usr/local/etc/letsencrypt + - src: homepage_www + dst: /var/www + - src: homepage_agate_tls + dst: /usr/local/etc/agate/tls + - src: homepage_gemini + dst: /var/gemini + - src: ingress_letsencrypt dst: /mnt/letsencrypt mode: ro diff --git a/roles/jails/06_hommabot/defaults/main.yml b/roles/jails/06_hommabot/defaults/main.yml index 997ee86..7e9ffb4 100644 --- a/roles/jails/06_hommabot/defaults/main.yml +++ b/roles/jails/06_hommabot/defaults/main.yml @@ -1,8 +1,11 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/hommabot_data - mountpoint: /var/db/hommabot + - name: hommabot_data + +nullfs: + - src: hommabot_data + dst: /var/db/hommabot pkg_custom: - hommabot diff --git a/roles/jails/07_aggro/defaults/main.yml b/roles/jails/07_aggro/defaults/main.yml index 0077cf3..2904331 100644 --- a/roles/jails/07_aggro/defaults/main.yml +++ b/roles/jails/07_aggro/defaults/main.yml @@ -1,8 +1,11 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/aggro_data - mountpoint: /var/db/aggro + - name: aggro_data + +nullfs: + - src: aggro_data + dst: /var/db/aggro pkg_custom: - aggro diff --git a/roles/jails/08_diddle/defaults/main.yml b/roles/jails/08_diddle/defaults/main.yml index 090340b..7b1b0e6 100644 --- a/roles/jails/08_diddle/defaults/main.yml +++ b/roles/jails/08_diddle/defaults/main.yml @@ -1,8 +1,11 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/diddle_data - mountpoint: /var/db/diddle + - name: diddle_data + +nullfs: + - src: diddle_data + dst: /var/db/diddle pkg_custom: - diddle diff --git a/roles/jails/09_redis/defaults/main.yml b/roles/jails/09_redis/defaults/main.yml index 75df5f8..1acc74a 100644 --- a/roles/jails/09_redis/defaults/main.yml +++ b/roles/jails/09_redis/defaults/main.yml @@ -1,12 +1,15 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/valkey_data - mountpoint: /var/db/valkey + - name: valkey_data owner: "537" group: "537" mode: "0755" +nullfs: + - src: valkey_data + dst: /var/db/valkey + pkg: - valkey diff --git a/roles/jails/10_samba/defaults/main.yml b/roles/jails/10_samba/defaults/main.yml index 4416022..9900117 100644 --- a/roles/jails/10_samba/defaults/main.yml +++ b/roles/jails/10_samba/defaults/main.yml @@ -1,10 +1,15 @@ userland: "15.1-RELEASE" +jail_conf_options: + - "allow.mount" + - "allow.mount.fdescfs" + - "enforce_statfs = 1" + pkg: - samba419 nullfs: - - src: /usr/local/jails/volumes/storage + - src: storage dst: /mnt/storage services: diff --git a/roles/jails/11_spliit/defaults/main.yml b/roles/jails/11_spliit/defaults/main.yml index 7ce40cd..77a318e 100644 --- a/roles/jails/11_spliit/defaults/main.yml +++ b/roles/jails/11_spliit/defaults/main.yml @@ -1,8 +1,11 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/spliit_data - mountpoint: /var/db/expense-tracker + - name: spliit_data + +nullfs: + - src: spliit_data + dst: /var/db/expense-tracker pkg_custom: - expense-tracker diff --git a/roles/jails/12_goaccess/defaults/main.yml b/roles/jails/12_goaccess/defaults/main.yml index 98a2e43..5d9ef54 100644 --- a/roles/jails/12_goaccess/defaults/main.yml +++ b/roles/jails/12_goaccess/defaults/main.yml @@ -1,11 +1,12 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/goaccess_data - mountpoint: /var/db/goaccess + - name: goaccess_data nullfs: - - src: /usr/local/jails/containers/ingress/var/log/nginx + - src: goaccess_data + dst: /var/db/goaccess + - src: ingress_logs dst: /mnt/nginx_logs pkg: diff --git a/roles/jails/13_plex/defaults/main.yml b/roles/jails/13_plex/defaults/main.yml index 4ae644c..1a9f31b 100644 --- a/roles/jails/13_plex/defaults/main.yml +++ b/roles/jails/13_plex/defaults/main.yml @@ -1,14 +1,15 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/plex_data - mountpoint: /usr/local/plexdata + - name: plex_data owner: 972 group: 972 mode: 755 nullfs: - - src: /usr/local/jails/volumes/storage/media + - src: plex_data + dst: /usr/local/plexdata + - src: storage/media dst: /mnt/media pkg_custom: diff --git a/roles/jails/14_freshrss/defaults/main.yml b/roles/jails/14_freshrss/defaults/main.yml index e421c88..2de31c7 100644 --- a/roles/jails/14_freshrss/defaults/main.yml +++ b/roles/jails/14_freshrss/defaults/main.yml @@ -1,11 +1,14 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/freshrss_data - mountpoint: /var/db/freshrss + - name: freshrss_data owner: "80" group: "80" +nullfs: + - src: freshrss_data + dst: /var/db/freshrss + pkg: - nginx diff --git a/roles/jails/15_paste/defaults/main.yml b/roles/jails/15_paste/defaults/main.yml index d32f815..a2da2cd 100644 --- a/roles/jails/15_paste/defaults/main.yml +++ b/roles/jails/15_paste/defaults/main.yml @@ -1,8 +1,11 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/paste_data - mountpoint: /var/db/pastebin + - name: paste_data + +nullfs: + - src: paste_data + dst: /var/db/pastebin pkg_custom: - pastebin diff --git a/roles/jails/16_dl/defaults/main.yml b/roles/jails/16_dl/defaults/main.yml index c6ae55d..fef93a1 100644 --- a/roles/jails/16_dl/defaults/main.yml +++ b/roles/jails/16_dl/defaults/main.yml @@ -5,28 +5,32 @@ jail_conf_options: - "allow.raw_sockets" - "allow.mlock" -nullfs: - - src: /usr/local/jails/volumes/storage - dst: /mnt/storage - zfs: - - name: zroot/jails/volumes/dl_sonarr - mountpoint: /usr/local/sonarr + - name: dl_sonarr owner: 351 group: 351 - - name: zroot/jails/volumes/dl_radarr - mountpoint: /usr/local/radarr + - name: dl_radarr owner: 352 group: 352 - - name: zroot/jails/volumes/dl_lidarr - mountpoint: /usr/local/lidarr + - name: dl_lidarr owner: 356 group: 356 - - name: zroot/jails/volumes/dl_prowlarr - mountpoint: /usr/local/prowlarr + - name: dl_prowlarr owner: 359 group: 359 +nullfs: + - src: storage + dst: /mnt/storage + - src: dl_sonarr + dst: /usr/local/sonarr + - src: dl_radarr + dst: /usr/local/radarr + - src: dl_lidarr + dst: /usr/local/lidarr + - src: dl_prowlarr + dst: /usr/local/prowlarr + pkg: - transmission-daemon - transmission-web diff --git a/roles/jails/17_syncthing/defaults/main.yml b/roles/jails/17_syncthing/defaults/main.yml index b60c952..9e1f3f1 100644 --- a/roles/jails/17_syncthing/defaults/main.yml +++ b/roles/jails/17_syncthing/defaults/main.yml @@ -1,21 +1,24 @@ userland: "15.1-RELEASE" nullfs: - - src: /usr/local/jails/volumes/storage/docs + - src: storage/docs dst: /mnt/docs - - src: /usr/local/jails/volumes/storage/vault + - src: storage/vault dst: /mnt/vault - - src: /usr/local/jails/volumes/storage/jan-systems-2025-content + - src: storage/jan-systems-2025-content dst: /mnt/jan-systems-2025-content - - src: /usr/local/jails/volumes/storage/projects-ableton + - src: storage/projects-ableton dst: /mnt/projects-ableton zfs: - - name: zroot/jails/volumes/syncthing_config - mountpoint: /usr/local/etc/syncthing + - name: syncthing_config owner: "1001" group: "1001" +nullfs: + - src: syncthing_config + dst: /usr/local/etc/syncthing + pkg: - syncthing diff --git a/roles/jails/18_komga/defaults/main.yml b/roles/jails/18_komga/defaults/main.yml index 3010e4f..39b958d 100644 --- a/roles/jails/18_komga/defaults/main.yml +++ b/roles/jails/18_komga/defaults/main.yml @@ -1,11 +1,12 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/komga_data - mountpoint: /var/db/komga + - name: komga_data nullfs: - - src: /usr/local/jails/volumes/storage/media/manga + - src: komga_data + dst: /var/db/komga + - src: storage/media/manga dst: /mnt/manga pkg: diff --git a/roles/jails/19_leolalla_fi/defaults/main.yml b/roles/jails/19_leolalla_fi/defaults/main.yml index a9157e2..0e1b449 100644 --- a/roles/jails/19_leolalla_fi/defaults/main.yml +++ b/roles/jails/19_leolalla_fi/defaults/main.yml @@ -1,8 +1,11 @@ userland: "15.1-RELEASE" zfs: - - name: zroot/jails/volumes/leolalla_fi_data - mountpoint: /var/db/gdrive-photo-gallery + - name: leolalla_fi_data + +nullfs: + - src: leolalla_fi_data + dst: /var/db/gdrive-photo-gallery pkg_custom: - gdrive-photo-gallery diff --git a/roles/jails/19_leolalla_fi/tasks/main.yml b/roles/jails/19_leolalla_fi/tasks/main.yml index 021b2cf..6cb6177 100644 --- a/roles/jails/19_leolalla_fi/tasks/main.yml +++ b/roles/jails/19_leolalla_fi/tasks/main.yml @@ -11,14 +11,6 @@ mode: "0600" no_log: true -- name: Deploy wrapper script - template: - src: "{{ jail_role_dir }}/templates/usr_local_bin_gdrive_photo_gallery" - dest: /usr/local/bin/gdrive-photo-gallery - owner: root - group: wheel - mode: "0755" - - import_role: name: jail tasks_from: jail_launch diff --git a/roles/jails/20_immich/defaults/main.yml b/roles/jails/20_immich/defaults/main.yml index b3347d3..6cb9390 100644 --- a/roles/jails/20_immich/defaults/main.yml +++ b/roles/jails/20_immich/defaults/main.yml @@ -8,22 +8,22 @@ oci_images: - name: immich-server image: "ghcr.io/immich-app/immich-server:{{ immich_version }}" check_path: /usr/src/app/server/dist/main.js - mounts: - - { type: nullfs, src: /immich_data, dst: /data } - name: immich-ml image: "ghcr.io/immich-app/immich-machine-learning:{{ immich_version }}" check_path: /opt/venv/bin/python - mounts: - - { type: nullfs, src: /model_cache, dst: /cache } zfs: - - name: zroot/jails/volumes/immich_data - mountpoint: /immich_data + - name: immich_data mode: "0755" - - name: zroot/jails/volumes/immich_model_cache - mountpoint: /model_cache + - name: immich_model_cache mode: "0755" +nullfs: + - src: immich_data + dst: /image/immich-server/data + - src: immich_model_cache + dst: /image/immich-server/cache + pkg: - skopeo diff --git a/roles/jails/20_immich/tasks/main.yml b/roles/jails/20_immich/tasks/main.yml index 835cd50..39decd1 100644 --- a/roles/jails/20_immich/tasks/main.yml +++ b/roles/jails/20_immich/tasks/main.yml @@ -46,28 +46,29 @@ label: "{{ item.item.name }}" when: not item.stat.exists -# Create mount points in each rootfs -- name: Create base mount points in image rootfs +- name: Create volume mount points in image rootfs file: - path: "/image/{{ item[0].name }}/{{ item[1] }}" + path: "/image/{{ item[0].name }}{{ item[1].dst }}" state: directory owner: root group: wheel mode: "0755" - loop: "{{ oci_images | product(['proc', 'sys', 'dev', 'tmp', 'run']) | list }}" + loop: "{{ oci_images | product(nullfs | default([]) | normalize_nullfs) | list }}" loop_control: - label: "{{ item[0].name }}/{{ item[1] }}" + label: "{{ item[0].name }}:{{ item[1].dst }}" + when: "item[1].dst.startswith('/image/' + item[0].name)" -- name: Create nullfs mount points in image rootfs +# Create mount points in each rootfs +- name: Create base mount points in image rootfs file: - path: "/image/{{ item.0.name }}{{ item.1.dst }}" + path: "/image/{{ item[0].name }}/{{ item[1] }}" state: directory owner: root group: wheel mode: "0755" - loop: "{{ oci_images | subelements('mounts', skip_missing=True) }}" + loop: "{{ oci_images | product(['proc', 'sys', 'dev', 'tmp', 'run']) | list }}" loop_control: - label: "{{ item.0.name }}:{{ item.1.dst }}" + label: "{{ item[0].name }}/{{ item[1] }}" # Deploy resolv.conf into each rootfs - name: Deploy resolv.conf into image rootfs diff --git a/roles/jails/20_immich/templates/jail.conf.j2 b/roles/jails/20_immich/templates/jail.conf.j2 index 13680af..8e3c626 100644 --- a/roles/jails/20_immich/templates/jail.conf.j2 +++ b/roles/jails/20_immich/templates/jail.conf.j2 @@ -13,26 +13,22 @@ immich { enforce_statfs = 1; devfs_ruleset = 4; - host.hostname = "immich"; - path = "/usr/local/jails/containers/immich"; + host.hostname = "${name}"; + path = "/usr/local/jails/containers/${name}"; exec.start = "/bin/sh /etc/rc"; exec.stop = "/bin/sh /etc/rc.shutdown"; - # LAN epair - exec.prestart += "ifconfig epl{{ jail_num }}a destroy 2>/dev/null || true"; - exec.prestart += "ifconfig epair{{ jail_num }}000 create"; - exec.prestart += "ifconfig epair{{ jail_num }}000a name epl{{ jail_num }}a"; - exec.prestart += "ifconfig epair{{ jail_num }}000b name epl{{ jail_num }}b"; - exec.prestart += "ifconfig epl{{ jail_num }}b ether random"; - exec.prestart += "ifconfig brlan0 addm epl{{ jail_num }}a"; - exec.poststart += "ifconfig epl{{ jail_num }}b vnet immich"; - exec.poststart += "ifconfig epl{{ jail_num }}a up"; - exec.poststart += "jexec immich ifconfig epl{{ jail_num }}b up"; - exec.poststart += "jexec immich ifconfig epl{{ jail_num }}b {{ jail_lan_cidr | ipv4_nth_cidr(jail_num | int + jail_lan_offset | int) }}"; - exec.poststart += "jexec immich route delete default || true"; - exec.poststart += "jexec immich route add default {{ ingress_ip }} || true"; - exec.poststop += "ifconfig epl{{ jail_num }}a destroy 2>/dev/null || true"; + # Networking + exec.prestart += "jail_net prestart {{ jail_num }} ${name}"; + exec.poststart += "jail_net poststart {{ jail_num }} ${name}"; + exec.prestop += "jail_net prestop {{ jail_num }} ${name}"; + exec.poststop += "jail_net poststop {{ jail_num }} ${name}"; + + # nullfs mounts +{% for mount in nullfs | default([]) | normalize_nullfs %} + mount += "{{ mount.src }} ${path}{{ mount.dst }} nullfs {{ mount.mode | default('rw') }} 0 0"; +{% endfor %} {% for img in oci_images %} # Mounts for {{ img.name }} rootfs @@ -41,8 +37,5 @@ immich { mount += "devfs $path/image/{{ img.name }}/dev devfs rw 0 0"; mount += "tmpfs $path/image/{{ img.name }}/tmp tmpfs rw 0 0"; mount += "tmpfs $path/image/{{ img.name }}/run tmpfs rw 0 0"; -{% for mnt in img.mounts | default([]) %} - mount += "$path{{ mnt.src }} $path/image/{{ img.name }}{{ mnt.dst }} nullfs rw 0 0"; -{% endfor %} {% endfor %} } |