From cc1b02f99e457d2a5a440698e9bae1ca30f408e1 Mon Sep 17 00:00:00 2001
From: Jan Tuomi <jan@jantuomi.fi>
Date: Fri, 25 Jul 2025 00:09:01 +0300
Subject: Initial impl of vnet jails

---
 playbook.yml                                       | 25 +++++++--
 tasks/general.yml                                  |  2 -
 tasks/jail_ingress.yml                             | 64 ++++++++++++++++++++++
 tasks/jails.yml                                    | 20 +++++--
 tasks/network.yml                                  | 49 +----------------
 tasks/pkg_jail_install.yml                         |  9 +++
 templates/etc_devd_lan0_jail_restore.conf.j2       |  6 --
 templates/etc_jail.conf.j2                         | 23 ++++++--
 templates/etc_pf.conf.j2                           | 10 ++++
 templates/etc_rc.conf.j2                           |  2 +
 templates/usr_local_bin_restore_jail_aliases.sh.j2 | 27 ---------
 templates/usr_local_etc_nginx_nginx.conf.j2        |  6 +-
 12 files changed, 139 insertions(+), 104 deletions(-)
 create mode 100644 tasks/jail_ingress.yml
 create mode 100644 tasks/pkg_jail_install.yml
 delete mode 100644 templates/etc_devd_lan0_jail_restore.conf.j2
 delete mode 100644 templates/usr_local_bin_restore_jail_aliases.sh.j2

diff --git a/playbook.yml b/playbook.yml
index 975d792..743cba8 100644
--- a/playbook.yml
+++ b/playbook.yml
@@ -19,19 +19,32 @@
     smtp_port: "{{ lookup('env', 'SMTP_PORT') }}"
     dma_mail_hostname: "{{ lookup('env', 'DMA_MAILNAME') }}"
     dma_to_address: "{{ lookup('env', 'DMA_TO_ADDRESS') }}"
-    jan_systems_html_dir: "{{ lookup('env', 'JAN_SYSTEMS_HTML_DIR') }}"
     do_api_token: "{{ lookup('env', 'DO_API_TOKEN') }}"
     lan_ipv4_cidr: 192.168.0.10/16
     lan_ipv4_gateway: 192.168.0.1
     lan_search_domain: local.jan.systems
     jail_userland_version: 14.3-RELEASE
+    jail_ingress_ip: "192.168.2.1"
     jails:
-      - { name: nginx_test, ip: "192.168.2.1" }
-      - { name: postgres, ip: "192.168.2.2" }
+      - { name: ingress, num: 1 }
+      - { name: postgres, num: 2 }
+      - { name: irc_thelounge, num: 3 }
+      - { name: taulubot, num: 4 }
+      - { name: veetibot, num: 5 }
+      - { name: hommabot, num: 6 }
+      - { name: aggro, num: 7 }
+      - { name: diddle, num: 8 }
+      - { name: gallery_sakari, num: 9 }
+      - { name: gallery_leo, num: 10 }
+      - { name: spliit, num: 11 }
+      - { name: stirling-pdf, num: 12 }
+      - { name: ente, num: 13 }
     static_sites:
-      - { site: "jan.systems", host: "jan.systems", tls: false }
-      - { site: "jan.systems", host: "pursotin.jan.systems", tls: true }
-      - { site: "jan.systems", host: "local.jan.systems", tls: false }
+      - { site: "homepage", host: "jan.systems", tls: false }
+      - { site: "homepage", host: "pursotin.jan.systems", tls: true }
+      - { site: "homepage", host: "local.jan.systems", tls: false }
+    static_site_dirs:
+      homepage: "{{ lookup('env', 'JAN_SYSTEMS_HTML_DIR') }}"
 
   tasks:
     - name: Run general tasks
diff --git a/tasks/general.yml b/tasks/general.yml
index 3c55117..9361217 100644
--- a/tasks/general.yml
+++ b/tasks/general.yml
@@ -4,11 +4,9 @@
     state: present
   loop:
     - rsync
-    - nginx
     - dma
     - jq
     - curl
-    - py311-certbot
 
 - name: Set up periodic.conf
   template:
diff --git a/tasks/jail_ingress.yml b/tasks/jail_ingress.yml
new file mode 100644
index 0000000..1a87722
--- /dev/null
+++ b/tasks/jail_ingress.yml
@@ -0,0 +1,64 @@
+- name: Install packages inside ingress jail
+  loop:
+    - { jail: ingress, package: nginx }
+    - { jail: ingress, package: py311-certbot }
+  include_tasks: pkg_jail_install.yml
+
+- name: Configure nginx.conf
+  template:
+    src: usr_local_etc_nginx_nginx.conf.j2
+    dest: /usr/local/jails/containers/ingress/usr/local/etc/nginx/nginx.conf
+    owner: root
+    group: wheel
+    mode: "0644"
+  register: nginx_conf
+
+- name: Create static sites log dir
+  file:
+    path: /usr/local/jails/containers/ingress/var/log/nginx/{{ item }}
+    state: directory
+    owner: root
+    group: wheel
+    mode: "0755"
+  loop: "{{ static_sites | map(attribute='site') | unique }}"
+
+- name: Create access.log for static sites
+  file:
+    path: /usr/local/jails/containers/ingress/var/log/nginx/{{ item }}/access.log
+    state: touch
+    owner: root
+    group: wheel
+    mode: "0644"
+  loop: "{{ static_sites | map(attribute='site') | unique }}"
+
+- name: Create error.log for static sites
+  file:
+    path: /usr/local/jails/containers/ingress/var/log/nginx/{{ item }}/error.log
+    state: touch
+    owner: root
+    group: wheel
+    mode: "0644"
+  loop: "{{ static_sites | map(attribute='site') | unique }}"
+
+#- name: Generate LetsEncrypt certs
+#  shell: jexec ingress certbot certonly --standalone -n -m jan@jantuomi.fi --agree-tos -d {{ item }}
+#  loop: "{{ static_sites | selectattr('tls') | map(attribute='host') }}"
+#  when: nginx_conf.changed
+
+- name: Copy over static HTML
+  ansible.posix.synchronize:
+    src: "{{ item.value.rstrip('/') + '/' }}"
+    dest: /usr/local/jails/containers/ingress/usr/local/www/{{ item.key }}
+  loop: "{{ static_site_dirs | dict2items }}"
+
+- name: Check if nginx is enabled
+  shell: service -j ingress nginx status
+  changed_when: false
+  failed_when: false
+  register: ingress_nginx_enabled
+
+- name: Enable and start nginx
+  shell: |
+    service -j ingress nginx enable
+    service -j ingress nginx onestart
+  when: ingress_nginx_enabled.rc != 0
diff --git a/tasks/jails.yml b/tasks/jails.yml
index 8233dc5..7cbe707 100644
--- a/tasks/jails.yml
+++ b/tasks/jails.yml
@@ -37,18 +37,23 @@
     - name: Unarchive userland
       shell: tar -xzf /usr/local/jails/media/{{ jail_userland_version }}-base.txz -C /usr/local/jails/templates/{{ jail_userland_version }}
 
-    - name: Copy localtime and resolv.conf to jail userland
+    - name: Copy localtime to jail userland
       copy:
         remote_src: true
-        src: "{{ item }}"
-        dest: "/usr/local/jails/templates/{{ jail_userland_version }}{{ item }}"
-      loop:
-        - /etc/resolv.conf
-        - /etc/localtime
+        src: /etc/localtime
+        dest: /usr/local/jails/templates/{{ jail_userland_version }}/etc/localtime
+
+    - name: Copy resolv.conf to jail userland
+      shell: |
+        cat /etc/resolv.conf > /usr/local/jails/templates/{{ jail_userland_version }}/etc/resolv.conf
+        chmod 644 /usr/local/jails/templates/{{ jail_userland_version }}/etc/resolv.conf
 
     - name: Update userland to latest patch level
       shell: freebsd-update -b /usr/local/jails/templates/{{ jail_userland_version }}/ fetch install
 
+    - name: Disable syslogd in the template
+      shell: echo 'syslogd_enable="NO"' >> /usr/local/jails/templates/{{ jail_userland_version }}/etc/rc.conf
+
     - name: Create userland ZFS snapshot
       shell: zfs snapshot zroot/jails/templates/{{ jail_userland_version }}@base
 
@@ -67,3 +72,6 @@
       loop_control:
         loop_var: jail
       include_tasks: jail_single.yml
+
+- name: Configure ingress jail
+  include_tasks: jail_ingress.yml
diff --git a/tasks/network.yml b/tasks/network.yml
index ed57a13..a8749a7 100644
--- a/tasks/network.yml
+++ b/tasks/network.yml
@@ -12,7 +12,7 @@
   when: resolv_conf.changed
 
 - name: Restart networking if interface configuration changed
-  shell: service netif restart && service routing restart
+  shell: service netif restart && service routing restart && service dhclient restart wan0
   when: rc_conf.changed or resolv_conf.changed
 
 - name: Set up sshd
@@ -67,53 +67,6 @@
   shell: pfctl -f /etc/pf.conf
   when: pf_conf.changed
 
-- name: Configure nginx.conf
-  template:
-    src: usr_local_etc_nginx_nginx.conf.j2
-    dest: /usr/local/etc/nginx/nginx.conf
-    owner: root
-    group: wheel
-    mode: "0644"
-  register: nginx_conf
-
-- name: Generate LetsEncrypt certs
-  shell: certbot certonly --standalone -n -m jan@jantuomi.fi --agree-tos -d {{ item }}
-  loop: "{{ static_sites | selectattr('tls') | map(attribute='host') }}"
-  when: nginx_conf.changed
-
-- name: Copy over jan.systems HTML
-  ansible.posix.synchronize:
-    src: "{{ jan_systems_html_dir.rstrip('/') + '/' }}"
-    dest: /usr/local/www/jan.systems
-
-- name: Start nginx
-  service:
-    name: nginx
-    state: started
-  register: started_nginx
-
-- name: Restart nginx
-  service:
-    name: nginx
-    state: restarted
-  when: not started_nginx.changed and nginx_conf.changed
-
-- name: Set up restore_jail_aliases.sh
-  template:
-    src: usr_local_bin_restore_jail_aliases.sh.j2
-    dest: /usr/local/bin/restore_jail_aliases.sh
-    owner: root
-    group: wheel
-    mode: "0755"
-
-- name: Set up devd lan0_jail_restore.conf
-  template:
-    src: etc_devd_lan0_jail_restore.conf.j2
-    dest: /etc/devd/lan0_jail_restore.conf
-    owner: root
-    group: wheel
-    mode: "0644"
-
 - name: Set up /usr/local/etc/do_dyndns_auth
   template:
     src: usr_local_etc_do_dyndns_auth.j2
diff --git a/tasks/pkg_jail_install.yml b/tasks/pkg_jail_install.yml
new file mode 100644
index 0000000..0315a03
--- /dev/null
+++ b/tasks/pkg_jail_install.yml
@@ -0,0 +1,9 @@
+- name: Check if {{ item.package }} is installed
+  shell: pkg -j {{ item.jail }} info {{ item.package }}
+  register: pkg_jail_installed
+  changed_when: false
+  failed_when: false
+
+- name: Install {{ item.package }}
+  shell: pkg -j {{ item.jail }} install -y {{ item.package }}
+  when: pkg_jail_installed.rc != 0
diff --git a/templates/etc_devd_lan0_jail_restore.conf.j2 b/templates/etc_devd_lan0_jail_restore.conf.j2
deleted file mode 100644
index c59b428..0000000
--- a/templates/etc_devd_lan0_jail_restore.conf.j2
+++ /dev/null
@@ -1,6 +0,0 @@
-notify 10 {
-    match "system"          "IFNET";
-    match "subsystem"       "lan0";
-    match "type"            "LINK_UP";
-    action "/usr/local/bin/restore_jail_aliases.sh";
-};
diff --git a/templates/etc_jail.conf.j2 b/templates/etc_jail.conf.j2
index bcfcef0..86da8de 100644
--- a/templates/etc_jail.conf.j2
+++ b/templates/etc_jail.conf.j2
@@ -1,7 +1,19 @@
-# STARTUP/LOGGING
-exec.start = "/bin/sh /etc/rc";
-exec.stop = "/bin/sh /etc/rc.shutdown";
-exec.consolelog = "/var/log/jail_console_${name}.log";
+# STARTUP/LOGGING/VNET
+$epair            = "epair${num}"; # epairXa is host end, epairXb is jail end
+vnet;
+vnet.interface    = "${epair}b";
+persist;
+exec.prestart    += "ifconfig $epair create || echo 'Failed to create $epair'";
+exec.prestart    += "ifconfig ${epair}a up";
+exec.prestart    += "ifconfig bridge0 addm ${epair}a";
+exec.clean;
+exec.start        = "/bin/sh /etc/rc";
+exec.poststart    = "jexec ${name} ifconfig ${epair}b 192.168.2.${num}/16";
+exec.poststart   += "jexec ${name} route add default 192.168.0.1 || echo 'Failed to add default route'";
+exec.stop         = "/bin/sh /etc/rc.shutdown";
+exec.poststop    += "ifconfig ${epair}b -vnet $name"; # workaround to bug 238326: move ${epair}b from the jail to the host when stopping jail services
+exec.poststop    += "ifconfig ${epair}b destroy";     # and then destroy the pair by destroying one end of it
+exec.consolelog   = "/var/log/jail_console_${name}.log";
 
 # PERMISSIONS
 allow.raw_sockets;
@@ -15,8 +27,7 @@ path = "/usr/local/jails/containers/${name}";
 # JAILS
 {% for jail in jails %}
 {{ jail.name }} {
-  ip4.addr = {{ jail.ip }};
-  interface = lan0;
+  $num           = "{{ jail.num }}";
 }
 
 {% endfor %}
diff --git a/templates/etc_pf.conf.j2 b/templates/etc_pf.conf.j2
index 0b5a756..0bb7fef 100644
--- a/templates/etc_pf.conf.j2
+++ b/templates/etc_pf.conf.j2
@@ -1,10 +1,20 @@
 lan_if = "lan0"
 wan_if = "wan0"
+jail_ingress_ip = "{{ jail_ingress_ip }}"
 
 # Default policy
 set skip on lo
 set block-policy return
 scrub in
+
+# Redirect HTTP to ingress jail
+# TODO: fix these redirects, they seem to not work
+rdr on $wan_if proto tcp from any to ($wan_if) port 80 -> $jail_ingress_ip port 80
+rdr on $wan_if proto tcp from any to ($wan_if) port 443 -> $jail_ingress_ip port 443
+rdr on $lan_if proto tcp from any to ($lan_if) port 80 -> $jail_ingress_ip port 80
+rdr on $lan_if proto tcp from any to ($lan_if) port 443 -> $jail_ingress_ip port 443
+
+# Default block all
 block in all
 
 # Allow all outbound traffic
diff --git a/templates/etc_rc.conf.j2 b/templates/etc_rc.conf.j2
index 641125b..195e599 100644
--- a/templates/etc_rc.conf.j2
+++ b/templates/etc_rc.conf.j2
@@ -14,6 +14,8 @@ ifconfig_lan0="inet {{ lan_ipv4_cidr }}"
 ifconfig_lan0_ipv6="inet6 accept_rtadv"
 ifconfig_wan0="DHCP"
 ifconfig_wan0_ipv6="inet6 accept_rtadv"
+cloned_interfaces="bridge0"
+ifconfig_bridge0="addm lan0 up"
 zpool_gpt_labels_enable="YES"
 smartd_enable="YES"
 pf_enable="YES"
diff --git a/templates/usr_local_bin_restore_jail_aliases.sh.j2 b/templates/usr_local_bin_restore_jail_aliases.sh.j2
deleted file mode 100644
index 25d317f..0000000
--- a/templates/usr_local_bin_restore_jail_aliases.sh.j2
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/sh
-
-# The point of this script is to restore jail aliases
-# if the lan0 interface flaps
-
-if_lan="lan0"
-ipv4_out="$(ifconfig lan0 | grep "inet " | grep -v "0xffffffff")"
-ipv4_lan="$(echo "${ipv4_out}" | awk '{ print $2 }')"
-ipv4_mask="$(echo "${ipv4_out}" | awk '{ print $4 }')"
-
-if ! [ "$ipv4_mask" = "0xffff0000" ]; then
-    >&2 echo "Unexpected netmask on lan0: ${ipv4_mask}"
-    exit 1
-fi
-
-jls ip4.addr | while read ip; do
-    case "$ip" in
-        "192.168"*) ;;
-        *) continue ;;
-    esac
-    if ifconfig "$if_lan" | grep -q "$ip"; then
-        continue
-    fi
-
-    echo "Re-adding $ip to $if_lan"
-    ifconfig "$if_lan" inet "$ip/32" alias
-done
diff --git a/templates/usr_local_etc_nginx_nginx.conf.j2 b/templates/usr_local_etc_nginx_nginx.conf.j2
index fcdaf02..55c55df 100644
--- a/templates/usr_local_etc_nginx_nginx.conf.j2
+++ b/templates/usr_local_etc_nginx_nginx.conf.j2
@@ -22,7 +22,7 @@ http {
 
     {% for site in static_sites -%}
     {% if site.tls -%}
-    server {
+    {#server {
         listen 443 ssl;
         server_name {{ site.host }};
 
@@ -41,7 +41,7 @@ http {
         include /usr/local/etc/letsencrypt/options-ssl-nginx.conf;
         ssl_dhparam /usr/local/etc/letsencrypt/ssl-dhparams.pem;
     }
-
+    #}
     {% endif -%}
 
     server {
@@ -61,7 +61,7 @@ http {
         server_name {{ jail.name }}.jan.systems {{ jail.name }}.local.jan.systems;
 
         location / {
-            proxy_pass http://{{ jail.ip }};
+            proxy_pass http://192.168.2.{{ jail.num }};
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-- 
cgit v1.3