diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | .kiroignore | 1 | ||||
| -rw-r--r-- | host_vars/pursotin.yml | 7 | ||||
| -rw-r--r-- | host_vars/test_vm.yml | 12 | ||||
| -rw-r--r-- | inventory | 3 | ||||
| -rw-r--r-- | playbook.yml | 6 | ||||
| -rw-r--r-- | principles.md | 7 | ||||
| -rw-r--r-- | tasks/general.yml | 19 | ||||
| -rw-r--r-- | tasks/jail_ingress.yml | 17 | ||||
| -rw-r--r-- | tasks/jails_dataset.yml | 11 | ||||
| -rw-r--r-- | tasks/network.yml | 9 | ||||
| -rw-r--r-- | templates/etc_rc.conf.j2 | 16 | ||||
| -rw-r--r-- | templates/etc_resolv.conf.j2 | 2 | ||||
| -rw-r--r-- | templates/jail_confs/_epl_network.j2 | 5 | ||||
| -rw-r--r-- | templates/jail_confs/ingress.j2 | 5 | ||||
| -rwxr-xr-x | vm.sh | 246 |
16 files changed, 346 insertions, 21 deletions
@@ -1,2 +1,3 @@ .envrc secrets.yml +vm/ diff --git a/.kiroignore b/.kiroignore new file mode 100644 index 0000000..769c244 --- /dev/null +++ b/.kiroignore @@ -0,0 +1 @@ +secrets.yml diff --git a/host_vars/pursotin.yml b/host_vars/pursotin.yml new file mode 100644 index 0000000..0f7e173 --- /dev/null +++ b/host_vars/pursotin.yml @@ -0,0 +1,7 @@ +nic_lan: igc0 +nic_wan: igc1 +arch: amd64/amd64 +is_test_vm: false +jail_lan_prefix: "192.168.2" +jail_lan_prefixlen: "16" +jail_ip_offset: 0 diff --git a/host_vars/test_vm.yml b/host_vars/test_vm.yml new file mode 100644 index 0000000..821f137 --- /dev/null +++ b/host_vars/test_vm.yml @@ -0,0 +1,12 @@ +ansible_host: 10.0.20.2 +ansible_user: root +ansible_ssh_private_key_file: vm/id_ed25519 +ansible_ssh_common_args: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +nic_lan: vtnet0 +nic_wan: vtnet1 +arch: arm64/aarch64 +is_test_vm: true +dns_nameserver: "10.0.20.1" +jail_lan_prefix: "10.0.20" +jail_lan_prefixlen: "24" +jail_ip_offset: 100 @@ -1,2 +1,3 @@ -[pursotin] +[hosts] pursotin +test_vm diff --git a/playbook.yml b/playbook.yml index 3ff7ebd..9269a44 100644 --- a/playbook.yml +++ b/playbook.yml @@ -1,6 +1,6 @@ --- - name: Configure my FreeBSD home server - hosts: pursotin + hosts: hosts pre_tasks: - name: Fail if secrets.yml wasn't loaded and smtp_user is missing @@ -41,7 +41,7 @@ - { userland: "{{ jail_userland_15_0 }}", num: 17, name: syncthing } - { userland: "{{ jail_userland_15_0 }}", num: 18, name: komga } - { userland: "{{ jail_userland_15_0 }}", num: 19, name: leolalla_fi } - ingress_ip: 192.168.2.1 + ingress_ip: "{{ jail_lan_prefix }}.{{ 1 + jail_ip_offset }}" ingress_routes: - { host: jan.systems, jail: homepage } - { host: jantuomi.fi, redirect: jan.systems } @@ -77,10 +77,12 @@ - name: Run email tasks tags: [email] import_tasks: tasks/email.yml + when: not is_test_vm - name: Run ZFS tasks tags: [zfs] import_tasks: tasks/zfs.yml + when: not is_test_vm - name: Run common jails tasks tags: [jails] diff --git a/principles.md b/principles.md new file mode 100644 index 0000000..245846c --- /dev/null +++ b/principles.md @@ -0,0 +1,7 @@ +# Migration principles + +1. Jail definitions should be localized, e.g. jails/ingress.yml. A jail definition should define zfs mounts, nullfs mounts, files to install, networking scripts (pre_start etc) to insert into jail.conf. +2. Jails should be defined as hosts in the inventory file. To reach them, ansible should use a ssh conn to the host and then `jexec $jailname`. +3. Each structure in the jail definition yml (e.g. `pkg: ["nginx"]`) should be handled by an ansible role that activates when a relevant structure is defined in the jail definition. +4. Both the host (pursotin) and the jails inside it should be defined in this project, so ansible should be configured to use correct hosts and correct connection methods depending on host. +5. The main playbooks should be very light and just dispatch. diff --git a/tasks/general.yml b/tasks/general.yml index 76315cd..0ec9219 100644 --- a/tasks/general.yml +++ b/tasks/general.yml @@ -10,9 +10,16 @@ - bash - python - py311-pip - - vm-bhyve - fastfetch + +- name: Install amd64-specific packages + package: + name: "{{ item }}" + state: present + loop: + - vm-bhyve - drm-kmod + when: not is_test_vm - name: Set up periodic.conf template: @@ -39,10 +46,11 @@ group: wheel mode: "0644" register: fstab + when: not is_test_vm - name: Run mount -a shell: mount -a - when: fstab.changed + when: not is_test_vm and fstab.changed - name: Start auditd service: @@ -105,6 +113,7 @@ owner: root group: wheel mode: "0600" + when: not is_test_vm - name: Copy backup SSH public key copy: @@ -113,6 +122,7 @@ owner: root group: wheel mode: "0644" + when: not is_test_vm - name: Copy SSH config template: @@ -121,6 +131,7 @@ owner: root group: wheel mode: "0644" + when: not is_test_vm - name: Copy backup script template: @@ -129,15 +140,17 @@ owner: root group: wheel mode: "0755" + when: not is_test_vm - name: Check if zroot/vm exists shell: zfs list zroot/vm register: zroot_vm_check failed_when: false changed_when: false + when: not is_test_vm - name: Add vm dataset for bhyve and init - when: zroot_vm_check.rc != 0 + when: not is_test_vm and zroot_vm_check.rc != 0 block: - name: Create vm dataset shell: zfs create zroot/vm diff --git a/tasks/jail_ingress.yml b/tasks/jail_ingress.yml index 1848177..5945e42 100644 --- a/tasks/jail_ingress.yml +++ b/tasks/jail_ingress.yml @@ -1,3 +1,20 @@ +- name: Ensure ingress goaccess mount points exist + file: + path: "{{ item }}" + state: directory + owner: root + group: wheel + mode: "0755" + loop: + - /usr/local/jails/containers/goaccess/var/www/goaccess + - /usr/local/jails/containers/ingress/mnt/www_goaccess + +- name: Start ingress jail + shell: service jail start ingress + register: ingress_jail_start + failed_when: false + changed_when: "'already running' not in ingress_jail_start.stderr" + - name: Install packages inside jail loop: - { jail: ingress, package: nginx } diff --git a/tasks/jails_dataset.yml b/tasks/jails_dataset.yml index 3925a8c..9e2748c 100644 --- a/tasks/jails_dataset.yml +++ b/tasks/jails_dataset.yml @@ -25,7 +25,7 @@ block: - name: Download userland get_url: - url: https://download.freebsd.org/ftp/releases/amd64/amd64/{{ dataset.userland }}/base.txz + url: https://download.freebsd.org/ftp/releases/{{ arch }}/{{ dataset.userland }}/base.txz dest: /usr/local/jails/media/{{ dataset.userland }}-base.txz owner: root group: wheel @@ -41,9 +41,12 @@ dest: /usr/local/jails/templates/{{ dataset.userland }}/etc/localtime - name: Copy resolv.conf to jail userland - shell: | - cat /etc/resolv.conf > /usr/local/jails/templates/{{ dataset.userland }}/etc/resolv.conf - chmod 644 /usr/local/jails/templates/{{ dataset.userland }}/etc/resolv.conf + template: + src: etc_resolv.conf.j2 + dest: /usr/local/jails/templates/{{ dataset.userland }}/etc/resolv.conf + owner: root + group: wheel + mode: "0644" - name: Disable resolvconf in the template shell: echo 'resolvconf=NO' >> /usr/local/jails/templates/{{ dataset.userland }}/etc/resolvconf.conf diff --git a/tasks/network.yml b/tasks/network.yml index 5f60d05..86cbf46 100644 --- a/tasks/network.yml +++ b/tasks/network.yml @@ -5,17 +5,16 @@ owner: root group: wheel mode: "0644" - register: resolv_conf - -- name: Make resolv.conf immutable - shell: chflags schg /etc/resolv.conf - when: resolv_conf.changed # TODO: this doesn't work without rc_conf having been run #- name: Restart networking if interface configuration changed # shell: service netif restart && service routing restart # when: rc_conf.changed or resolv_conf.changed +- name: Apply network configuration + shell: service netif restart && service routing restart + when: rc_conf.changed + - name: Set up sshd template: src: etc_ssh_sshd_config.j2 diff --git a/templates/etc_rc.conf.j2 b/templates/etc_rc.conf.j2 index 6ee2583..5380056 100644 --- a/templates/etc_rc.conf.j2 +++ b/templates/etc_rc.conf.j2 @@ -8,13 +8,21 @@ moused_nondefault_enable="NO" # Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable dumpdev="AUTO" zfs_enable="YES" -defaultrouter="192.168.0.1" +{% if is_test_vm %} +defaultrouter="10.0.20.1" +{% else %} +defaultrouter="{{ lan_ipv4_gateway }}" +{% endif %} cloned_interfaces="bridge0 bridge1" -ifconfig_igc0_name="lan0" -ifconfig_igc1_name="wan0" +ifconfig_{{ nic_lan }}_name="lan0" +ifconfig_{{ nic_wan }}_name="wan0" ifconfig_bridge0_name="brlan0" ifconfig_bridge1_name="brwan0" +{% if is_test_vm %} +ifconfig_lan0="inet 10.0.20.2/24" +{% else %} ifconfig_lan0="inet {{ lan_ipv4_cidr }}" +{% endif %} ifconfig_lan0_ipv6="inet6 accept_rtadv" ifconfig_wan0="up" ifconfig_brlan0="addm lan0 up" @@ -31,8 +39,10 @@ clear_tmp_enable="YES" cleanvar_enable="YES" auditd_enable="YES" devd_enable="YES" +{% if not is_test_vm %} vm_enable="YES" vm_dir="zfs:zroot/vm" vm_list="alpine0" vm_delay="5" kld_list="if_wg i915kms" +{% endif %} diff --git a/templates/etc_resolv.conf.j2 b/templates/etc_resolv.conf.j2 index 9232f25..b382ecf 100644 --- a/templates/etc_resolv.conf.j2 +++ b/templates/etc_resolv.conf.j2 @@ -1,2 +1,2 @@ search {{ lan_search_domain }} -nameserver {{ lan_ipv4_gateway }} +nameserver {{ dns_nameserver | default(lan_ipv4_gateway) }} diff --git a/templates/jail_confs/_epl_network.j2 b/templates/jail_confs/_epl_network.j2 index e80aebe..fa3744a 100644 --- a/templates/jail_confs/_epl_network.j2 +++ b/templates/jail_confs/_epl_network.j2 @@ -8,10 +8,11 @@ exec.prestart += "ifconfig brlan0 addm epl{{ jail.num }}a"; exec.poststart += "ifconfig epl{{ jail.num }}b vnet ${name}"; exec.poststart += "jexec ${name} ifconfig epl{{ jail.num }}b up"; exec.poststart += "ifconfig epl{{ jail.num }}a up"; -exec.poststart += "jexec ${name} ifconfig epl{{ jail.num }}b 192.168.2.{{ jail.num }}/16"; +exec.poststart += "jexec ${name} ifconfig epl{{ jail.num }}b {{ jail_lan_prefix }}.{{ jail.num + jail_ip_offset }}/{{ jail_lan_prefixlen }}"; exec.poststart += "jexec ${name} route delete default || echo 'No default route to delete'"; +{% if not is_test_vm %} exec.poststart += "jexec ${name} route add 10.6.210.0/24 {{ lan_ipv4_gateway }} || echo 'Failed to add Wireguard return route'"; - +{% endif %} exec.poststop += "ifconfig epl{{ jail.num }}a destroy"; ## _epl_network end diff --git a/templates/jail_confs/ingress.j2 b/templates/jail_confs/ingress.j2 index 59db3f1..5964d06 100644 --- a/templates/jail_confs/ingress.j2 +++ b/templates/jail_confs/ingress.j2 @@ -10,7 +10,12 @@ exec.prestart += "ifconfig brwan0 addm epw{{ jail.num }}a"; exec.poststart += "ifconfig epw{{ jail.num }}b vnet ${name}"; exec.poststart += "jexec ${name} ifconfig epw{{ jail.num }}b up"; exec.poststart += "ifconfig epw{{ jail.num }}a up"; +{% if is_test_vm %} +exec.poststart += "jexec ${name} ifconfig epw{{ jail.num }}b inet 10.0.20.3/24"; +exec.poststart += "jexec ${name} route add default 10.0.20.1"; +{% else %} exec.poststart += "jexec ${name} service dhclient restart epw{{ jail.num }}b"; +{% endif %} exec.prestart += "mount -t nullfs /usr/local/jails/containers/goaccess/var/www/goaccess /usr/local/jails/containers/ingress/mnt/www_goaccess || echo 'Failed to mount'"; exec.poststop += "umount /usr/local/jails/containers/ingress/mnt/www_goaccess || echo 'Failed to umount'"; @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +set -euo pipefail + +# vm.sh — Manage a QEMU FreeBSD aarch64 VM for testing Ansible playbooks. +# +# Usage: +# ./vm.sh dl Download the base image (one-time) +# ./vm.sh init Extract disk, create seed ISO and SSH key +# ./vm.sh up Start the VM (requires sudo for vmnet) +# ./vm.sh down Shut down the VM gracefully +# ./vm.sh kill Force-kill the VM +# ./vm.sh reset Delete disk & state, keeping the downloaded base image +# ./vm.sh ssh SSH into the VM +# ./vm.sh status Check if the VM is running +# ./vm.sh console Attach to serial console (ctrl-a x to exit) +# +# Prerequisites: brew install qemu socat cdrtools +# +# Networking (vmnet-shared, subnet 10.0.20.0/24): +# .1 Mac (vmnet gateway, provides NAT + DNS) +# .2 VM host LAN (lan0) +# .3 Ingress jail WAN interface +# .101-XXX Jails LAN (jail number + 100) +# +# All IPs are directly reachable from the Mac without extra routes. +# Run playbook: ansible-playbook -i inventory playbook.yml --limit test_vm + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +VM_DIR="${SCRIPT_DIR}/vm" +DISK="${VM_DIR}/disk.qcow2" +PFLASH_VARS="${VM_DIR}/pflash_vars.fd" +PID_FILE="${VM_DIR}/qemu.pid" +MONITOR_SOCK="${VM_DIR}/qemu-monitor.sock" +SERIAL_SOCK="${VM_DIR}/qemu-serial.sock" + +FREEBSD_VERSION="${FREEBSD_VERSION:-15.0}" +IMAGE_NAME="FreeBSD-${FREEBSD_VERSION}-RELEASE-arm64-aarch64-BASIC-CLOUDINIT-zfs.qcow2" +IMAGE_XZ="${VM_DIR}/${IMAGE_NAME}.xz" +IMAGE_URL="https://download.freebsd.org/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/aarch64/Latest/${IMAGE_NAME}.xz" + +PFLASH_CODE="$(brew --prefix)/share/qemu/edk2-aarch64-code.fd" +SEED_ISO="${VM_DIR}/seed.iso" +SSH_KEY="${VM_DIR}/id_ed25519" + +SSH_PORT_LAN=22 +VM_RAM=2048 +VM_CPUS=2 +DISK_SIZE=20G + +cmd_dl() { + mkdir -p "$VM_DIR" + if [[ -f "$IMAGE_XZ" ]]; then + echo "Base image already downloaded: $IMAGE_XZ" + return + fi + echo "Downloading FreeBSD ${FREEBSD_VERSION} aarch64 VM image..." + curl --fail -C - -L -o "$IMAGE_XZ" "$IMAGE_URL" || { rm -f "$IMAGE_XZ"; echo "Download failed: $IMAGE_URL"; exit 1; } + echo "Done." +} + +prepare_disk() { + if [[ ! -f "$IMAGE_XZ" ]]; then + echo "Base image not found. Run: ./vm.sh dl" + exit 1 + fi + echo "Extracting disk from base image..." + xz -dk "$IMAGE_XZ" + mv "${VM_DIR}/${IMAGE_NAME}" "$DISK" + qemu-img resize "$DISK" "$DISK_SIZE" +} + +prepare_seed() { + [[ -f "$SEED_ISO" ]] && return + if [[ ! -f "$SSH_KEY" ]]; then + ssh-keygen -t ed25519 -f "$SSH_KEY" -N "" -q + fi + local pubkey + pubkey=$(cat "${SSH_KEY}.pub") + local seed_dir="${VM_DIR}/seed" + mkdir -p "$seed_dir" + cat > "$seed_dir/meta-data" <<EOF +instance-id: pursotin-vm +local-hostname: pursotin +EOF + cat > "$seed_dir/user-data" <<EOF +#cloud-config +ssh_pwauth: true +disable_root: false + +users: + - name: root + ssh_authorized_keys: + - ${pubkey} + +packages: + - python3 + +runcmd: + - pkg install -y python3 + - cp /usr/share/zoneinfo/Europe/Helsinki /etc/localtime + - sysrc ifconfig_vtnet0="inet 10.0.20.2/24" + - sysrc defaultrouter="10.0.20.1" + - ifconfig vtnet0 inet 10.0.20.2/24 + - route add default 10.0.20.1 + - sed -i '' 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + - sed -i '' 's/^PermitRootLogin no/PermitRootLogin yes/' /etc/ssh/sshd_config + - service sshd restart +EOF + mkisofs -output "$SEED_ISO" -volid cidata -joliet -rock \ + "$seed_dir/user-data" "$seed_dir/meta-data" + rm -rf "$seed_dir" +} + +cmd_init() { + mkdir -p "$VM_DIR" + if [[ -f "$DISK" ]]; then + echo "Already initialized. Use './vm.sh reset' first to re-initialize." + exit 0 + fi + [[ -f "$IMAGE_XZ" ]] || cmd_dl + prepare_disk + prepare_seed + if [[ ! -f "$PFLASH_VARS" ]]; then + dd if=/dev/zero of="$PFLASH_VARS" bs=1m count=64 2>/dev/null + fi + echo "Initialized. Run './vm.sh up' to start." +} + +cmd_up() { + if [[ -f "$PID_FILE" ]] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "VM already running (PID $(cat "$PID_FILE"))" + exit 0 + fi + + if [[ ! -f "$DISK" ]]; then + echo "Not initialized. Run: ./vm.sh init" + exit 1 + fi + + echo "Starting VM (requires sudo for vmnet)..." + sudo -v || { echo "sudo required for vmnet networking"; exit 1; } + sudo qemu-system-aarch64 \ + -M virt,highmem=on \ + -accel hvf \ + -cpu host \ + -m "$VM_RAM" \ + -smp "$VM_CPUS" \ + -drive if=pflash,format=raw,file="$PFLASH_CODE",readonly=on \ + -drive if=pflash,format=raw,file="$PFLASH_VARS" \ + -drive file="$DISK",format=qcow2,if=virtio \ + -cdrom "$SEED_ISO" \ + -netdev vmnet-shared,id=lan,start-address=10.0.20.1,end-address=10.0.20.254,subnet-mask=255.255.255.0 \ + -device virtio-net-pci,netdev=lan \ + -netdev vmnet-shared,id=wan,start-address=10.0.20.1,end-address=10.0.20.254,subnet-mask=255.255.255.0 \ + -device virtio-net-pci,netdev=wan \ + -serial unix:"$SERIAL_SOCK",server,nowait \ + -monitor unix:"$MONITOR_SOCK",server,nowait \ + -pidfile "$PID_FILE" \ + -daemonize \ + -display none \ + && sudo chmod 644 "$PID_FILE" \ + && sudo chmod 777 "$SERIAL_SOCK" "$MONITOR_SOCK" \ + && echo "VM started (PID $(cat "$PID_FILE"))" \ + && echo "SSH: ./vm.sh ssh" \ + && echo "Console: ./vm.sh console" +} + +cmd_down() { + if [[ ! -S "$MONITOR_SOCK" ]]; then + echo "VM not running (no monitor socket)" + exit 1 + fi + echo "system_powerdown" | sudo socat - UNIX-CONNECT:"$MONITOR_SOCK" + echo "Sent powerdown signal." + local pid + pid=$(cat "$PID_FILE" 2>/dev/null || true) + if [[ -n "$pid" ]]; then + for _ in $(seq 1 30); do + sudo kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + if sudo kill -0 "$pid" 2>/dev/null; then + echo "VM still running after 30s. Use './vm.sh kill' to force." + else + rm -f "$PID_FILE" + echo "VM stopped." + fi + fi +} + +cmd_kill() { + if [[ -f "$PID_FILE" ]]; then + local pid + pid=$(cat "$PID_FILE") + sudo kill -9 "$pid" 2>/dev/null && echo "Killed VM (PID $pid)" || echo "Process not found" + rm -f "$PID_FILE" + else + echo "No PID file found" + fi +} + +cmd_reset() { + cmd_kill 2>/dev/null || true + rm -f "$DISK" "$PFLASH_VARS" "$SEED_ISO" "$SSH_KEY" "${SSH_KEY}.pub" + rm -f "$MONITOR_SOCK" "$SERIAL_SOCK" "$PID_FILE" + echo "VM state reset. Base image kept at $IMAGE_XZ" +} + +cmd_ssh() { + exec ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -i "$SSH_KEY" root@10.0.20.2 "$@" +} + +cmd_console() { + if [[ ! -S "$SERIAL_SOCK" ]]; then + echo "VM not running (no serial socket)" + exit 1 + fi + echo "Connecting to serial console (ctrl-a x to exit)..." + exec socat -,rawer,escape=0x01 UNIX-CONNECT:"$SERIAL_SOCK" +} + +cmd_status() { + if [[ -f "$PID_FILE" ]] && sudo kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "VM running (PID $(cat "$PID_FILE"))" + else + echo "VM not running" + rm -f "$PID_FILE" 2>/dev/null + fi +} + +case "${1:-}" in + dl) cmd_dl ;; + init) cmd_init ;; + up) cmd_up ;; + down) cmd_down ;; + kill) cmd_kill ;; + reset) cmd_reset ;; + ssh) shift; cmd_ssh "$@" ;; + console) cmd_console ;; + status) cmd_status ;; + *) + echo "Usage: $0 {dl|init|up|down|kill|reset|ssh|console|status}" + exit 1 + ;; +esac |
