Added logic and script to test for PKGBUILD poisoning

arch_upgrade.yml

@@ -1,26 +0,0 @@
-- name: All Arch hosts up-to-date
-  hosts: arch
-  tasks:
-    - name: Get current Python minor version
-      ansible.builtin.shell:
-        cmd: python --version | grep -Po '\d+\.\d+'
-      register:  orig_python
-    - name: Full repository upgrade
-      become: true
-      community.general.pacman:
-        update_cache: true
-        upgrade: true
-    - name: Get new Python minor version
-      ansible.builtin.shell:
-        cmd: python --version | grep -Po '\d+\.\d+'
-      register:  new_python
-    - name:  AUR upgrade
-      aur:
-        use:  pikaur
-        upgrade:  true
-        aur_only:  true
-    - import_tasks:  ../tasks/aur_rebuild.yml
-      vars:
-        package_pattern:  python
-      when:
-        - new_python.stdout is version(orig_python.stdout, '>', version_type='strict')

aur_rebuild.yml

@@ -1,14 +0,0 @@
-- name:  Get list of AUR Python packages that need to be rebuilt
-  ansible.builtin.shell:
-    cmd:
-      comm -12 <(pactree -lrud1 {{ package_pattern }} | sort -u) <(pacman -Qqm | sort -u)
-  register: aur_packages
-- name:  Rebuild AUR Python packages
-  aur:
-    use:  pikaur
-    name:  '{{ item }}'
-    aur_only:  true
-    extra_args:  --rebuild
-  loop:  '{{ aur_packages.stdout.split() }}'
-
-

build_needs_reboot.yml

@@ -1,22 +0,0 @@
-- name:  Create test script to determine if reboot is necessary
-  hosts:  all
-  tasks:
-    - name: Create ~/bin if it doesn't already exist
-      ansible.builtin.shell:
-        cmd:  mkdir  ~/bin
-      args:
-        creates:  ~/bin/
-    - name:  Build needs_reboot
-      ansible.builtin.shell: 
-        cmd:  |
-          print '#!/usr/bin/env zsh' > needs_reboot
-          source ~/.zsh_functions
-          declare -f kernel_func >> needs_reboot
-          perl -pi -e 'if (/OK/) { $_ = "false\n" }' needs_reboot
-          perl -pi -e 'if (/needs reboot/) { $_ = "true\n" }' needs_reboot
-          printf "\n\n\nkernel_func\n" >> needs_reboot
-          chmod +x needs_reboot
-      args:
-        creates:  needs_reboot
-        executable:  /usr/bin/zsh
-        chdir:  ~/bin/

debian_upgrade.yml

@@ -1,11 +0,0 @@
-- name: All Debian hosts up-to-date
-  hosts: debian
-  tasks:
-    - name: Full system upgrade
-      become: true
-      ansible.builtin.apt:
-        #executable:  /usr/bin/pikaur
-        update_cache: true
-        name: "*"
-        state:  latest
-        #upgrade: true

kernel_func

@@ -1,13 +0,0 @@
-kernel_func () {
-	machine_id=$(cat /etc/machine-id) 
-	current_kernel=$(uname -r) 
-	current_ucode=$(awk -F'[[:space:]]*:[[:space:]]*' '/microcode/ {print $2}' /proc/cpuinfo | uniq) 
-	[[ -f /run/next_kernel ]] && next_kernel="$(sudo cat /run/next_kernel)" 
-	next_ucode=$(iucode_tool -lqS /lib/firmware/intel-ucode/ | grep -Po 'rev 0x\d+' | tr -d '[rev ]' | tail -1) 
-	if [[ "${current_kernel}" == ${next_kernel} ]] && [[ "${current_ucode}" == ${next_ucode} ]] || [[ -z "${next_kernel}" ]]
-	then
-		print -P "[%F{#00ff00}OK%f]"
-	else
-		print -P "[%F{yellow}needs reboot%f]"
-	fi
-}

reboot.yml

@@ -1,13 +0,0 @@
-- name:  Stop mollyguard if active
-  ansible.builtin.script:  stop_mollyguard
-  register:  mg
-- name:  Determine if a reboot is necessary
-  ansible.builtin.command:
-    cmd:  ~/bin/needs_reboot
-  register:  needs_reboot
-- name:  Conditionally reboot
-  ansible.builtin.reboot:
-  when:
-    - inventory_hostname not in group['controller']
-    - mgc succeeded
-    - needs_reboot succeeded

roles/arch_update/files/aur_diff_check.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+import urllib.request
+import urllib.parse
+import json
+import subprocess
+import os
+import re
+import tempfile
+import sys
+import shutil
+
+def run_cmd(cmd, cwd=None):
+    return subprocess.run(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+def get_local_foreign_packages():
+    res = run_cmd(['pacman', '-Qm'])
+    if res.returncode != 0:
+        return {}
+    pkgs = {}
+    for line in res.stdout.strip().split('\n'):
+        if line:
+            parts = line.split()
+            if len(parts) == 2:
+                pkgs[parts[0]] = parts[1]
+    return pkgs
+
+def get_aur_info(pkgnames):
+    if not pkgnames:
+        return {}
+    results = {}
+    chunk_size = 100
+    pkgs = list(pkgnames)
+    for i in range(0, len(pkgs), chunk_size):
+        chunk = pkgs[i:i+chunk_size]
+        url = "https://aur.archlinux.org/rpc/v5/info?" + "&".join([f"arg[]={urllib.parse.quote(p)}" for p in chunk])
+        try:
+            req = urllib.request.urlopen(url, timeout=10)
+            data = json.loads(req.read().decode('utf-8'))
+            if data.get('type') == 'multiinfo':
+                for item in data.get('results', []):
+                    results[item['Name']] = item
+        except Exception:
+            pass
+    return results
+
+def needs_upgrade(local_ver, aur_ver):
+    res = run_cmd(['vercmp', local_ver, aur_ver])
+    try:
+        return int(res.stdout.strip()) < 0
+    except Exception:
+        return False
+
+def extract_ver(content):
+    pkgver, pkgrel = None, None
+    for line in content.split('\n'):
+        line = line.strip()
+        if line.startswith('pkgver='):
+            pkgver = line.split('=', 1)[1].strip('"\'')
+        elif line.startswith('pkgrel='):
+            pkgrel = line.split('=', 1)[1].strip('"\'')
+    return pkgver, pkgrel
+
+def strip_volatile_fields(content):
+    content = re.sub(r'^pkgver=.*$', '', content, flags=re.MULTILINE)
+    content = re.sub(r'^pkgrel=.*$', '', content, flags=re.MULTILINE)
+    sum_patterns = [
+        r'md5sums', r'sha1sums', r'sha224sums', r'sha256sums', 
+        r'sha384sums', r'sha512sums', r'b2sums', r'cksums'
+    ]
+    for sp in sum_patterns:
+        pattern = r'^' + sp + r'(?:_[a-zA-Z0-9_]+)?=\(.*?\)'
+        content = re.sub(pattern, '', content, flags=re.MULTILINE | re.DOTALL)
+    
+    lines = [line.rstrip() for line in content.split('\n') if line.strip()]
+    return '\n'.join(lines)
+
+def main():
+    local_pkgs = get_local_foreign_packages()
+    if not local_pkgs:
+        sys.exit(0)
+
+    aur_info = get_aur_info(list(local_pkgs.keys()))
+    
+    upgrades = []
+    for pkg, local_ver in local_pkgs.items():
+        if pkg in aur_info:
+            aur_ver = aur_info[pkg]['Version']
+            if needs_upgrade(local_ver, aur_ver):
+                upgrades.append((pkg, local_ver, aur_ver))
+                
+    if not upgrades:
+        sys.exit(0)
+        
+    print(f"Checking PKGBUILD diffs for {len(upgrades)} upgrades...")
+    
+    temp_dir = tempfile.mkdtemp()
+    try:
+        for pkg, local_ver, aur_ver in upgrades:
+            print(f"Checking {pkg} ({local_ver} -> {aur_ver})")
+            repo_dir = os.path.join(temp_dir, pkg)
+            res = run_cmd(['git', 'clone', '--quiet', f'https://aur.archlinux.org/{pkg}.git', repo_dir])
+            if res.returncode != 0:
+                print(f"Failed to clone {pkg}", file=sys.stderr)
+                sys.exit(1)
+            
+            res = run_cmd(['git', 'log', '--pretty=format:%H'], cwd=repo_dir)
+            commits = res.stdout.strip().split('\n')
+            
+            old_pkgbuild_content = None
+            new_pkgbuild_content = None
+            
+            with open(os.path.join(repo_dir, 'PKGBUILD'), 'r', encoding='utf-8', errors='ignore') as f:
+                new_pkgbuild_content = f.read()
+                
+            local_ver_no_epoch = local_ver.split(':', 1)[-1]
+            parts = local_ver_no_epoch.rsplit('-', 1)
+            expected_ver, expected_rel = parts if len(parts) == 2 else (parts[0], "")
+            
+            for commit in commits:
+                res = run_cmd(['git', 'show', f"{commit}:PKGBUILD"], cwd=repo_dir)
+                if res.returncode == 0:
+                    content = res.stdout
+                    p_ver, p_rel = extract_ver(content)
+                    if p_ver == expected_ver and p_rel == expected_rel:
+                        old_pkgbuild_content = content
+                        break
+            
+            if old_pkgbuild_content is None:
+                print(f"ERROR: Could not find old PKGBUILD for {pkg} version {local_ver}. Cannot safely verify diff.", file=sys.stderr)
+                sys.exit(1)
+            
+            old_stripped = strip_volatile_fields(old_pkgbuild_content)
+            new_stripped = strip_volatile_fields(new_pkgbuild_content)
+            
+            if old_stripped != new_stripped:
+                print(f"ERROR: Unsafe changes detected in PKGBUILD for {pkg}!", file=sys.stderr)
+                with open(os.path.join(temp_dir, 'old'), 'w') as f:
+                    f.write(old_stripped)
+                with open(os.path.join(temp_dir, 'new'), 'w') as f:
+                    f.write(new_stripped)
+                subprocess.run(['diff', '-u', os.path.join(temp_dir, 'old'), os.path.join(temp_dir, 'new')])
+                sys.exit(1)
+                
+    finally:
+        shutil.rmtree(temp_dir)
+
+if __name__ == '__main__':
+    main()

roles/arch_update/tasks/aur_rebuild.yaml

@@ -0,0 +1,17 @@
+- name: Get list of AUR Python packages that need to be rebuilt
+  ansible.builtin.shell:
+    cmd:
+      comm -12 <(pactree -lrud1 {{ package_pattern }} | sort -u) <(pacman -Qqm | sort -u)
+    executable: /bin/bash
+  register: aur_packages
+  changed_when: false
+  failed_when: false
+
+- name: Rebuild AUR Python packages
+  aur:
+    use: "{{ aur_helper }}"
+    name: '{{ item }}'
+    aur_only: true
+    extra_args: --rebuild
+  loop: '{{ aur_packages.stdout_lines | default([]) }}'
+  when: aur_packages.stdout_lines | default([]) | length > 0

roles/arch_update/tasks/aur_upgrade.yaml

@@ -0,0 +1,18 @@
+---
+- name: Check AUR package diffs for poisoning
+  ansible.builtin.script: aur_diff_check.py
+  changed_when: false
+
+- name: AUR upgrade
+  aur:
+    use: "{{ aur_helper }}"
+    upgrade: true
+    aur_only: true
+
+- name: Rebuild AUR Python packages if version changed
+  ansible.builtin.import_tasks: aur_rebuild.yaml
+  check_mode:  false
+  vars:
+    package_pattern: python
+  when: new_python is version(orig_python, '>', version_type='strict')
+

roles/arch_update/tasks/main.yaml

@@ -0,0 +1,20 @@
+---
+- name: Get original Python version
+  ansible.builtin.include_tasks: python_check.yaml
+  vars:
+    target_var: orig_python
+
+- name: Perform official repository updates
+  ansible.builtin.import_tasks: repo_upgrade.yaml
+
+- name: Get new Python version
+  ansible.builtin.include_tasks: python_check.yaml
+  vars:
+    target_var: new_python
+
+- name: Perform AUR updates and rebuilds
+  ansible.builtin.import_tasks: aur_upgrade.yaml
+
+- name:  Report on any .pacnew files that need to be merged
+  ansible.builtin.include_tasks:  pacnew.yaml
+

roles/arch_update/tasks/pacnew.yaml

@@ -0,0 +1,12 @@
+- name: Check for pending .pacnew files
+  ansible.builtin.find:
+    paths: /etc
+    patterns: "*.pacnew"
+    recurse: yes
+  register: pacnew_files
+
+- name: Alert if .pacnew files exist
+  ansible.builtin.debug:
+    msg: "Warning: The following .pacnew files require merging: {{ pacnew_files.files | map(attribute='path') | list }}"
+  when: pacnew_files.matched > 0
+

roles/arch_update/tasks/python_check.yaml

@@ -0,0 +1,12 @@
+---
+- name: Run Python version command
+  ansible.builtin.shell:
+    cmd: python --version | grep -Po '\d+\.\d+'
+  register: _temp_python_version
+  changed_when: false
+  check_mode:  false
+
+- name: Assign version to dynamic variable
+  ansible.builtin.set_fact:
+    "{{ target_var }}": "{{ _temp_python_version.stdout }}"
+

roles/arch_update/tasks/repo_upgrade.yaml

@@ -0,0 +1,15 @@
+---
+- name: Perform full system upgrade
+  become:  true
+  community.general.pacman:
+    update_cache: true
+    upgrade: true
+    extra_args: "--noconfirm"
+  register: arch_upgrade_result
+  failed_when: 
+    - arch_upgrade_result.failed == true
+    - "'there is nothing to do' not in (arch_upgrade_result.stdout | default('') | lower)"
+- name: Debug full Arch upgrade output
+  ansible.builtin.debug:
+    var: arch_upgrade_result
+

roles/debian_update/tasks/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: Full system upgrade
+  become: true
+  ansible.builtin.apt:
+    update_cache: true
+    upgrade: full

roles/docker_update/tasks/main.yaml

@@ -0,0 +1,10 @@
+---
+- name: Pull latest images and update Docker Compose stacks
+  become: true
+  community.docker.docker_compose_v2:
+    project_src: "{{ item }}"
+    state: present
+    pull: always
+    build: always
+  loop: "{{ docker_compose_dirs | default([]) }}"
+

roles/mailcow_update/handlers/main.yaml

@@ -0,0 +1,19 @@
+---
+- name: Tear down Mailcow and main Docker stack(s) (LIFO)
+  become: true
+  community.docker.docker_compose_v2:
+    project_src: "{{ item }}"
+    state: absent
+  # Drops Mailcow first, THEN loops through your main stacks
+  loop: "{{ [mailcow_dir | default('/data/mailcow')] + (docker_compose_dirs | default([])) }}"
+  listen: Restart main Docker stack
+
+- name: Bring main Docker stack(s) and Mailcow back up (FIFO)
+  become: true
+  community.docker.docker_compose_v2:
+    project_src: "{{ item }}"
+    state: present
+  # Brings your main stacks up first (recreating the network), THEN Mailcow
+  loop: "{{ (docker_compose_dirs | default([])) + [mailcow_dir | default('/data/mailcow')] }}"
+  listen: Restart main Docker stack
+

roles/mailcow_update/tasks/main.yaml

@@ -0,0 +1,25 @@
+---
+- name: Execute unattended Mailcow update
+  become: true
+  ansible.builtin.command:
+    cmd: ./update.sh --force
+    chdir: "{{ mailcow_dir | default('/data/mailcow') }}"
+  register: mailcow_first_pass
+  # Exit code 2 means "modules updated, run me again". Anything else is a real failure.
+  failed_when: mailcow_first_pass.rc not in [0, 2]
+  changed_when: 
+    - mailcow_first_pass.rc == 0
+    - "'Updated code is available' in mailcow_first_pass.stdout or 'Pulling' in mailcow_first_pass.stdout or 'Upgrading' in mailcow_first_pass.stdout"
+  notify: Restart main Docker stack
+
+- name: Re-run unattended Mailcow update (if modules changed)
+  become: true
+  ansible.builtin.command:
+    cmd: ./update.sh --force
+    chdir: "{{ mailcow_dir | default('/data/mailcow') }}"
+  # Only trigger this second pass if the first pass threw the specific restart code
+  when: mailcow_first_pass.rc == 2
+  register: mailcow_second_pass
+  changed_when: 
+    - "'Updated code is available' in mailcow_second_pass.stdout or 'Pulling' in mailcow_second_pass.stdout or 'Upgrading' in mailcow_second_pass.stdout"
+  notify: Restart main Docker stack

roles/reboot/files/check_reboot.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+# Exit code 0 = Reboot required
+# Exit code 1 = System is up to date / No reboot needed
+
+# 1. Branch based on detected distro
+if [[ -f /etc/os-release ]]; then
+    source /etc/os-release
+else
+    echo "Cannot determine OS."
+    exit 1
+fi
+
+# ==========================================
+# DEBIAN LOGIC
+# ==========================================
+if [[ "$ID" == "debian" || "$ID_LIKE" == *"debian"* ]]; then
+    # Debian automatically creates this file when a kernel, microcode, 
+    # or core library update requires a reboot.
+    if [[ -f /var/run/reboot-required ]]; then
+        exit 0
+    else
+        exit 1
+    fi
+
+# ==========================================
+# ARCH LINUX LOGIC
+# ==========================================
+elif [[ "$ID" == "arch" || "$ID_LIKE" == *"arch"* ]]; then
+    
+    # --- KERNEL CHECK ---
+    # Get the currently running kernel version
+    current_kernel=$(uname -r)
+    
+    # Determine the installed kernel package based on the running kernel's name
+    # e.g., "6.1.60-1-lts" -> "linux-lts", "6.5.9-arch1-1" -> "linux"
+    if [[ "$current_kernel" == *"-lts" ]]; then
+        kernel_pkg="linux-lts"
+    elif [[ "$current_kernel" == *"-zen" ]]; then
+        kernel_pkg="linux-zen"
+    elif [[ "$current_kernel" == *"-hardened" ]]; then
+        kernel_pkg="linux-hardened"
+    else
+        kernel_pkg="linux"
+    fi
+
+    # Get the installed version from pacman.
+    # Note: pacman uses periods (6.5.9.arch1-1) while uname uses dashes (6.5.9-arch1-1).
+    # We strip the package name and format it to match `uname -r` for a clean comparison.
+    installed_kernel=$(pacman -Q "$kernel_pkg" | awk '{print $2}' | sed 's/\./-/g' | sed 's/-\([^-]*\)$/.\1/')
+
+    # In some UKI setups or specific Arch versions, the sed replacement above might not 
+    # perfectly match `uname -r` format. A bulletproof fallback is checking if the 
+    # modules directory for the currently running kernel has been removed by a pacman update.
+    if [[ ! -d "/usr/lib/modules/$current_kernel" ]]; then
+        exit 0
+    fi
+
+    # --- MICROCODE CHECK ---
+    # Check CPU vendor to determine which microcode package to look for
+    vendor=$(grep -m 1 'vendor_id' /proc/cpuinfo | awk '{print $3}')
+    if [[ "$vendor" == "GenuineIntel" ]]; then
+        ucode_pkg="intel-ucode"
+    elif [[ "$vendor" == "AuthenticAMD" ]]; then
+        ucode_pkg="amd-ucode"
+    else
+        ucode_pkg=""
+    fi
+
+    if [[ -n "$ucode_pkg" ]]; then
+        # Check if the microcode package was updated recently by looking at the 
+        # build date of the installed package vs the boot time of the system.
+        
+        # Get system boot time in seconds since epoch
+        boot_time=$(awk '{print int($1)}' /proc/uptime)
+        boot_epoch=$(( $(date +%s) - boot_time ))
+        
+        # Get the install time of the microcode package
+        ucode_install_epoch=$(expac '%time' "$ucode_pkg" 2>/dev/null || echo 0)
+
+        # If the microcode was installed AFTER the system booted, reboot needed
+        if (( ucode_install_epoch > boot_epoch )); then
+            exit 0
+        fi
+    fi
+
+    # If we made it this far, neither kernel nor microcode triggered a reboot
+    exit 1
+
+else
+    # Fallback for unsupported OS
+    exit 1
+fi
+

roles/reboot/handlers/main.yaml

@@ -0,0 +1,16 @@
+---
+- name: Temporarily stop Mollyguard to allow reboot
+  become: true
+  ansible.builtin.systemd:
+    name: mollyguard.service
+    state: stopped
+  listen: Reboot system
+  failed_when: false
+
+- name: Execute System Reboot
+  become: true
+  ansible.builtin.reboot:
+    msg: "Reboot initiated by Ansible"
+    reboot_timeout: 600
+  listen: Reboot system
+

roles/reboot/tasks/main.yaml

@@ -0,0 +1,12 @@
+---
+- name: Check if kernel or microcode update requires reboot
+  ansible.builtin.script: check_reboot.sh  # Your script placed in files/
+  register: reboot_check
+  # Prevent Ansible from failing if the script returns false (exit code 1)
+  failed_when: false
+  # Trigger the handler ONLY if the script returns true (exit code 0)
+  changed_when: reboot_check.rc == 0
+  # Forces this specific task to run during a dry run
+  check_mode: false  
+  notify: Reboot system
+

stop_mollyguard

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-if systemctl is-active mollyguard; then
-    systemctl stop mollyguard
-fi
-

systemd-creds.yaml

@@ -0,0 +1,27 @@
+- name:  Establish systemd-creds for use with multiple services
+  hosts:  arch debian
+  become:  true
+
+  vars_prompt:
+    - name:  passphrase
+      prompt:  "Enter password manager passphrase"
+      private:  true
+      confirm:  true
+    - name:  creds_var
+      prompt:  "Enter the environment variable where systemd will store the credential PATH"
+      private:  false
+    - name:  creds_name
+      prompt:  "Enter the credential name"
+      private:  false
+
+  tasks:
+    - name:  Create shared systemd runtime override directory
+      ansible.builtin.command:
+        cmd:  mkdir /etc/systemd/system/shared.d
+        creates:  /etc/systemd/system/shared.d
+    - name:  Create override
+      ansible.builtin.shell:
+        cmd: |
+          printf '%s' {{ passphrase | quote }} | (echo "[Service]"; systemd-creds encrypt --name={{ creds_name }} --pretty - -) >> /etc/systemd/system/shared.d/00-systemd-creds.conf
+          printf "Environment=%s=%%d/%s\n" {{ creds_var }} {{ creds_name }} >> /etc/systemd/system/shared.d/00-systemd-creds.conf
+        #creates:  /etc/systemd/system/shared.d/00-systemd-creds.conf