From d086e7bcaf103c21933c351ed2f329835615e8aa Mon Sep 17 00:00:00 2001 From: WolfyDelta Date: Fri, 3 Jul 2026 14:35:47 -0400 Subject: [PATCH] created update-prod-servers.sh , .service , .timer to trigger update-rocky-prod.yml playbook every friday at 9:00PM then send a report to a discord webhook --- playbooks/apt-update-test.yml | 32 ------- playbooks/update-prod-servers.yml | 94 ------------------- playbooks/update-rocky-prod.yml | 131 +++++++++++++++++++++++++++ playbooks/update-ubuntu-servers.yml | 95 ------------------- run/update-prod-servers.sh | 51 +++++++---- services/update-prod-servers.service | 8 +- services/update-prod-servers.timer | 4 +- 7 files changed, 172 insertions(+), 243 deletions(-) delete mode 100644 playbooks/apt-update-test.yml delete mode 100644 playbooks/update-prod-servers.yml create mode 100644 playbooks/update-rocky-prod.yml delete mode 100644 playbooks/update-ubuntu-servers.yml diff --git a/playbooks/apt-update-test.yml b/playbooks/apt-update-test.yml deleted file mode 100644 index 3ddadd6..0000000 --- a/playbooks/apt-update-test.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -- name: Update and upgrade wolfywebserver02 - hosts: wolfywebserver02 - become: true - gather_facts: true - - tasks: - - name: Update apt cache - ansible.builtin.apt: - update_cache: true - cache_valid_time: 3600 - - - name: Upgrade installed packages - ansible.builtin.apt: - upgrade: dist - - - name: Remove unneeded packages - ansible.builtin.apt: - autoremove: true - - - name: Check if reboot is required - ansible.builtin.stat: - path: /var/run/reboot-required - register: reboot_required - - - name: Reboot server if required - ansible.builtin.reboot: - pre_reboot_delay: 60 - post_reboot_delay: 60 - msg: "Rebooting in 60 seconds (intiated by Ansible)" - reboot_timeout: 3600 - when: reboot_required.stat.exists \ No newline at end of file diff --git a/playbooks/update-prod-servers.yml b/playbooks/update-prod-servers.yml deleted file mode 100644 index 0114dcb..0000000 --- a/playbooks/update-prod-servers.yml +++ /dev/null @@ -1,94 +0,0 @@ ---- -- name: Patch Prod Ubuntu servers and report to Discord - hosts: prod_ubuntu_servers - become: true - gather_facts: true - - vars: - discord_webhook_id: "1507643788326338630" - discord_webhook_token: "TkbDnf6naR7KxhqLDIvGJyeesq9OTglxHrLHyy3d6_1WiyI-bbzUGDeVu1yCjZN7fCdd" - - tasks: - - name: Update apt cache - ansible.builtin.apt: - update_cache: true - cache_valid_time: 3600 - register: apt_cache_result - - - name: Upgrade installed packages - ansible.builtin.apt: - upgrade: dist - register: apt_upgrade_result - - - name: Remove unneeded packages - ansible.builtin.apt: - autoremove: true - register: apt_autoremove_result - - - name: Check if reboot is required - ansible.builtin.stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Reboot server if required - ansible.builtin.reboot: - pre_reboot_delay: 60 - post_reboot_delay: 60 - reboot_timeout: 600 - msg: "Rebooting in 60 seconds (intiated by Ansible)" - when: reboot_required_file.stat.exists - register: reboot_result - - - name: Build per-host summary - ansible.builtin.set_fact: - patch_summary: | - Host: {{ inventory_hostname }} - Apt Cache Updated: {{ 'Yes' if (apt_cache_result.changed | default(false)) else 'No' }} - Packages Upgraded: {{ 'Yes' if (apt_upgrade_result.changed | default(false)) else 'No' }} - Autoremove Changed: {{ 'Yes' if (apt_autoremove_result.changed | default(false)) else 'No' }} - Reboot Required: {{ 'Yes' if (reboot_required_file.stat.exists | default(false)) else 'No' }} - Reboot Performed: {{ 'Yes' if (reboot_result.changed | default(false)) else 'No' }} - - - name: Send Discord report - community.general.discord: - webhook_id: "{{ discord_webhook_id }}" - webhook_token: "{{ discord_webhook_token }}" - username: "Ansible" - avatar_url: "https://opreviews.anime-pictures.net/49f/49f1117eec83be2bd5af12772a70d2e4_bp.avif" - content: | - ====================================== - Production Servers Update Report - Generated: {{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%SZ') }} - ====================================== - - Total Hosts: {{ ansible_play_hosts_all | length }} - - {% for host in ansible_play_hosts_all | sort %} - {{ hostvars[host].patch_summary | default('Host: ' ~ host ~ '\n No summary generated\n') }} - - {% endfor %} - ====================================== - Summary - ====================================== - Hosts Requiring Reboot: {{ - ansible_play_hosts_all - | select('in', ansible_play_hosts_all) - | map('extract', hostvars) - | selectattr('reboot_required_file.stat.exists', 'defined') - | selectattr('reboot_required_file.stat.exists') - | list - | length - }} - - Hosts Rebooted: {{ - ansible_play_hosts_all - | select('in', ansible_play_hosts_all) - | map('extract', hostvars) - | selectattr('reboot_result.changed', 'defined') - | selectattr('reboot_result.changed') - | list - | length - }} - delegate_to: localhost - run_once: true - become: false \ No newline at end of file diff --git a/playbooks/update-rocky-prod.yml b/playbooks/update-rocky-prod.yml new file mode 100644 index 0000000..9bec143 --- /dev/null +++ b/playbooks/update-rocky-prod.yml @@ -0,0 +1,131 @@ +--- +- name: Patch Test Rocky servers and report to generic webhook + hosts: prod_rocky_servers + become: true + gather_facts: true + + vars: + discord_webhook_id: "1396804024942198794" + discord_webhook_token: "zdT2wisqhAZuZ_UPKmAmtV0z_rX-PZnyHIXsjpPHQO3uakBO2xQ3R44kOxb9QZ4QxebL" + + tasks: + - name: Verify target is RedHat family + ansible.builtin.assert: + that: + - ansible_os_family == "RedHat" + fail_msg: "This playbook is intended for Rocky/RHEL-family systems only." + success_msg: "Target is RedHat family." + + - name: Ensure dnf-utils is installed for needs-restarting + ansible.builtin.dnf: + name: dnf-utils + state: present + register: dnf_utils_result + + - name: Update DNF metadata cache + ansible.builtin.dnf: + update_cache: true + register: dnf_cache_result + + - name: Upgrade installed packages + ansible.builtin.dnf: + name: "*" + state: latest + update_only: true + register: dnf_upgrade_result + + - name: Remove unneeded packages + ansible.builtin.dnf: + autoremove: true + register: dnf_autoremove_result + + - name: Check if reboot is required + ansible.builtin.command: + cmd: needs-restarting -r + register: reboot_required_check + changed_when: false + failed_when: reboot_required_check.rc not in [0, 1] + + - name: Set reboot required fact + ansible.builtin.set_fact: + reboot_required: "{{ reboot_required_check.rc == 1 }}" + + - name: Reboot server if required + ansible.builtin.reboot: + pre_reboot_delay: 60 + post_reboot_delay: 60 + reboot_timeout: 600 + msg: "Rebooting in 60 seconds, initiated by Ansible" + when: reboot_required + register: reboot_result + + - name: Build per-host summary + ansible.builtin.set_fact: + patch_summary: | + Host: {{ inventory_hostname }} + OS: {{ ansible_distribution }} {{ ansible_distribution_version }} + DNF Utils Changed: {{ 'Yes' if (dnf_utils_result.changed | default(false)) else 'No' }} + DNF Cache Updated: {{ 'Yes' if (dnf_cache_result.changed | default(false)) else 'No' }} + Packages Upgraded: {{ 'Yes' if (dnf_upgrade_result.changed | default(false)) else 'No' }} + Autoremove Changed: {{ 'Yes' if (dnf_autoremove_result.changed | default(false)) else 'No' }} + Reboot Required: {{ 'Yes' if (reboot_required | default(false)) else 'No' }} + Reboot Performed: {{ 'Yes' if (reboot_result.changed | default(false)) else 'No' }} + + Reboot Check Output: + {{ reboot_required_check.stdout | default('') | indent(10, true) }} + + - name: Build per-host summary + ansible.builtin.set_fact: + patch_summary: | + Host: {{ inventory_hostname }} + OS: {{ ansible_distribution }} {{ ansible_distribution_version }} + DNF Utils Changed: {{ 'Yes' if (dnf_utils_result.changed | default(false)) else 'No' }} + DNF Cache Updated: {{ 'Yes' if (dnf_cache_result.changed | default(false)) else 'No' }} + Packages Upgraded: {{ 'Yes' if (dnf_upgrade_result.changed | default(false)) else 'No' }} + Autoremove Changed: {{ 'Yes' if (dnf_autoremove_result.changed | default(false)) else 'No' }} + Reboot Required: {{ 'Yes' if (reboot_required | default(false)) else 'No' }} + Reboot Performed: {{ 'Yes' if (reboot_result.changed | default(false)) else 'No' }} + + Reboot Check Output: + {{ reboot_required_check.stdout | default('') | indent(10, true) }} + + - name: Send Discord Rocky update report + community.general.discord: + webhook_id: "{{ discord_webhook_id }}" + webhook_token: "{{ discord_webhook_token }}" + username: "Ansible" + content: | + ====================================== + Rocky Servers Update Report + Generated: {{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%SZ') }} + ====================================== + + Total Hosts: {{ ansible_play_hosts_all | length }} + + {% for host in ansible_play_hosts_all | sort %} + {{ hostvars[host].patch_summary | default('Host: ' ~ host ~ '\n No summary generated\n') }} + + {% endfor %} + ====================================== + Summary + ====================================== + Hosts Requiring Reboot: {{ + ansible_play_hosts_all + | map('extract', hostvars) + | selectattr('reboot_required', 'defined') + | selectattr('reboot_required') + | list + | length + }} + + Hosts Rebooted: {{ + ansible_play_hosts_all + | map('extract', hostvars) + | selectattr('reboot_result.changed', 'defined') + | selectattr('reboot_result.changed') + | list + | length + }} + delegate_to: localhost + run_once: true + become: false \ No newline at end of file diff --git a/playbooks/update-ubuntu-servers.yml b/playbooks/update-ubuntu-servers.yml deleted file mode 100644 index 4c8f10d..0000000 --- a/playbooks/update-ubuntu-servers.yml +++ /dev/null @@ -1,95 +0,0 @@ ---- -- name: Patch Ubuntu servers and report to Discord - hosts: test_ubuntu_servers - become: true - gather_facts: true - - vars: -# discord_webhook_url: "https://discord.com/api/webhooks/1447800027941437554/WyjT6AH3Pn_1Wgwah7AjMm47l2UVpZCMZ0DbMnpEr3vKPImHZkkmZDtRZrlhPl4MnW73 - discord_webhook_id: "1507643788326338630" - discord_webhook_token: "TkbDnf6naR7KxhqLDIvGJyeesq9OTglxHrLHyy3d6_1WiyI-bbzUGDeVu1yCjZN7fCdd" - - tasks: - - name: Update apt cache - ansible.builtin.apt: - update_cache: true - cache_valid_time: 3600 - register: apt_cache_result - - - name: Upgrade installed packages - ansible.builtin.apt: - upgrade: dist - register: apt_upgrade_result - - - name: Remove unneeded packages - ansible.builtin.apt: - autoremove: true - register: apt_autoremove_result - - - name: Check if reboot is required - ansible.builtin.stat: - path: /var/run/reboot-required - register: reboot_required_file - - - name: Reboot server if required - ansible.builtin.reboot: - pre_reboot_delay: 60 - post_reboot_delay: 60 - reboot_timeout: 600 - msg: "Rebooting in 60 seconds (intiated by Ansible)" - when: reboot_required_file.stat.exists - register: reboot_result - - - name: Build per-host summary - ansible.builtin.set_fact: - patch_summary: | - Host: {{ inventory_hostname }} - Apt Cache Updated: {{ 'Yes' if (apt_cache_result.changed | default(false)) else 'No' }} - Packages Upgraded: {{ 'Yes' if (apt_upgrade_result.changed | default(false)) else 'No' }} - Autoremove Changed: {{ 'Yes' if (apt_autoremove_result.changed | default(false)) else 'No' }} - Reboot Required: {{ 'Yes' if (reboot_required_file.stat.exists | default(false)) else 'No' }} - Reboot Performed: {{ 'Yes' if (reboot_result.changed | default(false)) else 'No' }} - - - name: Send Discord report - community.general.discord: - webhook_id: "{{ discord_webhook_id }}" - webhook_token: "{{ discord_webhook_token }}" - username: "Ansible" - avatar_url: "https://opreviews.anime-pictures.net/49f/49f1117eec83be2bd5af12772a70d2e4_bp.avif" - content: | - ====================================== - Test Servers Update Report - Generated: {{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%SZ') }} - ====================================== - - Total Hosts: {{ ansible_play_hosts_all | length }} - - {% for host in ansible_play_hosts_all | sort %} - {{ hostvars[host].patch_summary | default('Host: ' ~ host ~ '\n No summary generated\n') }} - - {% endfor %} - ====================================== - Summary - ====================================== - Hosts Requiring Reboot: {{ - ansible_play_hosts_all - | select('in', ansible_play_hosts_all) - | map('extract', hostvars) - | selectattr('reboot_required_file.stat.exists', 'defined') - | selectattr('reboot_required_file.stat.exists') - | list - | length - }} - - Hosts Rebooted: {{ - ansible_play_hosts_all - | select('in', ansible_play_hosts_all) - | map('extract', hostvars) - | selectattr('reboot_result.changed', 'defined') - | selectattr('reboot_result.changed') - | list - | length - }} - delegate_to: localhost - run_once: true - become: false \ No newline at end of file diff --git a/run/update-prod-servers.sh b/run/update-prod-servers.sh index 0b88039..c73f1fb 100644 --- a/run/update-prod-servers.sh +++ b/run/update-prod-servers.sh @@ -1,23 +1,42 @@ -#!/bin/bash -set -euo pipefail +#!/usr/bin/env bash +set -Eeuo pipefail -LOG_DIR="/home/wolfy/log/ansible-prod" -mkdir -p "$LOG_DIR" +# ------------------------------------------------------------ +# Ansible Rocky production patching wrapper +# Intended to be run by systemd service/timer +# ------------------------------------------------------------ -TIMESTAMP="$(date '+%Y-%m-%d_%H-%M-%S')" -LOG_FILE="$LOG_DIR/ubuntu-prod-update-$TIMESTAMP.log" +ANSIBLE_PROJECT="/opt/Ansible" +VENV_PATH="${ANSIBLE_PROJECT}/.venv" -exec >>"$LOG_FILE" 2>&1 +PLAYBOOK="${ANSIBLE_PROJECT}/playbooks/update-rocky-prod.yml" +LOCK_FILE="/tmp/update-rocky-prod.lock" -echo "========================================" -echo "Prod patching run started: $(date -Is)" -echo "========================================" +ANSIBLE_PLAYBOOK="${VENV_PATH}/bin/ansible-playbook" -cd /home/wolfy/ansible +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" +} -/home/wolfy/ansible-env/bin/ansible-playbook playbooks/update-prod-servers.yml +fail() { + log "ERROR: $*" + exit 1 +} -echo -echo "========================================" -echo "Prod patching run completed: $(date -Is)" -echo "========================================" \ No newline at end of file +log "Starting Rocky production patching" + +[[ -d "$VENV_PATH" ]] || fail "Python virtual environment not found: $VENV_PATH" +[[ -x "$ANSIBLE_PLAYBOOK" ]] || fail "ansible-playbook not found or not executable: $ANSIBLE_PLAYBOOK" +[[ -f "$PLAYBOOK" ]] || fail "Playbook not found: $PLAYBOOK" + +( + flock -n 200 || fail "Another patching run is already active" + + cd "$ANSIBLE_PROJECT" + + "$ANSIBLE_PLAYBOOK" \ + "$PLAYBOOK" + +) 200>"$LOCK_FILE" + +log "Finished Rocky production patching" \ No newline at end of file diff --git a/services/update-prod-servers.service b/services/update-prod-servers.service index 1a0717d..b190aaa 100644 --- a/services/update-prod-servers.service +++ b/services/update-prod-servers.service @@ -5,7 +5,7 @@ After=network-online.target [Service] Type=oneshot -User=wolfy -Group=wolfy -ExecStart=/home/wolfy/ansible/run/update-prod-servers.sh -WorkingDirectory=/home/wolfy/ansible \ No newline at end of file +User=ansible +Group=ansible-admins +ExecStart=/opt/ansible/run/update-prod-servers.sh +WorkingDirectory=/opt/ansible \ No newline at end of file diff --git a/services/update-prod-servers.timer b/services/update-prod-servers.timer index 1df4c40..abdf36e 100644 --- a/services/update-prod-servers.timer +++ b/services/update-prod-servers.timer @@ -1,8 +1,8 @@ [Unit] -Description=Weekly production patching timer +Description=Weekly Rocky patching timer Updates @ 9:00 PM every Friday [Timer] -OnCalendar=Mon *-*-* 07:30:00 +OnCalendar=Fri *-*-* 21:00:00 Persistent=true Unit=update-prod-servers.service