Post-Tyranny-Tech-Infrastructure/scripts/run-maintenance-all-servers.sh

#!/usr/bin/env bash
#
# Run Nextcloud maintenance playbook on all servers
# Created: 2026-01-24
#
# This script runs the nextcloud maintenance playbook on each server
# with its corresponding SSH key.
#
# Usage:
#   cd infrastructure/
#   HCLOUD_TOKEN="..." ./scripts/run-maintenance-all-servers.sh
#
# Or with SOPS_AGE_KEY_FILE if needed:
#   SOPS_AGE_KEY_FILE="keys/age-key.txt" HCLOUD_TOKEN="..." ./scripts/run-maintenance-all-servers.sh

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
ANSIBLE_DIR="$PROJECT_ROOT/ansible"
KEYS_DIR="$PROJECT_ROOT/keys/ssh"
PLAYBOOK="playbooks/260124-nextcloud-maintenance.yml"

# Check required environment variables
if [ -z "${HCLOUD_TOKEN:-}" ]; then
    echo -e "${RED}Error: HCLOUD_TOKEN environment variable is required${NC}"
    exit 1
fi

# Change to ansible directory
cd "$ANSIBLE_DIR"

echo -e "${BLUE}============================================================${NC}"
echo -e "${BLUE}Nextcloud Maintenance - All Servers${NC}"
echo -e "${BLUE}============================================================${NC}"
echo ""
echo "Playbook: $PLAYBOOK"
echo "Ansible directory: $ANSIBLE_DIR"
echo ""

# Get list of all servers with SSH keys
SERVERS=()
for keyfile in "$KEYS_DIR"/*.pub; do
    if [ -f "$keyfile" ]; then
        server=$(basename "$keyfile" .pub)
        # Skip special servers
        if [[ "$server" != "README" ]] && [[ "$server" != "edge" ]]; then
            SERVERS+=("$server")
        fi
    fi
done

echo -e "${BLUE}Found ${#SERVERS[@]} servers:${NC}"
printf '%s\n' "${SERVERS[@]}" | sort
echo ""

# Counters
SUCCESS_COUNT=0
FAILED_COUNT=0
SKIPPED_COUNT=0
declare -a SUCCESS_SERVERS
declare -a FAILED_SERVERS
declare -a SKIPPED_SERVERS

echo -e "${BLUE}============================================================${NC}"
echo -e "${BLUE}Starting maintenance run...${NC}"
echo -e "${BLUE}============================================================${NC}"
echo ""

# Run playbook for each server
for server in "${SERVERS[@]}"; do
    echo -e "${YELLOW}-----------------------------------------------------------${NC}"
    echo -e "${YELLOW}Processing: $server${NC}"
    echo -e "${YELLOW}-----------------------------------------------------------${NC}"

    SSH_KEY="$KEYS_DIR/$server"

    if [ ! -f "$SSH_KEY" ]; then
        echo -e "${RED}✗ SSH key not found: $SSH_KEY${NC}"
        SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
        SKIPPED_SERVERS+=("$server")
        echo ""
        continue
    fi

    # Run the playbook (with SSH options to prevent agent key issues)
    if env HCLOUD_TOKEN="$HCLOUD_TOKEN" \
        ANSIBLE_SSH_ARGS="-o IdentitiesOnly=yes" \
        ~/.local/bin/ansible-playbook \
        -i hcloud.yml \
        "$PLAYBOOK" \
        --limit "$server" \
        --private-key "$SSH_KEY" 2>&1; then

        echo -e "${GREEN}✓ Success: $server${NC}"
        SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
        SUCCESS_SERVERS+=("$server")
    else
        echo -e "${RED}✗ Failed: $server${NC}"
        FAILED_COUNT=$((FAILED_COUNT + 1))
        FAILED_SERVERS+=("$server")
    fi

    echo ""
done

# Summary
echo -e "${BLUE}============================================================${NC}"
echo -e "${BLUE}MAINTENANCE RUN SUMMARY${NC}"
echo -e "${BLUE}============================================================${NC}"
echo ""
echo "Total servers: ${#SERVERS[@]}"
echo -e "${GREEN}Successful: $SUCCESS_COUNT${NC}"
echo -e "${RED}Failed: $FAILED_COUNT${NC}"
echo -e "${YELLOW}Skipped: $SKIPPED_COUNT${NC}"
echo ""

if [ $SUCCESS_COUNT -gt 0 ]; then
    echo -e "${GREEN}Successful servers:${NC}"
    printf '  %s\n' "${SUCCESS_SERVERS[@]}"
    echo ""
fi

if [ $FAILED_COUNT -gt 0 ]; then
    echo -e "${RED}Failed servers:${NC}"
    printf '  %s\n' "${FAILED_SERVERS[@]}"
    echo ""
fi

if [ $SKIPPED_COUNT -gt 0 ]; then
    echo -e "${YELLOW}Skipped servers:${NC}"
    printf '  %s\n' "${SKIPPED_SERVERS[@]}"
    echo ""
fi

echo -e "${BLUE}============================================================${NC}"

# Exit with error if any failures
if [ $FAILED_COUNT -gt 0 ]; then
    exit 1
fi

exit 0