#!/usr/bin/env bash # # EMERGENCY FIX: Systemd Service Architecture # Stops infinite restart cycles and fixes broken service architecture # set -e # Script configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # Remote connection configuration REMOTE_HOST="${1:-192.168.20.65}" REMOTE_USER="${2:-thrillwiki}" REMOTE_PORT="${3:-22}" SSH_KEY="${SSH_KEY:-$HOME/.ssh/thrillwiki_vm}" SSH_OPTIONS="-i $SSH_KEY -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=30" echo -e "${RED}๐Ÿšจ EMERGENCY SYSTEMD ARCHITECTURE FIX${NC}" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" echo "" echo "Target: ${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_PORT}" echo "" echo -e "${YELLOW}โš ๏ธ This will fix critical issues:${NC}" echo "โ€ข Stop infinite restart cycles (currently at 32+ restarts)" echo "โ€ข Disable problematic continuous deployment service" echo "โ€ข Clean up stale lock files" echo "โ€ข Fix broken timer configuration" echo "โ€ข Deploy correct service architecture" echo "โ€ข Create missing smart-deploy.sh script" echo "" # Function to run remote commands with error handling run_remote() { local cmd="$1" local description="$2" local use_sudo="${3:-false}" echo -e "${YELLOW}Executing: ${description}${NC}" if [ "$use_sudo" = "true" ]; then if ssh $SSH_OPTIONS -p $REMOTE_PORT -t $REMOTE_USER@$REMOTE_HOST "sudo $cmd" 2>/dev/null; then echo -e "${GREEN}โœ… SUCCESS: ${description}${NC}" return 0 else echo -e "${RED}โŒ FAILED: ${description}${NC}" return 1 fi else if ssh $SSH_OPTIONS -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST "$cmd" 2>/dev/null; then echo -e "${GREEN}โœ… SUCCESS: ${description}${NC}" return 0 else echo -e "${RED}โŒ FAILED: ${description}${NC}" return 1 fi fi } # Step 1: Emergency stop of problematic service echo -e "${RED}๐Ÿ›‘ STEP 1: EMERGENCY STOP OF PROBLEMATIC SERVICE${NC}" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" run_remote "systemctl stop thrillwiki-deployment.service" "Stop problematic deployment service" true run_remote "systemctl disable thrillwiki-deployment.service" "Disable problematic deployment service" true echo "" echo -e "${GREEN}โœ… Infinite restart cycle STOPPED${NC}" echo "" # Step 2: Clean up system state echo -e "${YELLOW}๐Ÿงน STEP 2: CLEANUP SYSTEM STATE${NC}" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" # Remove stale lock file run_remote "rm -f /tmp/thrillwiki-deployment.lock" "Remove stale lock file" # Kill any remaining deployment processes (non-critical if it fails) ssh $SSH_OPTIONS -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST "pkill -f 'deploy-automation.sh' || true" 2>/dev/null || echo -e "${YELLOW}โš ๏ธ No deployment processes to kill (this is fine)${NC}" echo "" # Step 3: Create missing smart-deploy.sh script echo -e "${BLUE}๐Ÿ“ STEP 3: CREATE MISSING SMART-DEPLOY.SH SCRIPT${NC}" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" # Create the smart-deploy.sh script on the remote server cat > /tmp/smart-deploy.sh << 'SMART_DEPLOY_EOF' #!/usr/bin/env bash # # ThrillWiki Smart Deployment Script # One-shot deployment automation for timer-based execution # set -e # Configuration PROJECT_DIR="/home/thrillwiki/thrillwiki" LOG_DIR="$PROJECT_DIR/logs" LOG_FILE="$LOG_DIR/smart-deploy.log" # Ensure log directory exists mkdir -p "$LOG_DIR" # Logging function log_message() { local level="$1" local message="$2" local timestamp="$(date '+%Y-%m-%d %H:%M:%S')" echo "[$timestamp] [$level] [SMART-DEPLOY] $message" | tee -a "$LOG_FILE" } log_message "INFO" "Smart deployment started" # Change to project directory cd "$PROJECT_DIR" # Check for updates log_message "INFO" "Checking for repository updates" if git fetch origin main; then LOCAL_COMMIT=$(git rev-parse HEAD) REMOTE_COMMIT=$(git rev-parse origin/main) if [ "$LOCAL_COMMIT" != "$REMOTE_COMMIT" ]; then log_message "INFO" "Updates found, pulling changes" git pull origin main # Check if requirements changed if git diff --name-only HEAD~1 | grep -E "(pyproject.toml|requirements.*\.txt)" > /dev/null; then log_message "INFO" "Dependencies changed, updating packages" if command -v uv > /dev/null; then uv sync else pip install -r requirements.txt fi fi # Check if migrations are needed if command -v uv > /dev/null; then MIGRATION_CHECK=$(uv run manage.py showmigrations --plan | grep '\[ \]' || true) else MIGRATION_CHECK=$(python manage.py showmigrations --plan | grep '\[ \]' || true) fi if [ -n "$MIGRATION_CHECK" ]; then log_message "INFO" "Running database migrations" if command -v uv > /dev/null; then uv run manage.py migrate else python manage.py migrate fi fi # Collect static files if needed log_message "INFO" "Collecting static files" if command -v uv > /dev/null; then uv run manage.py collectstatic --noinput else python manage.py collectstatic --noinput fi log_message "INFO" "Deployment completed successfully" else log_message "INFO" "No updates available" fi else log_message "WARNING" "Failed to fetch updates" fi log_message "INFO" "Smart deployment finished" SMART_DEPLOY_EOF # Upload the smart-deploy.sh script echo -e "${YELLOW}Uploading smart-deploy.sh script...${NC}" if scp $SSH_OPTIONS -P $REMOTE_PORT /tmp/smart-deploy.sh "$REMOTE_USER@$REMOTE_HOST:[AWS-SECRET-REMOVED]t-deploy.sh" 2>/dev/null; then echo -e "${GREEN}โœ… smart-deploy.sh uploaded successfully${NC}" rm -f /tmp/smart-deploy.sh else echo -e "${RED}โŒ Failed to upload smart-deploy.sh${NC}" exit 1 fi # Make it executable run_remote "chmod +x [AWS-SECRET-REMOVED]t-deploy.sh" "Make smart-deploy.sh executable" echo "" # Step 4: Fix timer configuration echo -e "${BLUE}โฐ STEP 4: FIX TIMER CONFIGURATION${NC}" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" # Stop and disable timer first run_remote "systemctl stop thrillwiki-smart-deploy.timer" "Stop smart deploy timer" true run_remote "systemctl disable thrillwiki-smart-deploy.timer" "Disable smart deploy timer" true # Upload corrected service files echo -e "${YELLOW}Uploading corrected service files...${NC}" # Upload thrillwiki-smart-deploy.service if scp $SSH_OPTIONS -P $REMOTE_PORT "$PROJECT_DIR/scripts/systemd/thrillwiki-smart-deploy.service" "$REMOTE_USER@$REMOTE_HOST:/tmp/thrillwiki-smart-deploy.service" 2>/dev/null; then run_remote "sudo cp /tmp/thrillwiki-smart-deploy.service /etc/systemd/system/" "Install smart deploy service" run_remote "rm -f /tmp/thrillwiki-smart-deploy.service" "Clean up temp service file" else echo -e "${RED}โŒ Failed to upload smart deploy service${NC}" fi # Upload thrillwiki-smart-deploy.timer if scp $SSH_OPTIONS -P $REMOTE_PORT "$PROJECT_DIR/scripts/systemd/thrillwiki-smart-deploy.timer" "$REMOTE_USER@$REMOTE_HOST:/tmp/thrillwiki-smart-deploy.timer" 2>/dev/null; then run_remote "sudo cp /tmp/thrillwiki-smart-deploy.timer /etc/systemd/system/" "Install smart deploy timer" run_remote "rm -f /tmp/thrillwiki-smart-deploy.timer" "Clean up temp timer file" else echo -e "${RED}โŒ Failed to upload smart deploy timer${NC}" fi echo "" # Step 5: Reload systemd and enable proper services echo -e "${GREEN}๐Ÿ”„ STEP 5: RELOAD SYSTEMD AND ENABLE PROPER SERVICES${NC}" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" run_remote "systemctl daemon-reload" "Reload systemd configuration" true run_remote "systemctl enable thrillwiki-smart-deploy.service" "Enable smart deploy service" true run_remote "systemctl enable thrillwiki-smart-deploy.timer" "Enable smart deploy timer" true run_remote "systemctl start thrillwiki-smart-deploy.timer" "Start smart deploy timer" true echo "" # Step 6: Verify the fix echo -e "${GREEN}โœ… STEP 6: VERIFY THE FIX${NC}" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" echo -e "${YELLOW}Checking service status...${NC}" ssh $SSH_OPTIONS -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST "systemctl status thrillwiki-deployment.service --no-pager -l" || echo "โœ… Problematic service is stopped (expected)" echo "" ssh $SSH_OPTIONS -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST "systemctl status thrillwiki-smart-deploy.timer --no-pager -l" echo "" ssh $SSH_OPTIONS -p $REMOTE_PORT $REMOTE_USER@$REMOTE_HOST "systemctl status thrillwiki-smart-deploy.service --no-pager -l" echo "" echo -e "${GREEN}๐ŸŽ‰ EMERGENCY FIX COMPLETED SUCCESSFULLY!${NC}" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" echo "" echo -e "${GREEN}โœ… FIXED ISSUES:${NC}" echo "โ€ข Stopped infinite restart cycles" echo "โ€ข Disabled problematic continuous deployment service" echo "โ€ข Cleaned up stale lock files and processes" echo "โ€ข Created missing smart-deploy.sh script" echo "โ€ข Fixed timer configuration" echo "โ€ข Enabled proper timer-based automation" echo "" echo -e "${BLUE}๐Ÿ“‹ MONITORING COMMANDS:${NC}" echo "โ€ข Check timer status: ssh $REMOTE_USER@$REMOTE_HOST 'sudo systemctl status thrillwiki-smart-deploy.timer'" echo "โ€ข View deployment logs: ssh $REMOTE_USER@$REMOTE_HOST 'tail -f /home/thrillwiki/thrillwiki/logs/smart-deploy.log'" echo "โ€ข Test manual deployment: ssh $REMOTE_USER@$REMOTE_HOST '[AWS-SECRET-REMOVED]t-deploy.sh'" echo "" echo -e "${GREEN}โœ… System is now properly configured with timer-based automation!${NC}"