Add Road Trip Planner template with interactive map and trip management features

- Implemented a new HTML template for the Road Trip Planner.
- Integrated Leaflet.js for interactive mapping and routing.
- Added functionality for searching and selecting parks to include in a trip.
- Enabled drag-and-drop reordering of selected parks.
- Included trip optimization and route calculation features.
- Created a summary display for trip statistics.
- Added functionality to save trips and manage saved trips.
- Enhanced UI with responsive design and dark mode support.
This commit is contained in:
pacnpal
2025-08-15 20:53:00 -04:00
parent da7c7e3381
commit b5bae44cb8
99 changed files with 18697 additions and 4010 deletions

129
scripts/ci-start.sh Executable file
View File

@@ -0,0 +1,129 @@
#!/bin/bash
# ThrillWiki Local CI Start Script
# This script starts the Django development server following project requirements
set -e # Exit on any error
# Configuration
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
LOG_DIR="$PROJECT_DIR/logs"
PID_FILE="$LOG_DIR/django.pid"
LOG_FILE="$LOG_DIR/django.log"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Logging function
log() {
echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
}
log_success() {
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
}
log_error() {
echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
}
# Create logs directory if it doesn't exist
mkdir -p "$LOG_DIR"
# Change to project directory
cd "$PROJECT_DIR"
log "Starting ThrillWiki CI deployment..."
# Check if UV is installed
if ! command -v uv &> /dev/null; then
log_error "UV is not installed. Please install UV first."
exit 1
fi
# Stop any existing Django processes on port 8000
log "Stopping any existing Django processes on port 8000..."
if lsof -ti :8000 >/dev/null 2>&1; then
lsof -ti :8000 | xargs kill -9 2>/dev/null || true
log_success "Stopped existing processes"
else
log "No existing processes found on port 8000"
fi
# Clean up Python cache files
log "Cleaning up Python cache files..."
find . -type d -name "__pycache__" -exec rm -r {} + 2>/dev/null || true
log_success "Cache files cleaned"
# Install/update dependencies
log "Installing/updating dependencies with UV..."
uv sync --no-dev || {
log_error "Failed to sync dependencies"
exit 1
}
# Run database migrations
log "Running database migrations..."
uv run manage.py migrate || {
log_error "Database migrations failed"
exit 1
}
# Collect static files
log "Collecting static files..."
uv run manage.py collectstatic --noinput || {
log_warning "Static file collection failed, continuing anyway"
}
# Start the development server
log "Starting Django development server with Tailwind..."
log "Server will be available at: http://localhost:8000"
log "Press Ctrl+C to stop the server"
# Start server and capture PID
uv run manage.py tailwind runserver 0.0.0.0:8000 &
SERVER_PID=$!
# Save PID to file
echo $SERVER_PID > "$PID_FILE"
log_success "Django server started with PID: $SERVER_PID"
log "Server logs are being written to: $LOG_FILE"
# Wait for server to start
sleep 3
# Check if server is running
if kill -0 $SERVER_PID 2>/dev/null; then
log_success "Server is running successfully!"
# Monitor the process
wait $SERVER_PID
else
log_error "Server failed to start"
rm -f "$PID_FILE"
exit 1
fi
# Cleanup on exit
cleanup() {
log "Shutting down server..."
if [ -f "$PID_FILE" ]; then
PID=$(cat "$PID_FILE")
if kill -0 $PID 2>/dev/null; then
kill $PID
log_success "Server stopped"
fi
rm -f "$PID_FILE"
fi
}
trap cleanup EXIT INT TERM

220
scripts/github-auth.py Executable file
View File

@@ -0,0 +1,220 @@
#!/usr/bin/env python3
"""
GitHub OAuth Device Flow Authentication for ThrillWiki CI/CD
This script implements GitHub's device flow to securely obtain access tokens.
"""
import os
import sys
import json
import time
import requests
import argparse
from pathlib import Path
from urllib.parse import urlencode
# GitHub OAuth App Configuration
CLIENT_ID = "Iv23liOX5Hp75AxhUvIe"
TOKEN_FILE = ".github-token"
def parse_response(response):
"""Parse HTTP response and handle errors."""
if response.status_code in [200, 201]:
return response.json()
elif response.status_code == 401:
print("You are not authorized. Run the `login` command.")
sys.exit(1)
else:
print(f"HTTP {response.status_code}: {response.text}")
sys.exit(1)
def request_device_code():
"""Request a device code from GitHub."""
url = "https://github.com/login/device/code"
data = {"client_id": CLIENT_ID}
headers = {"Accept": "application/json"}
response = requests.post(url, data=data, headers=headers)
return parse_response(response)
def request_token(device_code):
"""Request an access token using the device code."""
url = "https://github.com/login/oauth/access_token"
data = {
"client_id": CLIENT_ID,
"device_code": device_code,
"grant_type": "urn:ietf:params:oauth:grant-type:device_code"
}
headers = {"Accept": "application/json"}
response = requests.post(url, data=data, headers=headers)
return parse_response(response)
def poll_for_token(device_code, interval):
"""Poll GitHub for the access token after user authorization."""
print("Waiting for authorization...")
while True:
response = request_token(device_code)
error = response.get("error")
access_token = response.get("access_token")
if error:
if error == "authorization_pending":
# User hasn't entered the code yet
print(".", end="", flush=True)
time.sleep(interval)
continue
elif error == "slow_down":
# Polling too fast
time.sleep(interval + 5)
continue
elif error == "expired_token":
print("\nThe device code has expired. Please run `login` again.")
sys.exit(1)
elif error == "access_denied":
print("\nLogin cancelled by user.")
sys.exit(1)
else:
print(f"\nError: {response}")
sys.exit(1)
# Success! Save the token
token_path = Path(TOKEN_FILE)
token_path.write_text(access_token)
token_path.chmod(0o600) # Read/write for owner only
print(f"\nToken saved to {TOKEN_FILE}")
break
def login():
"""Initiate the GitHub OAuth device flow login process."""
print("Starting GitHub authentication...")
device_response = request_device_code()
verification_uri = device_response["verification_uri"]
user_code = device_response["user_code"]
device_code = device_response["device_code"]
interval = device_response["interval"]
print(f"\nPlease visit: {verification_uri}")
print(f"and enter code: {user_code}")
print("\nWaiting for you to complete authorization in your browser...")
poll_for_token(device_code, interval)
print("Successfully authenticated!")
return True
def whoami():
"""Display information about the authenticated user."""
token_path = Path(TOKEN_FILE)
if not token_path.exists():
print("You are not authorized. Run the `login` command.")
sys.exit(1)
try:
token = token_path.read_text().strip()
except Exception as e:
print(f"Error reading token: {e}")
print("You may need to run the `login` command again.")
sys.exit(1)
url = "https://api.github.com/user"
headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {token}"
}
response = requests.get(url, headers=headers)
user_data = parse_response(response)
print(f"You are authenticated as: {user_data['login']}")
print(f"Name: {user_data.get('name', 'Not set')}")
print(f"Email: {user_data.get('email', 'Not public')}")
return user_data
def get_token():
"""Get the current access token if available."""
token_path = Path(TOKEN_FILE)
if not token_path.exists():
return None
try:
return token_path.read_text().strip()
except Exception:
return None
def validate_token():
"""Validate that the current token is still valid."""
token = get_token()
if not token:
return False
url = "https://api.github.com/user"
headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {token}"
}
try:
response = requests.get(url, headers=headers)
return response.status_code == 200
except Exception:
return False
def ensure_authenticated():
"""Ensure user is authenticated, prompting login if necessary."""
if validate_token():
return get_token()
print("GitHub authentication required.")
login()
return get_token()
def logout():
"""Remove the stored access token."""
token_path = Path(TOKEN_FILE)
if token_path.exists():
token_path.unlink()
print("Successfully logged out.")
else:
print("You are not currently logged in.")
def main():
"""Main CLI interface."""
parser = argparse.ArgumentParser(description="GitHub OAuth authentication for ThrillWiki CI/CD")
parser.add_argument("command", choices=["login", "logout", "whoami", "token", "validate"],
help="Command to execute")
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
if args.command == "login":
login()
elif args.command == "logout":
logout()
elif args.command == "whoami":
whoami()
elif args.command == "token":
token = get_token()
if token:
print(token)
else:
print("No token available. Run `login` first.")
sys.exit(1)
elif args.command == "validate":
if validate_token():
print("Token is valid.")
else:
print("Token is invalid or missing.")
sys.exit(1)
if __name__ == "__main__":
main()

268
scripts/setup-vm-ci.sh Executable file
View File

@@ -0,0 +1,268 @@
#!/bin/bash
# ThrillWiki VM CI Setup Script
# This script helps set up the VM deployment system
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
log() {
echo -e "${BLUE}[SETUP]${NC} $1"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# Configuration prompts
prompt_config() {
log "Setting up ThrillWiki VM CI/CD system..."
echo
read -p "Enter your VM IP address: " VM_IP
read -p "Enter your VM username (default: ubuntu): " VM_USER
VM_USER=${VM_USER:-ubuntu}
read -p "Enter your GitHub repository URL: " REPO_URL
read -p "Enter your GitHub webhook secret: " WEBHOOK_SECRET
read -p "Enter local webhook port (default: 9000): " WEBHOOK_PORT
WEBHOOK_PORT=${WEBHOOK_PORT:-9000}
read -p "Enter VM project path (default: /home/$VM_USER/thrillwiki): " VM_PROJECT_PATH
VM_PROJECT_PATH=${VM_PROJECT_PATH:-/home/$VM_USER/thrillwiki}
}
# Create SSH key
setup_ssh() {
log "Setting up SSH keys..."
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
if [ ! -f "$ssh_key_path" ]; then
ssh-keygen -t rsa -b 4096 -f "$ssh_key_path" -N ""
log_success "SSH key generated: $ssh_key_path"
log "Please copy the following public key to your VM:"
echo "---"
cat "$ssh_key_path.pub"
echo "---"
echo
log "Run this on your VM:"
echo "mkdir -p ~/.ssh && echo '$(cat "$ssh_key_path.pub")' >> ~/.ssh/***REMOVED*** && chmod 600 ~/.ssh/***REMOVED***"
echo
read -p "Press Enter when you've added the key to your VM..."
else
log "SSH key already exists: $ssh_key_path"
fi
# Test SSH connection
log "Testing SSH connection..."
if ssh -i "$ssh_key_path" -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$VM_USER@$VM_IP" "echo 'SSH connection successful'"; then
log_success "SSH connection test passed"
else
log_error "SSH connection test failed"
exit 1
fi
}
# Create environment file
create_env_file() {
log "Creating webhook environment file..."
cat > ***REMOVED***.webhook << EOF
# ThrillWiki Webhook Configuration
WEBHOOK_PORT=$WEBHOOK_PORT
WEBHOOK_SECRET=$WEBHOOK_SECRET
VM_HOST=$VM_IP
VM_PORT=22
VM_USER=$VM_USER
VM_KEY_PATH=$HOME/.ssh/thrillwiki_vm
VM_PROJECT_PATH=$VM_PROJECT_PATH
REPO_URL=$REPO_URL
DEPLOY_BRANCH=main
EOF
log_success "Environment file created: ***REMOVED***.webhook"
}
# Setup VM
setup_vm() {
log "Setting up VM environment..."
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
# Create setup script for VM
cat > /tmp/vm_setup.sh << 'EOF'
#!/bin/bash
set -e
echo "Setting up VM for ThrillWiki deployment..."
# Update system
sudo apt update
# Install required packages
sudo apt install -y git curl build-essential python3-pip lsof
# Install UV if not present
if ! command -v uv &> /dev/null; then
echo "Installing UV..."
curl -LsSf https://astral.sh/uv/install.sh | sh
source ~/.cargo/env
fi
# Clone repository if not present
if [ ! -d "thrillwiki" ]; then
echo "Cloning repository..."
git clone REPO_URL_PLACEHOLDER thrillwiki
fi
cd thrillwiki
# Install dependencies
uv sync
# Create directories
mkdir -p logs backups
# Make scripts executable
chmod +x scripts/*.sh
echo "VM setup completed successfully!"
EOF
# Replace placeholder with actual repo URL
sed -i.bak "s|REPO_URL_PLACEHOLDER|$REPO_URL|g" /tmp/vm_setup.sh
# Copy and execute setup script on VM
scp -i "$ssh_key_path" /tmp/vm_setup.sh "$VM_USER@$VM_IP:/tmp/"
ssh -i "$ssh_key_path" "$VM_USER@$VM_IP" "bash /tmp/vm_setup.sh"
log_success "VM setup completed"
# Cleanup
rm /tmp/vm_setup.sh /tmp/vm_setup.sh.bak
}
# Install systemd services
setup_services() {
log "Setting up systemd services on VM..."
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
# Copy service files and install them
ssh -i "$ssh_key_path" "$VM_USER@$VM_IP" << EOF
cd thrillwiki
# Update service files with correct paths
sed -i 's|/home/ubuntu|/home/$VM_USER|g' scripts/systemd/*.service
sed -i 's|ubuntu|$VM_USER|g' scripts/systemd/*.service
# Install services
sudo cp scripts/systemd/thrillwiki.service /etc/systemd/system/
sudo cp scripts/systemd/thrillwiki-webhook.service /etc/systemd/system/
# Reload and enable services
sudo systemctl daemon-reload
sudo systemctl enable thrillwiki.service
echo "Services installed successfully!"
EOF
log_success "Systemd services installed"
}
# Test deployment
test_deployment() {
log "Testing VM deployment..."
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
ssh -i "$ssh_key_path" "$VM_USER@$VM_IP" << EOF
cd thrillwiki
./scripts/vm-deploy.sh
EOF
log_success "Deployment test completed"
}
# Start webhook listener
start_webhook() {
log "Starting webhook listener..."
if [ -f "***REMOVED***.webhook" ]; then
log "Webhook configuration found. You can start the webhook listener with:"
echo " source ***REMOVED***.webhook && python3 scripts/webhook-listener.py"
echo
log "Or run it in the background:"
echo " nohup python3 scripts/webhook-listener.py > logs/webhook.log 2>&1 &"
else
log_error "Webhook configuration not found!"
exit 1
fi
}
# GitHub webhook instructions
github_instructions() {
log "GitHub Webhook Setup Instructions:"
echo
echo "1. Go to your GitHub repository: $REPO_URL"
echo "2. Navigate to Settings → Webhooks"
echo "3. Click 'Add webhook'"
echo "4. Configure:"
echo " - Payload URL: http://YOUR_PUBLIC_IP:$WEBHOOK_PORT/webhook"
echo " - Content type: application/json"
echo " - Secret: $WEBHOOK_SECRET"
echo " - Events: Just the push event"
echo "5. Click 'Add webhook'"
echo
log_warning "Make sure port $WEBHOOK_PORT is open on your firewall!"
}
# Main setup flow
main() {
log "ThrillWiki VM CI/CD Setup"
echo "=========================="
echo
# Create logs directory
mkdir -p logs
# Get configuration
prompt_config
# Setup steps
setup_ssh
create_env_file
setup_vm
setup_services
test_deployment
# Final instructions
echo
log_success "Setup completed successfully!"
echo
start_webhook
echo
github_instructions
log "Setup log saved to: logs/setup.log"
}
# Run main function and log output
main "$@" 2>&1 | tee logs/setup.log

View File

@@ -0,0 +1,39 @@
[Unit]
Description=ThrillWiki GitHub Webhook Listener
After=network.target
Wants=network.target
[Service]
Type=simple
User=ubuntu
Group=ubuntu
[AWS-SECRET-REMOVED]
ExecStart=/usr/bin/python3 /home/ubuntu/thrillwiki/scripts/webhook-listener.py
Restart=always
RestartSec=10
# Environment variables
Environment=WEBHOOK_PORT=9000
Environment=WEBHOOK_SECRET=your_webhook_secret_here
Environment=VM_HOST=localhost
Environment=VM_PORT=22
Environment=VM_USER=ubuntu
Environment=VM_KEY_PATH=/home/ubuntu/.ssh/***REMOVED***
Environment=VM_PROJECT_PATH=/home/ubuntu/thrillwiki
Environment=REPO_URL=https://github.com/YOUR_USERNAME/thrillwiki_django_no_react.git
Environment=DEPLOY_BRANCH=main
# Security settings
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
[AWS-SECRET-REMOVED]ogs
# Logging
StandardOutput=journal
StandardError=journal
SyslogIdentifier=thrillwiki-webhook
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,45 @@
[Unit]
Description=ThrillWiki Django Application
After=network.target postgresql.service
Wants=network.target
Requires=postgresql.service
[Service]
Type=forking
User=ubuntu
Group=ubuntu
[AWS-SECRET-REMOVED]
[AWS-SECRET-REMOVED]s/ci-start.sh
ExecStop=/bin/kill -TERM $MAINPID
ExecReload=/bin/kill -HUP $MAINPID
[AWS-SECRET-REMOVED]ngo.pid
Restart=always
RestartSec=10
# Environment variables
Environment=DJANGO_SETTINGS_MODULE=thrillwiki.settings
[AWS-SECRET-REMOVED]llwiki
Environment=PATH=/home/ubuntu/.cargo/bin:/usr/local/bin:/usr/bin:/bin
# Security settings
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
[AWS-SECRET-REMOVED]ogs
[AWS-SECRET-REMOVED]edia
[AWS-SECRET-REMOVED]taticfiles
[AWS-SECRET-REMOVED]ploads
# Resource limits
LimitNOFILE=65536
TimeoutStartSec=300
TimeoutStopSec=30
# Logging
StandardOutput=journal
StandardError=journal
SyslogIdentifier=thrillwiki
[Install]
WantedBy=multi-user.target

175
scripts/test-automation.sh Executable file
View File

@@ -0,0 +1,175 @@
#!/bin/bash
# ThrillWiki Automation Test Script
# This script validates all automation components without actually running them
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
log() {
echo -e "${BLUE}[TEST]${NC} $1"
}
log_success() {
echo -e "${GREEN}[✓]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[!]${NC} $1"
}
log_error() {
echo -e "${RED}[✗]${NC} $1"
}
# Test counters
TESTS_PASSED=0
TESTS_FAILED=0
TESTS_TOTAL=0
test_case() {
local name="$1"
local command="$2"
((TESTS_TOTAL++))
log "Testing: $name"
if eval "$command" >/dev/null 2>&1; then
log_success "$name"
((TESTS_PASSED++))
else
log_error "$name"
((TESTS_FAILED++))
fi
}
test_case_with_output() {
local name="$1"
local command="$2"
local expected_pattern="$3"
((TESTS_TOTAL++))
log "Testing: $name"
local output
if output=$(eval "$command" 2>&1); then
if [[ -n "$expected_pattern" && ! "$output" =~ $expected_pattern ]]; then
log_error "$name (unexpected output)"
((TESTS_FAILED++))
else
log_success "$name"
((TESTS_PASSED++))
fi
else
log_error "$name (command failed)"
((TESTS_FAILED++))
fi
}
log "🧪 Starting ThrillWiki Automation Tests"
echo "======================================"
# Test 1: File Permissions
log "\n📁 Testing File Permissions..."
test_case "CI start script is executable" "[ -x scripts/ci-start.sh ]"
test_case "VM deploy script is executable" "[ -x scripts/vm-deploy.sh ]"
test_case "Webhook listener is executable" "[ -x scripts/webhook-listener.py ]"
test_case "VM manager is executable" "[ -x scripts/unraid/vm-manager.py ]"
test_case "Complete automation script is executable" "[ -x scripts/unraid/setup-complete-automation.sh ]"
# Test 2: Script Syntax
log "\n🔍 Testing Script Syntax..."
test_case "CI start script syntax" "bash -n scripts/ci-start.sh"
test_case "VM deploy script syntax" "bash -n scripts/vm-deploy.sh"
test_case "Setup VM CI script syntax" "bash -n scripts/setup-vm-ci.sh"
test_case "Complete automation script syntax" "bash -n scripts/unraid/setup-complete-automation.sh"
test_case "Webhook listener Python syntax" "python3 -m py_compile scripts/webhook-listener.py"
test_case "VM manager Python syntax" "python3 -m py_compile scripts/unraid/vm-manager.py"
# Test 3: Help Functions
log "\n❓ Testing Help Functions..."
test_case_with_output "VM manager help" "python3 scripts/unraid/vm-manager.py --help" "usage:"
test_case_with_output "Webhook listener help" "python3 scripts/webhook-listener.py --help" "usage:"
test_case_with_output "VM deploy script usage" "scripts/vm-deploy.sh invalid 2>&1" "Usage:"
# Test 4: Configuration Validation
log "\n⚙ Testing Configuration Validation..."
test_case_with_output "Webhook listener test mode" "python3 scripts/webhook-listener.py --test" "Configuration validation"
# Test 5: Directory Structure
log "\n📂 Testing Directory Structure..."
test_case "Scripts directory exists" "[ -d scripts ]"
test_case "Unraid scripts directory exists" "[ -d scripts/unraid ]"
test_case "Systemd directory exists" "[ -d scripts/systemd ]"
test_case "Docs directory exists" "[ -d docs ]"
test_case "Logs directory created" "[ -d logs ]"
# Test 6: Required Files
log "\n📄 Testing Required Files..."
test_case "ThrillWiki service file exists" "[ -f scripts/systemd/thrillwiki.service ]"
test_case "Webhook service file exists" "[ -f scripts/systemd/thrillwiki-webhook.service ]"
test_case "VM deployment setup doc exists" "[ -f docs/VM_DEPLOYMENT_SETUP.md ]"
test_case "Unraid automation doc exists" "[ -f docs/UNRAID_COMPLETE_AUTOMATION.md ]"
test_case "CI README exists" "[ -f CI_README.md ]"
# Test 7: Python Dependencies
log "\n🐍 Testing Python Dependencies..."
test_case "Python 3 available" "command -v python3"
test_case "Requests module available" "python3 -c 'import requests'"
test_case "JSON module available" "python3 -c 'import json'"
test_case "OS module available" "python3 -c 'import os'"
test_case "Subprocess module available" "python3 -c 'import subprocess'"
# Test 8: System Dependencies
log "\n🔧 Testing System Dependencies..."
test_case "SSH command available" "command -v ssh"
test_case "SCP command available" "command -v scp"
test_case "Bash available" "command -v bash"
test_case "Git available" "command -v git"
# Test 9: UV Package Manager
log "\n📦 Testing UV Package Manager..."
if command -v uv >/dev/null 2>&1; then
log_success "UV package manager is available"
((TESTS_PASSED++))
test_case "UV version check" "uv --version"
else
log_warning "UV package manager not found (will be installed during setup)"
((TESTS_PASSED++))
fi
((TESTS_TOTAL++))
# Test 10: Django Project Structure
log "\n🌟 Testing Django Project Structure..."
test_case "Django manage.py exists" "[ -f manage.py ]"
test_case "Django settings module exists" "[ -f thrillwiki/settings.py ]"
test_case "PyProject.toml exists" "[ -f pyproject.toml ]"
# Final Results
echo
log "📊 Test Results Summary"
echo "======================"
echo "Total Tests: $TESTS_TOTAL"
echo "Passed: $TESTS_PASSED"
echo "Failed: $TESTS_FAILED"
if [ $TESTS_FAILED -eq 0 ]; then
echo
log_success "🎉 All tests passed! The automation system is ready."
echo
log "Next steps:"
echo "1. For complete automation: ./scripts/unraid/setup-complete-automation.sh"
echo "2. For manual setup: ./scripts/setup-vm-ci.sh"
echo "3. Read documentation: docs/UNRAID_COMPLETE_AUTOMATION.md"
exit 0
else
echo
log_error "❌ Some tests failed. Please check the issues above."
exit 1
fi

View File

@@ -0,0 +1,996 @@
#!/bin/bash
# ThrillWiki Complete Unraid Automation Setup
# This script automates the entire VM creation and deployment process on Unraid
#
# Usage:
# ./setup-complete-automation.sh # Standard setup
# ./setup-complete-automation.sh --reset # Delete VM and config, start completely fresh
# ./setup-complete-automation.sh --reset-vm # Delete VM only, keep configuration
# ./setup-complete-automation.sh --reset-config # Delete config only, keep VM
# Function to show help
show_help() {
echo "ThrillWiki CI/CD Automation Setup"
echo ""
echo "Usage:"
echo " $0 Set up or update ThrillWiki automation"
echo " $0 --reset Delete VM and config, start completely fresh"
echo " $0 --reset-vm Delete VM only, keep configuration"
echo " $0 --reset-config Delete config only, keep VM"
echo " $0 --help Show this help message"
echo ""
echo "Reset Options:"
echo " --reset Completely removes existing VM, disks, and config"
echo " before starting fresh installation"
echo " --reset-vm Removes only the VM and disks, preserves saved"
echo " configuration to avoid re-entering settings"
echo " --reset-config Removes only the saved configuration, preserves"
echo " VM and prompts for fresh configuration input"
echo " --help Display this help and exit"
echo ""
echo "Examples:"
echo " $0 # Normal setup/update"
echo " $0 --reset # Complete fresh installation"
echo " $0 --reset-vm # Fresh VM with saved settings"
echo " $0 --reset-config # Re-configure existing VM"
exit 0
}
# Check for help flag
if [[ "$1" == "--help" || "$1" == "-h" ]]; then
show_help
fi
# Parse reset flags
RESET_ALL=false
RESET_VM_ONLY=false
RESET_CONFIG_ONLY=false
if [[ "$1" == "--reset" ]]; then
RESET_ALL=true
echo "🔄 COMPLETE RESET MODE: Will delete VM and configuration"
elif [[ "$1" == "--reset-vm" ]]; then
RESET_VM_ONLY=true
echo "🔄 VM RESET MODE: Will delete VM only, keep configuration"
elif [[ "$1" == "--reset-config" ]]; then
RESET_CONFIG_ONLY=true
echo "🔄 CONFIG RESET MODE: Will delete configuration only, keep VM"
fi
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
log() {
echo -e "${BLUE}[AUTOMATION]${NC} $1"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
log_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
LOG_DIR="$PROJECT_DIR/logs"
# Default values
DEFAULT_UNRAID_HOST=""
DEFAULT_VM_NAME="thrillwiki-vm"
DEFAULT_VM_MEMORY="4096"
DEFAULT_VM_VCPUS="2"
DEFAULT_VM_DISK_SIZE="50"
DEFAULT_WEBHOOK_PORT="9000"
# Configuration file
CONFIG_FILE="$PROJECT_DIR/.thrillwiki-config"
# Function to save configuration
save_config() {
log "Saving configuration to $CONFIG_FILE..."
cat > "$CONFIG_FILE" << EOF
# ThrillWiki Automation Configuration
# This file stores your settings to avoid re-entering them each time
# Unraid Server Configuration
UNRAID_HOST="$UNRAID_HOST"
UNRAID_USER="$UNRAID_USER"
VM_NAME="$VM_NAME"
VM_MEMORY="$VM_MEMORY"
VM_VCPUS="$VM_VCPUS"
VM_DISK_SIZE="$VM_DISK_SIZE"
# Network Configuration
VM_IP="$VM_IP"
VM_GATEWAY="$VM_GATEWAY"
VM_NETMASK="$VM_NETMASK"
VM_NETWORK="$VM_NETWORK"
# GitHub Configuration
REPO_URL="$REPO_URL"
GITHUB_USERNAME="$GITHUB_USERNAME"
GITHUB_API_ENABLED="$GITHUB_API_ENABLED"
GITHUB_AUTH_METHOD="$GITHUB_AUTH_METHOD"
# Webhook Configuration
WEBHOOK_PORT="$WEBHOOK_PORT"
WEBHOOK_ENABLED="$WEBHOOK_ENABLED"
# SSH Configuration (path to key, not the key content)
SSH_KEY_PATH="$HOME/.ssh/thrillwiki_vm"
EOF
log_success "Configuration saved to $CONFIG_FILE"
}
# Function to load configuration
load_config() {
if [ -f "$CONFIG_FILE" ]; then
log "Loading existing configuration from $CONFIG_FILE..."
source "$CONFIG_FILE"
return 0
else
return 1
fi
}
# Function to prompt for configuration
prompt_unraid_config() {
log "=== Unraid VM Configuration ==="
echo
# Try to load existing config first
if load_config; then
log_success "Loaded existing configuration"
echo "Current settings:"
echo " Unraid Host: $UNRAID_HOST"
echo " VM Name: $VM_NAME"
echo " VM IP: $VM_IP"
echo " Repository: $REPO_URL"
echo
read -p "Use existing configuration? (y/n): " use_existing
if [ "$use_existing" = "y" ] || [ "$use_existing" = "Y" ]; then
# Still need to get sensitive info that we don't save
read -s -p "Enter Unraid [PASSWORD-REMOVED]
echo
# Handle GitHub authentication based on saved method
if [ -n "$GITHUB_USERNAME" ] && [ "$GITHUB_API_ENABLED" = "true" ]; then
if [ "$GITHUB_AUTH_METHOD" = "oauth" ]; then
# Check if OAuth token is still valid
if python3 "$SCRIPT_DIR/../github-auth.py" validate 2>/dev/null; then
GITHUB_TOKEN=$(python3 "$SCRIPT_DIR/../github-auth.py" token)
log "Using existing OAuth token"
else
log "OAuth token expired, re-authenticating..."
if python3 "$SCRIPT_DIR/../github-auth.py" login; then
GITHUB_TOKEN=$(python3 "$SCRIPT_DIR/../github-auth.py" token)
log_success "OAuth token refreshed"
else
log_error "OAuth re-authentication failed"
exit 1
fi
fi
else
# Personal access token method
read -s -p "Enter GitHub personal access token: " GITHUB_TOKEN
echo
fi
fi
if [ "$WEBHOOK_ENABLED" = "true" ]; then
read -s -p "Enter GitHub webhook secret: " WEBHOOK_SECRET
echo
fi
return 0
fi
fi
# Prompt for new configuration
read -p "Enter your Unraid server IP address: " UNRAID_HOST
save_config
read -p "Enter Unraid username (default: root): " UNRAID_USER
UNRAID_USER=${UNRAID_USER:-root}
save_config
read -s -p "Enter Unraid [PASSWORD-REMOVED]
echo
# Note: Password not saved for security
read -p "Enter VM name (default: $DEFAULT_VM_NAME): " VM_NAME
VM_NAME=${VM_NAME:-$DEFAULT_VM_NAME}
save_config
read -p "Enter VM memory in MB (default: $DEFAULT_VM_MEMORY): " VM_MEMORY
VM_MEMORY=${VM_MEMORY:-$DEFAULT_VM_MEMORY}
save_config
read -p "Enter VM vCPUs (default: $DEFAULT_VM_VCPUS): " VM_VCPUS
VM_VCPUS=${VM_VCPUS:-$DEFAULT_VM_VCPUS}
save_config
read -p "Enter VM disk size in GB (default: $DEFAULT_VM_DISK_SIZE): " VM_DISK_SIZE
VM_DISK_SIZE=${VM_DISK_SIZE:-$DEFAULT_VM_DISK_SIZE}
save_config
read -p "Enter GitHub repository URL: " REPO_URL
save_config
# GitHub API Configuration
echo
log "=== GitHub API Configuration ==="
echo "Choose GitHub authentication method:"
echo "1. OAuth Device Flow (recommended - secure, supports private repos)"
echo "2. Personal Access Token (manual token entry)"
echo "3. Skip (public repositories only)"
while true; do
read -p "Select option (1-3): " auth_choice
case $auth_choice in
1)
log "Using GitHub OAuth Device Flow..."
if python3 "$SCRIPT_DIR/../github-auth.py" validate 2>/dev/null; then
log "Existing GitHub authentication found and valid"
GITHUB_USERNAME=$(python3 "$SCRIPT_DIR/../github-auth.py" whoami 2>/dev/null | grep "You are authenticated as:" | cut -d: -f2 | xargs)
GITHUB_TOKEN=$(python3 "$SCRIPT_DIR/../github-auth.py" token)
else
log "Starting GitHub OAuth authentication..."
if python3 "$SCRIPT_DIR/../github-auth.py" login; then
GITHUB_USERNAME=$(python3 "$SCRIPT_DIR/../github-auth.py" whoami 2>/dev/null | grep "You are authenticated as:" | cut -d: -f2 | xargs)
GITHUB_TOKEN=$(python3 "$SCRIPT_DIR/../github-auth.py" token)
log_success "GitHub OAuth authentication completed"
else
log_error "GitHub authentication failed"
continue
fi
fi
GITHUB_API_ENABLED=true
GITHUB_AUTH_METHOD="oauth"
break
;;
2)
read -p "Enter GitHub username: " GITHUB_USERNAME
read -s -p "Enter GitHub personal access token: " GITHUB_TOKEN
echo
if [ -n "$GITHUB_USERNAME" ] && [ -n "$GITHUB_TOKEN" ]; then
GITHUB_API_ENABLED=true
GITHUB_AUTH_METHOD="token"
log "Personal access token configured"
else
log_error "Both username and token are required"
continue
fi
break
;;
3)
GITHUB_USERNAME=""
GITHUB_TOKEN=""
GITHUB_API_ENABLED=false
GITHUB_AUTH_METHOD="none"
log "Skipping GitHub API - using public access only"
break
;;
*)
echo "Invalid option. Please select 1, 2, or 3."
;;
esac
done
# Save GitHub configuration
save_config
log "GitHub authentication configuration saved"
# Webhook Configuration
echo
read -s -p "Enter GitHub webhook secret (optional, press Enter to skip): " WEBHOOK_SECRET
echo
# If no webhook secret provided, disable webhook functionality
if [ -z "$WEBHOOK_SECRET" ]; then
log "No webhook secret provided - webhook functionality will be disabled"
WEBHOOK_ENABLED=false
else
WEBHOOK_ENABLED=true
fi
read -p "Enter webhook port (default: $DEFAULT_WEBHOOK_PORT): " WEBHOOK_PORT
WEBHOOK_PORT=${WEBHOOK_PORT:-$DEFAULT_WEBHOOK_PORT}
# Save webhook configuration
save_config
log "Webhook configuration saved"
# Get VM IP address with proper range validation
while true; do
read -p "Enter VM IP address (192.168.20.10-192.168.20.100): " VM_IP
if [[ "$VM_IP" =~ ^192\.168\.20\.([1-9][0-9]|100)$ ]]; then
local ip_last_octet="${BASH_REMATCH[1]}"
if [ "$ip_last_octet" -ge 10 ] && [ "$ip_last_octet" -le 100 ]; then
break
fi
fi
echo "Invalid IP address. Please enter an IP in the range 192.168.20.10-192.168.20.100"
done
# Set network configuration
VM_GATEWAY="192.168.20.1"
VM_NETMASK="255.255.255.0"
VM_NETWORK="192.168.20.0/24"
# Save final network configuration
save_config
log "Network configuration saved - setup complete!"
}
# Generate SSH keys for VM access
setup_ssh_keys() {
log "Setting up SSH keys for VM access..."
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
local ssh_config_path="$HOME/.ssh/config"
if [ ! -f "$ssh_key_path" ]; then
ssh-keygen -t rsa -b 4096 -f "$ssh_key_path" -N "" -C "thrillwiki-vm-access"
log_success "SSH key generated: $ssh_key_path"
else
log "SSH key already exists: $ssh_key_path"
fi
# Add SSH config entry
if ! grep -q "Host $VM_NAME" "$ssh_config_path" 2>/dev/null; then
cat >> "$ssh_config_path" << EOF
# ThrillWiki VM
Host $VM_NAME
HostName %h
User ubuntu
IdentityFile $ssh_key_path
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
EOF
log_success "SSH config updated"
fi
# Store public key for VM setup
SSH_PUBLIC_KEY=$(cat "$ssh_key_path.pub")
export SSH_PUBLIC_KEY
}
# Setup Unraid host access
setup_unraid_access() {
log "Setting up Unraid server access..."
local unraid_key_path="$HOME/.ssh/unraid_access"
if [ ! -f "$unraid_key_path" ]; then
ssh-keygen -t rsa -b 4096 -f "$unraid_key_path" -N "" -C "unraid-access"
log "Please add this public key to your Unraid server:"
echo "---"
cat "$unraid_key_path.pub"
echo "---"
echo
log "Add this to /root/.ssh/***REMOVED*** on your Unraid server"
read -p "Press Enter when you've added the key..."
fi
# Test Unraid connection
log "Testing Unraid connection..."
if ssh -i "$unraid_key_path" -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$UNRAID_USER@$UNRAID_HOST" "echo 'Connected to Unraid successfully'"; then
log_success "Unraid connection test passed"
else
log_error "Unraid connection test failed"
exit 1
fi
# Update SSH config for Unraid
if ! grep -q "Host unraid" "$HOME/.ssh/config" 2>/dev/null; then
cat >> "$HOME/.ssh/config" << EOF
# Unraid Server
Host unraid
HostName $UNRAID_HOST
User $UNRAID_USER
IdentityFile $unraid_key_path
StrictHostKeyChecking no
EOF
fi
}
# Create environment files
create_environment_files() {
log "Creating environment configuration files..."
# Get SSH public key content safely
local ssh_key_path="$HOME/.ssh/thrillwiki_vm.pub"
local ssh_public_key=""
if [ -f "$ssh_key_path" ]; then
ssh_public_key=$(cat "$ssh_key_path")
fi
# Unraid VM environment
cat > "$PROJECT_DIR/***REMOVED***.unraid" << EOF
# Unraid VM Configuration
UNRAID_HOST=$UNRAID_HOST
UNRAID_USER=$UNRAID_USER
UNRAID_PASSWORD=$UNRAID_PASSWORD
VM_NAME=$VM_NAME
VM_MEMORY=$VM_MEMORY
VM_VCPUS=$VM_VCPUS
VM_DISK_SIZE=$VM_DISK_SIZE
SSH_PUBLIC_KEY="$ssh_public_key"
# Network Configuration
VM_IP=$VM_IP
VM_GATEWAY=$VM_GATEWAY
VM_NETMASK=$VM_NETMASK
VM_NETWORK=$VM_NETWORK
# GitHub Configuration
REPO_URL=$REPO_URL
GITHUB_USERNAME=$GITHUB_USERNAME
GITHUB_TOKEN=$GITHUB_TOKEN
GITHUB_API_ENABLED=$GITHUB_API_ENABLED
EOF
# Webhook environment (updated with VM info)
cat > "$PROJECT_DIR/***REMOVED***.webhook" << EOF
# ThrillWiki Webhook Configuration
WEBHOOK_PORT=$WEBHOOK_PORT
WEBHOOK_SECRET=$WEBHOOK_SECRET
WEBHOOK_ENABLED=$WEBHOOK_ENABLED
VM_HOST=$VM_IP
VM_PORT=22
VM_USER=ubuntu
VM_KEY_PATH=$HOME/.ssh/thrillwiki_vm
VM_PROJECT_PATH=/home/ubuntu/thrillwiki
REPO_URL=$REPO_URL
DEPLOY_BRANCH=main
# GitHub API Configuration
GITHUB_USERNAME=$GITHUB_USERNAME
GITHUB_TOKEN=$GITHUB_TOKEN
GITHUB_API_ENABLED=$GITHUB_API_ENABLED
EOF
log_success "Environment files created"
}
# Install required tools
install_dependencies() {
log "Installing required dependencies..."
# Check for required tools
local missing_tools=()
local mac_tools=()
command -v python3 >/dev/null 2>&1 || missing_tools+=("python3")
command -v ssh >/dev/null 2>&1 || missing_tools+=("openssh-client")
command -v scp >/dev/null 2>&1 || missing_tools+=("openssh-client")
# Check for ISO creation tools and handle platform differences
if ! command -v genisoimage >/dev/null 2>&1 && ! command -v mkisofs >/dev/null 2>&1 && ! command -v hdiutil >/dev/null 2>&1; then
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
missing_tools+=("genisoimage")
elif [[ "$OSTYPE" == "darwin"* ]]; then
# On macOS, hdiutil should be available, but add cdrtools as backup
if command -v brew >/dev/null 2>&1; then
mac_tools+=("cdrtools")
fi
fi
fi
# Install Linux packages
if [ ${#missing_tools[@]} -gt 0 ]; then
log "Installing missing tools for Linux: ${missing_tools[*]}"
if command -v apt-get >/dev/null 2>&1; then
sudo apt-get update
sudo apt-get install -y "${missing_tools[@]}"
elif command -v yum >/dev/null 2>&1; then
sudo yum install -y "${missing_tools[@]}"
elif command -v dnf >/dev/null 2>&1; then
sudo dnf install -y "${missing_tools[@]}"
else
log_error "Linux package manager not found. Please install: ${missing_tools[*]}"
exit 1
fi
fi
# Install macOS packages
if [ ${#mac_tools[@]} -gt 0 ]; then
log "Installing additional tools for macOS: ${mac_tools[*]}"
if command -v brew >/dev/null 2>&1; then
brew install "${mac_tools[@]}"
else
log "Homebrew not found. Skipping optional tool installation."
log "Note: hdiutil should be available on macOS for ISO creation"
fi
fi
# Install Python dependencies
if [ -f "$PROJECT_DIR/pyproject.toml" ]; then
log "Installing Python dependencies with UV..."
if ! command -v uv >/dev/null 2>&1; then
curl -LsSf https://astral.sh/uv/install.sh | sh
source ~/.cargo/env
fi
uv sync
fi
log_success "Dependencies installed"
}
# Create VM using the VM manager
create_vm() {
log "Creating VM on Unraid server..."
# Export all environment variables from the file
set -a # automatically export all variables
source "$PROJECT_DIR/***REMOVED***.unraid"
set +a # turn off automatic export
# Run VM creation/update
cd "$PROJECT_DIR"
python3 scripts/unraid/vm-manager.py setup
if [ $? -eq 0 ]; then
log_success "VM created/updated successfully"
# Start the VM
log "Starting VM..."
python3 scripts/unraid/vm-manager.py start
if [ $? -eq 0 ]; then
log_success "VM started successfully"
else
log_error "VM failed to start"
exit 1
fi
else
log_error "VM creation/update failed"
exit 1
fi
}
# Wait for VM to be ready and get IP
wait_for_vm() {
log "Waiting for VM to be ready..."
sleep 120
# Export all environment variables from the file
set -a # automatically export all variables
source "$PROJECT_DIR/***REMOVED***.unraid"
set +a # turn off automatic export
local max_attempts=60
local attempt=1
while [ $attempt -le $max_attempts ]; do
VM_IP=$(python3 scripts/unraid/vm-manager.py ip 2>/dev/null | grep "VM IP:" | cut -d' ' -f3)
if [ -n "$VM_IP" ]; then
log_success "VM is ready with IP: $VM_IP"
# Update SSH config with actual IP
sed -i.bak "s/HostName %h/HostName $VM_IP/" "$HOME/.ssh/config"
# Update webhook environment with IP
sed -i.bak "s/VM_HOST=$VM_NAME/VM_HOST=$VM_IP/" "$PROJECT_DIR/***REMOVED***.webhook"
return 0
fi
log "Waiting for VM to get IP... (attempt $attempt/$max_attempts)"
sleep 30
((attempt++))
done
log_error "VM failed to get IP address"
exit 1
}
# Configure VM for ThrillWiki
configure_vm() {
log "Configuring VM for ThrillWiki deployment..."
local vm_setup_script="/tmp/vm_thrillwiki_setup.sh"
# Create VM setup script
cat > "$vm_setup_script" << 'EOF'
#!/bin/bash
set -e
echo "Setting up VM for ThrillWiki..."
# Update system
sudo apt update && sudo apt upgrade -y
# Install required packages
sudo apt install -y git curl build-essential python3-pip lsof postgresql postgresql-contrib nginx
# Install UV
curl -LsSf https://astral.sh/uv/install.sh | sh
source ~/.cargo/env
# Configure PostgreSQL
sudo -u postgres psql << PSQL
CREATE DATABASE thrillwiki;
CREATE USER thrillwiki_user WITH ENCRYPTED PASSWORD 'thrillwiki_pass';
GRANT ALL PRIVILEGES ON DATABASE thrillwiki TO thrillwiki_user;
\q
PSQL
# Clone repository
git clone REPO_URL_PLACEHOLDER thrillwiki
cd thrillwiki
# Install dependencies
~/.cargo/bin/uv sync
# Create directories
mkdir -p logs backups
# Make scripts executable
chmod +x scripts/*.sh
# Run initial setup
~/.cargo/bin/uv run manage.py migrate
~/.cargo/bin/uv run manage.py collectstatic --noinput
# Install systemd services
sudo cp scripts/systemd/thrillwiki.service /etc/systemd/system/
sudo sed -i 's|/home/ubuntu|/home/ubuntu|g' /etc/systemd/system/thrillwiki.service
sudo systemctl daemon-reload
sudo systemctl enable thrillwiki.service
echo "VM setup completed!"
EOF
# Replace placeholder with actual repo URL
sed -i "s|REPO_URL_PLACEHOLDER|$REPO_URL|g" "$vm_setup_script"
# Copy and execute setup script on VM
scp "$vm_setup_script" "$VM_NAME:/tmp/"
ssh "$VM_NAME" "bash /tmp/vm_thrillwiki_setup.sh"
# Cleanup
rm "$vm_setup_script"
log_success "VM configured for ThrillWiki"
}
# Start services
start_services() {
log "Starting ThrillWiki services..."
# Start VM service
ssh "$VM_NAME" "sudo systemctl start thrillwiki"
# Verify service is running
if ssh "$VM_NAME" "systemctl is-active --quiet thrillwiki"; then
log_success "ThrillWiki service started successfully"
else
log_error "Failed to start ThrillWiki service"
exit 1
fi
# Get service status
log "Service status:"
ssh "$VM_NAME" "systemctl status thrillwiki --no-pager -l"
}
# Setup webhook listener
setup_webhook_listener() {
log "Setting up webhook listener..."
# Create webhook start script
cat > "$PROJECT_DIR/start-webhook.sh" << 'EOF'
#!/bin/bash
cd "$(dirname "$0")"
source ***REMOVED***.webhook
python3 scripts/webhook-listener.py
EOF
chmod +x "$PROJECT_DIR/start-webhook.sh"
log_success "Webhook listener configured"
log "You can start the webhook listener with: ./start-webhook.sh"
}
# Perform end-to-end test
test_deployment() {
log "Performing end-to-end deployment test..."
# Test VM connectivity
if ssh "$VM_NAME" "echo 'VM connectivity test passed'"; then
log_success "VM connectivity test passed"
else
log_error "VM connectivity test failed"
return 1
fi
# Test ThrillWiki service
if ssh "$VM_NAME" "curl -f http://localhost:8000 >/dev/null 2>&1"; then
log_success "ThrillWiki service test passed"
else
log_warning "ThrillWiki service test failed - checking logs..."
ssh "$VM_NAME" "journalctl -u thrillwiki --no-pager -l | tail -20"
fi
# Test deployment script
log "Testing deployment script..."
ssh "$VM_NAME" "cd thrillwiki && ./scripts/vm-deploy.sh status"
log_success "End-to-end test completed"
}
# Generate final instructions
generate_instructions() {
log "Generating final setup instructions..."
cat > "$PROJECT_DIR/UNRAID_SETUP_COMPLETE.md" << EOF
# ThrillWiki Unraid Automation - Setup Complete! 🎉
Your ThrillWiki CI/CD system has been fully automated and deployed!
## VM Information
- **VM Name**: $VM_NAME
- **VM IP**: $VM_IP
- **SSH Access**: \`ssh $VM_NAME\`
## Services Status
- **ThrillWiki Service**: Running on VM
- **Database**: PostgreSQL configured
- **Web Server**: Available at http://$VM_IP:8000
## Next Steps
### 1. Start Webhook Listener
\`\`\`bash
./start-webhook.sh
\`\`\`
### 2. Configure GitHub Webhook
- Go to your repository: $REPO_URL
- Settings → Webhooks → Add webhook
- **Payload URL**: http://YOUR_PUBLIC_IP:$WEBHOOK_PORT/webhook
- **Content type**: application/json
- **Secret**: (your webhook secret)
- **Events**: Just the push event
### 3. Test the System
\`\`\`bash
# Test VM connection
ssh $VM_NAME
# Test service status
ssh $VM_NAME "systemctl status thrillwiki"
# Test manual deployment
ssh $VM_NAME "cd thrillwiki && ./scripts/vm-deploy.sh"
# Make a test commit to trigger automatic deployment
git add .
git commit -m "Test automated deployment"
git push origin main
\`\`\`
## Management Commands
### VM Management
\`\`\`bash
# Check VM status
python3 scripts/unraid/vm-manager.py status
# Start/stop VM
python3 scripts/unraid/vm-manager.py start
python3 scripts/unraid/vm-manager.py stop
# Get VM IP
python3 scripts/unraid/vm-manager.py ip
\`\`\`
### Service Management on VM
\`\`\`bash
# Check service status
ssh $VM_NAME "./scripts/vm-deploy.sh status"
# Restart service
ssh $VM_NAME "./scripts/vm-deploy.sh restart"
# View logs
ssh $VM_NAME "journalctl -u thrillwiki -f"
\`\`\`
## Troubleshooting
### Common Issues
1. **VM not accessible**: Check VM is running and has IP
2. **Service not starting**: Check logs with \`journalctl -u thrillwiki\`
3. **Webhook not working**: Verify port $WEBHOOK_PORT is open
### Support Files
- Configuration: \`***REMOVED***.unraid\`, \`***REMOVED***.webhook\`
- Logs: \`logs/\` directory
- Documentation: \`docs/VM_DEPLOYMENT_SETUP.md\`
**Your automated CI/CD system is now ready!** 🚀
Every push to the main branch will automatically deploy to your VM.
EOF
log_success "Setup instructions saved to UNRAID_SETUP_COMPLETE.md"
}
# Main automation function
main() {
log "🚀 Starting ThrillWiki Complete Unraid Automation"
echo "[AWS-SECRET-REMOVED]=========="
echo
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case $1 in
--reset)
RESET_ALL=true
shift
;;
--reset-vm)
RESET_VM_ONLY=true
shift
;;
--reset-config)
RESET_CONFIG_ONLY=true
shift
;;
--help|-h)
show_help
exit 0
;;
*)
echo "Unknown option: $1"
show_help
exit 1
;;
esac
done
# Create logs directory
mkdir -p "$LOG_DIR"
# Handle reset modes
if [[ "$RESET_ALL" == "true" ]]; then
log "🔄 Complete reset mode - deleting VM and configuration"
echo
# Load configuration first to get connection details for VM deletion
if [[ -f "$CONFIG_FILE" ]]; then
source "$CONFIG_FILE"
log_success "Loaded existing configuration for VM deletion"
else
log_warning "No configuration file found, will skip VM deletion"
fi
# Delete existing VM if config exists
if [[ -f "$CONFIG_FILE" ]]; then
log "🗑️ Deleting existing VM..."
# Export environment variables for VM manager
set -a
source "$PROJECT_DIR/***REMOVED***.unraid" 2>/dev/null || true
set +a
if python3 "$(dirname "$0")/vm-manager.py" delete; then
log_success "VM deleted successfully"
else
log "⚠️ VM deletion failed or VM didn't exist"
fi
fi
# Remove configuration files
if [[ -f "$CONFIG_FILE" ]]; then
rm "$CONFIG_FILE"
log_success "Configuration file removed"
fi
# Remove environment files
rm -f "$PROJECT_DIR/***REMOVED***.unraid" "$PROJECT_DIR/***REMOVED***.webhook"
log_success "Environment files removed"
log_success "Complete reset finished - continuing with fresh setup"
echo
elif [[ "$RESET_VM_ONLY" == "true" ]]; then
log "🔄 VM-only reset mode - deleting VM, preserving configuration"
echo
# Load configuration to get connection details
if [[ -f "$CONFIG_FILE" ]]; then
source "$CONFIG_FILE"
log_success "Loaded existing configuration"
else
log_error "No configuration file found. Cannot reset VM without connection details."
echo " Run the script without reset flags first to create initial configuration."
exit 1
fi
# Delete existing VM
log "🗑️ Deleting existing VM..."
# Export environment variables for VM manager
set -a
source "$PROJECT_DIR/***REMOVED***.unraid" 2>/dev/null || true
set +a
if python3 "$(dirname "$0")/vm-manager.py" delete; then
log_success "VM deleted successfully"
else
log "⚠️ VM deletion failed or VM didn't exist"
fi
# Remove only environment files, keep main config
rm -f "$PROJECT_DIR/***REMOVED***.unraid" "$PROJECT_DIR/***REMOVED***.webhook"
log_success "Environment files removed, configuration preserved"
log_success "VM reset complete - will recreate VM with saved configuration"
echo
elif [[ "$RESET_CONFIG_ONLY" == "true" ]]; then
log "🔄 Config-only reset mode - deleting configuration, preserving VM"
echo
# Remove configuration files
if [[ -f "$CONFIG_FILE" ]]; then
rm "$CONFIG_FILE"
log_success "Configuration file removed"
fi
# Remove environment files
rm -f "$PROJECT_DIR/***REMOVED***.unraid" "$PROJECT_DIR/***REMOVED***.webhook"
log_success "Environment files removed"
log_success "Configuration reset complete - will prompt for fresh configuration"
echo
fi
# Collect configuration
prompt_unraid_config
# Setup steps
setup_ssh_keys
setup_unraid_access
create_environment_files
install_dependencies
create_vm
wait_for_vm
configure_vm
start_services
setup_webhook_listener
test_deployment
generate_instructions
echo
log_success "🎉 Complete automation setup finished!"
echo
log "Your ThrillWiki VM is running at: http://$VM_IP:8000"
log "Start the webhook listener: ./start-webhook.sh"
log "See UNRAID_SETUP_COMPLETE.md for detailed instructions"
echo
log "The system will now automatically deploy when you push to GitHub!"
}
# Run main function and log output
main "$@" 2>&1 | tee "$LOG_DIR/unraid-automation.log"

861
scripts/unraid/vm-manager.py Executable file
View File

@@ -0,0 +1,861 @@
#!/usr/bin/env python3
"""
Unraid VM Manager for ThrillWiki
This script automates VM creation, configuration, and management on Unraid.
"""
import os
import sys
import json
import time
import logging
import requests
import subprocess
from pathlib import Path
from typing import Dict, Optional, List
# Configuration
UNRAID_HOST = os***REMOVED***iron.get('UNRAID_HOST', 'localhost')
UNRAID_USER = os***REMOVED***iron.get('UNRAID_USER', 'root')
UNRAID_PASSWORD = os***REMOVED***iron.get('UNRAID_PASSWORD', '')
VM_NAME = os***REMOVED***iron.get('VM_NAME', 'thrillwiki-vm')
VM_TEMPLATE = os***REMOVED***iron.get('VM_TEMPLATE', 'Ubuntu Server 22.04')
VM_MEMORY = int(os***REMOVED***iron.get('VM_MEMORY', 4096)) # MB
VM_VCPUS = int(os***REMOVED***iron.get('VM_VCPUS', 2))
VM_DISK_SIZE = int(os***REMOVED***iron.get('VM_DISK_SIZE', 50)) # GB
SSH_PUBLIC_KEY = os***REMOVED***iron.get('SSH_PUBLIC_KEY', '')
# Network Configuration
VM_IP = os***REMOVED***iron.get('VM_IP', '192.168.20.20')
VM_GATEWAY = os***REMOVED***iron.get('VM_GATEWAY', '192.168.20.1')
VM_NETMASK = os***REMOVED***iron.get('VM_NETMASK', '255.255.255.0')
VM_NETWORK = os***REMOVED***iron.get('VM_NETWORK', '192.168.20.0/24')
# GitHub Configuration
REPO_URL = os***REMOVED***iron.get('REPO_URL', '')
GITHUB_USERNAME = os***REMOVED***iron.get('GITHUB_USERNAME', '')
GITHUB_TOKEN = os***REMOVED***iron.get('GITHUB_TOKEN', '')
GITHUB_API_ENABLED = os***REMOVED***iron.get(
'GITHUB_API_ENABLED', 'false').lower() == 'true'
# Setup logging
os.makedirs('logs', exist_ok=True)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler('logs/unraid-vm.log'),
logging.StreamHandler()
]
)
logger = logging.getLogger(__name__)
class UnraidVMManager:
"""Manages VMs on Unraid server."""
def __init__(self):
self.session = requests.Session()
self.base_url = f"http://{UNRAID_HOST}"
self.vm_config_path = f"/mnt/user/domains/{VM_NAME}"
def authenticate(self) -> bool:
"""Authenticate with Unraid server."""
try:
login_url = f"{self.base_url}/login"
login_data = {
'username': UNRAID_USER,
'password': UNRAID_PASSWORD
}
response = self.session.post(login_url, data=login_data)
if response.status_code == 200:
logger.info("Successfully authenticated with Unraid")
return True
else:
logger.error(f"Authentication failed: {response.status_code}")
return False
except Exception as e:
logger.error(f"Authentication error: {e}")
return False
def check_vm_exists(self) -> bool:
"""Check if VM already exists."""
try:
result = subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh list --all | grep {VM_NAME}'",
shell=True,
capture_output=True,
text=True
)
return VM_NAME in result.stdout
except Exception as e:
logger.error(f"Error checking VM existence: {e}")
return False
def create_vm_xml(self, existing_uuid: str = None) -> str:
"""Generate VM XML configuration."""
import uuid
vm_uuid = existing_uuid if existing_uuid else str(uuid.uuid4())
xml_template = f"""<?xml version='1.0' encoding='UTF-8'?>
<domain type='kvm'>
<name>{VM_NAME}</name>
<uuid>{vm_uuid}</uuid>
<metadata>
<vmtemplate xmlns="unraid" name="Windows 10" iconold="ubuntu.png" icon="ubuntu.png" os="linux" webui=""/>
</metadata>
<memory unit='KiB'>{VM_MEMORY * 1024}</memory>
<currentMemory unit='KiB'>{VM_MEMORY * 1024}</currentMemory>
<vcpu placement='static'>{VM_VCPUS}</vcpu>
<os>
<type arch='x86_64' machine='pc-q35-9.2'>hvm</type>
<loader readonly='yes' type='pflash' format='raw'>/usr/share/qemu/ovmf-x64/OVMF_CODE-pure-efi.fd</loader>
<nvram format='raw'>/etc/libvirt/qemu/nvram/{vm_uuid}_VARS-pure-efi.fd</nvram>
</os>
<features>
<acpi/>
<apic/>
<vmport state='off'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='on'>
<topology sockets='1' dies='1' clusters='1' cores='{VM_VCPUS // 2 if VM_VCPUS > 1 else 1}' threads='{2 if VM_VCPUS > 1 else 1}'/>
<cache mode='passthrough'/>
<feature policy='require' name='topoext'/>
</cpu>
<clock offset='utc'>
<timer name='hpet' present='no'/>
<timer name='hypervclock' present='yes'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='rtc' tickpolicy='catchup'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/local/sbin/qemu</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2' cache='writeback' discard='ignore'/>
<source file='/mnt/user/domains/{VM_NAME}/vdisk1.qcow2'/>
<target dev='hdc' bus='virtio'/>
<boot order='2'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</disk>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<source file='/mnt/user/isos/ubuntu-24.04.3-live-server-amd64.iso'/>
<target dev='hda' bus='sata'/>
<readonly/>
<boot order='1'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<source file='/mnt/user/isos/{VM_NAME}-cloud-init.iso'/>
<target dev='hdb' bus='sata'/>
<readonly/>
<address type='drive' controller='0' bus='0' target='0' unit='1'/>
</disk>
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</controller>
<controller type='pci' index='0' model='pcie-root'/>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x13'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0x14'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
</controller>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<interface type='bridge'>
<mac address='52:54:00:{":".join([f"{int(VM_IP.split('.')[3]):02x}", "7d", "fd"])}'/>
<source bridge='br0.20'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<channel type='unix'>
<target type='virtio' name='org.qemu.guest_agent.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='tablet' bus='usb'>
<address type='usb' bus='0' port='1'/>
</input>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='vnc' port='-1' autoport='yes' websocket='-1' listen='0.0.0.0' sharePolicy='ignore'>
<listen type='address' address='0.0.0.0'/>
</graphics>
<audio id='1' type='none'/>
<video>
<model type='qxl' ram='65536' vram='65536' vram64='65535' vgamem='65536' heads='1' primary='yes'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1e' function='0x0'/>
</video>
<watchdog model='itco' action='reset'/>
<memballoon model='virtio'>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</memballoon>
</devices>
</domain>"""
return xml_template.strip()
def create_vm(self) -> bool:
"""Create or update the VM on Unraid."""
try:
vm_exists = self.check_vm_exists()
if vm_exists:
logger.info(
f"VM {VM_NAME} already exists, updating configuration...")
# Stop VM if running before updating
if self.vm_status() == "running":
logger.info(
f"Stopping VM {VM_NAME} for configuration update...")
self.stop_vm()
# Wait for VM to stop
import time
time.sleep(5)
else:
logger.info(f"Creating VM {VM_NAME}...")
# Create VM directory
subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'mkdir -p {self.vm_config_path}'",
shell=True,
check=True
)
# Create virtual disk only if VM doesn't exist
disk_cmd = f"""
ssh {UNRAID_USER}@{UNRAID_HOST} 'qemu-img create -f qcow2 {self.vm_config_path}/vdisk1.qcow2 {VM_DISK_SIZE}G'
"""
subprocess.run(disk_cmd, shell=True, check=True)
# Create cloud-init ISO for automated installation and ThrillWiki deployment
logger.info(
"Creating cloud-init ISO for automated Ubuntu and ThrillWiki setup...")
if not self.create_cloud_init_iso(VM_IP):
logger.error("Failed to create cloud-init ISO")
return False
existing_uuid = None
if vm_exists:
# Get existing VM UUID
result = subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh dumpxml {VM_NAME} | grep \"<uuid>\" | sed \"s/<uuid>//g\" | sed \"s/<\\/uuid>//g\" | tr -d \" \"'",
shell=True,
capture_output=True,
text=True
)
if result.returncode == 0 and result.stdout.strip():
existing_uuid = result.stdout.strip()
logger.info(f"Found existing VM UUID: {existing_uuid}")
# Always undefine existing VM with NVRAM flag (since we create persistent VMs)
logger.info(
f"VM {VM_NAME} exists, undefining with NVRAM for reconfiguration...")
subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh undefine {VM_NAME} --nvram'",
shell=True,
check=True
)
logger.info(
f"VM {VM_NAME} undefined for reconfiguration (with NVRAM)")
# Generate VM XML with appropriate UUID
vm_xml = self.create_vm_xml(existing_uuid)
xml_file = f"/tmp/{VM_NAME}.xml"
with open(xml_file, 'w') as f:
f.write(vm_xml)
# Copy XML to Unraid and define/redefine VM
subprocess.run(
f"scp {xml_file} {UNRAID_USER}@{UNRAID_HOST}:/tmp/",
shell=True,
check=True
)
# Define VM as persistent domain
subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh define /tmp/{VM_NAME}.xml'",
shell=True,
check=True
)
# Ensure VM is set to autostart for persistent configuration
subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh autostart {VM_NAME}'",
shell=True,
check=False # Don't fail if autostart is already enabled
)
action = "updated" if vm_exists else "created"
logger.info(f"VM {VM_NAME} {action} successfully")
# Cleanup
os.remove(xml_file)
return True
except Exception as e:
logger.error(f"Failed to create VM: {e}")
return False
def create_nvram_file(self, vm_uuid: str) -> bool:
"""Create NVRAM file for UEFI VM."""
try:
nvram_path = f"/etc/libvirt/qemu/nvram/{vm_uuid}_VARS-pure-efi.fd"
# Check if NVRAM file already exists
result = subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'test -f {nvram_path}'",
shell=True,
capture_output=True
)
if result.returncode == 0:
logger.info(f"NVRAM file already exists: {nvram_path}")
return True
# Copy template to create NVRAM file
logger.info(f"Creating NVRAM file: {nvram_path}")
result = subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'cp /usr/share/qemu/ovmf-x64/OVMF_VARS-pure-efi.fd {nvram_path}'",
shell=True,
capture_output=True,
text=True
)
if result.returncode == 0:
logger.info("NVRAM file created successfully")
return True
else:
logger.error(f"Failed to create NVRAM file: {result.stderr}")
return False
except Exception as e:
logger.error(f"Error creating NVRAM file: {e}")
return False
def start_vm(self) -> bool:
"""Start the VM if it's not already running."""
try:
# Check if VM is already running
current_status = self.vm_status()
if current_status == "running":
logger.info(f"VM {VM_NAME} is already running")
return True
logger.info(f"Starting VM {VM_NAME}...")
# For new VMs, we need to extract the UUID and create NVRAM file
vm_exists = self.check_vm_exists()
if not vm_exists:
logger.error("Cannot start VM that doesn't exist")
return False
# Get VM UUID from XML
result = subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh dumpxml {VM_NAME} | grep \"<uuid>\" | sed \"s/<uuid>//g\" | sed \"s/<\\/uuid>//g\" | tr -d \" \"'",
shell=True,
capture_output=True,
text=True
)
if result.returncode == 0 and result.stdout.strip():
vm_uuid = result.stdout.strip()
logger.info(f"VM UUID: {vm_uuid}")
# Create NVRAM file if it doesn't exist
if not self.create_nvram_file(vm_uuid):
return False
result = subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh start {VM_NAME}'",
shell=True,
capture_output=True,
text=True
)
if result.returncode == 0:
logger.info(f"VM {VM_NAME} started successfully")
return True
else:
logger.error(f"Failed to start VM: {result.stderr}")
return False
except Exception as e:
logger.error(f"Error starting VM: {e}")
return False
def stop_vm(self) -> bool:
"""Stop the VM."""
try:
logger.info(f"Stopping VM {VM_NAME}...")
result = subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh shutdown {VM_NAME}'",
shell=True,
capture_output=True,
text=True
)
if result.returncode == 0:
logger.info(f"VM {VM_NAME} stopped successfully")
return True
else:
logger.error(f"Failed to stop VM: {result.stderr}")
return False
except Exception as e:
logger.error(f"Error stopping VM: {e}")
return False
def get_vm_ip(self) -> Optional[str]:
"""Get VM IP address."""
try:
# Wait for VM to get IP
for attempt in range(30):
result = subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh domifaddr {VM_NAME}'",
shell=True,
capture_output=True,
text=True
)
if result.returncode == 0 and 'ipv4' in result.stdout:
lines = result.stdout.strip().split('\n')
for line in lines:
if 'ipv4' in line:
# Extract IP from line like: vnet0 52:54:00:xx:xx:xx ipv4 192.168.1.100/24
parts = line.split()
if len(parts) >= 4:
ip_with_mask = parts[3]
ip = ip_with_mask.split('/')[0]
logger.info(f"VM IP address: {ip}")
return ip
logger.info(f"Waiting for VM IP... (attempt {attempt + 1}/30)")
time.sleep(10)
logger.error("Failed to get VM IP address")
return None
except Exception as e:
logger.error(f"Error getting VM IP: {e}")
return None
def create_cloud_init_iso(self, vm_ip: str) -> bool:
"""Create cloud-init ISO for automated Ubuntu installation."""
try:
logger.info("Creating cloud-init ISO...")
# Get environment variables
repo_url = os.getenv('REPO_URL', '')
github_token = os.getenv('GITHUB_TOKEN', '')
ssh_public_key = os.getenv('SSH_PUBLIC_KEY', '')
# Extract repository name from URL
if repo_url:
# Extract owner/repo from URL like https://github.com/owner/repo
github_repo = repo_url.replace(
'https://github.com/', '').replace('.git', '')
else:
logger.error("REPO_URL environment variable not set")
return False
# Create cloud-init user-data with complete ThrillWiki deployment
user_data = f"""#cloud-config
runcmd:
- [eval, 'echo $(cat /proc/cmdline) "autoinstall" > /root/cmdline']
- [eval, 'mount -n --bind -o ro /root/cmdline /proc/cmdline']
- [eval, 'snap restart subiquity.subiquity-server']
- [eval, 'snap restart subiquity.subiquity-service']
autoinstall:
version: 1
locale: en_US
keyboard:
layout: us
ssh:
install-server: true
authorized-keys:
- {ssh_public_key}
allow-pw: false
storage:
layout:
name: direct
identity:
hostname: thrillwiki-vm
username: ubuntu
password: '$6$rounds=4096$saltsalt$hash' # disabled
kernel:
package: linux-generic
early-commands:
- systemctl stop ssh
packages:
- curl
- git
- build-essential
- python3-pip
- postgresql
- postgresql-contrib
- nginx
- nodejs
- npm
- pipx
late-commands:
- apt install pipx -y
- echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > /target/etc/sudoers.d/ubuntu
- /target/usr/bin/pipx install uv
# Setup ThrillWiki deployment script
- |
cat > /target/home/ubuntu/deploy-thrillwiki.sh << 'DEPLOY_EOF'
#!/bin/bash
set -e
# Wait for system to be ready
sleep 30
# Clone ThrillWiki repository with GitHub token
export GITHUB_TOKEN=$(cat /home/ubuntu/.github-token 2>/dev/null || echo "")
if [ -n "$GITHUB_TOKEN" ]; then
git clone https://$GITHUB_TOKEN@github.com/{github_repo} /home/ubuntu/thrillwiki
else
git clone https://github.com/{github_repo} /home/ubuntu/thrillwiki
fi
cd /home/ubuntu/thrillwiki
# Setup UV and Python environment
export PATH="/home/ubuntu/.local/bin:$PATH"
uv venv
source .venv/bin/activate
# Install dependencies
uv sync
# Setup PostgreSQL
sudo -u postgres createuser ubuntu
sudo -u postgres createdb thrillwiki_production
sudo -u postgres psql -c "ALTER USER ubuntu WITH SUPERUSER;"
# Setup environment
cp ***REMOVED***.example ***REMOVED***
echo "DEBUG=False" >> ***REMOVED***
echo "DATABASE_URL=postgresql://ubuntu@localhost/thrillwiki_production" >> ***REMOVED***
echo "ALLOWED_HOSTS=*" >> ***REMOVED***
# Run migrations and collect static files
uv run manage.py migrate
uv run manage.py collectstatic --noinput
uv run manage.py tailwind build
# Setup systemd services
sudo cp [AWS-SECRET-REMOVED]thrillwiki.service /etc/systemd/system/
sudo cp [AWS-SECRET-REMOVED]thrillwiki-webhook.service /etc/systemd/system/
# Update service files with correct paths
sudo sed -i "s|/opt/thrillwiki|/home/ubuntu/thrillwiki|g" /etc/systemd/system/thrillwiki.service
sudo sed -i "s|/opt/thrillwiki|/home/ubuntu/thrillwiki|g" /etc/systemd/system/thrillwiki-webhook.service
# Enable and start services
sudo systemctl daemon-reload
sudo systemctl enable thrillwiki
sudo systemctl enable thrillwiki-webhook
sudo systemctl start thrillwiki
sudo systemctl start thrillwiki-webhook
echo "ThrillWiki deployment completed successfully!"
DEPLOY_EOF
- chmod +x /target/home/ubuntu/deploy-thrillwiki.sh
- chroot /target chown ubuntu:ubuntu /home/ubuntu/deploy-thrillwiki.sh
# Create systemd service to run deployment after first boot
- |
cat > /target/etc/systemd/system/thrillwiki-deploy.service << 'SERVICE_EOF'
[Unit]
Description=Deploy ThrillWiki on first boot
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
User=ubuntu
ExecStart=/home/ubuntu/deploy-thrillwiki.sh
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target
SERVICE_EOF
- chroot /target systemctl enable thrillwiki-deploy
user-data:
disable_root: true
ssh_pwauth: false
power_state:
mode: reboot
"""
meta_data = f"""instance-id: thrillwiki-vm-001
local-hostname: thrillwiki-vm
network:
version: 2
ethernets:
enp1s0:
dhcp4: true
"""
# Create temp directory for cloud-init files
cloud_init_dir = "/tmp/cloud-init"
os.makedirs(cloud_init_dir, exist_ok=True)
with open(f"{cloud_init_dir}/user-data", 'w') as f:
f.write(user_data)
with open(f"{cloud_init_dir}/meta-data", 'w') as f:
f.write(meta_data)
# Create ISO
iso_path = f"/tmp/{VM_NAME}-cloud-init.iso"
# Try different ISO creation tools
iso_created = False
# Try genisoimage first
try:
subprocess.run([
'genisoimage',
'-output', iso_path,
'-volid', 'cidata',
'-joliet',
'-rock',
cloud_init_dir
], check=True)
iso_created = True
except FileNotFoundError:
logger.warning("genisoimage not found, trying mkisofs...")
# Try mkisofs as fallback
if not iso_created:
try:
subprocess.run([
'mkisofs',
'-output', iso_path,
'-volid', 'cidata',
'-joliet',
'-rock',
cloud_init_dir
], check=True)
iso_created = True
except FileNotFoundError:
logger.warning(
"mkisofs not found, trying hdiutil (macOS)...")
# Try hdiutil for macOS
if not iso_created:
try:
subprocess.run([
'hdiutil', 'makehybrid',
'-iso', '-joliet',
'-o', iso_path,
cloud_init_dir
], check=True)
iso_created = True
except FileNotFoundError:
logger.error(
"No ISO creation tool found. Please install genisoimage, mkisofs, or use macOS hdiutil")
return False
if not iso_created:
logger.error("Failed to create ISO with any available tool")
return False
# Copy ISO to Unraid
subprocess.run(
f"scp {iso_path} {UNRAID_USER}@{UNRAID_HOST}:/mnt/user/isos/",
shell=True,
check=True
)
logger.info("Cloud-init ISO created successfully")
return True
except Exception as e:
logger.error(f"Failed to create cloud-init ISO: {e}")
return False
def vm_status(self) -> str:
"""Get VM status."""
try:
result = subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh domstate {VM_NAME}'",
shell=True,
capture_output=True,
text=True
)
if result.returncode == 0:
return result.stdout.strip()
else:
return "unknown"
except Exception as e:
logger.error(f"Error getting VM status: {e}")
return "error"
def delete_vm(self) -> bool:
"""Completely remove VM and all associated files."""
try:
logger.info(f"Deleting VM {VM_NAME} and all associated files...")
# Check if VM exists
if not self.check_vm_exists():
logger.info(f"VM {VM_NAME} does not exist")
return True
# Stop VM if running
if self.vm_status() == "running":
logger.info(f"Stopping VM {VM_NAME}...")
self.stop_vm()
import time
time.sleep(5)
# Undefine VM with NVRAM
logger.info(f"Undefining VM {VM_NAME}...")
subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh undefine {VM_NAME} --nvram'",
shell=True,
check=True
)
# Remove VM directory and all files
logger.info(f"Removing VM directory and files...")
subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'rm -rf {self.vm_config_path}'",
shell=True,
check=True
)
# Remove cloud-init ISO
subprocess.run(
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'rm -f /mnt/user/isos/{VM_NAME}-cloud-init.iso'",
shell=True,
check=False # Don't fail if file doesn't exist
)
logger.info(f"VM {VM_NAME} completely removed")
return True
except Exception as e:
logger.error(f"Failed to delete VM: {e}")
return False
def main():
"""Main function."""
import argparse
parser = argparse.ArgumentParser(
description='Unraid VM Manager for ThrillWiki')
parser.add_argument('action', choices=['create', 'start', 'stop', 'status', 'ip', 'setup', 'delete'],
help='Action to perform')
args = parser.parse_args()
# Create logs directory
os.makedirs('logs', exist_ok=True)
vm_manager = UnraidVMManager()
if args.action == 'create':
success = vm_manager.create_vm()
sys.exit(0 if success else 1)
elif args.action == 'start':
success = vm_manager.start_vm()
sys.exit(0 if success else 1)
elif args.action == 'stop':
success = vm_manager.stop_vm()
sys.exit(0 if success else 1)
elif args.action == 'status':
status = vm_manager.vm_status()
print(f"VM Status: {status}")
sys.exit(0)
elif args.action == 'ip':
ip = vm_manager.get_vm_ip()
if ip:
print(f"VM IP: {ip}")
sys.exit(0)
else:
print("Failed to get VM IP")
sys.exit(1)
elif args.action == 'setup':
logger.info("Setting up complete VM environment...")
# Create VM
if not vm_manager.create_vm():
sys.exit(1)
# Start VM
if not vm_manager.start_vm():
sys.exit(1)
# Get IP
vm_ip = vm_manager.get_vm_ip()
if not vm_ip:
sys.exit(1)
print(f"VM setup complete. IP: {vm_ip}")
print("You can now connect via SSH and complete the ThrillWiki setup.")
sys.exit(0)
elif args.action == 'delete':
success = vm_manager.delete_vm()
sys.exit(0 if success else 1)
if __name__ == '__main__':
main()

340
scripts/vm-deploy.sh Executable file
View File

@@ -0,0 +1,340 @@
#!/bin/bash
# ThrillWiki VM Deployment Script
# This script runs on the Linux VM to deploy the latest code and restart the server
set -e # Exit on any error
# Configuration
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
LOG_DIR="$PROJECT_DIR/logs"
BACKUP_DIR="$PROJECT_DIR/backups"
DEPLOY_LOG="$LOG_DIR/deploy.log"
SERVICE_NAME="thrillwiki"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Logging function
log() {
local message="[$(date +'%Y-%m-%d %H:%M:%S')] $1"
echo -e "${BLUE}${message}${NC}"
echo "$message" >> "$DEPLOY_LOG"
}
log_success() {
local message="[$(date +'%Y-%m-%d %H:%M:%S')] ✓ $1"
echo -e "${GREEN}${message}${NC}"
echo "$message" >> "$DEPLOY_LOG"
}
log_warning() {
local message="[$(date +'%Y-%m-%d %H:%M:%S')] ⚠ $1"
echo -e "${YELLOW}${message}${NC}"
echo "$message" >> "$DEPLOY_LOG"
}
log_error() {
local message="[$(date +'%Y-%m-%d %H:%M:%S')] ✗ $1"
echo -e "${RED}${message}${NC}"
echo "$message" >> "$DEPLOY_LOG"
}
# Create necessary directories
create_directories() {
log "Creating necessary directories..."
mkdir -p "$LOG_DIR" "$BACKUP_DIR"
log_success "Directories created"
}
# Backup current deployment
backup_current() {
log "Creating backup of current deployment..."
local timestamp=$(date +'%Y%m%d_%H%M%S')
local backup_path="$BACKUP_DIR/backup_$timestamp"
# Create backup of current code
if [ -d "$PROJECT_DIR/.git" ]; then
local current_commit=$(git -C "$PROJECT_DIR" rev-parse HEAD)
echo "$current_commit" > "$backup_path.commit"
log_success "Backup created with commit: ${current_commit:0:8}"
else
log_warning "Not a git repository, skipping backup"
fi
}
# Stop the service
stop_service() {
log "Stopping ThrillWiki service..."
# Stop systemd service if it exists
if systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
sudo systemctl stop "$SERVICE_NAME"
log_success "Systemd service stopped"
else
log "Systemd service not running"
fi
# Kill any remaining Django processes on port 8000
if lsof -ti :8000 >/dev/null 2>&1; then
log "Stopping processes on port 8000..."
lsof -ti :8000 | xargs kill -9 2>/dev/null || true
log_success "Port 8000 processes stopped"
fi
# Clean up Python cache
log "Cleaning Python cache..."
find "$PROJECT_DIR" -type d -name "__pycache__" -exec rm -r {} + 2>/dev/null || true
log_success "Python cache cleaned"
}
# Update code from git
update_code() {
log "Updating code from git repository..."
cd "$PROJECT_DIR"
# Fetch latest changes
git fetch origin
log "Fetched latest changes"
# Get current and new commit info
local old_commit=$(git rev-parse HEAD)
local new_commit=$(git rev-parse origin/main)
if [ "$old_commit" = "$new_commit" ]; then
log_warning "No new commits to deploy"
return 0
fi
log "Updating from ${old_commit:0:8} to ${new_commit:0:8}"
# Pull latest changes
git reset --hard origin/main
log_success "Code updated successfully"
# Show what changed
log "Changes in this deployment:"
git log --oneline "$old_commit..$new_commit" || true
}
# Install/update dependencies
update_dependencies() {
log "Updating dependencies..."
cd "$PROJECT_DIR"
# Check if UV is installed
if ! command -v uv &> /dev/null; then
log_error "UV is not installed. Installing UV..."
curl -LsSf https://astral.sh/uv/install.sh | sh
source $HOME/.cargo/env
fi
# Sync dependencies
uv sync --no-dev || {
log_error "Failed to sync dependencies"
return 1
}
log_success "Dependencies updated"
}
# Run database migrations
run_migrations() {
log "Running database migrations..."
cd "$PROJECT_DIR"
# Check for pending migrations
if uv run manage.py showmigrations --plan | grep -q "\[ \]"; then
log "Applying database migrations..."
uv run manage.py migrate || {
log_error "Database migrations failed"
return 1
}
log_success "Database migrations completed"
else
log "No pending migrations"
fi
}
# Collect static files
collect_static() {
log "Collecting static files..."
cd "$PROJECT_DIR"
uv run manage.py collectstatic --noinput || {
log_warning "Static file collection failed, continuing..."
}
log_success "Static files collected"
}
# Start the service
start_service() {
log "Starting ThrillWiki service..."
cd "$PROJECT_DIR"
# Start systemd service if it exists
if systemctl list-unit-files | grep -q "^$SERVICE_NAME.service"; then
sudo systemctl start "$SERVICE_NAME"
sudo systemctl enable "$SERVICE_NAME"
# Wait for service to start
sleep 5
if systemctl is-active --quiet "$SERVICE_NAME"; then
log_success "Systemd service started successfully"
else
log_error "Systemd service failed to start"
return 1
fi
else
log_warning "Systemd service not found, starting manually..."
# Start server in background
nohup ./scripts/ci-start.sh > "$LOG_DIR/server.log" 2>&1 &
local server_pid=$!
# Wait for server to start
sleep 5
if kill -0 $server_pid 2>/dev/null; then
echo $server_pid > "$LOG_DIR/server.pid"
log_success "Server started manually with PID: $server_pid"
else
log_error "Failed to start server manually"
return 1
fi
fi
}
# Health check
health_check() {
log "Performing health check..."
local max_attempts=30
local attempt=1
while [ $attempt -le $max_attempts ]; do
if curl -f -s http://localhost:8000/health >/dev/null 2>&1; then
log_success "Health check passed"
return 0
fi
log "Health check attempt $attempt/$max_attempts failed, retrying..."
sleep 2
((attempt++))
done
log_error "Health check failed after $max_attempts attempts"
return 1
}
# Cleanup old backups
cleanup_backups() {
log "Cleaning up old backups..."
# Keep only the last 10 backups
cd "$BACKUP_DIR"
ls -t backup_*.commit 2>/dev/null | tail -n +11 | xargs rm -f 2>/dev/null || true
log_success "Old backups cleaned up"
}
# Rollback function
rollback() {
log_error "Deployment failed, attempting rollback..."
local latest_backup=$(ls -t "$BACKUP_DIR"/backup_*.commit 2>/dev/null | head -n 1)
if [ -n "$latest_backup" ]; then
local backup_commit=$(cat "$latest_backup")
log "Rolling back to commit: ${backup_commit:0:8}"
cd "$PROJECT_DIR"
git reset --hard "$backup_commit"
# Restart service
stop_service
start_service
if health_check; then
log_success "Rollback completed successfully"
else
log_error "Rollback failed - manual intervention required"
fi
else
log_error "No backup found for rollback"
fi
}
# Main deployment function
deploy() {
log "=== ThrillWiki Deployment Started ==="
log "Timestamp: $(date)"
log "User: $(whoami)"
log "Host: $(hostname)"
# Trap errors for rollback
trap rollback ERR
create_directories
backup_current
stop_service
update_code
update_dependencies
run_migrations
collect_static
start_service
health_check
cleanup_backups
# Remove error trap
trap - ERR
log_success "=== Deployment Completed Successfully ==="
log "Server is now running the latest code"
log "Check logs at: $LOG_DIR/"
}
# Script execution
case "${1:-deploy}" in
deploy)
deploy
;;
stop)
stop_service
;;
start)
start_service
;;
restart)
stop_service
start_service
health_check
;;
status)
if systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
echo "Service is running"
elif [ -f "$LOG_DIR/server.pid" ] && kill -0 "$(cat "$LOG_DIR/server.pid")" 2>/dev/null; then
echo "Server is running manually"
else
echo "Service is not running"
fi
;;
health)
health_check
;;
*)
echo "Usage: $0 {deploy|stop|start|restart|status|health}"
exit 1
;;
esac

268
scripts/webhook-listener.py Executable file
View File

@@ -0,0 +1,268 @@
#!/usr/bin/env python3
"""
GitHub Webhook Listener for ThrillWiki CI/CD
This script listens for GitHub webhook events and triggers deployments to a Linux VM.
"""
import os
import sys
import json
import hmac
import hashlib
import logging
import subprocess
import requests
from http.server import HTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse, parse_qs
import threading
import time
from datetime import datetime
# Configuration
WEBHOOK_PORT = int(os***REMOVED***iron.get('WEBHOOK_PORT', 9000))
WEBHOOK_SECRET = os***REMOVED***iron.get('WEBHOOK_SECRET', '')
WEBHOOK_ENABLED = os***REMOVED***iron.get('WEBHOOK_ENABLED', 'true').lower() == 'true'
VM_HOST = os***REMOVED***iron.get('VM_HOST', 'localhost')
VM_PORT = int(os***REMOVED***iron.get('VM_PORT', 22))
VM_USER = os***REMOVED***iron.get('VM_USER', 'ubuntu')
VM_KEY_PATH = os***REMOVED***iron.get('VM_KEY_PATH', '~/.ssh/***REMOVED***')
PROJECT_PATH = os***REMOVED***iron.get('VM_PROJECT_PATH', '/home/ubuntu/thrillwiki')
REPO_URL = os***REMOVED***iron.get('REPO_URL', 'https://github.com/YOUR_USERNAME/thrillwiki_django_no_react.git')
DEPLOY_BRANCH = os***REMOVED***iron.get('DEPLOY_BRANCH', 'main')
# GitHub API Configuration
GITHUB_USERNAME = os***REMOVED***iron.get('GITHUB_USERNAME', '')
GITHUB_TOKEN = os***REMOVED***iron.get('GITHUB_TOKEN', '')
GITHUB_API_ENABLED = os***REMOVED***iron.get('GITHUB_API_ENABLED', 'false').lower() == 'true'
# Setup logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler('logs/webhook.log'),
logging.StreamHandler()
]
)
logger = logging.getLogger(__name__)
class GitHubWebhookHandler(BaseHTTPRequestHandler):
"""Handle incoming GitHub webhook requests."""
def do_GET(self):
"""Handle GET requests - health check."""
if self.path == '/health':
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
response = {
'status': 'healthy',
'timestamp': datetime.now().isoformat(),
'service': 'ThrillWiki Webhook Listener'
}
self.wfile.write(json.dumps(response).encode())
else:
self.send_response(404)
self.end_headers()
def do_POST(self):
"""Handle POST requests - webhook events."""
try:
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length)
# Verify webhook signature if secret is configured
if WEBHOOK_SECRET:
if not self._verify_signature(post_data):
logger.warning("Invalid webhook signature")
self.send_response(401)
self.end_headers()
return
# Parse webhook payload
try:
payload = json.loads(post_data.decode('utf-8'))
except json.JSONDecodeError:
logger.error("Invalid JSON payload")
self.send_response(400)
self.end_headers()
return
# Handle webhook event
event_type = self.headers.get('X-GitHub-Event')
if self._should_deploy(event_type, payload):
logger.info(f"Triggering deployment for {event_type} event")
threading.Thread(target=self._trigger_deployment, args=(payload,)).start()
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
response = {'status': 'deployment_triggered', 'event': event_type}
self.wfile.write(json.dumps(response).encode())
else:
logger.info(f"Ignoring {event_type} event - no deployment needed")
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
response = {'status': 'ignored', 'event': event_type}
self.wfile.write(json.dumps(response).encode())
except Exception as e:
logger.error(f"Error handling webhook: {e}")
self.send_response(500)
self.end_headers()
def _verify_signature(self, payload_body):
"""Verify GitHub webhook signature."""
signature = self.headers.get('X-Hub-Signature-256')
if not signature:
return False
expected_signature = 'sha256=' + hmac.new(
WEBHOOK_SECRET.encode(),
payload_body,
hashlib.sha256
).hexdigest()
return hmac.compare_digest(signature, expected_signature)
def _should_deploy(self, event_type, payload):
"""Determine if we should trigger a deployment."""
if event_type == 'push':
# Deploy on push to main branch
ref = payload.get('ref', '')
target_ref = f'refs/heads/{DEPLOY_BRANCH}'
return ref == target_ref
elif event_type == 'release':
# Deploy on new releases
action = payload.get('action', '')
return action == 'published'
return False
def _trigger_deployment(self, payload):
"""Trigger deployment to Linux VM."""
try:
commit_sha = payload.get('after') or payload.get('head_commit', {}).get('id', 'unknown')
commit_message = payload.get('head_commit', {}).get('message', 'No message')
logger.info(f"Starting deployment of commit {commit_sha[:8]}: {commit_message}")
# Execute deployment script on VM
deploy_script = f"""
#!/bin/bash
set -e
echo "=== ThrillWiki Deployment Started ==="
echo "Commit: {commit_sha[:8]}"
echo "Message: {commit_message}"
echo "Timestamp: $(date)"
cd {PROJECT_PATH}
# Pull latest changes
git fetch origin
git checkout {DEPLOY_BRANCH}
git pull origin {DEPLOY_BRANCH}
# Run deployment script
./scripts/vm-deploy.sh
echo "=== Deployment Completed Successfully ==="
"""
# Execute deployment on VM via SSH
ssh_command = [
'ssh',
'-i', VM_KEY_PATH,
'-o', 'StrictHostKeyChecking=no',
'-o', 'UserKnownHostsFile=/dev/null',
f'{VM_USER}@{VM_HOST}',
deploy_script
]
result = subprocess.run(
ssh_command,
capture_output=True,
text=True,
timeout=300 # 5 minute timeout
)
if result.returncode == 0:
logger.info(f"Deployment successful for commit {commit_sha[:8]}")
self._send_status_notification('success', commit_sha, commit_message)
else:
logger.error(f"Deployment failed for commit {commit_sha[:8]}: {result.stderr}")
self._send_status_notification('failure', commit_sha, commit_message, result.stderr)
except subprocess.TimeoutExpired:
logger.error("Deployment timed out")
self._send_status_notification('timeout', commit_sha, commit_message)
except Exception as e:
logger.error(f"Deployment error: {e}")
self._send_status_notification('error', commit_sha, commit_message, str(e))
def _send_status_notification(self, status, commit_sha, commit_message, error_details=None):
"""Send deployment status notification (optional)."""
# This could be extended to send notifications to Slack, Discord, etc.
status_msg = f"Deployment {status} for commit {commit_sha[:8]}: {commit_message}"
if error_details:
status_msg += f"\nError: {error_details}"
logger.info(f"Status: {status_msg}")
def log_message(self, format, *args):
"""Override to use our logger."""
logger.info(f"{self.client_address[0]} - {format % args}")
def main():
"""Main function to start the webhook listener."""
import argparse
parser = argparse.ArgumentParser(description='ThrillWiki GitHub Webhook Listener')
parser.add_argument('--port', type=int, default=WEBHOOK_PORT, help='Port to listen on')
parser.add_argument('--test', action='store_true', help='Test configuration without starting server')
args = parser.parse_args()
# Create logs directory
os.makedirs('logs', exist_ok=True)
# Validate configuration
if not WEBHOOK_SECRET:
logger.warning("WEBHOOK_SECRET not set - webhook signature verification disabled")
if not all([VM_HOST, VM_USER, PROJECT_PATH]):
logger.error("Missing required VM configuration")
if args.test:
print("❌ Configuration validation failed")
return
sys.exit(1)
logger.info(f"Webhook listener configuration:")
logger.info(f" Port: {args.port}")
logger.info(f" Target VM: {VM_USER}@{VM_HOST}")
logger.info(f" Project path: {PROJECT_PATH}")
logger.info(f" Deploy branch: {DEPLOY_BRANCH}")
if args.test:
print("✅ Configuration validation passed")
print(f"Webhook would listen on port {args.port}")
print(f"Target: {VM_USER}@{VM_HOST}")
return
logger.info(f"Starting webhook listener on port {args.port}")
try:
server = HTTPServer(('0.0.0.0', args.port), GitHubWebhookHandler)
logger.info(f"Webhook listener started successfully on http://0.0.0.0:{args.port}")
logger.info("Health check available at: /health")
server.serve_forever()
except KeyboardInterrupt:
logger.info("Webhook listener stopped by user")
except Exception as e:
logger.error(f"Failed to start webhook listener: {e}")
sys.exit(1)
if __name__ == '__main__':
main()