mirror of
https://github.com/pacnpal/thrillwiki_django_no_react.git
synced 2025-12-20 10:51:09 -05:00
Add Road Trip Planner template with interactive map and trip management features
- Implemented a new HTML template for the Road Trip Planner. - Integrated Leaflet.js for interactive mapping and routing. - Added functionality for searching and selecting parks to include in a trip. - Enabled drag-and-drop reordering of selected parks. - Included trip optimization and route calculation features. - Created a summary display for trip statistics. - Added functionality to save trips and manage saved trips. - Enhanced UI with responsive design and dark mode support.
This commit is contained in:
129
scripts/ci-start.sh
Executable file
129
scripts/ci-start.sh
Executable file
@@ -0,0 +1,129 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ThrillWiki Local CI Start Script
|
||||
# This script starts the Django development server following project requirements
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
# Configuration
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
LOG_DIR="$PROJECT_DIR/logs"
|
||||
PID_FILE="$LOG_DIR/django.pid"
|
||||
LOG_FILE="$LOG_DIR/django.log"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Logging function
|
||||
log() {
|
||||
echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
|
||||
}
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Change to project directory
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
log "Starting ThrillWiki CI deployment..."
|
||||
|
||||
# Check if UV is installed
|
||||
if ! command -v uv &> /dev/null; then
|
||||
log_error "UV is not installed. Please install UV first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Stop any existing Django processes on port 8000
|
||||
log "Stopping any existing Django processes on port 8000..."
|
||||
if lsof -ti :8000 >/dev/null 2>&1; then
|
||||
lsof -ti :8000 | xargs kill -9 2>/dev/null || true
|
||||
log_success "Stopped existing processes"
|
||||
else
|
||||
log "No existing processes found on port 8000"
|
||||
fi
|
||||
|
||||
# Clean up Python cache files
|
||||
log "Cleaning up Python cache files..."
|
||||
find . -type d -name "__pycache__" -exec rm -r {} + 2>/dev/null || true
|
||||
log_success "Cache files cleaned"
|
||||
|
||||
# Install/update dependencies
|
||||
log "Installing/updating dependencies with UV..."
|
||||
uv sync --no-dev || {
|
||||
log_error "Failed to sync dependencies"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Run database migrations
|
||||
log "Running database migrations..."
|
||||
uv run manage.py migrate || {
|
||||
log_error "Database migrations failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Collect static files
|
||||
log "Collecting static files..."
|
||||
uv run manage.py collectstatic --noinput || {
|
||||
log_warning "Static file collection failed, continuing anyway"
|
||||
}
|
||||
|
||||
# Start the development server
|
||||
log "Starting Django development server with Tailwind..."
|
||||
log "Server will be available at: http://localhost:8000"
|
||||
log "Press Ctrl+C to stop the server"
|
||||
|
||||
# Start server and capture PID
|
||||
uv run manage.py tailwind runserver 0.0.0.0:8000 &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Save PID to file
|
||||
echo $SERVER_PID > "$PID_FILE"
|
||||
|
||||
log_success "Django server started with PID: $SERVER_PID"
|
||||
log "Server logs are being written to: $LOG_FILE"
|
||||
|
||||
# Wait for server to start
|
||||
sleep 3
|
||||
|
||||
# Check if server is running
|
||||
if kill -0 $SERVER_PID 2>/dev/null; then
|
||||
log_success "Server is running successfully!"
|
||||
|
||||
# Monitor the process
|
||||
wait $SERVER_PID
|
||||
else
|
||||
log_error "Server failed to start"
|
||||
rm -f "$PID_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Cleanup on exit
|
||||
cleanup() {
|
||||
log "Shutting down server..."
|
||||
if [ -f "$PID_FILE" ]; then
|
||||
PID=$(cat "$PID_FILE")
|
||||
if kill -0 $PID 2>/dev/null; then
|
||||
kill $PID
|
||||
log_success "Server stopped"
|
||||
fi
|
||||
rm -f "$PID_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
220
scripts/github-auth.py
Executable file
220
scripts/github-auth.py
Executable file
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
GitHub OAuth Device Flow Authentication for ThrillWiki CI/CD
|
||||
This script implements GitHub's device flow to securely obtain access tokens.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import requests
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
|
||||
# GitHub OAuth App Configuration
|
||||
CLIENT_ID = "Iv23liOX5Hp75AxhUvIe"
|
||||
TOKEN_FILE = ".github-token"
|
||||
|
||||
def parse_response(response):
|
||||
"""Parse HTTP response and handle errors."""
|
||||
if response.status_code in [200, 201]:
|
||||
return response.json()
|
||||
elif response.status_code == 401:
|
||||
print("You are not authorized. Run the `login` command.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"HTTP {response.status_code}: {response.text}")
|
||||
sys.exit(1)
|
||||
|
||||
def request_device_code():
|
||||
"""Request a device code from GitHub."""
|
||||
url = "https://github.com/login/device/code"
|
||||
data = {"client_id": CLIENT_ID}
|
||||
headers = {"Accept": "application/json"}
|
||||
|
||||
response = requests.post(url, data=data, headers=headers)
|
||||
return parse_response(response)
|
||||
|
||||
def request_token(device_code):
|
||||
"""Request an access token using the device code."""
|
||||
url = "https://github.com/login/oauth/access_token"
|
||||
data = {
|
||||
"client_id": CLIENT_ID,
|
||||
"device_code": device_code,
|
||||
"grant_type": "urn:ietf:params:oauth:grant-type:device_code"
|
||||
}
|
||||
headers = {"Accept": "application/json"}
|
||||
|
||||
response = requests.post(url, data=data, headers=headers)
|
||||
return parse_response(response)
|
||||
|
||||
def poll_for_token(device_code, interval):
|
||||
"""Poll GitHub for the access token after user authorization."""
|
||||
print("Waiting for authorization...")
|
||||
|
||||
while True:
|
||||
response = request_token(device_code)
|
||||
error = response.get("error")
|
||||
access_token = response.get("access_token")
|
||||
|
||||
if error:
|
||||
if error == "authorization_pending":
|
||||
# User hasn't entered the code yet
|
||||
print(".", end="", flush=True)
|
||||
time.sleep(interval)
|
||||
continue
|
||||
elif error == "slow_down":
|
||||
# Polling too fast
|
||||
time.sleep(interval + 5)
|
||||
continue
|
||||
elif error == "expired_token":
|
||||
print("\nThe device code has expired. Please run `login` again.")
|
||||
sys.exit(1)
|
||||
elif error == "access_denied":
|
||||
print("\nLogin cancelled by user.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"\nError: {response}")
|
||||
sys.exit(1)
|
||||
|
||||
# Success! Save the token
|
||||
token_path = Path(TOKEN_FILE)
|
||||
token_path.write_text(access_token)
|
||||
token_path.chmod(0o600) # Read/write for owner only
|
||||
|
||||
print(f"\nToken saved to {TOKEN_FILE}")
|
||||
break
|
||||
|
||||
def login():
|
||||
"""Initiate the GitHub OAuth device flow login process."""
|
||||
print("Starting GitHub authentication...")
|
||||
|
||||
device_response = request_device_code()
|
||||
verification_uri = device_response["verification_uri"]
|
||||
user_code = device_response["user_code"]
|
||||
device_code = device_response["device_code"]
|
||||
interval = device_response["interval"]
|
||||
|
||||
print(f"\nPlease visit: {verification_uri}")
|
||||
print(f"and enter code: {user_code}")
|
||||
print("\nWaiting for you to complete authorization in your browser...")
|
||||
|
||||
poll_for_token(device_code, interval)
|
||||
print("Successfully authenticated!")
|
||||
return True
|
||||
|
||||
def whoami():
|
||||
"""Display information about the authenticated user."""
|
||||
token_path = Path(TOKEN_FILE)
|
||||
|
||||
if not token_path.exists():
|
||||
print("You are not authorized. Run the `login` command.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
token = token_path.read_text().strip()
|
||||
except Exception as e:
|
||||
print(f"Error reading token: {e}")
|
||||
print("You may need to run the `login` command again.")
|
||||
sys.exit(1)
|
||||
|
||||
url = "https://api.github.com/user"
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"Authorization": f"Bearer {token}"
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers)
|
||||
user_data = parse_response(response)
|
||||
|
||||
print(f"You are authenticated as: {user_data['login']}")
|
||||
print(f"Name: {user_data.get('name', 'Not set')}")
|
||||
print(f"Email: {user_data.get('email', 'Not public')}")
|
||||
|
||||
return user_data
|
||||
|
||||
def get_token():
|
||||
"""Get the current access token if available."""
|
||||
token_path = Path(TOKEN_FILE)
|
||||
|
||||
if not token_path.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
return token_path.read_text().strip()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def validate_token():
|
||||
"""Validate that the current token is still valid."""
|
||||
token = get_token()
|
||||
if not token:
|
||||
return False
|
||||
|
||||
url = "https://api.github.com/user"
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"Authorization": f"Bearer {token}"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers)
|
||||
return response.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def ensure_authenticated():
|
||||
"""Ensure user is authenticated, prompting login if necessary."""
|
||||
if validate_token():
|
||||
return get_token()
|
||||
|
||||
print("GitHub authentication required.")
|
||||
login()
|
||||
return get_token()
|
||||
|
||||
def logout():
|
||||
"""Remove the stored access token."""
|
||||
token_path = Path(TOKEN_FILE)
|
||||
|
||||
if token_path.exists():
|
||||
token_path.unlink()
|
||||
print("Successfully logged out.")
|
||||
else:
|
||||
print("You are not currently logged in.")
|
||||
|
||||
def main():
|
||||
"""Main CLI interface."""
|
||||
parser = argparse.ArgumentParser(description="GitHub OAuth authentication for ThrillWiki CI/CD")
|
||||
parser.add_argument("command", choices=["login", "logout", "whoami", "token", "validate"],
|
||||
help="Command to execute")
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "login":
|
||||
login()
|
||||
elif args.command == "logout":
|
||||
logout()
|
||||
elif args.command == "whoami":
|
||||
whoami()
|
||||
elif args.command == "token":
|
||||
token = get_token()
|
||||
if token:
|
||||
print(token)
|
||||
else:
|
||||
print("No token available. Run `login` first.")
|
||||
sys.exit(1)
|
||||
elif args.command == "validate":
|
||||
if validate_token():
|
||||
print("Token is valid.")
|
||||
else:
|
||||
print("Token is invalid or missing.")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
268
scripts/setup-vm-ci.sh
Executable file
268
scripts/setup-vm-ci.sh
Executable file
@@ -0,0 +1,268 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ThrillWiki VM CI Setup Script
|
||||
# This script helps set up the VM deployment system
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log() {
|
||||
echo -e "${BLUE}[SETUP]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Configuration prompts
|
||||
prompt_config() {
|
||||
log "Setting up ThrillWiki VM CI/CD system..."
|
||||
echo
|
||||
|
||||
read -p "Enter your VM IP address: " VM_IP
|
||||
read -p "Enter your VM username (default: ubuntu): " VM_USER
|
||||
VM_USER=${VM_USER:-ubuntu}
|
||||
|
||||
read -p "Enter your GitHub repository URL: " REPO_URL
|
||||
read -p "Enter your GitHub webhook secret: " WEBHOOK_SECRET
|
||||
|
||||
read -p "Enter local webhook port (default: 9000): " WEBHOOK_PORT
|
||||
WEBHOOK_PORT=${WEBHOOK_PORT:-9000}
|
||||
|
||||
read -p "Enter VM project path (default: /home/$VM_USER/thrillwiki): " VM_PROJECT_PATH
|
||||
VM_PROJECT_PATH=${VM_PROJECT_PATH:-/home/$VM_USER/thrillwiki}
|
||||
}
|
||||
|
||||
# Create SSH key
|
||||
setup_ssh() {
|
||||
log "Setting up SSH keys..."
|
||||
|
||||
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
|
||||
|
||||
if [ ! -f "$ssh_key_path" ]; then
|
||||
ssh-keygen -t rsa -b 4096 -f "$ssh_key_path" -N ""
|
||||
log_success "SSH key generated: $ssh_key_path"
|
||||
|
||||
log "Please copy the following public key to your VM:"
|
||||
echo "---"
|
||||
cat "$ssh_key_path.pub"
|
||||
echo "---"
|
||||
echo
|
||||
log "Run this on your VM:"
|
||||
echo "mkdir -p ~/.ssh && echo '$(cat "$ssh_key_path.pub")' >> ~/.ssh/***REMOVED*** && chmod 600 ~/.ssh/***REMOVED***"
|
||||
echo
|
||||
read -p "Press Enter when you've added the key to your VM..."
|
||||
else
|
||||
log "SSH key already exists: $ssh_key_path"
|
||||
fi
|
||||
|
||||
# Test SSH connection
|
||||
log "Testing SSH connection..."
|
||||
if ssh -i "$ssh_key_path" -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$VM_USER@$VM_IP" "echo 'SSH connection successful'"; then
|
||||
log_success "SSH connection test passed"
|
||||
else
|
||||
log_error "SSH connection test failed"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Create environment file
|
||||
create_env_file() {
|
||||
log "Creating webhook environment file..."
|
||||
|
||||
cat > ***REMOVED***.webhook << EOF
|
||||
# ThrillWiki Webhook Configuration
|
||||
WEBHOOK_PORT=$WEBHOOK_PORT
|
||||
WEBHOOK_SECRET=$WEBHOOK_SECRET
|
||||
VM_HOST=$VM_IP
|
||||
VM_PORT=22
|
||||
VM_USER=$VM_USER
|
||||
VM_KEY_PATH=$HOME/.ssh/thrillwiki_vm
|
||||
VM_PROJECT_PATH=$VM_PROJECT_PATH
|
||||
REPO_URL=$REPO_URL
|
||||
DEPLOY_BRANCH=main
|
||||
EOF
|
||||
|
||||
log_success "Environment file created: ***REMOVED***.webhook"
|
||||
}
|
||||
|
||||
# Setup VM
|
||||
setup_vm() {
|
||||
log "Setting up VM environment..."
|
||||
|
||||
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
|
||||
|
||||
# Create setup script for VM
|
||||
cat > /tmp/vm_setup.sh << 'EOF'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "Setting up VM for ThrillWiki deployment..."
|
||||
|
||||
# Update system
|
||||
sudo apt update
|
||||
|
||||
# Install required packages
|
||||
sudo apt install -y git curl build-essential python3-pip lsof
|
||||
|
||||
# Install UV if not present
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "Installing UV..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
source ~/.cargo/env
|
||||
fi
|
||||
|
||||
# Clone repository if not present
|
||||
if [ ! -d "thrillwiki" ]; then
|
||||
echo "Cloning repository..."
|
||||
git clone REPO_URL_PLACEHOLDER thrillwiki
|
||||
fi
|
||||
|
||||
cd thrillwiki
|
||||
|
||||
# Install dependencies
|
||||
uv sync
|
||||
|
||||
# Create directories
|
||||
mkdir -p logs backups
|
||||
|
||||
# Make scripts executable
|
||||
chmod +x scripts/*.sh
|
||||
|
||||
echo "VM setup completed successfully!"
|
||||
EOF
|
||||
|
||||
# Replace placeholder with actual repo URL
|
||||
sed -i.bak "s|REPO_URL_PLACEHOLDER|$REPO_URL|g" /tmp/vm_setup.sh
|
||||
|
||||
# Copy and execute setup script on VM
|
||||
scp -i "$ssh_key_path" /tmp/vm_setup.sh "$VM_USER@$VM_IP:/tmp/"
|
||||
ssh -i "$ssh_key_path" "$VM_USER@$VM_IP" "bash /tmp/vm_setup.sh"
|
||||
|
||||
log_success "VM setup completed"
|
||||
|
||||
# Cleanup
|
||||
rm /tmp/vm_setup.sh /tmp/vm_setup.sh.bak
|
||||
}
|
||||
|
||||
# Install systemd services
|
||||
setup_services() {
|
||||
log "Setting up systemd services on VM..."
|
||||
|
||||
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
|
||||
|
||||
# Copy service files and install them
|
||||
ssh -i "$ssh_key_path" "$VM_USER@$VM_IP" << EOF
|
||||
cd thrillwiki
|
||||
|
||||
# Update service files with correct paths
|
||||
sed -i 's|/home/ubuntu|/home/$VM_USER|g' scripts/systemd/*.service
|
||||
sed -i 's|ubuntu|$VM_USER|g' scripts/systemd/*.service
|
||||
|
||||
# Install services
|
||||
sudo cp scripts/systemd/thrillwiki.service /etc/systemd/system/
|
||||
sudo cp scripts/systemd/thrillwiki-webhook.service /etc/systemd/system/
|
||||
|
||||
# Reload and enable services
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable thrillwiki.service
|
||||
|
||||
echo "Services installed successfully!"
|
||||
EOF
|
||||
|
||||
log_success "Systemd services installed"
|
||||
}
|
||||
|
||||
# Test deployment
|
||||
test_deployment() {
|
||||
log "Testing VM deployment..."
|
||||
|
||||
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
|
||||
|
||||
ssh -i "$ssh_key_path" "$VM_USER@$VM_IP" << EOF
|
||||
cd thrillwiki
|
||||
./scripts/vm-deploy.sh
|
||||
EOF
|
||||
|
||||
log_success "Deployment test completed"
|
||||
}
|
||||
|
||||
# Start webhook listener
|
||||
start_webhook() {
|
||||
log "Starting webhook listener..."
|
||||
|
||||
if [ -f "***REMOVED***.webhook" ]; then
|
||||
log "Webhook configuration found. You can start the webhook listener with:"
|
||||
echo " source ***REMOVED***.webhook && python3 scripts/webhook-listener.py"
|
||||
echo
|
||||
log "Or run it in the background:"
|
||||
echo " nohup python3 scripts/webhook-listener.py > logs/webhook.log 2>&1 &"
|
||||
else
|
||||
log_error "Webhook configuration not found!"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# GitHub webhook instructions
|
||||
github_instructions() {
|
||||
log "GitHub Webhook Setup Instructions:"
|
||||
echo
|
||||
echo "1. Go to your GitHub repository: $REPO_URL"
|
||||
echo "2. Navigate to Settings → Webhooks"
|
||||
echo "3. Click 'Add webhook'"
|
||||
echo "4. Configure:"
|
||||
echo " - Payload URL: http://YOUR_PUBLIC_IP:$WEBHOOK_PORT/webhook"
|
||||
echo " - Content type: application/json"
|
||||
echo " - Secret: $WEBHOOK_SECRET"
|
||||
echo " - Events: Just the push event"
|
||||
echo "5. Click 'Add webhook'"
|
||||
echo
|
||||
log_warning "Make sure port $WEBHOOK_PORT is open on your firewall!"
|
||||
}
|
||||
|
||||
# Main setup flow
|
||||
main() {
|
||||
log "ThrillWiki VM CI/CD Setup"
|
||||
echo "=========================="
|
||||
echo
|
||||
|
||||
# Create logs directory
|
||||
mkdir -p logs
|
||||
|
||||
# Get configuration
|
||||
prompt_config
|
||||
|
||||
# Setup steps
|
||||
setup_ssh
|
||||
create_env_file
|
||||
setup_vm
|
||||
setup_services
|
||||
test_deployment
|
||||
|
||||
# Final instructions
|
||||
echo
|
||||
log_success "Setup completed successfully!"
|
||||
echo
|
||||
start_webhook
|
||||
echo
|
||||
github_instructions
|
||||
|
||||
log "Setup log saved to: logs/setup.log"
|
||||
}
|
||||
|
||||
# Run main function and log output
|
||||
main "$@" 2>&1 | tee logs/setup.log
|
||||
39
scripts/systemd/thrillwiki-webhook.service
Normal file
39
scripts/systemd/thrillwiki-webhook.service
Normal file
@@ -0,0 +1,39 @@
|
||||
[Unit]
|
||||
Description=ThrillWiki GitHub Webhook Listener
|
||||
After=network.target
|
||||
Wants=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=ubuntu
|
||||
Group=ubuntu
|
||||
[AWS-SECRET-REMOVED]
|
||||
ExecStart=/usr/bin/python3 /home/ubuntu/thrillwiki/scripts/webhook-listener.py
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
# Environment variables
|
||||
Environment=WEBHOOK_PORT=9000
|
||||
Environment=WEBHOOK_SECRET=your_webhook_secret_here
|
||||
Environment=VM_HOST=localhost
|
||||
Environment=VM_PORT=22
|
||||
Environment=VM_USER=ubuntu
|
||||
Environment=VM_KEY_PATH=/home/ubuntu/.ssh/***REMOVED***
|
||||
Environment=VM_PROJECT_PATH=/home/ubuntu/thrillwiki
|
||||
Environment=REPO_URL=https://github.com/YOUR_USERNAME/thrillwiki_django_no_react.git
|
||||
Environment=DEPLOY_BRANCH=main
|
||||
|
||||
# Security settings
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
[AWS-SECRET-REMOVED]ogs
|
||||
|
||||
# Logging
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=thrillwiki-webhook
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
45
scripts/systemd/thrillwiki.service
Normal file
45
scripts/systemd/thrillwiki.service
Normal file
@@ -0,0 +1,45 @@
|
||||
[Unit]
|
||||
Description=ThrillWiki Django Application
|
||||
After=network.target postgresql.service
|
||||
Wants=network.target
|
||||
Requires=postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=forking
|
||||
User=ubuntu
|
||||
Group=ubuntu
|
||||
[AWS-SECRET-REMOVED]
|
||||
[AWS-SECRET-REMOVED]s/ci-start.sh
|
||||
ExecStop=/bin/kill -TERM $MAINPID
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
[AWS-SECRET-REMOVED]ngo.pid
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
# Environment variables
|
||||
Environment=DJANGO_SETTINGS_MODULE=thrillwiki.settings
|
||||
[AWS-SECRET-REMOVED]llwiki
|
||||
Environment=PATH=/home/ubuntu/.cargo/bin:/usr/local/bin:/usr/bin:/bin
|
||||
|
||||
# Security settings
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
[AWS-SECRET-REMOVED]ogs
|
||||
[AWS-SECRET-REMOVED]edia
|
||||
[AWS-SECRET-REMOVED]taticfiles
|
||||
[AWS-SECRET-REMOVED]ploads
|
||||
|
||||
# Resource limits
|
||||
LimitNOFILE=65536
|
||||
TimeoutStartSec=300
|
||||
TimeoutStopSec=30
|
||||
|
||||
# Logging
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=thrillwiki
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
175
scripts/test-automation.sh
Executable file
175
scripts/test-automation.sh
Executable file
@@ -0,0 +1,175 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ThrillWiki Automation Test Script
|
||||
# This script validates all automation components without actually running them
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log() {
|
||||
echo -e "${BLUE}[TEST]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[✓]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[!]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[✗]${NC} $1"
|
||||
}
|
||||
|
||||
# Test counters
|
||||
TESTS_PASSED=0
|
||||
TESTS_FAILED=0
|
||||
TESTS_TOTAL=0
|
||||
|
||||
test_case() {
|
||||
local name="$1"
|
||||
local command="$2"
|
||||
|
||||
((TESTS_TOTAL++))
|
||||
log "Testing: $name"
|
||||
|
||||
if eval "$command" >/dev/null 2>&1; then
|
||||
log_success "$name"
|
||||
((TESTS_PASSED++))
|
||||
else
|
||||
log_error "$name"
|
||||
((TESTS_FAILED++))
|
||||
fi
|
||||
}
|
||||
|
||||
test_case_with_output() {
|
||||
local name="$1"
|
||||
local command="$2"
|
||||
local expected_pattern="$3"
|
||||
|
||||
((TESTS_TOTAL++))
|
||||
log "Testing: $name"
|
||||
|
||||
local output
|
||||
if output=$(eval "$command" 2>&1); then
|
||||
if [[ -n "$expected_pattern" && ! "$output" =~ $expected_pattern ]]; then
|
||||
log_error "$name (unexpected output)"
|
||||
((TESTS_FAILED++))
|
||||
else
|
||||
log_success "$name"
|
||||
((TESTS_PASSED++))
|
||||
fi
|
||||
else
|
||||
log_error "$name (command failed)"
|
||||
((TESTS_FAILED++))
|
||||
fi
|
||||
}
|
||||
|
||||
log "🧪 Starting ThrillWiki Automation Tests"
|
||||
echo "======================================"
|
||||
|
||||
# Test 1: File Permissions
|
||||
log "\n📁 Testing File Permissions..."
|
||||
test_case "CI start script is executable" "[ -x scripts/ci-start.sh ]"
|
||||
test_case "VM deploy script is executable" "[ -x scripts/vm-deploy.sh ]"
|
||||
test_case "Webhook listener is executable" "[ -x scripts/webhook-listener.py ]"
|
||||
test_case "VM manager is executable" "[ -x scripts/unraid/vm-manager.py ]"
|
||||
test_case "Complete automation script is executable" "[ -x scripts/unraid/setup-complete-automation.sh ]"
|
||||
|
||||
# Test 2: Script Syntax
|
||||
log "\n🔍 Testing Script Syntax..."
|
||||
test_case "CI start script syntax" "bash -n scripts/ci-start.sh"
|
||||
test_case "VM deploy script syntax" "bash -n scripts/vm-deploy.sh"
|
||||
test_case "Setup VM CI script syntax" "bash -n scripts/setup-vm-ci.sh"
|
||||
test_case "Complete automation script syntax" "bash -n scripts/unraid/setup-complete-automation.sh"
|
||||
test_case "Webhook listener Python syntax" "python3 -m py_compile scripts/webhook-listener.py"
|
||||
test_case "VM manager Python syntax" "python3 -m py_compile scripts/unraid/vm-manager.py"
|
||||
|
||||
# Test 3: Help Functions
|
||||
log "\n❓ Testing Help Functions..."
|
||||
test_case_with_output "VM manager help" "python3 scripts/unraid/vm-manager.py --help" "usage:"
|
||||
test_case_with_output "Webhook listener help" "python3 scripts/webhook-listener.py --help" "usage:"
|
||||
test_case_with_output "VM deploy script usage" "scripts/vm-deploy.sh invalid 2>&1" "Usage:"
|
||||
|
||||
# Test 4: Configuration Validation
|
||||
log "\n⚙️ Testing Configuration Validation..."
|
||||
test_case_with_output "Webhook listener test mode" "python3 scripts/webhook-listener.py --test" "Configuration validation"
|
||||
|
||||
# Test 5: Directory Structure
|
||||
log "\n📂 Testing Directory Structure..."
|
||||
test_case "Scripts directory exists" "[ -d scripts ]"
|
||||
test_case "Unraid scripts directory exists" "[ -d scripts/unraid ]"
|
||||
test_case "Systemd directory exists" "[ -d scripts/systemd ]"
|
||||
test_case "Docs directory exists" "[ -d docs ]"
|
||||
test_case "Logs directory created" "[ -d logs ]"
|
||||
|
||||
# Test 6: Required Files
|
||||
log "\n📄 Testing Required Files..."
|
||||
test_case "ThrillWiki service file exists" "[ -f scripts/systemd/thrillwiki.service ]"
|
||||
test_case "Webhook service file exists" "[ -f scripts/systemd/thrillwiki-webhook.service ]"
|
||||
test_case "VM deployment setup doc exists" "[ -f docs/VM_DEPLOYMENT_SETUP.md ]"
|
||||
test_case "Unraid automation doc exists" "[ -f docs/UNRAID_COMPLETE_AUTOMATION.md ]"
|
||||
test_case "CI README exists" "[ -f CI_README.md ]"
|
||||
|
||||
# Test 7: Python Dependencies
|
||||
log "\n🐍 Testing Python Dependencies..."
|
||||
test_case "Python 3 available" "command -v python3"
|
||||
test_case "Requests module available" "python3 -c 'import requests'"
|
||||
test_case "JSON module available" "python3 -c 'import json'"
|
||||
test_case "OS module available" "python3 -c 'import os'"
|
||||
test_case "Subprocess module available" "python3 -c 'import subprocess'"
|
||||
|
||||
# Test 8: System Dependencies
|
||||
log "\n🔧 Testing System Dependencies..."
|
||||
test_case "SSH command available" "command -v ssh"
|
||||
test_case "SCP command available" "command -v scp"
|
||||
test_case "Bash available" "command -v bash"
|
||||
test_case "Git available" "command -v git"
|
||||
|
||||
# Test 9: UV Package Manager
|
||||
log "\n📦 Testing UV Package Manager..."
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
log_success "UV package manager is available"
|
||||
((TESTS_PASSED++))
|
||||
test_case "UV version check" "uv --version"
|
||||
else
|
||||
log_warning "UV package manager not found (will be installed during setup)"
|
||||
((TESTS_PASSED++))
|
||||
fi
|
||||
((TESTS_TOTAL++))
|
||||
|
||||
# Test 10: Django Project Structure
|
||||
log "\n🌟 Testing Django Project Structure..."
|
||||
test_case "Django manage.py exists" "[ -f manage.py ]"
|
||||
test_case "Django settings module exists" "[ -f thrillwiki/settings.py ]"
|
||||
test_case "PyProject.toml exists" "[ -f pyproject.toml ]"
|
||||
|
||||
# Final Results
|
||||
echo
|
||||
log "📊 Test Results Summary"
|
||||
echo "======================"
|
||||
echo "Total Tests: $TESTS_TOTAL"
|
||||
echo "Passed: $TESTS_PASSED"
|
||||
echo "Failed: $TESTS_FAILED"
|
||||
|
||||
if [ $TESTS_FAILED -eq 0 ]; then
|
||||
echo
|
||||
log_success "🎉 All tests passed! The automation system is ready."
|
||||
echo
|
||||
log "Next steps:"
|
||||
echo "1. For complete automation: ./scripts/unraid/setup-complete-automation.sh"
|
||||
echo "2. For manual setup: ./scripts/setup-vm-ci.sh"
|
||||
echo "3. Read documentation: docs/UNRAID_COMPLETE_AUTOMATION.md"
|
||||
exit 0
|
||||
else
|
||||
echo
|
||||
log_error "❌ Some tests failed. Please check the issues above."
|
||||
exit 1
|
||||
fi
|
||||
996
scripts/unraid/setup-complete-automation.sh
Executable file
996
scripts/unraid/setup-complete-automation.sh
Executable file
@@ -0,0 +1,996 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ThrillWiki Complete Unraid Automation Setup
|
||||
# This script automates the entire VM creation and deployment process on Unraid
|
||||
#
|
||||
# Usage:
|
||||
# ./setup-complete-automation.sh # Standard setup
|
||||
# ./setup-complete-automation.sh --reset # Delete VM and config, start completely fresh
|
||||
# ./setup-complete-automation.sh --reset-vm # Delete VM only, keep configuration
|
||||
# ./setup-complete-automation.sh --reset-config # Delete config only, keep VM
|
||||
|
||||
# Function to show help
|
||||
show_help() {
|
||||
echo "ThrillWiki CI/CD Automation Setup"
|
||||
echo ""
|
||||
echo "Usage:"
|
||||
echo " $0 Set up or update ThrillWiki automation"
|
||||
echo " $0 --reset Delete VM and config, start completely fresh"
|
||||
echo " $0 --reset-vm Delete VM only, keep configuration"
|
||||
echo " $0 --reset-config Delete config only, keep VM"
|
||||
echo " $0 --help Show this help message"
|
||||
echo ""
|
||||
echo "Reset Options:"
|
||||
echo " --reset Completely removes existing VM, disks, and config"
|
||||
echo " before starting fresh installation"
|
||||
echo " --reset-vm Removes only the VM and disks, preserves saved"
|
||||
echo " configuration to avoid re-entering settings"
|
||||
echo " --reset-config Removes only the saved configuration, preserves"
|
||||
echo " VM and prompts for fresh configuration input"
|
||||
echo " --help Display this help and exit"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Normal setup/update"
|
||||
echo " $0 --reset # Complete fresh installation"
|
||||
echo " $0 --reset-vm # Fresh VM with saved settings"
|
||||
echo " $0 --reset-config # Re-configure existing VM"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Check for help flag
|
||||
if [[ "$1" == "--help" || "$1" == "-h" ]]; then
|
||||
show_help
|
||||
fi
|
||||
|
||||
# Parse reset flags
|
||||
RESET_ALL=false
|
||||
RESET_VM_ONLY=false
|
||||
RESET_CONFIG_ONLY=false
|
||||
|
||||
if [[ "$1" == "--reset" ]]; then
|
||||
RESET_ALL=true
|
||||
echo "🔄 COMPLETE RESET MODE: Will delete VM and configuration"
|
||||
elif [[ "$1" == "--reset-vm" ]]; then
|
||||
RESET_VM_ONLY=true
|
||||
echo "🔄 VM RESET MODE: Will delete VM only, keep configuration"
|
||||
elif [[ "$1" == "--reset-config" ]]; then
|
||||
RESET_CONFIG_ONLY=true
|
||||
echo "🔄 CONFIG RESET MODE: Will delete configuration only, keep VM"
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log() {
|
||||
echo -e "${BLUE}[AUTOMATION]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Configuration
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
LOG_DIR="$PROJECT_DIR/logs"
|
||||
|
||||
# Default values
|
||||
DEFAULT_UNRAID_HOST=""
|
||||
DEFAULT_VM_NAME="thrillwiki-vm"
|
||||
DEFAULT_VM_MEMORY="4096"
|
||||
DEFAULT_VM_VCPUS="2"
|
||||
DEFAULT_VM_DISK_SIZE="50"
|
||||
DEFAULT_WEBHOOK_PORT="9000"
|
||||
|
||||
# Configuration file
|
||||
CONFIG_FILE="$PROJECT_DIR/.thrillwiki-config"
|
||||
|
||||
# Function to save configuration
|
||||
save_config() {
|
||||
log "Saving configuration to $CONFIG_FILE..."
|
||||
cat > "$CONFIG_FILE" << EOF
|
||||
# ThrillWiki Automation Configuration
|
||||
# This file stores your settings to avoid re-entering them each time
|
||||
|
||||
# Unraid Server Configuration
|
||||
UNRAID_HOST="$UNRAID_HOST"
|
||||
UNRAID_USER="$UNRAID_USER"
|
||||
VM_NAME="$VM_NAME"
|
||||
VM_MEMORY="$VM_MEMORY"
|
||||
VM_VCPUS="$VM_VCPUS"
|
||||
VM_DISK_SIZE="$VM_DISK_SIZE"
|
||||
|
||||
# Network Configuration
|
||||
VM_IP="$VM_IP"
|
||||
VM_GATEWAY="$VM_GATEWAY"
|
||||
VM_NETMASK="$VM_NETMASK"
|
||||
VM_NETWORK="$VM_NETWORK"
|
||||
|
||||
# GitHub Configuration
|
||||
REPO_URL="$REPO_URL"
|
||||
GITHUB_USERNAME="$GITHUB_USERNAME"
|
||||
GITHUB_API_ENABLED="$GITHUB_API_ENABLED"
|
||||
GITHUB_AUTH_METHOD="$GITHUB_AUTH_METHOD"
|
||||
|
||||
# Webhook Configuration
|
||||
WEBHOOK_PORT="$WEBHOOK_PORT"
|
||||
WEBHOOK_ENABLED="$WEBHOOK_ENABLED"
|
||||
|
||||
# SSH Configuration (path to key, not the key content)
|
||||
SSH_KEY_PATH="$HOME/.ssh/thrillwiki_vm"
|
||||
EOF
|
||||
|
||||
log_success "Configuration saved to $CONFIG_FILE"
|
||||
}
|
||||
|
||||
# Function to load configuration
|
||||
load_config() {
|
||||
if [ -f "$CONFIG_FILE" ]; then
|
||||
log "Loading existing configuration from $CONFIG_FILE..."
|
||||
source "$CONFIG_FILE"
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to prompt for configuration
|
||||
prompt_unraid_config() {
|
||||
log "=== Unraid VM Configuration ==="
|
||||
echo
|
||||
|
||||
# Try to load existing config first
|
||||
if load_config; then
|
||||
log_success "Loaded existing configuration"
|
||||
echo "Current settings:"
|
||||
echo " Unraid Host: $UNRAID_HOST"
|
||||
echo " VM Name: $VM_NAME"
|
||||
echo " VM IP: $VM_IP"
|
||||
echo " Repository: $REPO_URL"
|
||||
echo
|
||||
read -p "Use existing configuration? (y/n): " use_existing
|
||||
if [ "$use_existing" = "y" ] || [ "$use_existing" = "Y" ]; then
|
||||
# Still need to get sensitive info that we don't save
|
||||
read -s -p "Enter Unraid [PASSWORD-REMOVED]
|
||||
echo
|
||||
|
||||
# Handle GitHub authentication based on saved method
|
||||
if [ -n "$GITHUB_USERNAME" ] && [ "$GITHUB_API_ENABLED" = "true" ]; then
|
||||
if [ "$GITHUB_AUTH_METHOD" = "oauth" ]; then
|
||||
# Check if OAuth token is still valid
|
||||
if python3 "$SCRIPT_DIR/../github-auth.py" validate 2>/dev/null; then
|
||||
GITHUB_TOKEN=$(python3 "$SCRIPT_DIR/../github-auth.py" token)
|
||||
log "Using existing OAuth token"
|
||||
else
|
||||
log "OAuth token expired, re-authenticating..."
|
||||
if python3 "$SCRIPT_DIR/../github-auth.py" login; then
|
||||
GITHUB_TOKEN=$(python3 "$SCRIPT_DIR/../github-auth.py" token)
|
||||
log_success "OAuth token refreshed"
|
||||
else
|
||||
log_error "OAuth re-authentication failed"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
# Personal access token method
|
||||
read -s -p "Enter GitHub personal access token: " GITHUB_TOKEN
|
||||
echo
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$WEBHOOK_ENABLED" = "true" ]; then
|
||||
read -s -p "Enter GitHub webhook secret: " WEBHOOK_SECRET
|
||||
echo
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Prompt for new configuration
|
||||
read -p "Enter your Unraid server IP address: " UNRAID_HOST
|
||||
save_config
|
||||
|
||||
read -p "Enter Unraid username (default: root): " UNRAID_USER
|
||||
UNRAID_USER=${UNRAID_USER:-root}
|
||||
save_config
|
||||
|
||||
read -s -p "Enter Unraid [PASSWORD-REMOVED]
|
||||
echo
|
||||
# Note: Password not saved for security
|
||||
|
||||
read -p "Enter VM name (default: $DEFAULT_VM_NAME): " VM_NAME
|
||||
VM_NAME=${VM_NAME:-$DEFAULT_VM_NAME}
|
||||
save_config
|
||||
|
||||
read -p "Enter VM memory in MB (default: $DEFAULT_VM_MEMORY): " VM_MEMORY
|
||||
VM_MEMORY=${VM_MEMORY:-$DEFAULT_VM_MEMORY}
|
||||
save_config
|
||||
|
||||
read -p "Enter VM vCPUs (default: $DEFAULT_VM_VCPUS): " VM_VCPUS
|
||||
VM_VCPUS=${VM_VCPUS:-$DEFAULT_VM_VCPUS}
|
||||
save_config
|
||||
|
||||
read -p "Enter VM disk size in GB (default: $DEFAULT_VM_DISK_SIZE): " VM_DISK_SIZE
|
||||
VM_DISK_SIZE=${VM_DISK_SIZE:-$DEFAULT_VM_DISK_SIZE}
|
||||
save_config
|
||||
|
||||
read -p "Enter GitHub repository URL: " REPO_URL
|
||||
save_config
|
||||
|
||||
# GitHub API Configuration
|
||||
echo
|
||||
log "=== GitHub API Configuration ==="
|
||||
echo "Choose GitHub authentication method:"
|
||||
echo "1. OAuth Device Flow (recommended - secure, supports private repos)"
|
||||
echo "2. Personal Access Token (manual token entry)"
|
||||
echo "3. Skip (public repositories only)"
|
||||
|
||||
while true; do
|
||||
read -p "Select option (1-3): " auth_choice
|
||||
case $auth_choice in
|
||||
1)
|
||||
log "Using GitHub OAuth Device Flow..."
|
||||
if python3 "$SCRIPT_DIR/../github-auth.py" validate 2>/dev/null; then
|
||||
log "Existing GitHub authentication found and valid"
|
||||
GITHUB_USERNAME=$(python3 "$SCRIPT_DIR/../github-auth.py" whoami 2>/dev/null | grep "You are authenticated as:" | cut -d: -f2 | xargs)
|
||||
GITHUB_TOKEN=$(python3 "$SCRIPT_DIR/../github-auth.py" token)
|
||||
else
|
||||
log "Starting GitHub OAuth authentication..."
|
||||
if python3 "$SCRIPT_DIR/../github-auth.py" login; then
|
||||
GITHUB_USERNAME=$(python3 "$SCRIPT_DIR/../github-auth.py" whoami 2>/dev/null | grep "You are authenticated as:" | cut -d: -f2 | xargs)
|
||||
GITHUB_TOKEN=$(python3 "$SCRIPT_DIR/../github-auth.py" token)
|
||||
log_success "GitHub OAuth authentication completed"
|
||||
else
|
||||
log_error "GitHub authentication failed"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
GITHUB_API_ENABLED=true
|
||||
GITHUB_AUTH_METHOD="oauth"
|
||||
break
|
||||
;;
|
||||
2)
|
||||
read -p "Enter GitHub username: " GITHUB_USERNAME
|
||||
read -s -p "Enter GitHub personal access token: " GITHUB_TOKEN
|
||||
echo
|
||||
if [ -n "$GITHUB_USERNAME" ] && [ -n "$GITHUB_TOKEN" ]; then
|
||||
GITHUB_API_ENABLED=true
|
||||
GITHUB_AUTH_METHOD="token"
|
||||
log "Personal access token configured"
|
||||
else
|
||||
log_error "Both username and token are required"
|
||||
continue
|
||||
fi
|
||||
break
|
||||
;;
|
||||
3)
|
||||
GITHUB_USERNAME=""
|
||||
GITHUB_TOKEN=""
|
||||
GITHUB_API_ENABLED=false
|
||||
GITHUB_AUTH_METHOD="none"
|
||||
log "Skipping GitHub API - using public access only"
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Invalid option. Please select 1, 2, or 3."
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Save GitHub configuration
|
||||
save_config
|
||||
log "GitHub authentication configuration saved"
|
||||
|
||||
# Webhook Configuration
|
||||
echo
|
||||
read -s -p "Enter GitHub webhook secret (optional, press Enter to skip): " WEBHOOK_SECRET
|
||||
echo
|
||||
|
||||
# If no webhook secret provided, disable webhook functionality
|
||||
if [ -z "$WEBHOOK_SECRET" ]; then
|
||||
log "No webhook secret provided - webhook functionality will be disabled"
|
||||
WEBHOOK_ENABLED=false
|
||||
else
|
||||
WEBHOOK_ENABLED=true
|
||||
fi
|
||||
|
||||
read -p "Enter webhook port (default: $DEFAULT_WEBHOOK_PORT): " WEBHOOK_PORT
|
||||
WEBHOOK_PORT=${WEBHOOK_PORT:-$DEFAULT_WEBHOOK_PORT}
|
||||
|
||||
# Save webhook configuration
|
||||
save_config
|
||||
log "Webhook configuration saved"
|
||||
|
||||
# Get VM IP address with proper range validation
|
||||
while true; do
|
||||
read -p "Enter VM IP address (192.168.20.10-192.168.20.100): " VM_IP
|
||||
if [[ "$VM_IP" =~ ^192\.168\.20\.([1-9][0-9]|100)$ ]]; then
|
||||
local ip_last_octet="${BASH_REMATCH[1]}"
|
||||
if [ "$ip_last_octet" -ge 10 ] && [ "$ip_last_octet" -le 100 ]; then
|
||||
break
|
||||
fi
|
||||
fi
|
||||
echo "Invalid IP address. Please enter an IP in the range 192.168.20.10-192.168.20.100"
|
||||
done
|
||||
|
||||
# Set network configuration
|
||||
VM_GATEWAY="192.168.20.1"
|
||||
VM_NETMASK="255.255.255.0"
|
||||
VM_NETWORK="192.168.20.0/24"
|
||||
|
||||
# Save final network configuration
|
||||
save_config
|
||||
log "Network configuration saved - setup complete!"
|
||||
}
|
||||
|
||||
# Generate SSH keys for VM access
|
||||
setup_ssh_keys() {
|
||||
log "Setting up SSH keys for VM access..."
|
||||
|
||||
local ssh_key_path="$HOME/.ssh/thrillwiki_vm"
|
||||
local ssh_config_path="$HOME/.ssh/config"
|
||||
|
||||
if [ ! -f "$ssh_key_path" ]; then
|
||||
ssh-keygen -t rsa -b 4096 -f "$ssh_key_path" -N "" -C "thrillwiki-vm-access"
|
||||
log_success "SSH key generated: $ssh_key_path"
|
||||
else
|
||||
log "SSH key already exists: $ssh_key_path"
|
||||
fi
|
||||
|
||||
# Add SSH config entry
|
||||
if ! grep -q "Host $VM_NAME" "$ssh_config_path" 2>/dev/null; then
|
||||
cat >> "$ssh_config_path" << EOF
|
||||
|
||||
# ThrillWiki VM
|
||||
Host $VM_NAME
|
||||
HostName %h
|
||||
User ubuntu
|
||||
IdentityFile $ssh_key_path
|
||||
StrictHostKeyChecking no
|
||||
UserKnownHostsFile /dev/null
|
||||
EOF
|
||||
log_success "SSH config updated"
|
||||
fi
|
||||
|
||||
# Store public key for VM setup
|
||||
SSH_PUBLIC_KEY=$(cat "$ssh_key_path.pub")
|
||||
export SSH_PUBLIC_KEY
|
||||
}
|
||||
|
||||
# Setup Unraid host access
|
||||
setup_unraid_access() {
|
||||
log "Setting up Unraid server access..."
|
||||
|
||||
local unraid_key_path="$HOME/.ssh/unraid_access"
|
||||
|
||||
if [ ! -f "$unraid_key_path" ]; then
|
||||
ssh-keygen -t rsa -b 4096 -f "$unraid_key_path" -N "" -C "unraid-access"
|
||||
|
||||
log "Please add this public key to your Unraid server:"
|
||||
echo "---"
|
||||
cat "$unraid_key_path.pub"
|
||||
echo "---"
|
||||
echo
|
||||
log "Add this to /root/.ssh/***REMOVED*** on your Unraid server"
|
||||
read -p "Press Enter when you've added the key..."
|
||||
fi
|
||||
|
||||
# Test Unraid connection
|
||||
log "Testing Unraid connection..."
|
||||
if ssh -i "$unraid_key_path" -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$UNRAID_USER@$UNRAID_HOST" "echo 'Connected to Unraid successfully'"; then
|
||||
log_success "Unraid connection test passed"
|
||||
else
|
||||
log_error "Unraid connection test failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Update SSH config for Unraid
|
||||
if ! grep -q "Host unraid" "$HOME/.ssh/config" 2>/dev/null; then
|
||||
cat >> "$HOME/.ssh/config" << EOF
|
||||
|
||||
# Unraid Server
|
||||
Host unraid
|
||||
HostName $UNRAID_HOST
|
||||
User $UNRAID_USER
|
||||
IdentityFile $unraid_key_path
|
||||
StrictHostKeyChecking no
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# Create environment files
|
||||
create_environment_files() {
|
||||
log "Creating environment configuration files..."
|
||||
|
||||
# Get SSH public key content safely
|
||||
local ssh_key_path="$HOME/.ssh/thrillwiki_vm.pub"
|
||||
local ssh_public_key=""
|
||||
if [ -f "$ssh_key_path" ]; then
|
||||
ssh_public_key=$(cat "$ssh_key_path")
|
||||
fi
|
||||
|
||||
# Unraid VM environment
|
||||
cat > "$PROJECT_DIR/***REMOVED***.unraid" << EOF
|
||||
# Unraid VM Configuration
|
||||
UNRAID_HOST=$UNRAID_HOST
|
||||
UNRAID_USER=$UNRAID_USER
|
||||
UNRAID_PASSWORD=$UNRAID_PASSWORD
|
||||
VM_NAME=$VM_NAME
|
||||
VM_MEMORY=$VM_MEMORY
|
||||
VM_VCPUS=$VM_VCPUS
|
||||
VM_DISK_SIZE=$VM_DISK_SIZE
|
||||
SSH_PUBLIC_KEY="$ssh_public_key"
|
||||
|
||||
# Network Configuration
|
||||
VM_IP=$VM_IP
|
||||
VM_GATEWAY=$VM_GATEWAY
|
||||
VM_NETMASK=$VM_NETMASK
|
||||
VM_NETWORK=$VM_NETWORK
|
||||
|
||||
# GitHub Configuration
|
||||
REPO_URL=$REPO_URL
|
||||
GITHUB_USERNAME=$GITHUB_USERNAME
|
||||
GITHUB_TOKEN=$GITHUB_TOKEN
|
||||
GITHUB_API_ENABLED=$GITHUB_API_ENABLED
|
||||
EOF
|
||||
|
||||
# Webhook environment (updated with VM info)
|
||||
cat > "$PROJECT_DIR/***REMOVED***.webhook" << EOF
|
||||
# ThrillWiki Webhook Configuration
|
||||
WEBHOOK_PORT=$WEBHOOK_PORT
|
||||
WEBHOOK_SECRET=$WEBHOOK_SECRET
|
||||
WEBHOOK_ENABLED=$WEBHOOK_ENABLED
|
||||
VM_HOST=$VM_IP
|
||||
VM_PORT=22
|
||||
VM_USER=ubuntu
|
||||
VM_KEY_PATH=$HOME/.ssh/thrillwiki_vm
|
||||
VM_PROJECT_PATH=/home/ubuntu/thrillwiki
|
||||
REPO_URL=$REPO_URL
|
||||
DEPLOY_BRANCH=main
|
||||
|
||||
# GitHub API Configuration
|
||||
GITHUB_USERNAME=$GITHUB_USERNAME
|
||||
GITHUB_TOKEN=$GITHUB_TOKEN
|
||||
GITHUB_API_ENABLED=$GITHUB_API_ENABLED
|
||||
EOF
|
||||
|
||||
log_success "Environment files created"
|
||||
}
|
||||
|
||||
# Install required tools
|
||||
install_dependencies() {
|
||||
log "Installing required dependencies..."
|
||||
|
||||
# Check for required tools
|
||||
local missing_tools=()
|
||||
local mac_tools=()
|
||||
|
||||
command -v python3 >/dev/null 2>&1 || missing_tools+=("python3")
|
||||
command -v ssh >/dev/null 2>&1 || missing_tools+=("openssh-client")
|
||||
command -v scp >/dev/null 2>&1 || missing_tools+=("openssh-client")
|
||||
|
||||
# Check for ISO creation tools and handle platform differences
|
||||
if ! command -v genisoimage >/dev/null 2>&1 && ! command -v mkisofs >/dev/null 2>&1 && ! command -v hdiutil >/dev/null 2>&1; then
|
||||
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||
missing_tools+=("genisoimage")
|
||||
elif [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
# On macOS, hdiutil should be available, but add cdrtools as backup
|
||||
if command -v brew >/dev/null 2>&1; then
|
||||
mac_tools+=("cdrtools")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Install Linux packages
|
||||
if [ ${#missing_tools[@]} -gt 0 ]; then
|
||||
log "Installing missing tools for Linux: ${missing_tools[*]}"
|
||||
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y "${missing_tools[@]}"
|
||||
elif command -v yum >/dev/null 2>&1; then
|
||||
sudo yum install -y "${missing_tools[@]}"
|
||||
elif command -v dnf >/dev/null 2>&1; then
|
||||
sudo dnf install -y "${missing_tools[@]}"
|
||||
else
|
||||
log_error "Linux package manager not found. Please install: ${missing_tools[*]}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Install macOS packages
|
||||
if [ ${#mac_tools[@]} -gt 0 ]; then
|
||||
log "Installing additional tools for macOS: ${mac_tools[*]}"
|
||||
if command -v brew >/dev/null 2>&1; then
|
||||
brew install "${mac_tools[@]}"
|
||||
else
|
||||
log "Homebrew not found. Skipping optional tool installation."
|
||||
log "Note: hdiutil should be available on macOS for ISO creation"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Install Python dependencies
|
||||
if [ -f "$PROJECT_DIR/pyproject.toml" ]; then
|
||||
log "Installing Python dependencies with UV..."
|
||||
if ! command -v uv >/dev/null 2>&1; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
source ~/.cargo/env
|
||||
fi
|
||||
uv sync
|
||||
fi
|
||||
|
||||
log_success "Dependencies installed"
|
||||
}
|
||||
|
||||
# Create VM using the VM manager
|
||||
create_vm() {
|
||||
log "Creating VM on Unraid server..."
|
||||
|
||||
# Export all environment variables from the file
|
||||
set -a # automatically export all variables
|
||||
source "$PROJECT_DIR/***REMOVED***.unraid"
|
||||
set +a # turn off automatic export
|
||||
|
||||
# Run VM creation/update
|
||||
cd "$PROJECT_DIR"
|
||||
python3 scripts/unraid/vm-manager.py setup
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_success "VM created/updated successfully"
|
||||
|
||||
# Start the VM
|
||||
log "Starting VM..."
|
||||
python3 scripts/unraid/vm-manager.py start
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_success "VM started successfully"
|
||||
else
|
||||
log_error "VM failed to start"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
log_error "VM creation/update failed"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Wait for VM to be ready and get IP
|
||||
wait_for_vm() {
|
||||
log "Waiting for VM to be ready..."
|
||||
sleep 120
|
||||
# Export all environment variables from the file
|
||||
set -a # automatically export all variables
|
||||
source "$PROJECT_DIR/***REMOVED***.unraid"
|
||||
set +a # turn off automatic export
|
||||
|
||||
local max_attempts=60
|
||||
local attempt=1
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
VM_IP=$(python3 scripts/unraid/vm-manager.py ip 2>/dev/null | grep "VM IP:" | cut -d' ' -f3)
|
||||
|
||||
if [ -n "$VM_IP" ]; then
|
||||
log_success "VM is ready with IP: $VM_IP"
|
||||
|
||||
# Update SSH config with actual IP
|
||||
sed -i.bak "s/HostName %h/HostName $VM_IP/" "$HOME/.ssh/config"
|
||||
|
||||
# Update webhook environment with IP
|
||||
sed -i.bak "s/VM_HOST=$VM_NAME/VM_HOST=$VM_IP/" "$PROJECT_DIR/***REMOVED***.webhook"
|
||||
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Waiting for VM to get IP... (attempt $attempt/$max_attempts)"
|
||||
sleep 30
|
||||
((attempt++))
|
||||
done
|
||||
|
||||
log_error "VM failed to get IP address"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Configure VM for ThrillWiki
|
||||
configure_vm() {
|
||||
log "Configuring VM for ThrillWiki deployment..."
|
||||
|
||||
local vm_setup_script="/tmp/vm_thrillwiki_setup.sh"
|
||||
|
||||
# Create VM setup script
|
||||
cat > "$vm_setup_script" << 'EOF'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "Setting up VM for ThrillWiki..."
|
||||
|
||||
# Update system
|
||||
sudo apt update && sudo apt upgrade -y
|
||||
|
||||
# Install required packages
|
||||
sudo apt install -y git curl build-essential python3-pip lsof postgresql postgresql-contrib nginx
|
||||
|
||||
# Install UV
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
source ~/.cargo/env
|
||||
|
||||
# Configure PostgreSQL
|
||||
sudo -u postgres psql << PSQL
|
||||
CREATE DATABASE thrillwiki;
|
||||
CREATE USER thrillwiki_user WITH ENCRYPTED PASSWORD 'thrillwiki_pass';
|
||||
GRANT ALL PRIVILEGES ON DATABASE thrillwiki TO thrillwiki_user;
|
||||
\q
|
||||
PSQL
|
||||
|
||||
# Clone repository
|
||||
git clone REPO_URL_PLACEHOLDER thrillwiki
|
||||
cd thrillwiki
|
||||
|
||||
# Install dependencies
|
||||
~/.cargo/bin/uv sync
|
||||
|
||||
# Create directories
|
||||
mkdir -p logs backups
|
||||
|
||||
# Make scripts executable
|
||||
chmod +x scripts/*.sh
|
||||
|
||||
# Run initial setup
|
||||
~/.cargo/bin/uv run manage.py migrate
|
||||
~/.cargo/bin/uv run manage.py collectstatic --noinput
|
||||
|
||||
# Install systemd services
|
||||
sudo cp scripts/systemd/thrillwiki.service /etc/systemd/system/
|
||||
sudo sed -i 's|/home/ubuntu|/home/ubuntu|g' /etc/systemd/system/thrillwiki.service
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable thrillwiki.service
|
||||
|
||||
echo "VM setup completed!"
|
||||
EOF
|
||||
|
||||
# Replace placeholder with actual repo URL
|
||||
sed -i "s|REPO_URL_PLACEHOLDER|$REPO_URL|g" "$vm_setup_script"
|
||||
|
||||
# Copy and execute setup script on VM
|
||||
scp "$vm_setup_script" "$VM_NAME:/tmp/"
|
||||
ssh "$VM_NAME" "bash /tmp/vm_thrillwiki_setup.sh"
|
||||
|
||||
# Cleanup
|
||||
rm "$vm_setup_script"
|
||||
|
||||
log_success "VM configured for ThrillWiki"
|
||||
}
|
||||
|
||||
# Start services
|
||||
start_services() {
|
||||
log "Starting ThrillWiki services..."
|
||||
|
||||
# Start VM service
|
||||
ssh "$VM_NAME" "sudo systemctl start thrillwiki"
|
||||
|
||||
# Verify service is running
|
||||
if ssh "$VM_NAME" "systemctl is-active --quiet thrillwiki"; then
|
||||
log_success "ThrillWiki service started successfully"
|
||||
else
|
||||
log_error "Failed to start ThrillWiki service"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get service status
|
||||
log "Service status:"
|
||||
ssh "$VM_NAME" "systemctl status thrillwiki --no-pager -l"
|
||||
}
|
||||
|
||||
# Setup webhook listener
|
||||
setup_webhook_listener() {
|
||||
log "Setting up webhook listener..."
|
||||
|
||||
# Create webhook start script
|
||||
cat > "$PROJECT_DIR/start-webhook.sh" << 'EOF'
|
||||
#!/bin/bash
|
||||
cd "$(dirname "$0")"
|
||||
source ***REMOVED***.webhook
|
||||
python3 scripts/webhook-listener.py
|
||||
EOF
|
||||
|
||||
chmod +x "$PROJECT_DIR/start-webhook.sh"
|
||||
|
||||
log_success "Webhook listener configured"
|
||||
log "You can start the webhook listener with: ./start-webhook.sh"
|
||||
}
|
||||
|
||||
# Perform end-to-end test
|
||||
test_deployment() {
|
||||
log "Performing end-to-end deployment test..."
|
||||
|
||||
# Test VM connectivity
|
||||
if ssh "$VM_NAME" "echo 'VM connectivity test passed'"; then
|
||||
log_success "VM connectivity test passed"
|
||||
else
|
||||
log_error "VM connectivity test failed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Test ThrillWiki service
|
||||
if ssh "$VM_NAME" "curl -f http://localhost:8000 >/dev/null 2>&1"; then
|
||||
log_success "ThrillWiki service test passed"
|
||||
else
|
||||
log_warning "ThrillWiki service test failed - checking logs..."
|
||||
ssh "$VM_NAME" "journalctl -u thrillwiki --no-pager -l | tail -20"
|
||||
fi
|
||||
|
||||
# Test deployment script
|
||||
log "Testing deployment script..."
|
||||
ssh "$VM_NAME" "cd thrillwiki && ./scripts/vm-deploy.sh status"
|
||||
|
||||
log_success "End-to-end test completed"
|
||||
}
|
||||
|
||||
# Generate final instructions
|
||||
generate_instructions() {
|
||||
log "Generating final setup instructions..."
|
||||
|
||||
cat > "$PROJECT_DIR/UNRAID_SETUP_COMPLETE.md" << EOF
|
||||
# ThrillWiki Unraid Automation - Setup Complete! 🎉
|
||||
|
||||
Your ThrillWiki CI/CD system has been fully automated and deployed!
|
||||
|
||||
## VM Information
|
||||
- **VM Name**: $VM_NAME
|
||||
- **VM IP**: $VM_IP
|
||||
- **SSH Access**: \`ssh $VM_NAME\`
|
||||
|
||||
## Services Status
|
||||
- **ThrillWiki Service**: Running on VM
|
||||
- **Database**: PostgreSQL configured
|
||||
- **Web Server**: Available at http://$VM_IP:8000
|
||||
|
||||
## Next Steps
|
||||
|
||||
### 1. Start Webhook Listener
|
||||
\`\`\`bash
|
||||
./start-webhook.sh
|
||||
\`\`\`
|
||||
|
||||
### 2. Configure GitHub Webhook
|
||||
- Go to your repository: $REPO_URL
|
||||
- Settings → Webhooks → Add webhook
|
||||
- **Payload URL**: http://YOUR_PUBLIC_IP:$WEBHOOK_PORT/webhook
|
||||
- **Content type**: application/json
|
||||
- **Secret**: (your webhook secret)
|
||||
- **Events**: Just the push event
|
||||
|
||||
### 3. Test the System
|
||||
\`\`\`bash
|
||||
# Test VM connection
|
||||
ssh $VM_NAME
|
||||
|
||||
# Test service status
|
||||
ssh $VM_NAME "systemctl status thrillwiki"
|
||||
|
||||
# Test manual deployment
|
||||
ssh $VM_NAME "cd thrillwiki && ./scripts/vm-deploy.sh"
|
||||
|
||||
# Make a test commit to trigger automatic deployment
|
||||
git add .
|
||||
git commit -m "Test automated deployment"
|
||||
git push origin main
|
||||
\`\`\`
|
||||
|
||||
## Management Commands
|
||||
|
||||
### VM Management
|
||||
\`\`\`bash
|
||||
# Check VM status
|
||||
python3 scripts/unraid/vm-manager.py status
|
||||
|
||||
# Start/stop VM
|
||||
python3 scripts/unraid/vm-manager.py start
|
||||
python3 scripts/unraid/vm-manager.py stop
|
||||
|
||||
# Get VM IP
|
||||
python3 scripts/unraid/vm-manager.py ip
|
||||
\`\`\`
|
||||
|
||||
### Service Management on VM
|
||||
\`\`\`bash
|
||||
# Check service status
|
||||
ssh $VM_NAME "./scripts/vm-deploy.sh status"
|
||||
|
||||
# Restart service
|
||||
ssh $VM_NAME "./scripts/vm-deploy.sh restart"
|
||||
|
||||
# View logs
|
||||
ssh $VM_NAME "journalctl -u thrillwiki -f"
|
||||
\`\`\`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
1. **VM not accessible**: Check VM is running and has IP
|
||||
2. **Service not starting**: Check logs with \`journalctl -u thrillwiki\`
|
||||
3. **Webhook not working**: Verify port $WEBHOOK_PORT is open
|
||||
|
||||
### Support Files
|
||||
- Configuration: \`***REMOVED***.unraid\`, \`***REMOVED***.webhook\`
|
||||
- Logs: \`logs/\` directory
|
||||
- Documentation: \`docs/VM_DEPLOYMENT_SETUP.md\`
|
||||
|
||||
**Your automated CI/CD system is now ready!** 🚀
|
||||
|
||||
Every push to the main branch will automatically deploy to your VM.
|
||||
EOF
|
||||
|
||||
log_success "Setup instructions saved to UNRAID_SETUP_COMPLETE.md"
|
||||
}
|
||||
|
||||
# Main automation function
|
||||
main() {
|
||||
log "🚀 Starting ThrillWiki Complete Unraid Automation"
|
||||
echo "[AWS-SECRET-REMOVED]=========="
|
||||
echo
|
||||
|
||||
# Parse command line arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--reset)
|
||||
RESET_ALL=true
|
||||
shift
|
||||
;;
|
||||
--reset-vm)
|
||||
RESET_VM_ONLY=true
|
||||
shift
|
||||
;;
|
||||
--reset-config)
|
||||
RESET_CONFIG_ONLY=true
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
show_help
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
show_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Create logs directory
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Handle reset modes
|
||||
if [[ "$RESET_ALL" == "true" ]]; then
|
||||
log "🔄 Complete reset mode - deleting VM and configuration"
|
||||
echo
|
||||
|
||||
# Load configuration first to get connection details for VM deletion
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
source "$CONFIG_FILE"
|
||||
log_success "Loaded existing configuration for VM deletion"
|
||||
else
|
||||
log_warning "No configuration file found, will skip VM deletion"
|
||||
fi
|
||||
|
||||
# Delete existing VM if config exists
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
log "🗑️ Deleting existing VM..."
|
||||
# Export environment variables for VM manager
|
||||
set -a
|
||||
source "$PROJECT_DIR/***REMOVED***.unraid" 2>/dev/null || true
|
||||
set +a
|
||||
|
||||
if python3 "$(dirname "$0")/vm-manager.py" delete; then
|
||||
log_success "VM deleted successfully"
|
||||
else
|
||||
log "⚠️ VM deletion failed or VM didn't exist"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Remove configuration files
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
rm "$CONFIG_FILE"
|
||||
log_success "Configuration file removed"
|
||||
fi
|
||||
|
||||
# Remove environment files
|
||||
rm -f "$PROJECT_DIR/***REMOVED***.unraid" "$PROJECT_DIR/***REMOVED***.webhook"
|
||||
log_success "Environment files removed"
|
||||
|
||||
log_success "Complete reset finished - continuing with fresh setup"
|
||||
echo
|
||||
|
||||
elif [[ "$RESET_VM_ONLY" == "true" ]]; then
|
||||
log "🔄 VM-only reset mode - deleting VM, preserving configuration"
|
||||
echo
|
||||
|
||||
# Load configuration to get connection details
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
source "$CONFIG_FILE"
|
||||
log_success "Loaded existing configuration"
|
||||
else
|
||||
log_error "No configuration file found. Cannot reset VM without connection details."
|
||||
echo " Run the script without reset flags first to create initial configuration."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Delete existing VM
|
||||
log "🗑️ Deleting existing VM..."
|
||||
# Export environment variables for VM manager
|
||||
set -a
|
||||
source "$PROJECT_DIR/***REMOVED***.unraid" 2>/dev/null || true
|
||||
set +a
|
||||
|
||||
if python3 "$(dirname "$0")/vm-manager.py" delete; then
|
||||
log_success "VM deleted successfully"
|
||||
else
|
||||
log "⚠️ VM deletion failed or VM didn't exist"
|
||||
fi
|
||||
|
||||
# Remove only environment files, keep main config
|
||||
rm -f "$PROJECT_DIR/***REMOVED***.unraid" "$PROJECT_DIR/***REMOVED***.webhook"
|
||||
log_success "Environment files removed, configuration preserved"
|
||||
|
||||
log_success "VM reset complete - will recreate VM with saved configuration"
|
||||
echo
|
||||
|
||||
elif [[ "$RESET_CONFIG_ONLY" == "true" ]]; then
|
||||
log "🔄 Config-only reset mode - deleting configuration, preserving VM"
|
||||
echo
|
||||
|
||||
# Remove configuration files
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
rm "$CONFIG_FILE"
|
||||
log_success "Configuration file removed"
|
||||
fi
|
||||
|
||||
# Remove environment files
|
||||
rm -f "$PROJECT_DIR/***REMOVED***.unraid" "$PROJECT_DIR/***REMOVED***.webhook"
|
||||
log_success "Environment files removed"
|
||||
|
||||
log_success "Configuration reset complete - will prompt for fresh configuration"
|
||||
echo
|
||||
fi
|
||||
|
||||
# Collect configuration
|
||||
prompt_unraid_config
|
||||
|
||||
# Setup steps
|
||||
setup_ssh_keys
|
||||
setup_unraid_access
|
||||
create_environment_files
|
||||
install_dependencies
|
||||
create_vm
|
||||
wait_for_vm
|
||||
configure_vm
|
||||
start_services
|
||||
setup_webhook_listener
|
||||
test_deployment
|
||||
generate_instructions
|
||||
|
||||
echo
|
||||
log_success "🎉 Complete automation setup finished!"
|
||||
echo
|
||||
log "Your ThrillWiki VM is running at: http://$VM_IP:8000"
|
||||
log "Start the webhook listener: ./start-webhook.sh"
|
||||
log "See UNRAID_SETUP_COMPLETE.md for detailed instructions"
|
||||
echo
|
||||
log "The system will now automatically deploy when you push to GitHub!"
|
||||
}
|
||||
|
||||
# Run main function and log output
|
||||
main "$@" 2>&1 | tee "$LOG_DIR/unraid-automation.log"
|
||||
861
scripts/unraid/vm-manager.py
Executable file
861
scripts/unraid/vm-manager.py
Executable file
@@ -0,0 +1,861 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Unraid VM Manager for ThrillWiki
|
||||
This script automates VM creation, configuration, and management on Unraid.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
import requests
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, List
|
||||
|
||||
# Configuration
|
||||
UNRAID_HOST = os***REMOVED***iron.get('UNRAID_HOST', 'localhost')
|
||||
UNRAID_USER = os***REMOVED***iron.get('UNRAID_USER', 'root')
|
||||
UNRAID_PASSWORD = os***REMOVED***iron.get('UNRAID_PASSWORD', '')
|
||||
VM_NAME = os***REMOVED***iron.get('VM_NAME', 'thrillwiki-vm')
|
||||
VM_TEMPLATE = os***REMOVED***iron.get('VM_TEMPLATE', 'Ubuntu Server 22.04')
|
||||
VM_MEMORY = int(os***REMOVED***iron.get('VM_MEMORY', 4096)) # MB
|
||||
VM_VCPUS = int(os***REMOVED***iron.get('VM_VCPUS', 2))
|
||||
VM_DISK_SIZE = int(os***REMOVED***iron.get('VM_DISK_SIZE', 50)) # GB
|
||||
SSH_PUBLIC_KEY = os***REMOVED***iron.get('SSH_PUBLIC_KEY', '')
|
||||
|
||||
# Network Configuration
|
||||
VM_IP = os***REMOVED***iron.get('VM_IP', '192.168.20.20')
|
||||
VM_GATEWAY = os***REMOVED***iron.get('VM_GATEWAY', '192.168.20.1')
|
||||
VM_NETMASK = os***REMOVED***iron.get('VM_NETMASK', '255.255.255.0')
|
||||
VM_NETWORK = os***REMOVED***iron.get('VM_NETWORK', '192.168.20.0/24')
|
||||
|
||||
# GitHub Configuration
|
||||
REPO_URL = os***REMOVED***iron.get('REPO_URL', '')
|
||||
GITHUB_USERNAME = os***REMOVED***iron.get('GITHUB_USERNAME', '')
|
||||
GITHUB_TOKEN = os***REMOVED***iron.get('GITHUB_TOKEN', '')
|
||||
GITHUB_API_ENABLED = os***REMOVED***iron.get(
|
||||
'GITHUB_API_ENABLED', 'false').lower() == 'true'
|
||||
|
||||
# Setup logging
|
||||
os.makedirs('logs', exist_ok=True)
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('logs/unraid-vm.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UnraidVMManager:
|
||||
"""Manages VMs on Unraid server."""
|
||||
|
||||
def __init__(self):
|
||||
self.session = requests.Session()
|
||||
self.base_url = f"http://{UNRAID_HOST}"
|
||||
self.vm_config_path = f"/mnt/user/domains/{VM_NAME}"
|
||||
|
||||
def authenticate(self) -> bool:
|
||||
"""Authenticate with Unraid server."""
|
||||
try:
|
||||
login_url = f"{self.base_url}/login"
|
||||
login_data = {
|
||||
'username': UNRAID_USER,
|
||||
'password': UNRAID_PASSWORD
|
||||
}
|
||||
|
||||
response = self.session.post(login_url, data=login_data)
|
||||
if response.status_code == 200:
|
||||
logger.info("Successfully authenticated with Unraid")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Authentication failed: {response.status_code}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Authentication error: {e}")
|
||||
return False
|
||||
|
||||
def check_vm_exists(self) -> bool:
|
||||
"""Check if VM already exists."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh list --all | grep {VM_NAME}'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
return VM_NAME in result.stdout
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking VM existence: {e}")
|
||||
return False
|
||||
|
||||
def create_vm_xml(self, existing_uuid: str = None) -> str:
|
||||
"""Generate VM XML configuration."""
|
||||
import uuid
|
||||
vm_uuid = existing_uuid if existing_uuid else str(uuid.uuid4())
|
||||
|
||||
xml_template = f"""<?xml version='1.0' encoding='UTF-8'?>
|
||||
<domain type='kvm'>
|
||||
<name>{VM_NAME}</name>
|
||||
<uuid>{vm_uuid}</uuid>
|
||||
<metadata>
|
||||
<vmtemplate xmlns="unraid" name="Windows 10" iconold="ubuntu.png" icon="ubuntu.png" os="linux" webui=""/>
|
||||
</metadata>
|
||||
<memory unit='KiB'>{VM_MEMORY * 1024}</memory>
|
||||
<currentMemory unit='KiB'>{VM_MEMORY * 1024}</currentMemory>
|
||||
<vcpu placement='static'>{VM_VCPUS}</vcpu>
|
||||
<os>
|
||||
<type arch='x86_64' machine='pc-q35-9.2'>hvm</type>
|
||||
<loader readonly='yes' type='pflash' format='raw'>/usr/share/qemu/ovmf-x64/OVMF_CODE-pure-efi.fd</loader>
|
||||
<nvram format='raw'>/etc/libvirt/qemu/nvram/{vm_uuid}_VARS-pure-efi.fd</nvram>
|
||||
</os>
|
||||
<features>
|
||||
<acpi/>
|
||||
<apic/>
|
||||
<vmport state='off'/>
|
||||
</features>
|
||||
<cpu mode='host-passthrough' check='none' migratable='on'>
|
||||
<topology sockets='1' dies='1' clusters='1' cores='{VM_VCPUS // 2 if VM_VCPUS > 1 else 1}' threads='{2 if VM_VCPUS > 1 else 1}'/>
|
||||
<cache mode='passthrough'/>
|
||||
<feature policy='require' name='topoext'/>
|
||||
</cpu>
|
||||
<clock offset='utc'>
|
||||
<timer name='hpet' present='no'/>
|
||||
<timer name='hypervclock' present='yes'/>
|
||||
<timer name='pit' tickpolicy='delay'/>
|
||||
<timer name='rtc' tickpolicy='catchup'/>
|
||||
</clock>
|
||||
<on_poweroff>destroy</on_poweroff>
|
||||
<on_reboot>restart</on_reboot>
|
||||
<on_crash>restart</on_crash>
|
||||
<pm>
|
||||
<suspend-to-mem enabled='no'/>
|
||||
<suspend-to-disk enabled='no'/>
|
||||
</pm>
|
||||
<devices>
|
||||
<emulator>/usr/local/sbin/qemu</emulator>
|
||||
<disk type='file' device='disk'>
|
||||
<driver name='qemu' type='qcow2' cache='writeback' discard='ignore'/>
|
||||
<source file='/mnt/user/domains/{VM_NAME}/vdisk1.qcow2'/>
|
||||
<target dev='hdc' bus='virtio'/>
|
||||
<boot order='2'/>
|
||||
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
|
||||
</disk>
|
||||
<disk type='file' device='cdrom'>
|
||||
<driver name='qemu' type='raw'/>
|
||||
<source file='/mnt/user/isos/ubuntu-24.04.3-live-server-amd64.iso'/>
|
||||
<target dev='hda' bus='sata'/>
|
||||
<readonly/>
|
||||
<boot order='1'/>
|
||||
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
|
||||
</disk>
|
||||
<disk type='file' device='cdrom'>
|
||||
<driver name='qemu' type='raw'/>
|
||||
<source file='/mnt/user/isos/{VM_NAME}-cloud-init.iso'/>
|
||||
<target dev='hdb' bus='sata'/>
|
||||
<readonly/>
|
||||
<address type='drive' controller='0' bus='0' target='0' unit='1'/>
|
||||
</disk>
|
||||
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
|
||||
</controller>
|
||||
<controller type='pci' index='0' model='pcie-root'/>
|
||||
<controller type='pci' index='1' model='pcie-root-port'>
|
||||
<model name='pcie-root-port'/>
|
||||
<target chassis='1' port='0x10'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
|
||||
</controller>
|
||||
<controller type='pci' index='2' model='pcie-root-port'>
|
||||
<model name='pcie-root-port'/>
|
||||
<target chassis='2' port='0x11'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
|
||||
</controller>
|
||||
<controller type='pci' index='3' model='pcie-root-port'>
|
||||
<model name='pcie-root-port'/>
|
||||
<target chassis='3' port='0x12'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
|
||||
</controller>
|
||||
<controller type='pci' index='4' model='pcie-root-port'>
|
||||
<model name='pcie-root-port'/>
|
||||
<target chassis='4' port='0x13'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
|
||||
</controller>
|
||||
<controller type='pci' index='5' model='pcie-root-port'>
|
||||
<model name='pcie-root-port'/>
|
||||
<target chassis='5' port='0x14'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
|
||||
</controller>
|
||||
<controller type='virtio-serial' index='0'>
|
||||
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
|
||||
</controller>
|
||||
<controller type='sata' index='0'>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
|
||||
</controller>
|
||||
<interface type='bridge'>
|
||||
<mac address='52:54:00:{":".join([f"{int(VM_IP.split('.')[3]):02x}", "7d", "fd"])}'/>
|
||||
<source bridge='br0.20'/>
|
||||
<model type='virtio'/>
|
||||
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
|
||||
</interface>
|
||||
<serial type='pty'>
|
||||
<target type='isa-serial' port='0'>
|
||||
<model name='isa-serial'/>
|
||||
</target>
|
||||
</serial>
|
||||
<console type='pty'>
|
||||
<target type='serial' port='0'/>
|
||||
</console>
|
||||
<channel type='unix'>
|
||||
<target type='virtio' name='org.qemu.guest_agent.0'/>
|
||||
<address type='virtio-serial' controller='0' bus='0' port='1'/>
|
||||
</channel>
|
||||
<input type='tablet' bus='usb'>
|
||||
<address type='usb' bus='0' port='1'/>
|
||||
</input>
|
||||
<input type='mouse' bus='ps2'/>
|
||||
<input type='keyboard' bus='ps2'/>
|
||||
<graphics type='vnc' port='-1' autoport='yes' websocket='-1' listen='0.0.0.0' sharePolicy='ignore'>
|
||||
<listen type='address' address='0.0.0.0'/>
|
||||
</graphics>
|
||||
<audio id='1' type='none'/>
|
||||
<video>
|
||||
<model type='qxl' ram='65536' vram='65536' vram64='65535' vgamem='65536' heads='1' primary='yes'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x1e' function='0x0'/>
|
||||
</video>
|
||||
<watchdog model='itco' action='reset'/>
|
||||
<memballoon model='virtio'>
|
||||
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
|
||||
</memballoon>
|
||||
</devices>
|
||||
</domain>"""
|
||||
return xml_template.strip()
|
||||
|
||||
def create_vm(self) -> bool:
|
||||
"""Create or update the VM on Unraid."""
|
||||
try:
|
||||
vm_exists = self.check_vm_exists()
|
||||
|
||||
if vm_exists:
|
||||
logger.info(
|
||||
f"VM {VM_NAME} already exists, updating configuration...")
|
||||
# Stop VM if running before updating
|
||||
if self.vm_status() == "running":
|
||||
logger.info(
|
||||
f"Stopping VM {VM_NAME} for configuration update...")
|
||||
self.stop_vm()
|
||||
# Wait for VM to stop
|
||||
import time
|
||||
time.sleep(5)
|
||||
else:
|
||||
logger.info(f"Creating VM {VM_NAME}...")
|
||||
|
||||
# Create VM directory
|
||||
subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'mkdir -p {self.vm_config_path}'",
|
||||
shell=True,
|
||||
check=True
|
||||
)
|
||||
|
||||
# Create virtual disk only if VM doesn't exist
|
||||
disk_cmd = f"""
|
||||
ssh {UNRAID_USER}@{UNRAID_HOST} 'qemu-img create -f qcow2 {self.vm_config_path}/vdisk1.qcow2 {VM_DISK_SIZE}G'
|
||||
"""
|
||||
subprocess.run(disk_cmd, shell=True, check=True)
|
||||
|
||||
# Create cloud-init ISO for automated installation and ThrillWiki deployment
|
||||
logger.info(
|
||||
"Creating cloud-init ISO for automated Ubuntu and ThrillWiki setup...")
|
||||
if not self.create_cloud_init_iso(VM_IP):
|
||||
logger.error("Failed to create cloud-init ISO")
|
||||
return False
|
||||
|
||||
existing_uuid = None
|
||||
|
||||
if vm_exists:
|
||||
# Get existing VM UUID
|
||||
result = subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh dumpxml {VM_NAME} | grep \"<uuid>\" | sed \"s/<uuid>//g\" | sed \"s/<\\/uuid>//g\" | tr -d \" \"'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
existing_uuid = result.stdout.strip()
|
||||
logger.info(f"Found existing VM UUID: {existing_uuid}")
|
||||
|
||||
# Always undefine existing VM with NVRAM flag (since we create persistent VMs)
|
||||
logger.info(
|
||||
f"VM {VM_NAME} exists, undefining with NVRAM for reconfiguration...")
|
||||
subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh undefine {VM_NAME} --nvram'",
|
||||
shell=True,
|
||||
check=True
|
||||
)
|
||||
logger.info(
|
||||
f"VM {VM_NAME} undefined for reconfiguration (with NVRAM)")
|
||||
|
||||
# Generate VM XML with appropriate UUID
|
||||
vm_xml = self.create_vm_xml(existing_uuid)
|
||||
xml_file = f"/tmp/{VM_NAME}.xml"
|
||||
|
||||
with open(xml_file, 'w') as f:
|
||||
f.write(vm_xml)
|
||||
|
||||
# Copy XML to Unraid and define/redefine VM
|
||||
subprocess.run(
|
||||
f"scp {xml_file} {UNRAID_USER}@{UNRAID_HOST}:/tmp/",
|
||||
shell=True,
|
||||
check=True
|
||||
)
|
||||
|
||||
# Define VM as persistent domain
|
||||
subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh define /tmp/{VM_NAME}.xml'",
|
||||
shell=True,
|
||||
check=True
|
||||
)
|
||||
|
||||
# Ensure VM is set to autostart for persistent configuration
|
||||
subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh autostart {VM_NAME}'",
|
||||
shell=True,
|
||||
check=False # Don't fail if autostart is already enabled
|
||||
)
|
||||
|
||||
action = "updated" if vm_exists else "created"
|
||||
logger.info(f"VM {VM_NAME} {action} successfully")
|
||||
|
||||
# Cleanup
|
||||
os.remove(xml_file)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create VM: {e}")
|
||||
return False
|
||||
|
||||
def create_nvram_file(self, vm_uuid: str) -> bool:
|
||||
"""Create NVRAM file for UEFI VM."""
|
||||
try:
|
||||
nvram_path = f"/etc/libvirt/qemu/nvram/{vm_uuid}_VARS-pure-efi.fd"
|
||||
|
||||
# Check if NVRAM file already exists
|
||||
result = subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'test -f {nvram_path}'",
|
||||
shell=True,
|
||||
capture_output=True
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info(f"NVRAM file already exists: {nvram_path}")
|
||||
return True
|
||||
|
||||
# Copy template to create NVRAM file
|
||||
logger.info(f"Creating NVRAM file: {nvram_path}")
|
||||
result = subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'cp /usr/share/qemu/ovmf-x64/OVMF_VARS-pure-efi.fd {nvram_path}'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info("NVRAM file created successfully")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Failed to create NVRAM file: {result.stderr}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating NVRAM file: {e}")
|
||||
return False
|
||||
|
||||
def start_vm(self) -> bool:
|
||||
"""Start the VM if it's not already running."""
|
||||
try:
|
||||
# Check if VM is already running
|
||||
current_status = self.vm_status()
|
||||
if current_status == "running":
|
||||
logger.info(f"VM {VM_NAME} is already running")
|
||||
return True
|
||||
|
||||
logger.info(f"Starting VM {VM_NAME}...")
|
||||
|
||||
# For new VMs, we need to extract the UUID and create NVRAM file
|
||||
vm_exists = self.check_vm_exists()
|
||||
if not vm_exists:
|
||||
logger.error("Cannot start VM that doesn't exist")
|
||||
return False
|
||||
|
||||
# Get VM UUID from XML
|
||||
result = subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh dumpxml {VM_NAME} | grep \"<uuid>\" | sed \"s/<uuid>//g\" | sed \"s/<\\/uuid>//g\" | tr -d \" \"'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
vm_uuid = result.stdout.strip()
|
||||
logger.info(f"VM UUID: {vm_uuid}")
|
||||
|
||||
# Create NVRAM file if it doesn't exist
|
||||
if not self.create_nvram_file(vm_uuid):
|
||||
return False
|
||||
|
||||
result = subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh start {VM_NAME}'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info(f"VM {VM_NAME} started successfully")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Failed to start VM: {result.stderr}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting VM: {e}")
|
||||
return False
|
||||
|
||||
def stop_vm(self) -> bool:
|
||||
"""Stop the VM."""
|
||||
try:
|
||||
logger.info(f"Stopping VM {VM_NAME}...")
|
||||
|
||||
result = subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh shutdown {VM_NAME}'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info(f"VM {VM_NAME} stopped successfully")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Failed to stop VM: {result.stderr}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error stopping VM: {e}")
|
||||
return False
|
||||
|
||||
def get_vm_ip(self) -> Optional[str]:
|
||||
"""Get VM IP address."""
|
||||
try:
|
||||
# Wait for VM to get IP
|
||||
for attempt in range(30):
|
||||
result = subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh domifaddr {VM_NAME}'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode == 0 and 'ipv4' in result.stdout:
|
||||
lines = result.stdout.strip().split('\n')
|
||||
for line in lines:
|
||||
if 'ipv4' in line:
|
||||
# Extract IP from line like: vnet0 52:54:00:xx:xx:xx ipv4 192.168.1.100/24
|
||||
parts = line.split()
|
||||
if len(parts) >= 4:
|
||||
ip_with_mask = parts[3]
|
||||
ip = ip_with_mask.split('/')[0]
|
||||
logger.info(f"VM IP address: {ip}")
|
||||
return ip
|
||||
|
||||
logger.info(f"Waiting for VM IP... (attempt {attempt + 1}/30)")
|
||||
time.sleep(10)
|
||||
|
||||
logger.error("Failed to get VM IP address")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting VM IP: {e}")
|
||||
return None
|
||||
|
||||
def create_cloud_init_iso(self, vm_ip: str) -> bool:
|
||||
"""Create cloud-init ISO for automated Ubuntu installation."""
|
||||
try:
|
||||
logger.info("Creating cloud-init ISO...")
|
||||
|
||||
# Get environment variables
|
||||
repo_url = os.getenv('REPO_URL', '')
|
||||
github_token = os.getenv('GITHUB_TOKEN', '')
|
||||
ssh_public_key = os.getenv('SSH_PUBLIC_KEY', '')
|
||||
|
||||
# Extract repository name from URL
|
||||
if repo_url:
|
||||
# Extract owner/repo from URL like https://github.com/owner/repo
|
||||
github_repo = repo_url.replace(
|
||||
'https://github.com/', '').replace('.git', '')
|
||||
else:
|
||||
logger.error("REPO_URL environment variable not set")
|
||||
return False
|
||||
|
||||
# Create cloud-init user-data with complete ThrillWiki deployment
|
||||
user_data = f"""#cloud-config
|
||||
runcmd:
|
||||
- [eval, 'echo $(cat /proc/cmdline) "autoinstall" > /root/cmdline']
|
||||
- [eval, 'mount -n --bind -o ro /root/cmdline /proc/cmdline']
|
||||
- [eval, 'snap restart subiquity.subiquity-server']
|
||||
- [eval, 'snap restart subiquity.subiquity-service']
|
||||
|
||||
autoinstall:
|
||||
version: 1
|
||||
locale: en_US
|
||||
keyboard:
|
||||
layout: us
|
||||
ssh:
|
||||
install-server: true
|
||||
authorized-keys:
|
||||
- {ssh_public_key}
|
||||
allow-pw: false
|
||||
storage:
|
||||
layout:
|
||||
name: direct
|
||||
identity:
|
||||
hostname: thrillwiki-vm
|
||||
username: ubuntu
|
||||
password: '$6$rounds=4096$saltsalt$hash' # disabled
|
||||
kernel:
|
||||
package: linux-generic
|
||||
early-commands:
|
||||
- systemctl stop ssh
|
||||
packages:
|
||||
- curl
|
||||
- git
|
||||
- build-essential
|
||||
- python3-pip
|
||||
- postgresql
|
||||
- postgresql-contrib
|
||||
- nginx
|
||||
- nodejs
|
||||
- npm
|
||||
- pipx
|
||||
late-commands:
|
||||
- apt install pipx -y
|
||||
- echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > /target/etc/sudoers.d/ubuntu
|
||||
- /target/usr/bin/pipx install uv
|
||||
# Setup ThrillWiki deployment script
|
||||
- |
|
||||
cat > /target/home/ubuntu/deploy-thrillwiki.sh << 'DEPLOY_EOF'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Wait for system to be ready
|
||||
sleep 30
|
||||
|
||||
# Clone ThrillWiki repository with GitHub token
|
||||
export GITHUB_TOKEN=$(cat /home/ubuntu/.github-token 2>/dev/null || echo "")
|
||||
if [ -n "$GITHUB_TOKEN" ]; then
|
||||
git clone https://$GITHUB_TOKEN@github.com/{github_repo} /home/ubuntu/thrillwiki
|
||||
else
|
||||
git clone https://github.com/{github_repo} /home/ubuntu/thrillwiki
|
||||
fi
|
||||
|
||||
cd /home/ubuntu/thrillwiki
|
||||
|
||||
# Setup UV and Python environment
|
||||
export PATH="/home/ubuntu/.local/bin:$PATH"
|
||||
uv venv
|
||||
source .venv/bin/activate
|
||||
|
||||
# Install dependencies
|
||||
uv sync
|
||||
|
||||
# Setup PostgreSQL
|
||||
sudo -u postgres createuser ubuntu
|
||||
sudo -u postgres createdb thrillwiki_production
|
||||
sudo -u postgres psql -c "ALTER USER ubuntu WITH SUPERUSER;"
|
||||
|
||||
# Setup environment
|
||||
cp ***REMOVED***.example ***REMOVED***
|
||||
echo "DEBUG=False" >> ***REMOVED***
|
||||
echo "DATABASE_URL=postgresql://ubuntu@localhost/thrillwiki_production" >> ***REMOVED***
|
||||
echo "ALLOWED_HOSTS=*" >> ***REMOVED***
|
||||
|
||||
# Run migrations and collect static files
|
||||
uv run manage.py migrate
|
||||
uv run manage.py collectstatic --noinput
|
||||
uv run manage.py tailwind build
|
||||
|
||||
# Setup systemd services
|
||||
sudo cp [AWS-SECRET-REMOVED]thrillwiki.service /etc/systemd/system/
|
||||
sudo cp [AWS-SECRET-REMOVED]thrillwiki-webhook.service /etc/systemd/system/
|
||||
|
||||
# Update service files with correct paths
|
||||
sudo sed -i "s|/opt/thrillwiki|/home/ubuntu/thrillwiki|g" /etc/systemd/system/thrillwiki.service
|
||||
sudo sed -i "s|/opt/thrillwiki|/home/ubuntu/thrillwiki|g" /etc/systemd/system/thrillwiki-webhook.service
|
||||
|
||||
# Enable and start services
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable thrillwiki
|
||||
sudo systemctl enable thrillwiki-webhook
|
||||
sudo systemctl start thrillwiki
|
||||
sudo systemctl start thrillwiki-webhook
|
||||
|
||||
echo "ThrillWiki deployment completed successfully!"
|
||||
DEPLOY_EOF
|
||||
- chmod +x /target/home/ubuntu/deploy-thrillwiki.sh
|
||||
- chroot /target chown ubuntu:ubuntu /home/ubuntu/deploy-thrillwiki.sh
|
||||
# Create systemd service to run deployment after first boot
|
||||
- |
|
||||
cat > /target/etc/systemd/system/thrillwiki-deploy.service << 'SERVICE_EOF'
|
||||
[Unit]
|
||||
Description=Deploy ThrillWiki on first boot
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=ubuntu
|
||||
ExecStart=/home/ubuntu/deploy-thrillwiki.sh
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
SERVICE_EOF
|
||||
- chroot /target systemctl enable thrillwiki-deploy
|
||||
user-data:
|
||||
disable_root: true
|
||||
ssh_pwauth: false
|
||||
power_state:
|
||||
mode: reboot
|
||||
"""
|
||||
|
||||
meta_data = f"""instance-id: thrillwiki-vm-001
|
||||
local-hostname: thrillwiki-vm
|
||||
network:
|
||||
version: 2
|
||||
ethernets:
|
||||
enp1s0:
|
||||
dhcp4: true
|
||||
"""
|
||||
|
||||
# Create temp directory for cloud-init files
|
||||
cloud_init_dir = "/tmp/cloud-init"
|
||||
os.makedirs(cloud_init_dir, exist_ok=True)
|
||||
|
||||
with open(f"{cloud_init_dir}/user-data", 'w') as f:
|
||||
f.write(user_data)
|
||||
|
||||
with open(f"{cloud_init_dir}/meta-data", 'w') as f:
|
||||
f.write(meta_data)
|
||||
|
||||
# Create ISO
|
||||
iso_path = f"/tmp/{VM_NAME}-cloud-init.iso"
|
||||
|
||||
# Try different ISO creation tools
|
||||
iso_created = False
|
||||
|
||||
# Try genisoimage first
|
||||
try:
|
||||
subprocess.run([
|
||||
'genisoimage',
|
||||
'-output', iso_path,
|
||||
'-volid', 'cidata',
|
||||
'-joliet',
|
||||
'-rock',
|
||||
cloud_init_dir
|
||||
], check=True)
|
||||
iso_created = True
|
||||
except FileNotFoundError:
|
||||
logger.warning("genisoimage not found, trying mkisofs...")
|
||||
|
||||
# Try mkisofs as fallback
|
||||
if not iso_created:
|
||||
try:
|
||||
subprocess.run([
|
||||
'mkisofs',
|
||||
'-output', iso_path,
|
||||
'-volid', 'cidata',
|
||||
'-joliet',
|
||||
'-rock',
|
||||
cloud_init_dir
|
||||
], check=True)
|
||||
iso_created = True
|
||||
except FileNotFoundError:
|
||||
logger.warning(
|
||||
"mkisofs not found, trying hdiutil (macOS)...")
|
||||
|
||||
# Try hdiutil for macOS
|
||||
if not iso_created:
|
||||
try:
|
||||
subprocess.run([
|
||||
'hdiutil', 'makehybrid',
|
||||
'-iso', '-joliet',
|
||||
'-o', iso_path,
|
||||
cloud_init_dir
|
||||
], check=True)
|
||||
iso_created = True
|
||||
except FileNotFoundError:
|
||||
logger.error(
|
||||
"No ISO creation tool found. Please install genisoimage, mkisofs, or use macOS hdiutil")
|
||||
return False
|
||||
|
||||
if not iso_created:
|
||||
logger.error("Failed to create ISO with any available tool")
|
||||
return False
|
||||
|
||||
# Copy ISO to Unraid
|
||||
subprocess.run(
|
||||
f"scp {iso_path} {UNRAID_USER}@{UNRAID_HOST}:/mnt/user/isos/",
|
||||
shell=True,
|
||||
check=True
|
||||
)
|
||||
|
||||
logger.info("Cloud-init ISO created successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create cloud-init ISO: {e}")
|
||||
return False
|
||||
|
||||
def vm_status(self) -> str:
|
||||
"""Get VM status."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh domstate {VM_NAME}'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
return result.stdout.strip()
|
||||
else:
|
||||
return "unknown"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting VM status: {e}")
|
||||
return "error"
|
||||
|
||||
def delete_vm(self) -> bool:
|
||||
"""Completely remove VM and all associated files."""
|
||||
try:
|
||||
logger.info(f"Deleting VM {VM_NAME} and all associated files...")
|
||||
|
||||
# Check if VM exists
|
||||
if not self.check_vm_exists():
|
||||
logger.info(f"VM {VM_NAME} does not exist")
|
||||
return True
|
||||
|
||||
# Stop VM if running
|
||||
if self.vm_status() == "running":
|
||||
logger.info(f"Stopping VM {VM_NAME}...")
|
||||
self.stop_vm()
|
||||
import time
|
||||
time.sleep(5)
|
||||
|
||||
# Undefine VM with NVRAM
|
||||
logger.info(f"Undefining VM {VM_NAME}...")
|
||||
subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh undefine {VM_NAME} --nvram'",
|
||||
shell=True,
|
||||
check=True
|
||||
)
|
||||
|
||||
# Remove VM directory and all files
|
||||
logger.info(f"Removing VM directory and files...")
|
||||
subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'rm -rf {self.vm_config_path}'",
|
||||
shell=True,
|
||||
check=True
|
||||
)
|
||||
|
||||
# Remove cloud-init ISO
|
||||
subprocess.run(
|
||||
f"ssh {UNRAID_USER}@{UNRAID_HOST} 'rm -f /mnt/user/isos/{VM_NAME}-cloud-init.iso'",
|
||||
shell=True,
|
||||
check=False # Don't fail if file doesn't exist
|
||||
)
|
||||
|
||||
logger.info(f"VM {VM_NAME} completely removed")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete VM: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Unraid VM Manager for ThrillWiki')
|
||||
parser.add_argument('action', choices=['create', 'start', 'stop', 'status', 'ip', 'setup', 'delete'],
|
||||
help='Action to perform')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Create logs directory
|
||||
os.makedirs('logs', exist_ok=True)
|
||||
|
||||
vm_manager = UnraidVMManager()
|
||||
|
||||
if args.action == 'create':
|
||||
success = vm_manager.create_vm()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
elif args.action == 'start':
|
||||
success = vm_manager.start_vm()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
elif args.action == 'stop':
|
||||
success = vm_manager.stop_vm()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
elif args.action == 'status':
|
||||
status = vm_manager.vm_status()
|
||||
print(f"VM Status: {status}")
|
||||
sys.exit(0)
|
||||
|
||||
elif args.action == 'ip':
|
||||
ip = vm_manager.get_vm_ip()
|
||||
if ip:
|
||||
print(f"VM IP: {ip}")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Failed to get VM IP")
|
||||
sys.exit(1)
|
||||
|
||||
elif args.action == 'setup':
|
||||
logger.info("Setting up complete VM environment...")
|
||||
|
||||
# Create VM
|
||||
if not vm_manager.create_vm():
|
||||
sys.exit(1)
|
||||
|
||||
# Start VM
|
||||
if not vm_manager.start_vm():
|
||||
sys.exit(1)
|
||||
|
||||
# Get IP
|
||||
vm_ip = vm_manager.get_vm_ip()
|
||||
if not vm_ip:
|
||||
sys.exit(1)
|
||||
|
||||
print(f"VM setup complete. IP: {vm_ip}")
|
||||
print("You can now connect via SSH and complete the ThrillWiki setup.")
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
elif args.action == 'delete':
|
||||
success = vm_manager.delete_vm()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
340
scripts/vm-deploy.sh
Executable file
340
scripts/vm-deploy.sh
Executable file
@@ -0,0 +1,340 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ThrillWiki VM Deployment Script
|
||||
# This script runs on the Linux VM to deploy the latest code and restart the server
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
# Configuration
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
LOG_DIR="$PROJECT_DIR/logs"
|
||||
BACKUP_DIR="$PROJECT_DIR/backups"
|
||||
DEPLOY_LOG="$LOG_DIR/deploy.log"
|
||||
SERVICE_NAME="thrillwiki"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Logging function
|
||||
log() {
|
||||
local message="[$(date +'%Y-%m-%d %H:%M:%S')] $1"
|
||||
echo -e "${BLUE}${message}${NC}"
|
||||
echo "$message" >> "$DEPLOY_LOG"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
local message="[$(date +'%Y-%m-%d %H:%M:%S')] ✓ $1"
|
||||
echo -e "${GREEN}${message}${NC}"
|
||||
echo "$message" >> "$DEPLOY_LOG"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
local message="[$(date +'%Y-%m-%d %H:%M:%S')] ⚠ $1"
|
||||
echo -e "${YELLOW}${message}${NC}"
|
||||
echo "$message" >> "$DEPLOY_LOG"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
local message="[$(date +'%Y-%m-%d %H:%M:%S')] ✗ $1"
|
||||
echo -e "${RED}${message}${NC}"
|
||||
echo "$message" >> "$DEPLOY_LOG"
|
||||
}
|
||||
|
||||
# Create necessary directories
|
||||
create_directories() {
|
||||
log "Creating necessary directories..."
|
||||
mkdir -p "$LOG_DIR" "$BACKUP_DIR"
|
||||
log_success "Directories created"
|
||||
}
|
||||
|
||||
# Backup current deployment
|
||||
backup_current() {
|
||||
log "Creating backup of current deployment..."
|
||||
local timestamp=$(date +'%Y%m%d_%H%M%S')
|
||||
local backup_path="$BACKUP_DIR/backup_$timestamp"
|
||||
|
||||
# Create backup of current code
|
||||
if [ -d "$PROJECT_DIR/.git" ]; then
|
||||
local current_commit=$(git -C "$PROJECT_DIR" rev-parse HEAD)
|
||||
echo "$current_commit" > "$backup_path.commit"
|
||||
log_success "Backup created with commit: ${current_commit:0:8}"
|
||||
else
|
||||
log_warning "Not a git repository, skipping backup"
|
||||
fi
|
||||
}
|
||||
|
||||
# Stop the service
|
||||
stop_service() {
|
||||
log "Stopping ThrillWiki service..."
|
||||
|
||||
# Stop systemd service if it exists
|
||||
if systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
|
||||
sudo systemctl stop "$SERVICE_NAME"
|
||||
log_success "Systemd service stopped"
|
||||
else
|
||||
log "Systemd service not running"
|
||||
fi
|
||||
|
||||
# Kill any remaining Django processes on port 8000
|
||||
if lsof -ti :8000 >/dev/null 2>&1; then
|
||||
log "Stopping processes on port 8000..."
|
||||
lsof -ti :8000 | xargs kill -9 2>/dev/null || true
|
||||
log_success "Port 8000 processes stopped"
|
||||
fi
|
||||
|
||||
# Clean up Python cache
|
||||
log "Cleaning Python cache..."
|
||||
find "$PROJECT_DIR" -type d -name "__pycache__" -exec rm -r {} + 2>/dev/null || true
|
||||
log_success "Python cache cleaned"
|
||||
}
|
||||
|
||||
# Update code from git
|
||||
update_code() {
|
||||
log "Updating code from git repository..."
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# Fetch latest changes
|
||||
git fetch origin
|
||||
log "Fetched latest changes"
|
||||
|
||||
# Get current and new commit info
|
||||
local old_commit=$(git rev-parse HEAD)
|
||||
local new_commit=$(git rev-parse origin/main)
|
||||
|
||||
if [ "$old_commit" = "$new_commit" ]; then
|
||||
log_warning "No new commits to deploy"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Updating from ${old_commit:0:8} to ${new_commit:0:8}"
|
||||
|
||||
# Pull latest changes
|
||||
git reset --hard origin/main
|
||||
log_success "Code updated successfully"
|
||||
|
||||
# Show what changed
|
||||
log "Changes in this deployment:"
|
||||
git log --oneline "$old_commit..$new_commit" || true
|
||||
}
|
||||
|
||||
# Install/update dependencies
|
||||
update_dependencies() {
|
||||
log "Updating dependencies..."
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# Check if UV is installed
|
||||
if ! command -v uv &> /dev/null; then
|
||||
log_error "UV is not installed. Installing UV..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
source $HOME/.cargo/env
|
||||
fi
|
||||
|
||||
# Sync dependencies
|
||||
uv sync --no-dev || {
|
||||
log_error "Failed to sync dependencies"
|
||||
return 1
|
||||
}
|
||||
|
||||
log_success "Dependencies updated"
|
||||
}
|
||||
|
||||
# Run database migrations
|
||||
run_migrations() {
|
||||
log "Running database migrations..."
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# Check for pending migrations
|
||||
if uv run manage.py showmigrations --plan | grep -q "\[ \]"; then
|
||||
log "Applying database migrations..."
|
||||
uv run manage.py migrate || {
|
||||
log_error "Database migrations failed"
|
||||
return 1
|
||||
}
|
||||
log_success "Database migrations completed"
|
||||
else
|
||||
log "No pending migrations"
|
||||
fi
|
||||
}
|
||||
|
||||
# Collect static files
|
||||
collect_static() {
|
||||
log "Collecting static files..."
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
uv run manage.py collectstatic --noinput || {
|
||||
log_warning "Static file collection failed, continuing..."
|
||||
}
|
||||
|
||||
log_success "Static files collected"
|
||||
}
|
||||
|
||||
# Start the service
|
||||
start_service() {
|
||||
log "Starting ThrillWiki service..."
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# Start systemd service if it exists
|
||||
if systemctl list-unit-files | grep -q "^$SERVICE_NAME.service"; then
|
||||
sudo systemctl start "$SERVICE_NAME"
|
||||
sudo systemctl enable "$SERVICE_NAME"
|
||||
|
||||
# Wait for service to start
|
||||
sleep 5
|
||||
|
||||
if systemctl is-active --quiet "$SERVICE_NAME"; then
|
||||
log_success "Systemd service started successfully"
|
||||
else
|
||||
log_error "Systemd service failed to start"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log_warning "Systemd service not found, starting manually..."
|
||||
|
||||
# Start server in background
|
||||
nohup ./scripts/ci-start.sh > "$LOG_DIR/server.log" 2>&1 &
|
||||
local server_pid=$!
|
||||
|
||||
# Wait for server to start
|
||||
sleep 5
|
||||
|
||||
if kill -0 $server_pid 2>/dev/null; then
|
||||
echo $server_pid > "$LOG_DIR/server.pid"
|
||||
log_success "Server started manually with PID: $server_pid"
|
||||
else
|
||||
log_error "Failed to start server manually"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Health check
|
||||
health_check() {
|
||||
log "Performing health check..."
|
||||
|
||||
local max_attempts=30
|
||||
local attempt=1
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if curl -f -s http://localhost:8000/health >/dev/null 2>&1; then
|
||||
log_success "Health check passed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Health check attempt $attempt/$max_attempts failed, retrying..."
|
||||
sleep 2
|
||||
((attempt++))
|
||||
done
|
||||
|
||||
log_error "Health check failed after $max_attempts attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Cleanup old backups
|
||||
cleanup_backups() {
|
||||
log "Cleaning up old backups..."
|
||||
|
||||
# Keep only the last 10 backups
|
||||
cd "$BACKUP_DIR"
|
||||
ls -t backup_*.commit 2>/dev/null | tail -n +11 | xargs rm -f 2>/dev/null || true
|
||||
|
||||
log_success "Old backups cleaned up"
|
||||
}
|
||||
|
||||
# Rollback function
|
||||
rollback() {
|
||||
log_error "Deployment failed, attempting rollback..."
|
||||
|
||||
local latest_backup=$(ls -t "$BACKUP_DIR"/backup_*.commit 2>/dev/null | head -n 1)
|
||||
|
||||
if [ -n "$latest_backup" ]; then
|
||||
local backup_commit=$(cat "$latest_backup")
|
||||
log "Rolling back to commit: ${backup_commit:0:8}"
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
git reset --hard "$backup_commit"
|
||||
|
||||
# Restart service
|
||||
stop_service
|
||||
start_service
|
||||
|
||||
if health_check; then
|
||||
log_success "Rollback completed successfully"
|
||||
else
|
||||
log_error "Rollback failed - manual intervention required"
|
||||
fi
|
||||
else
|
||||
log_error "No backup found for rollback"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main deployment function
|
||||
deploy() {
|
||||
log "=== ThrillWiki Deployment Started ==="
|
||||
log "Timestamp: $(date)"
|
||||
log "User: $(whoami)"
|
||||
log "Host: $(hostname)"
|
||||
|
||||
# Trap errors for rollback
|
||||
trap rollback ERR
|
||||
|
||||
create_directories
|
||||
backup_current
|
||||
stop_service
|
||||
update_code
|
||||
update_dependencies
|
||||
run_migrations
|
||||
collect_static
|
||||
start_service
|
||||
health_check
|
||||
cleanup_backups
|
||||
|
||||
# Remove error trap
|
||||
trap - ERR
|
||||
|
||||
log_success "=== Deployment Completed Successfully ==="
|
||||
log "Server is now running the latest code"
|
||||
log "Check logs at: $LOG_DIR/"
|
||||
}
|
||||
|
||||
# Script execution
|
||||
case "${1:-deploy}" in
|
||||
deploy)
|
||||
deploy
|
||||
;;
|
||||
stop)
|
||||
stop_service
|
||||
;;
|
||||
start)
|
||||
start_service
|
||||
;;
|
||||
restart)
|
||||
stop_service
|
||||
start_service
|
||||
health_check
|
||||
;;
|
||||
status)
|
||||
if systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then
|
||||
echo "Service is running"
|
||||
elif [ -f "$LOG_DIR/server.pid" ] && kill -0 "$(cat "$LOG_DIR/server.pid")" 2>/dev/null; then
|
||||
echo "Server is running manually"
|
||||
else
|
||||
echo "Service is not running"
|
||||
fi
|
||||
;;
|
||||
health)
|
||||
health_check
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {deploy|stop|start|restart|status|health}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
268
scripts/webhook-listener.py
Executable file
268
scripts/webhook-listener.py
Executable file
@@ -0,0 +1,268 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
GitHub Webhook Listener for ThrillWiki CI/CD
|
||||
This script listens for GitHub webhook events and triggers deployments to a Linux VM.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import hmac
|
||||
import hashlib
|
||||
import logging
|
||||
import subprocess
|
||||
import requests
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
# Configuration
|
||||
WEBHOOK_PORT = int(os***REMOVED***iron.get('WEBHOOK_PORT', 9000))
|
||||
WEBHOOK_SECRET = os***REMOVED***iron.get('WEBHOOK_SECRET', '')
|
||||
WEBHOOK_ENABLED = os***REMOVED***iron.get('WEBHOOK_ENABLED', 'true').lower() == 'true'
|
||||
VM_HOST = os***REMOVED***iron.get('VM_HOST', 'localhost')
|
||||
VM_PORT = int(os***REMOVED***iron.get('VM_PORT', 22))
|
||||
VM_USER = os***REMOVED***iron.get('VM_USER', 'ubuntu')
|
||||
VM_KEY_PATH = os***REMOVED***iron.get('VM_KEY_PATH', '~/.ssh/***REMOVED***')
|
||||
PROJECT_PATH = os***REMOVED***iron.get('VM_PROJECT_PATH', '/home/ubuntu/thrillwiki')
|
||||
REPO_URL = os***REMOVED***iron.get('REPO_URL', 'https://github.com/YOUR_USERNAME/thrillwiki_django_no_react.git')
|
||||
DEPLOY_BRANCH = os***REMOVED***iron.get('DEPLOY_BRANCH', 'main')
|
||||
|
||||
# GitHub API Configuration
|
||||
GITHUB_USERNAME = os***REMOVED***iron.get('GITHUB_USERNAME', '')
|
||||
GITHUB_TOKEN = os***REMOVED***iron.get('GITHUB_TOKEN', '')
|
||||
GITHUB_API_ENABLED = os***REMOVED***iron.get('GITHUB_API_ENABLED', 'false').lower() == 'true'
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('logs/webhook.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class GitHubWebhookHandler(BaseHTTPRequestHandler):
|
||||
"""Handle incoming GitHub webhook requests."""
|
||||
|
||||
def do_GET(self):
|
||||
"""Handle GET requests - health check."""
|
||||
if self.path == '/health':
|
||||
self.send_response(200)
|
||||
self.send_header('Content-type', 'application/json')
|
||||
self.end_headers()
|
||||
response = {
|
||||
'status': 'healthy',
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'service': 'ThrillWiki Webhook Listener'
|
||||
}
|
||||
self.wfile.write(json.dumps(response).encode())
|
||||
else:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def do_POST(self):
|
||||
"""Handle POST requests - webhook events."""
|
||||
try:
|
||||
content_length = int(self.headers['Content-Length'])
|
||||
post_data = self.rfile.read(content_length)
|
||||
|
||||
# Verify webhook signature if secret is configured
|
||||
if WEBHOOK_SECRET:
|
||||
if not self._verify_signature(post_data):
|
||||
logger.warning("Invalid webhook signature")
|
||||
self.send_response(401)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
# Parse webhook payload
|
||||
try:
|
||||
payload = json.loads(post_data.decode('utf-8'))
|
||||
except json.JSONDecodeError:
|
||||
logger.error("Invalid JSON payload")
|
||||
self.send_response(400)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
# Handle webhook event
|
||||
event_type = self.headers.get('X-GitHub-Event')
|
||||
if self._should_deploy(event_type, payload):
|
||||
logger.info(f"Triggering deployment for {event_type} event")
|
||||
threading.Thread(target=self._trigger_deployment, args=(payload,)).start()
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header('Content-type', 'application/json')
|
||||
self.end_headers()
|
||||
response = {'status': 'deployment_triggered', 'event': event_type}
|
||||
self.wfile.write(json.dumps(response).encode())
|
||||
else:
|
||||
logger.info(f"Ignoring {event_type} event - no deployment needed")
|
||||
self.send_response(200)
|
||||
self.send_header('Content-type', 'application/json')
|
||||
self.end_headers()
|
||||
response = {'status': 'ignored', 'event': event_type}
|
||||
self.wfile.write(json.dumps(response).encode())
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error handling webhook: {e}")
|
||||
self.send_response(500)
|
||||
self.end_headers()
|
||||
|
||||
def _verify_signature(self, payload_body):
|
||||
"""Verify GitHub webhook signature."""
|
||||
signature = self.headers.get('X-Hub-Signature-256')
|
||||
if not signature:
|
||||
return False
|
||||
|
||||
expected_signature = 'sha256=' + hmac.new(
|
||||
WEBHOOK_SECRET.encode(),
|
||||
payload_body,
|
||||
hashlib.sha256
|
||||
).hexdigest()
|
||||
|
||||
return hmac.compare_digest(signature, expected_signature)
|
||||
|
||||
def _should_deploy(self, event_type, payload):
|
||||
"""Determine if we should trigger a deployment."""
|
||||
if event_type == 'push':
|
||||
# Deploy on push to main branch
|
||||
ref = payload.get('ref', '')
|
||||
target_ref = f'refs/heads/{DEPLOY_BRANCH}'
|
||||
return ref == target_ref
|
||||
elif event_type == 'release':
|
||||
# Deploy on new releases
|
||||
action = payload.get('action', '')
|
||||
return action == 'published'
|
||||
|
||||
return False
|
||||
|
||||
def _trigger_deployment(self, payload):
|
||||
"""Trigger deployment to Linux VM."""
|
||||
try:
|
||||
commit_sha = payload.get('after') or payload.get('head_commit', {}).get('id', 'unknown')
|
||||
commit_message = payload.get('head_commit', {}).get('message', 'No message')
|
||||
|
||||
logger.info(f"Starting deployment of commit {commit_sha[:8]}: {commit_message}")
|
||||
|
||||
# Execute deployment script on VM
|
||||
deploy_script = f"""
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "=== ThrillWiki Deployment Started ==="
|
||||
echo "Commit: {commit_sha[:8]}"
|
||||
echo "Message: {commit_message}"
|
||||
echo "Timestamp: $(date)"
|
||||
|
||||
cd {PROJECT_PATH}
|
||||
|
||||
# Pull latest changes
|
||||
git fetch origin
|
||||
git checkout {DEPLOY_BRANCH}
|
||||
git pull origin {DEPLOY_BRANCH}
|
||||
|
||||
# Run deployment script
|
||||
./scripts/vm-deploy.sh
|
||||
|
||||
echo "=== Deployment Completed Successfully ==="
|
||||
"""
|
||||
|
||||
# Execute deployment on VM via SSH
|
||||
ssh_command = [
|
||||
'ssh',
|
||||
'-i', VM_KEY_PATH,
|
||||
'-o', 'StrictHostKeyChecking=no',
|
||||
'-o', 'UserKnownHostsFile=/dev/null',
|
||||
f'{VM_USER}@{VM_HOST}',
|
||||
deploy_script
|
||||
]
|
||||
|
||||
result = subprocess.run(
|
||||
ssh_command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300 # 5 minute timeout
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info(f"Deployment successful for commit {commit_sha[:8]}")
|
||||
self._send_status_notification('success', commit_sha, commit_message)
|
||||
else:
|
||||
logger.error(f"Deployment failed for commit {commit_sha[:8]}: {result.stderr}")
|
||||
self._send_status_notification('failure', commit_sha, commit_message, result.stderr)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("Deployment timed out")
|
||||
self._send_status_notification('timeout', commit_sha, commit_message)
|
||||
except Exception as e:
|
||||
logger.error(f"Deployment error: {e}")
|
||||
self._send_status_notification('error', commit_sha, commit_message, str(e))
|
||||
|
||||
def _send_status_notification(self, status, commit_sha, commit_message, error_details=None):
|
||||
"""Send deployment status notification (optional)."""
|
||||
# This could be extended to send notifications to Slack, Discord, etc.
|
||||
status_msg = f"Deployment {status} for commit {commit_sha[:8]}: {commit_message}"
|
||||
if error_details:
|
||||
status_msg += f"\nError: {error_details}"
|
||||
|
||||
logger.info(f"Status: {status_msg}")
|
||||
|
||||
def log_message(self, format, *args):
|
||||
"""Override to use our logger."""
|
||||
logger.info(f"{self.client_address[0]} - {format % args}")
|
||||
|
||||
def main():
|
||||
"""Main function to start the webhook listener."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='ThrillWiki GitHub Webhook Listener')
|
||||
parser.add_argument('--port', type=int, default=WEBHOOK_PORT, help='Port to listen on')
|
||||
parser.add_argument('--test', action='store_true', help='Test configuration without starting server')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Create logs directory
|
||||
os.makedirs('logs', exist_ok=True)
|
||||
|
||||
# Validate configuration
|
||||
if not WEBHOOK_SECRET:
|
||||
logger.warning("WEBHOOK_SECRET not set - webhook signature verification disabled")
|
||||
|
||||
if not all([VM_HOST, VM_USER, PROJECT_PATH]):
|
||||
logger.error("Missing required VM configuration")
|
||||
if args.test:
|
||||
print("❌ Configuration validation failed")
|
||||
return
|
||||
sys.exit(1)
|
||||
|
||||
logger.info(f"Webhook listener configuration:")
|
||||
logger.info(f" Port: {args.port}")
|
||||
logger.info(f" Target VM: {VM_USER}@{VM_HOST}")
|
||||
logger.info(f" Project path: {PROJECT_PATH}")
|
||||
logger.info(f" Deploy branch: {DEPLOY_BRANCH}")
|
||||
|
||||
if args.test:
|
||||
print("✅ Configuration validation passed")
|
||||
print(f"Webhook would listen on port {args.port}")
|
||||
print(f"Target: {VM_USER}@{VM_HOST}")
|
||||
return
|
||||
|
||||
logger.info(f"Starting webhook listener on port {args.port}")
|
||||
|
||||
try:
|
||||
server = HTTPServer(('0.0.0.0', args.port), GitHubWebhookHandler)
|
||||
logger.info(f"Webhook listener started successfully on http://0.0.0.0:{args.port}")
|
||||
logger.info("Health check available at: /health")
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Webhook listener stopped by user")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start webhook listener: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user