#!/usr/bin/env python3 """ Unraid VM Manager for ThrillWiki - Modular Ubuntu Autoinstall Follows the Ubuntu autoinstall guide exactly: 1. Creates modified Ubuntu ISO with autoinstall configuration 2. Manages VM lifecycle on Unraid server 3. Handles ThrillWiki deployment automation """ import os import sys import time import logging import subprocess import shutil from pathlib import Path from typing import Optional # Import our modular components # Note: UnraidVMManager is defined locally in this file # Configuration UNRAID_HOST = os.environ.get("UNRAID_HOST", "localhost") UNRAID_USER = os.environ.get("UNRAID_USER", "root") VM_NAME = os.environ.get("VM_NAME", "thrillwiki-vm") VM_MEMORY = int(os.environ.get("VM_MEMORY", 4096)) # MB VM_VCPUS = int(os.environ.get("VM_VCPUS", 2)) VM_DISK_SIZE = int(os.environ.get("VM_DISK_SIZE", 50)) # GB SSH_PUBLIC_KEY = os.environ.get("SSH_PUBLIC_KEY", "") # Network Configuration VM_IP = os.environ.get("VM_IP", "dhcp") VM_GATEWAY = os.environ.get("VM_GATEWAY", "192.168.20.1") VM_NETMASK = os.environ.get("VM_NETMASK", "255.255.255.0") VM_NETWORK = os.environ.get("VM_NETWORK", "192.168.20.0/24") # GitHub Configuration REPO_URL = os.environ.get("REPO_URL", "") GITHUB_USERNAME = os.environ.get("GITHUB_USERNAME", "") GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "") # Ubuntu version preference UBUNTU_VERSION = os.environ.get("UBUNTU_VERSION", "24.04") # Setup logging os.makedirs("logs", exist_ok=True) logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", handlers=[ logging.FileHandler("logs/unraid-vm.log"), logging.StreamHandler(), ], ) logger = logging.getLogger(__name__) class UnraidVMManager: """Manages VMs on Unraid server.""" def __init__(self): self.vm_config_path = f"/mnt/user/domains/{VM_NAME}" def authenticate(self) -> bool: """Test SSH connectivity to Unraid server.""" try: result = subprocess.run( f"ssh -o ConnectTimeout=10 {UNRAID_USER}@{UNRAID_HOST} 'echo Connected'", shell=True, capture_output=True, text=True, timeout=15, ) if result.returncode == 0 and "Connected" in result.stdout: logger.info("Successfully connected to Unraid via SSH") return True else: logger.error(f"SSH connection failed: {result.stderr}") return False except Exception as e: logger.error(f"SSH authentication error: {e}") return False def check_vm_exists(self) -> bool: """Check if VM already exists.""" try: result = subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh list --all | grep {VM_NAME}'", shell=True, capture_output=True, text=True, ) return VM_NAME in result.stdout except Exception as e: logger.error(f"Error checking VM existence: {e}") return False def _generate_mac_suffix(self) -> str: """Generate MAC address suffix based on VM IP or name.""" if VM_IP.lower() != "dhcp" and "." in VM_IP: # Use last octet of static IP for MAC generation last_octet = int(VM_IP.split(".")[-1]) return f"{last_octet:02x}:7d:fd" else: # Use hash of VM name for consistent MAC generation import hashlib hash_obj = hashlib.md5(VM_NAME.encode()) hash_bytes = hash_obj.digest()[:3] return ":".join([f"{b:02x}" for b in hash_bytes]) def create_vm_xml(self, existing_uuid: str = None) -> str: """Generate VM XML configuration from template file.""" import uuid vm_uuid = existing_uuid if existing_uuid else str(uuid.uuid4()) # Detect Ubuntu ISO dynamically ubuntu_iso_path = self._detect_ubuntu_iso() if not ubuntu_iso_path: raise FileNotFoundError("No Ubuntu ISO found for VM template") # Read XML template from file template_path = Path(__file__).parent / "thrillwiki-vm-template.xml" if not template_path.exists(): raise FileNotFoundError(f"VM XML template not found at {template_path}") with open(template_path, "r", encoding="utf-8") as f: xml_template = f.read() # Calculate CPU topology cpu_cores = VM_VCPUS // 2 if VM_VCPUS > 1 else 1 cpu_threads = 2 if VM_VCPUS > 1 else 1 mac_suffix = self._generate_mac_suffix() # Replace placeholders with actual values xml_content = xml_template.format( VM_NAME=VM_NAME, VM_UUID=vm_uuid, VM_MEMORY_KIB=VM_MEMORY * 1024, VM_VCPUS=VM_VCPUS, CPU_CORES=cpu_cores, CPU_THREADS=cpu_threads, MAC_SUFFIX=mac_suffix, UBUNTU_ISO_PATH=ubuntu_iso_path, ) return xml_content.strip() def _detect_ubuntu_iso(self) -> Optional[str]: """Detect and return the path of the best available Ubuntu ISO.""" try: # Find all Ubuntu ISOs find_all_result = subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'find /mnt/user/isos -name \"ubuntu*.iso\" -type f | sort -V'", shell=True, capture_output=True, text=True, ) if find_all_result.returncode != 0 or not find_all_result.stdout.strip(): return None available_isos = find_all_result.stdout.strip().split("\n") # Prioritize ISOs by version and type # Sort by preference: 24.04 LTS > 22.04 LTS > 23.x > 20.04 > others # Within each version, prefer the latest point release priority_versions = [ "24.04", # Ubuntu 24.04 LTS (highest priority) "22.04", # Ubuntu 22.04 LTS "23.10", # Ubuntu 23.10 "23.04", # Ubuntu 23.04 "20.04", # Ubuntu 20.04 LTS ] # Find the best ISO based on priority, preferring latest point # releases for version in priority_versions: # Find all ISOs for this version version_isos = [] for iso in available_isos: if version in iso and ( "server" in iso.lower() or "live" in iso.lower() ): version_isos.append(iso) if version_isos: # Sort by version number (reverse to get latest first) # This will put 24.04.3 before 24.04.2 before 24.04.1 # before 24.04 version_isos.sort(reverse=True) return version_isos[0] # If no priority match, use the first server/live ISO found for iso in available_isos: if "server" in iso.lower() or "live" in iso.lower(): return iso # If still no match, use the first Ubuntu ISO found (any type) if available_isos: return available_isos[0] return None except Exception as e: logger.error(f"Error detecting Ubuntu ISO: {e}") return None def create_vm(self) -> bool: """Create or update the VM on Unraid.""" try: vm_exists = self.check_vm_exists() if vm_exists: logger.info(f"VM {VM_NAME} already exists, updating configuration...") # Always try to stop VM before updating (force stop) current_status = self.vm_status() logger.info(f"Current VM status: {current_status}") if current_status not in ["shut off", "unknown"]: logger.info(f"Stopping VM {VM_NAME} for configuration update...") self.stop_vm() # Wait for VM to stop time.sleep(3) else: logger.info(f"VM {VM_NAME} is already stopped") else: logger.info(f"Creating VM {VM_NAME}...") # Ensure VM directory exists (for both new and updated VMs) subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'mkdir -p {self.vm_config_path}'", shell=True, check=True, ) # Create virtual disk if it doesn't exist (for both new and updated # VMs) disk_check = subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'test -f {self.vm_config_path}/vdisk1.qcow2'", shell=True, capture_output=True, ) if disk_check.returncode != 0: logger.info(f"Creating virtual disk for VM {VM_NAME}...") disk_cmd = f""" ssh {UNRAID_USER}@{UNRAID_HOST} 'qemu-img create -f qcow2 {self.vm_config_path}/vdisk1.qcow2 {VM_DISK_SIZE}G' """ subprocess.run(disk_cmd, shell=True, check=True) else: logger.info(f"Virtual disk already exists for VM {VM_NAME}") # Always create/recreate cloud-init ISO for automated installation and ThrillWiki deployment # This ensures the latest configuration is used whether creating or # updating the VM logger.info( "Creating cloud-init ISO for automated Ubuntu and ThrillWiki setup..." ) if not self.create_cloud_init_iso(VM_IP): logger.error("Failed to create cloud-init ISO") return False # For Ubuntu 24.04, use UEFI boot instead of kernel extraction # Ubuntu 24.04 has issues with direct kernel boot autoinstall logger.info("Using UEFI boot for Ubuntu 24.04 compatibility...") if not self.fallback_to_uefi_boot(): logger.error("UEFI boot setup failed") return False existing_uuid = None if vm_exists: # Get existing VM UUID result = subprocess.run( f'ssh {UNRAID_USER}@{UNRAID_HOST} \'virsh dumpxml {VM_NAME} | grep "" | sed "s///g" | sed "s/<\\/uuid>//g" | tr -d " "\'', shell=True, capture_output=True, text=True, ) if result.returncode == 0 and result.stdout.strip(): existing_uuid = result.stdout.strip() logger.info(f"Found existing VM UUID: {existing_uuid}") # Check if VM is persistent or transient persistent_check = subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh list --persistent --all | grep {VM_NAME}'", shell=True, capture_output=True, text=True, ) is_persistent = VM_NAME in persistent_check.stdout if is_persistent: # Undefine persistent VM with NVRAM flag logger.info( f"VM {VM_NAME} is persistent, undefining with NVRAM for reconfiguration..." ) subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh undefine {VM_NAME} --nvram'", shell=True, check=True, ) logger.info( f"Persistent VM {VM_NAME} undefined for reconfiguration" ) else: # Handle transient VM - just destroy it logger.info( f"VM {VM_NAME} is transient, destroying for reconfiguration..." ) # Stop the VM first if it's running if self.vm_status() == "running": subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh destroy {VM_NAME}'", shell=True, check=True, ) logger.info(f"Transient VM {VM_NAME} destroyed for reconfiguration") # Generate VM XML with appropriate UUID vm_xml = self.create_vm_xml(existing_uuid) xml_file = f"/tmp/{VM_NAME}.xml" with open(xml_file, "w", encoding="utf-8") as f: f.write(vm_xml) # Copy XML to Unraid and define/redefine VM subprocess.run( f"scp {xml_file} {UNRAID_USER}@{UNRAID_HOST}:/tmp/", shell=True, check=True, ) # Define VM as persistent domain subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh define /tmp/{VM_NAME}.xml'", shell=True, check=True, ) # Ensure VM is set to autostart for persistent configuration subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'virsh autostart {VM_NAME}'", shell=True, check=False, # Don't fail if autostart is already enabled ) action = "updated" if vm_exists else "created" logger.info(f"VM {VM_NAME} {action} successfully") # Cleanup os.remove(xml_file) return True except Exception as e: logger.error(f"Failed to create VM: {e}") return False def extract_ubuntu_kernel(self) -> bool: """Extract Ubuntu kernel and initrd from ISO for direct boot.""" try: # Check available Ubuntu ISOs and select the correct one iso_mount_point = "/tmp/ubuntu-iso" logger.info("Checking for available Ubuntu ISOs...") # List available Ubuntu ISOs with detailed information result = subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'ls -la /mnt/user/isos/ubuntu*.iso 2>/dev/null || echo \"No Ubuntu ISOs found\"'", shell=True, capture_output=True, text=True, ) logger.info(f"Available ISOs: {result.stdout}") # First, try to find ANY existing Ubuntu ISOs dynamically # This will find all Ubuntu ISOs regardless of naming convention find_all_result = subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'find /mnt/user/isos -name \"ubuntu*.iso\" -type f | sort -V'", shell=True, capture_output=True, text=True, ) ubuntu_iso_path = None available_isos = [] if find_all_result.returncode == 0 and find_all_result.stdout.strip(): available_isos = find_all_result.stdout.strip().split("\n") logger.info( f"Found { len(available_isos)} Ubuntu ISOs: {available_isos}" ) # Prioritize ISOs by version and type (prefer LTS, prefer newer versions) # Sort by preference: 24.04 LTS > 22.04 LTS > 23.x > 20.04 > others # Within each version, prefer the latest point release priority_versions = [ "24.04", # Ubuntu 24.04 LTS (highest priority) "22.04", # Ubuntu 22.04 LTS "23.10", # Ubuntu 23.10 "23.04", # Ubuntu 23.04 "20.04", # Ubuntu 20.04 LTS ] # Find the best ISO based on priority, preferring latest point # releases for version in priority_versions: # Find all ISOs for this version version_isos = [] for iso in available_isos: if version in iso and ( "server" in iso.lower() or "live" in iso.lower() ): version_isos.append(iso) if version_isos: # Sort by version number (reverse to get latest first) # This will put 24.04.3 before 24.04.2 before 24.04.1 # before 24.04 version_isos.sort(reverse=True) ubuntu_iso_path = version_isos[0] logger.info( f"Selected latest Ubuntu {version} ISO: {ubuntu_iso_path}" ) break # If no priority match, use the first server/live ISO found if not ubuntu_iso_path: for iso in available_isos: if "server" in iso.lower() or "live" in iso.lower(): ubuntu_iso_path = iso logger.info( f"Selected Ubuntu server/live ISO: {ubuntu_iso_path}" ) break # If still no match, use the first Ubuntu ISO found (any type) if not ubuntu_iso_path and available_isos: ubuntu_iso_path = available_isos[0] logger.info( f"Selected first available Ubuntu ISO: {ubuntu_iso_path}" ) logger.warning( f"Using non-server Ubuntu ISO - this may not support autoinstall" ) if not ubuntu_iso_path: logger.error("No Ubuntu server ISO found in /mnt/user/isos/") logger.error("") logger.error("🔥 MISSING UBUNTU ISO - ACTION REQUIRED 🔥") logger.error("") logger.error( "Please download Ubuntu LTS Server ISO to your Unraid server:" ) logger.error("") logger.error( "📦 RECOMMENDED: Ubuntu 24.04 LTS (Noble Numbat) - Latest LTS:" ) logger.error(" 1. Go to: https://releases.ubuntu.com/24.04/") logger.error(" 2. Download: ubuntu-24.04-live-server-amd64.iso") logger.error(" 3. Upload to: /mnt/user/isos/ on your Unraid server") logger.error("") logger.error( "📦 ALTERNATIVE: Ubuntu 22.04 LTS (Jammy Jellyfish) - Stable:" ) logger.error(" 1. Go to: https://releases.ubuntu.com/22.04/") logger.error(" 2. Download: ubuntu-22.04-live-server-amd64.iso") logger.error(" 3. Upload to: /mnt/user/isos/ on your Unraid server") logger.error("") logger.error("💡 Quick download via wget on Unraid server:") logger.error(" # For Ubuntu 24.04 LTS (recommended):") logger.error( " wget -P /mnt/user/isos/ https://releases.ubuntu.com/24.04/ubuntu-24.04-live-server-amd64.iso" ) logger.error(" # For Ubuntu 22.04 LTS (stable):") logger.error( " wget -P /mnt/user/isos/ https://releases.ubuntu.com/22.04/ubuntu-22.04-live-server-amd64.iso" ) logger.error("") logger.error("Then re-run this script.") logger.error("") return False # Verify ISO file integrity logger.info(f"Verifying ISO file: {ubuntu_iso_path}") stat_result = subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'stat {ubuntu_iso_path}'", shell=True, capture_output=True, text=True, ) if stat_result.returncode != 0: logger.error(f"Cannot access ISO file: {ubuntu_iso_path}") return False logger.info(f"ISO file stats: {stat_result.stdout.strip()}") # Clean up any previous mount points subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'umount {iso_mount_point} 2>/dev/null || true'", shell=True, check=False, ) # Remove mount point if it exists subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'rmdir {iso_mount_point} 2>/dev/null || true'", shell=True, check=False, ) # Create mount point logger.info(f"Creating mount point: {iso_mount_point}") subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'mkdir -p {iso_mount_point}'", shell=True, check=True, ) # Check if loop module is loaded logger.info("Checking loop module availability...") loop_check = subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'lsmod | grep loop || modprobe loop'", shell=True, capture_output=True, text=True, ) logger.info(f"Loop module check: {loop_check.stdout}") # Mount ISO with more verbose output logger.info(f"Mounting ISO: {ubuntu_iso_path} to {iso_mount_point}") mount_result = subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'mount -o loop,ro {ubuntu_iso_path} {iso_mount_point}'", shell=True, capture_output=True, text=True, ) if mount_result.returncode != 0: logger.error( f"Failed to mount ISO. Return code: { mount_result.returncode}" ) logger.error(f"STDOUT: {mount_result.stdout}") logger.error(f"STDERR: {mount_result.stderr}") return False logger.info("ISO mounted successfully") # Create directory for extracted kernel files kernel_dir = f"/mnt/user/domains/{VM_NAME}/kernel" subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'mkdir -p {kernel_dir}'", shell=True, check=True, ) # Extract kernel and initrd subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'cp {iso_mount_point}/casper/vmlinuz {kernel_dir}/'", shell=True, check=True, ) subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'cp {iso_mount_point}/casper/initrd {kernel_dir}/'", shell=True, check=True, ) # Unmount ISO subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'umount {iso_mount_point}'", shell=True, check=True, ) # Remove mount point subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'rmdir {iso_mount_point}'", shell=True, check=True, ) logger.info("Ubuntu kernel and initrd extracted successfully") return True except Exception as e: logger.error(f"Failed to extract Ubuntu kernel: {e}") # Clean up on failure subprocess.run( f"ssh {UNRAID_USER}@{UNRAID_HOST} 'umount {iso_mount_point} 2>/dev/null || true'", shell=True, check=False, ) return False def fallback_to_uefi_boot(self) -> bool: """Fallback to UEFI boot when kernel extraction fails.""" try: logger.info("Setting up fallback UEFI boot configuration...") # First, detect available Ubuntu ISO for the fallback template ubuntu_iso_path = self._detect_ubuntu_iso() if not ubuntu_iso_path: logger.error("Cannot create UEFI fallback without Ubuntu ISO") return False # Create a fallback VM XML template path fallback_template_path = ( Path(__file__).parent / "thrillwiki-vm-uefi-fallback-template.xml" ) # Create fallback UEFI template with detected Ubuntu ISO logger.info( f"Creating fallback UEFI template with detected ISO: {ubuntu_iso_path}" ) uefi_template = f""" {{VM_NAME}} {{VM_UUID}} {{VM_MEMORY_KIB}} {{VM_MEMORY_KIB}} {{VM_VCPUS}} hvm /usr/share/qemu/ovmf-x64/OVMF_CODE-pure-efi.fd /etc/libvirt/qemu/nvram/{{VM_UUID}}_VARS-pure-efi.fd destroy restart restart /usr/local/sbin/qemu