mirror of
https://github.com/pacnpal/thrillwiki_django_no_react.git
synced 2025-12-21 07:51:09 -05:00
Refactor test utilities and enhance ASGI settings
- Cleaned up and standardized assertions in ApiTestMixin for API response validation. - Updated ASGI settings to use os.environ for setting the DJANGO_SETTINGS_MODULE. - Removed unused imports and improved formatting in settings.py. - Refactored URL patterns in urls.py for better readability and organization. - Enhanced view functions in views.py for consistency and clarity. - Added .flake8 configuration for linting and style enforcement. - Introduced type stubs for django-environ to improve type checking with Pylance.
This commit is contained in:
pacnpal
@@ -5,7 +5,6 @@ Handles VM creation using pre-built template disks instead of autoinstall.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
import subprocess
|
||||
@@ -60,23 +59,28 @@ class UnraidTemplateVMManager:
|
||||
hash_bytes = hash_obj.digest()[:3]
|
||||
return ":".join([f"{b:02x}" for b in hash_bytes])
|
||||
|
||||
def create_vm_xml(self, vm_memory: int, vm_vcpus: int, vm_ip: str,
|
||||
existing_uuid: str = None) -> str:
|
||||
def create_vm_xml(
|
||||
self,
|
||||
vm_memory: int,
|
||||
vm_vcpus: int,
|
||||
vm_ip: str,
|
||||
existing_uuid: str = None,
|
||||
) -> str:
|
||||
"""Generate VM XML configuration from template file."""
|
||||
vm_uuid = existing_uuid if existing_uuid else str(uuid.uuid4())
|
||||
|
||||
|
||||
# Use simplified template for template-based VMs
|
||||
template_path = Path(__file__).parent / "thrillwiki-vm-template-simple.xml"
|
||||
if not template_path.exists():
|
||||
raise FileNotFoundError(f"VM XML template not found at {template_path}")
|
||||
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
|
||||
with open(template_path, "r", encoding="utf-8") as f:
|
||||
xml_template = f.read()
|
||||
|
||||
|
||||
# Calculate CPU topology
|
||||
cpu_cores = vm_vcpus // 2 if vm_vcpus > 1 else 1
|
||||
cpu_threads = 2 if vm_vcpus > 1 else 1
|
||||
|
||||
|
||||
# Replace placeholders with actual values
|
||||
xml_content = xml_template.format(
|
||||
VM_NAME=self.vm_name,
|
||||
@@ -85,25 +89,32 @@ class UnraidTemplateVMManager:
|
||||
VM_VCPUS=vm_vcpus,
|
||||
CPU_CORES=cpu_cores,
|
||||
CPU_THREADS=cpu_threads,
|
||||
MAC_SUFFIX=self._generate_mac_suffix(vm_ip)
|
||||
MAC_SUFFIX=self._generate_mac_suffix(vm_ip),
|
||||
)
|
||||
|
||||
|
||||
return xml_content.strip()
|
||||
|
||||
def create_vm_from_template(self, vm_memory: int, vm_vcpus: int,
|
||||
vm_disk_size: int, vm_ip: str) -> bool:
|
||||
def create_vm_from_template(
|
||||
self, vm_memory: int, vm_vcpus: int, vm_disk_size: int, vm_ip: str
|
||||
) -> bool:
|
||||
"""Create VM from template disk."""
|
||||
try:
|
||||
vm_exists = self.check_vm_exists()
|
||||
|
||||
if vm_exists:
|
||||
logger.info(f"VM {self.vm_name} already exists, updating configuration...")
|
||||
logger.info(
|
||||
f"VM {
|
||||
self.vm_name} already exists, updating configuration..."
|
||||
)
|
||||
# Always try to stop VM before updating
|
||||
current_status = self.vm_status()
|
||||
logger.info(f"Current VM status: {current_status}")
|
||||
|
||||
|
||||
if current_status not in ["shut off", "unknown"]:
|
||||
logger.info(f"Stopping VM {self.vm_name} for configuration update...")
|
||||
logger.info(
|
||||
f"Stopping VM {
|
||||
self.vm_name} for configuration update..."
|
||||
)
|
||||
self.stop_vm()
|
||||
time.sleep(3)
|
||||
else:
|
||||
@@ -123,7 +134,10 @@ class UnraidTemplateVMManager:
|
||||
|
||||
if vm_exists:
|
||||
# Get existing VM UUID
|
||||
cmd = f"ssh {self.unraid_user}@{self.unraid_host} 'virsh dumpxml {self.vm_name} | grep \"<uuid>\" | sed \"s/<uuid>//g\" | sed \"s/<\\/uuid>//g\" | tr -d \" \"'"
|
||||
cmd = f'ssh {
|
||||
self.unraid_user}@{
|
||||
self.unraid_host} \'virsh dumpxml {
|
||||
self.vm_name} | grep "<uuid>" | sed "s/<uuid>//g" | sed "s/<\\/uuid>//g" | tr -d " "\''
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
shell=True,
|
||||
@@ -142,34 +156,49 @@ class UnraidTemplateVMManager:
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
is_persistent = self.vm_name in persistent_check.stdout
|
||||
|
||||
|
||||
if is_persistent:
|
||||
# Undefine persistent VM with NVRAM flag
|
||||
logger.info(f"VM {self.vm_name} is persistent, undefining with NVRAM for reconfiguration...")
|
||||
logger.info(
|
||||
f"VM {
|
||||
self.vm_name} is persistent, undefining with NVRAM for reconfiguration..."
|
||||
)
|
||||
subprocess.run(
|
||||
f"ssh {self.unraid_user}@{self.unraid_host} 'virsh undefine {self.vm_name} --nvram'",
|
||||
f"ssh {
|
||||
self.unraid_user}@{
|
||||
self.unraid_host} 'virsh undefine {
|
||||
self.vm_name} --nvram'",
|
||||
shell=True,
|
||||
check=True,
|
||||
)
|
||||
logger.info(f"Persistent VM {self.vm_name} undefined for reconfiguration")
|
||||
logger.info(
|
||||
f"Persistent VM {
|
||||
self.vm_name} undefined for reconfiguration"
|
||||
)
|
||||
else:
|
||||
# Handle transient VM - just destroy it
|
||||
logger.info(f"VM {self.vm_name} is transient, destroying for reconfiguration...")
|
||||
logger.info(
|
||||
f"VM {
|
||||
self.vm_name} is transient, destroying for reconfiguration..."
|
||||
)
|
||||
if self.vm_status() == "running":
|
||||
subprocess.run(
|
||||
f"ssh {self.unraid_user}@{self.unraid_host} 'virsh destroy {self.vm_name}'",
|
||||
shell=True,
|
||||
check=True,
|
||||
)
|
||||
logger.info(f"Transient VM {self.vm_name} destroyed for reconfiguration")
|
||||
logger.info(
|
||||
f"Transient VM {
|
||||
self.vm_name} destroyed for reconfiguration"
|
||||
)
|
||||
|
||||
# Step 2: Generate VM XML with appropriate UUID
|
||||
vm_xml = self.create_vm_xml(vm_memory, vm_vcpus, vm_ip, existing_uuid)
|
||||
xml_file = f"/tmp/{self.vm_name}.xml"
|
||||
|
||||
with open(xml_file, "w", encoding='utf-8') as f:
|
||||
with open(xml_file, "w", encoding="utf-8") as f:
|
||||
f.write(vm_xml)
|
||||
|
||||
# Step 3: Copy XML to Unraid and define VM
|
||||
@@ -188,13 +217,19 @@ class UnraidTemplateVMManager:
|
||||
|
||||
# Ensure VM is set to autostart for persistent configuration
|
||||
subprocess.run(
|
||||
f"ssh {self.unraid_user}@{self.unraid_host} 'virsh autostart {self.vm_name}'",
|
||||
f"ssh {
|
||||
self.unraid_user}@{
|
||||
self.unraid_host} 'virsh autostart {
|
||||
self.vm_name}'",
|
||||
shell=True,
|
||||
check=False, # Don't fail if autostart is already enabled
|
||||
)
|
||||
|
||||
action = "updated" if vm_exists else "created"
|
||||
logger.info(f"VM {self.vm_name} {action} successfully from template")
|
||||
logger.info(
|
||||
f"VM {
|
||||
self.vm_name} {action} successfully from template"
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
os.remove(xml_file)
|
||||
@@ -224,7 +259,9 @@ class UnraidTemplateVMManager:
|
||||
# Copy template to create NVRAM file
|
||||
logger.info(f"Creating NVRAM file: {nvram_path}")
|
||||
result = subprocess.run(
|
||||
f"ssh {self.unraid_user}@{self.unraid_host} 'cp /usr/share/qemu/ovmf-x64/OVMF_VARS-pure-efi.fd {nvram_path}'",
|
||||
f"ssh {
|
||||
self.unraid_user}@{
|
||||
self.unraid_host} 'cp /usr/share/qemu/ovmf-x64/OVMF_VARS-pure-efi.fd {nvram_path}'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
@@ -259,7 +296,10 @@ class UnraidTemplateVMManager:
|
||||
return False
|
||||
|
||||
# Get VM UUID from XML
|
||||
cmd = f"ssh {self.unraid_user}@{self.unraid_host} 'virsh dumpxml {self.vm_name} | grep \"<uuid>\" | sed \"s/<uuid>//g\" | sed \"s/<\\/uuid>//g\" | tr -d \" \"'"
|
||||
cmd = f'ssh {
|
||||
self.unraid_user}@{
|
||||
self.unraid_host} \'virsh dumpxml {
|
||||
self.vm_name} | grep "<uuid>" | sed "s/<uuid>//g" | sed "s/<\\/uuid>//g" | tr -d " "\''
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
shell=True,
|
||||
@@ -284,7 +324,9 @@ class UnraidTemplateVMManager:
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info(f"VM {self.vm_name} started successfully")
|
||||
logger.info("VM is booting from template disk - should be ready quickly!")
|
||||
logger.info(
|
||||
"VM is booting from template disk - should be ready quickly!"
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Failed to start VM: {result.stderr}")
|
||||
@@ -305,37 +347,49 @@ class UnraidTemplateVMManager:
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
# Wait up to 30 seconds for graceful shutdown
|
||||
logger.info(f"Waiting for VM {self.vm_name} to shutdown gracefully...")
|
||||
logger.info(
|
||||
f"Waiting for VM {
|
||||
self.vm_name} to shutdown gracefully..."
|
||||
)
|
||||
for i in range(30):
|
||||
status = self.vm_status()
|
||||
if status in ["shut off", "unknown"]:
|
||||
logger.info(f"VM {self.vm_name} stopped gracefully")
|
||||
return True
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
# If still running after 30 seconds, force destroy
|
||||
logger.warning(f"VM {self.vm_name} didn't shutdown gracefully, forcing destroy...")
|
||||
logger.warning(
|
||||
f"VM {
|
||||
self.vm_name} didn't shutdown gracefully, forcing destroy..."
|
||||
)
|
||||
destroy_result = subprocess.run(
|
||||
f"ssh {self.unraid_user}@{self.unraid_host} 'virsh destroy {self.vm_name}'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
|
||||
if destroy_result.returncode == 0:
|
||||
logger.info(f"VM {self.vm_name} forcefully destroyed")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Failed to destroy VM: {destroy_result.stderr}")
|
||||
logger.error(
|
||||
f"Failed to destroy VM: {
|
||||
destroy_result.stderr}"
|
||||
)
|
||||
return False
|
||||
else:
|
||||
logger.error(f"Failed to initiate VM shutdown: {result.stderr}")
|
||||
logger.error(
|
||||
f"Failed to initiate VM shutdown: {
|
||||
result.stderr}"
|
||||
)
|
||||
return False
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
@@ -350,94 +404,121 @@ class UnraidTemplateVMManager:
|
||||
try:
|
||||
# Method 1: Try guest agent first (most reliable for template VMs)
|
||||
logger.info("Trying guest agent for IP detection...")
|
||||
ssh_cmd = f"ssh -o StrictHostKeyChecking=no {self.unraid_user}@{self.unraid_host} 'virsh guestinfo {self.vm_name} --interface 2>/dev/null || echo FAILED'"
|
||||
ssh_cmd = f"ssh -o StrictHostKeyChecking=no {
|
||||
self.unraid_user}@{
|
||||
self.unraid_host} 'virsh guestinfo {
|
||||
self.vm_name} --interface 2>/dev/null || echo FAILED'"
|
||||
logger.info(f"Running SSH command: {ssh_cmd}")
|
||||
result = subprocess.run(
|
||||
ssh_cmd,
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10
|
||||
ssh_cmd, shell=True, capture_output=True, text=True, timeout=10
|
||||
)
|
||||
|
||||
logger.info(f"Guest agent result (returncode={result.returncode}): {result.stdout[:200]}...")
|
||||
|
||||
if result.returncode == 0 and "FAILED" not in result.stdout and "addr" in result.stdout:
|
||||
|
||||
logger.info(
|
||||
f"Guest agent result (returncode={result.returncode}): {result.stdout[:200]}..."
|
||||
)
|
||||
|
||||
if (
|
||||
result.returncode == 0
|
||||
and "FAILED" not in result.stdout
|
||||
and "addr" in result.stdout
|
||||
):
|
||||
# Parse guest agent output for IP addresses
|
||||
lines = result.stdout.strip().split("\n")
|
||||
import re
|
||||
|
||||
for line in lines:
|
||||
logger.info(f"Processing line: {line}")
|
||||
# Look for lines like: if.1.addr.0.addr : 192.168.20.65
|
||||
if ".addr." in line and "addr :" in line and "127.0.0.1" not in line:
|
||||
if (
|
||||
".addr." in line
|
||||
and "addr :" in line
|
||||
and "127.0.0.1" not in line
|
||||
):
|
||||
# Extract IP address from the line
|
||||
ip_match = re.search(r':\s*([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})\s*$', line)
|
||||
ip_match = re.search(
|
||||
r":\s*([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})\s*$",
|
||||
line,
|
||||
)
|
||||
if ip_match:
|
||||
ip = ip_match.group(1)
|
||||
logger.info(f"Found potential IP: {ip}")
|
||||
# Skip localhost and Docker bridge IPs
|
||||
if not ip.startswith('127.') and not ip.startswith('172.'):
|
||||
if not ip.startswith("127.") and not ip.startswith("172."):
|
||||
logger.info(f"Found IP via guest agent: {ip}")
|
||||
return ip
|
||||
|
||||
|
||||
# Method 2: Try domifaddr (network interface detection)
|
||||
logger.info("Trying domifaddr for IP detection...")
|
||||
result = subprocess.run(
|
||||
f"ssh {self.unraid_user}@{self.unraid_host} 'virsh domifaddr {self.vm_name} 2>/dev/null || echo FAILED'",
|
||||
f"ssh {
|
||||
self.unraid_user}@{
|
||||
self.unraid_host} 'virsh domifaddr {
|
||||
self.vm_name} 2>/dev/null || echo FAILED'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0 and "FAILED" not in result.stdout and "ipv4" in result.stdout:
|
||||
|
||||
if (
|
||||
result.returncode == 0
|
||||
and "FAILED" not in result.stdout
|
||||
and "ipv4" in result.stdout
|
||||
):
|
||||
lines = result.stdout.strip().split("\n")
|
||||
for line in lines:
|
||||
if "ipv4" in line:
|
||||
# Extract IP from line like: vnet0 52:54:00:xx:xx:xx ipv4 192.168.1.100/24
|
||||
# Extract IP from line like: vnet0
|
||||
# 52:54:00:xx:xx:xx ipv4 192.168.1.100/24
|
||||
parts = line.split()
|
||||
if len(parts) >= 4:
|
||||
ip_with_mask = parts[3]
|
||||
ip = ip_with_mask.split("/")[0]
|
||||
logger.info(f"Found IP via domifaddr: {ip}")
|
||||
return ip
|
||||
|
||||
# Method 3: Try ARP table lookup (fallback for when guest agent isn't ready)
|
||||
|
||||
# Method 3: Try ARP table lookup (fallback for when guest agent
|
||||
# isn't ready)
|
||||
logger.info("Trying ARP table lookup...")
|
||||
# Get VM MAC address first
|
||||
mac_result = subprocess.run(
|
||||
f"ssh {self.unraid_user}@{self.unraid_host} 'virsh dumpxml {self.vm_name} | grep \"mac address\" | head -1 | sed \"s/.*address=.\\([^'\"]*\\).*/\\1/\"'",
|
||||
f'ssh {
|
||||
self.unraid_user}@{
|
||||
self.unraid_host} \'virsh dumpxml {
|
||||
self.vm_name} | grep "mac address" | head -1 | sed "s/.*address=.\\([^\'"]*\\).*/\\1/"\'',
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
|
||||
if mac_result.returncode == 0 and mac_result.stdout.strip():
|
||||
mac_addr = mac_result.stdout.strip()
|
||||
logger.info(f"VM MAC address: {mac_addr}")
|
||||
|
||||
|
||||
# Look up IP by MAC in ARP table
|
||||
arp_result = subprocess.run(
|
||||
f"ssh {self.unraid_user}@{self.unraid_host} 'arp -a | grep {mac_addr} || echo NOTFOUND'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
|
||||
if arp_result.returncode == 0 and "NOTFOUND" not in arp_result.stdout:
|
||||
# Parse ARP output like: (192.168.1.100) at 52:54:00:xx:xx:xx
|
||||
# Parse ARP output like: (192.168.1.100) at
|
||||
# 52:54:00:xx:xx:xx
|
||||
import re
|
||||
ip_match = re.search(r'\(([0-9.]+)\)', arp_result.stdout)
|
||||
|
||||
ip_match = re.search(r"\(([0-9.]+)\)", arp_result.stdout)
|
||||
if ip_match:
|
||||
ip = ip_match.group(1)
|
||||
logger.info(f"Found IP via ARP lookup: {ip}")
|
||||
return ip
|
||||
|
||||
|
||||
logger.warning("All IP detection methods failed")
|
||||
return None
|
||||
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("Timeout getting VM IP - guest agent may not be ready")
|
||||
return None
|
||||
@@ -467,7 +548,10 @@ class UnraidTemplateVMManager:
|
||||
def delete_vm(self) -> bool:
|
||||
"""Completely remove VM and all associated files."""
|
||||
try:
|
||||
logger.info(f"Deleting VM {self.vm_name} and all associated files...")
|
||||
logger.info(
|
||||
f"Deleting VM {
|
||||
self.vm_name} and all associated files..."
|
||||
)
|
||||
|
||||
# Check if VM exists
|
||||
if not self.check_vm_exists():
|
||||
@@ -483,7 +567,10 @@ class UnraidTemplateVMManager:
|
||||
# Undefine VM with NVRAM
|
||||
logger.info(f"Undefining VM {self.vm_name}...")
|
||||
subprocess.run(
|
||||
f"ssh {self.unraid_user}@{self.unraid_host} 'virsh undefine {self.vm_name} --nvram'",
|
||||
f"ssh {
|
||||
self.unraid_user}@{
|
||||
self.unraid_host} 'virsh undefine {
|
||||
self.vm_name} --nvram'",
|
||||
shell=True,
|
||||
check=True,
|
||||
)
|
||||
@@ -503,11 +590,13 @@ class UnraidTemplateVMManager:
|
||||
logger.error(f"Failed to delete VM: {e}")
|
||||
return False
|
||||
|
||||
def customize_vm_for_thrillwiki(self, repo_url: str, github_token: str = "") -> bool:
|
||||
def customize_vm_for_thrillwiki(
|
||||
self, repo_url: str, github_token: str = ""
|
||||
) -> bool:
|
||||
"""Customize the VM for ThrillWiki after it boots."""
|
||||
try:
|
||||
logger.info("Waiting for VM to be accessible via SSH...")
|
||||
|
||||
|
||||
# Wait for VM to get an IP and be SSH accessible
|
||||
vm_ip = None
|
||||
max_attempts = 20
|
||||
@@ -524,36 +613,42 @@ class UnraidTemplateVMManager:
|
||||
if ssh_test.returncode == 0:
|
||||
logger.info(f"VM is SSH accessible at {vm_ip}")
|
||||
break
|
||||
|
||||
logger.info(f"Waiting for SSH access... (attempt {attempt + 1}/{max_attempts})")
|
||||
|
||||
logger.info(
|
||||
f"Waiting for SSH access... (attempt {
|
||||
attempt + 1}/{max_attempts})"
|
||||
)
|
||||
time.sleep(15)
|
||||
|
||||
|
||||
if not vm_ip:
|
||||
logger.error("VM failed to become SSH accessible")
|
||||
return False
|
||||
|
||||
|
||||
# Run ThrillWiki deployment on the VM
|
||||
logger.info("Running ThrillWiki deployment on VM...")
|
||||
|
||||
|
||||
deploy_cmd = f"cd /home/thrillwiki && /home/thrillwiki/deploy-thrillwiki.sh '{repo_url}'"
|
||||
if github_token:
|
||||
deploy_cmd = f"cd /home/thrillwiki && GITHUB_TOKEN='{github_token}' /home/thrillwiki/deploy-thrillwiki.sh '{repo_url}'"
|
||||
|
||||
|
||||
deploy_result = subprocess.run(
|
||||
f"ssh -o StrictHostKeyChecking=no thrillwiki@{vm_ip} '{deploy_cmd}'",
|
||||
shell=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
if deploy_result.returncode == 0:
|
||||
logger.info("ThrillWiki deployment completed successfully!")
|
||||
logger.info(f"ThrillWiki should be accessible at http://{vm_ip}:8000")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"ThrillWiki deployment failed: {deploy_result.stderr}")
|
||||
logger.error(
|
||||
f"ThrillWiki deployment failed: {
|
||||
deploy_result.stderr}"
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error customizing VM: {e}")
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user