mirror of
https://github.com/pacnpal/Pac-cogs.git
synced 2025-12-20 02:41:06 -05:00
Core Systems:
Component-based architecture with lifecycle management Enhanced error handling and recovery mechanisms Comprehensive state management and tracking Event-driven architecture with monitoring Queue Management: Multiple processing strategies for different scenarios Advanced state management with recovery Comprehensive metrics and health monitoring Sophisticated cleanup system with multiple strategies Processing Pipeline: Enhanced message handling with validation Improved URL extraction and processing Better queue management and monitoring Advanced cleanup mechanisms Overall Benefits: Better code organization and maintainability Improved error handling and recovery Enhanced monitoring and reporting More robust and reliable system
This commit is contained in:
pacnpal
@@ -2,19 +2,151 @@
|
||||
|
||||
import logging
|
||||
import asyncio
|
||||
from enum import Enum
|
||||
from typing import Optional, Tuple, Dict, Any
|
||||
from datetime import datetime
|
||||
import discord
|
||||
from discord.ext import commands
|
||||
from discord import app_commands
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
|
||||
from .message_handler import MessageHandler
|
||||
from .queue_handler import QueueHandler
|
||||
from .progress_tracker import ProgressTracker
|
||||
from .status_display import StatusDisplay
|
||||
from .cleanup_manager import CleanupManager
|
||||
from .reactions import REACTIONS
|
||||
|
||||
logger = logging.getLogger("VideoArchiver")
|
||||
|
||||
class ProcessorState(Enum):
|
||||
"""Possible states of the video processor"""
|
||||
INITIALIZING = "initializing"
|
||||
READY = "ready"
|
||||
PROCESSING = "processing"
|
||||
PAUSED = "paused"
|
||||
ERROR = "error"
|
||||
SHUTDOWN = "shutdown"
|
||||
|
||||
class OperationType(Enum):
|
||||
"""Types of processor operations"""
|
||||
MESSAGE_PROCESSING = "message_processing"
|
||||
VIDEO_PROCESSING = "video_processing"
|
||||
QUEUE_MANAGEMENT = "queue_management"
|
||||
CLEANUP = "cleanup"
|
||||
|
||||
class OperationTracker:
|
||||
"""Tracks processor operations"""
|
||||
|
||||
def __init__(self):
|
||||
self.operations: Dict[str, Dict[str, Any]] = {}
|
||||
self.operation_history: List[Dict[str, Any]] = []
|
||||
self.error_count = 0
|
||||
self.success_count = 0
|
||||
|
||||
def start_operation(
|
||||
self,
|
||||
op_type: OperationType,
|
||||
details: Dict[str, Any]
|
||||
) -> str:
|
||||
"""Start tracking an operation"""
|
||||
op_id = f"{op_type.value}_{datetime.utcnow().timestamp()}"
|
||||
self.operations[op_id] = {
|
||||
"type": op_type.value,
|
||||
"start_time": datetime.utcnow(),
|
||||
"status": "running",
|
||||
"details": details
|
||||
}
|
||||
return op_id
|
||||
|
||||
def end_operation(
|
||||
self,
|
||||
op_id: str,
|
||||
success: bool,
|
||||
error: Optional[str] = None
|
||||
) -> None:
|
||||
"""End tracking an operation"""
|
||||
if op_id in self.operations:
|
||||
self.operations[op_id].update({
|
||||
"end_time": datetime.utcnow(),
|
||||
"status": "success" if success else "error",
|
||||
"error": error
|
||||
})
|
||||
# Move to history
|
||||
self.operation_history.append(self.operations.pop(op_id))
|
||||
# Update counts
|
||||
if success:
|
||||
self.success_count += 1
|
||||
else:
|
||||
self.error_count += 1
|
||||
|
||||
def get_active_operations(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Get currently active operations"""
|
||||
return self.operations.copy()
|
||||
|
||||
def get_operation_stats(self) -> Dict[str, Any]:
|
||||
"""Get operation statistics"""
|
||||
return {
|
||||
"total_operations": len(self.operation_history) + len(self.operations),
|
||||
"active_operations": len(self.operations),
|
||||
"success_count": self.success_count,
|
||||
"error_count": self.error_count,
|
||||
"success_rate": (
|
||||
self.success_count / (self.success_count + self.error_count)
|
||||
if (self.success_count + self.error_count) > 0
|
||||
else 0
|
||||
)
|
||||
}
|
||||
|
||||
class HealthMonitor:
|
||||
"""Monitors processor health"""
|
||||
|
||||
def __init__(self, processor: 'VideoProcessor'):
|
||||
self.processor = processor
|
||||
self.last_check: Optional[datetime] = None
|
||||
self.health_status: Dict[str, bool] = {}
|
||||
self._monitor_task: Optional[asyncio.Task] = None
|
||||
|
||||
async def start_monitoring(self) -> None:
|
||||
"""Start health monitoring"""
|
||||
self._monitor_task = asyncio.create_task(self._monitor_health())
|
||||
|
||||
async def stop_monitoring(self) -> None:
|
||||
"""Stop health monitoring"""
|
||||
if self._monitor_task:
|
||||
self._monitor_task.cancel()
|
||||
try:
|
||||
await self._monitor_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _monitor_health(self) -> None:
|
||||
"""Monitor processor health"""
|
||||
while True:
|
||||
try:
|
||||
self.last_check = datetime.utcnow()
|
||||
|
||||
# Check component health
|
||||
self.health_status.update({
|
||||
"queue_handler": self.processor.queue_handler.is_healthy(),
|
||||
"message_handler": self.processor.message_handler.is_healthy(),
|
||||
"progress_tracker": self.processor.progress_tracker.is_healthy()
|
||||
})
|
||||
|
||||
# Check operation health
|
||||
op_stats = self.processor.operation_tracker.get_operation_stats()
|
||||
self.health_status["operations"] = (
|
||||
op_stats["success_rate"] >= 0.9 # 90% success rate threshold
|
||||
)
|
||||
|
||||
await asyncio.sleep(60) # Check every minute
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Health monitoring error: {e}")
|
||||
await asyncio.sleep(30) # Shorter interval on error
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
"""Check if processor is healthy"""
|
||||
return all(self.health_status.values())
|
||||
|
||||
class VideoProcessor:
|
||||
"""Handles video processing operations"""
|
||||
|
||||
@@ -34,91 +166,101 @@ class VideoProcessor:
|
||||
self.db = db
|
||||
self.queue_manager = queue_manager
|
||||
|
||||
# Initialize state
|
||||
self.state = ProcessorState.INITIALIZING
|
||||
self.operation_tracker = OperationTracker()
|
||||
self.health_monitor = HealthMonitor(self)
|
||||
|
||||
# Initialize handlers
|
||||
self.queue_handler = QueueHandler(bot, config_manager, components)
|
||||
self.message_handler = MessageHandler(bot, config_manager, queue_manager)
|
||||
self.progress_tracker = ProgressTracker()
|
||||
self.cleanup_manager = CleanupManager(self.queue_handler, ffmpeg_mgr)
|
||||
|
||||
# Pass db to queue handler if it exists
|
||||
if self.db:
|
||||
self.queue_handler.db = self.db
|
||||
|
||||
# Store queue task reference but don't start processing here
|
||||
# Queue processing is managed by VideoArchiver class
|
||||
# Store queue task reference
|
||||
self._queue_task = None
|
||||
|
||||
# Mark as ready
|
||||
self.state = ProcessorState.READY
|
||||
logger.info("VideoProcessor initialized successfully")
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Start processor operations"""
|
||||
await self.health_monitor.start_monitoring()
|
||||
|
||||
async def process_video(self, item) -> Tuple[bool, Optional[str]]:
|
||||
"""Process a video from the queue by delegating to queue handler"""
|
||||
return await self.queue_handler.process_video(item)
|
||||
"""Process a video from the queue"""
|
||||
op_id = self.operation_tracker.start_operation(
|
||||
OperationType.VIDEO_PROCESSING,
|
||||
{"item": str(item)}
|
||||
)
|
||||
|
||||
try:
|
||||
self.state = ProcessorState.PROCESSING
|
||||
result = await self.queue_handler.process_video(item)
|
||||
success = result[0]
|
||||
error = None if success else result[1]
|
||||
self.operation_tracker.end_operation(op_id, success, error)
|
||||
return result
|
||||
except Exception as e:
|
||||
self.operation_tracker.end_operation(op_id, False, str(e))
|
||||
raise
|
||||
finally:
|
||||
self.state = ProcessorState.READY
|
||||
|
||||
async def process_message(self, message: discord.Message) -> None:
|
||||
"""Process a message for video content"""
|
||||
await self.message_handler.process_message(message)
|
||||
|
||||
async def cleanup(self):
|
||||
"""Clean up resources and stop processing"""
|
||||
op_id = self.operation_tracker.start_operation(
|
||||
OperationType.MESSAGE_PROCESSING,
|
||||
{"message_id": message.id}
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info("Starting VideoProcessor cleanup...")
|
||||
|
||||
# Clean up queue handler
|
||||
try:
|
||||
await self.queue_handler.cleanup()
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning up queue handler: {e}")
|
||||
|
||||
# Clean up FFmpeg manager
|
||||
if self.ffmpeg_mgr:
|
||||
try:
|
||||
self.ffmpeg_mgr.kill_all_processes()
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning up FFmpeg manager: {e}")
|
||||
|
||||
# Cancel queue processing task if we have one
|
||||
if self._queue_task and not self._queue_task.done():
|
||||
self._queue_task.cancel()
|
||||
try:
|
||||
await self._queue_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"Error cancelling queue task: {e}")
|
||||
|
||||
logger.info("VideoProcessor cleanup completed successfully")
|
||||
|
||||
await self.message_handler.process_message(message)
|
||||
self.operation_tracker.end_operation(op_id, True)
|
||||
except Exception as e:
|
||||
logger.error(f"Error during VideoProcessor cleanup: {str(e)}", exc_info=True)
|
||||
self.operation_tracker.end_operation(op_id, False, str(e))
|
||||
raise
|
||||
|
||||
async def force_cleanup(self):
|
||||
"""Force cleanup of resources when normal cleanup fails"""
|
||||
async def cleanup(self) -> None:
|
||||
"""Clean up resources and stop processing"""
|
||||
op_id = self.operation_tracker.start_operation(
|
||||
OperationType.CLEANUP,
|
||||
{"type": "normal"}
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info("Starting force cleanup of VideoProcessor...")
|
||||
|
||||
# Force cleanup queue handler
|
||||
try:
|
||||
await self.queue_handler.force_cleanup()
|
||||
except Exception as e:
|
||||
logger.error(f"Error force cleaning queue handler: {e}")
|
||||
|
||||
# Force cleanup FFmpeg
|
||||
if self.ffmpeg_mgr:
|
||||
try:
|
||||
self.ffmpeg_mgr.kill_all_processes()
|
||||
except Exception as e:
|
||||
logger.error(f"Error force cleaning FFmpeg manager: {e}")
|
||||
|
||||
# Force cancel queue task
|
||||
if self._queue_task and not self._queue_task.done():
|
||||
self._queue_task.cancel()
|
||||
|
||||
logger.info("VideoProcessor force cleanup completed")
|
||||
|
||||
self.state = ProcessorState.SHUTDOWN
|
||||
await self.health_monitor.stop_monitoring()
|
||||
await self.cleanup_manager.cleanup()
|
||||
self.operation_tracker.end_operation(op_id, True)
|
||||
except Exception as e:
|
||||
logger.error(f"Error during VideoProcessor force cleanup: {str(e)}", exc_info=True)
|
||||
self.operation_tracker.end_operation(op_id, False, str(e))
|
||||
logger.error(f"Error during cleanup: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
async def show_queue_details(self, ctx: commands.Context):
|
||||
"""Display detailed queue status and progress information"""
|
||||
async def force_cleanup(self) -> None:
|
||||
"""Force cleanup of resources"""
|
||||
op_id = self.operation_tracker.start_operation(
|
||||
OperationType.CLEANUP,
|
||||
{"type": "force"}
|
||||
)
|
||||
|
||||
try:
|
||||
self.state = ProcessorState.SHUTDOWN
|
||||
await self.health_monitor.stop_monitoring()
|
||||
await self.cleanup_manager.force_cleanup()
|
||||
self.operation_tracker.end_operation(op_id, True)
|
||||
except Exception as e:
|
||||
self.operation_tracker.end_operation(op_id, False, str(e))
|
||||
raise
|
||||
|
||||
async def show_queue_details(self, ctx: commands.Context) -> None:
|
||||
"""Display detailed queue status"""
|
||||
try:
|
||||
if not self.queue_manager:
|
||||
await ctx.send("Queue manager is not initialized.")
|
||||
@@ -126,111 +268,37 @@ class VideoProcessor:
|
||||
|
||||
# Get queue status
|
||||
queue_status = self.queue_manager.get_queue_status(ctx.guild.id)
|
||||
|
||||
# Get active operations
|
||||
active_ops = self.operation_tracker.get_active_operations()
|
||||
|
||||
# Create embed for queue overview
|
||||
embed = discord.Embed(
|
||||
title="Queue Status Details",
|
||||
color=discord.Color.blue(),
|
||||
timestamp=datetime.utcnow(),
|
||||
# Create and send status embed
|
||||
embed = await StatusDisplay.create_queue_status_embed(
|
||||
queue_status,
|
||||
active_ops
|
||||
)
|
||||
|
||||
# Queue statistics
|
||||
embed.add_field(
|
||||
name="Queue Statistics",
|
||||
value=f"```\n"
|
||||
f"Pending: {queue_status['pending']}\n"
|
||||
f"Processing: {queue_status['processing']}\n"
|
||||
f"Completed: {queue_status['completed']}\n"
|
||||
f"Failed: {queue_status['failed']}\n"
|
||||
f"Success Rate: {queue_status['metrics']['success_rate']:.1%}\n"
|
||||
f"Avg Processing Time: {queue_status['metrics']['avg_processing_time']:.1f}s\n"
|
||||
f"```",
|
||||
inline=False,
|
||||
)
|
||||
|
||||
# Active operations
|
||||
active_ops = self.progress_tracker.get_active_operations()
|
||||
|
||||
# Active downloads
|
||||
downloads = active_ops['downloads']
|
||||
if downloads:
|
||||
active_downloads = ""
|
||||
for url, progress in downloads.items():
|
||||
active_downloads += (
|
||||
f"URL: {url[:50]}...\n"
|
||||
f"Progress: {progress.get('percent', 0):.1f}%\n"
|
||||
f"Speed: {progress.get('speed', 'N/A')}\n"
|
||||
f"ETA: {progress.get('eta', 'N/A')}\n"
|
||||
f"Size: {progress.get('downloaded_bytes', 0)}/{progress.get('total_bytes', 0)} bytes\n"
|
||||
f"Started: {progress.get('start_time', 'N/A')}\n"
|
||||
f"Retries: {progress.get('retries', 0)}\n"
|
||||
f"-------------------\n"
|
||||
)
|
||||
embed.add_field(
|
||||
name="Active Downloads",
|
||||
value=f"```\n{active_downloads}```",
|
||||
inline=False,
|
||||
)
|
||||
else:
|
||||
embed.add_field(
|
||||
name="Active Downloads",
|
||||
value="```\nNo active downloads```",
|
||||
inline=False,
|
||||
)
|
||||
|
||||
# Active compressions
|
||||
compressions = active_ops['compressions']
|
||||
if compressions:
|
||||
active_compressions = ""
|
||||
for file_id, progress in compressions.items():
|
||||
active_compressions += (
|
||||
f"File: {progress.get('filename', 'Unknown')}\n"
|
||||
f"Progress: {progress.get('percent', 0):.1f}%\n"
|
||||
f"Time Elapsed: {progress.get('elapsed_time', 'N/A')}\n"
|
||||
f"Input Size: {progress.get('input_size', 0)} bytes\n"
|
||||
f"Current Size: {progress.get('current_size', 0)} bytes\n"
|
||||
f"Target Size: {progress.get('target_size', 0)} bytes\n"
|
||||
f"Codec: {progress.get('codec', 'Unknown')}\n"
|
||||
f"Hardware Accel: {progress.get('hardware_accel', False)}\n"
|
||||
f"-------------------\n"
|
||||
)
|
||||
embed.add_field(
|
||||
name="Active Compressions",
|
||||
value=f"```\n{active_compressions}```",
|
||||
inline=False,
|
||||
)
|
||||
else:
|
||||
embed.add_field(
|
||||
name="Active Compressions",
|
||||
value="```\nNo active compressions```",
|
||||
inline=False,
|
||||
)
|
||||
|
||||
# Error statistics
|
||||
if queue_status["metrics"]["errors_by_type"]:
|
||||
error_stats = "\n".join(
|
||||
f"{error_type}: {count}"
|
||||
for error_type, count in queue_status["metrics"]["errors_by_type"].items()
|
||||
)
|
||||
embed.add_field(
|
||||
name="Error Statistics",
|
||||
value=f"```\n{error_stats}```",
|
||||
inline=False,
|
||||
)
|
||||
|
||||
# Hardware acceleration statistics
|
||||
embed.add_field(
|
||||
name="Hardware Statistics",
|
||||
value=f"```\n"
|
||||
f"Hardware Accel Failures: {queue_status['metrics']['hardware_accel_failures']}\n"
|
||||
f"Compression Failures: {queue_status['metrics']['compression_failures']}\n"
|
||||
f"Peak Memory Usage: {queue_status['metrics']['peak_memory_usage']:.1f}MB\n"
|
||||
f"```",
|
||||
inline=False,
|
||||
)
|
||||
|
||||
await ctx.send(embed=embed)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error showing queue details: {str(e)}", exc_info=True)
|
||||
logger.error(f"Error showing queue details: {e}", exc_info=True)
|
||||
await ctx.send(f"Error getting queue details: {str(e)}")
|
||||
|
||||
def set_queue_task(self, task: asyncio.Task) -> None:
|
||||
"""Set the queue processing task"""
|
||||
self._queue_task = task
|
||||
self.cleanup_manager.set_queue_task(task)
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
"""Get processor status"""
|
||||
return {
|
||||
"state": self.state.value,
|
||||
"health": self.health_monitor.is_healthy(),
|
||||
"operations": self.operation_tracker.get_operation_stats(),
|
||||
"active_operations": self.operation_tracker.get_active_operations(),
|
||||
"last_health_check": (
|
||||
self.health_monitor.last_check.isoformat()
|
||||
if self.health_monitor.last_check
|
||||
else None
|
||||
),
|
||||
"health_status": self.health_monitor.health_status
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user