Removed the problematic singleton pattern from queue manager

Added proper activity tracking in the monitoring system
Reduced timeouts and deadlock thresholds
Implemented more aggressive cleanup procedures
Added system-wide FFmpeg process cleanup

videoarchiver/core/cleanup.py

@@ -2,7 +2,10 @@
 
 import logging
 import asyncio
+import signal
+import os
 from typing import TYPE_CHECKING
+from pathlib import Path
 
 from ..utils.file_ops import cleanup_downloads
 
@@ -11,48 +14,61 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger("VideoArchiver")
 
-CLEANUP_TIMEOUT = 15  # seconds
+CLEANUP_TIMEOUT = 5  # Reduced timeout to 5 seconds
+FORCE_CLEANUP_TIMEOUT = 3  # Even shorter timeout for force cleanup
 
 async def cleanup_resources(cog: "VideoArchiver") -> None:
     """Clean up all resources with proper handling"""
     try:
+        logger.info("Starting resource cleanup...")
+
         # Cancel initialization if still running
         if cog._init_task and not cog._init_task.done():
+            logger.info("Cancelling initialization task")
             cog._init_task.cancel()
             try:
                 await asyncio.wait_for(cog._init_task, timeout=CLEANUP_TIMEOUT)
             except (asyncio.TimeoutError, asyncio.CancelledError):
-                pass
+                logger.warning("Initialization task cancellation timed out")
 
         # Stop update checker
-        if hasattr(cog, "update_checker"):
+        if hasattr(cog, "update_checker") and cog.update_checker:
+            logger.info("Stopping update checker")
             try:
                 await asyncio.wait_for(
                     cog.update_checker.stop(), timeout=CLEANUP_TIMEOUT
                 )
             except asyncio.TimeoutError:
-                pass
+                logger.warning("Update checker stop timed out")
+            cog.update_checker = None
 
         # Clean up processor
-        if hasattr(cog, "processor"):
+        if hasattr(cog, "processor") and cog.processor:
+            logger.info("Cleaning up processor")
             try:
                 await asyncio.wait_for(
                     cog.processor.cleanup(), timeout=CLEANUP_TIMEOUT
                 )
             except asyncio.TimeoutError:
+                logger.warning("Processor cleanup timed out, forcing cleanup")
                 await cog.processor.force_cleanup()
+            cog.processor = None
 
         # Clean up queue manager
-        if hasattr(cog, "queue_manager"):
+        if hasattr(cog, "queue_manager") and cog.queue_manager:
+            logger.info("Cleaning up queue manager")
             try:
                 await asyncio.wait_for(
                     cog.queue_manager.cleanup(), timeout=CLEANUP_TIMEOUT
                 )
             except asyncio.TimeoutError:
+                logger.warning("Queue manager cleanup timed out, forcing stop")
                 cog.queue_manager.force_stop()
+            cog.queue_manager = None
 
         # Clean up components for each guild
         if hasattr(cog, "components"):
+            logger.info("Cleaning up guild components")
             for guild_id, components in cog.components.items():
                 try:
                     if "message_manager" in components:
@@ -66,44 +82,106 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
 
             cog.components.clear()
 
+        # Kill any FFmpeg processes
+        if hasattr(cog, "ffmpeg_mgr") and cog.ffmpeg_mgr:
+            logger.info("Killing FFmpeg processes")
+            cog.ffmpeg_mgr.kill_all_processes()
+            cog.ffmpeg_mgr = None
+
         # Clean up download directory
         if hasattr(cog, "download_path") and cog.download_path.exists():
+            logger.info("Cleaning up download directory")
             try:
-                await cleanup_downloads(str(cog.download_path))
-                cog.download_path.rmdir()
+                await asyncio.wait_for(
+                    cleanup_downloads(str(cog.download_path)),
+                    timeout=CLEANUP_TIMEOUT
+                )
+                if cog.download_path.exists():
+                    cog.download_path.rmdir()
             except Exception as e:
                 logger.error(f"Error cleaning up download directory: {str(e)}")
 
+        # Kill any remaining FFmpeg processes system-wide
+        try:
+            if os.name != 'nt':  # Unix-like systems
+                os.system("pkill -9 ffmpeg")
+            else:  # Windows
+                os.system("taskkill /F /IM ffmpeg.exe")
+        except Exception as e:
+            logger.error(f"Error killing FFmpeg processes: {str(e)}")
+
     except Exception as e:
         logger.error(f"Error during cleanup: {str(e)}")
         raise
     finally:
+        logger.info("Clearing ready flag")
         cog.ready.clear()
 
 async def force_cleanup_resources(cog: "VideoArchiver") -> None:
     """Force cleanup of resources when timeout occurs"""
     try:
-        # Cancel all tasks
-        if hasattr(cog, "processor"):
+        logger.info("Starting force cleanup...")
+
+        # Cancel all tasks immediately
+        if hasattr(cog, "processor") and cog.processor:
+            logger.info("Force cleaning processor")
             await cog.processor.force_cleanup()
+            cog.processor = None
 
         # Force stop queue manager
-        if hasattr(cog, "queue_manager"):
+        if hasattr(cog, "queue_manager") and cog.queue_manager:
+            logger.info("Force stopping queue manager")
             cog.queue_manager.force_stop()
+            cog.queue_manager = None
 
-        # Kill any remaining FFmpeg processes
-        if hasattr(cog, "ffmpeg_mgr"):
+        # Kill FFmpeg processes
+        if hasattr(cog, "ffmpeg_mgr") and cog.ffmpeg_mgr:
+            logger.info("Force killing FFmpeg processes")
             cog.ffmpeg_mgr.kill_all_processes()
+            cog.ffmpeg_mgr = None
+
+        # Force kill any remaining FFmpeg processes system-wide
+        try:
+            if os.name != 'nt':  # Unix-like systems
+                os.system("pkill -9 ffmpeg")
+            else:  # Windows
+                os.system("taskkill /F /IM ffmpeg.exe")
+        except Exception as e:
+            logger.error(f"Error force killing FFmpeg processes: {str(e)}")
 
         # Clean up download directory
         if hasattr(cog, "download_path") and cog.download_path.exists():
+            logger.info("Force cleaning download directory")
             try:
-                await cleanup_downloads(str(cog.download_path))
-                cog.download_path.rmdir()
+                await asyncio.wait_for(
+                    cleanup_downloads(str(cog.download_path)),
+                    timeout=FORCE_CLEANUP_TIMEOUT
+                )
+                if cog.download_path.exists():
+                    cog.download_path.rmdir()
             except Exception as e:
                 logger.error(f"Error force cleaning download directory: {str(e)}")
 
+        # Clear all components
+        if hasattr(cog, "components"):
+            logger.info("Force clearing components")
+            cog.components.clear()
+
     except Exception as e:
         logger.error(f"Error during force cleanup: {str(e)}")
     finally:
+        logger.info("Clearing ready flag")
         cog.ready.clear()
+        
+        # Clear all references
+        cog.bot = None
+        cog.processor = None
+        cog.queue_manager = None
+        cog.update_checker = None
+        cog.ffmpeg_mgr = None
+        cog.components = {}
+        cog.db = None
+        cog._init_task = None
+        cog._cleanup_task = None
+        if hasattr(cog, '_queue_task'):
+            cog._queue_task = None