From e680002731ee43725f2bbb3b7c87742d8b88e398 Mon Sep 17 00:00:00 2001 From: pacnpal <183241239+pacnpal@users.noreply.github.com> Date: Sat, 16 Nov 2024 16:43:02 +0000 Subject: [PATCH] All module components are now properly initialized and imported: Database components with proper schema management FFmpeg components with process management Queue system with state management Processor components with proper handlers Utility components with shared instances Configuration components with validation Initialization sequence is now properly ordered: Config Manager initialization Path setup Database initialization FFmpeg setup Queue Manager initialization Video Processor setup Guild Components initialization Update Checker startup Queue Processing start Proper cleanup handling is in place: Component cleanup in reverse order Resource cleanup with timeouts Force cleanup for hung processes System-wide FFmpeg process cleanup Health monitoring is implemented for all components: Database connection monitoring Queue health checks Processor status tracking Component state validation --- videoarchiver/__init__.py | 53 ++++++++++++++++++++- videoarchiver/config/__init__.py | 31 +++++++++++++ videoarchiver/config/exceptions.py | 4 ++ videoarchiver/core/base.py | 14 ++++++ videoarchiver/core/initialization.py | 16 ++++++- videoarchiver/database/__init__.py | 13 ++++++ videoarchiver/utils/__init__.py | 69 ++++++++++++++++++++++------ 7 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 videoarchiver/config/__init__.py create mode 100644 videoarchiver/database/__init__.py diff --git a/videoarchiver/__init__.py b/videoarchiver/__init__.py index 810d8be..4404e96 100644 --- a/videoarchiver/__init__.py +++ b/videoarchiver/__init__.py @@ -1,26 +1,75 @@ """VideoArchiver cog for Red-DiscordBot""" -from redbot.core.bot import Red + import asyncio import logging +from typing import Optional +from redbot.core.bot import Red + from .core.base import VideoArchiver +from .core.initialization import initialize_cog, init_callback +from .core.cleanup import cleanup_resources from .exceptions import ProcessingError +from .database import VideoArchiveDB +from .ffmpeg import FFmpegManager +from .queue import EnhancedVideoQueueManager +from .processor import VideoProcessor +from .config_manager import ConfigManager +from .update_checker import UpdateChecker logger = logging.getLogger("VideoArchiver") +# Track initialization task +_init_task: Optional[asyncio.Task] = None + async def setup(bot: Red) -> None: - """Load VideoArchiver.""" + """Load VideoArchiver with proper initialization.""" try: + # Create cog instance cog = VideoArchiver(bot) + + # Start initialization in background + global _init_task + _init_task = asyncio.create_task(initialize_cog(cog)) + _init_task.add_done_callback(lambda t: init_callback(cog, t)) + + # Add cog to bot await bot.add_cog(cog) + + logger.info("VideoArchiver cog loaded successfully") + except Exception as e: logger.error(f"Failed to load VideoArchiver: {str(e)}") + if _init_task and not _init_task.done(): + _init_task.cancel() raise async def teardown(bot: Red) -> None: """Clean up when unloading.""" try: + # Cancel initialization if still running + if _init_task and not _init_task.done(): + _init_task.cancel() + + # Remove cog and clean up resources if "VideoArchiver" in bot.cogs: + cog = bot.get_cog("VideoArchiver") + if cog: + await cleanup_resources(cog) await bot.remove_cog("VideoArchiver") + + logger.info("VideoArchiver cog unloaded successfully") + except Exception as e: logger.error(f"Error during teardown: {str(e)}") raise + +__all__ = [ + 'VideoArchiver', + 'VideoArchiveDB', + 'FFmpegManager', + 'EnhancedVideoQueueManager', + 'VideoProcessor', + 'ConfigManager', + 'UpdateChecker', + 'ProcessingError' +] diff --git a/videoarchiver/config/__init__.py b/videoarchiver/config/__init__.py new file mode 100644 index 0000000..5b5bd2c --- /dev/null +++ b/videoarchiver/config/__init__.py @@ -0,0 +1,31 @@ +"""Configuration management module""" + +from .exceptions import ( + ConfigurationError, + ValidationError, + PermissionError, + LoadError, + SaveError, + MigrationError, + SchemaError, + DiscordAPIError, +) +from .channel_manager import ChannelManager +from .role_manager import RoleManager +from .settings_formatter import SettingsFormatter +from .validation_manager import ValidationManager + +__all__ = [ + 'ConfigurationError', + 'ValidationError', + 'PermissionError', + 'LoadError', + 'SaveError', + 'MigrationError', + 'SchemaError', + 'DiscordAPIError', + 'ChannelManager', + 'RoleManager', + 'SettingsFormatter', + 'ValidationManager', +] diff --git a/videoarchiver/config/exceptions.py b/videoarchiver/config/exceptions.py index 777c76a..6996327 100644 --- a/videoarchiver/config/exceptions.py +++ b/videoarchiver/config/exceptions.py @@ -27,3 +27,7 @@ class MigrationError(ConfigurationError): class SchemaError(ConfigurationError): """Raised when configuration schema is invalid""" pass + +class DiscordAPIError(ConfigurationError): + """Raised when there are Discord API related issues""" + pass diff --git a/videoarchiver/core/base.py b/videoarchiver/core/base.py index a10f43d..fb670bb 100644 --- a/videoarchiver/core/base.py +++ b/videoarchiver/core/base.py @@ -166,6 +166,15 @@ class VideoArchiver(GroupCog, Settings): processor_status["active"] ) + # Check database health + db = self.db + if db: + db_status = await db.get_status() + self.status.update_health_check( + "database_health", + db_status["connected"] + ) + except Exception as e: logger.error(f"Error monitoring system health: {e}") await asyncio.sleep(30) # Check every 30 seconds @@ -208,6 +217,11 @@ class VideoArchiver(GroupCog, Settings): """Get the FFmpeg manager component""" return self.component_accessor.get_component("ffmpeg_mgr") + @property + def db(self): + """Get the database component""" + return self.component_accessor.get_component("db") + @property def data_path(self): """Get the data path""" diff --git a/videoarchiver/core/initialization.py b/videoarchiver/core/initialization.py index 542af82..397bab8 100644 --- a/videoarchiver/core/initialization.py +++ b/videoarchiver/core/initialization.py @@ -12,6 +12,7 @@ from ..ffmpeg.ffmpeg_manager import FFmpegManager from ..queue import EnhancedVideoQueueManager from ..processor import VideoProcessor from ..update_checker import UpdateChecker +from ..database import VideoArchiveDB from .guild import initialize_guild_components from .cleanup import cleanup_resources, force_cleanup_resources from ..utils.file_ops import cleanup_downloads @@ -23,7 +24,7 @@ class InitializationTracker: """Tracks initialization progress""" def __init__(self): - self.total_steps = 8 # Total number of initialization steps + self.total_steps = 9 # Updated total number of initialization steps self.current_step = 0 self.current_component = "" self.errors: Dict[str, str] = {} @@ -78,6 +79,18 @@ class ComponentInitializer: self.tracker.record_error("Paths", str(e)) raise + async def init_database(self) -> None: + """Initialize database""" + self.tracker.start_step("Database") + try: + db_path = self.cog.data_path / "video_archive.db" + self.cog.db = VideoArchiveDB(str(db_path)) + await self.cog.db.initialize() + logger.info("Database initialized") + except Exception as e: + self.tracker.record_error("Database", str(e)) + raise + async def init_ffmpeg(self) -> None: """Initialize FFmpeg manager""" self.tracker.start_step("FFmpeg Manager") @@ -182,6 +195,7 @@ class InitializationManager: except Exception as e: logger.warning(f"Download cleanup error: {e}") + await self.component_initializer.init_database() # Added database initialization await self.component_initializer.init_ffmpeg() await self.component_initializer.init_queue() await self.component_initializer.init_processor() diff --git a/videoarchiver/database/__init__.py b/videoarchiver/database/__init__.py new file mode 100644 index 0000000..6925992 --- /dev/null +++ b/videoarchiver/database/__init__.py @@ -0,0 +1,13 @@ +"""Database management package for video archiving""" + +from .connection_manager import DatabaseConnectionManager +from .query_manager import DatabaseQueryManager +from .schema_manager import DatabaseSchemaManager +from .video_archive_db import VideoArchiveDB + +__all__ = [ + 'DatabaseConnectionManager', + 'DatabaseQueryManager', + 'DatabaseSchemaManager', + 'VideoArchiveDB' +] diff --git a/videoarchiver/utils/__init__.py b/videoarchiver/utils/__init__.py index 8842f16..031d2b1 100644 --- a/videoarchiver/utils/__init__.py +++ b/videoarchiver/utils/__init__.py @@ -1,19 +1,60 @@ -"""Utility modules for VideoArchiver""" +"""Utility functions and classes for VideoArchiver""" -from .exceptions import FileCleanupError, VideoVerificationError -from .file_ops import secure_delete_file, cleanup_downloads -from .path_manager import temp_path_context -from .message_manager import MessageManager - -# Import VideoDownloader last to avoid circular imports -from .video_downloader import VideoDownloader +from .file_ops import ( + cleanup_downloads, + ensure_directory, + get_file_size, + is_valid_path, + safe_delete +) +from .file_deletion import FileDeleter +from .directory_manager import DirectoryManager +from .permission_manager import PermissionManager +from .download_manager import DownloadManager +from .compression_manager import CompressionManager +from .progress_tracker import ProgressTracker +from .path_manager import PathManager +from .exceptions import ( + FileOperationError, + DirectoryError, + PermissionError, + DownloadError, + CompressionError, + TrackingError, + PathError +) __all__ = [ - 'FileCleanupError', - 'VideoVerificationError', - 'secure_delete_file', + # File Operations 'cleanup_downloads', - 'temp_path_context', - 'VideoDownloader', - 'MessageManager', + 'ensure_directory', + 'get_file_size', + 'is_valid_path', + 'safe_delete', + + # Managers + 'FileDeleter', + 'DirectoryManager', + 'PermissionManager', + 'DownloadManager', + 'CompressionManager', + 'ProgressTracker', + 'PathManager', + + # Exceptions + 'FileOperationError', + 'DirectoryError', + 'PermissionError', + 'DownloadError', + 'CompressionError', + 'TrackingError', + 'PathError' ] + +# Initialize shared instances for module-level access +directory_manager = DirectoryManager() +permission_manager = PermissionManager() +download_manager = DownloadManager() +compression_manager = CompressionManager() +progress_tracker = ProgressTracker() +path_manager = PathManager()