All module components are now properly initialized and imported:

Database components with proper schema management
FFmpeg components with process management
Queue system with state management
Processor components with proper handlers
Utility components with shared instances
Configuration components with validation
Initialization sequence is now properly ordered:

Config Manager initialization
Path setup
Database initialization
FFmpeg setup
Queue Manager initialization
Video Processor setup
Guild Components initialization
Update Checker startup
Queue Processing start
Proper cleanup handling is in place:

Component cleanup in reverse order
Resource cleanup with timeouts
Force cleanup for hung processes
System-wide FFmpeg process cleanup
Health monitoring is implemented for all components:

Database connection monitoring
Queue health checks
Processor status tracking
Component state validation
This commit is contained in:
pacnpal
2024-11-16 16:43:02 +00:00
parent 871eccd978
commit e680002731
7 changed files with 183 additions and 17 deletions

View File

@@ -1,26 +1,75 @@
"""VideoArchiver cog for Red-DiscordBot""" """VideoArchiver cog for Red-DiscordBot"""
from redbot.core.bot import Red
import asyncio import asyncio
import logging import logging
from typing import Optional
from redbot.core.bot import Red
from .core.base import VideoArchiver from .core.base import VideoArchiver
from .core.initialization import initialize_cog, init_callback
from .core.cleanup import cleanup_resources
from .exceptions import ProcessingError from .exceptions import ProcessingError
from .database import VideoArchiveDB
from .ffmpeg import FFmpegManager
from .queue import EnhancedVideoQueueManager
from .processor import VideoProcessor
from .config_manager import ConfigManager
from .update_checker import UpdateChecker
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
# Track initialization task
_init_task: Optional[asyncio.Task] = None
async def setup(bot: Red) -> None: async def setup(bot: Red) -> None:
"""Load VideoArchiver.""" """Load VideoArchiver with proper initialization."""
try: try:
# Create cog instance
cog = VideoArchiver(bot) cog = VideoArchiver(bot)
# Start initialization in background
global _init_task
_init_task = asyncio.create_task(initialize_cog(cog))
_init_task.add_done_callback(lambda t: init_callback(cog, t))
# Add cog to bot
await bot.add_cog(cog) await bot.add_cog(cog)
logger.info("VideoArchiver cog loaded successfully")
except Exception as e: except Exception as e:
logger.error(f"Failed to load VideoArchiver: {str(e)}") logger.error(f"Failed to load VideoArchiver: {str(e)}")
if _init_task and not _init_task.done():
_init_task.cancel()
raise raise
async def teardown(bot: Red) -> None: async def teardown(bot: Red) -> None:
"""Clean up when unloading.""" """Clean up when unloading."""
try: try:
# Cancel initialization if still running
if _init_task and not _init_task.done():
_init_task.cancel()
# Remove cog and clean up resources
if "VideoArchiver" in bot.cogs: if "VideoArchiver" in bot.cogs:
cog = bot.get_cog("VideoArchiver")
if cog:
await cleanup_resources(cog)
await bot.remove_cog("VideoArchiver") await bot.remove_cog("VideoArchiver")
logger.info("VideoArchiver cog unloaded successfully")
except Exception as e: except Exception as e:
logger.error(f"Error during teardown: {str(e)}") logger.error(f"Error during teardown: {str(e)}")
raise raise
__all__ = [
'VideoArchiver',
'VideoArchiveDB',
'FFmpegManager',
'EnhancedVideoQueueManager',
'VideoProcessor',
'ConfigManager',
'UpdateChecker',
'ProcessingError'
]

View File

@@ -0,0 +1,31 @@
"""Configuration management module"""
from .exceptions import (
ConfigurationError,
ValidationError,
PermissionError,
LoadError,
SaveError,
MigrationError,
SchemaError,
DiscordAPIError,
)
from .channel_manager import ChannelManager
from .role_manager import RoleManager
from .settings_formatter import SettingsFormatter
from .validation_manager import ValidationManager
__all__ = [
'ConfigurationError',
'ValidationError',
'PermissionError',
'LoadError',
'SaveError',
'MigrationError',
'SchemaError',
'DiscordAPIError',
'ChannelManager',
'RoleManager',
'SettingsFormatter',
'ValidationManager',
]

View File

@@ -27,3 +27,7 @@ class MigrationError(ConfigurationError):
class SchemaError(ConfigurationError): class SchemaError(ConfigurationError):
"""Raised when configuration schema is invalid""" """Raised when configuration schema is invalid"""
pass pass
class DiscordAPIError(ConfigurationError):
"""Raised when there are Discord API related issues"""
pass

View File

@@ -166,6 +166,15 @@ class VideoArchiver(GroupCog, Settings):
processor_status["active"] processor_status["active"]
) )
# Check database health
db = self.db
if db:
db_status = await db.get_status()
self.status.update_health_check(
"database_health",
db_status["connected"]
)
except Exception as e: except Exception as e:
logger.error(f"Error monitoring system health: {e}") logger.error(f"Error monitoring system health: {e}")
await asyncio.sleep(30) # Check every 30 seconds await asyncio.sleep(30) # Check every 30 seconds
@@ -208,6 +217,11 @@ class VideoArchiver(GroupCog, Settings):
"""Get the FFmpeg manager component""" """Get the FFmpeg manager component"""
return self.component_accessor.get_component("ffmpeg_mgr") return self.component_accessor.get_component("ffmpeg_mgr")
@property
def db(self):
"""Get the database component"""
return self.component_accessor.get_component("db")
@property @property
def data_path(self): def data_path(self):
"""Get the data path""" """Get the data path"""

View File

@@ -12,6 +12,7 @@ from ..ffmpeg.ffmpeg_manager import FFmpegManager
from ..queue import EnhancedVideoQueueManager from ..queue import EnhancedVideoQueueManager
from ..processor import VideoProcessor from ..processor import VideoProcessor
from ..update_checker import UpdateChecker from ..update_checker import UpdateChecker
from ..database import VideoArchiveDB
from .guild import initialize_guild_components from .guild import initialize_guild_components
from .cleanup import cleanup_resources, force_cleanup_resources from .cleanup import cleanup_resources, force_cleanup_resources
from ..utils.file_ops import cleanup_downloads from ..utils.file_ops import cleanup_downloads
@@ -23,7 +24,7 @@ class InitializationTracker:
"""Tracks initialization progress""" """Tracks initialization progress"""
def __init__(self): def __init__(self):
self.total_steps = 8 # Total number of initialization steps self.total_steps = 9 # Updated total number of initialization steps
self.current_step = 0 self.current_step = 0
self.current_component = "" self.current_component = ""
self.errors: Dict[str, str] = {} self.errors: Dict[str, str] = {}
@@ -78,6 +79,18 @@ class ComponentInitializer:
self.tracker.record_error("Paths", str(e)) self.tracker.record_error("Paths", str(e))
raise raise
async def init_database(self) -> None:
"""Initialize database"""
self.tracker.start_step("Database")
try:
db_path = self.cog.data_path / "video_archive.db"
self.cog.db = VideoArchiveDB(str(db_path))
await self.cog.db.initialize()
logger.info("Database initialized")
except Exception as e:
self.tracker.record_error("Database", str(e))
raise
async def init_ffmpeg(self) -> None: async def init_ffmpeg(self) -> None:
"""Initialize FFmpeg manager""" """Initialize FFmpeg manager"""
self.tracker.start_step("FFmpeg Manager") self.tracker.start_step("FFmpeg Manager")
@@ -182,6 +195,7 @@ class InitializationManager:
except Exception as e: except Exception as e:
logger.warning(f"Download cleanup error: {e}") logger.warning(f"Download cleanup error: {e}")
await self.component_initializer.init_database() # Added database initialization
await self.component_initializer.init_ffmpeg() await self.component_initializer.init_ffmpeg()
await self.component_initializer.init_queue() await self.component_initializer.init_queue()
await self.component_initializer.init_processor() await self.component_initializer.init_processor()

View File

@@ -0,0 +1,13 @@
"""Database management package for video archiving"""
from .connection_manager import DatabaseConnectionManager
from .query_manager import DatabaseQueryManager
from .schema_manager import DatabaseSchemaManager
from .video_archive_db import VideoArchiveDB
__all__ = [
'DatabaseConnectionManager',
'DatabaseQueryManager',
'DatabaseSchemaManager',
'VideoArchiveDB'
]

View File

@@ -1,19 +1,60 @@
"""Utility modules for VideoArchiver""" """Utility functions and classes for VideoArchiver"""
from .exceptions import FileCleanupError, VideoVerificationError from .file_ops import (
from .file_ops import secure_delete_file, cleanup_downloads cleanup_downloads,
from .path_manager import temp_path_context ensure_directory,
from .message_manager import MessageManager get_file_size,
is_valid_path,
# Import VideoDownloader last to avoid circular imports safe_delete
from .video_downloader import VideoDownloader )
from .file_deletion import FileDeleter
from .directory_manager import DirectoryManager
from .permission_manager import PermissionManager
from .download_manager import DownloadManager
from .compression_manager import CompressionManager
from .progress_tracker import ProgressTracker
from .path_manager import PathManager
from .exceptions import (
FileOperationError,
DirectoryError,
PermissionError,
DownloadError,
CompressionError,
TrackingError,
PathError
)
__all__ = [ __all__ = [
'FileCleanupError', # File Operations
'VideoVerificationError',
'secure_delete_file',
'cleanup_downloads', 'cleanup_downloads',
'temp_path_context', 'ensure_directory',
'VideoDownloader', 'get_file_size',
'MessageManager', 'is_valid_path',
'safe_delete',
# Managers
'FileDeleter',
'DirectoryManager',
'PermissionManager',
'DownloadManager',
'CompressionManager',
'ProgressTracker',
'PathManager',
# Exceptions
'FileOperationError',
'DirectoryError',
'PermissionError',
'DownloadError',
'CompressionError',
'TrackingError',
'PathError'
] ]
# Initialize shared instances for module-level access
directory_manager = DirectoryManager()
permission_manager = PermissionManager()
download_manager = DownloadManager()
compression_manager = CompressionManager()
progress_tracker = ProgressTracker()
path_manager = PathManager()