mirror of
https://github.com/pacnpal/Pac-cogs.git
synced 2025-12-20 02:41:06 -05:00
All module components are now properly initialized and imported:
Database components with proper schema management FFmpeg components with process management Queue system with state management Processor components with proper handlers Utility components with shared instances Configuration components with validation Initialization sequence is now properly ordered: Config Manager initialization Path setup Database initialization FFmpeg setup Queue Manager initialization Video Processor setup Guild Components initialization Update Checker startup Queue Processing start Proper cleanup handling is in place: Component cleanup in reverse order Resource cleanup with timeouts Force cleanup for hung processes System-wide FFmpeg process cleanup Health monitoring is implemented for all components: Database connection monitoring Queue health checks Processor status tracking Component state validation
This commit is contained in:
@@ -1,26 +1,75 @@
|
|||||||
"""VideoArchiver cog for Red-DiscordBot"""
|
"""VideoArchiver cog for Red-DiscordBot"""
|
||||||
from redbot.core.bot import Red
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
from redbot.core.bot import Red
|
||||||
|
|
||||||
from .core.base import VideoArchiver
|
from .core.base import VideoArchiver
|
||||||
|
from .core.initialization import initialize_cog, init_callback
|
||||||
|
from .core.cleanup import cleanup_resources
|
||||||
from .exceptions import ProcessingError
|
from .exceptions import ProcessingError
|
||||||
|
from .database import VideoArchiveDB
|
||||||
|
from .ffmpeg import FFmpegManager
|
||||||
|
from .queue import EnhancedVideoQueueManager
|
||||||
|
from .processor import VideoProcessor
|
||||||
|
from .config_manager import ConfigManager
|
||||||
|
from .update_checker import UpdateChecker
|
||||||
|
|
||||||
logger = logging.getLogger("VideoArchiver")
|
logger = logging.getLogger("VideoArchiver")
|
||||||
|
|
||||||
|
# Track initialization task
|
||||||
|
_init_task: Optional[asyncio.Task] = None
|
||||||
|
|
||||||
async def setup(bot: Red) -> None:
|
async def setup(bot: Red) -> None:
|
||||||
"""Load VideoArchiver."""
|
"""Load VideoArchiver with proper initialization."""
|
||||||
try:
|
try:
|
||||||
|
# Create cog instance
|
||||||
cog = VideoArchiver(bot)
|
cog = VideoArchiver(bot)
|
||||||
|
|
||||||
|
# Start initialization in background
|
||||||
|
global _init_task
|
||||||
|
_init_task = asyncio.create_task(initialize_cog(cog))
|
||||||
|
_init_task.add_done_callback(lambda t: init_callback(cog, t))
|
||||||
|
|
||||||
|
# Add cog to bot
|
||||||
await bot.add_cog(cog)
|
await bot.add_cog(cog)
|
||||||
|
|
||||||
|
logger.info("VideoArchiver cog loaded successfully")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Failed to load VideoArchiver: {str(e)}")
|
logger.error(f"Failed to load VideoArchiver: {str(e)}")
|
||||||
|
if _init_task and not _init_task.done():
|
||||||
|
_init_task.cancel()
|
||||||
raise
|
raise
|
||||||
|
|
||||||
async def teardown(bot: Red) -> None:
|
async def teardown(bot: Red) -> None:
|
||||||
"""Clean up when unloading."""
|
"""Clean up when unloading."""
|
||||||
try:
|
try:
|
||||||
|
# Cancel initialization if still running
|
||||||
|
if _init_task and not _init_task.done():
|
||||||
|
_init_task.cancel()
|
||||||
|
|
||||||
|
# Remove cog and clean up resources
|
||||||
if "VideoArchiver" in bot.cogs:
|
if "VideoArchiver" in bot.cogs:
|
||||||
|
cog = bot.get_cog("VideoArchiver")
|
||||||
|
if cog:
|
||||||
|
await cleanup_resources(cog)
|
||||||
await bot.remove_cog("VideoArchiver")
|
await bot.remove_cog("VideoArchiver")
|
||||||
|
|
||||||
|
logger.info("VideoArchiver cog unloaded successfully")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error during teardown: {str(e)}")
|
logger.error(f"Error during teardown: {str(e)}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'VideoArchiver',
|
||||||
|
'VideoArchiveDB',
|
||||||
|
'FFmpegManager',
|
||||||
|
'EnhancedVideoQueueManager',
|
||||||
|
'VideoProcessor',
|
||||||
|
'ConfigManager',
|
||||||
|
'UpdateChecker',
|
||||||
|
'ProcessingError'
|
||||||
|
]
|
||||||
|
|||||||
31
videoarchiver/config/__init__.py
Normal file
31
videoarchiver/config/__init__.py
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
"""Configuration management module"""
|
||||||
|
|
||||||
|
from .exceptions import (
|
||||||
|
ConfigurationError,
|
||||||
|
ValidationError,
|
||||||
|
PermissionError,
|
||||||
|
LoadError,
|
||||||
|
SaveError,
|
||||||
|
MigrationError,
|
||||||
|
SchemaError,
|
||||||
|
DiscordAPIError,
|
||||||
|
)
|
||||||
|
from .channel_manager import ChannelManager
|
||||||
|
from .role_manager import RoleManager
|
||||||
|
from .settings_formatter import SettingsFormatter
|
||||||
|
from .validation_manager import ValidationManager
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'ConfigurationError',
|
||||||
|
'ValidationError',
|
||||||
|
'PermissionError',
|
||||||
|
'LoadError',
|
||||||
|
'SaveError',
|
||||||
|
'MigrationError',
|
||||||
|
'SchemaError',
|
||||||
|
'DiscordAPIError',
|
||||||
|
'ChannelManager',
|
||||||
|
'RoleManager',
|
||||||
|
'SettingsFormatter',
|
||||||
|
'ValidationManager',
|
||||||
|
]
|
||||||
@@ -27,3 +27,7 @@ class MigrationError(ConfigurationError):
|
|||||||
class SchemaError(ConfigurationError):
|
class SchemaError(ConfigurationError):
|
||||||
"""Raised when configuration schema is invalid"""
|
"""Raised when configuration schema is invalid"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
class DiscordAPIError(ConfigurationError):
|
||||||
|
"""Raised when there are Discord API related issues"""
|
||||||
|
pass
|
||||||
|
|||||||
@@ -166,6 +166,15 @@ class VideoArchiver(GroupCog, Settings):
|
|||||||
processor_status["active"]
|
processor_status["active"]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Check database health
|
||||||
|
db = self.db
|
||||||
|
if db:
|
||||||
|
db_status = await db.get_status()
|
||||||
|
self.status.update_health_check(
|
||||||
|
"database_health",
|
||||||
|
db_status["connected"]
|
||||||
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error monitoring system health: {e}")
|
logger.error(f"Error monitoring system health: {e}")
|
||||||
await asyncio.sleep(30) # Check every 30 seconds
|
await asyncio.sleep(30) # Check every 30 seconds
|
||||||
@@ -208,6 +217,11 @@ class VideoArchiver(GroupCog, Settings):
|
|||||||
"""Get the FFmpeg manager component"""
|
"""Get the FFmpeg manager component"""
|
||||||
return self.component_accessor.get_component("ffmpeg_mgr")
|
return self.component_accessor.get_component("ffmpeg_mgr")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def db(self):
|
||||||
|
"""Get the database component"""
|
||||||
|
return self.component_accessor.get_component("db")
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def data_path(self):
|
def data_path(self):
|
||||||
"""Get the data path"""
|
"""Get the data path"""
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from ..ffmpeg.ffmpeg_manager import FFmpegManager
|
|||||||
from ..queue import EnhancedVideoQueueManager
|
from ..queue import EnhancedVideoQueueManager
|
||||||
from ..processor import VideoProcessor
|
from ..processor import VideoProcessor
|
||||||
from ..update_checker import UpdateChecker
|
from ..update_checker import UpdateChecker
|
||||||
|
from ..database import VideoArchiveDB
|
||||||
from .guild import initialize_guild_components
|
from .guild import initialize_guild_components
|
||||||
from .cleanup import cleanup_resources, force_cleanup_resources
|
from .cleanup import cleanup_resources, force_cleanup_resources
|
||||||
from ..utils.file_ops import cleanup_downloads
|
from ..utils.file_ops import cleanup_downloads
|
||||||
@@ -23,7 +24,7 @@ class InitializationTracker:
|
|||||||
"""Tracks initialization progress"""
|
"""Tracks initialization progress"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.total_steps = 8 # Total number of initialization steps
|
self.total_steps = 9 # Updated total number of initialization steps
|
||||||
self.current_step = 0
|
self.current_step = 0
|
||||||
self.current_component = ""
|
self.current_component = ""
|
||||||
self.errors: Dict[str, str] = {}
|
self.errors: Dict[str, str] = {}
|
||||||
@@ -78,6 +79,18 @@ class ComponentInitializer:
|
|||||||
self.tracker.record_error("Paths", str(e))
|
self.tracker.record_error("Paths", str(e))
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
async def init_database(self) -> None:
|
||||||
|
"""Initialize database"""
|
||||||
|
self.tracker.start_step("Database")
|
||||||
|
try:
|
||||||
|
db_path = self.cog.data_path / "video_archive.db"
|
||||||
|
self.cog.db = VideoArchiveDB(str(db_path))
|
||||||
|
await self.cog.db.initialize()
|
||||||
|
logger.info("Database initialized")
|
||||||
|
except Exception as e:
|
||||||
|
self.tracker.record_error("Database", str(e))
|
||||||
|
raise
|
||||||
|
|
||||||
async def init_ffmpeg(self) -> None:
|
async def init_ffmpeg(self) -> None:
|
||||||
"""Initialize FFmpeg manager"""
|
"""Initialize FFmpeg manager"""
|
||||||
self.tracker.start_step("FFmpeg Manager")
|
self.tracker.start_step("FFmpeg Manager")
|
||||||
@@ -182,6 +195,7 @@ class InitializationManager:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Download cleanup error: {e}")
|
logger.warning(f"Download cleanup error: {e}")
|
||||||
|
|
||||||
|
await self.component_initializer.init_database() # Added database initialization
|
||||||
await self.component_initializer.init_ffmpeg()
|
await self.component_initializer.init_ffmpeg()
|
||||||
await self.component_initializer.init_queue()
|
await self.component_initializer.init_queue()
|
||||||
await self.component_initializer.init_processor()
|
await self.component_initializer.init_processor()
|
||||||
|
|||||||
13
videoarchiver/database/__init__.py
Normal file
13
videoarchiver/database/__init__.py
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
"""Database management package for video archiving"""
|
||||||
|
|
||||||
|
from .connection_manager import DatabaseConnectionManager
|
||||||
|
from .query_manager import DatabaseQueryManager
|
||||||
|
from .schema_manager import DatabaseSchemaManager
|
||||||
|
from .video_archive_db import VideoArchiveDB
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'DatabaseConnectionManager',
|
||||||
|
'DatabaseQueryManager',
|
||||||
|
'DatabaseSchemaManager',
|
||||||
|
'VideoArchiveDB'
|
||||||
|
]
|
||||||
@@ -1,19 +1,60 @@
|
|||||||
"""Utility modules for VideoArchiver"""
|
"""Utility functions and classes for VideoArchiver"""
|
||||||
|
|
||||||
from .exceptions import FileCleanupError, VideoVerificationError
|
from .file_ops import (
|
||||||
from .file_ops import secure_delete_file, cleanup_downloads
|
cleanup_downloads,
|
||||||
from .path_manager import temp_path_context
|
ensure_directory,
|
||||||
from .message_manager import MessageManager
|
get_file_size,
|
||||||
|
is_valid_path,
|
||||||
# Import VideoDownloader last to avoid circular imports
|
safe_delete
|
||||||
from .video_downloader import VideoDownloader
|
)
|
||||||
|
from .file_deletion import FileDeleter
|
||||||
|
from .directory_manager import DirectoryManager
|
||||||
|
from .permission_manager import PermissionManager
|
||||||
|
from .download_manager import DownloadManager
|
||||||
|
from .compression_manager import CompressionManager
|
||||||
|
from .progress_tracker import ProgressTracker
|
||||||
|
from .path_manager import PathManager
|
||||||
|
from .exceptions import (
|
||||||
|
FileOperationError,
|
||||||
|
DirectoryError,
|
||||||
|
PermissionError,
|
||||||
|
DownloadError,
|
||||||
|
CompressionError,
|
||||||
|
TrackingError,
|
||||||
|
PathError
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
'FileCleanupError',
|
# File Operations
|
||||||
'VideoVerificationError',
|
|
||||||
'secure_delete_file',
|
|
||||||
'cleanup_downloads',
|
'cleanup_downloads',
|
||||||
'temp_path_context',
|
'ensure_directory',
|
||||||
'VideoDownloader',
|
'get_file_size',
|
||||||
'MessageManager',
|
'is_valid_path',
|
||||||
|
'safe_delete',
|
||||||
|
|
||||||
|
# Managers
|
||||||
|
'FileDeleter',
|
||||||
|
'DirectoryManager',
|
||||||
|
'PermissionManager',
|
||||||
|
'DownloadManager',
|
||||||
|
'CompressionManager',
|
||||||
|
'ProgressTracker',
|
||||||
|
'PathManager',
|
||||||
|
|
||||||
|
# Exceptions
|
||||||
|
'FileOperationError',
|
||||||
|
'DirectoryError',
|
||||||
|
'PermissionError',
|
||||||
|
'DownloadError',
|
||||||
|
'CompressionError',
|
||||||
|
'TrackingError',
|
||||||
|
'PathError'
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Initialize shared instances for module-level access
|
||||||
|
directory_manager = DirectoryManager()
|
||||||
|
permission_manager = PermissionManager()
|
||||||
|
download_manager = DownloadManager()
|
||||||
|
compression_manager = CompressionManager()
|
||||||
|
progress_tracker = ProgressTracker()
|
||||||
|
path_manager = PathManager()
|
||||||
|
|||||||
Reference in New Issue
Block a user