pacnpal/Pac-cogs
1
0
Fork 0
mirror of https://github.com/pacnpal/Pac-cogs.git synced 2025-12-20 02:41:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

diabolical

Browse Source
This commit is contained in:
pacnpal
2024-11-18 05:17:37 +00:00
parent fc06e54d8a
commit df9099f2c8
75 changed files with 744 additions and 719 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
info.json
View File

@@ -5,5 +5,5 @@
"install_msg": "Thank you for installing the Pac-cogs repo!", "install_msg": "Thank you for installing the Pac-cogs repo!",
"name": "Pac-cogs", "name": "Pac-cogs",
"short": "Very cool cogs!", "short": "Very cool cogs!",
"description": "Right now, just a birthday cog." "description": "Cogs that I made for my own use but you can use them too if you want!."
} }

131
videoarchiver/__init__.py
View File

@@ -9,58 +9,58 @@ from redbot.core.bot import Red # type: ignore
# Force reload of all modules # Force reload of all modules
modules_to_reload = [ modules_to_reload = [
".utils.exceptions", "utils.exceptions",
".utils", "utils",
".processor", "processor",
".processor.core", "processor.core",
".processor.queue_processor", "processor.queue_processor",
".queue", "queue",
".queue.types", # Added types module "queue.types", # Added types module
".queue.models", "queue.models",
".queue.manager", "queue.manager",
".queue.cleaners", "queue.cleaners",
".queue.cleaners.guild_cleaner", "queue.cleaners.guild_cleaner",
".queue.cleaners.history_cleaner", "queue.cleaners.history_cleaner",
".queue.cleaners.tracking_cleaner", "queue.cleaners.tracking_cleaner",
".queue.monitoring", "queue.monitoring",
".queue.recovery_manager", "queue.recovery_manager",
".queue.state_manager", "queue.state_manager",
".ffmpeg", "ffmpeg",
".ffmpeg.binary_manager", "ffmpeg.binary_manager",
".ffmpeg.encoder_params", "ffmpeg.encoder_params",
".ffmpeg.exceptions", "ffmpeg.exceptions",
".ffmpeg.ffmpeg_downloader", "ffmpeg.ffmpeg_downloader",
".ffmpeg.ffmpeg_manager", "ffmpeg.ffmpeg_manager",
".ffmpeg.gpu_detector", "ffmpeg.gpu_detector",
".ffmpeg.process_manager", "ffmpeg.process_manager",
".ffmpeg.verification_manager", "ffmpeg.verification_manager",
".ffmpeg.video_analyzer", "ffmpeg.video_analyzer",
".database", "database",
".database.connection_manager", "database.connection_manager",
".database.query_manager", "database.query_manager",
".database.schema_manager", "database.schema_manager",
".database.video_archive_db", "database.video_archive_db",
".config", "config",
".config.channel_manager", "config.channel_manager",
".config.exceptions", "config.exceptions",
".config.role_manager", "config.role_manager",
".config.settings_formatter", "config.settings_formatter",
".config.validation_manager", "config.validation_manager",
".core", "core",
".core.base", "core.base",
".core.cleanup", "core.cleanup",
".core.commands", "core.commands",
".core.commands.archiver_commands", "core.commands.archiver_commands",
".core.commands.database_commands", "core.commands.database_commands",
".core.commands.settings_commands", "core.commands.settings_commands",
".core.component_manager", "core.component_manager",
".core.error_handler", "core.error_handler",
".core.events", "core.events",
".core.guild", "core.guild",
".core.initialization", "core.initialization",
".core.lifecycle", "core.lifecycle",
".core.response_handler", "core.response_handler",
".core.settings", "core.settings",
] ]
# Remove modules to force fresh import # Remove modules to force fresh import
@@ -68,7 +68,7 @@ for module in modules_to_reload:
if module in sys.modules: if module in sys.modules:
del sys.modules[module] del sys.modules[module]
try: # try:
# Try relative imports first # Try relative imports first
from . import utils from . import utils
from . import processor from . import processor
@@ -89,7 +89,8 @@ try:
ErrorSeverity, ErrorSeverity,
ProcessingError, ProcessingError,
) )
except ImportError:
# except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail
# from videoarchiver import utils # from videoarchiver import utils
# from videoarchiver import processor # from videoarchiver import processor
@@ -121,17 +122,17 @@ except ImportError:
importlib.reload(core) importlib.reload(core)
# Import all submodules # Import all submodules
from .database import * from database import *
from .ffmpeg import * from ffmpeg import *
from .queue import * from queue import *
from .processor import * from processor import *
from .config_manager import * from config_manager import *
from .update_checker import * from update_checker import *
from .queue.cleaners import * from queue.cleaners import *
from .database import * from database import *
from .utils import * from utils import *
from .core import * from core import *
from .config import * from config import *
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

10
videoarchiver/config/__init__.py
View File

@@ -2,7 +2,7 @@
# try: # try:
# Try relative imports first # Try relative imports first
from .exceptions import ( from exceptions import (
ConfigurationError, ConfigurationError,
ValidationError, ValidationError,
PermissionError, PermissionError,
@@ -12,10 +12,10 @@ from .exceptions import (
SchemaError, SchemaError,
DiscordAPIError, DiscordAPIError,
) )
from .channel_manager import ChannelManager from channel_manager import ChannelManager
from .role_manager import RoleManager from role_manager import RoleManager
from .settings_formatter import SettingsFormatter from settings_formatter import SettingsFormatter
from .validation_manager import ValidationManager from validation_manager import ValidationManager
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

2
videoarchiver/config/channel_manager.py
View File

@@ -6,7 +6,7 @@ import discord # type: ignore
# try: # try:
# Try relative imports first # Try relative imports first
from .exceptions import ( from exceptions import (
ConfigurationError as ConfigError, ConfigurationError as ConfigError,
DiscordAPIError, DiscordAPIError,
) )

2
videoarchiver/config/role_manager.py
View File

@@ -6,7 +6,7 @@ import discord # type: ignore
# try: # try:
# Try relative imports first # Try relative imports first
from .exceptions import ConfigurationError as ConfigError from exceptions import ConfigurationError as ConfigError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

2
videoarchiver/config/settings_formatter.py
View File

@@ -7,7 +7,7 @@ import discord # type: ignore
# try: # try:
# Try relative imports first # Try relative imports first
from .exceptions import ConfigurationError as ConfigError from exceptions import ConfigurationError as ConfigError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

2
videoarchiver/config/validation_manager.py
View File

@@ -5,7 +5,7 @@ from typing import Any, Dict, List, Union
# try: # try:
# Try relative imports first # Try relative imports first
from .exceptions import ConfigurationError as ConfigError from exceptions import ConfigurationError as ConfigError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

10
videoarchiver/config_manager.py
View File

@@ -8,11 +8,11 @@ from redbot.core import Config # type: ignore
# try: # try:
# Try relative imports first # Try relative imports first
from .config.validation_manager import ValidationManager from config.validation_manager import ValidationManager
from .config.settings_formatter import SettingsFormatter from config.settings_formatter import SettingsFormatter
from .config.channel_manager import ChannelManager from config.channel_manager import ChannelManager
from .config.role_manager import RoleManager from config.role_manager import RoleManager
from .utils.exceptions import ConfigurationError as ConfigError from utils.exceptions import ConfigurationError as ConfigError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

2
videoarchiver/core/__init__.py
View File

@@ -2,7 +2,7 @@
# try: # try:
# Try relative imports first # Try relative imports first
from .base import VideoArchiver from base import VideoArchiver
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

30
videoarchiver/core/base.py
View File

@@ -14,21 +14,21 @@ from redbot.core.commands import GroupCog, Context # type: ignore
# try: # try:
# Try relative imports first # Try relative imports first
from .settings import Settings from settings import Settings
from .lifecycle import LifecycleManager, LifecycleState from lifecycle import LifecycleManager, LifecycleState
from .component_manager import ComponentManager, ComponentState from component_manager import ComponentManager, ComponentState
from .error_handler import error_manager, handle_command_error from error_handler import error_manager, handle_command_error
from .response_handler import ResponseManager from response_handler import ResponseManager
from .commands.archiver_commands import setup_archiver_commands from commands.archiver_commands import setup_archiver_commands
from .commands.database_commands import setup_database_commands from commands.database_commands import setup_database_commands
from .commands.settings_commands import setup_settings_commands from commands.settings_commands import setup_settings_commands
from .events import setup_events, EventManager from events import setup_events, EventManager
from ..processor.core import VideoProcessor from processor.core import VideoProcessor
from ..queue.manager import EnhancedVideoQueueManager from queue.manager import EnhancedVideoQueueManager
from ..ffmpeg.ffmpeg_manager import FFmpegManager from ffmpeg.ffmpeg_manager import FFmpegManager
from ..database.video_archive_db import VideoArchiveDB from database.video_archive_db import VideoArchiveDB
from ..config_manager import ConfigManager from config_manager import ConfigManager
from ..utils.exceptions import CogError, ErrorContext, ErrorSeverity from utils.exceptions import CogError, ErrorContext, ErrorSeverity
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

0
videoarchiver/core/types.py → videoarchiver/core/c_types.py
View File

120
videoarchiver/core/cleanup.py
View File

@@ -8,25 +8,29 @@ from enum import Enum, auto
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar
#try: # try:
# Try relative imports first # Try relative imports first
from ..utils.file_ops import cleanup_downloads from utils.file_ops import cleanup_downloads
from ..utils.exceptions import CleanupError, ErrorContext, ErrorSeverity from utils.exceptions import CleanupError, ErrorContext, ErrorSeverity
#except ImportError:
# Fall back to absolute imports if relative imports fail # except ImportError:
# from videoarchiver.utils.file_ops import cleanup_downloads # Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import CleanupError, ErrorContext, ErrorSeverity # from videoarchiver.utils.file_ops import cleanup_downloads
# from videoarchiver.utils.exceptions import CleanupError, ErrorContext, ErrorSeverity
if TYPE_CHECKING: if TYPE_CHECKING:
#try: # try:
from .base import VideoArchiver from .base import VideoArchiver
#except ImportError:
# from videoarchiver.core.base import VideoArchiver # except ImportError:
# from videoarchiver.core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class CleanupPhase(Enum): class CleanupPhase(Enum):
"""Cleanup phases""" """Cleanup phases"""
INITIALIZATION = auto() INITIALIZATION = auto()
UPDATE_CHECKER = auto() UPDATE_CHECKER = auto()
PROCESSOR = auto() PROCESSOR = auto()
@@ -36,21 +40,26 @@ class CleanupPhase(Enum):
DOWNLOADS = auto() DOWNLOADS = auto()
REFERENCES = auto() REFERENCES = auto()
class CleanupStatus(Enum): class CleanupStatus(Enum):
"""Cleanup status""" """Cleanup status"""
SUCCESS = auto() SUCCESS = auto()
TIMEOUT = auto() TIMEOUT = auto()
ERROR = auto() ERROR = auto()
SKIPPED = auto() SKIPPED = auto()
class CleanupResult(TypedDict): class CleanupResult(TypedDict):
"""Type definition for cleanup result""" """Type definition for cleanup result"""
phase: CleanupPhase phase: CleanupPhase
status: CleanupStatus status: CleanupStatus
error: Optional[str] error: Optional[str]
duration: float duration: float
timestamp: str timestamp: str
class CleanupManager: class CleanupManager:
"""Manages cleanup operations""" """Manages cleanup operations"""
@@ -65,7 +74,7 @@ class CleanupManager:
phase: CleanupPhase, phase: CleanupPhase,
status: CleanupStatus, status: CleanupStatus,
error: Optional[str] = None, error: Optional[str] = None,
duration: float = 0.0 duration: float = 0.0,
) -> None: ) -> None:
"""Record result of a cleanup phase""" """Record result of a cleanup phase"""
self.results[phase] = CleanupResult( self.results[phase] = CleanupResult(
@@ -73,13 +82,14 @@ class CleanupManager:
status=status, status=status,
error=error, error=error,
duration=duration, duration=duration,
timestamp=datetime.utcnow().isoformat() timestamp=datetime.utcnow().isoformat(),
) )
def get_results(self) -> Dict[CleanupPhase, CleanupResult]: def get_results(self) -> Dict[CleanupPhase, CleanupResult]:
"""Get cleanup results""" """Get cleanup results"""
return self.results.copy() return self.results.copy()
async def cleanup_resources(cog: "VideoArchiver") -> None: async def cleanup_resources(cog: "VideoArchiver") -> None:
""" """
Clean up all resources with proper handling. Clean up all resources with proper handling.
@@ -102,11 +112,13 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
try: try:
logger.info("Cancelling initialization task") logger.info("Cancelling initialization task")
cog._init_task.cancel() cog._init_task.cancel()
await asyncio.wait_for(cog._init_task, timeout=cleanup_manager.CLEANUP_TIMEOUT) await asyncio.wait_for(
cog._init_task, timeout=cleanup_manager.CLEANUP_TIMEOUT
)
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.INITIALIZATION, CleanupPhase.INITIALIZATION,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except (asyncio.TimeoutError, asyncio.CancelledError) as e: except (asyncio.TimeoutError, asyncio.CancelledError) as e:
logger.warning("Initialization task cancellation timed out") logger.warning("Initialization task cancellation timed out")
@@ -114,7 +126,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.INITIALIZATION, CleanupPhase.INITIALIZATION,
CleanupStatus.TIMEOUT, CleanupStatus.TIMEOUT,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
# Stop update checker # Stop update checker
@@ -123,13 +135,12 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
try: try:
logger.info("Stopping update checker") logger.info("Stopping update checker")
await asyncio.wait_for( await asyncio.wait_for(
cog.update_checker.stop(), cog.update_checker.stop(), timeout=cleanup_manager.CLEANUP_TIMEOUT
timeout=cleanup_manager.CLEANUP_TIMEOUT
) )
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.UPDATE_CHECKER, CleanupPhase.UPDATE_CHECKER,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except asyncio.TimeoutError as e: except asyncio.TimeoutError as e:
logger.warning("Update checker stop timed out") logger.warning("Update checker stop timed out")
@@ -137,7 +148,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.UPDATE_CHECKER, CleanupPhase.UPDATE_CHECKER,
CleanupStatus.TIMEOUT, CleanupStatus.TIMEOUT,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
cog.update_checker = None cog.update_checker = None
@@ -147,13 +158,12 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
try: try:
logger.info("Cleaning up processor") logger.info("Cleaning up processor")
await asyncio.wait_for( await asyncio.wait_for(
cog.processor.cleanup(), cog.processor.cleanup(), timeout=cleanup_manager.CLEANUP_TIMEOUT
timeout=cleanup_manager.CLEANUP_TIMEOUT
) )
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.PROCESSOR, CleanupPhase.PROCESSOR,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except asyncio.TimeoutError as e: except asyncio.TimeoutError as e:
logger.warning("Processor cleanup timed out, forcing cleanup") logger.warning("Processor cleanup timed out, forcing cleanup")
@@ -162,7 +172,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.PROCESSOR, CleanupPhase.PROCESSOR,
CleanupStatus.TIMEOUT, CleanupStatus.TIMEOUT,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
cog.processor = None cog.processor = None
@@ -172,13 +182,12 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
try: try:
logger.info("Cleaning up queue manager") logger.info("Cleaning up queue manager")
await asyncio.wait_for( await asyncio.wait_for(
cog.queue_manager.cleanup(), cog.queue_manager.cleanup(), timeout=cleanup_manager.CLEANUP_TIMEOUT
timeout=cleanup_manager.CLEANUP_TIMEOUT
) )
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.QUEUE_MANAGER, CleanupPhase.QUEUE_MANAGER,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except asyncio.TimeoutError as e: except asyncio.TimeoutError as e:
logger.warning("Queue manager cleanup timed out, forcing stop") logger.warning("Queue manager cleanup timed out, forcing stop")
@@ -187,7 +196,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.QUEUE_MANAGER, CleanupPhase.QUEUE_MANAGER,
CleanupStatus.TIMEOUT, CleanupStatus.TIMEOUT,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
cog.queue_manager = None cog.queue_manager = None
@@ -214,14 +223,14 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.COMPONENTS, CleanupPhase.COMPONENTS,
status, status,
"\n".join(errors) if errors else None, "\n".join(errors) if errors else None,
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.COMPONENTS, CleanupPhase.COMPONENTS,
CleanupStatus.ERROR, CleanupStatus.ERROR,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
# Kill any FFmpeg processes # Kill any FFmpeg processes
@@ -233,7 +242,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
cog.ffmpeg_mgr = None cog.ffmpeg_mgr = None
# Kill any remaining FFmpeg processes system-wide # Kill any remaining FFmpeg processes system-wide
if os.name != 'nt': # Unix-like systems if os.name != "nt": # Unix-like systems
os.system("pkill -9 ffmpeg") os.system("pkill -9 ffmpeg")
else: # Windows else: # Windows
os.system("taskkill /F /IM ffmpeg.exe") os.system("taskkill /F /IM ffmpeg.exe")
@@ -241,14 +250,14 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.FFMPEG, CleanupPhase.FFMPEG,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.FFMPEG, CleanupPhase.FFMPEG,
CleanupStatus.ERROR, CleanupStatus.ERROR,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
# Clean up download directory # Clean up download directory
@@ -258,21 +267,21 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
logger.info("Cleaning up download directory") logger.info("Cleaning up download directory")
await asyncio.wait_for( await asyncio.wait_for(
cleanup_downloads(str(cog.download_path)), cleanup_downloads(str(cog.download_path)),
timeout=cleanup_manager.CLEANUP_TIMEOUT timeout=cleanup_manager.CLEANUP_TIMEOUT,
) )
if cog.download_path.exists(): if cog.download_path.exists():
cog.download_path.rmdir() cog.download_path.rmdir()
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.DOWNLOADS, CleanupPhase.DOWNLOADS,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.DOWNLOADS, CleanupPhase.DOWNLOADS,
CleanupStatus.ERROR, CleanupStatus.ERROR,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
@@ -284,8 +293,8 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
"Cleanup", "Cleanup",
"cleanup_resources", "cleanup_resources",
{"duration": (datetime.utcnow() - start_time).total_seconds()}, {"duration": (datetime.utcnow() - start_time).total_seconds()},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
finally: finally:
logger.info("Clearing ready flag") logger.info("Clearing ready flag")
@@ -294,13 +303,14 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
# Log cleanup results # Log cleanup results
for phase, result in cleanup_manager.get_results().items(): for phase, result in cleanup_manager.get_results().items():
status_str = f"{result['status'].name}" status_str = f"{result['status'].name}"
if result['error']: if result["error"]:
status_str += f" ({result['error']})" status_str += f" ({result['error']})"
logger.info( logger.info(
f"Cleanup phase {phase.name}: {status_str} " f"Cleanup phase {phase.name}: {status_str} "
f"(Duration: {result['duration']:.2f}s)" f"(Duration: {result['duration']:.2f}s)"
) )
async def force_cleanup_resources(cog: "VideoArchiver") -> None: async def force_cleanup_resources(cog: "VideoArchiver") -> None:
""" """
Force cleanup of resources when timeout occurs. Force cleanup of resources when timeout occurs.
@@ -323,14 +333,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.PROCESSOR, CleanupPhase.PROCESSOR,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.PROCESSOR, CleanupPhase.PROCESSOR,
CleanupStatus.ERROR, CleanupStatus.ERROR,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
cog.processor = None cog.processor = None
@@ -343,14 +353,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.QUEUE_MANAGER, CleanupPhase.QUEUE_MANAGER,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.QUEUE_MANAGER, CleanupPhase.QUEUE_MANAGER,
CleanupStatus.ERROR, CleanupStatus.ERROR,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
cog.queue_manager = None cog.queue_manager = None
@@ -363,7 +373,7 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cog.ffmpeg_mgr = None cog.ffmpeg_mgr = None
# Force kill any remaining FFmpeg processes system-wide # Force kill any remaining FFmpeg processes system-wide
if os.name != 'nt': # Unix-like systems if os.name != "nt": # Unix-like systems
os.system("pkill -9 ffmpeg") os.system("pkill -9 ffmpeg")
else: # Windows else: # Windows
os.system("taskkill /F /IM ffmpeg.exe") os.system("taskkill /F /IM ffmpeg.exe")
@@ -371,14 +381,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.FFMPEG, CleanupPhase.FFMPEG,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.FFMPEG, CleanupPhase.FFMPEG,
CleanupStatus.ERROR, CleanupStatus.ERROR,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
# Clean up download directory # Clean up download directory
@@ -388,21 +398,21 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
logger.info("Force cleaning download directory") logger.info("Force cleaning download directory")
await asyncio.wait_for( await asyncio.wait_for(
cleanup_downloads(str(cog.download_path)), cleanup_downloads(str(cog.download_path)),
timeout=cleanup_manager.FORCE_CLEANUP_TIMEOUT timeout=cleanup_manager.FORCE_CLEANUP_TIMEOUT,
) )
if cog.download_path.exists(): if cog.download_path.exists():
cog.download_path.rmdir() cog.download_path.rmdir()
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.DOWNLOADS, CleanupPhase.DOWNLOADS,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.DOWNLOADS, CleanupPhase.DOWNLOADS,
CleanupStatus.ERROR, CleanupStatus.ERROR,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
# Clear all components # Clear all components
@@ -414,14 +424,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.COMPONENTS, CleanupPhase.COMPONENTS,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.COMPONENTS, CleanupPhase.COMPONENTS,
CleanupStatus.ERROR, CleanupStatus.ERROR,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
@@ -443,25 +453,25 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cog.db = None cog.db = None
cog._init_task = None cog._init_task = None
cog._cleanup_task = None cog._cleanup_task = None
if hasattr(cog, '_queue_task'): if hasattr(cog, "_queue_task"):
cog._queue_task = None cog._queue_task = None
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.REFERENCES, CleanupPhase.REFERENCES,
CleanupStatus.SUCCESS, CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds() duration=(datetime.utcnow() - phase_start).total_seconds(),
) )
except Exception as e: except Exception as e:
cleanup_manager.record_result( cleanup_manager.record_result(
CleanupPhase.REFERENCES, CleanupPhase.REFERENCES,
CleanupStatus.ERROR, CleanupStatus.ERROR,
str(e), str(e),
(datetime.utcnow() - phase_start).total_seconds() (datetime.utcnow() - phase_start).total_seconds(),
) )
# Log cleanup results # Log cleanup results
for phase, result in cleanup_manager.get_results().items(): for phase, result in cleanup_manager.get_results().items():
status_str = f"{result['status'].name}" status_str = f"{result['status'].name}"
if result['error']: if result["error"]:
status_str += f" ({result['error']})" status_str += f" ({result['error']})"
logger.info( logger.info(
f"Force cleanup phase {phase.name}: {status_str} " f"Force cleanup phase {phase.name}: {status_str} "

7
videoarchiver/core/commands.py
View File

@@ -7,9 +7,10 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING: if TYPE_CHECKING:
# try: # try:
from .base import VideoArchiver from base import VideoArchiver
# except ImportError: # except ImportError:
# from videoarchiver.core.base import VideoArchiver # from videoarchiver.core.base import VideoArchiver
def setup_commands(cog: "VideoArchiver") -> None: def setup_commands(cog: "VideoArchiver") -> None:
"""Command setup is now handled in the VideoArchiver class""" """Command setup is now handled in the VideoArchiver class"""

6
videoarchiver/core/commands/__init__.py
View File

@@ -1,8 +1,8 @@
"""Command handlers for VideoArchiver""" """Command handlers for VideoArchiver"""
from .archiver_commands import setup_archiver_commands from archiver_commands import setup_archiver_commands
from .database_commands import setup_database_commands from database_commands import setup_database_commands
from .settings_commands import setup_settings_commands from settings_commands import setup_settings_commands
__all__ = [ __all__ = [
"setup_archiver_commands", "setup_archiver_commands",

4
videoarchiver/core/commands/archiver_commands.py
View File

@@ -9,8 +9,8 @@ from discord import app_commands # type: ignore
from redbot.core import commands # type: ignore from redbot.core import commands # type: ignore
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore
from ...core.response_handler import handle_response, ResponseType from core.response_handler import handle_response, ResponseType
from ...utils.exceptions import CommandError, ErrorContext, ErrorSeverity from utils.exceptions import CommandError, ErrorContext, ErrorSeverity
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

6
videoarchiver/core/commands/database_commands.py
View File

@@ -10,9 +10,9 @@ from discord import app_commands # type: ignore
from redbot.core import commands # type: ignore from redbot.core import commands # type: ignore
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore
from ...core.response_handler import handle_response, ResponseType from core.response_handler import handle_response, ResponseType
from ...utils.exceptions import CommandError, ErrorContext, ErrorSeverity, DatabaseError from utils.exceptions import CommandError, ErrorContext, ErrorSeverity, DatabaseError
from ...database.video_archive_db import VideoArchiveDB from database.video_archive_db import VideoArchiveDB
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

6
videoarchiver/core/commands/settings_commands.py
View File

@@ -9,9 +9,9 @@ from discord import app_commands # type: ignore
from redbot.core import commands # type: ignore from redbot.core import commands # type: ignore
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore
from ...core.settings import VideoFormat, VideoQuality from core.settings import VideoFormat, VideoQuality
from ...core.response_handler import handle_response, ResponseType from core.response_handler import handle_response, ResponseType
from ...utils.exceptions import CommandError, ErrorContext, ErrorSeverity from utils.exceptions import CommandError, ErrorContext, ErrorSeverity
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

12
videoarchiver/core/component_manager.py
View File

@@ -22,12 +22,12 @@ import importlib
# try: # try:
# Try relative imports first # Try relative imports first
from ..utils.exceptions import ComponentError, ErrorContext, ErrorSeverity from utils.exceptions import ComponentError, ErrorContext, ErrorSeverity
from ..utils.path_manager import PathManager from utils.path_manager import PathManager
from ..config_manager import ConfigManager from config_manager import ConfigManager
from ..processor.core import VideoProcessor from processor.core import VideoProcessor
from ..queue.manager import EnhancedVideoQueueManager from queue.manager import EnhancedVideoQueueManager
from ..ffmpeg.ffmpeg_manager import FFmpegManager from ffmpeg.ffmpeg_manager import FFmpegManager
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

69
videoarchiver/core/error_handler.py
View File

@@ -4,19 +4,19 @@ import logging
import traceback import traceback
from typing import Dict, Optional, Tuple, Type, TypedDict, ClassVar from typing import Dict, Optional, Tuple, Type, TypedDict, ClassVar
from enum import Enum, auto from enum import Enum, auto
import discord # type: ignore import discord # type: ignore
from redbot.core.commands import ( # type: ignore from redbot.core.commands import ( # type: ignore
Context, Context,
MissingPermissions, MissingPermissions,
BotMissingPermissions, BotMissingPermissions,
MissingRequiredArgument, MissingRequiredArgument,
BadArgument, BadArgument,
CommandError CommandError,
) )
#try: # try:
# Try relative imports first # Try relative imports first
from ..utils.exceptions import ( from utils.exceptions import (
VideoArchiverError, VideoArchiverError,
ErrorSeverity, ErrorSeverity,
ErrorContext, ErrorContext,
@@ -33,9 +33,10 @@ from ..utils.exceptions import (
TrackingError, TrackingError,
NetworkError, NetworkError,
ResourceExhaustedError, ResourceExhaustedError,
ConfigurationError ConfigurationError,
) )
from ..core.response_handler import response_manager from core.response_handler import response_manager
# except ImportError: # except ImportError:
# # Fall back to absolute imports if relative imports fail # # Fall back to absolute imports if relative imports fail
# # from videoarchiver.utils.exceptions import ( # # from videoarchiver.utils.exceptions import (
@@ -57,12 +58,14 @@ from ..core.response_handler import response_manager
# ResourceExhaustedError, # ResourceExhaustedError,
# ConfigurationError # ConfigurationError
# ) # )
# from videoarchiver.core.response_handler import response_manager # from videoarchiver.core.response_handler import response_manager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class ErrorCategory(Enum): class ErrorCategory(Enum):
"""Categories of errors""" """Categories of errors"""
PERMISSION = auto() PERMISSION = auto()
ARGUMENT = auto() ARGUMENT = auto()
CONFIGURATION = auto() CONFIGURATION = auto()
@@ -76,17 +79,22 @@ class ErrorCategory(Enum):
HEALTH = auto() HEALTH = auto()
UNEXPECTED = auto() UNEXPECTED = auto()
class ErrorStats(TypedDict): class ErrorStats(TypedDict):
"""Type definition for error statistics""" """Type definition for error statistics"""
counts: Dict[str, int] counts: Dict[str, int]
patterns: Dict[str, Dict[str, int]] patterns: Dict[str, Dict[str, int]]
severities: Dict[str, Dict[str, int]] severities: Dict[str, Dict[str, int]]
class ErrorFormatter: class ErrorFormatter:
"""Formats error messages for display""" """Formats error messages for display"""
@staticmethod @staticmethod
def format_error_message(error: Exception, context: Optional[ErrorContext] = None) -> str: def format_error_message(
error: Exception, context: Optional[ErrorContext] = None
) -> str:
"""Format error message with context""" """Format error message with context"""
base_message = str(error) base_message = str(error)
if context: if context:
@@ -110,16 +118,18 @@ class ErrorFormatter:
return "An unexpected error occurred. Please check the logs for details." return "An unexpected error occurred. Please check the logs for details."
return str(error) return str(error)
class ErrorCategorizer: class ErrorCategorizer:
"""Categorizes errors and determines handling strategy""" """Categorizes errors and determines handling strategy"""
ERROR_MAPPING: ClassVar[Dict[Type[Exception], Tuple[ErrorCategory, ErrorSeverity]]] = { ERROR_MAPPING: ClassVar[
Dict[Type[Exception], Tuple[ErrorCategory, ErrorSeverity]]
] = {
# Discord command errors # Discord command errors
MissingPermissions: (ErrorCategory.PERMISSION, ErrorSeverity.MEDIUM), MissingPermissions: (ErrorCategory.PERMISSION, ErrorSeverity.MEDIUM),
BotMissingPermissions: (ErrorCategory.PERMISSION, ErrorSeverity.HIGH), BotMissingPermissions: (ErrorCategory.PERMISSION, ErrorSeverity.HIGH),
MissingRequiredArgument: (ErrorCategory.ARGUMENT, ErrorSeverity.LOW), MissingRequiredArgument: (ErrorCategory.ARGUMENT, ErrorSeverity.LOW),
BadArgument: (ErrorCategory.ARGUMENT, ErrorSeverity.LOW), BadArgument: (ErrorCategory.ARGUMENT, ErrorSeverity.LOW),
# VideoArchiver errors # VideoArchiver errors
ProcessorError: (ErrorCategory.PROCESSING, ErrorSeverity.HIGH), ProcessorError: (ErrorCategory.PROCESSING, ErrorSeverity.HIGH),
ValidationError: (ErrorCategory.VALIDATION, ErrorSeverity.MEDIUM), ValidationError: (ErrorCategory.VALIDATION, ErrorSeverity.MEDIUM),
@@ -134,7 +144,7 @@ class ErrorCategorizer:
TrackingError: (ErrorCategory.PROCESSING, ErrorSeverity.MEDIUM), TrackingError: (ErrorCategory.PROCESSING, ErrorSeverity.MEDIUM),
NetworkError: (ErrorCategory.NETWORK, ErrorSeverity.MEDIUM), NetworkError: (ErrorCategory.NETWORK, ErrorSeverity.MEDIUM),
ResourceExhaustedError: (ErrorCategory.RESOURCE, ErrorSeverity.HIGH), ResourceExhaustedError: (ErrorCategory.RESOURCE, ErrorSeverity.HIGH),
ConfigurationError: (ErrorCategory.CONFIGURATION, ErrorSeverity.HIGH) ConfigurationError: (ErrorCategory.CONFIGURATION, ErrorSeverity.HIGH),
} }
@classmethod @classmethod
@@ -153,6 +163,7 @@ class ErrorCategorizer:
return category, severity return category, severity
return ErrorCategory.UNEXPECTED, ErrorSeverity.HIGH return ErrorCategory.UNEXPECTED, ErrorSeverity.HIGH
class ErrorTracker: class ErrorTracker:
"""Tracks error occurrences and patterns""" """Tracks error occurrences and patterns"""
@@ -162,10 +173,7 @@ class ErrorTracker:
self.error_severities: Dict[str, Dict[str, int]] = {} self.error_severities: Dict[str, Dict[str, int]] = {}
def track_error( def track_error(
self, self, error: Exception, category: ErrorCategory, severity: ErrorSeverity
error: Exception,
category: ErrorCategory,
severity: ErrorSeverity
) -> None: ) -> None:
""" """
Track an error occurrence. Track an error occurrence.
@@ -204,9 +212,10 @@ class ErrorTracker:
return ErrorStats( return ErrorStats(
counts=self.error_counts.copy(), counts=self.error_counts.copy(),
patterns=self.error_patterns.copy(), patterns=self.error_patterns.copy(),
severities=self.error_severities.copy() severities=self.error_severities.copy(),
) )
class ErrorManager: class ErrorManager:
"""Manages error handling and reporting""" """Manages error handling and reporting"""
@@ -215,11 +224,7 @@ class ErrorManager:
self.categorizer = ErrorCategorizer() self.categorizer = ErrorCategorizer()
self.tracker = ErrorTracker() self.tracker = ErrorTracker()
async def handle_error( async def handle_error(self, ctx: Context, error: Exception) -> None:
self,
ctx: Context,
error: Exception
) -> None:
""" """
Handle a command error. Handle a command error.
@@ -238,9 +243,9 @@ class ErrorManager:
details={ details={
"guild_id": str(ctx.guild.id) if ctx.guild else "DM", "guild_id": str(ctx.guild.id) if ctx.guild else "DM",
"channel_id": str(ctx.channel.id), "channel_id": str(ctx.channel.id),
"user_id": str(ctx.author.id) "user_id": str(ctx.author.id),
}, },
severity=severity severity=severity,
) )
# Track error # Track error
@@ -255,9 +260,7 @@ class ErrorManager:
# Send response # Send response
await response_manager.send_response( await response_manager.send_response(
ctx, ctx, content=user_message, response_type=severity.name.lower()
content=user_message,
response_type=severity.name.lower()
) )
except Exception as e: except Exception as e:
@@ -269,16 +272,12 @@ class ErrorManager:
await response_manager.send_response( await response_manager.send_response(
ctx, ctx,
content="An error occurred while handling another error. Please check the logs.", content="An error occurred while handling another error. Please check the logs.",
response_type="error" response_type="error",
) )
except Exception: except Exception:
pass pass
def _log_error( def _log_error(self, message: str, severity: ErrorSeverity) -> None:
self,
message: str,
severity: ErrorSeverity
) -> None:
""" """
Log error details. Log error details.
@@ -296,9 +295,11 @@ class ErrorManager:
except Exception as e: except Exception as e:
logger.error(f"Error logging error details: {e}") logger.error(f"Error logging error details: {e}")
# Global error manager instance # Global error manager instance
error_manager = ErrorManager() error_manager = ErrorManager()
async def handle_command_error(ctx: Context, error: Exception) -> None: async def handle_command_error(ctx: Context, error: Exception) -> None:
""" """
Helper function to handle command errors using the error manager. Helper function to handle command errors using the error manager.

41
videoarchiver/core/events.py
View File

@@ -9,28 +9,29 @@ from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar, List
import discord # type: ignore import discord # type: ignore
#try: # try:
# Try relative imports first # Try relative imports first
from ..processor.constants import REACTIONS from processor.constants import REACTIONS
from ..processor.reactions import handle_archived_reaction from processor.reactions import handle_archived_reaction
from .guild import initialize_guild_components, cleanup_guild_components from guild import initialize_guild_components, cleanup_guild_components
from .error_handler import ErrorManager from error_handler import ErrorManager
from .response_handler import response_manager from response_handler import response_manager
from ..utils.exceptions import EventError, ErrorContext, ErrorSeverity from utils.exceptions import EventError, ErrorContext, ErrorSeverity
#except ImportError:
# Fall back to absolute imports if relative imports fail # except ImportError:
# from videoarchiver.processor.constants import REACTIONS # Fall back to absolute imports if relative imports fail
# from videoarchiver.processor.reactions import handle_archived_reaction # from videoarchiver.processor.constants import REACTIONS
# from videoarchiver.core.guild import initialize_guild_components, cleanup_guild_components # from videoarchiver.processor.reactions import handle_archived_reaction
# from videoarchiver.core.error_handler import ErrorManager # from videoarchiver.core.guild import initialize_guild_components, cleanup_guild_components
# from videoarchiver.core.response_handler import response_manager # from videoarchiver.core.error_handler import ErrorManager
# from videoarchiver.utils.exceptions import EventError, ErrorContext, ErrorSeverity # from videoarchiver.core.response_handler import response_manager
# from videoarchiver.utils.exceptions import EventError, ErrorContext, ErrorSeverity
if TYPE_CHECKING: if TYPE_CHECKING:
#try: # try:
from .base import VideoArchiver from base import VideoArchiver
# except ImportError: # except ImportError:
# from videoarchiver.core.base import VideoArchiver # from videoarchiver.core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

10
videoarchiver/core/guild.py
View File

@@ -6,10 +6,10 @@ from typing import TYPE_CHECKING, Dict, Any, Optional
# try: # try:
# Try relative imports first # Try relative imports first
from ..utils.download_core import DownloadCore from utils.download_core import DownloadCore
from ..utils.message_manager import MessageManager from utils.message_manager import MessageManager
from ..utils.file_ops import cleanup_downloads from utils.file_ops import cleanup_downloads
from ..utils.exceptions import VideoArchiverError as ProcessingError from utils.exceptions import VideoArchiverError as ProcessingError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail
@@ -20,7 +20,7 @@ from ..utils.exceptions import VideoArchiverError as ProcessingError
if TYPE_CHECKING: if TYPE_CHECKING:
# try: # try:
from .base import VideoArchiver from base import VideoArchiver
# except ImportError: # except ImportError:
# from videoarchiver.core.base import VideoArchiver # from videoarchiver.core.base import VideoArchiver

6
videoarchiver/core/initialization.py
View File

@@ -6,8 +6,8 @@ import logging
# try: # try:
# Try relative imports first # Try relative imports first
from ..utils.exceptions import ComponentError, ErrorContext, ErrorSeverity from utils.exceptions import ComponentError, ErrorContext, ErrorSeverity
from .lifecycle import LifecycleState from lifecycle import LifecycleState
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail
@@ -16,7 +16,7 @@ from .lifecycle import LifecycleState
if TYPE_CHECKING: if TYPE_CHECKING:
# try: # try:
from .base import VideoArchiver from base import VideoArchiver
# except ImportError: # except ImportError:
# from videoarchiver.core.base import VideoArchiver # from videoarchiver.core.base import VideoArchiver

27
videoarchiver/core/lifecycle.py
View File

@@ -8,25 +8,26 @@ from enum import Enum, auto
from datetime import datetime from datetime import datetime
# try: # try:
# Try relative imports first # Try relative imports first
from .cleanup import cleanup_resources, force_cleanup_resources from cleanup import cleanup_resources, force_cleanup_resources
from ..utils.exceptions import ( from utils.exceptions import (
VideoArchiverError, VideoArchiverError,
ErrorContext, ErrorContext,
ErrorSeverity, ErrorSeverity,
ComponentError, ComponentError,
CleanupError, CleanupError,
) )
#except ImportError:
# Fall back to absolute imports if relative imports fail # except ImportError:
# from videoarchiver.core.cleanup import cleanup_resources, force_cleanup_resources # Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import ( # from videoarchiver.core.cleanup import cleanup_resources, force_cleanup_resources
# VideoArchiverError, # from videoarchiver.utils.exceptions import (
# ErrorContext, # VideoArchiverError,
# ErrorSeverity, # ErrorContext,
# ComponentError, # ErrorSeverity,
# CleanupError, # ComponentError,
# ) # CleanupError,
# )
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/core/response_handler.py
View File

@@ -9,7 +9,7 @@ from redbot.core.commands import Context # type: ignore
# try: # try:
# Try relative imports first # Try relative imports first
from ..utils.exceptions import ErrorSeverity from utils.exceptions import ErrorSeverity
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

2
videoarchiver/core/settings.py
View File

@@ -6,7 +6,7 @@ from enum import Enum, auto
# try: # try:
# Try relative imports first # Try relative imports first
from ..utils.exceptions import ConfigurationError, ErrorContext, ErrorSeverity from utils.exceptions import ConfigurationError, ErrorContext, ErrorSeverity
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

8
videoarchiver/database/__init__.py
View File

@@ -2,10 +2,10 @@
# try: # try:
# Try relative imports first # Try relative imports first
from .connection_manager import DatabaseConnectionManager from connection_manager import DatabaseConnectionManager
from .query_manager import DatabaseQueryManager from query_manager import DatabaseQueryManager
from .schema_manager import DatabaseSchemaManager from schema_manager import DatabaseSchemaManager
from .video_archive_db import VideoArchiveDB from video_archive_db import VideoArchiveDB
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

128
videoarchiver/database/connection_manager.py
View File

@@ -10,24 +10,29 @@ import threading
from queue import Queue, Empty from queue import Queue, Empty
from datetime import datetime from datetime import datetime
#try: # try:
# Try relative imports first # Try relative imports first
from ..utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity from utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
#except ImportError:
# Fall back to absolute imports if relative imports fail # except ImportError:
# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity # Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
logger = logging.getLogger("DBConnectionManager") logger = logging.getLogger("DBConnectionManager")
class ConnectionState(Enum): class ConnectionState(Enum):
"""Connection states""" """Connection states"""
AVAILABLE = auto() AVAILABLE = auto()
IN_USE = auto() IN_USE = auto()
CLOSED = auto() CLOSED = auto()
ERROR = auto() ERROR = auto()
class ConnectionStatus(TypedDict): class ConnectionStatus(TypedDict):
"""Type definition for connection status""" """Type definition for connection status"""
state: str state: str
created_at: str created_at: str
last_used: str last_used: str
@@ -36,8 +41,10 @@ class ConnectionStatus(TypedDict):
pool_size: int pool_size: int
available_connections: int available_connections: int
class ConnectionMetrics(TypedDict): class ConnectionMetrics(TypedDict):
"""Type definition for connection metrics""" """Type definition for connection metrics"""
total_connections: int total_connections: int
active_connections: int active_connections: int
idle_connections: int idle_connections: int
@@ -46,6 +53,7 @@ class ConnectionMetrics(TypedDict):
failed_transactions: int failed_transactions: int
average_transaction_time: float average_transaction_time: float
class ConnectionInfo: class ConnectionInfo:
"""Tracks connection information""" """Tracks connection information"""
@@ -73,6 +81,7 @@ class ConnectionInfo:
return 0.0 return 0.0
return self.total_transaction_time / self.transaction_count return self.total_transaction_time / self.transaction_count
class DatabaseConnectionManager: class DatabaseConnectionManager:
"""Manages SQLite database connections and connection pooling""" """Manages SQLite database connections and connection pooling"""
@@ -123,8 +132,8 @@ class DatabaseConnectionManager:
"ConnectionManager", "ConnectionManager",
"initialize_pool", "initialize_pool",
{"pool_size": self.pool_size}, {"pool_size": self.pool_size},
ErrorSeverity.CRITICAL ErrorSeverity.CRITICAL,
) ),
) )
def _create_connection(self) -> Optional[sqlite3.Connection]: def _create_connection(self) -> Optional[sqlite3.Connection]:
@@ -141,7 +150,7 @@ class DatabaseConnectionManager:
conn = sqlite3.connect( conn = sqlite3.connect(
self.db_path, self.db_path,
detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
timeout=self.CONNECTION_TIMEOUT timeout=self.CONNECTION_TIMEOUT,
) )
# Enable foreign keys # Enable foreign keys
@@ -167,8 +176,8 @@ class DatabaseConnectionManager:
"ConnectionManager", "ConnectionManager",
"create_connection", "create_connection",
{"path": str(self.db_path)}, {"path": str(self.db_path)},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@contextmanager @contextmanager
@@ -186,7 +195,7 @@ class DatabaseConnectionManager:
start_time = datetime.utcnow() start_time = datetime.utcnow()
try: try:
# Check if we have a transaction-bound connection # Check if we have a transaction-bound connection
conn = getattr(self._local, 'transaction_connection', None) conn = getattr(self._local, "transaction_connection", None)
if conn is not None: if conn is not None:
yield conn yield conn
return return
@@ -204,8 +213,8 @@ class DatabaseConnectionManager:
"ConnectionManager", "ConnectionManager",
"get_connection", "get_connection",
None, None,
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Update connection info # Update connection info
@@ -229,15 +238,12 @@ class DatabaseConnectionManager:
raise DatabaseError( raise DatabaseError(
error, error,
context=ErrorContext( context=ErrorContext(
"ConnectionManager", "ConnectionManager", "get_connection", None, ErrorSeverity.HIGH
"get_connection", ),
None,
ErrorSeverity.HIGH
)
) )
finally: finally:
if conn and not hasattr(self._local, 'transaction_connection'): if conn and not hasattr(self._local, "transaction_connection"):
try: try:
conn.rollback() # Reset connection state conn.rollback() # Reset connection state
self._connection_pool.put(conn) self._connection_pool.put(conn)
@@ -254,7 +260,9 @@ class DatabaseConnectionManager:
try: try:
conn.close() conn.close()
if id(conn) in self._connection_info: if id(conn) in self._connection_info:
self._connection_info[id(conn)].state = ConnectionState.CLOSED self._connection_info[id(conn)].state = (
ConnectionState.CLOSED
)
except Exception: except Exception:
pass pass
@@ -269,15 +277,12 @@ class DatabaseConnectionManager:
Raises: Raises:
DatabaseError: If unable to start transaction DatabaseError: If unable to start transaction
""" """
if hasattr(self._local, 'transaction_connection'): if hasattr(self._local, "transaction_connection"):
raise DatabaseError( raise DatabaseError(
"Nested transactions are not supported", "Nested transactions are not supported",
context=ErrorContext( context=ErrorContext(
"ConnectionManager", "ConnectionManager", "transaction", None, ErrorSeverity.HIGH
"transaction", ),
None,
ErrorSeverity.HIGH
)
) )
conn = None conn = None
@@ -293,11 +298,8 @@ class DatabaseConnectionManager:
raise DatabaseError( raise DatabaseError(
"Failed to create database connection", "Failed to create database connection",
context=ErrorContext( context=ErrorContext(
"ConnectionManager", "ConnectionManager", "transaction", None, ErrorSeverity.HIGH
"transaction", ),
None,
ErrorSeverity.HIGH
)
) )
# Update connection info # Update connection info
@@ -330,18 +332,15 @@ class DatabaseConnectionManager:
raise DatabaseError( raise DatabaseError(
error, error,
context=ErrorContext( context=ErrorContext(
"ConnectionManager", "ConnectionManager", "transaction", None, ErrorSeverity.HIGH
"transaction", ),
None,
ErrorSeverity.HIGH
)
) )
finally: finally:
if conn: if conn:
try: try:
# Remove thread-local binding # Remove thread-local binding
delattr(self._local, 'transaction_connection') delattr(self._local, "transaction_connection")
# Return connection to pool # Return connection to pool
self._connection_pool.put(conn) self._connection_pool.put(conn)
@@ -358,7 +357,9 @@ class DatabaseConnectionManager:
try: try:
conn.close() conn.close()
if id(conn) in self._connection_info: if id(conn) in self._connection_info:
self._connection_info[id(conn)].state = ConnectionState.CLOSED self._connection_info[id(conn)].state = (
ConnectionState.CLOSED
)
except Exception: except Exception:
pass pass
@@ -377,7 +378,9 @@ class DatabaseConnectionManager:
try: try:
conn.close() conn.close()
if id(conn) in self._connection_info: if id(conn) in self._connection_info:
self._connection_info[id(conn)].state = ConnectionState.CLOSED self._connection_info[id(conn)].state = (
ConnectionState.CLOSED
)
except Exception as e: except Exception as e:
logger.error(f"Error closing connection: {e}") logger.error(f"Error closing connection: {e}")
except Empty: except Empty:
@@ -388,11 +391,8 @@ class DatabaseConnectionManager:
raise DatabaseError( raise DatabaseError(
error, error,
context=ErrorContext( context=ErrorContext(
"ConnectionManager", "ConnectionManager", "close_all", None, ErrorSeverity.HIGH
"close_all", ),
None,
ErrorSeverity.HIGH
)
) )
def get_status(self) -> ConnectionStatus: def get_status(self) -> ConnectionStatus:
@@ -403,27 +403,25 @@ class DatabaseConnectionManager:
Connection status information Connection status information
""" """
active_connections = sum( active_connections = sum(
1 for info in self._connection_info.values() 1
for info in self._connection_info.values()
if info.state == ConnectionState.IN_USE if info.state == ConnectionState.IN_USE
) )
return ConnectionStatus( return ConnectionStatus(
state="healthy" if active_connections < self.pool_size else "exhausted", state="healthy" if active_connections < self.pool_size else "exhausted",
created_at=min( created_at=min(
info.created_at.isoformat() info.created_at.isoformat() for info in self._connection_info.values()
for info in self._connection_info.values()
), ),
last_used=max( last_used=max(
info.last_used.isoformat() info.last_used.isoformat() for info in self._connection_info.values()
for info in self._connection_info.values()
), ),
error=None, error=None,
transaction_count=sum( transaction_count=sum(
info.transaction_count info.transaction_count for info in self._connection_info.values()
for info in self._connection_info.values()
), ),
pool_size=self.pool_size, pool_size=self.pool_size,
available_connections=self.pool_size - active_connections available_connections=self.pool_size - active_connections,
) )
def get_metrics(self) -> ConnectionMetrics: def get_metrics(self) -> ConnectionMetrics:
@@ -434,33 +432,33 @@ class DatabaseConnectionManager:
Connection metrics information Connection metrics information
""" """
total_transactions = sum( total_transactions = sum(
info.transaction_count info.transaction_count for info in self._connection_info.values()
for info in self._connection_info.values()
)
total_errors = sum(
info.error_count
for info in self._connection_info.values()
) )
total_errors = sum(info.error_count for info in self._connection_info.values())
total_time = sum( total_time = sum(
info.total_transaction_time info.total_transaction_time for info in self._connection_info.values()
for info in self._connection_info.values()
) )
return ConnectionMetrics( return ConnectionMetrics(
total_connections=len(self._connection_info), total_connections=len(self._connection_info),
active_connections=sum( active_connections=sum(
1 for info in self._connection_info.values() 1
for info in self._connection_info.values()
if info.state == ConnectionState.IN_USE if info.state == ConnectionState.IN_USE
), ),
idle_connections=sum( idle_connections=sum(
1 for info in self._connection_info.values() 1
for info in self._connection_info.values()
if info.state == ConnectionState.AVAILABLE if info.state == ConnectionState.AVAILABLE
), ),
failed_connections=sum( failed_connections=sum(
1 for info in self._connection_info.values() 1
for info in self._connection_info.values()
if info.state == ConnectionState.ERROR if info.state == ConnectionState.ERROR
), ),
total_transactions=total_transactions, total_transactions=total_transactions,
failed_transactions=total_errors, failed_transactions=total_errors,
average_transaction_time=total_time / total_transactions if total_transactions > 0 else 0.0 average_transaction_time=(
total_time / total_transactions if total_transactions > 0 else 0.0
),
) )

13
videoarchiver/database/schema_manager.py
View File

@@ -7,12 +7,13 @@ from typing import List, Dict, Any, Optional, TypedDict, ClassVar, Union
from enum import Enum, auto from enum import Enum, auto
from datetime import datetime from datetime import datetime
#try: # try:
# Try relative imports first # Try relative imports first
from ..utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity from utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
#except ImportError:
# Fall back to absolute imports if relative imports fail # except ImportError:
# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity # Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
logger = logging.getLogger("DBSchemaManager") logger = logging.getLogger("DBSchemaManager")

6
videoarchiver/database/video_archive_db.py
View File

@@ -6,9 +6,9 @@ from typing import Optional, Dict, Any, List
# try: # try:
# Try relative imports first # Try relative imports first
from .schema_manager import DatabaseSchemaManager from schema_manager import DatabaseSchemaManager
from .query_manager import DatabaseQueryManager from query_manager import DatabaseQueryManager
from .connection_manager import DatabaseConnectionManager from connection_manager import DatabaseConnectionManager
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

12
videoarchiver/ffmpeg/__init__.py
View File

@@ -20,12 +20,12 @@ logger = logging.getLogger("VideoArchiver")
# Import components after logging is configured # Import components after logging is configured
#try: #try:
# Try relative imports first # Try relative imports first
from .ffmpeg_manager import FFmpegManager from ffmpeg_manager import FFmpegManager
from .video_analyzer import VideoAnalyzer from video_analyzer import VideoAnalyzer
from .gpu_detector import GPUDetector from gpu_detector import GPUDetector
from .encoder_params import EncoderParams from encoder_params import EncoderParams
from .ffmpeg_downloader import FFmpegDownloader from ffmpeg_downloader import FFmpegDownloader
from .exceptions import ( from exceptions import (
FFmpegError, FFmpegError,
DownloadError, DownloadError,
VerificationError, VerificationError,

BIN
videoarchiver/ffmpeg/__pycache__/__init__.cpython-312.pyc
View File

Binary file not shown.

BIN
videoarchiver/ffmpeg/__pycache__/exceptions.cpython-312.pyc
View File

Binary file not shown.

BIN
videoarchiver/ffmpeg/__pycache__/ffmpeg_manager.cpython-312.pyc
View File

Binary file not shown.

BIN
videoarchiver/ffmpeg/__pycache__/gpu_detector.cpython-312.pyc
View File

Binary file not shown.

BIN
videoarchiver/ffmpeg/__pycache__/video_analyzer.cpython-312.pyc
View File

Binary file not shown.

6
videoarchiver/ffmpeg/binary_manager.py
View File

@@ -7,15 +7,15 @@ from typing import Dict, Optional
#try: #try:
# Try relative imports first # Try relative imports first
from .exceptions import ( from exceptions import (
FFmpegError, FFmpegError,
DownloadError, DownloadError,
VerificationError, VerificationError,
PermissionError, PermissionError,
FFmpegNotFoundError FFmpegNotFoundError
) )
from .ffmpeg_downloader import FFmpegDownloader from ffmpeg_downloader import FFmpegDownloader
from .verification_manager import VerificationManager from verification_manager import VerificationManager
#except ImportError: #except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail
# from videoarchiver.ffmpeg.exceptions import ( # from videoarchiver.ffmpeg.exceptions import (

2
videoarchiver/ffmpeg/encoder_params.py
View File

@@ -6,7 +6,7 @@ from typing import Dict, Any
#try: #try:
# Try relative imports first # Try relative imports first
from .exceptions import CompressionError, QualityError, BitrateError from exceptions import CompressionError, QualityError, BitrateError
#except ImportError: #except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail
# from videoarchiver.ffmpeg.exceptions import CompressionError, QualityError, BitrateError # from videoarchiver.ffmpeg.exceptions import CompressionError, QualityError, BitrateError

2
videoarchiver/ffmpeg/ffmpeg_downloader.py
View File

@@ -18,7 +18,7 @@ import lzma
# try: # try:
# Try relative imports first # Try relative imports first
from .exceptions import DownloadError from exceptions import DownloadError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

14
videoarchiver/ffmpeg/ffmpeg_manager.py
View File

@@ -8,13 +8,13 @@ from typing import Dict, Any, Optional
# try: # try:
# Try relative imports first # Try relative imports first
from .exceptions import FFmpegError, AnalysisError, FFmpegNotFoundError from exceptions import FFmpegError, AnalysisError, FFmpegNotFoundError
from .gpu_detector import GPUDetector from gpu_detector import GPUDetector
from .video_analyzer import VideoAnalyzer from video_analyzer import VideoAnalyzer
from .encoder_params import EncoderParams from encoder_params import EncoderParams
from .process_manager import ProcessManager from process_manager import ProcessManager
from .verification_manager import VerificationManager from verification_manager import VerificationManager
from .binary_manager import BinaryManager from binary_manager import BinaryManager
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

9
videoarchiver/info.json
View File

@@ -23,10 +23,15 @@
"download", "download",
"automation", "automation",
"queue", "queue",
"ffmpeg" "ffmpeg",
"discord"
], ],
"min_bot_version": "3.5.0", "min_bot_version": "3.5.0",
"min_python_version": [3, 8, 0], "min_python_version": [
3,
8,
0
],
"hidden": false, "hidden": false,
"disabled": false, "disabled": false,
"type": "COG", "type": "COG",

20
videoarchiver/processor/__init__.py
View File

@@ -1,10 +1,10 @@
"""Video processing module for VideoArchiver""" """Video processing module for VideoArchiver"""
from typing import Dict, Any, Optional, Union, List, Tuple from typing import Dict, Any, Optional, Union, List, Tuple
import discord # type: ignore import discord # type: ignore
# Import constants first since they have no dependencies # Import constants first since they have no dependencies
from .constants import ( from constants import (
REACTIONS, REACTIONS,
ReactionType, ReactionType,
ReactionEmojis, ReactionEmojis,
@@ -14,10 +14,10 @@ from .constants import (
) )
# Import core components # Import core components
from .core import VideoProcessor from core import VideoProcessor
# Import URL related components # Import URL related components
from .url_extractor import ( from url_extractor import (
URLExtractor, URLExtractor,
URLMetadata, URLMetadata,
URLPattern, URLPattern,
@@ -28,7 +28,7 @@ from .url_extractor import (
) )
# Import validation components # Import validation components
from .message_validator import ( from message_validator import (
MessageValidator, MessageValidator,
ValidationContext, ValidationContext,
ValidationRule, ValidationRule,
@@ -41,7 +41,7 @@ from .message_validator import (
) )
# Import reaction handlers # Import reaction handlers
from .reactions import ( from reactions import (
handle_archived_reaction, handle_archived_reaction,
update_queue_position_reaction, update_queue_position_reaction,
update_progress_reaction, update_progress_reaction,
@@ -49,12 +49,12 @@ from .reactions import (
) )
# Import progress tracking # Import progress tracking
from ..utils.progress_tracker import ProgressTracker from utils.progress_tracker import ProgressTracker
# Import handlers after other dependencies are loaded # Import handlers after other dependencies are loaded
from .message_handler import MessageHandler from message_handler import MessageHandler
from .queue_handler import QueueHandler from queue_handler import QueueHandler
from .queue_processor import QueueProcessor from queue_processor import QueueProcessor
# Export public classes and constants # Export public classes and constants
__all__ = [ __all__ = [

6
videoarchiver/processor/cleanup_manager.py
View File

@@ -20,12 +20,12 @@ from typing import (
from datetime import datetime, timedelta from datetime import datetime, timedelta
if TYPE_CHECKING: if TYPE_CHECKING:
from .queue_handler import QueueHandler from queue_handler import QueueHandler
# try: # try:
# Try relative imports first # Try relative imports first
from ..ffmpeg.ffmpeg_manager import FFmpegManager from ffmpeg.ffmpeg_manager import FFmpegManager
from ..utils.exceptions import CleanupError from utils.exceptions import CleanupError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail
# # from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager # # from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager

8
videoarchiver/processor/core.py
View File

@@ -8,7 +8,7 @@ from typing import Any, ClassVar, Dict, List, Optional, Tuple, TYPE_CHECKING
import discord # type: ignore import discord # type: ignore
from discord.ext import commands # type: ignore from discord.ext import commands # type: ignore
from ..core.types import ( from core.c_types import (
ComponentState, ComponentState,
ProcessorState, ProcessorState,
ComponentStatus, ComponentStatus,
@@ -16,9 +16,9 @@ from ..core.types import (
IConfigManager, IConfigManager,
IQueueManager, IQueueManager,
) )
from .constants import REACTIONS from constants import REACTIONS
from ..utils.progress_tracker import ProgressTracker from utils.progress_tracker import ProgressTracker
from ..utils.exceptions import ProcessorError from utils.exceptions import ProcessorError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

12
videoarchiver/processor/message_handler.py
View File

@@ -21,12 +21,12 @@ from discord.ext import commands # type: ignore
# try: # try:
# Try relative imports first # Try relative imports first
from ..config_manager import ConfigManager from config_manager import ConfigManager
from .constants import REACTIONS from constants import REACTIONS
from .message_validator import MessageValidator, ValidationError from message_validator import MessageValidator, ValidationError
from .url_extractor import URLExtractor, URLMetadata from url_extractor import URLExtractor, URLMetadata
from ..queue.types import QueuePriority from queue.q_types import QueuePriority
from ..utils.exceptions import MessageHandlerError from utils.exceptions import MessageHandlerError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

2
videoarchiver/processor/message_validator.py
View File

@@ -9,7 +9,7 @@ import discord # type: ignore
#try: #try:
# Try relative imports first # Try relative imports first
from ..utils.exceptions import ValidationError from utils.exceptions import ValidationError
#except ImportError: #except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import ValidationError # from videoarchiver.utils.exceptions import ValidationError

31
videoarchiver/processor/queue_handler.py
View File

@@ -10,15 +10,16 @@ import discord # type: ignore
# try: # try:
# Try relative imports first # Try relative imports first
from .. import utils # from . import utils
from ..database.video_archive_db import VideoArchiveDB from database.video_archive_db import VideoArchiveDB
from ..utils.download_manager import DownloadManager from utils.download_manager import DownloadManager
from ..utils.message_manager import MessageManager from utils.message_manager import MessageManager
from ..utils.exceptions import QueueHandlerError from utils.exceptions import QueueHandlerError
from ..queue.models import QueueItem from queue.models import QueueItem
from ..config_manager import ConfigManager from config_manager import ConfigManager
from . import progress_tracker # Import from processor package
from .constants import REACTIONS from utils.progress_tracker import ProgressTracker # Import from processor package
from constants import REACTIONS
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail
@@ -66,11 +67,13 @@ class QueueHandler:
self, self,
bot: discord.Client, bot: discord.Client,
config_manager: ConfigManager, config_manager: ConfigManager,
progress_tracker: ProgressTracker,
components: Dict[int, Dict[str, Any]], components: Dict[int, Dict[str, Any]],
db: Optional[VideoArchiveDB] = None, db: Optional[VideoArchiveDB] = None,
) -> None: ) -> None:
self.bot = bot self.bot = bot
self.config_manager = config_manager self.config_manager = config_manager
self.progress_tracker = progress_tracker
self.components = components self.components = components
self.db = db self.db = db
self._unloading = False self._unloading = False
@@ -392,11 +395,13 @@ class QueueHandler:
return return
# Update progress tracking # Update progress tracking
progress_tracker.update_download_progress( ProgressTracker.update_download_progress(
url, url,
{ {
"percent": progress, "percent": progress,
"last_update": datetime.utcnow().isoformat(), "last_update": datetime.now(
datetime.timezone.utc
)().isoformat(),
}, },
) )
@@ -439,9 +444,9 @@ class QueueHandler:
download_task, timeout=self.DOWNLOAD_TIMEOUT download_task, timeout=self.DOWNLOAD_TIMEOUT
) )
if success: if success:
progress_tracker.complete_download(url) ProgressTracker.complete_download(url)
else: else:
progress_tracker.increment_download_retries(url) ProgressTracker.increment_download_retries(url)
return success, file_path, error return success, file_path, error
except asyncio.TimeoutError: except asyncio.TimeoutError:

149
videoarchiver/processor/queue_processor.py
View File

@@ -1,97 +1,110 @@
"""Queue processing functionality for video processing"""
import logging import logging
import asyncio import asyncio
from typing import List, Optional, Dict, Any, Set, ClassVar from typing import List, Optional, Dict, Any, Set, ClassVar
from datetime import datetime from datetime import datetime
import sys
from pathlib import Path
#try: # Get the parent directory (videoarchiver root)
# Try relative imports first BASE_DIR = Path(__file__).resolve().parent.parent
from ..queue.types import QueuePriority, QueueMetrics, ProcessingMetrics if str(BASE_DIR) not in sys.path:
from ..queue.models import QueueItem sys.path.insert(0, str(BASE_DIR))
#except ImportError:
# Fall back to absolute imports if relative imports fail # Use non-relative imports
# from videoarchiver.queue.types import QueuePriority, QueueMetrics, ProcessingMetrics from queue.q_types import QueuePriority, QueueMetrics, ProcessingMetrics
# from videoarchiver.queue.models import QueueItem from queue.models import QueueItem
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class QueueProcessor: class QueueProcessor:
"""Handles processing of video queue items""" """
Handles the processing of queue items with priority-based scheduling.
"""
_active_items: ClassVar[Set[int]] = set() # Class variables for tracking global state
_processing_lock: ClassVar[asyncio.Lock] = asyncio.Lock() active_items: ClassVar[Set[str]] = set()
processing_metrics: ClassVar[Dict[str, ProcessingMetrics]] = {}
def __init__(self, queue_manager): def __init__(self):
"""Initialize queue processor self.queue_metrics = QueueMetrics()
self.processing_lock = asyncio.Lock()
self.is_running = False
self._current_item: Optional[QueueItem] = None
self._priority_queues: Dict[QueuePriority, List[QueueItem]] = {
priority: [] for priority in QueuePriority
}
Args: @property
queue_manager: Queue manager instance to handle queue operations def current_item(self) -> Optional[QueueItem]:
"""Get the currently processing item."""
return self._current_item
def add_item(self, item: QueueItem) -> bool:
""" """
self.queue_manager = queue_manager Add an item to the appropriate priority queue.
self._metrics = ProcessingMetrics()
async def process_urls(self, message, urls, priority: QueuePriority = QueuePriority.NORMAL) -> None:
"""Process URLs from a message
Args: Args:
message: Discord message containing URLs item: QueueItem to add
urls: List of URLs to process
priority: Processing priority level
"""
for url_metadata in urls:
await self.queue_manager.add_to_queue(
url=url_metadata.url,
message_id=message.id,
channel_id=message.channel.id,
guild_id=message.guild.id,
author_id=message.author.id,
priority=priority.value
)
async def process_item(self, item: QueueItem) -> bool:
"""Process a single queue item
Args:
item: Queue item to process
Returns: Returns:
bool: Success status bool: True if item was added successfully
""" """
if item.id in self._active_items: if item.id in self.active_items:
logger.warning(f"Item {item.id} is already being processed") logger.warning(f"Item {item.id} is already in queue")
return False return False
try: self._priority_queues[item.priority].append(item)
self._active_items.add(item.id) self.active_items.add(item.id)
start_time = datetime.now() self.queue_metrics.total_items += 1
logger.info(f"Added item {item.id} to {item.priority.name} priority queue")
return True
# Process item logic here def remove_item(self, item_id: str) -> Optional[QueueItem]:
# Placeholder for actual video processing """
await asyncio.sleep(1) Remove an item from any priority queue.
processing_time = (datetime.now() - start_time).total_seconds() Args:
self._update_metrics(processing_time, True, item.size) item_id: ID of item to remove
return True
except Exception as e: Returns:
logger.error(f"Error processing item {item.id}: {str(e)}") Optional[QueueItem]: Removed item if found, None otherwise
self._update_metrics(0, False, 0) """
return False for priority in QueuePriority:
queue = self._priority_queues[priority]
finally: for item in queue:
self._active_items.remove(item.id) if item.id == item_id:
queue.remove(item)
self.active_items.discard(item_id)
self.queue_metrics.total_items -= 1
logger.info(
f"Removed item {item_id} from {priority.name} priority queue"
)
return item
return None
def _update_metrics(self, processing_time: float, success: bool, size: int) -> None: def _update_metrics(self, processing_time: float, success: bool, size: int) -> None:
"""Update processing metrics""" """
Update processing metrics.
Args:
processing_time: Time taken to process the item
success: Whether processing was successful
size: Size of the processed item
"""
if success: if success:
self._metrics.record_success(processing_time) self.queue_metrics.record_success(processing_time)
else: else:
self._metrics.record_failure("Processing error") self.queue_metrics.record_failure("Processing error")
def get_metrics(self) -> QueueMetrics: def get_metrics(self) -> QueueMetrics:
"""Get current processing metrics""" """
total = self._metrics.total_processed Get current processing metrics.
Returns:
QueueMetrics: Current queue processing metrics
"""
total = self.queue_metrics.total_processed
if total == 0: if total == 0:
return QueueMetrics( return QueueMetrics(
total_items=0, total_items=0,
@@ -103,8 +116,8 @@ class QueueProcessor:
return QueueMetrics( return QueueMetrics(
total_items=total, total_items=total,
processing_time=self._metrics.avg_processing_time, processing_time=self.queue_metrics.avg_processing_time,
success_rate=self._metrics.successful / total, success_rate=self.queue_metrics.successful / total,
error_rate=self._metrics.failed / total, error_rate=self.queue_metrics.failed / total,
average_size=0, # This would need to be tracked separately if needed average_size=0, # This would need to be tracked separately if needed
) )

4
videoarchiver/processor/reactions.py
View File

@@ -9,13 +9,13 @@ from urllib.parse import urlparse
# try: # try:
# Try relative imports first # Try relative imports first
from ..processor.constants import ( from processor.constants import (
REACTIONS, REACTIONS,
ReactionType, ReactionType,
get_reaction, get_reaction,
get_progress_emoji, get_progress_emoji,
) )
from ..database.video_archive_db import VideoArchiveDB from database.video_archive_db import VideoArchiveDB
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

2
videoarchiver/processor/status_display.py
View File

@@ -20,7 +20,7 @@ import discord # type: ignore
# try: # try:
# Try relative imports first # Try relative imports first
from ..utils.exceptions import DisplayError from utils.exceptions import DisplayError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

86
videoarchiver/queue/__init__.py
View File

@@ -1,58 +1,58 @@
"""Queue management package for video processing""" """Queue management package for video processing"""
from .models import QueueItem, QueueMetrics from models import QueueItem, QueueMetrics
from .types import QueuePriority, ProcessingMetrics from q_types import QueuePriority, ProcessingMetrics
from .manager import EnhancedVideoQueueManager from manager import EnhancedVideoQueueManager
from .persistence import QueuePersistenceManager, QueueError from persistence import QueuePersistenceManager, QueueError
from .monitoring import QueueMonitor, MonitoringError from monitoring import QueueMonitor, MonitoringError
from .cleanup import QueueCleaner, CleanupError from cleanup import QueueCleaner, CleanupError
from .recovery_manager import RecoveryManager from recovery_manager import RecoveryManager
from .state_manager import StateManager from state_manager import QueueStateManager
from .metrics_manager import MetricsManager from metrics_manager import QueueMetricsManager
from .processor import QueueProcessor from processor import QueueProcessor
from .health_checker import HealthChecker from health_checker import HealthChecker
# Importing from cleaners subdirectory # Importing from cleaners subdirectory
from .cleaners import GuildCleaner, HistoryCleaner, TrackingCleaner from cleaners import GuildCleaner, HistoryCleaner, TrackingCleaner
# Corrected relative imports from utils # Corrected relative imports from utils
from ..utils.compression_handler import CompressionHandler from utils.compression_handler import CompressionHandler
from ..utils.directory_manager import DirectoryManager from utils.directory_manager import DirectoryManager
from ..utils.download_manager import DownloadManager from utils.download_manager import DownloadManager
from ..utils.file_operations import FileOperations from utils.file_operations import FileOperations
from ..utils.progress_tracker import ProgressTracker from utils.progress_tracker import ProgressTracker
from ..utils.url_validator import UrlValidator from processor.url_extractor import URLValidator
__all__ = [ __all__ = [
# Queue models and types # Queue models and types
'QueueItem', "QueueItem",
'QueueMetrics', "QueueMetrics",
'QueuePriority', "QueuePriority",
'ProcessingMetrics', "ProcessingMetrics",
# Core components # Core components
'EnhancedVideoQueueManager', "EnhancedVideoQueueManager",
'QueuePersistenceManager', "QueuePersistenceManager",
'QueueMonitor', "QueueMonitor",
'QueueCleaner', "QueueCleaner",
'QueueProcessor', "QueueProcessor",
'HealthChecker', "HealthChecker",
# Managers # Managers
'RecoveryManager', "RecoveryManager",
'StateManager', "QueueStateManager",
'MetricsManager', "QueueMetricsManager",
# Cleaners # Cleaners
'GuildCleaner', "GuildCleaner",
'HistoryCleaner', "HistoryCleaner",
'TrackingCleaner', "TrackingCleaner",
# Utility handlers # Utility handlers
'CompressionHandler', "CompressionHandler",
'DirectoryManager', "DirectoryManager",
'DownloadManager', "DownloadManager",
'FileOperations', "FileOperations",
'ProgressTracker', "ProgressTracker",
'UrlValidator', "URLValidator",
# Errors # Errors
'QueueError', "QueueError",
'MonitoringError', "MonitoringError",
'CleanupError', "CleanupError",
] ]

6
videoarchiver/queue/cleaners/__init__.py
View File

@@ -1,8 +1,8 @@
"""Queue cleaning functionality""" """Queue cleaning functionality"""
from .guild_cleaner import GuildCleaner from guild_cleaner import GuildCleaner
from .history_cleaner import HistoryCleaner from history_cleaner import HistoryCleaner
from .tracking_cleaner import TrackingCleaner from tracking_cleaner import TrackingCleaner
__all__ = [ __all__ = [
'GuildCleaner', 'GuildCleaner',

2
videoarchiver/queue/cleaners/guild_cleaner.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass, field
from typing import Dict, List, Set, Tuple, Any, Optional from typing import Dict, List, Set, Tuple, Any, Optional
from datetime import datetime from datetime import datetime
from ..models import QueueItem from models import QueueItem
logger = logging.getLogger("GuildCleaner") logger = logging.getLogger("GuildCleaner")

2
videoarchiver/queue/cleaners/history_cleaner.py
View File

@@ -6,7 +6,7 @@ from dataclasses import dataclass, field
from typing import Dict, Optional, List, Any, Set from typing import Dict, Optional, List, Any, Set
from datetime import datetime, timedelta from datetime import datetime, timedelta
from ..models import QueueItem from models import QueueItem
logger = logging.getLogger("HistoryCleaner") logger = logging.getLogger("HistoryCleaner")

162
videoarchiver/queue/cleaners/tracking_cleaner.py
View File

@@ -7,34 +7,42 @@ from dataclasses import dataclass, field
from typing import Dict, List, Set, Tuple, Any, Optional from typing import Dict, List, Set, Tuple, Any, Optional
from datetime import datetime from datetime import datetime
from ..models import QueueItem from models import QueueItem
logger = logging.getLogger("TrackingCleaner") logger = logging.getLogger("TrackingCleaner")
class TrackingCleanupStrategy(Enum): class TrackingCleanupStrategy(Enum):
"""Tracking cleanup strategies""" """Tracking cleanup strategies"""
AGGRESSIVE = "aggressive" # Remove all invalid entries
AGGRESSIVE = "aggressive" # Remove all invalid entries
CONSERVATIVE = "conservative" # Keep recent invalid entries CONSERVATIVE = "conservative" # Keep recent invalid entries
BALANCED = "balanced" # Balance between cleanup and retention BALANCED = "balanced" # Balance between cleanup and retention
class TrackingType(Enum): class TrackingType(Enum):
"""Types of tracking data""" """Types of tracking data"""
GUILD = "guild" GUILD = "guild"
CHANNEL = "channel" CHANNEL = "channel"
URL = "url" URL = "url"
@dataclass @dataclass
class TrackingCleanupConfig: class TrackingCleanupConfig:
"""Configuration for tracking cleanup""" """Configuration for tracking cleanup"""
batch_size: int = 100 batch_size: int = 100
retention_period: int = 3600 # 1 hour retention_period: int = 3600 # 1 hour
validate_urls: bool = True validate_urls: bool = True
cleanup_empty: bool = True cleanup_empty: bool = True
max_invalid_ratio: float = 0.5 # 50% invalid threshold max_invalid_ratio: float = 0.5 # 50% invalid threshold
@dataclass @dataclass
class TrackingCleanupResult: class TrackingCleanupResult:
"""Result of a tracking cleanup operation""" """Result of a tracking cleanup operation"""
timestamp: datetime timestamp: datetime
strategy: TrackingCleanupStrategy strategy: TrackingCleanupStrategy
items_cleaned: int items_cleaned: int
@@ -45,6 +53,7 @@ class TrackingCleanupResult:
final_counts: Dict[str, int] final_counts: Dict[str, int]
error: Optional[str] = None error: Optional[str] = None
class TrackingValidator: class TrackingValidator:
"""Validates tracking data""" """Validates tracking data"""
@@ -64,6 +73,7 @@ class TrackingValidator:
except Exception: except Exception:
return False return False
class TrackingCleanupTracker: class TrackingCleanupTracker:
"""Tracks cleanup operations""" """Tracks cleanup operations"""
@@ -94,9 +104,7 @@ class TrackingCleanupTracker:
"total_guilds_cleaned": self.total_guilds_cleaned, "total_guilds_cleaned": self.total_guilds_cleaned,
"total_channels_cleaned": self.total_channels_cleaned, "total_channels_cleaned": self.total_channels_cleaned,
"last_cleanup": ( "last_cleanup": (
self.last_cleanup.isoformat() self.last_cleanup.isoformat() if self.last_cleanup else None
if self.last_cleanup
else None
), ),
"recent_cleanups": [ "recent_cleanups": [
{ {
@@ -105,19 +113,20 @@ class TrackingCleanupTracker:
"items_cleaned": r.items_cleaned, "items_cleaned": r.items_cleaned,
"guilds_cleaned": r.guilds_cleaned, "guilds_cleaned": r.guilds_cleaned,
"channels_cleaned": r.channels_cleaned, "channels_cleaned": r.channels_cleaned,
"duration": r.duration "duration": r.duration,
} }
for r in self.history[-5:] # Last 5 cleanups for r in self.history[-5:] # Last 5 cleanups
] ],
} }
class TrackingCleaner: class TrackingCleaner:
"""Handles cleanup of queue tracking data""" """Handles cleanup of queue tracking data"""
def __init__( def __init__(
self, self,
strategy: TrackingCleanupStrategy = TrackingCleanupStrategy.BALANCED, strategy: TrackingCleanupStrategy = TrackingCleanupStrategy.BALANCED,
config: Optional[TrackingCleanupConfig] = None config: Optional[TrackingCleanupConfig] = None,
): ):
self.strategy = strategy self.strategy = strategy
self.config = config or TrackingCleanupConfig() self.config = config or TrackingCleanupConfig()
@@ -129,17 +138,14 @@ class TrackingCleaner:
guild_queues: Dict[int, Set[str]], guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]],
queue: List[QueueItem], queue: List[QueueItem],
processing: Dict[str, QueueItem] processing: Dict[str, QueueItem],
) -> Tuple[int, Dict[str, int]]: ) -> Tuple[int, Dict[str, int]]:
"""Clean up tracking data""" """Clean up tracking data"""
start_time = datetime.utcnow() start_time = datetime.utcnow()
try: try:
# Get initial counts # Get initial counts
initial_counts = self._get_tracking_counts( initial_counts = self._get_tracking_counts(guild_queues, channel_queues)
guild_queues,
channel_queues
)
# Get valid URLs # Get valid URLs
valid_urls = self._get_valid_urls(queue, processing) valid_urls = self._get_valid_urls(queue, processing)
@@ -151,21 +157,15 @@ class TrackingCleaner:
if self.strategy == TrackingCleanupStrategy.AGGRESSIVE: if self.strategy == TrackingCleanupStrategy.AGGRESSIVE:
cleaned = await self._aggressive_cleanup( cleaned = await self._aggressive_cleanup(
guild_queues, guild_queues, channel_queues, valid_urls
channel_queues,
valid_urls
) )
elif self.strategy == TrackingCleanupStrategy.CONSERVATIVE: elif self.strategy == TrackingCleanupStrategy.CONSERVATIVE:
cleaned = await self._conservative_cleanup( cleaned = await self._conservative_cleanup(
guild_queues, guild_queues, channel_queues, valid_urls
channel_queues,
valid_urls
) )
else: # BALANCED else: # BALANCED
cleaned = await self._balanced_cleanup( cleaned = await self._balanced_cleanup(
guild_queues, guild_queues, channel_queues, valid_urls
channel_queues,
valid_urls
) )
items_cleaned = cleaned[0] items_cleaned = cleaned[0]
@@ -173,10 +173,7 @@ class TrackingCleaner:
channels_cleaned = cleaned[2] channels_cleaned = cleaned[2]
# Get final counts # Get final counts
final_counts = self._get_tracking_counts( final_counts = self._get_tracking_counts(guild_queues, channel_queues)
guild_queues,
channel_queues
)
# Record cleanup result # Record cleanup result
duration = (datetime.utcnow() - start_time).total_seconds() duration = (datetime.utcnow() - start_time).total_seconds()
@@ -188,37 +185,39 @@ class TrackingCleaner:
channels_cleaned=channels_cleaned, channels_cleaned=channels_cleaned,
duration=duration, duration=duration,
initial_counts=initial_counts, initial_counts=initial_counts,
final_counts=final_counts final_counts=final_counts,
) )
self.tracker.record_cleanup(result) self.tracker.record_cleanup(result)
logger.info(self.format_tracking_cleanup_report( logger.info(
initial_counts, self.format_tracking_cleanup_report(
final_counts, initial_counts, final_counts, duration
duration )
)) )
return items_cleaned, initial_counts return items_cleaned, initial_counts
except Exception as e: except Exception as e:
logger.error(f"Error cleaning tracking data: {e}") logger.error(f"Error cleaning tracking data: {e}")
self.tracker.record_cleanup(TrackingCleanupResult( self.tracker.record_cleanup(
timestamp=datetime.utcnow(), TrackingCleanupResult(
strategy=self.strategy, timestamp=datetime.utcnow(),
items_cleaned=0, strategy=self.strategy,
guilds_cleaned=0, items_cleaned=0,
channels_cleaned=0, guilds_cleaned=0,
duration=0, channels_cleaned=0,
initial_counts={}, duration=0,
final_counts={}, initial_counts={},
error=str(e) final_counts={},
)) error=str(e),
)
)
raise raise
async def _aggressive_cleanup( async def _aggressive_cleanup(
self, self,
guild_queues: Dict[int, Set[str]], guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]],
valid_urls: Set[str] valid_urls: Set[str],
) -> Tuple[int, int, int]: ) -> Tuple[int, int, int]:
"""Perform aggressive cleanup""" """Perform aggressive cleanup"""
items_cleaned = 0 items_cleaned = 0
@@ -227,18 +226,14 @@ class TrackingCleaner:
# Clean guild tracking # Clean guild tracking
guild_cleaned = await self._cleanup_guild_tracking( guild_cleaned = await self._cleanup_guild_tracking(
guild_queues, guild_queues, valid_urls, validate_all=True
valid_urls,
validate_all=True
) )
items_cleaned += guild_cleaned[0] items_cleaned += guild_cleaned[0]
guilds_cleaned += guild_cleaned[1] guilds_cleaned += guild_cleaned[1]
# Clean channel tracking # Clean channel tracking
channel_cleaned = await self._cleanup_channel_tracking( channel_cleaned = await self._cleanup_channel_tracking(
channel_queues, channel_queues, valid_urls, validate_all=True
valid_urls,
validate_all=True
) )
items_cleaned += channel_cleaned[0] items_cleaned += channel_cleaned[0]
channels_cleaned += channel_cleaned[1] channels_cleaned += channel_cleaned[1]
@@ -249,7 +244,7 @@ class TrackingCleaner:
self, self,
guild_queues: Dict[int, Set[str]], guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]],
valid_urls: Set[str] valid_urls: Set[str],
) -> Tuple[int, int, int]: ) -> Tuple[int, int, int]:
"""Perform conservative cleanup""" """Perform conservative cleanup"""
items_cleaned = 0 items_cleaned = 0
@@ -261,9 +256,7 @@ class TrackingCleaner:
invalid_ratio = len(urls - valid_urls) / len(urls) if urls else 0 invalid_ratio = len(urls - valid_urls) / len(urls) if urls else 0
if invalid_ratio > self.config.max_invalid_ratio: if invalid_ratio > self.config.max_invalid_ratio:
cleaned = await self._cleanup_guild_tracking( cleaned = await self._cleanup_guild_tracking(
{guild_id: urls}, {guild_id: urls}, valid_urls, validate_all=False
valid_urls,
validate_all=False
) )
items_cleaned += cleaned[0] items_cleaned += cleaned[0]
guilds_cleaned += cleaned[1] guilds_cleaned += cleaned[1]
@@ -272,9 +265,7 @@ class TrackingCleaner:
invalid_ratio = len(urls - valid_urls) / len(urls) if urls else 0 invalid_ratio = len(urls - valid_urls) / len(urls) if urls else 0
if invalid_ratio > self.config.max_invalid_ratio: if invalid_ratio > self.config.max_invalid_ratio:
cleaned = await self._cleanup_channel_tracking( cleaned = await self._cleanup_channel_tracking(
{channel_id: urls}, {channel_id: urls}, valid_urls, validate_all=False
valid_urls,
validate_all=False
) )
items_cleaned += cleaned[0] items_cleaned += cleaned[0]
channels_cleaned += cleaned[1] channels_cleaned += cleaned[1]
@@ -285,7 +276,7 @@ class TrackingCleaner:
self, self,
guild_queues: Dict[int, Set[str]], guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]],
valid_urls: Set[str] valid_urls: Set[str],
) -> Tuple[int, int, int]: ) -> Tuple[int, int, int]:
"""Perform balanced cleanup""" """Perform balanced cleanup"""
items_cleaned = 0 items_cleaned = 0
@@ -294,18 +285,14 @@ class TrackingCleaner:
# Clean guild tracking with validation # Clean guild tracking with validation
guild_cleaned = await self._cleanup_guild_tracking( guild_cleaned = await self._cleanup_guild_tracking(
guild_queues, guild_queues, valid_urls, validate_all=self.config.validate_urls
valid_urls,
validate_all=self.config.validate_urls
) )
items_cleaned += guild_cleaned[0] items_cleaned += guild_cleaned[0]
guilds_cleaned += guild_cleaned[1] guilds_cleaned += guild_cleaned[1]
# Clean channel tracking with validation # Clean channel tracking with validation
channel_cleaned = await self._cleanup_channel_tracking( channel_cleaned = await self._cleanup_channel_tracking(
channel_queues, channel_queues, valid_urls, validate_all=self.config.validate_urls
valid_urls,
validate_all=self.config.validate_urls
) )
items_cleaned += channel_cleaned[0] items_cleaned += channel_cleaned[0]
channels_cleaned += channel_cleaned[1] channels_cleaned += channel_cleaned[1]
@@ -316,7 +303,7 @@ class TrackingCleaner:
self, self,
guild_queues: Dict[int, Set[str]], guild_queues: Dict[int, Set[str]],
valid_urls: Set[str], valid_urls: Set[str],
validate_all: bool validate_all: bool,
) -> Tuple[int, int]: ) -> Tuple[int, int]:
"""Clean up guild tracking data""" """Clean up guild tracking data"""
items_cleaned = 0 items_cleaned = 0
@@ -331,10 +318,11 @@ class TrackingCleaner:
original_size = len(guild_queues[guild_id]) original_size = len(guild_queues[guild_id])
guild_queues[guild_id] = { guild_queues[guild_id] = {
url for url in guild_queues[guild_id] url
for url in guild_queues[guild_id]
if ( if (
(not validate_all or self.validator.validate_url(url)) and (not validate_all or self.validator.validate_url(url))
url in valid_urls and url in valid_urls
) )
} }
items_cleaned += original_size - len(guild_queues[guild_id]) items_cleaned += original_size - len(guild_queues[guild_id])
@@ -355,7 +343,7 @@ class TrackingCleaner:
self, self,
channel_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]],
valid_urls: Set[str], valid_urls: Set[str],
validate_all: bool validate_all: bool,
) -> Tuple[int, int]: ) -> Tuple[int, int]:
"""Clean up channel tracking data""" """Clean up channel tracking data"""
items_cleaned = 0 items_cleaned = 0
@@ -370,10 +358,11 @@ class TrackingCleaner:
original_size = len(channel_queues[channel_id]) original_size = len(channel_queues[channel_id])
channel_queues[channel_id] = { channel_queues[channel_id] = {
url for url in channel_queues[channel_id] url
for url in channel_queues[channel_id]
if ( if (
(not validate_all or self.validator.validate_url(url)) and (not validate_all or self.validator.validate_url(url))
url in valid_urls and url in valid_urls
) )
} }
items_cleaned += original_size - len(channel_queues[channel_id]) items_cleaned += original_size - len(channel_queues[channel_id])
@@ -391,9 +380,7 @@ class TrackingCleaner:
return items_cleaned, channels_cleaned return items_cleaned, channels_cleaned
def _get_valid_urls( def _get_valid_urls(
self, self, queue: List[QueueItem], processing: Dict[str, QueueItem]
queue: List[QueueItem],
processing: Dict[str, QueueItem]
) -> Set[str]: ) -> Set[str]:
"""Get set of valid URLs""" """Get set of valid URLs"""
valid_urls = {item.url for item in queue} valid_urls = {item.url for item in queue}
@@ -401,28 +388,25 @@ class TrackingCleaner:
return valid_urls return valid_urls
def _get_tracking_counts( def _get_tracking_counts(
self, self, guild_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]]
guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]]
) -> Dict[str, int]: ) -> Dict[str, int]:
"""Get tracking data counts""" """Get tracking data counts"""
return { return {
'guilds': len(guild_queues), "guilds": len(guild_queues),
'channels': len(channel_queues), "channels": len(channel_queues),
'guild_urls': sum(len(urls) for urls in guild_queues.values()), "guild_urls": sum(len(urls) for urls in guild_queues.values()),
'channel_urls': sum(len(urls) for urls in channel_queues.values()) "channel_urls": sum(len(urls) for urls in channel_queues.values()),
} }
def format_tracking_cleanup_report( def format_tracking_cleanup_report(
self, self,
initial_counts: Dict[str, int], initial_counts: Dict[str, int],
final_counts: Dict[str, int], final_counts: Dict[str, int],
duration: float duration: float,
) -> str: ) -> str:
"""Format a tracking cleanup report""" """Format a tracking cleanup report"""
total_cleaned = ( total_cleaned = (initial_counts["guild_urls"] - final_counts["guild_urls"]) + (
(initial_counts['guild_urls'] - final_counts['guild_urls']) + initial_counts["channel_urls"] - final_counts["channel_urls"]
(initial_counts['channel_urls'] - final_counts['channel_urls'])
) )
return ( return (
@@ -446,7 +430,7 @@ class TrackingCleaner:
"retention_period": self.config.retention_period, "retention_period": self.config.retention_period,
"validate_urls": self.config.validate_urls, "validate_urls": self.config.validate_urls,
"cleanup_empty": self.config.cleanup_empty, "cleanup_empty": self.config.cleanup_empty,
"max_invalid_ratio": self.config.max_invalid_ratio "max_invalid_ratio": self.config.max_invalid_ratio,
}, },
"tracker": self.tracker.get_stats() "tracker": self.tracker.get_stats(),
} }

8
videoarchiver/queue/cleanup.py
View File

@@ -7,16 +7,16 @@ from dataclasses import dataclass, field
from typing import Dict, List, Set, Optional, Any, Tuple from typing import Dict, List, Set, Optional, Any, Tuple
from datetime import datetime, timedelta from datetime import datetime, timedelta
from .models import QueueItem, QueueMetrics from models import QueueItem, QueueMetrics
from .cleaners.history_cleaner import ( from cleaners.history_cleaner import (
HistoryCleaner, HistoryCleaner,
CleanupStrategy as HistoryStrategy CleanupStrategy as HistoryStrategy
) )
from .cleaners.guild_cleaner import ( from cleaners.guild_cleaner import (
GuildCleaner, GuildCleaner,
GuildCleanupStrategy GuildCleanupStrategy
) )
from .cleaners.tracking_cleaner import ( from cleaners.tracking_cleaner import (
TrackingCleaner, TrackingCleaner,
TrackingCleanupStrategy TrackingCleanupStrategy
) )

26
videoarchiver/queue/manager.py
View File

@@ -7,21 +7,22 @@ from dataclasses import dataclass, field
from typing import Optional, Tuple, Dict, Any, List, Set, Callable from typing import Optional, Tuple, Dict, Any, List, Set, Callable
from datetime import datetime, timedelta from datetime import datetime, timedelta
from ..core.types import IQueueManager, QueueState, ComponentStatus from core.c_types import IQueueManager, QueueState, ComponentStatus
from .state_manager import QueueStateManager from state_manager import QueueStateManager
from .processor import QueueProcessor from processor import QueueProcessor
from .metrics_manager import QueueMetricsManager from metrics_manager import QueueMetricsManager
from .persistence import QueuePersistenceManager from persistence import QueuePersistenceManager, QueueError
from .monitoring import QueueMonitor, MonitoringLevel from monitoring import QueueMonitor, MonitoringLevel
from .cleanup import QueueCleaner from cleanup import QueueCleaner, CleanupError
from .models import QueueItem, QueueError, CleanupError from models import QueueItem, QueueError
from .types import ProcessingStrategy from q_types import ProcessingStrategy
logger = logging.getLogger("QueueManager") logger = logging.getLogger("QueueManager")
class QueueMode(Enum): class QueueMode(Enum):
"""Queue processing modes""" """Queue processing modes"""
NORMAL = "normal" # Standard processing NORMAL = "normal" # Standard processing
BATCH = "batch" # Batch processing BATCH = "batch" # Batch processing
PRIORITY = "priority" # Priority-based processing PRIORITY = "priority" # Priority-based processing
@@ -31,6 +32,7 @@ class QueueMode(Enum):
@dataclass @dataclass
class QueueConfig: class QueueConfig:
"""Queue configuration settings""" """Queue configuration settings"""
max_retries: int = 3 max_retries: int = 3
retry_delay: int = 5 retry_delay: int = 5
max_queue_size: int = 1000 max_queue_size: int = 1000
@@ -47,6 +49,7 @@ class QueueConfig:
@dataclass @dataclass
class QueueStats: class QueueStats:
"""Queue statistics""" """Queue statistics"""
start_time: datetime = field(default_factory=datetime.utcnow) start_time: datetime = field(default_factory=datetime.utcnow)
total_processed: int = 0 total_processed: int = 0
total_failed: int = 0 total_failed: int = 0
@@ -230,7 +233,8 @@ class EnhancedVideoQueueManager(IQueueManager):
"monitoring": monitor_stats, "monitoring": monitor_stats,
"state": self.coordinator.state.value, "state": self.coordinator.state.value,
"mode": self.coordinator.mode.value, "mode": self.coordinator.mode.value,
"active": self.coordinator.state == QueueState.RUNNING and bool(processor_stats["active_tasks"]), "active": self.coordinator.state == QueueState.RUNNING
and bool(processor_stats["active_tasks"]),
"stalled": monitor_stats.get("stalled", False), "stalled": monitor_stats.get("stalled", False),
"stats": { "stats": {
"uptime": self.stats.uptime.total_seconds(), "uptime": self.stats.uptime.total_seconds(),
@@ -301,7 +305,7 @@ class EnhancedVideoQueueManager(IQueueManager):
"total_processed": self.stats.total_processed, "total_processed": self.stats.total_processed,
"total_failed": self.stats.total_failed, "total_failed": self.stats.total_failed,
}, },
} },
) )
# Helper methods below... # Helper methods below...

4
videoarchiver/queue/monitoring.py
View File

@@ -8,8 +8,8 @@ from dataclasses import dataclass, field
from typing import Optional, Dict, Any, List, Set from typing import Optional, Dict, Any, List, Set
from datetime import datetime, timedelta from datetime import datetime, timedelta
from .health_checker import HealthChecker, HealthStatus, HealthCategory from health_checker import HealthChecker, HealthStatus, HealthCategory
from .recovery_manager import RecoveryManager, RecoveryStrategy from recovery_manager import RecoveryManager, RecoveryStrategy
logger = logging.getLogger("QueueMonitoring") logger = logging.getLogger("QueueMonitoring")

2
videoarchiver/queue/persistence.py
View File

@@ -9,7 +9,7 @@ import asyncio
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Dict, Any, Optional from typing import Dict, Any, Optional
from .models import QueueItem, QueueMetrics from models import QueueItem, QueueMetrics
# Configure logging # Configure logging
logging.basicConfig( logging.basicConfig(

6
videoarchiver/queue/processor.py
View File

@@ -8,9 +8,9 @@ from dataclasses import dataclass
from typing import Callable, Optional, Tuple, List, Set, Dict, Any from typing import Callable, Optional, Tuple, List, Set, Dict, Any
from datetime import datetime, timedelta from datetime import datetime, timedelta
from .models import QueueItem from models import QueueItem
from .state_manager import QueueStateManager, ItemState from state_manager import QueueStateManager, ItemState
from .monitoring import QueueMonitor from monitoring import QueueMonitor
logger = logging.getLogger("QueueProcessor") logger = logging.getLogger("QueueProcessor")

0
videoarchiver/queue/types.py → videoarchiver/queue/q_types.py
View File

2
videoarchiver/queue/recovery_manager.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass, field
from typing import List, Tuple, Dict, Optional, Any, Set from typing import List, Tuple, Dict, Optional, Any, Set
from datetime import datetime, timedelta from datetime import datetime, timedelta
from .models import QueueItem from models import QueueItem
logger = logging.getLogger("QueueRecoveryManager") logger = logging.getLogger("QueueRecoveryManager")

2
videoarchiver/queue/state_manager.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass
from typing import Dict, Set, List, Optional, Any from typing import Dict, Set, List, Optional, Any
from datetime import datetime from datetime import datetime
from .models import QueueItem, QueueMetrics from models import QueueItem, QueueMetrics
logger = logging.getLogger("QueueStateManager") logger = logging.getLogger("QueueStateManager")

26
videoarchiver/shared/__init__.py
View File

@@ -1,6 +1,6 @@
"""Shared functionality for the videoarchiver package""" """Shared functionality for the videoarchiver package"""
from .progress import ( from progress import (
compression_progress, compression_progress,
download_progress, download_progress,
processing_progress, processing_progress,
@@ -16,16 +16,16 @@ from .progress import (
) )
__all__ = [ __all__ = [
'compression_progress', "compression_progress",
'download_progress', "download_progress",
'processing_progress', "processing_progress",
'get_compression_progress', "get_compression_progress",
'update_compression_progress', "update_compression_progress",
'clear_compression_progress', "clear_compression_progress",
'get_download_progress', "get_download_progress",
'update_download_progress', "update_download_progress",
'clear_download_progress', "clear_download_progress",
'get_processing_progress', "get_processing_progress",
'update_processing_progress', "update_processing_progress",
'clear_processing_progress', "clear_processing_progress",
] ]

2
videoarchiver/update_checker.py
View File

@@ -18,7 +18,7 @@ import shutil
# try: # try:
# Try relative imports first # Try relative imports first
from .utils.exceptions import UpdateError from utils.exceptions import UpdateError
# except ImportError: # except ImportError:
# Fall back to absolute imports if relative imports fail # Fall back to absolute imports if relative imports fail

20
videoarchiver/utils/__init__.py
View File

@@ -2,27 +2,27 @@
from typing import Dict, Optional, Any, Union, List from typing import Dict, Optional, Any, Union, List
from .file_ops import ( from file_ops import (
cleanup_downloads, cleanup_downloads,
ensure_directory, ensure_directory,
get_file_size, get_file_size,
is_valid_path, is_valid_path,
safe_delete safe_delete
) )
from .file_deletion import FileDeleter from file_deletion import FileDeleter
from .directory_manager import DirectoryManager from directory_manager import DirectoryManager
from .permission_manager import PermissionManager from permission_manager import PermissionManager
from .download_manager import DownloadManager from download_manager import DownloadManager
from .compression_manager import CompressionManager from compression_manager import CompressionManager
from .progress_tracker import ( from progress_tracker import (
ProgressTracker, ProgressTracker,
ProgressStatus, ProgressStatus,
DownloadProgress, DownloadProgress,
CompressionProgress, CompressionProgress,
CompressionParams CompressionParams
) )
from .path_manager import PathManager from path_manager import PathManager
from .exceptions import ( from exceptions import (
# Base exception # Base exception
VideoArchiverError, VideoArchiverError,
ErrorSeverity, ErrorSeverity,
@@ -86,7 +86,7 @@ from .exceptions import (
) )
# Import progress_tracker from processor # Import progress_tracker from processor
from ..processor import progress_tracker from processor import progress_tracker
__all__ = [ __all__ = [
# File Operations # File Operations

10
videoarchiver/utils/compression_handler.py
View File

@@ -7,11 +7,11 @@ import subprocess
from datetime import datetime from datetime import datetime
from typing import Dict, Optional, Callable, Set, Tuple from typing import Dict, Optional, Callable, Set, Tuple
from ..ffmpeg.ffmpeg_manager import FFmpegManager from ffmpeg.ffmpeg_manager import FFmpegManager
from ..ffmpeg.exceptions import CompressionError from ffmpeg.exceptions import CompressionError
from ..utils.exceptions import VideoVerificationError from utils.exceptions import VideoVerificationError
from ..utils.file_operations import FileOperations from utils.file_operations import FileOperations
from ..utils.progress_handler import ProgressHandler from utils.progress_handler import ProgressHandler
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

10
videoarchiver/utils/compression_manager.py
View File

@@ -7,11 +7,11 @@ import subprocess
from datetime import datetime from datetime import datetime
from typing import Dict, Any, Optional, Callable, List, Set, Tuple from typing import Dict, Any, Optional, Callable, List, Set, Tuple
from ..shared.progress import update_compression_progress from shared.progress import update_compression_progress
from ..utils.compression_handler import CompressionHandler from utils.compression_handler import CompressionHandler
from ..utils.progress_handler import ProgressHandler from utils.progress_handler import ProgressHandler
from ..utils.file_operations import FileOperations from utils.file_operations import FileOperations
from ..utils.exceptions import CompressionError, VideoVerificationError from utils.exceptions import CompressionError, VideoVerificationError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

4
videoarchiver/utils/directory_manager.py
View File

@@ -6,8 +6,8 @@ import asyncio
from pathlib import Path from pathlib import Path
from typing import List, Optional, Tuple from typing import List, Optional, Tuple
from ..utils.exceptions import FileCleanupError from utils.exceptions import FileCleanupError
from ..utils.file_deletion import SecureFileDeleter from utils.file_deletion import SecureFileDeleter
logger = logging.getLogger("DirectoryManager") logger = logging.getLogger("DirectoryManager")

12
videoarchiver/utils/download_core.py
View File

@@ -7,12 +7,12 @@ import yt_dlp # type: ignore
from typing import Dict, Optional, Callable, Tuple from typing import Dict, Optional, Callable, Tuple
from pathlib import Path from pathlib import Path
from ..utils.url_validator import check_url_support from utils.url_validator import check_url_support
from ..utils.progress_handler import ProgressHandler, CancellableYTDLLogger from utils.progress_handler import ProgressHandler, CancellableYTDLLogger
from ..utils.file_operations import FileOperations from utils.file_operations import FileOperations
from ..utils.compression_handler import CompressionHandler from utils.compression_handler import CompressionHandler
from ..utils.process_manager import ProcessManager from utils.process_manager import ProcessManager
from ..ffmpeg.ffmpeg_manager import FFmpegManager from ffmpeg.ffmpeg_manager import FFmpegManager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

6
videoarchiver/utils/download_manager.py
View File

@@ -9,9 +9,9 @@ from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional, Tuple, Callable, Any from typing import Dict, List, Optional, Tuple, Callable, Any
from pathlib import Path from pathlib import Path
from ..ffmpeg.verification_manager import VerificationManager from ffmpeg.verification_manager import VerificationManager
from ..utils.compression_manager import CompressionManager from utils.compression_manager import CompressionManager
from ..processor import progress_tracker # Import from processor instead of utils from processor import progress_tracker # Import from processor instead of utils
logger = logging.getLogger("DownloadManager") logger = logging.getLogger("DownloadManager")

2
videoarchiver/utils/file_deletion.py
View File

@@ -7,7 +7,7 @@ import logging
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
from ..utils.exceptions import FileCleanupError from utils.exceptions import FileCleanupError
logger = logging.getLogger("FileDeleter") logger = logging.getLogger("FileDeleter")

4
videoarchiver/utils/file_operations.py
View File

@@ -9,8 +9,8 @@ import subprocess
from typing import Tuple from typing import Tuple
from pathlib import Path from pathlib import Path
from ..utils.exceptions import VideoVerificationError from utils.exceptions import VideoVerificationError
from ..utils.file_deletion import SecureFileDeleter from utils.file_deletion import SecureFileDeleter
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

8
videoarchiver/utils/file_ops.py
View File

@@ -4,10 +4,10 @@ import logging
from pathlib import Path from pathlib import Path
from typing import List, Tuple, Optional from typing import List, Tuple, Optional
from ..utils.exceptions import FileCleanupError from utils.exceptions import FileCleanupError
from ..utils.file_deletion import SecureFileDeleter from utils.file_deletion import SecureFileDeleter
from ..utils.directory_manager import DirectoryManager from utils.directory_manager import DirectoryManager
from ..utils.permission_manager import PermissionManager from utils.permission_manager import PermissionManager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

4
videoarchiver/utils/path_manager.py
View File

@@ -11,8 +11,8 @@ import time
from typing import List, Optional, AsyncGenerator from typing import List, Optional, AsyncGenerator
from pathlib import Path from pathlib import Path
from ..utils.exceptions import FileCleanupError from utils.exceptions import FileCleanupError
from ..utils.permission_manager import PermissionManager from utils.permission_manager import PermissionManager
logger = logging.getLogger("PathManager") logger = logging.getLogger("PathManager")

2
videoarchiver/utils/permission_manager.py
View File

@@ -6,7 +6,7 @@ import logging
from pathlib import Path from pathlib import Path
from typing import Optional, Union, List from typing import Optional, Union, List
from ..utils.exceptions import FileCleanupError from utils.exceptions import FileCleanupError
logger = logging.getLogger("PermissionManager") logger = logging.getLogger("PermissionManager")