From df9099f2c849810a60109841e1378af06b86cc7c Mon Sep 17 00:00:00 2001 From: pacnpal <183241239+pacnpal@users.noreply.github.com> Date: Mon, 18 Nov 2024 05:17:37 +0000 Subject: [PATCH] diabolical --- info.json | 2 +- videoarchiver/__init__.py | 131 ++++++------- videoarchiver/config/__init__.py | 10 +- videoarchiver/config/channel_manager.py | 2 +- videoarchiver/config/role_manager.py | 2 +- videoarchiver/config/settings_formatter.py | 2 +- videoarchiver/config/validation_manager.py | 2 +- videoarchiver/config_manager.py | 10 +- videoarchiver/core/__init__.py | 2 +- videoarchiver/core/base.py | 30 +-- videoarchiver/core/{types.py => c_types.py} | 0 videoarchiver/core/cleanup.py | 126 ++++++------ videoarchiver/core/commands.py | 7 +- videoarchiver/core/commands/__init__.py | 6 +- .../core/commands/archiver_commands.py | 4 +- .../core/commands/database_commands.py | 6 +- .../core/commands/settings_commands.py | 6 +- videoarchiver/core/component_manager.py | 12 +- videoarchiver/core/error_handler.py | 99 +++++----- videoarchiver/core/events.py | 41 ++-- videoarchiver/core/guild.py | 10 +- videoarchiver/core/initialization.py | 6 +- videoarchiver/core/lifecycle.py | 27 +-- videoarchiver/core/response_handler.py | 2 +- videoarchiver/core/settings.py | 2 +- videoarchiver/database/__init__.py | 8 +- videoarchiver/database/connection_manager.py | 180 +++++++++--------- videoarchiver/database/schema_manager.py | 13 +- videoarchiver/database/video_archive_db.py | 6 +- videoarchiver/ffmpeg/__init__.py | 12 +- .../__pycache__/__init__.cpython-312.pyc | Bin 396 -> 0 bytes .../__pycache__/exceptions.cpython-312.pyc | Bin 823 -> 0 bytes .../ffmpeg_manager.cpython-312.pyc | Bin 5501 -> 0 bytes .../__pycache__/gpu_detector.cpython-312.pyc | Bin 5996 -> 0 bytes .../video_analyzer.cpython-312.pyc | Bin 6804 -> 0 bytes videoarchiver/ffmpeg/binary_manager.py | 6 +- videoarchiver/ffmpeg/encoder_params.py | 2 +- videoarchiver/ffmpeg/ffmpeg_downloader.py | 2 +- videoarchiver/ffmpeg/ffmpeg_manager.py | 14 +- videoarchiver/info.json | 11 +- videoarchiver/processor/__init__.py | 20 +- videoarchiver/processor/cleanup_manager.py | 6 +- videoarchiver/processor/core.py | 8 +- videoarchiver/processor/message_handler.py | 12 +- videoarchiver/processor/message_validator.py | 2 +- videoarchiver/processor/queue_handler.py | 31 +-- videoarchiver/processor/queue_processor.py | 151 ++++++++------- videoarchiver/processor/reactions.py | 4 +- videoarchiver/processor/status_display.py | 2 +- videoarchiver/queue/__init__.py | 86 ++++----- videoarchiver/queue/cleaners/__init__.py | 6 +- videoarchiver/queue/cleaners/guild_cleaner.py | 2 +- .../queue/cleaners/history_cleaner.py | 2 +- .../queue/cleaners/tracking_cleaner.py | 170 ++++++++--------- videoarchiver/queue/cleanup.py | 8 +- videoarchiver/queue/manager.py | 26 +-- videoarchiver/queue/monitoring.py | 4 +- videoarchiver/queue/persistence.py | 2 +- videoarchiver/queue/processor.py | 6 +- videoarchiver/queue/{types.py => q_types.py} | 0 videoarchiver/queue/recovery_manager.py | 2 +- videoarchiver/queue/state_manager.py | 2 +- videoarchiver/shared/__init__.py | 26 +-- videoarchiver/update_checker.py | 2 +- videoarchiver/utils/__init__.py | 20 +- videoarchiver/utils/compression_handler.py | 10 +- videoarchiver/utils/compression_manager.py | 10 +- videoarchiver/utils/directory_manager.py | 4 +- videoarchiver/utils/download_core.py | 12 +- videoarchiver/utils/download_manager.py | 6 +- videoarchiver/utils/file_deletion.py | 2 +- videoarchiver/utils/file_operations.py | 4 +- videoarchiver/utils/file_ops.py | 8 +- videoarchiver/utils/path_manager.py | 4 +- videoarchiver/utils/permission_manager.py | 2 +- 75 files changed, 744 insertions(+), 719 deletions(-) rename videoarchiver/core/{types.py => c_types.py} (100%) delete mode 100644 videoarchiver/ffmpeg/__pycache__/__init__.cpython-312.pyc delete mode 100644 videoarchiver/ffmpeg/__pycache__/exceptions.cpython-312.pyc delete mode 100644 videoarchiver/ffmpeg/__pycache__/ffmpeg_manager.cpython-312.pyc delete mode 100644 videoarchiver/ffmpeg/__pycache__/gpu_detector.cpython-312.pyc delete mode 100644 videoarchiver/ffmpeg/__pycache__/video_analyzer.cpython-312.pyc rename videoarchiver/queue/{types.py => q_types.py} (100%) diff --git a/info.json b/info.json index 55d2fc2..06a889b 100644 --- a/info.json +++ b/info.json @@ -5,5 +5,5 @@ "install_msg": "Thank you for installing the Pac-cogs repo!", "name": "Pac-cogs", "short": "Very cool cogs!", - "description": "Right now, just a birthday cog." + "description": "Cogs that I made for my own use but you can use them too if you want!." } \ No newline at end of file diff --git a/videoarchiver/__init__.py b/videoarchiver/__init__.py index 35c3349..3e79ef5 100644 --- a/videoarchiver/__init__.py +++ b/videoarchiver/__init__.py @@ -9,58 +9,58 @@ from redbot.core.bot import Red # type: ignore # Force reload of all modules modules_to_reload = [ - ".utils.exceptions", - ".utils", - ".processor", - ".processor.core", - ".processor.queue_processor", - ".queue", - ".queue.types", # Added types module - ".queue.models", - ".queue.manager", - ".queue.cleaners", - ".queue.cleaners.guild_cleaner", - ".queue.cleaners.history_cleaner", - ".queue.cleaners.tracking_cleaner", - ".queue.monitoring", - ".queue.recovery_manager", - ".queue.state_manager", - ".ffmpeg", - ".ffmpeg.binary_manager", - ".ffmpeg.encoder_params", - ".ffmpeg.exceptions", - ".ffmpeg.ffmpeg_downloader", - ".ffmpeg.ffmpeg_manager", - ".ffmpeg.gpu_detector", - ".ffmpeg.process_manager", - ".ffmpeg.verification_manager", - ".ffmpeg.video_analyzer", - ".database", - ".database.connection_manager", - ".database.query_manager", - ".database.schema_manager", - ".database.video_archive_db", - ".config", - ".config.channel_manager", - ".config.exceptions", - ".config.role_manager", - ".config.settings_formatter", - ".config.validation_manager", - ".core", - ".core.base", - ".core.cleanup", - ".core.commands", - ".core.commands.archiver_commands", - ".core.commands.database_commands", - ".core.commands.settings_commands", - ".core.component_manager", - ".core.error_handler", - ".core.events", - ".core.guild", - ".core.initialization", - ".core.lifecycle", - ".core.response_handler", - ".core.settings", + "utils.exceptions", + "utils", + "processor", + "processor.core", + "processor.queue_processor", + "queue", + "queue.types", # Added types module + "queue.models", + "queue.manager", + "queue.cleaners", + "queue.cleaners.guild_cleaner", + "queue.cleaners.history_cleaner", + "queue.cleaners.tracking_cleaner", + "queue.monitoring", + "queue.recovery_manager", + "queue.state_manager", + "ffmpeg", + "ffmpeg.binary_manager", + "ffmpeg.encoder_params", + "ffmpeg.exceptions", + "ffmpeg.ffmpeg_downloader", + "ffmpeg.ffmpeg_manager", + "ffmpeg.gpu_detector", + "ffmpeg.process_manager", + "ffmpeg.verification_manager", + "ffmpeg.video_analyzer", + "database", + "database.connection_manager", + "database.query_manager", + "database.schema_manager", + "database.video_archive_db", + "config", + "config.channel_manager", + "config.exceptions", + "config.role_manager", + "config.settings_formatter", + "config.validation_manager", + "core", + "core.base", + "core.cleanup", + "core.commands", + "core.commands.archiver_commands", + "core.commands.database_commands", + "core.commands.settings_commands", + "core.component_manager", + "core.error_handler", + "core.events", + "core.guild", + "core.initialization", + "core.lifecycle", + "core.response_handler", + "core.settings", ] # Remove modules to force fresh import @@ -68,7 +68,7 @@ for module in modules_to_reload: if module in sys.modules: del sys.modules[module] -try: + # try: # Try relative imports first from . import utils from . import processor @@ -89,7 +89,8 @@ try: ErrorSeverity, ProcessingError, ) -except ImportError: + + # except ImportError: # Fall back to absolute imports if relative imports fail # from videoarchiver import utils # from videoarchiver import processor @@ -121,17 +122,17 @@ except ImportError: importlib.reload(core) # Import all submodules -from .database import * -from .ffmpeg import * -from .queue import * -from .processor import * -from .config_manager import * -from .update_checker import * -from .queue.cleaners import * -from .database import * -from .utils import * -from .core import * -from .config import * +from database import * +from ffmpeg import * +from queue import * +from processor import * +from config_manager import * +from update_checker import * +from queue.cleaners import * +from database import * +from utils import * +from core import * +from config import * logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/config/__init__.py b/videoarchiver/config/__init__.py index 537921c..7dfab14 100644 --- a/videoarchiver/config/__init__.py +++ b/videoarchiver/config/__init__.py @@ -2,7 +2,7 @@ # try: # Try relative imports first -from .exceptions import ( +from exceptions import ( ConfigurationError, ValidationError, PermissionError, @@ -12,10 +12,10 @@ from .exceptions import ( SchemaError, DiscordAPIError, ) -from .channel_manager import ChannelManager -from .role_manager import RoleManager -from .settings_formatter import SettingsFormatter -from .validation_manager import ValidationManager +from channel_manager import ChannelManager +from role_manager import RoleManager +from settings_formatter import SettingsFormatter +from validation_manager import ValidationManager # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/config/channel_manager.py b/videoarchiver/config/channel_manager.py index bee17e7..42b290d 100644 --- a/videoarchiver/config/channel_manager.py +++ b/videoarchiver/config/channel_manager.py @@ -6,7 +6,7 @@ import discord # type: ignore # try: # Try relative imports first -from .exceptions import ( +from exceptions import ( ConfigurationError as ConfigError, DiscordAPIError, ) diff --git a/videoarchiver/config/role_manager.py b/videoarchiver/config/role_manager.py index 29df9bc..acb96f2 100644 --- a/videoarchiver/config/role_manager.py +++ b/videoarchiver/config/role_manager.py @@ -6,7 +6,7 @@ import discord # type: ignore # try: # Try relative imports first -from .exceptions import ConfigurationError as ConfigError +from exceptions import ConfigurationError as ConfigError # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/config/settings_formatter.py b/videoarchiver/config/settings_formatter.py index 977d144..1bf4764 100644 --- a/videoarchiver/config/settings_formatter.py +++ b/videoarchiver/config/settings_formatter.py @@ -7,7 +7,7 @@ import discord # type: ignore # try: # Try relative imports first -from .exceptions import ConfigurationError as ConfigError +from exceptions import ConfigurationError as ConfigError # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/config/validation_manager.py b/videoarchiver/config/validation_manager.py index d3e2c96..b604904 100644 --- a/videoarchiver/config/validation_manager.py +++ b/videoarchiver/config/validation_manager.py @@ -5,7 +5,7 @@ from typing import Any, Dict, List, Union # try: # Try relative imports first -from .exceptions import ConfigurationError as ConfigError +from exceptions import ConfigurationError as ConfigError # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/config_manager.py b/videoarchiver/config_manager.py index 4bf65a1..27ec4be 100644 --- a/videoarchiver/config_manager.py +++ b/videoarchiver/config_manager.py @@ -8,11 +8,11 @@ from redbot.core import Config # type: ignore # try: # Try relative imports first -from .config.validation_manager import ValidationManager -from .config.settings_formatter import SettingsFormatter -from .config.channel_manager import ChannelManager -from .config.role_manager import RoleManager -from .utils.exceptions import ConfigurationError as ConfigError +from config.validation_manager import ValidationManager +from config.settings_formatter import SettingsFormatter +from config.channel_manager import ChannelManager +from config.role_manager import RoleManager +from utils.exceptions import ConfigurationError as ConfigError # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/core/__init__.py b/videoarchiver/core/__init__.py index 01dd7cd..b9556f5 100644 --- a/videoarchiver/core/__init__.py +++ b/videoarchiver/core/__init__.py @@ -2,7 +2,7 @@ # try: # Try relative imports first -from .base import VideoArchiver +from base import VideoArchiver # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/core/base.py b/videoarchiver/core/base.py index ae2e3d1..9cf69da 100644 --- a/videoarchiver/core/base.py +++ b/videoarchiver/core/base.py @@ -14,21 +14,21 @@ from redbot.core.commands import GroupCog, Context # type: ignore # try: # Try relative imports first -from .settings import Settings -from .lifecycle import LifecycleManager, LifecycleState -from .component_manager import ComponentManager, ComponentState -from .error_handler import error_manager, handle_command_error -from .response_handler import ResponseManager -from .commands.archiver_commands import setup_archiver_commands -from .commands.database_commands import setup_database_commands -from .commands.settings_commands import setup_settings_commands -from .events import setup_events, EventManager -from ..processor.core import VideoProcessor -from ..queue.manager import EnhancedVideoQueueManager -from ..ffmpeg.ffmpeg_manager import FFmpegManager -from ..database.video_archive_db import VideoArchiveDB -from ..config_manager import ConfigManager -from ..utils.exceptions import CogError, ErrorContext, ErrorSeverity +from settings import Settings +from lifecycle import LifecycleManager, LifecycleState +from component_manager import ComponentManager, ComponentState +from error_handler import error_manager, handle_command_error +from response_handler import ResponseManager +from commands.archiver_commands import setup_archiver_commands +from commands.database_commands import setup_database_commands +from commands.settings_commands import setup_settings_commands +from events import setup_events, EventManager +from processor.core import VideoProcessor +from queue.manager import EnhancedVideoQueueManager +from ffmpeg.ffmpeg_manager import FFmpegManager +from database.video_archive_db import VideoArchiveDB +from config_manager import ConfigManager +from utils.exceptions import CogError, ErrorContext, ErrorSeverity # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/core/types.py b/videoarchiver/core/c_types.py similarity index 100% rename from videoarchiver/core/types.py rename to videoarchiver/core/c_types.py diff --git a/videoarchiver/core/cleanup.py b/videoarchiver/core/cleanup.py index 1c09bba..facd94e 100644 --- a/videoarchiver/core/cleanup.py +++ b/videoarchiver/core/cleanup.py @@ -8,25 +8,29 @@ from enum import Enum, auto from pathlib import Path from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar -#try: - # Try relative imports first -from ..utils.file_ops import cleanup_downloads -from ..utils.exceptions import CleanupError, ErrorContext, ErrorSeverity -#except ImportError: - # Fall back to absolute imports if relative imports fail - # from videoarchiver.utils.file_ops import cleanup_downloads - # from videoarchiver.utils.exceptions import CleanupError, ErrorContext, ErrorSeverity +# try: +# Try relative imports first +from utils.file_ops import cleanup_downloads +from utils.exceptions import CleanupError, ErrorContext, ErrorSeverity + +# except ImportError: +# Fall back to absolute imports if relative imports fail +# from videoarchiver.utils.file_ops import cleanup_downloads +# from videoarchiver.utils.exceptions import CleanupError, ErrorContext, ErrorSeverity if TYPE_CHECKING: - #try: + # try: from .base import VideoArchiver - #except ImportError: - # from videoarchiver.core.base import VideoArchiver + + # except ImportError: + # from videoarchiver.core.base import VideoArchiver logger = logging.getLogger("VideoArchiver") + class CleanupPhase(Enum): """Cleanup phases""" + INITIALIZATION = auto() UPDATE_CHECKER = auto() PROCESSOR = auto() @@ -36,21 +40,26 @@ class CleanupPhase(Enum): DOWNLOADS = auto() REFERENCES = auto() + class CleanupStatus(Enum): """Cleanup status""" + SUCCESS = auto() TIMEOUT = auto() ERROR = auto() SKIPPED = auto() + class CleanupResult(TypedDict): """Type definition for cleanup result""" + phase: CleanupPhase status: CleanupStatus error: Optional[str] duration: float timestamp: str + class CleanupManager: """Manages cleanup operations""" @@ -65,7 +74,7 @@ class CleanupManager: phase: CleanupPhase, status: CleanupStatus, error: Optional[str] = None, - duration: float = 0.0 + duration: float = 0.0, ) -> None: """Record result of a cleanup phase""" self.results[phase] = CleanupResult( @@ -73,20 +82,21 @@ class CleanupManager: status=status, error=error, duration=duration, - timestamp=datetime.utcnow().isoformat() + timestamp=datetime.utcnow().isoformat(), ) def get_results(self) -> Dict[CleanupPhase, CleanupResult]: """Get cleanup results""" return self.results.copy() + async def cleanup_resources(cog: "VideoArchiver") -> None: """ Clean up all resources with proper handling. - + Args: cog: VideoArchiver cog instance - + Raises: CleanupError: If cleanup fails """ @@ -102,11 +112,13 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: try: logger.info("Cancelling initialization task") cog._init_task.cancel() - await asyncio.wait_for(cog._init_task, timeout=cleanup_manager.CLEANUP_TIMEOUT) + await asyncio.wait_for( + cog._init_task, timeout=cleanup_manager.CLEANUP_TIMEOUT + ) cleanup_manager.record_result( CleanupPhase.INITIALIZATION, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except (asyncio.TimeoutError, asyncio.CancelledError) as e: logger.warning("Initialization task cancellation timed out") @@ -114,7 +126,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: CleanupPhase.INITIALIZATION, CleanupStatus.TIMEOUT, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) # Stop update checker @@ -123,13 +135,12 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: try: logger.info("Stopping update checker") await asyncio.wait_for( - cog.update_checker.stop(), - timeout=cleanup_manager.CLEANUP_TIMEOUT + cog.update_checker.stop(), timeout=cleanup_manager.CLEANUP_TIMEOUT ) cleanup_manager.record_result( CleanupPhase.UPDATE_CHECKER, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except asyncio.TimeoutError as e: logger.warning("Update checker stop timed out") @@ -137,7 +148,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: CleanupPhase.UPDATE_CHECKER, CleanupStatus.TIMEOUT, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) cog.update_checker = None @@ -147,13 +158,12 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: try: logger.info("Cleaning up processor") await asyncio.wait_for( - cog.processor.cleanup(), - timeout=cleanup_manager.CLEANUP_TIMEOUT + cog.processor.cleanup(), timeout=cleanup_manager.CLEANUP_TIMEOUT ) cleanup_manager.record_result( CleanupPhase.PROCESSOR, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except asyncio.TimeoutError as e: logger.warning("Processor cleanup timed out, forcing cleanup") @@ -162,7 +172,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: CleanupPhase.PROCESSOR, CleanupStatus.TIMEOUT, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) cog.processor = None @@ -172,13 +182,12 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: try: logger.info("Cleaning up queue manager") await asyncio.wait_for( - cog.queue_manager.cleanup(), - timeout=cleanup_manager.CLEANUP_TIMEOUT + cog.queue_manager.cleanup(), timeout=cleanup_manager.CLEANUP_TIMEOUT ) cleanup_manager.record_result( CleanupPhase.QUEUE_MANAGER, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except asyncio.TimeoutError as e: logger.warning("Queue manager cleanup timed out, forcing stop") @@ -187,7 +196,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: CleanupPhase.QUEUE_MANAGER, CleanupStatus.TIMEOUT, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) cog.queue_manager = None @@ -214,14 +223,14 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: CleanupPhase.COMPONENTS, status, "\n".join(errors) if errors else None, - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: cleanup_manager.record_result( CleanupPhase.COMPONENTS, CleanupStatus.ERROR, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) # Kill any FFmpeg processes @@ -233,7 +242,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: cog.ffmpeg_mgr = None # Kill any remaining FFmpeg processes system-wide - if os.name != 'nt': # Unix-like systems + if os.name != "nt": # Unix-like systems os.system("pkill -9 ffmpeg") else: # Windows os.system("taskkill /F /IM ffmpeg.exe") @@ -241,14 +250,14 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: cleanup_manager.record_result( CleanupPhase.FFMPEG, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: cleanup_manager.record_result( CleanupPhase.FFMPEG, CleanupStatus.ERROR, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) # Clean up download directory @@ -258,21 +267,21 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: logger.info("Cleaning up download directory") await asyncio.wait_for( cleanup_downloads(str(cog.download_path)), - timeout=cleanup_manager.CLEANUP_TIMEOUT + timeout=cleanup_manager.CLEANUP_TIMEOUT, ) if cog.download_path.exists(): cog.download_path.rmdir() cleanup_manager.record_result( CleanupPhase.DOWNLOADS, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: cleanup_manager.record_result( CleanupPhase.DOWNLOADS, CleanupStatus.ERROR, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: @@ -284,8 +293,8 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: "Cleanup", "cleanup_resources", {"duration": (datetime.utcnow() - start_time).total_seconds()}, - ErrorSeverity.HIGH - ) + ErrorSeverity.HIGH, + ), ) finally: logger.info("Clearing ready flag") @@ -294,17 +303,18 @@ async def cleanup_resources(cog: "VideoArchiver") -> None: # Log cleanup results for phase, result in cleanup_manager.get_results().items(): status_str = f"{result['status'].name}" - if result['error']: + if result["error"]: status_str += f" ({result['error']})" logger.info( f"Cleanup phase {phase.name}: {status_str} " f"(Duration: {result['duration']:.2f}s)" ) + async def force_cleanup_resources(cog: "VideoArchiver") -> None: """ Force cleanup of resources when timeout occurs. - + Args: cog: VideoArchiver cog instance """ @@ -323,14 +333,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None: cleanup_manager.record_result( CleanupPhase.PROCESSOR, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: cleanup_manager.record_result( CleanupPhase.PROCESSOR, CleanupStatus.ERROR, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) cog.processor = None @@ -343,14 +353,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None: cleanup_manager.record_result( CleanupPhase.QUEUE_MANAGER, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: cleanup_manager.record_result( CleanupPhase.QUEUE_MANAGER, CleanupStatus.ERROR, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) cog.queue_manager = None @@ -363,7 +373,7 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None: cog.ffmpeg_mgr = None # Force kill any remaining FFmpeg processes system-wide - if os.name != 'nt': # Unix-like systems + if os.name != "nt": # Unix-like systems os.system("pkill -9 ffmpeg") else: # Windows os.system("taskkill /F /IM ffmpeg.exe") @@ -371,14 +381,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None: cleanup_manager.record_result( CleanupPhase.FFMPEG, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: cleanup_manager.record_result( CleanupPhase.FFMPEG, CleanupStatus.ERROR, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) # Clean up download directory @@ -388,21 +398,21 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None: logger.info("Force cleaning download directory") await asyncio.wait_for( cleanup_downloads(str(cog.download_path)), - timeout=cleanup_manager.FORCE_CLEANUP_TIMEOUT + timeout=cleanup_manager.FORCE_CLEANUP_TIMEOUT, ) if cog.download_path.exists(): cog.download_path.rmdir() cleanup_manager.record_result( CleanupPhase.DOWNLOADS, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: cleanup_manager.record_result( CleanupPhase.DOWNLOADS, CleanupStatus.ERROR, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) # Clear all components @@ -414,14 +424,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None: cleanup_manager.record_result( CleanupPhase.COMPONENTS, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: cleanup_manager.record_result( CleanupPhase.COMPONENTS, CleanupStatus.ERROR, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: @@ -443,25 +453,25 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None: cog.db = None cog._init_task = None cog._cleanup_task = None - if hasattr(cog, '_queue_task'): + if hasattr(cog, "_queue_task"): cog._queue_task = None cleanup_manager.record_result( CleanupPhase.REFERENCES, CleanupStatus.SUCCESS, - duration=(datetime.utcnow() - phase_start).total_seconds() + duration=(datetime.utcnow() - phase_start).total_seconds(), ) except Exception as e: cleanup_manager.record_result( CleanupPhase.REFERENCES, CleanupStatus.ERROR, str(e), - (datetime.utcnow() - phase_start).total_seconds() + (datetime.utcnow() - phase_start).total_seconds(), ) # Log cleanup results for phase, result in cleanup_manager.get_results().items(): status_str = f"{result['status'].name}" - if result['error']: + if result["error"]: status_str += f" ({result['error']})" logger.info( f"Force cleanup phase {phase.name}: {status_str} " diff --git a/videoarchiver/core/commands.py b/videoarchiver/core/commands.py index 518a508..8425443 100644 --- a/videoarchiver/core/commands.py +++ b/videoarchiver/core/commands.py @@ -7,9 +7,10 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: # try: - from .base import VideoArchiver - # except ImportError: - # from videoarchiver.core.base import VideoArchiver + from base import VideoArchiver +# except ImportError: +# from videoarchiver.core.base import VideoArchiver + def setup_commands(cog: "VideoArchiver") -> None: """Command setup is now handled in the VideoArchiver class""" diff --git a/videoarchiver/core/commands/__init__.py b/videoarchiver/core/commands/__init__.py index a4624b3..b018da5 100644 --- a/videoarchiver/core/commands/__init__.py +++ b/videoarchiver/core/commands/__init__.py @@ -1,8 +1,8 @@ """Command handlers for VideoArchiver""" -from .archiver_commands import setup_archiver_commands -from .database_commands import setup_database_commands -from .settings_commands import setup_settings_commands +from archiver_commands import setup_archiver_commands +from database_commands import setup_database_commands +from settings_commands import setup_settings_commands __all__ = [ "setup_archiver_commands", diff --git a/videoarchiver/core/commands/archiver_commands.py b/videoarchiver/core/commands/archiver_commands.py index 08cc9a1..e3ebd99 100644 --- a/videoarchiver/core/commands/archiver_commands.py +++ b/videoarchiver/core/commands/archiver_commands.py @@ -9,8 +9,8 @@ from discord import app_commands # type: ignore from redbot.core import commands # type: ignore from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore -from ...core.response_handler import handle_response, ResponseType -from ...utils.exceptions import CommandError, ErrorContext, ErrorSeverity +from core.response_handler import handle_response, ResponseType +from utils.exceptions import CommandError, ErrorContext, ErrorSeverity logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/core/commands/database_commands.py b/videoarchiver/core/commands/database_commands.py index 647ad7c..a5b55df 100644 --- a/videoarchiver/core/commands/database_commands.py +++ b/videoarchiver/core/commands/database_commands.py @@ -10,9 +10,9 @@ from discord import app_commands # type: ignore from redbot.core import commands # type: ignore from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore -from ...core.response_handler import handle_response, ResponseType -from ...utils.exceptions import CommandError, ErrorContext, ErrorSeverity, DatabaseError -from ...database.video_archive_db import VideoArchiveDB +from core.response_handler import handle_response, ResponseType +from utils.exceptions import CommandError, ErrorContext, ErrorSeverity, DatabaseError +from database.video_archive_db import VideoArchiveDB logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/core/commands/settings_commands.py b/videoarchiver/core/commands/settings_commands.py index ddf0dc6..d426854 100644 --- a/videoarchiver/core/commands/settings_commands.py +++ b/videoarchiver/core/commands/settings_commands.py @@ -9,9 +9,9 @@ from discord import app_commands # type: ignore from redbot.core import commands # type: ignore from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore -from ...core.settings import VideoFormat, VideoQuality -from ...core.response_handler import handle_response, ResponseType -from ...utils.exceptions import CommandError, ErrorContext, ErrorSeverity +from core.settings import VideoFormat, VideoQuality +from core.response_handler import handle_response, ResponseType +from utils.exceptions import CommandError, ErrorContext, ErrorSeverity logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/core/component_manager.py b/videoarchiver/core/component_manager.py index d3940b6..82c38b6 100644 --- a/videoarchiver/core/component_manager.py +++ b/videoarchiver/core/component_manager.py @@ -22,12 +22,12 @@ import importlib # try: # Try relative imports first -from ..utils.exceptions import ComponentError, ErrorContext, ErrorSeverity -from ..utils.path_manager import PathManager -from ..config_manager import ConfigManager -from ..processor.core import VideoProcessor -from ..queue.manager import EnhancedVideoQueueManager -from ..ffmpeg.ffmpeg_manager import FFmpegManager +from utils.exceptions import ComponentError, ErrorContext, ErrorSeverity +from utils.path_manager import PathManager +from config_manager import ConfigManager +from processor.core import VideoProcessor +from queue.manager import EnhancedVideoQueueManager +from ffmpeg.ffmpeg_manager import FFmpegManager # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/core/error_handler.py b/videoarchiver/core/error_handler.py index 689222e..0c62a5b 100644 --- a/videoarchiver/core/error_handler.py +++ b/videoarchiver/core/error_handler.py @@ -4,19 +4,19 @@ import logging import traceback from typing import Dict, Optional, Tuple, Type, TypedDict, ClassVar from enum import Enum, auto -import discord # type: ignore -from redbot.core.commands import ( # type: ignore +import discord # type: ignore +from redbot.core.commands import ( # type: ignore Context, MissingPermissions, BotMissingPermissions, MissingRequiredArgument, BadArgument, - CommandError + CommandError, ) -#try: - # Try relative imports first -from ..utils.exceptions import ( +# try: +# Try relative imports first +from utils.exceptions import ( VideoArchiverError, ErrorSeverity, ErrorContext, @@ -33,9 +33,10 @@ from ..utils.exceptions import ( TrackingError, NetworkError, ResourceExhaustedError, - ConfigurationError + ConfigurationError, ) -from ..core.response_handler import response_manager +from core.response_handler import response_manager + # except ImportError: # # Fall back to absolute imports if relative imports fail # # from videoarchiver.utils.exceptions import ( @@ -57,12 +58,14 @@ from ..core.response_handler import response_manager # ResourceExhaustedError, # ConfigurationError # ) - # from videoarchiver.core.response_handler import response_manager +# from videoarchiver.core.response_handler import response_manager logger = logging.getLogger("VideoArchiver") + class ErrorCategory(Enum): """Categories of errors""" + PERMISSION = auto() ARGUMENT = auto() CONFIGURATION = auto() @@ -76,17 +79,22 @@ class ErrorCategory(Enum): HEALTH = auto() UNEXPECTED = auto() + class ErrorStats(TypedDict): """Type definition for error statistics""" + counts: Dict[str, int] patterns: Dict[str, Dict[str, int]] severities: Dict[str, Dict[str, int]] + class ErrorFormatter: """Formats error messages for display""" @staticmethod - def format_error_message(error: Exception, context: Optional[ErrorContext] = None) -> str: + def format_error_message( + error: Exception, context: Optional[ErrorContext] = None + ) -> str: """Format error message with context""" base_message = str(error) if context: @@ -110,16 +118,18 @@ class ErrorFormatter: return "An unexpected error occurred. Please check the logs for details." return str(error) + class ErrorCategorizer: """Categorizes errors and determines handling strategy""" - ERROR_MAPPING: ClassVar[Dict[Type[Exception], Tuple[ErrorCategory, ErrorSeverity]]] = { + ERROR_MAPPING: ClassVar[ + Dict[Type[Exception], Tuple[ErrorCategory, ErrorSeverity]] + ] = { # Discord command errors MissingPermissions: (ErrorCategory.PERMISSION, ErrorSeverity.MEDIUM), BotMissingPermissions: (ErrorCategory.PERMISSION, ErrorSeverity.HIGH), MissingRequiredArgument: (ErrorCategory.ARGUMENT, ErrorSeverity.LOW), BadArgument: (ErrorCategory.ARGUMENT, ErrorSeverity.LOW), - # VideoArchiver errors ProcessorError: (ErrorCategory.PROCESSING, ErrorSeverity.HIGH), ValidationError: (ErrorCategory.VALIDATION, ErrorSeverity.MEDIUM), @@ -134,17 +144,17 @@ class ErrorCategorizer: TrackingError: (ErrorCategory.PROCESSING, ErrorSeverity.MEDIUM), NetworkError: (ErrorCategory.NETWORK, ErrorSeverity.MEDIUM), ResourceExhaustedError: (ErrorCategory.RESOURCE, ErrorSeverity.HIGH), - ConfigurationError: (ErrorCategory.CONFIGURATION, ErrorSeverity.HIGH) + ConfigurationError: (ErrorCategory.CONFIGURATION, ErrorSeverity.HIGH), } @classmethod def categorize_error(cls, error: Exception) -> Tuple[ErrorCategory, ErrorSeverity]: """ Categorize an error and determine its severity. - + Args: error: Exception to categorize - + Returns: Tuple of (Error category, Severity level) """ @@ -153,6 +163,7 @@ class ErrorCategorizer: return category, severity return ErrorCategory.UNEXPECTED, ErrorSeverity.HIGH + class ErrorTracker: """Tracks error occurrences and patterns""" @@ -162,31 +173,28 @@ class ErrorTracker: self.error_severities: Dict[str, Dict[str, int]] = {} def track_error( - self, - error: Exception, - category: ErrorCategory, - severity: ErrorSeverity + self, error: Exception, category: ErrorCategory, severity: ErrorSeverity ) -> None: """ Track an error occurrence. - + Args: error: Exception that occurred category: Error category severity: Error severity """ error_type = type(error).__name__ - + # Track error counts self.error_counts[error_type] = self.error_counts.get(error_type, 0) + 1 - + # Track error patterns by category if category.value not in self.error_patterns: self.error_patterns[category.value] = {} self.error_patterns[category.value][error_type] = ( self.error_patterns[category.value].get(error_type, 0) + 1 ) - + # Track error severities if severity.value not in self.error_severities: self.error_severities[severity.value] = {} @@ -197,16 +205,17 @@ class ErrorTracker: def get_error_stats(self) -> ErrorStats: """ Get error statistics. - + Returns: Dictionary containing error statistics """ return ErrorStats( counts=self.error_counts.copy(), patterns=self.error_patterns.copy(), - severities=self.error_severities.copy() + severities=self.error_severities.copy(), ) + class ErrorManager: """Manages error handling and reporting""" @@ -215,14 +224,10 @@ class ErrorManager: self.categorizer = ErrorCategorizer() self.tracker = ErrorTracker() - async def handle_error( - self, - ctx: Context, - error: Exception - ) -> None: + async def handle_error(self, ctx: Context, error: Exception) -> None: """ Handle a command error. - + Args: ctx: Command context error: The error that occurred @@ -230,7 +235,7 @@ class ErrorManager: try: # Categorize error category, severity = self.categorizer.categorize_error(error) - + # Create error context context = ErrorContext( component=ctx.command.qualified_name if ctx.command else "unknown", @@ -238,26 +243,24 @@ class ErrorManager: details={ "guild_id": str(ctx.guild.id) if ctx.guild else "DM", "channel_id": str(ctx.channel.id), - "user_id": str(ctx.author.id) + "user_id": str(ctx.author.id), }, - severity=severity + severity=severity, ) - + # Track error self.tracker.track_error(error, category, severity) - + # Format error messages log_message = self.formatter.format_error_message(error, context) user_message = self.formatter.format_user_message(error, category) - + # Log error details self._log_error(log_message, severity) - + # Send response await response_manager.send_response( - ctx, - content=user_message, - response_type=severity.name.lower() + ctx, content=user_message, response_type=severity.name.lower() ) except Exception as e: @@ -269,19 +272,15 @@ class ErrorManager: await response_manager.send_response( ctx, content="An error occurred while handling another error. Please check the logs.", - response_type="error" + response_type="error", ) except Exception: pass - def _log_error( - self, - message: str, - severity: ErrorSeverity - ) -> None: + def _log_error(self, message: str, severity: ErrorSeverity) -> None: """ Log error details. - + Args: message: Error message to log severity: Error severity @@ -296,13 +295,15 @@ class ErrorManager: except Exception as e: logger.error(f"Error logging error details: {e}") + # Global error manager instance error_manager = ErrorManager() + async def handle_command_error(ctx: Context, error: Exception) -> None: """ Helper function to handle command errors using the error manager. - + Args: ctx: Command context error: Exception to handle diff --git a/videoarchiver/core/events.py b/videoarchiver/core/events.py index 44f892b..a0befc7 100644 --- a/videoarchiver/core/events.py +++ b/videoarchiver/core/events.py @@ -9,28 +9,29 @@ from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar, List import discord # type: ignore -#try: - # Try relative imports first -from ..processor.constants import REACTIONS -from ..processor.reactions import handle_archived_reaction -from .guild import initialize_guild_components, cleanup_guild_components -from .error_handler import ErrorManager -from .response_handler import response_manager -from ..utils.exceptions import EventError, ErrorContext, ErrorSeverity -#except ImportError: - # Fall back to absolute imports if relative imports fail - # from videoarchiver.processor.constants import REACTIONS - # from videoarchiver.processor.reactions import handle_archived_reaction - # from videoarchiver.core.guild import initialize_guild_components, cleanup_guild_components - # from videoarchiver.core.error_handler import ErrorManager - # from videoarchiver.core.response_handler import response_manager - # from videoarchiver.utils.exceptions import EventError, ErrorContext, ErrorSeverity +# try: +# Try relative imports first +from processor.constants import REACTIONS +from processor.reactions import handle_archived_reaction +from guild import initialize_guild_components, cleanup_guild_components +from error_handler import ErrorManager +from response_handler import response_manager +from utils.exceptions import EventError, ErrorContext, ErrorSeverity + +# except ImportError: +# Fall back to absolute imports if relative imports fail +# from videoarchiver.processor.constants import REACTIONS +# from videoarchiver.processor.reactions import handle_archived_reaction +# from videoarchiver.core.guild import initialize_guild_components, cleanup_guild_components +# from videoarchiver.core.error_handler import ErrorManager +# from videoarchiver.core.response_handler import response_manager +# from videoarchiver.utils.exceptions import EventError, ErrorContext, ErrorSeverity if TYPE_CHECKING: - #try: - from .base import VideoArchiver - # except ImportError: - # from videoarchiver.core.base import VideoArchiver + # try: + from base import VideoArchiver +# except ImportError: +# from videoarchiver.core.base import VideoArchiver logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/core/guild.py b/videoarchiver/core/guild.py index 39e3909..dfa60c6 100644 --- a/videoarchiver/core/guild.py +++ b/videoarchiver/core/guild.py @@ -6,10 +6,10 @@ from typing import TYPE_CHECKING, Dict, Any, Optional # try: # Try relative imports first -from ..utils.download_core import DownloadCore -from ..utils.message_manager import MessageManager -from ..utils.file_ops import cleanup_downloads -from ..utils.exceptions import VideoArchiverError as ProcessingError +from utils.download_core import DownloadCore +from utils.message_manager import MessageManager +from utils.file_ops import cleanup_downloads +from utils.exceptions import VideoArchiverError as ProcessingError # except ImportError: # Fall back to absolute imports if relative imports fail @@ -20,7 +20,7 @@ from ..utils.exceptions import VideoArchiverError as ProcessingError if TYPE_CHECKING: # try: - from .base import VideoArchiver + from base import VideoArchiver # except ImportError: # from videoarchiver.core.base import VideoArchiver diff --git a/videoarchiver/core/initialization.py b/videoarchiver/core/initialization.py index 7fa6359..c05c1e7 100644 --- a/videoarchiver/core/initialization.py +++ b/videoarchiver/core/initialization.py @@ -6,8 +6,8 @@ import logging # try: # Try relative imports first -from ..utils.exceptions import ComponentError, ErrorContext, ErrorSeverity -from .lifecycle import LifecycleState +from utils.exceptions import ComponentError, ErrorContext, ErrorSeverity +from lifecycle import LifecycleState # except ImportError: # Fall back to absolute imports if relative imports fail @@ -16,7 +16,7 @@ from .lifecycle import LifecycleState if TYPE_CHECKING: # try: - from .base import VideoArchiver + from base import VideoArchiver # except ImportError: # from videoarchiver.core.base import VideoArchiver diff --git a/videoarchiver/core/lifecycle.py b/videoarchiver/core/lifecycle.py index 234fe05..291e04e 100644 --- a/videoarchiver/core/lifecycle.py +++ b/videoarchiver/core/lifecycle.py @@ -8,25 +8,26 @@ from enum import Enum, auto from datetime import datetime # try: - # Try relative imports first -from .cleanup import cleanup_resources, force_cleanup_resources -from ..utils.exceptions import ( +# Try relative imports first +from cleanup import cleanup_resources, force_cleanup_resources +from utils.exceptions import ( VideoArchiverError, ErrorContext, ErrorSeverity, ComponentError, CleanupError, ) -#except ImportError: - # Fall back to absolute imports if relative imports fail - # from videoarchiver.core.cleanup import cleanup_resources, force_cleanup_resources - # from videoarchiver.utils.exceptions import ( - # VideoArchiverError, - # ErrorContext, - # ErrorSeverity, - # ComponentError, - # CleanupError, - # ) + +# except ImportError: +# Fall back to absolute imports if relative imports fail +# from videoarchiver.core.cleanup import cleanup_resources, force_cleanup_resources +# from videoarchiver.utils.exceptions import ( +# VideoArchiverError, +# ErrorContext, +# ErrorSeverity, +# ComponentError, +# CleanupError, +# ) logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/core/response_handler.py b/videoarchiver/core/response_handler.py index 3e8e2bc..f1b4ede 100644 --- a/videoarchiver/core/response_handler.py +++ b/videoarchiver/core/response_handler.py @@ -9,7 +9,7 @@ from redbot.core.commands import Context # type: ignore # try: # Try relative imports first -from ..utils.exceptions import ErrorSeverity +from utils.exceptions import ErrorSeverity # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/core/settings.py b/videoarchiver/core/settings.py index 590523e..e1e9c51 100644 --- a/videoarchiver/core/settings.py +++ b/videoarchiver/core/settings.py @@ -6,7 +6,7 @@ from enum import Enum, auto # try: # Try relative imports first -from ..utils.exceptions import ConfigurationError, ErrorContext, ErrorSeverity +from utils.exceptions import ConfigurationError, ErrorContext, ErrorSeverity # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/database/__init__.py b/videoarchiver/database/__init__.py index 2f44a29..bac57ad 100644 --- a/videoarchiver/database/__init__.py +++ b/videoarchiver/database/__init__.py @@ -2,10 +2,10 @@ # try: # Try relative imports first -from .connection_manager import DatabaseConnectionManager -from .query_manager import DatabaseQueryManager -from .schema_manager import DatabaseSchemaManager -from .video_archive_db import VideoArchiveDB +from connection_manager import DatabaseConnectionManager +from query_manager import DatabaseQueryManager +from schema_manager import DatabaseSchemaManager +from video_archive_db import VideoArchiveDB # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/database/connection_manager.py b/videoarchiver/database/connection_manager.py index 84143b1..6d78f5f 100644 --- a/videoarchiver/database/connection_manager.py +++ b/videoarchiver/database/connection_manager.py @@ -10,24 +10,29 @@ import threading from queue import Queue, Empty from datetime import datetime -#try: - # Try relative imports first -from ..utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity -#except ImportError: - # Fall back to absolute imports if relative imports fail - # from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity +# try: +# Try relative imports first +from utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity + +# except ImportError: +# Fall back to absolute imports if relative imports fail +# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity logger = logging.getLogger("DBConnectionManager") + class ConnectionState(Enum): """Connection states""" + AVAILABLE = auto() IN_USE = auto() CLOSED = auto() ERROR = auto() + class ConnectionStatus(TypedDict): """Type definition for connection status""" + state: str created_at: str last_used: str @@ -36,8 +41,10 @@ class ConnectionStatus(TypedDict): pool_size: int available_connections: int + class ConnectionMetrics(TypedDict): """Type definition for connection metrics""" + total_connections: int active_connections: int idle_connections: int @@ -46,6 +53,7 @@ class ConnectionMetrics(TypedDict): failed_transactions: int average_transaction_time: float + class ConnectionInfo: """Tracks connection information""" @@ -73,6 +81,7 @@ class ConnectionInfo: return 0.0 return self.total_transaction_time / self.transaction_count + class DatabaseConnectionManager: """Manages SQLite database connections and connection pooling""" @@ -83,11 +92,11 @@ class DatabaseConnectionManager: def __init__(self, db_path: Path, pool_size: int = DEFAULT_POOL_SIZE) -> None: """ Initialize the connection manager. - + Args: db_path: Path to the SQLite database file pool_size: Maximum number of connections in the pool - + Raises: DatabaseError: If initialization fails """ @@ -97,14 +106,14 @@ class DatabaseConnectionManager: self._connection_info: Dict[int, ConnectionInfo] = {} self._local = threading.local() self._lock = threading.Lock() - + # Initialize connection pool self._initialize_pool() def _initialize_pool(self) -> None: """ Initialize the connection pool. - + Raises: DatabaseError: If pool initialization fails """ @@ -123,17 +132,17 @@ class DatabaseConnectionManager: "ConnectionManager", "initialize_pool", {"pool_size": self.pool_size}, - ErrorSeverity.CRITICAL - ) + ErrorSeverity.CRITICAL, + ), ) def _create_connection(self) -> Optional[sqlite3.Connection]: """ Create a new database connection with proper settings. - + Returns: New database connection or None if creation fails - + Raises: DatabaseError: If connection creation fails """ @@ -141,23 +150,23 @@ class DatabaseConnectionManager: conn = sqlite3.connect( self.db_path, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, - timeout=self.CONNECTION_TIMEOUT + timeout=self.CONNECTION_TIMEOUT, ) - + # Enable foreign keys conn.execute("PRAGMA foreign_keys = ON") - + # Set journal mode to WAL for better concurrency conn.execute("PRAGMA journal_mode = WAL") - + # Set synchronous mode to NORMAL for better performance conn.execute("PRAGMA synchronous = NORMAL") - + # Enable extended result codes for better error handling conn.execute("PRAGMA extended_result_codes = ON") - + return conn - + except sqlite3.Error as e: error = f"Failed to create database connection: {str(e)}" logger.error(error, exc_info=True) @@ -167,18 +176,18 @@ class DatabaseConnectionManager: "ConnectionManager", "create_connection", {"path": str(self.db_path)}, - ErrorSeverity.HIGH - ) + ErrorSeverity.HIGH, + ), ) @contextmanager def get_connection(self) -> Generator[sqlite3.Connection, None, None]: """ Get a database connection from the pool. - + Yields: Database connection - + Raises: DatabaseError: If unable to get a connection """ @@ -186,7 +195,7 @@ class DatabaseConnectionManager: start_time = datetime.utcnow() try: # Check if we have a transaction-bound connection - conn = getattr(self._local, 'transaction_connection', None) + conn = getattr(self._local, "transaction_connection", None) if conn is not None: yield conn return @@ -204,8 +213,8 @@ class DatabaseConnectionManager: "ConnectionManager", "get_connection", None, - ErrorSeverity.HIGH - ) + ErrorSeverity.HIGH, + ), ) # Update connection info @@ -229,32 +238,31 @@ class DatabaseConnectionManager: raise DatabaseError( error, context=ErrorContext( - "ConnectionManager", - "get_connection", - None, - ErrorSeverity.HIGH - ) + "ConnectionManager", "get_connection", None, ErrorSeverity.HIGH + ), ) finally: - if conn and not hasattr(self._local, 'transaction_connection'): + if conn and not hasattr(self._local, "transaction_connection"): try: conn.rollback() # Reset connection state self._connection_pool.put(conn) - + # Update connection info if id(conn) in self._connection_info: conn_info = self._connection_info[id(conn)] conn_info.state = ConnectionState.AVAILABLE duration = (datetime.utcnow() - start_time).total_seconds() conn_info.total_transaction_time += duration - + except Exception as e: logger.error(f"Error returning connection to pool: {e}") try: conn.close() if id(conn) in self._connection_info: - self._connection_info[id(conn)].state = ConnectionState.CLOSED + self._connection_info[id(conn)].state = ( + ConnectionState.CLOSED + ) except Exception: pass @@ -262,22 +270,19 @@ class DatabaseConnectionManager: def transaction(self) -> Generator[sqlite3.Connection, None, None]: """ Start a database transaction. - + Yields: Database connection for the transaction - + Raises: DatabaseError: If unable to start transaction """ - if hasattr(self._local, 'transaction_connection'): + if hasattr(self._local, "transaction_connection"): raise DatabaseError( "Nested transactions are not supported", context=ErrorContext( - "ConnectionManager", - "transaction", - None, - ErrorSeverity.HIGH - ) + "ConnectionManager", "transaction", None, ErrorSeverity.HIGH + ), ) conn = None @@ -293,11 +298,8 @@ class DatabaseConnectionManager: raise DatabaseError( "Failed to create database connection", context=ErrorContext( - "ConnectionManager", - "transaction", - None, - ErrorSeverity.HIGH - ) + "ConnectionManager", "transaction", None, ErrorSeverity.HIGH + ), ) # Update connection info @@ -330,42 +332,41 @@ class DatabaseConnectionManager: raise DatabaseError( error, context=ErrorContext( - "ConnectionManager", - "transaction", - None, - ErrorSeverity.HIGH - ) + "ConnectionManager", "transaction", None, ErrorSeverity.HIGH + ), ) finally: if conn: try: # Remove thread-local binding - delattr(self._local, 'transaction_connection') - + delattr(self._local, "transaction_connection") + # Return connection to pool self._connection_pool.put(conn) - + # Update connection info if id(conn) in self._connection_info: conn_info = self._connection_info[id(conn)] conn_info.state = ConnectionState.AVAILABLE duration = (datetime.utcnow() - start_time).total_seconds() conn_info.total_transaction_time += duration - + except Exception as e: logger.error(f"Error cleaning up transaction: {e}") try: conn.close() if id(conn) in self._connection_info: - self._connection_info[id(conn)].state = ConnectionState.CLOSED + self._connection_info[id(conn)].state = ( + ConnectionState.CLOSED + ) except Exception: pass def close_all(self) -> None: """ Close all connections in the pool. - + Raises: DatabaseError: If cleanup fails """ @@ -377,7 +378,9 @@ class DatabaseConnectionManager: try: conn.close() if id(conn) in self._connection_info: - self._connection_info[id(conn)].state = ConnectionState.CLOSED + self._connection_info[id(conn)].state = ( + ConnectionState.CLOSED + ) except Exception as e: logger.error(f"Error closing connection: {e}") except Empty: @@ -388,79 +391,74 @@ class DatabaseConnectionManager: raise DatabaseError( error, context=ErrorContext( - "ConnectionManager", - "close_all", - None, - ErrorSeverity.HIGH - ) + "ConnectionManager", "close_all", None, ErrorSeverity.HIGH + ), ) def get_status(self) -> ConnectionStatus: """ Get current connection manager status. - + Returns: Connection status information """ active_connections = sum( - 1 for info in self._connection_info.values() + 1 + for info in self._connection_info.values() if info.state == ConnectionState.IN_USE ) - + return ConnectionStatus( state="healthy" if active_connections < self.pool_size else "exhausted", created_at=min( - info.created_at.isoformat() - for info in self._connection_info.values() + info.created_at.isoformat() for info in self._connection_info.values() ), last_used=max( - info.last_used.isoformat() - for info in self._connection_info.values() + info.last_used.isoformat() for info in self._connection_info.values() ), error=None, transaction_count=sum( - info.transaction_count - for info in self._connection_info.values() + info.transaction_count for info in self._connection_info.values() ), pool_size=self.pool_size, - available_connections=self.pool_size - active_connections + available_connections=self.pool_size - active_connections, ) def get_metrics(self) -> ConnectionMetrics: """ Get connection metrics. - + Returns: Connection metrics information """ total_transactions = sum( - info.transaction_count - for info in self._connection_info.values() - ) - total_errors = sum( - info.error_count - for info in self._connection_info.values() + info.transaction_count for info in self._connection_info.values() ) + total_errors = sum(info.error_count for info in self._connection_info.values()) total_time = sum( - info.total_transaction_time - for info in self._connection_info.values() + info.total_transaction_time for info in self._connection_info.values() ) - + return ConnectionMetrics( total_connections=len(self._connection_info), active_connections=sum( - 1 for info in self._connection_info.values() + 1 + for info in self._connection_info.values() if info.state == ConnectionState.IN_USE ), idle_connections=sum( - 1 for info in self._connection_info.values() + 1 + for info in self._connection_info.values() if info.state == ConnectionState.AVAILABLE ), failed_connections=sum( - 1 for info in self._connection_info.values() + 1 + for info in self._connection_info.values() if info.state == ConnectionState.ERROR ), total_transactions=total_transactions, failed_transactions=total_errors, - average_transaction_time=total_time / total_transactions if total_transactions > 0 else 0.0 + average_transaction_time=( + total_time / total_transactions if total_transactions > 0 else 0.0 + ), ) diff --git a/videoarchiver/database/schema_manager.py b/videoarchiver/database/schema_manager.py index 09d7ac5..110a7eb 100644 --- a/videoarchiver/database/schema_manager.py +++ b/videoarchiver/database/schema_manager.py @@ -7,12 +7,13 @@ from typing import List, Dict, Any, Optional, TypedDict, ClassVar, Union from enum import Enum, auto from datetime import datetime -#try: - # Try relative imports first -from ..utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity -#except ImportError: - # Fall back to absolute imports if relative imports fail - # from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity +# try: +# Try relative imports first +from utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity + +# except ImportError: +# Fall back to absolute imports if relative imports fail +# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity logger = logging.getLogger("DBSchemaManager") diff --git a/videoarchiver/database/video_archive_db.py b/videoarchiver/database/video_archive_db.py index 92ef704..5194d2c 100644 --- a/videoarchiver/database/video_archive_db.py +++ b/videoarchiver/database/video_archive_db.py @@ -6,9 +6,9 @@ from typing import Optional, Dict, Any, List # try: # Try relative imports first -from .schema_manager import DatabaseSchemaManager -from .query_manager import DatabaseQueryManager -from .connection_manager import DatabaseConnectionManager +from schema_manager import DatabaseSchemaManager +from query_manager import DatabaseQueryManager +from connection_manager import DatabaseConnectionManager # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/ffmpeg/__init__.py b/videoarchiver/ffmpeg/__init__.py index e73895b..b242264 100644 --- a/videoarchiver/ffmpeg/__init__.py +++ b/videoarchiver/ffmpeg/__init__.py @@ -20,12 +20,12 @@ logger = logging.getLogger("VideoArchiver") # Import components after logging is configured #try: # Try relative imports first -from .ffmpeg_manager import FFmpegManager -from .video_analyzer import VideoAnalyzer -from .gpu_detector import GPUDetector -from .encoder_params import EncoderParams -from .ffmpeg_downloader import FFmpegDownloader -from .exceptions import ( +from ffmpeg_manager import FFmpegManager +from video_analyzer import VideoAnalyzer +from gpu_detector import GPUDetector +from encoder_params import EncoderParams +from ffmpeg_downloader import FFmpegDownloader +from exceptions import ( FFmpegError, DownloadError, VerificationError, diff --git a/videoarchiver/ffmpeg/__pycache__/__init__.cpython-312.pyc b/videoarchiver/ffmpeg/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 093539e3432920bb5d765461e6b0a635ae9b7f2b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 396 zcmZ8cy-ve07_{@RKsq8Z(KSOc6H|m3pq)T0EMBY-yJWC#4 z7!Vs$Hzw=^7QW$pXWw@^KSxmr^gJvM^R+(TjrbeumuwC+Spf<#ppaT6LU>$ wMOCzf?AJ>bBwWy&k{!u`4mAI5b~Jq15W*L*_W_bMNM1qw7EbPcxvigm08G|xtN;K2 diff --git a/videoarchiver/ffmpeg/__pycache__/exceptions.cpython-312.pyc b/videoarchiver/ffmpeg/__pycache__/exceptions.cpython-312.pyc deleted file mode 100644 index a63a869289f238eaa115876f48a64297fd509fcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 823 zcmb7CyH3L}6t$C-ra%RZOazI62udUdCd7a!NFY@RiKUBWVz-Ty*1_q6!rFn6jc-7F z3O@i@82ACCmW2slr%eH|;NkfAT-*2DYd;l>vq+8C-cEfDBlHG~$;ge#@K}+1#84F( z=m;@9K+NFy(M;o2L#w7zEpBA0snwiPXTGUcKkwSt%ZG=}h}TP!hqTR^!>@cEwFA+L z{Yi3|l_5Qbi)y+@996M0VK7`Z`-W>KbLll@DTTaV-={I3-sIGTbbhCSjdA6{N^C+} zw8;rcW(jEu)(L^n6LQ(1;W&~fgbAOJ2Q&m1pZi`zG`Sa1-(x}S3&~2|fN@c}5N*#F zbx*IT^cz8!ORrXg61{0@TTx#^%QOIqZ3P4LVr@TLwuE=kp?|f&J~};5>t8veL9BY& zYw(t%R7XTyQr%RiMuV^-Eyz#+(7EP-6(^8aBs1h@h9LegL&XEpYlVWc)Ya;DSLt&& zY`jC;ivRYt0KNdgW#(%m^97-bo0HJOq)?vxpR~ccD1F1vY7=`med|5Bq|U+jiZ>O! fn;7F4v@)^`T)1;bNKGSK!R^hI-<^Dbu6y?dtl7Jg diff --git a/videoarchiver/ffmpeg/__pycache__/ffmpeg_manager.cpython-312.pyc b/videoarchiver/ffmpeg/__pycache__/ffmpeg_manager.cpython-312.pyc deleted file mode 100644 index f638d3d8ab0c9f40a27d6fc037bf9446d88c4102..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5501 zcmbVQO>7&-6`ox#DgKF+MOhLlTUtx9Vp^7%$nsATJC0(-ZWCK_tHyDdIt0aC$rMN~ zJG-(iLjHl_gA4?4bCQFihz~tDb%82}A}UazP18e)UPw>{-Hn0<2ztnkwF0zAd+2+! zKO|)-LAsD;-pqS5^XARG@4ew)nwz}@zF%HFJiFRV$QRhKKEWh5eh0)dkw~0KoWy6i zILB%}&a>JPcd%NB3s5^U&a5l$;!sw|G-cg!ch(d4WW8}O^f@J0#+UWS{futPG-m_x z0HfWRV74XR!e~!sN47QI%4lyUlx>T*afBzbFt^`Qu2xIFhn&%hxAWvQk^Hxb)clY) z`#Ki*XrQqBY%-k_Po2u<d=MpbV}15 zW4Q(0J(<_iN-mlC5?28kpE1kJlqwXPr_a4ME^Bg1W6kFCX-QVbKz^YhBft6OTuPB- zdM-(mSrs~3jPc{jm0U(i0!KH;(A1^$WtpZ7#f{Q!Ha30%#4?cy*q)%>ZxgUOFAFog zyJNv0ZPuHO4W4CIrYZAq*hnXo4*1`A486-lvw0F{ z9pWNM*u5IcvrlWXs5#3dRtMCUT6I$FTGxrxNiC8&^K#Vlf5ksU&ENwKTn2^oT+pIx zwY45d+S+I#iw|R3EV@?L?a$Ut>#+FWI>uio*=EtTdakQRTU4Ev0jp2zv8XxGqH1;B zh#gk1w%4LS$6kxB)zG`E*0Lze{lG%DT^1Zsp%DIlF0G}LnRG#JbVpk7#k6cbHZy}h z3O?vk7gSBo>P^`sI7Uv^-P1``PDp7w5pAP*RureEXc^s|&m^@Og=Q%#PFu0hE7e@A zHIcU0O&Ck)KBG?L!RM(5NBC#+^9jjxW{St82QH&OC*UqKhlWWRI{}iq2TPcEM$v=W z`HYs%Qza#z=`b{{Xt9(s8)L1ci58~4%qn3(8y(IAa3$ED6Ny55gF_Bm{b)V) zH&EOpU%5zl_q~hd&cP+&?Ph3o4X%f}mQTEMq8#d9Z`)OA>o2wSm)iy&?F=tpeCOg< zO+uF!M1$1;*(FxGMoL{H<*v~sVa?qMqj&FnAXN4pE$us6-t{bYhS$3XABm42*F6_6uWyyjzVO@wePPapy$EH&F5oRD44v-_V*bwlebr=2J|6UJvJ|;Yj zMh(u~IaBO9y1HZ4Q4Akj^B-St-CY!quO?Tgirr)7)|ZO@m;O_AqV(4)<~V=s7$YY; z$;X|Zalhl^Jp!_ar zy3o}a2^vNo#}PQ=P4am#R0;N$g1wdC)1~0kE0J>WSkZlq_5)WBXh}LNYY7!Xcp^KU zs^6WR-(3PpW)C};gAVhn?fseq0I=Skro>i2yy#ejw6_2&7^RhAfc^Nw^QUFawp2>V z<{{RqkhO~}!a~HRDl?{bNfu`z$Gk)XYgo z7?YWqMST?pqAwZFq_^X+y2Z^Y3jK{n-&$%6rc&`1F&}G>RCYd9+WFK2Z+YjzC2!q% z43&aI59U^fKN&4OH(C7Q`Oixm&s~L}lZoD>|zPWD*myko-Bf#J`U7KR7 zbd~xEcBjIXi65TJY+>8 zQJL9v(p`p(CD0f*_tat`03J+biPhrasi>PqK|{J93lhseA5sUf7=pqAz+s^4 z0Z0Q#mwP^bcCmh>d2!=-BEHR!9UmQb0^Q)z2$gufHV#Aa|9CVDHX-y#_|*b15H_7nVoCy5!xyA3q?G)^6Q1U_ z-Z`6}5?22O7^|YsGW2R`yL} zie&gHg?7W(CzLf`ULaBEXgp&38FmrIsrV+g)V*=%M$z5P{CpzXichCpGAkz%@ODcW zKM?`#Pb7XapUfB>+J|f74pVq zrrH910ogA?@jhY88+jReCB*_Cvg!&Hx5;C^-FcR)KHcIx$=x5{AXr!XdFSy5jw*p_ zMOc|y?OVC z{cdOYzOX@{skz4!X>srSe4Exa3p+{(Fk!(lP1Zj#T`C$LLuNpOGm0i+%2iEOk^o ak&-8J??l-%c+>SSVS?j&9uq9tTK@xr*|JIi diff --git a/videoarchiver/ffmpeg/__pycache__/gpu_detector.cpython-312.pyc b/videoarchiver/ffmpeg/__pycache__/gpu_detector.cpython-312.pyc deleted file mode 100644 index 07ddbf6b4fc15c3447be0ea5b7b720835a5c7da8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5996 zcmb_gT})eNn*NT@@h=A30b?5i91=)yio;JRp$sHT10+Ey=}t188Ptj8_#Bht@n64l z_=yE=HR=UuBbCezM4eUWUd&Ln7nOD|a&6rrs&+sf3W(0`&ggI%Mv2fT{NLZ7$85>7TL>A(QjDh!J$^0=_%FQ@T9ubtlAIo+(HqLVOrxcdOJQY$amcR|nZU*fZc653Cdu?3Zitu7vPHJW zOaW*Er2>@NBq)O#YfOLz3p4f?T3Mmg@j|H$N^5S~gATn0wlvOml%{%3EQae3XJFaU z^48e`G0ns8DW2RqS>l#Rm{nkKlLR@~Eww>9Xwn5$PQ+M7#crbbY`o`2Iw|*LBGDd6 zQKM-ph34@nLf6w;Pc%K-!*Y@G!Ss|C(KDNw3zyD1O;2WS>b7uLNhw-5oNd^*=95Ng zEzG5&2)#=R&Bs=RKim9IAxCrzm9;sVik7bz%ZL?zY(GDSQV=KjbTq^Ou>>`A=FL7~mrC}wpMB+I5f$!sT-Vibl&hX0Lnnz&sw0-Y9_5+hZw)#9r ziN=%*+jkJ!y=>B)M&G#0NbSXvSpyX;eM=@|%^#3|t)w~fb5N0G)7vK8M=75DP>GTGh?0n0Psk$bP9!CXil&OuNG5V!0ku(Nb@C%Yx1>M>l!(qpl9Fy# zQktBAG)?N}Oj=bIbyG^qHXcwu7+02t#p$43H>FiwNJWyeZkfwS5lz-TWzh*Il+@f} z7-UJWH;NXNl$2gjS)qMmF)C+Jn{`VfJv%E?-MSE=DJ3Itk!;gz>{HG^uLf; zC!jYK9Rn@c-xBKaGb9Og%_Q-XAeom?>nxIW)&VkE(Wbbi)P8@sWYP>PN#bZ~4tzgc zE7?A=e_Zar%rEiriazWOBRNptt`N|cigtju@F?pyYbvts%!*fKRTGt%7!lQs993dU zR4kv5xR9p5RI{c~OgASY^DzaIO14Xb4l0d~N^>-V!ZzABtSVV~w6A}tcky(8?=Te0 zqo;baLMS>s53)R$Na$SXYdn#Uc-^9EQhH8fu}nckH6J2 z4a!^02A(s&`>Svn>M{iHlA_?T2mZ%hzc!w4=*Ts6?6^A%?uNBl2(}ylso-whuwp(| zaMu@Wt^T@|3%hR8D6U_-eGO#$v9Epoo!jq}qk=!MYd1I6txOi3q@gKa-;t~D*y3{a z!EI;o+s7_H%6j07dVi6J_?e0z^{cTXW1ZxFr+uuCzu(J2_#6PBfHBd10$~*Zs0ea3 zF`gw7|8Lwrkf;bIj9UJFxKZ=I+JfRJsW@v;Ce3H#j zG*B=52-JdVq7hGN zSy&RJTDHgU+ee+T0@#NO_aV9Q&zPTk&g4|PDakYYkGm)HlDXPF#M{4rk8#wm17BtT z7U|HE1#a=MRFAPLNDVCieW&5wcN&NAH1_O$Qtv+kG)tI558C?H5%hnv*d#l z_v0K0?_dxs&Epl*SScb}oso{2z+eh!FxR+SKQc__QPM|LZo!1*9Q@sST%rju?qQ>+ z)@r2K7|iJ2D(;Q&cdek*d(G$_3}g?N=?IuERRrpO32A|--c&U?sZLT6tIC%JU3=_RU~;6B~a8IH~I$$!l`+%kQ9kiw!TM_=X)fo z^iVkkCU+bRefD?{%-0i5L{t^3lKZN#wGx$kU;qhdI7f9$?_ytXZ@+G1BS(@kibic3 zKt;_&6CpjIu8`w8bLBWIG~$>L(QMQN*kbk&`U!Q(UTbT zV9<-f2@Fm_px0e3p~}Q!MxnB#dnSQlO{cZV^ju1spfpV>ngLxv_dvJEiwc06E~GOu zpbV8Gz}$c_DH+{*@iJ?!TQ6TJB`lJR^Z}NiUI2<^o?~Rqnv8%il#&6ovtVYm;Si|Llq`OsmkvjF~O<}`K>2vzVgLPndQHrw(XMwO{pg$oWq6ZJpA z#=k(wEDpQpap36P);q1b`{imd)Ks@J1u*1p%DXypE`XtjuHaVu-c)|%wH*APe67$K z%y*v7b)LTG-07THvu*fu?juEq6$UObz4zZD*^m9(gbR>7NTN zt#{wJ^Tr21yL;`53l>hC%%2#^ofx?{vvcCYhHcY($MubsG){3v zcMS~N(7HZ-d%EE9ui2}u*ju+U0V{57UVroUn|W_n&fB%)J(2hJ=DfW--u{)zf~#@k zSk5JG%3Jdvrtbwli{)OqROsl+cMRq_20zX2bi4}N@a5d?yAI;4dw*zk=wF8OH7&WC zmV8ZDuBK}{IQppOOriD2%7wL`tX^i+^4s0B#M7L22XpS=qnAduU89K79^L6il=eLP z239o1ZF?_%w`(IF|8IQTu6EYqi*k$ab`Mt@z^HzvUWQv-9qXEOll%QIO^%S?w+v18 z@(;W=NIz&DJ9FMcK6eKuL;UC69EQE-$ycnOzifr-&qrM6>-ay`au|97=R^D-yEzD- z;|LWeu^VXCiyT4XfwKM}PhbSsG|L0Q1&aHLt%CSSK&x;D$!}@Wz-UQZDZKgx}dU&wMt+xaNzZ#9@Gxyi3 z`Hg%vh9cE#Ek^y4)fn{!_qSo$`nHw1zbv)S{q<&B58%yDOFrlGh?5;SFKeNnCY8(A zClu4j;bSrMpV5aClXm(=v+FP1h@L? z*XYp%dA$yNYk2RF!zKHu3!1#B(Qc?@xTOoz=qD3~Q*;oE5m|uwN8m*!MPI=Zyl*-R zLAHb0ZxQ?!@Q87N=s7c2zGPFumq8cYwGUblRdgYaD+O~OkyHjTxNb+gRC2EB=(Z9g1j)Nr)4UD}TgnO6?}K0F+N`%UpuDY7vY z=`w?a3MwEG&9+rTg?-NfQS94Q4HtxfeTsV$xV$p;*xSaO6i>@Wc58UsJy>WvTJQ!6 z{wBjm0e8JI?-J3Vjy-gBK5lNiJA7w&GxodWhsmAhfi)W#>CLWB#vl3m3S!qrcJ0Df z%}3YH6`EV`p1yN>b8xGFr}<>Qc_7z3u+u!aHeT?xZ4Tso#|xppeCTX0boMjnPUtm& z0e{YaqToBcDcoM(vVA(9>l&(t6a`=FX2-AJDRlSdyU*mh&)m!IbWcHV-$Q?}=pu(Z ziv9!722&iq8(dnwwB3C65Bz6i4=ncyB_ZnU>}fn0AK4Z_Py#r5cLf7)5eP(K0fR zRu`_P(+PS6^Grk#+9}@TM}(r4D*--QONs&ZR z{4l&rAX}OmCxoq8!(k~M4Tl*uqBBs)R6)1mYZAU9S&=OzlvXTEE*W^nT diff --git a/videoarchiver/ffmpeg/__pycache__/video_analyzer.cpython-312.pyc b/videoarchiver/ffmpeg/__pycache__/video_analyzer.cpython-312.pyc deleted file mode 100644 index 2979b923f67373554992a03a77d7a1a6fba7e682..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6804 zcmb_hTWlNGnLfjrAvqK;k}Qi9B~g?mUuavB9mjTK%W^2mcH)bY!o4_JqyvgGk|^`$ zGb76)D6e7oK`ye0x>(e5n?;m|A_|;hRcv>mqWcg#Ss;sj5lwH!PE>3b-G};(i3~J# z9=iX3hNLJ*?QXC=GXHb^&-v$^|MH#Ve|NbY1k%21N5&4+6Y^WEl#@^@tS>_04pGQ3 zQ8>jC=Y~0!TZSzx=ZATg3&R5Bd|XUehpilr72>vpeb}Cmh9y=PyMMedb4hifh53{mX2h$7vyl*ZIeTSAV^?(f7D zH5H5`Bk^f1rUgeQlTkgEf@(~k4vwa1@bu|KS{-|gTtgww5MGVw;~|S74923m!H1L6 z(CCh)lDc|LPry85Dm7ee)-a8Z$F8b0S`ve063`C#t-l4ro+BJ=mR=H@kcpCT)eH*v zzNP$OGejb~Sni#$mP;FXmMDDr6FHJ4KeXK8IOM{0S(;#nSgx5la0%~=8+_CD4x&5D zV>$`lIm7Aha($w%T-wMLYnJ?&D7Gy3WAgVxSzeC32HQTvUFW9lWGWQyzf z+EPuy#{Ffbnkq@RA(biHPx&b>BxQ~cn8z9{oh#D^T}`A@G(x9?Y0!jVl&TR_7*HW4 z7>%oupzhLt9Cs;Y)` z>RMDyqhN+Lo*F}?6xouXx@;yHRxmhKLyZdtpPud+PbJizbR^oN#I$INDqUC6GP*`m zdQUVp*28L%l1BHK-)A<;%y=>??M_b{btssOJ0q8Dg8ErAKBXr?#A!Civlk}hqM083%4)K?^){n zmGkHqYaZX{ge$e5zkPnCxp(=}^4SkYKCb(iUuinO>b#J1Uih-en;M_WOhf zURL%=16JFE_T~Z3_MlgU-VZquT0XS$SeB4$lfNk2_<9^fxI9P3xOM7^dw@1I!9s@T55Wz}63E{kHv@7PjKX^RzY|3_ZO7QFT>nLy)zCT01` zE=Vw9i-1yf4vwmGsM+cq6leK+XKG-!ZO<$Q)oe{=J!e+Pidk#cmbGW4tV3}r?zik& z=TEqc7+xC6yb$!dW?Xt>xjung_DoVdI%+jbSyy=l2!NG<&5;<1S|A8IXI$4^7fbm@ zxCzx}4uwrs4>AuMWL`6=vrv-)2cC$;gK9FGQZPDbs;)z>MJp-{Yol@>g5UacKukoD zC9lJ#&v04r6)3mDUETxc!@c3mazEq{5xh|+kZB8xbG+ez>D8#LPp4HwWMT}xN8Ym- zmhJ-vr){!Rsdr3Gs@KxAFVnr*P)kujX8hf~@l-St*ZR6E<2ykPtqlkZ0V}PSa-kO% z!pq-W&i+DPdG)*b6W@I+0!@Edx5Jp5E-yD{^KynwgK!o}K%AS3Df&1?S7TVm)!5j$ zZaAqtN+StXrV(9zDjYh{bI`B?W=lkL!>&w1E5<+j2>2Bo@t55-iXUW%kx3<%8lt

Rp3AlDpS!ZeE$#oA=Yym9184uP4qE;|_rZ{%(;++U zf&%RZV(_DBjiMG9Huz@zWWzHa(d2R1Ejf{5yTKZi2)!(8Q8-D>sK*h}R3aT$ua(*| zUK5p=M~aHa4)s_t!p0trP;H9h_?IY9dEnux)-LF zxV-PVe8V9a3yuD_19t<9_NDfG)ARZ5hi6Z()i=&%?p*)D^@ZfpXx@7wU*9)7u;y)< z|MtQ+?#ipJ2Xd_kmi)`#%(tG*dtaP=X|1t&{?x*ryRWRa^yFH4mK@9Bd`n-x@#O58 zqLs9DtOkzc0!NlI`9S~dD}}(0)xgnQ;OMd=9~hW@1qAa4eqDPGW(YlO*`KdHSEy}R z7kR&X_H5Bkn%h?W&*%KlFAeAYFV3EW`TT7Kf3V>77g{ta4_U4 zGm4vo&Os0P$RnK+`Hy@%P7(g2V;q$Km54y;Y1sNr(I4Y7i~deH4~+gW&m0|O))_?i5H#IG zCH#y;G*t=0wT3y^Ao`NZCE0LCu8viCUfKgf(E~speLBO_H-ZrG+GGzkueo`TMhXj(cRs$pYmf`4On%}f;+W0uM+l=ee4)4tg* zw{So&!7%OnK+rm^(jZsxG`)HK_VtxO_ajfwnm;g~S=oCm*LFPb@13))HTvg|z1?@W zZ?SuMSH3x%Z|tAr*S!AuqYD@Bp2&Mci{YHNYo)fUXd|@;icV6y{fj3eQ)>?9M!g=o zJ+#vL!oPGZM}E0$*|*Z%zv>)VaSjwb4WQUK=&6QRq?=(!*hB7nq<)^i?`seD@%Q^g zC~s0{R0F2YzbmQp2AI7EU6>A&O=?Oe1hc)GR@4s>0;sG4_XP!DqhcR(%v6EdiHZVl zaA$c%Ql$4C_e!QPBPca9BK)jbVXL==i3;%Cm;(S?$p{C@>k#l}Y>F#uyNU?Ku24e)A1l$w5)hm0_@Ib{d z-Ts$DyZ3AG?f`&HkLX&))uoN6rerm#QwXw|b`wPDkumK=JaTn3)?GEY`?c_Q&cFhr z5b7u6af9pn7Tn_8(MTF@TB@9y)ZzAJ2>AMCf-}xx2*_h_Cm!P=Ki&gm>_>Mo{oBc* zOv_+J@QO#Nc!$%W8unhm?4R8d4Tm;40*D~0YMQ~*NqEoDbOoL=3{gwNBOMLH^i?=v z2&y4R6-2<}fJq_o*!I1OGqnd@nln%3%eJgVzlwEuU5cV1Za?aBG} zeKKNiFerA}A;{1PUFy3r($0Y=Xaj z&I;bAEjU+G0OBmP?0Eaa-3z6VFZ`s;h=zR+D~v4*rBDY}WTb0L!R?tbVUTKuc! z-dxw2j}PVc5B>fiso%L;*ZHun^UEjCmqL2emagxSb6{ zp#LC$e_t=q2fHO`d2mqdKW2MygvatRMo&ls2l)@J?F0My5BG~ure}bElY2yW#oXi1 zp@0t&niXG-(ZRjNU#zRu9;;&c+B~+eq6AaIkUbMR z1(pOlrggBxHNLUAe)H19(}26pi>^cYF(#xFtZdBbJG*)sTdo7L!eB$-s>no(!g zOSXV9or3d)$Ct{8nyS8#h4w)&3)WC(pc{95(+vk4C&_d?qxxGxc^x@l2Xce_m#68| z_8o;C9fh5HKeGvl1_hV>+TI5Mb`6^LzT5pyHv_agPUZtIuC;eA z48QxmcfPlD;e*5ZoiFFx&%z|N4wyu8P`qFcg`W9ib*-0N5^zRF4B_C=*gAY0PcF)KEI-LQVjc|Nl^xFK$xCzjeGEB@Z6a-8FHf|Pyz{{TlYTLS<9 diff --git a/videoarchiver/ffmpeg/binary_manager.py b/videoarchiver/ffmpeg/binary_manager.py index 292e594..8cc51bb 100644 --- a/videoarchiver/ffmpeg/binary_manager.py +++ b/videoarchiver/ffmpeg/binary_manager.py @@ -7,15 +7,15 @@ from typing import Dict, Optional #try: # Try relative imports first -from .exceptions import ( +from exceptions import ( FFmpegError, DownloadError, VerificationError, PermissionError, FFmpegNotFoundError ) -from .ffmpeg_downloader import FFmpegDownloader -from .verification_manager import VerificationManager +from ffmpeg_downloader import FFmpegDownloader +from verification_manager import VerificationManager #except ImportError: # Fall back to absolute imports if relative imports fail # from videoarchiver.ffmpeg.exceptions import ( diff --git a/videoarchiver/ffmpeg/encoder_params.py b/videoarchiver/ffmpeg/encoder_params.py index fc2ebbb..db9fe53 100644 --- a/videoarchiver/ffmpeg/encoder_params.py +++ b/videoarchiver/ffmpeg/encoder_params.py @@ -6,7 +6,7 @@ from typing import Dict, Any #try: # Try relative imports first -from .exceptions import CompressionError, QualityError, BitrateError +from exceptions import CompressionError, QualityError, BitrateError #except ImportError: # Fall back to absolute imports if relative imports fail # from videoarchiver.ffmpeg.exceptions import CompressionError, QualityError, BitrateError diff --git a/videoarchiver/ffmpeg/ffmpeg_downloader.py b/videoarchiver/ffmpeg/ffmpeg_downloader.py index 55670db..3c8be3e 100644 --- a/videoarchiver/ffmpeg/ffmpeg_downloader.py +++ b/videoarchiver/ffmpeg/ffmpeg_downloader.py @@ -18,7 +18,7 @@ import lzma # try: # Try relative imports first -from .exceptions import DownloadError +from exceptions import DownloadError # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/ffmpeg/ffmpeg_manager.py b/videoarchiver/ffmpeg/ffmpeg_manager.py index 87945ba..f4c5bb4 100644 --- a/videoarchiver/ffmpeg/ffmpeg_manager.py +++ b/videoarchiver/ffmpeg/ffmpeg_manager.py @@ -8,13 +8,13 @@ from typing import Dict, Any, Optional # try: # Try relative imports first -from .exceptions import FFmpegError, AnalysisError, FFmpegNotFoundError -from .gpu_detector import GPUDetector -from .video_analyzer import VideoAnalyzer -from .encoder_params import EncoderParams -from .process_manager import ProcessManager -from .verification_manager import VerificationManager -from .binary_manager import BinaryManager +from exceptions import FFmpegError, AnalysisError, FFmpegNotFoundError +from gpu_detector import GPUDetector +from video_analyzer import VideoAnalyzer +from encoder_params import EncoderParams +from process_manager import ProcessManager +from verification_manager import VerificationManager +from binary_manager import BinaryManager # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/info.json b/videoarchiver/info.json index 08a838f..530a55d 100644 --- a/videoarchiver/info.json +++ b/videoarchiver/info.json @@ -23,10 +23,15 @@ "download", "automation", "queue", - "ffmpeg" + "ffmpeg", + "discord" ], "min_bot_version": "3.5.0", - "min_python_version": [3, 8, 0], + "min_python_version": [ + 3, + 8, + 0 + ], "hidden": false, "disabled": false, "type": "COG", @@ -52,4 +57,4 @@ "read_message_history", "add_reactions" ] -} +} \ No newline at end of file diff --git a/videoarchiver/processor/__init__.py b/videoarchiver/processor/__init__.py index e6f07a4..b8b5497 100644 --- a/videoarchiver/processor/__init__.py +++ b/videoarchiver/processor/__init__.py @@ -1,10 +1,10 @@ """Video processing module for VideoArchiver""" from typing import Dict, Any, Optional, Union, List, Tuple -import discord # type: ignore +import discord # type: ignore # Import constants first since they have no dependencies -from .constants import ( +from constants import ( REACTIONS, ReactionType, ReactionEmojis, @@ -14,10 +14,10 @@ from .constants import ( ) # Import core components -from .core import VideoProcessor +from core import VideoProcessor # Import URL related components -from .url_extractor import ( +from url_extractor import ( URLExtractor, URLMetadata, URLPattern, @@ -28,7 +28,7 @@ from .url_extractor import ( ) # Import validation components -from .message_validator import ( +from message_validator import ( MessageValidator, ValidationContext, ValidationRule, @@ -41,7 +41,7 @@ from .message_validator import ( ) # Import reaction handlers -from .reactions import ( +from reactions import ( handle_archived_reaction, update_queue_position_reaction, update_progress_reaction, @@ -49,12 +49,12 @@ from .reactions import ( ) # Import progress tracking -from ..utils.progress_tracker import ProgressTracker +from utils.progress_tracker import ProgressTracker # Import handlers after other dependencies are loaded -from .message_handler import MessageHandler -from .queue_handler import QueueHandler -from .queue_processor import QueueProcessor +from message_handler import MessageHandler +from queue_handler import QueueHandler +from queue_processor import QueueProcessor # Export public classes and constants __all__ = [ diff --git a/videoarchiver/processor/cleanup_manager.py b/videoarchiver/processor/cleanup_manager.py index 0de5594..f350da8 100644 --- a/videoarchiver/processor/cleanup_manager.py +++ b/videoarchiver/processor/cleanup_manager.py @@ -20,12 +20,12 @@ from typing import ( from datetime import datetime, timedelta if TYPE_CHECKING: - from .queue_handler import QueueHandler + from queue_handler import QueueHandler # try: # Try relative imports first - from ..ffmpeg.ffmpeg_manager import FFmpegManager - from ..utils.exceptions import CleanupError + from ffmpeg.ffmpeg_manager import FFmpegManager + from utils.exceptions import CleanupError # except ImportError: # Fall back to absolute imports if relative imports fail # # from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager diff --git a/videoarchiver/processor/core.py b/videoarchiver/processor/core.py index dd277e9..4bd0552 100644 --- a/videoarchiver/processor/core.py +++ b/videoarchiver/processor/core.py @@ -8,7 +8,7 @@ from typing import Any, ClassVar, Dict, List, Optional, Tuple, TYPE_CHECKING import discord # type: ignore from discord.ext import commands # type: ignore -from ..core.types import ( +from core.c_types import ( ComponentState, ProcessorState, ComponentStatus, @@ -16,9 +16,9 @@ from ..core.types import ( IConfigManager, IQueueManager, ) -from .constants import REACTIONS -from ..utils.progress_tracker import ProgressTracker -from ..utils.exceptions import ProcessorError +from constants import REACTIONS +from utils.progress_tracker import ProgressTracker +from utils.exceptions import ProcessorError logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/processor/message_handler.py b/videoarchiver/processor/message_handler.py index 7ba392d..45b36ef 100644 --- a/videoarchiver/processor/message_handler.py +++ b/videoarchiver/processor/message_handler.py @@ -21,12 +21,12 @@ from discord.ext import commands # type: ignore # try: # Try relative imports first -from ..config_manager import ConfigManager -from .constants import REACTIONS -from .message_validator import MessageValidator, ValidationError -from .url_extractor import URLExtractor, URLMetadata -from ..queue.types import QueuePriority -from ..utils.exceptions import MessageHandlerError +from config_manager import ConfigManager +from constants import REACTIONS +from message_validator import MessageValidator, ValidationError +from url_extractor import URLExtractor, URLMetadata +from queue.q_types import QueuePriority +from utils.exceptions import MessageHandlerError # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/processor/message_validator.py b/videoarchiver/processor/message_validator.py index 85e279f..8c04122 100644 --- a/videoarchiver/processor/message_validator.py +++ b/videoarchiver/processor/message_validator.py @@ -9,7 +9,7 @@ import discord # type: ignore #try: # Try relative imports first -from ..utils.exceptions import ValidationError +from utils.exceptions import ValidationError #except ImportError: # Fall back to absolute imports if relative imports fail # from videoarchiver.utils.exceptions import ValidationError diff --git a/videoarchiver/processor/queue_handler.py b/videoarchiver/processor/queue_handler.py index 9f76713..d31d047 100644 --- a/videoarchiver/processor/queue_handler.py +++ b/videoarchiver/processor/queue_handler.py @@ -10,15 +10,16 @@ import discord # type: ignore # try: # Try relative imports first -from .. import utils -from ..database.video_archive_db import VideoArchiveDB -from ..utils.download_manager import DownloadManager -from ..utils.message_manager import MessageManager -from ..utils.exceptions import QueueHandlerError -from ..queue.models import QueueItem -from ..config_manager import ConfigManager -from . import progress_tracker # Import from processor package -from .constants import REACTIONS +# from . import utils +from database.video_archive_db import VideoArchiveDB +from utils.download_manager import DownloadManager +from utils.message_manager import MessageManager +from utils.exceptions import QueueHandlerError +from queue.models import QueueItem +from config_manager import ConfigManager + +from utils.progress_tracker import ProgressTracker # Import from processor package +from constants import REACTIONS # except ImportError: # Fall back to absolute imports if relative imports fail @@ -66,11 +67,13 @@ class QueueHandler: self, bot: discord.Client, config_manager: ConfigManager, + progress_tracker: ProgressTracker, components: Dict[int, Dict[str, Any]], db: Optional[VideoArchiveDB] = None, ) -> None: self.bot = bot self.config_manager = config_manager + self.progress_tracker = progress_tracker self.components = components self.db = db self._unloading = False @@ -392,11 +395,13 @@ class QueueHandler: return # Update progress tracking - progress_tracker.update_download_progress( + ProgressTracker.update_download_progress( url, { "percent": progress, - "last_update": datetime.utcnow().isoformat(), + "last_update": datetime.now( + datetime.timezone.utc + )().isoformat(), }, ) @@ -439,9 +444,9 @@ class QueueHandler: download_task, timeout=self.DOWNLOAD_TIMEOUT ) if success: - progress_tracker.complete_download(url) + ProgressTracker.complete_download(url) else: - progress_tracker.increment_download_retries(url) + ProgressTracker.increment_download_retries(url) return success, file_path, error except asyncio.TimeoutError: diff --git a/videoarchiver/processor/queue_processor.py b/videoarchiver/processor/queue_processor.py index a79caaa..ef89fce 100644 --- a/videoarchiver/processor/queue_processor.py +++ b/videoarchiver/processor/queue_processor.py @@ -1,97 +1,110 @@ -"""Queue processing functionality for video processing""" - import logging import asyncio from typing import List, Optional, Dict, Any, Set, ClassVar from datetime import datetime +import sys +from pathlib import Path -#try: - # Try relative imports first -from ..queue.types import QueuePriority, QueueMetrics, ProcessingMetrics -from ..queue.models import QueueItem -#except ImportError: - # Fall back to absolute imports if relative imports fail - # from videoarchiver.queue.types import QueuePriority, QueueMetrics, ProcessingMetrics - # from videoarchiver.queue.models import QueueItem +# Get the parent directory (videoarchiver root) +BASE_DIR = Path(__file__).resolve().parent.parent +if str(BASE_DIR) not in sys.path: + sys.path.insert(0, str(BASE_DIR)) + +# Use non-relative imports +from queue.q_types import QueuePriority, QueueMetrics, ProcessingMetrics +from queue.models import QueueItem logger = logging.getLogger("VideoArchiver") + class QueueProcessor: - """Handles processing of video queue items""" + """ + Handles the processing of queue items with priority-based scheduling. + """ - _active_items: ClassVar[Set[int]] = set() - _processing_lock: ClassVar[asyncio.Lock] = asyncio.Lock() + # Class variables for tracking global state + active_items: ClassVar[Set[str]] = set() + processing_metrics: ClassVar[Dict[str, ProcessingMetrics]] = {} - def __init__(self, queue_manager): - """Initialize queue processor - - Args: - queue_manager: Queue manager instance to handle queue operations + def __init__(self): + self.queue_metrics = QueueMetrics() + self.processing_lock = asyncio.Lock() + self.is_running = False + self._current_item: Optional[QueueItem] = None + self._priority_queues: Dict[QueuePriority, List[QueueItem]] = { + priority: [] for priority in QueuePriority + } + + @property + def current_item(self) -> Optional[QueueItem]: + """Get the currently processing item.""" + return self._current_item + + def add_item(self, item: QueueItem) -> bool: """ - self.queue_manager = queue_manager - self._metrics = ProcessingMetrics() - - async def process_urls(self, message, urls, priority: QueuePriority = QueuePriority.NORMAL) -> None: - """Process URLs from a message - - Args: - message: Discord message containing URLs - urls: List of URLs to process - priority: Processing priority level - """ - for url_metadata in urls: - await self.queue_manager.add_to_queue( - url=url_metadata.url, - message_id=message.id, - channel_id=message.channel.id, - guild_id=message.guild.id, - author_id=message.author.id, - priority=priority.value - ) - - async def process_item(self, item: QueueItem) -> bool: - """Process a single queue item + Add an item to the appropriate priority queue. Args: - item: Queue item to process + item: QueueItem to add Returns: - bool: Success status + bool: True if item was added successfully """ - if item.id in self._active_items: - logger.warning(f"Item {item.id} is already being processed") + if item.id in self.active_items: + logger.warning(f"Item {item.id} is already in queue") return False - try: - self._active_items.add(item.id) - start_time = datetime.now() + self._priority_queues[item.priority].append(item) + self.active_items.add(item.id) + self.queue_metrics.total_items += 1 + logger.info(f"Added item {item.id} to {item.priority.name} priority queue") + return True - # Process item logic here - # Placeholder for actual video processing - await asyncio.sleep(1) + def remove_item(self, item_id: str) -> Optional[QueueItem]: + """ + Remove an item from any priority queue. - processing_time = (datetime.now() - start_time).total_seconds() - self._update_metrics(processing_time, True, item.size) - return True + Args: + item_id: ID of item to remove - except Exception as e: - logger.error(f"Error processing item {item.id}: {str(e)}") - self._update_metrics(0, False, 0) - return False - - finally: - self._active_items.remove(item.id) + Returns: + Optional[QueueItem]: Removed item if found, None otherwise + """ + for priority in QueuePriority: + queue = self._priority_queues[priority] + for item in queue: + if item.id == item_id: + queue.remove(item) + self.active_items.discard(item_id) + self.queue_metrics.total_items -= 1 + logger.info( + f"Removed item {item_id} from {priority.name} priority queue" + ) + return item + return None def _update_metrics(self, processing_time: float, success: bool, size: int) -> None: - """Update processing metrics""" + """ + Update processing metrics. + + Args: + processing_time: Time taken to process the item + success: Whether processing was successful + size: Size of the processed item + """ if success: - self._metrics.record_success(processing_time) + self.queue_metrics.record_success(processing_time) else: - self._metrics.record_failure("Processing error") + self.queue_metrics.record_failure("Processing error") def get_metrics(self) -> QueueMetrics: - """Get current processing metrics""" - total = self._metrics.total_processed + """ + Get current processing metrics. + + Returns: + QueueMetrics: Current queue processing metrics + """ + total = self.queue_metrics.total_processed if total == 0: return QueueMetrics( total_items=0, @@ -103,8 +116,8 @@ class QueueProcessor: return QueueMetrics( total_items=total, - processing_time=self._metrics.avg_processing_time, - success_rate=self._metrics.successful / total, - error_rate=self._metrics.failed / total, + processing_time=self.queue_metrics.avg_processing_time, + success_rate=self.queue_metrics.successful / total, + error_rate=self.queue_metrics.failed / total, average_size=0, # This would need to be tracked separately if needed ) diff --git a/videoarchiver/processor/reactions.py b/videoarchiver/processor/reactions.py index f040b0c..525cb84 100644 --- a/videoarchiver/processor/reactions.py +++ b/videoarchiver/processor/reactions.py @@ -9,13 +9,13 @@ from urllib.parse import urlparse # try: # Try relative imports first -from ..processor.constants import ( +from processor.constants import ( REACTIONS, ReactionType, get_reaction, get_progress_emoji, ) -from ..database.video_archive_db import VideoArchiveDB +from database.video_archive_db import VideoArchiveDB # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/processor/status_display.py b/videoarchiver/processor/status_display.py index 3b942b3..3dc27df 100644 --- a/videoarchiver/processor/status_display.py +++ b/videoarchiver/processor/status_display.py @@ -20,7 +20,7 @@ import discord # type: ignore # try: # Try relative imports first -from ..utils.exceptions import DisplayError +from utils.exceptions import DisplayError # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/queue/__init__.py b/videoarchiver/queue/__init__.py index 05a5204..95d6f58 100644 --- a/videoarchiver/queue/__init__.py +++ b/videoarchiver/queue/__init__.py @@ -1,58 +1,58 @@ """Queue management package for video processing""" -from .models import QueueItem, QueueMetrics -from .types import QueuePriority, ProcessingMetrics -from .manager import EnhancedVideoQueueManager -from .persistence import QueuePersistenceManager, QueueError -from .monitoring import QueueMonitor, MonitoringError -from .cleanup import QueueCleaner, CleanupError -from .recovery_manager import RecoveryManager -from .state_manager import StateManager -from .metrics_manager import MetricsManager -from .processor import QueueProcessor -from .health_checker import HealthChecker +from models import QueueItem, QueueMetrics +from q_types import QueuePriority, ProcessingMetrics +from manager import EnhancedVideoQueueManager +from persistence import QueuePersistenceManager, QueueError +from monitoring import QueueMonitor, MonitoringError +from cleanup import QueueCleaner, CleanupError +from recovery_manager import RecoveryManager +from state_manager import QueueStateManager +from metrics_manager import QueueMetricsManager +from processor import QueueProcessor +from health_checker import HealthChecker # Importing from cleaners subdirectory -from .cleaners import GuildCleaner, HistoryCleaner, TrackingCleaner +from cleaners import GuildCleaner, HistoryCleaner, TrackingCleaner # Corrected relative imports from utils -from ..utils.compression_handler import CompressionHandler -from ..utils.directory_manager import DirectoryManager -from ..utils.download_manager import DownloadManager -from ..utils.file_operations import FileOperations -from ..utils.progress_tracker import ProgressTracker -from ..utils.url_validator import UrlValidator +from utils.compression_handler import CompressionHandler +from utils.directory_manager import DirectoryManager +from utils.download_manager import DownloadManager +from utils.file_operations import FileOperations +from utils.progress_tracker import ProgressTracker +from processor.url_extractor import URLValidator __all__ = [ # Queue models and types - 'QueueItem', - 'QueueMetrics', - 'QueuePriority', - 'ProcessingMetrics', + "QueueItem", + "QueueMetrics", + "QueuePriority", + "ProcessingMetrics", # Core components - 'EnhancedVideoQueueManager', - 'QueuePersistenceManager', - 'QueueMonitor', - 'QueueCleaner', - 'QueueProcessor', - 'HealthChecker', + "EnhancedVideoQueueManager", + "QueuePersistenceManager", + "QueueMonitor", + "QueueCleaner", + "QueueProcessor", + "HealthChecker", # Managers - 'RecoveryManager', - 'StateManager', - 'MetricsManager', + "RecoveryManager", + "QueueStateManager", + "QueueMetricsManager", # Cleaners - 'GuildCleaner', - 'HistoryCleaner', - 'TrackingCleaner', + "GuildCleaner", + "HistoryCleaner", + "TrackingCleaner", # Utility handlers - 'CompressionHandler', - 'DirectoryManager', - 'DownloadManager', - 'FileOperations', - 'ProgressTracker', - 'UrlValidator', + "CompressionHandler", + "DirectoryManager", + "DownloadManager", + "FileOperations", + "ProgressTracker", + "URLValidator", # Errors - 'QueueError', - 'MonitoringError', - 'CleanupError', + "QueueError", + "MonitoringError", + "CleanupError", ] diff --git a/videoarchiver/queue/cleaners/__init__.py b/videoarchiver/queue/cleaners/__init__.py index a51cb5b..97d701d 100644 --- a/videoarchiver/queue/cleaners/__init__.py +++ b/videoarchiver/queue/cleaners/__init__.py @@ -1,8 +1,8 @@ """Queue cleaning functionality""" -from .guild_cleaner import GuildCleaner -from .history_cleaner import HistoryCleaner -from .tracking_cleaner import TrackingCleaner +from guild_cleaner import GuildCleaner +from history_cleaner import HistoryCleaner +from tracking_cleaner import TrackingCleaner __all__ = [ 'GuildCleaner', diff --git a/videoarchiver/queue/cleaners/guild_cleaner.py b/videoarchiver/queue/cleaners/guild_cleaner.py index 54914f4..b537b5f 100644 --- a/videoarchiver/queue/cleaners/guild_cleaner.py +++ b/videoarchiver/queue/cleaners/guild_cleaner.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, field from typing import Dict, List, Set, Tuple, Any, Optional from datetime import datetime -from ..models import QueueItem +from models import QueueItem logger = logging.getLogger("GuildCleaner") diff --git a/videoarchiver/queue/cleaners/history_cleaner.py b/videoarchiver/queue/cleaners/history_cleaner.py index 6284d05..11aa758 100644 --- a/videoarchiver/queue/cleaners/history_cleaner.py +++ b/videoarchiver/queue/cleaners/history_cleaner.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from typing import Dict, Optional, List, Any, Set from datetime import datetime, timedelta -from ..models import QueueItem +from models import QueueItem logger = logging.getLogger("HistoryCleaner") diff --git a/videoarchiver/queue/cleaners/tracking_cleaner.py b/videoarchiver/queue/cleaners/tracking_cleaner.py index 83e15dd..fb2e3db 100644 --- a/videoarchiver/queue/cleaners/tracking_cleaner.py +++ b/videoarchiver/queue/cleaners/tracking_cleaner.py @@ -7,34 +7,42 @@ from dataclasses import dataclass, field from typing import Dict, List, Set, Tuple, Any, Optional from datetime import datetime -from ..models import QueueItem +from models import QueueItem logger = logging.getLogger("TrackingCleaner") + class TrackingCleanupStrategy(Enum): """Tracking cleanup strategies""" - AGGRESSIVE = "aggressive" # Remove all invalid entries + + AGGRESSIVE = "aggressive" # Remove all invalid entries CONSERVATIVE = "conservative" # Keep recent invalid entries - BALANCED = "balanced" # Balance between cleanup and retention + BALANCED = "balanced" # Balance between cleanup and retention + class TrackingType(Enum): """Types of tracking data""" + GUILD = "guild" CHANNEL = "channel" URL = "url" + @dataclass class TrackingCleanupConfig: """Configuration for tracking cleanup""" + batch_size: int = 100 retention_period: int = 3600 # 1 hour validate_urls: bool = True cleanup_empty: bool = True max_invalid_ratio: float = 0.5 # 50% invalid threshold + @dataclass class TrackingCleanupResult: """Result of a tracking cleanup operation""" + timestamp: datetime strategy: TrackingCleanupStrategy items_cleaned: int @@ -45,6 +53,7 @@ class TrackingCleanupResult: final_counts: Dict[str, int] error: Optional[str] = None + class TrackingValidator: """Validates tracking data""" @@ -64,6 +73,7 @@ class TrackingValidator: except Exception: return False + class TrackingCleanupTracker: """Tracks cleanup operations""" @@ -94,9 +104,7 @@ class TrackingCleanupTracker: "total_guilds_cleaned": self.total_guilds_cleaned, "total_channels_cleaned": self.total_channels_cleaned, "last_cleanup": ( - self.last_cleanup.isoformat() - if self.last_cleanup - else None + self.last_cleanup.isoformat() if self.last_cleanup else None ), "recent_cleanups": [ { @@ -105,19 +113,20 @@ class TrackingCleanupTracker: "items_cleaned": r.items_cleaned, "guilds_cleaned": r.guilds_cleaned, "channels_cleaned": r.channels_cleaned, - "duration": r.duration + "duration": r.duration, } for r in self.history[-5:] # Last 5 cleanups - ] + ], } + class TrackingCleaner: """Handles cleanup of queue tracking data""" def __init__( self, strategy: TrackingCleanupStrategy = TrackingCleanupStrategy.BALANCED, - config: Optional[TrackingCleanupConfig] = None + config: Optional[TrackingCleanupConfig] = None, ): self.strategy = strategy self.config = config or TrackingCleanupConfig() @@ -129,17 +138,14 @@ class TrackingCleaner: guild_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]], queue: List[QueueItem], - processing: Dict[str, QueueItem] + processing: Dict[str, QueueItem], ) -> Tuple[int, Dict[str, int]]: """Clean up tracking data""" start_time = datetime.utcnow() - + try: # Get initial counts - initial_counts = self._get_tracking_counts( - guild_queues, - channel_queues - ) + initial_counts = self._get_tracking_counts(guild_queues, channel_queues) # Get valid URLs valid_urls = self._get_valid_urls(queue, processing) @@ -151,21 +157,15 @@ class TrackingCleaner: if self.strategy == TrackingCleanupStrategy.AGGRESSIVE: cleaned = await self._aggressive_cleanup( - guild_queues, - channel_queues, - valid_urls + guild_queues, channel_queues, valid_urls ) elif self.strategy == TrackingCleanupStrategy.CONSERVATIVE: cleaned = await self._conservative_cleanup( - guild_queues, - channel_queues, - valid_urls + guild_queues, channel_queues, valid_urls ) else: # BALANCED cleaned = await self._balanced_cleanup( - guild_queues, - channel_queues, - valid_urls + guild_queues, channel_queues, valid_urls ) items_cleaned = cleaned[0] @@ -173,10 +173,7 @@ class TrackingCleaner: channels_cleaned = cleaned[2] # Get final counts - final_counts = self._get_tracking_counts( - guild_queues, - channel_queues - ) + final_counts = self._get_tracking_counts(guild_queues, channel_queues) # Record cleanup result duration = (datetime.utcnow() - start_time).total_seconds() @@ -188,37 +185,39 @@ class TrackingCleaner: channels_cleaned=channels_cleaned, duration=duration, initial_counts=initial_counts, - final_counts=final_counts + final_counts=final_counts, ) self.tracker.record_cleanup(result) - logger.info(self.format_tracking_cleanup_report( - initial_counts, - final_counts, - duration - )) + logger.info( + self.format_tracking_cleanup_report( + initial_counts, final_counts, duration + ) + ) return items_cleaned, initial_counts except Exception as e: logger.error(f"Error cleaning tracking data: {e}") - self.tracker.record_cleanup(TrackingCleanupResult( - timestamp=datetime.utcnow(), - strategy=self.strategy, - items_cleaned=0, - guilds_cleaned=0, - channels_cleaned=0, - duration=0, - initial_counts={}, - final_counts={}, - error=str(e) - )) + self.tracker.record_cleanup( + TrackingCleanupResult( + timestamp=datetime.utcnow(), + strategy=self.strategy, + items_cleaned=0, + guilds_cleaned=0, + channels_cleaned=0, + duration=0, + initial_counts={}, + final_counts={}, + error=str(e), + ) + ) raise async def _aggressive_cleanup( self, guild_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]], - valid_urls: Set[str] + valid_urls: Set[str], ) -> Tuple[int, int, int]: """Perform aggressive cleanup""" items_cleaned = 0 @@ -227,18 +226,14 @@ class TrackingCleaner: # Clean guild tracking guild_cleaned = await self._cleanup_guild_tracking( - guild_queues, - valid_urls, - validate_all=True + guild_queues, valid_urls, validate_all=True ) items_cleaned += guild_cleaned[0] guilds_cleaned += guild_cleaned[1] # Clean channel tracking channel_cleaned = await self._cleanup_channel_tracking( - channel_queues, - valid_urls, - validate_all=True + channel_queues, valid_urls, validate_all=True ) items_cleaned += channel_cleaned[0] channels_cleaned += channel_cleaned[1] @@ -249,7 +244,7 @@ class TrackingCleaner: self, guild_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]], - valid_urls: Set[str] + valid_urls: Set[str], ) -> Tuple[int, int, int]: """Perform conservative cleanup""" items_cleaned = 0 @@ -261,9 +256,7 @@ class TrackingCleaner: invalid_ratio = len(urls - valid_urls) / len(urls) if urls else 0 if invalid_ratio > self.config.max_invalid_ratio: cleaned = await self._cleanup_guild_tracking( - {guild_id: urls}, - valid_urls, - validate_all=False + {guild_id: urls}, valid_urls, validate_all=False ) items_cleaned += cleaned[0] guilds_cleaned += cleaned[1] @@ -272,9 +265,7 @@ class TrackingCleaner: invalid_ratio = len(urls - valid_urls) / len(urls) if urls else 0 if invalid_ratio > self.config.max_invalid_ratio: cleaned = await self._cleanup_channel_tracking( - {channel_id: urls}, - valid_urls, - validate_all=False + {channel_id: urls}, valid_urls, validate_all=False ) items_cleaned += cleaned[0] channels_cleaned += cleaned[1] @@ -285,7 +276,7 @@ class TrackingCleaner: self, guild_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]], - valid_urls: Set[str] + valid_urls: Set[str], ) -> Tuple[int, int, int]: """Perform balanced cleanup""" items_cleaned = 0 @@ -294,18 +285,14 @@ class TrackingCleaner: # Clean guild tracking with validation guild_cleaned = await self._cleanup_guild_tracking( - guild_queues, - valid_urls, - validate_all=self.config.validate_urls + guild_queues, valid_urls, validate_all=self.config.validate_urls ) items_cleaned += guild_cleaned[0] guilds_cleaned += guild_cleaned[1] # Clean channel tracking with validation channel_cleaned = await self._cleanup_channel_tracking( - channel_queues, - valid_urls, - validate_all=self.config.validate_urls + channel_queues, valid_urls, validate_all=self.config.validate_urls ) items_cleaned += channel_cleaned[0] channels_cleaned += channel_cleaned[1] @@ -316,7 +303,7 @@ class TrackingCleaner: self, guild_queues: Dict[int, Set[str]], valid_urls: Set[str], - validate_all: bool + validate_all: bool, ) -> Tuple[int, int]: """Clean up guild tracking data""" items_cleaned = 0 @@ -331,14 +318,15 @@ class TrackingCleaner: original_size = len(guild_queues[guild_id]) guild_queues[guild_id] = { - url for url in guild_queues[guild_id] + url + for url in guild_queues[guild_id] if ( - (not validate_all or self.validator.validate_url(url)) and - url in valid_urls + (not validate_all or self.validator.validate_url(url)) + and url in valid_urls ) } items_cleaned += original_size - len(guild_queues[guild_id]) - + if self.config.cleanup_empty and not guild_queues[guild_id]: guild_queues.pop(guild_id) guilds_cleaned += 1 @@ -355,7 +343,7 @@ class TrackingCleaner: self, channel_queues: Dict[int, Set[str]], valid_urls: Set[str], - validate_all: bool + validate_all: bool, ) -> Tuple[int, int]: """Clean up channel tracking data""" items_cleaned = 0 @@ -370,14 +358,15 @@ class TrackingCleaner: original_size = len(channel_queues[channel_id]) channel_queues[channel_id] = { - url for url in channel_queues[channel_id] + url + for url in channel_queues[channel_id] if ( - (not validate_all or self.validator.validate_url(url)) and - url in valid_urls + (not validate_all or self.validator.validate_url(url)) + and url in valid_urls ) } items_cleaned += original_size - len(channel_queues[channel_id]) - + if self.config.cleanup_empty and not channel_queues[channel_id]: channel_queues.pop(channel_id) channels_cleaned += 1 @@ -391,9 +380,7 @@ class TrackingCleaner: return items_cleaned, channels_cleaned def _get_valid_urls( - self, - queue: List[QueueItem], - processing: Dict[str, QueueItem] + self, queue: List[QueueItem], processing: Dict[str, QueueItem] ) -> Set[str]: """Get set of valid URLs""" valid_urls = {item.url for item in queue} @@ -401,30 +388,27 @@ class TrackingCleaner: return valid_urls def _get_tracking_counts( - self, - guild_queues: Dict[int, Set[str]], - channel_queues: Dict[int, Set[str]] + self, guild_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]] ) -> Dict[str, int]: """Get tracking data counts""" return { - 'guilds': len(guild_queues), - 'channels': len(channel_queues), - 'guild_urls': sum(len(urls) for urls in guild_queues.values()), - 'channel_urls': sum(len(urls) for urls in channel_queues.values()) + "guilds": len(guild_queues), + "channels": len(channel_queues), + "guild_urls": sum(len(urls) for urls in guild_queues.values()), + "channel_urls": sum(len(urls) for urls in channel_queues.values()), } def format_tracking_cleanup_report( self, initial_counts: Dict[str, int], final_counts: Dict[str, int], - duration: float + duration: float, ) -> str: """Format a tracking cleanup report""" - total_cleaned = ( - (initial_counts['guild_urls'] - final_counts['guild_urls']) + - (initial_counts['channel_urls'] - final_counts['channel_urls']) + total_cleaned = (initial_counts["guild_urls"] - final_counts["guild_urls"]) + ( + initial_counts["channel_urls"] - final_counts["channel_urls"] ) - + return ( f"Tracking Cleanup Results:\n" f"Strategy: {self.strategy.value}\n" @@ -446,7 +430,7 @@ class TrackingCleaner: "retention_period": self.config.retention_period, "validate_urls": self.config.validate_urls, "cleanup_empty": self.config.cleanup_empty, - "max_invalid_ratio": self.config.max_invalid_ratio + "max_invalid_ratio": self.config.max_invalid_ratio, }, - "tracker": self.tracker.get_stats() + "tracker": self.tracker.get_stats(), } diff --git a/videoarchiver/queue/cleanup.py b/videoarchiver/queue/cleanup.py index 10c9816..912b3ab 100644 --- a/videoarchiver/queue/cleanup.py +++ b/videoarchiver/queue/cleanup.py @@ -7,16 +7,16 @@ from dataclasses import dataclass, field from typing import Dict, List, Set, Optional, Any, Tuple from datetime import datetime, timedelta -from .models import QueueItem, QueueMetrics -from .cleaners.history_cleaner import ( +from models import QueueItem, QueueMetrics +from cleaners.history_cleaner import ( HistoryCleaner, CleanupStrategy as HistoryStrategy ) -from .cleaners.guild_cleaner import ( +from cleaners.guild_cleaner import ( GuildCleaner, GuildCleanupStrategy ) -from .cleaners.tracking_cleaner import ( +from cleaners.tracking_cleaner import ( TrackingCleaner, TrackingCleanupStrategy ) diff --git a/videoarchiver/queue/manager.py b/videoarchiver/queue/manager.py index 09a868d..24e44be 100644 --- a/videoarchiver/queue/manager.py +++ b/videoarchiver/queue/manager.py @@ -7,21 +7,22 @@ from dataclasses import dataclass, field from typing import Optional, Tuple, Dict, Any, List, Set, Callable from datetime import datetime, timedelta -from ..core.types import IQueueManager, QueueState, ComponentStatus -from .state_manager import QueueStateManager -from .processor import QueueProcessor -from .metrics_manager import QueueMetricsManager -from .persistence import QueuePersistenceManager -from .monitoring import QueueMonitor, MonitoringLevel -from .cleanup import QueueCleaner -from .models import QueueItem, QueueError, CleanupError -from .types import ProcessingStrategy +from core.c_types import IQueueManager, QueueState, ComponentStatus +from state_manager import QueueStateManager +from processor import QueueProcessor +from metrics_manager import QueueMetricsManager +from persistence import QueuePersistenceManager, QueueError +from monitoring import QueueMonitor, MonitoringLevel +from cleanup import QueueCleaner, CleanupError +from models import QueueItem, QueueError +from q_types import ProcessingStrategy logger = logging.getLogger("QueueManager") class QueueMode(Enum): """Queue processing modes""" + NORMAL = "normal" # Standard processing BATCH = "batch" # Batch processing PRIORITY = "priority" # Priority-based processing @@ -31,6 +32,7 @@ class QueueMode(Enum): @dataclass class QueueConfig: """Queue configuration settings""" + max_retries: int = 3 retry_delay: int = 5 max_queue_size: int = 1000 @@ -47,6 +49,7 @@ class QueueConfig: @dataclass class QueueStats: """Queue statistics""" + start_time: datetime = field(default_factory=datetime.utcnow) total_processed: int = 0 total_failed: int = 0 @@ -230,7 +233,8 @@ class EnhancedVideoQueueManager(IQueueManager): "monitoring": monitor_stats, "state": self.coordinator.state.value, "mode": self.coordinator.mode.value, - "active": self.coordinator.state == QueueState.RUNNING and bool(processor_stats["active_tasks"]), + "active": self.coordinator.state == QueueState.RUNNING + and bool(processor_stats["active_tasks"]), "stalled": monitor_stats.get("stalled", False), "stats": { "uptime": self.stats.uptime.total_seconds(), @@ -301,7 +305,7 @@ class EnhancedVideoQueueManager(IQueueManager): "total_processed": self.stats.total_processed, "total_failed": self.stats.total_failed, }, - } + }, ) # Helper methods below... diff --git a/videoarchiver/queue/monitoring.py b/videoarchiver/queue/monitoring.py index b81a19d..4021a7b 100644 --- a/videoarchiver/queue/monitoring.py +++ b/videoarchiver/queue/monitoring.py @@ -8,8 +8,8 @@ from dataclasses import dataclass, field from typing import Optional, Dict, Any, List, Set from datetime import datetime, timedelta -from .health_checker import HealthChecker, HealthStatus, HealthCategory -from .recovery_manager import RecoveryManager, RecoveryStrategy +from health_checker import HealthChecker, HealthStatus, HealthCategory +from recovery_manager import RecoveryManager, RecoveryStrategy logger = logging.getLogger("QueueMonitoring") diff --git a/videoarchiver/queue/persistence.py b/videoarchiver/queue/persistence.py index 5078bc0..afe8c00 100644 --- a/videoarchiver/queue/persistence.py +++ b/videoarchiver/queue/persistence.py @@ -9,7 +9,7 @@ import asyncio from datetime import datetime, timedelta from typing import Dict, Any, Optional -from .models import QueueItem, QueueMetrics +from models import QueueItem, QueueMetrics # Configure logging logging.basicConfig( diff --git a/videoarchiver/queue/processor.py b/videoarchiver/queue/processor.py index 305e337..8666ab7 100644 --- a/videoarchiver/queue/processor.py +++ b/videoarchiver/queue/processor.py @@ -8,9 +8,9 @@ from dataclasses import dataclass from typing import Callable, Optional, Tuple, List, Set, Dict, Any from datetime import datetime, timedelta -from .models import QueueItem -from .state_manager import QueueStateManager, ItemState -from .monitoring import QueueMonitor +from models import QueueItem +from state_manager import QueueStateManager, ItemState +from monitoring import QueueMonitor logger = logging.getLogger("QueueProcessor") diff --git a/videoarchiver/queue/types.py b/videoarchiver/queue/q_types.py similarity index 100% rename from videoarchiver/queue/types.py rename to videoarchiver/queue/q_types.py diff --git a/videoarchiver/queue/recovery_manager.py b/videoarchiver/queue/recovery_manager.py index 8998210..d97a3f1 100644 --- a/videoarchiver/queue/recovery_manager.py +++ b/videoarchiver/queue/recovery_manager.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, field from typing import List, Tuple, Dict, Optional, Any, Set from datetime import datetime, timedelta -from .models import QueueItem +from models import QueueItem logger = logging.getLogger("QueueRecoveryManager") diff --git a/videoarchiver/queue/state_manager.py b/videoarchiver/queue/state_manager.py index f6e325a..eb96ed1 100644 --- a/videoarchiver/queue/state_manager.py +++ b/videoarchiver/queue/state_manager.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import Dict, Set, List, Optional, Any from datetime import datetime -from .models import QueueItem, QueueMetrics +from models import QueueItem, QueueMetrics logger = logging.getLogger("QueueStateManager") diff --git a/videoarchiver/shared/__init__.py b/videoarchiver/shared/__init__.py index f4cca8d..7ec75e8 100644 --- a/videoarchiver/shared/__init__.py +++ b/videoarchiver/shared/__init__.py @@ -1,6 +1,6 @@ """Shared functionality for the videoarchiver package""" -from .progress import ( +from progress import ( compression_progress, download_progress, processing_progress, @@ -16,16 +16,16 @@ from .progress import ( ) __all__ = [ - 'compression_progress', - 'download_progress', - 'processing_progress', - 'get_compression_progress', - 'update_compression_progress', - 'clear_compression_progress', - 'get_download_progress', - 'update_download_progress', - 'clear_download_progress', - 'get_processing_progress', - 'update_processing_progress', - 'clear_processing_progress', + "compression_progress", + "download_progress", + "processing_progress", + "get_compression_progress", + "update_compression_progress", + "clear_compression_progress", + "get_download_progress", + "update_download_progress", + "clear_download_progress", + "get_processing_progress", + "update_processing_progress", + "clear_processing_progress", ] diff --git a/videoarchiver/update_checker.py b/videoarchiver/update_checker.py index d8cc352..4fc8af9 100644 --- a/videoarchiver/update_checker.py +++ b/videoarchiver/update_checker.py @@ -18,7 +18,7 @@ import shutil # try: # Try relative imports first -from .utils.exceptions import UpdateError +from utils.exceptions import UpdateError # except ImportError: # Fall back to absolute imports if relative imports fail diff --git a/videoarchiver/utils/__init__.py b/videoarchiver/utils/__init__.py index 97cb694..9a58b06 100644 --- a/videoarchiver/utils/__init__.py +++ b/videoarchiver/utils/__init__.py @@ -2,27 +2,27 @@ from typing import Dict, Optional, Any, Union, List -from .file_ops import ( +from file_ops import ( cleanup_downloads, ensure_directory, get_file_size, is_valid_path, safe_delete ) -from .file_deletion import FileDeleter -from .directory_manager import DirectoryManager -from .permission_manager import PermissionManager -from .download_manager import DownloadManager -from .compression_manager import CompressionManager -from .progress_tracker import ( +from file_deletion import FileDeleter +from directory_manager import DirectoryManager +from permission_manager import PermissionManager +from download_manager import DownloadManager +from compression_manager import CompressionManager +from progress_tracker import ( ProgressTracker, ProgressStatus, DownloadProgress, CompressionProgress, CompressionParams ) -from .path_manager import PathManager -from .exceptions import ( +from path_manager import PathManager +from exceptions import ( # Base exception VideoArchiverError, ErrorSeverity, @@ -86,7 +86,7 @@ from .exceptions import ( ) # Import progress_tracker from processor -from ..processor import progress_tracker +from processor import progress_tracker __all__ = [ # File Operations diff --git a/videoarchiver/utils/compression_handler.py b/videoarchiver/utils/compression_handler.py index 744eead..690cf68 100644 --- a/videoarchiver/utils/compression_handler.py +++ b/videoarchiver/utils/compression_handler.py @@ -7,11 +7,11 @@ import subprocess from datetime import datetime from typing import Dict, Optional, Callable, Set, Tuple -from ..ffmpeg.ffmpeg_manager import FFmpegManager -from ..ffmpeg.exceptions import CompressionError -from ..utils.exceptions import VideoVerificationError -from ..utils.file_operations import FileOperations -from ..utils.progress_handler import ProgressHandler +from ffmpeg.ffmpeg_manager import FFmpegManager +from ffmpeg.exceptions import CompressionError +from utils.exceptions import VideoVerificationError +from utils.file_operations import FileOperations +from utils.progress_handler import ProgressHandler logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/utils/compression_manager.py b/videoarchiver/utils/compression_manager.py index 501aebe..aacafaf 100644 --- a/videoarchiver/utils/compression_manager.py +++ b/videoarchiver/utils/compression_manager.py @@ -7,11 +7,11 @@ import subprocess from datetime import datetime from typing import Dict, Any, Optional, Callable, List, Set, Tuple -from ..shared.progress import update_compression_progress -from ..utils.compression_handler import CompressionHandler -from ..utils.progress_handler import ProgressHandler -from ..utils.file_operations import FileOperations -from ..utils.exceptions import CompressionError, VideoVerificationError +from shared.progress import update_compression_progress +from utils.compression_handler import CompressionHandler +from utils.progress_handler import ProgressHandler +from utils.file_operations import FileOperations +from utils.exceptions import CompressionError, VideoVerificationError logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/utils/directory_manager.py b/videoarchiver/utils/directory_manager.py index 1b06f8f..1dea183 100644 --- a/videoarchiver/utils/directory_manager.py +++ b/videoarchiver/utils/directory_manager.py @@ -6,8 +6,8 @@ import asyncio from pathlib import Path from typing import List, Optional, Tuple -from ..utils.exceptions import FileCleanupError -from ..utils.file_deletion import SecureFileDeleter +from utils.exceptions import FileCleanupError +from utils.file_deletion import SecureFileDeleter logger = logging.getLogger("DirectoryManager") diff --git a/videoarchiver/utils/download_core.py b/videoarchiver/utils/download_core.py index a1fa109..86d4f02 100644 --- a/videoarchiver/utils/download_core.py +++ b/videoarchiver/utils/download_core.py @@ -7,12 +7,12 @@ import yt_dlp # type: ignore from typing import Dict, Optional, Callable, Tuple from pathlib import Path -from ..utils.url_validator import check_url_support -from ..utils.progress_handler import ProgressHandler, CancellableYTDLLogger -from ..utils.file_operations import FileOperations -from ..utils.compression_handler import CompressionHandler -from ..utils.process_manager import ProcessManager -from ..ffmpeg.ffmpeg_manager import FFmpegManager +from utils.url_validator import check_url_support +from utils.progress_handler import ProgressHandler, CancellableYTDLLogger +from utils.file_operations import FileOperations +from utils.compression_handler import CompressionHandler +from utils.process_manager import ProcessManager +from ffmpeg.ffmpeg_manager import FFmpegManager logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/utils/download_manager.py b/videoarchiver/utils/download_manager.py index 0947fa8..5a5ef90 100644 --- a/videoarchiver/utils/download_manager.py +++ b/videoarchiver/utils/download_manager.py @@ -9,9 +9,9 @@ from concurrent.futures import ThreadPoolExecutor from typing import Dict, List, Optional, Tuple, Callable, Any from pathlib import Path -from ..ffmpeg.verification_manager import VerificationManager -from ..utils.compression_manager import CompressionManager -from ..processor import progress_tracker # Import from processor instead of utils +from ffmpeg.verification_manager import VerificationManager +from utils.compression_manager import CompressionManager +from processor import progress_tracker # Import from processor instead of utils logger = logging.getLogger("DownloadManager") diff --git a/videoarchiver/utils/file_deletion.py b/videoarchiver/utils/file_deletion.py index 5a57cf7..06d500e 100644 --- a/videoarchiver/utils/file_deletion.py +++ b/videoarchiver/utils/file_deletion.py @@ -7,7 +7,7 @@ import logging from pathlib import Path from typing import Optional -from ..utils.exceptions import FileCleanupError +from utils.exceptions import FileCleanupError logger = logging.getLogger("FileDeleter") diff --git a/videoarchiver/utils/file_operations.py b/videoarchiver/utils/file_operations.py index 2cd7e05..616ccdd 100644 --- a/videoarchiver/utils/file_operations.py +++ b/videoarchiver/utils/file_operations.py @@ -9,8 +9,8 @@ import subprocess from typing import Tuple from pathlib import Path -from ..utils.exceptions import VideoVerificationError -from ..utils.file_deletion import SecureFileDeleter +from utils.exceptions import VideoVerificationError +from utils.file_deletion import SecureFileDeleter logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/utils/file_ops.py b/videoarchiver/utils/file_ops.py index b53086b..d762a92 100644 --- a/videoarchiver/utils/file_ops.py +++ b/videoarchiver/utils/file_ops.py @@ -4,10 +4,10 @@ import logging from pathlib import Path from typing import List, Tuple, Optional -from ..utils.exceptions import FileCleanupError -from ..utils.file_deletion import SecureFileDeleter -from ..utils.directory_manager import DirectoryManager -from ..utils.permission_manager import PermissionManager +from utils.exceptions import FileCleanupError +from utils.file_deletion import SecureFileDeleter +from utils.directory_manager import DirectoryManager +from utils.permission_manager import PermissionManager logger = logging.getLogger("VideoArchiver") diff --git a/videoarchiver/utils/path_manager.py b/videoarchiver/utils/path_manager.py index ef2bcd2..67e4b91 100644 --- a/videoarchiver/utils/path_manager.py +++ b/videoarchiver/utils/path_manager.py @@ -11,8 +11,8 @@ import time from typing import List, Optional, AsyncGenerator from pathlib import Path -from ..utils.exceptions import FileCleanupError -from ..utils.permission_manager import PermissionManager +from utils.exceptions import FileCleanupError +from utils.permission_manager import PermissionManager logger = logging.getLogger("PathManager") diff --git a/videoarchiver/utils/permission_manager.py b/videoarchiver/utils/permission_manager.py index d5c9a88..c214983 100644 --- a/videoarchiver/utils/permission_manager.py +++ b/videoarchiver/utils/permission_manager.py @@ -6,7 +6,7 @@ import logging from pathlib import Path from typing import Optional, Union, List -from ..utils.exceptions import FileCleanupError +from utils.exceptions import FileCleanupError logger = logging.getLogger("PermissionManager")