pacnpal/Pac-cogs
1
0
Fork 0
mirror of https://github.com/pacnpal/Pac-cogs.git synced 2025-12-20 10:51:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Converted all 'from videoarchiver.' imports to relative imports

Browse Source
This commit is contained in:
pacnpal
2024-11-17 16:05:12 +00:00
parent c830be2841
commit 62c97f0b01
69 changed files with 417 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
videoarchiver/__init__.py
View File

@@ -39,10 +39,10 @@ importlib.reload(videoarchiver.processor)
import videoarchiver.queue import videoarchiver.queue
importlib.reload(videoarchiver.queue) importlib.reload(videoarchiver.queue)
from videoarchiver.core.base import VideoArchiver from .core.base import VideoArchiver
from videoarchiver.core.initialization import initialize_cog, init_callback from .core.initialization import initialize_cog, init_callback
from videoarchiver.core.cleanup import cleanup_resources from .core.cleanup import cleanup_resources
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
VideoArchiverError, VideoArchiverError,
CommandError, CommandError,
EventError, EventError,
@@ -51,12 +51,12 @@ from videoarchiver.utils.exceptions import (
ErrorSeverity, ErrorSeverity,
ProcessingError ProcessingError
) )
from videoarchiver.database import VideoArchiveDB from .database import VideoArchiveDB
from videoarchiver.ffmpeg import FFmpegManager from .ffmpeg import FFmpegManager
from videoarchiver.queue import EnhancedVideoQueueManager from .queue import EnhancedVideoQueueManager
from videoarchiver.processor import VideoProcessor from .processor import VideoProcessor
from videoarchiver.config_manager import ConfigManager from .config_manager import ConfigManager
from videoarchiver.update_checker import UpdateChecker from .update_checker import UpdateChecker
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/commands.py
View File

@@ -1,4 +1,4 @@
"""Re-export commands from core.base""" """Re-export commands from core.base"""
from videoarchiver.core.base import VideoArchiver from .core.base import VideoArchiver
__all__ = ['VideoArchiver'] __all__ = ['VideoArchiver']

10
videoarchiver/config/__init__.py
View File

@@ -1,6 +1,6 @@
"""Configuration management module""" """Configuration management module"""
from videoarchiver.config.exceptions import ( from .exceptions import (
ConfigurationError, ConfigurationError,
ValidationError, ValidationError,
PermissionError, PermissionError,
@@ -10,10 +10,10 @@ from videoarchiver.config.exceptions import (
SchemaError, SchemaError,
DiscordAPIError, DiscordAPIError,
) )
from videoarchiver.config.channel_manager import ChannelManager from .channel_manager import ChannelManager
from videoarchiver.config.role_manager import RoleManager from .role_manager import RoleManager
from videoarchiver.config.settings_formatter import SettingsFormatter from .settings_formatter import SettingsFormatter
from videoarchiver.config.validation_manager import ValidationManager from .validation_manager import ValidationManager
__all__ = [ __all__ = [
'ConfigurationError', 'ConfigurationError',

2
videoarchiver/config/channel_manager.py
View File

@@ -4,7 +4,7 @@ import logging
from typing import Dict, List, Optional, Tuple from typing import Dict, List, Optional, Tuple
import discord import discord
from videoarchiver.config.exceptions import ( from .exceptions import (
ConfigurationError as ConfigError, ConfigurationError as ConfigError,
DiscordAPIError, DiscordAPIError,
) )

2
videoarchiver/config/role_manager.py
View File

@@ -4,7 +4,7 @@ import logging
from typing import Dict, List, Set, Tuple, Optional, Any from typing import Dict, List, Set, Tuple, Optional, Any
import discord import discord
from videoarchiver.config.exceptions import ConfigurationError as ConfigError from .exceptions import ConfigurationError as ConfigError
logger = logging.getLogger("RoleManager") logger = logging.getLogger("RoleManager")

2
videoarchiver/config/settings_formatter.py
View File

@@ -5,7 +5,7 @@ from typing import Dict, Any, List
from datetime import datetime from datetime import datetime
import discord import discord
from videoarchiver.config.exceptions import ConfigurationError as ConfigError from .config.exceptions import ConfigurationError as ConfigError
logger = logging.getLogger("SettingsFormatter") logger = logging.getLogger("SettingsFormatter")

2
videoarchiver/config/validation_manager.py
View File

@@ -2,7 +2,7 @@
import logging import logging
from typing import Any, Dict, List, Union from typing import Any, Dict, List, Union
from videoarchiver.config.exceptions import ConfigurationError as ConfigError from .exceptions import ConfigurationError as ConfigError
logger = logging.getLogger("ConfigValidation") logger = logging.getLogger("ConfigValidation")

10
videoarchiver/config_manager.py
View File

@@ -6,11 +6,11 @@ from typing import Dict, Any, Optional, List, Union
import discord import discord
from redbot.core import Config from redbot.core import Config
from videoarchiver.config.validation_manager import ValidationManager from .config.validation_manager import ValidationManager
from videoarchiver.config.settings_formatter import SettingsFormatter from .config.settings_formatter import SettingsFormatter
from videoarchiver.config.channel_manager import ChannelManager from .config.channel_manager import ChannelManager
from videoarchiver.config.role_manager import RoleManager from .config.role_manager import RoleManager
from videoarchiver.utils.exceptions import ConfigurationError as ConfigError from .utils.exceptions import ConfigurationError as ConfigError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/core/__init__.py
View File

@@ -1,5 +1,5 @@
"""Core module for VideoArchiver cog""" """Core module for VideoArchiver cog"""
from videoarchiver.core.base import VideoArchiver from .core.base import VideoArchiver
__all__ = ["VideoArchiver"] __all__ = ["VideoArchiver"]

30
videoarchiver/core/base.py
View File

@@ -12,22 +12,22 @@ import discord
from redbot.core.bot import Red from redbot.core.bot import Red
from redbot.core.commands import GroupCog, Context from redbot.core.commands import GroupCog, Context
from videoarchiver.core.settings import Settings from .settings import Settings
from videoarchiver.core.lifecycle import LifecycleManager, LifecycleState from .lifecycle import LifecycleManager, LifecycleState
from videoarchiver.core.component_manager import ComponentManager, ComponentState from .component_manager import ComponentManager, ComponentState
from videoarchiver.core.error_handler import error_manager, handle_command_error from .error_handler import error_manager, handle_command_error
from videoarchiver.core.response_handler import response_manager from .response_handler import response_manager
from videoarchiver.core.commands.archiver_commands import setup_archiver_commands from .commands.archiver_commands import setup_archiver_commands
from videoarchiver.core.commands.database_commands import setup_database_commands from .commands.database_commands import setup_database_commands
from videoarchiver.core.commands.settings_commands import setup_settings_commands from .commands.settings_commands import setup_settings_commands
from videoarchiver.core.events import setup_events, EventManager from .events import setup_events, EventManager
from videoarchiver.processor.core import Processor from .processor.core import Processor
from videoarchiver.queue.manager import QueueManager from .queue.manager import QueueManager
from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager from .ffmpeg.ffmpeg_manager import FFmpegManager
from videoarchiver.database.video_archive_db import VideoArchiveDB from .database.video_archive_db import VideoArchiveDB
from videoarchiver.config_manager import ConfigManager from .config_manager import ConfigManager
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
CogError, CogError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity

6
videoarchiver/core/cleanup.py
View File

@@ -8,15 +8,15 @@ from enum import Enum, auto
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar
from videoarchiver.utils.file_ops import cleanup_downloads from .utils.file_ops import cleanup_downloads
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
CleanupError, CleanupError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity
) )
if TYPE_CHECKING: if TYPE_CHECKING:
from videoarchiver.core.base import VideoArchiver from .base import VideoArchiver
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/core/commands.py
View File

@@ -6,7 +6,7 @@
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
if TYPE_CHECKING: if TYPE_CHECKING:
from videoarchiver.core.base import VideoArchiver from .base import VideoArchiver
def setup_commands(cog: "VideoArchiver") -> None: def setup_commands(cog: "VideoArchiver") -> None:
"""Command setup is now handled in the VideoArchiver class""" """Command setup is now handled in the VideoArchiver class"""

6
videoarchiver/core/commands/__init__.py
View File

@@ -1,8 +1,8 @@
"""Command handlers for VideoArchiver""" """Command handlers for VideoArchiver"""
from videoarchiver.core.commands.archiver_commands import setup_archiver_commands from .core.commands.archiver_commands import setup_archiver_commands
from videoarchiver.core.commands.database_commands import setup_database_commands from .core.commands.database_commands import setup_database_commands
from videoarchiver.core.commands.settings_commands import setup_settings_commands from .core.commands.settings_commands import setup_settings_commands
__all__ = [ __all__ = [
'setup_archiver_commands', 'setup_archiver_commands',

4
videoarchiver/core/commands/archiver_commands.py
View File

@@ -9,8 +9,8 @@ from discord import app_commands
from redbot.core import commands from redbot.core import commands
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions
from videoarchiver.core.response_handler import handle_response, ResponseType from .core.response_handler import handle_response, ResponseType
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
CommandError, CommandError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity

6
videoarchiver/core/commands/database_commands.py
View File

@@ -10,14 +10,14 @@ from discord import app_commands
from redbot.core import commands from redbot.core import commands
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions
from videoarchiver.core.response_handler import handle_response, ResponseType from .core.response_handler import handle_response, ResponseType
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
CommandError, CommandError,
ErrorContext, ErrorContext,
ErrorSeverity, ErrorSeverity,
DatabaseError DatabaseError
) )
from videoarchiver.database.video_archive_db import VideoArchiveDB from .database.video_archive_db import VideoArchiveDB
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

6
videoarchiver/core/commands/settings_commands.py
View File

@@ -9,9 +9,9 @@ from discord import app_commands
from redbot.core import commands from redbot.core import commands
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions
from videoarchiver.core.settings import VideoFormat, VideoQuality from .core.settings import VideoFormat, VideoQuality
from videoarchiver.core.response_handler import handle_response, ResponseType from .core.response_handler import handle_response, ResponseType
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
CommandError, CommandError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity

139
videoarchiver/core/component_manager.py
View File

@@ -2,27 +2,36 @@
import logging import logging
import asyncio import asyncio
from typing import Dict, Any, Optional, Set, List, TypedDict, ClassVar, Type, Union, Protocol from typing import (
Dict,
Any,
Optional,
Set,
List,
TypedDict,
ClassVar,
Type,
Union,
Protocol,
)
from enum import Enum, auto from enum import Enum, auto
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
import importlib import importlib
from videoarchiver.utils.exceptions import ( from .utils.exceptions import ComponentError, ErrorContext, ErrorSeverity
ComponentError, from .utils.path_manager import ensure_directory
ErrorContext, from .config_manager import ConfigManager
ErrorSeverity from .processor.core import Processor
) from .queue.manager import EnhancedVideoQueueManager
from videoarchiver.utils.path_manager import ensure_directory from .ffmpeg.ffmpeg_manager import FFmpegManager
from videoarchiver.config_manager import ConfigManager
from videoarchiver.processor.core import Processor
from videoarchiver.queue.manager import EnhancedVideoQueueManager
from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class ComponentState(Enum): class ComponentState(Enum):
"""Possible states of a component""" """Possible states of a component"""
UNREGISTERED = auto() UNREGISTERED = auto()
REGISTERED = auto() REGISTERED = auto()
INITIALIZING = auto() INITIALIZING = auto()
@@ -30,16 +39,20 @@ class ComponentState(Enum):
ERROR = auto() ERROR = auto()
SHUTDOWN = auto() SHUTDOWN = auto()
class ComponentHistory(TypedDict): class ComponentHistory(TypedDict):
"""Type definition for component history entry""" """Type definition for component history entry"""
component: str component: str
state: str state: str
timestamp: str timestamp: str
error: Optional[str] error: Optional[str]
duration: float duration: float
class ComponentStatus(TypedDict): class ComponentStatus(TypedDict):
"""Type definition for component status""" """Type definition for component status"""
state: str state: str
registration_time: Optional[str] registration_time: Optional[str]
initialization_time: Optional[str] initialization_time: Optional[str]
@@ -48,8 +61,10 @@ class ComponentStatus(TypedDict):
error: Optional[str] error: Optional[str]
health: bool health: bool
class Initializable(Protocol): class Initializable(Protocol):
"""Protocol for initializable components""" """Protocol for initializable components"""
async def initialize(self) -> None: async def initialize(self) -> None:
"""Initialize the component""" """Initialize the component"""
... ...
@@ -58,6 +73,7 @@ class Initializable(Protocol):
"""Shutdown the component""" """Shutdown the component"""
... ...
class Component: class Component:
"""Base class for managed components""" """Base class for managed components"""
@@ -98,6 +114,7 @@ class Component:
"""Check if component is healthy""" """Check if component is healthy"""
return self.state == ComponentState.READY and not self.error return self.state == ComponentState.READY and not self.error
class ComponentTracker: class ComponentTracker:
"""Tracks component states and relationships""" """Tracks component states and relationships"""
@@ -108,10 +125,7 @@ class ComponentTracker:
self.history: List[ComponentHistory] = [] self.history: List[ComponentHistory] = []
def update_state( def update_state(
self, self, name: str, state: ComponentState, error: Optional[str] = None
name: str,
state: ComponentState,
error: Optional[str] = None
) -> None: ) -> None:
"""Update component state""" """Update component state"""
self.states[name] = state self.states[name] = state
@@ -124,24 +138,24 @@ class ComponentTracker:
last_time = datetime.fromisoformat(last_entry["timestamp"]) last_time = datetime.fromisoformat(last_entry["timestamp"])
duration = (now - last_time).total_seconds() duration = (now - last_time).total_seconds()
self.history.append(ComponentHistory( self.history.append(
ComponentHistory(
component=name, component=name,
state=state.name, state=state.name,
timestamp=now.isoformat(), timestamp=now.isoformat(),
error=error, error=error,
duration=duration duration=duration,
)) )
)
# Cleanup old history # Cleanup old history
if len(self.history) > self.MAX_HISTORY: if len(self.history) > self.MAX_HISTORY:
self.history = self.history[-self.MAX_HISTORY:] self.history = self.history[-self.MAX_HISTORY :]
def get_component_history(self, name: str) -> List[ComponentHistory]: def get_component_history(self, name: str) -> List[ComponentHistory]:
"""Get state history for a component""" """Get state history for a component"""
return [ return [entry for entry in self.history if entry["component"] == name]
entry for entry in self.history
if entry["component"] == name
]
class DependencyManager: class DependencyManager:
"""Manages component dependencies""" """Manages component dependencies"""
@@ -169,8 +183,8 @@ class DependencyManager:
"DependencyManager", "DependencyManager",
"add_dependency", "add_dependency",
{"component": component, "dependency": dependency}, {"component": component, "dependency": dependency},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
if component not in self.dependencies: if component not in self.dependencies:
@@ -192,8 +206,7 @@ class DependencyManager:
return False return False
visited.add(start) visited.add(start)
return any( return any(
has_path(dep, end) has_path(dep, end) for dep in self.dependencies.get(start, set())
for dep in self.dependencies.get(start, set())
) )
return has_path(dependency, component) return has_path(dependency, component)
@@ -223,8 +236,7 @@ class DependencyManager:
def visit(component: str) -> None: def visit(component: str) -> None:
if component in temp_visited: if component in temp_visited:
cycle = " -> ".join( cycle = " -> ".join(
name for name in self.dependencies name for name in self.dependencies if name in temp_visited
if name in temp_visited
) )
raise ComponentError( raise ComponentError(
f"Dependency cycle detected: {cycle}", f"Dependency cycle detected: {cycle}",
@@ -232,8 +244,8 @@ class DependencyManager:
"DependencyManager", "DependencyManager",
"get_initialization_order", "get_initialization_order",
{"cycle": cycle}, {"cycle": cycle},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
if component in visited: if component in visited:
return return
@@ -256,12 +268,13 @@ class DependencyManager:
"DependencyManager", "DependencyManager",
"get_initialization_order", "get_initialization_order",
None, None,
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
return order return order
class ComponentManager: class ComponentManager:
"""Manages VideoArchiver components""" """Manages VideoArchiver components"""
@@ -269,7 +282,7 @@ class ComponentManager:
"config_manager": (ConfigManager, set()), "config_manager": (ConfigManager, set()),
"processor": (Processor, {"config_manager"}), "processor": (Processor, {"config_manager"}),
"queue_manager": (EnhancedVideoQueueManager, {"config_manager"}), "queue_manager": (EnhancedVideoQueueManager, {"config_manager"}),
"ffmpeg_mgr": (FFmpegManager, set()) "ffmpeg_mgr": (FFmpegManager, set()),
} }
def __init__(self, cog: Any) -> None: def __init__(self, cog: Any) -> None:
@@ -282,7 +295,7 @@ class ComponentManager:
self, self,
name: str, name: str,
component: Union[Component, Any], component: Union[Component, Any],
dependencies: Optional[Set[str]] = None dependencies: Optional[Set[str]] = None,
) -> None: ) -> None:
""" """
Register a component with dependencies. Register a component with dependencies.
@@ -314,8 +327,8 @@ class ComponentManager:
"ComponentManager", "ComponentManager",
"register", "register",
{"component": name, "dependency": dep}, {"component": name, "dependency": dep},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
self.dependency_manager.add_dependency(name, dep) self.dependency_manager.add_dependency(name, dep)
@@ -335,8 +348,8 @@ class ComponentManager:
"ComponentManager", "ComponentManager",
"register", "register",
{"component": name}, {"component": name},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
async def initialize_components(self) -> None: async def initialize_components(self) -> None:
@@ -374,8 +387,8 @@ class ComponentManager:
"ComponentManager", "ComponentManager",
"initialize_components", "initialize_components",
{"component": name}, {"component": name},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
except Exception as e: except Exception as e:
@@ -387,8 +400,8 @@ class ComponentManager:
"ComponentManager", "ComponentManager",
"initialize_components", "initialize_components",
None, None,
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
async def _initialize_core_components(self) -> None: async def _initialize_core_components(self) -> None:
@@ -421,8 +434,8 @@ class ComponentManager:
"ComponentManager", "ComponentManager",
"_initialize_core_components", "_initialize_core_components",
None, None,
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
async def _initialize_paths(self) -> None: async def _initialize_paths(self) -> None:
@@ -450,11 +463,8 @@ class ComponentManager:
raise ComponentError( raise ComponentError(
error, error,
context=ErrorContext( context=ErrorContext(
"ComponentManager", "ComponentManager", "_initialize_paths", None, ErrorSeverity.HIGH
"_initialize_paths", ),
None,
ErrorSeverity.HIGH
)
) )
def get(self, name: str) -> Optional[Component]: def get(self, name: str) -> Optional[Component]:
@@ -469,7 +479,9 @@ class ComponentManager:
ComponentError: If shutdown fails ComponentError: If shutdown fails
""" """
try: try:
shutdown_order = reversed(self.dependency_manager.get_initialization_order()) shutdown_order = reversed(
self.dependency_manager.get_initialization_order()
)
for name in shutdown_order: for name in shutdown_order:
if name not in self._components: if name not in self._components:
@@ -489,8 +501,8 @@ class ComponentManager:
"ComponentManager", "ComponentManager",
"shutdown_components", "shutdown_components",
{"component": name}, {"component": name},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
except Exception as e: except Exception as e:
@@ -499,11 +511,8 @@ class ComponentManager:
raise ComponentError( raise ComponentError(
error, error,
context=ErrorContext( context=ErrorContext(
"ComponentManager", "ComponentManager", "shutdown_components", None, ErrorSeverity.HIGH
"shutdown_components", ),
None,
ErrorSeverity.HIGH
)
) )
def clear(self) -> None: def clear(self) -> None:
@@ -521,12 +530,20 @@ class ComponentManager:
return { return {
name: ComponentStatus( name: ComponentStatus(
state=self.tracker.states.get(name, ComponentState.UNREGISTERED).name, state=self.tracker.states.get(name, ComponentState.UNREGISTERED).name,
registration_time=component.registration_time.isoformat() if component.registration_time else None, registration_time=(
initialization_time=component.initialization_time.isoformat() if component.initialization_time else None, component.registration_time.isoformat()
if component.registration_time
else None
),
initialization_time=(
component.initialization_time.isoformat()
if component.initialization_time
else None
),
dependencies=self.dependency_manager.get_dependencies(name), dependencies=self.dependency_manager.get_dependencies(name),
dependents=self.dependency_manager.get_dependents(name), dependents=self.dependency_manager.get_dependents(name),
error=component.error, error=component.error,
health=component.is_healthy() health=component.is_healthy(),
) )
for name, component in self._components.items() for name, component in self._components.items()
} }

4
videoarchiver/core/error_handler.py
View File

@@ -14,7 +14,7 @@ from redbot.core.commands import (
CommandError CommandError
) )
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
VideoArchiverError, VideoArchiverError,
ErrorSeverity, ErrorSeverity,
ErrorContext, ErrorContext,
@@ -33,7 +33,7 @@ from videoarchiver.utils.exceptions import (
ResourceExhaustedError, ResourceExhaustedError,
ConfigurationError ConfigurationError
) )
from videoarchiver.core.response_handler import response_manager from .core.response_handler import response_manager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

14
videoarchiver/core/events.py
View File

@@ -9,15 +9,15 @@ from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar, List
import discord import discord
from videoarchiver.processor.constants import REACTIONS from .processor.constants import REACTIONS
from videoarchiver.processor.reactions import handle_archived_reaction from .processor.reactions import handle_archived_reaction
from videoarchiver.core.guild import initialize_guild_components, cleanup_guild_components from .core.guild import initialize_guild_components, cleanup_guild_components
from videoarchiver.core.error_handler import error_manager from .core.error_handler import error_manager
from videoarchiver.core.response_handler import response_manager from .core.response_handler import response_manager
from videoarchiver.utils.exceptions import EventError, ErrorContext, ErrorSeverity from .utils.exceptions import EventError, ErrorContext, ErrorSeverity
if TYPE_CHECKING: if TYPE_CHECKING:
from videoarchiver.core.base import VideoArchiver from .core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

10
videoarchiver/core/guild.py
View File

@@ -4,13 +4,13 @@ import logging
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Dict, Any, Optional from typing import TYPE_CHECKING, Dict, Any, Optional
from videoarchiver.utils.download_core import DownloadCore from .utils.download_core import DownloadCore
from videoarchiver.utils.message_manager import MessageManager from .utils.message_manager import MessageManager
from videoarchiver.utils.file_ops import cleanup_downloads from .utils.file_ops import cleanup_downloads
from videoarchiver.utils.exceptions import VideoArchiverError as ProcessingError from .utils.exceptions import VideoArchiverError as ProcessingError
if TYPE_CHECKING: if TYPE_CHECKING:
from videoarchiver.core.base import VideoArchiver from .core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

6
videoarchiver/core/initialization.py
View File

@@ -4,15 +4,15 @@ from typing import TYPE_CHECKING, Optional, Dict, Any
import asyncio import asyncio
import logging import logging
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
ComponentError, ComponentError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity
) )
from videoarchiver.core.lifecycle import LifecycleState from .core.lifecycle import LifecycleState
if TYPE_CHECKING: if TYPE_CHECKING:
from videoarchiver.core.base import VideoArchiver from .core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

4
videoarchiver/core/lifecycle.py
View File

@@ -7,8 +7,8 @@ from typing import Optional, Dict, Any, Set, List, Callable, TypedDict, ClassVar
from enum import Enum, auto from enum import Enum, auto
from datetime import datetime from datetime import datetime
from videoarchiver.core.cleanup import cleanup_resources, force_cleanup_resources from .core.cleanup import cleanup_resources, force_cleanup_resources
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
VideoArchiverError, VideoArchiverError,
ErrorContext, ErrorContext,
ErrorSeverity, ErrorSeverity,

2
videoarchiver/core/response_handler.py
View File

@@ -7,7 +7,7 @@ from datetime import datetime
import discord import discord
from redbot.core.commands import Context from redbot.core.commands import Context
from videoarchiver.utils.exceptions import ErrorSeverity from .utils.exceptions import ErrorSeverity
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/core/settings.py
View File

@@ -4,7 +4,7 @@ from typing import Dict, Any, List, Optional, Union, TypedDict, ClassVar
from dataclasses import dataclass, field from dataclasses import dataclass, field
from enum import Enum, auto from enum import Enum, auto
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
ConfigurationError, ConfigurationError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity

8
videoarchiver/database/__init__.py
View File

@@ -1,9 +1,9 @@
"""Database management package for video archiving""" """Database management package for video archiving"""
from videoarchiver.database.connection_manager import DatabaseConnectionManager from .connection_manager import DatabaseConnectionManager
from videoarchiver.database.query_manager import DatabaseQueryManager from .query_manager import DatabaseQueryManager
from videoarchiver.database.schema_manager import DatabaseSchemaManager from .schema_manager import DatabaseSchemaManager
from videoarchiver.database.video_archive_db import VideoArchiveDB from .video_archive_db import VideoArchiveDB
__all__ = [ __all__ = [
'DatabaseConnectionManager', 'DatabaseConnectionManager',

2
videoarchiver/database/connection_manager.py
View File

@@ -10,7 +10,7 @@ import threading
from queue import Queue, Empty from queue import Queue, Empty
from datetime import datetime from datetime import datetime
from videoarchiver.utils.exceptions import ( from ..utils.exceptions import (
DatabaseError, DatabaseError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity

2
videoarchiver/database/schema_manager.py
View File

@@ -7,7 +7,7 @@ from typing import List, Dict, Any, Optional, TypedDict, ClassVar, Union
from enum import Enum, auto from enum import Enum, auto
from datetime import datetime from datetime import datetime
from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity from ..utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
logger = logging.getLogger("DBSchemaManager") logger = logging.getLogger("DBSchemaManager")

6
videoarchiver/database/video_archive_db.py
View File

@@ -4,9 +4,9 @@ import logging
from pathlib import Path from pathlib import Path
from typing import Optional, Dict, Any, List from typing import Optional, Dict, Any, List
from videoarchiver.database.schema_manager import DatabaseSchemaManager from .schema_manager import DatabaseSchemaManager
from videoarchiver.database.query_manager import DatabaseQueryManager from .query_manager import DatabaseQueryManager
from videoarchiver.database.connection_manager import DatabaseConnectionManager from .connection_manager import DatabaseConnectionManager
logger = logging.getLogger("VideoArchiverDB") logger = logging.getLogger("VideoArchiverDB")

38
videoarchiver/exceptions.py
View File

@@ -1,6 +1,6 @@
"""Base exceptions for VideoArchiver""" """Base exceptions for VideoArchiver"""
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
VideoArchiverError, VideoArchiverError,
ConfigurationError, ConfigurationError,
VideoVerificationError, VideoVerificationError,
@@ -23,24 +23,24 @@ from videoarchiver.utils.exceptions import (
# Re-export all exceptions # Re-export all exceptions
__all__ = [ __all__ = [
'VideoArchiverError', "VideoArchiverError",
'ConfigurationError', "ConfigurationError",
'VideoVerificationError', "VideoVerificationError",
'QueueError', "QueueError",
'FileCleanupError', "FileCleanupError",
'ResourceExhaustedError', "ResourceExhaustedError",
'ProcessingError', "ProcessingError",
'CleanupError', "CleanupError",
'FileOperationError', "FileOperationError",
'VideoDownloadError', "VideoDownloadError",
'VideoProcessingError', "VideoProcessingError",
'VideoUploadError', "VideoUploadError",
'VideoCleanupError', "VideoCleanupError",
'PermissionError', "PermissionError",
'NetworkError', "NetworkError",
'ResourceError', "ResourceError",
'ComponentError', "ComponentError",
'DiscordAPIError', "DiscordAPIError",
] ]
# Alias exceptions for backward compatibility # Alias exceptions for backward compatibility

12
videoarchiver/ffmpeg/__init__.py
View File

@@ -18,12 +18,12 @@ logging.basicConfig(
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
# Import components after logging is configured # Import components after logging is configured
from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager from .ffmpeg_manager import FFmpegManager
from videoarchiver.ffmpeg.video_analyzer import VideoAnalyzer from .video_analyzer import VideoAnalyzer
from videoarchiver.ffmpeg.gpu_detector import GPUDetector from .gpu_detector import GPUDetector
from videoarchiver.ffmpeg.encoder_params import EncoderParams from .encoder_params import EncoderParams
from videoarchiver.ffmpeg.ffmpeg_downloader import FFmpegDownloader from .ffmpeg_downloader import FFmpegDownloader
from videoarchiver.ffmpeg.exceptions import ( from .exceptions import (
FFmpegError, FFmpegError,
DownloadError, DownloadError,
VerificationError, VerificationError,

6
videoarchiver/ffmpeg/binary_manager.py
View File

@@ -5,15 +5,15 @@ import os
from pathlib import Path from pathlib import Path
from typing import Dict, Optional from typing import Dict, Optional
from videoarchiver.ffmpeg.exceptions import ( from .exceptions import (
FFmpegError, FFmpegError,
DownloadError, DownloadError,
VerificationError, VerificationError,
PermissionError, PermissionError,
FFmpegNotFoundError FFmpegNotFoundError
) )
from videoarchiver.ffmpeg.ffmpeg_downloader import FFmpegDownloader from .ffmpeg_downloader import FFmpegDownloader
from videoarchiver.ffmpeg.verification_manager import VerificationManager from .verification_manager import VerificationManager
logger = logging.getLogger("FFmpegBinaryManager") logger = logging.getLogger("FFmpegBinaryManager")

2
videoarchiver/ffmpeg/encoder_params.py
View File

@@ -3,7 +3,7 @@
import os import os
import logging import logging
from typing import Dict, Any from typing import Dict, Any
from videoarchiver.ffmpeg.exceptions import CompressionError, QualityError, BitrateError from .exceptions import CompressionError, QualityError, BitrateError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/ffmpeg/ffmpeg_downloader.py
View File

@@ -16,7 +16,7 @@ from typing import Optional, Dict, List
import time import time
import lzma import lzma
from videoarchiver.ffmpeg.exceptions import DownloadError from .exceptions import DownloadError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

14
videoarchiver/ffmpeg/ffmpeg_manager.py
View File

@@ -6,17 +6,17 @@ import multiprocessing
from pathlib import Path from pathlib import Path
from typing import Dict, Any, Optional from typing import Dict, Any, Optional
from videoarchiver.ffmpeg.exceptions import ( from .exceptions import (
FFmpegError, FFmpegError,
AnalysisError, AnalysisError,
FFmpegNotFoundError FFmpegNotFoundError
) )
from videoarchiver.ffmpeg.gpu_detector import GPUDetector from .gpu_detector import GPUDetector
from videoarchiver.ffmpeg.video_analyzer import VideoAnalyzer from .video_analyzer import VideoAnalyzer
from videoarchiver.ffmpeg.encoder_params import EncoderParams from .encoder_params import EncoderParams
from videoarchiver.ffmpeg.process_manager import ProcessManager from .process_manager import ProcessManager
from videoarchiver.ffmpeg.verification_manager import VerificationManager from .verification_manager import VerificationManager
from videoarchiver.ffmpeg.binary_manager import BinaryManager from .binary_manager import BinaryManager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/ffmpeg/verification_manager.py
View File

@@ -6,7 +6,7 @@ import subprocess
from pathlib import Path from pathlib import Path
from typing import Dict, List, Optional from typing import Dict, List, Optional
from videoarchiver.ffmpeg.exceptions import ( from .exceptions import (
TimeoutError, TimeoutError,
VerificationError, VerificationError,
EncodingError, EncodingError,

2
videoarchiver/processor.py
View File

@@ -1,6 +1,6 @@
"""Re-export video processing components from processor module""" """Re-export video processing components from processor module"""
from videoarchiver.processor import ( from .processor import (
VideoProcessor, VideoProcessor,
REACTIONS, REACTIONS,
MessageHandler, MessageHandler,

16
videoarchiver/processor/__init__.py
View File

@@ -3,8 +3,8 @@
from typing import Dict, Any, Optional, Union, List, Tuple from typing import Dict, Any, Optional, Union, List, Tuple
import discord import discord
from videoarchiver.processor.core import VideoProcessor from .processor.core import VideoProcessor
from videoarchiver.processor.constants import ( from .processor.constants import (
REACTIONS, REACTIONS,
ReactionType, ReactionType,
ReactionEmojis, ReactionEmojis,
@@ -12,7 +12,7 @@ from videoarchiver.processor.constants import (
get_reaction, get_reaction,
get_progress_emoji get_progress_emoji
) )
from videoarchiver.processor.url_extractor import ( from .processor.url_extractor import (
URLExtractor, URLExtractor,
URLMetadata, URLMetadata,
URLPattern, URLPattern,
@@ -21,7 +21,7 @@ from videoarchiver.processor.url_extractor import (
URLValidator, URLValidator,
URLMetadataExtractor URLMetadataExtractor
) )
from videoarchiver.processor.message_validator import ( from .processor.message_validator import (
MessageValidator, MessageValidator,
ValidationContext, ValidationContext,
ValidationRule, ValidationRule,
@@ -32,15 +32,15 @@ from videoarchiver.processor.message_validator import (
ValidationCacheEntry, ValidationCacheEntry,
ValidationError ValidationError
) )
from videoarchiver.processor.message_handler import MessageHandler from .processor.message_handler import MessageHandler
from videoarchiver.processor.queue_handler import QueueHandler from .processor.queue_handler import QueueHandler
from videoarchiver.processor.reactions import ( from .processor.reactions import (
handle_archived_reaction, handle_archived_reaction,
update_queue_position_reaction, update_queue_position_reaction,
update_progress_reaction, update_progress_reaction,
update_download_progress_reaction update_download_progress_reaction
) )
from videoarchiver.utils import progress_tracker from .utils import progress_tracker
# Export public classes and constants # Export public classes and constants
__all__ = [ __all__ = [

6
videoarchiver/processor/cleanup_manager.py
View File

@@ -18,9 +18,9 @@ from typing import (
) )
from datetime import datetime, timedelta from datetime import datetime, timedelta
from videoarchiver.processor.queue_handler import QueueHandler from .processor.queue_handler import QueueHandler
from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager from .ffmpeg.ffmpeg_manager import FFmpegManager
from videoarchiver.utils.exceptions import CleanupError from .utils.exceptions import CleanupError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

22
videoarchiver/processor/core.py
View File

@@ -8,17 +8,17 @@ from datetime import datetime, timedelta
import discord import discord
from discord.ext import commands from discord.ext import commands
from videoarchiver.processor.message_handler import MessageHandler from .processor.message_handler import MessageHandler
from videoarchiver.processor.queue_handler import QueueHandler from .processor.queue_handler import QueueHandler
from videoarchiver.utils import progress_tracker from .utils import progress_tracker
from videoarchiver.processor.status_display import StatusDisplay from .processor.status_display import StatusDisplay
from videoarchiver.processor.cleanup_manager import CleanupManager, CleanupStrategy from .processor.cleanup_manager import CleanupManager, CleanupStrategy
from videoarchiver.processor.constants import REACTIONS from .processor.constants import REACTIONS
from videoarchiver.queue.manager import EnhancedVideoQueueManager from .queue.manager import EnhancedVideoQueueManager
from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager from .ffmpeg.ffmpeg_manager import FFmpegManager
from videoarchiver.database.video_archive_db import VideoArchiveDB from .database.video_archive_db import VideoArchiveDB
from videoarchiver.config_manager import ConfigManager from .config_manager import ConfigManager
from videoarchiver.utils.exceptions import ProcessorError from .utils.exceptions import ProcessorError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

14
videoarchiver/processor/message_handler.py
View File

@@ -8,13 +8,13 @@ from datetime import datetime, timedelta
import discord import discord
from discord.ext import commands from discord.ext import commands
from videoarchiver.processor.url_extractor import URLExtractor, URLMetadata from .processor.url_extractor import URLExtractor, URLMetadata
from videoarchiver.processor.message_validator import MessageValidator, ValidationError from .processor.message_validator import MessageValidator, ValidationError
from videoarchiver.processor.queue_processor import QueueProcessor, QueuePriority from .processor.queue_processor import QueueProcessor, QueuePriority
from videoarchiver.processor.constants import REACTIONS from .processor.constants import REACTIONS
from videoarchiver.queue.manager import EnhancedVideoQueueManager from .queue.manager import EnhancedVideoQueueManager
from videoarchiver.config_manager import ConfigManager from .config_manager import ConfigManager
from videoarchiver.utils.exceptions import MessageHandlerError from .utils.exceptions import MessageHandlerError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/processor/message_validator.py
View File

@@ -7,7 +7,7 @@ from typing import Dict, Optional, Tuple, List, Any, Callable, Set, TypedDict, C
from datetime import datetime from datetime import datetime
import discord import discord
from videoarchiver.utils.exceptions import ValidationError from .utils.exceptions import ValidationError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

16
videoarchiver/processor/queue_handler.py
View File

@@ -8,14 +8,14 @@ from typing import Optional, Dict, Any, List, Tuple, Set, TypedDict, ClassVar, C
from datetime import datetime from datetime import datetime
import discord import discord
from videoarchiver.utils import progress_tracker from .utils import progress_tracker
from videoarchiver.database.video_archive_db import VideoArchiveDB from .database.video_archive_db import VideoArchiveDB
from videoarchiver.utils.download_manager import DownloadManager from .utils.download_manager import DownloadManager
from videoarchiver.utils.message_manager import MessageManager from .utils.message_manager import MessageManager
from videoarchiver.utils.exceptions import QueueHandlerError from .utils.exceptions import QueueHandlerError
from videoarchiver.queue.models import QueueItem from .queue.models import QueueItem
from videoarchiver.config_manager import ConfigManager from .config_manager import ConfigManager
from videoarchiver.processor.constants import REACTIONS from .processor.constants import REACTIONS
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

10
videoarchiver/processor/queue_processor.py
View File

@@ -7,11 +7,11 @@ from typing import List, Optional, Dict, Any, Set, Union, TypedDict, ClassVar
from datetime import datetime from datetime import datetime
import discord import discord
from videoarchiver.queue.models import QueueItem from .queue.models import QueueItem
from videoarchiver.queue.manager import EnhancedVideoQueueManager from .queue.manager import EnhancedVideoQueueManager
from videoarchiver.processor.constants import REACTIONS from .processor.constants import REACTIONS
from videoarchiver.processor.url_extractor import URLMetadata from .processor.url_extractor import URLMetadata
from videoarchiver.utils.exceptions import QueueProcessingError from .utils.exceptions import QueueProcessingError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

4
videoarchiver/processor/reactions.py
View File

@@ -7,8 +7,8 @@ from typing import List, Optional
import discord import discord
from urllib.parse import urlparse from urllib.parse import urlparse
from videoarchiver.processor.constants import REACTIONS, ReactionType, get_reaction, get_progress_emoji from .processor.constants import REACTIONS, ReactionType, get_reaction, get_progress_emoji
from videoarchiver.database.video_archive_db import VideoArchiveDB from .database.video_archive_db import VideoArchiveDB
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/processor/status_display.py
View File

@@ -7,7 +7,7 @@ from datetime import datetime
from typing import Dict, Any, List, Optional, Callable, TypeVar, Union, TypedDict, ClassVar, Tuple from typing import Dict, Any, List, Optional, Callable, TypeVar, Union, TypedDict, ClassVar, Tuple
import discord import discord
from videoarchiver.utils.exceptions import DisplayError from .utils.exceptions import DisplayError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

10
videoarchiver/queue/__init__.py
View File

@@ -1,10 +1,10 @@
"""Queue management package for video processing""" """Queue management package for video processing"""
from videoarchiver.queue.models import QueueItem, QueueMetrics from .models import QueueItem, QueueMetrics
from videoarchiver.queue.manager import EnhancedVideoQueueManager from .manager import EnhancedVideoQueueManager
from videoarchiver.queue.persistence import QueuePersistenceManager, QueueError from .persistence import QueuePersistenceManager, QueueError
from videoarchiver.queue.monitoring import QueueMonitor, MonitoringError from .monitoring import QueueMonitor, MonitoringError
from videoarchiver.queue.cleanup import QueueCleaner, CleanupError from .cleanup import QueueCleaner, CleanupError
__all__ = [ __all__ = [
'QueueItem', 'QueueItem',

6
videoarchiver/queue/cleaners/__init__.py
View File

@@ -1,8 +1,8 @@
"""Queue cleaning functionality""" """Queue cleaning functionality"""
from videoarchiver.queue.cleaners.guild_cleaner import GuildCleaner from .guild_cleaner import GuildCleaner
from videoarchiver.queue.cleaners.history_cleaner import HistoryCleaner from .history_cleaner import HistoryCleaner
from videoarchiver.queue.cleaners.tracking_cleaner import TrackingCleaner from .tracking_cleaner import TrackingCleaner
__all__ = [ __all__ = [
'GuildCleaner', 'GuildCleaner',

2
videoarchiver/queue/cleaners/guild_cleaner.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass, field
from typing import Dict, List, Set, Tuple, Any, Optional from typing import Dict, List, Set, Tuple, Any, Optional
from datetime import datetime from datetime import datetime
from videoarchiver.queue.models import QueueItem from ..models import QueueItem
logger = logging.getLogger("GuildCleaner") logger = logging.getLogger("GuildCleaner")

2
videoarchiver/queue/cleaners/history_cleaner.py
View File

@@ -6,7 +6,7 @@ from dataclasses import dataclass, field
from typing import Dict, Optional, List, Any, Set from typing import Dict, Optional, List, Any, Set
from datetime import datetime, timedelta from datetime import datetime, timedelta
from videoarchiver.queue.models import QueueItem from ..models import QueueItem
logger = logging.getLogger("HistoryCleaner") logger = logging.getLogger("HistoryCleaner")

2
videoarchiver/queue/cleaners/tracking_cleaner.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass, field
from typing import Dict, List, Set, Tuple, Any, Optional from typing import Dict, List, Set, Tuple, Any, Optional
from datetime import datetime from datetime import datetime
from videoarchiver.queue.models import QueueItem from ..models import QueueItem
logger = logging.getLogger("TrackingCleaner") logger = logging.getLogger("TrackingCleaner")

8
videoarchiver/queue/cleanup.py
View File

@@ -7,16 +7,16 @@ from dataclasses import dataclass, field
from typing import Dict, List, Set, Optional, Any, Tuple from typing import Dict, List, Set, Optional, Any, Tuple
from datetime import datetime, timedelta from datetime import datetime, timedelta
from videoarchiver.queue.models import QueueItem, QueueMetrics from .models import QueueItem, QueueMetrics
from videoarchiver.queue.cleaners.history_cleaner import ( from .cleaners.history_cleaner import (
HistoryCleaner, HistoryCleaner,
CleanupStrategy as HistoryStrategy CleanupStrategy as HistoryStrategy
) )
from videoarchiver.queue.cleaners.guild_cleaner import ( from .cleaners.guild_cleaner import (
GuildCleaner, GuildCleaner,
GuildCleanupStrategy GuildCleanupStrategy
) )
from videoarchiver.queue.cleaners.tracking_cleaner import ( from .cleaners.tracking_cleaner import (
TrackingCleaner, TrackingCleaner,
TrackingCleanupStrategy TrackingCleanupStrategy
) )

14
videoarchiver/queue/manager.py
View File

@@ -7,13 +7,13 @@ from dataclasses import dataclass, field
from typing import Optional, Tuple, Dict, Any, List, Set, Callable from typing import Optional, Tuple, Dict, Any, List, Set, Callable
from datetime import datetime, timedelta from datetime import datetime, timedelta
from videoarchiver.queue.state_manager import QueueStateManager from .state_manager import QueueStateManager
from videoarchiver.queue.processor import QueueProcessor from .processor import QueueProcessor
from videoarchiver.queue.metrics_manager import QueueMetricsManager from .metrics_manager import QueueMetricsManager
from videoarchiver.queue.persistence import QueuePersistenceManager from .persistence import QueuePersistenceManager
from videoarchiver.queue.monitoring import QueueMonitor, MonitoringLevel from .monitoring import QueueMonitor, MonitoringLevel
from videoarchiver.queue.cleanup import QueueCleaner from .cleanup import QueueCleaner
from videoarchiver.queue.models import QueueItem, QueueError, CleanupError from .models import QueueItem, QueueError, CleanupError
logger = logging.getLogger("QueueManager") logger = logging.getLogger("QueueManager")

4
videoarchiver/queue/monitoring.py
View File

@@ -8,8 +8,8 @@ from dataclasses import dataclass, field
from typing import Optional, Dict, Any, List, Set from typing import Optional, Dict, Any, List, Set
from datetime import datetime, timedelta from datetime import datetime, timedelta
from videoarchiver.queue.health_checker import HealthChecker, HealthStatus, HealthCategory from .health_checker import HealthChecker, HealthStatus, HealthCategory
from videoarchiver.queue.recovery_manager import RecoveryManager, RecoveryStrategy from .recovery_manager import RecoveryManager, RecoveryStrategy
logger = logging.getLogger("QueueMonitoring") logger = logging.getLogger("QueueMonitoring")

97
videoarchiver/queue/persistence.py
View File

@@ -8,7 +8,8 @@ import fcntl
import asyncio import asyncio
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Dict, Any, Optional from typing import Dict, Any, Optional
from videoarchiver.queue.models import QueueItem, QueueMetrics
from .models import QueueItem, QueueMetrics
# Configure logging # Configure logging
logging.basicConfig( logging.basicConfig(
@@ -16,6 +17,7 @@ logging.basicConfig(
) )
logger = logging.getLogger("QueuePersistence") logger = logging.getLogger("QueuePersistence")
class QueuePersistenceManager: class QueuePersistenceManager:
"""Manages persistence of queue state to disk""" """Manages persistence of queue state to disk"""
@@ -25,7 +27,7 @@ class QueuePersistenceManager:
max_retries: int = 3, max_retries: int = 3,
retry_delay: int = 1, retry_delay: int = 1,
backup_interval: int = 3600, # 1 hour backup_interval: int = 3600, # 1 hour
max_backups: int = 24 # Keep last 24 backups max_backups: int = 24, # Keep last 24 backups
): ):
"""Initialize the persistence manager """Initialize the persistence manager
@@ -50,7 +52,7 @@ class QueuePersistenceManager:
processing: Dict[str, QueueItem], processing: Dict[str, QueueItem],
completed: Dict[str, QueueItem], completed: Dict[str, QueueItem],
failed: Dict[str, QueueItem], failed: Dict[str, QueueItem],
metrics: QueueMetrics metrics: QueueMetrics,
) -> None: ) -> None:
"""Persist queue state to disk with improved error handling """Persist queue state to disk with improved error handling
@@ -87,14 +89,14 @@ class QueuePersistenceManager:
"compression_failures": metrics.compression_failures, "compression_failures": metrics.compression_failures,
"hardware_accel_failures": metrics.hardware_accel_failures, "hardware_accel_failures": metrics.hardware_accel_failures,
}, },
"timestamp": datetime.utcnow().isoformat() "timestamp": datetime.utcnow().isoformat(),
} }
# Ensure directory exists # Ensure directory exists
os.makedirs(os.path.dirname(self.persistence_path), exist_ok=True) os.makedirs(os.path.dirname(self.persistence_path), exist_ok=True)
# Acquire file lock # Acquire file lock
lock_fd = open(self._lock_file, 'w') lock_fd = open(self._lock_file, "w")
fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX) fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX)
# Write with retries # Write with retries
@@ -120,7 +122,9 @@ class QueuePersistenceManager:
except Exception as e: except Exception as e:
if attempt == self.max_retries - 1: if attempt == self.max_retries - 1:
raise raise
logger.warning(f"Retry {attempt + 1}/{self.max_retries} failed: {e}") logger.warning(
f"Retry {attempt + 1}/{self.max_retries} failed: {e}"
)
await asyncio.sleep(self.retry_delay) await asyncio.sleep(self.retry_delay)
except Exception as e: except Exception as e:
@@ -140,18 +144,25 @@ class QueuePersistenceManager:
# Create backup # Create backup
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S") timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
backup_path = f"{self.persistence_path}.bak.{timestamp}" backup_path = f"{self.persistence_path}.bak.{timestamp}"
with open(self.persistence_path, "rb") as src, open(backup_path, "wb") as dst: with open(self.persistence_path, "rb") as src, open(
backup_path, "wb"
) as dst:
dst.write(src.read()) dst.write(src.read())
dst.flush() dst.flush()
os.fsync(dst.fileno()) os.fsync(dst.fileno())
# Clean old backups # Clean old backups
backup_files = sorted([ backup_files = sorted(
f for f in os.listdir(os.path.dirname(self.persistence_path)) [
f
for f in os.listdir(os.path.dirname(self.persistence_path))
if f.startswith(os.path.basename(self.persistence_path) + ".bak.") if f.startswith(os.path.basename(self.persistence_path) + ".bak.")
]) ]
)
while len(backup_files) > self.max_backups: while len(backup_files) > self.max_backups:
old_backup = os.path.join(os.path.dirname(self.persistence_path), backup_files.pop(0)) old_backup = os.path.join(
os.path.dirname(self.persistence_path), backup_files.pop(0)
)
try: try:
os.remove(old_backup) os.remove(old_backup)
except Exception as e: except Exception as e:
@@ -175,7 +186,7 @@ class QueuePersistenceManager:
lock_fd = None lock_fd = None
try: try:
# Acquire file lock # Acquire file lock
lock_fd = open(self._lock_file, 'w') lock_fd = open(self._lock_file, "w")
fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX) fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX)
# Try loading main file # Try loading main file
@@ -188,18 +199,28 @@ class QueuePersistenceManager:
break break
except Exception as e: except Exception as e:
last_error = e last_error = e
logger.warning(f"Retry {attempt + 1}/{self.max_retries} failed: {e}") logger.warning(
f"Retry {attempt + 1}/{self.max_retries} failed: {e}"
)
time.sleep(self.retry_delay) time.sleep(self.retry_delay)
# If main file failed, try loading latest backup # If main file failed, try loading latest backup
if state is None: if state is None:
backup_files = sorted([ backup_files = sorted(
f for f in os.listdir(os.path.dirname(self.persistence_path)) [
if f.startswith(os.path.basename(self.persistence_path) + ".bak.") f
], reverse=True) for f in os.listdir(os.path.dirname(self.persistence_path))
if f.startswith(
os.path.basename(self.persistence_path) + ".bak."
)
],
reverse=True,
)
if backup_files: if backup_files:
latest_backup = os.path.join(os.path.dirname(self.persistence_path), backup_files[0]) latest_backup = os.path.join(
os.path.dirname(self.persistence_path), backup_files[0]
)
try: try:
with open(latest_backup, "r") as f: with open(latest_backup, "r") as f:
state = json.load(f) state = json.load(f)
@@ -207,7 +228,9 @@ class QueuePersistenceManager:
except Exception as e: except Exception as e:
logger.error(f"Failed to load backup: {e}") logger.error(f"Failed to load backup: {e}")
if last_error: if last_error:
raise QueueError(f"Failed to load queue state: {last_error}") raise QueueError(
f"Failed to load queue state: {last_error}"
)
raise raise
if state is None: if state is None:
@@ -218,22 +241,34 @@ class QueuePersistenceManager:
try: try:
if isinstance(item_data, dict): if isinstance(item_data, dict):
# Ensure datetime fields are properly formatted # Ensure datetime fields are properly formatted
for field in ['added_at', 'last_retry', 'last_error_time']: for field in ["added_at", "last_retry", "last_error_time"]:
if field in item_data and item_data[field]: if field in item_data and item_data[field]:
if isinstance(item_data[field], str): if isinstance(item_data[field], str):
try: try:
item_data[field] = datetime.fromisoformat(item_data[field]) item_data[field] = datetime.fromisoformat(
item_data[field]
)
except ValueError: except ValueError:
item_data[field] = datetime.utcnow() if field == 'added_at' else None item_data[field] = (
datetime.utcnow()
if field == "added_at"
else None
)
elif not isinstance(item_data[field], datetime): elif not isinstance(item_data[field], datetime):
item_data[field] = datetime.utcnow() if field == 'added_at' else None item_data[field] = (
datetime.utcnow()
if field == "added_at"
else None
)
# Ensure processing_time is a float # Ensure processing_time is a float
if 'processing_time' in item_data: if "processing_time" in item_data:
try: try:
item_data['processing_time'] = float(item_data['processing_time']) item_data["processing_time"] = float(
item_data["processing_time"]
)
except (ValueError, TypeError): except (ValueError, TypeError):
item_data['processing_time'] = 0.0 item_data["processing_time"] = 0.0
return QueueItem(**item_data) return QueueItem(**item_data)
return None return None
@@ -283,15 +318,21 @@ class QueuePersistenceManager:
backup_path = f"{self.persistence_path}.corrupted.{int(time.time())}" backup_path = f"{self.persistence_path}.corrupted.{int(time.time())}"
try: try:
os.rename(self.persistence_path, backup_path) os.rename(self.persistence_path, backup_path)
logger.info(f"Created backup of corrupted state file: {backup_path}") logger.info(
f"Created backup of corrupted state file: {backup_path}"
)
except Exception as be: except Exception as be:
logger.error(f"Failed to create backup of corrupted state file: {str(be)}") logger.error(
f"Failed to create backup of corrupted state file: {str(be)}"
)
raise QueueError(f"Failed to load queue state: {str(e)}") raise QueueError(f"Failed to load queue state: {str(e)}")
finally: finally:
if lock_fd: if lock_fd:
fcntl.flock(lock_fd.fileno(), fcntl.LOCK_UN) fcntl.flock(lock_fd.fileno(), fcntl.LOCK_UN)
lock_fd.close() lock_fd.close()
class QueueError(Exception): class QueueError(Exception):
"""Base exception for queue-related errors""" """Base exception for queue-related errors"""
pass pass

6
videoarchiver/queue/processor.py
View File

@@ -8,9 +8,9 @@ from dataclasses import dataclass
from typing import Callable, Optional, Tuple, List, Set, Dict, Any from typing import Callable, Optional, Tuple, List, Set, Dict, Any
from datetime import datetime, timedelta from datetime import datetime, timedelta
from videoarchiver.queue.models import QueueItem from .models import QueueItem
from videoarchiver.queue.state_manager import QueueStateManager, ItemState from .state_manager import QueueStateManager, ItemState
from videoarchiver.queue.monitoring import QueueMonitor from .monitoring import QueueMonitor
logger = logging.getLogger("QueueProcessor") logger = logging.getLogger("QueueProcessor")

2
videoarchiver/queue/recovery_manager.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass, field
from typing import List, Tuple, Dict, Optional, Any, Set from typing import List, Tuple, Dict, Optional, Any, Set
from datetime import datetime, timedelta from datetime import datetime, timedelta
from videoarchiver.queue.models import QueueItem from .models import QueueItem
logger = logging.getLogger("QueueRecoveryManager") logger = logging.getLogger("QueueRecoveryManager")

2
videoarchiver/queue/state_manager.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass
from typing import Dict, Set, List, Optional, Any from typing import Dict, Set, List, Optional, Any
from datetime import datetime from datetime import datetime
from videoarchiver.queue.models import QueueItem, QueueMetrics from .models import QueueItem, QueueMetrics
logger = logging.getLogger("QueueStateManager") logger = logging.getLogger("QueueStateManager")

2
videoarchiver/update_checker.py
View File

@@ -15,7 +15,7 @@ import tempfile
import os import os
import shutil import shutil
from videoarchiver.exceptions import UpdateError from .exceptions import UpdateError
logger = logging.getLogger('VideoArchiver') logger = logging.getLogger('VideoArchiver')

18
videoarchiver/utils/__init__.py
View File

@@ -2,27 +2,27 @@
from typing import Dict, Optional, Any, Union, List from typing import Dict, Optional, Any, Union, List
from videoarchiver.utils.file_ops import ( from .utils.file_ops import (
cleanup_downloads, cleanup_downloads,
ensure_directory, ensure_directory,
get_file_size, get_file_size,
is_valid_path, is_valid_path,
safe_delete safe_delete
) )
from videoarchiver.utils.file_deletion import FileDeleter from .utils.file_deletion import FileDeleter
from videoarchiver.utils.directory_manager import DirectoryManager from .utils.directory_manager import DirectoryManager
from videoarchiver.utils.permission_manager import PermissionManager from .utils.permission_manager import PermissionManager
from videoarchiver.utils.download_manager import DownloadManager from .utils.download_manager import DownloadManager
from videoarchiver.utils.compression_manager import CompressionManager from .utils.compression_manager import CompressionManager
from videoarchiver.utils.progress_tracker import ( from .utils.progress_tracker import (
ProgressTracker, ProgressTracker,
ProgressStatus, ProgressStatus,
DownloadProgress, DownloadProgress,
CompressionProgress, CompressionProgress,
CompressionParams CompressionParams
) )
from videoarchiver.utils.path_manager import PathManager from .utils.path_manager import PathManager
from videoarchiver.utils.exceptions import ( from .utils.exceptions import (
# Base exception # Base exception
VideoArchiverError, VideoArchiverError,
ErrorSeverity, ErrorSeverity,

10
videoarchiver/utils/compression_handler.py
View File

@@ -7,11 +7,11 @@ import subprocess
from datetime import datetime from datetime import datetime
from typing import Dict, Optional, Callable, Set, Tuple from typing import Dict, Optional, Callable, Set, Tuple
from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager from .ffmpeg.ffmpeg_manager import FFmpegManager
from videoarchiver.ffmpeg.exceptions import CompressionError from .ffmpeg.exceptions import CompressionError
from videoarchiver.utils.exceptions import VideoVerificationError from .utils.exceptions import VideoVerificationError
from videoarchiver.utils.file_operations import FileOperations from .utils.file_operations import FileOperations
from videoarchiver.utils.progress_handler import ProgressHandler from .utils.progress_handler import ProgressHandler
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

10
videoarchiver/utils/compression_manager.py
View File

@@ -7,11 +7,11 @@ import subprocess
from datetime import datetime from datetime import datetime
from typing import Dict, Any, Optional, Callable, List, Set, Tuple from typing import Dict, Any, Optional, Callable, List, Set, Tuple
from videoarchiver.processor import _compression_progress from .processor import _compression_progress
from videoarchiver.utils.compression_handler import CompressionHandler from .utils.compression_handler import CompressionHandler
from videoarchiver.utils.progress_handler import ProgressHandler from .utils.progress_handler import ProgressHandler
from videoarchiver.utils.file_operations import FileOperations from .utils.file_operations import FileOperations
from videoarchiver.utils.exceptions import CompressionError, VideoVerificationError from .utils.exceptions import CompressionError, VideoVerificationError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

4
videoarchiver/utils/directory_manager.py
View File

@@ -6,8 +6,8 @@ import asyncio
from pathlib import Path from pathlib import Path
from typing import List, Optional, Tuple from typing import List, Optional, Tuple
from videoarchiver.utils.exceptions import FileCleanupError from .utils.exceptions import FileCleanupError
from videoarchiver.utils.file_deletion import SecureFileDeleter from .utils.file_deletion import SecureFileDeleter
logger = logging.getLogger("DirectoryManager") logger = logging.getLogger("DirectoryManager")

12
videoarchiver/utils/download_core.py
View File

@@ -7,12 +7,12 @@ import yt_dlp
from typing import Dict, Optional, Callable, Tuple from typing import Dict, Optional, Callable, Tuple
from pathlib import Path from pathlib import Path
from videoarchiver.utils.url_validator import check_url_support from .utils.url_validator import check_url_support
from videoarchiver.utils.progress_handler import ProgressHandler, CancellableYTDLLogger from .utils.progress_handler import ProgressHandler, CancellableYTDLLogger
from videoarchiver.utils.file_operations import FileOperations from .utils.file_operations import FileOperations
from videoarchiver.utils.compression_handler import CompressionHandler from .utils.compression_handler import CompressionHandler
from videoarchiver.utils.process_manager import ProcessManager from .utils.process_manager import ProcessManager
from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager from .ffmpeg.ffmpeg_manager import FFmpegManager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

6
videoarchiver/utils/download_manager.py
View File

@@ -9,9 +9,9 @@ from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional, Tuple, Callable, Any from typing import Dict, List, Optional, Tuple, Callable, Any
from pathlib import Path from pathlib import Path
from videoarchiver.utils.verification_manager import VideoVerificationManager from .utils.verification_manager import VideoVerificationManager
from videoarchiver.utils.compression_manager import CompressionManager from .utils.compression_manager import CompressionManager
from videoarchiver.utils import progress_tracker from .utils import progress_tracker
logger = logging.getLogger("DownloadManager") logger = logging.getLogger("DownloadManager")

2
videoarchiver/utils/file_deletion.py
View File

@@ -7,7 +7,7 @@ import logging
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
from videoarchiver.utils.exceptions import FileCleanupError from .utils.exceptions import FileCleanupError
logger = logging.getLogger("FileDeleter") logger = logging.getLogger("FileDeleter")

4
videoarchiver/utils/file_operations.py
View File

@@ -9,8 +9,8 @@ import subprocess
from typing import Tuple from typing import Tuple
from pathlib import Path from pathlib import Path
from videoarchiver.utils.exceptions import VideoVerificationError from .utils.exceptions import VideoVerificationError
from videoarchiver.utils.file_deletion import secure_delete_file from .utils.file_deletion import secure_delete_file
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

8
videoarchiver/utils/file_ops.py
View File

@@ -4,10 +4,10 @@ import logging
from pathlib import Path from pathlib import Path
from typing import List, Tuple, Optional from typing import List, Tuple, Optional
from videoarchiver.utils.exceptions import FileCleanupError from .utils.exceptions import FileCleanupError
from videoarchiver.utils.file_deletion import SecureFileDeleter from .utils.file_deletion import SecureFileDeleter
from videoarchiver.utils.directory_manager import DirectoryManager from .utils.directory_manager import DirectoryManager
from videoarchiver.utils.permission_manager import PermissionManager from .utils.permission_manager import PermissionManager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

4
videoarchiver/utils/path_manager.py
View File

@@ -10,8 +10,8 @@ import time
from typing import Generator, List, Optional from typing import Generator, List, Optional
from pathlib import Path from pathlib import Path
from videoarchiver.utils.exceptions import FileCleanupError from .utils.exceptions import FileCleanupError
from videoarchiver.utils.permission_manager import PermissionManager from .utils.permission_manager import PermissionManager
logger = logging.getLogger("PathManager") logger = logging.getLogger("PathManager")

2
videoarchiver/utils/permission_manager.py
View File

@@ -6,7 +6,7 @@ import logging
from pathlib import Path from pathlib import Path
from typing import Optional, Union, List from typing import Optional, Union, List
from videoarchiver.utils.exceptions import FileCleanupError from .utils.exceptions import FileCleanupError
logger = logging.getLogger("PermissionManager") logger = logging.getLogger("PermissionManager")