pacnpal/Pac-cogs
1
0
Fork 0
mirror of https://github.com/pacnpal/Pac-cogs.git synced 2025-12-20 02:41:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

diabolical

Browse Source
This commit is contained in:
pacnpal
2024-11-18 05:17:37 +00:00
parent fc06e54d8a
commit df9099f2c8
75 changed files with 744 additions and 719 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
videoarchiver/__init__.py
View File

@@ -9,58 +9,58 @@ from redbot.core.bot import Red # type: ignore
# Force reload of all modules
modules_to_reload = [
".utils.exceptions",
".utils",
".processor",
".processor.core",
".processor.queue_processor",
".queue",
".queue.types", # Added types module
".queue.models",
".queue.manager",
".queue.cleaners",
".queue.cleaners.guild_cleaner",
".queue.cleaners.history_cleaner",
".queue.cleaners.tracking_cleaner",
".queue.monitoring",
".queue.recovery_manager",
".queue.state_manager",
".ffmpeg",
".ffmpeg.binary_manager",
".ffmpeg.encoder_params",
".ffmpeg.exceptions",
".ffmpeg.ffmpeg_downloader",
".ffmpeg.ffmpeg_manager",
".ffmpeg.gpu_detector",
".ffmpeg.process_manager",
".ffmpeg.verification_manager",
".ffmpeg.video_analyzer",
".database",
".database.connection_manager",
".database.query_manager",
".database.schema_manager",
".database.video_archive_db",
".config",
".config.channel_manager",
".config.exceptions",
".config.role_manager",
".config.settings_formatter",
".config.validation_manager",
".core",
".core.base",
".core.cleanup",
".core.commands",
".core.commands.archiver_commands",
".core.commands.database_commands",
".core.commands.settings_commands",
".core.component_manager",
".core.error_handler",
".core.events",
".core.guild",
".core.initialization",
".core.lifecycle",
".core.response_handler",
".core.settings",
"utils.exceptions",
"utils",
"processor",
"processor.core",
"processor.queue_processor",
"queue",
"queue.types", # Added types module
"queue.models",
"queue.manager",
"queue.cleaners",
"queue.cleaners.guild_cleaner",
"queue.cleaners.history_cleaner",
"queue.cleaners.tracking_cleaner",
"queue.monitoring",
"queue.recovery_manager",
"queue.state_manager",
"ffmpeg",
"ffmpeg.binary_manager",
"ffmpeg.encoder_params",
"ffmpeg.exceptions",
"ffmpeg.ffmpeg_downloader",
"ffmpeg.ffmpeg_manager",
"ffmpeg.gpu_detector",
"ffmpeg.process_manager",
"ffmpeg.verification_manager",
"ffmpeg.video_analyzer",
"database",
"database.connection_manager",
"database.query_manager",
"database.schema_manager",
"database.video_archive_db",
"config",
"config.channel_manager",
"config.exceptions",
"config.role_manager",
"config.settings_formatter",
"config.validation_manager",
"core",
"core.base",
"core.cleanup",
"core.commands",
"core.commands.archiver_commands",
"core.commands.database_commands",
"core.commands.settings_commands",
"core.component_manager",
"core.error_handler",
"core.events",
"core.guild",
"core.initialization",
"core.lifecycle",
"core.response_handler",
"core.settings",
]
# Remove modules to force fresh import
@@ -68,7 +68,7 @@ for module in modules_to_reload:
if module in sys.modules:
del sys.modules[module]
try:
# try:
# Try relative imports first
from . import utils
from . import processor
@@ -89,7 +89,8 @@ try:
ErrorSeverity,
ProcessingError,
)
except ImportError:
# except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver import utils
# from videoarchiver import processor
@@ -121,17 +122,17 @@ except ImportError:
importlib.reload(core)
# Import all submodules
from .database import *
from .ffmpeg import *
from .queue import *
from .processor import *
from .config_manager import *
from .update_checker import *
from .queue.cleaners import *
from .database import *
from .utils import *
from .core import *
from .config import *
from database import *
from ffmpeg import *
from queue import *
from processor import *
from config_manager import *
from update_checker import *
from queue.cleaners import *
from database import *
from utils import *
from core import *
from config import *
logger = logging.getLogger("VideoArchiver")

10
videoarchiver/config/__init__.py
View File

@@ -2,7 +2,7 @@
# try:
# Try relative imports first
from .exceptions import (
from exceptions import (
ConfigurationError,
ValidationError,
PermissionError,
@@ -12,10 +12,10 @@ from .exceptions import (
SchemaError,
DiscordAPIError,
)
from .channel_manager import ChannelManager
from .role_manager import RoleManager
from .settings_formatter import SettingsFormatter
from .validation_manager import ValidationManager
from channel_manager import ChannelManager
from role_manager import RoleManager
from settings_formatter import SettingsFormatter
from validation_manager import ValidationManager
# except ImportError:
# Fall back to absolute imports if relative imports fail

2
videoarchiver/config/channel_manager.py
View File

@@ -6,7 +6,7 @@ import discord # type: ignore
# try:
# Try relative imports first
from .exceptions import (
from exceptions import (
ConfigurationError as ConfigError,
DiscordAPIError,
)

2
videoarchiver/config/role_manager.py
View File

@@ -6,7 +6,7 @@ import discord # type: ignore
# try:
# Try relative imports first
from .exceptions import ConfigurationError as ConfigError
from exceptions import ConfigurationError as ConfigError
# except ImportError:
# Fall back to absolute imports if relative imports fail

2
videoarchiver/config/settings_formatter.py
View File

@@ -7,7 +7,7 @@ import discord # type: ignore
# try:
# Try relative imports first
from .exceptions import ConfigurationError as ConfigError
from exceptions import ConfigurationError as ConfigError
# except ImportError:
# Fall back to absolute imports if relative imports fail

2
videoarchiver/config/validation_manager.py
View File

@@ -5,7 +5,7 @@ from typing import Any, Dict, List, Union
# try:
# Try relative imports first
from .exceptions import ConfigurationError as ConfigError
from exceptions import ConfigurationError as ConfigError
# except ImportError:
# Fall back to absolute imports if relative imports fail

10
videoarchiver/config_manager.py
View File

@@ -8,11 +8,11 @@ from redbot.core import Config # type: ignore
# try:
# Try relative imports first
from .config.validation_manager import ValidationManager
from .config.settings_formatter import SettingsFormatter
from .config.channel_manager import ChannelManager
from .config.role_manager import RoleManager
from .utils.exceptions import ConfigurationError as ConfigError
from config.validation_manager import ValidationManager
from config.settings_formatter import SettingsFormatter
from config.channel_manager import ChannelManager
from config.role_manager import RoleManager
from utils.exceptions import ConfigurationError as ConfigError
# except ImportError:
# Fall back to absolute imports if relative imports fail

2
videoarchiver/core/__init__.py
View File

@@ -2,7 +2,7 @@
# try:
# Try relative imports first
from .base import VideoArchiver
from base import VideoArchiver
# except ImportError:
# Fall back to absolute imports if relative imports fail

30
videoarchiver/core/base.py
View File

@@ -14,21 +14,21 @@ from redbot.core.commands import GroupCog, Context # type: ignore
# try:
# Try relative imports first
from .settings import Settings
from .lifecycle import LifecycleManager, LifecycleState
from .component_manager import ComponentManager, ComponentState
from .error_handler import error_manager, handle_command_error
from .response_handler import ResponseManager
from .commands.archiver_commands import setup_archiver_commands
from .commands.database_commands import setup_database_commands
from .commands.settings_commands import setup_settings_commands
from .events import setup_events, EventManager
from ..processor.core import VideoProcessor
from ..queue.manager import EnhancedVideoQueueManager
from ..ffmpeg.ffmpeg_manager import FFmpegManager
from ..database.video_archive_db import VideoArchiveDB
from ..config_manager import ConfigManager
from ..utils.exceptions import CogError, ErrorContext, ErrorSeverity
from settings import Settings
from lifecycle import LifecycleManager, LifecycleState
from component_manager import ComponentManager, ComponentState
from error_handler import error_manager, handle_command_error
from response_handler import ResponseManager
from commands.archiver_commands import setup_archiver_commands
from commands.database_commands import setup_database_commands
from commands.settings_commands import setup_settings_commands
from events import setup_events, EventManager
from processor.core import VideoProcessor
from queue.manager import EnhancedVideoQueueManager
from ffmpeg.ffmpeg_manager import FFmpegManager
from database.video_archive_db import VideoArchiveDB
from config_manager import ConfigManager
from utils.exceptions import CogError, ErrorContext, ErrorSeverity
# except ImportError:
# Fall back to absolute imports if relative imports fail

0
videoarchiver/core/types.py → videoarchiver/core/c_types.py
View File

126
videoarchiver/core/cleanup.py
View File

@@ -8,25 +8,29 @@ from enum import Enum, auto
from pathlib import Path
from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar
#try:
# Try relative imports first
from ..utils.file_ops import cleanup_downloads
from ..utils.exceptions import CleanupError, ErrorContext, ErrorSeverity
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.file_ops import cleanup_downloads
# from videoarchiver.utils.exceptions import CleanupError, ErrorContext, ErrorSeverity
# try:
# Try relative imports first
from utils.file_ops import cleanup_downloads
from utils.exceptions import CleanupError, ErrorContext, ErrorSeverity
# except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.file_ops import cleanup_downloads
# from videoarchiver.utils.exceptions import CleanupError, ErrorContext, ErrorSeverity
if TYPE_CHECKING:
#try:
# try:
from .base import VideoArchiver
#except ImportError:
# from videoarchiver.core.base import VideoArchiver
# except ImportError:
# from videoarchiver.core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver")
class CleanupPhase(Enum):
"""Cleanup phases"""
INITIALIZATION = auto()
UPDATE_CHECKER = auto()
PROCESSOR = auto()
@@ -36,21 +40,26 @@ class CleanupPhase(Enum):
DOWNLOADS = auto()
REFERENCES = auto()
class CleanupStatus(Enum):
"""Cleanup status"""
SUCCESS = auto()
TIMEOUT = auto()
ERROR = auto()
SKIPPED = auto()
class CleanupResult(TypedDict):
"""Type definition for cleanup result"""
phase: CleanupPhase
status: CleanupStatus
error: Optional[str]
duration: float
timestamp: str
class CleanupManager:
"""Manages cleanup operations"""
@@ -65,7 +74,7 @@ class CleanupManager:
phase: CleanupPhase,
status: CleanupStatus,
error: Optional[str] = None,
duration: float = 0.0
duration: float = 0.0,
) -> None:
"""Record result of a cleanup phase"""
self.results[phase] = CleanupResult(
@@ -73,20 +82,21 @@ class CleanupManager:
status=status,
error=error,
duration=duration,
timestamp=datetime.utcnow().isoformat()
timestamp=datetime.utcnow().isoformat(),
)
def get_results(self) -> Dict[CleanupPhase, CleanupResult]:
"""Get cleanup results"""
return self.results.copy()
async def cleanup_resources(cog: "VideoArchiver") -> None:
"""
Clean up all resources with proper handling.
Args:
cog: VideoArchiver cog instance
Raises:
CleanupError: If cleanup fails
"""
@@ -102,11 +112,13 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
try:
logger.info("Cancelling initialization task")
cog._init_task.cancel()
await asyncio.wait_for(cog._init_task, timeout=cleanup_manager.CLEANUP_TIMEOUT)
await asyncio.wait_for(
cog._init_task, timeout=cleanup_manager.CLEANUP_TIMEOUT
)
cleanup_manager.record_result(
CleanupPhase.INITIALIZATION,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except (asyncio.TimeoutError, asyncio.CancelledError) as e:
logger.warning("Initialization task cancellation timed out")
@@ -114,7 +126,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.INITIALIZATION,
CleanupStatus.TIMEOUT,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
# Stop update checker
@@ -123,13 +135,12 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
try:
logger.info("Stopping update checker")
await asyncio.wait_for(
cog.update_checker.stop(),
timeout=cleanup_manager.CLEANUP_TIMEOUT
cog.update_checker.stop(), timeout=cleanup_manager.CLEANUP_TIMEOUT
)
cleanup_manager.record_result(
CleanupPhase.UPDATE_CHECKER,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except asyncio.TimeoutError as e:
logger.warning("Update checker stop timed out")
@@ -137,7 +148,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.UPDATE_CHECKER,
CleanupStatus.TIMEOUT,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
cog.update_checker = None
@@ -147,13 +158,12 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
try:
logger.info("Cleaning up processor")
await asyncio.wait_for(
cog.processor.cleanup(),
timeout=cleanup_manager.CLEANUP_TIMEOUT
cog.processor.cleanup(), timeout=cleanup_manager.CLEANUP_TIMEOUT
)
cleanup_manager.record_result(
CleanupPhase.PROCESSOR,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except asyncio.TimeoutError as e:
logger.warning("Processor cleanup timed out, forcing cleanup")
@@ -162,7 +172,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.PROCESSOR,
CleanupStatus.TIMEOUT,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
cog.processor = None
@@ -172,13 +182,12 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
try:
logger.info("Cleaning up queue manager")
await asyncio.wait_for(
cog.queue_manager.cleanup(),
timeout=cleanup_manager.CLEANUP_TIMEOUT
cog.queue_manager.cleanup(), timeout=cleanup_manager.CLEANUP_TIMEOUT
)
cleanup_manager.record_result(
CleanupPhase.QUEUE_MANAGER,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except asyncio.TimeoutError as e:
logger.warning("Queue manager cleanup timed out, forcing stop")
@@ -187,7 +196,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.QUEUE_MANAGER,
CleanupStatus.TIMEOUT,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
cog.queue_manager = None
@@ -214,14 +223,14 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
CleanupPhase.COMPONENTS,
status,
"\n".join(errors) if errors else None,
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
cleanup_manager.record_result(
CleanupPhase.COMPONENTS,
CleanupStatus.ERROR,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
# Kill any FFmpeg processes
@@ -233,7 +242,7 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
cog.ffmpeg_mgr = None
# Kill any remaining FFmpeg processes system-wide
if os.name != 'nt': # Unix-like systems
if os.name != "nt": # Unix-like systems
os.system("pkill -9 ffmpeg")
else: # Windows
os.system("taskkill /F /IM ffmpeg.exe")
@@ -241,14 +250,14 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result(
CleanupPhase.FFMPEG,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
cleanup_manager.record_result(
CleanupPhase.FFMPEG,
CleanupStatus.ERROR,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
# Clean up download directory
@@ -258,21 +267,21 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
logger.info("Cleaning up download directory")
await asyncio.wait_for(
cleanup_downloads(str(cog.download_path)),
timeout=cleanup_manager.CLEANUP_TIMEOUT
timeout=cleanup_manager.CLEANUP_TIMEOUT,
)
if cog.download_path.exists():
cog.download_path.rmdir()
cleanup_manager.record_result(
CleanupPhase.DOWNLOADS,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
cleanup_manager.record_result(
CleanupPhase.DOWNLOADS,
CleanupStatus.ERROR,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
@@ -284,8 +293,8 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
"Cleanup",
"cleanup_resources",
{"duration": (datetime.utcnow() - start_time).total_seconds()},
ErrorSeverity.HIGH
)
ErrorSeverity.HIGH,
),
)
finally:
logger.info("Clearing ready flag")
@@ -294,17 +303,18 @@ async def cleanup_resources(cog: "VideoArchiver") -> None:
# Log cleanup results
for phase, result in cleanup_manager.get_results().items():
status_str = f"{result['status'].name}"
if result['error']:
if result["error"]:
status_str += f" ({result['error']})"
logger.info(
f"Cleanup phase {phase.name}: {status_str} "
f"(Duration: {result['duration']:.2f}s)"
)
async def force_cleanup_resources(cog: "VideoArchiver") -> None:
"""
Force cleanup of resources when timeout occurs.
Args:
cog: VideoArchiver cog instance
"""
@@ -323,14 +333,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result(
CleanupPhase.PROCESSOR,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
cleanup_manager.record_result(
CleanupPhase.PROCESSOR,
CleanupStatus.ERROR,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
cog.processor = None
@@ -343,14 +353,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result(
CleanupPhase.QUEUE_MANAGER,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
cleanup_manager.record_result(
CleanupPhase.QUEUE_MANAGER,
CleanupStatus.ERROR,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
cog.queue_manager = None
@@ -363,7 +373,7 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cog.ffmpeg_mgr = None
# Force kill any remaining FFmpeg processes system-wide
if os.name != 'nt': # Unix-like systems
if os.name != "nt": # Unix-like systems
os.system("pkill -9 ffmpeg")
else: # Windows
os.system("taskkill /F /IM ffmpeg.exe")
@@ -371,14 +381,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result(
CleanupPhase.FFMPEG,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
cleanup_manager.record_result(
CleanupPhase.FFMPEG,
CleanupStatus.ERROR,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
# Clean up download directory
@@ -388,21 +398,21 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
logger.info("Force cleaning download directory")
await asyncio.wait_for(
cleanup_downloads(str(cog.download_path)),
timeout=cleanup_manager.FORCE_CLEANUP_TIMEOUT
timeout=cleanup_manager.FORCE_CLEANUP_TIMEOUT,
)
if cog.download_path.exists():
cog.download_path.rmdir()
cleanup_manager.record_result(
CleanupPhase.DOWNLOADS,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
cleanup_manager.record_result(
CleanupPhase.DOWNLOADS,
CleanupStatus.ERROR,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
# Clear all components
@@ -414,14 +424,14 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cleanup_manager.record_result(
CleanupPhase.COMPONENTS,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
cleanup_manager.record_result(
CleanupPhase.COMPONENTS,
CleanupStatus.ERROR,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
@@ -443,25 +453,25 @@ async def force_cleanup_resources(cog: "VideoArchiver") -> None:
cog.db = None
cog._init_task = None
cog._cleanup_task = None
if hasattr(cog, '_queue_task'):
if hasattr(cog, "_queue_task"):
cog._queue_task = None
cleanup_manager.record_result(
CleanupPhase.REFERENCES,
CleanupStatus.SUCCESS,
duration=(datetime.utcnow() - phase_start).total_seconds()
duration=(datetime.utcnow() - phase_start).total_seconds(),
)
except Exception as e:
cleanup_manager.record_result(
CleanupPhase.REFERENCES,
CleanupStatus.ERROR,
str(e),
(datetime.utcnow() - phase_start).total_seconds()
(datetime.utcnow() - phase_start).total_seconds(),
)
# Log cleanup results
for phase, result in cleanup_manager.get_results().items():
status_str = f"{result['status'].name}"
if result['error']:
if result["error"]:
status_str += f" ({result['error']})"
logger.info(
f"Force cleanup phase {phase.name}: {status_str} "

7
videoarchiver/core/commands.py
View File

@@ -7,9 +7,10 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
# try:
from .base import VideoArchiver
# except ImportError:
# from videoarchiver.core.base import VideoArchiver
from base import VideoArchiver
# except ImportError:
# from videoarchiver.core.base import VideoArchiver
def setup_commands(cog: "VideoArchiver") -> None:
"""Command setup is now handled in the VideoArchiver class"""

6
videoarchiver/core/commands/__init__.py
View File

@@ -1,8 +1,8 @@
"""Command handlers for VideoArchiver"""
from .archiver_commands import setup_archiver_commands
from .database_commands import setup_database_commands
from .settings_commands import setup_settings_commands
from archiver_commands import setup_archiver_commands
from database_commands import setup_database_commands
from settings_commands import setup_settings_commands
__all__ = [
"setup_archiver_commands",

4
videoarchiver/core/commands/archiver_commands.py
View File

@@ -9,8 +9,8 @@ from discord import app_commands # type: ignore
from redbot.core import commands # type: ignore
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore
from ...core.response_handler import handle_response, ResponseType
from ...utils.exceptions import CommandError, ErrorContext, ErrorSeverity
from core.response_handler import handle_response, ResponseType
from utils.exceptions import CommandError, ErrorContext, ErrorSeverity
logger = logging.getLogger("VideoArchiver")

6
videoarchiver/core/commands/database_commands.py
View File

@@ -10,9 +10,9 @@ from discord import app_commands # type: ignore
from redbot.core import commands # type: ignore
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore
from ...core.response_handler import handle_response, ResponseType
from ...utils.exceptions import CommandError, ErrorContext, ErrorSeverity, DatabaseError
from ...database.video_archive_db import VideoArchiveDB
from core.response_handler import handle_response, ResponseType
from utils.exceptions import CommandError, ErrorContext, ErrorSeverity, DatabaseError
from database.video_archive_db import VideoArchiveDB
logger = logging.getLogger("VideoArchiver")

6
videoarchiver/core/commands/settings_commands.py
View File

@@ -9,9 +9,9 @@ from discord import app_commands # type: ignore
from redbot.core import commands # type: ignore
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore
from ...core.settings import VideoFormat, VideoQuality
from ...core.response_handler import handle_response, ResponseType
from ...utils.exceptions import CommandError, ErrorContext, ErrorSeverity
from core.settings import VideoFormat, VideoQuality
from core.response_handler import handle_response, ResponseType
from utils.exceptions import CommandError, ErrorContext, ErrorSeverity
logger = logging.getLogger("VideoArchiver")

12
videoarchiver/core/component_manager.py
View File

@@ -22,12 +22,12 @@ import importlib
# try:
# Try relative imports first
from ..utils.exceptions import ComponentError, ErrorContext, ErrorSeverity
from ..utils.path_manager import PathManager
from ..config_manager import ConfigManager
from ..processor.core import VideoProcessor
from ..queue.manager import EnhancedVideoQueueManager
from ..ffmpeg.ffmpeg_manager import FFmpegManager
from utils.exceptions import ComponentError, ErrorContext, ErrorSeverity
from utils.path_manager import PathManager
from config_manager import ConfigManager
from processor.core import VideoProcessor
from queue.manager import EnhancedVideoQueueManager
from ffmpeg.ffmpeg_manager import FFmpegManager
# except ImportError:
# Fall back to absolute imports if relative imports fail

99
videoarchiver/core/error_handler.py
View File

@@ -4,19 +4,19 @@ import logging
import traceback
from typing import Dict, Optional, Tuple, Type, TypedDict, ClassVar
from enum import Enum, auto
import discord # type: ignore
from redbot.core.commands import ( # type: ignore
import discord # type: ignore
from redbot.core.commands import ( # type: ignore
Context,
MissingPermissions,
BotMissingPermissions,
MissingRequiredArgument,
BadArgument,
CommandError
CommandError,
)
#try:
# Try relative imports first
from ..utils.exceptions import (
# try:
# Try relative imports first
from utils.exceptions import (
VideoArchiverError,
ErrorSeverity,
ErrorContext,
@@ -33,9 +33,10 @@ from ..utils.exceptions import (
TrackingError,
NetworkError,
ResourceExhaustedError,
ConfigurationError
ConfigurationError,
)
from ..core.response_handler import response_manager
from core.response_handler import response_manager
# except ImportError:
# # Fall back to absolute imports if relative imports fail
# # from videoarchiver.utils.exceptions import (
@@ -57,12 +58,14 @@ from ..core.response_handler import response_manager
# ResourceExhaustedError,
# ConfigurationError
# )
# from videoarchiver.core.response_handler import response_manager
# from videoarchiver.core.response_handler import response_manager
logger = logging.getLogger("VideoArchiver")
class ErrorCategory(Enum):
"""Categories of errors"""
PERMISSION = auto()
ARGUMENT = auto()
CONFIGURATION = auto()
@@ -76,17 +79,22 @@ class ErrorCategory(Enum):
HEALTH = auto()
UNEXPECTED = auto()
class ErrorStats(TypedDict):
"""Type definition for error statistics"""
counts: Dict[str, int]
patterns: Dict[str, Dict[str, int]]
severities: Dict[str, Dict[str, int]]
class ErrorFormatter:
"""Formats error messages for display"""
@staticmethod
def format_error_message(error: Exception, context: Optional[ErrorContext] = None) -> str:
def format_error_message(
error: Exception, context: Optional[ErrorContext] = None
) -> str:
"""Format error message with context"""
base_message = str(error)
if context:
@@ -110,16 +118,18 @@ class ErrorFormatter:
return "An unexpected error occurred. Please check the logs for details."
return str(error)
class ErrorCategorizer:
"""Categorizes errors and determines handling strategy"""
ERROR_MAPPING: ClassVar[Dict[Type[Exception], Tuple[ErrorCategory, ErrorSeverity]]] = {
ERROR_MAPPING: ClassVar[
Dict[Type[Exception], Tuple[ErrorCategory, ErrorSeverity]]
] = {
# Discord command errors
MissingPermissions: (ErrorCategory.PERMISSION, ErrorSeverity.MEDIUM),
BotMissingPermissions: (ErrorCategory.PERMISSION, ErrorSeverity.HIGH),
MissingRequiredArgument: (ErrorCategory.ARGUMENT, ErrorSeverity.LOW),
BadArgument: (ErrorCategory.ARGUMENT, ErrorSeverity.LOW),
# VideoArchiver errors
ProcessorError: (ErrorCategory.PROCESSING, ErrorSeverity.HIGH),
ValidationError: (ErrorCategory.VALIDATION, ErrorSeverity.MEDIUM),
@@ -134,17 +144,17 @@ class ErrorCategorizer:
TrackingError: (ErrorCategory.PROCESSING, ErrorSeverity.MEDIUM),
NetworkError: (ErrorCategory.NETWORK, ErrorSeverity.MEDIUM),
ResourceExhaustedError: (ErrorCategory.RESOURCE, ErrorSeverity.HIGH),
ConfigurationError: (ErrorCategory.CONFIGURATION, ErrorSeverity.HIGH)
ConfigurationError: (ErrorCategory.CONFIGURATION, ErrorSeverity.HIGH),
}
@classmethod
def categorize_error(cls, error: Exception) -> Tuple[ErrorCategory, ErrorSeverity]:
"""
Categorize an error and determine its severity.
Args:
error: Exception to categorize
Returns:
Tuple of (Error category, Severity level)
"""
@@ -153,6 +163,7 @@ class ErrorCategorizer:
return category, severity
return ErrorCategory.UNEXPECTED, ErrorSeverity.HIGH
class ErrorTracker:
"""Tracks error occurrences and patterns"""
@@ -162,31 +173,28 @@ class ErrorTracker:
self.error_severities: Dict[str, Dict[str, int]] = {}
def track_error(
self,
error: Exception,
category: ErrorCategory,
severity: ErrorSeverity
self, error: Exception, category: ErrorCategory, severity: ErrorSeverity
) -> None:
"""
Track an error occurrence.
Args:
error: Exception that occurred
category: Error category
severity: Error severity
"""
error_type = type(error).__name__
# Track error counts
self.error_counts[error_type] = self.error_counts.get(error_type, 0) + 1
# Track error patterns by category
if category.value not in self.error_patterns:
self.error_patterns[category.value] = {}
self.error_patterns[category.value][error_type] = (
self.error_patterns[category.value].get(error_type, 0) + 1
)
# Track error severities
if severity.value not in self.error_severities:
self.error_severities[severity.value] = {}
@@ -197,16 +205,17 @@ class ErrorTracker:
def get_error_stats(self) -> ErrorStats:
"""
Get error statistics.
Returns:
Dictionary containing error statistics
"""
return ErrorStats(
counts=self.error_counts.copy(),
patterns=self.error_patterns.copy(),
severities=self.error_severities.copy()
severities=self.error_severities.copy(),
)
class ErrorManager:
"""Manages error handling and reporting"""
@@ -215,14 +224,10 @@ class ErrorManager:
self.categorizer = ErrorCategorizer()
self.tracker = ErrorTracker()
async def handle_error(
self,
ctx: Context,
error: Exception
) -> None:
async def handle_error(self, ctx: Context, error: Exception) -> None:
"""
Handle a command error.
Args:
ctx: Command context
error: The error that occurred
@@ -230,7 +235,7 @@ class ErrorManager:
try:
# Categorize error
category, severity = self.categorizer.categorize_error(error)
# Create error context
context = ErrorContext(
component=ctx.command.qualified_name if ctx.command else "unknown",
@@ -238,26 +243,24 @@ class ErrorManager:
details={
"guild_id": str(ctx.guild.id) if ctx.guild else "DM",
"channel_id": str(ctx.channel.id),
"user_id": str(ctx.author.id)
"user_id": str(ctx.author.id),
},
severity=severity
severity=severity,
)
# Track error
self.tracker.track_error(error, category, severity)
# Format error messages
log_message = self.formatter.format_error_message(error, context)
user_message = self.formatter.format_user_message(error, category)
# Log error details
self._log_error(log_message, severity)
# Send response
await response_manager.send_response(
ctx,
content=user_message,
response_type=severity.name.lower()
ctx, content=user_message, response_type=severity.name.lower()
)
except Exception as e:
@@ -269,19 +272,15 @@ class ErrorManager:
await response_manager.send_response(
ctx,
content="An error occurred while handling another error. Please check the logs.",
response_type="error"
response_type="error",
)
except Exception:
pass
def _log_error(
self,
message: str,
severity: ErrorSeverity
) -> None:
def _log_error(self, message: str, severity: ErrorSeverity) -> None:
"""
Log error details.
Args:
message: Error message to log
severity: Error severity
@@ -296,13 +295,15 @@ class ErrorManager:
except Exception as e:
logger.error(f"Error logging error details: {e}")
# Global error manager instance
error_manager = ErrorManager()
async def handle_command_error(ctx: Context, error: Exception) -> None:
"""
Helper function to handle command errors using the error manager.
Args:
ctx: Command context
error: Exception to handle

41
videoarchiver/core/events.py
View File

@@ -9,28 +9,29 @@ from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar, List
import discord # type: ignore
#try:
# Try relative imports first
from ..processor.constants import REACTIONS
from ..processor.reactions import handle_archived_reaction
from .guild import initialize_guild_components, cleanup_guild_components
from .error_handler import ErrorManager
from .response_handler import response_manager
from ..utils.exceptions import EventError, ErrorContext, ErrorSeverity
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.processor.constants import REACTIONS
# from videoarchiver.processor.reactions import handle_archived_reaction
# from videoarchiver.core.guild import initialize_guild_components, cleanup_guild_components
# from videoarchiver.core.error_handler import ErrorManager
# from videoarchiver.core.response_handler import response_manager
# from videoarchiver.utils.exceptions import EventError, ErrorContext, ErrorSeverity
# try:
# Try relative imports first
from processor.constants import REACTIONS
from processor.reactions import handle_archived_reaction
from guild import initialize_guild_components, cleanup_guild_components
from error_handler import ErrorManager
from response_handler import response_manager
from utils.exceptions import EventError, ErrorContext, ErrorSeverity
# except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.processor.constants import REACTIONS
# from videoarchiver.processor.reactions import handle_archived_reaction
# from videoarchiver.core.guild import initialize_guild_components, cleanup_guild_components
# from videoarchiver.core.error_handler import ErrorManager
# from videoarchiver.core.response_handler import response_manager
# from videoarchiver.utils.exceptions import EventError, ErrorContext, ErrorSeverity
if TYPE_CHECKING:
#try:
from .base import VideoArchiver
# except ImportError:
# from videoarchiver.core.base import VideoArchiver
# try:
from base import VideoArchiver
# except ImportError:
# from videoarchiver.core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver")

10
videoarchiver/core/guild.py
View File

@@ -6,10 +6,10 @@ from typing import TYPE_CHECKING, Dict, Any, Optional
# try:
# Try relative imports first
from ..utils.download_core import DownloadCore
from ..utils.message_manager import MessageManager
from ..utils.file_ops import cleanup_downloads
from ..utils.exceptions import VideoArchiverError as ProcessingError
from utils.download_core import DownloadCore
from utils.message_manager import MessageManager
from utils.file_ops import cleanup_downloads
from utils.exceptions import VideoArchiverError as ProcessingError
# except ImportError:
# Fall back to absolute imports if relative imports fail
@@ -20,7 +20,7 @@ from ..utils.exceptions import VideoArchiverError as ProcessingError
if TYPE_CHECKING:
# try:
from .base import VideoArchiver
from base import VideoArchiver
# except ImportError:
# from videoarchiver.core.base import VideoArchiver

6
videoarchiver/core/initialization.py
View File

@@ -6,8 +6,8 @@ import logging
# try:
# Try relative imports first
from ..utils.exceptions import ComponentError, ErrorContext, ErrorSeverity
from .lifecycle import LifecycleState
from utils.exceptions import ComponentError, ErrorContext, ErrorSeverity
from lifecycle import LifecycleState
# except ImportError:
# Fall back to absolute imports if relative imports fail
@@ -16,7 +16,7 @@ from .lifecycle import LifecycleState
if TYPE_CHECKING:
# try:
from .base import VideoArchiver
from base import VideoArchiver
# except ImportError:
# from videoarchiver.core.base import VideoArchiver

27
videoarchiver/core/lifecycle.py
View File

@@ -8,25 +8,26 @@ from enum import Enum, auto
from datetime import datetime
# try:
# Try relative imports first
from .cleanup import cleanup_resources, force_cleanup_resources
from ..utils.exceptions import (
# Try relative imports first
from cleanup import cleanup_resources, force_cleanup_resources
from utils.exceptions import (
VideoArchiverError,
ErrorContext,
ErrorSeverity,
ComponentError,
CleanupError,
)
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.core.cleanup import cleanup_resources, force_cleanup_resources
# from videoarchiver.utils.exceptions import (
# VideoArchiverError,
# ErrorContext,
# ErrorSeverity,
# ComponentError,
# CleanupError,
# )
# except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.core.cleanup import cleanup_resources, force_cleanup_resources
# from videoarchiver.utils.exceptions import (
# VideoArchiverError,
# ErrorContext,
# ErrorSeverity,
# ComponentError,
# CleanupError,
# )
logger = logging.getLogger("VideoArchiver")

2
videoarchiver/core/response_handler.py
View File

@@ -9,7 +9,7 @@ from redbot.core.commands import Context # type: ignore
# try:
# Try relative imports first
from ..utils.exceptions import ErrorSeverity
from utils.exceptions import ErrorSeverity
# except ImportError:
# Fall back to absolute imports if relative imports fail

2
videoarchiver/core/settings.py
View File

@@ -6,7 +6,7 @@ from enum import Enum, auto
# try:
# Try relative imports first
from ..utils.exceptions import ConfigurationError, ErrorContext, ErrorSeverity
from utils.exceptions import ConfigurationError, ErrorContext, ErrorSeverity
# except ImportError:
# Fall back to absolute imports if relative imports fail

8
videoarchiver/database/__init__.py
View File

@@ -2,10 +2,10 @@
# try:
# Try relative imports first
from .connection_manager import DatabaseConnectionManager
from .query_manager import DatabaseQueryManager
from .schema_manager import DatabaseSchemaManager
from .video_archive_db import VideoArchiveDB
from connection_manager import DatabaseConnectionManager
from query_manager import DatabaseQueryManager
from schema_manager import DatabaseSchemaManager
from video_archive_db import VideoArchiveDB
# except ImportError:
# Fall back to absolute imports if relative imports fail

180
videoarchiver/database/connection_manager.py
View File

@@ -10,24 +10,29 @@ import threading
from queue import Queue, Empty
from datetime import datetime
#try:
# Try relative imports first
from ..utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
# try:
# Try relative imports first
from utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
# except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
logger = logging.getLogger("DBConnectionManager")
class ConnectionState(Enum):
"""Connection states"""
AVAILABLE = auto()
IN_USE = auto()
CLOSED = auto()
ERROR = auto()
class ConnectionStatus(TypedDict):
"""Type definition for connection status"""
state: str
created_at: str
last_used: str
@@ -36,8 +41,10 @@ class ConnectionStatus(TypedDict):
pool_size: int
available_connections: int
class ConnectionMetrics(TypedDict):
"""Type definition for connection metrics"""
total_connections: int
active_connections: int
idle_connections: int
@@ -46,6 +53,7 @@ class ConnectionMetrics(TypedDict):
failed_transactions: int
average_transaction_time: float
class ConnectionInfo:
"""Tracks connection information"""
@@ -73,6 +81,7 @@ class ConnectionInfo:
return 0.0
return self.total_transaction_time / self.transaction_count
class DatabaseConnectionManager:
"""Manages SQLite database connections and connection pooling"""
@@ -83,11 +92,11 @@ class DatabaseConnectionManager:
def __init__(self, db_path: Path, pool_size: int = DEFAULT_POOL_SIZE) -> None:
"""
Initialize the connection manager.
Args:
db_path: Path to the SQLite database file
pool_size: Maximum number of connections in the pool
Raises:
DatabaseError: If initialization fails
"""
@@ -97,14 +106,14 @@ class DatabaseConnectionManager:
self._connection_info: Dict[int, ConnectionInfo] = {}
self._local = threading.local()
self._lock = threading.Lock()
# Initialize connection pool
self._initialize_pool()
def _initialize_pool(self) -> None:
"""
Initialize the connection pool.
Raises:
DatabaseError: If pool initialization fails
"""
@@ -123,17 +132,17 @@ class DatabaseConnectionManager:
"ConnectionManager",
"initialize_pool",
{"pool_size": self.pool_size},
ErrorSeverity.CRITICAL
)
ErrorSeverity.CRITICAL,
),
)
def _create_connection(self) -> Optional[sqlite3.Connection]:
"""
Create a new database connection with proper settings.
Returns:
New database connection or None if creation fails
Raises:
DatabaseError: If connection creation fails
"""
@@ -141,23 +150,23 @@ class DatabaseConnectionManager:
conn = sqlite3.connect(
self.db_path,
detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
timeout=self.CONNECTION_TIMEOUT
timeout=self.CONNECTION_TIMEOUT,
)
# Enable foreign keys
conn.execute("PRAGMA foreign_keys = ON")
# Set journal mode to WAL for better concurrency
conn.execute("PRAGMA journal_mode = WAL")
# Set synchronous mode to NORMAL for better performance
conn.execute("PRAGMA synchronous = NORMAL")
# Enable extended result codes for better error handling
conn.execute("PRAGMA extended_result_codes = ON")
return conn
except sqlite3.Error as e:
error = f"Failed to create database connection: {str(e)}"
logger.error(error, exc_info=True)
@@ -167,18 +176,18 @@ class DatabaseConnectionManager:
"ConnectionManager",
"create_connection",
{"path": str(self.db_path)},
ErrorSeverity.HIGH
)
ErrorSeverity.HIGH,
),
)
@contextmanager
def get_connection(self) -> Generator[sqlite3.Connection, None, None]:
"""
Get a database connection from the pool.
Yields:
Database connection
Raises:
DatabaseError: If unable to get a connection
"""
@@ -186,7 +195,7 @@ class DatabaseConnectionManager:
start_time = datetime.utcnow()
try:
# Check if we have a transaction-bound connection
conn = getattr(self._local, 'transaction_connection', None)
conn = getattr(self._local, "transaction_connection", None)
if conn is not None:
yield conn
return
@@ -204,8 +213,8 @@ class DatabaseConnectionManager:
"ConnectionManager",
"get_connection",
None,
ErrorSeverity.HIGH
)
ErrorSeverity.HIGH,
),
)
# Update connection info
@@ -229,32 +238,31 @@ class DatabaseConnectionManager:
raise DatabaseError(
error,
context=ErrorContext(
"ConnectionManager",
"get_connection",
None,
ErrorSeverity.HIGH
)
"ConnectionManager", "get_connection", None, ErrorSeverity.HIGH
),
)
finally:
if conn and not hasattr(self._local, 'transaction_connection'):
if conn and not hasattr(self._local, "transaction_connection"):
try:
conn.rollback() # Reset connection state
self._connection_pool.put(conn)
# Update connection info
if id(conn) in self._connection_info:
conn_info = self._connection_info[id(conn)]
conn_info.state = ConnectionState.AVAILABLE
duration = (datetime.utcnow() - start_time).total_seconds()
conn_info.total_transaction_time += duration
except Exception as e:
logger.error(f"Error returning connection to pool: {e}")
try:
conn.close()
if id(conn) in self._connection_info:
self._connection_info[id(conn)].state = ConnectionState.CLOSED
self._connection_info[id(conn)].state = (
ConnectionState.CLOSED
)
except Exception:
pass
@@ -262,22 +270,19 @@ class DatabaseConnectionManager:
def transaction(self) -> Generator[sqlite3.Connection, None, None]:
"""
Start a database transaction.
Yields:
Database connection for the transaction
Raises:
DatabaseError: If unable to start transaction
"""
if hasattr(self._local, 'transaction_connection'):
if hasattr(self._local, "transaction_connection"):
raise DatabaseError(
"Nested transactions are not supported",
context=ErrorContext(
"ConnectionManager",
"transaction",
None,
ErrorSeverity.HIGH
)
"ConnectionManager", "transaction", None, ErrorSeverity.HIGH
),
)
conn = None
@@ -293,11 +298,8 @@ class DatabaseConnectionManager:
raise DatabaseError(
"Failed to create database connection",
context=ErrorContext(
"ConnectionManager",
"transaction",
None,
ErrorSeverity.HIGH
)
"ConnectionManager", "transaction", None, ErrorSeverity.HIGH
),
)
# Update connection info
@@ -330,42 +332,41 @@ class DatabaseConnectionManager:
raise DatabaseError(
error,
context=ErrorContext(
"ConnectionManager",
"transaction",
None,
ErrorSeverity.HIGH
)
"ConnectionManager", "transaction", None, ErrorSeverity.HIGH
),
)
finally:
if conn:
try:
# Remove thread-local binding
delattr(self._local, 'transaction_connection')
delattr(self._local, "transaction_connection")
# Return connection to pool
self._connection_pool.put(conn)
# Update connection info
if id(conn) in self._connection_info:
conn_info = self._connection_info[id(conn)]
conn_info.state = ConnectionState.AVAILABLE
duration = (datetime.utcnow() - start_time).total_seconds()
conn_info.total_transaction_time += duration
except Exception as e:
logger.error(f"Error cleaning up transaction: {e}")
try:
conn.close()
if id(conn) in self._connection_info:
self._connection_info[id(conn)].state = ConnectionState.CLOSED
self._connection_info[id(conn)].state = (
ConnectionState.CLOSED
)
except Exception:
pass
def close_all(self) -> None:
"""
Close all connections in the pool.
Raises:
DatabaseError: If cleanup fails
"""
@@ -377,7 +378,9 @@ class DatabaseConnectionManager:
try:
conn.close()
if id(conn) in self._connection_info:
self._connection_info[id(conn)].state = ConnectionState.CLOSED
self._connection_info[id(conn)].state = (
ConnectionState.CLOSED
)
except Exception as e:
logger.error(f"Error closing connection: {e}")
except Empty:
@@ -388,79 +391,74 @@ class DatabaseConnectionManager:
raise DatabaseError(
error,
context=ErrorContext(
"ConnectionManager",
"close_all",
None,
ErrorSeverity.HIGH
)
"ConnectionManager", "close_all", None, ErrorSeverity.HIGH
),
)
def get_status(self) -> ConnectionStatus:
"""
Get current connection manager status.
Returns:
Connection status information
"""
active_connections = sum(
1 for info in self._connection_info.values()
1
for info in self._connection_info.values()
if info.state == ConnectionState.IN_USE
)
return ConnectionStatus(
state="healthy" if active_connections < self.pool_size else "exhausted",
created_at=min(
info.created_at.isoformat()
for info in self._connection_info.values()
info.created_at.isoformat() for info in self._connection_info.values()
),
last_used=max(
info.last_used.isoformat()
for info in self._connection_info.values()
info.last_used.isoformat() for info in self._connection_info.values()
),
error=None,
transaction_count=sum(
info.transaction_count
for info in self._connection_info.values()
info.transaction_count for info in self._connection_info.values()
),
pool_size=self.pool_size,
available_connections=self.pool_size - active_connections
available_connections=self.pool_size - active_connections,
)
def get_metrics(self) -> ConnectionMetrics:
"""
Get connection metrics.
Returns:
Connection metrics information
"""
total_transactions = sum(
info.transaction_count
for info in self._connection_info.values()
)
total_errors = sum(
info.error_count
for info in self._connection_info.values()
info.transaction_count for info in self._connection_info.values()
)
total_errors = sum(info.error_count for info in self._connection_info.values())
total_time = sum(
info.total_transaction_time
for info in self._connection_info.values()
info.total_transaction_time for info in self._connection_info.values()
)
return ConnectionMetrics(
total_connections=len(self._connection_info),
active_connections=sum(
1 for info in self._connection_info.values()
1
for info in self._connection_info.values()
if info.state == ConnectionState.IN_USE
),
idle_connections=sum(
1 for info in self._connection_info.values()
1
for info in self._connection_info.values()
if info.state == ConnectionState.AVAILABLE
),
failed_connections=sum(
1 for info in self._connection_info.values()
1
for info in self._connection_info.values()
if info.state == ConnectionState.ERROR
),
total_transactions=total_transactions,
failed_transactions=total_errors,
average_transaction_time=total_time / total_transactions if total_transactions > 0 else 0.0
average_transaction_time=(
total_time / total_transactions if total_transactions > 0 else 0.0
),
)

13
videoarchiver/database/schema_manager.py
View File

@@ -7,12 +7,13 @@ from typing import List, Dict, Any, Optional, TypedDict, ClassVar, Union
from enum import Enum, auto
from datetime import datetime
#try:
# Try relative imports first
from ..utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
# try:
# Try relative imports first
from utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
# except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import DatabaseError, ErrorContext, ErrorSeverity
logger = logging.getLogger("DBSchemaManager")

6
videoarchiver/database/video_archive_db.py
View File

@@ -6,9 +6,9 @@ from typing import Optional, Dict, Any, List
# try:
# Try relative imports first
from .schema_manager import DatabaseSchemaManager
from .query_manager import DatabaseQueryManager
from .connection_manager import DatabaseConnectionManager
from schema_manager import DatabaseSchemaManager
from query_manager import DatabaseQueryManager
from connection_manager import DatabaseConnectionManager
# except ImportError:
# Fall back to absolute imports if relative imports fail

12
videoarchiver/ffmpeg/__init__.py
View File

@@ -20,12 +20,12 @@ logger = logging.getLogger("VideoArchiver")
# Import components after logging is configured
#try:
# Try relative imports first
from .ffmpeg_manager import FFmpegManager
from .video_analyzer import VideoAnalyzer
from .gpu_detector import GPUDetector
from .encoder_params import EncoderParams
from .ffmpeg_downloader import FFmpegDownloader
from .exceptions import (
from ffmpeg_manager import FFmpegManager
from video_analyzer import VideoAnalyzer
from gpu_detector import GPUDetector
from encoder_params import EncoderParams
from ffmpeg_downloader import FFmpegDownloader
from exceptions import (
FFmpegError,
DownloadError,
VerificationError,

BIN
videoarchiver/ffmpeg/__pycache__/__init__.cpython-312.pyc
View File

Binary file not shown.

BIN
videoarchiver/ffmpeg/__pycache__/exceptions.cpython-312.pyc
View File

Binary file not shown.

BIN
videoarchiver/ffmpeg/__pycache__/ffmpeg_manager.cpython-312.pyc
View File

Binary file not shown.

BIN
videoarchiver/ffmpeg/__pycache__/gpu_detector.cpython-312.pyc
View File

Binary file not shown.

BIN
videoarchiver/ffmpeg/__pycache__/video_analyzer.cpython-312.pyc
View File

Binary file not shown.

6
videoarchiver/ffmpeg/binary_manager.py
View File

@@ -7,15 +7,15 @@ from typing import Dict, Optional
#try:
# Try relative imports first
from .exceptions import (
from exceptions import (
FFmpegError,
DownloadError,
VerificationError,
PermissionError,
FFmpegNotFoundError
)
from .ffmpeg_downloader import FFmpegDownloader
from .verification_manager import VerificationManager
from ffmpeg_downloader import FFmpegDownloader
from verification_manager import VerificationManager
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.ffmpeg.exceptions import (

2
videoarchiver/ffmpeg/encoder_params.py
View File

@@ -6,7 +6,7 @@ from typing import Dict, Any
#try:
# Try relative imports first
from .exceptions import CompressionError, QualityError, BitrateError
from exceptions import CompressionError, QualityError, BitrateError
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.ffmpeg.exceptions import CompressionError, QualityError, BitrateError

2
videoarchiver/ffmpeg/ffmpeg_downloader.py
View File

@@ -18,7 +18,7 @@ import lzma
# try:
# Try relative imports first
from .exceptions import DownloadError
from exceptions import DownloadError
# except ImportError:
# Fall back to absolute imports if relative imports fail

14
videoarchiver/ffmpeg/ffmpeg_manager.py
View File

@@ -8,13 +8,13 @@ from typing import Dict, Any, Optional
# try:
# Try relative imports first
from .exceptions import FFmpegError, AnalysisError, FFmpegNotFoundError
from .gpu_detector import GPUDetector
from .video_analyzer import VideoAnalyzer
from .encoder_params import EncoderParams
from .process_manager import ProcessManager
from .verification_manager import VerificationManager
from .binary_manager import BinaryManager
from exceptions import FFmpegError, AnalysisError, FFmpegNotFoundError
from gpu_detector import GPUDetector
from video_analyzer import VideoAnalyzer
from encoder_params import EncoderParams
from process_manager import ProcessManager
from verification_manager import VerificationManager
from binary_manager import BinaryManager
# except ImportError:
# Fall back to absolute imports if relative imports fail

11
videoarchiver/info.json
View File

@@ -23,10 +23,15 @@
"download",
"automation",
"queue",
"ffmpeg"
"ffmpeg",
"discord"
],
"min_bot_version": "3.5.0",
"min_python_version": [3, 8, 0],
"min_python_version": [
3,
8,
0
],
"hidden": false,
"disabled": false,
"type": "COG",
@@ -52,4 +57,4 @@
"read_message_history",
"add_reactions"
]
}
}

20
videoarchiver/processor/__init__.py
View File

@@ -1,10 +1,10 @@
"""Video processing module for VideoArchiver"""
from typing import Dict, Any, Optional, Union, List, Tuple
import discord # type: ignore
import discord # type: ignore
# Import constants first since they have no dependencies
from .constants import (
from constants import (
REACTIONS,
ReactionType,
ReactionEmojis,
@@ -14,10 +14,10 @@ from .constants import (
)
# Import core components
from .core import VideoProcessor
from core import VideoProcessor
# Import URL related components
from .url_extractor import (
from url_extractor import (
URLExtractor,
URLMetadata,
URLPattern,
@@ -28,7 +28,7 @@ from .url_extractor import (
)
# Import validation components
from .message_validator import (
from message_validator import (
MessageValidator,
ValidationContext,
ValidationRule,
@@ -41,7 +41,7 @@ from .message_validator import (
)
# Import reaction handlers
from .reactions import (
from reactions import (
handle_archived_reaction,
update_queue_position_reaction,
update_progress_reaction,
@@ -49,12 +49,12 @@ from .reactions import (
)
# Import progress tracking
from ..utils.progress_tracker import ProgressTracker
from utils.progress_tracker import ProgressTracker
# Import handlers after other dependencies are loaded
from .message_handler import MessageHandler
from .queue_handler import QueueHandler
from .queue_processor import QueueProcessor
from message_handler import MessageHandler
from queue_handler import QueueHandler
from queue_processor import QueueProcessor
# Export public classes and constants
__all__ = [

6
videoarchiver/processor/cleanup_manager.py
View File

@@ -20,12 +20,12 @@ from typing import (
from datetime import datetime, timedelta
if TYPE_CHECKING:
from .queue_handler import QueueHandler
from queue_handler import QueueHandler
# try:
# Try relative imports first
from ..ffmpeg.ffmpeg_manager import FFmpegManager
from ..utils.exceptions import CleanupError
from ffmpeg.ffmpeg_manager import FFmpegManager
from utils.exceptions import CleanupError
# except ImportError:
# Fall back to absolute imports if relative imports fail
# # from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager

8
videoarchiver/processor/core.py
View File

@@ -8,7 +8,7 @@ from typing import Any, ClassVar, Dict, List, Optional, Tuple, TYPE_CHECKING
import discord # type: ignore
from discord.ext import commands # type: ignore
from ..core.types import (
from core.c_types import (
ComponentState,
ProcessorState,
ComponentStatus,
@@ -16,9 +16,9 @@ from ..core.types import (
IConfigManager,
IQueueManager,
)
from .constants import REACTIONS
from ..utils.progress_tracker import ProgressTracker
from ..utils.exceptions import ProcessorError
from constants import REACTIONS
from utils.progress_tracker import ProgressTracker
from utils.exceptions import ProcessorError
logger = logging.getLogger("VideoArchiver")

12
videoarchiver/processor/message_handler.py
View File

@@ -21,12 +21,12 @@ from discord.ext import commands # type: ignore
# try:
# Try relative imports first
from ..config_manager import ConfigManager
from .constants import REACTIONS
from .message_validator import MessageValidator, ValidationError
from .url_extractor import URLExtractor, URLMetadata
from ..queue.types import QueuePriority
from ..utils.exceptions import MessageHandlerError
from config_manager import ConfigManager
from constants import REACTIONS
from message_validator import MessageValidator, ValidationError
from url_extractor import URLExtractor, URLMetadata
from queue.q_types import QueuePriority
from utils.exceptions import MessageHandlerError
# except ImportError:
# Fall back to absolute imports if relative imports fail

2
videoarchiver/processor/message_validator.py
View File

@@ -9,7 +9,7 @@ import discord # type: ignore
#try:
# Try relative imports first
from ..utils.exceptions import ValidationError
from utils.exceptions import ValidationError
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import ValidationError

31
videoarchiver/processor/queue_handler.py
View File

@@ -10,15 +10,16 @@ import discord # type: ignore
# try:
# Try relative imports first
from .. import utils
from ..database.video_archive_db import VideoArchiveDB
from ..utils.download_manager import DownloadManager
from ..utils.message_manager import MessageManager
from ..utils.exceptions import QueueHandlerError
from ..queue.models import QueueItem
from ..config_manager import ConfigManager
from . import progress_tracker # Import from processor package
from .constants import REACTIONS
# from . import utils
from database.video_archive_db import VideoArchiveDB
from utils.download_manager import DownloadManager
from utils.message_manager import MessageManager
from utils.exceptions import QueueHandlerError
from queue.models import QueueItem
from config_manager import ConfigManager
from utils.progress_tracker import ProgressTracker # Import from processor package
from constants import REACTIONS
# except ImportError:
# Fall back to absolute imports if relative imports fail
@@ -66,11 +67,13 @@ class QueueHandler:
self,
bot: discord.Client,
config_manager: ConfigManager,
progress_tracker: ProgressTracker,
components: Dict[int, Dict[str, Any]],
db: Optional[VideoArchiveDB] = None,
) -> None:
self.bot = bot
self.config_manager = config_manager
self.progress_tracker = progress_tracker
self.components = components
self.db = db
self._unloading = False
@@ -392,11 +395,13 @@ class QueueHandler:
return
# Update progress tracking
progress_tracker.update_download_progress(
ProgressTracker.update_download_progress(
url,
{
"percent": progress,
"last_update": datetime.utcnow().isoformat(),
"last_update": datetime.now(
datetime.timezone.utc
)().isoformat(),
},
)
@@ -439,9 +444,9 @@ class QueueHandler:
download_task, timeout=self.DOWNLOAD_TIMEOUT
)
if success:
progress_tracker.complete_download(url)
ProgressTracker.complete_download(url)
else:
progress_tracker.increment_download_retries(url)
ProgressTracker.increment_download_retries(url)
return success, file_path, error
except asyncio.TimeoutError:

151
videoarchiver/processor/queue_processor.py
View File

@@ -1,97 +1,110 @@
"""Queue processing functionality for video processing"""
import logging
import asyncio
from typing import List, Optional, Dict, Any, Set, ClassVar
from datetime import datetime
import sys
from pathlib import Path
#try:
# Try relative imports first
from ..queue.types import QueuePriority, QueueMetrics, ProcessingMetrics
from ..queue.models import QueueItem
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.queue.types import QueuePriority, QueueMetrics, ProcessingMetrics
# from videoarchiver.queue.models import QueueItem
# Get the parent directory (videoarchiver root)
BASE_DIR = Path(__file__).resolve().parent.parent
if str(BASE_DIR) not in sys.path:
sys.path.insert(0, str(BASE_DIR))
# Use non-relative imports
from queue.q_types import QueuePriority, QueueMetrics, ProcessingMetrics
from queue.models import QueueItem
logger = logging.getLogger("VideoArchiver")
class QueueProcessor:
"""Handles processing of video queue items"""
"""
Handles the processing of queue items with priority-based scheduling.
"""
_active_items: ClassVar[Set[int]] = set()
_processing_lock: ClassVar[asyncio.Lock] = asyncio.Lock()
# Class variables for tracking global state
active_items: ClassVar[Set[str]] = set()
processing_metrics: ClassVar[Dict[str, ProcessingMetrics]] = {}
def __init__(self, queue_manager):
"""Initialize queue processor
Args:
queue_manager: Queue manager instance to handle queue operations
def __init__(self):
self.queue_metrics = QueueMetrics()
self.processing_lock = asyncio.Lock()
self.is_running = False
self._current_item: Optional[QueueItem] = None
self._priority_queues: Dict[QueuePriority, List[QueueItem]] = {
priority: [] for priority in QueuePriority
}
@property
def current_item(self) -> Optional[QueueItem]:
"""Get the currently processing item."""
return self._current_item
def add_item(self, item: QueueItem) -> bool:
"""
self.queue_manager = queue_manager
self._metrics = ProcessingMetrics()
async def process_urls(self, message, urls, priority: QueuePriority = QueuePriority.NORMAL) -> None:
"""Process URLs from a message
Args:
message: Discord message containing URLs
urls: List of URLs to process
priority: Processing priority level
"""
for url_metadata in urls:
await self.queue_manager.add_to_queue(
url=url_metadata.url,
message_id=message.id,
channel_id=message.channel.id,
guild_id=message.guild.id,
author_id=message.author.id,
priority=priority.value
)
async def process_item(self, item: QueueItem) -> bool:
"""Process a single queue item
Add an item to the appropriate priority queue.
Args:
item: Queue item to process
item: QueueItem to add
Returns:
bool: Success status
bool: True if item was added successfully
"""
if item.id in self._active_items:
logger.warning(f"Item {item.id} is already being processed")
if item.id in self.active_items:
logger.warning(f"Item {item.id} is already in queue")
return False
try:
self._active_items.add(item.id)
start_time = datetime.now()
self._priority_queues[item.priority].append(item)
self.active_items.add(item.id)
self.queue_metrics.total_items += 1
logger.info(f"Added item {item.id} to {item.priority.name} priority queue")
return True
# Process item logic here
# Placeholder for actual video processing
await asyncio.sleep(1)
def remove_item(self, item_id: str) -> Optional[QueueItem]:
"""
Remove an item from any priority queue.
processing_time = (datetime.now() - start_time).total_seconds()
self._update_metrics(processing_time, True, item.size)
return True
Args:
item_id: ID of item to remove
except Exception as e:
logger.error(f"Error processing item {item.id}: {str(e)}")
self._update_metrics(0, False, 0)
return False
finally:
self._active_items.remove(item.id)
Returns:
Optional[QueueItem]: Removed item if found, None otherwise
"""
for priority in QueuePriority:
queue = self._priority_queues[priority]
for item in queue:
if item.id == item_id:
queue.remove(item)
self.active_items.discard(item_id)
self.queue_metrics.total_items -= 1
logger.info(
f"Removed item {item_id} from {priority.name} priority queue"
)
return item
return None
def _update_metrics(self, processing_time: float, success: bool, size: int) -> None:
"""Update processing metrics"""
"""
Update processing metrics.
Args:
processing_time: Time taken to process the item
success: Whether processing was successful
size: Size of the processed item
"""
if success:
self._metrics.record_success(processing_time)
self.queue_metrics.record_success(processing_time)
else:
self._metrics.record_failure("Processing error")
self.queue_metrics.record_failure("Processing error")
def get_metrics(self) -> QueueMetrics:
"""Get current processing metrics"""
total = self._metrics.total_processed
"""
Get current processing metrics.
Returns:
QueueMetrics: Current queue processing metrics
"""
total = self.queue_metrics.total_processed
if total == 0:
return QueueMetrics(
total_items=0,
@@ -103,8 +116,8 @@ class QueueProcessor:
return QueueMetrics(
total_items=total,
processing_time=self._metrics.avg_processing_time,
success_rate=self._metrics.successful / total,
error_rate=self._metrics.failed / total,
processing_time=self.queue_metrics.avg_processing_time,
success_rate=self.queue_metrics.successful / total,
error_rate=self.queue_metrics.failed / total,
average_size=0, # This would need to be tracked separately if needed
)

4
videoarchiver/processor/reactions.py
View File

@@ -9,13 +9,13 @@ from urllib.parse import urlparse
# try:
# Try relative imports first
from ..processor.constants import (
from processor.constants import (
REACTIONS,
ReactionType,
get_reaction,
get_progress_emoji,
)
from ..database.video_archive_db import VideoArchiveDB
from database.video_archive_db import VideoArchiveDB
# except ImportError:
# Fall back to absolute imports if relative imports fail

2
videoarchiver/processor/status_display.py
View File

@@ -20,7 +20,7 @@ import discord # type: ignore
# try:
# Try relative imports first
from ..utils.exceptions import DisplayError
from utils.exceptions import DisplayError
# except ImportError:
# Fall back to absolute imports if relative imports fail

86
videoarchiver/queue/__init__.py
View File

@@ -1,58 +1,58 @@
"""Queue management package for video processing"""
from .models import QueueItem, QueueMetrics
from .types import QueuePriority, ProcessingMetrics
from .manager import EnhancedVideoQueueManager
from .persistence import QueuePersistenceManager, QueueError
from .monitoring import QueueMonitor, MonitoringError
from .cleanup import QueueCleaner, CleanupError
from .recovery_manager import RecoveryManager
from .state_manager import StateManager
from .metrics_manager import MetricsManager
from .processor import QueueProcessor
from .health_checker import HealthChecker
from models import QueueItem, QueueMetrics
from q_types import QueuePriority, ProcessingMetrics
from manager import EnhancedVideoQueueManager
from persistence import QueuePersistenceManager, QueueError
from monitoring import QueueMonitor, MonitoringError
from cleanup import QueueCleaner, CleanupError
from recovery_manager import RecoveryManager
from state_manager import QueueStateManager
from metrics_manager import QueueMetricsManager
from processor import QueueProcessor
from health_checker import HealthChecker
# Importing from cleaners subdirectory
from .cleaners import GuildCleaner, HistoryCleaner, TrackingCleaner
from cleaners import GuildCleaner, HistoryCleaner, TrackingCleaner
# Corrected relative imports from utils
from ..utils.compression_handler import CompressionHandler
from ..utils.directory_manager import DirectoryManager
from ..utils.download_manager import DownloadManager
from ..utils.file_operations import FileOperations
from ..utils.progress_tracker import ProgressTracker
from ..utils.url_validator import UrlValidator
from utils.compression_handler import CompressionHandler
from utils.directory_manager import DirectoryManager
from utils.download_manager import DownloadManager
from utils.file_operations import FileOperations
from utils.progress_tracker import ProgressTracker
from processor.url_extractor import URLValidator
__all__ = [
# Queue models and types
'QueueItem',
'QueueMetrics',
'QueuePriority',
'ProcessingMetrics',
"QueueItem",
"QueueMetrics",
"QueuePriority",
"ProcessingMetrics",
# Core components
'EnhancedVideoQueueManager',
'QueuePersistenceManager',
'QueueMonitor',
'QueueCleaner',
'QueueProcessor',
'HealthChecker',
"EnhancedVideoQueueManager",
"QueuePersistenceManager",
"QueueMonitor",
"QueueCleaner",
"QueueProcessor",
"HealthChecker",
# Managers
'RecoveryManager',
'StateManager',
'MetricsManager',
"RecoveryManager",
"QueueStateManager",
"QueueMetricsManager",
# Cleaners
'GuildCleaner',
'HistoryCleaner',
'TrackingCleaner',
"GuildCleaner",
"HistoryCleaner",
"TrackingCleaner",
# Utility handlers
'CompressionHandler',
'DirectoryManager',
'DownloadManager',
'FileOperations',
'ProgressTracker',
'UrlValidator',
"CompressionHandler",
"DirectoryManager",
"DownloadManager",
"FileOperations",
"ProgressTracker",
"URLValidator",
# Errors
'QueueError',
'MonitoringError',
'CleanupError',
"QueueError",
"MonitoringError",
"CleanupError",
]

6
videoarchiver/queue/cleaners/__init__.py
View File

@@ -1,8 +1,8 @@
"""Queue cleaning functionality"""
from .guild_cleaner import GuildCleaner
from .history_cleaner import HistoryCleaner
from .tracking_cleaner import TrackingCleaner
from guild_cleaner import GuildCleaner
from history_cleaner import HistoryCleaner
from tracking_cleaner import TrackingCleaner
__all__ = [
'GuildCleaner',

2
videoarchiver/queue/cleaners/guild_cleaner.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass, field
from typing import Dict, List, Set, Tuple, Any, Optional
from datetime import datetime
from ..models import QueueItem
from models import QueueItem
logger = logging.getLogger("GuildCleaner")

2
videoarchiver/queue/cleaners/history_cleaner.py
View File

@@ -6,7 +6,7 @@ from dataclasses import dataclass, field
from typing import Dict, Optional, List, Any, Set
from datetime import datetime, timedelta
from ..models import QueueItem
from models import QueueItem
logger = logging.getLogger("HistoryCleaner")

170
videoarchiver/queue/cleaners/tracking_cleaner.py
View File

@@ -7,34 +7,42 @@ from dataclasses import dataclass, field
from typing import Dict, List, Set, Tuple, Any, Optional
from datetime import datetime
from ..models import QueueItem
from models import QueueItem
logger = logging.getLogger("TrackingCleaner")
class TrackingCleanupStrategy(Enum):
"""Tracking cleanup strategies"""
AGGRESSIVE = "aggressive" # Remove all invalid entries
AGGRESSIVE = "aggressive" # Remove all invalid entries
CONSERVATIVE = "conservative" # Keep recent invalid entries
BALANCED = "balanced" # Balance between cleanup and retention
BALANCED = "balanced" # Balance between cleanup and retention
class TrackingType(Enum):
"""Types of tracking data"""
GUILD = "guild"
CHANNEL = "channel"
URL = "url"
@dataclass
class TrackingCleanupConfig:
"""Configuration for tracking cleanup"""
batch_size: int = 100
retention_period: int = 3600 # 1 hour
validate_urls: bool = True
cleanup_empty: bool = True
max_invalid_ratio: float = 0.5 # 50% invalid threshold
@dataclass
class TrackingCleanupResult:
"""Result of a tracking cleanup operation"""
timestamp: datetime
strategy: TrackingCleanupStrategy
items_cleaned: int
@@ -45,6 +53,7 @@ class TrackingCleanupResult:
final_counts: Dict[str, int]
error: Optional[str] = None
class TrackingValidator:
"""Validates tracking data"""
@@ -64,6 +73,7 @@ class TrackingValidator:
except Exception:
return False
class TrackingCleanupTracker:
"""Tracks cleanup operations"""
@@ -94,9 +104,7 @@ class TrackingCleanupTracker:
"total_guilds_cleaned": self.total_guilds_cleaned,
"total_channels_cleaned": self.total_channels_cleaned,
"last_cleanup": (
self.last_cleanup.isoformat()
if self.last_cleanup
else None
self.last_cleanup.isoformat() if self.last_cleanup else None
),
"recent_cleanups": [
{
@@ -105,19 +113,20 @@ class TrackingCleanupTracker:
"items_cleaned": r.items_cleaned,
"guilds_cleaned": r.guilds_cleaned,
"channels_cleaned": r.channels_cleaned,
"duration": r.duration
"duration": r.duration,
}
for r in self.history[-5:] # Last 5 cleanups
]
],
}
class TrackingCleaner:
"""Handles cleanup of queue tracking data"""
def __init__(
self,
strategy: TrackingCleanupStrategy = TrackingCleanupStrategy.BALANCED,
config: Optional[TrackingCleanupConfig] = None
config: Optional[TrackingCleanupConfig] = None,
):
self.strategy = strategy
self.config = config or TrackingCleanupConfig()
@@ -129,17 +138,14 @@ class TrackingCleaner:
guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]],
queue: List[QueueItem],
processing: Dict[str, QueueItem]
processing: Dict[str, QueueItem],
) -> Tuple[int, Dict[str, int]]:
"""Clean up tracking data"""
start_time = datetime.utcnow()
try:
# Get initial counts
initial_counts = self._get_tracking_counts(
guild_queues,
channel_queues
)
initial_counts = self._get_tracking_counts(guild_queues, channel_queues)
# Get valid URLs
valid_urls = self._get_valid_urls(queue, processing)
@@ -151,21 +157,15 @@ class TrackingCleaner:
if self.strategy == TrackingCleanupStrategy.AGGRESSIVE:
cleaned = await self._aggressive_cleanup(
guild_queues,
channel_queues,
valid_urls
guild_queues, channel_queues, valid_urls
)
elif self.strategy == TrackingCleanupStrategy.CONSERVATIVE:
cleaned = await self._conservative_cleanup(
guild_queues,
channel_queues,
valid_urls
guild_queues, channel_queues, valid_urls
)
else: # BALANCED
cleaned = await self._balanced_cleanup(
guild_queues,
channel_queues,
valid_urls
guild_queues, channel_queues, valid_urls
)
items_cleaned = cleaned[0]
@@ -173,10 +173,7 @@ class TrackingCleaner:
channels_cleaned = cleaned[2]
# Get final counts
final_counts = self._get_tracking_counts(
guild_queues,
channel_queues
)
final_counts = self._get_tracking_counts(guild_queues, channel_queues)
# Record cleanup result
duration = (datetime.utcnow() - start_time).total_seconds()
@@ -188,37 +185,39 @@ class TrackingCleaner:
channels_cleaned=channels_cleaned,
duration=duration,
initial_counts=initial_counts,
final_counts=final_counts
final_counts=final_counts,
)
self.tracker.record_cleanup(result)
logger.info(self.format_tracking_cleanup_report(
initial_counts,
final_counts,
duration
))
logger.info(
self.format_tracking_cleanup_report(
initial_counts, final_counts, duration
)
)
return items_cleaned, initial_counts
except Exception as e:
logger.error(f"Error cleaning tracking data: {e}")
self.tracker.record_cleanup(TrackingCleanupResult(
timestamp=datetime.utcnow(),
strategy=self.strategy,
items_cleaned=0,
guilds_cleaned=0,
channels_cleaned=0,
duration=0,
initial_counts={},
final_counts={},
error=str(e)
))
self.tracker.record_cleanup(
TrackingCleanupResult(
timestamp=datetime.utcnow(),
strategy=self.strategy,
items_cleaned=0,
guilds_cleaned=0,
channels_cleaned=0,
duration=0,
initial_counts={},
final_counts={},
error=str(e),
)
)
raise
async def _aggressive_cleanup(
self,
guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]],
valid_urls: Set[str]
valid_urls: Set[str],
) -> Tuple[int, int, int]:
"""Perform aggressive cleanup"""
items_cleaned = 0
@@ -227,18 +226,14 @@ class TrackingCleaner:
# Clean guild tracking
guild_cleaned = await self._cleanup_guild_tracking(
guild_queues,
valid_urls,
validate_all=True
guild_queues, valid_urls, validate_all=True
)
items_cleaned += guild_cleaned[0]
guilds_cleaned += guild_cleaned[1]
# Clean channel tracking
channel_cleaned = await self._cleanup_channel_tracking(
channel_queues,
valid_urls,
validate_all=True
channel_queues, valid_urls, validate_all=True
)
items_cleaned += channel_cleaned[0]
channels_cleaned += channel_cleaned[1]
@@ -249,7 +244,7 @@ class TrackingCleaner:
self,
guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]],
valid_urls: Set[str]
valid_urls: Set[str],
) -> Tuple[int, int, int]:
"""Perform conservative cleanup"""
items_cleaned = 0
@@ -261,9 +256,7 @@ class TrackingCleaner:
invalid_ratio = len(urls - valid_urls) / len(urls) if urls else 0
if invalid_ratio > self.config.max_invalid_ratio:
cleaned = await self._cleanup_guild_tracking(
{guild_id: urls},
valid_urls,
validate_all=False
{guild_id: urls}, valid_urls, validate_all=False
)
items_cleaned += cleaned[0]
guilds_cleaned += cleaned[1]
@@ -272,9 +265,7 @@ class TrackingCleaner:
invalid_ratio = len(urls - valid_urls) / len(urls) if urls else 0
if invalid_ratio > self.config.max_invalid_ratio:
cleaned = await self._cleanup_channel_tracking(
{channel_id: urls},
valid_urls,
validate_all=False
{channel_id: urls}, valid_urls, validate_all=False
)
items_cleaned += cleaned[0]
channels_cleaned += cleaned[1]
@@ -285,7 +276,7 @@ class TrackingCleaner:
self,
guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]],
valid_urls: Set[str]
valid_urls: Set[str],
) -> Tuple[int, int, int]:
"""Perform balanced cleanup"""
items_cleaned = 0
@@ -294,18 +285,14 @@ class TrackingCleaner:
# Clean guild tracking with validation
guild_cleaned = await self._cleanup_guild_tracking(
guild_queues,
valid_urls,
validate_all=self.config.validate_urls
guild_queues, valid_urls, validate_all=self.config.validate_urls
)
items_cleaned += guild_cleaned[0]
guilds_cleaned += guild_cleaned[1]
# Clean channel tracking with validation
channel_cleaned = await self._cleanup_channel_tracking(
channel_queues,
valid_urls,
validate_all=self.config.validate_urls
channel_queues, valid_urls, validate_all=self.config.validate_urls
)
items_cleaned += channel_cleaned[0]
channels_cleaned += channel_cleaned[1]
@@ -316,7 +303,7 @@ class TrackingCleaner:
self,
guild_queues: Dict[int, Set[str]],
valid_urls: Set[str],
validate_all: bool
validate_all: bool,
) -> Tuple[int, int]:
"""Clean up guild tracking data"""
items_cleaned = 0
@@ -331,14 +318,15 @@ class TrackingCleaner:
original_size = len(guild_queues[guild_id])
guild_queues[guild_id] = {
url for url in guild_queues[guild_id]
url
for url in guild_queues[guild_id]
if (
(not validate_all or self.validator.validate_url(url)) and
url in valid_urls
(not validate_all or self.validator.validate_url(url))
and url in valid_urls
)
}
items_cleaned += original_size - len(guild_queues[guild_id])
if self.config.cleanup_empty and not guild_queues[guild_id]:
guild_queues.pop(guild_id)
guilds_cleaned += 1
@@ -355,7 +343,7 @@ class TrackingCleaner:
self,
channel_queues: Dict[int, Set[str]],
valid_urls: Set[str],
validate_all: bool
validate_all: bool,
) -> Tuple[int, int]:
"""Clean up channel tracking data"""
items_cleaned = 0
@@ -370,14 +358,15 @@ class TrackingCleaner:
original_size = len(channel_queues[channel_id])
channel_queues[channel_id] = {
url for url in channel_queues[channel_id]
url
for url in channel_queues[channel_id]
if (
(not validate_all or self.validator.validate_url(url)) and
url in valid_urls
(not validate_all or self.validator.validate_url(url))
and url in valid_urls
)
}
items_cleaned += original_size - len(channel_queues[channel_id])
if self.config.cleanup_empty and not channel_queues[channel_id]:
channel_queues.pop(channel_id)
channels_cleaned += 1
@@ -391,9 +380,7 @@ class TrackingCleaner:
return items_cleaned, channels_cleaned
def _get_valid_urls(
self,
queue: List[QueueItem],
processing: Dict[str, QueueItem]
self, queue: List[QueueItem], processing: Dict[str, QueueItem]
) -> Set[str]:
"""Get set of valid URLs"""
valid_urls = {item.url for item in queue}
@@ -401,30 +388,27 @@ class TrackingCleaner:
return valid_urls
def _get_tracking_counts(
self,
guild_queues: Dict[int, Set[str]],
channel_queues: Dict[int, Set[str]]
self, guild_queues: Dict[int, Set[str]], channel_queues: Dict[int, Set[str]]
) -> Dict[str, int]:
"""Get tracking data counts"""
return {
'guilds': len(guild_queues),
'channels': len(channel_queues),
'guild_urls': sum(len(urls) for urls in guild_queues.values()),
'channel_urls': sum(len(urls) for urls in channel_queues.values())
"guilds": len(guild_queues),
"channels": len(channel_queues),
"guild_urls": sum(len(urls) for urls in guild_queues.values()),
"channel_urls": sum(len(urls) for urls in channel_queues.values()),
}
def format_tracking_cleanup_report(
self,
initial_counts: Dict[str, int],
final_counts: Dict[str, int],
duration: float
duration: float,
) -> str:
"""Format a tracking cleanup report"""
total_cleaned = (
(initial_counts['guild_urls'] - final_counts['guild_urls']) +
(initial_counts['channel_urls'] - final_counts['channel_urls'])
total_cleaned = (initial_counts["guild_urls"] - final_counts["guild_urls"]) + (
initial_counts["channel_urls"] - final_counts["channel_urls"]
)
return (
f"Tracking Cleanup Results:\n"
f"Strategy: {self.strategy.value}\n"
@@ -446,7 +430,7 @@ class TrackingCleaner:
"retention_period": self.config.retention_period,
"validate_urls": self.config.validate_urls,
"cleanup_empty": self.config.cleanup_empty,
"max_invalid_ratio": self.config.max_invalid_ratio
"max_invalid_ratio": self.config.max_invalid_ratio,
},
"tracker": self.tracker.get_stats()
"tracker": self.tracker.get_stats(),
}

8
videoarchiver/queue/cleanup.py
View File

@@ -7,16 +7,16 @@ from dataclasses import dataclass, field
from typing import Dict, List, Set, Optional, Any, Tuple
from datetime import datetime, timedelta
from .models import QueueItem, QueueMetrics
from .cleaners.history_cleaner import (
from models import QueueItem, QueueMetrics
from cleaners.history_cleaner import (
HistoryCleaner,
CleanupStrategy as HistoryStrategy
)
from .cleaners.guild_cleaner import (
from cleaners.guild_cleaner import (
GuildCleaner,
GuildCleanupStrategy
)
from .cleaners.tracking_cleaner import (
from cleaners.tracking_cleaner import (
TrackingCleaner,
TrackingCleanupStrategy
)

26
videoarchiver/queue/manager.py
View File

@@ -7,21 +7,22 @@ from dataclasses import dataclass, field
from typing import Optional, Tuple, Dict, Any, List, Set, Callable
from datetime import datetime, timedelta
from ..core.types import IQueueManager, QueueState, ComponentStatus
from .state_manager import QueueStateManager
from .processor import QueueProcessor
from .metrics_manager import QueueMetricsManager
from .persistence import QueuePersistenceManager
from .monitoring import QueueMonitor, MonitoringLevel
from .cleanup import QueueCleaner
from .models import QueueItem, QueueError, CleanupError
from .types import ProcessingStrategy
from core.c_types import IQueueManager, QueueState, ComponentStatus
from state_manager import QueueStateManager
from processor import QueueProcessor
from metrics_manager import QueueMetricsManager
from persistence import QueuePersistenceManager, QueueError
from monitoring import QueueMonitor, MonitoringLevel
from cleanup import QueueCleaner, CleanupError
from models import QueueItem, QueueError
from q_types import ProcessingStrategy
logger = logging.getLogger("QueueManager")
class QueueMode(Enum):
"""Queue processing modes"""
NORMAL = "normal" # Standard processing
BATCH = "batch" # Batch processing
PRIORITY = "priority" # Priority-based processing
@@ -31,6 +32,7 @@ class QueueMode(Enum):
@dataclass
class QueueConfig:
"""Queue configuration settings"""
max_retries: int = 3
retry_delay: int = 5
max_queue_size: int = 1000
@@ -47,6 +49,7 @@ class QueueConfig:
@dataclass
class QueueStats:
"""Queue statistics"""
start_time: datetime = field(default_factory=datetime.utcnow)
total_processed: int = 0
total_failed: int = 0
@@ -230,7 +233,8 @@ class EnhancedVideoQueueManager(IQueueManager):
"monitoring": monitor_stats,
"state": self.coordinator.state.value,
"mode": self.coordinator.mode.value,
"active": self.coordinator.state == QueueState.RUNNING and bool(processor_stats["active_tasks"]),
"active": self.coordinator.state == QueueState.RUNNING
and bool(processor_stats["active_tasks"]),
"stalled": monitor_stats.get("stalled", False),
"stats": {
"uptime": self.stats.uptime.total_seconds(),
@@ -301,7 +305,7 @@ class EnhancedVideoQueueManager(IQueueManager):
"total_processed": self.stats.total_processed,
"total_failed": self.stats.total_failed,
},
}
},
)
# Helper methods below...

4
videoarchiver/queue/monitoring.py
View File

@@ -8,8 +8,8 @@ from dataclasses import dataclass, field
from typing import Optional, Dict, Any, List, Set
from datetime import datetime, timedelta
from .health_checker import HealthChecker, HealthStatus, HealthCategory
from .recovery_manager import RecoveryManager, RecoveryStrategy
from health_checker import HealthChecker, HealthStatus, HealthCategory
from recovery_manager import RecoveryManager, RecoveryStrategy
logger = logging.getLogger("QueueMonitoring")

2
videoarchiver/queue/persistence.py
View File

@@ -9,7 +9,7 @@ import asyncio
from datetime import datetime, timedelta
from typing import Dict, Any, Optional
from .models import QueueItem, QueueMetrics
from models import QueueItem, QueueMetrics
# Configure logging
logging.basicConfig(

6
videoarchiver/queue/processor.py
View File

@@ -8,9 +8,9 @@ from dataclasses import dataclass
from typing import Callable, Optional, Tuple, List, Set, Dict, Any
from datetime import datetime, timedelta
from .models import QueueItem
from .state_manager import QueueStateManager, ItemState
from .monitoring import QueueMonitor
from models import QueueItem
from state_manager import QueueStateManager, ItemState
from monitoring import QueueMonitor
logger = logging.getLogger("QueueProcessor")

0
videoarchiver/queue/types.py → videoarchiver/queue/q_types.py
View File

2
videoarchiver/queue/recovery_manager.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass, field
from typing import List, Tuple, Dict, Optional, Any, Set
from datetime import datetime, timedelta
from .models import QueueItem
from models import QueueItem
logger = logging.getLogger("QueueRecoveryManager")

2
videoarchiver/queue/state_manager.py
View File

@@ -7,7 +7,7 @@ from dataclasses import dataclass
from typing import Dict, Set, List, Optional, Any
from datetime import datetime
from .models import QueueItem, QueueMetrics
from models import QueueItem, QueueMetrics
logger = logging.getLogger("QueueStateManager")

26
videoarchiver/shared/__init__.py
View File

@@ -1,6 +1,6 @@
"""Shared functionality for the videoarchiver package"""
from .progress import (
from progress import (
compression_progress,
download_progress,
processing_progress,
@@ -16,16 +16,16 @@ from .progress import (
)
__all__ = [
'compression_progress',
'download_progress',
'processing_progress',
'get_compression_progress',
'update_compression_progress',
'clear_compression_progress',
'get_download_progress',
'update_download_progress',
'clear_download_progress',
'get_processing_progress',
'update_processing_progress',
'clear_processing_progress',
"compression_progress",
"download_progress",
"processing_progress",
"get_compression_progress",
"update_compression_progress",
"clear_compression_progress",
"get_download_progress",
"update_download_progress",
"clear_download_progress",
"get_processing_progress",
"update_processing_progress",
"clear_processing_progress",
]

2
videoarchiver/update_checker.py
View File

@@ -18,7 +18,7 @@ import shutil
# try:
# Try relative imports first
from .utils.exceptions import UpdateError
from utils.exceptions import UpdateError
# except ImportError:
# Fall back to absolute imports if relative imports fail

20
videoarchiver/utils/__init__.py
View File

@@ -2,27 +2,27 @@
from typing import Dict, Optional, Any, Union, List
from .file_ops import (
from file_ops import (
cleanup_downloads,
ensure_directory,
get_file_size,
is_valid_path,
safe_delete
)
from .file_deletion import FileDeleter
from .directory_manager import DirectoryManager
from .permission_manager import PermissionManager
from .download_manager import DownloadManager
from .compression_manager import CompressionManager
from .progress_tracker import (
from file_deletion import FileDeleter
from directory_manager import DirectoryManager
from permission_manager import PermissionManager
from download_manager import DownloadManager
from compression_manager import CompressionManager
from progress_tracker import (
ProgressTracker,
ProgressStatus,
DownloadProgress,
CompressionProgress,
CompressionParams
)
from .path_manager import PathManager
from .exceptions import (
from path_manager import PathManager
from exceptions import (
# Base exception
VideoArchiverError,
ErrorSeverity,
@@ -86,7 +86,7 @@ from .exceptions import (
)
# Import progress_tracker from processor
from ..processor import progress_tracker
from processor import progress_tracker
__all__ = [
# File Operations

10
videoarchiver/utils/compression_handler.py
View File

@@ -7,11 +7,11 @@ import subprocess
from datetime import datetime
from typing import Dict, Optional, Callable, Set, Tuple
from ..ffmpeg.ffmpeg_manager import FFmpegManager
from ..ffmpeg.exceptions import CompressionError
from ..utils.exceptions import VideoVerificationError
from ..utils.file_operations import FileOperations
from ..utils.progress_handler import ProgressHandler
from ffmpeg.ffmpeg_manager import FFmpegManager
from ffmpeg.exceptions import CompressionError
from utils.exceptions import VideoVerificationError
from utils.file_operations import FileOperations
from utils.progress_handler import ProgressHandler
logger = logging.getLogger("VideoArchiver")

10
videoarchiver/utils/compression_manager.py
View File

@@ -7,11 +7,11 @@ import subprocess
from datetime import datetime
from typing import Dict, Any, Optional, Callable, List, Set, Tuple
from ..shared.progress import update_compression_progress
from ..utils.compression_handler import CompressionHandler
from ..utils.progress_handler import ProgressHandler
from ..utils.file_operations import FileOperations
from ..utils.exceptions import CompressionError, VideoVerificationError
from shared.progress import update_compression_progress
from utils.compression_handler import CompressionHandler
from utils.progress_handler import ProgressHandler
from utils.file_operations import FileOperations
from utils.exceptions import CompressionError, VideoVerificationError
logger = logging.getLogger("VideoArchiver")

4
videoarchiver/utils/directory_manager.py
View File

@@ -6,8 +6,8 @@ import asyncio
from pathlib import Path
from typing import List, Optional, Tuple
from ..utils.exceptions import FileCleanupError
from ..utils.file_deletion import SecureFileDeleter
from utils.exceptions import FileCleanupError
from utils.file_deletion import SecureFileDeleter
logger = logging.getLogger("DirectoryManager")

12
videoarchiver/utils/download_core.py
View File

@@ -7,12 +7,12 @@ import yt_dlp # type: ignore
from typing import Dict, Optional, Callable, Tuple
from pathlib import Path
from ..utils.url_validator import check_url_support
from ..utils.progress_handler import ProgressHandler, CancellableYTDLLogger
from ..utils.file_operations import FileOperations
from ..utils.compression_handler import CompressionHandler
from ..utils.process_manager import ProcessManager
from ..ffmpeg.ffmpeg_manager import FFmpegManager
from utils.url_validator import check_url_support
from utils.progress_handler import ProgressHandler, CancellableYTDLLogger
from utils.file_operations import FileOperations
from utils.compression_handler import CompressionHandler
from utils.process_manager import ProcessManager
from ffmpeg.ffmpeg_manager import FFmpegManager
logger = logging.getLogger("VideoArchiver")

6
videoarchiver/utils/download_manager.py
View File

@@ -9,9 +9,9 @@ from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional, Tuple, Callable, Any
from pathlib import Path
from ..ffmpeg.verification_manager import VerificationManager
from ..utils.compression_manager import CompressionManager
from ..processor import progress_tracker # Import from processor instead of utils
from ffmpeg.verification_manager import VerificationManager
from utils.compression_manager import CompressionManager
from processor import progress_tracker # Import from processor instead of utils
logger = logging.getLogger("DownloadManager")

2
videoarchiver/utils/file_deletion.py
View File

@@ -7,7 +7,7 @@ import logging
from pathlib import Path
from typing import Optional
from ..utils.exceptions import FileCleanupError
from utils.exceptions import FileCleanupError
logger = logging.getLogger("FileDeleter")

4
videoarchiver/utils/file_operations.py
View File

@@ -9,8 +9,8 @@ import subprocess
from typing import Tuple
from pathlib import Path
from ..utils.exceptions import VideoVerificationError
from ..utils.file_deletion import SecureFileDeleter
from utils.exceptions import VideoVerificationError
from utils.file_deletion import SecureFileDeleter
logger = logging.getLogger("VideoArchiver")

8
videoarchiver/utils/file_ops.py
View File

@@ -4,10 +4,10 @@ import logging
from pathlib import Path
from typing import List, Tuple, Optional
from ..utils.exceptions import FileCleanupError
from ..utils.file_deletion import SecureFileDeleter
from ..utils.directory_manager import DirectoryManager
from ..utils.permission_manager import PermissionManager
from utils.exceptions import FileCleanupError
from utils.file_deletion import SecureFileDeleter
from utils.directory_manager import DirectoryManager
from utils.permission_manager import PermissionManager
logger = logging.getLogger("VideoArchiver")

4
videoarchiver/utils/path_manager.py
View File

@@ -11,8 +11,8 @@ import time
from typing import List, Optional, AsyncGenerator
from pathlib import Path
from ..utils.exceptions import FileCleanupError
from ..utils.permission_manager import PermissionManager
from utils.exceptions import FileCleanupError
from utils.permission_manager import PermissionManager
logger = logging.getLogger("PathManager")

2
videoarchiver/utils/permission_manager.py
View File

@@ -6,7 +6,7 @@ import logging
from pathlib import Path
from typing import Optional, Union, List
from ..utils.exceptions import FileCleanupError
from utils.exceptions import FileCleanupError
logger = logging.getLogger("PermissionManager")