pacnpal/Pac-cogs
1
0
Fork 0
mirror of https://github.com/pacnpal/Pac-cogs.git synced 2025-12-19 18:31:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

loads of import fixes

Browse Source
This commit is contained in:
pacnpal
2024-11-17 19:47:18 +00:00
parent f71e174c0d
commit 97dd6d72f2
49 changed files with 1061 additions and 1062 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
birthday/__init__.py
View File

@@ -1,8 +1,8 @@
"""Birthday cog for Red-DiscordBot""" """Birthday cog for Red-DiscordBot"""
from redbot.core.bot import Red from redbot.core.bot import Red # type: ignore
import logging import logging
import discord import discord # type: ignore
from discord.app_commands.errors import CommandAlreadyRegistered from discord.app_commands.errors import CommandAlreadyRegistered # type: ignore
from .birthday import Birthday, birthday_context_menu from .birthday import Birthday, birthday_context_menu
logger = logging.getLogger("Birthday") logger = logging.getLogger("Birthday")

8
birthday/birthday.py
View File

@@ -1,7 +1,7 @@
import discord import discord # type: ignore
from redbot.core import commands, checks, app_commands from redbot.core import commands, checks, app_commands # type: ignore
from redbot.core.bot import Red from redbot.core.bot import Red # type: ignore
from redbot.core.config import Config from redbot.core.config import Config # type: ignore
from datetime import datetime, time, timedelta from datetime import datetime, time, timedelta
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import random import random

2
overseerr/__init__.py
View File

@@ -1,5 +1,5 @@
"""Overseerr cog for Red-DiscordBot""" """Overseerr cog for Red-DiscordBot"""
from redbot.core.bot import Red from redbot.core.bot import Red # type: ignore
import logging import logging
from .overseerr import Overseerr from .overseerr import Overseerr

6
overseerr/overseerr.py
View File

@@ -1,10 +1,10 @@
import aiohttp import aiohttp
from redbot.core import commands, Config, app_commands from redbot.core import commands, Config, app_commands # type: ignore
from redbot.core.bot import Red from redbot.core.bot import Red # type: ignore
import asyncio import asyncio
import json import json
import urllib.parse import urllib.parse
import discord import discord # type: ignore
class Overseerr(commands.Cog): class Overseerr(commands.Cog):
def __init__(self, bot: Red): def __init__(self, bot: Red):

2
videoarchiver/__init__.py
View File

@@ -5,7 +5,7 @@ import asyncio
import logging import logging
import importlib import importlib
from typing import Optional from typing import Optional
from redbot.core.bot import Red from redbot.core.bot import Red # type: ignore
# Force reload of all modules # Force reload of all modules
modules_to_reload = [ modules_to_reload = [

2
videoarchiver/config/channel_manager.py
View File

@@ -2,7 +2,7 @@
import logging import logging
from typing import Dict, List, Optional, Tuple from typing import Dict, List, Optional, Tuple
import discord import discord # type: ignore
from .exceptions import ( from .exceptions import (
ConfigurationError as ConfigError, ConfigurationError as ConfigError,

2
videoarchiver/config/role_manager.py
View File

@@ -2,7 +2,7 @@
import logging import logging
from typing import Dict, List, Set, Tuple, Optional, Any from typing import Dict, List, Set, Tuple, Optional, Any
import discord import discord # type: ignore
from .exceptions import ConfigurationError as ConfigError from .exceptions import ConfigurationError as ConfigError

4
videoarchiver/config/settings_formatter.py
View File

@@ -3,9 +3,9 @@
import logging import logging
from typing import Dict, Any, List from typing import Dict, Any, List
from datetime import datetime from datetime import datetime
import discord import discord # type: ignore
from .config.exceptions import ConfigurationError as ConfigError from ..config.exceptions import ConfigurationError as ConfigError
logger = logging.getLogger("SettingsFormatter") logger = logging.getLogger("SettingsFormatter")

4
videoarchiver/config_manager.py
View File

@@ -3,8 +3,8 @@
import logging import logging
import asyncio import asyncio
from typing import Dict, Any, Optional, List, Union from typing import Dict, Any, Optional, List, Union
import discord import discord # type: ignore
from redbot.core import Config from redbot.core import Config # type: ignore
from .config.validation_manager import ValidationManager from .config.validation_manager import ValidationManager
from .config.settings_formatter import SettingsFormatter from .config.settings_formatter import SettingsFormatter

2
videoarchiver/core/__init__.py
View File

@@ -1,5 +1,5 @@
"""Core module for VideoArchiver cog""" """Core module for VideoArchiver cog"""
from .core.base import VideoArchiver from ..core.base import VideoArchiver
__all__ = ["VideoArchiver"] __all__ = ["VideoArchiver"]

18
videoarchiver/core/base.py
View File

@@ -8,9 +8,9 @@ from typing import Dict, Any, Optional, TypedDict, ClassVar, List, Set, Union
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
import discord import discord # type: ignore
from redbot.core.bot import Red from redbot.core.bot import Red # type: ignore
from redbot.core.commands import GroupCog, Context from redbot.core.commands import GroupCog, Context # type: ignore
from .settings import Settings from .settings import Settings
from .lifecycle import LifecycleManager, LifecycleState from .lifecycle import LifecycleManager, LifecycleState
@@ -22,12 +22,12 @@ from .commands.database_commands import setup_database_commands
from .commands.settings_commands import setup_settings_commands from .commands.settings_commands import setup_settings_commands
from .events import setup_events, EventManager from .events import setup_events, EventManager
from .processor.core import Processor from ..processor.core import Processor
from .queue.manager import QueueManager from ..queue.manager import QueueManager
from .ffmpeg.ffmpeg_manager import FFmpegManager from ..ffmpeg.ffmpeg_manager import FFmpegManager
from .database.video_archive_db import VideoArchiveDB from ..database.video_archive_db import VideoArchiveDB
from .config_manager import ConfigManager from ..config_manager import ConfigManager
from .utils.exceptions import ( from ..utils.exceptions import (
CogError, CogError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity

4
videoarchiver/core/cleanup.py
View File

@@ -8,8 +8,8 @@ from enum import Enum, auto
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar
from .utils.file_ops import cleanup_downloads from ..utils.file_ops import cleanup_downloads
from .utils.exceptions import ( from ..utils.exceptions import (
CleanupError, CleanupError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity

12
videoarchiver/core/commands/__init__.py
View File

@@ -1,11 +1,11 @@
"""Command handlers for VideoArchiver""" """Command handlers for VideoArchiver"""
from .core.commands.archiver_commands import setup_archiver_commands from .archiver_commands import setup_archiver_commands
from .core.commands.database_commands import setup_database_commands from .database_commands import setup_database_commands
from .core.commands.settings_commands import setup_settings_commands from .settings_commands import setup_settings_commands
__all__ = [ __all__ = [
'setup_archiver_commands', "setup_archiver_commands",
'setup_database_commands', "setup_database_commands",
'setup_settings_commands' "setup_settings_commands",
] ]

104
videoarchiver/core/commands/archiver_commands.py
View File

@@ -4,47 +4,44 @@ import logging
from enum import Enum, auto from enum import Enum, auto
from typing import Optional, Any, Dict, TypedDict, Callable, Awaitable from typing import Optional, Any, Dict, TypedDict, Callable, Awaitable
import discord import discord # type: ignore
from discord import app_commands from discord import app_commands # type: ignore
from redbot.core import commands from redbot.core import commands # type: ignore
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore
from .core.response_handler import handle_response, ResponseType from core.response_handler import handle_response, ResponseType
from .utils.exceptions import ( from utils.exceptions import CommandError, ErrorContext, ErrorSeverity
CommandError,
ErrorContext,
ErrorSeverity
)
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class CommandCategory(Enum): class CommandCategory(Enum):
"""Command categories""" """Command categories"""
MANAGEMENT = auto() MANAGEMENT = auto()
STATUS = auto() STATUS = auto()
UTILITY = auto() UTILITY = auto()
class CommandResult(TypedDict): class CommandResult(TypedDict):
"""Type definition for command result""" """Type definition for command result"""
success: bool success: bool
message: str message: str
details: Optional[Dict[str, Any]] details: Optional[Dict[str, Any]]
error: Optional[str] error: Optional[str]
class CommandContext: class CommandContext:
"""Context manager for command execution""" """Context manager for command execution"""
def __init__(
self, def __init__(self, ctx: Context, category: CommandCategory, operation: str) -> None:
ctx: Context,
category: CommandCategory,
operation: str
) -> None:
self.ctx = ctx self.ctx = ctx
self.category = category self.category = category
self.operation = operation self.operation = operation
self.start_time = None self.start_time = None
async def __aenter__(self) -> 'CommandContext': async def __aenter__(self) -> "CommandContext":
"""Set up command context""" """Set up command context"""
self.start_time = self.ctx.message.created_at self.start_time = self.ctx.message.created_at
logger.debug( logger.debug(
@@ -62,22 +59,23 @@ class CommandContext:
await handle_response( await handle_response(
self.ctx, self.ctx,
f"An error occurred: {str(exc_val)}", f"An error occurred: {str(exc_val)}",
response_type=ResponseType.ERROR response_type=ResponseType.ERROR,
) )
return True return True
return False return False
def setup_archiver_commands(cog: Any) -> Callable: def setup_archiver_commands(cog: Any) -> Callable:
""" """
Set up archiver commands for the cog. Set up archiver commands for the cog.
Args: Args:
cog: VideoArchiver cog instance cog: VideoArchiver cog instance
Returns: Returns:
Main archiver command group Main archiver command group
""" """
@hybrid_group(name="archiver", fallback="help") @hybrid_group(name="archiver", fallback="help")
@guild_only() @guild_only()
async def archiver(ctx: Context) -> None: async def archiver(ctx: Context) -> None:
@@ -86,7 +84,7 @@ def setup_archiver_commands(cog: Any) -> Callable:
await handle_response( await handle_response(
ctx, ctx,
"Use `/help archiver` for a list of commands.", "Use `/help archiver` for a list of commands.",
response_type=ResponseType.INFO response_type=ResponseType.INFO,
) )
@archiver.command(name="enable") @archiver.command(name="enable")
@@ -104,8 +102,8 @@ def setup_archiver_commands(cog: Any) -> Callable:
"ArchiverCommands", "ArchiverCommands",
"enable_archiver", "enable_archiver",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Check current setting # Check current setting
@@ -116,7 +114,7 @@ def setup_archiver_commands(cog: Any) -> Callable:
await handle_response( await handle_response(
ctx, ctx,
"Video archiving is already enabled.", "Video archiving is already enabled.",
response_type=ResponseType.WARNING response_type=ResponseType.WARNING,
) )
return return
@@ -125,7 +123,7 @@ def setup_archiver_commands(cog: Any) -> Callable:
await handle_response( await handle_response(
ctx, ctx,
"Video archiving has been enabled.", "Video archiving has been enabled.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -137,8 +135,8 @@ def setup_archiver_commands(cog: Any) -> Callable:
"ArchiverCommands", "ArchiverCommands",
"enable_archiver", "enable_archiver",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@archiver.command(name="disable") @archiver.command(name="disable")
@@ -156,8 +154,8 @@ def setup_archiver_commands(cog: Any) -> Callable:
"ArchiverCommands", "ArchiverCommands",
"disable_archiver", "disable_archiver",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Check current setting # Check current setting
@@ -168,7 +166,7 @@ def setup_archiver_commands(cog: Any) -> Callable:
await handle_response( await handle_response(
ctx, ctx,
"Video archiving is already disabled.", "Video archiving is already disabled.",
response_type=ResponseType.WARNING response_type=ResponseType.WARNING,
) )
return return
@@ -177,7 +175,7 @@ def setup_archiver_commands(cog: Any) -> Callable:
await handle_response( await handle_response(
ctx, ctx,
"Video archiving has been disabled.", "Video archiving has been disabled.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -189,8 +187,8 @@ def setup_archiver_commands(cog: Any) -> Callable:
"ArchiverCommands", "ArchiverCommands",
"disable_archiver", "disable_archiver",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@archiver.command(name="queue") @archiver.command(name="queue")
@@ -207,8 +205,8 @@ def setup_archiver_commands(cog: Any) -> Callable:
"ArchiverCommands", "ArchiverCommands",
"show_queue", "show_queue",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
await cog.processor.show_queue_details(ctx) await cog.processor.show_queue_details(ctx)
@@ -222,8 +220,8 @@ def setup_archiver_commands(cog: Any) -> Callable:
"ArchiverCommands", "ArchiverCommands",
"show_queue", "show_queue",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
@archiver.command(name="status") @archiver.command(name="status")
@@ -235,22 +233,32 @@ def setup_archiver_commands(cog: Any) -> Callable:
try: try:
# Get comprehensive status # Get comprehensive status
status = { status = {
"enabled": await cog.config_manager.get_setting(ctx.guild.id, "enabled"), "enabled": await cog.config_manager.get_setting(
"queue": cog.queue_manager.get_queue_status(ctx.guild.id) if cog.queue_manager else None, ctx.guild.id, "enabled"
),
"queue": (
cog.queue_manager.get_queue_status(ctx.guild.id)
if cog.queue_manager
else None
),
"processor": cog.processor.get_status() if cog.processor else None, "processor": cog.processor.get_status() if cog.processor else None,
"components": cog.component_manager.get_component_status(), "components": cog.component_manager.get_component_status(),
"health": cog.status_tracker.get_status() "health": cog.status_tracker.get_status(),
} }
# Create status embed # Create status embed
embed = discord.Embed( embed = discord.Embed(
title="VideoArchiver Status", title="VideoArchiver Status",
color=discord.Color.blue() if status["enabled"] else discord.Color.red() color=(
discord.Color.blue()
if status["enabled"]
else discord.Color.red()
),
) )
embed.add_field( embed.add_field(
name="Status", name="Status",
value="Enabled" if status["enabled"] else "Disabled", value="Enabled" if status["enabled"] else "Disabled",
inline=False inline=False,
) )
if status["queue"]: if status["queue"]:
@@ -261,7 +269,7 @@ def setup_archiver_commands(cog: Any) -> Callable:
f"Processing: {status['queue']['processing']}\n" f"Processing: {status['queue']['processing']}\n"
f"Completed: {status['queue']['completed']}" f"Completed: {status['queue']['completed']}"
), ),
inline=True inline=True,
) )
if status["processor"]: if status["processor"]:
@@ -271,7 +279,7 @@ def setup_archiver_commands(cog: Any) -> Callable:
f"Active: {status['processor']['active']}\n" f"Active: {status['processor']['active']}\n"
f"Health: {status['processor']['health']}" f"Health: {status['processor']['health']}"
), ),
inline=True inline=True,
) )
embed.add_field( embed.add_field(
@@ -280,7 +288,7 @@ def setup_archiver_commands(cog: Any) -> Callable:
f"State: {status['health']['state']}\n" f"State: {status['health']['state']}\n"
f"Errors: {status['health']['error_count']}" f"Errors: {status['health']['error_count']}"
), ),
inline=True inline=True,
) )
await ctx.send(embed=embed) await ctx.send(embed=embed)
@@ -294,8 +302,8 @@ def setup_archiver_commands(cog: Any) -> Callable:
"ArchiverCommands", "ArchiverCommands",
"show_status", "show_status",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
# Store commands in cog for access # Store commands in cog for access

196
videoarchiver/core/commands/database_commands.py
View File

@@ -5,31 +5,30 @@ from datetime import datetime
from enum import Enum, auto from enum import Enum, auto
from typing import Optional, Any, Dict, TypedDict from typing import Optional, Any, Dict, TypedDict
import discord import discord # type: ignore
from discord import app_commands from discord import app_commands # type: ignore
from redbot.core import commands from redbot.core import commands # type: ignore
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore
from .core.response_handler import handle_response, ResponseType from core.response_handler import handle_response, ResponseType
from .utils.exceptions import ( from utils.exceptions import CommandError, ErrorContext, ErrorSeverity, DatabaseError
CommandError, from database.video_archive_db import VideoArchiveDB
ErrorContext,
ErrorSeverity,
DatabaseError
)
from .database.video_archive_db import VideoArchiveDB
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class DatabaseOperation(Enum): class DatabaseOperation(Enum):
"""Database operation types""" """Database operation types"""
ENABLE = auto() ENABLE = auto()
DISABLE = auto() DISABLE = auto()
QUERY = auto() QUERY = auto()
MAINTENANCE = auto() MAINTENANCE = auto()
class DatabaseStatus(TypedDict): class DatabaseStatus(TypedDict):
"""Type definition for database status""" """Type definition for database status"""
enabled: bool enabled: bool
connected: bool connected: bool
initialized: bool initialized: bool
@@ -37,8 +36,10 @@ class DatabaseStatus(TypedDict):
last_operation: Optional[str] last_operation: Optional[str]
operation_time: Optional[str] operation_time: Optional[str]
class ArchivedVideo(TypedDict): class ArchivedVideo(TypedDict):
"""Type definition for archived video data""" """Type definition for archived video data"""
url: str url: str
discord_url: str discord_url: str
message_id: int message_id: int
@@ -46,20 +47,23 @@ class ArchivedVideo(TypedDict):
guild_id: int guild_id: int
archived_at: str archived_at: str
async def check_database_status(cog: Any) -> DatabaseStatus: async def check_database_status(cog: Any) -> DatabaseStatus:
""" """
Check database status. Check database status.
Args: Args:
cog: VideoArchiver cog instance cog: VideoArchiver cog instance
Returns: Returns:
Database status information Database status information
""" """
try: try:
enabled = await cog.config_manager.get_setting( enabled = (
None, "use_database" await cog.config_manager.get_setting(None, "use_database")
) if cog.config_manager else False if cog.config_manager
else False
)
return DatabaseStatus( return DatabaseStatus(
enabled=enabled, enabled=enabled,
@@ -67,7 +71,7 @@ async def check_database_status(cog: Any) -> DatabaseStatus:
initialized=cog.db is not None, initialized=cog.db is not None,
error=None, error=None,
last_operation=None, last_operation=None,
operation_time=datetime.utcnow().isoformat() operation_time=datetime.utcnow().isoformat(),
) )
except Exception as e: except Exception as e:
return DatabaseStatus( return DatabaseStatus(
@@ -76,16 +80,17 @@ async def check_database_status(cog: Any) -> DatabaseStatus:
initialized=False, initialized=False,
error=str(e), error=str(e),
last_operation=None, last_operation=None,
operation_time=datetime.utcnow().isoformat() operation_time=datetime.utcnow().isoformat(),
) )
def setup_database_commands(cog: Any) -> Any: def setup_database_commands(cog: Any) -> Any:
""" """
Set up database commands for the cog. Set up database commands for the cog.
Args: Args:
cog: VideoArchiver cog instance cog: VideoArchiver cog instance
Returns: Returns:
Main database command group Main database command group
""" """
@@ -98,39 +103,39 @@ def setup_database_commands(cog: Any) -> Any:
try: try:
# Get database status # Get database status
status = await check_database_status(cog) status = await check_database_status(cog)
# Create status embed # Create status embed
embed = discord.Embed( embed = discord.Embed(
title="Video Archive Database Status", title="Video Archive Database Status",
color=discord.Color.blue() if status["enabled"] else discord.Color.red() color=(
discord.Color.blue()
if status["enabled"]
else discord.Color.red()
),
) )
embed.add_field( embed.add_field(
name="Status", name="Status",
value="Enabled" if status["enabled"] else "Disabled", value="Enabled" if status["enabled"] else "Disabled",
inline=False inline=False,
) )
embed.add_field( embed.add_field(
name="Connection", name="Connection",
value="Connected" if status["connected"] else "Disconnected", value="Connected" if status["connected"] else "Disconnected",
inline=True inline=True,
) )
embed.add_field( embed.add_field(
name="Initialization", name="Initialization",
value="Initialized" if status["initialized"] else "Not Initialized", value="Initialized" if status["initialized"] else "Not Initialized",
inline=True inline=True,
) )
if status["error"]: if status["error"]:
embed.add_field( embed.add_field(name="Error", value=status["error"], inline=False)
name="Error",
value=status["error"],
inline=False
)
await handle_response( await handle_response(
ctx, ctx,
"Use `/help archivedb` for a list of commands.", "Use `/help archivedb` for a list of commands.",
embed=embed, embed=embed,
response_type=ResponseType.INFO response_type=ResponseType.INFO,
) )
except Exception as e: except Exception as e:
error = f"Failed to get database status: {str(e)}" error = f"Failed to get database status: {str(e)}"
@@ -141,8 +146,8 @@ def setup_database_commands(cog: Any) -> Any:
"DatabaseCommands", "DatabaseCommands",
"show_status", "show_status",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
@archivedb.command(name="enable") @archivedb.command(name="enable")
@@ -159,8 +164,8 @@ def setup_database_commands(cog: Any) -> Any:
"DatabaseCommands", "DatabaseCommands",
"enable_database", "enable_database",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -175,7 +180,7 @@ def setup_database_commands(cog: Any) -> Any:
await handle_response( await handle_response(
ctx, ctx,
"The video archive database is already enabled.", "The video archive database is already enabled.",
response_type=ResponseType.WARNING response_type=ResponseType.WARNING,
) )
return return
@@ -190,8 +195,8 @@ def setup_database_commands(cog: Any) -> Any:
"DatabaseCommands", "DatabaseCommands",
"enable_database", "enable_database",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Update processor with database # Update processor with database
@@ -201,17 +206,13 @@ def setup_database_commands(cog: Any) -> Any:
cog.processor.queue_handler.db = cog.db cog.processor.queue_handler.db = cog.db
# Update setting # Update setting
await cog.config_manager.update_setting( await cog.config_manager.update_setting(ctx.guild.id, "use_database", True)
ctx.guild.id,
"use_database",
True
)
# Send success message # Send success message
await handle_response( await handle_response(
ctx, ctx,
"Video archive database has been enabled.", "Video archive database has been enabled.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -223,8 +224,8 @@ def setup_database_commands(cog: Any) -> Any:
"DatabaseCommands", "DatabaseCommands",
"enable_database", "enable_database",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@archivedb.command(name="disable") @archivedb.command(name="disable")
@@ -241,8 +242,8 @@ def setup_database_commands(cog: Any) -> Any:
"DatabaseCommands", "DatabaseCommands",
"disable_database", "disable_database",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -250,14 +251,13 @@ def setup_database_commands(cog: Any) -> Any:
await ctx.defer() await ctx.defer()
current_setting = await cog.config_manager.get_setting( current_setting = await cog.config_manager.get_setting(
ctx.guild.id, ctx.guild.id, "use_database"
"use_database"
) )
if not current_setting: if not current_setting:
await handle_response( await handle_response(
ctx, ctx,
"The video archive database is already disabled.", "The video archive database is already disabled.",
response_type=ResponseType.WARNING response_type=ResponseType.WARNING,
) )
return return
@@ -275,15 +275,11 @@ def setup_database_commands(cog: Any) -> Any:
if cog.processor.queue_handler: if cog.processor.queue_handler:
cog.processor.queue_handler.db = None cog.processor.queue_handler.db = None
await cog.config_manager.update_setting( await cog.config_manager.update_setting(ctx.guild.id, "use_database", False)
ctx.guild.id,
"use_database",
False
)
await handle_response( await handle_response(
ctx, ctx,
"Video archive database has been disabled.", "Video archive database has been disabled.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -295,8 +291,8 @@ def setup_database_commands(cog: Any) -> Any:
"DatabaseCommands", "DatabaseCommands",
"disable_database", "disable_database",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@archivedb.command(name="check") @archivedb.command(name="check")
@@ -310,7 +306,7 @@ def setup_database_commands(cog: Any) -> Any:
await handle_response( await handle_response(
ctx, ctx,
"The archive database is not enabled. Ask an admin to enable it with `/archivedb enable`", "The archive database is not enabled. Ask an admin to enable it with `/archivedb enable`",
response_type=ResponseType.ERROR response_type=ResponseType.ERROR,
) )
return return
@@ -327,8 +323,8 @@ def setup_database_commands(cog: Any) -> Any:
"DatabaseCommands", "DatabaseCommands",
"checkarchived", "checkarchived",
{"guild_id": ctx.guild.id, "url": url}, {"guild_id": ctx.guild.id, "url": url},
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
if result: if result:
@@ -338,30 +334,12 @@ def setup_database_commands(cog: Any) -> Any:
description=f"This video has been archived!\n\nOriginal URL: {url}", description=f"This video has been archived!\n\nOriginal URL: {url}",
color=discord.Color.green(), color=discord.Color.green(),
) )
embed.add_field( embed.add_field(name="Archived Link", value=discord_url, inline=False)
name="Archived Link", embed.add_field(name="Message ID", value=str(message_id), inline=True)
value=discord_url, embed.add_field(name="Channel ID", value=str(channel_id), inline=True)
inline=False embed.add_field(name="Guild ID", value=str(guild_id), inline=True)
)
embed.add_field(
name="Message ID",
value=str(message_id),
inline=True
)
embed.add_field(
name="Channel ID",
value=str(channel_id),
inline=True
)
embed.add_field(
name="Guild ID",
value=str(guild_id),
inline=True
)
await handle_response( await handle_response(
ctx, ctx, embed=embed, response_type=ResponseType.SUCCESS
embed=embed,
response_type=ResponseType.SUCCESS
) )
else: else:
embed = discord.Embed( embed = discord.Embed(
@@ -370,9 +348,7 @@ def setup_database_commands(cog: Any) -> Any:
color=discord.Color.red(), color=discord.Color.red(),
) )
await handle_response( await handle_response(
ctx, ctx, embed=embed, response_type=ResponseType.WARNING
embed=embed,
response_type=ResponseType.WARNING
) )
except Exception as e: except Exception as e:
@@ -384,8 +360,8 @@ def setup_database_commands(cog: Any) -> Any:
"DatabaseCommands", "DatabaseCommands",
"checkarchived", "checkarchived",
{"guild_id": ctx.guild.id, "url": url}, {"guild_id": ctx.guild.id, "url": url},
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
@archivedb.command(name="status") @archivedb.command(name="status")
@@ -399,7 +375,7 @@ def setup_database_commands(cog: Any) -> Any:
await ctx.defer() await ctx.defer()
status = await check_database_status(cog) status = await check_database_status(cog)
# Get additional stats if database is enabled # Get additional stats if database is enabled
stats = {} stats = {}
if cog.db and status["connected"]: if cog.db and status["connected"]:
@@ -410,53 +386,49 @@ def setup_database_commands(cog: Any) -> Any:
embed = discord.Embed( embed = discord.Embed(
title="Database Status", title="Database Status",
color=discord.Color.green() if status["connected"] else discord.Color.red() color=(
discord.Color.green()
if status["connected"]
else discord.Color.red()
),
) )
embed.add_field( embed.add_field(
name="Status", name="Status",
value="Enabled" if status["enabled"] else "Disabled", value="Enabled" if status["enabled"] else "Disabled",
inline=False inline=False,
) )
embed.add_field( embed.add_field(
name="Connection", name="Connection",
value="Connected" if status["connected"] else "Disconnected", value="Connected" if status["connected"] else "Disconnected",
inline=True inline=True,
) )
embed.add_field( embed.add_field(
name="Initialization", name="Initialization",
value="Initialized" if status["initialized"] else "Not Initialized", value="Initialized" if status["initialized"] else "Not Initialized",
inline=True inline=True,
) )
if stats: if stats:
embed.add_field( embed.add_field(
name="Total Videos", name="Total Videos",
value=str(stats.get("total_videos", 0)), value=str(stats.get("total_videos", 0)),
inline=True inline=True,
) )
embed.add_field( embed.add_field(
name="Total Size", name="Total Size",
value=f"{stats.get('total_size', 0)} MB", value=f"{stats.get('total_size', 0)} MB",
inline=True inline=True,
) )
embed.add_field( embed.add_field(
name="Last Update", name="Last Update",
value=stats.get("last_update", "Never"), value=stats.get("last_update", "Never"),
inline=True inline=True,
) )
if status["error"]: if status["error"]:
embed.add_field( embed.add_field(name="Error", value=status["error"], inline=False)
name="Error",
value=status["error"],
inline=False
)
await handle_response( await handle_response(ctx, embed=embed, response_type=ResponseType.INFO)
ctx,
embed=embed,
response_type=ResponseType.INFO
)
except Exception as e: except Exception as e:
error = f"Failed to get database status: {str(e)}" error = f"Failed to get database status: {str(e)}"
@@ -467,8 +439,8 @@ def setup_database_commands(cog: Any) -> Any:
"DatabaseCommands", "DatabaseCommands",
"database_status", "database_status",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
# Store commands in cog for access # Store commands in cog for access

358
videoarchiver/core/commands/settings_commands.py
View File

@@ -4,122 +4,130 @@ import logging
from enum import Enum, auto from enum import Enum, auto
from typing import Optional, Any, Dict, TypedDict from typing import Optional, Any, Dict, TypedDict
import discord import discord # type: ignore
from discord import app_commands from discord import app_commands # type: ignore
from redbot.core import commands from redbot.core import commands # type: ignore
from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions from redbot.core.commands import Context, hybrid_group, guild_only, admin_or_permissions # type: ignore
from .core.settings import VideoFormat, VideoQuality from core.settings import VideoFormat, VideoQuality
from .core.response_handler import handle_response, ResponseType from core.response_handler import handle_response, ResponseType
from .utils.exceptions import ( from utils.exceptions import CommandError, ErrorContext, ErrorSeverity
CommandError,
ErrorContext,
ErrorSeverity
)
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class SettingCategory(Enum): class SettingCategory(Enum):
"""Setting categories""" """Setting categories"""
CHANNELS = auto() CHANNELS = auto()
VIDEO = auto() VIDEO = auto()
MESSAGES = auto() MESSAGES = auto()
PERFORMANCE = auto() PERFORMANCE = auto()
class SettingValidation(TypedDict): class SettingValidation(TypedDict):
"""Type definition for setting validation""" """Type definition for setting validation"""
valid: bool valid: bool
error: Optional[str] error: Optional[str]
details: Dict[str, Any] details: Dict[str, Any]
class SettingUpdate(TypedDict): class SettingUpdate(TypedDict):
"""Type definition for setting update""" """Type definition for setting update"""
setting: str setting: str
old_value: Any old_value: Any
new_value: Any new_value: Any
category: SettingCategory category: SettingCategory
async def validate_setting( async def validate_setting(
category: SettingCategory, category: SettingCategory, setting: str, value: Any
setting: str,
value: Any
) -> SettingValidation: ) -> SettingValidation:
""" """
Validate a setting value. Validate a setting value.
Args: Args:
category: Setting category category: Setting category
setting: Setting name setting: Setting name
value: Value to validate value: Value to validate
Returns: Returns:
Validation result Validation result
""" """
validation = SettingValidation( validation = SettingValidation(
valid=True, valid=True,
error=None, error=None,
details={"category": category.name, "setting": setting, "value": value} details={"category": category.name, "setting": setting, "value": value},
) )
try: try:
if category == SettingCategory.VIDEO: if category == SettingCategory.VIDEO:
if setting == "format": if setting == "format":
if value not in [f.value for f in VideoFormat]: if value not in [f.value for f in VideoFormat]:
validation.update({ validation.update(
"valid": False, {
"error": f"Invalid format. Must be one of: {', '.join(f.value for f in VideoFormat)}" "valid": False,
}) "error": f"Invalid format. Must be one of: {', '.join(f.value for f in VideoFormat)}",
}
)
elif setting == "quality": elif setting == "quality":
if not 144 <= value <= 4320: if not 144 <= value <= 4320:
validation.update({ validation.update(
"valid": False, {
"error": "Quality must be between 144 and 4320" "valid": False,
}) "error": "Quality must be between 144 and 4320",
}
)
elif setting == "max_file_size": elif setting == "max_file_size":
if not 1 <= value <= 100: if not 1 <= value <= 100:
validation.update({ validation.update(
"valid": False, {"valid": False, "error": "Size must be between 1 and 100 MB"}
"error": "Size must be between 1 and 100 MB" )
})
elif category == SettingCategory.MESSAGES: elif category == SettingCategory.MESSAGES:
if setting == "duration": if setting == "duration":
if not 0 <= value <= 168: if not 0 <= value <= 168:
validation.update({ validation.update(
"valid": False, {
"error": "Duration must be between 0 and 168 hours (1 week)" "valid": False,
}) "error": "Duration must be between 0 and 168 hours (1 week)",
}
)
elif setting == "template": elif setting == "template":
placeholders = ["{author}", "{channel}", "{original_message}"] placeholders = ["{author}", "{channel}", "{original_message}"]
if not any(ph in value for ph in placeholders): if not any(ph in value for ph in placeholders):
validation.update({ validation.update(
"valid": False, {
"error": f"Template must include at least one placeholder: {', '.join(placeholders)}" "valid": False,
}) "error": f"Template must include at least one placeholder: {', '.join(placeholders)}",
}
)
elif category == SettingCategory.PERFORMANCE: elif category == SettingCategory.PERFORMANCE:
if setting == "concurrent_downloads": if setting == "concurrent_downloads":
if not 1 <= value <= 5: if not 1 <= value <= 5:
validation.update({ validation.update(
"valid": False, {
"error": "Concurrent downloads must be between 1 and 5" "valid": False,
}) "error": "Concurrent downloads must be between 1 and 5",
}
)
except Exception as e: except Exception as e:
validation.update({ validation.update({"valid": False, "error": f"Validation error: {str(e)}"})
"valid": False,
"error": f"Validation error: {str(e)}"
})
return validation return validation
def setup_settings_commands(cog: Any) -> Any: def setup_settings_commands(cog: Any) -> Any:
""" """
Set up settings commands for the cog. Set up settings commands for the cog.
Args: Args:
cog: VideoArchiver cog instance cog: VideoArchiver cog instance
Returns: Returns:
Main settings command group Main settings command group
""" """
@@ -137,8 +145,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"show_settings", "show_settings",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -146,11 +154,7 @@ def setup_settings_commands(cog: Any) -> Any:
await ctx.defer() await ctx.defer()
embed = await cog.config_manager.format_settings_embed(ctx.guild) embed = await cog.config_manager.format_settings_embed(ctx.guild)
await handle_response( await handle_response(ctx, embed=embed, response_type=ResponseType.INFO)
ctx,
embed=embed,
response_type=ResponseType.INFO
)
except Exception as e: except Exception as e:
error = f"Failed to show settings: {str(e)}" error = f"Failed to show settings: {str(e)}"
@@ -161,8 +165,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"show_settings", "show_settings",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
@settings.command(name="setchannel") @settings.command(name="setchannel")
@@ -180,8 +184,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_archive_channel", "set_archive_channel",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -194,7 +198,7 @@ def setup_settings_commands(cog: Any) -> Any:
send_messages=True, send_messages=True,
embed_links=True, embed_links=True,
attach_files=True, attach_files=True,
read_message_history=True read_message_history=True,
) )
channel_perms = channel.permissions_for(bot_member) channel_perms = channel.permissions_for(bot_member)
if not all(getattr(channel_perms, perm) for perm in required_perms): if not all(getattr(channel_perms, perm) for perm in required_perms):
@@ -207,23 +211,22 @@ def setup_settings_commands(cog: Any) -> Any:
"guild_id": ctx.guild.id, "guild_id": ctx.guild.id,
"channel_id": channel.id, "channel_id": channel.id,
"missing_perms": [ "missing_perms": [
perm for perm in required_perms perm
for perm in required_perms
if not getattr(channel_perms, perm) if not getattr(channel_perms, perm)
] ],
}, },
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
await cog.config_manager.update_setting( await cog.config_manager.update_setting(
ctx.guild.id, ctx.guild.id, "archive_channel", channel.id
"archive_channel",
channel.id
) )
await handle_response( await handle_response(
ctx, ctx,
f"Archive channel has been set to {channel.mention}.", f"Archive channel has been set to {channel.mention}.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -235,8 +238,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_archive_channel", "set_archive_channel",
{"guild_id": ctx.guild.id, "channel_id": channel.id}, {"guild_id": ctx.guild.id, "channel_id": channel.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@settings.command(name="setlog") @settings.command(name="setlog")
@@ -254,8 +257,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_log_channel", "set_log_channel",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -265,9 +268,7 @@ def setup_settings_commands(cog: Any) -> Any:
# Check channel permissions # Check channel permissions
bot_member = ctx.guild.me bot_member = ctx.guild.me
required_perms = discord.Permissions( required_perms = discord.Permissions(
send_messages=True, send_messages=True, embed_links=True, read_message_history=True
embed_links=True,
read_message_history=True
) )
channel_perms = channel.permissions_for(bot_member) channel_perms = channel.permissions_for(bot_member)
if not all(getattr(channel_perms, perm) for perm in required_perms): if not all(getattr(channel_perms, perm) for perm in required_perms):
@@ -280,23 +281,22 @@ def setup_settings_commands(cog: Any) -> Any:
"guild_id": ctx.guild.id, "guild_id": ctx.guild.id,
"channel_id": channel.id, "channel_id": channel.id,
"missing_perms": [ "missing_perms": [
perm for perm in required_perms perm
for perm in required_perms
if not getattr(channel_perms, perm) if not getattr(channel_perms, perm)
] ],
}, },
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
await cog.config_manager.update_setting( await cog.config_manager.update_setting(
ctx.guild.id, ctx.guild.id, "log_channel", channel.id
"log_channel",
channel.id
) )
await handle_response( await handle_response(
ctx, ctx,
f"Log channel has been set to {channel.mention}.", f"Log channel has been set to {channel.mention}.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -308,8 +308,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_log_channel", "set_log_channel",
{"guild_id": ctx.guild.id, "channel_id": channel.id}, {"guild_id": ctx.guild.id, "channel_id": channel.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@settings.command(name="addchannel") @settings.command(name="addchannel")
@@ -327,8 +327,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"add_enabled_channel", "add_enabled_channel",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -338,8 +338,7 @@ def setup_settings_commands(cog: Any) -> Any:
# Check channel permissions # Check channel permissions
bot_member = ctx.guild.me bot_member = ctx.guild.me
required_perms = discord.Permissions( required_perms = discord.Permissions(
read_messages=True, read_messages=True, read_message_history=True
read_message_history=True
) )
channel_perms = channel.permissions_for(bot_member) channel_perms = channel.permissions_for(bot_member)
if not all(getattr(channel_perms, perm) for perm in required_perms): if not all(getattr(channel_perms, perm) for perm in required_perms):
@@ -352,36 +351,34 @@ def setup_settings_commands(cog: Any) -> Any:
"guild_id": ctx.guild.id, "guild_id": ctx.guild.id,
"channel_id": channel.id, "channel_id": channel.id,
"missing_perms": [ "missing_perms": [
perm for perm in required_perms perm
for perm in required_perms
if not getattr(channel_perms, perm) if not getattr(channel_perms, perm)
] ],
}, },
ErrorSeverity.MEDIUM ErrorSeverity.MEDIUM,
) ),
) )
enabled_channels = await cog.config_manager.get_setting( enabled_channels = await cog.config_manager.get_setting(
ctx.guild.id, ctx.guild.id, "enabled_channels"
"enabled_channels"
) )
if channel.id in enabled_channels: if channel.id in enabled_channels:
await handle_response( await handle_response(
ctx, ctx,
f"{channel.mention} is already being monitored.", f"{channel.mention} is already being monitored.",
response_type=ResponseType.WARNING response_type=ResponseType.WARNING,
) )
return return
enabled_channels.append(channel.id) enabled_channels.append(channel.id)
await cog.config_manager.update_setting( await cog.config_manager.update_setting(
ctx.guild.id, ctx.guild.id, "enabled_channels", enabled_channels
"enabled_channels",
enabled_channels
) )
await handle_response( await handle_response(
ctx, ctx,
f"Now monitoring {channel.mention} for videos.", f"Now monitoring {channel.mention} for videos.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -393,15 +390,17 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"add_enabled_channel", "add_enabled_channel",
{"guild_id": ctx.guild.id, "channel_id": channel.id}, {"guild_id": ctx.guild.id, "channel_id": channel.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@settings.command(name="removechannel") @settings.command(name="removechannel")
@guild_only() @guild_only()
@admin_or_permissions(administrator=True) @admin_or_permissions(administrator=True)
@app_commands.describe(channel="The channel to stop monitoring") @app_commands.describe(channel="The channel to stop monitoring")
async def remove_enabled_channel(ctx: Context, channel: discord.TextChannel) -> None: async def remove_enabled_channel(
ctx: Context, channel: discord.TextChannel
) -> None:
"""Remove a channel from video monitoring.""" """Remove a channel from video monitoring."""
try: try:
# Check if config manager is ready # Check if config manager is ready
@@ -412,8 +411,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"remove_enabled_channel", "remove_enabled_channel",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -421,27 +420,24 @@ def setup_settings_commands(cog: Any) -> Any:
await ctx.defer() await ctx.defer()
enabled_channels = await cog.config_manager.get_setting( enabled_channels = await cog.config_manager.get_setting(
ctx.guild.id, ctx.guild.id, "enabled_channels"
"enabled_channels"
) )
if channel.id not in enabled_channels: if channel.id not in enabled_channels:
await handle_response( await handle_response(
ctx, ctx,
f"{channel.mention} is not being monitored.", f"{channel.mention} is not being monitored.",
response_type=ResponseType.WARNING response_type=ResponseType.WARNING,
) )
return return
enabled_channels.remove(channel.id) enabled_channels.remove(channel.id)
await cog.config_manager.update_setting( await cog.config_manager.update_setting(
ctx.guild.id, ctx.guild.id, "enabled_channels", enabled_channels
"enabled_channels",
enabled_channels
) )
await handle_response( await handle_response(
ctx, ctx,
f"Stopped monitoring {channel.mention} for videos.", f"Stopped monitoring {channel.mention} for videos.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -453,8 +449,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"remove_enabled_channel", "remove_enabled_channel",
{"guild_id": ctx.guild.id, "channel_id": channel.id}, {"guild_id": ctx.guild.id, "channel_id": channel.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@settings.command(name="setformat") @settings.command(name="setformat")
@@ -472,8 +468,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_video_format", "set_video_format",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -482,28 +478,20 @@ def setup_settings_commands(cog: Any) -> Any:
# Validate format # Validate format
format = format.lower() format = format.lower()
validation = await validate_setting( validation = await validate_setting(SettingCategory.VIDEO, "format", format)
SettingCategory.VIDEO,
"format",
format
)
if not validation["valid"]: if not validation["valid"]:
await handle_response( await handle_response(
ctx, ctx, validation["error"], response_type=ResponseType.ERROR
validation["error"],
response_type=ResponseType.ERROR
) )
return return
await cog.config_manager.update_setting( await cog.config_manager.update_setting(
ctx.guild.id, ctx.guild.id, "video_format", format
"video_format",
format
) )
await handle_response( await handle_response(
ctx, ctx,
f"Video format has been set to {format}.", f"Video format has been set to {format}.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -515,8 +503,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_video_format", "set_video_format",
{"guild_id": ctx.guild.id, "format": format}, {"guild_id": ctx.guild.id, "format": format},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@settings.command(name="setquality") @settings.command(name="setquality")
@@ -534,8 +522,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_video_quality", "set_video_quality",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -544,27 +532,21 @@ def setup_settings_commands(cog: Any) -> Any:
# Validate quality # Validate quality
validation = await validate_setting( validation = await validate_setting(
SettingCategory.VIDEO, SettingCategory.VIDEO, "quality", quality
"quality",
quality
) )
if not validation["valid"]: if not validation["valid"]:
await handle_response( await handle_response(
ctx, ctx, validation["error"], response_type=ResponseType.ERROR
validation["error"],
response_type=ResponseType.ERROR
) )
return return
await cog.config_manager.update_setting( await cog.config_manager.update_setting(
ctx.guild.id, ctx.guild.id, "video_quality", quality
"video_quality",
quality
) )
await handle_response( await handle_response(
ctx, ctx,
f"Video quality has been set to {quality}p.", f"Video quality has been set to {quality}p.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -576,8 +558,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_video_quality", "set_video_quality",
{"guild_id": ctx.guild.id, "quality": quality}, {"guild_id": ctx.guild.id, "quality": quality},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@settings.command(name="setmaxsize") @settings.command(name="setmaxsize")
@@ -595,8 +577,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_max_file_size", "set_max_file_size",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -605,27 +587,19 @@ def setup_settings_commands(cog: Any) -> Any:
# Validate size # Validate size
validation = await validate_setting( validation = await validate_setting(
SettingCategory.VIDEO, SettingCategory.VIDEO, "max_file_size", size
"max_file_size",
size
) )
if not validation["valid"]: if not validation["valid"]:
await handle_response( await handle_response(
ctx, ctx, validation["error"], response_type=ResponseType.ERROR
validation["error"],
response_type=ResponseType.ERROR
) )
return return
await cog.config_manager.update_setting( await cog.config_manager.update_setting(ctx.guild.id, "max_file_size", size)
ctx.guild.id,
"max_file_size",
size
)
await handle_response( await handle_response(
ctx, ctx,
f"Maximum file size has been set to {size}MB.", f"Maximum file size has been set to {size}MB.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -637,8 +611,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_max_file_size", "set_max_file_size",
{"guild_id": ctx.guild.id, "size": size}, {"guild_id": ctx.guild.id, "size": size},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@settings.command(name="setmessageduration") @settings.command(name="setmessageduration")
@@ -656,8 +630,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_message_duration", "set_message_duration",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -666,27 +640,21 @@ def setup_settings_commands(cog: Any) -> Any:
# Validate duration # Validate duration
validation = await validate_setting( validation = await validate_setting(
SettingCategory.MESSAGES, SettingCategory.MESSAGES, "duration", hours
"duration",
hours
) )
if not validation["valid"]: if not validation["valid"]:
await handle_response( await handle_response(
ctx, ctx, validation["error"], response_type=ResponseType.ERROR
validation["error"],
response_type=ResponseType.ERROR
) )
return return
await cog.config_manager.update_setting( await cog.config_manager.update_setting(
ctx.guild.id, ctx.guild.id, "message_duration", hours
"message_duration",
hours
) )
await handle_response( await handle_response(
ctx, ctx,
f"Message duration has been set to {hours} hours.", f"Message duration has been set to {hours} hours.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -698,8 +666,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_message_duration", "set_message_duration",
{"guild_id": ctx.guild.id, "hours": hours}, {"guild_id": ctx.guild.id, "hours": hours},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@settings.command(name="settemplate") @settings.command(name="settemplate")
@@ -719,8 +687,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_message_template", "set_message_template",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -729,27 +697,21 @@ def setup_settings_commands(cog: Any) -> Any:
# Validate template # Validate template
validation = await validate_setting( validation = await validate_setting(
SettingCategory.MESSAGES, SettingCategory.MESSAGES, "template", template
"template",
template
) )
if not validation["valid"]: if not validation["valid"]:
await handle_response( await handle_response(
ctx, ctx, validation["error"], response_type=ResponseType.ERROR
validation["error"],
response_type=ResponseType.ERROR
) )
return return
await cog.config_manager.update_setting( await cog.config_manager.update_setting(
ctx.guild.id, ctx.guild.id, "message_template", template
"message_template",
template
) )
await handle_response( await handle_response(
ctx, ctx,
f"Message template has been set to: {template}", f"Message template has been set to: {template}",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -761,8 +723,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_message_template", "set_message_template",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
@settings.command(name="setconcurrent") @settings.command(name="setconcurrent")
@@ -780,8 +742,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_concurrent_downloads", "set_concurrent_downloads",
{"guild_id": ctx.guild.id}, {"guild_id": ctx.guild.id},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Defer the response immediately for slash commands # Defer the response immediately for slash commands
@@ -790,27 +752,21 @@ def setup_settings_commands(cog: Any) -> Any:
# Validate count # Validate count
validation = await validate_setting( validation = await validate_setting(
SettingCategory.PERFORMANCE, SettingCategory.PERFORMANCE, "concurrent_downloads", count
"concurrent_downloads",
count
) )
if not validation["valid"]: if not validation["valid"]:
await handle_response( await handle_response(
ctx, ctx, validation["error"], response_type=ResponseType.ERROR
validation["error"],
response_type=ResponseType.ERROR
) )
return return
await cog.config_manager.update_setting( await cog.config_manager.update_setting(
ctx.guild.id, ctx.guild.id, "concurrent_downloads", count
"concurrent_downloads",
count
) )
await handle_response( await handle_response(
ctx, ctx,
f"Concurrent downloads has been set to {count}.", f"Concurrent downloads has been set to {count}.",
response_type=ResponseType.SUCCESS response_type=ResponseType.SUCCESS,
) )
except Exception as e: except Exception as e:
@@ -822,8 +778,8 @@ def setup_settings_commands(cog: Any) -> Any:
"SettingsCommands", "SettingsCommands",
"set_concurrent_downloads", "set_concurrent_downloads",
{"guild_id": ctx.guild.id, "count": count}, {"guild_id": ctx.guild.id, "count": count},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
# Store commands in cog for access # Store commands in cog for access

13
videoarchiver/core/component_manager.py
View File

@@ -8,6 +8,7 @@ from typing import (
Optional, Optional,
Set, Set,
List, List,
Tuple,
TypedDict, TypedDict,
ClassVar, ClassVar,
Type, Type,
@@ -19,12 +20,12 @@ from datetime import datetime
from pathlib import Path from pathlib import Path
import importlib import importlib
from .utils.exceptions import ComponentError, ErrorContext, ErrorSeverity from ..utils.exceptions import ComponentError, ErrorContext, ErrorSeverity
from .utils.path_manager import ensure_directory from ..utils.path_manager import ensure_directory
from .config_manager import ConfigManager from ..config_manager import ConfigManager
from .processor.core import Processor from ..processor.core import Processor
from .queue.manager import EnhancedVideoQueueManager from ..queue.manager import EnhancedVideoQueueManager
from .ffmpeg.ffmpeg_manager import FFmpegManager from ..ffmpeg.ffmpeg_manager import FFmpegManager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

8
videoarchiver/core/error_handler.py
View File

@@ -4,8 +4,8 @@ import logging
import traceback import traceback
from typing import Dict, Optional, Tuple, Type, TypedDict, ClassVar from typing import Dict, Optional, Tuple, Type, TypedDict, ClassVar
from enum import Enum, auto from enum import Enum, auto
import discord import discord # type: ignore
from redbot.core.commands import ( from redbot.core.commands import ( # type: ignore
Context, Context,
MissingPermissions, MissingPermissions,
BotMissingPermissions, BotMissingPermissions,
@@ -14,7 +14,7 @@ from redbot.core.commands import (
CommandError CommandError
) )
from .utils.exceptions import ( from ..utils.exceptions import (
VideoArchiverError, VideoArchiverError,
ErrorSeverity, ErrorSeverity,
ErrorContext, ErrorContext,
@@ -33,7 +33,7 @@ from .utils.exceptions import (
ResourceExhaustedError, ResourceExhaustedError,
ConfigurationError ConfigurationError
) )
from .core.response_handler import response_manager from ..core.response_handler import response_manager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

21
videoarchiver/core/events.py
View File

@@ -7,23 +7,24 @@ from datetime import datetime
from enum import Enum, auto from enum import Enum, auto
from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar, List from typing import TYPE_CHECKING, Dict, Any, Optional, TypedDict, ClassVar, List
import discord import discord # type: ignore
from .processor.constants import REACTIONS from ..processor.constants import REACTIONS
from .processor.reactions import handle_archived_reaction from ..processor.reactions import handle_archived_reaction
from .core.guild import initialize_guild_components, cleanup_guild_components from ..core.guild import initialize_guild_components, cleanup_guild_components
from .core.error_handler import error_manager from ..core.error_handler import error_manager
from .core.response_handler import response_manager from ..core.response_handler import response_manager
from .utils.exceptions import EventError, ErrorContext, ErrorSeverity from ..utils.exceptions import EventError, ErrorContext, ErrorSeverity
if TYPE_CHECKING: if TYPE_CHECKING:
from .core.base import VideoArchiver from ..core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class EventType(Enum): class EventType(Enum):
"""Types of Discord events""" """Types of Discord events"""
GUILD_JOIN = auto() GUILD_JOIN = auto()
GUILD_REMOVE = auto() GUILD_REMOVE = auto()
MESSAGE = auto() MESSAGE = auto()
@@ -34,6 +35,7 @@ class EventType(Enum):
class EventStats(TypedDict): class EventStats(TypedDict):
"""Type definition for event statistics""" """Type definition for event statistics"""
counts: Dict[str, int] counts: Dict[str, int]
last_events: Dict[str, str] last_events: Dict[str, str]
errors: Dict[str, int] errors: Dict[str, int]
@@ -43,6 +45,7 @@ class EventStats(TypedDict):
class EventHistory(TypedDict): class EventHistory(TypedDict):
"""Type definition for event history entry""" """Type definition for event history entry"""
event_type: str event_type: str
timestamp: str timestamp: str
guild_id: Optional[int] guild_id: Optional[int]
@@ -94,7 +97,7 @@ class EventTracker:
# Cleanup old history # Cleanup old history
if len(self.history) > self.MAX_HISTORY: if len(self.history) > self.MAX_HISTORY:
self.history = self.history[-self.MAX_HISTORY:] self.history = self.history[-self.MAX_HISTORY :]
def record_error( def record_error(
self, event_type: EventType, error: str, duration: float = 0.0 self, event_type: EventType, error: str, duration: float = 0.0

12
videoarchiver/core/guild.py
View File

@@ -4,16 +4,17 @@ import logging
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Dict, Any, Optional from typing import TYPE_CHECKING, Dict, Any, Optional
from .utils.download_core import DownloadCore from ..utils.download_core import DownloadCore
from .utils.message_manager import MessageManager from ..utils.message_manager import MessageManager
from .utils.file_ops import cleanup_downloads from ..utils.file_ops import cleanup_downloads
from .utils.exceptions import VideoArchiverError as ProcessingError from ..utils.exceptions import VideoArchiverError as ProcessingError
if TYPE_CHECKING: if TYPE_CHECKING:
from .core.base import VideoArchiver from ..core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
async def initialize_guild_components(cog: "VideoArchiver", guild_id: int) -> None: async def initialize_guild_components(cog: "VideoArchiver", guild_id: int) -> None:
"""Initialize or update components for a guild with error handling""" """Initialize or update components for a guild with error handling"""
try: try:
@@ -53,6 +54,7 @@ async def initialize_guild_components(cog: "VideoArchiver", guild_id: int) -> No
logger.error(f"Failed to initialize guild {guild_id}: {str(e)}") logger.error(f"Failed to initialize guild {guild_id}: {str(e)}")
raise ProcessingError(f"Guild initialization failed: {str(e)}") raise ProcessingError(f"Guild initialization failed: {str(e)}")
async def cleanup_guild_components(cog: "VideoArchiver", guild_id: int) -> None: async def cleanup_guild_components(cog: "VideoArchiver", guild_id: int) -> None:
"""Clean up components for a specific guild""" """Clean up components for a specific guild"""
try: try:

6
videoarchiver/core/initialization.py
View File

@@ -4,15 +4,15 @@ from typing import TYPE_CHECKING, Optional, Dict, Any
import asyncio import asyncio
import logging import logging
from .utils.exceptions import ( from ..utils.exceptions import (
ComponentError, ComponentError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity
) )
from .core.lifecycle import LifecycleState from ..core.lifecycle import LifecycleState
if TYPE_CHECKING: if TYPE_CHECKING:
from .core.base import VideoArchiver from ..core.base import VideoArchiver
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

179
videoarchiver/core/lifecycle.py
View File

@@ -7,54 +7,65 @@ from typing import Optional, Dict, Any, Set, List, Callable, TypedDict, ClassVar
from enum import Enum, auto from enum import Enum, auto
from datetime import datetime from datetime import datetime
from .core.cleanup import cleanup_resources, force_cleanup_resources from ..core.cleanup import cleanup_resources, force_cleanup_resources
from .utils.exceptions import ( from ..utils.exceptions import (
VideoArchiverError, VideoArchiverError,
ErrorContext, ErrorContext,
ErrorSeverity, ErrorSeverity,
ComponentError, ComponentError,
CleanupError CleanupError,
) )
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class LifecycleState(Enum): class LifecycleState(Enum):
"""Possible states in the cog lifecycle""" """Possible states in the cog lifecycle"""
UNINITIALIZED = auto() UNINITIALIZED = auto()
INITIALIZING = auto() INITIALIZING = auto()
READY = auto() READY = auto()
UNLOADING = auto() UNLOADING = auto()
ERROR = auto() ERROR = auto()
class TaskStatus(Enum): class TaskStatus(Enum):
"""Task execution status""" """Task execution status"""
RUNNING = auto() RUNNING = auto()
COMPLETED = auto() COMPLETED = auto()
CANCELLED = auto() CANCELLED = auto()
FAILED = auto() FAILED = auto()
class TaskHistory(TypedDict): class TaskHistory(TypedDict):
"""Type definition for task history entry""" """Type definition for task history entry"""
start_time: str start_time: str
end_time: Optional[str] end_time: Optional[str]
status: str status: str
error: Optional[str] error: Optional[str]
duration: float duration: float
class StateHistory(TypedDict): class StateHistory(TypedDict):
"""Type definition for state history entry""" """Type definition for state history entry"""
state: str state: str
timestamp: str timestamp: str
duration: float duration: float
details: Optional[Dict[str, Any]] details: Optional[Dict[str, Any]]
class LifecycleStatus(TypedDict): class LifecycleStatus(TypedDict):
"""Type definition for lifecycle status""" """Type definition for lifecycle status"""
state: str state: str
state_history: List[StateHistory] state_history: List[StateHistory]
tasks: Dict[str, Any] tasks: Dict[str, Any]
health: bool health: bool
class TaskManager: class TaskManager:
"""Manages asyncio tasks""" """Manages asyncio tasks"""
@@ -69,20 +80,20 @@ class TaskManager:
name: str, name: str,
coro: Callable[..., Any], coro: Callable[..., Any],
callback: Optional[Callable[[asyncio.Task], None]] = None, callback: Optional[Callable[[asyncio.Task], None]] = None,
timeout: Optional[float] = None timeout: Optional[float] = None,
) -> asyncio.Task: ) -> asyncio.Task:
""" """
Create and track a task. Create and track a task.
Args: Args:
name: Task name name: Task name
coro: Coroutine to run coro: Coroutine to run
callback: Optional completion callback callback: Optional completion callback
timeout: Optional timeout in seconds timeout: Optional timeout in seconds
Returns: Returns:
Created task Created task
Raises: Raises:
ComponentError: If task creation fails ComponentError: If task creation fails
""" """
@@ -94,14 +105,16 @@ class TaskManager:
end_time=None, end_time=None,
status=TaskStatus.RUNNING.name, status=TaskStatus.RUNNING.name,
error=None, error=None,
duration=0.0 duration=0.0,
) )
if timeout: if timeout:
asyncio.create_task(self._handle_timeout(name, task, timeout)) asyncio.create_task(self._handle_timeout(name, task, timeout))
if callback: if callback:
task.add_done_callback(lambda t: self._handle_completion(name, t, callback)) task.add_done_callback(
lambda t: self._handle_completion(name, t, callback)
)
else: else:
task.add_done_callback(lambda t: self._handle_completion(name, t)) task.add_done_callback(lambda t: self._handle_completion(name, t))
@@ -116,15 +129,12 @@ class TaskManager:
"TaskManager", "TaskManager",
"create_task", "create_task",
{"task_name": name}, {"task_name": name},
ErrorSeverity.HIGH ErrorSeverity.HIGH,
) ),
) )
async def _handle_timeout( async def _handle_timeout(
self, self, name: str, task: asyncio.Task, timeout: float
name: str,
task: asyncio.Task,
timeout: float
) -> None: ) -> None:
"""Handle task timeout""" """Handle task timeout"""
try: try:
@@ -134,16 +144,14 @@ class TaskManager:
logger.warning(f"Task {name} timed out after {timeout}s") logger.warning(f"Task {name} timed out after {timeout}s")
task.cancel() task.cancel()
self._update_task_history( self._update_task_history(
name, name, TaskStatus.FAILED, f"Task timed out after {timeout}s"
TaskStatus.FAILED,
f"Task timed out after {timeout}s"
) )
def _handle_completion( def _handle_completion(
self, self,
name: str, name: str,
task: asyncio.Task, task: asyncio.Task,
callback: Optional[Callable[[asyncio.Task], None]] = None callback: Optional[Callable[[asyncio.Task], None]] = None,
) -> None: ) -> None:
"""Handle task completion""" """Handle task completion"""
try: try:
@@ -169,26 +177,25 @@ class TaskManager:
self._tasks.pop(name, None) self._tasks.pop(name, None)
def _update_task_history( def _update_task_history(
self, self, name: str, status: TaskStatus, error: Optional[str] = None
name: str,
status: TaskStatus,
error: Optional[str] = None
) -> None: ) -> None:
"""Update task history entry""" """Update task history entry"""
if name in self._task_history: if name in self._task_history:
end_time = datetime.utcnow() end_time = datetime.utcnow()
start_time = datetime.fromisoformat(self._task_history[name]["start_time"]) start_time = datetime.fromisoformat(self._task_history[name]["start_time"])
self._task_history[name].update({ self._task_history[name].update(
"end_time": end_time.isoformat(), {
"status": status.name, "end_time": end_time.isoformat(),
"error": error, "status": status.name,
"duration": (end_time - start_time).total_seconds() "error": error,
}) "duration": (end_time - start_time).total_seconds(),
}
)
async def cancel_task(self, name: str) -> None: async def cancel_task(self, name: str) -> None:
""" """
Cancel a specific task. Cancel a specific task.
Args: Args:
name: Task name to cancel name: Task name to cancel
""" """
@@ -210,15 +217,16 @@ class TaskManager:
def get_task_status(self) -> Dict[str, Any]: def get_task_status(self) -> Dict[str, Any]:
""" """
Get status of all tasks. Get status of all tasks.
Returns: Returns:
Dictionary containing task status information Dictionary containing task status information
""" """
return { return {
"active_tasks": list(self._tasks.keys()), "active_tasks": list(self._tasks.keys()),
"history": self._task_history.copy() "history": self._task_history.copy(),
} }
class StateTracker: class StateTracker:
"""Tracks lifecycle state and transitions""" """Tracks lifecycle state and transitions"""
@@ -228,13 +236,11 @@ class StateTracker:
self._record_state() self._record_state()
def set_state( def set_state(
self, self, state: LifecycleState, details: Optional[Dict[str, Any]] = None
state: LifecycleState,
details: Optional[Dict[str, Any]] = None
) -> None: ) -> None:
""" """
Set current state. Set current state.
Args: Args:
state: New state state: New state
details: Optional state transition details details: Optional state transition details
@@ -242,10 +248,7 @@ class StateTracker:
self.state = state self.state = state
self._record_state(details) self._record_state(details)
def _record_state( def _record_state(self, details: Optional[Dict[str, Any]] = None) -> None:
self,
details: Optional[Dict[str, Any]] = None
) -> None:
"""Record state transition""" """Record state transition"""
now = datetime.utcnow() now = datetime.utcnow()
duration = 0.0 duration = 0.0
@@ -253,17 +256,20 @@ class StateTracker:
last_state = datetime.fromisoformat(self.state_history[-1]["timestamp"]) last_state = datetime.fromisoformat(self.state_history[-1]["timestamp"])
duration = (now - last_state).total_seconds() duration = (now - last_state).total_seconds()
self.state_history.append(StateHistory( self.state_history.append(
state=self.state.name, StateHistory(
timestamp=now.isoformat(), state=self.state.name,
duration=duration, timestamp=now.isoformat(),
details=details duration=duration,
)) details=details,
)
)
def get_state_history(self) -> List[StateHistory]: def get_state_history(self) -> List[StateHistory]:
"""Get state transition history""" """Get state transition history"""
return self.state_history.copy() return self.state_history.copy()
class LifecycleManager: class LifecycleManager:
"""Manages the lifecycle of the VideoArchiver cog""" """Manages the lifecycle of the VideoArchiver cog"""
@@ -278,12 +284,11 @@ class LifecycleManager:
self._cleanup_handlers: Set[Callable] = set() self._cleanup_handlers: Set[Callable] = set()
def register_cleanup_handler( def register_cleanup_handler(
self, self, handler: Union[Callable[[], None], Callable[[], Any]]
handler: Union[Callable[[], None], Callable[[], Any]]
) -> None: ) -> None:
""" """
Register a cleanup handler. Register a cleanup handler.
Args: Args:
handler: Cleanup handler function handler: Cleanup handler function
""" """
@@ -292,14 +297,14 @@ class LifecycleManager:
async def initialize_cog(self) -> None: async def initialize_cog(self) -> None:
""" """
Initialize all components with proper error handling. Initialize all components with proper error handling.
Raises: Raises:
ComponentError: If initialization fails ComponentError: If initialization fails
""" """
try: try:
# Initialize components in sequence # Initialize components in sequence
await self.cog.component_manager.initialize_components() await self.cog.component_manager.initialize_components()
# Set ready flag # Set ready flag
self.cog.ready.set() self.cog.ready.set()
logger.info("VideoArchiver initialization completed successfully") logger.info("VideoArchiver initialization completed successfully")
@@ -311,11 +316,8 @@ class LifecycleManager:
raise ComponentError( raise ComponentError(
error, error,
context=ErrorContext( context=ErrorContext(
"LifecycleManager", "LifecycleManager", "initialize_cog", None, ErrorSeverity.HIGH
"initialize_cog", ),
None,
ErrorSeverity.HIGH
)
) )
def init_callback(self, task: asyncio.Task) -> None: def init_callback(self, task: asyncio.Task) -> None:
@@ -326,67 +328,57 @@ class LifecycleManager:
self.state_tracker.set_state(LifecycleState.READY) self.state_tracker.set_state(LifecycleState.READY)
except asyncio.CancelledError: except asyncio.CancelledError:
logger.warning("Initialization was cancelled") logger.warning("Initialization was cancelled")
self.state_tracker.set_state( self.state_tracker.set_state(LifecycleState.ERROR, {"reason": "cancelled"})
LifecycleState.ERROR,
{"reason": "cancelled"}
)
asyncio.create_task(cleanup_resources(self.cog)) asyncio.create_task(cleanup_resources(self.cog))
except Exception as e: except Exception as e:
logger.error(f"Initialization failed: {str(e)}", exc_info=True) logger.error(f"Initialization failed: {str(e)}", exc_info=True)
self.state_tracker.set_state( self.state_tracker.set_state(LifecycleState.ERROR, {"error": str(e)})
LifecycleState.ERROR,
{"error": str(e)}
)
asyncio.create_task(cleanup_resources(self.cog)) asyncio.create_task(cleanup_resources(self.cog))
async def handle_load(self) -> None: async def handle_load(self) -> None:
""" """
Handle cog loading without blocking. Handle cog loading without blocking.
Raises: Raises:
VideoArchiverError: If load fails VideoArchiverError: If load fails
""" """
try: try:
self.state_tracker.set_state(LifecycleState.INITIALIZING) self.state_tracker.set_state(LifecycleState.INITIALIZING)
# Start initialization as background task # Start initialization as background task
await self.task_manager.create_task( await self.task_manager.create_task(
"initialization", "initialization",
self.initialize_cog(), self.initialize_cog(),
self.init_callback, self.init_callback,
timeout=self.INIT_TIMEOUT timeout=self.INIT_TIMEOUT,
) )
logger.info("Initialization started in background") logger.info("Initialization started in background")
except Exception as e: except Exception as e:
self.state_tracker.set_state(LifecycleState.ERROR) self.state_tracker.set_state(LifecycleState.ERROR)
# Ensure cleanup on any error # Ensure cleanup on any error
try: try:
await asyncio.wait_for( await asyncio.wait_for(
force_cleanup_resources(self.cog), force_cleanup_resources(self.cog), timeout=self.CLEANUP_TIMEOUT
timeout=self.CLEANUP_TIMEOUT
) )
except asyncio.TimeoutError: except asyncio.TimeoutError:
logger.error("Force cleanup during load error timed out") logger.error("Force cleanup during load error timed out")
raise VideoArchiverError( raise VideoArchiverError(
f"Error during cog load: {str(e)}", f"Error during cog load: {str(e)}",
context=ErrorContext( context=ErrorContext(
"LifecycleManager", "LifecycleManager", "handle_load", None, ErrorSeverity.HIGH
"handle_load", ),
None,
ErrorSeverity.HIGH
)
) )
async def handle_unload(self) -> None: async def handle_unload(self) -> None:
""" """
Clean up when cog is unloaded. Clean up when cog is unloaded.
Raises: Raises:
CleanupError: If cleanup fails CleanupError: If cleanup fails
""" """
self.state_tracker.set_state(LifecycleState.UNLOADING) self.state_tracker.set_state(LifecycleState.UNLOADING)
try: try:
# Cancel all tasks # Cancel all tasks
await self.task_manager.cancel_all_tasks() await self.task_manager.cancel_all_tasks()
@@ -397,13 +389,11 @@ class LifecycleManager:
# Try normal cleanup # Try normal cleanup
try: try:
cleanup_task = await self.task_manager.create_task( cleanup_task = await self.task_manager.create_task(
"cleanup", "cleanup", cleanup_resources(self.cog), timeout=self.UNLOAD_TIMEOUT
cleanup_resources(self.cog),
timeout=self.UNLOAD_TIMEOUT
) )
await cleanup_task await cleanup_task
logger.info("Normal cleanup completed") logger.info("Normal cleanup completed")
except (asyncio.TimeoutError, Exception) as e: except (asyncio.TimeoutError, Exception) as e:
if isinstance(e, asyncio.TimeoutError): if isinstance(e, asyncio.TimeoutError):
logger.warning("Normal cleanup timed out, forcing cleanup") logger.warning("Normal cleanup timed out, forcing cleanup")
@@ -413,8 +403,7 @@ class LifecycleManager:
# Force cleanup # Force cleanup
try: try:
await asyncio.wait_for( await asyncio.wait_for(
force_cleanup_resources(self.cog), force_cleanup_resources(self.cog), timeout=self.CLEANUP_TIMEOUT
timeout=self.CLEANUP_TIMEOUT
) )
logger.info("Force cleanup completed") logger.info("Force cleanup completed")
except asyncio.TimeoutError: except asyncio.TimeoutError:
@@ -426,8 +415,8 @@ class LifecycleManager:
"LifecycleManager", "LifecycleManager",
"handle_unload", "handle_unload",
None, None,
ErrorSeverity.CRITICAL ErrorSeverity.CRITICAL,
) ),
) )
except Exception as e: except Exception as e:
error = f"Error during force cleanup: {str(e)}" error = f"Error during force cleanup: {str(e)}"
@@ -438,25 +427,19 @@ class LifecycleManager:
"LifecycleManager", "LifecycleManager",
"handle_unload", "handle_unload",
None, None,
ErrorSeverity.CRITICAL ErrorSeverity.CRITICAL,
) ),
) )
except Exception as e: except Exception as e:
error = f"Error during cog unload: {str(e)}" error = f"Error during cog unload: {str(e)}"
logger.error(error, exc_info=True) logger.error(error, exc_info=True)
self.state_tracker.set_state( self.state_tracker.set_state(LifecycleState.ERROR, {"error": str(e)})
LifecycleState.ERROR,
{"error": str(e)}
)
raise CleanupError( raise CleanupError(
error, error,
context=ErrorContext( context=ErrorContext(
"LifecycleManager", "LifecycleManager", "handle_unload", None, ErrorSeverity.CRITICAL
"handle_unload", ),
None,
ErrorSeverity.CRITICAL
)
) )
finally: finally:
# Clear all references # Clear all references
@@ -487,7 +470,7 @@ class LifecycleManager:
def get_status(self) -> LifecycleStatus: def get_status(self) -> LifecycleStatus:
""" """
Get current lifecycle status. Get current lifecycle status.
Returns: Returns:
Dictionary containing lifecycle status information Dictionary containing lifecycle status information
""" """
@@ -495,5 +478,5 @@ class LifecycleManager:
state=self.state_tracker.state.name, state=self.state_tracker.state.name,
state_history=self.state_tracker.get_state_history(), state_history=self.state_tracker.get_state_history(),
tasks=self.task_manager.get_task_status(), tasks=self.task_manager.get_task_status(),
health=self.state_tracker.state == LifecycleState.READY health=self.state_tracker.state == LifecycleState.READY,
) )

6
videoarchiver/core/response_handler.py
View File

@@ -4,10 +4,10 @@ import logging
from enum import Enum, auto from enum import Enum, auto
from typing import Optional, Union, Dict, Any, TypedDict, ClassVar from typing import Optional, Union, Dict, Any, TypedDict, ClassVar
from datetime import datetime from datetime import datetime
import discord import discord # type: ignore
from redbot.core.commands import Context from redbot.core.commands import Context # type: ignore
from .utils.exceptions import ErrorSeverity from ..utils.exceptions import ErrorSeverity
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/core/settings.py
View File

@@ -4,7 +4,7 @@ from typing import Dict, Any, List, Optional, Union, TypedDict, ClassVar
from dataclasses import dataclass, field from dataclasses import dataclass, field
from enum import Enum, auto from enum import Enum, auto
from .utils.exceptions import ( from ..utils.exceptions import (
ConfigurationError, ConfigurationError,
ErrorContext, ErrorContext,
ErrorSeverity ErrorSeverity

2
videoarchiver/ffmpeg/process_manager.py
View File

@@ -1,7 +1,7 @@
"""Module for managing FFmpeg processes""" """Module for managing FFmpeg processes"""
import logging import logging
import psutil import psutil # type: ignore
import subprocess import subprocess
import time import time
from typing import Set, Optional from typing import Set, Optional

76
videoarchiver/processor/__init__.py
View File

@@ -1,27 +1,27 @@
"""Video processing module for VideoArchiver""" """Video processing module for VideoArchiver"""
from typing import Dict, Any, Optional, Union, List, Tuple from typing import Dict, Any, Optional, Union, List, Tuple
import discord import discord # type: ignore
from .processor.core import VideoProcessor from ..processor.core import VideoProcessor
from .processor.constants import ( from ..processor.constants import (
REACTIONS, REACTIONS,
ReactionType, ReactionType,
ReactionEmojis, ReactionEmojis,
ProgressEmojis, ProgressEmojis,
get_reaction, get_reaction,
get_progress_emoji get_progress_emoji,
) )
from .processor.url_extractor import ( from ..processor.url_extractor import (
URLExtractor, URLExtractor,
URLMetadata, URLMetadata,
URLPattern, URLPattern,
URLType, URLType,
URLPatternManager, URLPatternManager,
URLValidator, URLValidator,
URLMetadataExtractor URLMetadataExtractor,
) )
from .processor.message_validator import ( from ..processor.message_validator import (
MessageValidator, MessageValidator,
ValidationContext, ValidationContext,
ValidationRule, ValidationRule,
@@ -30,17 +30,17 @@ from .processor.message_validator import (
ValidationCache, ValidationCache,
ValidationStats, ValidationStats,
ValidationCacheEntry, ValidationCacheEntry,
ValidationError ValidationError,
) )
from .processor.message_handler import MessageHandler from ..processor.message_handler import MessageHandler
from .processor.queue_handler import QueueHandler from ..processor.queue_handler import QueueHandler
from .processor.reactions import ( from ..processor.reactions import (
handle_archived_reaction, handle_archived_reaction,
update_queue_position_reaction, update_queue_position_reaction,
update_progress_reaction, update_progress_reaction,
update_download_progress_reaction update_download_progress_reaction,
) )
from .utils import progress_tracker from ..utils import progress_tracker
# Export public classes and constants # Export public classes and constants
__all__ = [ __all__ = [
@@ -48,7 +48,6 @@ __all__ = [
"VideoProcessor", "VideoProcessor",
"MessageHandler", "MessageHandler",
"QueueHandler", "QueueHandler",
# URL Extraction # URL Extraction
"URLExtractor", "URLExtractor",
"URLMetadata", "URLMetadata",
@@ -57,7 +56,6 @@ __all__ = [
"URLPatternManager", "URLPatternManager",
"URLValidator", "URLValidator",
"URLMetadataExtractor", "URLMetadataExtractor",
# Message Validation # Message Validation
"MessageValidator", "MessageValidator",
"ValidationContext", "ValidationContext",
@@ -68,13 +66,11 @@ __all__ = [
"ValidationStats", "ValidationStats",
"ValidationCacheEntry", "ValidationCacheEntry",
"ValidationError", "ValidationError",
# Constants and enums # Constants and enums
"REACTIONS", "REACTIONS",
"ReactionType", "ReactionType",
"ReactionEmojis", "ReactionEmojis",
"ProgressEmojis", "ProgressEmojis",
# Helper functions # Helper functions
"get_reaction", "get_reaction",
"get_progress_emoji", "get_progress_emoji",
@@ -87,7 +83,6 @@ __all__ = [
"get_active_operations", "get_active_operations",
"get_validation_stats", "get_validation_stats",
"clear_caches", "clear_caches",
# Reaction handlers # Reaction handlers
"handle_archived_reaction", "handle_archived_reaction",
"update_queue_position_reaction", "update_queue_position_reaction",
@@ -104,105 +99,114 @@ __description__ = "Video processing module for archiving Discord videos"
url_extractor = URLExtractor() url_extractor = URLExtractor()
message_validator = MessageValidator() message_validator = MessageValidator()
# URL extraction helper functions # URL extraction helper functions
async def extract_urls( async def extract_urls(
message: discord.Message, message: discord.Message, enabled_sites: Optional[List[str]] = None
enabled_sites: Optional[List[str]] = None
) -> List[URLMetadata]: ) -> List[URLMetadata]:
""" """
Extract video URLs from a Discord message. Extract video URLs from a Discord message.
Args: Args:
message: Discord message to extract URLs from message: Discord message to extract URLs from
enabled_sites: Optional list of enabled site identifiers enabled_sites: Optional list of enabled site identifiers
Returns: Returns:
List of URLMetadata objects for extracted URLs List of URLMetadata objects for extracted URLs
""" """
return await url_extractor.extract_urls(message, enabled_sites) return await url_extractor.extract_urls(message, enabled_sites)
async def validate_message( async def validate_message(
message: discord.Message, message: discord.Message, settings: Dict[str, Any]
settings: Dict[str, Any]
) -> Tuple[bool, Optional[str]]: ) -> Tuple[bool, Optional[str]]:
""" """
Validate a Discord message. Validate a Discord message.
Args: Args:
message: Discord message to validate message: Discord message to validate
settings: Guild settings dictionary settings: Guild settings dictionary
Returns: Returns:
Tuple of (is_valid, reason) Tuple of (is_valid, reason)
Raises: Raises:
ValidationError: If validation fails unexpectedly ValidationError: If validation fails unexpectedly
""" """
return await message_validator.validate_message(message, settings) return await message_validator.validate_message(message, settings)
# Progress tracking helper functions # Progress tracking helper functions
def update_download_progress(url: str, progress_data: Dict[str, Any]) -> None: def update_download_progress(url: str, progress_data: Dict[str, Any]) -> None:
""" """
Update download progress for a specific URL. Update download progress for a specific URL.
Args: Args:
url: The URL being downloaded url: The URL being downloaded
progress_data: Dictionary containing progress information progress_data: Dictionary containing progress information
""" """
progress_tracker.update_download_progress(url, progress_data) progress_tracker.update_download_progress(url, progress_data)
def complete_download(url: str) -> None: def complete_download(url: str) -> None:
""" """
Mark a download as complete. Mark a download as complete.
Args: Args:
url: The URL that completed downloading url: The URL that completed downloading
""" """
progress_tracker.complete_download(url) progress_tracker.complete_download(url)
def increment_download_retries(url: str) -> None: def increment_download_retries(url: str) -> None:
""" """
Increment retry count for a download. Increment retry count for a download.
Args: Args:
url: The URL being retried url: The URL being retried
""" """
progress_tracker.increment_download_retries(url) progress_tracker.increment_download_retries(url)
def get_download_progress(url: Optional[str] = None) -> Union[Dict[str, Any], Dict[str, Dict[str, Any]]]:
def get_download_progress(
url: Optional[str] = None,
) -> Union[Dict[str, Any], Dict[str, Dict[str, Any]]]:
""" """
Get download progress for a specific URL or all downloads. Get download progress for a specific URL or all downloads.
Args: Args:
url: Optional URL to get progress for. If None, returns all download progress. url: Optional URL to get progress for. If None, returns all download progress.
Returns: Returns:
Dictionary containing progress information for one or all downloads Dictionary containing progress information for one or all downloads
""" """
return progress_tracker.get_download_progress(url) return progress_tracker.get_download_progress(url)
def get_active_operations() -> Dict[str, Dict[str, Any]]: def get_active_operations() -> Dict[str, Dict[str, Any]]:
""" """
Get all active operations. Get all active operations.
Returns: Returns:
Dictionary containing information about all active operations Dictionary containing information about all active operations
""" """
return progress_tracker.get_active_operations() return progress_tracker.get_active_operations()
def get_validation_stats() -> ValidationStats: def get_validation_stats() -> ValidationStats:
""" """
Get message validation statistics. Get message validation statistics.
Returns: Returns:
Dictionary containing validation statistics and rule information Dictionary containing validation statistics and rule information
""" """
return message_validator.get_stats() return message_validator.get_stats()
def clear_caches(message_id: Optional[int] = None) -> None: def clear_caches(message_id: Optional[int] = None) -> None:
""" """
Clear URL and validation caches. Clear URL and validation caches.
Args: Args:
message_id: Optional message ID to clear caches for. If None, clears all caches. message_id: Optional message ID to clear caches for. If None, clears all caches.
""" """

6
videoarchiver/processor/cleanup_manager.py
View File

@@ -18,9 +18,9 @@ from typing import (
) )
from datetime import datetime, timedelta from datetime import datetime, timedelta
from .processor.queue_handler import QueueHandler from ..processor.queue_handler import QueueHandler
from .ffmpeg.ffmpeg_manager import FFmpegManager from ..ffmpeg.ffmpeg_manager import FFmpegManager
from .utils.exceptions import CleanupError from ..utils.exceptions import CleanupError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

161
videoarchiver/processor/core.py
View File

@@ -1,29 +1,33 @@
"""Core VideoProcessor class that manages video processing operations""" """Core VideoProcessor class that manages video processing operations"""
import logging
import asyncio import asyncio
from enum import Enum, auto import logging
from typing import Optional, Tuple, Dict, Any, List, TypedDict, ClassVar
from datetime import datetime, timedelta from datetime import datetime, timedelta
import discord from enum import auto, Enum
from discord.ext import commands from typing import Any, ClassVar, Dict, List, Optional, Tuple, TypedDict
from .processor.message_handler import MessageHandler import discord # type: ignore
from .processor.queue_handler import QueueHandler from discord.ext import commands # type: ignore
from .utils import progress_tracker
from .processor.status_display import StatusDisplay from ..config_manager import ConfigManager
from .processor.cleanup_manager import CleanupManager, CleanupStrategy from ..database.video_archive_db import VideoArchiveDB
from .processor.constants import REACTIONS from ..ffmpeg.ffmpeg_manager import FFmpegManager
from .queue.manager import EnhancedVideoQueueManager from ..processor.cleanup_manager import CleanupManager, CleanupStrategy
from .ffmpeg.ffmpeg_manager import FFmpegManager from ..processor.constants import REACTIONS
from .database.video_archive_db import VideoArchiveDB
from .config_manager import ConfigManager from ..processor.message_handler import MessageHandler
from .utils.exceptions import ProcessorError from ..processor.queue_handler import QueueHandler
from ..processor.status_display import StatusDisplay
from ..queue.manager import EnhancedVideoQueueManager
from ..utils import progress_tracker
from ..utils.exceptions import ProcessorError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class ProcessorState(Enum): class ProcessorState(Enum):
"""Possible states of the video processor""" """Possible states of the video processor"""
INITIALIZING = auto() INITIALIZING = auto()
READY = auto() READY = auto()
PROCESSING = auto() PROCESSING = auto()
@@ -31,15 +35,19 @@ class ProcessorState(Enum):
ERROR = auto() ERROR = auto()
SHUTDOWN = auto() SHUTDOWN = auto()
class OperationType(Enum): class OperationType(Enum):
"""Types of processor operations""" """Types of processor operations"""
MESSAGE_PROCESSING = auto() MESSAGE_PROCESSING = auto()
VIDEO_PROCESSING = auto() VIDEO_PROCESSING = auto()
QUEUE_MANAGEMENT = auto() QUEUE_MANAGEMENT = auto()
CLEANUP = auto() CLEANUP = auto()
class OperationDetails(TypedDict): class OperationDetails(TypedDict):
"""Type definition for operation details""" """Type definition for operation details"""
type: str type: str
start_time: datetime start_time: datetime
end_time: Optional[datetime] end_time: Optional[datetime]
@@ -47,16 +55,20 @@ class OperationDetails(TypedDict):
details: Dict[str, Any] details: Dict[str, Any]
error: Optional[str] error: Optional[str]
class OperationStats(TypedDict): class OperationStats(TypedDict):
"""Type definition for operation statistics""" """Type definition for operation statistics"""
total_operations: int total_operations: int
active_operations: int active_operations: int
success_count: int success_count: int
error_count: int error_count: int
success_rate: float success_rate: float
class ProcessorStatus(TypedDict): class ProcessorStatus(TypedDict):
"""Type definition for processor status""" """Type definition for processor status"""
state: str state: str
health: bool health: bool
operations: OperationStats operations: OperationStats
@@ -64,6 +76,7 @@ class ProcessorStatus(TypedDict):
last_health_check: Optional[str] last_health_check: Optional[str]
health_status: Dict[str, bool] health_status: Dict[str, bool]
class OperationTracker: class OperationTracker:
"""Tracks processor operations""" """Tracks processor operations"""
@@ -75,18 +88,14 @@ class OperationTracker:
self.error_count = 0 self.error_count = 0
self.success_count = 0 self.success_count = 0
def start_operation( def start_operation(self, op_type: OperationType, details: Dict[str, Any]) -> str:
self,
op_type: OperationType,
details: Dict[str, Any]
) -> str:
""" """
Start tracking an operation. Start tracking an operation.
Args: Args:
op_type: Type of operation op_type: Type of operation
details: Operation details details: Operation details
Returns: Returns:
Operation ID string Operation ID string
""" """
@@ -97,30 +106,29 @@ class OperationTracker:
end_time=None, end_time=None,
status="running", status="running",
details=details, details=details,
error=None error=None,
) )
return op_id return op_id
def end_operation( def end_operation(
self, self, op_id: str, success: bool, error: Optional[str] = None
op_id: str,
success: bool,
error: Optional[str] = None
) -> None: ) -> None:
""" """
End tracking an operation. End tracking an operation.
Args: Args:
op_id: Operation ID op_id: Operation ID
success: Whether operation succeeded success: Whether operation succeeded
error: Optional error message error: Optional error message
""" """
if op_id in self.operations: if op_id in self.operations:
self.operations[op_id].update({ self.operations[op_id].update(
"end_time": datetime.utcnow(), {
"status": "success" if success else "error", "end_time": datetime.utcnow(),
"error": error "status": "success" if success else "error",
}) "error": error,
}
)
# Move to history # Move to history
self.operation_history.append(self.operations.pop(op_id)) self.operation_history.append(self.operations.pop(op_id))
# Update counts # Update counts
@@ -131,12 +139,12 @@ class OperationTracker:
# Cleanup old history if needed # Cleanup old history if needed
if len(self.operation_history) > self.MAX_HISTORY: if len(self.operation_history) > self.MAX_HISTORY:
self.operation_history = self.operation_history[-self.MAX_HISTORY:] self.operation_history = self.operation_history[-self.MAX_HISTORY :]
def get_active_operations(self) -> Dict[str, OperationDetails]: def get_active_operations(self) -> Dict[str, OperationDetails]:
""" """
Get currently active operations. Get currently active operations.
Returns: Returns:
Dictionary of active operations Dictionary of active operations
""" """
@@ -145,7 +153,7 @@ class OperationTracker:
def get_operation_stats(self) -> OperationStats: def get_operation_stats(self) -> OperationStats:
""" """
Get operation statistics. Get operation statistics.
Returns: Returns:
Dictionary containing operation statistics Dictionary containing operation statistics
""" """
@@ -155,9 +163,10 @@ class OperationTracker:
active_operations=len(self.operations), active_operations=len(self.operations),
success_count=self.success_count, success_count=self.success_count,
error_count=self.error_count, error_count=self.error_count,
success_rate=self.success_count / total if total > 0 else 0.0 success_rate=self.success_count / total if total > 0 else 0.0,
) )
class HealthMonitor: class HealthMonitor:
"""Monitors processor health""" """Monitors processor health"""
@@ -165,7 +174,7 @@ class HealthMonitor:
ERROR_CHECK_INTERVAL: ClassVar[int] = 30 # Seconds between checks after error ERROR_CHECK_INTERVAL: ClassVar[int] = 30 # Seconds between checks after error
SUCCESS_RATE_THRESHOLD: ClassVar[float] = 0.9 # 90% success rate threshold SUCCESS_RATE_THRESHOLD: ClassVar[float] = 0.9 # 90% success rate threshold
def __init__(self, processor: 'VideoProcessor') -> None: def __init__(self, processor: "VideoProcessor") -> None:
self.processor = processor self.processor = processor
self.last_check: Optional[datetime] = None self.last_check: Optional[datetime] = None
self.health_status: Dict[str, bool] = {} self.health_status: Dict[str, bool] = {}
@@ -191,13 +200,15 @@ class HealthMonitor:
while True: while True:
try: try:
self.last_check = datetime.utcnow() self.last_check = datetime.utcnow()
# Check component health # Check component health
self.health_status.update({ self.health_status.update(
"queue_handler": self.processor.queue_handler.is_healthy(), {
"message_handler": self.processor.message_handler.is_healthy(), "queue_handler": self.processor.queue_handler.is_healthy(),
"progress_tracker": progress_tracker.is_healthy() "message_handler": self.processor.message_handler.is_healthy(),
}) "progress_tracker": progress_tracker.is_healthy(),
}
)
# Check operation health # Check operation health
op_stats = self.processor.operation_tracker.get_operation_stats() op_stats = self.processor.operation_tracker.get_operation_stats()
@@ -214,12 +225,13 @@ class HealthMonitor:
def is_healthy(self) -> bool: def is_healthy(self) -> bool:
""" """
Check if processor is healthy. Check if processor is healthy.
Returns: Returns:
True if all components are healthy, False otherwise True if all components are healthy, False otherwise
""" """
return all(self.health_status.values()) return all(self.health_status.values())
class VideoProcessor: class VideoProcessor:
"""Handles video processing operations""" """Handles video processing operations"""
@@ -230,7 +242,7 @@ class VideoProcessor:
components: Dict[int, Dict[str, Any]], components: Dict[int, Dict[str, Any]],
queue_manager: Optional[EnhancedVideoQueueManager] = None, queue_manager: Optional[EnhancedVideoQueueManager] = None,
ffmpeg_mgr: Optional[FFmpegManager] = None, ffmpeg_mgr: Optional[FFmpegManager] = None,
db: Optional[VideoArchiveDB] = None db: Optional[VideoArchiveDB] = None,
) -> None: ) -> None:
self.bot = bot self.bot = bot
self.config = config_manager self.config = config_manager
@@ -249,9 +261,7 @@ class VideoProcessor:
self.queue_handler = QueueHandler(bot, config_manager, components) self.queue_handler = QueueHandler(bot, config_manager, components)
self.message_handler = MessageHandler(bot, config_manager, queue_manager) self.message_handler = MessageHandler(bot, config_manager, queue_manager)
self.cleanup_manager = CleanupManager( self.cleanup_manager = CleanupManager(
self.queue_handler, self.queue_handler, ffmpeg_mgr, CleanupStrategy.NORMAL
ffmpeg_mgr,
CleanupStrategy.NORMAL
) )
# Pass db to queue handler if it exists # Pass db to queue handler if it exists
@@ -260,7 +270,7 @@ class VideoProcessor:
# Store queue task reference # Store queue task reference
self._queue_task: Optional[asyncio.Task] = None self._queue_task: Optional[asyncio.Task] = None
# Mark as ready # Mark as ready
self.state = ProcessorState.READY self.state = ProcessorState.READY
logger.info("VideoProcessor initialized successfully") logger.info("VideoProcessor initialized successfully")
@@ -273,7 +283,7 @@ class VideoProcessor:
async def start(self) -> None: async def start(self) -> None:
""" """
Start processor operations. Start processor operations.
Raises: Raises:
ProcessorError: If startup fails ProcessorError: If startup fails
""" """
@@ -288,21 +298,20 @@ class VideoProcessor:
async def process_video(self, item: Any) -> Tuple[bool, Optional[str]]: async def process_video(self, item: Any) -> Tuple[bool, Optional[str]]:
""" """
Process a video from the queue. Process a video from the queue.
Args: Args:
item: Queue item to process item: Queue item to process
Returns: Returns:
Tuple of (success, error_message) Tuple of (success, error_message)
Raises: Raises:
ProcessorError: If processing fails ProcessorError: If processing fails
""" """
op_id = self.operation_tracker.start_operation( op_id = self.operation_tracker.start_operation(
OperationType.VIDEO_PROCESSING, OperationType.VIDEO_PROCESSING, {"item": str(item)}
{"item": str(item)}
) )
try: try:
self.state = ProcessorState.PROCESSING self.state = ProcessorState.PROCESSING
result = await self.queue_handler.process_video(item) result = await self.queue_handler.process_video(item)
@@ -321,18 +330,17 @@ class VideoProcessor:
async def process_message(self, message: discord.Message) -> None: async def process_message(self, message: discord.Message) -> None:
""" """
Process a message for video content. Process a message for video content.
Args: Args:
message: Discord message to process message: Discord message to process
Raises: Raises:
ProcessorError: If processing fails ProcessorError: If processing fails
""" """
op_id = self.operation_tracker.start_operation( op_id = self.operation_tracker.start_operation(
OperationType.MESSAGE_PROCESSING, OperationType.MESSAGE_PROCESSING, {"message_id": message.id}
{"message_id": message.id}
) )
try: try:
await self.message_handler.process_message(message) await self.message_handler.process_message(message)
self.operation_tracker.end_operation(op_id, True) self.operation_tracker.end_operation(op_id, True)
@@ -345,15 +353,14 @@ class VideoProcessor:
async def cleanup(self) -> None: async def cleanup(self) -> None:
""" """
Clean up resources and stop processing. Clean up resources and stop processing.
Raises: Raises:
ProcessorError: If cleanup fails ProcessorError: If cleanup fails
""" """
op_id = self.operation_tracker.start_operation( op_id = self.operation_tracker.start_operation(
OperationType.CLEANUP, OperationType.CLEANUP, {"type": "normal"}
{"type": "normal"}
) )
try: try:
self.state = ProcessorState.SHUTDOWN self.state = ProcessorState.SHUTDOWN
await self.health_monitor.stop_monitoring() await self.health_monitor.stop_monitoring()
@@ -368,15 +375,14 @@ class VideoProcessor:
async def force_cleanup(self) -> None: async def force_cleanup(self) -> None:
""" """
Force cleanup of resources. Force cleanup of resources.
Raises: Raises:
ProcessorError: If force cleanup fails ProcessorError: If force cleanup fails
""" """
op_id = self.operation_tracker.start_operation( op_id = self.operation_tracker.start_operation(
OperationType.CLEANUP, OperationType.CLEANUP, {"type": "force"}
{"type": "force"}
) )
try: try:
self.state = ProcessorState.SHUTDOWN self.state = ProcessorState.SHUTDOWN
await self.health_monitor.stop_monitoring() await self.health_monitor.stop_monitoring()
@@ -391,7 +397,7 @@ class VideoProcessor:
async def show_queue_details(self, ctx: commands.Context) -> None: async def show_queue_details(self, ctx: commands.Context) -> None:
""" """
Display detailed queue status. Display detailed queue status.
Args: Args:
ctx: Command context ctx: Command context
""" """
@@ -402,14 +408,13 @@ class VideoProcessor:
# Get queue status # Get queue status
queue_status = self.queue_manager.get_queue_status(ctx.guild.id) queue_status = self.queue_manager.get_queue_status(ctx.guild.id)
# Get active operations # Get active operations
active_ops = self.operation_tracker.get_active_operations() active_ops = self.operation_tracker.get_active_operations()
# Create and send status embed # Create and send status embed
embed = await StatusDisplay.create_queue_status_embed( embed = await StatusDisplay.create_queue_status_embed(
queue_status, queue_status, active_ops
active_ops
) )
await ctx.send(embed=embed) await ctx.send(embed=embed)
@@ -421,7 +426,7 @@ class VideoProcessor:
def set_queue_task(self, task: asyncio.Task) -> None: def set_queue_task(self, task: asyncio.Task) -> None:
""" """
Set the queue processing task. Set the queue processing task.
Args: Args:
task: Queue processing task task: Queue processing task
""" """
@@ -431,7 +436,7 @@ class VideoProcessor:
def get_status(self) -> ProcessorStatus: def get_status(self) -> ProcessorStatus:
""" """
Get processor status. Get processor status.
Returns: Returns:
Dictionary containing processor status information Dictionary containing processor status information
""" """
@@ -445,5 +450,5 @@ class VideoProcessor:
if self.health_monitor.last_check if self.health_monitor.last_check
else None else None
), ),
health_status=self.health_monitor.health_status health_status=self.health_monitor.health_status,
) )

155
videoarchiver/processor/message_handler.py
View File

@@ -1,25 +1,29 @@
"""Message processing and URL extraction for VideoProcessor""" """Message processing and URL extraction for VideoProcessor"""
import logging
import asyncio import asyncio
from enum import Enum, auto import logging
from typing import Optional, Dict, Any, List, Tuple, Set, TypedDict, ClassVar
from datetime import datetime, timedelta from datetime import datetime, timedelta
import discord from enum import auto, Enum
from discord.ext import commands from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, TypedDict
from .processor.url_extractor import URLExtractor, URLMetadata import discord # type: ignore
from .processor.message_validator import MessageValidator, ValidationError from discord.ext import commands # type: ignore
from .processor.queue_processor import QueueProcessor, QueuePriority
from .processor.constants import REACTIONS from ..config_manager import ConfigManager
from .queue.manager import EnhancedVideoQueueManager from ..processor.constants import REACTIONS
from .config_manager import ConfigManager from ..processor.message_validator import MessageValidator, ValidationError
from .utils.exceptions import MessageHandlerError from ..processor.queue_processor import QueuePriority, QueueProcessor
from ..processor.url_extractor import URLExtractor, URLMetadata
from ..queue.manager import EnhancedVideoQueueManager
from ..utils.exceptions import MessageHandlerError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class MessageState(Enum): class MessageState(Enum):
"""Possible states of message processing""" """Possible states of message processing"""
RECEIVED = auto() RECEIVED = auto()
VALIDATING = auto() VALIDATING = auto()
EXTRACTING = auto() EXTRACTING = auto()
@@ -28,21 +32,27 @@ class MessageState(Enum):
FAILED = auto() FAILED = auto()
IGNORED = auto() IGNORED = auto()
class ProcessingStage(Enum): class ProcessingStage(Enum):
"""Message processing stages""" """Message processing stages"""
VALIDATION = auto() VALIDATION = auto()
EXTRACTION = auto() EXTRACTION = auto()
QUEUEING = auto() QUEUEING = auto()
COMPLETION = auto() COMPLETION = auto()
class MessageCacheEntry(TypedDict): class MessageCacheEntry(TypedDict):
"""Type definition for message cache entry""" """Type definition for message cache entry"""
valid: bool valid: bool
reason: Optional[str] reason: Optional[str]
timestamp: str timestamp: str
class MessageStatus(TypedDict): class MessageStatus(TypedDict):
"""Type definition for message status""" """Type definition for message status"""
state: Optional[MessageState] state: Optional[MessageState]
stage: Optional[ProcessingStage] stage: Optional[ProcessingStage]
error: Optional[str] error: Optional[str]
@@ -50,6 +60,7 @@ class MessageStatus(TypedDict):
end_time: Optional[datetime] end_time: Optional[datetime]
duration: Optional[float] duration: Optional[float]
class MessageCache: class MessageCache:
"""Caches message validation results""" """Caches message validation results"""
@@ -61,7 +72,7 @@ class MessageCache:
def add(self, message_id: int, result: MessageCacheEntry) -> None: def add(self, message_id: int, result: MessageCacheEntry) -> None:
""" """
Add a result to cache. Add a result to cache.
Args: Args:
message_id: Discord message ID message_id: Discord message ID
result: Validation result entry result: Validation result entry
@@ -74,10 +85,10 @@ class MessageCache:
def get(self, message_id: int) -> Optional[MessageCacheEntry]: def get(self, message_id: int) -> Optional[MessageCacheEntry]:
""" """
Get a cached result. Get a cached result.
Args: Args:
message_id: Discord message ID message_id: Discord message ID
Returns: Returns:
Cached validation entry or None if not found Cached validation entry or None if not found
""" """
@@ -94,6 +105,7 @@ class MessageCache:
del self._cache[oldest] del self._cache[oldest]
del self._access_times[oldest] del self._access_times[oldest]
class ProcessingTracker: class ProcessingTracker:
"""Tracks message processing state and progress""" """Tracks message processing state and progress"""
@@ -109,7 +121,7 @@ class ProcessingTracker:
def start_processing(self, message_id: int) -> None: def start_processing(self, message_id: int) -> None:
""" """
Start tracking a message. Start tracking a message.
Args: Args:
message_id: Discord message ID message_id: Discord message ID
""" """
@@ -121,11 +133,11 @@ class ProcessingTracker:
message_id: int, message_id: int,
state: MessageState, state: MessageState,
stage: Optional[ProcessingStage] = None, stage: Optional[ProcessingStage] = None,
error: Optional[str] = None error: Optional[str] = None,
) -> None: ) -> None:
""" """
Update message state. Update message state.
Args: Args:
message_id: Discord message ID message_id: Discord message ID
state: New message state state: New message state
@@ -143,16 +155,16 @@ class ProcessingTracker:
def get_status(self, message_id: int) -> MessageStatus: def get_status(self, message_id: int) -> MessageStatus:
""" """
Get processing status for a message. Get processing status for a message.
Args: Args:
message_id: Discord message ID message_id: Discord message ID
Returns: Returns:
Dictionary containing message status information Dictionary containing message status information
""" """
end_time = self.end_times.get(message_id) end_time = self.end_times.get(message_id)
start_time = self.start_times.get(message_id) start_time = self.start_times.get(message_id)
return MessageStatus( return MessageStatus(
state=self.states.get(message_id), state=self.states.get(message_id),
stage=self.stages.get(message_id), stage=self.stages.get(message_id),
@@ -163,29 +175,32 @@ class ProcessingTracker:
(end_time - start_time).total_seconds() (end_time - start_time).total_seconds()
if end_time and start_time if end_time and start_time
else None else None
) ),
) )
def is_message_stuck(self, message_id: int) -> bool: def is_message_stuck(self, message_id: int) -> bool:
""" """
Check if a message is stuck in processing. Check if a message is stuck in processing.
Args: Args:
message_id: Discord message ID message_id: Discord message ID
Returns: Returns:
True if message is stuck, False otherwise True if message is stuck, False otherwise
""" """
if message_id not in self.states or message_id not in self.start_times: if message_id not in self.states or message_id not in self.start_times:
return False return False
state = self.states[message_id] state = self.states[message_id]
if state in (MessageState.COMPLETED, MessageState.FAILED, MessageState.IGNORED): if state in (MessageState.COMPLETED, MessageState.FAILED, MessageState.IGNORED):
return False return False
processing_time = (datetime.utcnow() - self.start_times[message_id]).total_seconds() processing_time = (
datetime.utcnow() - self.start_times[message_id]
).total_seconds()
return processing_time > self.MAX_PROCESSING_TIME return processing_time > self.MAX_PROCESSING_TIME
class MessageHandler: class MessageHandler:
"""Handles processing of messages for video content""" """Handles processing of messages for video content"""
@@ -193,14 +208,14 @@ class MessageHandler:
self, self,
bot: discord.Client, bot: discord.Client,
config_manager: ConfigManager, config_manager: ConfigManager,
queue_manager: EnhancedVideoQueueManager queue_manager: EnhancedVideoQueueManager,
) -> None: ) -> None:
self.bot = bot self.bot = bot
self.config_manager = config_manager self.config_manager = config_manager
self.url_extractor = URLExtractor() self.url_extractor = URLExtractor()
self.message_validator = MessageValidator() self.message_validator = MessageValidator()
self.queue_processor = QueueProcessor(queue_manager) self.queue_processor = QueueProcessor(queue_manager)
# Initialize tracking and caching # Initialize tracking and caching
self.tracker = ProcessingTracker() self.tracker = ProcessingTracker()
self.validation_cache = MessageCache() self.validation_cache = MessageCache()
@@ -209,10 +224,10 @@ class MessageHandler:
async def process_message(self, message: discord.Message) -> None: async def process_message(self, message: discord.Message) -> None:
""" """
Process a message for video content. Process a message for video content.
Args: Args:
message: Discord message to process message: Discord message to process
Raises: Raises:
MessageHandlerError: If there's an error during processing MessageHandlerError: If there's an error during processing
""" """
@@ -224,11 +239,7 @@ class MessageHandler:
await self._process_message_internal(message) await self._process_message_internal(message)
except Exception as e: except Exception as e:
logger.error(f"Error processing message: {str(e)}", exc_info=True) logger.error(f"Error processing message: {str(e)}", exc_info=True)
self.tracker.update_state( self.tracker.update_state(message.id, MessageState.FAILED, error=str(e))
message.id,
MessageState.FAILED,
error=str(e)
)
try: try:
await message.add_reaction(REACTIONS["error"]) await message.add_reaction(REACTIONS["error"])
except Exception as react_error: except Exception as react_error:
@@ -237,10 +248,10 @@ class MessageHandler:
async def _process_message_internal(self, message: discord.Message) -> None: async def _process_message_internal(self, message: discord.Message) -> None:
""" """
Internal message processing logic. Internal message processing logic.
Args: Args:
message: Discord message to process message: Discord message to process
Raises: Raises:
MessageHandlerError: If there's an error during processing MessageHandlerError: If there's an error during processing
""" """
@@ -260,43 +271,38 @@ class MessageHandler:
else: else:
# Validate message # Validate message
self.tracker.update_state( self.tracker.update_state(
message.id, message.id, MessageState.VALIDATING, ProcessingStage.VALIDATION
MessageState.VALIDATING,
ProcessingStage.VALIDATION
) )
try: try:
is_valid, reason = await self.message_validator.validate_message( is_valid, reason = await self.message_validator.validate_message(
message, message, settings
settings
) )
# Cache result # Cache result
self.validation_cache.add(message.id, MessageCacheEntry( self.validation_cache.add(
valid=is_valid, message.id,
reason=reason, MessageCacheEntry(
timestamp=datetime.utcnow().isoformat() valid=is_valid,
)) reason=reason,
timestamp=datetime.utcnow().isoformat(),
),
)
except ValidationError as e: except ValidationError as e:
raise MessageHandlerError(f"Validation failed: {str(e)}") raise MessageHandlerError(f"Validation failed: {str(e)}")
if not is_valid: if not is_valid:
logger.debug(f"Message validation failed: {reason}") logger.debug(f"Message validation failed: {reason}")
self.tracker.update_state( self.tracker.update_state(
message.id, message.id, MessageState.IGNORED, error=reason
MessageState.IGNORED,
error=reason
) )
return return
# Extract URLs # Extract URLs
self.tracker.update_state( self.tracker.update_state(
message.id, message.id, MessageState.EXTRACTING, ProcessingStage.EXTRACTION
MessageState.EXTRACTING,
ProcessingStage.EXTRACTION
) )
try: try:
urls: List[URLMetadata] = await self.url_extractor.extract_urls( urls: List[URLMetadata] = await self.url_extractor.extract_urls(
message, message, enabled_sites=settings.get("enabled_sites")
enabled_sites=settings.get("enabled_sites")
) )
if not urls: if not urls:
logger.debug("No valid URLs found in message") logger.debug("No valid URLs found in message")
@@ -307,24 +313,18 @@ class MessageHandler:
# Process URLs # Process URLs
self.tracker.update_state( self.tracker.update_state(
message.id, message.id, MessageState.PROCESSING, ProcessingStage.QUEUEING
MessageState.PROCESSING,
ProcessingStage.QUEUEING
) )
try: try:
await self.queue_processor.process_urls( await self.queue_processor.process_urls(
message, message, urls, priority=QueuePriority.NORMAL
urls,
priority=QueuePriority.NORMAL
) )
except Exception as e: except Exception as e:
raise MessageHandlerError(f"Queue processing failed: {str(e)}") raise MessageHandlerError(f"Queue processing failed: {str(e)}")
# Mark completion # Mark completion
self.tracker.update_state( self.tracker.update_state(
message.id, message.id, MessageState.COMPLETED, ProcessingStage.COMPLETION
MessageState.COMPLETED,
ProcessingStage.COMPLETION
) )
except MessageHandlerError: except MessageHandlerError:
@@ -333,35 +333,28 @@ class MessageHandler:
raise MessageHandlerError(f"Unexpected error: {str(e)}") raise MessageHandlerError(f"Unexpected error: {str(e)}")
async def format_archive_message( async def format_archive_message(
self, self, author: Optional[discord.Member], channel: discord.TextChannel, url: str
author: Optional[discord.Member],
channel: discord.TextChannel,
url: str
) -> str: ) -> str:
""" """
Format message for archive channel. Format message for archive channel.
Args: Args:
author: Optional message author author: Optional message author
channel: Channel the message was posted in channel: Channel the message was posted in
url: URL being archived url: URL being archived
Returns: Returns:
Formatted message string Formatted message string
""" """
return await self.queue_processor.format_archive_message( return await self.queue_processor.format_archive_message(author, channel, url)
author,
channel,
url
)
def get_message_status(self, message_id: int) -> MessageStatus: def get_message_status(self, message_id: int) -> MessageStatus:
""" """
Get processing status for a message. Get processing status for a message.
Args: Args:
message_id: Discord message ID message_id: Discord message ID
Returns: Returns:
Dictionary containing message status information Dictionary containing message status information
""" """
@@ -370,7 +363,7 @@ class MessageHandler:
def is_healthy(self) -> bool: def is_healthy(self) -> bool:
""" """
Check if handler is healthy. Check if handler is healthy.
Returns: Returns:
True if handler is healthy, False otherwise True if handler is healthy, False otherwise
""" """
@@ -378,7 +371,9 @@ class MessageHandler:
# Check for any stuck messages # Check for any stuck messages
for message_id in self.tracker.states: for message_id in self.tracker.states:
if self.tracker.is_message_stuck(message_id): if self.tracker.is_message_stuck(message_id):
logger.warning(f"Message {message_id} appears to be stuck in processing") logger.warning(
f"Message {message_id} appears to be stuck in processing"
)
return False return False
return True return True
except Exception as e: except Exception as e:

4
videoarchiver/processor/message_validator.py
View File

@@ -5,9 +5,9 @@ from enum import Enum, auto
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Dict, Optional, Tuple, List, Any, Callable, Set, TypedDict, ClassVar from typing import Dict, Optional, Tuple, List, Any, Callable, Set, TypedDict, ClassVar
from datetime import datetime from datetime import datetime
import discord import discord # type: ignore
from .utils.exceptions import ValidationError from ..utils.exceptions import ValidationError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

27
videoarchiver/processor/queue_handler.py
View File

@@ -6,29 +6,33 @@ import os
from enum import Enum, auto from enum import Enum, auto
from typing import Optional, Dict, Any, List, Tuple, Set, TypedDict, ClassVar, Callable from typing import Optional, Dict, Any, List, Tuple, Set, TypedDict, ClassVar, Callable
from datetime import datetime from datetime import datetime
import discord import discord # type: ignore
from .utils import progress_tracker from ..utils import progress_tracker
from .database.video_archive_db import VideoArchiveDB from ..database.video_archive_db import VideoArchiveDB
from .utils.download_manager import DownloadManager from ..utils.download_manager import DownloadManager
from .utils.message_manager import MessageManager from ..utils.message_manager import MessageManager
from .utils.exceptions import QueueHandlerError from ..utils.exceptions import QueueHandlerError
from .queue.models import QueueItem from ..queue.models import QueueItem
from .config_manager import ConfigManager from ..config_manager import ConfigManager
from .processor.constants import REACTIONS from ..processor.constants import REACTIONS
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class QueueItemStatus(Enum): class QueueItemStatus(Enum):
"""Status of a queue item""" """Status of a queue item"""
PENDING = auto() PENDING = auto()
PROCESSING = auto() PROCESSING = auto()
COMPLETED = auto() COMPLETED = auto()
FAILED = auto() FAILED = auto()
CANCELLED = auto() CANCELLED = auto()
class QueueStats(TypedDict): class QueueStats(TypedDict):
"""Type definition for queue statistics""" """Type definition for queue statistics"""
active_downloads: int active_downloads: int
processing_items: int processing_items: int
completed_items: int completed_items: int
@@ -37,6 +41,7 @@ class QueueStats(TypedDict):
last_processed: Optional[str] last_processed: Optional[str]
is_healthy: bool is_healthy: bool
class QueueHandler: class QueueHandler:
"""Handles queue processing and video operations""" """Handles queue processing and video operations"""
@@ -48,7 +53,7 @@ class QueueHandler:
bot: discord.Client, bot: discord.Client,
config_manager: ConfigManager, config_manager: ConfigManager,
components: Dict[int, Dict[str, Any]], components: Dict[int, Dict[str, Any]],
db: Optional[VideoArchiveDB] = None db: Optional[VideoArchiveDB] = None,
) -> None: ) -> None:
self.bot = bot self.bot = bot
self.config_manager = config_manager self.config_manager = config_manager
@@ -64,7 +69,7 @@ class QueueHandler:
"failed_items": 0, "failed_items": 0,
"average_processing_time": 0.0, "average_processing_time": 0.0,
"last_processed": None, "last_processed": None,
"is_healthy": True "is_healthy": True,
} }
async def process_video(self, item: QueueItem) -> Tuple[bool, Optional[str]]: async def process_video(self, item: QueueItem) -> Tuple[bool, Optional[str]]:

75
videoarchiver/processor/queue_processor.py
View File

@@ -5,106 +5,111 @@ import asyncio
from enum import Enum, auto from enum import Enum, auto
from typing import List, Optional, Dict, Any, Set, Union, TypedDict, ClassVar from typing import List, Optional, Dict, Any, Set, Union, TypedDict, ClassVar
from datetime import datetime from datetime import datetime
import discord import discord # type: ignore
from .queue.models import QueueItem from ..queue.models import QueueItem
from .queue.manager import EnhancedVideoQueueManager from ..queue.manager import EnhancedVideoQueueManager
from .processor.constants import REACTIONS from ..processor.constants import REACTIONS
from .processor.url_extractor import URLMetadata from ..processor.url_extractor import URLMetadata
from .utils.exceptions import QueueProcessingError from ..utils.exceptions import QueueProcessingError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class QueuePriority(Enum): class QueuePriority(Enum):
"""Priority levels for queue processing""" """Priority levels for queue processing"""
HIGH = auto() HIGH = auto()
NORMAL = auto() NORMAL = auto()
LOW = auto() LOW = auto()
class QueueMetrics(TypedDict): class QueueMetrics(TypedDict):
"""Type definition for queue metrics""" """Type definition for queue metrics"""
total_items: int total_items: int
processing_time: float processing_time: float
success_rate: float success_rate: float
error_rate: float error_rate: float
average_size: float average_size: float
class QueueProcessor: class QueueProcessor:
"""Handles processing of video queue items""" """Handles processing of video queue items"""
_active_items: ClassVar[Set[int]] = set() _active_items: ClassVar[Set[int]] = set()
_processing_lock: ClassVar[asyncio.Lock] = asyncio.Lock() _processing_lock: ClassVar[asyncio.Lock] = asyncio.Lock()
def __init__(self, queue_manager: EnhancedVideoQueueManager): def __init__(self, queue_manager: EnhancedVideoQueueManager):
self.queue_manager = queue_manager self.queue_manager = queue_manager
self._metrics: Dict[str, Any] = { self._metrics: Dict[str, Any] = {
'processed_count': 0, "processed_count": 0,
'error_count': 0, "error_count": 0,
'total_size': 0, "total_size": 0,
'total_time': 0 "total_time": 0,
} }
async def process_item(self, item: QueueItem) -> bool: async def process_item(self, item: QueueItem) -> bool:
""" """
Process a single queue item Process a single queue item
Args: Args:
item: Queue item to process item: Queue item to process
Returns: Returns:
bool: Success status bool: Success status
""" """
if item.id in self._active_items: if item.id in self._active_items:
logger.warning(f"Item {item.id} is already being processed") logger.warning(f"Item {item.id} is already being processed")
return False return False
try: try:
self._active_items.add(item.id) self._active_items.add(item.id)
start_time = datetime.now() start_time = datetime.now()
# Process item logic here # Process item logic here
# Placeholder for actual video processing # Placeholder for actual video processing
await asyncio.sleep(1) await asyncio.sleep(1)
processing_time = (datetime.now() - start_time).total_seconds() processing_time = (datetime.now() - start_time).total_seconds()
self._update_metrics(processing_time, True, item.size) self._update_metrics(processing_time, True, item.size)
return True return True
except Exception as e: except Exception as e:
logger.error(f"Error processing item {item.id}: {str(e)}") logger.error(f"Error processing item {item.id}: {str(e)}")
self._update_metrics(0, False, 0) self._update_metrics(0, False, 0)
return False return False
finally: finally:
self._active_items.remove(item.id) self._active_items.remove(item.id)
def _update_metrics(self, processing_time: float, success: bool, size: int) -> None: def _update_metrics(self, processing_time: float, success: bool, size: int) -> None:
"""Update processing metrics""" """Update processing metrics"""
self._metrics['processed_count'] += 1 self._metrics["processed_count"] += 1
self._metrics['total_time'] += processing_time self._metrics["total_time"] += processing_time
if not success: if not success:
self._metrics['error_count'] += 1 self._metrics["error_count"] += 1
if size > 0: if size > 0:
self._metrics['total_size'] += size self._metrics["total_size"] += size
def get_metrics(self) -> QueueMetrics: def get_metrics(self) -> QueueMetrics:
"""Get current processing metrics""" """Get current processing metrics"""
total = self._metrics['processed_count'] total = self._metrics["processed_count"]
if total == 0: if total == 0:
return QueueMetrics( return QueueMetrics(
total_items=0, total_items=0,
processing_time=0, processing_time=0,
success_rate=0, success_rate=0,
error_rate=0, error_rate=0,
average_size=0 average_size=0,
) )
return QueueMetrics( return QueueMetrics(
total_items=total, total_items=total,
processing_time=self._metrics['total_time'], processing_time=self._metrics["total_time"],
success_rate=(total - self._metrics['error_count']) / total, success_rate=(total - self._metrics["error_count"]) / total,
error_rate=self._metrics['error_count'] / total, error_rate=self._metrics["error_count"] / total,
average_size=self._metrics['total_size'] / total average_size=self._metrics["total_size"] / total,
) )

49
videoarchiver/processor/reactions.py
View File

@@ -4,22 +4,26 @@ import logging
import asyncio import asyncio
import re import re
from typing import List, Optional from typing import List, Optional
import discord import discord # type: ignore
from urllib.parse import urlparse from urllib.parse import urlparse
from .processor.constants import REACTIONS, ReactionType, get_reaction, get_progress_emoji from ..processor.constants import (
from .database.video_archive_db import VideoArchiveDB REACTIONS,
ReactionType,
get_reaction,
get_progress_emoji,
)
from ..database.video_archive_db import VideoArchiveDB
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
async def handle_archived_reaction( async def handle_archived_reaction(
message: discord.Message, message: discord.Message, user: discord.User, db: VideoArchiveDB
user: discord.User,
db: VideoArchiveDB
) -> None: ) -> None:
""" """
Handle reaction to archived video message. Handle reaction to archived video message.
Args: Args:
message: The Discord message that was reacted to message: The Discord message that was reacted to
user: The user who added the reaction user: The user who added the reaction
@@ -27,7 +31,9 @@ async def handle_archived_reaction(
""" """
try: try:
# Check if the reaction is from a user (not the bot) and is the archived reaction # Check if the reaction is from a user (not the bot) and is the archived reaction
if user.bot or str(message.reactions[0].emoji) != get_reaction(ReactionType.ARCHIVED): if user.bot or str(message.reactions[0].emoji) != get_reaction(
ReactionType.ARCHIVED
):
return return
# Extract URLs from the message using regex # Extract URLs from the message using regex
@@ -37,9 +43,9 @@ async def handle_archived_reaction(
# Check each URL in the database # Check each URL in the database
for url in urls: for url in urls:
# Ensure URL has proper scheme # Ensure URL has proper scheme
if url.startswith('www.'): if url.startswith("www."):
url = 'http://' + url url = "http://" + url
# Validate URL # Validate URL
try: try:
result = urlparse(url) result = urlparse(url)
@@ -59,14 +65,13 @@ async def handle_archived_reaction(
except Exception as e: except Exception as e:
logger.error(f"Error handling archived reaction: {e}", exc_info=True) logger.error(f"Error handling archived reaction: {e}", exc_info=True)
async def update_queue_position_reaction( async def update_queue_position_reaction(
message: discord.Message, message: discord.Message, position: int, bot_user: discord.ClientUser
position: int,
bot_user: discord.ClientUser
) -> None: ) -> None:
""" """
Update queue position reaction. Update queue position reaction.
Args: Args:
message: The Discord message to update reactions on message: The Discord message to update reactions on
position: Queue position (0-based index) position: Queue position (0-based index)
@@ -100,14 +105,13 @@ async def update_queue_position_reaction(
except Exception as e: except Exception as e:
logger.error(f"Failed to update queue position reaction: {e}", exc_info=True) logger.error(f"Failed to update queue position reaction: {e}", exc_info=True)
async def update_progress_reaction( async def update_progress_reaction(
message: discord.Message, message: discord.Message, progress: float, bot_user: discord.ClientUser
progress: float,
bot_user: discord.ClientUser
) -> None: ) -> None:
""" """
Update progress reaction based on FFmpeg progress. Update progress reaction based on FFmpeg progress.
Args: Args:
message: The Discord message to update reactions on message: The Discord message to update reactions on
progress: Progress value between 0 and 100 progress: Progress value between 0 and 100
@@ -142,14 +146,13 @@ async def update_progress_reaction(
except Exception as e: except Exception as e:
logger.error(f"Failed to update progress reaction: {e}", exc_info=True) logger.error(f"Failed to update progress reaction: {e}", exc_info=True)
async def update_download_progress_reaction( async def update_download_progress_reaction(
message: discord.Message, message: discord.Message, progress: float, bot_user: discord.ClientUser
progress: float,
bot_user: discord.ClientUser
) -> None: ) -> None:
""" """
Update download progress reaction. Update download progress reaction.
Args: Args:
message: The Discord message to update reactions on message: The Discord message to update reactions on
progress: Progress value between 0 and 100 progress: Progress value between 0 and 100

260
videoarchiver/processor/status_display.py
View File

@@ -4,40 +4,59 @@ import logging
from enum import Enum, auto from enum import Enum, auto
from dataclasses import dataclass, field from dataclasses import dataclass, field
from datetime import datetime from datetime import datetime
from typing import Dict, Any, List, Optional, Callable, TypeVar, Union, TypedDict, ClassVar, Tuple from typing import (
import discord Dict,
Any,
List,
Optional,
Callable,
TypeVar,
Union,
TypedDict,
ClassVar,
Tuple,
)
import discord # type: ignore
from .utils.exceptions import DisplayError from ..utils.exceptions import DisplayError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
T = TypeVar('T') T = TypeVar("T")
class DisplayTheme(TypedDict): class DisplayTheme(TypedDict):
"""Type definition for display theme""" """Type definition for display theme"""
title_color: discord.Color title_color: discord.Color
success_color: discord.Color success_color: discord.Color
warning_color: discord.Color warning_color: discord.Color
error_color: discord.Color error_color: discord.Color
info_color: discord.Color info_color: discord.Color
class DisplaySection(Enum): class DisplaySection(Enum):
"""Available display sections""" """Available display sections"""
QUEUE_STATS = auto() QUEUE_STATS = auto()
DOWNLOADS = auto() DOWNLOADS = auto()
COMPRESSIONS = auto() COMPRESSIONS = auto()
ERRORS = auto() ERRORS = auto()
HARDWARE = auto() HARDWARE = auto()
class DisplayCondition(Enum): class DisplayCondition(Enum):
"""Display conditions for sections""" """Display conditions for sections"""
HAS_ERRORS = "has_errors" HAS_ERRORS = "has_errors"
HAS_DOWNLOADS = "has_downloads" HAS_DOWNLOADS = "has_downloads"
HAS_COMPRESSIONS = "has_compressions" HAS_COMPRESSIONS = "has_compressions"
@dataclass @dataclass
class DisplayTemplate: class DisplayTemplate:
"""Template for status display sections""" """Template for status display sections"""
name: str name: str
format_string: str format_string: str
inline: bool = False inline: bool = False
@@ -46,27 +65,28 @@ class DisplayTemplate:
formatter: Optional[Callable[[Dict[str, Any]], str]] = None formatter: Optional[Callable[[Dict[str, Any]], str]] = None
max_items: int = field(default=5) # Maximum items to display in lists max_items: int = field(default=5) # Maximum items to display in lists
class StatusFormatter: class StatusFormatter:
"""Formats status information for display""" """Formats status information for display"""
BYTE_UNITS: ClassVar[List[str]] = ['B', 'KB', 'MB', 'GB', 'TB'] BYTE_UNITS: ClassVar[List[str]] = ["B", "KB", "MB", "GB", "TB"]
TIME_THRESHOLDS: ClassVar[List[Tuple[float, str]]] = [ TIME_THRESHOLDS: ClassVar[List[Tuple[float, str]]] = [
(60, 's'), (60, "s"),
(3600, 'm'), (3600, "m"),
(float('inf'), 'h') (float("inf"), "h"),
] ]
@staticmethod @staticmethod
def format_bytes(bytes_value: Union[int, float]) -> str: def format_bytes(bytes_value: Union[int, float]) -> str:
""" """
Format bytes into human readable format. Format bytes into human readable format.
Args: Args:
bytes_value: Number of bytes to format bytes_value: Number of bytes to format
Returns: Returns:
Formatted string with appropriate unit Formatted string with appropriate unit
Raises: Raises:
ValueError: If bytes_value is negative ValueError: If bytes_value is negative
""" """
@@ -84,13 +104,13 @@ class StatusFormatter:
def format_time(seconds: float) -> str: def format_time(seconds: float) -> str:
""" """
Format time duration. Format time duration.
Args: Args:
seconds: Number of seconds to format seconds: Number of seconds to format
Returns: Returns:
Formatted time string Formatted time string
Raises: Raises:
ValueError: If seconds is negative ValueError: If seconds is negative
""" """
@@ -107,13 +127,13 @@ class StatusFormatter:
def format_percentage(value: float) -> str: def format_percentage(value: float) -> str:
""" """
Format percentage value. Format percentage value.
Args: Args:
value: Percentage value to format (0-100) value: Percentage value to format (0-100)
Returns: Returns:
Formatted percentage string Formatted percentage string
Raises: Raises:
ValueError: If value is outside valid range ValueError: If value is outside valid range
""" """
@@ -125,14 +145,14 @@ class StatusFormatter:
def truncate_url(url: str, max_length: int = 50) -> str: def truncate_url(url: str, max_length: int = 50) -> str:
""" """
Truncate URL to specified length. Truncate URL to specified length.
Args: Args:
url: URL to truncate url: URL to truncate
max_length: Maximum length for URL max_length: Maximum length for URL
Returns: Returns:
Truncated URL string Truncated URL string
Raises: Raises:
ValueError: If max_length is less than 4 ValueError: If max_length is less than 4
""" """
@@ -140,6 +160,7 @@ class StatusFormatter:
raise ValueError("max_length must be at least 4") raise ValueError("max_length must be at least 4")
return f"{url[:max_length]}..." if len(url) > max_length else url return f"{url[:max_length]}..." if len(url) > max_length else url
class DisplayManager: class DisplayManager:
"""Manages status display configuration""" """Manages status display configuration"""
@@ -148,7 +169,7 @@ class DisplayManager:
success_color=discord.Color.green(), success_color=discord.Color.green(),
warning_color=discord.Color.gold(), warning_color=discord.Color.gold(),
error_color=discord.Color.red(), error_color=discord.Color.red(),
info_color=discord.Color.blurple() info_color=discord.Color.blurple(),
) )
def __init__(self) -> None: def __init__(self) -> None:
@@ -165,7 +186,7 @@ class DisplayManager:
"Avg Processing Time: {avg_processing_time}\n" "Avg Processing Time: {avg_processing_time}\n"
"```" "```"
), ),
order=1 order=1,
), ),
DisplaySection.DOWNLOADS: DisplayTemplate( DisplaySection.DOWNLOADS: DisplayTemplate(
name="Active Downloads", name="Active Downloads",
@@ -181,7 +202,7 @@ class DisplayManager:
"```" "```"
), ),
order=2, order=2,
condition=DisplayCondition.HAS_DOWNLOADS condition=DisplayCondition.HAS_DOWNLOADS,
), ),
DisplaySection.COMPRESSIONS: DisplayTemplate( DisplaySection.COMPRESSIONS: DisplayTemplate(
name="Active Compressions", name="Active Compressions",
@@ -198,13 +219,13 @@ class DisplayManager:
"```" "```"
), ),
order=3, order=3,
condition=DisplayCondition.HAS_COMPRESSIONS condition=DisplayCondition.HAS_COMPRESSIONS,
), ),
DisplaySection.ERRORS: DisplayTemplate( DisplaySection.ERRORS: DisplayTemplate(
name="Error Statistics", name="Error Statistics",
format_string="```\n{error_stats}```", format_string="```\n{error_stats}```",
condition=DisplayCondition.HAS_ERRORS, condition=DisplayCondition.HAS_ERRORS,
order=4 order=4,
), ),
DisplaySection.HARDWARE: DisplayTemplate( DisplaySection.HARDWARE: DisplayTemplate(
name="Hardware Statistics", name="Hardware Statistics",
@@ -215,11 +236,12 @@ class DisplayManager:
"Peak Memory Usage: {memory_usage}\n" "Peak Memory Usage: {memory_usage}\n"
"```" "```"
), ),
order=5 order=5,
) ),
} }
self.theme = self.DEFAULT_THEME.copy() self.theme = self.DEFAULT_THEME.copy()
class StatusDisplay: class StatusDisplay:
"""Handles formatting and display of queue status information""" """Handles formatting and display of queue status information"""
@@ -229,20 +251,18 @@ class StatusDisplay:
@classmethod @classmethod
async def create_queue_status_embed( async def create_queue_status_embed(
cls, cls, queue_status: Dict[str, Any], active_ops: Dict[str, Any]
queue_status: Dict[str, Any],
active_ops: Dict[str, Any]
) -> discord.Embed: ) -> discord.Embed:
""" """
Create an embed displaying queue status and active operations. Create an embed displaying queue status and active operations.
Args: Args:
queue_status: Dictionary containing queue status information queue_status: Dictionary containing queue status information
active_ops: Dictionary containing active operations information active_ops: Dictionary containing active operations information
Returns: Returns:
Discord embed containing formatted status information Discord embed containing formatted status information
Raises: Raises:
DisplayError: If there's an error creating the embed DisplayError: If there's an error creating the embed
""" """
@@ -251,13 +271,12 @@ class StatusDisplay:
embed = discord.Embed( embed = discord.Embed(
title="Queue Status Details", title="Queue Status Details",
color=display.display_manager.theme["title_color"], color=display.display_manager.theme["title_color"],
timestamp=datetime.utcnow() timestamp=datetime.utcnow(),
) )
# Add sections in order # Add sections in order
sections = sorted( sections = sorted(
display.display_manager.templates.items(), display.display_manager.templates.items(), key=lambda x: x[1].order
key=lambda x: x[1].order
) )
for section, template in sections: for section, template in sections:
@@ -265,9 +284,7 @@ class StatusDisplay:
# Check condition if exists # Check condition if exists
if template.condition: if template.condition:
if not display._check_condition( if not display._check_condition(
template.condition, template.condition, queue_status, active_ops
queue_status,
active_ops
): ):
continue continue
@@ -275,9 +292,13 @@ class StatusDisplay:
if section == DisplaySection.QUEUE_STATS: if section == DisplaySection.QUEUE_STATS:
display._add_queue_statistics(embed, queue_status, template) display._add_queue_statistics(embed, queue_status, template)
elif section == DisplaySection.DOWNLOADS: elif section == DisplaySection.DOWNLOADS:
display._add_active_downloads(embed, active_ops.get('downloads', {}), template) display._add_active_downloads(
embed, active_ops.get("downloads", {}), template
)
elif section == DisplaySection.COMPRESSIONS: elif section == DisplaySection.COMPRESSIONS:
display._add_active_compressions(embed, active_ops.get('compressions', {}), template) display._add_active_compressions(
embed, active_ops.get("compressions", {}), template
)
elif section == DisplaySection.ERRORS: elif section == DisplaySection.ERRORS:
display._add_error_statistics(embed, queue_status, template) display._add_error_statistics(embed, queue_status, template)
elif section == DisplaySection.HARDWARE: elif section == DisplaySection.HARDWARE:
@@ -297,7 +318,7 @@ class StatusDisplay:
self, self,
condition: DisplayCondition, condition: DisplayCondition,
queue_status: Dict[str, Any], queue_status: Dict[str, Any],
active_ops: Dict[str, Any] active_ops: Dict[str, Any],
) -> bool: ) -> bool:
"""Check if condition for displaying section is met""" """Check if condition for displaying section is met"""
try: try:
@@ -316,185 +337,214 @@ class StatusDisplay:
self, self,
embed: discord.Embed, embed: discord.Embed,
queue_status: Dict[str, Any], queue_status: Dict[str, Any],
template: DisplayTemplate template: DisplayTemplate,
) -> None: ) -> None:
"""Add queue statistics to the embed""" """Add queue statistics to the embed"""
try: try:
metrics = queue_status.get('metrics', {}) metrics = queue_status.get("metrics", {})
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value=template.format_string.format( value=template.format_string.format(
pending=queue_status.get('pending', 0), pending=queue_status.get("pending", 0),
processing=queue_status.get('processing', 0), processing=queue_status.get("processing", 0),
completed=queue_status.get('completed', 0), completed=queue_status.get("completed", 0),
failed=queue_status.get('failed', 0), failed=queue_status.get("failed", 0),
success_rate=self.formatter.format_percentage( success_rate=self.formatter.format_percentage(
metrics.get('success_rate', 0) * 100 metrics.get("success_rate", 0) * 100
), ),
avg_processing_time=self.formatter.format_time( avg_processing_time=self.formatter.format_time(
metrics.get('avg_processing_time', 0) metrics.get("avg_processing_time", 0)
) ),
), ),
inline=template.inline inline=template.inline,
) )
except Exception as e: except Exception as e:
logger.error(f"Error adding queue statistics: {e}") logger.error(f"Error adding queue statistics: {e}")
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value="```\nError displaying queue statistics```", value="```\nError displaying queue statistics```",
inline=template.inline inline=template.inline,
) )
def _add_active_downloads( def _add_active_downloads(
self, self, embed: discord.Embed, downloads: Dict[str, Any], template: DisplayTemplate
embed: discord.Embed,
downloads: Dict[str, Any],
template: DisplayTemplate
) -> None: ) -> None:
"""Add active downloads information to the embed""" """Add active downloads information to the embed"""
try: try:
if downloads: if downloads:
content = [] content = []
for url, progress in list(downloads.items())[:template.max_items]: for url, progress in list(downloads.items())[: template.max_items]:
try: try:
content.append(template.format_string.format( content.append(
url=self.formatter.truncate_url(url), template.format_string.format(
percent=self.formatter.format_percentage(progress.get('percent', 0)), url=self.formatter.truncate_url(url),
speed=progress.get('speed', 'N/A'), percent=self.formatter.format_percentage(
eta=progress.get('eta', 'N/A'), progress.get("percent", 0)
size=f"{self.formatter.format_bytes(progress.get('downloaded_bytes', 0))}/" ),
f"{self.formatter.format_bytes(progress.get('total_bytes', 0))}", speed=progress.get("speed", "N/A"),
start_time=progress.get('start_time', 'N/A'), eta=progress.get("eta", "N/A"),
retries=progress.get('retries', 0) size=f"{self.formatter.format_bytes(progress.get('downloaded_bytes', 0))}/"
)) f"{self.formatter.format_bytes(progress.get('total_bytes', 0))}",
start_time=progress.get("start_time", "N/A"),
retries=progress.get("retries", 0),
)
)
except Exception as e: except Exception as e:
logger.error(f"Error formatting download {url}: {e}") logger.error(f"Error formatting download {url}: {e}")
continue continue
if len(downloads) > template.max_items: if len(downloads) > template.max_items:
content.append(f"\n... and {len(downloads) - template.max_items} more") content.append(
f"\n... and {len(downloads) - template.max_items} more"
)
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value="".join(content) if content else "```\nNo active downloads```", value=(
inline=template.inline "".join(content) if content else "```\nNo active downloads```"
),
inline=template.inline,
) )
else: else:
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value="```\nNo active downloads```", value="```\nNo active downloads```",
inline=template.inline inline=template.inline,
) )
except Exception as e: except Exception as e:
logger.error(f"Error adding active downloads: {e}") logger.error(f"Error adding active downloads: {e}")
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value="```\nError displaying downloads```", value="```\nError displaying downloads```",
inline=template.inline inline=template.inline,
) )
def _add_active_compressions( def _add_active_compressions(
self, self,
embed: discord.Embed, embed: discord.Embed,
compressions: Dict[str, Any], compressions: Dict[str, Any],
template: DisplayTemplate template: DisplayTemplate,
) -> None: ) -> None:
"""Add active compressions information to the embed""" """Add active compressions information to the embed"""
try: try:
if compressions: if compressions:
content = [] content = []
for file_id, progress in list(compressions.items())[:template.max_items]: for file_id, progress in list(compressions.items())[
: template.max_items
]:
try: try:
content.append(template.format_string.format( content.append(
filename=progress.get('filename', 'Unknown'), template.format_string.format(
percent=self.formatter.format_percentage(progress.get('percent', 0)), filename=progress.get("filename", "Unknown"),
elapsed_time=progress.get('elapsed_time', 'N/A'), percent=self.formatter.format_percentage(
input_size=self.formatter.format_bytes(progress.get('input_size', 0)), progress.get("percent", 0)
current_size=self.formatter.format_bytes(progress.get('current_size', 0)), ),
target_size=self.formatter.format_bytes(progress.get('target_size', 0)), elapsed_time=progress.get("elapsed_time", "N/A"),
codec=progress.get('codec', 'Unknown'), input_size=self.formatter.format_bytes(
hardware_accel=progress.get('hardware_accel', False) progress.get("input_size", 0)
)) ),
current_size=self.formatter.format_bytes(
progress.get("current_size", 0)
),
target_size=self.formatter.format_bytes(
progress.get("target_size", 0)
),
codec=progress.get("codec", "Unknown"),
hardware_accel=progress.get("hardware_accel", False),
)
)
except Exception as e: except Exception as e:
logger.error(f"Error formatting compression {file_id}: {e}") logger.error(f"Error formatting compression {file_id}: {e}")
continue continue
if len(compressions) > template.max_items: if len(compressions) > template.max_items:
content.append(f"\n... and {len(compressions) - template.max_items} more") content.append(
f"\n... and {len(compressions) - template.max_items} more"
)
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value="".join(content) if content else "```\nNo active compressions```", value=(
inline=template.inline "".join(content)
if content
else "```\nNo active compressions```"
),
inline=template.inline,
) )
else: else:
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value="```\nNo active compressions```", value="```\nNo active compressions```",
inline=template.inline inline=template.inline,
) )
except Exception as e: except Exception as e:
logger.error(f"Error adding active compressions: {e}") logger.error(f"Error adding active compressions: {e}")
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value="```\nError displaying compressions```", value="```\nError displaying compressions```",
inline=template.inline inline=template.inline,
) )
def _add_error_statistics( def _add_error_statistics(
self, self,
embed: discord.Embed, embed: discord.Embed,
queue_status: Dict[str, Any], queue_status: Dict[str, Any],
template: DisplayTemplate template: DisplayTemplate,
) -> None: ) -> None:
"""Add error statistics to the embed""" """Add error statistics to the embed"""
try: try:
metrics = queue_status.get('metrics', {}) metrics = queue_status.get("metrics", {})
errors_by_type = metrics.get('errors_by_type', {}) errors_by_type = metrics.get("errors_by_type", {})
if errors_by_type: if errors_by_type:
error_stats = "\n".join( error_stats = "\n".join(
f"{error_type}: {count}" f"{error_type}: {count}"
for error_type, count in list(errors_by_type.items())[:template.max_items] for error_type, count in list(errors_by_type.items())[
: template.max_items
]
) )
if len(errors_by_type) > template.max_items: if len(errors_by_type) > template.max_items:
error_stats += f"\n... and {len(errors_by_type) - template.max_items} more" error_stats += (
f"\n... and {len(errors_by_type) - template.max_items} more"
)
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value=template.format_string.format(error_stats=error_stats), value=template.format_string.format(error_stats=error_stats),
inline=template.inline inline=template.inline,
) )
except Exception as e: except Exception as e:
logger.error(f"Error adding error statistics: {e}") logger.error(f"Error adding error statistics: {e}")
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value="```\nError displaying error statistics```", value="```\nError displaying error statistics```",
inline=template.inline inline=template.inline,
) )
def _add_hardware_statistics( def _add_hardware_statistics(
self, self,
embed: discord.Embed, embed: discord.Embed,
queue_status: Dict[str, Any], queue_status: Dict[str, Any],
template: DisplayTemplate template: DisplayTemplate,
) -> None: ) -> None:
"""Add hardware statistics to the embed""" """Add hardware statistics to the embed"""
try: try:
metrics = queue_status.get('metrics', {}) metrics = queue_status.get("metrics", {})
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value=template.format_string.format( value=template.format_string.format(
hw_failures=metrics.get('hardware_accel_failures', 0), hw_failures=metrics.get("hardware_accel_failures", 0),
comp_failures=metrics.get('compression_failures', 0), comp_failures=metrics.get("compression_failures", 0),
memory_usage=self.formatter.format_bytes( memory_usage=self.formatter.format_bytes(
metrics.get('peak_memory_usage', 0) * 1024 * 1024 # Convert MB to bytes metrics.get("peak_memory_usage", 0)
) * 1024
* 1024 # Convert MB to bytes
),
), ),
inline=template.inline inline=template.inline,
) )
except Exception as e: except Exception as e:
logger.error(f"Error adding hardware statistics: {e}") logger.error(f"Error adding hardware statistics: {e}")
embed.add_field( embed.add_field(
name=template.name, name=template.name,
value="```\nError displaying hardware statistics```", value="```\nError displaying hardware statistics```",
inline=template.inline inline=template.inline,
) )

2
videoarchiver/processor/url_extractor.py
View File

@@ -6,7 +6,7 @@ from enum import Enum
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import List, Dict, Optional, Set, Pattern, ClassVar from typing import List, Dict, Optional, Set, Pattern, ClassVar
from datetime import datetime from datetime import datetime
import discord import discord # type: ignore
from urllib.parse import urlparse, parse_qs, ParseResult from urllib.parse import urlparse, parse_qs, ParseResult
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

2
videoarchiver/queue/health_checker.py
View File

@@ -1,7 +1,7 @@
"""Module for queue health checks""" """Module for queue health checks"""
import logging import logging
import psutil import psutil # type: ignore
import time import time
from enum import Enum from enum import Enum
from dataclasses import dataclass, field from dataclasses import dataclass, field

126
videoarchiver/update_checker.py
View File

@@ -1,10 +1,11 @@
"""Update checker for yt-dlp""" """Update checker for yt-dlp"""
import logging import logging
from importlib.metadata import version as get_package_version from importlib.metadata import version as get_package_version
from datetime import datetime, timedelta from datetime import datetime, timedelta
import aiohttp import aiohttp
from packaging import version from packaging import version
import discord import discord # type: ignore
from typing import Optional, Tuple, Dict, Any from typing import Optional, Tuple, Dict, Any
import asyncio import asyncio
import sys import sys
@@ -15,20 +16,21 @@ import tempfile
import os import os
import shutil import shutil
from .exceptions import UpdateError from .utils.exceptions import UpdateError
logger = logging.getLogger("VideoArchiver")
logger = logging.getLogger('VideoArchiver')
class UpdateChecker: class UpdateChecker:
"""Handles checking for yt-dlp updates""" """Handles checking for yt-dlp updates"""
GITHUB_API_URL = 'https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest' GITHUB_API_URL = "https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest"
UPDATE_CHECK_INTERVAL = 21600 # 6 hours in seconds UPDATE_CHECK_INTERVAL = 21600 # 6 hours in seconds
MAX_RETRIES = 3 MAX_RETRIES = 3
RETRY_DELAY = 5 RETRY_DELAY = 5
REQUEST_TIMEOUT = 30 REQUEST_TIMEOUT = 30
SUBPROCESS_TIMEOUT = 300 # 5 minutes SUBPROCESS_TIMEOUT = 300 # 5 minutes
def __init__(self, bot, config_manager): def __init__(self, bot, config_manager):
self.bot = bot self.bot = bot
self.config_manager = config_manager self.config_manager = config_manager
@@ -44,10 +46,10 @@ class UpdateChecker:
if self._session is None or self._session.closed: if self._session is None or self._session.closed:
self._session = aiohttp.ClientSession( self._session = aiohttp.ClientSession(
headers={ headers={
'Accept': 'application/vnd.github.v3+json', "Accept": "application/vnd.github.v3+json",
'User-Agent': 'VideoArchiver-Bot' "User-Agent": "VideoArchiver-Bot",
}, },
timeout=aiohttp.ClientTimeout(total=self.REQUEST_TIMEOUT) timeout=aiohttp.ClientTimeout(total=self.REQUEST_TIMEOUT),
) )
async def start(self) -> None: async def start(self) -> None:
@@ -67,30 +69,36 @@ class UpdateChecker:
except asyncio.CancelledError: except asyncio.CancelledError:
pass pass
self._check_task = None self._check_task = None
if self._session and not self._session.closed: if self._session and not self._session.closed:
await self._session.close() await self._session.close()
self._session = None self._session = None
logger.info("Update checker task stopped") logger.info("Update checker task stopped")
async def _check_loop(self) -> None: async def _check_loop(self) -> None:
"""Periodic update check loop with improved error handling""" """Periodic update check loop with improved error handling"""
await self.bot.wait_until_ready() await self.bot.wait_until_ready()
while not self._shutdown: while not self._shutdown:
try: try:
for guild in self.bot.guilds: for guild in self.bot.guilds:
try: try:
settings = await self.config_manager.get_guild_settings(guild.id) settings = await self.config_manager.get_guild_settings(
if settings.get('disable_update_check', False): guild.id
)
if settings.get("disable_update_check", False):
continue continue
current_time = datetime.utcnow() current_time = datetime.utcnow()
# Check if we've checked recently # Check if we've checked recently
last_check = self._last_version_check.get(guild.id) last_check = self._last_version_check.get(guild.id)
if last_check and (current_time - last_check).total_seconds() < self.UPDATE_CHECK_INTERVAL: if (
last_check
and (current_time - last_check).total_seconds()
< self.UPDATE_CHECK_INTERVAL
):
continue continue
# Check rate limits # Check rate limits
@@ -105,7 +113,9 @@ class UpdateChecker:
self._last_version_check[guild.id] = current_time self._last_version_check[guild.id] = current_time
except Exception as e: except Exception as e:
logger.error(f"Error checking updates for guild {guild.id}: {str(e)}") logger.error(
f"Error checking updates for guild {guild.id}: {str(e)}"
)
continue continue
except asyncio.CancelledError: except asyncio.CancelledError:
@@ -124,7 +134,7 @@ class UpdateChecker:
await self._log_error( await self._log_error(
guild, guild,
UpdateError("Could not determine current yt-dlp version"), UpdateError("Could not determine current yt-dlp version"),
"checking current version" "checking current version",
) )
return return
@@ -134,14 +144,14 @@ class UpdateChecker:
# Update last check time # Update last check time
await self.config_manager.update_setting( await self.config_manager.update_setting(
guild.id, guild.id, "last_update_check", datetime.utcnow().isoformat()
"last_update_check",
datetime.utcnow().isoformat()
) )
# Compare versions # Compare versions
if version.parse(current_version) < version.parse(latest_version): if version.parse(current_version) < version.parse(latest_version):
await self._notify_update(guild, current_version, latest_version, settings) await self._notify_update(
guild, current_version, latest_version, settings
)
except Exception as e: except Exception as e:
await self._log_error(guild, e, "checking for updates") await self._log_error(guild, e, "checking for updates")
@@ -149,7 +159,7 @@ class UpdateChecker:
async def _get_current_version(self) -> Optional[str]: async def _get_current_version(self) -> Optional[str]:
"""Get current yt-dlp version with error handling""" """Get current yt-dlp version with error handling"""
try: try:
return get_package_version('yt-dlp') return get_package_version("yt-dlp")
except Exception as e: except Exception as e:
logger.error(f"Error getting current version: {str(e)}") logger.error(f"Error getting current version: {str(e)}")
return None return None
@@ -157,35 +167,48 @@ class UpdateChecker:
async def _get_latest_version(self) -> Optional[str]: async def _get_latest_version(self) -> Optional[str]:
"""Get the latest version from GitHub with retries and rate limit handling""" """Get the latest version from GitHub with retries and rate limit handling"""
await self._init_session() await self._init_session()
for attempt in range(self.MAX_RETRIES): for attempt in range(self.MAX_RETRIES):
try: try:
async with self._session.get(self.GITHUB_API_URL) as response: async with self._session.get(self.GITHUB_API_URL) as response:
# Update rate limit info # Update rate limit info
self._remaining_requests = int(response.headers.get('X-RateLimit-Remaining', 0)) self._remaining_requests = int(
self._rate_limit_reset = int(response.headers.get('X-RateLimit-Reset', 0)) response.headers.get("X-RateLimit-Remaining", 0)
)
self._rate_limit_reset = int(
response.headers.get("X-RateLimit-Reset", 0)
)
if response.status == 200: if response.status == 200:
data = await response.json() data = await response.json()
return data['tag_name'].lstrip('v') return data["tag_name"].lstrip("v")
elif response.status == 403 and 'X-RateLimit-Remaining' in response.headers: elif (
response.status == 403
and "X-RateLimit-Remaining" in response.headers
):
logger.warning("GitHub API rate limit reached") logger.warning("GitHub API rate limit reached")
return None return None
elif response.status == 404: elif response.status == 404:
raise UpdateError("GitHub API endpoint not found") raise UpdateError("GitHub API endpoint not found")
else: else:
raise UpdateError(f"GitHub API returned status {response.status}") raise UpdateError(
f"GitHub API returned status {response.status}"
)
except asyncio.TimeoutError: except asyncio.TimeoutError:
logger.error(f"Timeout getting latest version (attempt {attempt + 1}/{self.MAX_RETRIES})") logger.error(
f"Timeout getting latest version (attempt {attempt + 1}/{self.MAX_RETRIES})"
)
if attempt == self.MAX_RETRIES - 1: if attempt == self.MAX_RETRIES - 1:
return None return None
except Exception as e: except Exception as e:
logger.error(f"Error getting latest version (attempt {attempt + 1}/{self.MAX_RETRIES}): {str(e)}") logger.error(
f"Error getting latest version (attempt {attempt + 1}/{self.MAX_RETRIES}): {str(e)}"
)
if attempt == self.MAX_RETRIES - 1: if attempt == self.MAX_RETRIES - 1:
return None return None
await asyncio.sleep(self.RETRY_DELAY * (attempt + 1)) await asyncio.sleep(self.RETRY_DELAY * (attempt + 1))
return None return None
@@ -195,7 +218,7 @@ class UpdateChecker:
guild: discord.Guild, guild: discord.Guild,
current_version: str, current_version: str,
latest_version: str, latest_version: str,
settings: dict settings: dict,
) -> None: ) -> None:
"""Notify about available updates with retry mechanism""" """Notify about available updates with retry mechanism"""
owner = self.bot.get_user(self.bot.owner_id) owner = self.bot.get_user(self.bot.owner_id)
@@ -203,7 +226,7 @@ class UpdateChecker:
await self._log_error( await self._log_error(
guild, guild,
UpdateError("Could not find bot owner"), UpdateError("Could not find bot owner"),
"sending update notification" "sending update notification",
) )
return return
@@ -223,23 +246,25 @@ class UpdateChecker:
await self._log_error( await self._log_error(
guild, guild,
UpdateError(f"Failed to send update notification: {str(e)}"), UpdateError(f"Failed to send update notification: {str(e)}"),
"sending update notification" "sending update notification",
) )
else: else:
await asyncio.sleep(settings.get("discord_retry_delay", 5)) await asyncio.sleep(settings.get("discord_retry_delay", 5))
async def _log_error(self, guild: discord.Guild, error: Exception, context: str) -> None: async def _log_error(
self, guild: discord.Guild, error: Exception, context: str
) -> None:
"""Log an error to the guild's log channel with enhanced formatting""" """Log an error to the guild's log channel with enhanced formatting"""
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
error_message = f"[{timestamp}] Error {context}: {str(error)}" error_message = f"[{timestamp}] Error {context}: {str(error)}"
log_channel = await self.config_manager.get_channel(guild, "log") log_channel = await self.config_manager.get_channel(guild, "log")
if log_channel: if log_channel:
try: try:
await log_channel.send(f"```\n{error_message}\n```") await log_channel.send(f"```\n{error_message}\n```")
except discord.HTTPException as e: except discord.HTTPException as e:
logger.error(f"Failed to send error to log channel: {str(e)}") logger.error(f"Failed to send error to log channel: {str(e)}")
logger.error(f"Guild {guild.id} - {error_message}") logger.error(f"Guild {guild.id} - {error_message}")
async def update_yt_dlp(self) -> Tuple[bool, str]: async def update_yt_dlp(self) -> Tuple[bool, str]:
@@ -247,32 +272,29 @@ class UpdateChecker:
temp_dir = None temp_dir = None
try: try:
# Create temporary directory for pip output # Create temporary directory for pip output
temp_dir = tempfile.mkdtemp(prefix='ytdlp_update_') temp_dir = tempfile.mkdtemp(prefix="ytdlp_update_")
log_file = Path(temp_dir) / 'pip_log.txt' log_file = Path(temp_dir) / "pip_log.txt"
# Prepare pip command # Prepare pip command
cmd = [ cmd = [
sys.executable, sys.executable,
'-m', "-m",
'pip', "pip",
'install', "install",
'--upgrade', "--upgrade",
'yt-dlp', "yt-dlp",
'--log', "--log",
str(log_file) str(log_file),
] ]
# Run pip in subprocess with timeout # Run pip in subprocess with timeout
process = await asyncio.create_subprocess_exec( process = await asyncio.create_subprocess_exec(
*cmd, *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
) )
try: try:
stdout, stderr = await asyncio.wait_for( stdout, stderr = await asyncio.wait_for(
process.communicate(), process.communicate(), timeout=self.SUBPROCESS_TIMEOUT
timeout=self.SUBPROCESS_TIMEOUT
) )
except asyncio.TimeoutError: except asyncio.TimeoutError:
process.kill() process.kill()
@@ -288,7 +310,7 @@ class UpdateChecker:
error_details = "Unknown error" error_details = "Unknown error"
if log_file.exists(): if log_file.exists():
try: try:
error_details = log_file.read_text(errors='ignore') error_details = log_file.read_text(errors="ignore")
except Exception: except Exception:
pass pass
return False, f"Failed to update: {error_details}" return False, f"Failed to update: {error_details}"

10
videoarchiver/utils/compression_handler.py
View File

@@ -7,11 +7,11 @@ import subprocess
from datetime import datetime from datetime import datetime
from typing import Dict, Optional, Callable, Set, Tuple from typing import Dict, Optional, Callable, Set, Tuple
from .ffmpeg.ffmpeg_manager import FFmpegManager from ..ffmpeg.ffmpeg_manager import FFmpegManager
from .ffmpeg.exceptions import CompressionError from ..ffmpeg.exceptions import CompressionError
from .utils.exceptions import VideoVerificationError from ..utils.exceptions import VideoVerificationError
from .utils.file_operations import FileOperations from ..utils.file_operations import FileOperations
from .utils.progress_handler import ProgressHandler from ..utils.progress_handler import ProgressHandler
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

11
videoarchiver/utils/compression_manager.py
View File

@@ -7,14 +7,15 @@ import subprocess
from datetime import datetime from datetime import datetime
from typing import Dict, Any, Optional, Callable, List, Set, Tuple from typing import Dict, Any, Optional, Callable, List, Set, Tuple
from .processor import _compression_progress from ..processor import _compression_progress
from .utils.compression_handler import CompressionHandler from ..utils.compression_handler import CompressionHandler
from .utils.progress_handler import ProgressHandler from ..utils.progress_handler import ProgressHandler
from .utils.file_operations import FileOperations from ..utils.file_operations import FileOperations
from .utils.exceptions import CompressionError, VideoVerificationError from ..utils.exceptions import CompressionError, VideoVerificationError
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class CompressionManager: class CompressionManager:
"""Manages video compression operations""" """Manages video compression operations"""

43
videoarchiver/utils/directory_manager.py
View File

@@ -6,11 +6,12 @@ import asyncio
from pathlib import Path from pathlib import Path
from typing import List, Optional, Tuple from typing import List, Optional, Tuple
from .utils.exceptions import FileCleanupError from ..utils.exceptions import FileCleanupError
from .utils.file_deletion import SecureFileDeleter from ..utils.file_deletion import SecureFileDeleter
logger = logging.getLogger("DirectoryManager") logger = logging.getLogger("DirectoryManager")
class DirectoryManager: class DirectoryManager:
"""Handles directory operations and cleanup""" """Handles directory operations and cleanup"""
@@ -18,21 +19,18 @@ class DirectoryManager:
self.file_deleter = SecureFileDeleter() self.file_deleter = SecureFileDeleter()
async def cleanup_directory( async def cleanup_directory(
self, self, directory_path: str, recursive: bool = True, delete_empty: bool = True
directory_path: str,
recursive: bool = True,
delete_empty: bool = True
) -> Tuple[int, List[str]]: ) -> Tuple[int, List[str]]:
"""Clean up a directory by removing files and optionally empty subdirectories """Clean up a directory by removing files and optionally empty subdirectories
Args: Args:
directory_path: Path to the directory to clean directory_path: Path to the directory to clean
recursive: Whether to clean subdirectories recursive: Whether to clean subdirectories
delete_empty: Whether to delete empty directories delete_empty: Whether to delete empty directories
Returns: Returns:
Tuple[int, List[str]]: (Number of files deleted, List of errors) Tuple[int, List[str]]: (Number of files deleted, List of errors)
Raises: Raises:
FileCleanupError: If cleanup fails critically FileCleanupError: If cleanup fails critically
""" """
@@ -45,9 +43,7 @@ class DirectoryManager:
try: try:
# Process files and directories # Process files and directories
deleted, errs = await self._process_directory_contents( deleted, errs = await self._process_directory_contents(
directory_path, directory_path, recursive, delete_empty
recursive,
delete_empty
) )
deleted_count += deleted deleted_count += deleted
errors.extend(errs) errors.extend(errs)
@@ -69,10 +65,7 @@ class DirectoryManager:
raise FileCleanupError(f"Directory cleanup failed: {str(e)}") raise FileCleanupError(f"Directory cleanup failed: {str(e)}")
async def _process_directory_contents( async def _process_directory_contents(
self, self, directory_path: str, recursive: bool, delete_empty: bool
directory_path: str,
recursive: bool,
delete_empty: bool
) -> Tuple[int, List[str]]: ) -> Tuple[int, List[str]]:
"""Process contents of a directory""" """Process contents of a directory"""
deleted_count = 0 deleted_count = 0
@@ -90,9 +83,7 @@ class DirectoryManager:
elif entry.is_dir() and recursive: elif entry.is_dir() and recursive:
# Process subdirectory # Process subdirectory
subdir_deleted, subdir_errors = await self.cleanup_directory( subdir_deleted, subdir_errors = await self.cleanup_directory(
entry.path, entry.path, recursive=True, delete_empty=delete_empty
recursive=True,
delete_empty=delete_empty
) )
deleted_count += subdir_deleted deleted_count += subdir_deleted
errors.extend(subdir_errors) errors.extend(subdir_errors)
@@ -133,31 +124,31 @@ class DirectoryManager:
async def ensure_directory(self, directory_path: str) -> None: async def ensure_directory(self, directory_path: str) -> None:
"""Ensure a directory exists and is accessible """Ensure a directory exists and is accessible
Args: Args:
directory_path: Path to the directory to ensure directory_path: Path to the directory to ensure
Raises: Raises:
FileCleanupError: If directory cannot be created or accessed FileCleanupError: If directory cannot be created or accessed
""" """
try: try:
path = Path(directory_path) path = Path(directory_path)
path.mkdir(parents=True, exist_ok=True) path.mkdir(parents=True, exist_ok=True)
# Verify directory is writable # Verify directory is writable
if not os.access(directory_path, os.W_OK): if not os.access(directory_path, os.W_OK):
raise FileCleanupError(f"Directory {directory_path} is not writable") raise FileCleanupError(f"Directory {directory_path} is not writable")
except Exception as e: except Exception as e:
logger.error(f"Error ensuring directory {directory_path}: {e}") logger.error(f"Error ensuring directory {directory_path}: {e}")
raise FileCleanupError(f"Failed to ensure directory: {str(e)}") raise FileCleanupError(f"Failed to ensure directory: {str(e)}")
async def get_directory_size(self, directory_path: str) -> int: async def get_directory_size(self, directory_path: str) -> int:
"""Get total size of a directory in bytes """Get total size of a directory in bytes
Args: Args:
directory_path: Path to the directory directory_path: Path to the directory
Returns: Returns:
int: Total size in bytes int: Total size in bytes
""" """
@@ -173,5 +164,5 @@ class DirectoryManager:
logger.warning(f"Error getting size for {entry.path}: {e}") logger.warning(f"Error getting size for {entry.path}: {e}")
except Exception as e: except Exception as e:
logger.error(f"Error calculating directory size: {e}") logger.error(f"Error calculating directory size: {e}")
return total_size return total_size

14
videoarchiver/utils/download_core.py
View File

@@ -3,16 +3,16 @@
import os import os
import asyncio import asyncio
import logging import logging
import yt_dlp import yt_dlp # type: ignore
from typing import Dict, Optional, Callable, Tuple from typing import Dict, Optional, Callable, Tuple
from pathlib import Path from pathlib import Path
from .utils.url_validator import check_url_support from ..utils.url_validator import check_url_support
from .utils.progress_handler import ProgressHandler, CancellableYTDLLogger from ..utils.progress_handler import ProgressHandler, CancellableYTDLLogger
from .utils.file_operations import FileOperations from ..utils.file_operations import FileOperations
from .utils.compression_handler import CompressionHandler from ..utils.compression_handler import CompressionHandler
from .utils.process_manager import ProcessManager from ..utils.process_manager import ProcessManager
from .ffmpeg.ffmpeg_manager import FFmpegManager from ..ffmpeg.ffmpeg_manager import FFmpegManager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

59
videoarchiver/utils/download_manager.py
View File

@@ -3,21 +3,22 @@
import os import os
import logging import logging
import asyncio import asyncio
import yt_dlp import yt_dlp # type: ignore
from datetime import datetime from datetime import datetime
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from typing import Dict, List, Optional, Tuple, Callable, Any from typing import Dict, List, Optional, Tuple, Callable, Any
from pathlib import Path from pathlib import Path
from .utils.verification_manager import VideoVerificationManager from ..ffmpeg.verification_manager import VerificationManager
from .utils.compression_manager import CompressionManager from ..utils.compression_manager import CompressionManager
from .utils import progress_tracker from ..utils import progress_tracker
logger = logging.getLogger("DownloadManager") logger = logging.getLogger("DownloadManager")
class CancellableYTDLLogger: class CancellableYTDLLogger:
"""Custom yt-dlp logger that can be cancelled""" """Custom yt-dlp logger that can be cancelled"""
def __init__(self): def __init__(self):
self.cancelled = False self.cancelled = False
@@ -36,6 +37,7 @@ class CancellableYTDLLogger:
raise Exception("Download cancelled") raise Exception("Download cancelled")
logger.error(msg) logger.error(msg)
class DownloadManager: class DownloadManager:
"""Manages video downloads and processing""" """Manages video downloads and processing"""
@@ -53,40 +55,33 @@ class DownloadManager:
max_file_size: int, max_file_size: int,
enabled_sites: Optional[List[str]] = None, enabled_sites: Optional[List[str]] = None,
concurrent_downloads: int = 2, concurrent_downloads: int = 2,
ffmpeg_mgr = None ffmpeg_mgr=None,
): ):
self.download_path = Path(download_path) self.download_path = Path(download_path)
self.download_path.mkdir(parents=True, exist_ok=True) self.download_path.mkdir(parents=True, exist_ok=True)
os.chmod(str(self.download_path), 0o755) os.chmod(str(self.download_path), 0o755)
# Initialize components # Initialize components
self.verification_manager = VideoVerificationManager(ffmpeg_mgr) self.verification_manager = VerificationManager(ffmpeg_mgr)
self.compression_manager = CompressionManager(ffmpeg_mgr, max_file_size) self.compression_manager = CompressionManager(ffmpeg_mgr, max_file_size)
# Create thread pool # Create thread pool
self.download_pool = ThreadPoolExecutor( self.download_pool = ThreadPoolExecutor(
max_workers=max(1, min(3, concurrent_downloads)), max_workers=max(1, min(3, concurrent_downloads)),
thread_name_prefix="videoarchiver_download" thread_name_prefix="videoarchiver_download",
) )
# Initialize state # Initialize state
self._shutting_down = False self._shutting_down = False
self.ytdl_logger = CancellableYTDLLogger() self.ytdl_logger = CancellableYTDLLogger()
# Configure yt-dlp options # Configure yt-dlp options
self.ydl_opts = self._configure_ydl_opts( self.ydl_opts = self._configure_ydl_opts(
video_format, video_format, max_quality, max_file_size, ffmpeg_mgr
max_quality,
max_file_size,
ffmpeg_mgr
) )
def _configure_ydl_opts( def _configure_ydl_opts(
self, self, video_format: str, max_quality: int, max_file_size: int, ffmpeg_mgr
video_format: str,
max_quality: int,
max_file_size: int,
ffmpeg_mgr
) -> Dict[str, Any]: ) -> Dict[str, Any]:
"""Configure yt-dlp options""" """Configure yt-dlp options"""
return { return {
@@ -124,7 +119,9 @@ class DownloadManager:
try: try:
size = os.path.getsize(info["filepath"]) size = os.path.getsize(info["filepath"])
if size > self.compression_manager.max_file_size: if size > self.compression_manager.max_file_size:
logger.info(f"File exceeds size limit, will compress: {info['filepath']}") logger.info(
f"File exceeds size limit, will compress: {info['filepath']}"
)
except OSError as e: except OSError as e:
logger.error(f"Error checking file size: {str(e)}") logger.error(f"Error checking file size: {str(e)}")
@@ -155,30 +152,24 @@ class DownloadManager:
progress_tracker.clear_progress() progress_tracker.clear_progress()
async def download_video( async def download_video(
self, self, url: str, progress_callback: Optional[Callable[[float], None]] = None
url: str,
progress_callback: Optional[Callable[[float], None]] = None
) -> Tuple[bool, str, str]: ) -> Tuple[bool, str, str]:
"""Download and process a video""" """Download and process a video"""
if self._shutting_down: if self._shutting_down:
return False, "", "Downloader is shutting down" return False, "", "Downloader is shutting down"
progress_tracker.start_download(url) progress_tracker.start_download(url)
try: try:
# Download video # Download video
success, file_path, error = await self._safe_download( success, file_path, error = await self._safe_download(
url, url, progress_callback
progress_callback
) )
if not success: if not success:
return False, "", error return False, "", error
# Verify and compress if needed # Verify and compress if needed
return await self._process_downloaded_file( return await self._process_downloaded_file(file_path, progress_callback)
file_path,
progress_callback
)
except Exception as e: except Exception as e:
logger.error(f"Download error: {str(e)}") logger.error(f"Download error: {str(e)}")
@@ -188,18 +179,14 @@ class DownloadManager:
progress_tracker.end_download(url) progress_tracker.end_download(url)
async def _safe_download( async def _safe_download(
self, self, url: str, progress_callback: Optional[Callable[[float], None]]
url: str,
progress_callback: Optional[Callable[[float], None]]
) -> Tuple[bool, str, str]: ) -> Tuple[bool, str, str]:
"""Safely download video with retries""" """Safely download video with retries"""
# Implementation moved to separate method for clarity # Implementation moved to separate method for clarity
pass # Implementation would be similar to original but using new components pass # Implementation would be similar to original but using new components
async def _process_downloaded_file( async def _process_downloaded_file(
self, self, file_path: str, progress_callback: Optional[Callable[[float], None]]
file_path: str,
progress_callback: Optional[Callable[[float], None]]
) -> Tuple[bool, str, str]: ) -> Tuple[bool, str, str]:
"""Process a downloaded file (verify and compress if needed)""" """Process a downloaded file (verify and compress if needed)"""
# Implementation moved to separate method for clarity # Implementation moved to separate method for clarity

21
videoarchiver/utils/file_deletion.py
View File

@@ -7,16 +7,17 @@ import logging
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
from .utils.exceptions import FileCleanupError from ..utils.exceptions import FileCleanupError
logger = logging.getLogger("FileDeleter") logger = logging.getLogger("FileDeleter")
class SecureFileDeleter: class SecureFileDeleter:
"""Handles secure file deletion operations""" """Handles secure file deletion operations"""
def __init__(self, max_size: int = 100 * 1024 * 1024): def __init__(self, max_size: int = 100 * 1024 * 1024):
"""Initialize the file deleter """Initialize the file deleter
Args: Args:
max_size: Maximum file size in bytes for secure deletion (default: 100MB) max_size: Maximum file size in bytes for secure deletion (default: 100MB)
""" """
@@ -24,13 +25,13 @@ class SecureFileDeleter:
async def delete_file(self, file_path: str) -> bool: async def delete_file(self, file_path: str) -> bool:
"""Delete a file securely """Delete a file securely
Args: Args:
file_path: Path to the file to delete file_path: Path to the file to delete
Returns: Returns:
bool: True if file was successfully deleted bool: True if file was successfully deleted
Raises: Raises:
FileCleanupError: If file deletion fails after all attempts FileCleanupError: If file deletion fails after all attempts
""" """
@@ -65,7 +66,9 @@ class SecureFileDeleter:
async def _delete_large_file(self, file_path: str) -> bool: async def _delete_large_file(self, file_path: str) -> bool:
"""Delete a large file directly""" """Delete a large file directly"""
try: try:
logger.debug(f"File {file_path} exceeds max size for secure deletion, performing direct removal") logger.debug(
f"File {file_path} exceeds max size for secure deletion, performing direct removal"
)
os.remove(file_path) os.remove(file_path)
return True return True
except OSError as e: except OSError as e:
@@ -84,11 +87,13 @@ class SecureFileDeleter:
async def _zero_file_content(self, file_path: str, file_size: int) -> None: async def _zero_file_content(self, file_path: str, file_size: int) -> None:
"""Zero out file content in chunks""" """Zero out file content in chunks"""
try: try:
chunk_size = min(1024 * 1024, file_size) # 1MB chunks or file size if smaller chunk_size = min(
1024 * 1024, file_size
) # 1MB chunks or file size if smaller
with open(file_path, "wb") as f: with open(file_path, "wb") as f:
for offset in range(0, file_size, chunk_size): for offset in range(0, file_size, chunk_size):
write_size = min(chunk_size, file_size - offset) write_size = min(chunk_size, file_size - offset)
f.write(b'\0' * write_size) f.write(b"\0" * write_size)
await asyncio.sleep(0) # Allow other tasks to run await asyncio.sleep(0) # Allow other tasks to run
f.flush() f.flush()
os.fsync(f.fileno()) os.fsync(f.fileno())

11
videoarchiver/utils/file_operations.py
View File

@@ -9,14 +9,15 @@ import subprocess
from typing import Tuple from typing import Tuple
from pathlib import Path from pathlib import Path
from .utils.exceptions import VideoVerificationError from ..utils.exceptions import VideoVerificationError
from .utils.file_deletion import secure_delete_file from ..utils.file_deletion import SecureFileDeleter
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
class FileOperations: class FileOperations:
"""Handles safe file operations with retries""" """Handles safe file operations with retries"""
def __init__(self, max_retries: int = 3, retry_delay: int = 1): def __init__(self, max_retries: int = 3, retry_delay: int = 1):
self.max_retries = max_retries self.max_retries = max_retries
self.retry_delay = retry_delay self.retry_delay = retry_delay
@@ -26,7 +27,7 @@ class FileOperations:
for attempt in range(self.max_retries): for attempt in range(self.max_retries):
try: try:
if os.path.exists(file_path): if os.path.exists(file_path):
await secure_delete_file(file_path) await SecureFileDeleter(file_path)
return True return True
except Exception as e: except Exception as e:
logger.error(f"Delete attempt {attempt + 1} failed: {str(e)}") logger.error(f"Delete attempt {attempt + 1} failed: {str(e)}")
@@ -122,7 +123,7 @@ class FileOperations:
result = subprocess.run(cmd, capture_output=True, text=True) result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0: if result.returncode != 0:
raise Exception(f"FFprobe failed: {result.stderr}") raise Exception(f"FFprobe failed: {result.stderr}")
data = json.loads(result.stdout) data = json.loads(result.stdout)
return float(data["format"]["duration"]) return float(data["format"]["duration"])
except Exception as e: except Exception as e:

8
videoarchiver/utils/file_ops.py
View File

@@ -4,10 +4,10 @@ import logging
from pathlib import Path from pathlib import Path
from typing import List, Tuple, Optional from typing import List, Tuple, Optional
from .utils.exceptions import FileCleanupError from ..utils.exceptions import FileCleanupError
from .utils.file_deletion import SecureFileDeleter from ..utils.file_deletion import SecureFileDeleter
from .utils.directory_manager import DirectoryManager from ..utils.directory_manager import DirectoryManager
from .utils.permission_manager import PermissionManager from ..utils.permission_manager import PermissionManager
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")

9
videoarchiver/utils/path_manager.py
View File

@@ -1,5 +1,6 @@
"""Path management utilities""" """Path management utilities"""
import asyncio
import os import os
import tempfile import tempfile
import shutil import shutil
@@ -7,11 +8,11 @@ import stat
import logging import logging
import contextlib import contextlib
import time import time
from typing import Generator, List, Optional from typing import List, Optional, AsyncGenerator
from pathlib import Path from pathlib import Path
from .utils.exceptions import FileCleanupError from ..utils.exceptions import FileCleanupError
from .utils.permission_manager import PermissionManager from ..utils.permission_manager import PermissionManager
logger = logging.getLogger("PathManager") logger = logging.getLogger("PathManager")
@@ -162,7 +163,7 @@ class PathManager:
async def temp_path_context( async def temp_path_context(
self, self,
prefix: str = "videoarchiver_" prefix: str = "videoarchiver_"
) -> Generator[str, None, None]: ) -> AsyncGenerator[str, None]:
"""Async context manager for temporary path creation and cleanup """Async context manager for temporary path creation and cleanup
Args: Args:

2
videoarchiver/utils/permission_manager.py
View File

@@ -6,7 +6,7 @@ import logging
from pathlib import Path from pathlib import Path
from typing import Optional, Union, List from typing import Optional, Union, List
from .utils.exceptions import FileCleanupError from ..utils.exceptions import FileCleanupError
logger = logging.getLogger("PermissionManager") logger = logging.getLogger("PermissionManager")

11
videoarchiver/utils/progress_handler.py
View File

@@ -1,8 +1,7 @@
"""Progress tracking and logging utilities for video downloads"""
import logging import logging
from datetime import datetime from datetime import datetime
from typing import Dict, Any, Optional, Callable from typing import Dict, Any, Optional, Callable
import os
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
@@ -13,17 +12,17 @@ class CancellableYTDLLogger:
def debug(self, msg): def debug(self, msg):
if self.cancelled: if self.cancelled:
raise yt_dlp.utils.DownloadError("Download cancelled") raise yt_dlp.utils.DownloadError("Download cancelled") # type: ignore
logger.debug(msg) logger.debug(msg)
def warning(self, msg): def warning(self, msg):
if self.cancelled: if self.cancelled:
raise yt_dlp.utils.DownloadError("Download cancelled") raise yt_dlp.utils.DownloadError("Download cancelled") # type: ignore
logger.warning(msg) logger.warning(msg)
def error(self, msg): def error(self, msg):
if self.cancelled: if self.cancelled:
raise yt_dlp.utils.DownloadError("Download cancelled") raise yt_dlp.utils.DownloadError("Download cancelled") # type: ignore
logger.error(msg) logger.error(msg)
class ProgressHandler: class ProgressHandler:
@@ -123,4 +122,4 @@ class ProgressHandler:
progress_callback(progress) progress_callback(progress)
except Exception as e: except Exception as e:
logger.error(f"Error upda logger.error(f"Error updating compression progress: {str(e)}")

2
videoarchiver/utils/url_validator.py
View File

@@ -2,7 +2,7 @@
import re import re
import logging import logging
import yt_dlp import yt_dlp # type: ignore
from typing import List, Optional from typing import List, Optional
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")