pacnpal/Pac-cogs
1
0
Fork 0
mirror of https://github.com/pacnpal/Pac-cogs.git synced 2025-12-20 10:51:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

diabolical

Browse Source
This commit is contained in:
pacnpal
2024-11-18 05:17:37 +00:00
parent fc06e54d8a
commit df9099f2c8
75 changed files with 744 additions and 719 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
videoarchiver/processor/__init__.py
View File

@@ -1,10 +1,10 @@
"""Video processing module for VideoArchiver"""
from typing import Dict, Any, Optional, Union, List, Tuple
import discord # type: ignore
import discord # type: ignore
# Import constants first since they have no dependencies
from .constants import (
from constants import (
REACTIONS,
ReactionType,
ReactionEmojis,
@@ -14,10 +14,10 @@ from .constants import (
)
# Import core components
from .core import VideoProcessor
from core import VideoProcessor
# Import URL related components
from .url_extractor import (
from url_extractor import (
URLExtractor,
URLMetadata,
URLPattern,
@@ -28,7 +28,7 @@ from .url_extractor import (
)
# Import validation components
from .message_validator import (
from message_validator import (
MessageValidator,
ValidationContext,
ValidationRule,
@@ -41,7 +41,7 @@ from .message_validator import (
)
# Import reaction handlers
from .reactions import (
from reactions import (
handle_archived_reaction,
update_queue_position_reaction,
update_progress_reaction,
@@ -49,12 +49,12 @@ from .reactions import (
)
# Import progress tracking
from ..utils.progress_tracker import ProgressTracker
from utils.progress_tracker import ProgressTracker
# Import handlers after other dependencies are loaded
from .message_handler import MessageHandler
from .queue_handler import QueueHandler
from .queue_processor import QueueProcessor
from message_handler import MessageHandler
from queue_handler import QueueHandler
from queue_processor import QueueProcessor
# Export public classes and constants
__all__ = [

6
videoarchiver/processor/cleanup_manager.py
View File

@@ -20,12 +20,12 @@ from typing import (
from datetime import datetime, timedelta
if TYPE_CHECKING:
from .queue_handler import QueueHandler
from queue_handler import QueueHandler
# try:
# Try relative imports first
from ..ffmpeg.ffmpeg_manager import FFmpegManager
from ..utils.exceptions import CleanupError
from ffmpeg.ffmpeg_manager import FFmpegManager
from utils.exceptions import CleanupError
# except ImportError:
# Fall back to absolute imports if relative imports fail
# # from videoarchiver.ffmpeg.ffmpeg_manager import FFmpegManager

8
videoarchiver/processor/core.py
View File

@@ -8,7 +8,7 @@ from typing import Any, ClassVar, Dict, List, Optional, Tuple, TYPE_CHECKING
import discord # type: ignore
from discord.ext import commands # type: ignore
from ..core.types import (
from core.c_types import (
ComponentState,
ProcessorState,
ComponentStatus,
@@ -16,9 +16,9 @@ from ..core.types import (
IConfigManager,
IQueueManager,
)
from .constants import REACTIONS
from ..utils.progress_tracker import ProgressTracker
from ..utils.exceptions import ProcessorError
from constants import REACTIONS
from utils.progress_tracker import ProgressTracker
from utils.exceptions import ProcessorError
logger = logging.getLogger("VideoArchiver")

12
videoarchiver/processor/message_handler.py
View File

@@ -21,12 +21,12 @@ from discord.ext import commands # type: ignore
# try:
# Try relative imports first
from ..config_manager import ConfigManager
from .constants import REACTIONS
from .message_validator import MessageValidator, ValidationError
from .url_extractor import URLExtractor, URLMetadata
from ..queue.types import QueuePriority
from ..utils.exceptions import MessageHandlerError
from config_manager import ConfigManager
from constants import REACTIONS
from message_validator import MessageValidator, ValidationError
from url_extractor import URLExtractor, URLMetadata
from queue.q_types import QueuePriority
from utils.exceptions import MessageHandlerError
# except ImportError:
# Fall back to absolute imports if relative imports fail

2
videoarchiver/processor/message_validator.py
View File

@@ -9,7 +9,7 @@ import discord # type: ignore
#try:
# Try relative imports first
from ..utils.exceptions import ValidationError
from utils.exceptions import ValidationError
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.utils.exceptions import ValidationError

31
videoarchiver/processor/queue_handler.py
View File

@@ -10,15 +10,16 @@ import discord # type: ignore
# try:
# Try relative imports first
from .. import utils
from ..database.video_archive_db import VideoArchiveDB
from ..utils.download_manager import DownloadManager
from ..utils.message_manager import MessageManager
from ..utils.exceptions import QueueHandlerError
from ..queue.models import QueueItem
from ..config_manager import ConfigManager
from . import progress_tracker # Import from processor package
from .constants import REACTIONS
# from . import utils
from database.video_archive_db import VideoArchiveDB
from utils.download_manager import DownloadManager
from utils.message_manager import MessageManager
from utils.exceptions import QueueHandlerError
from queue.models import QueueItem
from config_manager import ConfigManager
from utils.progress_tracker import ProgressTracker # Import from processor package
from constants import REACTIONS
# except ImportError:
# Fall back to absolute imports if relative imports fail
@@ -66,11 +67,13 @@ class QueueHandler:
self,
bot: discord.Client,
config_manager: ConfigManager,
progress_tracker: ProgressTracker,
components: Dict[int, Dict[str, Any]],
db: Optional[VideoArchiveDB] = None,
) -> None:
self.bot = bot
self.config_manager = config_manager
self.progress_tracker = progress_tracker
self.components = components
self.db = db
self._unloading = False
@@ -392,11 +395,13 @@ class QueueHandler:
return
# Update progress tracking
progress_tracker.update_download_progress(
ProgressTracker.update_download_progress(
url,
{
"percent": progress,
"last_update": datetime.utcnow().isoformat(),
"last_update": datetime.now(
datetime.timezone.utc
)().isoformat(),
},
)
@@ -439,9 +444,9 @@ class QueueHandler:
download_task, timeout=self.DOWNLOAD_TIMEOUT
)
if success:
progress_tracker.complete_download(url)
ProgressTracker.complete_download(url)
else:
progress_tracker.increment_download_retries(url)
ProgressTracker.increment_download_retries(url)
return success, file_path, error
except asyncio.TimeoutError:

151
videoarchiver/processor/queue_processor.py
View File

@@ -1,97 +1,110 @@
"""Queue processing functionality for video processing"""
import logging
import asyncio
from typing import List, Optional, Dict, Any, Set, ClassVar
from datetime import datetime
import sys
from pathlib import Path
#try:
# Try relative imports first
from ..queue.types import QueuePriority, QueueMetrics, ProcessingMetrics
from ..queue.models import QueueItem
#except ImportError:
# Fall back to absolute imports if relative imports fail
# from videoarchiver.queue.types import QueuePriority, QueueMetrics, ProcessingMetrics
# from videoarchiver.queue.models import QueueItem
# Get the parent directory (videoarchiver root)
BASE_DIR = Path(__file__).resolve().parent.parent
if str(BASE_DIR) not in sys.path:
sys.path.insert(0, str(BASE_DIR))
# Use non-relative imports
from queue.q_types import QueuePriority, QueueMetrics, ProcessingMetrics
from queue.models import QueueItem
logger = logging.getLogger("VideoArchiver")
class QueueProcessor:
"""Handles processing of video queue items"""
"""
Handles the processing of queue items with priority-based scheduling.
"""
_active_items: ClassVar[Set[int]] = set()
_processing_lock: ClassVar[asyncio.Lock] = asyncio.Lock()
# Class variables for tracking global state
active_items: ClassVar[Set[str]] = set()
processing_metrics: ClassVar[Dict[str, ProcessingMetrics]] = {}
def __init__(self, queue_manager):
"""Initialize queue processor
Args:
queue_manager: Queue manager instance to handle queue operations
def __init__(self):
self.queue_metrics = QueueMetrics()
self.processing_lock = asyncio.Lock()
self.is_running = False
self._current_item: Optional[QueueItem] = None
self._priority_queues: Dict[QueuePriority, List[QueueItem]] = {
priority: [] for priority in QueuePriority
}
@property
def current_item(self) -> Optional[QueueItem]:
"""Get the currently processing item."""
return self._current_item
def add_item(self, item: QueueItem) -> bool:
"""
self.queue_manager = queue_manager
self._metrics = ProcessingMetrics()
async def process_urls(self, message, urls, priority: QueuePriority = QueuePriority.NORMAL) -> None:
"""Process URLs from a message
Args:
message: Discord message containing URLs
urls: List of URLs to process
priority: Processing priority level
"""
for url_metadata in urls:
await self.queue_manager.add_to_queue(
url=url_metadata.url,
message_id=message.id,
channel_id=message.channel.id,
guild_id=message.guild.id,
author_id=message.author.id,
priority=priority.value
)
async def process_item(self, item: QueueItem) -> bool:
"""Process a single queue item
Add an item to the appropriate priority queue.
Args:
item: Queue item to process
item: QueueItem to add
Returns:
bool: Success status
bool: True if item was added successfully
"""
if item.id in self._active_items:
logger.warning(f"Item {item.id} is already being processed")
if item.id in self.active_items:
logger.warning(f"Item {item.id} is already in queue")
return False
try:
self._active_items.add(item.id)
start_time = datetime.now()
self._priority_queues[item.priority].append(item)
self.active_items.add(item.id)
self.queue_metrics.total_items += 1
logger.info(f"Added item {item.id} to {item.priority.name} priority queue")
return True
# Process item logic here
# Placeholder for actual video processing
await asyncio.sleep(1)
def remove_item(self, item_id: str) -> Optional[QueueItem]:
"""
Remove an item from any priority queue.
processing_time = (datetime.now() - start_time).total_seconds()
self._update_metrics(processing_time, True, item.size)
return True
Args:
item_id: ID of item to remove
except Exception as e:
logger.error(f"Error processing item {item.id}: {str(e)}")
self._update_metrics(0, False, 0)
return False
finally:
self._active_items.remove(item.id)
Returns:
Optional[QueueItem]: Removed item if found, None otherwise
"""
for priority in QueuePriority:
queue = self._priority_queues[priority]
for item in queue:
if item.id == item_id:
queue.remove(item)
self.active_items.discard(item_id)
self.queue_metrics.total_items -= 1
logger.info(
f"Removed item {item_id} from {priority.name} priority queue"
)
return item
return None
def _update_metrics(self, processing_time: float, success: bool, size: int) -> None:
"""Update processing metrics"""
"""
Update processing metrics.
Args:
processing_time: Time taken to process the item
success: Whether processing was successful
size: Size of the processed item
"""
if success:
self._metrics.record_success(processing_time)
self.queue_metrics.record_success(processing_time)
else:
self._metrics.record_failure("Processing error")
self.queue_metrics.record_failure("Processing error")
def get_metrics(self) -> QueueMetrics:
"""Get current processing metrics"""
total = self._metrics.total_processed
"""
Get current processing metrics.
Returns:
QueueMetrics: Current queue processing metrics
"""
total = self.queue_metrics.total_processed
if total == 0:
return QueueMetrics(
total_items=0,
@@ -103,8 +116,8 @@ class QueueProcessor:
return QueueMetrics(
total_items=total,
processing_time=self._metrics.avg_processing_time,
success_rate=self._metrics.successful / total,
error_rate=self._metrics.failed / total,
processing_time=self.queue_metrics.avg_processing_time,
success_rate=self.queue_metrics.successful / total,
error_rate=self.queue_metrics.failed / total,
average_size=0, # This would need to be tracked separately if needed
)

4
videoarchiver/processor/reactions.py
View File

@@ -9,13 +9,13 @@ from urllib.parse import urlparse
# try:
# Try relative imports first
from ..processor.constants import (
from processor.constants import (
REACTIONS,
ReactionType,
get_reaction,
get_progress_emoji,
)
from ..database.video_archive_db import VideoArchiveDB
from database.video_archive_db import VideoArchiveDB
# except ImportError:
# Fall back to absolute imports if relative imports fail

2
videoarchiver/processor/status_display.py
View File

@@ -20,7 +20,7 @@ import discord # type: ignore
# try:
# Try relative imports first
from ..utils.exceptions import DisplayError
from utils.exceptions import DisplayError
# except ImportError:
# Fall back to absolute imports if relative imports fail