mirror of
https://github.com/pacnpal/Pac-cogs.git
synced 2025-12-20 02:41:06 -05:00
Core Systems:
Component-based architecture with lifecycle management Enhanced error handling and recovery mechanisms Comprehensive state management and tracking Event-driven architecture with monitoring Queue Management: Multiple processing strategies for different scenarios Advanced state management with recovery Comprehensive metrics and health monitoring Sophisticated cleanup system with multiple strategies Processing Pipeline: Enhanced message handling with validation Improved URL extraction and processing Better queue management and monitoring Advanced cleanup mechanisms Overall Benefits: Better code organization and maintainability Improved error handling and recovery Enhanced monitoring and reporting More robust and reliable system
This commit is contained in:
pacnpal
237
videoarchiver/processor/queue_processor.py
Normal file
237
videoarchiver/processor/queue_processor.py
Normal file
@@ -0,0 +1,237 @@
|
||||
"""Queue processing functionality for video processing"""
|
||||
|
||||
import logging
|
||||
import asyncio
|
||||
from enum import Enum
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Dict, Any, Set
|
||||
from datetime import datetime
|
||||
import discord
|
||||
|
||||
from .reactions import REACTIONS
|
||||
|
||||
logger = logging.getLogger("VideoArchiver")
|
||||
|
||||
class QueuePriority(Enum):
|
||||
"""Queue item priorities"""
|
||||
HIGH = 0
|
||||
NORMAL = 1
|
||||
LOW = 2
|
||||
|
||||
@dataclass
|
||||
class QueueItem:
|
||||
"""Represents an item in the processing queue"""
|
||||
url: str
|
||||
message_id: int
|
||||
channel_id: int
|
||||
guild_id: int
|
||||
author_id: int
|
||||
priority: QueuePriority
|
||||
added_at: datetime
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
attempts: int = 0
|
||||
last_attempt: Optional[datetime] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
class ProcessingStrategy(Enum):
|
||||
"""Available processing strategies"""
|
||||
FIFO = "fifo" # First in, first out
|
||||
PRIORITY = "priority" # Process by priority
|
||||
SMART = "smart" # Smart processing based on various factors
|
||||
|
||||
class QueueMetrics:
|
||||
"""Tracks queue processing metrics"""
|
||||
|
||||
def __init__(self):
|
||||
self.total_processed = 0
|
||||
self.successful = 0
|
||||
self.failed = 0
|
||||
self.processing_times: List[float] = []
|
||||
self.errors: Dict[str, int] = {}
|
||||
self.last_processed: Optional[datetime] = None
|
||||
|
||||
def record_success(self, processing_time: float) -> None:
|
||||
"""Record successful processing"""
|
||||
self.total_processed += 1
|
||||
self.successful += 1
|
||||
self.processing_times.append(processing_time)
|
||||
self.last_processed = datetime.utcnow()
|
||||
|
||||
def record_failure(self, error: str) -> None:
|
||||
"""Record processing failure"""
|
||||
self.total_processed += 1
|
||||
self.failed += 1
|
||||
self.errors[error] = self.errors.get(error, 0) + 1
|
||||
self.last_processed = datetime.utcnow()
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get queue metrics"""
|
||||
avg_time = (
|
||||
sum(self.processing_times) / len(self.processing_times)
|
||||
if self.processing_times
|
||||
else 0
|
||||
)
|
||||
return {
|
||||
"total_processed": self.total_processed,
|
||||
"successful": self.successful,
|
||||
"failed": self.failed,
|
||||
"success_rate": (
|
||||
self.successful / self.total_processed
|
||||
if self.total_processed > 0
|
||||
else 0
|
||||
),
|
||||
"average_processing_time": avg_time,
|
||||
"error_counts": self.errors.copy(),
|
||||
"last_processed": self.last_processed
|
||||
}
|
||||
|
||||
class QueueProcessor:
|
||||
"""Handles adding videos to the processing queue"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
queue_manager,
|
||||
strategy: ProcessingStrategy = ProcessingStrategy.SMART,
|
||||
max_retries: int = 3
|
||||
):
|
||||
self.queue_manager = queue_manager
|
||||
self.strategy = strategy
|
||||
self.max_retries = max_retries
|
||||
self.metrics = QueueMetrics()
|
||||
self._processing: Set[str] = set()
|
||||
self._processing_lock = asyncio.Lock()
|
||||
|
||||
async def process_urls(
|
||||
self,
|
||||
message: discord.Message,
|
||||
urls: List[str],
|
||||
priority: QueuePriority = QueuePriority.NORMAL
|
||||
) -> None:
|
||||
"""Process extracted URLs by adding them to the queue"""
|
||||
for url in urls:
|
||||
try:
|
||||
logger.info(f"Adding URL to queue: {url}")
|
||||
await message.add_reaction(REACTIONS['queued'])
|
||||
|
||||
# Create queue item
|
||||
item = QueueItem(
|
||||
url=url,
|
||||
message_id=message.id,
|
||||
channel_id=message.channel.id,
|
||||
guild_id=message.guild.id,
|
||||
author_id=message.author.id,
|
||||
priority=priority,
|
||||
added_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
# Add to queue with appropriate strategy
|
||||
await self._add_to_queue(item)
|
||||
logger.info(f"Successfully added video to queue: {url}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to add video to queue: {str(e)}")
|
||||
await message.add_reaction(REACTIONS['error'])
|
||||
continue
|
||||
|
||||
async def _add_to_queue(self, item: QueueItem) -> None:
|
||||
"""Add item to queue using current strategy"""
|
||||
async with self._processing_lock:
|
||||
if item.url in self._processing:
|
||||
logger.debug(f"URL already being processed: {item.url}")
|
||||
return
|
||||
|
||||
self._processing.add(item.url)
|
||||
|
||||
try:
|
||||
# Apply processing strategy
|
||||
if self.strategy == ProcessingStrategy.PRIORITY:
|
||||
await self._add_with_priority(item)
|
||||
elif self.strategy == ProcessingStrategy.SMART:
|
||||
await self._add_with_smart_strategy(item)
|
||||
else: # FIFO
|
||||
await self._add_fifo(item)
|
||||
|
||||
finally:
|
||||
async with self._processing_lock:
|
||||
self._processing.remove(item.url)
|
||||
|
||||
async def _add_with_priority(self, item: QueueItem) -> None:
|
||||
"""Add item with priority handling"""
|
||||
await self.queue_manager.add_to_queue(
|
||||
url=item.url,
|
||||
message_id=item.message_id,
|
||||
channel_id=item.channel_id,
|
||||
guild_id=item.guild_id,
|
||||
author_id=item.author_id,
|
||||
priority=item.priority.value
|
||||
)
|
||||
|
||||
async def _add_with_smart_strategy(self, item: QueueItem) -> None:
|
||||
"""Add item using smart processing strategy"""
|
||||
# Calculate priority based on various factors
|
||||
priority = await self._calculate_smart_priority(item)
|
||||
|
||||
await self.queue_manager.add_to_queue(
|
||||
url=item.url,
|
||||
message_id=item.message_id,
|
||||
channel_id=item.channel_id,
|
||||
guild_id=item.guild_id,
|
||||
author_id=item.author_id,
|
||||
priority=priority
|
||||
)
|
||||
|
||||
async def _add_fifo(self, item: QueueItem) -> None:
|
||||
"""Add item using FIFO strategy"""
|
||||
await self.queue_manager.add_to_queue(
|
||||
url=item.url,
|
||||
message_id=item.message_id,
|
||||
channel_id=item.channel_id,
|
||||
guild_id=item.guild_id,
|
||||
author_id=item.author_id,
|
||||
priority=QueuePriority.NORMAL.value
|
||||
)
|
||||
|
||||
async def _calculate_smart_priority(self, item: QueueItem) -> int:
|
||||
"""Calculate priority using smart strategy"""
|
||||
base_priority = item.priority.value
|
||||
|
||||
# Adjust based on queue metrics
|
||||
stats = self.metrics.get_stats()
|
||||
if stats["total_processed"] > 0:
|
||||
# Boost priority if queue is processing efficiently
|
||||
if stats["success_rate"] > 0.9: # 90% success rate
|
||||
base_priority -= 1
|
||||
# Lower priority if having issues
|
||||
elif stats["success_rate"] < 0.5: # 50% success rate
|
||||
base_priority += 1
|
||||
|
||||
# Adjust based on retries
|
||||
if item.attempts > 0:
|
||||
base_priority += item.attempts
|
||||
|
||||
# Ensure priority stays in valid range
|
||||
return max(0, min(base_priority, len(QueuePriority) - 1))
|
||||
|
||||
async def format_archive_message(
|
||||
self,
|
||||
author: Optional[discord.Member],
|
||||
channel: discord.TextChannel,
|
||||
url: str
|
||||
) -> str:
|
||||
"""Format message for archive channel"""
|
||||
author_mention = author.mention if author else "Unknown User"
|
||||
channel_mention = channel.mention if channel else "Unknown Channel"
|
||||
|
||||
return (
|
||||
f"Video archived from {author_mention} in {channel_mention}\n"
|
||||
f"Original URL: {url}"
|
||||
)
|
||||
|
||||
def get_metrics(self) -> Dict[str, Any]:
|
||||
"""Get queue processing metrics"""
|
||||
return {
|
||||
"metrics": self.metrics.get_stats(),
|
||||
"strategy": self.strategy.value,
|
||||
"active_processing": len(self._processing),
|
||||
"max_retries": self.max_retries
|
||||
}
|
||||
Reference in New Issue
Block a user