pacnpal/Pac-cogs
1
0
Fork 0
mirror of https://github.com/pacnpal/Pac-cogs.git synced 2025-12-20 02:41:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

📥 Pending: Shows items waiting to be processed

Browse Source 
⚙️ Processing: Shows items currently being processed
✅ Completed: Shows successfully processed items
❌ Failed: Shows failed items
This commit is contained in:
pacnpal
2024-11-15 14:12:43 +00:00
parent 0d7b1d449f
commit f8d383a55a
3 changed files with 101 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
videoarchiver/enhanced_queue.py
View File

@@ -115,6 +115,7 @@ class QueueItem:
last_error_time: Optional[datetime] = None last_error_time: Optional[datetime] = None
hardware_accel_attempted: bool = False hardware_accel_attempted: bool = False
compression_attempted: bool = False compression_attempted: bool = False
original_message: Optional[Any] = None # Store the original message reference
class EnhancedVideoQueueManager: class EnhancedVideoQueueManager:
@@ -190,7 +191,9 @@ class EnhancedVideoQueueManager:
channel_id: int, channel_id: int,
guild_id: int, guild_id: int,
author_id: int, author_id: int,
callback: Optional[Callable[[str, bool, str], Any]] = None, # Make callback optional callback: Optional[
Callable[[str, bool, str], Any]
] = None, # Make callback optional
priority: int = 0, priority: int = 0,
) -> bool: ) -> bool:
"""Add a video to the processing queue with priority support""" """Add a video to the processing queue with priority support"""
@@ -268,7 +271,9 @@ class EnhancedVideoQueueManager:
start_time = time.time() start_time = time.time()
logger.info(f"Calling processor for item: {item.url}") logger.info(f"Calling processor for item: {item.url}")
success, error = await processor(item) success, error = await processor(item)
logger.info(f"Processor result for {item.url}: success={success}, error={error}") logger.info(
f"Processor result for {item.url}: success={success}, error={error}"
)
processing_time = time.time() - start_time processing_time = time.time() - start_time
# Update metrics # Update metrics
@@ -299,7 +304,9 @@ class EnhancedVideoQueueManager:
item.compression_attempted = True item.compression_attempted = True
# Add back to queue with adjusted priority # Add back to queue with adjusted priority
item.priority = max(0, item.priority - 1) # Lower priority for retries item.priority = max(
0, item.priority - 1
) # Lower priority for retries
self._queue.append(item) self._queue.append(item)
logger.warning( logger.warning(
f"Retrying item: {item.url} (attempt {item.retry_count})" f"Retrying item: {item.url} (attempt {item.retry_count})"
@@ -336,7 +343,9 @@ class EnhancedVideoQueueManager:
# Continue processing even if persistence fails # Continue processing even if persistence fails
except Exception as e: except Exception as e:
logger.error(f"Critical error in queue processor: {traceback.format_exc()}") logger.error(
f"Critical error in queue processor: {traceback.format_exc()}"
)
# Ensure we don't get stuck in a tight loop on critical errors # Ensure we don't get stuck in a tight loop on critical errors
await asyncio.sleep(1) await asyncio.sleep(1)
continue # Continue to next iteration to process remaining items continue # Continue to next iteration to process remaining items
@@ -421,7 +430,9 @@ class EnhancedVideoQueueManager:
if item.get("last_retry"): if item.get("last_retry"):
item["last_retry"] = datetime.fromisoformat(item["last_retry"]) item["last_retry"] = datetime.fromisoformat(item["last_retry"])
if item.get("last_error_time"): if item.get("last_error_time"):
item["last_error_time"] = datetime.fromisoformat(item["last_error_time"]) item["last_error_time"] = datetime.fromisoformat(
item["last_error_time"]
)
self._queue.append(QueueItem(**item)) self._queue.append(QueueItem(**item))
self._processing = { self._processing = {
@@ -438,8 +449,12 @@ class EnhancedVideoQueueManager:
self.metrics.success_rate = metrics_data["success_rate"] self.metrics.success_rate = metrics_data["success_rate"]
self.metrics.errors_by_type = metrics_data["errors_by_type"] self.metrics.errors_by_type = metrics_data["errors_by_type"]
self.metrics.last_error = metrics_data["last_error"] self.metrics.last_error = metrics_data["last_error"]
self.metrics.compression_failures = metrics_data.get("compression_failures", 0) self.metrics.compression_failures = metrics_data.get(
self.metrics.hardware_accel_failures = metrics_data.get("hardware_accel_failures", 0) "compression_failures", 0
)
self.metrics.hardware_accel_failures = metrics_data.get(
"hardware_accel_failures", 0
)
if metrics_data["last_error_time"]: if metrics_data["last_error_time"]:
self.metrics.last_error_time = datetime.fromisoformat( self.metrics.last_error_time = datetime.fromisoformat(
@@ -477,6 +492,7 @@ class EnhancedVideoQueueManager:
logger.warning(f"High memory usage detected: {memory_usage:.2f}MB") logger.warning(f"High memory usage detected: {memory_usage:.2f}MB")
# Force garbage collection # Force garbage collection
import gc import gc
gc.collect() gc.collect()
# Check for potential deadlocks with reduced threshold # Check for potential deadlocks with reduced threshold
@@ -524,7 +540,8 @@ class EnhancedVideoQueueManager:
for url, item in list(self._processing.items()): for url, item in list(self._processing.items()):
if ( if (
item.processing_time > 0 item.processing_time > 0
and (current_time - item.processing_time) > self.deadlock_threshold and (current_time - item.processing_time)
> self.deadlock_threshold
): ):
# Move to failed queue if max retries reached # Move to failed queue if max retries reached
if item.retry_count >= self.max_retries: if item.retry_count >= self.max_retries:
@@ -586,9 +603,19 @@ class EnhancedVideoQueueManager:
try: try:
# Count items for this guild # Count items for this guild
pending = len([item for item in self._queue if item.guild_id == guild_id]) pending = len([item for item in self._queue if item.guild_id == guild_id])
processing = len([item for item in self._processing.values() if item.guild_id == guild_id]) processing = len(
completed = len([item for item in self._completed.values() if item.guild_id == guild_id]) [
failed = len([item for item in self._failed.values() if item.guild_id == guild_id]) item
for item in self._processing.values()
if item.guild_id == guild_id
]
)
completed = len(
[item for item in self._completed.values() if item.guild_id == guild_id]
)
failed = len(
[item for item in self._failed.values() if item.guild_id == guild_id]
)
# Get metrics # Get metrics
metrics = { metrics = {
@@ -600,7 +627,7 @@ class EnhancedVideoQueueManager:
"last_cleanup": self.metrics.last_cleanup.strftime("%Y-%m-%d %H:%M:%S"), "last_cleanup": self.metrics.last_cleanup.strftime("%Y-%m-%d %H:%M:%S"),
"errors_by_type": self.metrics.errors_by_type, "errors_by_type": self.metrics.errors_by_type,
"compression_failures": self.metrics.compression_failures, "compression_failures": self.metrics.compression_failures,
"hardware_accel_failures": self.metrics.hardware_accel_failures "hardware_accel_failures": self.metrics.hardware_accel_failures,
} }
return { return {
@@ -608,7 +635,7 @@ class EnhancedVideoQueueManager:
"processing": processing, "processing": processing,
"completed": completed, "completed": completed,
"failed": failed, "failed": failed,
"metrics": metrics "metrics": metrics,
} }
except Exception as e: except Exception as e:
@@ -628,8 +655,8 @@ class EnhancedVideoQueueManager:
"last_cleanup": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), "last_cleanup": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
"errors_by_type": {}, "errors_by_type": {},
"compression_failures": 0, "compression_failures": 0,
"hardware_accel_failures": 0 "hardware_accel_failures": 0,
} },
} }
async def clear_guild_queue(self, guild_id: int) -> int: async def clear_guild_queue(self, guild_id: int) -> int:
@@ -648,7 +675,9 @@ class EnhancedVideoQueueManager:
guild_urls = self._guild_queues.get(guild_id, set()) guild_urls = self._guild_queues.get(guild_id, set())
# Clear from pending queue # Clear from pending queue
self._queue = [item for item in self._queue if item.guild_id != guild_id] self._queue = [
item for item in self._queue if item.guild_id != guild_id
]
# Clear from processing # Clear from processing
for url in list(self._processing.keys()): for url in list(self._processing.keys()):
@@ -676,7 +705,8 @@ class EnhancedVideoQueueManager:
# Clear channel tracking for this guild's channels # Clear channel tracking for this guild's channels
for channel_id in list(self._channel_queues.keys()): for channel_id in list(self._channel_queues.keys()):
self._channel_queues[channel_id] = { self._channel_queues[channel_id] = {
url for url in self._channel_queues[channel_id] url
for url in self._channel_queues[channel_id]
if url not in guild_urls if url not in guild_urls
} }

50
videoarchiver/processor.py
View File

@@ -3,7 +3,7 @@
import os import os
import logging import logging
import asyncio import asyncio
import discord # Added missing import import discord
from pathlib import Path from pathlib import Path
from typing import Dict, Any, Optional, Tuple from typing import Dict, Any, Optional, Tuple
import traceback import traceback
@@ -80,6 +80,12 @@ class VideoProcessor:
if not content or not downloader.is_supported_url(content): if not content or not downloader.is_supported_url(content):
return return
# Add video camera reaction to indicate processing
try:
await message.add_reaction("📹")
except Exception as e:
logger.error(f"Failed to add video camera reaction: {e}")
# Add to processing queue # Add to processing queue
await self.queue_manager.add_to_queue( await self.queue_manager.add_to_queue(
url=content, url=content,
@@ -108,6 +114,18 @@ class VideoProcessor:
if not downloader or not message_manager: if not downloader or not message_manager:
return False, f"Missing required components for guild {guild_id}" return False, f"Missing required components for guild {guild_id}"
# Get original message
try:
channel = self.bot.get_channel(item.channel_id)
if not channel:
return False, f"Channel {item.channel_id} not found"
original_message = await channel.fetch_message(item.message_id)
except discord.NotFound:
original_message = None
except Exception as e:
logger.error(f"Error fetching original message: {e}")
original_message = None
# Download and process video # Download and process video
try: try:
success, file_path, error = await downloader.download_video(item.url) success, file_path, error = await downloader.download_video(item.url)
@@ -126,30 +144,12 @@ class VideoProcessor:
if not archive_channel: if not archive_channel:
return False, "Archive channel not configured" return False, "Archive channel not configured"
# Get original message
try:
channel = self.bot.get_channel(item.channel_id)
if not channel:
return False, f"Channel {item.channel_id} not found"
original_message = await channel.fetch_message(item.message_id)
except discord.NotFound:
original_message = None
except Exception as e:
logger.error(f"Error fetching original message: {e}")
original_message = None
# Format message # Format message
try: try:
author = original_message.author if original_message else None author = original_message.author if original_message else None
message = await message_manager.format_message( message = await message_manager.format_message(
author=author, author=author,
channel=channel, channel=channel,
original_message=original_message
)
except Exception as e:
logger.error(f"Error formatting message: {e}")
message = f"Video from {item.url}"
# Upload to archive channel # Upload to archive channel
try: try:
if not os.path.exists(file_path): if not os.path.exists(file_path):
@@ -159,21 +159,12 @@ class VideoProcessor:
content=message, content=message,
file=discord.File(file_path) file=discord.File(file_path)
) )
except discord.HTTPException as e: except discord.HTTPException as e:
return False, f"Failed to upload to Discord: {str(e)}" return False, f"Failed to upload to Discord: {str(e)}"
except Exception as e: except Exception as e:
return False, f"Failed to archive video: {str(e)}" return False, f"Failed to archive video: {str(e)}"
# Delete original if configured
if original_message:
try:
settings = await self.config.get_guild_settings(guild_id)
if settings.get("delete_after_repost", False):
await original_message.delete()
except Exception as e:
logger.warning(f"Failed to delete original message: {e}")
# Don't fail the process for deletion errors
return True, None return True, None
except Exception as e: except Exception as e:
@@ -186,7 +177,6 @@ class VideoProcessor:
os.unlink(file_path) os.unlink(file_path)
except Exception as e: except Exception as e:
logger.error(f"Failed to clean up file {file_path}: {e}") logger.error(f"Failed to clean up file {file_path}: {e}")
# Don't fail the process for cleanup errors
except Exception as e: except Exception as e:
logger.error(f"Error processing video: {traceback.format_exc()}") logger.error(f"Error processing video: {traceback.format_exc()}")

21
videoarchiver/utils/message_manager.py
View File

@@ -2,7 +2,7 @@
import asyncio import asyncio
import logging import logging
from typing import Dict from typing import Dict, List
logger = logging.getLogger("VideoArchiver") logger = logging.getLogger("VideoArchiver")
@@ -20,6 +20,25 @@ class MessageManager:
author=author, url=url, original_message=original_message author=author, url=url, original_message=original_message
) )
async def get_message_reactions(self, message) -> List[Dict]:
"""Get all reactions from a message"""
reactions = []
for reaction in message.reactions:
reactions.append({
'emoji': str(reaction.emoji),
'count': reaction.count,
'users': [user.id async for user in reaction.users()]
})
return reactions
async def restore_reactions(self, message, reactions: List[Dict]) -> None:
"""Restore reactions to a message"""
for reaction_data in reactions:
try:
await message.add_reaction(reaction_data['emoji'])
except Exception as e:
logger.error(f"Failed to restore reaction {reaction_data['emoji']}: {str(e)}")
async def schedule_message_deletion(self, message_id: int, delete_func) -> None: async def schedule_message_deletion(self, message_id: int, delete_func) -> None:
if self.message_duration <= 0: if self.message_duration <= 0:
return return