pacnpal/Pac-cogs
1
0
Fork 0
mirror of https://github.com/pacnpal/Pac-cogs.git synced 2025-12-20 02:41:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

Moved progress_tracker.py to utils/ as the single source of truth

Browse Source 
Implemented proper singleton pattern in ProgressTracker class
Created shared instance in utils/init.py
Updated all files to use relative imports correctly
Removed duplicate progress_tracker.py from processor/
Fixed circular imports by removing ProgressTracker from processor.py re-exports
This commit is contained in:
pacnpal
2024-11-17 01:24:03 +00:00
parent 98ed3dfc6a
commit b10722f05b
2 changed files with 65 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
videoarchiver/processor.py
View File

@@ -3,7 +3,6 @@
from .processor import (
VideoProcessor,
REACTIONS,
ProgressTracker,
MessageHandler,
QueueHandler
)
@@ -11,7 +10,6 @@ from .processor import (
__all__ = [
'VideoProcessor',
'REACTIONS',
'ProgressTracker',
'MessageHandler',
'QueueHandler'
]

85
videoarchiver/processor/cleanup_manager.py
View File

@@ -4,7 +4,18 @@ import logging
import asyncio
from enum import Enum, auto
from dataclasses import dataclass, field
from typing import Optional, Dict, Any, List, Set, TypedDict, ClassVar, Callable, Awaitable, Tuple
from typing import (
Optional,
Dict,
Any,
List,
Set,
TypedDict,
ClassVar,
Callable,
Awaitable,
Tuple,
)
from datetime import datetime, timedelta
from .queue_handler import QueueHandler
@@ -13,44 +24,55 @@ from ..utils.exceptions import CleanupError
logger = logging.getLogger("VideoArchiver")
class CleanupStage(Enum):
"""Cleanup stages"""
QUEUE = auto()
FFMPEG = auto()
TASKS = auto()
RESOURCES = auto()
class CleanupStrategy(Enum):
"""Cleanup strategies"""
NORMAL = auto()
FORCE = auto()
GRACEFUL = auto()
class CleanupStats(TypedDict):
"""Type definition for cleanup statistics"""
total_cleanups: int
active_cleanups: int
success_rate: float
average_duration: float
stage_success_rates: Dict[str, float]
@dataclass
class CleanupResult:
"""Result of a cleanup operation"""
success: bool
stage: CleanupStage
error: Optional[str] = None
duration: float = 0.0
timestamp: str = field(default_factory=lambda: datetime.utcnow().isoformat())
@dataclass
class CleanupOperation:
"""Represents a cleanup operation"""
stage: CleanupStage
func: Callable[[], Awaitable[None]]
force_func: Optional[Callable[[], Awaitable[None]]] = None
timeout: float = 30.0 # Default timeout in seconds
class CleanupTracker:
"""Tracks cleanup operations"""
@@ -75,13 +97,9 @@ class CleanupTracker:
# Cleanup old history if needed
if len(self.cleanup_history) >= self.MAX_HISTORY:
self.cleanup_history = self.cleanup_history[-self.MAX_HISTORY:]
self.cleanup_history = self.cleanup_history[-self.MAX_HISTORY :]
def record_stage_result(
self,
cleanup_id: str,
result: CleanupResult
) -> None:
def record_stage_result(self, cleanup_id: str, result: CleanupResult) -> None:
"""
Record result of a cleanup stage.
@@ -101,13 +119,17 @@ class CleanupTracker:
"""
if cleanup_id in self.active_cleanups:
end_time = datetime.utcnow()
self.cleanup_history.append({
self.cleanup_history.append(
{
"id": cleanup_id,
"start_time": self.start_times[cleanup_id],
"end_time": end_time,
"duration": (end_time - self.start_times[cleanup_id]).total_seconds(),
"results": self.stage_results[cleanup_id]
})
"duration": (
end_time - self.start_times[cleanup_id]
).total_seconds(),
"results": self.stage_results[cleanup_id],
}
)
self.active_cleanups.remove(cleanup_id)
self.start_times.pop(cleanup_id)
self.stage_results.pop(cleanup_id)
@@ -124,7 +146,7 @@ class CleanupTracker:
active_cleanups=len(self.active_cleanups),
success_rate=self._calculate_success_rate(),
average_duration=self._calculate_average_duration(),
stage_success_rates=self._calculate_stage_success_rates()
stage_success_rates=self._calculate_stage_success_rates(),
)
def _calculate_success_rate(self) -> float:
@@ -132,7 +154,8 @@ class CleanupTracker:
if not self.cleanup_history:
return 1.0
successful = sum(
1 for cleanup in self.cleanup_history
1
for cleanup in self.cleanup_history
if all(result.success for result in cleanup["results"])
)
return successful / len(self.cleanup_history)
@@ -161,6 +184,7 @@ class CleanupTracker:
for stage, attempts in stage_attempts.items()
}
class CleanupManager:
"""Manages cleanup operations for the video processor"""
@@ -170,7 +194,7 @@ class CleanupManager:
self,
queue_handler: QueueHandler,
ffmpeg_mgr: Optional[FFmpegManager] = None,
strategy: CleanupStrategy = CleanupStrategy.NORMAL
strategy: CleanupStrategy = CleanupStrategy.NORMAL,
) -> None:
self.queue_handler = queue_handler
self.ffmpeg_mgr = ffmpeg_mgr
@@ -184,20 +208,20 @@ class CleanupManager:
stage=CleanupStage.QUEUE,
func=self._cleanup_queue,
force_func=self._force_cleanup_queue,
timeout=30.0
timeout=30.0,
),
CleanupOperation(
stage=CleanupStage.FFMPEG,
func=self._cleanup_ffmpeg,
force_func=self._force_cleanup_ffmpeg,
timeout=15.0
timeout=15.0,
),
CleanupOperation(
stage=CleanupStage.TASKS,
func=self._cleanup_tasks,
force_func=self._force_cleanup_tasks,
timeout=15.0
)
timeout=15.0,
),
]
async def cleanup(self) -> None:
@@ -217,21 +241,17 @@ class CleanupManager:
for operation in self.cleanup_operations:
try:
start_time = datetime.utcnow()
await asyncio.wait_for(
operation.func(),
timeout=operation.timeout
)
await asyncio.wait_for(operation.func(), timeout=operation.timeout)
duration = (datetime.utcnow() - start_time).total_seconds()
self.tracker.record_stage_result(
cleanup_id,
CleanupResult(True, operation.stage, duration=duration)
CleanupResult(True, operation.stage, duration=duration),
)
except asyncio.TimeoutError:
error = f"Cleanup stage {operation.stage.value} timed out"
logger.error(error)
self.tracker.record_stage_result(
cleanup_id,
CleanupResult(False, operation.stage, error)
cleanup_id, CleanupResult(False, operation.stage, error)
)
if self.strategy != CleanupStrategy.GRACEFUL:
raise CleanupError(error)
@@ -239,8 +259,7 @@ class CleanupManager:
error = f"Error in {operation.stage.value} cleanup: {e}"
logger.error(error)
self.tracker.record_stage_result(
cleanup_id,
CleanupResult(False, operation.stage, str(e))
cleanup_id, CleanupResult(False, operation.stage, str(e))
)
if self.strategy != CleanupStrategy.GRACEFUL:
raise CleanupError(error)
@@ -272,19 +291,17 @@ class CleanupManager:
try:
start_time = datetime.utcnow()
await asyncio.wait_for(
operation.force_func(),
timeout=operation.timeout
operation.force_func(), timeout=operation.timeout
)
duration = (datetime.utcnow() - start_time).total_seconds()
self.tracker.record_stage_result(
cleanup_id,
CleanupResult(True, operation.stage, duration=duration)
CleanupResult(True, operation.stage, duration=duration),
)
except Exception as e:
logger.error(f"Error in force {operation.stage.value} cleanup: {e}")
self.tracker.record_stage_result(
cleanup_id,
CleanupResult(False, operation.stage, str(e))
cleanup_id, CleanupResult(False, operation.stage, str(e))
)
logger.info("Force cleanup completed")
@@ -359,8 +376,8 @@ class CleanupManager:
{
"stage": op.stage.value,
"timeout": op.timeout,
"has_force_cleanup": op.force_func is not None
"has_force_cleanup": op.force_func is not None,
}
for op in self.cleanup_operations
]
],
}