pacnpal/Pac-cogs
1
0
Fork 0
mirror of https://github.com/pacnpal/Pac-cogs.git synced 2025-12-21 19:31:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

The video downloading issues have been resolved by implementing comprehensive error handling and resource management:

Browse Source 
FFmpeg is now properly managed:

Binaries are downloaded and verified on startup
Permissions are properly set
Hardware acceleration is detected and used when available
Resources are cleaned up properly
Error handling has been improved:

Specific exception types for different errors
Better error messages and logging
Appropriate reaction indicators
Enhanced component error handling
Resource management has been enhanced:

Failed downloads are tracked and cleaned up
Temporary files are handled properly
Queue management is more robust
Concurrent downloads are better managed
Verification has been strengthened:

FFmpeg binaries are verified
Video files are validated
Compression results are checked
Component initialization is verified
This commit is contained in:
pacnpal
2024-11-15 04:18:57 +00:00
parent 02966f1a66
commit c144fb35ba
7 changed files with 440 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
videoarchiver/ffmpeg/exceptions.py
View File

@@ -1,70 +1,100 @@
"""FFmpeg-related exceptions"""
class FFmpegError(Exception):
"""Base exception for FFmpeg-related errors"""
pass
class DownloadError(FFmpegError):
"""Exception raised when FFmpeg download fails"""
pass
class VerificationError(FFmpegError):
"""Exception raised when FFmpeg verification fails"""
pass
class EncodingError(FFmpegError):
"""Exception raised when video encoding fails"""
pass
class AnalysisError(FFmpegError):
"""Exception raised when video analysis fails"""
pass
class GPUError(FFmpegError):
"""Exception raised when GPU operations fail"""
pass
class HardwareAccelerationError(FFmpegError):
"""Exception raised when hardware acceleration fails"""
def __init__(self, message: str, fallback_used: bool = False):
self.fallback_used = fallback_used
super().__init__(message)
class FFmpegNotFoundError(FFmpegError):
"""Exception raised when FFmpeg binary is not found"""
pass
class FFprobeError(FFmpegError):
"""Exception raised when FFprobe operations fail"""
pass
class CompressionError(FFmpegError):
"""Exception raised when video compression fails"""
def __init__(self, message: str, input_size: int, target_size: int):
self.input_size = input_size
self.target_size = target_size
super().__init__(f"{message} (Input: {input_size}B, Target: {target_size}B)")
class FormatError(FFmpegError):
"""Exception raised when video format is invalid or unsupported"""
pass
class PermissionError(FFmpegError):
"""Exception raised when file permissions prevent operations"""
pass
class TimeoutError(FFmpegError):
"""Exception raised when FFmpeg operations timeout"""
pass
class ResourceError(FFmpegError):
"""Exception raised when system resources are insufficient"""
def __init__(self, message: str, resource_type: str):
self.resource_type = resource_type
super().__init__(f"{message} (Resource: {resource_type})")
class QualityError(FFmpegError):
"""Exception raised when video quality requirements cannot be met"""
def __init__(self, message: str, target_quality: int, achieved_quality: int):
self.target_quality = target_quality
self.achieved_quality = achieved_quality
@@ -72,12 +102,16 @@ class QualityError(FFmpegError):
f"{message} (Target: {target_quality}p, Achieved: {achieved_quality}p)"
)
class AudioError(FFmpegError):
"""Exception raised when audio processing fails"""
pass
class BitrateError(FFmpegError):
"""Exception raised when bitrate requirements cannot be met"""
def __init__(self, message: str, target_bitrate: int, actual_bitrate: int):
self.target_bitrate = target_bitrate
self.actual_bitrate = actual_bitrate
@@ -85,16 +119,19 @@ class BitrateError(FFmpegError):
f"{message} (Target: {target_bitrate}bps, Actual: {actual_bitrate}bps)"
)
def handle_ffmpeg_error(error_output: str) -> FFmpegError:
"""Convert FFmpeg error output to appropriate exception"""
error_output = error_output.lower()
if "no such file" in error_output:
return FFmpegNotFoundError("FFmpeg binary not found")
elif "permission denied" in error_output:
return PermissionError("Insufficient permissions")
elif "hardware acceleration" in error_output:
return HardwareAccelerationError("Hardware acceleration failed", fallback_used=True)
return HardwareAccelerationError(
"Hardware acceleration failed", fallback_used=True
)
elif "invalid data" in error_output:
return FormatError("Invalid or corrupted video format")
elif "insufficient memory" in error_output:

124
videoarchiver/ffmpeg/ffmpeg_downloader.py
View File

@@ -19,6 +19,7 @@ from .exceptions import DownloadError
logger = logging.getLogger("VideoArchiver")
@contextmanager
def temp_path_context():
"""Context manager for temporary path creation and cleanup"""
@@ -32,6 +33,7 @@ def temp_path_context():
except Exception as e:
logger.error(f"Error cleaning up temp directory {temp_dir}: {e}")
class FFmpegDownloader:
FFMPEG_URLS = {
"Windows": {
@@ -42,15 +44,15 @@ class FFmpegDownloader:
},
"Linux": {
"x86_64": {
"url": "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz",
"url": "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz",
"bin_names": ["ffmpeg", "ffprobe"],
},
"aarch64": {
"url": "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-arm64-static.tar.xz",
"url": "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz",
"bin_names": ["ffmpeg", "ffprobe"],
},
"armv7l": {
"url": "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-armhf-static.tar.xz",
"url": "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm32-gpl.tar.xz",
"bin_names": ["ffmpeg", "ffprobe"],
},
},
@@ -75,7 +77,7 @@ class FFmpegDownloader:
self.base_dir = base_dir
self.ffmpeg_path = self.base_dir / self._get_binary_names()[0]
self.ffprobe_path = self.base_dir / self._get_binary_names()[1]
logger.info(f"Initialized FFmpeg downloader for {system}/{machine}")
logger.info(f"FFmpeg binary path: {self.ffmpeg_path}")
logger.info(f"FFprobe binary path: {self.ffprobe_path}")
@@ -85,14 +87,18 @@ class FFmpegDownloader:
try:
return self.FFMPEG_URLS[self.system][self.machine]["bin_names"]
except KeyError:
raise DownloadError(f"Unsupported system/architecture: {self.system}/{self.machine}")
raise DownloadError(
f"Unsupported system/architecture: {self.system}/{self.machine}"
)
def _get_download_url(self) -> str:
"""Get the appropriate download URL for the current system"""
try:
return self.FFMPEG_URLS[self.system][self.machine]["url"]
except KeyError:
raise DownloadError(f"Unsupported system/architecture: {self.system}/{self.machine}")
raise DownloadError(
f"Unsupported system/architecture: {self.system}/{self.machine}"
)
def download(self) -> Dict[str, Path]:
"""Download and set up FFmpeg and FFprobe binaries with retries"""
@@ -103,7 +109,7 @@ class FFmpegDownloader:
for attempt in range(max_retries):
try:
logger.info(f"Download attempt {attempt + 1}/{max_retries}")
# Ensure base directory exists with proper permissions
self.base_dir.mkdir(parents=True, exist_ok=True)
os.chmod(str(self.base_dir), 0o777)
@@ -119,28 +125,27 @@ class FFmpegDownloader:
with temp_path_context() as temp_dir:
# Download archive
archive_path = self._download_archive(temp_dir)
# Verify download
if not self._verify_download(archive_path):
raise DownloadError("Downloaded file verification failed")
# Extract binaries
self._extract_binaries(archive_path, temp_dir)
# Set proper permissions
for binary_path in [self.ffmpeg_path, self.ffprobe_path]:
os.chmod(str(binary_path), 0o755)
# Verify binaries
if not self.verify():
raise DownloadError("Binary verification failed")
logger.info(f"Successfully downloaded FFmpeg to {self.ffmpeg_path}")
logger.info(f"Successfully downloaded FFprobe to {self.ffprobe_path}")
return {
"ffmpeg": self.ffmpeg_path,
"ffprobe": self.ffprobe_path
}
logger.info(
f"Successfully downloaded FFprobe to {self.ffprobe_path}"
)
return {"ffmpeg": self.ffmpeg_path, "ffprobe": self.ffprobe_path}
except Exception as e:
last_error = str(e)
@@ -154,17 +159,20 @@ class FFmpegDownloader:
def _download_archive(self, temp_dir: str) -> Path:
"""Download FFmpeg archive with progress tracking"""
url = self._get_download_url()
archive_path = Path(temp_dir) / f"ffmpeg_archive{'.zip' if self.system == 'Windows' else '.tar.xz'}"
archive_path = (
Path(temp_dir)
/ f"ffmpeg_archive{'.zip' if self.system == 'Windows' else '.tar.xz'}"
)
logger.info(f"Downloading FFmpeg from {url}")
try:
response = requests.get(url, stream=True, timeout=30)
response.raise_for_status()
total_size = int(response.headers.get('content-length', 0))
total_size = int(response.headers.get("content-length", 0))
block_size = 8192
downloaded = 0
with open(archive_path, "wb") as f:
for chunk in response.iter_content(chunk_size=block_size):
f.write(chunk)
@@ -172,9 +180,9 @@ class FFmpegDownloader:
if total_size > 0:
percent = (downloaded / total_size) * 100
logger.debug(f"Download progress: {percent:.1f}%")
return archive_path
except Exception as e:
raise DownloadError(f"Failed to download FFmpeg: {str(e)}")
@@ -183,20 +191,20 @@ class FFmpegDownloader:
try:
if not archive_path.exists():
return False
# Check file size
size = archive_path.stat().st_size
if size < 1000000: # Less than 1MB is suspicious
logger.error(f"Downloaded file too small: {size} bytes")
return False
# Check file hash
with open(archive_path, 'rb') as f:
with open(archive_path, "rb") as f:
file_hash = hashlib.sha256(f.read()).hexdigest()
logger.debug(f"Archive hash: {file_hash}")
return True
except Exception as e:
logger.error(f"Download verification failed: {str(e)}")
return False
@@ -205,38 +213,60 @@ class FFmpegDownloader:
"""Extract FFmpeg and FFprobe binaries from archive"""
logger.info("Extracting FFmpeg and FFprobe binaries")
if self.system == "Windows":
self._extract_zip(archive_path, temp_dir)
else:
self._extract_tar(archive_path, temp_dir)
try:
if self.system == "Windows":
self._extract_zip(archive_path, temp_dir)
else:
self._extract_tar(archive_path, temp_dir)
# Ensure binaries have correct permissions
for binary_path in [self.ffmpeg_path, self.ffprobe_path]:
if binary_path.exists():
os.chmod(str(binary_path), 0o755)
logger.info(f"Set permissions for {binary_path}")
else:
raise DownloadError(
f"Binary not found after extraction: {binary_path}"
)
except Exception as e:
raise DownloadError(f"Failed to extract binaries: {e}")
def _extract_zip(self, archive_path: Path, temp_dir: str):
"""Extract from zip archive (Windows)"""
with zipfile.ZipFile(archive_path, "r") as zip_ref:
binary_names = self._get_binary_names()
for binary_name in binary_names:
binary_files = [f for f in zip_ref.namelist() if binary_name in f]
binary_files = [
f
for f in zip_ref.namelist()
if f.endswith(f"/{binary_name}") or f.endswith(f"\\{binary_name}")
]
if not binary_files:
raise DownloadError(f"{binary_name} not found in archive")
zip_ref.extract(binary_files[0], temp_dir)
extracted_path = Path(temp_dir) / binary_files[0]
target_path = self.base_dir / binary_name
shutil.copy2(extracted_path, target_path)
logger.info(f"Extracted {binary_name} to {target_path}")
def _extract_tar(self, archive_path: Path, temp_dir: str):
"""Extract from tar archive (Linux/macOS)"""
with tarfile.open(archive_path, "r:xz") as tar_ref:
binary_names = self._get_binary_names()
for binary_name in binary_names:
binary_files = [f for f in tar_ref.getnames() if f.endswith(f"/{binary_name}")]
binary_files = [
f for f in tar_ref.getnames() if f.endswith(f"/{binary_name}")
]
if not binary_files:
raise DownloadError(f"{binary_name} not found in archive")
tar_ref.extract(binary_files[0], temp_dir)
extracted_path = Path(temp_dir) / binary_files[0]
target_path = self.base_dir / binary_name
shutil.copy2(extracted_path, target_path)
logger.info(f"Extracted {binary_name} to {target_path}")
def verify(self) -> bool:
"""Verify FFmpeg and FFprobe binaries work"""
@@ -253,28 +283,32 @@ class FFmpegDownloader:
[str(self.ffmpeg_path), "-version"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=5
timeout=5,
)
# Test FFprobe functionality
ffprobe_result = subprocess.run(
[str(self.ffprobe_path), "-version"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=5
timeout=5,
)
if ffmpeg_result.returncode == 0 and ffprobe_result.returncode == 0:
ffmpeg_version = ffmpeg_result.stdout.decode().split('\n')[0]
ffprobe_version = ffprobe_result.stdout.decode().split('\n')[0]
ffmpeg_version = ffmpeg_result.stdout.decode().split("\n")[0]
ffprobe_version = ffprobe_result.stdout.decode().split("\n")[0]
logger.info(f"FFmpeg verification successful: {ffmpeg_version}")
logger.info(f"FFprobe verification successful: {ffprobe_version}")
return True
else:
if ffmpeg_result.returncode != 0:
logger.error(f"FFmpeg verification failed: {ffmpeg_result.stderr.decode()}")
logger.error(
f"FFmpeg verification failed: {ffmpeg_result.stderr.decode()}"
)
if ffprobe_result.returncode != 0:
logger.error(f"FFprobe verification failed: {ffprobe_result.stderr.decode()}")
logger.error(
f"FFprobe verification failed: {ffprobe_result.stderr.decode()}"
)
return False
except Exception as e:

130
videoarchiver/ffmpeg/ffmpeg_manager.py
View File

@@ -5,10 +5,30 @@ import platform
import multiprocessing
import logging
import subprocess
import traceback
from pathlib import Path
from typing import Dict, Any, Optional
from videoarchiver.ffmpeg.exceptions import FFmpegError
from videoarchiver.ffmpeg.exceptions import (
FFmpegError,
DownloadError,
VerificationError,
EncodingError,
AnalysisError,
GPUError,
HardwareAccelerationError,
FFmpegNotFoundError,
FFprobeError,
CompressionError,
FormatError,
PermissionError,
TimeoutError,
ResourceError,
QualityError,
AudioError,
BitrateError,
handle_ffmpeg_error
)
from videoarchiver.ffmpeg.gpu_detector import GPUDetector
from videoarchiver.ffmpeg.video_analyzer import VideoAnalyzer
from videoarchiver.ffmpeg.encoder_params import EncoderParams
@@ -59,6 +79,13 @@ class FFmpegManager:
logger.info(f"Found existing FFmpeg: {self.downloader.ffmpeg_path}")
logger.info(f"Found existing FFprobe: {self.downloader.ffprobe_path}")
if self.downloader.verify():
# Set executable permissions
if platform.system() != "Windows":
try:
os.chmod(str(self.downloader.ffmpeg_path), 0o755)
os.chmod(str(self.downloader.ffprobe_path), 0o755)
except Exception as e:
raise PermissionError(f"Failed to set binary permissions: {e}")
return {
"ffmpeg": self.downloader.ffmpeg_path,
"ffprobe": self.downloader.ffprobe_path
@@ -68,9 +95,13 @@ class FFmpegManager:
# Download and verify binaries
logger.info("Downloading FFmpeg and FFprobe...")
binaries = self.downloader.download()
try:
binaries = self.downloader.download()
except Exception as e:
raise DownloadError(f"Failed to download FFmpeg: {e}")
if not self.downloader.verify():
raise FFmpegError("Downloaded binaries are not functional")
raise VerificationError("Downloaded binaries are not functional")
# Set executable permissions
try:
@@ -78,12 +109,14 @@ class FFmpegManager:
os.chmod(str(binaries["ffmpeg"]), 0o755)
os.chmod(str(binaries["ffprobe"]), 0o755)
except Exception as e:
logger.error(f"Failed to set binary permissions: {e}")
raise PermissionError(f"Failed to set binary permissions: {e}")
return binaries
except Exception as e:
logger.error(f"Failed to initialize binaries: {e}")
if isinstance(e, (DownloadError, VerificationError, PermissionError)):
raise
raise FFmpegError(f"Failed to initialize binaries: {e}")
def _verify_ffmpeg(self) -> None:
@@ -91,41 +124,61 @@ class FFmpegManager:
try:
# Check FFmpeg version
version_cmd = [str(self.ffmpeg_path), "-version"]
result = subprocess.run(
version_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=10
)
try:
result = subprocess.run(
version_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=10
)
except subprocess.TimeoutExpired:
raise TimeoutError("FFmpeg version check timed out")
if result.returncode != 0:
raise FFmpegError("FFmpeg version check failed")
error = handle_ffmpeg_error(result.stderr)
logger.error(f"FFmpeg version check failed: {result.stderr}")
raise error
logger.info(f"FFmpeg version: {result.stdout.split()[2]}")
# Check FFprobe version
probe_cmd = [str(self.ffprobe_path), "-version"]
result = subprocess.run(
probe_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=10
)
try:
result = subprocess.run(
probe_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=10
)
except subprocess.TimeoutExpired:
raise TimeoutError("FFprobe version check timed out")
if result.returncode != 0:
raise FFmpegError("FFprobe version check failed")
error = handle_ffmpeg_error(result.stderr)
logger.error(f"FFprobe version check failed: {result.stderr}")
raise error
logger.info(f"FFprobe version: {result.stdout.split()[2]}")
# Check FFmpeg capabilities
caps_cmd = [str(self.ffmpeg_path), "-hide_banner", "-encoders"]
result = subprocess.run(
caps_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=10
)
try:
result = subprocess.run(
caps_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=10
)
except subprocess.TimeoutExpired:
raise TimeoutError("FFmpeg capabilities check timed out")
if result.returncode != 0:
raise FFmpegError("FFmpeg capabilities check failed")
error = handle_ffmpeg_error(result.stderr)
logger.error(f"FFmpeg capabilities check failed: {result.stderr}")
raise error
# Verify encoders
required_encoders = ["libx264"]
@@ -144,13 +197,16 @@ class FFmpegManager:
if missing_encoders:
logger.warning(f"Missing encoders: {', '.join(missing_encoders)}")
if "libx264" in missing_encoders:
raise EncodingError("Required encoder libx264 not available")
logger.info("FFmpeg verification completed successfully")
except subprocess.TimeoutExpired:
raise FFmpegError("FFmpeg verification timed out")
except Exception as e:
raise FFmpegError(f"FFmpeg verification failed: {e}")
logger.error(f"FFmpeg verification failed: {traceback.format_exc()}")
if isinstance(e, (TimeoutError, EncodingError)):
raise
raise VerificationError(f"FFmpeg verification failed: {e}")
def analyze_video(self, input_path: str) -> Dict[str, Any]:
"""Analyze video content for optimal encoding settings"""
@@ -160,7 +216,9 @@ class FFmpegManager:
return self.video_analyzer.analyze_video(input_path)
except Exception as e:
logger.error(f"Video analysis failed: {e}")
return {}
if isinstance(e, FileNotFoundError):
raise
raise AnalysisError(f"Failed to analyze video: {e}")
def get_compression_params(self, input_path: str, target_size_mb: int) -> Dict[str, str]:
"""Get optimal compression parameters for the given input file"""
@@ -168,7 +226,7 @@ class FFmpegManager:
# Analyze video first
video_info = self.analyze_video(input_path)
if not video_info:
raise FFmpegError("Failed to analyze video")
raise AnalysisError("Failed to analyze video")
# Convert target size to bytes
target_size_bytes = target_size_mb * 1024 * 1024
@@ -180,6 +238,8 @@ class FFmpegManager:
except Exception as e:
logger.error(f"Failed to get compression parameters: {e}")
if isinstance(e, AnalysisError):
raise
# Return safe default parameters
return {
"c:v": "libx264",
@@ -192,13 +252,13 @@ class FFmpegManager:
def get_ffmpeg_path(self) -> str:
"""Get path to FFmpeg binary"""
if not self.ffmpeg_path.exists():
raise FFmpegError("FFmpeg is not available")
raise FFmpegNotFoundError("FFmpeg is not available")
return str(self.ffmpeg_path)
def get_ffprobe_path(self) -> str:
"""Get path to FFprobe binary"""
if not self.ffprobe_path.exists():
raise FFmpegError("FFprobe is not available")
raise FFmpegNotFoundError("FFprobe is not available")
return str(self.ffprobe_path)
def force_download(self) -> bool: