mirror of
https://github.com/pacnpal/Pac-cogs.git
synced 2025-12-20 02:41:06 -05:00
Core Systems:
Component-based architecture with lifecycle management Enhanced error handling and recovery mechanisms Comprehensive state management and tracking Event-driven architecture with monitoring Queue Management: Multiple processing strategies for different scenarios Advanced state management with recovery Comprehensive metrics and health monitoring Sophisticated cleanup system with multiple strategies Processing Pipeline: Enhanced message handling with validation Improved URL extraction and processing Better queue management and monitoring Advanced cleanup mechanisms Overall Benefits: Better code organization and maintainability Improved error handling and recovery Enhanced monitoring and reporting More robust and reliable system
This commit is contained in:
pacnpal
@@ -1,135 +1,150 @@
|
||||
"""File operation utilities"""
|
||||
|
||||
import os
|
||||
import stat
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
from .exceptions import FileCleanupError
|
||||
from .file_deletion import SecureFileDeleter
|
||||
from .directory_manager import DirectoryManager
|
||||
from .permission_manager import PermissionManager
|
||||
|
||||
logger = logging.getLogger("VideoArchiver")
|
||||
|
||||
async def secure_delete_file(file_path: str, max_size: int = 100 * 1024 * 1024) -> bool:
|
||||
"""Delete a file securely
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to delete
|
||||
max_size: Maximum file size in bytes to attempt secure deletion (default: 100MB)
|
||||
class FileOperations:
|
||||
"""Manages file and directory operations"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize file operation managers"""
|
||||
self.file_deleter = SecureFileDeleter()
|
||||
self.directory_manager = DirectoryManager()
|
||||
self.permission_manager = PermissionManager()
|
||||
|
||||
async def secure_delete_file(
|
||||
self,
|
||||
file_path: str,
|
||||
max_size: Optional[int] = None
|
||||
) -> bool:
|
||||
"""Delete a file securely
|
||||
|
||||
Returns:
|
||||
bool: True if file was successfully deleted, False otherwise
|
||||
Args:
|
||||
file_path: Path to the file to delete
|
||||
max_size: Optional maximum file size for secure deletion
|
||||
|
||||
Returns:
|
||||
bool: True if file was successfully deleted
|
||||
|
||||
Raises:
|
||||
FileCleanupError: If file deletion fails
|
||||
"""
|
||||
try:
|
||||
# Ensure file is writable before deletion
|
||||
await self.permission_manager.ensure_writable(file_path)
|
||||
|
||||
# Perform secure deletion
|
||||
if max_size:
|
||||
self.file_deleter.max_size = max_size
|
||||
return await self.file_deleter.delete_file(file_path)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during secure file deletion: {e}")
|
||||
raise FileCleanupError(f"Secure deletion failed: {str(e)}")
|
||||
|
||||
async def cleanup_downloads(
|
||||
self,
|
||||
download_path: str,
|
||||
recursive: bool = True,
|
||||
delete_empty: bool = True
|
||||
) -> None:
|
||||
"""Clean up the downloads directory
|
||||
|
||||
Raises:
|
||||
FileCleanupError: If file deletion fails after all attempts
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
return True
|
||||
|
||||
try:
|
||||
# Get file size
|
||||
Args:
|
||||
download_path: Path to the downloads directory
|
||||
recursive: Whether to clean subdirectories
|
||||
delete_empty: Whether to delete empty directories
|
||||
|
||||
Raises:
|
||||
FileCleanupError: If cleanup fails
|
||||
"""
|
||||
try:
|
||||
file_size = os.path.getsize(file_path)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not get size of {file_path}: {e}")
|
||||
file_size = 0
|
||||
# Ensure we have necessary permissions
|
||||
await self.permission_manager.ensure_writable(
|
||||
download_path,
|
||||
recursive=recursive
|
||||
)
|
||||
|
||||
# For large files, skip secure deletion and just remove
|
||||
if file_size > max_size:
|
||||
logger.debug(f"File {file_path} exceeds max size for secure deletion, performing direct removal")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
return True
|
||||
except OSError as e:
|
||||
logger.error(f"Failed to remove large file {file_path}: {e}")
|
||||
return False
|
||||
# Perform cleanup
|
||||
deleted_count, errors = await self.directory_manager.cleanup_directory(
|
||||
download_path,
|
||||
recursive=recursive,
|
||||
delete_empty=delete_empty
|
||||
)
|
||||
|
||||
# Ensure file is writable
|
||||
try:
|
||||
current_mode = os.stat(file_path).st_mode
|
||||
os.chmod(file_path, current_mode | stat.S_IWRITE)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not modify permissions of {file_path}: {e}")
|
||||
raise FileCleanupError(f"Permission error: {str(e)}")
|
||||
# Log results
|
||||
if errors:
|
||||
error_msg = "\n".join(errors)
|
||||
logger.error(f"Cleanup completed with errors:\n{error_msg}")
|
||||
raise FileCleanupError(f"Cleanup completed with {len(errors)} errors")
|
||||
else:
|
||||
logger.info(f"Successfully cleaned up {deleted_count} files")
|
||||
|
||||
# Zero out file content in chunks to avoid memory issues
|
||||
if file_size > 0:
|
||||
try:
|
||||
chunk_size = min(1024 * 1024, file_size) # 1MB chunks or file size if smaller
|
||||
with open(file_path, "wb") as f:
|
||||
for offset in range(0, file_size, chunk_size):
|
||||
write_size = min(chunk_size, file_size - offset)
|
||||
f.write(b'\0' * write_size)
|
||||
# Allow other tasks to run
|
||||
await asyncio.sleep(0)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
except OSError as e:
|
||||
logger.warning(f"Error zeroing file {file_path}: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error during downloads cleanup: {e}")
|
||||
raise FileCleanupError(f"Downloads cleanup failed: {str(e)}")
|
||||
|
||||
# Delete the file
|
||||
try:
|
||||
Path(file_path).unlink(missing_ok=True)
|
||||
return True
|
||||
except OSError as e:
|
||||
logger.error(f"Failed to delete file {file_path}: {e}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during deletion of {file_path}: {e}")
|
||||
# Last resort: try force delete
|
||||
try:
|
||||
if os.path.exists(file_path):
|
||||
os.chmod(file_path, stat.S_IWRITE | stat.S_IREAD)
|
||||
Path(file_path).unlink(missing_ok=True)
|
||||
except Exception as e2:
|
||||
logger.error(f"Force delete failed for {file_path}: {e2}")
|
||||
raise FileCleanupError(f"Force delete failed: {str(e2)}")
|
||||
return not os.path.exists(file_path)
|
||||
|
||||
async def cleanup_downloads(download_path: str) -> None:
|
||||
"""Clean up the downloads directory
|
||||
|
||||
Args:
|
||||
download_path: Path to the downloads directory to clean
|
||||
async def ensure_directory(self, directory_path: str) -> None:
|
||||
"""Ensure a directory exists with proper permissions
|
||||
|
||||
Raises:
|
||||
FileCleanupError: If cleanup fails
|
||||
"""
|
||||
try:
|
||||
if not os.path.exists(download_path):
|
||||
return
|
||||
Args:
|
||||
directory_path: Path to the directory
|
||||
|
||||
Raises:
|
||||
FileCleanupError: If directory cannot be created or accessed
|
||||
"""
|
||||
try:
|
||||
# Create directory if needed
|
||||
await self.directory_manager.ensure_directory(directory_path)
|
||||
|
||||
# Set proper permissions
|
||||
await self.permission_manager.fix_permissions(directory_path)
|
||||
|
||||
# Verify it's writable
|
||||
if not await self.permission_manager.check_permissions(
|
||||
directory_path,
|
||||
require_writable=True,
|
||||
require_readable=True,
|
||||
require_executable=True
|
||||
):
|
||||
raise FileCleanupError(f"Directory {directory_path} has incorrect permissions")
|
||||
|
||||
errors = []
|
||||
# Delete all files in the directory
|
||||
for entry in os.scandir(download_path):
|
||||
try:
|
||||
path = entry.path
|
||||
if entry.is_file():
|
||||
if not await secure_delete_file(path):
|
||||
errors.append(f"Failed to delete file: {path}")
|
||||
elif entry.is_dir():
|
||||
await asyncio.to_thread(lambda: os.rmdir(path) if not os.listdir(path) else None)
|
||||
except Exception as e:
|
||||
errors.append(f"Error processing {entry.path}: {str(e)}")
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"Error ensuring directory: {e}")
|
||||
raise FileCleanupError(f"Failed to ensure directory: {str(e)}")
|
||||
|
||||
# Clean up empty subdirectories
|
||||
for root, dirs, files in os.walk(download_path, topdown=False):
|
||||
for name in dirs:
|
||||
try:
|
||||
dir_path = os.path.join(root, name)
|
||||
if not os.listdir(dir_path): # Check if directory is empty
|
||||
await asyncio.to_thread(os.rmdir, dir_path)
|
||||
except Exception as e:
|
||||
errors.append(f"Error removing directory {name}: {str(e)}")
|
||||
async def get_directory_info(
|
||||
self,
|
||||
directory_path: str
|
||||
) -> Tuple[int, List[str]]:
|
||||
"""Get directory size and any permission issues
|
||||
|
||||
Args:
|
||||
directory_path: Path to the directory
|
||||
|
||||
Returns:
|
||||
Tuple[int, List[str]]: (Total size in bytes, List of permission issues)
|
||||
"""
|
||||
try:
|
||||
# Get directory size
|
||||
total_size = await self.directory_manager.get_directory_size(directory_path)
|
||||
|
||||
# Check permissions
|
||||
permission_issues = await self.permission_manager.fix_permissions(
|
||||
directory_path,
|
||||
recursive=True
|
||||
)
|
||||
|
||||
return total_size, permission_issues
|
||||
|
||||
if errors:
|
||||
raise FileCleanupError("\n".join(errors))
|
||||
|
||||
except FileCleanupError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error during cleanup of {download_path}: {e}")
|
||||
raise FileCleanupError(f"Cleanup failed: {str(e)}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting directory info: {e}")
|
||||
return 0, [f"Error: {str(e)}"]
|
||||
|
||||
Reference in New Issue
Block a user