Core Systems:

Component-based architecture with lifecycle management
Enhanced error handling and recovery mechanisms
Comprehensive state management and tracking
Event-driven architecture with monitoring
Queue Management:

Multiple processing strategies for different scenarios
Advanced state management with recovery
Comprehensive metrics and health monitoring
Sophisticated cleanup system with multiple strategies
Processing Pipeline:

Enhanced message handling with validation
Improved URL extraction and processing
Better queue management and monitoring
Advanced cleanup mechanisms
Overall Benefits:

Better code organization and maintainability
Improved error handling and recovery
Enhanced monitoring and reporting
More robust and reliable system
This commit is contained in:
pacnpal
2024-11-16 05:01:29 +00:00
parent 537a325807
commit a4ca6e8ea6
47 changed files with 11085 additions and 2110 deletions

View File

@@ -0,0 +1,190 @@
"""Module for managing database connections"""
import logging
import sqlite3
from pathlib import Path
from contextlib import contextmanager
from typing import Generator, Optional
import threading
from queue import Queue, Empty
logger = logging.getLogger("DBConnectionManager")
class ConnectionManager:
"""Manages SQLite database connections and connection pooling"""
def __init__(self, db_path: Path, pool_size: int = 5):
"""Initialize the connection manager
Args:
db_path: Path to the SQLite database file
pool_size: Maximum number of connections in the pool
"""
self.db_path = db_path
self.pool_size = pool_size
self._connection_pool: Queue[sqlite3.Connection] = Queue(maxsize=pool_size)
self._local = threading.local()
self._lock = threading.Lock()
# Initialize connection pool
self._initialize_pool()
def _initialize_pool(self) -> None:
"""Initialize the connection pool"""
try:
for _ in range(self.pool_size):
conn = self._create_connection()
if conn:
self._connection_pool.put(conn)
except Exception as e:
logger.error(f"Error initializing connection pool: {e}")
raise
def _create_connection(self) -> Optional[sqlite3.Connection]:
"""Create a new database connection with proper settings"""
try:
conn = sqlite3.connect(
self.db_path,
detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES,
timeout=30.0 # 30 second timeout
)
# Enable foreign keys
conn.execute("PRAGMA foreign_keys = ON")
# Set journal mode to WAL for better concurrency
conn.execute("PRAGMA journal_mode = WAL")
# Set synchronous mode to NORMAL for better performance
conn.execute("PRAGMA synchronous = NORMAL")
# Enable extended result codes for better error handling
conn.execute("PRAGMA extended_result_codes = ON")
return conn
except sqlite3.Error as e:
logger.error(f"Error creating database connection: {e}")
return None
@contextmanager
def get_connection(self) -> Generator[sqlite3.Connection, None, None]:
"""Get a database connection from the pool
Yields:
sqlite3.Connection: A database connection
Raises:
sqlite3.Error: If unable to get a connection
"""
conn = None
try:
# Check if we have a transaction-bound connection
conn = getattr(self._local, 'transaction_connection', None)
if conn is not None:
yield conn
return
# Get connection from pool or create new one
try:
conn = self._connection_pool.get(timeout=5.0)
except Empty:
logger.warning("Connection pool exhausted, creating new connection")
conn = self._create_connection()
if not conn:
raise sqlite3.Error("Failed to create database connection")
yield conn
except Exception as e:
logger.error(f"Error getting database connection: {e}")
if conn:
try:
conn.rollback()
except Exception:
pass
raise
finally:
if conn and not hasattr(self._local, 'transaction_connection'):
try:
conn.rollback() # Reset connection state
self._connection_pool.put(conn)
except Exception as e:
logger.error(f"Error returning connection to pool: {e}")
try:
conn.close()
except Exception:
pass
@contextmanager
def transaction(self) -> Generator[sqlite3.Connection, None, None]:
"""Start a database transaction
Yields:
sqlite3.Connection: A database connection for the transaction
Raises:
sqlite3.Error: If unable to start transaction
"""
if hasattr(self._local, 'transaction_connection'):
raise sqlite3.Error("Nested transactions are not supported")
conn = None
try:
# Get connection from pool
try:
conn = self._connection_pool.get(timeout=5.0)
except Empty:
logger.warning("Connection pool exhausted, creating new connection")
conn = self._create_connection()
if not conn:
raise sqlite3.Error("Failed to create database connection")
# Bind connection to current thread
self._local.transaction_connection = conn
# Start transaction
conn.execute("BEGIN")
yield conn
# Commit transaction
conn.commit()
except Exception as e:
logger.error(f"Error in database transaction: {e}")
if conn:
try:
conn.rollback()
except Exception:
pass
raise
finally:
if conn:
try:
# Remove thread-local binding
delattr(self._local, 'transaction_connection')
# Return connection to pool
self._connection_pool.put(conn)
except Exception as e:
logger.error(f"Error cleaning up transaction: {e}")
try:
conn.close()
except Exception:
pass
def close_all(self) -> None:
"""Close all connections in the pool"""
with self._lock:
while not self._connection_pool.empty():
try:
conn = self._connection_pool.get_nowait()
try:
conn.close()
except Exception as e:
logger.error(f"Error closing connection: {e}")
except Empty:
break

View File

@@ -0,0 +1,197 @@
"""Module for managing database queries"""
import logging
import sqlite3
from typing import Optional, Tuple, List, Dict, Any
from datetime import datetime
logger = logging.getLogger("DBQueryManager")
class QueryManager:
"""Manages database queries and operations"""
def __init__(self, connection_manager):
self.connection_manager = connection_manager
async def add_archived_video(
self,
original_url: str,
discord_url: str,
message_id: int,
channel_id: int,
guild_id: int,
metadata: Optional[Dict[str, Any]] = None
) -> bool:
"""Add a newly archived video to the database"""
try:
with self.connection_manager.get_connection() as conn:
cursor = conn.cursor()
# Prepare query and parameters
query = """
INSERT OR REPLACE INTO archived_videos
(original_url, discord_url, message_id, channel_id, guild_id,
file_size, duration, format, resolution, bitrate)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""
# Extract metadata values with defaults
metadata = metadata or {}
params = (
original_url,
discord_url,
message_id,
channel_id,
guild_id,
metadata.get('file_size'),
metadata.get('duration'),
metadata.get('format'),
metadata.get('resolution'),
metadata.get('bitrate')
)
cursor.execute(query, params)
conn.commit()
return True
except sqlite3.Error as e:
logger.error(f"Error adding archived video: {e}")
return False
async def get_archived_video(
self,
url: str
) -> Optional[Dict[str, Any]]:
"""Get archived video information by original URL"""
try:
with self.connection_manager.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
SELECT discord_url, message_id, channel_id, guild_id,
file_size, duration, format, resolution, bitrate,
archived_at
FROM archived_videos
WHERE original_url = ?
""", (url,))
result = cursor.fetchone()
if not result:
return None
return {
'discord_url': result[0],
'message_id': result[1],
'channel_id': result[2],
'guild_id': result[3],
'file_size': result[4],
'duration': result[5],
'format': result[6],
'resolution': result[7],
'bitrate': result[8],
'archived_at': result[9]
}
except sqlite3.Error as e:
logger.error(f"Error retrieving archived video: {e}")
return None
async def is_url_archived(self, url: str) -> bool:
"""Check if a URL has already been archived"""
try:
with self.connection_manager.get_connection() as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT 1 FROM archived_videos WHERE original_url = ?",
(url,)
)
return cursor.fetchone() is not None
except sqlite3.Error as e:
logger.error(f"Error checking archived status: {e}")
return False
async def get_guild_stats(self, guild_id: int) -> Dict[str, Any]:
"""Get archiving statistics for a guild"""
try:
with self.connection_manager.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
SELECT
COUNT(*) as total_videos,
SUM(file_size) as total_size,
AVG(duration) as avg_duration,
MAX(archived_at) as last_archived
FROM archived_videos
WHERE guild_id = ?
""", (guild_id,))
result = cursor.fetchone()
return {
'total_videos': result[0],
'total_size': result[1] or 0,
'avg_duration': result[2] or 0,
'last_archived': result[3]
}
except sqlite3.Error as e:
logger.error(f"Error getting guild stats: {e}")
return {
'total_videos': 0,
'total_size': 0,
'avg_duration': 0,
'last_archived': None
}
async def get_channel_videos(
self,
channel_id: int,
limit: int = 100,
offset: int = 0
) -> List[Dict[str, Any]]:
"""Get archived videos for a channel"""
try:
with self.connection_manager.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
SELECT original_url, discord_url, message_id,
file_size, duration, format, resolution,
archived_at
FROM archived_videos
WHERE channel_id = ?
ORDER BY archived_at DESC
LIMIT ? OFFSET ?
""", (channel_id, limit, offset))
results = cursor.fetchall()
return [{
'original_url': row[0],
'discord_url': row[1],
'message_id': row[2],
'file_size': row[3],
'duration': row[4],
'format': row[5],
'resolution': row[6],
'archived_at': row[7]
} for row in results]
except sqlite3.Error as e:
logger.error(f"Error getting channel videos: {e}")
return []
async def cleanup_old_records(self, days: int) -> int:
"""Clean up records older than specified days"""
try:
with self.connection_manager.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
DELETE FROM archived_videos
WHERE archived_at < datetime('now', ? || ' days')
""", (-days,))
deleted = cursor.rowcount
conn.commit()
return deleted
except sqlite3.Error as e:
logger.error(f"Error cleaning up old records: {e}")
return 0

View File

@@ -0,0 +1,109 @@
"""Module for managing database schema"""
import logging
import sqlite3
from pathlib import Path
from typing import List
logger = logging.getLogger("DBSchemaManager")
class SchemaManager:
"""Manages database schema creation and updates"""
SCHEMA_VERSION = 1 # Increment when schema changes
def __init__(self, db_path: Path):
self.db_path = db_path
def initialize_schema(self) -> None:
"""Initialize or update the database schema"""
try:
self._create_schema_version_table()
current_version = self._get_schema_version()
if current_version < self.SCHEMA_VERSION:
self._apply_migrations(current_version)
self._update_schema_version()
except sqlite3.Error as e:
logger.error(f"Schema initialization error: {e}")
raise
def _create_schema_version_table(self) -> None:
"""Create schema version tracking table"""
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("""
CREATE TABLE IF NOT EXISTS schema_version (
version INTEGER PRIMARY KEY
)
""")
# Insert initial version if table is empty
cursor.execute("INSERT OR IGNORE INTO schema_version VALUES (0)")
conn.commit()
def _get_schema_version(self) -> int:
"""Get current schema version"""
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("SELECT version FROM schema_version LIMIT 1")
result = cursor.fetchone()
return result[0] if result else 0
def _update_schema_version(self) -> None:
"""Update schema version to current"""
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute(
"UPDATE schema_version SET version = ?",
(self.SCHEMA_VERSION,)
)
conn.commit()
def _apply_migrations(self, current_version: int) -> None:
"""Apply necessary schema migrations"""
migrations = self._get_migrations(current_version)
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
for migration in migrations:
try:
cursor.executescript(migration)
conn.commit()
except sqlite3.Error as e:
logger.error(f"Migration failed: {e}")
raise
def _get_migrations(self, current_version: int) -> List[str]:
"""Get list of migrations to apply"""
migrations = []
# Version 0 to 1: Initial schema
if current_version < 1:
migrations.append("""
CREATE TABLE IF NOT EXISTS archived_videos (
original_url TEXT PRIMARY KEY,
discord_url TEXT NOT NULL,
message_id INTEGER NOT NULL,
channel_id INTEGER NOT NULL,
guild_id INTEGER NOT NULL,
archived_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
file_size INTEGER,
duration INTEGER,
format TEXT,
resolution TEXT,
bitrate INTEGER
);
CREATE INDEX IF NOT EXISTS idx_guild_channel
ON archived_videos(guild_id, channel_id);
CREATE INDEX IF NOT EXISTS idx_archived_at
ON archived_videos(archived_at);
""")
# Add more migrations here as schema evolves
# if current_version < 2:
# migrations.append(...)
return migrations

View File

@@ -1,8 +1,12 @@
"""Database management for archived videos"""
import sqlite3
import logging
from pathlib import Path
from typing import Optional, Tuple
from typing import Optional, Dict, Any, List
from .schema_manager import SchemaManager
from .query_manager import QueryManager
from .connection_manager import ConnectionManager
logger = logging.getLogger("VideoArchiverDB")
@@ -10,70 +14,84 @@ class VideoArchiveDB:
"""Manages the SQLite database for archived videos"""
def __init__(self, data_path: Path):
"""Initialize the database connection"""
"""Initialize the database and its components
Args:
data_path: Path to the data directory
"""
# Set up database path
self.db_path = data_path / "archived_videos.db"
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self._init_db()
# Initialize managers
self.connection_manager = ConnectionManager(self.db_path)
self.schema_manager = SchemaManager(self.db_path)
self.query_manager = QueryManager(self.connection_manager)
# Initialize database schema
self.schema_manager.initialize_schema()
logger.info("Video archive database initialized successfully")
def _init_db(self):
"""Initialize the database schema"""
try:
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("""
CREATE TABLE IF NOT EXISTS archived_videos (
original_url TEXT PRIMARY KEY,
discord_url TEXT NOT NULL,
message_id INTEGER NOT NULL,
channel_id INTEGER NOT NULL,
guild_id INTEGER NOT NULL,
archived_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
conn.commit()
except sqlite3.Error as e:
logger.error(f"Database initialization error: {e}")
raise
def add_archived_video(self, original_url: str, discord_url: str, message_id: int, channel_id: int, guild_id: int) -> bool:
async def add_archived_video(
self,
original_url: str,
discord_url: str,
message_id: int,
channel_id: int,
guild_id: int,
metadata: Optional[Dict[str, Any]] = None
) -> bool:
"""Add a newly archived video to the database"""
try:
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("""
INSERT OR REPLACE INTO archived_videos
(original_url, discord_url, message_id, channel_id, guild_id)
VALUES (?, ?, ?, ?, ?)
""", (original_url, discord_url, message_id, channel_id, guild_id))
conn.commit()
return True
except sqlite3.Error as e:
logger.error(f"Error adding archived video: {e}")
return False
return await self.query_manager.add_archived_video(
original_url,
discord_url,
message_id,
channel_id,
guild_id,
metadata
)
def get_archived_video(self, url: str) -> Optional[Tuple[str, int, int, int]]:
async def get_archived_video(self, url: str) -> Optional[Dict[str, Any]]:
"""Get archived video information by original URL"""
try:
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("""
SELECT discord_url, message_id, channel_id, guild_id
FROM archived_videos
WHERE original_url = ?
""", (url,))
result = cursor.fetchone()
return result if result else None
except sqlite3.Error as e:
logger.error(f"Error retrieving archived video: {e}")
return None
return await self.query_manager.get_archived_video(url)
def is_url_archived(self, url: str) -> bool:
async def is_url_archived(self, url: str) -> bool:
"""Check if a URL has already been archived"""
return await self.query_manager.is_url_archived(url)
async def get_guild_stats(self, guild_id: int) -> Dict[str, Any]:
"""Get archiving statistics for a guild"""
return await self.query_manager.get_guild_stats(guild_id)
async def get_channel_videos(
self,
channel_id: int,
limit: int = 100,
offset: int = 0
) -> List[Dict[str, Any]]:
"""Get archived videos for a channel"""
return await self.query_manager.get_channel_videos(
channel_id,
limit,
offset
)
async def cleanup_old_records(self, days: int) -> int:
"""Clean up records older than specified days"""
return await self.query_manager.cleanup_old_records(days)
def close(self) -> None:
"""Close all database connections"""
try:
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("SELECT 1 FROM archived_videos WHERE original_url = ?", (url,))
return cursor.fetchone() is not None
except sqlite3.Error as e:
logger.error(f"Error checking archived status: {e}")
return False
self.connection_manager.close_all()
logger.info("Database connections closed")
except Exception as e:
logger.error(f"Error closing database connections: {e}")
async def __aenter__(self):
"""Async context manager entry"""
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
"""Async context manager exit"""
self.close()