""" Celery tasks for data retention and cleanup. """ import logging import requests import os from celery import shared_task logger = logging.getLogger(__name__) SUPABASE_URL = os.environ.get('SUPABASE_URL', 'https://api.thrillwiki.com') SUPABASE_SERVICE_KEY = os.environ.get('SUPABASE_SERVICE_ROLE_KEY') @shared_task(bind=True, name='monitoring.run_data_retention_cleanup') def run_data_retention_cleanup(self): """ Run comprehensive data retention cleanup. Cleans up old metrics, anomaly detections, alerts, and incidents. Runs daily at 3 AM. """ logger.info("Starting data retention cleanup") if not SUPABASE_SERVICE_KEY: logger.error("SUPABASE_SERVICE_ROLE_KEY not configured") return {'success': False, 'error': 'Missing service key'} try: # Call the Supabase RPC function headers = { 'apikey': SUPABASE_SERVICE_KEY, 'Authorization': f'Bearer {SUPABASE_SERVICE_KEY}', 'Content-Type': 'application/json', } response = requests.post( f'{SUPABASE_URL}/rest/v1/rpc/run_data_retention_cleanup', headers=headers, timeout=60 ) if response.status_code == 200: result = response.json() logger.info(f"Data retention cleanup completed: {result}") return result else: logger.error(f"Data retention cleanup failed: {response.status_code} - {response.text}") return {'success': False, 'error': response.text} except Exception as e: logger.error(f"Error in data retention cleanup: {e}", exc_info=True) raise @shared_task(bind=True, name='monitoring.cleanup_old_metrics') def cleanup_old_metrics(self, retention_days: int = 30): """ Clean up old metric time series data. Runs daily to remove metrics older than retention period. """ logger.info(f"Cleaning up metrics older than {retention_days} days") if not SUPABASE_SERVICE_KEY: logger.error("SUPABASE_SERVICE_ROLE_KEY not configured") return {'success': False, 'error': 'Missing service key'} try: headers = { 'apikey': SUPABASE_SERVICE_KEY, 'Authorization': f'Bearer {SUPABASE_SERVICE_KEY}', 'Content-Type': 'application/json', } response = requests.post( f'{SUPABASE_URL}/rest/v1/rpc/cleanup_old_metrics', headers=headers, json={'retention_days': retention_days}, timeout=30 ) if response.status_code == 200: deleted_count = response.json() logger.info(f"Cleaned up {deleted_count} old metrics") return {'success': True, 'deleted_count': deleted_count} else: logger.error(f"Metrics cleanup failed: {response.status_code} - {response.text}") return {'success': False, 'error': response.text} except Exception as e: logger.error(f"Error in metrics cleanup: {e}", exc_info=True) raise @shared_task(bind=True, name='monitoring.cleanup_old_anomalies') def cleanup_old_anomalies(self, retention_days: int = 30): """ Clean up old anomaly detections. Archives resolved anomalies and deletes very old unresolved ones. """ logger.info(f"Cleaning up anomalies older than {retention_days} days") if not SUPABASE_SERVICE_KEY: logger.error("SUPABASE_SERVICE_ROLE_KEY not configured") return {'success': False, 'error': 'Missing service key'} try: headers = { 'apikey': SUPABASE_SERVICE_KEY, 'Authorization': f'Bearer {SUPABASE_SERVICE_KEY}', 'Content-Type': 'application/json', } response = requests.post( f'{SUPABASE_URL}/rest/v1/rpc/cleanup_old_anomalies', headers=headers, json={'retention_days': retention_days}, timeout=30 ) if response.status_code == 200: result = response.json() logger.info(f"Cleaned up anomalies: {result}") return {'success': True, 'result': result} else: logger.error(f"Anomalies cleanup failed: {response.status_code} - {response.text}") return {'success': False, 'error': response.text} except Exception as e: logger.error(f"Error in anomalies cleanup: {e}", exc_info=True) raise @shared_task(bind=True, name='monitoring.get_retention_stats') def get_retention_stats(self): """ Get current data retention statistics. Shows record counts and storage size for monitored tables. """ logger.info("Fetching data retention statistics") if not SUPABASE_SERVICE_KEY: logger.error("SUPABASE_SERVICE_ROLE_KEY not configured") return {'success': False, 'error': 'Missing service key'} try: headers = { 'apikey': SUPABASE_SERVICE_KEY, 'Authorization': f'Bearer {SUPABASE_SERVICE_KEY}', 'Content-Type': 'application/json', } response = requests.get( f'{SUPABASE_URL}/rest/v1/data_retention_stats', headers=headers, timeout=10 ) if response.status_code == 200: stats = response.json() logger.info(f"Retrieved retention stats for {len(stats)} tables") return {'success': True, 'stats': stats} else: logger.error(f"Failed to get retention stats: {response.status_code} - {response.text}") return {'success': False, 'error': response.text} except Exception as e: logger.error(f"Error getting retention stats: {e}", exc_info=True) raise