Set up auto metric collection

Add Django Celery tasks and utilities to periodically collect system metrics (error rates, response times, queue sizes) and record them into metric_time_series. Include monitoring app scaffolding, metrics collector, Celery beat schedule, middleware for live metrics, and a Supabase edge function for cross-source metrics.

django/apps/monitoring/tasks.py

@@ -0,0 +1,82 @@
+"""
+Celery tasks for periodic metric collection.
+"""
+import logging
+from celery import shared_task
+from .metrics_collector import MetricsCollector
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task(bind=True, name='monitoring.collect_system_metrics')
+def collect_system_metrics(self):
+    """
+    Periodic task to collect all system metrics.
+    Runs every minute to gather current system state.
+    """
+    logger.info("Starting system metrics collection")
+    
+    try:
+        metrics = MetricsCollector.collect_all_metrics()
+        logger.info(f"Collected metrics: {metrics}")
+        return {
+            'success': True,
+            'metrics_collected': len(metrics),
+            'metrics': metrics
+        }
+    except Exception as e:
+        logger.error(f"Error in collect_system_metrics task: {e}", exc_info=True)
+        raise
+
+
+@shared_task(bind=True, name='monitoring.collect_error_metrics')
+def collect_error_metrics(self):
+    """
+    Collect error-specific metrics.
+    Runs every minute to track error rates.
+    """
+    try:
+        error_rate = MetricsCollector.get_error_rate()
+        MetricsCollector.record_metric('error_rate', error_rate, 'performance')
+        return {'success': True, 'error_rate': error_rate}
+    except Exception as e:
+        logger.error(f"Error in collect_error_metrics task: {e}", exc_info=True)
+        raise
+
+
+@shared_task(bind=True, name='monitoring.collect_performance_metrics')
+def collect_performance_metrics(self):
+    """
+    Collect performance metrics (response times, cache hit rates).
+    Runs every minute.
+    """
+    try:
+        metrics = {}
+        
+        response_time = MetricsCollector.get_api_response_time()
+        MetricsCollector.record_metric('api_response_time', response_time, 'performance')
+        metrics['api_response_time'] = response_time
+        
+        cache_hit_rate = MetricsCollector.get_cache_hit_rate()
+        MetricsCollector.record_metric('cache_hit_rate', cache_hit_rate, 'performance')
+        metrics['cache_hit_rate'] = cache_hit_rate
+        
+        return {'success': True, 'metrics': metrics}
+    except Exception as e:
+        logger.error(f"Error in collect_performance_metrics task: {e}", exc_info=True)
+        raise
+
+
+@shared_task(bind=True, name='monitoring.collect_queue_metrics')
+def collect_queue_metrics(self):
+    """
+    Collect Celery queue metrics.
+    Runs every minute to monitor queue health.
+    """
+    try:
+        queue_size = MetricsCollector.get_celery_queue_size()
+        MetricsCollector.record_metric('celery_queue_size', queue_size, 'system')
+        return {'success': True, 'queue_size': queue_size}
+    except Exception as e:
+        logger.error(f"Error in collect_queue_metrics task: {e}", exc_info=True)
+        raise