Set up auto metric collection

Add Django Celery tasks and utilities to periodically collect system metrics (error rates, response times, queue sizes) and record them into metric_time_series. Include monitoring app scaffolding, metrics collector, Celery beat schedule, middleware for live metrics, and a Supabase edge function for cross-source metrics.

supabase/functions/collect-metrics/index.ts

@@ -0,0 +1,187 @@
+import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.57.4';
+
+const corsHeaders = {
+  'Access-Control-Allow-Origin': '*',
+  'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
+};
+
+interface MetricRecord {
+  metric_name: string;
+  metric_value: number;
+  metric_category: string;
+  timestamp: string;
+}
+
+Deno.serve(async (req) => {
+  if (req.method === 'OPTIONS') {
+    return new Response(null, { headers: corsHeaders });
+  }
+
+  try {
+    const supabaseUrl = Deno.env.get('SUPABASE_URL')!;
+    const supabaseKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')!;
+    const supabase = createClient(supabaseUrl, supabaseKey);
+
+    console.log('Starting metrics collection...');
+
+    const metrics: MetricRecord[] = [];
+    const timestamp = new Date().toISOString();
+
+    // 1. Collect API error rate from recent logs
+    const { data: recentErrors, error: errorQueryError } = await supabase
+      .from('system_alerts')
+      .select('id', { count: 'exact', head: true })
+      .gte('created_at', new Date(Date.now() - 60000).toISOString())
+      .in('severity', ['high', 'critical']);
+
+    if (!errorQueryError) {
+      const errorCount = recentErrors || 0;
+      metrics.push({
+        metric_name: 'api_error_count',
+        metric_value: errorCount as number,
+        metric_category: 'performance',
+        timestamp,
+      });
+    }
+
+    // 2. Collect rate limit violations
+    const { data: rateLimitViolations, error: rateLimitError } = await supabase
+      .from('rate_limit_logs')
+      .select('id', { count: 'exact', head: true })
+      .gte('timestamp', new Date(Date.now() - 60000).toISOString())
+      .eq('action_taken', 'blocked');
+
+    if (!rateLimitError) {
+      const violationCount = rateLimitViolations || 0;
+      metrics.push({
+        metric_name: 'rate_limit_violations',
+        metric_value: violationCount as number,
+        metric_category: 'security',
+        timestamp,
+      });
+    }
+
+    // 3. Collect pending submissions count
+    const { data: pendingSubmissions, error: submissionsError } = await supabase
+      .from('submissions')
+      .select('id', { count: 'exact', head: true })
+      .eq('moderation_status', 'pending');
+
+    if (!submissionsError) {
+      const pendingCount = pendingSubmissions || 0;
+      metrics.push({
+        metric_name: 'pending_submissions',
+        metric_value: pendingCount as number,
+        metric_category: 'workflow',
+        timestamp,
+      });
+    }
+
+    // 4. Collect active incidents count
+    const { data: activeIncidents, error: incidentsError } = await supabase
+      .from('incidents')
+      .select('id', { count: 'exact', head: true })
+      .in('status', ['open', 'investigating']);
+
+    if (!incidentsError) {
+      const incidentCount = activeIncidents || 0;
+      metrics.push({
+        metric_name: 'active_incidents',
+        metric_value: incidentCount as number,
+        metric_category: 'monitoring',
+        timestamp,
+      });
+    }
+
+    // 5. Collect unresolved alerts count
+    const { data: unresolvedAlerts, error: alertsError } = await supabase
+      .from('system_alerts')
+      .select('id', { count: 'exact', head: true })
+      .eq('resolved', false);
+
+    if (!alertsError) {
+      const alertCount = unresolvedAlerts || 0;
+      metrics.push({
+        metric_name: 'unresolved_alerts',
+        metric_value: alertCount as number,
+        metric_category: 'monitoring',
+        timestamp,
+      });
+    }
+
+    // 6. Calculate submission approval rate (last hour)
+    const { data: recentSubmissions, error: recentSubmissionsError } = await supabase
+      .from('submissions')
+      .select('moderation_status', { count: 'exact' })
+      .gte('created_at', new Date(Date.now() - 3600000).toISOString());
+
+    if (!recentSubmissionsError && recentSubmissions) {
+      const total = recentSubmissions.length;
+      const approved = recentSubmissions.filter(s => s.moderation_status === 'approved').length;
+      const approvalRate = total > 0 ? (approved / total) * 100 : 100;
+      
+      metrics.push({
+        metric_name: 'submission_approval_rate',
+        metric_value: approvalRate,
+        metric_category: 'workflow',
+        timestamp,
+      });
+    }
+
+    // 7. Calculate average moderation time (last hour)
+    const { data: moderatedSubmissions, error: moderatedError } = await supabase
+      .from('submissions')
+      .select('created_at, moderated_at')
+      .not('moderated_at', 'is', null)
+      .gte('moderated_at', new Date(Date.now() - 3600000).toISOString());
+
+    if (!moderatedError && moderatedSubmissions && moderatedSubmissions.length > 0) {
+      const totalTime = moderatedSubmissions.reduce((sum, sub) => {
+        const created = new Date(sub.created_at).getTime();
+        const moderated = new Date(sub.moderated_at).getTime();
+        return sum + (moderated - created);
+      }, 0);
+      
+      const avgTimeMinutes = (totalTime / moderatedSubmissions.length) / 60000;
+      
+      metrics.push({
+        metric_name: 'avg_moderation_time',
+        metric_value: avgTimeMinutes,
+        metric_category: 'workflow',
+        timestamp,
+      });
+    }
+
+    // Insert all collected metrics
+    if (metrics.length > 0) {
+      const { error: insertError } = await supabase
+        .from('metric_time_series')
+        .insert(metrics);
+
+      if (insertError) {
+        console.error('Error inserting metrics:', insertError);
+        throw insertError;
+      }
+
+      console.log(`Successfully recorded ${metrics.length} metrics`);
+    }
+
+    return new Response(
+      JSON.stringify({
+        success: true,
+        metrics_collected: metrics.length,
+        metrics: metrics.map(m => ({ name: m.metric_name, value: m.metric_value })),
+      }),
+      { headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+    );
+  } catch (error) {
+    console.error('Error in collect-metrics function:', error);
+    return new Response(
+      JSON.stringify({ error: error.message }),
+      {
+        status: 500,
+        headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+      }
+    );
+  }
+});