Add automated data retention cleanup

Implements edge function, Django tasks, and UI hooks/panels for automatic retention of old metrics, anomalies, alerts, and incidents, plus updates to query keys and monitoring dashboard to reflect data-retention workflows.

src/components/admin/DataRetentionPanel.tsx

@@ -0,0 +1,161 @@
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
+import { Button } from "@/components/ui/button";
+import { Badge } from "@/components/ui/badge";
+import { Trash2, Database, Clock, HardDrive, TrendingDown } from "lucide-react";
+import { useRetentionStats, useRunCleanup } from "@/hooks/admin/useDataRetention";
+import { formatDistanceToNow } from "date-fns";
+
+export function DataRetentionPanel() {
+  const { data: stats, isLoading } = useRetentionStats();
+  const runCleanup = useRunCleanup();
+
+  if (isLoading) {
+    return (
+      <Card>
+        <CardHeader>
+          <CardTitle>Data Retention</CardTitle>
+          <CardDescription>Loading retention statistics...</CardDescription>
+        </CardHeader>
+      </Card>
+    );
+  }
+
+  const totalRecords = stats?.reduce((sum, s) => sum + s.total_records, 0) || 0;
+  const totalSize = stats?.reduce((sum, s) => {
+    const size = s.table_size.replace(/[^0-9.]/g, '');
+    return sum + parseFloat(size);
+  }, 0) || 0;
+
+  return (
+    <Card>
+      <CardHeader>
+        <div className="flex items-center justify-between">
+          <div>
+            <CardTitle className="flex items-center gap-2">
+              <Database className="h-5 w-5" />
+              Data Retention Management
+            </CardTitle>
+            <CardDescription>
+              Automatic cleanup of old metrics and monitoring data
+            </CardDescription>
+          </div>
+          <Button
+            onClick={() => runCleanup.mutate()}
+            disabled={runCleanup.isPending}
+            variant="destructive"
+            size="sm"
+          >
+            <Trash2 className="h-4 w-4 mr-2" />
+            Run Cleanup Now
+          </Button>
+        </div>
+      </CardHeader>
+      <CardContent className="space-y-6">
+        {/* Summary Stats */}
+        <div className="grid gap-4 md:grid-cols-3">
+          <div className="space-y-2">
+            <div className="flex items-center gap-2 text-sm text-muted-foreground">
+              <Database className="h-4 w-4" />
+              Total Records
+            </div>
+            <div className="text-2xl font-bold">{totalRecords.toLocaleString()}</div>
+          </div>
+          <div className="space-y-2">
+            <div className="flex items-center gap-2 text-sm text-muted-foreground">
+              <HardDrive className="h-4 w-4" />
+              Total Size
+            </div>
+            <div className="text-2xl font-bold">{totalSize.toFixed(1)} MB</div>
+          </div>
+          <div className="space-y-2">
+            <div className="flex items-center gap-2 text-sm text-muted-foreground">
+              <TrendingDown className="h-4 w-4" />
+              Tables Monitored
+            </div>
+            <div className="text-2xl font-bold">{stats?.length || 0}</div>
+          </div>
+        </div>
+
+        {/* Retention Policies */}
+        <div>
+          <h3 className="font-semibold mb-3">Retention Policies</h3>
+          <div className="space-y-2 text-sm">
+            <div className="flex justify-between items-center p-2 bg-muted/50 rounded">
+              <span>Metrics (metric_time_series)</span>
+              <Badge variant="outline">30 days</Badge>
+            </div>
+            <div className="flex justify-between items-center p-2 bg-muted/50 rounded">
+              <span>Anomaly Detections</span>
+              <Badge variant="outline">30 days</Badge>
+            </div>
+            <div className="flex justify-between items-center p-2 bg-muted/50 rounded">
+              <span>Resolved Alerts</span>
+              <Badge variant="outline">90 days</Badge>
+            </div>
+            <div className="flex justify-between items-center p-2 bg-muted/50 rounded">
+              <span>Resolved Incidents</span>
+              <Badge variant="outline">90 days</Badge>
+            </div>
+          </div>
+        </div>
+
+        {/* Table Statistics */}
+        <div>
+          <h3 className="font-semibold mb-3">Storage Details</h3>
+          <div className="space-y-3">
+            {stats?.map((stat) => (
+              <div
+                key={stat.table_name}
+                className="border rounded-lg p-3 space-y-2"
+              >
+                <div className="flex items-center justify-between">
+                  <span className="font-medium">{stat.table_name}</span>
+                  <Badge variant="secondary">{stat.table_size}</Badge>
+                </div>
+                <div className="grid grid-cols-3 gap-2 text-xs text-muted-foreground">
+                  <div>
+                    <div>Total</div>
+                    <div className="font-medium text-foreground">
+                      {stat.total_records.toLocaleString()}
+                    </div>
+                  </div>
+                  <div>
+                    <div>Last 7 days</div>
+                    <div className="font-medium text-foreground">
+                      {stat.last_7_days.toLocaleString()}
+                    </div>
+                  </div>
+                  <div>
+                    <div>Last 30 days</div>
+                    <div className="font-medium text-foreground">
+                      {stat.last_30_days.toLocaleString()}
+                    </div>
+                  </div>
+                </div>
+                {stat.oldest_record && (
+                  <div className="flex items-center gap-1 text-xs text-muted-foreground">
+                    <Clock className="h-3 w-3" />
+                    Oldest:{" "}
+                    {formatDistanceToNow(new Date(stat.oldest_record), {
+                      addSuffix: true,
+                    })}
+                  </div>
+                )}
+              </div>
+            ))}
+          </div>
+        </div>
+
+        {/* Cleanup Schedule */}
+        <div className="bg-muted/50 rounded-lg p-4 space-y-2">
+          <h3 className="font-semibold text-sm">Automated Cleanup Schedule</h3>
+          <div className="space-y-1 text-sm text-muted-foreground">
+            <div>• Full cleanup runs daily at 3:00 AM</div>
+            <div>• Metrics cleanup at 3:30 AM</div>
+            <div>• Anomaly cleanup at 4:00 AM</div>
+          </div>
+        </div>
+      </CardContent>
+    </Card>
+  );
+}

src/hooks/admin/useDataRetention.ts

@@ -0,0 +1,134 @@
+import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
+import { supabase } from "@/integrations/supabase/client";
+import { toast } from "sonner";
+
+interface RetentionStats {
+  table_name: string;
+  total_records: number;
+  last_7_days: number;
+  last_30_days: number;
+  oldest_record: string;
+  newest_record: string;
+  table_size: string;
+}
+
+interface CleanupResult {
+  success: boolean;
+  cleanup_results: {
+    metrics_deleted: number;
+    anomalies_archived: number;
+    anomalies_deleted: number;
+    alerts_deleted: number;
+    incidents_deleted: number;
+  };
+  timestamp: string;
+}
+
+export function useRetentionStats() {
+  return useQuery({
+    queryKey: ["dataRetentionStats"],
+    queryFn: async () => {
+      const { data, error } = await supabase
+        .from("data_retention_stats")
+        .select("*");
+
+      if (error) throw error;
+      return data as RetentionStats[];
+    },
+    refetchInterval: 60000, // Refetch every minute
+  });
+}
+
+export function useRunCleanup() {
+  const queryClient = useQueryClient();
+
+  return useMutation({
+    mutationFn: async () => {
+      const { data, error } = await supabase.functions.invoke(
+        "data-retention-cleanup"
+      );
+
+      if (error) throw error;
+      return data as CleanupResult;
+    },
+    onSuccess: (data) => {
+      const results = data.cleanup_results;
+      const total =
+        results.metrics_deleted +
+        results.anomalies_archived +
+        results.anomalies_deleted +
+        results.alerts_deleted +
+        results.incidents_deleted;
+
+      toast.success(
+        `Cleanup completed: ${total} records removed`,
+        {
+          description: `Metrics: ${results.metrics_deleted}, Anomalies: ${results.anomalies_deleted}, Alerts: ${results.alerts_deleted}`,
+        }
+      );
+
+      // Invalidate relevant queries
+      queryClient.invalidateQueries({ queryKey: ["dataRetentionStats"] });
+      queryClient.invalidateQueries({ queryKey: ["anomalyDetections"] });
+      queryClient.invalidateQueries({ queryKey: ["systemAlerts"] });
+    },
+    onError: (error: Error) => {
+      toast.error("Failed to run cleanup", {
+        description: error.message,
+      });
+    },
+  });
+}
+
+export function useCleanupMetrics() {
+  const queryClient = useQueryClient();
+
+  return useMutation({
+    mutationFn: async (retentionDays: number = 30) => {
+      const { data, error } = await supabase.rpc("cleanup_old_metrics", {
+        retention_days: retentionDays,
+      });
+
+      if (error) throw error;
+      return data;
+    },
+    onSuccess: (deletedCount) => {
+      toast.success(`Cleaned up ${deletedCount} old metrics`);
+      queryClient.invalidateQueries({ queryKey: ["dataRetentionStats"] });
+    },
+    onError: (error: Error) => {
+      toast.error("Failed to cleanup metrics", {
+        description: error.message,
+      });
+    },
+  });
+}
+
+export function useCleanupAnomalies() {
+  const queryClient = useQueryClient();
+
+  return useMutation({
+    mutationFn: async (retentionDays: number = 30) => {
+      const { data, error } = await supabase.rpc("cleanup_old_anomalies", {
+        retention_days: retentionDays,
+      });
+
+      if (error) throw error;
+      return data;
+    },
+    onSuccess: (result) => {
+      // Result is returned as an array with one element
+      const cleanupResult = Array.isArray(result) ? result[0] : result;
+      toast.success(
+        `Cleaned up anomalies: ${cleanupResult.archived_count} archived, ${cleanupResult.deleted_count} deleted`
+      );
+      queryClient.invalidateQueries({ queryKey: ["dataRetentionStats"] });
+      queryClient.invalidateQueries({ queryKey: ["anomalyDetections"] });
+    },
+    onError: (error: Error) => {
+      toast.error("Failed to cleanup anomalies", {
+        description: error.message,
+      });
+    },
+  });
+}

src/lib/queryKeys.ts

@@ -96,5 +96,6 @@ export const queryKeys = {
     incidents: (status?: string) => ['monitoring', 'incidents', status] as const,
     incidentDetails: (incidentId: string) => ['monitoring', 'incident-details', incidentId] as const,
     anomalyDetections: () => ['monitoring', 'anomaly-detections'] as const,
+    dataRetentionStats: () => ['monitoring', 'data-retention-stats'] as const,
   },
 } as const;

src/pages/admin/MonitoringOverview.tsx

@@ -7,6 +7,7 @@ import { GroupedAlertsPanel } from '@/components/admin/GroupedAlertsPanel';
 import { CorrelatedAlertsPanel } from '@/components/admin/CorrelatedAlertsPanel';
 import { IncidentsPanel } from '@/components/admin/IncidentsPanel';
 import { AnomalyDetectionPanel } from '@/components/admin/AnomalyDetectionPanel';
+import { DataRetentionPanel } from '@/components/admin/DataRetentionPanel';
 import { MonitoringQuickStats } from '@/components/admin/MonitoringQuickStats';
 import { RecentActivityTimeline } from '@/components/admin/RecentActivityTimeline';
 import { MonitoringNavCards } from '@/components/admin/MonitoringNavCards';
@@ -150,6 +151,9 @@ export default function MonitoringOverview() {
           isLoading={anomalies.isLoading}
         />
 
+        {/* Data Retention Management */}
+        <DataRetentionPanel />
+
         {/* Quick Stats Grid */}
         <MonitoringQuickStats
           systemHealth={systemHealth.data ?? undefined}