mirror of
https://github.com/pacnpal/thrilltrack-explorer.git
synced 2025-12-20 13:31:12 -05:00
12a6bfdfab
Enhance detect-anomalies with advanced ML algorithms (Isolation Forest, seasonal decomposition, predictive modeling) and schedule frequent runs via pg_cron. Updates include implementing new detectors, ensemble logic, and plumbing to run and expose results through the anomaly detection UI and data hooks.
488 lines
18 KiB
TypeScript
488 lines
18 KiB
TypeScript
import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.57.4';
|
||
|
||
const corsHeaders = {
|
||
'Access-Control-Allow-Origin': '*',
|
||
'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
|
||
};
|
||
|
||
interface MetricData {
|
||
timestamp: string;
|
||
metric_value: number;
|
||
}
|
||
|
||
interface AnomalyDetectionConfig {
|
||
metric_name: string;
|
||
metric_category: string;
|
||
enabled: boolean;
|
||
sensitivity: number;
|
||
lookback_window_minutes: number;
|
||
detection_algorithms: string[];
|
||
min_data_points: number;
|
||
alert_threshold_score: number;
|
||
auto_create_alert: boolean;
|
||
}
|
||
|
||
interface AnomalyResult {
|
||
isAnomaly: boolean;
|
||
anomalyType: string;
|
||
deviationScore: number;
|
||
confidenceScore: number;
|
||
algorithm: string;
|
||
baselineValue: number;
|
||
anomalyValue: number;
|
||
}
|
||
|
||
// Advanced ML-based anomaly detection algorithms
|
||
class AnomalyDetector {
|
||
// Isolation Forest approximation: Detects outliers based on isolation score
|
||
static isolationForest(data: number[], currentValue: number, sensitivity: number = 0.6): AnomalyResult {
|
||
if (data.length < 10) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'isolation_forest', baselineValue: currentValue, anomalyValue: currentValue };
|
||
}
|
||
|
||
// Calculate isolation score (simplified version)
|
||
// Based on how different the value is from random samples
|
||
const samples = 20;
|
||
let isolationScore = 0;
|
||
|
||
for (let i = 0; i < samples; i++) {
|
||
const randomSample = data[Math.floor(Math.random() * data.length)];
|
||
const distance = Math.abs(currentValue - randomSample);
|
||
isolationScore += distance;
|
||
}
|
||
|
||
isolationScore = isolationScore / samples;
|
||
|
||
// Normalize by standard deviation
|
||
const mean = data.reduce((sum, val) => sum + val, 0) / data.length;
|
||
const variance = data.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / data.length;
|
||
const stdDev = Math.sqrt(variance);
|
||
|
||
const normalizedScore = stdDev > 0 ? isolationScore / stdDev : 0;
|
||
const isAnomaly = normalizedScore > (1 / sensitivity);
|
||
|
||
return {
|
||
isAnomaly,
|
||
anomalyType: currentValue > mean ? 'outlier_high' : 'outlier_low',
|
||
deviationScore: normalizedScore,
|
||
confidenceScore: Math.min(normalizedScore / 5, 1),
|
||
algorithm: 'isolation_forest',
|
||
baselineValue: mean,
|
||
anomalyValue: currentValue,
|
||
};
|
||
}
|
||
|
||
// Seasonal decomposition: Detects anomalies considering seasonal patterns
|
||
static seasonalDecomposition(data: number[], currentValue: number, sensitivity: number = 2.5, period: number = 24): AnomalyResult {
|
||
if (data.length < period * 2) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'seasonal', baselineValue: currentValue, anomalyValue: currentValue };
|
||
}
|
||
|
||
// Calculate seasonal component (average of values at same position in period)
|
||
const position = data.length % period;
|
||
const seasonalValues: number[] = [];
|
||
|
||
for (let i = position; i < data.length; i += period) {
|
||
seasonalValues.push(data[i]);
|
||
}
|
||
|
||
const seasonalMean = seasonalValues.reduce((sum, val) => sum + val, 0) / seasonalValues.length;
|
||
const seasonalStdDev = Math.sqrt(
|
||
seasonalValues.reduce((sum, val) => sum + Math.pow(val - seasonalMean, 2), 0) / seasonalValues.length
|
||
);
|
||
|
||
if (seasonalStdDev === 0) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'seasonal', baselineValue: seasonalMean, anomalyValue: currentValue };
|
||
}
|
||
|
||
const deviationScore = Math.abs(currentValue - seasonalMean) / seasonalStdDev;
|
||
const isAnomaly = deviationScore > sensitivity;
|
||
|
||
return {
|
||
isAnomaly,
|
||
anomalyType: currentValue > seasonalMean ? 'seasonal_spike' : 'seasonal_drop',
|
||
deviationScore,
|
||
confidenceScore: Math.min(deviationScore / (sensitivity * 2), 1),
|
||
algorithm: 'seasonal',
|
||
baselineValue: seasonalMean,
|
||
anomalyValue: currentValue,
|
||
};
|
||
}
|
||
|
||
// LSTM-inspired prediction: Simple exponential smoothing with trend detection
|
||
static predictiveAnomaly(data: number[], currentValue: number, sensitivity: number = 2.5): AnomalyResult {
|
||
if (data.length < 5) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'predictive', baselineValue: currentValue, anomalyValue: currentValue };
|
||
}
|
||
|
||
// Triple exponential smoothing (Holt-Winters approximation)
|
||
const alpha = 0.3; // Level smoothing
|
||
const beta = 0.1; // Trend smoothing
|
||
|
||
let level = data[0];
|
||
let trend = data[1] - data[0];
|
||
|
||
// Calculate smoothed values
|
||
for (let i = 1; i < data.length; i++) {
|
||
const prevLevel = level;
|
||
level = alpha * data[i] + (1 - alpha) * (level + trend);
|
||
trend = beta * (level - prevLevel) + (1 - beta) * trend;
|
||
}
|
||
|
||
// Predict next value
|
||
const prediction = level + trend;
|
||
|
||
// Calculate prediction error
|
||
const recentData = data.slice(-10);
|
||
const predictionErrors: number[] = [];
|
||
|
||
for (let i = 1; i < recentData.length; i++) {
|
||
const simplePrediction = recentData[i - 1];
|
||
predictionErrors.push(Math.abs(recentData[i] - simplePrediction));
|
||
}
|
||
|
||
const meanError = predictionErrors.reduce((sum, err) => sum + err, 0) / predictionErrors.length;
|
||
const errorStdDev = Math.sqrt(
|
||
predictionErrors.reduce((sum, err) => sum + Math.pow(err - meanError, 2), 0) / predictionErrors.length
|
||
);
|
||
|
||
const actualError = Math.abs(currentValue - prediction);
|
||
const deviationScore = errorStdDev > 0 ? actualError / errorStdDev : 0;
|
||
const isAnomaly = deviationScore > sensitivity;
|
||
|
||
return {
|
||
isAnomaly,
|
||
anomalyType: currentValue > prediction ? 'unexpected_spike' : 'unexpected_drop',
|
||
deviationScore,
|
||
confidenceScore: Math.min(deviationScore / (sensitivity * 2), 1),
|
||
algorithm: 'predictive',
|
||
baselineValue: prediction,
|
||
anomalyValue: currentValue,
|
||
};
|
||
}
|
||
|
||
// Ensemble method: Combines multiple algorithms for better accuracy
|
||
static ensemble(data: number[], currentValue: number, sensitivity: number = 2.5): AnomalyResult {
|
||
const results: AnomalyResult[] = [
|
||
this.zScore(data, currentValue, sensitivity),
|
||
this.movingAverage(data, currentValue, sensitivity),
|
||
this.rateOfChange(data, currentValue, sensitivity),
|
||
this.isolationForest(data, currentValue, 0.6),
|
||
this.predictiveAnomaly(data, currentValue, sensitivity),
|
||
];
|
||
|
||
// Count how many algorithms detected an anomaly
|
||
const anomalyCount = results.filter(r => r.isAnomaly).length;
|
||
const anomalyRatio = anomalyCount / results.length;
|
||
|
||
// Calculate average deviation and confidence
|
||
const avgDeviation = results.reduce((sum, r) => sum + r.deviationScore, 0) / results.length;
|
||
const avgConfidence = results.reduce((sum, r) => sum + r.confidenceScore, 0) / results.length;
|
||
|
||
// Determine anomaly type based on most common classification
|
||
const typeCount = new Map<string, number>();
|
||
results.forEach(r => {
|
||
typeCount.set(r.anomalyType, (typeCount.get(r.anomalyType) || 0) + 1);
|
||
});
|
||
|
||
let mostCommonType = 'none';
|
||
let maxCount = 0;
|
||
typeCount.forEach((count, type) => {
|
||
if (count > maxCount) {
|
||
maxCount = count;
|
||
mostCommonType = type;
|
||
}
|
||
});
|
||
|
||
const mean = data.reduce((sum, val) => sum + val, 0) / data.length;
|
||
|
||
return {
|
||
isAnomaly: anomalyRatio >= 0.4, // At least 40% of algorithms agree
|
||
anomalyType: mostCommonType,
|
||
deviationScore: avgDeviation,
|
||
confidenceScore: Math.min(avgConfidence * anomalyRatio * 2, 1),
|
||
algorithm: 'ensemble',
|
||
baselineValue: mean,
|
||
anomalyValue: currentValue,
|
||
};
|
||
}
|
||
|
||
// Z-Score algorithm: Detects outliers based on standard deviation
|
||
static zScore(data: number[], currentValue: number, sensitivity: number = 3.0): AnomalyResult {
|
||
if (data.length < 2) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'z_score', baselineValue: currentValue, anomalyValue: currentValue };
|
||
}
|
||
|
||
const mean = data.reduce((sum, val) => sum + val, 0) / data.length;
|
||
const variance = data.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / data.length;
|
||
const stdDev = Math.sqrt(variance);
|
||
|
||
if (stdDev === 0) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'z_score', baselineValue: mean, anomalyValue: currentValue };
|
||
}
|
||
|
||
const zScore = Math.abs((currentValue - mean) / stdDev);
|
||
const isAnomaly = zScore > sensitivity;
|
||
|
||
return {
|
||
isAnomaly,
|
||
anomalyType: currentValue > mean ? 'spike' : 'drop',
|
||
deviationScore: zScore,
|
||
confidenceScore: Math.min(zScore / (sensitivity * 2), 1),
|
||
algorithm: 'z_score',
|
||
baselineValue: mean,
|
||
anomalyValue: currentValue,
|
||
};
|
||
}
|
||
|
||
// Moving Average algorithm: Detects deviation from trend
|
||
static movingAverage(data: number[], currentValue: number, sensitivity: number = 2.5, window: number = 10): AnomalyResult {
|
||
if (data.length < window) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'moving_average', baselineValue: currentValue, anomalyValue: currentValue };
|
||
}
|
||
|
||
const recentData = data.slice(-window);
|
||
const ma = recentData.reduce((sum, val) => sum + val, 0) / recentData.length;
|
||
|
||
const mad = recentData.reduce((sum, val) => sum + Math.abs(val - ma), 0) / recentData.length;
|
||
|
||
if (mad === 0) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'moving_average', baselineValue: ma, anomalyValue: currentValue };
|
||
}
|
||
|
||
const deviation = Math.abs(currentValue - ma) / mad;
|
||
const isAnomaly = deviation > sensitivity;
|
||
|
||
return {
|
||
isAnomaly,
|
||
anomalyType: currentValue > ma ? 'spike' : 'drop',
|
||
deviationScore: deviation,
|
||
confidenceScore: Math.min(deviation / (sensitivity * 2), 1),
|
||
algorithm: 'moving_average',
|
||
baselineValue: ma,
|
||
anomalyValue: currentValue,
|
||
};
|
||
}
|
||
|
||
// Rate of Change algorithm: Detects sudden changes
|
||
static rateOfChange(data: number[], currentValue: number, sensitivity: number = 3.0): AnomalyResult {
|
||
if (data.length < 2) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'rate_of_change', baselineValue: currentValue, anomalyValue: currentValue };
|
||
}
|
||
|
||
const previousValue = data[data.length - 1];
|
||
|
||
if (previousValue === 0) {
|
||
return { isAnomaly: false, anomalyType: 'none', deviationScore: 0, confidenceScore: 0, algorithm: 'rate_of_change', baselineValue: previousValue, anomalyValue: currentValue };
|
||
}
|
||
|
||
const percentChange = Math.abs((currentValue - previousValue) / previousValue) * 100;
|
||
const isAnomaly = percentChange > (sensitivity * 10); // sensitivity * 10 = % threshold
|
||
|
||
return {
|
||
isAnomaly,
|
||
anomalyType: currentValue > previousValue ? 'trend_change' : 'drop',
|
||
deviationScore: percentChange / 10,
|
||
confidenceScore: Math.min(percentChange / (sensitivity * 20), 1),
|
||
algorithm: 'rate_of_change',
|
||
baselineValue: previousValue,
|
||
anomalyValue: currentValue,
|
||
};
|
||
}
|
||
}
|
||
|
||
Deno.serve(async (req) => {
|
||
if (req.method === 'OPTIONS') {
|
||
return new Response(null, { headers: corsHeaders });
|
||
}
|
||
|
||
try {
|
||
const supabaseUrl = Deno.env.get('SUPABASE_URL')!;
|
||
const supabaseKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')!;
|
||
const supabase = createClient(supabaseUrl, supabaseKey);
|
||
|
||
console.log('Starting anomaly detection run...');
|
||
|
||
// Get all enabled anomaly detection configurations
|
||
const { data: configs, error: configError } = await supabase
|
||
.from('anomaly_detection_config')
|
||
.select('*')
|
||
.eq('enabled', true);
|
||
|
||
if (configError) {
|
||
console.error('Error fetching configs:', configError);
|
||
throw configError;
|
||
}
|
||
|
||
console.log(`Processing ${configs?.length || 0} metric configurations`);
|
||
|
||
const anomaliesDetected: any[] = [];
|
||
|
||
for (const config of (configs as AnomalyDetectionConfig[])) {
|
||
try {
|
||
// Fetch historical data for this metric
|
||
const windowStart = new Date(Date.now() - config.lookback_window_minutes * 60 * 1000);
|
||
|
||
const { data: metricData, error: metricError } = await supabase
|
||
.from('metric_time_series')
|
||
.select('timestamp, metric_value')
|
||
.eq('metric_name', config.metric_name)
|
||
.gte('timestamp', windowStart.toISOString())
|
||
.order('timestamp', { ascending: true });
|
||
|
||
if (metricError) {
|
||
console.error(`Error fetching metric data for ${config.metric_name}:`, metricError);
|
||
continue;
|
||
}
|
||
|
||
const data = metricData as MetricData[];
|
||
|
||
if (!data || data.length < config.min_data_points) {
|
||
console.log(`Insufficient data for ${config.metric_name}: ${data?.length || 0} points`);
|
||
continue;
|
||
}
|
||
|
||
// Get current value (most recent)
|
||
const currentValue = data[data.length - 1].metric_value;
|
||
const historicalValues = data.slice(0, -1).map(d => d.metric_value);
|
||
|
||
// Run detection algorithms
|
||
const results: AnomalyResult[] = [];
|
||
|
||
for (const algorithm of config.detection_algorithms) {
|
||
let result: AnomalyResult;
|
||
|
||
switch (algorithm) {
|
||
case 'z_score':
|
||
result = AnomalyDetector.zScore(historicalValues, currentValue, config.sensitivity);
|
||
break;
|
||
case 'moving_average':
|
||
result = AnomalyDetector.movingAverage(historicalValues, currentValue, config.sensitivity);
|
||
break;
|
||
case 'rate_of_change':
|
||
result = AnomalyDetector.rateOfChange(historicalValues, currentValue, config.sensitivity);
|
||
break;
|
||
case 'isolation_forest':
|
||
result = AnomalyDetector.isolationForest(historicalValues, currentValue, 0.6);
|
||
break;
|
||
case 'seasonal':
|
||
result = AnomalyDetector.seasonalDecomposition(historicalValues, currentValue, config.sensitivity, 24);
|
||
break;
|
||
case 'predictive':
|
||
result = AnomalyDetector.predictiveAnomaly(historicalValues, currentValue, config.sensitivity);
|
||
break;
|
||
case 'ensemble':
|
||
result = AnomalyDetector.ensemble(historicalValues, currentValue, config.sensitivity);
|
||
break;
|
||
default:
|
||
continue;
|
||
}
|
||
|
||
if (result.isAnomaly && result.deviationScore >= config.alert_threshold_score) {
|
||
results.push(result);
|
||
}
|
||
}
|
||
|
||
// If any algorithm detected an anomaly
|
||
if (results.length > 0) {
|
||
// Use the result with highest confidence
|
||
const bestResult = results.reduce((best, current) =>
|
||
current.confidenceScore > best.confidenceScore ? current : best
|
||
);
|
||
|
||
// Determine severity based on deviation score
|
||
const severity =
|
||
bestResult.deviationScore >= 5 ? 'critical' :
|
||
bestResult.deviationScore >= 4 ? 'high' :
|
||
bestResult.deviationScore >= 3 ? 'medium' : 'low';
|
||
|
||
// Insert anomaly detection record
|
||
const { data: anomaly, error: anomalyError } = await supabase
|
||
.from('anomaly_detections')
|
||
.insert({
|
||
metric_name: config.metric_name,
|
||
metric_category: config.metric_category,
|
||
anomaly_type: bestResult.anomalyType,
|
||
severity,
|
||
baseline_value: bestResult.baselineValue,
|
||
anomaly_value: bestResult.anomalyValue,
|
||
deviation_score: bestResult.deviationScore,
|
||
confidence_score: bestResult.confidenceScore,
|
||
detection_algorithm: bestResult.algorithm,
|
||
time_window_start: windowStart.toISOString(),
|
||
time_window_end: new Date().toISOString(),
|
||
metadata: {
|
||
algorithms_run: config.detection_algorithms,
|
||
total_data_points: data.length,
|
||
sensitivity: config.sensitivity,
|
||
},
|
||
})
|
||
.select()
|
||
.single();
|
||
|
||
if (anomalyError) {
|
||
console.error(`Error inserting anomaly for ${config.metric_name}:`, anomalyError);
|
||
continue;
|
||
}
|
||
|
||
anomaliesDetected.push(anomaly);
|
||
|
||
// Auto-create alert if configured
|
||
if (config.auto_create_alert && severity in ['critical', 'high']) {
|
||
const { data: alert, error: alertError } = await supabase
|
||
.from('system_alerts')
|
||
.insert({
|
||
alert_type: 'anomaly_detected',
|
||
severity,
|
||
message: `Anomaly detected in ${config.metric_name}: ${bestResult.anomalyType} (${bestResult.deviationScore.toFixed(2)}σ deviation)`,
|
||
metadata: {
|
||
anomaly_id: anomaly.id,
|
||
metric_name: config.metric_name,
|
||
baseline_value: bestResult.baselineValue,
|
||
anomaly_value: bestResult.anomalyValue,
|
||
algorithm: bestResult.algorithm,
|
||
},
|
||
})
|
||
.select()
|
||
.single();
|
||
|
||
if (!alertError && alert) {
|
||
// Update anomaly with alert_id
|
||
await supabase
|
||
.from('anomaly_detections')
|
||
.update({ alert_created: true, alert_id: alert.id })
|
||
.eq('id', anomaly.id);
|
||
|
||
console.log(`Created alert for anomaly in ${config.metric_name}`);
|
||
}
|
||
}
|
||
|
||
console.log(`Anomaly detected: ${config.metric_name} - ${bestResult.anomalyType} (${bestResult.deviationScore.toFixed(2)}σ)`);
|
||
}
|
||
} catch (error) {
|
||
console.error(`Error processing metric ${config.metric_name}:`, error);
|
||
}
|
||
}
|
||
|
||
console.log(`Anomaly detection complete. Detected ${anomaliesDetected.length} anomalies`);
|
||
|
||
return new Response(
|
||
JSON.stringify({
|
||
success: true,
|
||
anomalies_detected: anomaliesDetected.length,
|
||
anomalies: anomaliesDetected,
|
||
}),
|
||
{ headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
|
||
);
|
||
} catch (error) {
|
||
console.error('Error in detect-anomalies function:', error);
|
||
return new Response(
|
||
JSON.stringify({ error: error.message }),
|
||
{
|
||
status: 500,
|
||
headers: { ...corsHeaders, 'Content-Type': 'application/json' },
|
||
}
|
||
);
|
||
}
|
||
});
|