mirror of
https://github.com/pacnpal/thrilltrack-explorer.git
synced 2025-12-20 12:31:26 -05:00
Implement orphaned image cleanup, temp refs cleanup, deadlock retry, and lock cleanup. These fixes address critical areas of data integrity, resource management, and system resilience within the submission pipeline.
163 lines
4.6 KiB
TypeScript
163 lines
4.6 KiB
TypeScript
/**
|
|
* Edge Function Retry Helper
|
|
* Provides exponential backoff retry logic for external API calls
|
|
*/
|
|
|
|
import { edgeLogger } from './logger.ts';
|
|
|
|
export interface EdgeRetryOptions {
|
|
maxAttempts?: number;
|
|
baseDelay?: number;
|
|
maxDelay?: number;
|
|
backoffMultiplier?: number;
|
|
jitter?: boolean;
|
|
shouldRetry?: (error: unknown) => boolean;
|
|
}
|
|
|
|
/**
|
|
* Determines if an error is transient and should be retried
|
|
*/
|
|
export function isRetryableError(error: unknown): boolean {
|
|
// Network errors
|
|
if (error instanceof TypeError && error.message.includes('fetch')) return true;
|
|
|
|
if (error instanceof Error) {
|
|
const msg = error.message.toLowerCase();
|
|
if (msg.includes('network') || msg.includes('timeout') || msg.includes('econnrefused')) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// HTTP status codes that should be retried
|
|
if (error && typeof error === 'object') {
|
|
const httpError = error as { status?: number; code?: string };
|
|
|
|
// Rate limiting
|
|
if (httpError.status === 429) return true;
|
|
|
|
// Service unavailable or gateway timeout
|
|
if (httpError.status === 503 || httpError.status === 504) return true;
|
|
|
|
// Server errors (5xx)
|
|
if (httpError.status && httpError.status >= 500 && httpError.status < 600) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Check if error is a database deadlock or serialization failure
|
|
*/
|
|
export function isDeadlockError(error: unknown): boolean {
|
|
if (!error || typeof error !== 'object') return false;
|
|
|
|
const dbError = error as { code?: string; message?: string };
|
|
|
|
// PostgreSQL deadlock error codes
|
|
if (dbError.code === '40P01') return true; // deadlock_detected
|
|
if (dbError.code === '40001') return true; // serialization_failure
|
|
|
|
// Check message for deadlock indicators
|
|
const message = dbError.message?.toLowerCase() || '';
|
|
if (message.includes('deadlock')) return true;
|
|
if (message.includes('could not serialize')) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Calculate exponential backoff delay with optional jitter
|
|
*/
|
|
function calculateBackoffDelay(
|
|
attempt: number,
|
|
baseDelay: number,
|
|
maxDelay: number,
|
|
backoffMultiplier: number,
|
|
jitter: boolean
|
|
): number {
|
|
const exponentialDelay = baseDelay * Math.pow(backoffMultiplier, attempt);
|
|
const cappedDelay = Math.min(exponentialDelay, maxDelay);
|
|
|
|
if (!jitter) return cappedDelay;
|
|
|
|
// Add random jitter (-30% to +30%)
|
|
const jitterAmount = cappedDelay * 0.3;
|
|
const randomJitter = (Math.random() * 2 - 1) * jitterAmount;
|
|
|
|
return Math.max(0, cappedDelay + randomJitter);
|
|
}
|
|
|
|
/**
|
|
* Retry wrapper for asynchronous operations with exponential backoff
|
|
*
|
|
* @param fn - Async function to retry
|
|
* @param options - Retry configuration
|
|
* @param requestId - Request ID for tracking
|
|
* @param context - Context description for logging
|
|
*/
|
|
export async function withEdgeRetry<T>(
|
|
fn: () => Promise<T>,
|
|
options: EdgeRetryOptions = {},
|
|
requestId: string,
|
|
context: string
|
|
): Promise<T> {
|
|
const maxAttempts = options.maxAttempts ?? 3;
|
|
const baseDelay = options.baseDelay ?? 1000;
|
|
const maxDelay = options.maxDelay ?? 10000;
|
|
const backoffMultiplier = options.backoffMultiplier ?? 2;
|
|
const jitter = options.jitter ?? true;
|
|
const shouldRetry = options.shouldRetry ?? isRetryableError;
|
|
|
|
let lastError: unknown;
|
|
|
|
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
try {
|
|
return await fn();
|
|
} catch (error) {
|
|
lastError = error;
|
|
|
|
// Don't retry if this is the last attempt
|
|
if (attempt === maxAttempts - 1) {
|
|
edgeLogger.error('All retry attempts exhausted', {
|
|
requestId,
|
|
context,
|
|
attempts: maxAttempts,
|
|
error: error instanceof Error ? error.message : 'Unknown error'
|
|
});
|
|
throw error;
|
|
}
|
|
|
|
// Don't retry if error is not retryable
|
|
if (!shouldRetry(error)) {
|
|
edgeLogger.info('Error not retryable, failing immediately', {
|
|
requestId,
|
|
context,
|
|
attempt: attempt + 1,
|
|
error: error instanceof Error ? error.message : 'Unknown error'
|
|
});
|
|
throw error;
|
|
}
|
|
|
|
// Calculate delay for next retry
|
|
const delay = calculateBackoffDelay(attempt, baseDelay, maxDelay, backoffMultiplier, jitter);
|
|
|
|
edgeLogger.info('Retrying after error', {
|
|
requestId,
|
|
context,
|
|
attempt: attempt + 1,
|
|
maxAttempts,
|
|
delay: Math.round(delay),
|
|
error: error instanceof Error ? error.message : 'Unknown error'
|
|
});
|
|
|
|
// Wait before retrying
|
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
}
|
|
}
|
|
|
|
// This should never be reached, but TypeScript needs it
|
|
throw lastError;
|
|
}
|