/** * Retry utility with exponential backoff * Handles transient failures gracefully with configurable retry logic */ import { logger } from './logger'; import { supabase } from './supabaseClient'; export interface RetryOptions { /** Maximum number of attempts (default: 3) */ maxAttempts?: number; /** Base delay in milliseconds (default: 1000) */ baseDelay?: number; /** Maximum delay in milliseconds (default: 10000) */ maxDelay?: number; /** Multiplier for exponential backoff (default: 2) */ backoffMultiplier?: number; /** Add jitter to prevent thundering herd (default: true) */ jitter?: boolean; /** Callback invoked before each retry attempt */ onRetry?: (attempt: number, error: unknown, delay: number) => void; /** Custom function to determine if error is retryable (default: isRetryableError) */ shouldRetry?: (error: unknown) => boolean; } /** * Extract Retry-After value from error headers * @param error - The error object * @returns Delay in milliseconds, or null if not found */ export function extractRetryAfter(error: unknown): number | null { if (!error || typeof error !== 'object') return null; // Check for Retry-After in error object const errorWithHeaders = error as { headers?: Headers | Record; retryAfter?: number | string }; // Direct retryAfter property if (errorWithHeaders.retryAfter) { const retryAfter = errorWithHeaders.retryAfter; if (typeof retryAfter === 'number') { return retryAfter * 1000; // Convert seconds to milliseconds } if (typeof retryAfter === 'string') { // Try parsing as number first (delay-seconds) const seconds = parseInt(retryAfter, 10); if (!isNaN(seconds)) { return seconds * 1000; } // Try parsing as HTTP-date const date = new Date(retryAfter); if (!isNaN(date.getTime())) { const delay = date.getTime() - Date.now(); return Math.max(0, delay); } } } // Check headers object if (errorWithHeaders.headers) { let retryAfterValue: string | null = null; if (errorWithHeaders.headers instanceof Headers) { retryAfterValue = errorWithHeaders.headers.get('retry-after'); } else if (typeof errorWithHeaders.headers === 'object') { // Check both lowercase and capitalized versions retryAfterValue = errorWithHeaders.headers['retry-after'] || errorWithHeaders.headers['Retry-After'] || null; } if (retryAfterValue) { // Try parsing as number first (delay-seconds) const seconds = parseInt(retryAfterValue, 10); if (!isNaN(seconds)) { return seconds * 1000; } // Try parsing as HTTP-date const date = new Date(retryAfterValue); if (!isNaN(date.getTime())) { const delay = date.getTime() - Date.now(); return Math.max(0, delay); } } } return null; } /** * Check if error is a rate limit (429) error * @param error - The error to check * @returns true if error is a rate limit error */ export function isRateLimitError(error: unknown): boolean { if (!error || typeof error !== 'object') return false; const errorWithStatus = error as { status?: number; code?: string }; // HTTP 429 status if (errorWithStatus.status === 429) return true; // Check error message for rate limit indicators if (error instanceof Error) { const message = error.message.toLowerCase(); if (message.includes('rate limit') || message.includes('too many requests') || message.includes('quota exceeded')) { return true; } } return false; } /** * Determines if an error is transient and retryable * @param error - The error to check * @returns true if error is retryable, false otherwise */ export function isRetryableError(error: unknown): boolean { // Network/timeout errors from fetch if (error instanceof TypeError && error.message.includes('fetch')) { return true; } // Network/timeout errors if (error instanceof Error) { const message = error.message.toLowerCase(); if (message.includes('network') || message.includes('timeout') || message.includes('connection') || message.includes('econnrefused') || message.includes('enotfound')) { return true; } } // Supabase/PostgreSQL errors if (error && typeof error === 'object') { const supabaseError = error as { code?: string; status?: number }; // Connection/timeout errors if (supabaseError.code === 'PGRST301') return true; // Connection timeout if (supabaseError.code === 'PGRST204') return true; // Temporary failure if (supabaseError.code === 'PGRST000') return true; // Connection error // HTTP status codes indicating transient failures if (supabaseError.status === 429) return true; // Rate limit - ALWAYS retry if (supabaseError.status === 503) return true; // Service unavailable if (supabaseError.status === 504) return true; // Gateway timeout if (supabaseError.status && supabaseError.status >= 500 && supabaseError.status < 600) { return true; // Server errors (5xx) } // Database-level transient errors if (supabaseError.code === '40001') return true; // Serialization failure if (supabaseError.code === '40P01') return true; // Deadlock detected if (supabaseError.code === '57014') return true; // Query cancelled if (supabaseError.code === '08000') return true; // Connection exception if (supabaseError.code === '08003') return true; // Connection does not exist if (supabaseError.code === '08006') return true; // Connection failure if (supabaseError.code === '08001') return true; // Unable to connect if (supabaseError.code === '08004') return true; // Server rejected connection } return false; } /** * Calculates delay for next retry attempt using exponential backoff or Retry-After header * @param attempt - Current attempt number (0-indexed) * @param options - Retry configuration * @param error - The error that triggered the retry (to check for Retry-After) * @returns Delay in milliseconds */ function calculateBackoffDelay( attempt: number, options: Required, error?: unknown ): number { // Check for rate limit with Retry-After header if (error && isRateLimitError(error)) { const retryAfter = extractRetryAfter(error); if (retryAfter !== null) { // Respect the Retry-After header, but cap it at maxDelay const cappedRetryAfter = Math.min(retryAfter, options.maxDelay); logger.info('[Retry] Rate limit detected - respecting Retry-After header', { retryAfterMs: retryAfter, cappedMs: cappedRetryAfter, attempt }); return cappedRetryAfter; } // No Retry-After header but is rate limit - use aggressive backoff const rateLimitDelay = options.baseDelay * Math.pow(options.backoffMultiplier, attempt + 1); const cappedDelay = Math.min(rateLimitDelay, options.maxDelay); logger.info('[Retry] Rate limit detected - using aggressive backoff', { delayMs: cappedDelay, attempt }); return cappedDelay; } // Standard exponential backoff const exponentialDelay = options.baseDelay * Math.pow(options.backoffMultiplier, attempt); const cappedDelay = Math.min(exponentialDelay, options.maxDelay); // Add jitter (randomness) to prevent thundering herd if (options.jitter) { const jitterAmount = cappedDelay * 0.3; // ±30% jitter const jitterOffset = (Math.random() * 2 - 1) * jitterAmount; return Math.max(0, cappedDelay + jitterOffset); } return cappedDelay; } /** * Load retry configuration from admin settings */ export async function loadRetryConfig(): Promise> { try { const { data: settings } = await supabase .from('admin_settings') .select('setting_key, setting_value') .in('setting_key', [ 'retry.max_attempts', 'retry.base_delay', 'retry.max_delay', 'retry.backoff_multiplier' ]); if (!settings || settings.length === 0) { return getDefaultRetryConfig(); } const config: any = {}; settings.forEach(s => { const key = s.setting_key.replace('retry.', ''); const camelKey = key.replace(/_([a-z])/g, (g) => g[1].toUpperCase()); if (key === 'backoff_multiplier') { config[camelKey] = parseFloat(String(s.setting_value)); } else { config[camelKey] = parseInt(String(s.setting_value)); } }); return { maxAttempts: config.maxAttempts ?? 3, baseDelay: config.baseDelay ?? 1000, maxDelay: config.maxDelay ?? 10000, backoffMultiplier: config.backoffMultiplier ?? 2, jitter: true, onRetry: () => {}, shouldRetry: isRetryableError }; } catch (error) { logger.error('Failed to load retry config', { error }); return getDefaultRetryConfig(); } } function getDefaultRetryConfig(): Required { return { maxAttempts: 3, baseDelay: 1000, maxDelay: 10000, backoffMultiplier: 2, jitter: true, onRetry: () => {}, shouldRetry: isRetryableError }; } // Cache admin config for 5 minutes let cachedRetryConfig: Required | null = null; let configCacheTime: number = 0; const CONFIG_CACHE_TTL = 5 * 60 * 1000; // 5 minutes async function getCachedRetryConfig(): Promise> { const now = Date.now(); if (cachedRetryConfig && (now - configCacheTime < CONFIG_CACHE_TTL)) { return cachedRetryConfig; } cachedRetryConfig = await loadRetryConfig(); configCacheTime = now; return cachedRetryConfig; } /** * Executes a function with retry logic and exponential backoff * * @param fn - Async function to execute * @param options - Retry configuration options * @returns Result of the function execution * @throws Last error if all retry attempts fail * * @example * ```typescript * const result = await withRetry( * async () => await supabase.rpc('my_function', { data }), * { * maxAttempts: 3, * onRetry: (attempt, error, delay) => { * toast.info(`Retrying... (${attempt}/3)`); * } * } * ); * ``` */ export async function withRetry( fn: () => Promise, options?: RetryOptions ): Promise { // Load config from admin settings const adminConfig = await getCachedRetryConfig(); // Merge: options override admin settings const config: Required = { maxAttempts: options?.maxAttempts ?? adminConfig.maxAttempts, baseDelay: options?.baseDelay ?? adminConfig.baseDelay, maxDelay: options?.maxDelay ?? adminConfig.maxDelay, backoffMultiplier: options?.backoffMultiplier ?? adminConfig.backoffMultiplier, jitter: options?.jitter ?? adminConfig.jitter, onRetry: options?.onRetry ?? adminConfig.onRetry, shouldRetry: options?.shouldRetry ?? adminConfig.shouldRetry, }; let lastError: unknown; for (let attempt = 0; attempt < config.maxAttempts; attempt++) { try { // Execute the function directly const result = await fn(); // Log successful retry if not first attempt if (attempt > 0) { logger.info('Retry succeeded', { attempt: attempt + 1, totalAttempts: config.maxAttempts }); } return result; } catch (error) { lastError = error; // Check if we should retry const isLastAttempt = attempt === config.maxAttempts - 1; const shouldRetry = config.shouldRetry(error); if (isLastAttempt || !shouldRetry) { // Log final failure logger.error('Retry exhausted or non-retryable error', { attempt: attempt + 1, maxAttempts: config.maxAttempts, isRetryable: shouldRetry, error: error instanceof Error ? error.message : String(error) }); throw error; } // Calculate delay for next attempt (respects Retry-After for rate limits) const delay = calculateBackoffDelay(attempt, config, error); // Log retry attempt with rate limit detection const isRateLimit = isRateLimitError(error); const retryAfter = isRateLimit ? extractRetryAfter(error) : null; logger.warn('Retrying after error', { attempt: attempt + 1, maxAttempts: config.maxAttempts, delay, isRateLimit, retryAfterMs: retryAfter, error: error instanceof Error ? error.message : String(error) }); // Invoke callback with additional context config.onRetry(attempt + 1, error, delay); // Wait before retrying await new Promise(resolve => setTimeout(resolve, delay)); } } // This should never be reached, but TypeScript requires it throw lastError; }