diff --git a/src/hooks/moderation/useModerationActions.ts b/src/hooks/moderation/useModerationActions.ts index 404f34c4..91a5c288 100644 --- a/src/hooks/moderation/useModerationActions.ts +++ b/src/hooks/moderation/useModerationActions.ts @@ -284,14 +284,23 @@ export function useModerationActions(config: ModerationActionsConfig): Moderatio } } - const { data, error, requestId } = await invokeWithTracking( + const { data, error, requestId, attempts } = await invokeWithTracking( 'process-selective-approval', { itemIds: submissionItems.map((i) => i.id), submissionId: item.id, }, - config.user?.id + config.user?.id, + undefined, + undefined, + 30000, // 30s timeout + { maxAttempts: 3, baseDelay: 1500 } // Critical operation - retry config ); + + // Log if retries were needed + if (attempts && attempts > 1) { + logger.log(`Approval succeeded after ${attempts} attempts for ${item.id}`); + } if (error) throw error; @@ -620,14 +629,22 @@ export function useModerationActions(config: ModerationActionsConfig): Moderatio failedItemsCount = failedItems.length; - const { data, error, requestId } = await invokeWithTracking( + const { data, error, requestId, attempts } = await invokeWithTracking( 'process-selective-approval', { itemIds: failedItems.map((i) => i.id), submissionId: item.id, }, - config.user?.id + config.user?.id, + undefined, + undefined, + 30000, + { maxAttempts: 3, baseDelay: 1500 } // Retry for failed items ); + + if (attempts && attempts > 1) { + logger.log(`Retry succeeded after ${attempts} attempts for ${item.id}`); + } if (error) throw error; @@ -699,16 +716,24 @@ export function useModerationActions(config: ModerationActionsConfig): Moderatio onActionStart(item.id); try { - // Call edge function for email notification - const { error: edgeFunctionError, requestId } = await invokeWithTracking( + // Call edge function for email notification with retry + const { error: edgeFunctionError, requestId, attempts } = await invokeWithTracking( 'send-escalation-notification', { submissionId: item.id, escalationReason: reason, escalatedBy: user.id, }, - user.id + user.id, + undefined, + undefined, + 45000, // Longer timeout for email sending + { maxAttempts: 3, baseDelay: 2000 } // Retry for email delivery ); + + if (attempts && attempts > 1) { + logger.log(`Escalation email sent after ${attempts} attempts`); + } if (edgeFunctionError) { // Edge function failed - log and show fallback toast diff --git a/src/lib/edgeFunctionTracking.ts b/src/lib/edgeFunctionTracking.ts index 54bf7719..330f5ef2 100644 --- a/src/lib/edgeFunctionTracking.ts +++ b/src/lib/edgeFunctionTracking.ts @@ -8,6 +8,8 @@ import { supabase } from '@/lib/supabaseClient'; import { trackRequest } from './requestTracking'; import { getErrorMessage } from './errorHandler'; +import { withRetry, isRetryableError, type RetryOptions } from './retryHelpers'; +import { breadcrumb } from './errorBreadcrumbs'; /** * Invoke a Supabase edge function with request tracking @@ -25,11 +27,32 @@ export async function invokeWithTracking( userId?: string, parentRequestId?: string, traceId?: string, - timeout: number = 30000 // Default 30s timeout -): Promise<{ data: T | null; error: any; requestId: string; duration: number }> { - // Create AbortController for timeout - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout); + timeout: number = 30000, // Default 30s timeout + retryOptions?: Partial // NEW: Optional retry configuration +): Promise<{ data: T | null; error: any; requestId: string; duration: number; attempts?: number }> { + // Configure retry options with defaults + const effectiveRetryOptions: RetryOptions = { + maxAttempts: retryOptions?.maxAttempts ?? 3, + baseDelay: retryOptions?.baseDelay ?? 1000, + maxDelay: retryOptions?.maxDelay ?? 10000, + backoffMultiplier: retryOptions?.backoffMultiplier ?? 2, + jitter: true, + shouldRetry: isRetryableError, + onRetry: (attempt, error, delay) => { + // Log retry attempt to breadcrumbs + breadcrumb.apiCall( + `/functions/${functionName}`, + 'POST', + undefined // status unknown during retry + ); + + console.info(`Retrying ${functionName} (attempt ${attempt}) after ${delay}ms:`, + getErrorMessage(error) + ); + }, + }; + + let attemptCount = 0; try { const { result, requestId, duration } = await trackRequest( @@ -41,22 +64,41 @@ export async function invokeWithTracking( traceId, }, async (context) => { - // Include client request ID in payload for correlation - const { data, error } = await supabase.functions.invoke(functionName, { - body: { ...payload, clientRequestId: context.requestId }, - signal: controller.signal, // Add abort signal for timeout - }); - - if (error) throw error; - return data; + return await withRetry( + async () => { + attemptCount++; + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + + try { + const { data, error } = await supabase.functions.invoke(functionName, { + body: { ...payload, clientRequestId: context.requestId }, + signal: controller.signal, + }); + + clearTimeout(timeoutId); + + if (error) { + // Enhance error with status for retry logic + const enhancedError = new Error(error.message || 'Edge function error'); + (enhancedError as any).status = error.status; + throw enhancedError; + } + + return data; + } catch (error) { + clearTimeout(timeoutId); + throw error; + } + }, + effectiveRetryOptions + ); } ); - clearTimeout(timeoutId); - return { data: result, error: null, requestId, duration }; + return { data: result, error: null, requestId, duration, attempts: attemptCount }; } catch (error: unknown) { - clearTimeout(timeoutId); - // Handle AbortError specifically if (error instanceof Error && error.name === 'AbortError') { return { @@ -67,6 +109,7 @@ export async function invokeWithTracking( }, requestId: 'timeout', duration: timeout, + attempts: attemptCount, }; } @@ -76,6 +119,7 @@ export async function invokeWithTracking( error: { message: errorMessage }, requestId: 'unknown', duration: 0, + attempts: attemptCount, }; } } @@ -93,6 +137,7 @@ export async function invokeBatchWithTracking( operations: Array<{ functionName: string; payload: any; + retryOptions?: Partial; }>, userId?: string ): Promise< @@ -102,6 +147,7 @@ export async function invokeBatchWithTracking( error: any; requestId: string; duration: number; + attempts?: number; }> > { const traceId = crypto.randomUUID(); @@ -113,7 +159,9 @@ export async function invokeBatchWithTracking( op.payload, userId, undefined, - traceId + traceId, + 30000, // default timeout + op.retryOptions // Pass through retry options ); return { functionName: op.functionName, ...result }; }) diff --git a/supabase/functions/_shared/retryHelper.ts b/supabase/functions/_shared/retryHelper.ts new file mode 100644 index 00000000..3f256c9d --- /dev/null +++ b/supabase/functions/_shared/retryHelper.ts @@ -0,0 +1,142 @@ +/** + * Edge Function Retry Helper + * Provides exponential backoff retry logic for external API calls + */ + +import { edgeLogger } from './logger.ts'; + +export interface EdgeRetryOptions { + maxAttempts?: number; + baseDelay?: number; + maxDelay?: number; + backoffMultiplier?: number; + jitter?: boolean; + shouldRetry?: (error: unknown) => boolean; +} + +/** + * Determines if an error is transient and should be retried + */ +export function isRetryableError(error: unknown): boolean { + // Network errors + if (error instanceof TypeError && error.message.includes('fetch')) return true; + + if (error instanceof Error) { + const msg = error.message.toLowerCase(); + if (msg.includes('network') || msg.includes('timeout') || msg.includes('econnrefused')) { + return true; + } + } + + // HTTP status codes that should be retried + if (error && typeof error === 'object') { + const httpError = error as { status?: number }; + + // Rate limiting + if (httpError.status === 429) return true; + + // Service unavailable or gateway timeout + if (httpError.status === 503 || httpError.status === 504) return true; + + // Server errors (5xx) + if (httpError.status && httpError.status >= 500 && httpError.status < 600) { + return true; + } + } + + return false; +} + +/** + * Calculate exponential backoff delay with optional jitter + */ +function calculateBackoffDelay( + attempt: number, + baseDelay: number, + maxDelay: number, + backoffMultiplier: number, + jitter: boolean +): number { + const exponentialDelay = baseDelay * Math.pow(backoffMultiplier, attempt); + const cappedDelay = Math.min(exponentialDelay, maxDelay); + + if (!jitter) return cappedDelay; + + // Add random jitter (-30% to +30%) + const jitterAmount = cappedDelay * 0.3; + const randomJitter = (Math.random() * 2 - 1) * jitterAmount; + + return Math.max(0, cappedDelay + randomJitter); +} + +/** + * Retry wrapper for asynchronous operations with exponential backoff + * + * @param fn - Async function to retry + * @param options - Retry configuration + * @param requestId - Request ID for tracking + * @param context - Context description for logging + */ +export async function withEdgeRetry( + fn: () => Promise, + options: EdgeRetryOptions = {}, + requestId: string, + context: string +): Promise { + const maxAttempts = options.maxAttempts ?? 3; + const baseDelay = options.baseDelay ?? 1000; + const maxDelay = options.maxDelay ?? 10000; + const backoffMultiplier = options.backoffMultiplier ?? 2; + const jitter = options.jitter ?? true; + const shouldRetry = options.shouldRetry ?? isRetryableError; + + let lastError: unknown; + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error; + + // Don't retry if this is the last attempt + if (attempt === maxAttempts - 1) { + edgeLogger.error('All retry attempts exhausted', { + requestId, + context, + attempts: maxAttempts, + error: error instanceof Error ? error.message : 'Unknown error' + }); + throw error; + } + + // Don't retry if error is not retryable + if (!shouldRetry(error)) { + edgeLogger.info('Error not retryable, failing immediately', { + requestId, + context, + attempt: attempt + 1, + error: error instanceof Error ? error.message : 'Unknown error' + }); + throw error; + } + + // Calculate delay for next retry + const delay = calculateBackoffDelay(attempt, baseDelay, maxDelay, backoffMultiplier, jitter); + + edgeLogger.info('Retrying after error', { + requestId, + context, + attempt: attempt + 1, + maxAttempts, + delay: Math.round(delay), + error: error instanceof Error ? error.message : 'Unknown error' + }); + + // Wait before retrying + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + + // This should never be reached, but TypeScript needs it + throw lastError; +} diff --git a/supabase/functions/manage-moderator-topic/index.ts b/supabase/functions/manage-moderator-topic/index.ts index ad6cc1d5..a6ef002d 100644 --- a/supabase/functions/manage-moderator-topic/index.ts +++ b/supabase/functions/manage-moderator-topic/index.ts @@ -2,6 +2,7 @@ import { serve } from "https://deno.land/std@0.190.0/http/server.ts"; import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4"; import { Novu } from "npm:@novu/api@1.6.0"; import { edgeLogger, startRequest, endRequest } from "../_shared/logger.ts"; +import { withEdgeRetry } from '../_shared/retryHelper.ts'; const corsHeaders = { 'Access-Control-Allow-Origin': '*', @@ -45,21 +46,28 @@ serve(async (req) => { for (const topicKey of topics) { try { - if (action === 'add') { - // Add subscriber to topic - await novu.topics.addSubscribers(topicKey, { - subscribers: [userId], - }); - edgeLogger.info('Added user to topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey }); - results.push({ topic: topicKey, action: 'added', success: true }); - } else { - // Remove subscriber from topic - await novu.topics.removeSubscribers(topicKey, { - subscribers: [userId], - }); - edgeLogger.info('Removed user from topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey }); - results.push({ topic: topicKey, action: 'removed', success: true }); - } + await withEdgeRetry( + async () => { + if (action === 'add') { + // Add subscriber to topic + await novu.topics.addSubscribers(topicKey, { + subscribers: [userId], + }); + edgeLogger.info('Added user to topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey }); + } else { + // Remove subscriber from topic + await novu.topics.removeSubscribers(topicKey, { + subscribers: [userId], + }); + edgeLogger.info('Removed user from topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey }); + } + }, + { maxAttempts: 3, baseDelay: 1000 }, + tracking.requestId, + `${action}-topic-${topicKey}` + ); + + results.push({ topic: topicKey, action: action === 'add' ? 'added' : 'removed', success: true }); } catch (error: any) { edgeLogger.error(`Error ${action}ing user ${userId} ${action === 'add' ? 'to' : 'from'} topic ${topicKey}`, { action: 'manage_moderator_topic', diff --git a/supabase/functions/notify-moderators-report/index.ts b/supabase/functions/notify-moderators-report/index.ts index 2d25a4d7..245c9b11 100644 --- a/supabase/functions/notify-moderators-report/index.ts +++ b/supabase/functions/notify-moderators-report/index.ts @@ -1,6 +1,7 @@ import { serve } from "https://deno.land/std@0.168.0/http/server.ts"; import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4"; import { edgeLogger, startRequest, endRequest } from "../_shared/logger.ts"; +import { withEdgeRetry } from '../_shared/retryHelper.ts'; const corsHeaders = { 'Access-Control-Allow-Origin': '*', @@ -150,22 +151,32 @@ serve(async (req) => { edgeLogger.info('Triggering notification with payload', { action: 'notify_moderators_report', requestId: tracking.requestId }); - // Invoke the trigger-notification function - const { data: result, error: notifyError } = await supabase.functions.invoke( - 'trigger-notification', - { - body: { - workflowId: template.workflow_id, - topicKey: 'moderation-reports', - payload: notificationPayload, - }, - } - ); + // Invoke the trigger-notification function with retry + const result = await withEdgeRetry( + async () => { + const { data, error } = await supabase.functions.invoke( + 'trigger-notification', + { + body: { + workflowId: template.workflow_id, + topicKey: 'moderation-reports', + payload: notificationPayload, + }, + } + ); - if (notifyError) { - edgeLogger.error('Error triggering notification', { action: 'notify_moderators_report', requestId: tracking.requestId, error: notifyError }); - throw notifyError; - } + if (error) { + const enhancedError = new Error(error.message || 'Notification trigger failed'); + (enhancedError as any).status = error.status; + throw enhancedError; + } + + return data; + }, + { maxAttempts: 3, baseDelay: 1000 }, + tracking.requestId, + 'trigger-report-notification' + ); edgeLogger.info('Notification triggered successfully', { action: 'notify_moderators_report', requestId: tracking.requestId, result }); diff --git a/supabase/functions/notify-moderators-submission/index.ts b/supabase/functions/notify-moderators-submission/index.ts index f3138db3..f803a3c4 100644 --- a/supabase/functions/notify-moderators-submission/index.ts +++ b/supabase/functions/notify-moderators-submission/index.ts @@ -1,6 +1,7 @@ import { serve } from "https://deno.land/std@0.168.0/http/server.ts"; import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4"; import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts'; +import { withEdgeRetry } from '../_shared/retryHelper.ts'; const corsHeaders = { 'Access-Control-Allow-Origin': '*', @@ -186,15 +187,30 @@ serve(async (req) => { isEscalated: is_escalated, }; - // Send ONE notification to the moderation-submissions topic + // Send ONE notification to the moderation-submissions topic with retry // All subscribers (moderators) will receive it automatically - const { data, error } = await supabase.functions.invoke('trigger-notification', { - body: { - workflowId: workflow.workflow_id, - topicKey: 'moderation-submissions', - payload: notificationPayload, + const data = await withEdgeRetry( + async () => { + const { data: result, error } = await supabase.functions.invoke('trigger-notification', { + body: { + workflowId: workflow.workflow_id, + topicKey: 'moderation-submissions', + payload: notificationPayload, + }, + }); + + if (error) { + const enhancedError = new Error(error.message || 'Notification trigger failed'); + (enhancedError as any).status = error.status; + throw enhancedError; + } + + return result; }, - }); + { maxAttempts: 3, baseDelay: 1000 }, + tracking.requestId, + 'trigger-submission-notification' + ); // Log notification in notification_logs with idempotency key await supabase.from('notification_logs').insert({ @@ -209,32 +225,6 @@ serve(async (req) => { } }); - if (error) { - const duration = endRequest(tracking); - edgeLogger.error('Failed to notify moderators via topic', { - action: 'notify_moderators', - requestId: tracking.requestId, - duration, - error: error.message - }); - return new Response( - JSON.stringify({ - success: false, - error: 'Failed to send notification to topic', - details: error.message, - requestId: tracking.requestId - }), - { - headers: { - ...corsHeaders, - 'Content-Type': 'application/json', - 'X-Request-ID': tracking.requestId - }, - status: 500, - } - ); - } - const duration = endRequest(tracking); edgeLogger.info('Successfully notified all moderators via topic', { action: 'notify_moderators', diff --git a/supabase/functions/send-escalation-notification/index.ts b/supabase/functions/send-escalation-notification/index.ts index 0a350220..fce0ae80 100644 --- a/supabase/functions/send-escalation-notification/index.ts +++ b/supabase/functions/send-escalation-notification/index.ts @@ -1,6 +1,7 @@ import { serve } from "https://deno.land/std@0.190.0/http/server.ts"; import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4"; import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts'; +import { withEdgeRetry } from '../_shared/retryHelper.ts'; const corsHeaders = { 'Access-Control-Allow-Origin': '*', @@ -169,7 +170,7 @@ serve(async (req) => { Please review this submission in the admin panel. `; - // Send email via ForwardEmail API + // Send email via ForwardEmail API with retry const forwardEmailApiKey = Deno.env.get('FORWARDEMAIL_API_KEY'); const adminEmail = Deno.env.get('ADMIN_EMAIL_ADDRESS'); const fromEmail = Deno.env.get('FROM_EMAIL_ADDRESS'); @@ -178,55 +179,43 @@ serve(async (req) => { throw new Error('Email configuration is incomplete. Please check environment variables.'); } - let emailResponse; - try { - emailResponse = await fetch('https://api.forwardemail.net/v1/emails', { - method: 'POST', - headers: { - 'Authorization': 'Basic ' + btoa(forwardEmailApiKey + ':'), - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - from: fromEmail, - to: adminEmail, - subject: emailSubject, - html: emailHtml, - text: emailText, - }), - }); - } catch (fetchError) { - edgeLogger.error('Network error sending email', { - requestId: tracking.requestId, - error: fetchError.message - }); - throw new Error('Network error: Unable to reach email service'); - } + const emailResult = await withEdgeRetry( + async () => { + const emailResponse = await fetch('https://api.forwardemail.net/v1/emails', { + method: 'POST', + headers: { + 'Authorization': 'Basic ' + btoa(forwardEmailApiKey + ':'), + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + from: fromEmail, + to: adminEmail, + subject: emailSubject, + html: emailHtml, + text: emailText, + }), + }); - if (!emailResponse.ok) { - let errorText; - try { - errorText = await emailResponse.text(); - } catch (parseError) { - errorText = 'Unable to parse error response'; - } - edgeLogger.error('ForwardEmail API error', { - requestId: tracking.requestId, - status: emailResponse.status, - errorText - }); - throw new Error(`Failed to send email: ${emailResponse.status} - ${errorText}`); - } + if (!emailResponse.ok) { + let errorText; + try { + errorText = await emailResponse.text(); + } catch (parseError) { + errorText = 'Unable to parse error response'; + } + + const error = new Error(`Failed to send email: ${emailResponse.status} - ${errorText}`); + (error as any).status = emailResponse.status; + throw error; + } - let emailResult; - try { - emailResult = await emailResponse.json(); - } catch (parseError) { - edgeLogger.error('Failed to parse email API response', { - requestId: tracking.requestId, - error: parseError.message - }); - throw new Error('Invalid response from email service'); - } + const result = await emailResponse.json(); + return result; + }, + { maxAttempts: 3, baseDelay: 1500, maxDelay: 10000 }, + tracking.requestId, + 'send-escalation-email' + ); edgeLogger.info('Email sent successfully', { requestId: tracking.requestId, emailId: emailResult.id diff --git a/supabase/functions/sync-all-moderators-to-topic/index.ts b/supabase/functions/sync-all-moderators-to-topic/index.ts index ca7f352c..eb759da7 100644 --- a/supabase/functions/sync-all-moderators-to-topic/index.ts +++ b/supabase/functions/sync-all-moderators-to-topic/index.ts @@ -2,6 +2,7 @@ import { serve } from "https://deno.land/std@0.168.0/http/server.ts"; import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4"; import { Novu } from "npm:@novu/api@1.6.0"; import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts'; +import { withEdgeRetry } from '../_shared/retryHelper.ts'; const corsHeaders = { 'Access-Control-Allow-Origin': '*', @@ -86,9 +87,17 @@ serve(async (req) => { const batch = uniqueUserIds.slice(i, i + batchSize); try { - await novu.topics.addSubscribers(topicKey, { - subscribers: batch, - }); + await withEdgeRetry( + async () => { + await novu.topics.addSubscribers(topicKey, { + subscribers: batch, + }); + }, + { maxAttempts: 3, baseDelay: 2000 }, + tracking.requestId, + `sync-batch-${topicKey}-${i}` + ); + successCount += batch.length; edgeLogger.info('Added batch of users to topic', { requestId: tracking.requestId,