feat: Implement retry logic and tracking

src/hooks/moderation/useModerationActions.ts

@@ -284,14 +284,23 @@ export function useModerationActions(config: ModerationActionsConfig): Moderatio
               }
             }
 
-            const { data, error, requestId } = await invokeWithTracking(
+            const { data, error, requestId, attempts } = await invokeWithTracking(
               'process-selective-approval',
               {
                 itemIds: submissionItems.map((i) => i.id),
                 submissionId: item.id,
               },
-              config.user?.id
+              config.user?.id,
+              undefined,
+              undefined,
+              30000, // 30s timeout
+              { maxAttempts: 3, baseDelay: 1500 } // Critical operation - retry config
             );
+            
+            // Log if retries were needed
+            if (attempts && attempts > 1) {
+              logger.log(`Approval succeeded after ${attempts} attempts for ${item.id}`);
+            }
 
             if (error) throw error;
 
@@ -620,14 +629,22 @@ export function useModerationActions(config: ModerationActionsConfig): Moderatio
 
         failedItemsCount = failedItems.length;
 
-        const { data, error, requestId } = await invokeWithTracking(
+        const { data, error, requestId, attempts } = await invokeWithTracking(
           'process-selective-approval',
           {
             itemIds: failedItems.map((i) => i.id),
             submissionId: item.id,
           },
-          config.user?.id
+          config.user?.id,
+          undefined,
+          undefined,
+          30000,
+          { maxAttempts: 3, baseDelay: 1500 } // Retry for failed items
         );
+        
+        if (attempts && attempts > 1) {
+          logger.log(`Retry succeeded after ${attempts} attempts for ${item.id}`);
+        }
 
         if (error) throw error;
 
@@ -699,16 +716,24 @@ export function useModerationActions(config: ModerationActionsConfig): Moderatio
       onActionStart(item.id);
 
       try {
-        // Call edge function for email notification
+        // Call edge function for email notification with retry
-        const { error: edgeFunctionError, requestId } = await invokeWithTracking(
+        const { error: edgeFunctionError, requestId, attempts } = await invokeWithTracking(
           'send-escalation-notification',
           {
             submissionId: item.id,
             escalationReason: reason,
             escalatedBy: user.id,
           },
-          user.id
+          user.id,
+          undefined,
+          undefined,
+          45000, // Longer timeout for email sending
+          { maxAttempts: 3, baseDelay: 2000 } // Retry for email delivery
         );
+        
+        if (attempts && attempts > 1) {
+          logger.log(`Escalation email sent after ${attempts} attempts`);
+        }
 
         if (edgeFunctionError) {
           // Edge function failed - log and show fallback toast

src/lib/edgeFunctionTracking.ts

@@ -8,6 +8,8 @@
 import { supabase } from '@/lib/supabaseClient';
 import { trackRequest } from './requestTracking';
 import { getErrorMessage } from './errorHandler';
+import { withRetry, isRetryableError, type RetryOptions } from './retryHelpers';
+import { breadcrumb } from './errorBreadcrumbs';
 
 /**
  * Invoke a Supabase edge function with request tracking
@@ -25,11 +27,32 @@ export async function invokeWithTracking<T = any>(
   userId?: string,
   parentRequestId?: string,
   traceId?: string,
-  timeout: number = 30000 // Default 30s timeout
+  timeout: number = 30000, // Default 30s timeout
-): Promise<{ data: T | null; error: any; requestId: string; duration: number }> {
+  retryOptions?: Partial<RetryOptions> // NEW: Optional retry configuration
-  // Create AbortController for timeout
+): Promise<{ data: T | null; error: any; requestId: string; duration: number; attempts?: number }> {
-  const controller = new AbortController();
+  // Configure retry options with defaults
-  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  const effectiveRetryOptions: RetryOptions = {
+    maxAttempts: retryOptions?.maxAttempts ?? 3,
+    baseDelay: retryOptions?.baseDelay ?? 1000,
+    maxDelay: retryOptions?.maxDelay ?? 10000,
+    backoffMultiplier: retryOptions?.backoffMultiplier ?? 2,
+    jitter: true,
+    shouldRetry: isRetryableError,
+    onRetry: (attempt, error, delay) => {
+      // Log retry attempt to breadcrumbs
+      breadcrumb.apiCall(
+        `/functions/${functionName}`,
+        'POST',
+        undefined // status unknown during retry
+      );
+      
+      console.info(`Retrying ${functionName} (attempt ${attempt}) after ${delay}ms:`, 
+        getErrorMessage(error)
+      );
+    },
+  };
+
+  let attemptCount = 0;
   
   try {
     const { result, requestId, duration } = await trackRequest(
@@ -41,22 +64,41 @@ export async function invokeWithTracking<T = any>(
         traceId,
       },
       async (context) => {
-        // Include client request ID in payload for correlation
+        return await withRetry(
-        const { data, error } = await supabase.functions.invoke<T>(functionName, {
+          async () => {
-          body: { ...payload, clientRequestId: context.requestId },
+            attemptCount++;
-          signal: controller.signal, // Add abort signal for timeout
+            
-        });
+            const controller = new AbortController();
-
+            const timeoutId = setTimeout(() => controller.abort(), timeout);
-        if (error) throw error;
+            
-        return data;
+            try {
+              const { data, error } = await supabase.functions.invoke<T>(functionName, {
+                body: { ...payload, clientRequestId: context.requestId },
+                signal: controller.signal,
+              });
+              
+              clearTimeout(timeoutId);
+              
+              if (error) {
+                // Enhance error with status for retry logic
+                const enhancedError = new Error(error.message || 'Edge function error');
+                (enhancedError as any).status = error.status;
+                throw enhancedError;
+              }
+              
+              return data;
+            } catch (error) {
+              clearTimeout(timeoutId);
+              throw error;
+            }
+          },
+          effectiveRetryOptions
+        );
       }
     );
 
-    clearTimeout(timeoutId);
+    return { data: result, error: null, requestId, duration, attempts: attemptCount };
-    return { data: result, error: null, requestId, duration };
   } catch (error: unknown) {
-    clearTimeout(timeoutId);
-    
     // Handle AbortError specifically
     if (error instanceof Error && error.name === 'AbortError') {
       return {
@@ -67,6 +109,7 @@ export async function invokeWithTracking<T = any>(
         },
         requestId: 'timeout',
         duration: timeout,
+        attempts: attemptCount,
       };
     }
     
@@ -76,6 +119,7 @@ export async function invokeWithTracking<T = any>(
       error: { message: errorMessage },
       requestId: 'unknown',
       duration: 0,
+      attempts: attemptCount,
     };
   }
 }
@@ -93,6 +137,7 @@ export async function invokeBatchWithTracking<T = any>(
   operations: Array<{
     functionName: string;
     payload: any;
+    retryOptions?: Partial<RetryOptions>;
   }>,
   userId?: string
 ): Promise<
@@ -102,6 +147,7 @@ export async function invokeBatchWithTracking<T = any>(
     error: any;
     requestId: string;
     duration: number;
+    attempts?: number;
   }>
 > {
   const traceId = crypto.randomUUID();
@@ -113,7 +159,9 @@ export async function invokeBatchWithTracking<T = any>(
         op.payload,
         userId,
         undefined,
-        traceId
+        traceId,
+        30000, // default timeout
+        op.retryOptions // Pass through retry options
       );
       return { functionName: op.functionName, ...result };
     })

supabase/functions/_shared/retryHelper.ts

@@ -0,0 +1,142 @@
+/**
+ * Edge Function Retry Helper
+ * Provides exponential backoff retry logic for external API calls
+ */
+
+import { edgeLogger } from './logger.ts';
+
+export interface EdgeRetryOptions {
+  maxAttempts?: number;
+  baseDelay?: number;
+  maxDelay?: number;
+  backoffMultiplier?: number;
+  jitter?: boolean;
+  shouldRetry?: (error: unknown) => boolean;
+}
+
+/**
+ * Determines if an error is transient and should be retried
+ */
+export function isRetryableError(error: unknown): boolean {
+  // Network errors
+  if (error instanceof TypeError && error.message.includes('fetch')) return true;
+  
+  if (error instanceof Error) {
+    const msg = error.message.toLowerCase();
+    if (msg.includes('network') || msg.includes('timeout') || msg.includes('econnrefused')) {
+      return true;
+    }
+  }
+  
+  // HTTP status codes that should be retried
+  if (error && typeof error === 'object') {
+    const httpError = error as { status?: number };
+    
+    // Rate limiting
+    if (httpError.status === 429) return true;
+    
+    // Service unavailable or gateway timeout
+    if (httpError.status === 503 || httpError.status === 504) return true;
+    
+    // Server errors (5xx)
+    if (httpError.status && httpError.status >= 500 && httpError.status < 600) {
+      return true;
+    }
+  }
+  
+  return false;
+}
+
+/**
+ * Calculate exponential backoff delay with optional jitter
+ */
+function calculateBackoffDelay(
+  attempt: number,
+  baseDelay: number,
+  maxDelay: number,
+  backoffMultiplier: number,
+  jitter: boolean
+): number {
+  const exponentialDelay = baseDelay * Math.pow(backoffMultiplier, attempt);
+  const cappedDelay = Math.min(exponentialDelay, maxDelay);
+  
+  if (!jitter) return cappedDelay;
+  
+  // Add random jitter (-30% to +30%)
+  const jitterAmount = cappedDelay * 0.3;
+  const randomJitter = (Math.random() * 2 - 1) * jitterAmount;
+  
+  return Math.max(0, cappedDelay + randomJitter);
+}
+
+/**
+ * Retry wrapper for asynchronous operations with exponential backoff
+ * 
+ * @param fn - Async function to retry
+ * @param options - Retry configuration
+ * @param requestId - Request ID for tracking
+ * @param context - Context description for logging
+ */
+export async function withEdgeRetry<T>(
+  fn: () => Promise<T>,
+  options: EdgeRetryOptions = {},
+  requestId: string,
+  context: string
+): Promise<T> {
+  const maxAttempts = options.maxAttempts ?? 3;
+  const baseDelay = options.baseDelay ?? 1000;
+  const maxDelay = options.maxDelay ?? 10000;
+  const backoffMultiplier = options.backoffMultiplier ?? 2;
+  const jitter = options.jitter ?? true;
+  const shouldRetry = options.shouldRetry ?? isRetryableError;
+  
+  let lastError: unknown;
+  
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      
+      // Don't retry if this is the last attempt
+      if (attempt === maxAttempts - 1) {
+        edgeLogger.error('All retry attempts exhausted', {
+          requestId,
+          context,
+          attempts: maxAttempts,
+          error: error instanceof Error ? error.message : 'Unknown error'
+        });
+        throw error;
+      }
+      
+      // Don't retry if error is not retryable
+      if (!shouldRetry(error)) {
+        edgeLogger.info('Error not retryable, failing immediately', {
+          requestId,
+          context,
+          attempt: attempt + 1,
+          error: error instanceof Error ? error.message : 'Unknown error'
+        });
+        throw error;
+      }
+      
+      // Calculate delay for next retry
+      const delay = calculateBackoffDelay(attempt, baseDelay, maxDelay, backoffMultiplier, jitter);
+      
+      edgeLogger.info('Retrying after error', {
+        requestId,
+        context,
+        attempt: attempt + 1,
+        maxAttempts,
+        delay: Math.round(delay),
+        error: error instanceof Error ? error.message : 'Unknown error'
+      });
+      
+      // Wait before retrying
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+  
+  // This should never be reached, but TypeScript needs it
+  throw lastError;
+}

supabase/functions/manage-moderator-topic/index.ts

@@ -2,6 +2,7 @@ import { serve } from "https://deno.land/std@0.190.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { Novu } from "npm:@novu/api@1.6.0";
 import { edgeLogger, startRequest, endRequest } from "../_shared/logger.ts";
+import { withEdgeRetry } from '../_shared/retryHelper.ts';
 
 const corsHeaders = {
   'Access-Control-Allow-Origin': '*',
@@ -45,21 +46,28 @@ serve(async (req) => {
 
     for (const topicKey of topics) {
       try {
-        if (action === 'add') {
+        await withEdgeRetry(
-          // Add subscriber to topic
+          async () => {
-          await novu.topics.addSubscribers(topicKey, {
+            if (action === 'add') {
-            subscribers: [userId],
+              // Add subscriber to topic
-          });
+              await novu.topics.addSubscribers(topicKey, {
-          edgeLogger.info('Added user to topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
+                subscribers: [userId],
-          results.push({ topic: topicKey, action: 'added', success: true });
+              });
-        } else {
+              edgeLogger.info('Added user to topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
-          // Remove subscriber from topic
+            } else {
-          await novu.topics.removeSubscribers(topicKey, {
+              // Remove subscriber from topic
-            subscribers: [userId],
+              await novu.topics.removeSubscribers(topicKey, {
-          });
+                subscribers: [userId],
-          edgeLogger.info('Removed user from topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
+              });
-          results.push({ topic: topicKey, action: 'removed', success: true });
+              edgeLogger.info('Removed user from topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
-        }
+            }
+          },
+          { maxAttempts: 3, baseDelay: 1000 },
+          tracking.requestId,
+          `${action}-topic-${topicKey}`
+        );
+        
+        results.push({ topic: topicKey, action: action === 'add' ? 'added' : 'removed', success: true });
       } catch (error: any) {
         edgeLogger.error(`Error ${action}ing user ${userId} ${action === 'add' ? 'to' : 'from'} topic ${topicKey}`, { 
           action: 'manage_moderator_topic', 

supabase/functions/notify-moderators-report/index.ts

@@ -1,6 +1,7 @@
 import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { edgeLogger, startRequest, endRequest } from "../_shared/logger.ts";
+import { withEdgeRetry } from '../_shared/retryHelper.ts';
 
 const corsHeaders = {
   'Access-Control-Allow-Origin': '*',
@@ -150,22 +151,32 @@ serve(async (req) => {
 
     edgeLogger.info('Triggering notification with payload', { action: 'notify_moderators_report', requestId: tracking.requestId });
 
-    // Invoke the trigger-notification function
+    // Invoke the trigger-notification function with retry
-    const { data: result, error: notifyError } = await supabase.functions.invoke(
+    const result = await withEdgeRetry(
-      'trigger-notification',
+      async () => {
-      {
+        const { data, error } = await supabase.functions.invoke(
-        body: {
+          'trigger-notification',
-          workflowId: template.workflow_id,
+          {
-          topicKey: 'moderation-reports',
+            body: {
-          payload: notificationPayload,
+              workflowId: template.workflow_id,
-        },
+              topicKey: 'moderation-reports',
-      }
+              payload: notificationPayload,
-    );
+            },
+          }
+        );
 
-    if (notifyError) {
+        if (error) {
-      edgeLogger.error('Error triggering notification', { action: 'notify_moderators_report', requestId: tracking.requestId, error: notifyError });
+          const enhancedError = new Error(error.message || 'Notification trigger failed');
-      throw notifyError;
+          (enhancedError as any).status = error.status;
-    }
+          throw enhancedError;
+        }
+
+        return data;
+      },
+      { maxAttempts: 3, baseDelay: 1000 },
+      tracking.requestId,
+      'trigger-report-notification'
+    );
 
     edgeLogger.info('Notification triggered successfully', { action: 'notify_moderators_report', requestId: tracking.requestId, result });
 

supabase/functions/notify-moderators-submission/index.ts

@@ -1,6 +1,7 @@
 import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts';
+import { withEdgeRetry } from '../_shared/retryHelper.ts';
 
 const corsHeaders = {
   'Access-Control-Allow-Origin': '*',
@@ -186,15 +187,30 @@ serve(async (req) => {
       isEscalated: is_escalated,
     };
 
-    // Send ONE notification to the moderation-submissions topic
+    // Send ONE notification to the moderation-submissions topic with retry
     // All subscribers (moderators) will receive it automatically
-    const { data, error } = await supabase.functions.invoke('trigger-notification', {
+    const data = await withEdgeRetry(
-      body: {
+      async () => {
-        workflowId: workflow.workflow_id,
+        const { data: result, error } = await supabase.functions.invoke('trigger-notification', {
-        topicKey: 'moderation-submissions',
+          body: {
-        payload: notificationPayload,
+            workflowId: workflow.workflow_id,
+            topicKey: 'moderation-submissions',
+            payload: notificationPayload,
+          },
+        });
+
+        if (error) {
+          const enhancedError = new Error(error.message || 'Notification trigger failed');
+          (enhancedError as any).status = error.status;
+          throw enhancedError;
+        }
+
+        return result;
       },
-    });
+      { maxAttempts: 3, baseDelay: 1000 },
+      tracking.requestId,
+      'trigger-submission-notification'
+    );
 
     // Log notification in notification_logs with idempotency key
     await supabase.from('notification_logs').insert({
@@ -209,32 +225,6 @@ serve(async (req) => {
       }
     });
 
-    if (error) {
-      const duration = endRequest(tracking);
-      edgeLogger.error('Failed to notify moderators via topic', { 
-        action: 'notify_moderators',
-        requestId: tracking.requestId,
-        duration,
-        error: error.message 
-      });
-      return new Response(
-        JSON.stringify({ 
-          success: false, 
-          error: 'Failed to send notification to topic',
-          details: error.message,
-          requestId: tracking.requestId 
-        }),
-        {
-          headers: { 
-            ...corsHeaders, 
-            'Content-Type': 'application/json',
-            'X-Request-ID': tracking.requestId 
-          },
-          status: 500,
-        }
-      );
-    }
-
     const duration = endRequest(tracking);
     edgeLogger.info('Successfully notified all moderators via topic', { 
       action: 'notify_moderators',

supabase/functions/send-escalation-notification/index.ts

@@ -1,6 +1,7 @@
 import { serve } from "https://deno.land/std@0.190.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts';
+import { withEdgeRetry } from '../_shared/retryHelper.ts';
 
 const corsHeaders = {
   'Access-Control-Allow-Origin': '*',
@@ -169,7 +170,7 @@ serve(async (req) => {
       Please review this submission in the admin panel.
     `;
 
-    // Send email via ForwardEmail API
+    // Send email via ForwardEmail API with retry
     const forwardEmailApiKey = Deno.env.get('FORWARDEMAIL_API_KEY');
     const adminEmail = Deno.env.get('ADMIN_EMAIL_ADDRESS');
     const fromEmail = Deno.env.get('FROM_EMAIL_ADDRESS');
@@ -178,55 +179,43 @@ serve(async (req) => {
       throw new Error('Email configuration is incomplete. Please check environment variables.');
     }
 
-    let emailResponse;
+    const emailResult = await withEdgeRetry(
-    try {
+      async () => {
-      emailResponse = await fetch('https://api.forwardemail.net/v1/emails', {
+        const emailResponse = await fetch('https://api.forwardemail.net/v1/emails', {
-        method: 'POST',
+          method: 'POST',
-        headers: {
+          headers: {
-          'Authorization': 'Basic ' + btoa(forwardEmailApiKey + ':'),
+            'Authorization': 'Basic ' + btoa(forwardEmailApiKey + ':'),
-          'Content-Type': 'application/json',
+            'Content-Type': 'application/json',
-        },
+          },
-        body: JSON.stringify({
+          body: JSON.stringify({
-          from: fromEmail,
+            from: fromEmail,
-          to: adminEmail,
+            to: adminEmail,
-          subject: emailSubject,
+            subject: emailSubject,
-          html: emailHtml,
+            html: emailHtml,
-          text: emailText,
+            text: emailText,
-        }),
+          }),
-      });
+        });
-    } catch (fetchError) {
-      edgeLogger.error('Network error sending email', { 
-        requestId: tracking.requestId,
-        error: fetchError.message
-      });
-      throw new Error('Network error: Unable to reach email service');
-    }
 
-    if (!emailResponse.ok) {
+        if (!emailResponse.ok) {
-      let errorText;
+          let errorText;
-      try {
+          try {
-        errorText = await emailResponse.text();
+            errorText = await emailResponse.text();
-      } catch (parseError) {
+          } catch (parseError) {
-        errorText = 'Unable to parse error response';
+            errorText = 'Unable to parse error response';
-      }
+          }
-      edgeLogger.error('ForwardEmail API error', { 
+          
-        requestId: tracking.requestId,
+          const error = new Error(`Failed to send email: ${emailResponse.status} - ${errorText}`);
-        status: emailResponse.status,
+          (error as any).status = emailResponse.status;
-        errorText
+          throw error;
-      });
+        }
-      throw new Error(`Failed to send email: ${emailResponse.status} - ${errorText}`);
-    }
 
-    let emailResult;
+        const result = await emailResponse.json();
-    try {
+        return result;
-      emailResult = await emailResponse.json();
+      },
-    } catch (parseError) {
+      { maxAttempts: 3, baseDelay: 1500, maxDelay: 10000 },
-      edgeLogger.error('Failed to parse email API response', { 
+      tracking.requestId,
-        requestId: tracking.requestId,
+      'send-escalation-email'
-        error: parseError.message
+    );
-      });
-      throw new Error('Invalid response from email service');
-    }
     edgeLogger.info('Email sent successfully', { 
       requestId: tracking.requestId,
       emailId: emailResult.id

supabase/functions/sync-all-moderators-to-topic/index.ts

@@ -2,6 +2,7 @@ import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { Novu } from "npm:@novu/api@1.6.0";
 import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts';
+import { withEdgeRetry } from '../_shared/retryHelper.ts';
 
 const corsHeaders = {
   'Access-Control-Allow-Origin': '*',
@@ -86,9 +87,17 @@ serve(async (req) => {
         const batch = uniqueUserIds.slice(i, i + batchSize);
         
         try {
-          await novu.topics.addSubscribers(topicKey, {
+          await withEdgeRetry(
-            subscribers: batch,
+            async () => {
-          });
+              await novu.topics.addSubscribers(topicKey, {
+                subscribers: batch,
+              });
+            },
+            { maxAttempts: 3, baseDelay: 2000 },
+            tracking.requestId,
+            `sync-batch-${topicKey}-${i}`
+          );
+          
           successCount += batch.length;
           edgeLogger.info('Added batch of users to topic', { 
             requestId: tracking.requestId,