feat: Implement retry logic and tracking

2025-12-20 14:11:13 -05:00 · 2025-11-05 20:19:43 +00:00
parent 028ea433bb
commit c8018b827e
8 changed files with 361 additions and 139 deletions
--- a/src/hooks/moderation/useModerationActions.ts
+++ b/src/hooks/moderation/useModerationActions.ts
@@ -284,15 +284,24 @@ export function useModerationActions(config: ModerationActionsConfig): Moderatio
              }
            }

-            const { data, error, requestId } = await invokeWithTracking(
+            const { data, error, requestId, attempts } = await invokeWithTracking(
              'process-selective-approval',
              {
                itemIds: submissionItems.map((i) => i.id),
                submissionId: item.id,
              },
-              config.user?.id
+              config.user?.id,
+              undefined,
+              undefined,
+              30000, // 30s timeout
+              { maxAttempts: 3, baseDelay: 1500 } // Critical operation - retry config
            );
            
+            // Log if retries were needed
+            if (attempts && attempts > 1) {
+              logger.log(`Approval succeeded after ${attempts} attempts for ${item.id}`);
+            }
+
            if (error) throw error;

            toast({
@@ -620,15 +629,23 @@ export function useModerationActions(config: ModerationActionsConfig): Moderatio

        failedItemsCount = failedItems.length;

-        const { data, error, requestId } = await invokeWithTracking(
+        const { data, error, requestId, attempts } = await invokeWithTracking(
          'process-selective-approval',
          {
            itemIds: failedItems.map((i) => i.id),
            submissionId: item.id,
          },
-          config.user?.id
+          config.user?.id,
+          undefined,
+          undefined,
+          30000,
+          { maxAttempts: 3, baseDelay: 1500 } // Retry for failed items
        );
        
+        if (attempts && attempts > 1) {
+          logger.log(`Retry succeeded after ${attempts} attempts for ${item.id}`);
+        }
+
        if (error) throw error;

        // Log audit trail for retry
@@ -699,17 +716,25 @@ export function useModerationActions(config: ModerationActionsConfig): Moderatio
      onActionStart(item.id);

      try {
-        // Call edge function for email notification
-        const { error: edgeFunctionError, requestId } = await invokeWithTracking(
+        // Call edge function for email notification with retry
+        const { error: edgeFunctionError, requestId, attempts } = await invokeWithTracking(
          'send-escalation-notification',
          {
            submissionId: item.id,
            escalationReason: reason,
            escalatedBy: user.id,
          },
-          user.id
+          user.id,
+          undefined,
+          undefined,
+          45000, // Longer timeout for email sending
+          { maxAttempts: 3, baseDelay: 2000 } // Retry for email delivery
        );
        
+        if (attempts && attempts > 1) {
+          logger.log(`Escalation email sent after ${attempts} attempts`);
+        }
+
        if (edgeFunctionError) {
          // Edge function failed - log and show fallback toast
          handleError(edgeFunctionError, {
--- a/src/lib/edgeFunctionTracking.ts
+++ b/src/lib/edgeFunctionTracking.ts
@@ -8,6 +8,8 @@
 import { supabase } from '@/lib/supabaseClient';
 import { trackRequest } from './requestTracking';
 import { getErrorMessage } from './errorHandler';
+import { withRetry, isRetryableError, type RetryOptions } from './retryHelpers';
+import { breadcrumb } from './errorBreadcrumbs';

 /**
 * Invoke a Supabase edge function with request tracking
@@ -25,11 +27,32 @@ export async function invokeWithTracking<T = any>(
  userId?: string,
  parentRequestId?: string,
  traceId?: string,
-  timeout: number = 30000 // Default 30s timeout
-): Promise<{ data: T | null; error: any; requestId: string; duration: number }> {
-  // Create AbortController for timeout
-  const controller = new AbortController();
-  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  timeout: number = 30000, // Default 30s timeout
+  retryOptions?: Partial<RetryOptions> // NEW: Optional retry configuration
+): Promise<{ data: T | null; error: any; requestId: string; duration: number; attempts?: number }> {
+  // Configure retry options with defaults
+  const effectiveRetryOptions: RetryOptions = {
+    maxAttempts: retryOptions?.maxAttempts ?? 3,
+    baseDelay: retryOptions?.baseDelay ?? 1000,
+    maxDelay: retryOptions?.maxDelay ?? 10000,
+    backoffMultiplier: retryOptions?.backoffMultiplier ?? 2,
+    jitter: true,
+    shouldRetry: isRetryableError,
+    onRetry: (attempt, error, delay) => {
+      // Log retry attempt to breadcrumbs
+      breadcrumb.apiCall(
+        `/functions/${functionName}`,
+        'POST',
+        undefined // status unknown during retry
+      );
+      
+      console.info(`Retrying ${functionName} (attempt ${attempt}) after ${delay}ms:`, 
+        getErrorMessage(error)
+      );
+    },
+  };
+
+  let attemptCount = 0;
  
  try {
    const { result, requestId, duration } = await trackRequest(
@@ -41,22 +64,41 @@ export async function invokeWithTracking<T = any>(
        traceId,
      },
      async (context) => {
-        // Include client request ID in payload for correlation
+        return await withRetry(
+          async () => {
+            attemptCount++;
+            
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), timeout);
+            
+            try {
              const { data, error } = await supabase.functions.invoke<T>(functionName, {
                body: { ...payload, clientRequestId: context.requestId },
-          signal: controller.signal, // Add abort signal for timeout
+                signal: controller.signal,
              });
              
-        if (error) throw error;
+              clearTimeout(timeoutId);
+              
+              if (error) {
+                // Enhance error with status for retry logic
+                const enhancedError = new Error(error.message || 'Edge function error');
+                (enhancedError as any).status = error.status;
+                throw enhancedError;
+              }
+              
              return data;
+            } catch (error) {
+              clearTimeout(timeoutId);
+              throw error;
+            }
+          },
+          effectiveRetryOptions
+        );
      }
    );

-    clearTimeout(timeoutId);
-    return { data: result, error: null, requestId, duration };
+    return { data: result, error: null, requestId, duration, attempts: attemptCount };
  } catch (error: unknown) {
-    clearTimeout(timeoutId);
-    
    // Handle AbortError specifically
    if (error instanceof Error && error.name === 'AbortError') {
      return {
@@ -67,6 +109,7 @@ export async function invokeWithTracking<T = any>(
        },
        requestId: 'timeout',
        duration: timeout,
+        attempts: attemptCount,
      };
    }
    
@@ -76,6 +119,7 @@ export async function invokeWithTracking<T = any>(
      error: { message: errorMessage },
      requestId: 'unknown',
      duration: 0,
+      attempts: attemptCount,
    };
  }
 }
@@ -93,6 +137,7 @@ export async function invokeBatchWithTracking<T = any>(
  operations: Array<{
    functionName: string;
    payload: any;
+    retryOptions?: Partial<RetryOptions>;
  }>,
  userId?: string
 ): Promise<
@@ -102,6 +147,7 @@ export async function invokeBatchWithTracking<T = any>(
    error: any;
    requestId: string;
    duration: number;
+    attempts?: number;
  }>
 > {
  const traceId = crypto.randomUUID();
@@ -113,7 +159,9 @@ export async function invokeBatchWithTracking<T = any>(
        op.payload,
        userId,
        undefined,
-        traceId
+        traceId,
+        30000, // default timeout
+        op.retryOptions // Pass through retry options
      );
      return { functionName: op.functionName, ...result };
    })
--- a/supabase/functions/_shared/retryHelper.ts
+++ b/supabase/functions/_shared/retryHelper.ts
@@ -0,0 +1,142 @@
+/**
+ * Edge Function Retry Helper
+ * Provides exponential backoff retry logic for external API calls
+ */
+
+import { edgeLogger } from './logger.ts';
+
+export interface EdgeRetryOptions {
+  maxAttempts?: number;
+  baseDelay?: number;
+  maxDelay?: number;
+  backoffMultiplier?: number;
+  jitter?: boolean;
+  shouldRetry?: (error: unknown) => boolean;
+}
+
+/**
+ * Determines if an error is transient and should be retried
+ */
+export function isRetryableError(error: unknown): boolean {
+  // Network errors
+  if (error instanceof TypeError && error.message.includes('fetch')) return true;
+  
+  if (error instanceof Error) {
+    const msg = error.message.toLowerCase();
+    if (msg.includes('network') || msg.includes('timeout') || msg.includes('econnrefused')) {
+      return true;
+    }
+  }
+  
+  // HTTP status codes that should be retried
+  if (error && typeof error === 'object') {
+    const httpError = error as { status?: number };
+    
+    // Rate limiting
+    if (httpError.status === 429) return true;
+    
+    // Service unavailable or gateway timeout
+    if (httpError.status === 503 || httpError.status === 504) return true;
+    
+    // Server errors (5xx)
+    if (httpError.status && httpError.status >= 500 && httpError.status < 600) {
+      return true;
+    }
+  }
+  
+  return false;
+}
+
+/**
+ * Calculate exponential backoff delay with optional jitter
+ */
+function calculateBackoffDelay(
+  attempt: number,
+  baseDelay: number,
+  maxDelay: number,
+  backoffMultiplier: number,
+  jitter: boolean
+): number {
+  const exponentialDelay = baseDelay * Math.pow(backoffMultiplier, attempt);
+  const cappedDelay = Math.min(exponentialDelay, maxDelay);
+  
+  if (!jitter) return cappedDelay;
+  
+  // Add random jitter (-30% to +30%)
+  const jitterAmount = cappedDelay * 0.3;
+  const randomJitter = (Math.random() * 2 - 1) * jitterAmount;
+  
+  return Math.max(0, cappedDelay + randomJitter);
+}
+
+/**
+ * Retry wrapper for asynchronous operations with exponential backoff
+ * 
+ * @param fn - Async function to retry
+ * @param options - Retry configuration
+ * @param requestId - Request ID for tracking
+ * @param context - Context description for logging
+ */
+export async function withEdgeRetry<T>(
+  fn: () => Promise<T>,
+  options: EdgeRetryOptions = {},
+  requestId: string,
+  context: string
+): Promise<T> {
+  const maxAttempts = options.maxAttempts ?? 3;
+  const baseDelay = options.baseDelay ?? 1000;
+  const maxDelay = options.maxDelay ?? 10000;
+  const backoffMultiplier = options.backoffMultiplier ?? 2;
+  const jitter = options.jitter ?? true;
+  const shouldRetry = options.shouldRetry ?? isRetryableError;
+  
+  let lastError: unknown;
+  
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      
+      // Don't retry if this is the last attempt
+      if (attempt === maxAttempts - 1) {
+        edgeLogger.error('All retry attempts exhausted', {
+          requestId,
+          context,
+          attempts: maxAttempts,
+          error: error instanceof Error ? error.message : 'Unknown error'
+        });
+        throw error;
+      }
+      
+      // Don't retry if error is not retryable
+      if (!shouldRetry(error)) {
+        edgeLogger.info('Error not retryable, failing immediately', {
+          requestId,
+          context,
+          attempt: attempt + 1,
+          error: error instanceof Error ? error.message : 'Unknown error'
+        });
+        throw error;
+      }
+      
+      // Calculate delay for next retry
+      const delay = calculateBackoffDelay(attempt, baseDelay, maxDelay, backoffMultiplier, jitter);
+      
+      edgeLogger.info('Retrying after error', {
+        requestId,
+        context,
+        attempt: attempt + 1,
+        maxAttempts,
+        delay: Math.round(delay),
+        error: error instanceof Error ? error.message : 'Unknown error'
+      });
+      
+      // Wait before retrying
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+  
+  // This should never be reached, but TypeScript needs it
+  throw lastError;
+}
--- a/supabase/functions/manage-moderator-topic/index.ts
+++ b/supabase/functions/manage-moderator-topic/index.ts
@@ -2,6 +2,7 @@ import { serve } from "https://deno.land/std@0.190.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { Novu } from "npm:@novu/api@1.6.0";
 import { edgeLogger, startRequest, endRequest } from "../_shared/logger.ts";
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -45,21 +46,28 @@ serve(async (req) => {

    for (const topicKey of topics) {
      try {
+        await withEdgeRetry(
+          async () => {
            if (action === 'add') {
              // Add subscriber to topic
              await novu.topics.addSubscribers(topicKey, {
                subscribers: [userId],
              });
              edgeLogger.info('Added user to topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
-          results.push({ topic: topicKey, action: 'added', success: true });
            } else {
              // Remove subscriber from topic
              await novu.topics.removeSubscribers(topicKey, {
                subscribers: [userId],
              });
              edgeLogger.info('Removed user from topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
-          results.push({ topic: topicKey, action: 'removed', success: true });
            }
+          },
+          { maxAttempts: 3, baseDelay: 1000 },
+          tracking.requestId,
+          `${action}-topic-${topicKey}`
+        );
+        
+        results.push({ topic: topicKey, action: action === 'add' ? 'added' : 'removed', success: true });
      } catch (error: any) {
        edgeLogger.error(`Error ${action}ing user ${userId} ${action === 'add' ? 'to' : 'from'} topic ${topicKey}`, { 
          action: 'manage_moderator_topic', 
--- a/supabase/functions/notify-moderators-report/index.ts
+++ b/supabase/functions/notify-moderators-report/index.ts
@@ -1,6 +1,7 @@
 import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { edgeLogger, startRequest, endRequest } from "../_shared/logger.ts";
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -150,8 +151,10 @@ serve(async (req) => {

    edgeLogger.info('Triggering notification with payload', { action: 'notify_moderators_report', requestId: tracking.requestId });

-    // Invoke the trigger-notification function
-    const { data: result, error: notifyError } = await supabase.functions.invoke(
+    // Invoke the trigger-notification function with retry
+    const result = await withEdgeRetry(
+      async () => {
+        const { data, error } = await supabase.functions.invoke(
          'trigger-notification',
          {
            body: {
@@ -162,11 +165,19 @@ serve(async (req) => {
          }
        );

-    if (notifyError) {
-      edgeLogger.error('Error triggering notification', { action: 'notify_moderators_report', requestId: tracking.requestId, error: notifyError });
-      throw notifyError;
+        if (error) {
+          const enhancedError = new Error(error.message || 'Notification trigger failed');
+          (enhancedError as any).status = error.status;
+          throw enhancedError;
        }

+        return data;
+      },
+      { maxAttempts: 3, baseDelay: 1000 },
+      tracking.requestId,
+      'trigger-report-notification'
+    );
+
    edgeLogger.info('Notification triggered successfully', { action: 'notify_moderators_report', requestId: tracking.requestId, result });

    endRequest(tracking, 200);
--- a/supabase/functions/notify-moderators-submission/index.ts
+++ b/supabase/functions/notify-moderators-submission/index.ts
@@ -1,6 +1,7 @@
 import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts';
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -186,9 +187,11 @@ serve(async (req) => {
      isEscalated: is_escalated,
    };

-    // Send ONE notification to the moderation-submissions topic
+    // Send ONE notification to the moderation-submissions topic with retry
    // All subscribers (moderators) will receive it automatically
-    const { data, error } = await supabase.functions.invoke('trigger-notification', {
+    const data = await withEdgeRetry(
+      async () => {
+        const { data: result, error } = await supabase.functions.invoke('trigger-notification', {
          body: {
            workflowId: workflow.workflow_id,
            topicKey: 'moderation-submissions',
@@ -196,6 +199,19 @@ serve(async (req) => {
          },
        });

+        if (error) {
+          const enhancedError = new Error(error.message || 'Notification trigger failed');
+          (enhancedError as any).status = error.status;
+          throw enhancedError;
+        }
+
+        return result;
+      },
+      { maxAttempts: 3, baseDelay: 1000 },
+      tracking.requestId,
+      'trigger-submission-notification'
+    );
+
    // Log notification in notification_logs with idempotency key
    await supabase.from('notification_logs').insert({
      user_id: '00000000-0000-0000-0000-000000000000', // Topic-based
@@ -209,32 +225,6 @@ serve(async (req) => {
      }
    });

-    if (error) {
-      const duration = endRequest(tracking);
-      edgeLogger.error('Failed to notify moderators via topic', { 
-        action: 'notify_moderators',
-        requestId: tracking.requestId,
-        duration,
-        error: error.message 
-      });
-      return new Response(
-        JSON.stringify({ 
-          success: false, 
-          error: 'Failed to send notification to topic',
-          details: error.message,
-          requestId: tracking.requestId 
-        }),
-        {
-          headers: { 
-            ...corsHeaders, 
-            'Content-Type': 'application/json',
-            'X-Request-ID': tracking.requestId 
-          },
-          status: 500,
-        }
-      );
-    }
-
    const duration = endRequest(tracking);
    edgeLogger.info('Successfully notified all moderators via topic', { 
      action: 'notify_moderators',
--- a/supabase/functions/send-escalation-notification/index.ts
+++ b/supabase/functions/send-escalation-notification/index.ts
@@ -1,6 +1,7 @@
 import { serve } from "https://deno.land/std@0.190.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts';
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -169,7 +170,7 @@ serve(async (req) => {
      Please review this submission in the admin panel.
    `;

-    // Send email via ForwardEmail API
+    // Send email via ForwardEmail API with retry
    const forwardEmailApiKey = Deno.env.get('FORWARDEMAIL_API_KEY');
    const adminEmail = Deno.env.get('ADMIN_EMAIL_ADDRESS');
    const fromEmail = Deno.env.get('FROM_EMAIL_ADDRESS');
@@ -178,9 +179,9 @@ serve(async (req) => {
      throw new Error('Email configuration is incomplete. Please check environment variables.');
    }

-    let emailResponse;
-    try {
-      emailResponse = await fetch('https://api.forwardemail.net/v1/emails', {
+    const emailResult = await withEdgeRetry(
+      async () => {
+        const emailResponse = await fetch('https://api.forwardemail.net/v1/emails', {
          method: 'POST',
          headers: {
            'Authorization': 'Basic ' + btoa(forwardEmailApiKey + ':'),
@@ -194,13 +195,6 @@ serve(async (req) => {
            text: emailText,
          }),
        });
-    } catch (fetchError) {
-      edgeLogger.error('Network error sending email', { 
-        requestId: tracking.requestId,
-        error: fetchError.message
-      });
-      throw new Error('Network error: Unable to reach email service');
-    }

        if (!emailResponse.ok) {
          let errorText;
@@ -209,24 +203,19 @@ serve(async (req) => {
          } catch (parseError) {
            errorText = 'Unable to parse error response';
          }
-      edgeLogger.error('ForwardEmail API error', { 
-        requestId: tracking.requestId,
-        status: emailResponse.status,
-        errorText
-      });
-      throw new Error(`Failed to send email: ${emailResponse.status} - ${errorText}`);
+          
+          const error = new Error(`Failed to send email: ${emailResponse.status} - ${errorText}`);
+          (error as any).status = emailResponse.status;
+          throw error;
        }

-    let emailResult;
-    try {
-      emailResult = await emailResponse.json();
-    } catch (parseError) {
-      edgeLogger.error('Failed to parse email API response', { 
-        requestId: tracking.requestId,
-        error: parseError.message
-      });
-      throw new Error('Invalid response from email service');
-    }
+        const result = await emailResponse.json();
+        return result;
+      },
+      { maxAttempts: 3, baseDelay: 1500, maxDelay: 10000 },
+      tracking.requestId,
+      'send-escalation-email'
+    );
    edgeLogger.info('Email sent successfully', { 
      requestId: tracking.requestId,
      emailId: emailResult.id
--- a/supabase/functions/sync-all-moderators-to-topic/index.ts
+++ b/supabase/functions/sync-all-moderators-to-topic/index.ts
@@ -2,6 +2,7 @@ import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { Novu } from "npm:@novu/api@1.6.0";
 import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts';
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -86,9 +87,17 @@ serve(async (req) => {
        const batch = uniqueUserIds.slice(i, i + batchSize);
        
        try {
+          await withEdgeRetry(
+            async () => {
              await novu.topics.addSubscribers(topicKey, {
                subscribers: batch,
              });
+            },
+            { maxAttempts: 3, baseDelay: 2000 },
+            tracking.requestId,
+            `sync-batch-${topicKey}-${i}`
+          );
+          
          successCount += batch.length;
          edgeLogger.info('Added batch of users to topic', { 
            requestId: tracking.requestId,