feat: Implement retry logic and tracking

2025-12-20 11:51:14 -05:00 · 2025-11-05 20:19:43 +00:00
parent 028ea433bb
commit c8018b827e
8 changed files with 361 additions and 139 deletions
--- a/supabase/functions/_shared/retryHelper.ts
+++ b/supabase/functions/_shared/retryHelper.ts
@@ -0,0 +1,142 @@
+/**
+ * Edge Function Retry Helper
+ * Provides exponential backoff retry logic for external API calls
+ */
+
+import { edgeLogger } from './logger.ts';
+
+export interface EdgeRetryOptions {
+  maxAttempts?: number;
+  baseDelay?: number;
+  maxDelay?: number;
+  backoffMultiplier?: number;
+  jitter?: boolean;
+  shouldRetry?: (error: unknown) => boolean;
+}
+
+/**
+ * Determines if an error is transient and should be retried
+ */
+export function isRetryableError(error: unknown): boolean {
+  // Network errors
+  if (error instanceof TypeError && error.message.includes('fetch')) return true;
+  
+  if (error instanceof Error) {
+    const msg = error.message.toLowerCase();
+    if (msg.includes('network') || msg.includes('timeout') || msg.includes('econnrefused')) {
+      return true;
+    }
+  }
+  
+  // HTTP status codes that should be retried
+  if (error && typeof error === 'object') {
+    const httpError = error as { status?: number };
+    
+    // Rate limiting
+    if (httpError.status === 429) return true;
+    
+    // Service unavailable or gateway timeout
+    if (httpError.status === 503 || httpError.status === 504) return true;
+    
+    // Server errors (5xx)
+    if (httpError.status && httpError.status >= 500 && httpError.status < 600) {
+      return true;
+    }
+  }
+  
+  return false;
+}
+
+/**
+ * Calculate exponential backoff delay with optional jitter
+ */
+function calculateBackoffDelay(
+  attempt: number,
+  baseDelay: number,
+  maxDelay: number,
+  backoffMultiplier: number,
+  jitter: boolean
+): number {
+  const exponentialDelay = baseDelay * Math.pow(backoffMultiplier, attempt);
+  const cappedDelay = Math.min(exponentialDelay, maxDelay);
+  
+  if (!jitter) return cappedDelay;
+  
+  // Add random jitter (-30% to +30%)
+  const jitterAmount = cappedDelay * 0.3;
+  const randomJitter = (Math.random() * 2 - 1) * jitterAmount;
+  
+  return Math.max(0, cappedDelay + randomJitter);
+}
+
+/**
+ * Retry wrapper for asynchronous operations with exponential backoff
+ * 
+ * @param fn - Async function to retry
+ * @param options - Retry configuration
+ * @param requestId - Request ID for tracking
+ * @param context - Context description for logging
+ */
+export async function withEdgeRetry<T>(
+  fn: () => Promise<T>,
+  options: EdgeRetryOptions = {},
+  requestId: string,
+  context: string
+): Promise<T> {
+  const maxAttempts = options.maxAttempts ?? 3;
+  const baseDelay = options.baseDelay ?? 1000;
+  const maxDelay = options.maxDelay ?? 10000;
+  const backoffMultiplier = options.backoffMultiplier ?? 2;
+  const jitter = options.jitter ?? true;
+  const shouldRetry = options.shouldRetry ?? isRetryableError;
+  
+  let lastError: unknown;
+  
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      
+      // Don't retry if this is the last attempt
+      if (attempt === maxAttempts - 1) {
+        edgeLogger.error('All retry attempts exhausted', {
+          requestId,
+          context,
+          attempts: maxAttempts,
+          error: error instanceof Error ? error.message : 'Unknown error'
+        });
+        throw error;
+      }
+      
+      // Don't retry if error is not retryable
+      if (!shouldRetry(error)) {
+        edgeLogger.info('Error not retryable, failing immediately', {
+          requestId,
+          context,
+          attempt: attempt + 1,
+          error: error instanceof Error ? error.message : 'Unknown error'
+        });
+        throw error;
+      }
+      
+      // Calculate delay for next retry
+      const delay = calculateBackoffDelay(attempt, baseDelay, maxDelay, backoffMultiplier, jitter);
+      
+      edgeLogger.info('Retrying after error', {
+        requestId,
+        context,
+        attempt: attempt + 1,
+        maxAttempts,
+        delay: Math.round(delay),
+        error: error instanceof Error ? error.message : 'Unknown error'
+      });
+      
+      // Wait before retrying
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+  
+  // This should never be reached, but TypeScript needs it
+  throw lastError;
+}
--- a/supabase/functions/manage-moderator-topic/index.ts
+++ b/supabase/functions/manage-moderator-topic/index.ts
@@ -2,6 +2,7 @@ import { serve } from "https://deno.land/std@0.190.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { Novu } from "npm:@novu/api@1.6.0";
 import { edgeLogger, startRequest, endRequest } from "../_shared/logger.ts";
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -45,21 +46,28 @@ serve(async (req) => {

    for (const topicKey of topics) {
      try {
-        if (action === 'add') {
-          // Add subscriber to topic
-          await novu.topics.addSubscribers(topicKey, {
-            subscribers: [userId],
-          });
-          edgeLogger.info('Added user to topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
-          results.push({ topic: topicKey, action: 'added', success: true });
-        } else {
-          // Remove subscriber from topic
-          await novu.topics.removeSubscribers(topicKey, {
-            subscribers: [userId],
-          });
-          edgeLogger.info('Removed user from topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
-          results.push({ topic: topicKey, action: 'removed', success: true });
-        }
+        await withEdgeRetry(
+          async () => {
+            if (action === 'add') {
+              // Add subscriber to topic
+              await novu.topics.addSubscribers(topicKey, {
+                subscribers: [userId],
+              });
+              edgeLogger.info('Added user to topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
+            } else {
+              // Remove subscriber from topic
+              await novu.topics.removeSubscribers(topicKey, {
+                subscribers: [userId],
+              });
+              edgeLogger.info('Removed user from topic', { action: 'manage_moderator_topic', requestId: tracking.requestId, userId, topicKey });
+            }
+          },
+          { maxAttempts: 3, baseDelay: 1000 },
+          tracking.requestId,
+          `${action}-topic-${topicKey}`
+        );
+        
+        results.push({ topic: topicKey, action: action === 'add' ? 'added' : 'removed', success: true });
      } catch (error: any) {
        edgeLogger.error(`Error ${action}ing user ${userId} ${action === 'add' ? 'to' : 'from'} topic ${topicKey}`, { 
          action: 'manage_moderator_topic', 
--- a/supabase/functions/notify-moderators-report/index.ts
+++ b/supabase/functions/notify-moderators-report/index.ts
@@ -1,6 +1,7 @@
 import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { edgeLogger, startRequest, endRequest } from "../_shared/logger.ts";
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -150,22 +151,32 @@ serve(async (req) => {

    edgeLogger.info('Triggering notification with payload', { action: 'notify_moderators_report', requestId: tracking.requestId });

-    // Invoke the trigger-notification function
-    const { data: result, error: notifyError } = await supabase.functions.invoke(
-      'trigger-notification',
-      {
-        body: {
-          workflowId: template.workflow_id,
-          topicKey: 'moderation-reports',
-          payload: notificationPayload,
-        },
-      }
-    );
+    // Invoke the trigger-notification function with retry
+    const result = await withEdgeRetry(
+      async () => {
+        const { data, error } = await supabase.functions.invoke(
+          'trigger-notification',
+          {
+            body: {
+              workflowId: template.workflow_id,
+              topicKey: 'moderation-reports',
+              payload: notificationPayload,
+            },
+          }
+        );

-    if (notifyError) {
-      edgeLogger.error('Error triggering notification', { action: 'notify_moderators_report', requestId: tracking.requestId, error: notifyError });
-      throw notifyError;
-    }
+        if (error) {
+          const enhancedError = new Error(error.message || 'Notification trigger failed');
+          (enhancedError as any).status = error.status;
+          throw enhancedError;
+        }
+
+        return data;
+      },
+      { maxAttempts: 3, baseDelay: 1000 },
+      tracking.requestId,
+      'trigger-report-notification'
+    );

    edgeLogger.info('Notification triggered successfully', { action: 'notify_moderators_report', requestId: tracking.requestId, result });

--- a/supabase/functions/notify-moderators-submission/index.ts
+++ b/supabase/functions/notify-moderators-submission/index.ts
@@ -1,6 +1,7 @@
 import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts';
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -186,15 +187,30 @@ serve(async (req) => {
      isEscalated: is_escalated,
    };

-    // Send ONE notification to the moderation-submissions topic
+    // Send ONE notification to the moderation-submissions topic with retry
    // All subscribers (moderators) will receive it automatically
-    const { data, error } = await supabase.functions.invoke('trigger-notification', {
-      body: {
-        workflowId: workflow.workflow_id,
-        topicKey: 'moderation-submissions',
-        payload: notificationPayload,
+    const data = await withEdgeRetry(
+      async () => {
+        const { data: result, error } = await supabase.functions.invoke('trigger-notification', {
+          body: {
+            workflowId: workflow.workflow_id,
+            topicKey: 'moderation-submissions',
+            payload: notificationPayload,
+          },
+        });
+
+        if (error) {
+          const enhancedError = new Error(error.message || 'Notification trigger failed');
+          (enhancedError as any).status = error.status;
+          throw enhancedError;
+        }
+
+        return result;
      },
-    });
+      { maxAttempts: 3, baseDelay: 1000 },
+      tracking.requestId,
+      'trigger-submission-notification'
+    );

    // Log notification in notification_logs with idempotency key
    await supabase.from('notification_logs').insert({
@@ -209,32 +225,6 @@ serve(async (req) => {
      }
    });

-    if (error) {
-      const duration = endRequest(tracking);
-      edgeLogger.error('Failed to notify moderators via topic', { 
-        action: 'notify_moderators',
-        requestId: tracking.requestId,
-        duration,
-        error: error.message 
-      });
-      return new Response(
-        JSON.stringify({ 
-          success: false, 
-          error: 'Failed to send notification to topic',
-          details: error.message,
-          requestId: tracking.requestId 
-        }),
-        {
-          headers: { 
-            ...corsHeaders, 
-            'Content-Type': 'application/json',
-            'X-Request-ID': tracking.requestId 
-          },
-          status: 500,
-        }
-      );
-    }
-
    const duration = endRequest(tracking);
    edgeLogger.info('Successfully notified all moderators via topic', { 
      action: 'notify_moderators',
--- a/supabase/functions/send-escalation-notification/index.ts
+++ b/supabase/functions/send-escalation-notification/index.ts
@@ -1,6 +1,7 @@
 import { serve } from "https://deno.land/std@0.190.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts';
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -169,7 +170,7 @@ serve(async (req) => {
      Please review this submission in the admin panel.
    `;

-    // Send email via ForwardEmail API
+    // Send email via ForwardEmail API with retry
    const forwardEmailApiKey = Deno.env.get('FORWARDEMAIL_API_KEY');
    const adminEmail = Deno.env.get('ADMIN_EMAIL_ADDRESS');
    const fromEmail = Deno.env.get('FROM_EMAIL_ADDRESS');
@@ -178,55 +179,43 @@ serve(async (req) => {
      throw new Error('Email configuration is incomplete. Please check environment variables.');
    }

-    let emailResponse;
-    try {
-      emailResponse = await fetch('https://api.forwardemail.net/v1/emails', {
-        method: 'POST',
-        headers: {
-          'Authorization': 'Basic ' + btoa(forwardEmailApiKey + ':'),
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          from: fromEmail,
-          to: adminEmail,
-          subject: emailSubject,
-          html: emailHtml,
-          text: emailText,
-        }),
-      });
-    } catch (fetchError) {
-      edgeLogger.error('Network error sending email', { 
-        requestId: tracking.requestId,
-        error: fetchError.message
-      });
-      throw new Error('Network error: Unable to reach email service');
-    }
+    const emailResult = await withEdgeRetry(
+      async () => {
+        const emailResponse = await fetch('https://api.forwardemail.net/v1/emails', {
+          method: 'POST',
+          headers: {
+            'Authorization': 'Basic ' + btoa(forwardEmailApiKey + ':'),
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({
+            from: fromEmail,
+            to: adminEmail,
+            subject: emailSubject,
+            html: emailHtml,
+            text: emailText,
+          }),
+        });

-    if (!emailResponse.ok) {
-      let errorText;
-      try {
-        errorText = await emailResponse.text();
-      } catch (parseError) {
-        errorText = 'Unable to parse error response';
-      }
-      edgeLogger.error('ForwardEmail API error', { 
-        requestId: tracking.requestId,
-        status: emailResponse.status,
-        errorText
-      });
-      throw new Error(`Failed to send email: ${emailResponse.status} - ${errorText}`);
-    }
+        if (!emailResponse.ok) {
+          let errorText;
+          try {
+            errorText = await emailResponse.text();
+          } catch (parseError) {
+            errorText = 'Unable to parse error response';
+          }
+          
+          const error = new Error(`Failed to send email: ${emailResponse.status} - ${errorText}`);
+          (error as any).status = emailResponse.status;
+          throw error;
+        }

-    let emailResult;
-    try {
-      emailResult = await emailResponse.json();
-    } catch (parseError) {
-      edgeLogger.error('Failed to parse email API response', { 
-        requestId: tracking.requestId,
-        error: parseError.message
-      });
-      throw new Error('Invalid response from email service');
-    }
+        const result = await emailResponse.json();
+        return result;
+      },
+      { maxAttempts: 3, baseDelay: 1500, maxDelay: 10000 },
+      tracking.requestId,
+      'send-escalation-email'
+    );
    edgeLogger.info('Email sent successfully', { 
      requestId: tracking.requestId,
      emailId: emailResult.id
--- a/supabase/functions/sync-all-moderators-to-topic/index.ts
+++ b/supabase/functions/sync-all-moderators-to-topic/index.ts
@@ -2,6 +2,7 @@ import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2.57.4";
 import { Novu } from "npm:@novu/api@1.6.0";
 import { edgeLogger, startRequest, endRequest } from '../_shared/logger.ts';
+import { withEdgeRetry } from '../_shared/retryHelper.ts';

 const corsHeaders = {
  'Access-Control-Allow-Origin': '*',
@@ -86,9 +87,17 @@ serve(async (req) => {
        const batch = uniqueUserIds.slice(i, i + batchSize);
        
        try {
-          await novu.topics.addSubscribers(topicKey, {
-            subscribers: batch,
-          });
+          await withEdgeRetry(
+            async () => {
+              await novu.topics.addSubscribers(topicKey, {
+                subscribers: batch,
+              });
+            },
+            { maxAttempts: 3, baseDelay: 2000 },
+            tracking.requestId,
+            `sync-batch-${topicKey}-${i}`
+          );
+          
          successCount += batch.length;
          edgeLogger.info('Added batch of users to topic', { 
            requestId: tracking.requestId,