feat: Implement Grafana Cloud fixes

.github/workflows/playwright.yml

@@ -29,51 +29,41 @@ jobs:
           fi
           echo "✅ Required secrets validated"
 
-      - name: Check Loki Connection
+      - name: Test Grafana Cloud Loki Connection
-        if: ${{ secrets.GRAFANA_LOKI_URL != '' }}
-        run: |
-          echo "🔍 Testing Loki connection..."
-          if [ -n "${{ secrets.GRAFANA_LOKI_USERNAME }}" ]; then
-            response=$(curl -s -o /dev/null -w "%{http_code}" \
-              -u "${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" \
-              "${{ secrets.GRAFANA_LOKI_URL }}/ready")
-          else
-            response=$(curl -s -o /dev/null -w "%{http_code}" \
-              "${{ secrets.GRAFANA_LOKI_URL }}/ready")
-          fi
-          
-          if [ "$response" = "200" ]; then
-            echo "✅ Loki is ready at ${{ secrets.GRAFANA_LOKI_URL }}"
-          else
-            echo "⚠️  Loki connection check returned HTTP $response"
-            echo "Tests will continue but logs may not be sent to Loki"
-          fi
-
-      - name: Send Pre-flight Event to Loki
         if: ${{ secrets.GRAFANA_LOKI_URL != '' }}
+        continue-on-error: true
         run: |
+          echo "🔍 Testing Grafana Cloud Loki connection..."
           timestamp=$(date +%s)000000000
-          auth_header=""
-          if [ -n "${{ secrets.GRAFANA_LOKI_USERNAME }}" ]; then
-            auth_header="-u ${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}"
-          fi
           
-          curl -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \
+          response=$(curl -s -w "\n%{http_code}" \
-            $auth_header \
+            --max-time 10 \
+            -u "${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" \
             -H "Content-Type: application/json" \
+            -H "User-Agent: ThrillWiki-Playwright-Tests/1.0" \
+            -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \
             -d "{
               \"streams\": [{
                 \"stream\": {
-                  \"job\": \"playwright-preflight\",
+                  \"job\": \"playwright_preflight\",
                   \"workflow\": \"${{ github.workflow }}\",
                   \"branch\": \"${{ github.ref_name }}\",
                   \"commit\": \"${{ github.sha }}\",
-                  \"run_id\": \"${{ github.run_id }}\",
+                  \"run_id\": \"${{ github.run_id }}\"
-                  \"event\": \"preflight_complete\"
                 },
-                \"values\": [[\"$timestamp\", \"Pre-flight checks completed successfully\"]]
+                \"values\": [[\"$timestamp\", \"Preflight check complete\"]]
               }]
-            }" || echo "⚠️  Failed to send pre-flight event to Loki"
+            }")
+          
+          http_code=$(echo "$response" | tail -n1)
+          
+          if [ "$http_code" = "204" ] || [ "$http_code" = "200" ]; then
+            echo "✅ Successfully connected to Grafana Cloud Loki"
+          else
+            echo "⚠️  Loki connection returned HTTP $http_code"
+            echo "Response: $(echo "$response" | head -n -1)"
+            echo "Tests will continue but logs may not be sent to Loki"
+          fi
 
   test:
     needs: preflight
@@ -101,20 +91,22 @@ jobs:
       
       - name: Send Test Start Event to Loki
         if: ${{ secrets.GRAFANA_LOKI_URL != '' }}
+        continue-on-error: true
         run: |
           timestamp=$(date +%s)000000000
-          auth_header=""
-          if [ -n "${{ secrets.GRAFANA_LOKI_USERNAME }}" ]; then
-            auth_header="-u ${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}"
-          fi
           
-          curl -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \
+          response=$(curl -s -w "\n%{http_code}" \
-            $auth_header \
+            --max-time 10 \
+            --retry 3 \
+            --retry-delay 2 \
+            -u "${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" \
             -H "Content-Type: application/json" \
+            -H "User-Agent: ThrillWiki-Playwright-Tests/1.0" \
+            -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \
             -d "{
               \"streams\": [{
                 \"stream\": {
-                  \"job\": \"playwright-tests\",
+                  \"job\": \"playwright_tests\",
                   \"browser\": \"${{ matrix.browser }}\",
                   \"workflow\": \"${{ github.workflow }}\",
                   \"branch\": \"${{ github.ref_name }}\",
@@ -124,7 +116,12 @@ jobs:
                 },
                 \"values\": [[\"$timestamp\", \"Starting Playwright tests for ${{ matrix.browser }}\"]]
               }]
-            }" || echo "⚠️  Failed to send start event to Loki"
+            }")
+          
+          http_code=$(echo "$response" | tail -n1)
+          if [ "$http_code" != "204" ] && [ "$http_code" != "200" ]; then
+            echo "⚠️  Failed to send to Loki (HTTP $http_code): $(echo "$response" | head -n -1)"
+          fi
 
       - name: Run Playwright tests
         id: playwright-run
@@ -172,21 +169,23 @@ jobs:
 
       - name: Send Test Results to Loki
         if: always() && secrets.GRAFANA_LOKI_URL != ''
+        continue-on-error: true
         run: |
-          timestamp=$(date +%s)000000000
           STATUS="${{ steps.playwright-run.outputs.test_exit_code == '0' && 'success' || 'failure' }}"
-          auth_header=""
+          timestamp=$(date +%s)000000000
-          if [ -n "${{ secrets.GRAFANA_LOKI_USERNAME }}" ]; then
-            auth_header="-u ${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}"
-          fi
           
-          curl -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \
+          response=$(curl -s -w "\n%{http_code}" \
-            $auth_header \
+            --max-time 10 \
+            --retry 3 \
+            --retry-delay 2 \
+            -u "${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" \
             -H "Content-Type: application/json" \
+            -H "User-Agent: ThrillWiki-Playwright-Tests/1.0" \
+            -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \
             -d "{
               \"streams\": [{
                 \"stream\": {
-                  \"job\": \"playwright-tests\",
+                  \"job\": \"playwright_tests\",
                   \"browser\": \"${{ matrix.browser }}\",
                   \"workflow\": \"${{ github.workflow }}\",
                   \"branch\": \"${{ github.ref_name }}\",
@@ -197,7 +196,12 @@ jobs:
                 },
                 \"values\": [[\"$timestamp\", \"{\\\"total\\\": ${{ steps.parse-results.outputs.total || 0 }}, \\\"passed\\\": ${{ steps.parse-results.outputs.passed || 0 }}, \\\"failed\\\": ${{ steps.parse-results.outputs.failed || 0 }}, \\\"skipped\\\": ${{ steps.parse-results.outputs.skipped || 0 }}, \\\"duration_ms\\\": ${{ steps.parse-results.outputs.duration || 0 }}}\"]]
               }]
-            }" || echo "⚠️  Failed to send results to Loki"
+            }")
+          
+          http_code=$(echo "$response" | tail -n1)
+          if [ "$http_code" != "204" ] && [ "$http_code" != "200" ]; then
+            echo "⚠️  Failed to send results to Loki (HTTP $http_code): $(echo "$response" | head -n -1)"
+          fi
       
       - name: Upload test results
         uses: actions/upload-artifact@v4

playwright.config.ts

@@ -25,12 +25,15 @@ export default defineConfig({
     ['html'],
     ['list'],
     ['json', { outputFile: 'test-results.json' }],
-    // Grafana Loki reporter for centralized logging
+    // Only include Loki reporter if Grafana Cloud credentials are configured
-    ['./tests/helpers/loki-reporter.ts', {
+    ...(process.env.GRAFANA_LOKI_URL && process.env.GRAFANA_LOKI_USERNAME && process.env.GRAFANA_LOKI_PASSWORD
+      ? [['./tests/helpers/loki-reporter.ts', {
           lokiUrl: process.env.GRAFANA_LOKI_URL,
           username: process.env.GRAFANA_LOKI_USERNAME,
           password: process.env.GRAFANA_LOKI_PASSWORD,
-    }]
+        }] as ['./tests/helpers/loki-reporter.ts', any]]
+      : []
+    )
   ],
   
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */

scripts/test-grafana-cloud.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+# Test Grafana Cloud Loki integration locally
+# Usage: ./scripts/test-grafana-cloud.sh
+
+set -e
+
+echo "🧪 ThrillWiki Grafana Cloud Loki Integration Test"
+echo "=================================================="
+echo ""
+
+# Check required environment variables
+if [ -z "$GRAFANA_LOKI_URL" ]; then
+  echo "❌ GRAFANA_LOKI_URL environment variable is not set"
+  echo ""
+  echo "Please set the following environment variables:"
+  echo "  export GRAFANA_LOKI_URL=\"https://logs-prod-us-central1.grafana.net\""
+  echo "  export GRAFANA_LOKI_USERNAME=\"123456\""
+  echo "  export GRAFANA_LOKI_PASSWORD=\"glc_xxxxxxxxxxxxx\""
+  exit 1
+fi
+
+if [ -z "$GRAFANA_LOKI_USERNAME" ]; then
+  echo "❌ GRAFANA_LOKI_USERNAME environment variable is not set"
+  exit 1
+fi
+
+if [ -z "$GRAFANA_LOKI_PASSWORD" ]; then
+  echo "❌ GRAFANA_LOKI_PASSWORD environment variable is not set"
+  exit 1
+fi
+
+echo "✅ Environment variables configured"
+echo "   Loki URL: $GRAFANA_LOKI_URL"
+echo "   Username: $GRAFANA_LOKI_USERNAME"
+echo ""
+
+# Test connection by sending a test log
+echo "🔍 Testing Grafana Cloud Loki connection..."
+timestamp=$(date +%s)000000000
+
+response=$(curl -s -w "\n%{http_code}" \
+  --max-time 10 \
+  -u "$GRAFANA_LOKI_USERNAME:$GRAFANA_LOKI_PASSWORD" \
+  -H "Content-Type: application/json" \
+  -H "User-Agent: ThrillWiki-Test-Script/1.0" \
+  -X POST "$GRAFANA_LOKI_URL/loki/api/v1/push" \
+  -d "{
+    \"streams\": [{
+      \"stream\": {
+        \"job\": \"test_script\",
+        \"source\": \"local\",
+        \"test_type\": \"connection_test\"
+      },
+      \"values\": [[\"$timestamp\", \"Test log from local machine at $(date)\"]]
+    }]
+  }")
+
+http_code=$(echo "$response" | tail -n1)
+body=$(echo "$response" | head -n -1)
+
+if [ "$http_code" = "204" ] || [ "$http_code" = "200" ]; then
+  echo "✅ Successfully sent test log to Grafana Cloud Loki!"
+  echo ""
+  echo "📊 View your logs in Grafana Cloud:"
+  echo "   1. Go to your Grafana Cloud instance"
+  echo "   2. Navigate to Explore"
+  echo "   3. Select your Loki data source"
+  echo "   4. Run query: {job=\"test_script\"}"
+  echo ""
+else
+  echo "❌ Failed to connect to Grafana Cloud Loki"
+  echo "   HTTP Status: $http_code"
+  if [ -n "$body" ]; then
+    echo "   Response: $body"
+  fi
+  echo ""
+  echo "Common issues:"
+  echo "  - Invalid API key (check GRAFANA_LOKI_PASSWORD)"
+  echo "  - Wrong instance ID (check GRAFANA_LOKI_USERNAME)"
+  echo "  - Incorrect region in URL (check GRAFANA_LOKI_URL)"
+  exit 1
+fi
+
+# Run a sample Playwright test with Loki reporter
+echo "🧪 Running sample Playwright test with Loki reporter..."
+echo ""
+
+if [ -d "tests/e2e/auth" ]; then
+  npx playwright test tests/e2e/auth/login.spec.ts --project=chromium --max-failures=1 || true
+  echo ""
+  echo "✅ Test completed (check above for test results)"
+  echo ""
+  echo "📊 View test logs in Grafana Cloud:"
+  echo "   Query: {job=\"playwright_tests\"}"
+  echo "   Filter by browser: {job=\"playwright_tests\", browser=\"chromium\"}"
+  echo "   Filter by status: {job=\"playwright_tests\", status=\"passed\"}"
+else
+  echo "⚠️  No tests found in tests/e2e/auth/"
+  echo "   Skipping Playwright test execution"
+fi
+
+echo ""
+echo "🎉 Grafana Cloud Loki integration test complete!"

tests/README.md

@@ -235,3 +235,82 @@ For questions or issues with tests, check:
 2. Playwright docs
 3. Test failure screenshots/videos in `test-results/`
 4. GitHub Actions logs for CI failures
+5. Grafana Cloud Loki for centralized test logs
+
+## Grafana Cloud Loki Integration
+
+All test runs automatically send logs to Grafana Cloud Loki for centralized monitoring and analysis.
+
+### Viewing Logs in Grafana Cloud
+
+1. **Access Grafana Cloud**: Go to your Grafana Cloud instance
+2. **Navigate to Explore**: Click "Explore" in the left sidebar
+3. **Select Loki Data Source**: Choose your Loki data source from the dropdown
+4. **Query Test Logs**: Use LogQL queries to filter logs
+
+### Common LogQL Queries
+
+```logql
+# All Playwright test logs
+{job="playwright_tests"}
+
+# Logs for specific browser
+{job="playwright_tests", browser="chromium"}
+
+# Failed tests only
+{job="playwright_tests", status="failed"}
+
+# Tests from specific branch
+{job="playwright_tests", branch="main"}
+
+# Tests from specific GitHub run
+{job="playwright_tests", run_id="1234567890"}
+
+# Logs from specific test file
+{job="playwright_tests"} |= "login.spec.ts"
+
+# Failed tests with error messages
+{job="playwright_tests", status="failed"} | json | line_format "{{.test_name}}: {{.error}}"
+```
+
+### Local Testing with Grafana Cloud
+
+Test your Grafana Cloud integration locally:
+
+```bash
+# Set environment variables
+export GRAFANA_LOKI_URL="https://logs-prod-us-central1.grafana.net"
+export GRAFANA_LOKI_USERNAME="123456"  # Your instance ID
+export GRAFANA_LOKI_PASSWORD="glc_xxxxxxxxxxxxx"  # Your API key
+
+# Run test script
+chmod +x scripts/test-grafana-cloud.sh
+./scripts/test-grafana-cloud.sh
+
+# Run tests with Loki reporter
+npx playwright test tests/e2e/auth/login.spec.ts --project=chromium
+```
+
+### Required GitHub Secrets
+
+For CI/CD integration, ensure these secrets are configured in your GitHub repository:
+
+- `GRAFANA_LOKI_URL` - Your Grafana Cloud Loki endpoint (e.g., `https://logs-prod-us-central1.grafana.net`)
+- `GRAFANA_LOKI_USERNAME` - Your Grafana Cloud instance ID
+- `GRAFANA_LOKI_PASSWORD` - Your Grafana Cloud API key (starts with `glc_`)
+
+### Troubleshooting Grafana Cloud Connection
+
+**401 Unauthorized Error:**
+- Check your `GRAFANA_LOKI_USERNAME` (should be your instance ID)
+- Verify your `GRAFANA_LOKI_PASSWORD` (API key starting with `glc_`)
+- Regenerate API key if needed (Security > API Keys in Grafana Cloud)
+
+**Logs Not Appearing:**
+- Verify the correct region in `GRAFANA_LOKI_URL`
+- Check time range in Grafana Explore (default is last 5 minutes)
+- Run test script to validate connection: `./scripts/test-grafana-cloud.sh`
+
+**429 Rate Limit Error:**
+- Reduce test concurrency in `playwright.config.ts`
+- Increase `flushInterval` in Loki reporter options

tests/helpers/loki-reporter.ts

@@ -44,10 +44,11 @@ export default class LokiReporter implements Reporter {
   private flushTimer?: NodeJS.Timeout;
   private labels: Record<string, string>;
   private testStartTime?: number;
+  private maxRetries: number = 3;
 
   constructor(options: LokiReporterOptions = {}) {
     this.lokiUrl = options.lokiUrl || process.env.GRAFANA_LOKI_URL || 'http://localhost:3100';
-    this.batchSize = options.batchSize || 10;
+    this.batchSize = options.batchSize || 5;
     this.flushInterval = options.flushInterval || 5000;
     
     // Setup basic auth if credentials provided
@@ -57,15 +58,15 @@ export default class LokiReporter implements Reporter {
       this.basicAuth = Buffer.from(`${username}:${password}`).toString('base64');
     }
 
-    // Base labels for all logs
+    // Base labels for all logs - sanitize to ensure Grafana Cloud compatibility
-    this.labels = {
+    this.labels = this.sanitizeLabels({
-      job: 'playwright-tests',
+      job: 'playwright_tests',
       workflow: process.env.GITHUB_WORKFLOW || 'local',
       branch: process.env.GITHUB_REF_NAME || 'local',
       commit: process.env.GITHUB_SHA || 'local',
       run_id: process.env.GITHUB_RUN_ID || 'local',
       ...options.labels,
-    };
+    });
 
     // Setup periodic flush
     this.flushTimer = setInterval(() => this.flush(), this.flushInterval);
@@ -183,6 +184,19 @@ export default class LokiReporter implements Reporter {
     }
   }
 
+  /**
+   * Sanitize label names to match Grafana Cloud requirements [a-zA-Z_][a-zA-Z0-9_]*
+   */
+  private sanitizeLabels(labels: Record<string, string>): Record<string, string> {
+    const sanitized: Record<string, string> = {};
+    for (const [key, value] of Object.entries(labels)) {
+      const sanitizedKey = key.replace(/[^a-zA-Z0-9_]/g, '_').replace(/^[0-9]/, '_$&');
+      const sanitizedValue = String(value).replace(/[\n\r\t]/g, ' ');
+      sanitized[sanitizedKey] = sanitizedValue;
+    }
+    return sanitized;
+  }
+
   /**
    * Log a message to Loki
    */
@@ -190,11 +204,11 @@ export default class LokiReporter implements Reporter {
     const timestamp = Date.now() * 1000000; // Convert to nanoseconds
     
     const stream: LokiStream = {
-      stream: {
+      stream: this.sanitizeLabels({
         ...this.labels,
         ...extraLabels,
         event: data.event || 'log',
-      },
+      }),
       values: [[timestamp.toString(), JSON.stringify(data)]],
     };
 
@@ -207,7 +221,7 @@ export default class LokiReporter implements Reporter {
   }
 
   /**
-   * Flush buffered logs to Loki
+   * Flush buffered logs to Loki with retry logic for Grafana Cloud
    */
   private async flush() {
     if (this.buffer.length === 0) {
@@ -220,9 +234,12 @@ export default class LokiReporter implements Reporter {
 
     this.buffer = [];
 
+    // Retry with exponential backoff
+    for (let attempt = 0; attempt < this.maxRetries; attempt++) {
       try {
         const headers: Record<string, string> = {
           'Content-Type': 'application/json',
+          'User-Agent': 'ThrillWiki-Playwright-Tests/1.0',
         };
 
         if (this.basicAuth) {
@@ -235,17 +252,39 @@ export default class LokiReporter implements Reporter {
           body: JSON.stringify(payload),
         });
 
-      if (!response.ok) {
+        if (response.ok || response.status === 204) {
+          // Success
+          return;
+        }
+
+        if (response.status === 401 || response.status === 403) {
+          console.error(`Loki authentication failed: ${response.status}. Check GRAFANA_LOKI_USERNAME and GRAFANA_LOKI_PASSWORD`);
+          return; // Don't retry auth errors
+        }
+
+        if (response.status === 429) {
+          console.warn(`Loki rate limit hit, retrying... (attempt ${attempt + 1}/${this.maxRetries})`);
+        } else {
           console.error(`Failed to send logs to Loki: ${response.status} ${response.statusText}`);
           const errorText = await response.text();
           console.error(`Response: ${errorText}`);
         }
+
+        // Don't retry on last attempt
+        if (attempt < this.maxRetries - 1) {
+          await new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 1000));
+        }
       } catch (error) {
-      console.error('Error sending logs to Loki:', error);
+        console.error(`Error sending logs to Loki (attempt ${attempt + 1}/${this.maxRetries}):`, error);
-      // Re-add to buffer to retry
+        if (attempt < this.maxRetries - 1) {
+          await new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 1000));
+        } else {
+          // Re-add to buffer on final failure
           this.buffer.push(...payload.streams);
         }
       }
+    }
+  }
 
   /**
    * Get relative path from project root