feat: Implement Grafana Cloud fixes

tests/README.md

@@ -235,3 +235,82 @@ For questions or issues with tests, check:
 2. Playwright docs
 3. Test failure screenshots/videos in `test-results/`
 4. GitHub Actions logs for CI failures
+5. Grafana Cloud Loki for centralized test logs
+
+## Grafana Cloud Loki Integration
+
+All test runs automatically send logs to Grafana Cloud Loki for centralized monitoring and analysis.
+
+### Viewing Logs in Grafana Cloud
+
+1. **Access Grafana Cloud**: Go to your Grafana Cloud instance
+2. **Navigate to Explore**: Click "Explore" in the left sidebar
+3. **Select Loki Data Source**: Choose your Loki data source from the dropdown
+4. **Query Test Logs**: Use LogQL queries to filter logs
+
+### Common LogQL Queries
+
+```logql
+# All Playwright test logs
+{job="playwright_tests"}
+
+# Logs for specific browser
+{job="playwright_tests", browser="chromium"}
+
+# Failed tests only
+{job="playwright_tests", status="failed"}
+
+# Tests from specific branch
+{job="playwright_tests", branch="main"}
+
+# Tests from specific GitHub run
+{job="playwright_tests", run_id="1234567890"}
+
+# Logs from specific test file
+{job="playwright_tests"} |= "login.spec.ts"
+
+# Failed tests with error messages
+{job="playwright_tests", status="failed"} | json | line_format "{{.test_name}}: {{.error}}"
+```
+
+### Local Testing with Grafana Cloud
+
+Test your Grafana Cloud integration locally:
+
+```bash
+# Set environment variables
+export GRAFANA_LOKI_URL="https://logs-prod-us-central1.grafana.net"
+export GRAFANA_LOKI_USERNAME="123456"  # Your instance ID
+export GRAFANA_LOKI_PASSWORD="glc_xxxxxxxxxxxxx"  # Your API key
+
+# Run test script
+chmod +x scripts/test-grafana-cloud.sh
+./scripts/test-grafana-cloud.sh
+
+# Run tests with Loki reporter
+npx playwright test tests/e2e/auth/login.spec.ts --project=chromium
+```
+
+### Required GitHub Secrets
+
+For CI/CD integration, ensure these secrets are configured in your GitHub repository:
+
+- `GRAFANA_LOKI_URL` - Your Grafana Cloud Loki endpoint (e.g., `https://logs-prod-us-central1.grafana.net`)
+- `GRAFANA_LOKI_USERNAME` - Your Grafana Cloud instance ID
+- `GRAFANA_LOKI_PASSWORD` - Your Grafana Cloud API key (starts with `glc_`)
+
+### Troubleshooting Grafana Cloud Connection
+
+**401 Unauthorized Error:**
+- Check your `GRAFANA_LOKI_USERNAME` (should be your instance ID)
+- Verify your `GRAFANA_LOKI_PASSWORD` (API key starting with `glc_`)
+- Regenerate API key if needed (Security > API Keys in Grafana Cloud)
+
+**Logs Not Appearing:**
+- Verify the correct region in `GRAFANA_LOKI_URL`
+- Check time range in Grafana Explore (default is last 5 minutes)
+- Run test script to validate connection: `./scripts/test-grafana-cloud.sh`
+
+**429 Rate Limit Error:**
+- Reduce test concurrency in `playwright.config.ts`
+- Increase `flushInterval` in Loki reporter options

tests/helpers/loki-reporter.ts

@@ -44,10 +44,11 @@ export default class LokiReporter implements Reporter {
   private flushTimer?: NodeJS.Timeout;
   private labels: Record<string, string>;
   private testStartTime?: number;
+  private maxRetries: number = 3;
 
   constructor(options: LokiReporterOptions = {}) {
     this.lokiUrl = options.lokiUrl || process.env.GRAFANA_LOKI_URL || 'http://localhost:3100';
-    this.batchSize = options.batchSize || 10;
+    this.batchSize = options.batchSize || 5;
     this.flushInterval = options.flushInterval || 5000;
     
     // Setup basic auth if credentials provided
@@ -57,15 +58,15 @@ export default class LokiReporter implements Reporter {
       this.basicAuth = Buffer.from(`${username}:${password}`).toString('base64');
     }
 
-    // Base labels for all logs
-    this.labels = {
-      job: 'playwright-tests',
+    // Base labels for all logs - sanitize to ensure Grafana Cloud compatibility
+    this.labels = this.sanitizeLabels({
+      job: 'playwright_tests',
       workflow: process.env.GITHUB_WORKFLOW || 'local',
       branch: process.env.GITHUB_REF_NAME || 'local',
       commit: process.env.GITHUB_SHA || 'local',
       run_id: process.env.GITHUB_RUN_ID || 'local',
       ...options.labels,
-    };
+    });
 
     // Setup periodic flush
     this.flushTimer = setInterval(() => this.flush(), this.flushInterval);
@@ -183,6 +184,19 @@ export default class LokiReporter implements Reporter {
     }
   }
 
+  /**
+   * Sanitize label names to match Grafana Cloud requirements [a-zA-Z_][a-zA-Z0-9_]*
+   */
+  private sanitizeLabels(labels: Record<string, string>): Record<string, string> {
+    const sanitized: Record<string, string> = {};
+    for (const [key, value] of Object.entries(labels)) {
+      const sanitizedKey = key.replace(/[^a-zA-Z0-9_]/g, '_').replace(/^[0-9]/, '_$&');
+      const sanitizedValue = String(value).replace(/[\n\r\t]/g, ' ');
+      sanitized[sanitizedKey] = sanitizedValue;
+    }
+    return sanitized;
+  }
+
   /**
    * Log a message to Loki
    */
@@ -190,11 +204,11 @@ export default class LokiReporter implements Reporter {
     const timestamp = Date.now() * 1000000; // Convert to nanoseconds
     
     const stream: LokiStream = {
-      stream: {
+      stream: this.sanitizeLabels({
         ...this.labels,
         ...extraLabels,
         event: data.event || 'log',
-      },
+      }),
       values: [[timestamp.toString(), JSON.stringify(data)]],
     };
 
@@ -207,7 +221,7 @@ export default class LokiReporter implements Reporter {
   }
 
   /**
-   * Flush buffered logs to Loki
+   * Flush buffered logs to Loki with retry logic for Grafana Cloud
    */
   private async flush() {
     if (this.buffer.length === 0) {
@@ -220,30 +234,55 @@ export default class LokiReporter implements Reporter {
 
     this.buffer = [];
 
-    try {
-      const headers: Record<string, string> = {
-        'Content-Type': 'application/json',
-      };
+    // Retry with exponential backoff
+    for (let attempt = 0; attempt < this.maxRetries; attempt++) {
+      try {
+        const headers: Record<string, string> = {
+          'Content-Type': 'application/json',
+          'User-Agent': 'ThrillWiki-Playwright-Tests/1.0',
+        };
 
-      if (this.basicAuth) {
-        headers['Authorization'] = `Basic ${this.basicAuth}`;
+        if (this.basicAuth) {
+          headers['Authorization'] = `Basic ${this.basicAuth}`;
+        }
+
+        const response = await fetch(`${this.lokiUrl}/loki/api/v1/push`, {
+          method: 'POST',
+          headers,
+          body: JSON.stringify(payload),
+        });
+
+        if (response.ok || response.status === 204) {
+          // Success
+          return;
+        }
+
+        if (response.status === 401 || response.status === 403) {
+          console.error(`Loki authentication failed: ${response.status}. Check GRAFANA_LOKI_USERNAME and GRAFANA_LOKI_PASSWORD`);
+          return; // Don't retry auth errors
+        }
+
+        if (response.status === 429) {
+          console.warn(`Loki rate limit hit, retrying... (attempt ${attempt + 1}/${this.maxRetries})`);
+        } else {
+          console.error(`Failed to send logs to Loki: ${response.status} ${response.statusText}`);
+          const errorText = await response.text();
+          console.error(`Response: ${errorText}`);
+        }
+
+        // Don't retry on last attempt
+        if (attempt < this.maxRetries - 1) {
+          await new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 1000));
+        }
+      } catch (error) {
+        console.error(`Error sending logs to Loki (attempt ${attempt + 1}/${this.maxRetries}):`, error);
+        if (attempt < this.maxRetries - 1) {
+          await new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 1000));
+        } else {
+          // Re-add to buffer on final failure
+          this.buffer.push(...payload.streams);
+        }
       }
-
-      const response = await fetch(`${this.lokiUrl}/loki/api/v1/push`, {
-        method: 'POST',
-        headers,
-        body: JSON.stringify(payload),
-      });
-
-      if (!response.ok) {
-        console.error(`Failed to send logs to Loki: ${response.status} ${response.statusText}`);
-        const errorText = await response.text();
-        console.error(`Response: ${errorText}`);
-      }
-    } catch (error) {
-      console.error('Error sending logs to Loki:', error);
-      // Re-add to buffer to retry
-      this.buffer.push(...payload.streams);
     }
   }