diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 3b289879..9790c6ab 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -29,51 +29,41 @@ jobs: fi echo "โœ… Required secrets validated" - - name: Check Loki Connection - if: ${{ secrets.GRAFANA_LOKI_URL != '' }} - run: | - echo "๐Ÿ” Testing Loki connection..." - if [ -n "${{ secrets.GRAFANA_LOKI_USERNAME }}" ]; then - response=$(curl -s -o /dev/null -w "%{http_code}" \ - -u "${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" \ - "${{ secrets.GRAFANA_LOKI_URL }}/ready") - else - response=$(curl -s -o /dev/null -w "%{http_code}" \ - "${{ secrets.GRAFANA_LOKI_URL }}/ready") - fi - - if [ "$response" = "200" ]; then - echo "โœ… Loki is ready at ${{ secrets.GRAFANA_LOKI_URL }}" - else - echo "โš ๏ธ Loki connection check returned HTTP $response" - echo "Tests will continue but logs may not be sent to Loki" - fi - - - name: Send Pre-flight Event to Loki + - name: Test Grafana Cloud Loki Connection if: ${{ secrets.GRAFANA_LOKI_URL != '' }} + continue-on-error: true run: | + echo "๐Ÿ” Testing Grafana Cloud Loki connection..." timestamp=$(date +%s)000000000 - auth_header="" - if [ -n "${{ secrets.GRAFANA_LOKI_USERNAME }}" ]; then - auth_header="-u ${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" - fi - curl -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \ - $auth_header \ + response=$(curl -s -w "\n%{http_code}" \ + --max-time 10 \ + -u "${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" \ -H "Content-Type: application/json" \ + -H "User-Agent: ThrillWiki-Playwright-Tests/1.0" \ + -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \ -d "{ \"streams\": [{ \"stream\": { - \"job\": \"playwright-preflight\", + \"job\": \"playwright_preflight\", \"workflow\": \"${{ github.workflow }}\", \"branch\": \"${{ github.ref_name }}\", \"commit\": \"${{ github.sha }}\", - \"run_id\": \"${{ github.run_id }}\", - \"event\": \"preflight_complete\" + \"run_id\": \"${{ github.run_id }}\" }, - \"values\": [[\"$timestamp\", \"Pre-flight checks completed successfully\"]] + \"values\": [[\"$timestamp\", \"Preflight check complete\"]] }] - }" || echo "โš ๏ธ Failed to send pre-flight event to Loki" + }") + + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "204" ] || [ "$http_code" = "200" ]; then + echo "โœ… Successfully connected to Grafana Cloud Loki" + else + echo "โš ๏ธ Loki connection returned HTTP $http_code" + echo "Response: $(echo "$response" | head -n -1)" + echo "Tests will continue but logs may not be sent to Loki" + fi test: needs: preflight @@ -101,20 +91,22 @@ jobs: - name: Send Test Start Event to Loki if: ${{ secrets.GRAFANA_LOKI_URL != '' }} + continue-on-error: true run: | timestamp=$(date +%s)000000000 - auth_header="" - if [ -n "${{ secrets.GRAFANA_LOKI_USERNAME }}" ]; then - auth_header="-u ${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" - fi - curl -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \ - $auth_header \ + response=$(curl -s -w "\n%{http_code}" \ + --max-time 10 \ + --retry 3 \ + --retry-delay 2 \ + -u "${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" \ -H "Content-Type: application/json" \ + -H "User-Agent: ThrillWiki-Playwright-Tests/1.0" \ + -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \ -d "{ \"streams\": [{ \"stream\": { - \"job\": \"playwright-tests\", + \"job\": \"playwright_tests\", \"browser\": \"${{ matrix.browser }}\", \"workflow\": \"${{ github.workflow }}\", \"branch\": \"${{ github.ref_name }}\", @@ -124,7 +116,12 @@ jobs: }, \"values\": [[\"$timestamp\", \"Starting Playwright tests for ${{ matrix.browser }}\"]] }] - }" || echo "โš ๏ธ Failed to send start event to Loki" + }") + + http_code=$(echo "$response" | tail -n1) + if [ "$http_code" != "204" ] && [ "$http_code" != "200" ]; then + echo "โš ๏ธ Failed to send to Loki (HTTP $http_code): $(echo "$response" | head -n -1)" + fi - name: Run Playwright tests id: playwright-run @@ -172,21 +169,23 @@ jobs: - name: Send Test Results to Loki if: always() && secrets.GRAFANA_LOKI_URL != '' + continue-on-error: true run: | - timestamp=$(date +%s)000000000 STATUS="${{ steps.playwright-run.outputs.test_exit_code == '0' && 'success' || 'failure' }}" - auth_header="" - if [ -n "${{ secrets.GRAFANA_LOKI_USERNAME }}" ]; then - auth_header="-u ${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" - fi + timestamp=$(date +%s)000000000 - curl -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \ - $auth_header \ + response=$(curl -s -w "\n%{http_code}" \ + --max-time 10 \ + --retry 3 \ + --retry-delay 2 \ + -u "${{ secrets.GRAFANA_LOKI_USERNAME }}:${{ secrets.GRAFANA_LOKI_PASSWORD }}" \ -H "Content-Type: application/json" \ + -H "User-Agent: ThrillWiki-Playwright-Tests/1.0" \ + -X POST "${{ secrets.GRAFANA_LOKI_URL }}/loki/api/v1/push" \ -d "{ \"streams\": [{ \"stream\": { - \"job\": \"playwright-tests\", + \"job\": \"playwright_tests\", \"browser\": \"${{ matrix.browser }}\", \"workflow\": \"${{ github.workflow }}\", \"branch\": \"${{ github.ref_name }}\", @@ -197,7 +196,12 @@ jobs: }, \"values\": [[\"$timestamp\", \"{\\\"total\\\": ${{ steps.parse-results.outputs.total || 0 }}, \\\"passed\\\": ${{ steps.parse-results.outputs.passed || 0 }}, \\\"failed\\\": ${{ steps.parse-results.outputs.failed || 0 }}, \\\"skipped\\\": ${{ steps.parse-results.outputs.skipped || 0 }}, \\\"duration_ms\\\": ${{ steps.parse-results.outputs.duration || 0 }}}\"]] }] - }" || echo "โš ๏ธ Failed to send results to Loki" + }") + + http_code=$(echo "$response" | tail -n1) + if [ "$http_code" != "204" ] && [ "$http_code" != "200" ]; then + echo "โš ๏ธ Failed to send results to Loki (HTTP $http_code): $(echo "$response" | head -n -1)" + fi - name: Upload test results uses: actions/upload-artifact@v4 diff --git a/playwright.config.ts b/playwright.config.ts index f8c2a8c0..d8b024c0 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -25,12 +25,15 @@ export default defineConfig({ ['html'], ['list'], ['json', { outputFile: 'test-results.json' }], - // Grafana Loki reporter for centralized logging - ['./tests/helpers/loki-reporter.ts', { - lokiUrl: process.env.GRAFANA_LOKI_URL, - username: process.env.GRAFANA_LOKI_USERNAME, - password: process.env.GRAFANA_LOKI_PASSWORD, - }] + // Only include Loki reporter if Grafana Cloud credentials are configured + ...(process.env.GRAFANA_LOKI_URL && process.env.GRAFANA_LOKI_USERNAME && process.env.GRAFANA_LOKI_PASSWORD + ? [['./tests/helpers/loki-reporter.ts', { + lokiUrl: process.env.GRAFANA_LOKI_URL, + username: process.env.GRAFANA_LOKI_USERNAME, + password: process.env.GRAFANA_LOKI_PASSWORD, + }] as ['./tests/helpers/loki-reporter.ts', any]] + : [] + ) ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ diff --git a/scripts/test-grafana-cloud.sh b/scripts/test-grafana-cloud.sh new file mode 100644 index 00000000..5fc30fd4 --- /dev/null +++ b/scripts/test-grafana-cloud.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Test Grafana Cloud Loki integration locally +# Usage: ./scripts/test-grafana-cloud.sh + +set -e + +echo "๐Ÿงช ThrillWiki Grafana Cloud Loki Integration Test" +echo "==================================================" +echo "" + +# Check required environment variables +if [ -z "$GRAFANA_LOKI_URL" ]; then + echo "โŒ GRAFANA_LOKI_URL environment variable is not set" + echo "" + echo "Please set the following environment variables:" + echo " export GRAFANA_LOKI_URL=\"https://logs-prod-us-central1.grafana.net\"" + echo " export GRAFANA_LOKI_USERNAME=\"123456\"" + echo " export GRAFANA_LOKI_PASSWORD=\"glc_xxxxxxxxxxxxx\"" + exit 1 +fi + +if [ -z "$GRAFANA_LOKI_USERNAME" ]; then + echo "โŒ GRAFANA_LOKI_USERNAME environment variable is not set" + exit 1 +fi + +if [ -z "$GRAFANA_LOKI_PASSWORD" ]; then + echo "โŒ GRAFANA_LOKI_PASSWORD environment variable is not set" + exit 1 +fi + +echo "โœ… Environment variables configured" +echo " Loki URL: $GRAFANA_LOKI_URL" +echo " Username: $GRAFANA_LOKI_USERNAME" +echo "" + +# Test connection by sending a test log +echo "๐Ÿ” Testing Grafana Cloud Loki connection..." +timestamp=$(date +%s)000000000 + +response=$(curl -s -w "\n%{http_code}" \ + --max-time 10 \ + -u "$GRAFANA_LOKI_USERNAME:$GRAFANA_LOKI_PASSWORD" \ + -H "Content-Type: application/json" \ + -H "User-Agent: ThrillWiki-Test-Script/1.0" \ + -X POST "$GRAFANA_LOKI_URL/loki/api/v1/push" \ + -d "{ + \"streams\": [{ + \"stream\": { + \"job\": \"test_script\", + \"source\": \"local\", + \"test_type\": \"connection_test\" + }, + \"values\": [[\"$timestamp\", \"Test log from local machine at $(date)\"]] + }] + }") + +http_code=$(echo "$response" | tail -n1) +body=$(echo "$response" | head -n -1) + +if [ "$http_code" = "204" ] || [ "$http_code" = "200" ]; then + echo "โœ… Successfully sent test log to Grafana Cloud Loki!" + echo "" + echo "๐Ÿ“Š View your logs in Grafana Cloud:" + echo " 1. Go to your Grafana Cloud instance" + echo " 2. Navigate to Explore" + echo " 3. Select your Loki data source" + echo " 4. Run query: {job=\"test_script\"}" + echo "" +else + echo "โŒ Failed to connect to Grafana Cloud Loki" + echo " HTTP Status: $http_code" + if [ -n "$body" ]; then + echo " Response: $body" + fi + echo "" + echo "Common issues:" + echo " - Invalid API key (check GRAFANA_LOKI_PASSWORD)" + echo " - Wrong instance ID (check GRAFANA_LOKI_USERNAME)" + echo " - Incorrect region in URL (check GRAFANA_LOKI_URL)" + exit 1 +fi + +# Run a sample Playwright test with Loki reporter +echo "๐Ÿงช Running sample Playwright test with Loki reporter..." +echo "" + +if [ -d "tests/e2e/auth" ]; then + npx playwright test tests/e2e/auth/login.spec.ts --project=chromium --max-failures=1 || true + echo "" + echo "โœ… Test completed (check above for test results)" + echo "" + echo "๐Ÿ“Š View test logs in Grafana Cloud:" + echo " Query: {job=\"playwright_tests\"}" + echo " Filter by browser: {job=\"playwright_tests\", browser=\"chromium\"}" + echo " Filter by status: {job=\"playwright_tests\", status=\"passed\"}" +else + echo "โš ๏ธ No tests found in tests/e2e/auth/" + echo " Skipping Playwright test execution" +fi + +echo "" +echo "๐ŸŽ‰ Grafana Cloud Loki integration test complete!" diff --git a/tests/README.md b/tests/README.md index 64d35a02..5f037c93 100644 --- a/tests/README.md +++ b/tests/README.md @@ -235,3 +235,82 @@ For questions or issues with tests, check: 2. Playwright docs 3. Test failure screenshots/videos in `test-results/` 4. GitHub Actions logs for CI failures +5. Grafana Cloud Loki for centralized test logs + +## Grafana Cloud Loki Integration + +All test runs automatically send logs to Grafana Cloud Loki for centralized monitoring and analysis. + +### Viewing Logs in Grafana Cloud + +1. **Access Grafana Cloud**: Go to your Grafana Cloud instance +2. **Navigate to Explore**: Click "Explore" in the left sidebar +3. **Select Loki Data Source**: Choose your Loki data source from the dropdown +4. **Query Test Logs**: Use LogQL queries to filter logs + +### Common LogQL Queries + +```logql +# All Playwright test logs +{job="playwright_tests"} + +# Logs for specific browser +{job="playwright_tests", browser="chromium"} + +# Failed tests only +{job="playwright_tests", status="failed"} + +# Tests from specific branch +{job="playwright_tests", branch="main"} + +# Tests from specific GitHub run +{job="playwright_tests", run_id="1234567890"} + +# Logs from specific test file +{job="playwright_tests"} |= "login.spec.ts" + +# Failed tests with error messages +{job="playwright_tests", status="failed"} | json | line_format "{{.test_name}}: {{.error}}" +``` + +### Local Testing with Grafana Cloud + +Test your Grafana Cloud integration locally: + +```bash +# Set environment variables +export GRAFANA_LOKI_URL="https://logs-prod-us-central1.grafana.net" +export GRAFANA_LOKI_USERNAME="123456" # Your instance ID +export GRAFANA_LOKI_PASSWORD="glc_xxxxxxxxxxxxx" # Your API key + +# Run test script +chmod +x scripts/test-grafana-cloud.sh +./scripts/test-grafana-cloud.sh + +# Run tests with Loki reporter +npx playwright test tests/e2e/auth/login.spec.ts --project=chromium +``` + +### Required GitHub Secrets + +For CI/CD integration, ensure these secrets are configured in your GitHub repository: + +- `GRAFANA_LOKI_URL` - Your Grafana Cloud Loki endpoint (e.g., `https://logs-prod-us-central1.grafana.net`) +- `GRAFANA_LOKI_USERNAME` - Your Grafana Cloud instance ID +- `GRAFANA_LOKI_PASSWORD` - Your Grafana Cloud API key (starts with `glc_`) + +### Troubleshooting Grafana Cloud Connection + +**401 Unauthorized Error:** +- Check your `GRAFANA_LOKI_USERNAME` (should be your instance ID) +- Verify your `GRAFANA_LOKI_PASSWORD` (API key starting with `glc_`) +- Regenerate API key if needed (Security > API Keys in Grafana Cloud) + +**Logs Not Appearing:** +- Verify the correct region in `GRAFANA_LOKI_URL` +- Check time range in Grafana Explore (default is last 5 minutes) +- Run test script to validate connection: `./scripts/test-grafana-cloud.sh` + +**429 Rate Limit Error:** +- Reduce test concurrency in `playwright.config.ts` +- Increase `flushInterval` in Loki reporter options diff --git a/tests/helpers/loki-reporter.ts b/tests/helpers/loki-reporter.ts index 2a1347fb..6c23e5b8 100644 --- a/tests/helpers/loki-reporter.ts +++ b/tests/helpers/loki-reporter.ts @@ -44,10 +44,11 @@ export default class LokiReporter implements Reporter { private flushTimer?: NodeJS.Timeout; private labels: Record; private testStartTime?: number; + private maxRetries: number = 3; constructor(options: LokiReporterOptions = {}) { this.lokiUrl = options.lokiUrl || process.env.GRAFANA_LOKI_URL || 'http://localhost:3100'; - this.batchSize = options.batchSize || 10; + this.batchSize = options.batchSize || 5; this.flushInterval = options.flushInterval || 5000; // Setup basic auth if credentials provided @@ -57,15 +58,15 @@ export default class LokiReporter implements Reporter { this.basicAuth = Buffer.from(`${username}:${password}`).toString('base64'); } - // Base labels for all logs - this.labels = { - job: 'playwright-tests', + // Base labels for all logs - sanitize to ensure Grafana Cloud compatibility + this.labels = this.sanitizeLabels({ + job: 'playwright_tests', workflow: process.env.GITHUB_WORKFLOW || 'local', branch: process.env.GITHUB_REF_NAME || 'local', commit: process.env.GITHUB_SHA || 'local', run_id: process.env.GITHUB_RUN_ID || 'local', ...options.labels, - }; + }); // Setup periodic flush this.flushTimer = setInterval(() => this.flush(), this.flushInterval); @@ -183,6 +184,19 @@ export default class LokiReporter implements Reporter { } } + /** + * Sanitize label names to match Grafana Cloud requirements [a-zA-Z_][a-zA-Z0-9_]* + */ + private sanitizeLabels(labels: Record): Record { + const sanitized: Record = {}; + for (const [key, value] of Object.entries(labels)) { + const sanitizedKey = key.replace(/[^a-zA-Z0-9_]/g, '_').replace(/^[0-9]/, '_$&'); + const sanitizedValue = String(value).replace(/[\n\r\t]/g, ' '); + sanitized[sanitizedKey] = sanitizedValue; + } + return sanitized; + } + /** * Log a message to Loki */ @@ -190,11 +204,11 @@ export default class LokiReporter implements Reporter { const timestamp = Date.now() * 1000000; // Convert to nanoseconds const stream: LokiStream = { - stream: { + stream: this.sanitizeLabels({ ...this.labels, ...extraLabels, event: data.event || 'log', - }, + }), values: [[timestamp.toString(), JSON.stringify(data)]], }; @@ -207,7 +221,7 @@ export default class LokiReporter implements Reporter { } /** - * Flush buffered logs to Loki + * Flush buffered logs to Loki with retry logic for Grafana Cloud */ private async flush() { if (this.buffer.length === 0) { @@ -220,30 +234,55 @@ export default class LokiReporter implements Reporter { this.buffer = []; - try { - const headers: Record = { - 'Content-Type': 'application/json', - }; + // Retry with exponential backoff + for (let attempt = 0; attempt < this.maxRetries; attempt++) { + try { + const headers: Record = { + 'Content-Type': 'application/json', + 'User-Agent': 'ThrillWiki-Playwright-Tests/1.0', + }; - if (this.basicAuth) { - headers['Authorization'] = `Basic ${this.basicAuth}`; + if (this.basicAuth) { + headers['Authorization'] = `Basic ${this.basicAuth}`; + } + + const response = await fetch(`${this.lokiUrl}/loki/api/v1/push`, { + method: 'POST', + headers, + body: JSON.stringify(payload), + }); + + if (response.ok || response.status === 204) { + // Success + return; + } + + if (response.status === 401 || response.status === 403) { + console.error(`Loki authentication failed: ${response.status}. Check GRAFANA_LOKI_USERNAME and GRAFANA_LOKI_PASSWORD`); + return; // Don't retry auth errors + } + + if (response.status === 429) { + console.warn(`Loki rate limit hit, retrying... (attempt ${attempt + 1}/${this.maxRetries})`); + } else { + console.error(`Failed to send logs to Loki: ${response.status} ${response.statusText}`); + const errorText = await response.text(); + console.error(`Response: ${errorText}`); + } + + // Don't retry on last attempt + if (attempt < this.maxRetries - 1) { + await new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 1000)); + } + } catch (error) { + console.error(`Error sending logs to Loki (attempt ${attempt + 1}/${this.maxRetries}):`, error); + if (attempt < this.maxRetries - 1) { + await new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 1000)); + } else { + // Re-add to buffer on final failure + this.buffer.push(...payload.streams); + } } - - const response = await fetch(`${this.lokiUrl}/loki/api/v1/push`, { - method: 'POST', - headers, - body: JSON.stringify(payload), - }); - - if (!response.ok) { - console.error(`Failed to send logs to Loki: ${response.status} ${response.statusText}`); - const errorText = await response.text(); - console.error(`Response: ${errorText}`); - } - } catch (error) { - console.error('Error sending logs to Loki:', error); - // Re-add to buffer to retry - this.buffer.push(...payload.streams); } }