ci: add flakiness report upload workflow for fork PRs (#2727)

.github/workflows/ci.yml

@@ -294,6 +294,13 @@ jobs:
           name: blob-report-${{ matrix.os.name }}-shard-${{ matrix.shard }}
           path: blob-report
           retention-days: 1
+      - name: Upload flakiness report (for fork PRs)
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ !cancelled() }}
+        with:
+          name: flakiness-report-${{ matrix.os.name }}-shard-${{ matrix.shard }}
+          path: flakiness-report
+          retention-days: 1
       - name: Cleanup (self-hosted macOS)
         if: always() && contains(matrix.os.image, 'self-hosted')
         run: bash scripts/ci-cleanup-macos.sh

.github/workflows/flakiness-upload.yml

+name: Upload to Flakiness.io
+
+# This workflow uploads Playwright test results to flakiness.io for PRs from forks.
+# Fork PRs don't have access to secrets during CI, so we use workflow_run to upload
+# results after CI completes (workflow_run has access to secrets from the base repo).
+# See: https://flakiness.io/docs/integrations/github-actions/#pull-requests-from-forks
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types: [completed]
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  upload:
+    # Only run for fork PRs (head_repository differs from repository)
+    # Skip if CI was cancelled
+    if: >
+      github.event.workflow_run.conclusion != 'cancelled' &&
+      github.event.workflow_run.head_repository.full_name != github.event.workflow_run.repository.full_name
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      # Download each shard's flakiness report separately to preserve structure
+      - name: Download flakiness reports
+        uses: actions/download-artifact@v4
+        with:
+          path: flakiness-reports
+          pattern: flakiness-report-*
+          github-token: ${{ github.token }}
+          repository: ${{ github.event.workflow_run.repository.full_name }}
+          run-id: ${{ github.event.workflow_run.id }}
+
+      - name: List downloaded reports
+        run: |
+          echo "Downloaded flakiness reports:"
+          ls -laR flakiness-reports/ || echo "No flakiness reports found"
+
+      - name: Upload to Flakiness.io
+        env:
+          FLAKINESS_ACCESS_TOKEN: ${{ secrets.FLAKINESS_ACCESS_TOKEN }}
+        run: |
+          # Install the flakiness SDK
+          npm install @flakiness/sdk
+
+          # Upload each shard's report
+          node --input-type=module << 'EOF'
+          import { readReport, uploadReport } from '@flakiness/sdk';
+          import fs from 'fs';
+          import path from 'path';
+
+          const reportsBaseDir = 'flakiness-reports';
+
+          // Each artifact is downloaded to its own subdirectory
+          // e.g., flakiness-reports/flakiness-report-macos-shard-1/report.json
+          const shardDirs = fs.readdirSync(reportsBaseDir).filter(name =>
+            fs.statSync(path.join(reportsBaseDir, name)).isDirectory()
+          );
+
+          if (shardDirs.length === 0) {
+            console.log('No flakiness report directories found');
+            process.exit(0);
+          }
+
+          console.log(`Found ${shardDirs.length} shard report(s):`, shardDirs);
+
+          for (const shardDir of shardDirs) {
+            const reportDir = path.join(reportsBaseDir, shardDir);
+            const reportJsonPath = path.join(reportDir, 'report.json');
+
+            if (!fs.existsSync(reportJsonPath)) {
+              console.log(`Skipping ${shardDir}: no report.json found`);
+              continue;
+            }
+
+            console.log(`\nUploading flakiness report from: ${reportDir}`);
+            try {
+              const { report, attachments, missingAttachments } = await readReport(reportDir);
+
+              if (missingAttachments.length > 0) {
+                console.warn('Missing attachments:', missingAttachments.map(a => a.id));
+              }
+
+              const result = await uploadReport(report, attachments, {
+                flakinessEndpoint: 'https://flakiness.io'
+              });
+
+              console.log('Upload result:', result);
+            } catch (error) {
+              console.error(`Failed to upload ${shardDir}:`, error.message);
+            }
+          }
+          EOF