chore: update Playwright summary upload (#2691)

.github/workflows/ci.yml

@@ -276,6 +276,17 @@ jobs:
         # by adding "DEBUG=pw:browser" in front.
         # Use blob reporter for sharding and merge capabilities
         run: DEBUG=pw:browser npx playwright test --shard=${{ matrix.shard }}/${{ matrix.shardTotal }}
+      - name: Record shard metadata
+        if: ${{ !cancelled() }}
+        run: |
+          mkdir -p blob-report
+          cat <<EOF > "blob-report/_playwright-shard-metadata-${{ matrix.os.name }}-shard-${{ matrix.shard }}-of-${{ matrix.shardTotal }}.json"
+          {
+            "os": "${{ matrix.os.name }}",
+            "shard": ${{ matrix.shard }},
+            "total": ${{ matrix.shardTotal }}
+          }
+          EOF
       - name: Upload shard results
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         if: ${{ !cancelled() }}

scripts/generate-playwright-summary.js

@@ -2,9 +2,7 @@
 // Used by the CI workflow's merge-reports job
 
 const fs = require("fs");
-
-// Expected number of shards per OS
-const EXPECTED_SHARDS_PER_OS = 4;
+const path = require("path");
 
 // Strip ANSI escape codes from terminal output
 function stripAnsi(str) {
@@ -27,29 +25,123 @@ function ensureOsBucket(resultsByOs, os) {
   }
 }
 
-// Check if any shards are missing based on blob report file counts.
-// Blob files are named report-${platform}-${timestamp}.zip (e.g., report-darwin-2024-01-01T12-00-00-000Z.zip)
-// We can't identify WHICH shards are missing, only that some are missing by counting files per OS.
+function collectBlobFiles(dir) {
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  const files = [];
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      files.push(...collectBlobFiles(fullPath));
+    } else {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+function detectOsFromPath(p) {
+  if (p.includes("darwin") || p.includes("macos")) {
+    return "macOS";
+  }
+  if (p.includes("win32") || p.includes("windows")) {
+    return "Windows";
+  }
+  return null;
+}
+
+function normalizeMetadataOs(rawOs) {
+  const os = (rawOs || "").toLowerCase();
+  if (os.includes("darwin") || os.includes("mac") || os.includes("macos")) {
+    return "macOS";
+  }
+  if (os.includes("win32") || os.includes("windows")) {
+    return "Windows";
+  }
+  return null;
+}
+
+function parseShardMetadata(filePath) {
+  const normalizedPath = filePath.toLowerCase();
+  // Example: _playwright-shard-metadata-macos-shard-1-of-4.json
+  const metadataMatch = normalizedPath.match(
+    /_playwright-shard-metadata-([a-z0-9_-]+)-shard-(\d+)-of-(\d+)\.json$/,
+  );
+  if (metadataMatch) {
+    return {
+      os: normalizeMetadataOs(metadataMatch[1]),
+      shard: Number(metadataMatch[2]),
+      total: Number(metadataMatch[3]),
+    };
+  }
+
+  const shardMatch = normalizedPath.match(/shard-(\d+)-of-(\d+)/);
+  if (!shardMatch) return null;
+
+  return {
+    os: detectOsFromPath(normalizedPath),
+    shard: Number(shardMatch[1]),
+    total: Number(shardMatch[2]),
+  };
+}
+
+// Check if any shards are missing based on blob report file metadata.
+// We can't identify WHICH shards are missing, only that some are missing by counting shard IDs per OS.
 function detectMissingShards(blobFiles) {
-  let macosCount = 0;
-  let windowsCount = 0;
+  const shardStateByOs = {
+    macOS: { found: new Set(), expected: 0 },
+    Windows: { found: new Set(), expected: 0 },
+  };
 
   for (const file of blobFiles) {
-    // Blob files are named: report-darwin-*.zip or report-win32-*.zip
-    if (file.includes("darwin")) {
-      macosCount++;
-    } else if (file.includes("win32")) {
-      windowsCount++;
+    const normalizedPath = file.toLowerCase();
+    const os = detectOsFromPath(normalizedPath);
+    if (!os) continue;
+
+    const parsed = parseShardMetadata(file);
+    if (!parsed || !parsed.total || !parsed.os) {
+      const state = shardStateByOs[os];
+      // Fall back to file-counting behavior if metadata isn't available.
+      state.found.add(state.found.size + 1);
+      continue;
+    }
+
+    const state = shardStateByOs[parsed.os];
+    if (!Number.isNaN(parsed.shard) && parsed.shard > 0) {
+      state.found.add(parsed.shard);
+    }
+    if (!Number.isNaN(parsed.total) && parsed.total > state.expected) {
+      state.expected = parsed.total;
+    }
+  }
+
+  for (const state of Object.values(shardStateByOs)) {
+    if (state.expected === 0) {
+      state.expected = state.found.size;
     }
   }
 
-  const macosMissing = Math.max(0, EXPECTED_SHARDS_PER_OS - macosCount);
-  const windowsMissing = Math.max(0, EXPECTED_SHARDS_PER_OS - windowsCount);
+  const macosFound = shardStateByOs.macOS.found.size;
+  const windowsFound = shardStateByOs.Windows.found.size;
+  const macosExpected = shardStateByOs.macOS.expected;
+  const windowsExpected = shardStateByOs.Windows.expected;
+  const macosMissing = Math.max(0, macosExpected - macosFound);
+  const windowsMissing = Math.max(0, windowsExpected - windowsFound);
 
   return {
     counts: {
-      macos: { found: macosCount, missing: macosMissing },
-      windows: { found: windowsCount, missing: windowsMissing },
+      macos: {
+        found: macosFound,
+        expected: macosExpected,
+        missing: macosMissing,
+      },
+      windows: {
+        found: windowsFound,
+        expected: windowsExpected,
+        missing: windowsMissing,
+      },
     },
     hasMissing: macosMissing > 0 || windowsMissing > 0,
   };
@@ -136,9 +228,11 @@ async function run({ github, context, core }) {
 
   // Identify which OS each blob report came from
   const blobDir = "all-blob-reports";
-  const blobFiles = fs.existsSync(blobDir) ? fs.readdirSync(blobDir) : [];
-  const hasMacOS = blobFiles.some((f) => f.includes("darwin"));
-  const hasWindows = blobFiles.some((f) => f.includes("win32"));
+  const blobFiles = fs.existsSync(blobDir) ? collectBlobFiles(blobDir) : [];
+  const hasMacOS = blobFiles.some((f) => detectOsFromPath(f.toLowerCase()));
+  const hasWindows = blobFiles.some(
+    (f) => detectOsFromPath(f.toLowerCase()) === "Windows",
+  );
 
   // Check for missing shards
   const { counts: shardCounts, hasMissing } = detectMissingShards(blobFiles);
@@ -284,10 +378,10 @@ async function run({ github, context, core }) {
     comment +=
       "Some test shards did not report results. This may indicate CI failures or timeouts.\n\n";
     if (shardCounts.macos.missing > 0) {
-      comment += `- 🍎 **macOS**: found ${shardCounts.macos.found}/${EXPECTED_SHARDS_PER_OS} shards (${shardCounts.macos.missing} missing)\n`;
+      comment += `- 🍎 **macOS**: found ${shardCounts.macos.found}/${shardCounts.macos.expected} shards (${shardCounts.macos.missing} missing)\n`;
     }
     if (shardCounts.windows.missing > 0) {
-      comment += `- 🪟 **Windows**: found ${shardCounts.windows.found}/${EXPECTED_SHARDS_PER_OS} shards (${shardCounts.windows.missing} missing)\n`;
+      comment += `- 🪟 **Windows**: found ${shardCounts.windows.found}/${shardCounts.windows.expected} shards (${shardCounts.windows.missing} missing)\n`;
     }
     comment += "\n";
   }