Simple playwright e2e test (#234)

644012c9 · Will Chen · GitHub · 9d046834 · 644012c9 · 644012c9
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,7 +17,11 @@ defaults:
 jobs:
  test:
-    runs-on: ubuntu-latest
+    # Why Mac?
+    # I can't run electron playwright on ubuntu-latest and
+    # Linux support for Dyad is experimental so not as important
+    # as Mac + Windows
+    runs-on: macos-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
@@ -33,3 +37,16 @@ jobs:
        run: npm run presubmit
      - name: Type-checking
        run: npm run ts
+      - name: Install Chromium browser for Playwright
+        run: npx playwright install chromium --with-deps
+      - name: Build
+        run: npm run pre:e2e
+      - name: E2E tests
+        # Add debug logging to make it easier to see what's failing
+        run: DEBUG=pw:browser npm run e2e
+      - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        if: failure()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 3
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,10 @@ yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
+# playwright
+playwright-report/
+test-results/
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json

--- a/e2e-tests/main.spec.ts
+++ b/e2e-tests/main.spec.ts
+/**
+ * Example Playwright script for Electron
+ * showing/testing various API features
+ * in both renderer and main processes
+ */
+import { expect, test as base } from "@playwright/test";
+import { findLatestBuild, parseElectronApp } from "electron-playwright-helpers";
+import { ElectronApplication, Page, _electron as electron } from "playwright";
+let electronApp: ElectronApplication;
+// From https://github.com/microsoft/playwright/issues/8208#issuecomment-1435475930
+//
+// Note how we mark the fixture as { auto: true }.
+// This way it is always instantiated, even if the test does not use it explicitly.
+export const test = base.extend<{ attachScreenshotsToReport: void }>({
+  attachScreenshotsToReport: [
+    async ({}, use, testInfo) => {
+      await use();
+      // After the test we can check whether the test passed or failed.
+      if (testInfo.status !== testInfo.expectedStatus) {
+        const screenshot = await page.screenshot();
+        await testInfo.attach("screenshot", {
+          body: screenshot,
+          contentType: "image/png",
+        });
+      }
+    },
+    { auto: true },
+  ],
+});
+test.beforeAll(async () => {
+  // find the latest build in the out directory
+  const latestBuild = findLatestBuild();
+  // parse the directory and find paths and other info
+  const appInfo = parseElectronApp(latestBuild);
+  process.env.E2E_TEST_BUILD = "true";
+  // This is just a hack to avoid the AI setup screen.
+  process.env.OPENAI_API_KEY = "sk-test";
+  electronApp = await electron.launch({
+    args: [
+      appInfo.main,
+      "--enable-logging",
+      "--user-data-dir=/tmp/dyad-e2e-tests",
+    ],
+    executablePath: appInfo.executable,
+  });
+  console.log("electronApp launched!");
+  // Listen to main process output immediately
+  electronApp.process().stdout?.on("data", (data) => {
+    console.log(`MAIN_PROCESS_STDOUT: ${data.toString()}`);
+  });
+  electronApp.process().stderr?.on("data", (data) => {
+    console.error(`MAIN_PROCESS_STDERR: ${data.toString()}`);
+  });
+  electronApp.on("close", () => {
+    console.log(`Electron app closed listener:`);
+  });
+  electronApp.on("window", async (page) => {
+    const filename = page.url()?.split("/").pop();
+    console.log(`Window opened: ${filename}`);
+    // capture errors
+    page.on("pageerror", (error) => {
+      console.error(error);
+    });
+    // capture console messages
+    page.on("console", (msg) => {
+      console.log(msg.text());
+    });
+  });
+});
+test.afterAll(async () => {
+  await electronApp.close();
+});
+let page: Page;
+test("renders the first page", async () => {
+  page = await electronApp.firstWindow();
+  await page.waitForSelector("h1");
+  const text = await page.$eval("h1", (el) => el.textContent);
+  expect(text).toBe("Build your dream app");
+});
--- a/forge.config.ts
+++ b/forge.config.ts
@@ -41,6 +41,8 @@ const ignore = (file: string) => {
  return true;
 };
+const isEndToEndTestBuild = process.env.E2E_TEST_BUILD === "true";
 const config: ForgeConfig = {
  packagerConfig: {
    protocols: [
@@ -51,10 +53,14 @@ const config: ForgeConfig = {
    ],
    icon: "./assets/icon/logo",
-    osxSign: {
+    osxSign: isEndToEndTestBuild
+      ? undefined
+      : {
          identity: process.env.APPLE_TEAM_ID,
        },
-    osxNotarize: {
+    osxNotarize: isEndToEndTestBuild
+      ? undefined
+      : {
          appleId: process.env.APPLE_ID!,
          appleIdPassword: process.env.APPLE_PASSWORD!,
          teamId: process.env.APPLE_TEAM_ID!,
@@ -124,7 +130,7 @@ const config: ForgeConfig = {
      [FuseV1Options.RunAsNode]: false,
      [FuseV1Options.EnableCookieEncryption]: true,
      [FuseV1Options.EnableNodeOptionsEnvironmentVariable]: false,
-      [FuseV1Options.EnableNodeCliInspectArguments]: false,
+      [FuseV1Options.EnableNodeCliInspectArguments]: isEndToEndTestBuild,
      [FuseV1Options.EnableEmbeddedAsarIntegrityValidation]: true,
      [FuseV1Options.OnlyLoadAppFromAsar]: true,
    }),

--- a/package-lock.json
+++ b/package-lock.json
 {
  "name": "dyad",
-  "version": "0.5.0",
+  "version": "0.6.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "dyad",
-      "version": "0.5.0",
+      "version": "0.6.0",
      "license": "MIT",
      "dependencies": {
        "@ai-sdk/anthropic": "^1.2.8",
@@ -45,6 +45,7 @@
        "dotenv": "^16.4.7",
        "drizzle-orm": "^0.41.0",
        "electron-log": "^5.3.3",
+        "electron-playwright-helpers": "^1.7.1",
        "electron-squirrel-startup": "^1.0.1",
        "esbuild-register": "^3.6.0",
        "fix-path": "^4.0.0",
@@ -85,6 +86,7 @@
        "@electron-forge/plugin-vite": "^7.8.0",
        "@electron-forge/publisher-github": "^7.8.0",
        "@electron/fuses": "^1.8.0",
+        "@playwright/test": "^1.52.0",
        "@testing-library/react": "^16.3.0",
        "@types/better-sqlite3": "^7.6.13",
        "@types/kill-port": "^2.0.3",
@@ -1140,7 +1142,6 @@
      "version": "3.4.1",
      "resolved": "https://registry.npmjs.org/@electron/asar/-/asar-3.4.1.tgz",
      "integrity": "sha512-i4/rNPRS84t0vSRa2HorerGRXWyF4vThfHesw0dmcWHp+cspK743UanA0suA5Q5y8kzY2y6YKrvbIUn69BCAiA==",
-      "dev": true,
      "license": "MIT",
      "dependencies": {
        "commander": "^5.0.0",
@@ -1158,7 +1159,6 @@
      "version": "5.1.0",
      "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",
      "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==",
-      "dev": true,
      "license": "MIT",
      "engines": {
        "node": ">= 6"
@@ -3726,6 +3726,22 @@
      "devOptional": true,
      "license": "MIT"
    },
+    "node_modules/@playwright/test": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.52.0.tgz",
+      "integrity": "sha512-uh6W7sb55hl7D6vsAeA+V2p5JnlAqzhqFyF0VcJkKZXkgnFcVG9PziERRHQfPLfNGx1C292a4JqbWzhR8L4R1g==",
+      "devOptional": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.52.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
    "node_modules/@polka/url": {
      "version": "1.0.0-next.29",
      "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz",
@@ -6925,7 +6941,6 @@
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
-      "dev": true,
      "license": "MIT"
    },
    "node_modules/base64-js": {
@@ -7055,7 +7070,6 @@
      "version": "1.1.11",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-      "dev": true,
      "license": "MIT",
      "dependencies": {
        "balanced-match": "^1.0.0",
@@ -7781,7 +7795,6 @@
      "version": "0.0.1",
      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
-      "dev": true,
      "license": "MIT"
    },
    "node_modules/content-disposition": {
@@ -9387,6 +9400,15 @@
        "node": ">= 14"
      }
    },
+    "node_modules/electron-playwright-helpers": {
+      "version": "1.7.1",
+      "resolved": "https://registry.npmjs.org/electron-playwright-helpers/-/electron-playwright-helpers-1.7.1.tgz",
+      "integrity": "sha512-S9mo7LfpERgub2WIuYVPpib4XKFeAqBP+mxYf5Bv7E0B5GUB+LUbSj6Fpu39h18Ar635Nf9nQYTmypjuvaYJng==",
+      "license": "MIT",
+      "dependencies": {
+        "@electron/asar": "^3.2.4"
+      }
+    },
    "node_modules/electron-squirrel-startup": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/electron-squirrel-startup/-/electron-squirrel-startup-1.0.1.tgz",
@@ -10894,7 +10916,6 @@
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
-      "dev": true,
      "license": "ISC"
    },
    "node_modules/fsevents": {
@@ -11243,7 +11264,6 @@
      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
      "deprecated": "Glob versions prior to v9 are no longer supported",
-      "dev": true,
      "license": "ISC",
      "dependencies": {
        "fs.realpath": "^1.0.0",
@@ -11842,7 +11862,6 @@
      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
      "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
-      "dev": true,
      "license": "ISC",
      "dependencies": {
        "once": "^1.3.0",
@@ -14619,7 +14638,6 @@
      "version": "3.1.2",
      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
-      "dev": true,
      "license": "ISC",
      "dependencies": {
        "brace-expansion": "^1.1.7"
@@ -15597,7 +15615,6 @@
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
-      "dev": true,
      "license": "MIT",
      "engines": {
        "node": ">=0.10.0"
@@ -15715,6 +15732,53 @@
        "node": ">=6"
      }
    },
+    "node_modules/playwright": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.52.0.tgz",
+      "integrity": "sha512-JAwMNMBlxJ2oD1kce4KPtMkDeKGHQstdpFPcPH3maElAXon/QZeTvtsfXmTMRyO9TslfoYOXkSsvao2nE1ilTw==",
+      "devOptional": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.52.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.52.0.tgz",
+      "integrity": "sha512-l2osTgLXSMeuLZOML9qYODUQoPPnUsKsb5/P6LJ2e6uPKXUdPK5WYhN4z03G+YNbWmGDY4YENauNu4ZKczreHg==",
+      "devOptional": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
    "node_modules/plist": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/plist/-/plist-3.1.0.tgz",

--- a/package.json
+++ b/package.json
@@ -33,7 +33,9 @@
    "test:watch": "vitest",
    "test:ui": "vitest --ui",
    "extract-codebase": "ts-node scripts/extract-codebase.ts",
-    "prepare": "husky install"
+    "prepare": "husky install",
+    "pre:e2e": "E2E_TEST_BUILD=true npm run package",
+    "e2e": "playwright test"
  },
  "keywords": [],
  "author": {
@@ -52,6 +54,7 @@
    "@electron-forge/plugin-vite": "^7.8.0",
    "@electron-forge/publisher-github": "^7.8.0",
    "@electron/fuses": "^1.8.0",
+    "@playwright/test": "^1.52.0",
    "@testing-library/react": "^16.3.0",
    "@types/better-sqlite3": "^7.6.13",
    "@types/kill-port": "^2.0.3",
@@ -111,6 +114,7 @@
    "dotenv": "^16.4.7",
    "drizzle-orm": "^0.41.0",
    "electron-log": "^5.3.3",
+    "electron-playwright-helpers": "^1.7.1",
    "electron-squirrel-startup": "^1.0.1",
    "esbuild-register": "^3.6.0",
    "fix-path": "^4.0.0",

--- a/playwright.config.ts
+++ b/playwright.config.ts
+import { PlaywrightTestConfig } from "@playwright/test";
+const config: PlaywrightTestConfig = {
+  testDir: "./e2e-tests",
+  maxFailures: 1,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* See https://playwright.dev/docs/trace-viewer */
+    trace: "retain-on-failure",
+    // These options do NOT work for electron playwright.
+    // Instead, you need to do a workaround.
+    // See https://github.com/microsoft/playwright/issues/8208
+    //
+    // screenshot: "on",
+    // video: "retain-on-failure",
+  },
+};
+export default config;
--- a/src/main.ts
+++ b/src/main.ts
@@ -65,6 +65,10 @@ export async function onFirstRunMaybe() {
 * applications folder.
 */
 async function promptMoveToApplicationsFolder(): Promise<void> {
+  // Why not in e2e tests?
+  // There's no way to stub this dialog in time, so we just skip it
+  // in e2e testing mode.
+  if (process.env.E2E_TEST_BUILD) return;
  if (process.platform !== "darwin") return;
  if (app.isInApplicationsFolder()) return;
  logger.log("Prompting user to move to applications folder");