test: deflake setup and context E2E flows (#3319)

e2e-tests/cloud_sandbox.spec.ts

@@ -64,7 +64,7 @@ testSkipIfWindows(
             .textContent({ timeout: Timeout.LONG });
           return digestText?.split(": ").at(-1)?.trim();
         },
-        { timeout: Timeout.LONG },
+        { timeout: Timeout.EXTRA_LONG },
       )
       .not.toBe(updatedDigest);
   },

e2e-tests/context_manage.spec.ts

@@ -286,6 +286,9 @@ test("manage context - smart context", async ({ po }) => {
   // the auto-includes.
   const proModesDialog = await po.openProModesDialog();
   await proModesDialog.setSmartContextMode("off");
+  await expect
+    .poll(() => po.settings.recordSettings().enableProSmartFilesContextMode)
+    .toBe(false);
   await proModesDialog.close();
 
   await po.sendPrompt("[dump]");

e2e-tests/helpers/fixtures.ts

@@ -95,6 +95,8 @@ export const test = base.extend<{
       if (!electronConfig.showSetupScreen) {
         // This is just a hack to avoid the AI setup screen.
         process.env.OPENAI_API_KEY = "sk-test";
+      } else {
+        delete process.env.OPENAI_API_KEY;
       }
       const baseTmpDir = os.tmpdir();
       const userDataDir = path.join(baseTmpDir, `dyad-e2e-tests-${Date.now()}`);

e2e-tests/helpers/page-objects/components/Settings.ts

@@ -205,7 +205,7 @@ export class Settings {
       .fill("test-api-key-12345");
     await this.page.getByRole("button", { name: "Save Key" }).click();
     // Wait for the key to be saved
-    await expect(this.page.getByText("test-api-key-12345")).toBeVisible();
+    await expect(this.page.getByText(/test.+2345/)).toBeVisible();
   }
 
   async setUpDyadProvider() {

rules/e2e-testing.md

@@ -124,6 +124,8 @@ If `npm run build` fails while rebuilding native modules with `ImportError` from
 - **Navigation to tabs**: Use `await expect(link).toBeVisible({ timeout: Timeout.EXTRA_LONG })` before clicking tab links (especially in `goToAppsTab()`). Electron sidebar links can take time to render during app initialization.
 - **Confirming flakiness**: Use `PLAYWRIGHT_RETRIES=0 PLAYWRIGHT_HTML_OPEN=never npm run e2e -- e2e-tests/<spec> --repeat-each=10` to reproduce flaky tests. `PLAYWRIGHT_RETRIES=0` is critical — CI defaults to 2 retries, hiding flakiness.
 - **`expect(...).toPass()` wrappers**: Give inner Playwright actions/assertions short explicit timeouts. Default 30s click/expect timeouts can consume the whole `toPass()` budget, so the retry wrapper never actually retries.
+- **Setup-screen tests and provider env vars**: E2E worker processes reuse `process.env`, so tests that set fake provider keys (for example `OPENAI_API_KEY`) can affect later tests in the same worker. When a fixture intentionally shows the setup screen, explicitly clear any env key that would make the provider appear configured.
+- **Settings-dependent prompts**: After toggling a setting that affects the next chat request (for example Smart Context mode), wait for the persisted settings state with `expect.poll(() => po.settings.recordSettings().someKey)` before sending the prompt. UI clicks can return before the main-process settings write is visible to the request path.
 - **Monaco file-switch assertions**: For code-editor tests, don't stop at waiting for the editor textbox to appear. Wait until Monaco's active model URI matches the file you clicked; otherwise the test can type into a still-switching editor model and miss real file-switch races.
 - **Monaco race repros**: If a file-editor bug only appears during quick tab/file changes, alternate between the affected files several times in one test before declaring it non-reproducible. A single switch often misses save-vs-switch timing bugs that show up immediately under `--repeat-each`.
 - **GitHub sync success assertions**: Scope "Successfully pushed to GitHub!" assertions to `getByTestId("github-connected-repo")`; the same text can also appear in a toast, causing Playwright strict-mode failures.