[codex] fix binary-safe cloud sandbox sync uploads (#3188)

src/ipc/utils/cloud_sandbox_provider.test.ts

@@ -42,6 +42,55 @@ import {
   uploadCloudSandboxFiles,
 } from "./cloud_sandbox_provider";
 
+type ParsedMultipartUpload = {
+  manifest: {
+    replaceAll: boolean;
+    deletedFiles: string[];
+    files: Array<{
+      path: string;
+      fieldName: string;
+    }>;
+  };
+  files: Record<string, Buffer>;
+};
+
+async function parseMultipartUpload(
+  init: RequestInit | undefined,
+): Promise<ParsedMultipartUpload> {
+  const request = new Request("https://dyad.test/upload", {
+    method: "POST",
+    body: init?.body as BodyInit,
+    headers: init?.headers,
+  });
+  const formData = await request.formData();
+  const manifestValue = formData.get("manifest");
+
+  if (typeof manifestValue !== "string") {
+    throw new Error("Expected manifest form field.");
+  }
+
+  const manifest = JSON.parse(
+    manifestValue,
+  ) as ParsedMultipartUpload["manifest"];
+  const files = Object.fromEntries(
+    await Promise.all(
+      manifest.files.map(async ({ path, fieldName }) => {
+        const filePart = formData.get(fieldName);
+        if (!(filePart instanceof File)) {
+          throw new Error(`Expected file part for ${fieldName}.`);
+        }
+
+        return [path, Buffer.from(await filePart.arrayBuffer())] as const;
+      }),
+    ),
+  );
+
+  return {
+    manifest,
+    files,
+  };
+}
+
 describe("cloud_sandbox_provider incremental sync", () => {
   let appPath: string;
   let fetchMock: ReturnType<typeof vi.fn>;
@@ -91,12 +140,19 @@ describe("cloud_sandbox_provider incremental sync", () => {
     expect(fetchMock).toHaveBeenCalledTimes(1);
     const [, init] = fetchMock.mock.calls[0];
     expect(init?.method).toBe("POST");
-    expect(JSON.parse(String(init?.body))).toEqual({
-      files: {
-        "src.ts": "console.log('hi');",
-      },
+    const upload = await parseMultipartUpload(init);
+    expect(upload.manifest).toEqual({
       replaceAll: false,
       deletedFiles: [],
+      files: [
+        {
+          path: "src.ts",
+          fieldName: "file_0",
+        },
+      ],
+    });
+    expect(upload.files).toEqual({
+      "src.ts": Buffer.from("console.log('hi');"),
     });
   });
 
@@ -113,12 +169,19 @@ describe("cloud_sandbox_provider incremental sync", () => {
 
     expect(fetchMock).toHaveBeenCalledTimes(1);
     const [, init] = fetchMock.mock.calls[0];
-    expect(JSON.parse(String(init?.body))).toEqual({
-      files: {
-        "keep.ts": "updated",
-      },
+    const upload = await parseMultipartUpload(init);
+    expect(upload.manifest).toEqual({
       replaceAll: false,
       deletedFiles: ["old.ts"],
+      files: [
+        {
+          path: "keep.ts",
+          fieldName: "file_0",
+        },
+      ],
+    });
+    expect(upload.files).toEqual({
+      "keep.ts": Buffer.from("updated"),
     });
   });
 
@@ -131,14 +194,51 @@ describe("cloud_sandbox_provider incremental sync", () => {
 
     expect(fetchMock).toHaveBeenCalledTimes(1);
     const [, init] = fetchMock.mock.calls[0];
-    expect(JSON.parse(String(init?.body))).toEqual({
-      files: {
-        "a.ts": "A",
-        "nested/b.ts": "B",
-      },
+    const upload = await parseMultipartUpload(init);
+    expect(upload.manifest).toEqual({
       replaceAll: true,
       deletedFiles: [],
+      files: [
+        {
+          path: "a.ts",
+          fieldName: "file_0",
+        },
+        {
+          path: "nested/b.ts",
+          fieldName: "file_1",
+        },
+      ],
+    });
+    expect(upload.files).toEqual({
+      "a.ts": Buffer.from("A"),
+      "nested/b.ts": Buffer.from("B"),
+    });
+  });
+
+  it("uploads binary files without utf-8 transcoding", async () => {
+    const originalBytes = Buffer.from([0x00, 0xff, 0x10, 0x80, 0x41, 0x42]);
+    await fs.writeFile(path.join(appPath, "assets.bin"), originalBytes);
+
+    await syncCloudSandboxDirtyPaths({
+      appId: 1,
+      changedPaths: ["assets.bin"],
+    });
+
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    const [, init] = fetchMock.mock.calls[0];
+    const upload = await parseMultipartUpload(init);
+
+    expect(upload.manifest).toEqual({
+      replaceAll: false,
+      deletedFiles: [],
+      files: [
+        {
+          path: "assets.bin",
+          fieldName: "file_0",
+        },
+      ],
     });
+    expect(upload.files["assets.bin"]).toEqual(originalBytes);
   });
 
   it("excludes gitignored paths, keeps root env files, and skips symlinks", async () => {
@@ -173,10 +273,10 @@ describe("cloud_sandbox_provider incremental sync", () => {
     });
 
     await expect(buildCloudSandboxFileMap(appPath)).resolves.toEqual({
-      ".env": "ROOT_ENV=1",
-      ".env.local": "ROOT_ENV_LOCAL=1",
-      "symlink-target.ts": "outside",
-      "visible.ts": "export const ok = true;",
+      ".env": Buffer.from("ROOT_ENV=1"),
+      ".env.local": Buffer.from("ROOT_ENV_LOCAL=1"),
+      "symlink-target.ts": Buffer.from("outside"),
+      "visible.ts": Buffer.from("export const ok = true;"),
     });
   });
 
@@ -201,13 +301,24 @@ describe("cloud_sandbox_provider incremental sync", () => {
 
     expect(fetchMock).toHaveBeenCalledTimes(1);
     const [, init] = fetchMock.mock.calls[0];
-    expect(JSON.parse(String(init?.body))).toEqual({
-      files: {
-        ".env.local": "SAFE_ENV=1",
-        "changed.ts": "updated",
-      },
+    const upload = await parseMultipartUpload(init);
+    expect(upload.manifest).toEqual({
       replaceAll: false,
       deletedFiles: ["ignored.ts", "linked.ts"],
+      files: [
+        {
+          path: ".env.local",
+          fieldName: "file_0",
+        },
+        {
+          path: "changed.ts",
+          fieldName: "file_1",
+        },
+      ],
+    });
+    expect(upload.files).toEqual({
+      ".env.local": Buffer.from("SAFE_ENV=1"),
+      "changed.ts": Buffer.from("updated"),
     });
   });
 
@@ -222,13 +333,24 @@ describe("cloud_sandbox_provider incremental sync", () => {
 
     expect(fetchMock).toHaveBeenCalledTimes(1);
     const [, init] = fetchMock.mock.calls[0];
-    expect(JSON.parse(String(init?.body))).toEqual({
-      files: {
-        ".gitignore": "dist\n",
-        "index.ts": "console.log('ok');",
-      },
+    const upload = await parseMultipartUpload(init);
+    expect(upload.manifest).toEqual({
       replaceAll: true,
       deletedFiles: [],
+      files: [
+        {
+          path: ".gitignore",
+          fieldName: "file_0",
+        },
+        {
+          path: "index.ts",
+          fieldName: "file_1",
+        },
+      ],
+    });
+    expect(upload.files).toEqual({
+      ".gitignore": Buffer.from("dist\n"),
+      "index.ts": Buffer.from("console.log('ok');"),
     });
   });
 
@@ -308,12 +430,19 @@ describe("cloud_sandbox_provider incremental sync", () => {
     expect(fetchMock).toHaveBeenCalledTimes(2);
 
     const [, secondInit] = fetchMock.mock.calls[1];
-    expect(JSON.parse(String(secondInit?.body))).toEqual({
-      files: {
-        "second.ts": "second",
-      },
+    const upload = await parseMultipartUpload(secondInit);
+    expect(upload.manifest).toEqual({
       replaceAll: false,
       deletedFiles: [],
+      files: [
+        {
+          path: "second.ts",
+          fieldName: "file_0",
+        },
+      ],
+    });
+    expect(upload.files).toEqual({
+      "second.ts": Buffer.from("second"),
     });
   });
 

src/ipc/utils/cloud_sandbox_provider.ts

@@ -14,12 +14,23 @@ const DYAD_ENGINE_URL =
 const CLOUD_SANDBOX_EXCLUDED_DIRS = new Set(["node_modules", ".git", ".next"]);
 const CLOUD_SANDBOX_ROOT_ALLOWLIST = new Set([".env", ".env.local"]);
 
-export type CloudSandboxFileMap = Record<string, string>;
+type CloudSandboxFileBytes = Uint8Array;
+
+export type CloudSandboxFileMap = Record<string, CloudSandboxFileBytes>;
 export type CloudSandboxSyncUpdate = {
   appId: number;
   errorMessage: string | null;
 };
 
+type CloudSandboxUploadManifest = {
+  replaceAll: boolean;
+  deletedFiles: string[];
+  files: Array<{
+    path: string;
+    fieldName: string;
+  }>;
+};
+
 const CloudSandboxCreateResponseSchema = z.object({
   sandboxId: z.string().trim().min(1),
   previewUrl: z.string().trim().min(1),
@@ -216,7 +227,10 @@ async function cloudSandboxFetch(
 ): Promise<Response> {
   const apiKey = getDyadEngineApiKey();
   const headers = new Headers(init.headers);
-  if (!headers.has("Content-Type") && init.body) {
+  const isMultipartBody =
+    typeof FormData !== "undefined" && init.body instanceof FormData;
+
+  if (!headers.has("Content-Type") && init.body && !isMultipartBody) {
     headers.set("Content-Type", "application/json");
   }
   if (apiKey) {
@@ -284,6 +298,40 @@ async function parseResponseJson<T>(
   return result.data;
 }
 
+function buildCloudSandboxUploadFormData(input: {
+  files: CloudSandboxFileMap;
+  replaceAll: boolean;
+  deletedFiles: string[];
+}): FormData {
+  const formData = new FormData();
+  const manifest: CloudSandboxUploadManifest = {
+    replaceAll: input.replaceAll,
+    deletedFiles: input.deletedFiles,
+    files: [],
+  };
+  const sortedFiles = Object.entries(input.files).sort(([left], [right]) =>
+    left.localeCompare(right),
+  );
+
+  for (const [filePath, content] of sortedFiles) {
+    const fieldName = `file_${manifest.files.length}`;
+    manifest.files.push({
+      path: filePath,
+      fieldName,
+    });
+    formData.append(
+      fieldName,
+      new Blob([Uint8Array.from(content)], {
+        type: "application/octet-stream",
+      }),
+      path.posix.basename(filePath) || fieldName,
+    );
+  }
+
+  formData.append("manifest", JSON.stringify(manifest));
+  return formData;
+}
+
 export async function buildCloudSandboxFileMap(
   appPath: string,
 ): Promise<CloudSandboxFileMap> {
@@ -292,7 +340,7 @@ export async function buildCloudSandboxFileMap(
     files.map(async (relativePath) => {
       const normalizedPath = normalizePath(relativePath);
       const fullPath = path.join(appPath, normalizedPath);
-      const content = await fsPromises.readFile(fullPath, "utf-8");
+      const content = await fsPromises.readFile(fullPath);
       return [normalizedPath, content] as const;
     }),
   );
@@ -448,7 +496,7 @@ async function buildCloudSandboxPartialFileMap(input: {
         continue;
       }
 
-      const content = await fsPromises.readFile(fullPath, "utf-8");
+      const content = await fsPromises.readFile(fullPath);
       files[normalizedPath] = content;
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code === "ENOENT") {
@@ -698,7 +746,7 @@ class DyadEngineCloudSandboxProvider implements CloudSandboxProvider {
   ) {
     const response = await cloudSandboxFetch(`/sandboxes/${sandboxId}/files`, {
       method: "POST",
-      body: JSON.stringify({
+      body: buildCloudSandboxUploadFormData({
         files,
         replaceAll: options?.replaceAll ?? false,
         deletedFiles: options?.deletedFiles ?? [],

testing/fake-llm-server/index.ts

@@ -77,7 +77,7 @@ export const CANNED_MESSAGE = `
 
 type FakeCloudSandbox = {
   id: string;
-  files: Record<string, string>;
+  files: Record<string, Buffer>;
   createdAt: number;
   previewAuthToken: string;
   syncRevision: number;
@@ -101,6 +101,81 @@ function createServiceResponse<T>(responseObject: T) {
   };
 }
 
+async function parseMultipartFormData(req: express.Request) {
+  const headers = new Headers();
+  for (const [key, value] of Object.entries(req.headers)) {
+    if (Array.isArray(value)) {
+      for (const entry of value) {
+        headers.append(key, entry);
+      }
+      continue;
+    }
+
+    if (value !== undefined) {
+      headers.set(key, value);
+    }
+  }
+
+  const chunks: Buffer[] = [];
+  for await (const chunk of req) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+
+  const request = new Request("http://localhost/fake-cloud-upload", {
+    method: req.method,
+    headers,
+    body: Buffer.concat(chunks),
+  });
+
+  return request.formData();
+}
+
+async function parseCloudSandboxUpload(req: express.Request) {
+  if (!req.is("multipart/form-data")) {
+    return {
+      replaceAll: req.body.replaceAll === true,
+      deletedFiles: Array.isArray(req.body.deletedFiles)
+        ? req.body.deletedFiles
+        : [],
+      files: Object.fromEntries(
+        Object.entries(req.body.files ?? {}).map(([filePath, content]) => [
+          filePath,
+          Buffer.from(String(content), "utf8"),
+        ]),
+      ) as Record<string, Buffer>,
+    };
+  }
+
+  const formData = await parseMultipartFormData(req);
+  const manifestValue = formData.get("manifest");
+
+  if (typeof manifestValue !== "string") {
+    throw new Error("Expected multipart sandbox upload manifest.");
+  }
+
+  const manifest = JSON.parse(manifestValue) as {
+    replaceAll: boolean;
+    deletedFiles?: string[];
+    files?: Array<{ path: string; fieldName: string }>;
+  };
+  const files: Record<string, Buffer> = {};
+
+  for (const entry of manifest.files ?? []) {
+    const filePart = formData.get(entry.fieldName);
+    if (!(filePart instanceof File)) {
+      throw new Error(`Expected multipart file part ${entry.fieldName}.`);
+    }
+
+    files[entry.path] = Buffer.from(await filePart.arrayBuffer());
+  }
+
+  return {
+    replaceAll: manifest.replaceAll === true,
+    deletedFiles: manifest.deletedFiles ?? [],
+    files,
+  };
+}
+
 function escapeHtml(text: string) {
   return text
     .replace(/&/g, "&amp;")
@@ -110,10 +185,10 @@ function escapeHtml(text: string) {
 
 function getSandboxPreviewHtml(sandbox: FakeCloudSandbox) {
   const interestingSource =
-    sandbox.files["src/App.tsx"] ??
-    sandbox.files["src/App.jsx"] ??
-    sandbox.files["app/page.tsx"] ??
-    sandbox.files["index.html"] ??
+    sandbox.files["src/App.tsx"]?.toString("utf8") ??
+    sandbox.files["src/App.jsx"]?.toString("utf8") ??
+    sandbox.files["app/page.tsx"]?.toString("utf8") ??
+    sandbox.files["index.html"]?.toString("utf8") ??
     "";
 
   const fileList = Object.keys(sandbox.files)
@@ -121,17 +196,16 @@ function getSandboxPreviewHtml(sandbox: FakeCloudSandbox) {
     .slice(0, 12)
     .map((file) => `<li>${escapeHtml(file)}</li>`)
     .join("");
-  const snapshotDigest = crypto
-    .createHash("sha1")
-    .update(
-      JSON.stringify(
-        Object.entries(sandbox.files).sort(([leftPath], [rightPath]) =>
-          leftPath.localeCompare(rightPath),
-        ),
-      ),
-    )
-    .digest("hex")
-    .slice(0, 12);
+  const snapshotHasher = crypto.createHash("sha1");
+  for (const [filePath, content] of Object.entries(sandbox.files).sort(
+    ([leftPath], [rightPath]) => leftPath.localeCompare(rightPath),
+  )) {
+    snapshotHasher.update(filePath);
+    snapshotHasher.update("\0");
+    snapshotHasher.update(content);
+    snapshotHasher.update("\0");
+  }
+  const snapshotDigest = snapshotHasher.digest("hex").slice(0, 12);
 
   return `<!doctype html>
 <html>
@@ -561,29 +635,31 @@ app.delete("/engine/v1/sandboxes/:sandboxId", (req, res) => {
   res.status(204).end();
 });
 
-app.post("/engine/v1/sandboxes/:sandboxId/files", (req, res) => {
+app.post("/engine/v1/sandboxes/:sandboxId/files", async (req, res) => {
   const sandbox = cloudSandboxes.get(req.params.sandboxId);
   if (!sandbox) {
     res.status(404).json({ error: "Sandbox not found" });
     return;
   }
 
+  const upload = await parseCloudSandboxUpload(req);
+
   console.log(
-    `[fake-cloud] upload sandbox=${sandbox.id} replaceAll=${String(req.body.replaceAll)} fileCount=${Object.keys(req.body.files ?? {}).length} deletedCount=${(req.body.deletedFiles ?? []).length}`,
+    `[fake-cloud] upload sandbox=${sandbox.id} replaceAll=${String(upload.replaceAll)} fileCount=${Object.keys(upload.files).length} deletedCount=${upload.deletedFiles.length}`,
   );
 
   sandbox.lastActiveAt = Date.now();
   sandbox.lastSuccessfulSyncAt = Date.now();
   sandbox.initialSyncCompleted = true;
   sandbox.syncRevision += 1;
-  sandbox.files = req.body.replaceAll
-    ? { ...req.body.files }
+  sandbox.files = upload.replaceAll
+    ? { ...upload.files }
     : {
         ...sandbox.files,
-        ...req.body.files,
+        ...upload.files,
       };
 
-  for (const deletedFile of req.body.deletedFiles ?? []) {
+  for (const deletedFile of upload.deletedFiles) {
     delete sandbox.files[deletedFile];
   }