Integrating web crawling in the custom theme generator (#2347)

e2e-tests/themes_management.spec.ts

@@ -271,3 +271,68 @@ test("themes management - AI generator flow", async ({ po }) => {
   await expect(po.page.getByText("AI Generated Theme")).toBeVisible();
   await expect(po.page.getByText("Created via AI generator")).toBeVisible();
 });
+
+test("themes management - AI generator from website URL", async ({ po }) => {
+  await po.setUpDyadPro();
+
+  // Navigate to Themes page via Library sidebar
+  await po.goToLibraryTab();
+  await po.page.getByRole("link", { name: "Themes" }).click();
+  await expect(po.page.getByRole("heading", { name: "Themes" })).toBeVisible();
+
+  // Click New Theme button
+  await po.page.getByRole("button", { name: "New Theme" }).click();
+
+  // Wait for dialog to open
+  await expect(
+    po.page.getByRole("dialog").getByText("Create Custom Theme"),
+  ).toBeVisible();
+
+  // Verify AI-Powered Generator tab is active by default
+  const aiTab = po.page.getByRole("tab", { name: "AI-Powered Generator" });
+  await expect(aiTab).toHaveAttribute("data-state", "active");
+
+  // Switch to Website URL input source
+  await po.page.getByRole("button", { name: "Website URL" }).click();
+
+  // Verify URL input is visible
+  const urlInput = po.page.getByLabel("Website URL");
+  await expect(urlInput).toBeVisible();
+
+  // Verify Generate button is disabled before entering URL
+  const generateButton = po.page.getByRole("button", {
+    name: "Generate Theme Prompt",
+  });
+  await expect(generateButton).toBeDisabled();
+
+  // Fill in theme details
+  await po.page.getByLabel("Theme Name").fill("Website Theme");
+  await po.page
+    .getByLabel("Description (optional)")
+    .fill("Generated from website");
+
+  // Enter a website URL
+  await urlInput.fill("https://example.com");
+
+  // Verify Generate button is now enabled
+  await expect(generateButton).toBeEnabled();
+
+  // Click Generate to get mock theme prompt (test mode returns mock response)
+  await generateButton.click();
+
+  // Wait for generation to complete - the generated prompt textarea should appear
+  await expect(po.page.locator("#ai-prompt")).toBeVisible({ timeout: 10000 });
+
+  // Verify the mock theme content is displayed (URL-specific mock)
+  await expect(po.page.getByText("Test Mode Theme (from URL)")).toBeVisible();
+
+  // Save the theme
+  await po.page.getByRole("button", { name: "Save Theme" }).click();
+
+  // Verify dialog closes and theme card appears
+  await expect(po.page.getByRole("dialog")).not.toBeVisible();
+  const themeCard = po.page.getByTestId("theme-card");
+  await expect(themeCard).toBeVisible();
+  await expect(themeCard.getByText("Website Theme")).toBeVisible();
+  await expect(themeCard.getByText("Generated from website")).toBeVisible();
+});

src/hooks/useCustomThemes.ts

@@ -6,6 +6,7 @@ import type {
   UpdateCustomThemeParams,
   GenerateThemePromptParams,
   GenerateThemePromptResult,
+  GenerateThemeFromUrlParams,
 } from "@/ipc/types";
 import { queryKeys } from "@/lib/queryKeys";
 
@@ -92,3 +93,13 @@ export function useGenerateThemePrompt() {
     },
   });
 }
+
+export function useGenerateThemeFromUrl() {
+  return useMutation({
+    mutationFn: async (
+      params: GenerateThemeFromUrlParams,
+    ): Promise<GenerateThemePromptResult> => {
+      return ipc.template.generateThemeFromUrl(params);
+    },
+  });
+}

src/ipc/types/index.ts

@@ -243,8 +243,11 @@ export type {
   DeleteCustomThemeParams,
   ThemeGenerationMode,
   ThemeGenerationModel,
+  ThemeInputSource,
+  CrawlStatus,
   GenerateThemePromptParams,
   GenerateThemePromptResult,
+  GenerateThemeFromUrlParams,
   SaveThemeImageParams,
   SaveThemeImageResult,
   CleanupThemeImagesParams,

src/ipc/types/templates.ts

@@ -100,6 +100,14 @@ export const ThemeGenerationModelSchema = z.enum([
 ]);
 export type ThemeGenerationModel = z.infer<typeof ThemeGenerationModelSchema>;
 
+// Theme input source (images or URL)
+export const ThemeInputSourceSchema = z.enum(["images", "url"]);
+export type ThemeInputSource = z.infer<typeof ThemeInputSourceSchema>;
+
+// Crawl status for UI feedback
+export const CrawlStatusSchema = z.enum(["crawling", "complete", "error"]);
+export type CrawlStatus = z.infer<typeof CrawlStatusSchema>;
+
 export const GenerateThemePromptParamsSchema = z.object({
   imagePaths: z.array(z.string()),
   keywords: z.string(),
@@ -119,6 +127,31 @@ export type GenerateThemePromptResult = z.infer<
   typeof GenerateThemePromptResultSchema
 >;
 
+// URL-based theme generation params
+export const GenerateThemeFromUrlParamsSchema = z.object({
+  url: z
+    .string()
+    .url()
+    .refine(
+      (url) => {
+        try {
+          const parsed = new URL(url);
+          return parsed.protocol === "http:" || parsed.protocol === "https:";
+        } catch {
+          return false;
+        }
+      },
+      { message: "Only HTTP and HTTPS URLs are supported" },
+    ),
+  keywords: z.string(),
+  generationMode: ThemeGenerationModeSchema,
+  model: ThemeGenerationModelSchema,
+});
+
+export type GenerateThemeFromUrlParams = z.infer<
+  typeof GenerateThemeFromUrlParamsSchema
+>;
+
 export const SaveThemeImageParamsSchema = z.object({
   data: z.string(),
   filename: z.string(),
@@ -201,6 +234,12 @@ export const templateContracts = {
     output: GenerateThemePromptResultSchema,
   }),
 
+  generateThemeFromUrl: defineContract({
+    channel: "generate-theme-from-url",
+    input: GenerateThemeFromUrlParamsSchema,
+    output: GenerateThemePromptResultSchema,
+  }),
+
   saveThemeImage: defineContract({
     channel: "save-theme-image",
     input: SaveThemeImageParamsSchema,

src/pro/main/ipc/handlers/local_agent/tools/web_crawl.ts

@@ -9,7 +9,7 @@ const webCrawlSchema = z.object({
   url: z.string().describe("URL to crawl"),
 });
 
-const webCrawlResponseSchema = z.object({
+export const webCrawlResponseSchema = z.object({
   rootUrl: z.string(),
   html: z.string().optional(),
   markdown: z.string().optional(),