Skip to content

B
bit-pm
Unverified 提交 7d5595f6 authored 作者: Will Chen's avatar Will Chen 提交者： GitHub
浏览文件
操作

Optimize codebase extract & fix proposal handler token counting logic (#36)

* remove deprecated script * Optimize codebase extract & fix proposal handler token counting logic * add caching + lock * sort by modified timestamp * cache cleanup
上级 813f170c
全部展开 隐藏空白字符变更
内嵌 并排
正在显示 包含 180 行增加 和 140 行删除
scripts/extract-codebase.ts deleted 100644 → 0
浏览文件 @ 813f170c
#!/usr/bin/env node
// Add type module declaration at the top
// @ts-check
// @ts-ignore
// eslint-disable-next-line
// @ts-nocheck
import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";
import { dirname } from "path";
import { isIgnored } from "isomorphic-git";
import log from "electron-log";
const logger = log.scope("extract-codebase");
// File extensions to include
const ALLOWED_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx", ".css"];
// Function to check if a path is ignored by gitignore
async function isGitIgnored(
filePath: string,
baseDir: string
): Promise<boolean> {
try {
const relativePath = path.relative(baseDir, filePath);
return await isIgnored({ fs, dir: baseDir, filepath: relativePath });
} catch (error) {
logger.error(`Error checking if path is git ignored: ${filePath}`, error);
return false;
}
}
// Function to recursively walk a directory
async function walkDirectory(dir: string, baseDir: string): Promise<string[]> {
const files: string[] = [];
// Read directory contents
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
// Skip if the entry is git ignored
if (await isGitIgnored(fullPath, baseDir)) {
continue;
}
if (entry.isDirectory()) {
// Recursively process subdirectories
const subDirFiles = await walkDirectory(fullPath, baseDir);
files.push(...subDirFiles);
} else if (entry.isFile()) {
// Check file extension
const ext = path.extname(entry.name).toLowerCase();
if (ALLOWED_EXTENSIONS.includes(ext)) {
files.push(fullPath);
}
}
}
return files;
}
// Function to read file contents and format for LLM consumption
function formatFile(filePath: string, baseDir: string): string {
try {
const relativePath = path.relative(baseDir, filePath);
// Check if the file is in components/ui directory
if (
relativePath.includes("eslint.config") ||
relativePath.includes("components/ui") ||
relativePath.includes("components\\ui")
) {
return `## File: ${relativePath}\n\n`;
}
const content = fs.readFileSync(filePath, "utf-8");
return `## File: ${relativePath}\n\`\`\`${path
.extname(filePath)
.substring(1)}\n${content}\n\`\`\`\n\n`;
} catch (error) {
logger.error(`Error reading file: ${filePath}`, error);
return `## File: ${filePath}\nError reading file: ${error}\n\n`;
}
}
async function main() {
// Get command line arguments
const args = process.argv.slice(2);
const scaffoldDir = args[0] || process.cwd();
const outputFile = args[1] || "codebase-extract.md";
console.log(`Extracting code from: ${scaffoldDir}`);
console.log(`Output will be written to: ${outputFile}`);
// Walk directory and get all files
const files = await walkDirectory(scaffoldDir, scaffoldDir);
console.log(`Found ${files.length} code files`);
// Format files
let output = `# Codebase Extract\nGenerated on: ${new Date().toISOString()}\nTotal files: ${
files.length
}\n\n`;
for (const file of files) {
output += formatFile(file, scaffoldDir);
}
// Write to output file
fs.writeFileSync(outputFile, output);
console.log(`Extraction complete. Output written to ${outputFile}`);
}
src/utils/codebase.ts
浏览文件 @ 7d5595f6
import fs from "node:fs";
import fsAsync from "node:fs/promises";
import path from "node:path";
import { isIgnored } from "isomorphic-git";
import log from "electron-log";
const logger = log.scope("utils/codebase");
// File extensions to include in the extraction
const ALLOWED_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx", ".css", ".html"];
......@@ -14,6 +18,23 @@ const ALWAYS_INCLUDE_FILES = ["package.json"];
// Maximum file size to include (in bytes) - 100KB
const MAX_FILE_SIZE = 100 * 1024;
// Maximum size for fileContentCache
const MAX_FILE_CACHE_SIZE = 500;
// File content cache with timestamps
type FileCache = {
content: string;
mtime: number;
};
// Cache for file contents
const fileContentCache = new Map<string, FileCache>();
// Cache for git ignored paths
const gitIgnoreCache = new Map<string, boolean>();
// Map to store .gitignore file paths and their modification times
const gitIgnoreMtimes = new Map<string, number>();
/**
* Check if a path should be ignored based on git ignore rules
*/
......@@ -22,14 +43,108 @@ async function isGitIgnored(
baseDir: string
): Promise<boolean> {
try {
// Check if any relevant .gitignore has been modified
// Git checks .gitignore files in the path from the repo root to the file
let currentDir = baseDir;
const pathParts = path.relative(baseDir, filePath).split(path.sep);
let shouldClearCache = false;
// Check root .gitignore
const rootGitIgnorePath = path.join(baseDir, ".gitignore");
try {
const stats = await fsAsync.stat(rootGitIgnorePath);
const lastMtime = gitIgnoreMtimes.get(rootGitIgnorePath) || 0;
if (stats.mtimeMs > lastMtime) {
gitIgnoreMtimes.set(rootGitIgnorePath, stats.mtimeMs);
shouldClearCache = true;
}
} catch (error) {
// Root .gitignore might not exist, which is fine
}
// Check .gitignore files in parent directories
for (let i = 0; i < pathParts.length - 1; i++) {
currentDir = path.join(currentDir, pathParts[i]);
const gitIgnorePath = path.join(currentDir, ".gitignore");
try {
const stats = await fsAsync.stat(gitIgnorePath);
const lastMtime = gitIgnoreMtimes.get(gitIgnorePath) || 0;
if (stats.mtimeMs > lastMtime) {
gitIgnoreMtimes.set(gitIgnorePath, stats.mtimeMs);
shouldClearCache = true;
}
} catch (error) {
// This directory might not have a .gitignore, which is fine
}
}
// Clear cache if any .gitignore was modified
if (shouldClearCache) {
gitIgnoreCache.clear();
}
const cacheKey = `${baseDir}:${filePath}`;
if (gitIgnoreCache.has(cacheKey)) {
return gitIgnoreCache.get(cacheKey)!;
}
const relativePath = path.relative(baseDir, filePath);
return await isIgnored({ fs, dir: baseDir, filepath: relativePath });
const result = await isIgnored({
fs,
dir: baseDir,
filepath: relativePath,
});
gitIgnoreCache.set(cacheKey, result);
return result;
} catch (error) {
console.error(`Error checking if path is git ignored: ${filePath}`, error);
logger.error(`Error checking if path is git ignored: ${filePath}`, error);
return false;
}
}
/**
* Read file contents with caching based on last modified time
*/
async function readFileWithCache(filePath: string): Promise<string | null> {
try {
// Get file stats to check the modification time
const stats = await fsAsync.stat(filePath);
const currentMtime = stats.mtimeMs;
// If file is in cache and hasn't been modified, use cached content
if (fileContentCache.has(filePath)) {
const cache = fileContentCache.get(filePath)!;
if (cache.mtime === currentMtime) {
return cache.content;
}
}
// Read file and update cache
const content = await fsAsync.readFile(filePath, "utf-8");
fileContentCache.set(filePath, { content, mtime: currentMtime });
// Manage cache size by clearing oldest entries when it gets too large
if (fileContentCache.size > MAX_FILE_CACHE_SIZE) {
// Get the oldest 25% of entries to remove
const entriesToDelete = Math.ceil(MAX_FILE_CACHE_SIZE * 0.25);
const keys = Array.from(fileContentCache.keys());
// Remove oldest entries (first in, first out)
for (let i = 0; i < entriesToDelete; i++) {
fileContentCache.delete(keys[i]);
}
}
return content;
} catch (error) {
logger.error(`Error reading file: ${filePath}`, error);
return null;
}
}
/**
* Recursively walk a directory and collect all relevant files
*/
......@@ -37,25 +152,29 @@ async function collectFiles(dir: string, baseDir: string): Promise<string[]> {
const files: string[] = [];
// Check if directory exists
if (!fs.existsSync(dir)) {
try {
await fsAsync.access(dir);
} catch {
// Directory doesn't exist or is not accessible
return files;
}
try {
// Read directory contents
const entries = fs.readdirSync(dir, { withFileTypes: true });
const entries = await fsAsync.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
// Process entries concurrently
const promises = entries.map(async (entry) => {
const fullPath = path.join(dir, entry.name);
// Skip excluded directories
if (entry.isDirectory() && EXCLUDED_DIRS.includes(entry.name)) {
continue;
return;
}
// Skip if the entry is git ignored
if (await isGitIgnored(fullPath, baseDir)) {
continue;
return;
}
if (entry.isDirectory()) {
......@@ -69,22 +188,24 @@ async function collectFiles(dir: string, baseDir: string): Promise<string[]> {
// Skip files that are too large
try {
const stats = fs.statSync(fullPath);
const stats = await fsAsync.stat(fullPath);
if (stats.size > MAX_FILE_SIZE) {
continue;
return;
}
} catch (error) {
console.error(`Error checking file size: ${fullPath}`, error);
continue;
logger.error(`Error checking file size: ${fullPath}`, error);
return;
}
if (ALLOWED_EXTENSIONS.includes(ext) || shouldAlwaysInclude) {
files.push(fullPath);
}
}
}
});
await Promise.all(promises);
} catch (error) {
console.error(`Error reading directory ${dir}:`, error);
logger.error(`Error reading directory ${dir}:`, error);
}
return files;
......@@ -93,7 +214,7 @@ async function collectFiles(dir: string, baseDir: string): Promise<string[]> {
/**
* Format a file for inclusion in the codebase extract
*/
function formatFile(filePath: string, baseDir: string): string {
async function formatFile(filePath: string, baseDir: string): Promise<string> {
try {
const relativePath = path.relative(baseDir, filePath);
......@@ -114,7 +235,15 @@ function formatFile(filePath: string, baseDir: string): string {
`;
}
const content = fs.readFileSync(filePath, "utf-8");
const content = await readFileWithCache(filePath);
if (content === null) {
return `<dyad-file path="${relativePath}">
// Error reading file
</dyad-file>
`;
}
return `<dyad-file path="${relativePath}">
${content}
......@@ -122,7 +251,7 @@ ${content}
`;
} catch (error) {
console.error(`Error reading file: ${filePath}`, error);
logger.error(`Error reading file: ${filePath}`, error);
return `<dyad-file path="${path.relative(baseDir, filePath)}">
// Error reading file: ${error}
</dyad-file>
......@@ -137,26 +266,53 @@ ${content}
* @returns A string containing formatted file contents
*/
export async function extractCodebase(appPath: string): Promise<string> {
if (!fs.existsSync(appPath)) {
return `# Error: Directory ${appPath} does not exist`;
try {
await fsAsync.access(appPath);
} catch {
return `# Error: Directory ${appPath} does not exist or is not accessible`;
}
const startTime = Date.now();
// Collect all relevant files
const files = await collectFiles(appPath, appPath);
// Sort files to prioritize important files
const sortedFiles = sortFilesByImportance(files, appPath);
// Sort files by modification time (oldest first)
// This is important for cache-ability.
const sortedFiles = await sortFilesByModificationTime(files);
// Format files
let output = "";
const formatPromises = sortedFiles.map((file) => formatFile(file, appPath));
const formattedFiles = await Promise.all(formatPromises);
output = formattedFiles.join("");
for (const file of sortedFiles) {
output += formatFile(file, appPath);
}
const endTime = Date.now();
logger.log("extractCodebase: time taken", endTime - startTime);
return output;
}
/**
* Sort files by their modification timestamp (oldest first)
*/
async function sortFilesByModificationTime(files: string[]): Promise<string[]> {
// Get stats for all files
const fileStats = await Promise.all(
files.map(async (file) => {
try {
const stats = await fsAsync.stat(file);
return { file, mtime: stats.mtimeMs };
} catch (error) {
// If there's an error getting stats, use current time as fallback
logger.error(`Error getting file stats for ${file}:`, error);
return { file, mtime: Date.now() };
}
})
);
// Sort by modification time (oldest first)
return fileStats.sort((a, b) => a.mtime - b.mtime).map((item) => item.file);
}
/**
* Sort files by their importance for context
*/
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论