repowiki-core 0.1.0

package/src/pipeline.ts

@@ -0,0 +1,225 @@
+import * as path from 'node:path';
+import * as fs from 'node:fs/promises';
+import { loadLLMConfig, LLMClient, validateConfig } from './llm/index.js';
+import { scanDirectory } from './scanner/index.js';
+import { buildTreeString } from './scanner/tree-builder.js';
+import { detectTechStack, detectEntrypoints, buildDependencyGraph } from './detector/index.js';
+import {
+    analyzeModules,
+    analyzeApiRoutes,
+    analyzeDatabaseModels,
+    analyzeConfigs,
+    analyzeWorkflows,
+} from './analyzer/index.js';
+import { WikiGenerator, generateHome, generateSidebar } from './generator/index.js';
+import type { AnalysisResult, ProjectProfile, WikiPage } from './models/index.js';
+
+export interface PipelineOptions {
+    workspacePath: string;
+    outputDir?: string;
+    modelName?: string;
+    concurrency?: number;
+    skipLlm?: boolean;
+    onProgress?: (event: PipelineEvent) => void;
+}
+
+export type PipelineEvent =
+    | { type: 'PROGRESS'; stage: string; progress: number; message: string }
+    | { type: 'DONE'; payload: { docsPath: string; pagesCount: number } }
+    | { type: 'ERROR'; code: number; message: string };
+
+/**
+ * 运行核心 Wiki 生成 Pipeline
+ */
+export async function runPipeline(options: PipelineOptions): Promise<AnalysisResult> {
+    const {
+        workspacePath,
+        outputDir = path.join(workspacePath, 'docs', 'wiki'),
+        modelName,
+        concurrency = 3,
+        skipLlm = false,
+        onProgress,
+    } = options;
+
+    const emitProgress = (stage: string, progress: number, message: string) => {
+        if (onProgress) {
+            onProgress({ type: 'PROGRESS', stage, progress, message });
+        } else {
+            // CLI 后台或未提供回调时输出规范的 JSONL 格式进程通知
+            console.log(JSON.stringify({ type: 'PROGRESS', stage, progress, message }));
+        }
+    };
+
+    try {
+        // ====================================================================
+        // 1. Pre-flight 阶段
+        // ====================================================================
+        emitProgress('Pre-flight', 5, '初始化环境与认证配置...');
+
+        let llmClient: LLMClient | null = null;
+
+        if (!skipLlm) {
+            const llmConfig = await loadLLMConfig(workspacePath);
+            if (modelName) {
+                llmConfig.modelName = modelName;
+            }
+
+            const validationErrors = validateConfig(llmConfig);
+            if (validationErrors.length > 0) {
+                // 如果 API Key 缺失，回退到免大模型模式
+                emitProgress('Pre-flight', 8, `未提供 API 凭证，切换至免大模型快速生成模式...`);
+            } else {
+                llmClient = new LLMClient({ config: llmConfig });
+                emitProgress('Pre-flight', 10, `成功加载 LLM 接口，使用模型: ${llmConfig.modelName}`);
+            }
+        } else {
+            emitProgress('Pre-flight', 10, '已开启免大模型（skip-llm）生成模式...');
+        }
+
+        // 确保输出目录存在
+        await fs.mkdir(outputDir, { recursive: true });
+
+        // ====================================================================
+        // 2. Scan Phase 阶段
+        // ====================================================================
+        emitProgress('Scanning', 15, '扫描项目文件结构中...');
+        const files = await scanDirectory(workspacePath);
+
+        emitProgress('Scanning', 25, `扫描完成，发现 ${files.length} 个非忽略文件。构建目录树...`);
+        const treeStr = buildTreeString(files, workspacePath);
+
+        emitProgress('Scanning', 30, '识别项目技术栈与框架特征...');
+        const techStackResult = await detectTechStack(workspacePath, files);
+
+        const projectProfile: ProjectProfile = {
+            name: path.basename(workspacePath) || 'unnamed-project',
+            rootPath: workspacePath,
+            languages: techStackResult.languages,
+            frameworks: techStackResult.frameworks,
+            packageManagers: techStackResult.packageManagers,
+            databases: techStackResult.databases,
+            services: techStackResult.services,
+            entrypoints: [],
+            configFiles: techStackResult.configFiles,
+        };
+
+        // ====================================================================
+        // 3. Analysis Phase 阶段
+        // ====================================================================
+        emitProgress('Analysis', 40, '分析项目入口及内部模块依赖图...');
+        const entrypoints = await detectEntrypoints(workspacePath, files);
+        projectProfile.entrypoints = entrypoints;
+
+        const dependencyGraph = await buildDependencyGraph(workspacePath, files);
+
+        emitProgress('Analysis', 50, '对模块结构聚类分析...');
+        const modules = await analyzeModules(workspacePath, files, llmClient);
+
+        emitProgress('Analysis', 60, '提取 API 路由及数据库模型定义...');
+        const apiRoutes = await analyzeApiRoutes(workspacePath, files);
+        const databaseModels = await analyzeDatabaseModels(workspacePath, files);
+
+        emitProgress('Analysis', 65, '分析 CI 流程、配置文件和 Agent 工作流...');
+        const configInfos = await analyzeConfigs(workspacePath, files);
+        const workflowInfos = await analyzeWorkflows(workspacePath, files);
+
+        const analysisResult: AnalysisResult = {
+            project: projectProfile,
+            tree: treeStr,
+            modules,
+            dependencies: dependencyGraph,
+            apiRoutes,
+            databaseModels,
+            wikiPages: [],
+        };
+
+        // ====================================================================
+        // 4. Planning Phase 阶段
+        // ====================================================================
+        emitProgress('Planning', 70, '制定 Wiki 页面规划方案...');
+        const generator = new WikiGenerator(llmClient, { outputDir, concurrency });
+        const plannedPages = await generator.planPages(analysisResult);
+
+        emitProgress('Planning', 75, `规划完成，共需要生成 ${plannedPages.length} 个 Wiki 页面。`);
+
+        // ====================================================================
+        // 5. Render Phase 阶段
+        // ====================================================================
+        const generatedPages: WikiPage[] = [];
+
+        // 构建快速交叉引用索引
+        const getExistingPagesSummary = () => {
+            return generatedPages.map((p) => `- ${p.title} (${p.filename})`).join('\n');
+        };
+
+        let currentProgress = 75;
+        const progressStep = (95 - 75) / Math.max(plannedPages.length, 1);
+
+        for (let i = 0; i < plannedPages.length; i++) {
+            const page = plannedPages[i];
+            currentProgress += progressStep;
+
+            emitProgress(
+                'LLM Inference',
+                Math.round(currentProgress),
+                `正在生成 (${i + 1}/${plannedPages.length}): ${page.title}...`
+            );
+
+            const wikiPage = await generator.generatePage(
+                workspacePath,
+                page,
+                analysisResult,
+                getExistingPagesSummary()
+            );
+
+            await generator.savePage(wikiPage);
+            generatedPages.push(wikiPage);
+        }
+
+        // ====================================================================
+        // 6. Post-flight 阶段
+        // ====================================================================
+        emitProgress('Post-flight', 96, '生成 Wiki 导航与侧边栏 _Sidebar.md...');
+
+        const sidebarContent = generateSidebar(generatedPages);
+        const homeContent = generateHome(generatedPages, projectProfile);
+
+        await fs.writeFile(path.join(outputDir, '_Sidebar.md'), sidebarContent, 'utf-8');
+        await fs.writeFile(path.join(outputDir, 'Home.md'), homeContent, 'utf-8');
+
+        // 将新页追加到分析结果中
+        analysisResult.wikiPages = generatedPages;
+
+        emitProgress('Post-flight', 100, '所有 Wiki 文档生成完成！');
+
+        const doneEvent = {
+            type: 'DONE' as const,
+            payload: {
+                docsPath: path.resolve(outputDir),
+                pagesCount: generatedPages.length + 2, // 包含 Home.md, _Sidebar.md
+            },
+        };
+
+        if (onProgress) {
+            onProgress(doneEvent);
+        } else {
+            console.log(JSON.stringify(doneEvent));
+        }
+
+        return analysisResult;
+    } catch (err: any) {
+        const errorEvent = {
+            type: 'ERROR' as const,
+            code: 500,
+            message: err?.message || String(err),
+        };
+
+        if (onProgress) {
+            onProgress(errorEvent);
+        } else {
+            console.log(JSON.stringify(errorEvent));
+        }
+
+        throw err;
+    }
+}

package/src/scanner/file-scanner.ts

@@ -0,0 +1,192 @@
+import fg from 'fast-glob';
+import { readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+
+import type { FileNode } from '../models/file-reference.js';
+import { createIgnoreFilter } from './ignore-rules.js';
+
+// ---------------------------------------------------------------------------
+// 语言映射表
+// ---------------------------------------------------------------------------
+
+/**
+ * 文件扩展名 → 语言标识的映射。
+ * 键为小写扩展名 (不含前导点)。
+ */
+const LANGUAGE_MAP: Record<string, string> = {
+    ts: 'typescript',
+    tsx: 'typescriptreact',
+    js: 'javascript',
+    jsx: 'javascriptreact',
+    py: 'python',
+    go: 'go',
+    rs: 'rust',
+    java: 'java',
+    rb: 'ruby',
+    css: 'css',
+    html: 'html',
+    md: 'markdown',
+    json: 'json',
+    yaml: 'yaml',
+    yml: 'yaml',
+    toml: 'toml',
+    sql: 'sql',
+    prisma: 'prisma',
+    vue: 'vue',
+    svelte: 'svelte',
+};
+
+/**
+ * 根据文件扩展名推断编程语言。
+ *
+ * @param filePath - 文件路径 (仅使用扩展名部分)
+ * @returns 语言标识字符串，无法识别时返回 `undefined`
+ */
+function detectLanguage(filePath: string): string | undefined {
+    const ext = path.extname(filePath).slice(1).toLowerCase();
+    return LANGUAGE_MAP[ext];
+}
+
+// ---------------------------------------------------------------------------
+// 行数统计
+// ---------------------------------------------------------------------------
+
+/**
+ * 统计文件中非空行数。
+ * 对大于 2 MB 的文件跳过行数统计以避免内存压力。
+ */
+async function countNonEmptyLines(
+    filePath: string,
+    sizeBytes: number,
+): Promise<number | undefined> {
+    // 跳过过大的文件
+    const MAX_SIZE_FOR_LINE_COUNT = 2 * 1024 * 1024; // 2 MB
+    if (sizeBytes > MAX_SIZE_FOR_LINE_COUNT) {
+        return undefined;
+    }
+
+    try {
+        const content = await readFile(filePath, 'utf-8');
+        return content.split(/\r?\n/).filter((line) => line.trim().length > 0).length;
+    } catch {
+        // 二进制文件或无法读取 —— 静默跳过
+        return undefined;
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Scanner 配置
+// ---------------------------------------------------------------------------
+
+/**
+ * 文件扫描器配置项。
+ */
+export interface ScannerConfig {
+    /** 最大递归深度 (默认 5) */
+    maxDepth: number;
+    /** 额外忽略模式 (glob) */
+    ignorePatterns: string[];
+    /** 仅包含指定扩展名 (不含前导点)。为空时包含所有文件 */
+    includeExtensions: string[];
+}
+
+/** 默认配置 */
+const DEFAULT_CONFIG: ScannerConfig = {
+    maxDepth: 5,
+    ignorePatterns: [],
+    includeExtensions: [],
+};
+
+// ---------------------------------------------------------------------------
+// 核心扫描函数
+// ---------------------------------------------------------------------------
+
+/**
+ * 递归扫描项目目录，返回扁平的 {@link FileNode} 列表。
+ *
+ * @param rootPath  - 项目根目录的绝对路径
+ * @param config    - 可选配置，覆盖默认值
+ * @returns 排序后的 FileNode 列表 (按相对路径字母序)
+ *
+ * @example
+ * ```ts
+ * const files = await scanDirectory('/path/to/project');
+ * console.log(files.length);
+ * ```
+ */
+export async function scanDirectory(
+    rootPath: string,
+    config?: Partial<ScannerConfig>,
+): Promise<FileNode[]> {
+    const cfg: ScannerConfig = { ...DEFAULT_CONFIG, ...config };
+
+    // 将 rootPath 规范化为 posix 风格以兼容 fast-glob
+    const normalizedRoot = rootPath.replace(/\\/g, '/');
+
+    // 构建忽略过滤器
+    const shouldIgnore = await createIgnoreFilter(rootPath);
+
+    // 构建 glob 模式
+    const pattern = `${normalizedRoot}/**/*`;
+
+    // fast-glob 选项
+    const entries = await fg(pattern, {
+        dot: false,
+        onlyFiles: true,
+        deep: cfg.maxDepth,
+        ignore: cfg.ignorePatterns,
+        absolute: true,
+        stats: false,
+    });
+
+    const nodes: FileNode[] = [];
+
+    for (const absPath of entries) {
+        // 计算相对路径 (posix 风格)
+        const relativePath = path
+            .relative(rootPath, absPath)
+            .replace(/\\/g, '/');
+
+        // 应用 ignore 过滤器
+        if (shouldIgnore(relativePath)) {
+            continue;
+        }
+
+        // 扩展名过滤
+        if (cfg.includeExtensions.length > 0) {
+            const ext = path.extname(absPath).slice(1).toLowerCase();
+            if (!cfg.includeExtensions.includes(ext)) {
+                continue;
+            }
+        }
+
+        // 获取文件信息
+        let fileStat;
+        try {
+            fileStat = await stat(absPath);
+        } catch {
+            // 文件在扫描期间被删除等异常情况 —— 静默跳过
+            continue;
+        }
+
+        const sizeBytes = fileStat.size;
+        const language = detectLanguage(absPath);
+        const lineCount = language
+            ? await countNonEmptyLines(absPath, sizeBytes)
+            : undefined;
+
+        nodes.push({
+            path: absPath,
+            relativePath,
+            nodeType: 'file',
+            sizeBytes,
+            language,
+            lineCount,
+        });
+    }
+
+    // 按相对路径排序
+    nodes.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
+
+    return nodes;
+}

package/src/scanner/ignore-rules.ts

@@ -0,0 +1,112 @@
+import ignore, { type Ignore } from 'ignore';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+// ---------------------------------------------------------------------------
+// 默认忽略规则
+// ---------------------------------------------------------------------------
+
+/**
+ * 默认忽略的目录 / 文件 glob 模式。
+ * 这些模式在任何项目中都应被忽略 (构建产物、依赖、缓存等)。
+ */
+export const DEFAULT_IGNORE_PATTERNS: readonly string[] = [
+    '.git/',
+    'node_modules/',
+    'dist/',
+    'build/',
+    '.next/',
+    'venv/',
+    '.venv/',
+    '__pycache__/',
+    '.pytest_cache/',
+    'coverage/',
+    '.cache/',
+    '.DS_Store',
+    '*.lock',
+    '*.min.js',
+    '*.map',
+];
+
+/**
+ * 敏感文件强制忽略模式。
+ * 为避免将密钥 / 凭据意外写入 Wiki，这些文件始终被排除。
+ */
+export const SENSITIVE_FILE_PATTERNS: readonly string[] = [
+    '.env',
+    '.env.local',
+    '.env.production',
+    '*.pem',
+    '*.key',
+    'id_rsa',
+    'id_ed25519',
+    'secrets.yaml',
+    'credentials.json',
+];
+
+// ---------------------------------------------------------------------------
+// 辅助函数
+// ---------------------------------------------------------------------------
+
+/**
+ * 尝试读取项目根目录下的 `.gitignore` 文件，并返回其内容行。
+ * 如果文件不存在或读取失败，返回空数组。
+ */
+async function loadGitignoreLines(projectRoot: string): Promise<string[]> {
+    try {
+        const content = await readFile(
+            path.join(projectRoot, '.gitignore'),
+            'utf-8',
+        );
+        return content
+            .split(/\r?\n/)
+            .map((line) => line.trim())
+            .filter((line) => line.length > 0 && !line.startsWith('#'));
+    } catch {
+        // .gitignore 不存在或不可读 —— 静默跳过
+        return [];
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 核心 API
+// ---------------------------------------------------------------------------
+
+/**
+ * 创建一个用于判断文件路径是否应被忽略的过滤函数。
+ *
+ * 合并以下三类规则:
+ * 1. 项目根目录下的 `.gitignore`
+ * 2. {@link DEFAULT_IGNORE_PATTERNS}
+ * 3. {@link SENSITIVE_FILE_PATTERNS}
+ *
+ * @param projectRoot - 项目根目录的绝对路径
+ * @returns 一个判定函数。传入**相对路径**，返回 `true` 代表该路径应被忽略。
+ *
+ * @example
+ * ```ts
+ * const shouldIgnore = await createIgnoreFilter('/path/to/project');
+ * if (shouldIgnore('node_modules/foo/index.js')) {
+ *     // 跳过
+ * }
+ * ```
+ */
+export async function createIgnoreFilter(
+    projectRoot: string,
+): Promise<(relativePath: string) => boolean> {
+    const ig: Ignore = (((ignore as any).default || ignore) as any)();
+
+    // 1. 加载 .gitignore
+    const gitignoreLines = await loadGitignoreLines(projectRoot);
+    if (gitignoreLines.length > 0) {
+        ig.add(gitignoreLines);
+    }
+
+    // 2. 默认忽略模式
+    ig.add(DEFAULT_IGNORE_PATTERNS as string[]);
+
+    // 3. 敏感文件模式
+    ig.add(SENSITIVE_FILE_PATTERNS as string[]);
+
+    return (relativePath: string): boolean => ig.ignores(relativePath);
+}

package/src/scanner/index.ts

@@ -0,0 +1,19 @@
+/**
+ * @module scanner
+ * 文件扫描器统一导出
+ */
+
+export {
+    DEFAULT_IGNORE_PATTERNS,
+    SENSITIVE_FILE_PATTERNS,
+    createIgnoreFilter,
+} from './ignore-rules.js';
+
+export {
+    type ScannerConfig,
+    scanDirectory,
+} from './file-scanner.js';
+
+export {
+    buildTreeString,
+} from './tree-builder.js';

package/src/scanner/tree-builder.ts

@@ -0,0 +1,156 @@
+import path from 'node:path';
+
+import type { FileNode } from '../models/file-reference.js';
+
+// ---------------------------------------------------------------------------
+// 内部数据结构
+// ---------------------------------------------------------------------------
+
+/** 目录树中的一个节点 (中间表示) */
+interface TreeNode {
+    /** 节点名称 (文件名或目录名) */
+    name: string;
+    /** 是否为目录 */
+    isDir: boolean;
+    /** 子节点 (仅目录有效) */
+    children: Map<string, TreeNode>;
+}
+
+// ---------------------------------------------------------------------------
+// 辅助函数
+// ---------------------------------------------------------------------------
+
+/**
+ * 将扁平文件列表构建为嵌套树结构。
+ */
+function buildTree(files: FileNode[], rootPath: string): TreeNode {
+    const root: TreeNode = {
+        name: path.basename(rootPath),
+        isDir: true,
+        children: new Map(),
+    };
+
+    for (const file of files) {
+        const segments = file.relativePath.split('/');
+        let current = root;
+
+        for (let i = 0; i < segments.length; i++) {
+            const seg = segments[i];
+            const isLast = i === segments.length - 1;
+
+            if (!current.children.has(seg)) {
+                current.children.set(seg, {
+                    name: seg,
+                    isDir: !isLast,
+                    children: new Map(),
+                });
+            }
+
+            current = current.children.get(seg)!;
+        }
+    }
+
+    return root;
+}
+
+/**
+ * 统计树中所有叶子文件数量。
+ */
+function countFiles(node: TreeNode): number {
+    if (!node.isDir || node.children.size === 0) {
+        return node.isDir ? 0 : 1;
+    }
+    let count = 0;
+    for (const child of node.children.values()) {
+        count += countFiles(child);
+    }
+    return count;
+}
+
+/**
+ * 统计树中所有目录数量 (不含根节点自身)。
+ */
+function countDirectories(node: TreeNode): number {
+    let count = 0;
+    for (const child of node.children.values()) {
+        if (child.isDir) {
+            count += 1 + countDirectories(child);
+        }
+    }
+    return count;
+}
+
+/**
+ * 递归渲染树字符串。
+ *
+ * @param node   - 当前节点
+ * @param prefix - 当前行前缀 (用于缩进)
+ * @param lines  - 输出行收集器
+ */
+function renderNode(
+    node: TreeNode,
+    prefix: string,
+    lines: string[],
+): void {
+    // 对子节点排序: 目录在前，同类按名称排序
+    const sorted = [...node.children.entries()].sort(([, a], [, b]) => {
+        if (a.isDir !== b.isDir) {
+            return a.isDir ? -1 : 1;
+        }
+        return a.name.localeCompare(b.name);
+    });
+
+    for (let i = 0; i < sorted.length; i++) {
+        const [, child] = sorted[i];
+        const isLast = i === sorted.length - 1;
+        const connector = isLast ? '\u2514\u2500\u2500 ' : '\u251C\u2500\u2500 ';
+        const childPrefix = isLast ? '    ' : '\u2502   ';
+
+        const displayName = child.isDir ? `${child.name}/` : child.name;
+        lines.push(`${prefix}${connector}${displayName}`);
+
+        if (child.isDir && child.children.size > 0) {
+            renderNode(child, prefix + childPrefix, lines);
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 核心 API
+// ---------------------------------------------------------------------------
+
+/**
+ * 将扁平的 {@link FileNode} 列表渲染为可读的树形文本，类似 `tree` 命令输出。
+ *
+ * 使用 Unicode box-drawing 字符绘制分支 (├──, └──, │)。
+ * 输出末尾附带文件数 / 目录数汇总。
+ *
+ * @param files    - 扁平的 FileNode 列表 (由 `scanDirectory` 返回)
+ * @param rootPath - 项目根目录的绝对路径
+ * @returns 多行字符串形式的目录树
+ *
+ * @example
+ * ```ts
+ * const files = await scanDirectory('/project');
+ * const tree  = buildTreeString(files, '/project');
+ * console.log(tree);
+ * ```
+ */
+export function buildTreeString(files: FileNode[], rootPath: string): string {
+    if (files.length === 0) {
+        const rootName = path.basename(rootPath);
+        return `${rootName}/\n\n0 directories, 0 files`;
+    }
+
+    const root = buildTree(files, rootPath);
+    const lines: string[] = [`${root.name}/`];
+
+    renderNode(root, '', lines);
+
+    const dirCount = countDirectories(root);
+    const fileCount = countFiles(root);
+    lines.push('');
+    lines.push(`${dirCount} directories, ${fileCount} files`);
+
+    return lines.join('\n');
+}