zengen 0.1.36 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zengen",
-  "version": "0.1.36",
+  "version": "0.2.0",
   "description": "ZEN - A minimalist Markdown documentation site builder",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -10,6 +10,7 @@
   "scripts": {
     "build": "npx rimraf dist && tsc",
     "dev": "ts-node src/cli.ts",
+    "build:doc": "npm run build && node dist/cli.js build --lang zh-Hans --lang en-US --verbose",
     "test": "npm run build && node --test dist/**/*.test.js",
     "test:types": "tsc --noEmit",
     "test:build": "npm run build && test -f dist/index.js && test -f dist/cli.js",
@@ -58,6 +59,7 @@
     "express": "^4.21.2",
     "highlight.js": "^11.11.1",
     "marked": "^17.0.1",
-    "minimatch": "^10.1.1"
+    "minimatch": "^10.1.1",
+    "yaml": "^2.8.2"
   }
 }

package/src/ai/extractMetadataFromMarkdown.ts

@@ -0,0 +1,95 @@
+import { completeMessages, OpenAIMessage } from '../services/openai';
+import { AIMetadata } from '../types';
+
+/**
+ * 从 markdown 内容中提取 metadata
+ * @param content Markdown 内容
+ * @returns Promise<AIMetadata> 提取的元数据，失败时抛出错误
+ */
+export async function extractMetadataFromMarkdown(content: string): Promise<AIMetadata> {
+  const prompt = buildMetadataPrompt(content);
+  const messages: OpenAIMessage[] = [
+    {
+      role: 'system',
+      content:
+        '你是一个专业的文档分析助手，擅长从文档中提取结构化信息。请严格按照要求的 JSON 格式返回结果。',
+    },
+    {
+      role: 'user',
+      content: prompt,
+    },
+  ];
+
+  const response = await completeMessages(messages, {
+    response_format: { type: 'json_object' },
+  });
+
+  const metadata = parseMetadataResponse(response.choices[0].message.content);
+
+  // 添加 tokens 使用情况
+  metadata.tokens_used = {
+    prompt: response.usage.prompt_tokens,
+    completion: response.usage.completion_tokens,
+    total: response.usage.total_tokens,
+  };
+
+  return metadata;
+}
+
+/**
+ * 构建提取 metadata 的 prompt
+ */
+function buildMetadataPrompt(content: string): string {
+  // 限制内容长度以避免 token 超限
+  const maxContentLength = 8000;
+  const truncatedContent =
+    content.length > maxContentLength
+      ? content.substring(0, maxContentLength) + '... [内容已截断]'
+      : content;
+
+  return `请分析以下文档内容，提取以下信息并返回 JSON 格式：
+
+文档内容：
+"""
+${truncatedContent}
+"""
+
+请提取：
+1. title: 文档的标题（简洁明了，不超过 20 个字）
+2. summary: 文档摘要（控制在 100 字以内，概括主要内容）
+3. tags: 关键词列表（3-8 个关键词，使用中文或英文）
+4. inferred_date: 文档中隐含的创建日期（如果有的话，格式：YYYY-MM-DD，没有就留空字符串）
+5. inferred_lang: 文档使用的语言代码（例如：zh-Hans 表示简体中文，en-US 表示美式英语）
+
+请严格按照以下 JSON 格式返回，不要包含任何其他文本：
+{
+  "title": "文档标题",
+  "summary": "文档摘要...",
+  "tags": ["关键词1", "关键词2", "关键词3"],
+  "inferred_date": "2023-01-01",
+  "inferred_lang": "zh-Hans"
+}`;
+}
+
+/**
+ * 解析 AI 返回的 metadata
+ */
+function parseMetadataResponse(responseContent: string): AIMetadata {
+  try {
+    const metadata = JSON.parse(responseContent);
+
+    // 验证和清理数据
+    return {
+      title: metadata.title?.trim() || '未命名文档',
+      summary: metadata.summary?.trim() || '',
+      tags: Array.isArray(metadata.tags)
+        ? metadata.tags.map((tag: string) => tag.trim()).filter(Boolean)
+        : [],
+      inferred_date: metadata.inferred_date?.trim() || undefined,
+      inferred_lang: metadata.inferred_lang?.trim() || 'zh-Hans',
+    };
+  } catch (error) {
+    console.error('❌ Failed to parse AI response:', error, 'Response:', responseContent);
+    throw error;
+  }
+}

package/src/ai/translateMarkdown.ts

@@ -0,0 +1,29 @@
+import { completeMessages, OpenAIMessage } from '../services/openai';
+
+/**
+ * 将 markdown 翻译为指定的语言
+ * @param content Markdown 内容
+ * @param targetLang 目标语言代码（例如：zh-Hans, en-US）
+ * @returns Promise<string> 翻译后的 Markdown 内容
+ */
+export async function translateMarkdown(content: string, targetLang: string): Promise<string> {
+  const messages: OpenAIMessage[] = [
+    {
+      role: 'system',
+      content: `你是一个专业的翻译助手，擅长将文档翻译成不同语言，同时保持原有的格式和结构。请将用户输入翻译成 ${targetLang}，注意保持 Markdown 格式不变，链接不变，不要翻译代码，但是可以翻译代码中的注释。`,
+    },
+    {
+      role: 'user',
+      content: content,
+    },
+  ];
+
+  const response = await completeMessages(messages);
+  const translatedContent = response.choices[0]?.message?.content?.trim() || '';
+
+  if (!translatedContent) {
+    throw new Error('Empty translation response');
+  }
+
+  return translatedContent;
+}

package/src/build/pipeline.ts

@@ -0,0 +1,211 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { translateMarkdown } from '../ai/translateMarkdown';
+import { findMarkdownEntries } from '../findEntries';
+import { loadMetaData, MetaData, saveMetaData } from '../metadata';
+import { INPUT_DIR, ZEN_DIR, ZEN_DIST_DIR, ZEN_SRC_DIR } from '../paths';
+import { extractMetadataByAI } from '../process/extractMetadataByAI';
+import { renderTemplates } from '../process/template';
+import { calculateFileHash } from '../scan/files';
+import { BuildOptions } from '../types';
+import { updateFrontmatter } from '../utils/frontmatter';
+
+/**
+ * 验证构建配置
+ */
+async function validateConfig(options: BuildOptions): Promise<void> {
+  const { verbose = false } = options;
+
+  if (verbose) {
+    console.log(`🚀 Starting ZEN build...`);
+    console.log(`🔗 Base URL: ${options.baseUrl || '(not set)'}`);
+    if (options.langs && options.langs.length > 0) {
+      console.log(`🌐 Target languages: ${options.langs.join(', ')}`);
+    }
+    console.log(`🔍 Verbose mode enabled`);
+  }
+
+  MetaData.options = options;
+}
+
+/**
+ * 扫描源文件
+ */
+async function scanSourceFiles(): Promise<void> {
+  console.log(`🔍 Scanning source directory...`);
+  const markdownFiles = await findMarkdownEntries(INPUT_DIR);
+  const hashes = new Set<string>();
+
+  for (const relativePath of markdownFiles) {
+    const fullPath = path.join(INPUT_DIR, relativePath);
+
+    try {
+      // 检查文件是否存在
+      await fs.access(fullPath);
+
+      const hash = await calculateFileHash(fullPath);
+
+      hashes.add(hash);
+
+      const metaWithSameHash = MetaData.files.find(f => f.hash === hash);
+      if (metaWithSameHash) {
+        metaWithSameHash.path = relativePath;
+      } else {
+        // 如果没有相同哈希的元数据，则添加一个新的占位符
+        MetaData.files.push({
+          hash,
+          path: relativePath,
+        });
+      }
+    } catch (error) {
+      console.warn(`⚠️ File not found or inaccessible: ${fullPath}`, error);
+    }
+  }
+  // 移除不再存在的文件元数据
+  MetaData.files = MetaData.files.filter(f => hashes.has(f.hash));
+
+  console.log(`✅ Found ${MetaData.files.length} Markdown files`);
+
+  if (MetaData.files.length === 0) {
+    console.warn(`⚠️ No Markdown files found in ${INPUT_DIR}`);
+  }
+}
+
+/**
+ * 存储母语文件到 .zen/src
+ */
+async function storeNativeFiles(): Promise<void> {
+  const {
+    options: { verbose },
+    files,
+  } = MetaData;
+  for (const file of MetaData.files) {
+    try {
+      if (!file.hash) throw new Error(`Missing hash`);
+      if (!file.metadata?.inferred_lang) throw new Error(`Missing inferred language`);
+      const filePath = path.join(ZEN_SRC_DIR, file.metadata.inferred_lang, file.hash + '.md');
+      const originalContent = await fs.readFile(path.join(INPUT_DIR, file.path), 'utf-8');
+
+      const enhancedContent = updateFrontmatter(originalContent, {
+        title: file.metadata.title,
+        summary: file.metadata.summary,
+        tags: file.metadata.tags,
+        inferred_date: file.metadata.inferred_date,
+        inferred_lang: file.metadata.inferred_lang,
+      });
+      await fs.mkdir(path.dirname(filePath), { recursive: true });
+
+      await fs.writeFile(filePath, enhancedContent, 'utf-8');
+    } catch (error) {
+      console.warn(`⚠️ Failed to store native file ${file.path}:`, error);
+    }
+  }
+
+  if (verbose && files.length > 0) {
+    console.log(`💾 Stored ${files.length} native language files to .zen/src`);
+  }
+}
+
+/**
+ * 处理翻译
+ */
+async function processTranslations(): Promise<void> {
+  const {
+    files,
+    options: { langs = [], verbose },
+  } = MetaData;
+
+  for (const file of files) {
+    if (verbose) console.info(`📄 Processing file for translation: ${file.path}`);
+    if (!file.metadata) {
+      console.warn(`⚠️ Missing metadata for file: ${file.path}, skipping translation.`);
+      continue;
+    }
+    for (const lang of langs) {
+      if (verbose) console.log(`🌐 Translating to ${lang}...`);
+      // 存储翻译文件到 .zen/src/{lang}
+      const sourcePath = path.join(ZEN_SRC_DIR, file.metadata.inferred_lang, file.hash + '.md'); // 使用已经加强的母语文件路径
+      const targetPath = path.join(ZEN_SRC_DIR, lang, file.hash + '.md');
+
+      try {
+        const content = await fs.readFile(sourcePath, 'utf-8');
+        if (file.metadata.inferred_lang === lang) {
+          if (verbose)
+            console.log(`ℹ️ Skipping translation for ${file.path}, already in target language`);
+          continue;
+        } else {
+          // 翻译
+          // 先检查是否已经有翻译文件存在
+
+          const exists = await fs.access(targetPath).then(
+            () => true,
+            () => false
+          );
+          if (exists) {
+            if (verbose) console.log(`ℹ️ Translation already exists for ${file.path} in ${lang}`);
+            continue;
+          }
+        }
+
+        const translatedContent = await translateMarkdown(content, lang);
+
+        await fs.mkdir(path.dirname(targetPath), { recursive: true });
+        await fs.writeFile(targetPath, translatedContent, 'utf-8');
+
+        if (verbose) console.log(`✅ Translated file saved: ${targetPath}`);
+      } catch (error) {
+        console.error(`❌ Failed to translate to ${lang}:`, error);
+      }
+    }
+  }
+}
+
+/**
+ * 构建管道（函数组合）
+ */
+async function buildPipeline(options: BuildOptions): Promise<void> {
+  // 验证配置
+  await validateConfig(options);
+
+  // 清理输出目录
+  await fs.rm(ZEN_DIST_DIR, { recursive: true, force: true });
+
+  // 确保 .zen/.gitignore 文件
+  await fs.writeFile(path.join(ZEN_DIR, '.gitignore'), 'dist\n', 'utf-8');
+
+  // 扫描源文件
+  await scanSourceFiles();
+
+  // 运行 AI 元数据提取
+  await extractMetadataByAI();
+
+  // 存储母语文件
+  await storeNativeFiles();
+
+  // 处理翻译
+  await processTranslations();
+
+  // 渲染模板
+  await renderTemplates();
+}
+
+/**
+ * 主构建函数
+ */
+export async function buildSite(options: BuildOptions): Promise<void> {
+  const startTime = Date.now();
+
+  try {
+    await loadMetaData();
+    await buildPipeline(options);
+
+    const endTime = Date.now();
+    const duration = ((endTime - startTime) / 1000).toFixed(2);
+    console.log(`🎉 Build completed in ${duration}s`);
+  } catch (error) {
+    console.error(`❌ Build failed:`, error);
+    throw error;
+  } finally {
+    await saveMetaData();
+  }
+}

package/src/cli.ts

@@ -1,65 +1,28 @@
 #!/usr/bin/env node
 
-import { config } from 'dotenv';
 import { Cli, Command, Option } from 'clipanion';
-import { ZenBuilder } from './builder';
-import { ZenConfig } from './types';
+import { config } from 'dotenv';
+import * as fs from 'fs';
 import * as path from 'path';
-import * as fs from 'fs/promises';
-import * as fsSync from 'fs';
-import * as url from 'url';
+import { buildSite } from './build/pipeline';
 
 // 加载 .env 文件中的环境变量
 config();
 
 // 获取版本号 - 从 package.json 读取
 function getVersion(): string {
-  try {
-    const packageJsonPath = path.join(__dirname, '..', 'package.json');
-    const packageJson = JSON.parse(fsSync.readFileSync(packageJsonPath, 'utf-8'));
-    return packageJson.version;
-  } catch {
-    return '0.1.32';
-  }
-}
-
-// 基础命令类
-abstract class BaseCommand extends Command {
-  protected async loadConfig(configPath?: string): Promise<ZenConfig> {
-    if (!configPath) {
-      return {};
-    }
-
-    try {
-      const resolvedPath = path.resolve(configPath);
-      const configContent = await fs.readFile(resolvedPath, 'utf-8');
-      return JSON.parse(configContent);
-    } catch (error) {
-      this.context.stderr.write(`❌ Failed to load config file: ${error}\n`);
-      throw error;
-    }
-  }
-
-  protected getOutDir(): string {
-    const currentDir = process.cwd();
-    return path.join(currentDir, '.zen', 'dist');
-  }
+  const packageJsonPath = path.join(__dirname, '..', 'package.json');
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
+  return packageJson.version;
 }
 
 // Build 命令
-class BuildCommand extends BaseCommand {
+class BuildCommand extends Command {
   static paths = [['build']];
 
   template = Option.String('-t,--template');
-  watch = Option.Boolean('-w,--watch');
-  serve = Option.Boolean('-s,--serve');
-  port = Option.String('-p,--port', '3000');
-  host = Option.String('--host', 'localhost');
   verbose = Option.Boolean('-v,--verbose');
-  config = Option.String('-c,--config');
   baseUrl = Option.String('--base-url');
-  clean = Option.Boolean('--clean');
-  ai = Option.Boolean('--ai', { description: 'Enable AI metadata extraction' });
   lang = Option.Array('--lang', {
     description: 'Target languages for translation (e.g., en-US, ja-JP)',
   });
@@ -72,103 +35,18 @@ class BuildCommand extends BaseCommand {
 
       Examples:
         $ zengen build
-        $ zengen build --watch
-        $ zengen build --watch --serve
-        $ zengen build --watch --serve --port 8080
-        $ zengen build --config zen.config.json
-        $ zengen build --clean
-        $ zengen build --ai (requires OPENAI_API_KEY environment variable)
         $ zengen build --lang en-US --lang ja-JP (translate to English and Japanese)
     `,
   });
 
   async execute() {
     try {
-      // 加载配置文件
-      const config = await this.loadConfig(this.config);
-
-      // 强制使用当前目录作为 src 目录，输出到 .zen/dist 目录
-      const currentDir = process.cwd();
-      const outDir = this.getOutDir();
-
-      // 处理 AI 配置：如果指定了 --ai 参数，启用 AI；否则使用配置中的设置
-      const aiConfig = {
-        ...config.ai,
-        enabled: this.ai ? true : config.ai?.enabled,
-      };
-
-      // 处理语言配置：命令行参数优先于配置文件
-      const targetLangs = this.lang && this.lang.length > 0 ? this.lang : config.i18n?.targetLangs;
-      const i18nConfig =
-        targetLangs && targetLangs.length > 0
-          ? {
-              ...config.i18n,
-              sourceLang: config.i18n?.sourceLang || 'zh-Hans', // 默认源语言为中文
-              targetLangs,
-            }
-          : undefined;
-
-      // 合并命令行参数和配置
-      const buildOptions = {
-        srcDir: currentDir,
-        outDir: outDir,
+      await buildSite({
         template: this.template ? path.resolve(this.template) : undefined,
-        watch: this.watch,
-        serve: this.serve,
-        port: parseInt(this.port, 10),
-        host: this.host,
         verbose: this.verbose,
-        baseUrl: this.baseUrl || config.baseUrl,
+        baseUrl: this.baseUrl,
         langs: this.lang,
-      };
-
-      // 创建最终的配置，包含 AI 和 i18n 设置
-      const finalConfig = {
-        ...config,
-        ai: aiConfig.enabled ? aiConfig : undefined,
-        i18n: i18nConfig,
-      };
-
-      const builder = new ZenBuilder(finalConfig);
-
-      // 验证配置
-      const errors = builder.validateConfig(finalConfig);
-      if (errors.length > 0) {
-        this.context.stderr.write('❌ Configuration errors:\n');
-        errors.forEach(error => this.context.stderr.write(`  - ${error}\n`));
-        return 1;
-      }
-
-      // 警告 --serve 选项需要 --watch 选项
-      if (this.serve && !this.watch) {
-        this.context.stdout.write(
-          '⚠️ Warning: --serve option requires --watch option, ignoring --serve\n'
-        );
-        buildOptions.serve = false;
-      }
-
-      // 清理输出目录
-      if (this.clean) {
-        await builder.clean(buildOptions.outDir);
-      }
-
-      // 构建或监听
-      if (this.watch) {
-        await builder.watch(buildOptions);
-      } else {
-        // 如果指定了语言参数，使用多语言构建
-        if (this.lang && this.lang.length > 0) {
-          const multiLangOptions = {
-            ...buildOptions,
-            langs: this.lang,
-            useMetaData: true,
-            filterOrphans: true,
-          };
-          await builder.buildMultiLang(multiLangOptions);
-        } else {
-          await builder.build(buildOptions);
-        }
-      }
+      });
 
       return 0;
     } catch (error) {

package/src/findEntries.ts

@@ -0,0 +1,37 @@
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+const execAsync = promisify(exec);
+
+/**
+ * 使用git命令查找所有Markdown文件
+ * 使用git ls-files --others --cached --exclude-standard获取所有文件
+ * 然后过滤掉.zen目录和只保留.md文件
+ *
+ * @param dirPath 要扫描的目录路径
+ * @returns Promise<string[]> 返回Markdown文件的相对路径数组
+ */
+export const findMarkdownEntries = async (dirPath: string): Promise<string[]> => {
+  try {
+    // 使用git命令获取所有文件（包括已跟踪和未跟踪的文件）
+    // 在指定的目录下执行git命令
+    const { stdout } = await execAsync('git ls-files --others --cached --exclude-standard', {
+      cwd: dirPath,
+    });
+
+    // 按行分割并过滤
+    const files = stdout
+      .split('\n')
+      .filter(line => line.trim() !== '') // 移除空行
+      .filter(file => !file.startsWith('.zen')) // 过滤掉.zen目录下的文件
+      .filter(file => file.endsWith('.md')); // 只保留.md文件
+
+    return files;
+  } catch (error) {
+    console.error('Error finding markdown entries:', error);
+
+    // 如果git命令失败，返回空数组
+    // 这可以处理没有git仓库或git不可用的情况
+    return [];
+  }
+};

package/src/index.ts

@@ -1,40 +1 @@
-export { ZenBuilder } from './builder';
-export { MarkdownConverter } from './markdown';
-export { TemplateEngine } from './template';
-export { NavigationGenerator } from './navigation';
-export type {
-  BuildOptions,
-  FileInfo,
-  NavigationItem,
-  TemplateData,
-  MarkdownProcessor,
-  ZenConfig,
-} from './types';
-
-/**
- * ZEN 文档构建工具
- *
- * 一个极简主义的 Markdown 文档站点生成器
- *
- * @example
- * ```typescript
- * import { ZenBuilder } from 'zengen';
- *
- * const builder = new ZenBuilder();
- * await builder.build({
- *   srcDir: './docs',
- *   outDir: './dist'
- * });
- * ```
- */
-import { ZenBuilder } from './builder';
-import { MarkdownConverter } from './markdown';
-import { TemplateEngine } from './template';
-import { NavigationGenerator } from './navigation';
-
-export default {
-  Builder: ZenBuilder,
-  MarkdownConverter: MarkdownConverter,
-  TemplateEngine: TemplateEngine,
-  NavigationGenerator: NavigationGenerator,
-};
+import './cli';

package/src/metadata.ts

@@ -0,0 +1,44 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { ZEN_META_PATH } from './paths';
+import { MetaDataStore } from './types';
+
+/**
+ * 全局 MetaDataStore 单例
+ */
+export const MetaData: MetaDataStore = {
+  // 稍后覆盖
+  version: '1.0.0',
+  options: {},
+  files: [],
+};
+
+/**
+ * 从文件中读取数据，覆盖 store，但是要保持它仍然是同一个对象
+ */
+export async function loadMetaData(): Promise<void> {
+  try {
+    await fs.access(ZEN_META_PATH);
+    const content = await fs.readFile(ZEN_META_PATH, 'utf-8');
+    const newData = JSON.parse(content);
+
+    // 使用 Object.assign 保持同一个对象引用
+    Object.assign(MetaData, newData);
+  } catch (error) {
+    // 如果文件不存在，初始化默认值
+    MetaData.version = '1.0.0';
+    MetaData.files = [];
+  }
+}
+
+/**
+ * 将 MetaData 写入 store
+ */
+export async function saveMetaData(): Promise<void> {
+  // 确保 .zen 目录存在
+  const zenDir = path.dirname(ZEN_META_PATH);
+  await fs.mkdir(zenDir, { recursive: true });
+
+  // 保存文件
+  await fs.writeFile(ZEN_META_PATH, JSON.stringify(MetaData, null, 2), 'utf-8');
+}

package/src/paths.ts

@@ -0,0 +1,7 @@
+import { join } from 'path';
+
+export const INPUT_DIR = process.cwd();
+export const ZEN_DIR = join(process.cwd(), '.zen');
+export const ZEN_DIST_DIR = join(ZEN_DIR, 'dist');
+export const ZEN_SRC_DIR = join(ZEN_DIR, 'src');
+export const ZEN_META_PATH = join(ZEN_DIR, 'meta.json');