solo-doc 0.1.0 → 0.2.0

package/README.md

@@ -71,6 +71,47 @@ solo-doc "http://10.1.2.3/docs/index.html" --type acp
 | `--limit <number>` | 限制爬取的页面数量 (用于测试/调试)。 | 无限制 |
 | `--no-headless` | 在可见模式下运行浏览器 (仅限 ACP，用于调试)。 | Headless (无头模式) |
 
+## 🧪 实验性功能 (Beta)
+
+### 🤖 文档对比 (AI VS 模式)
+
+使用本地 AI 模型对比两个文档的内容差异和结构差异。此功能处于 Beta 阶段。
+
+> **⚠️ 前置要求**:
+> 1. 本地已安装并运行 [Ollama](https://ollama.com/)。
+> 2. 已拉取所需的模型（推荐 `qwen3-vl:8b` 或类似多模态/大文本模型）。
+> 3. 确保 Ollama 服务监听在 `http://127.0.0.1:11434`。
+
+#### 用法
+
+```bash
+solo-doc vs <baseline-url> <target-url> [options]
+```
+
+#### 示例
+
+```bash
+# 对比 OpenShift 和 Alauda 的文档
+solo-doc vs \
+  "https://docs.redhat.com/en/documentation/openshift_container_platform/4.20/html-single/building_applications/index" \
+  "https://docs.alauda.io/container_platform/4.2/developer/building_application/index.html" \
+  --model qwen3-vl:8b
+```
+
+此命令将按顺序执行：
+1. **自动爬取**: 分别爬取两个 URL 并保存为 Markdown 文件（如果已存在则跳过）。
+2. **提取目录**: 提取两个文档的目录树结构。
+3. **AI 分析**: 调用本地 Ollama 模型，根据 `solo-doc-prompt.md` 定义的提示词进行两步分析：
+   - 生成 `vs-result.md`: 详细的内容与结构差异分析。
+   - 生成 `vs-tree.md`: 包含差异标注的合并目录树。
+
+#### VS 模式选项
+
+| 选项 | 描述 | 默认值 |
+|--------|-------------|---------|
+| `--model <name>` | 指定使用的 Ollama 模型名称。 | `qwen3-vl:8b` |
+| `-f, --force` | 强制重新爬取文档，即使文件已存在。 | false |
+
 ## ✅ 环境要求
 
 - Node.js >= 20

package/dist/bin/solo-doc.js

@@ -9,18 +9,59 @@ const CrawlerContext_1 = require("../src/CrawlerContext");
 const OCPStrategy_1 = require("../src/strategies/OCPStrategy");
 const ACPStrategy_1 = require("../src/strategies/ACPStrategy");
 const StrategyDetector_1 = require("../src/utils/StrategyDetector");
+const filename_1 = require("../src/utils/filename");
+const VSCommand_1 = require("../src/commands/VSCommand");
 const chalk_1 = __importDefault(require("chalk"));
 const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
 const program = new commander_1.Command();
+const getPackageVersion = () => {
+    try {
+        // Try production path first (dist/bin -> root)
+        const prodPath = path_1.default.resolve(__dirname, '../../package.json');
+        if (fs_1.default.existsSync(prodPath)) {
+            return JSON.parse(fs_1.default.readFileSync(prodPath, 'utf-8')).version;
+        }
+        // Try dev path (bin -> root)
+        const devPath = path_1.default.resolve(__dirname, '../package.json');
+        if (fs_1.default.existsSync(devPath)) {
+            return JSON.parse(fs_1.default.readFileSync(devPath, 'utf-8')).version;
+        }
+        return '1.0.0';
+    }
+    catch (e) {
+        return '1.0.0';
+    }
+};
 program
     .name('solo-doc')
     .description('CLI to crawl documentation sites and convert to single Markdown file')
-    .version('1.0.0')
+    .version(getPackageVersion());
+// VS Command
+program
+    .command('vs')
+    .description('Compare two documentation sites using AI (Beta)')
+    .argument('<baseline>', 'Baseline documentation URL')
+    .argument('<target>', 'Target documentation URL')
+    .option('--model <model>', 'Ollama model to use', 'qwen3-vl:8b')
+    .action(async (baseline, target, options) => {
+    try {
+        console.log(chalk_1.default.yellow('⚠️  [Beta Feature] This feature requires a local Ollama instance running at http://127.0.0.1:11434'));
+        await VSCommand_1.VSCommand.run(baseline, target, options);
+    }
+    catch (error) {
+        console.error(chalk_1.default.red(`[VS Mode] Failed: ${error.message}`));
+        process.exit(1);
+    }
+});
+// Default Crawl Command (Implicit)
+program
     .argument('<url>', 'The documentation URL to crawl')
     .option('-t, --type <type>', 'Force specify strategy type (ocp, acp)')
     .option('-o, --output <path>', 'Output file path')
     .option('--limit <number>', 'Limit number of pages (for debug)', parseInt)
     .option('--no-headless', 'Run in headful mode (show browser) - Only for ACP')
+    .option('-f, --force', 'Force overwrite existing file')
     .action(async (url, options) => {
     try {
         // 1. Determine Strategy
@@ -39,33 +80,27 @@ program
         }
         // 2. Instantiate Strategy
         let strategy;
-        // Helper function to generate default filename from URL
-        const generateDefaultFilename = (urlStr, typePrefix) => {
-            try {
-                const u = new URL(urlStr);
-                // Get the last path segment that isn't 'index.html' or 'index' or empty
-                const segments = u.pathname.split('/').filter(s => s && s !== 'index.html' && s !== 'index');
-                const lastSegment = segments.length > 0 ? segments[segments.length - 1] : 'docs';
-                // Sanitize filename
-                const safeName = lastSegment.replace(/[^a-zA-Z0-9-_]/g, '_');
-                return `${typePrefix}-${safeName}.md`;
-            }
-            catch (e) {
-                return `${typePrefix}-docs.md`;
-            }
-        };
         let defaultOutput;
         if (type === 'ocp' || type === StrategyDetector_1.StrategyType.OCP) {
             strategy = new OCPStrategy_1.OCPStrategy();
-            defaultOutput = generateDefaultFilename(url, 'ocp');
+            defaultOutput = (0, filename_1.generateDefaultFilename)(url, 'ocp');
         }
         else {
             strategy = new ACPStrategy_1.ACPStrategy();
-            defaultOutput = generateDefaultFilename(url, 'acp');
+            defaultOutput = (0, filename_1.generateDefaultFilename)(url, 'acp');
         }
         // 3. Prepare Context
         const context = new CrawlerContext_1.CrawlerContext(strategy);
         const outputPath = path_1.default.resolve(process.cwd(), options.output || defaultOutput);
+        // Check if file exists
+        if (fs_1.default.existsSync(outputPath) && !options.force) {
+            console.log(chalk_1.default.yellow('--------------------------------------------------'));
+            console.log(chalk_1.default.yellow(`ℹ  File already exists: ${outputPath}`));
+            console.log(chalk_1.default.yellow('   Skipping crawl to save time.'));
+            console.log(chalk_1.default.gray('   Use --force or -f to overwrite.'));
+            console.log(chalk_1.default.yellow('--------------------------------------------------'));
+            return;
+        }
         // 4. Run
         await context.run(url, {
             output: outputPath,

package/dist/src/ai/OllamaClient.js

@@ -0,0 +1,59 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OllamaClient = void 0;
+const axios_1 = __importDefault(require("axios"));
+class OllamaClient {
+    constructor(options) {
+        // Use 127.0.0.1 instead of localhost to avoid IPv6 issues (ECONNREFUSED ::1)
+        this.endpoint = options.endpoint || 'http://127.0.0.1:11434';
+        this.model = options.model;
+    }
+    async generate(prompt, onToken) {
+        try {
+            const response = await axios_1.default.post(`${this.endpoint}/api/generate`, {
+                model: this.model,
+                prompt: prompt,
+                stream: true
+            }, {
+                responseType: 'stream'
+            });
+            let fullResponse = '';
+            return new Promise((resolve, reject) => {
+                const stream = response.data;
+                stream.on('data', (chunk) => {
+                    const lines = chunk.toString().split('\n').filter(Boolean);
+                    for (const line of lines) {
+                        try {
+                            const json = JSON.parse(line);
+                            if (json.response) {
+                                fullResponse += json.response;
+                                if (onToken) {
+                                    onToken(json.response);
+                                }
+                            }
+                            if (json.done) {
+                                // stream ended
+                            }
+                        }
+                        catch (e) {
+                            // ignore partial JSON
+                        }
+                    }
+                });
+                stream.on('end', () => {
+                    resolve(fullResponse);
+                });
+                stream.on('error', (err) => {
+                    reject(err);
+                });
+            });
+        }
+        catch (error) {
+            throw new Error(`Ollama API call failed: ${error.message}. Is Ollama running at ${this.endpoint}?`);
+        }
+    }
+}
+exports.OllamaClient = OllamaClient;

package/dist/src/commands/VSCommand.js

@@ -0,0 +1,157 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VSCommand = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const chalk_1 = __importDefault(require("chalk"));
+const ora_1 = __importDefault(require("ora"));
+const CrawlerContext_1 = require("../CrawlerContext");
+const OCPStrategy_1 = require("../strategies/OCPStrategy");
+const ACPStrategy_1 = require("../strategies/ACPStrategy");
+const StrategyDetector_1 = require("../utils/StrategyDetector");
+const filename_1 = require("../utils/filename");
+const TocExtractor_1 = require("../utils/TocExtractor");
+const OllamaClient_1 = require("../ai/OllamaClient");
+class VSCommand {
+    static async run(baselineUrl, targetUrl, options) {
+        console.log(chalk_1.default.blue(`[VS Mode] Starting comparison between:`));
+        console.log(chalk_1.default.gray(`Baseline: ${baselineUrl}`));
+        console.log(chalk_1.default.gray(`Target: ${targetUrl}`));
+        console.log(chalk_1.default.gray(`Model: ${options.model}`));
+        // 1. Crawl Baseline
+        const baselineFile = await VSCommand.crawlUrl(baselineUrl, 'baseline');
+        // 2. Crawl Target
+        const targetFile = await VSCommand.crawlUrl(targetUrl, 'target');
+        // 3. Extract TOC
+        const baselineContent = fs_1.default.readFileSync(baselineFile, 'utf-8');
+        const targetContent = fs_1.default.readFileSync(targetFile, 'utf-8');
+        const baselineToc = TocExtractor_1.TocExtractor.extract(baselineContent);
+        const targetToc = TocExtractor_1.TocExtractor.extract(targetContent);
+        console.log(chalk_1.default.green(`[VS Mode] TOC extracted.`));
+        // 4. Load Prompts
+        const promptPath = path_1.default.resolve(process.cwd(), 'solo-doc-prompt.md');
+        let promptContent = '';
+        if (fs_1.default.existsSync(promptPath)) {
+            promptContent = fs_1.default.readFileSync(promptPath, 'utf-8');
+        }
+        else {
+            // Try to look in package root (assuming we might be running from bin)
+            // or just fail gracefully
+            const altPath = path_1.default.resolve(__dirname, '../../solo-doc-prompt.md');
+            if (fs_1.default.existsSync(altPath)) {
+                promptContent = fs_1.default.readFileSync(altPath, 'utf-8');
+            }
+            else {
+                console.warn(chalk_1.default.yellow('[VS Mode] Warning: solo-doc-prompt.md not found. Using default internal prompts.'));
+                // Define fallback prompts here if needed, or throw
+                // For now, let's throw to ensure user provides the file as requested
+                throw new Error('Could not find solo-doc-prompt.md in current directory.');
+            }
+        }
+        const prompts = VSCommand.parsePrompts(promptContent);
+        if (prompts.length < 2) {
+            throw new Error('Found fewer than 2 prompt templates in solo-doc-prompt.md');
+        }
+        const client = new OllamaClient_1.OllamaClient({ model: options.model });
+        // 5. Step 1: Independent Comparison
+        console.log(chalk_1.default.blue(`[VS Mode] Step 1: Analyzing differences...`));
+        // Replace placeholders
+        // Note: The prompt template has [看附件ocp的文档目录树]
+        let prompt1 = prompts[0];
+        prompt1 = prompt1.replace('[看附件ocp的文档目录树]', '\n' + baselineToc + '\n');
+        prompt1 = prompt1.replace('[看附件alauda的文档目录树]', '\n' + targetToc + '\n');
+        const spinner1 = (0, ora_1.default)('Waiting for AI response (this may take a while)...').start();
+        let hasStartedOutput1 = false;
+        const result1 = await client.generate(prompt1, (token) => {
+            if (!hasStartedOutput1) {
+                spinner1.stop();
+                process.stdout.write(chalk_1.default.cyan('AI Thinking: '));
+                hasStartedOutput1 = true;
+            }
+            process.stdout.write(token);
+        });
+        if (!hasStartedOutput1)
+            spinner1.stop();
+        process.stdout.write('\n');
+        const result1File = 'vs-result.md';
+        fs_1.default.writeFileSync(result1File, result1);
+        console.log(chalk_1.default.green(`[VS Mode] Step 1 complete. Saved to ${result1File}`));
+        // 6. Step 2: Integration
+        console.log(chalk_1.default.blue(`[VS Mode] Step 2: Integrating into documentation tree...`));
+        let prompt2 = prompts[1];
+        // The prompt says "基于OpenShift文档的目录树" - we should inject it if it's not explicitly placeholder
+        // Or we just prepend context.
+        // The template: "基于OpenShift文档的目录树，和上面的详细对比总结。"
+        // We construct the full prompt by prepending data.
+        const context2 = `
+OpenShift文档目录树：
+${baselineToc}
+
+详细对比总结：
+${result1}
+
+`;
+        prompt2 = context2 + prompt2;
+        const spinner2 = (0, ora_1.default)('Waiting for AI response (this may take a while)...').start();
+        let hasStartedOutput2 = false;
+        const result2 = await client.generate(prompt2, (token) => {
+            if (!hasStartedOutput2) {
+                spinner2.stop();
+                process.stdout.write(chalk_1.default.cyan('AI Thinking: '));
+                hasStartedOutput2 = true;
+            }
+            process.stdout.write(token);
+        });
+        if (!hasStartedOutput2)
+            spinner2.stop();
+        process.stdout.write('\n');
+        const result2File = 'vs-tree.md';
+        fs_1.default.writeFileSync(result2File, result2);
+        console.log(chalk_1.default.green(`[VS Mode] Step 2 complete. Saved to ${result2File}`));
+        console.log(chalk_1.default.green(`[VS Mode] All tasks finished.`));
+    }
+    static async crawlUrl(url, prefix) {
+        // Try to detect type
+        let type = StrategyDetector_1.StrategyDetector.detect(url);
+        let strategy;
+        // Simple logic: if detects OCP, use OCP. Else ACP (more generic).
+        if (type === StrategyDetector_1.StrategyType.OCP) {
+            strategy = new OCPStrategy_1.OCPStrategy();
+        }
+        else {
+            // Default to ACP which uses Puppeteer
+            strategy = new ACPStrategy_1.ACPStrategy();
+        }
+        const filename = (0, filename_1.generateDefaultFilename)(url, prefix);
+        const outputPath = path_1.default.resolve(process.cwd(), filename);
+        // Check if file exists
+        if (fs_1.default.existsSync(outputPath)) {
+            console.log(chalk_1.default.yellow('--------------------------------------------------'));
+            console.log(chalk_1.default.yellow(`ℹ  File already exists: ${outputPath}`));
+            console.log(chalk_1.default.yellow('   Using cached version for comparison.'));
+            console.log(chalk_1.default.yellow('--------------------------------------------------'));
+            return outputPath;
+        }
+        console.log(chalk_1.default.blue(`[VS Mode] Crawling ${url} -> ${filename}...`));
+        const context = new CrawlerContext_1.CrawlerContext(strategy);
+        // Suppress console log from crawler to keep output clean? 
+        // Or keep it to show progress. Keep it.
+        await context.run(url, { output: outputPath, headless: true });
+        return outputPath;
+    }
+    static parsePrompts(content) {
+        // Match content inside ``` ... ``` blocks that follow "Prompt模板"
+        // Regex: /Prompt模板.*?\n```([\s\S]*?)```/g
+        const regex = /Prompt模板.*?\n```([\s\S]*?)```/g;
+        const matches = [];
+        let match;
+        while ((match = regex.exec(content)) !== null) {
+            matches.push(match[1].trim());
+        }
+        return matches;
+    }
+}
+exports.VSCommand = VSCommand;

package/dist/src/utils/TocExtractor.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TocExtractor = void 0;
+/**
+ * Extracts the Table of Contents (headers) from Markdown content.
+ * Returns a string representation of the tree.
+ */
+class TocExtractor {
+    static extract(markdown) {
+        const lines = markdown.split('\n');
+        const tocLines = [];
+        let inCodeBlock = false;
+        for (const line of lines) {
+            // Simple code block detection to avoid headers inside code blocks
+            if (line.trim().startsWith('```')) {
+                inCodeBlock = !inCodeBlock;
+                continue;
+            }
+            if (inCodeBlock)
+                continue;
+            // Match headers H1 to H3 only
+            // Regex: ^#{1,3}\s
+            if (line.match(/^#{1,3}\s/)) {
+                tocLines.push(line.trim());
+            }
+        }
+        if (tocLines.length === 0) {
+            return "No headers found.";
+        }
+        return tocLines.join('\n');
+    }
+}
+exports.TocExtractor = TocExtractor;

package/dist/src/utils/filename.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateDefaultFilename = void 0;
+const generateDefaultFilename = (urlStr, typePrefix) => {
+    try {
+        const u = new URL(urlStr);
+        // Get the last path segment that isn't 'index.html' or 'index' or empty
+        const segments = u.pathname.split('/').filter(s => s && s !== 'index.html' && s !== 'index');
+        const lastSegment = segments.length > 0 ? segments[segments.length - 1] : 'docs';
+        // Sanitize filename
+        const safeName = lastSegment.replace(/[^a-zA-Z0-9-_]/g, '_');
+        return `${typePrefix}-${safeName}.md`;
+    }
+    catch (e) {
+        return `${typePrefix}-docs.md`;
+    }
+};
+exports.generateDefaultFilename = generateDefaultFilename;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "solo-doc",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "main": "dist/bin/solo-doc.js",
   "bin": {
     "solo-doc": "dist/bin/solo-doc.js"
@@ -13,6 +13,7 @@
     "clean": "rm -rf dist",
     "build": "tsc",
     "prepublishOnly": "npm run clean && npm run build",
+    "release": "npm run clean && npm run build && npm version patch --force && npm publish --access=public",
     "start": "node dist/bin/solo-doc.js",
     "dev": "ts-node bin/solo-doc.ts"
   },