skrypt-ai 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/auth/index.js +3 -3
  2. package/dist/cli.js +1 -1
  3. package/dist/commands/cron.js +0 -4
  4. package/dist/commands/generate/index.d.ts +3 -0
  5. package/dist/commands/generate/index.js +393 -0
  6. package/dist/commands/generate/scan.d.ts +41 -0
  7. package/dist/commands/generate/scan.js +256 -0
  8. package/dist/commands/generate/verify.d.ts +14 -0
  9. package/dist/commands/generate/verify.js +122 -0
  10. package/dist/commands/generate/write.d.ts +25 -0
  11. package/dist/commands/generate/write.js +120 -0
  12. package/dist/commands/import.js +4 -1
  13. package/dist/commands/llms-txt.js +6 -4
  14. package/dist/config/loader.d.ts +0 -1
  15. package/dist/config/loader.js +1 -1
  16. package/dist/generator/agents-md.d.ts +25 -0
  17. package/dist/generator/agents-md.js +122 -0
  18. package/dist/generator/index.d.ts +2 -0
  19. package/dist/generator/index.js +2 -0
  20. package/dist/generator/mdx-serializer.d.ts +11 -0
  21. package/dist/generator/mdx-serializer.js +135 -0
  22. package/dist/generator/organizer.d.ts +1 -16
  23. package/dist/generator/organizer.js +0 -38
  24. package/dist/generator/writer.js +5 -4
  25. package/dist/llm/proxy-client.d.ts +32 -0
  26. package/dist/llm/proxy-client.js +103 -0
  27. package/dist/scanner/csharp.d.ts +0 -4
  28. package/dist/scanner/csharp.js +9 -49
  29. package/dist/scanner/go.d.ts +0 -3
  30. package/dist/scanner/go.js +8 -35
  31. package/dist/scanner/java.d.ts +0 -4
  32. package/dist/scanner/java.js +9 -49
  33. package/dist/scanner/kotlin.d.ts +0 -3
  34. package/dist/scanner/kotlin.js +6 -33
  35. package/dist/scanner/php.d.ts +0 -10
  36. package/dist/scanner/php.js +11 -55
  37. package/dist/scanner/ruby.d.ts +0 -3
  38. package/dist/scanner/ruby.js +8 -38
  39. package/dist/scanner/rust.d.ts +0 -3
  40. package/dist/scanner/rust.js +10 -37
  41. package/dist/scanner/swift.d.ts +0 -3
  42. package/dist/scanner/swift.js +8 -35
  43. package/dist/scanner/utils.d.ts +41 -0
  44. package/dist/scanner/utils.js +97 -0
  45. package/dist/template/docs.json +5 -2
  46. package/dist/template/next.config.mjs +31 -0
  47. package/dist/template/package.json +5 -3
  48. package/dist/template/src/app/layout.tsx +13 -13
  49. package/dist/template/src/app/llms-full.md/route.ts +29 -0
  50. package/dist/template/src/app/llms.txt/route.ts +29 -0
  51. package/dist/template/src/app/md/[...slug]/route.ts +174 -0
  52. package/dist/template/src/app/reference/route.ts +22 -18
  53. package/dist/template/src/app/sitemap.ts +1 -1
  54. package/dist/template/src/components/ai-chat-impl.tsx +206 -0
  55. package/dist/template/src/components/ai-chat.tsx +20 -193
  56. package/dist/template/src/components/mdx/index.tsx +27 -4
  57. package/dist/template/src/lib/fonts.ts +135 -0
  58. package/dist/template/src/middleware.ts +101 -0
  59. package/dist/template/src/styles/globals.css +28 -20
  60. package/dist/utils/files.d.ts +0 -8
  61. package/dist/utils/files.js +0 -33
  62. package/package.json +1 -1
  63. package/dist/autofix/autofix.test.d.ts +0 -1
  64. package/dist/autofix/autofix.test.js +0 -487
  65. package/dist/commands/generate.d.ts +0 -9
  66. package/dist/commands/generate.js +0 -739
  67. package/dist/generator/generator.test.d.ts +0 -1
  68. package/dist/generator/generator.test.js +0 -259
  69. package/dist/generator/writer.test.d.ts +0 -1
  70. package/dist/generator/writer.test.js +0 -411
  71. package/dist/llm/llm.manual-test.d.ts +0 -1
  72. package/dist/llm/llm.manual-test.js +0 -112
  73. package/dist/llm/llm.mock-test.d.ts +0 -4
  74. package/dist/llm/llm.mock-test.js +0 -79
  75. package/dist/plugins/index.d.ts +0 -47
  76. package/dist/plugins/index.js +0 -181
  77. package/dist/scanner/content-type.test.d.ts +0 -1
  78. package/dist/scanner/content-type.test.js +0 -231
  79. package/dist/scanner/integration.test.d.ts +0 -4
  80. package/dist/scanner/integration.test.js +0 -180
  81. package/dist/scanner/scanner.test.d.ts +0 -1
  82. package/dist/scanner/scanner.test.js +0 -210
  83. package/dist/scanner/typescript.manual-test.d.ts +0 -1
  84. package/dist/scanner/typescript.manual-test.js +0 -112
  85. package/dist/template/src/app/docs/auth/page.mdx +0 -589
  86. package/dist/template/src/app/docs/autofix/page.mdx +0 -624
  87. package/dist/template/src/app/docs/cli/page.mdx +0 -217
  88. package/dist/template/src/app/docs/config/page.mdx +0 -428
  89. package/dist/template/src/app/docs/configuration/page.mdx +0 -86
  90. package/dist/template/src/app/docs/deployment/page.mdx +0 -112
  91. package/dist/template/src/app/docs/generator/generator.md +0 -504
  92. package/dist/template/src/app/docs/generator/organizer.md +0 -779
  93. package/dist/template/src/app/docs/generator/page.mdx +0 -613
  94. package/dist/template/src/app/docs/github/page.mdx +0 -502
  95. package/dist/template/src/app/docs/llm/anthropic-client.md +0 -549
  96. package/dist/template/src/app/docs/llm/index.md +0 -471
  97. package/dist/template/src/app/docs/llm/page.mdx +0 -428
  98. package/dist/template/src/app/docs/plugins/page.mdx +0 -1793
  99. package/dist/template/src/app/docs/pro/page.mdx +0 -121
  100. package/dist/template/src/app/docs/quickstart/page.mdx +0 -93
  101. package/dist/template/src/app/docs/scanner/content-type.md +0 -599
  102. package/dist/template/src/app/docs/scanner/index.md +0 -212
  103. package/dist/template/src/app/docs/scanner/page.mdx +0 -307
  104. package/dist/template/src/app/docs/scanner/python.md +0 -469
  105. package/dist/template/src/app/docs/scanner/python_parser.md +0 -1056
  106. package/dist/template/src/app/docs/scanner/rust.md +0 -325
  107. package/dist/template/src/app/docs/scanner/typescript.md +0 -201
  108. package/dist/template/src/app/icon.tsx +0 -29
  109. package/dist/utils/validation.d.ts +0 -1
  110. package/dist/utils/validation.js +0 -12
@@ -1,739 +0,0 @@
1
- import { Command } from 'commander';
2
- import { existsSync, copyFileSync, mkdirSync, readFileSync, rmSync } from 'fs';
3
- import { resolve, basename, dirname, join } from 'path';
4
- import { loadConfig, validateConfig, checkApiKey, resolveSourceEntries } from '../config/index.js';
5
- import { DEFAULT_MODELS } from '../config/types.js';
6
- import { scanDirectory } from '../scanner/index.js';
7
- import { createLLMClient } from '../llm/index.js';
8
- import { generateForElements, groupDocsByFile, writeDocsToDirectory, writeDocsByTopic, writeLlmsTxt } from '../generator/index.js';
9
- import { showSecurityNotice } from '../auth/notices.js';
10
- import { requirePro } from '../auth/index.js';
11
- import { runQA, printQAReport, fixQAIssues, printFixReport } from '../qa/index.js';
12
- import { extractDependencyIds, isChubInstalled, fetchContextHubDocs, exportToContextHub } from '../context-hub/index.js';
13
- import { discoverOrgRepos, cloneRepoToTemp } from '../github/org-discovery.js';
14
- import { extractSnippets, findDocFiles, runLocally, loadEnvFile } from '../testing/index.js';
15
- import { writeManifest, buildManifestEntries } from '../refresh/manifest.js';
16
- import { planSmartStructure, generateWithStructure } from '../structure/index.js';
17
- import * as readline from 'readline';
18
- /**
19
- * Parse source arguments with optional labels.
20
- * e.g. "./api:API" → { path: "./api", label: "API" }
21
- * e.g. "./src" → { path: "./src" }
22
- */
23
- export function parseSourceArgs(args) {
24
- return args.map(arg => {
25
- const colonIdx = arg.lastIndexOf(':');
26
- // Only treat as label separator if colon is not part of a path (e.g. C:\)
27
- if (colonIdx > 1 && !arg.substring(0, colonIdx).endsWith('\\')) {
28
- return {
29
- path: arg.substring(0, colonIdx),
30
- label: arg.substring(colonIdx + 1),
31
- };
32
- }
33
- return { path: arg };
34
- });
35
- }
36
- /**
37
- * Read .skryptignore patterns from source directory
38
- */
39
- function readIgnorePatterns(sourcePath) {
40
- const ignorePath = join(sourcePath, '.skryptignore');
41
- if (!existsSync(ignorePath))
42
- return [];
43
- const content = readFileSync(ignorePath, 'utf-8');
44
- return content
45
- .split('\n')
46
- .map(line => line.trim())
47
- .filter(line => line && !line.startsWith('#'));
48
- }
49
- /**
50
- * Auto-detect OpenAPI spec file in source directory
51
- */
52
- function findOpenAPISpec(sourcePath) {
53
- const candidates = [
54
- 'openapi.json',
55
- 'openapi.yaml',
56
- 'openapi.yml',
57
- 'swagger.json',
58
- 'swagger.yaml',
59
- 'swagger.yml',
60
- 'api.json',
61
- 'api.yaml',
62
- 'api.yml',
63
- ];
64
- for (const name of candidates) {
65
- const specPath = join(sourcePath, name);
66
- if (existsSync(specPath)) {
67
- return specPath;
68
- }
69
- }
70
- // Check common subdirectories
71
- const subdirs = ['docs', 'api', 'spec', '.'];
72
- for (const subdir of subdirs) {
73
- for (const name of candidates) {
74
- const specPath = join(sourcePath, subdir, name);
75
- if (existsSync(specPath)) {
76
- return specPath;
77
- }
78
- }
79
- }
80
- return null;
81
- }
82
- /**
83
- * Check if element should be excluded based on patterns
84
- */
85
- function shouldExcludeElement(element, patterns) {
86
- for (const pattern of patterns) {
87
- // Match by name
88
- if (pattern.startsWith('name:')) {
89
- const namePattern = pattern.slice(5);
90
- if (element.name === namePattern) {
91
- return true;
92
- }
93
- // Only use regex if the pattern contains regex metacharacters
94
- // Reject patterns with nested quantifiers to prevent catastrophic backtracking (ReDoS)
95
- if (/[*+?{}()|[\]\\^$.]/.test(namePattern)) {
96
- if (/(\+|\*|\?)\{|\(\?[^:)]|\(\.[*+].*\)\+|\([^)]*[+*][^)]*\)[+*]/.test(namePattern)) {
97
- continue; // Skip patterns prone to catastrophic backtracking
98
- }
99
- try {
100
- if (new RegExp(namePattern).test(element.name))
101
- return true;
102
- }
103
- catch {
104
- // Invalid regex — treat as literal match (already checked above)
105
- }
106
- }
107
- }
108
- // Match by file path
109
- else if (pattern.includes('/') || pattern.includes('*')) {
110
- const filePath = element.filePath;
111
- if (pattern.includes('**')) {
112
- const parts = pattern.split('**');
113
- const prefixMatch = !parts[0] || filePath.includes(parts[0].replace(/^\//, ''));
114
- const suffixMatch = !parts[1] || filePath.includes(parts[1].replace(/^\//, ''));
115
- if (prefixMatch && suffixMatch)
116
- return true;
117
- }
118
- else if (filePath.includes(pattern.replace(/\*/g, ''))) {
119
- return true;
120
- }
121
- }
122
- // Match by exact name
123
- else if (element.name === pattern) {
124
- return true;
125
- }
126
- }
127
- return false;
128
- }
129
- export const generateCommand = new Command('generate')
130
- .description('Generate documentation with code examples')
131
- .argument('[sources...]', 'Source directories to scan (use dir:Label for labels)')
132
- .option('-o, --output <dir>', 'Output directory')
133
- .option('-c, --config <file>', 'Config file path')
134
- .option('--provider <name>', 'LLM provider (deepseek, openai, anthropic, google, ollama, openrouter)')
135
- .option('--model <name>', 'LLM model name')
136
- .option('--base-url <url>', 'Custom API base URL (for Ollama or proxies)')
137
- .option('--dry-run', 'Scan only, do not generate docs')
138
- .option('--multi-lang', 'Generate TypeScript + Python examples')
139
- .option('--by-topic', 'Organize output by topic instead of file')
140
- .option('--openapi <file>', 'Include OpenAPI spec file for API Playground')
141
- .option('--public-only', 'Only document exported/public APIs')
142
- .option('--exclude <patterns...>', 'Exclude patterns (files, names, or name:pattern)')
143
- .option('--llms-txt', 'Generate llms.txt for Answer Engine Optimization (AEO)')
144
- .option('--project-name <name>', 'Project name for llms.txt header')
145
- .option('--org <name>', 'GitHub organization to discover repos from')
146
- .option('--repos <list>', 'Comma-separated list of repos to include (with --org)')
147
- .option('--exclude-repos <list>', 'Comma-separated list of repos to exclude (with --org)')
148
- .option('--verify', 'Verify generated code examples by running them (Pro)')
149
- .option('--env-file <file>', 'Load environment variables for code verification')
150
- .option('--smart-structure', 'Organize docs by user journey instead of file structure (Pro)')
151
- .option('--max-verify-iterations <n>', 'Max re-generation attempts for failing snippets', '2')
152
- .action(async (sources = [], options) => {
153
- try {
154
- const startTime = Date.now();
155
- // Require at least one source or --org
156
- if (sources.length === 0 && !options.org) {
157
- console.error('Error: At least one source directory is required (or use --org)');
158
- process.exit(1);
159
- }
160
- // Load config (file or defaults)
161
- const config = loadConfig(options.config);
162
- // CLI flags override config — first source is used for single-source compat
163
- if (sources.length > 0)
164
- config.source.path = sources[0];
165
- if (options.output)
166
- config.output.path = options.output;
167
- if (options.provider) {
168
- const validProviders = ['deepseek', 'openai', 'anthropic', 'google', 'ollama', 'openrouter'];
169
- if (!validProviders.includes(options.provider)) {
170
- console.error(`Error: Unknown provider "${options.provider}". Valid: ${validProviders.join(', ')}`);
171
- process.exit(1);
172
- }
173
- config.llm.provider = options.provider;
174
- // Use provider's default model unless explicitly specified
175
- if (!options.model) {
176
- config.llm.model = DEFAULT_MODELS[config.llm.provider];
177
- }
178
- }
179
- if (options.model)
180
- config.llm.model = options.model;
181
- if (options.baseUrl)
182
- config.llm.baseUrl = options.baseUrl;
183
- // Validate
184
- const errors = validateConfig(config);
185
- if (errors.length > 0) {
186
- console.error('Config errors:');
187
- errors.forEach(e => console.error(` - ${e}`));
188
- process.exit(1);
189
- }
190
- // Check for API key (not needed for Ollama or dry-run)
191
- if (!options.dryRun) {
192
- const { ok, envKey } = checkApiKey(config.llm.provider);
193
- if (!ok && envKey) {
194
- console.error(`Error: ${envKey} environment variable required for ${config.llm.provider}`);
195
- process.exit(1);
196
- }
197
- }
198
- // Pro-gated flags
199
- if (options.verify) {
200
- if (!await requirePro('generate --verify')) {
201
- process.exit(1);
202
- }
203
- }
204
- if (options.smartStructure) {
205
- if (!await requirePro('generate --smart-structure')) {
206
- process.exit(1);
207
- }
208
- }
209
- // First-run security notice
210
- showSecurityNotice();
211
- console.log('skrypt generate');
212
- if (options.org) {
213
- console.log(` source: org:${options.org}`);
214
- }
215
- else if (sources.length > 1) {
216
- console.log(` sources: ${sources.join(', ')}`);
217
- }
218
- else {
219
- console.log(` source: ${config.source.path}`);
220
- }
221
- console.log(` output: ${config.output.path}`);
222
- console.log(` provider: ${config.llm.provider}`);
223
- console.log(` model: ${config.llm.model}`);
224
- if (config.llm.baseUrl) {
225
- console.log(` base url: ${config.llm.baseUrl}`);
226
- }
227
- // Routing transparency
228
- const providerEnvKeys = {
229
- openai: 'OPENAI_API_KEY',
230
- anthropic: 'ANTHROPIC_API_KEY',
231
- google: 'GOOGLE_API_KEY',
232
- deepseek: 'DEEPSEEK_API_KEY',
233
- };
234
- const providerKey = providerEnvKeys[config.llm.provider];
235
- if (providerKey && process.env[providerKey]) {
236
- console.log(` routing: direct to ${config.llm.provider} (BYOK — your key never touches Skrypt)`);
237
- }
238
- else if (config.llm.provider !== 'ollama') {
239
- console.log(' routing: via Skrypt API proxy');
240
- }
241
- console.log('');
242
- // Resolve source entries: CLI args, --org, or config file
243
- let sourceEntries;
244
- const tempDirs = [];
245
- if (options.org) {
246
- // GitHub org discovery mode
247
- const token = process.env.GITHUB_TOKEN;
248
- if (!token) {
249
- console.error('Error: GITHUB_TOKEN environment variable required for --org');
250
- process.exit(1);
251
- }
252
- console.log(`Discovering repos in org "${options.org}"...`);
253
- const repoWhitelist = options.repos ? options.repos.split(',').map(r => r.trim()) : undefined;
254
- const repoBlacklist = options.excludeRepos ? options.excludeRepos.split(',').map(r => r.trim()) : undefined;
255
- let discoveredRepos = await discoverOrgRepos(options.org, token);
256
- if (repoWhitelist) {
257
- discoveredRepos = discoveredRepos.filter(r => repoWhitelist.includes(r.name));
258
- }
259
- if (repoBlacklist) {
260
- discoveredRepos = discoveredRepos.filter(r => !repoBlacklist.includes(r.name));
261
- }
262
- console.log(` Found ${discoveredRepos.length} repos`);
263
- sourceEntries = [];
264
- for (const repo of discoveredRepos) {
265
- console.log(` Cloning ${repo.full_name}...`);
266
- const tempDir = await cloneRepoToTemp(repo, token);
267
- tempDirs.push(tempDir);
268
- sourceEntries.push({
269
- path: tempDir,
270
- label: repo.name,
271
- include: config.source.include,
272
- exclude: config.source.exclude,
273
- });
274
- }
275
- }
276
- else if (sources.length > 1) {
277
- // Multiple CLI args
278
- sourceEntries = parseSourceArgs(sources);
279
- }
280
- else {
281
- // Single source or config file sources
282
- sourceEntries = resolveSourceEntries(config);
283
- }
284
- const isMultiSource = sourceEntries.length > 1 || sourceEntries.some(s => s.label);
285
- // Check all source paths exist
286
- for (const entry of sourceEntries) {
287
- const sourcePath = resolve(entry.path);
288
- if (!existsSync(sourcePath)) {
289
- console.error(`Error: Source directory not found: ${sourcePath}`);
290
- process.exit(1);
291
- }
292
- }
293
- // Step 1: Scan source code from all sources
294
- console.log('Step 1: Scanning source code...');
295
- let allElements = [];
296
- let totalFiles = 0;
297
- const allScanErrors = [];
298
- try {
299
- for (const entry of sourceEntries) {
300
- const sourcePath = resolve(entry.path);
301
- if (isMultiSource) {
302
- console.log(`\n Scanning ${entry.label || entry.path}...`);
303
- }
304
- const scanResult = await scanDirectory(sourcePath, {
305
- include: entry.include || config.source.include,
306
- exclude: entry.exclude || config.source.exclude,
307
- onProgress: (current, total, file) => {
308
- process.stdout.write(`\r [${current}/${total}] ${file.slice(-50).padStart(50)}`);
309
- }
310
- });
311
- console.log('');
312
- if (scanResult.errors.length > 0) {
313
- allScanErrors.push(...scanResult.errors);
314
- }
315
- totalFiles += scanResult.files.length;
316
- // Tag elements with source metadata
317
- for (const file of scanResult.files) {
318
- for (const el of file.elements) {
319
- if (entry.label) {
320
- el.sourceLabel = entry.label;
321
- }
322
- el.sourceRoot = sourcePath;
323
- allElements.push(el);
324
- }
325
- }
326
- }
327
- if (allScanErrors.length > 0) {
328
- console.log('\n Scan warnings:');
329
- allScanErrors.slice(0, 5).forEach(e => console.log(` - ${e}`));
330
- if (allScanErrors.length > 5) {
331
- console.log(` ... and ${allScanErrors.length - 5} more`);
332
- }
333
- }
334
- console.log(`\n Found ${allElements.length} API elements in ${totalFiles} files`);
335
- if (allElements.length === 0) {
336
- console.log(' No API elements found. Nothing to generate.');
337
- return;
338
- }
339
- // Apply privacy filters
340
- const initialCount = allElements.length;
341
- // 1. --public-only: filter to exported/public APIs only
342
- if (options.publicOnly) {
343
- allElements = allElements.filter(el => el.isExported === true || el.isPublic === true);
344
- console.log(` --public-only: filtered to ${allElements.length} exported APIs`);
345
- }
346
- // 2. Load .skryptignore patterns from all source dirs
347
- const ignorePatterns = [];
348
- for (const entry of sourceEntries) {
349
- ignorePatterns.push(...readIgnorePatterns(resolve(entry.path)));
350
- }
351
- if (ignorePatterns.length > 0) {
352
- console.log(` .skryptignore: loaded ${ignorePatterns.length} patterns`);
353
- }
354
- // 3. Combine with --exclude patterns
355
- const excludePatterns = [...ignorePatterns, ...(options.exclude || [])];
356
- if (excludePatterns.length > 0) {
357
- allElements = allElements.filter(el => !shouldExcludeElement(el, excludePatterns));
358
- if (options.exclude?.length) {
359
- console.log(` --exclude: applied ${options.exclude.length} additional patterns`);
360
- }
361
- }
362
- if (initialCount !== allElements.length) {
363
- console.log(` Filtered: ${initialCount} -> ${allElements.length} elements`);
364
- }
365
- // Show summary by kind
366
- const byKind = {};
367
- for (const el of allElements) {
368
- byKind[el.kind] = (byKind[el.kind] || 0) + 1;
369
- }
370
- const pluralize = (word, count) => {
371
- if (count === 1)
372
- return word;
373
- if (word === 'class')
374
- return 'classes';
375
- return word + 's';
376
- };
377
- console.log(' ' + Object.entries(byKind).map(([k, v]) => `${v} ${pluralize(k, v)}`).join(', '));
378
- // Dry run - stop here
379
- if (options.dryRun) {
380
- console.log('\n[dry run - stopping before generation]');
381
- return;
382
- }
383
- // Use first source path as the primary source for context/compat
384
- const primarySourcePath = resolve(sourceEntries[0].path);
385
- // Auto-read project context from README for richer doc generation
386
- let projectContext;
387
- const readmeCandidates = ['README.md', 'README.mdx', 'readme.md', 'README.rst', 'README.txt'];
388
- for (const candidate of readmeCandidates) {
389
- const readmePath = join(primarySourcePath, candidate);
390
- if (existsSync(readmePath)) {
391
- try {
392
- const raw = readFileSync(readmePath, 'utf-8');
393
- // Take the first ~1500 chars (intro/description section, not the whole file)
394
- projectContext = raw.slice(0, 1500);
395
- console.log(` Project context: loaded from ${candidate}`);
396
- }
397
- catch {
398
- // Skip if unreadable
399
- }
400
- break;
401
- }
402
- }
403
- // Also check parent directory (common for monorepos where source is a subdirectory)
404
- if (!projectContext) {
405
- const parentReadme = join(primarySourcePath, '..', 'README.md');
406
- if (existsSync(parentReadme)) {
407
- try {
408
- projectContext = readFileSync(parentReadme, 'utf-8').slice(0, 1500);
409
- console.log(' Project context: loaded from parent README.md');
410
- }
411
- catch {
412
- // Skip
413
- }
414
- }
415
- }
416
- // Context Hub enrichment: fetch third-party API docs if chub is installed
417
- let externalContext;
418
- const allImports = [];
419
- for (const el of allElements) {
420
- if (el.imports?.length) {
421
- allImports.push(...el.imports);
422
- }
423
- }
424
- const chubIds = extractDependencyIds(allImports);
425
- if (chubIds.length > 0 && isChubInstalled()) {
426
- console.log(`\n Context Hub: fetching docs for ${chubIds.length} dependencies...`);
427
- externalContext = fetchContextHubDocs(chubIds);
428
- if (externalContext.size > 0) {
429
- console.log(` Context Hub: enriching with ${externalContext.size} API references`);
430
- }
431
- }
432
- // Step 2: Generate docs
433
- console.log('\nStep 2: Generating documentation...');
434
- const client = createLLMClient({
435
- provider: config.llm.provider,
436
- model: config.llm.model,
437
- baseUrl: config.llm.baseUrl
438
- });
439
- let lastElement = '';
440
- const multiLanguage = options.multiLang ?? false;
441
- if (multiLanguage) {
442
- console.log(' mode: multi-language (TypeScript + Python)');
443
- }
444
- const genOptions = {
445
- multiLanguage,
446
- externalContext,
447
- projectContext,
448
- onProgress: (progress) => {
449
- if (progress.element !== lastElement) {
450
- if (lastElement)
451
- console.log('');
452
- lastElement = progress.element;
453
- }
454
- process.stdout.write(`\r [${progress.current}/${progress.total}] ${progress.element}: ${progress.status}`.padEnd(80));
455
- }
456
- };
457
- // Step 3: Write output
458
- const outputPath = resolve(config.output.path);
459
- let filesWritten;
460
- let totalDocs;
461
- let docs;
462
- let errorCount;
463
- if (options.smartStructure) {
464
- // Smart structure: LLM-planned page organization
465
- console.log(' mode: smart-structure (user-journey organization)');
466
- console.log('\n Planning documentation structure...');
467
- const structure = await planSmartStructure(allElements, client);
468
- console.log(` Planned ${structure.pages.length} page(s)`);
469
- for (const page of structure.pages) {
470
- console.log(` ${page.category}/${page.slug}: ${page.elements.length} elements`);
471
- }
472
- console.log('\nStep 3: Generating & writing structured documentation...');
473
- const result = await generateWithStructure(structure, client, outputPath, genOptions);
474
- filesWritten = result.filesWritten;
475
- totalDocs = result.totalDocs;
476
- docs = result.docs;
477
- errorCount = docs.filter(d => d.error).length;
478
- }
479
- else {
480
- docs = await generateForElements(allElements, client, genOptions);
481
- console.log('\n');
482
- console.log('Step 3: Writing documentation...');
483
- if (options.byTopic) {
484
- console.log(' mode: by-topic (grouped by concept)');
485
- const result = await writeDocsByTopic(docs, outputPath);
486
- filesWritten = result.filesWritten;
487
- totalDocs = result.totalDocs;
488
- console.log(` topics: ${result.topics.map(t => t.name).join(', ')}`);
489
- }
490
- else if (isMultiSource) {
491
- // Multi-source: write docs namespaced by source label
492
- filesWritten = 0;
493
- totalDocs = 0;
494
- // Group docs by source label
495
- const bySource = new Map();
496
- for (const doc of docs) {
497
- const label = doc.element.sourceLabel || '_default';
498
- if (!bySource.has(label))
499
- bySource.set(label, []);
500
- bySource.get(label).push(doc);
501
- }
502
- for (const [label, sourceDocs] of bySource) {
503
- const fileResults = groupDocsByFile(sourceDocs);
504
- const sourceOutputDir = label === '_default' ? outputPath : join(outputPath, label.toLowerCase());
505
- const sourceRoot = sourceDocs[0]?.element.sourceRoot || primarySourcePath;
506
- const result = await writeDocsToDirectory(fileResults, sourceOutputDir, sourceRoot);
507
- filesWritten += result.filesWritten;
508
- totalDocs += result.totalDocs;
509
- if (label !== '_default') {
510
- console.log(` ${label}: ${result.filesWritten} files`);
511
- }
512
- }
513
- }
514
- else {
515
- // Default: file-based output (single source)
516
- const fileResults = groupDocsByFile(docs);
517
- const result = await writeDocsToDirectory(fileResults, outputPath, primarySourcePath);
518
- filesWritten = result.filesWritten;
519
- totalDocs = result.totalDocs;
520
- }
521
- errorCount = docs.filter(d => d.error).length;
522
- }
523
- const duration = Math.round((Date.now() - startTime) / 1000);
524
- console.log(`\n Wrote ${filesWritten} documentation files to ${outputPath}`);
525
- // Copy OpenAPI spec (provided or auto-detected)
526
- let specPath = options.openapi ? resolve(options.openapi) : null;
527
- // Auto-detect if not provided
528
- if (!specPath) {
529
- const detected = findOpenAPISpec(primarySourcePath);
530
- if (detected) {
531
- specPath = detected;
532
- console.log(`\n Auto-detected OpenAPI spec: ${basename(detected)}`);
533
- }
534
- }
535
- if (specPath) {
536
- if (existsSync(specPath)) {
537
- const specFilename = basename(specPath);
538
- const contentDir = dirname(outputPath);
539
- const destPath = resolve(contentDir, specFilename);
540
- mkdirSync(dirname(destPath), { recursive: true });
541
- copyFileSync(specPath, destPath);
542
- console.log(` Copied OpenAPI spec: ${specFilename} -> ${destPath}`);
543
- console.log(' API Playground will be available at /reference');
544
- }
545
- else if (options.openapi) {
546
- console.log(`\n Warning: OpenAPI spec not found: ${specPath}`);
547
- }
548
- }
549
- // Always generate llms.txt for AEO (Answer Engine Optimization)
550
- await writeLlmsTxt(docs, outputPath, {
551
- projectName: options.projectName,
552
- description: `API documentation for ${options.projectName || basename(primarySourcePath)}`
553
- });
554
- console.log(`\n Generated llms.txt and llms-full.md for AEO`);
555
- // Step 4: Verify code examples (if --verify)
556
- if (options.verify) {
557
- console.log('\nStep 4: Verifying code examples...');
558
- let verifyEnvVars = {};
559
- if (options.envFile) {
560
- try {
561
- verifyEnvVars = loadEnvFile(resolve(options.envFile));
562
- console.log(` Loaded ${Object.keys(verifyEnvVars).length} env var(s) from ${options.envFile}`);
563
- }
564
- catch {
565
- console.log(` Warning: Could not load env file: ${options.envFile}`);
566
- }
567
- }
568
- const verifyConfig = {
569
- timeout: 15000,
570
- envVars: verifyEnvVars,
571
- installDeps: true,
572
- };
573
- const maxIterations = Math.max(1, parseInt(options.maxVerifyIterations ?? '2', 10) || 2);
574
- let failedCount = 0;
575
- let passedCount = 0;
576
- let skippedCount = 0;
577
- for (let iteration = 1; iteration <= maxIterations; iteration++) {
578
- const docFiles = findDocFiles(outputPath);
579
- const allSnippets = docFiles.flatMap(f => extractSnippets(f));
580
- if (allSnippets.length === 0) {
581
- console.log(' No code snippets found to verify');
582
- break;
583
- }
584
- if (iteration === 1) {
585
- console.log(` Found ${allSnippets.length} code snippet(s) to verify`);
586
- }
587
- else {
588
- console.log(`\n Retry ${iteration}/${maxIterations}: re-verifying ${allSnippets.length} snippet(s)...`);
589
- }
590
- failedCount = 0;
591
- passedCount = 0;
592
- skippedCount = 0;
593
- const failedSnippets = [];
594
- const snippetErrors = new Map();
595
- for (const snippet of allSnippets) {
596
- const result = await runLocally(snippet, verifyConfig);
597
- if (result.status === 'pass') {
598
- passedCount++;
599
- }
600
- else if (result.status === 'skip') {
601
- skippedCount++;
602
- }
603
- else {
604
- failedCount++;
605
- failedSnippets.push(snippet);
606
- const errorMsg = result.stderr?.trim().split('\n').slice(0, 5).join('\n') || `Exit code: ${result.exitCode}`;
607
- snippetErrors.set(snippet.filePath + ':' + snippet.lineNumber, errorMsg);
608
- console.log(` \x1b[31m✗\x1b[0m ${snippet.filePath}:${snippet.lineNumber} [${snippet.language}]`);
609
- if (result.stderr) {
610
- console.log(` ${result.stderr.trim().split('\n')[0]?.slice(0, 80)}`);
611
- }
612
- }
613
- }
614
- // If all passed or last iteration, stop
615
- if (failedCount === 0 || iteration === maxIterations) {
616
- break;
617
- }
618
- // Re-generate failing docs by re-running generation for elements whose docs had failing snippets
619
- console.log(`\n Re-generating ${failedSnippets.length} failing snippet(s)...`);
620
- const failedFiles = [...new Set(failedSnippets.map(s => s.filePath))];
621
- for (const failedFile of failedFiles) {
622
- // Find elements that map to this doc file
623
- const fileSnippets = failedSnippets.filter(s => s.filePath === failedFile);
624
- const matchingElements = allElements.filter(el => fileSnippets.some(s => {
625
- // Match element name in the snippet's surrounding code or filename
626
- const elNameLower = el.name.toLowerCase();
627
- return s.code.toLowerCase().includes(elNameLower) ||
628
- failedFile.toLowerCase().includes(elNameLower);
629
- }));
630
- if (matchingElements.length > 0) {
631
- // Build error context map: element name → error message
632
- const previousErrors = new Map();
633
- for (const snippet of fileSnippets) {
634
- const errKey = snippet.filePath + ':' + snippet.lineNumber;
635
- const errMsg = snippetErrors.get(errKey);
636
- if (errMsg) {
637
- // Map snippet back to element name
638
- const matchedEl = matchingElements.find(el => snippet.code.toLowerCase().includes(el.name.toLowerCase()));
639
- if (matchedEl) {
640
- previousErrors.set(matchedEl.name, errMsg);
641
- }
642
- }
643
- }
644
- const reDocs = await generateForElements(matchingElements, client, {
645
- ...genOptions,
646
- verify: true,
647
- previousErrors,
648
- onProgress: (p) => {
649
- process.stdout.write(`\r Re-generating: ${p.element} ${p.status}`.padEnd(80));
650
- },
651
- });
652
- console.log('');
653
- // Re-write the doc file with updated content
654
- const fileResults = groupDocsByFile(reDocs);
655
- await writeDocsToDirectory(fileResults, dirname(failedFile), primarySourcePath);
656
- }
657
- }
658
- }
659
- if (failedCount + passedCount + skippedCount > 0) {
660
- console.log(`\n Verification: ${passedCount} passed, ${failedCount} failed, ${skippedCount} skipped`);
661
- }
662
- }
663
- // Write manifest for staleness detection
664
- try {
665
- const manifestEntries = buildManifestEntries(allElements, outputPath);
666
- writeManifest(outputPath, manifestEntries);
667
- }
668
- catch {
669
- // Non-fatal — manifest is optional
670
- }
671
- // Step 5: Embedded QA (auto-fix then check)
672
- console.log(`\nStep ${options.verify ? '5' : '4'}: Running QA checks...`);
673
- const fixReport = fixQAIssues(outputPath);
674
- printFixReport(fixReport);
675
- const qaReport = runQA(outputPath);
676
- printQAReport(qaReport);
677
- // Context Hub export prompt (TTY only — skip in CI/piped mode)
678
- if (process.stdin.isTTY) {
679
- console.log('');
680
- console.log(' Context Hub: Make your docs discoverable by AI coding agents.');
681
- console.log(' Context Hub is a curated registry by Andrew Ng (7K+ stars)');
682
- console.log(' https://github.com/andrewyng/context-hub');
683
- console.log('');
684
- const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
685
- const answer = await new Promise((resolve) => {
686
- rl.question(' Export for Context Hub? (y/N) ', (ans) => {
687
- rl.close();
688
- resolve(ans.trim().toLowerCase());
689
- });
690
- });
691
- if (answer === 'y' || answer === 'yes') {
692
- const languages = multiLanguage ? ['typescript', 'python'] : ['typescript'];
693
- const projName = options.projectName || basename(primarySourcePath);
694
- const result = exportToContextHub(docs, outputPath, {
695
- projectName: projName,
696
- languages,
697
- description: `API documentation for ${projName}`,
698
- });
699
- console.log(`\n Exported ${result.filesWritten} files to ${result.outputDir}`);
700
- console.log(' See context-hub/README.md for submission instructions');
701
- }
702
- }
703
- console.log('\n=== Summary ===');
704
- console.log(` Total elements: ${totalDocs}`);
705
- console.log(` Generated: ${totalDocs - errorCount}`);
706
- if (errorCount > 0) {
707
- console.log(` Errors: ${errorCount}`);
708
- }
709
- console.log(` Duration: ${duration}s`);
710
- console.log(` Output: ${outputPath}`);
711
- if (errorCount > 0) {
712
- console.log('\n Elements with errors:');
713
- docs.filter(d => d.error).slice(0, 10).forEach(d => {
714
- console.log(` - ${d.element.name}: ${d.error?.slice(0, 50)}`);
715
- });
716
- if (errorCount > 10) {
717
- console.log(` ... and ${errorCount - 10} more`);
718
- }
719
- }
720
- console.log('\nDone!');
721
- }
722
- finally {
723
- // Clean up temp directories from --org clones
724
- for (const dir of tempDirs) {
725
- try {
726
- rmSync(dir, { recursive: true, force: true });
727
- }
728
- catch {
729
- // Ignore cleanup errors
730
- }
731
- }
732
- }
733
- }
734
- catch (err) {
735
- const message = err instanceof Error ? err.message : String(err);
736
- console.error(`Error: ${message}`);
737
- process.exit(1);
738
- }
739
- });