skrypt-ai 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/index.d.ts +13 -3
- package/dist/auth/index.js +101 -9
- package/dist/auth/keychain.d.ts +5 -0
- package/dist/auth/keychain.js +82 -0
- package/dist/auth/notices.d.ts +3 -0
- package/dist/auth/notices.js +42 -0
- package/dist/autofix/index.d.ts +0 -4
- package/dist/autofix/index.js +10 -24
- package/dist/capture/browser.d.ts +11 -0
- package/dist/capture/browser.js +173 -0
- package/dist/capture/diff.d.ts +23 -0
- package/dist/capture/diff.js +52 -0
- package/dist/capture/index.d.ts +23 -0
- package/dist/capture/index.js +210 -0
- package/dist/capture/naming.d.ts +17 -0
- package/dist/capture/naming.js +45 -0
- package/dist/capture/parser.d.ts +15 -0
- package/dist/capture/parser.js +80 -0
- package/dist/capture/types.d.ts +57 -0
- package/dist/capture/types.js +1 -0
- package/dist/cli.js +20 -3
- package/dist/commands/autofix.js +136 -120
- package/dist/commands/cron.js +58 -47
- package/dist/commands/deploy.js +123 -102
- package/dist/commands/generate.js +125 -7
- package/dist/commands/heal.d.ts +10 -0
- package/dist/commands/heal.js +201 -0
- package/dist/commands/i18n.js +146 -111
- package/dist/commands/import.d.ts +2 -0
- package/dist/commands/import.js +157 -0
- package/dist/commands/init.js +19 -7
- package/dist/commands/lint.js +50 -44
- package/dist/commands/llms-txt.js +59 -49
- package/dist/commands/login.js +63 -34
- package/dist/commands/mcp.js +6 -0
- package/dist/commands/monitor.js +13 -8
- package/dist/commands/qa.d.ts +2 -0
- package/dist/commands/qa.js +43 -0
- package/dist/commands/review-pr.js +108 -92
- package/dist/commands/sdk.js +128 -122
- package/dist/commands/security.d.ts +2 -0
- package/dist/commands/security.js +109 -0
- package/dist/commands/test.js +91 -92
- package/dist/commands/version.js +104 -75
- package/dist/commands/watch.js +130 -114
- package/dist/config/types.js +2 -2
- package/dist/context-hub/index.d.ts +23 -0
- package/dist/context-hub/index.js +179 -0
- package/dist/context-hub/mappings.d.ts +8 -0
- package/dist/context-hub/mappings.js +55 -0
- package/dist/context-hub/types.d.ts +33 -0
- package/dist/context-hub/types.js +1 -0
- package/dist/generator/generator.js +39 -6
- package/dist/generator/types.d.ts +7 -0
- package/dist/generator/writer.d.ts +3 -1
- package/dist/generator/writer.js +36 -7
- package/dist/importers/confluence.d.ts +5 -0
- package/dist/importers/confluence.js +137 -0
- package/dist/importers/detect.d.ts +20 -0
- package/dist/importers/detect.js +121 -0
- package/dist/importers/docusaurus.d.ts +5 -0
- package/dist/importers/docusaurus.js +279 -0
- package/dist/importers/gitbook.d.ts +5 -0
- package/dist/importers/gitbook.js +189 -0
- package/dist/importers/github.d.ts +8 -0
- package/dist/importers/github.js +99 -0
- package/dist/importers/index.d.ts +15 -0
- package/dist/importers/index.js +30 -0
- package/dist/importers/markdown.d.ts +6 -0
- package/dist/importers/markdown.js +105 -0
- package/dist/importers/mintlify.d.ts +5 -0
- package/dist/importers/mintlify.js +172 -0
- package/dist/importers/notion.d.ts +5 -0
- package/dist/importers/notion.js +174 -0
- package/dist/importers/readme.d.ts +5 -0
- package/dist/importers/readme.js +184 -0
- package/dist/importers/transform.d.ts +90 -0
- package/dist/importers/transform.js +457 -0
- package/dist/importers/types.d.ts +37 -0
- package/dist/importers/types.js +1 -0
- package/dist/llm/anthropic-client.d.ts +1 -0
- package/dist/llm/anthropic-client.js +3 -1
- package/dist/llm/index.d.ts +6 -4
- package/dist/llm/index.js +76 -261
- package/dist/llm/openai-client.d.ts +1 -0
- package/dist/llm/openai-client.js +7 -2
- package/dist/plugins/index.js +7 -0
- package/dist/qa/checks.d.ts +10 -0
- package/dist/qa/checks.js +492 -0
- package/dist/qa/fixes.d.ts +30 -0
- package/dist/qa/fixes.js +277 -0
- package/dist/qa/index.d.ts +29 -0
- package/dist/qa/index.js +187 -0
- package/dist/qa/types.d.ts +24 -0
- package/dist/qa/types.js +1 -0
- package/dist/scanner/csharp.d.ts +23 -0
- package/dist/scanner/csharp.js +421 -0
- package/dist/scanner/index.js +53 -26
- package/dist/scanner/java.d.ts +39 -0
- package/dist/scanner/java.js +318 -0
- package/dist/scanner/kotlin.d.ts +23 -0
- package/dist/scanner/kotlin.js +389 -0
- package/dist/scanner/php.d.ts +57 -0
- package/dist/scanner/php.js +351 -0
- package/dist/scanner/python.js +17 -0
- package/dist/scanner/ruby.d.ts +36 -0
- package/dist/scanner/ruby.js +431 -0
- package/dist/scanner/swift.d.ts +25 -0
- package/dist/scanner/swift.js +392 -0
- package/dist/scanner/types.d.ts +1 -1
- package/dist/template/content/docs/_navigation.json +46 -0
- package/dist/template/content/docs/_sidebars.json +684 -0
- package/dist/template/content/docs/core.md +4544 -0
- package/dist/template/content/docs/index.mdx +89 -0
- package/dist/template/content/docs/integrations.md +1158 -0
- package/dist/template/content/docs/llms-full.md +403 -0
- package/dist/template/content/docs/llms.txt +4588 -0
- package/dist/template/content/docs/other.md +10379 -0
- package/dist/template/content/docs/tools.md +746 -0
- package/dist/template/content/docs/types.md +531 -0
- package/dist/template/docs.json +13 -11
- package/dist/template/mdx-components.tsx +27 -2
- package/dist/template/package.json +6 -0
- package/dist/template/public/search-index.json +1 -1
- package/dist/template/scripts/build-search-index.mjs +149 -13
- package/dist/template/src/app/api/chat/route.ts +83 -128
- package/dist/template/src/app/docs/[...slug]/page.tsx +75 -20
- package/dist/template/src/app/docs/llms-full.md +151 -4
- package/dist/template/src/app/docs/llms.txt +2464 -847
- package/dist/template/src/app/docs/page.mdx +48 -38
- package/dist/template/src/app/layout.tsx +3 -1
- package/dist/template/src/app/page.tsx +22 -8
- package/dist/template/src/components/ai-chat.tsx +73 -64
- package/dist/template/src/components/breadcrumbs.tsx +21 -23
- package/dist/template/src/components/copy-button.tsx +13 -9
- package/dist/template/src/components/copy-page-button.tsx +54 -0
- package/dist/template/src/components/docs-layout.tsx +37 -25
- package/dist/template/src/components/header.tsx +51 -10
- package/dist/template/src/components/mdx/card.tsx +17 -3
- package/dist/template/src/components/mdx/code-block.tsx +13 -9
- package/dist/template/src/components/mdx/code-group.tsx +13 -8
- package/dist/template/src/components/mdx/heading.tsx +15 -2
- package/dist/template/src/components/mdx/highlighted-code.tsx +13 -8
- package/dist/template/src/components/mdx/index.tsx +2 -0
- package/dist/template/src/components/mdx/mermaid.tsx +110 -0
- package/dist/template/src/components/mdx/screenshot.tsx +150 -0
- package/dist/template/src/components/scroll-to-hash.tsx +48 -0
- package/dist/template/src/components/sidebar.tsx +12 -18
- package/dist/template/src/components/table-of-contents.tsx +9 -0
- package/dist/template/src/lib/highlight.ts +3 -88
- package/dist/template/src/lib/navigation.ts +159 -0
- package/dist/template/src/lib/search-types.ts +4 -1
- package/dist/template/src/lib/search.ts +30 -7
- package/dist/template/src/styles/globals.css +17 -6
- package/dist/utils/files.d.ts +9 -1
- package/dist/utils/files.js +59 -10
- package/dist/utils/validation.d.ts +0 -3
- package/dist/utils/validation.js +0 -26
- package/package.json +5 -1
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync, rmSync } from 'fs';
|
|
2
|
+
import { join, extname } from 'path';
|
|
3
|
+
import { tmpdir } from 'os';
|
|
4
|
+
import { detectFormat } from './detect.js';
|
|
5
|
+
import { runImport } from './index.js';
|
|
6
|
+
// Only fetch text-based files for import
|
|
7
|
+
const TEXT_EXTENSIONS = new Set([
|
|
8
|
+
'.md', '.mdx', '.json', '.yaml', '.yml', '.js', '.ts', '.txt',
|
|
9
|
+
'.html', '.htm', '.xml', '.css', '.toml', '.cfg', '.ini', '.rst',
|
|
10
|
+
]);
|
|
11
|
+
const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp', '.ico']);
|
|
12
|
+
const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB limit per file
|
|
13
|
+
const MAX_DEPTH = 20; // Prevent unbounded recursion
|
|
14
|
+
/**
|
|
15
|
+
* Fetch docs from a GitHub URL and import them.
|
|
16
|
+
*/
|
|
17
|
+
export async function importFromGitHub(owner, repo, path, ref, options) {
|
|
18
|
+
const token = process.env.GITHUB_TOKEN;
|
|
19
|
+
const headers = {
|
|
20
|
+
Accept: 'application/vnd.github.v3+json',
|
|
21
|
+
'User-Agent': 'skrypt-cli',
|
|
22
|
+
};
|
|
23
|
+
if (token) {
|
|
24
|
+
headers.Authorization = `Bearer ${token}`;
|
|
25
|
+
}
|
|
26
|
+
const tempDir = join(tmpdir(), `skrypt-import-${Date.now()}`);
|
|
27
|
+
mkdirSync(tempDir, { recursive: true });
|
|
28
|
+
try {
|
|
29
|
+
console.log(` Fetching from GitHub: ${owner}/${repo}/${path || ''}...`);
|
|
30
|
+
let fileCount = 0;
|
|
31
|
+
async function fetchDir(dirPath, localDir, depth) {
|
|
32
|
+
if (depth > MAX_DEPTH) {
|
|
33
|
+
console.warn(` Warning: Max directory depth (${MAX_DEPTH}) reached, skipping deeper paths`);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${dirPath}?ref=${ref}`;
|
|
37
|
+
const res = await fetch(apiUrl, { headers });
|
|
38
|
+
if (res.status === 403) {
|
|
39
|
+
const rateLimitRemaining = res.headers.get('X-RateLimit-Remaining');
|
|
40
|
+
if (rateLimitRemaining === '0') {
|
|
41
|
+
const resetTime = res.headers.get('X-RateLimit-Reset');
|
|
42
|
+
const resetDate = resetTime ? new Date(parseInt(resetTime) * 1000) : null;
|
|
43
|
+
throw new Error(`GitHub API rate limit exceeded. ${!token ? 'Set GITHUB_TOKEN for higher limits. ' : ''}${resetDate ? `Resets at ${resetDate.toLocaleTimeString()}.` : ''}`);
|
|
44
|
+
}
|
|
45
|
+
throw new Error(`GitHub API forbidden: ${res.statusText}`);
|
|
46
|
+
}
|
|
47
|
+
if (!res.ok) {
|
|
48
|
+
throw new Error(`GitHub API error: ${res.status} ${res.statusText}`);
|
|
49
|
+
}
|
|
50
|
+
const entries = await res.json();
|
|
51
|
+
for (const entry of entries) {
|
|
52
|
+
if (entry.type === 'dir') {
|
|
53
|
+
const subDir = join(localDir, entry.name);
|
|
54
|
+
mkdirSync(subDir, { recursive: true });
|
|
55
|
+
await fetchDir(entry.path, subDir, depth + 1);
|
|
56
|
+
}
|
|
57
|
+
else if (entry.type === 'file' && entry.download_url) {
|
|
58
|
+
// Skip files that are too large
|
|
59
|
+
if (entry.size && entry.size > MAX_FILE_SIZE) {
|
|
60
|
+
console.warn(` Skipping large file: ${entry.name} (${Math.round(entry.size / 1024)}KB)`);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
const ext = extname(entry.name).toLowerCase();
|
|
64
|
+
const isText = TEXT_EXTENSIONS.has(ext);
|
|
65
|
+
const isImage = IMAGE_EXTENSIONS.has(ext);
|
|
66
|
+
if (!isText && !isImage)
|
|
67
|
+
continue;
|
|
68
|
+
const fileRes = await fetch(entry.download_url, { headers });
|
|
69
|
+
if (fileRes.ok) {
|
|
70
|
+
if (isImage) {
|
|
71
|
+
// Fetch binary content properly for images
|
|
72
|
+
const buffer = Buffer.from(await fileRes.arrayBuffer());
|
|
73
|
+
writeFileSync(join(localDir, entry.name), buffer);
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
const content = await fileRes.text();
|
|
77
|
+
writeFileSync(join(localDir, entry.name), content);
|
|
78
|
+
}
|
|
79
|
+
fileCount++;
|
|
80
|
+
process.stdout.write(`\r Fetched ${fileCount} files...`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
await fetchDir(path, tempDir, 0);
|
|
86
|
+
console.log(`\r Fetched ${fileCount} files from GitHub`);
|
|
87
|
+
const format = options?.format || detectFormat(tempDir);
|
|
88
|
+
return runImport(tempDir, format, options?.name);
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
// Clean up temp directory
|
|
92
|
+
try {
|
|
93
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
// Best-effort cleanup
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export type { ImportFormat, ImportResult, ImportedGroup, ImportedPage, TransformStats } from './types.js';
|
|
2
|
+
export { detectFormat, isGitHubUrl, parseGitHubUrl } from './detect.js';
|
|
3
|
+
export { importMarkdown } from './markdown.js';
|
|
4
|
+
export { importMintlify } from './mintlify.js';
|
|
5
|
+
export { importDocusaurus } from './docusaurus.js';
|
|
6
|
+
export { importGitBook } from './gitbook.js';
|
|
7
|
+
export { importReadme } from './readme.js';
|
|
8
|
+
export { importNotion } from './notion.js';
|
|
9
|
+
export { importConfluence } from './confluence.js';
|
|
10
|
+
export { importFromGitHub } from './github.js';
|
|
11
|
+
import type { ImportFormat, ImportResult } from './types.js';
|
|
12
|
+
/**
|
|
13
|
+
* Run the appropriate importer for a detected format.
|
|
14
|
+
*/
|
|
15
|
+
export declare function runImport(dir: string, format: ImportFormat, name?: string): ImportResult;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export { detectFormat, isGitHubUrl, parseGitHubUrl } from './detect.js';
|
|
2
|
+
export { importMarkdown } from './markdown.js';
|
|
3
|
+
export { importMintlify } from './mintlify.js';
|
|
4
|
+
export { importDocusaurus } from './docusaurus.js';
|
|
5
|
+
export { importGitBook } from './gitbook.js';
|
|
6
|
+
export { importReadme } from './readme.js';
|
|
7
|
+
export { importNotion } from './notion.js';
|
|
8
|
+
export { importConfluence } from './confluence.js';
|
|
9
|
+
export { importFromGitHub } from './github.js';
|
|
10
|
+
import { importMarkdown } from './markdown.js';
|
|
11
|
+
import { importMintlify } from './mintlify.js';
|
|
12
|
+
import { importDocusaurus } from './docusaurus.js';
|
|
13
|
+
import { importGitBook } from './gitbook.js';
|
|
14
|
+
import { importReadme } from './readme.js';
|
|
15
|
+
import { importNotion } from './notion.js';
|
|
16
|
+
import { importConfluence } from './confluence.js';
|
|
17
|
+
/**
|
|
18
|
+
* Run the appropriate importer for a detected format.
|
|
19
|
+
*/
|
|
20
|
+
export function runImport(dir, format, name) {
|
|
21
|
+
switch (format) {
|
|
22
|
+
case 'mintlify': return importMintlify(dir, name);
|
|
23
|
+
case 'docusaurus': return importDocusaurus(dir, name);
|
|
24
|
+
case 'gitbook': return importGitBook(dir, name);
|
|
25
|
+
case 'readme': return importReadme(dir, name);
|
|
26
|
+
case 'notion': return importNotion(dir, name);
|
|
27
|
+
case 'confluence': return importConfluence(dir, name);
|
|
28
|
+
case 'markdown': return importMarkdown(dir, name);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { readFileSync, existsSync } from 'fs';
|
|
2
|
+
import { join, relative, dirname, basename, extname } from 'path';
|
|
3
|
+
import { findMdxFiles } from '../utils/files.js';
|
|
4
|
+
import { normalizeFrontmatter, getSortWeight } from './transform.js';
|
|
5
|
+
/**
|
|
6
|
+
* Import plain Markdown/MDX directory.
|
|
7
|
+
* Folder structure → groups, files → pages.
|
|
8
|
+
*/
|
|
9
|
+
export function importMarkdown(dir, name) {
|
|
10
|
+
const files = findMdxFiles(dir);
|
|
11
|
+
const result = createEmptyResult('markdown', name);
|
|
12
|
+
const stats = { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 };
|
|
13
|
+
if (files.length === 0) {
|
|
14
|
+
result.warnings.push('No .md or .mdx files found');
|
|
15
|
+
return result;
|
|
16
|
+
}
|
|
17
|
+
// Group files by directory
|
|
18
|
+
const grouped = new Map();
|
|
19
|
+
for (const filePath of files) {
|
|
20
|
+
const rel = relative(dir, filePath);
|
|
21
|
+
const dirName = dirname(rel);
|
|
22
|
+
const group = dirName === '.' ? 'Documentation' : dirName.split('/')[0];
|
|
23
|
+
if (!grouped.has(group))
|
|
24
|
+
grouped.set(group, []);
|
|
25
|
+
grouped.get(group).push({ path: filePath, content: readFileSync(filePath, 'utf-8') });
|
|
26
|
+
}
|
|
27
|
+
// Build navigation
|
|
28
|
+
for (const [groupName, groupFiles] of grouped) {
|
|
29
|
+
const pages = [];
|
|
30
|
+
// Sort: index/README first, then by frontmatter order, then alphabetical
|
|
31
|
+
const sorted = groupFiles.sort((a, b) => {
|
|
32
|
+
const aName = basename(a.path, extname(a.path)).toLowerCase();
|
|
33
|
+
const bName = basename(b.path, extname(b.path)).toLowerCase();
|
|
34
|
+
if (aName === 'index' || aName === 'readme')
|
|
35
|
+
return -1;
|
|
36
|
+
if (bName === 'index' || bName === 'readme')
|
|
37
|
+
return 1;
|
|
38
|
+
const aWeight = getSortWeight(a.content);
|
|
39
|
+
const bWeight = getSortWeight(b.content);
|
|
40
|
+
if (aWeight !== bWeight)
|
|
41
|
+
return aWeight - bWeight;
|
|
42
|
+
return aName.localeCompare(bName);
|
|
43
|
+
});
|
|
44
|
+
for (const file of sorted) {
|
|
45
|
+
const rel = relative(dir, file.path);
|
|
46
|
+
const slug = rel.replace(/\.(md|mdx)$/, '').replace(/\\/g, '/');
|
|
47
|
+
const title = extractTitle(file.content, file.path);
|
|
48
|
+
const transformed = normalizeFrontmatter(file.content, { title });
|
|
49
|
+
const outputPath = `content/docs/${slug}.mdx`;
|
|
50
|
+
pages.push({ title, slug, sourcePath: rel, content: transformed });
|
|
51
|
+
result.files.set(outputPath, transformed);
|
|
52
|
+
// Track images
|
|
53
|
+
const imgMatches = transformed.match(/!\[.*?\]\(([^)]+)\)/g);
|
|
54
|
+
if (imgMatches) {
|
|
55
|
+
for (const imgMatch of imgMatches) {
|
|
56
|
+
const src = imgMatch.match(/!\[.*?\]\(([^)]+)\)/)?.[1];
|
|
57
|
+
if (src && !src.startsWith('http')) {
|
|
58
|
+
const srcPath = join(dir, dirname(rel), src);
|
|
59
|
+
if (existsSync(srcPath)) {
|
|
60
|
+
const destPath = `public/images/${basename(src)}`;
|
|
61
|
+
result.assets.set(destPath, srcPath);
|
|
62
|
+
stats.images++;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
result.navigation.push({ group: titleCase(groupName), pages });
|
|
69
|
+
}
|
|
70
|
+
result.stats = {
|
|
71
|
+
pages: result.files.size,
|
|
72
|
+
groups: result.navigation.length,
|
|
73
|
+
transforms: stats,
|
|
74
|
+
};
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
function extractTitle(content, filePath) {
|
|
78
|
+
// Try frontmatter title
|
|
79
|
+
const fmMatch = content.match(/^---\n[\s\S]*?title:\s*["']?([^"'\n]+)["']?\s*\n[\s\S]*?---/m);
|
|
80
|
+
if (fmMatch)
|
|
81
|
+
return fmMatch[1].trim();
|
|
82
|
+
// Try first H1
|
|
83
|
+
const h1Match = content.match(/^#\s+(.+)$/m);
|
|
84
|
+
if (h1Match)
|
|
85
|
+
return h1Match[1].trim();
|
|
86
|
+
// Fallback to filename
|
|
87
|
+
return titleCase(basename(filePath, extname(filePath)).replace(/[-_]/g, ' '));
|
|
88
|
+
}
|
|
89
|
+
function titleCase(str) {
|
|
90
|
+
return str
|
|
91
|
+
.replace(/[-_]/g, ' ')
|
|
92
|
+
.replace(/\b\w/g, c => c.toUpperCase());
|
|
93
|
+
}
|
|
94
|
+
function createEmptyResult(format, name) {
|
|
95
|
+
return {
|
|
96
|
+
navigation: [],
|
|
97
|
+
name: name || 'Documentation',
|
|
98
|
+
description: 'Imported documentation',
|
|
99
|
+
files: new Map(),
|
|
100
|
+
assets: new Map(),
|
|
101
|
+
warnings: [],
|
|
102
|
+
stats: { pages: 0, groups: 0, transforms: { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 } },
|
|
103
|
+
sourceFormat: format,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { readFileSync, existsSync } from 'fs';
|
|
2
|
+
import { join, relative, basename, extname } from 'path';
|
|
3
|
+
import { findMdxFiles } from '../utils/files.js';
|
|
4
|
+
import { transformMintlifyCallouts, transformMintlifyTabs, normalizeFrontmatter, rewriteImagePaths } from './transform.js';
|
|
5
|
+
/**
|
|
6
|
+
* Import Mintlify documentation.
|
|
7
|
+
*/
|
|
8
|
+
export function importMintlify(dir, name) {
|
|
9
|
+
const result = createEmptyResult(name);
|
|
10
|
+
const stats = { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 };
|
|
11
|
+
// Load config
|
|
12
|
+
const config = loadMintlifyConfig(dir);
|
|
13
|
+
if (!config) {
|
|
14
|
+
result.warnings.push('Could not parse mint.json or docs.json');
|
|
15
|
+
return result;
|
|
16
|
+
}
|
|
17
|
+
result.name = config.name || name || 'Documentation';
|
|
18
|
+
result.description = config.description || 'Imported from Mintlify';
|
|
19
|
+
// Process navigation
|
|
20
|
+
if (config.navigation) {
|
|
21
|
+
for (const navGroup of config.navigation) {
|
|
22
|
+
const pages = [];
|
|
23
|
+
for (const pageRef of navGroup.pages) {
|
|
24
|
+
if (typeof pageRef === 'string') {
|
|
25
|
+
const page = processPage(dir, pageRef, stats, result);
|
|
26
|
+
if (page)
|
|
27
|
+
pages.push(page);
|
|
28
|
+
}
|
|
29
|
+
else if (typeof pageRef === 'object' && pageRef.pages) {
|
|
30
|
+
// Nested group — flatten into pages with prefix
|
|
31
|
+
for (const nestedPage of pageRef.pages) {
|
|
32
|
+
if (typeof nestedPage === 'string') {
|
|
33
|
+
const page = processPage(dir, nestedPage, stats, result);
|
|
34
|
+
if (page)
|
|
35
|
+
pages.push(page);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (pages.length > 0) {
|
|
41
|
+
result.navigation.push({ group: navGroup.group, pages });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
// No navigation config — walk files
|
|
47
|
+
const files = findMdxFiles(dir);
|
|
48
|
+
const pages = [];
|
|
49
|
+
for (const filePath of files) {
|
|
50
|
+
const rel = relative(dir, filePath).replace(/\\/g, '/').replace(/\.(md|mdx)$/, '');
|
|
51
|
+
const page = processPage(dir, rel, stats, result);
|
|
52
|
+
if (page)
|
|
53
|
+
pages.push(page);
|
|
54
|
+
}
|
|
55
|
+
if (pages.length > 0) {
|
|
56
|
+
result.navigation.push({ group: 'Documentation', pages });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// Copy images
|
|
60
|
+
collectImages(dir, result, stats);
|
|
61
|
+
result.stats = {
|
|
62
|
+
pages: result.files.size,
|
|
63
|
+
groups: result.navigation.length,
|
|
64
|
+
transforms: stats,
|
|
65
|
+
};
|
|
66
|
+
return result;
|
|
67
|
+
}
|
|
68
|
+
function processPage(dir, pageRef, stats, result) {
|
|
69
|
+
// Find file (try .mdx first, then .md)
|
|
70
|
+
let filePath = join(dir, `${pageRef}.mdx`);
|
|
71
|
+
if (!existsSync(filePath)) {
|
|
72
|
+
filePath = join(dir, `${pageRef}.md`);
|
|
73
|
+
}
|
|
74
|
+
if (!existsSync(filePath)) {
|
|
75
|
+
result.warnings.push(`Page not found: ${pageRef}`);
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
let content = readFileSync(filePath, 'utf-8');
|
|
79
|
+
// Handle <Snippet file="x.mdx" /> — inline referenced files
|
|
80
|
+
content = content.replace(/<Snippet\s+file="([^"]+)"\s*\/>/g, (_match, snippetPath) => {
|
|
81
|
+
const snippetFile = join(dir, snippetPath);
|
|
82
|
+
// Guard: prevent path traversal outside source directory
|
|
83
|
+
if (!snippetFile.startsWith(dir)) {
|
|
84
|
+
result.warnings.push(`Snippet path traversal blocked: ${snippetPath}`);
|
|
85
|
+
return `<!-- Snippet blocked: ${snippetPath} -->`;
|
|
86
|
+
}
|
|
87
|
+
if (existsSync(snippetFile)) {
|
|
88
|
+
return readFileSync(snippetFile, 'utf-8');
|
|
89
|
+
}
|
|
90
|
+
result.warnings.push(`Snippet not found: ${snippetPath}`);
|
|
91
|
+
return `<!-- Snippet not found: ${snippetPath} -->`;
|
|
92
|
+
});
|
|
93
|
+
// Apply transforms
|
|
94
|
+
const originalContent = content;
|
|
95
|
+
content = transformMintlifyCallouts(content);
|
|
96
|
+
content = transformMintlifyTabs(content);
|
|
97
|
+
content = normalizeFrontmatter(content);
|
|
98
|
+
// Count transforms
|
|
99
|
+
stats.callouts += countDiff(originalContent, content, '<Callout');
|
|
100
|
+
stats.tabs += countDiff(originalContent, content, '<TabPanel');
|
|
101
|
+
const title = extractTitle(content, filePath);
|
|
102
|
+
const slug = pageRef.replace(/\\/g, '/');
|
|
103
|
+
const outputPath = `content/docs/${slug}.mdx`;
|
|
104
|
+
result.files.set(outputPath, content);
|
|
105
|
+
return { title, slug, sourcePath: relative(dir, filePath), content };
|
|
106
|
+
}
|
|
107
|
+
function collectImages(dir, result, stats) {
|
|
108
|
+
// Scan all processed files for image references
|
|
109
|
+
const imageMapping = new Map();
|
|
110
|
+
for (const [, content] of result.files) {
|
|
111
|
+
const imgMatches = content.matchAll(/!\[.*?\]\(([^)]+)\)/g);
|
|
112
|
+
for (const match of imgMatches) {
|
|
113
|
+
const src = match[1];
|
|
114
|
+
if (src && !src.startsWith('http')) {
|
|
115
|
+
const srcPath = join(dir, src);
|
|
116
|
+
if (existsSync(srcPath)) {
|
|
117
|
+
const destPath = `/images/${basename(src)}`;
|
|
118
|
+
result.assets.set(`public${destPath}`, srcPath);
|
|
119
|
+
imageMapping.set(src, destPath);
|
|
120
|
+
stats.images++;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// Rewrite image paths in all files
|
|
126
|
+
if (imageMapping.size > 0) {
|
|
127
|
+
for (const [path, content] of result.files) {
|
|
128
|
+
result.files.set(path, rewriteImagePaths(content, imageMapping));
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
function loadMintlifyConfig(dir) {
|
|
133
|
+
for (const configName of ['mint.json', 'docs.json']) {
|
|
134
|
+
const configPath = join(dir, configName);
|
|
135
|
+
if (existsSync(configPath)) {
|
|
136
|
+
try {
|
|
137
|
+
return JSON.parse(readFileSync(configPath, 'utf-8'));
|
|
138
|
+
}
|
|
139
|
+
catch { /* try next */ }
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
function extractTitle(content, filePath) {
|
|
145
|
+
const fmMatch = content.match(/^---\n[\s\S]*?title:\s*["']?([^"'\n]+)["']?\s*\n[\s\S]*?---/m);
|
|
146
|
+
if (fmMatch)
|
|
147
|
+
return fmMatch[1].trim();
|
|
148
|
+
const h1Match = content.match(/^#\s+(.+)$/m);
|
|
149
|
+
if (h1Match)
|
|
150
|
+
return h1Match[1].trim();
|
|
151
|
+
return basename(filePath, extname(filePath)).replace(/[-_]/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
|
152
|
+
}
|
|
153
|
+
function countDiff(original, transformed, marker) {
|
|
154
|
+
const origCount = (original.match(new RegExp(escapeRegex(marker), 'g')) || []).length;
|
|
155
|
+
const newCount = (transformed.match(new RegExp(escapeRegex(marker), 'g')) || []).length;
|
|
156
|
+
return Math.max(0, newCount - origCount);
|
|
157
|
+
}
|
|
158
|
+
function escapeRegex(str) {
|
|
159
|
+
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
160
|
+
}
|
|
161
|
+
function createEmptyResult(name) {
|
|
162
|
+
return {
|
|
163
|
+
navigation: [],
|
|
164
|
+
name: name || 'Documentation',
|
|
165
|
+
description: 'Imported from Mintlify',
|
|
166
|
+
files: new Map(),
|
|
167
|
+
assets: new Map(),
|
|
168
|
+
warnings: [],
|
|
169
|
+
stats: { pages: 0, groups: 0, transforms: { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 } },
|
|
170
|
+
sourceFormat: 'mintlify',
|
|
171
|
+
};
|
|
172
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { readFileSync, readdirSync, statSync } from 'fs';
|
|
2
|
+
import { join, relative, basename, extname } from 'path';
|
|
3
|
+
import { transformNotionCallouts, transformNotionToggles, normalizeFrontmatter, stripNotionUUIDs, } from './transform.js';
|
|
4
|
+
/**
|
|
5
|
+
* Import Notion export (extracted folder with UUID-named files).
|
|
6
|
+
*/
|
|
7
|
+
export function importNotion(dir, name) {
|
|
8
|
+
const result = createEmptyResult(name);
|
|
9
|
+
const stats = { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 };
|
|
10
|
+
// Walk the export directory
|
|
11
|
+
const groups = discoverGroups(dir);
|
|
12
|
+
for (const group of groups) {
|
|
13
|
+
const pages = [];
|
|
14
|
+
for (const file of group.files) {
|
|
15
|
+
const page = processNotionPage(dir, file, stats, result);
|
|
16
|
+
if (page)
|
|
17
|
+
pages.push(page);
|
|
18
|
+
}
|
|
19
|
+
if (pages.length > 0) {
|
|
20
|
+
result.navigation.push({ group: group.label, pages });
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
// Copy images (strip UUIDs from filenames)
|
|
24
|
+
collectNotionImages(dir, result, stats);
|
|
25
|
+
result.stats = {
|
|
26
|
+
pages: result.files.size,
|
|
27
|
+
groups: result.navigation.length,
|
|
28
|
+
transforms: stats,
|
|
29
|
+
};
|
|
30
|
+
return result;
|
|
31
|
+
}
|
|
32
|
+
function discoverGroups(dir) {
|
|
33
|
+
const groups = [];
|
|
34
|
+
// Root-level markdown files
|
|
35
|
+
const rootFiles = [];
|
|
36
|
+
const entries = readdirSync(dir);
|
|
37
|
+
for (const entry of entries) {
|
|
38
|
+
const fullPath = join(dir, entry);
|
|
39
|
+
try {
|
|
40
|
+
if (statSync(fullPath).isFile() && /\.(md|mdx)$/.test(entry)) {
|
|
41
|
+
rootFiles.push(fullPath);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (rootFiles.length > 0) {
|
|
49
|
+
groups.push({ label: 'Documentation', files: rootFiles });
|
|
50
|
+
}
|
|
51
|
+
// Subdirectories (Notion exports create folders for sub-pages)
|
|
52
|
+
for (const entry of entries) {
|
|
53
|
+
const fullPath = join(dir, entry);
|
|
54
|
+
try {
|
|
55
|
+
if (!statSync(fullPath).isDirectory() || entry.startsWith('.'))
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const mdFiles = findMdFilesRecursive(fullPath);
|
|
62
|
+
if (mdFiles.length === 0)
|
|
63
|
+
continue;
|
|
64
|
+
// Clean UUID from folder name
|
|
65
|
+
const cleanName = stripNotionUUIDs(entry)
|
|
66
|
+
.replace(/[-_]/g, ' ')
|
|
67
|
+
.replace(/\b\w/g, c => c.toUpperCase())
|
|
68
|
+
.trim();
|
|
69
|
+
groups.push({ label: cleanName || entry, files: mdFiles });
|
|
70
|
+
}
|
|
71
|
+
return groups;
|
|
72
|
+
}
|
|
73
|
+
function findMdFilesRecursive(dir) {
|
|
74
|
+
const files = [];
|
|
75
|
+
try {
|
|
76
|
+
for (const entry of readdirSync(dir)) {
|
|
77
|
+
const fullPath = join(dir, entry);
|
|
78
|
+
try {
|
|
79
|
+
const stat = statSync(fullPath);
|
|
80
|
+
if (stat.isDirectory() && !entry.startsWith('.')) {
|
|
81
|
+
files.push(...findMdFilesRecursive(fullPath));
|
|
82
|
+
}
|
|
83
|
+
else if (stat.isFile() && /\.(md|mdx)$/.test(entry)) {
|
|
84
|
+
files.push(fullPath);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
catch { /* skip */ }
|
|
93
|
+
return files;
|
|
94
|
+
}
|
|
95
|
+
function processNotionPage(dir, filePath, stats, result) {
|
|
96
|
+
let content = readFileSync(filePath, 'utf-8');
|
|
97
|
+
const originalContent = content;
|
|
98
|
+
// Apply Notion transforms
|
|
99
|
+
content = transformNotionCallouts(content);
|
|
100
|
+
content = transformNotionToggles(content);
|
|
101
|
+
content = normalizeFrontmatter(content);
|
|
102
|
+
stats.callouts += countNew(originalContent, content, '<Callout');
|
|
103
|
+
stats.accordions += countNew(originalContent, content, '<Accordion');
|
|
104
|
+
const rel = relative(dir, filePath);
|
|
105
|
+
const cleanFilename = stripNotionUUIDs(rel.replace(/\.(md|mdx)$/, '')).replace(/\\/g, '/');
|
|
106
|
+
const slug = cleanFilename.toLowerCase().replace(/[^a-z0-9/]+/g, '-').replace(/^-|-$/g, '');
|
|
107
|
+
const title = extractTitle(content, filePath);
|
|
108
|
+
const outputPath = `content/docs/${slug}.mdx`;
|
|
109
|
+
result.files.set(outputPath, content);
|
|
110
|
+
return { title, slug, sourcePath: rel, content };
|
|
111
|
+
}
|
|
112
|
+
function collectNotionImages(dir, result, stats) {
|
|
113
|
+
const imgExts = new Set(['.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp']);
|
|
114
|
+
function walk(currentDir) {
|
|
115
|
+
try {
|
|
116
|
+
for (const entry of readdirSync(currentDir)) {
|
|
117
|
+
const fullPath = join(currentDir, entry);
|
|
118
|
+
try {
|
|
119
|
+
const stat = statSync(fullPath);
|
|
120
|
+
if (stat.isDirectory()) {
|
|
121
|
+
walk(fullPath);
|
|
122
|
+
}
|
|
123
|
+
else if (imgExts.has(extname(entry).toLowerCase())) {
|
|
124
|
+
const cleanName = stripNotionUUIDs(basename(entry, extname(entry))) + extname(entry);
|
|
125
|
+
const dest = `public/images/${cleanName}`;
|
|
126
|
+
result.assets.set(dest, fullPath);
|
|
127
|
+
// Rewrite paths in content
|
|
128
|
+
const origRef = relative(dir, fullPath);
|
|
129
|
+
for (const [path, content] of result.files) {
|
|
130
|
+
if (content.includes(origRef) || content.includes(entry)) {
|
|
131
|
+
result.files.set(path, content.replaceAll(origRef, `/images/${cleanName}`).replaceAll(entry, `/images/${cleanName}`));
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
stats.images++;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
catch { /* skip */ }
|
|
143
|
+
}
|
|
144
|
+
walk(dir);
|
|
145
|
+
}
|
|
146
|
+
function extractTitle(content, filePath) {
|
|
147
|
+
const fmMatch = content.match(/^---\n[\s\S]*?title:\s*["']?([^"'\n]+)["']?\s*\n[\s\S]*?---/m);
|
|
148
|
+
if (fmMatch)
|
|
149
|
+
return fmMatch[1].trim();
|
|
150
|
+
const h1Match = content.match(/^#\s+(.+)$/m);
|
|
151
|
+
if (h1Match)
|
|
152
|
+
return h1Match[1].trim();
|
|
153
|
+
// Strip UUID from filename for title
|
|
154
|
+
const cleanName = stripNotionUUIDs(basename(filePath, extname(filePath)));
|
|
155
|
+
return cleanName.replace(/[-_]/g, ' ').replace(/\b\w/g, c => c.toUpperCase()).trim();
|
|
156
|
+
}
|
|
157
|
+
function countNew(original, transformed, marker) {
|
|
158
|
+
const escaped = marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
159
|
+
const origCount = (original.match(new RegExp(escaped, 'g')) || []).length;
|
|
160
|
+
const newCount = (transformed.match(new RegExp(escaped, 'g')) || []).length;
|
|
161
|
+
return Math.max(0, newCount - origCount);
|
|
162
|
+
}
|
|
163
|
+
function createEmptyResult(name) {
|
|
164
|
+
return {
|
|
165
|
+
navigation: [],
|
|
166
|
+
name: name || 'Documentation',
|
|
167
|
+
description: 'Imported from Notion',
|
|
168
|
+
files: new Map(),
|
|
169
|
+
assets: new Map(),
|
|
170
|
+
warnings: [],
|
|
171
|
+
stats: { pages: 0, groups: 0, transforms: { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 } },
|
|
172
|
+
sourceFormat: 'notion',
|
|
173
|
+
};
|
|
174
|
+
}
|