skrypt-ai 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/index.d.ts +13 -3
- package/dist/auth/index.js +94 -9
- package/dist/auth/keychain.d.ts +5 -0
- package/dist/auth/keychain.js +82 -0
- package/dist/auth/notices.d.ts +3 -0
- package/dist/auth/notices.js +42 -0
- package/dist/autofix/index.js +10 -3
- package/dist/cli.js +16 -3
- package/dist/commands/generate.js +37 -1
- package/dist/commands/import.d.ts +2 -0
- package/dist/commands/import.js +157 -0
- package/dist/commands/init.js +19 -7
- package/dist/commands/login.js +15 -4
- package/dist/commands/review-pr.js +10 -0
- package/dist/commands/security.d.ts +2 -0
- package/dist/commands/security.js +103 -0
- package/dist/config/loader.js +2 -2
- package/dist/generator/writer.js +12 -3
- package/dist/importers/confluence.d.ts +5 -0
- package/dist/importers/confluence.js +137 -0
- package/dist/importers/detect.d.ts +20 -0
- package/dist/importers/detect.js +121 -0
- package/dist/importers/docusaurus.d.ts +5 -0
- package/dist/importers/docusaurus.js +279 -0
- package/dist/importers/gitbook.d.ts +5 -0
- package/dist/importers/gitbook.js +189 -0
- package/dist/importers/github.d.ts +8 -0
- package/dist/importers/github.js +99 -0
- package/dist/importers/index.d.ts +15 -0
- package/dist/importers/index.js +30 -0
- package/dist/importers/markdown.d.ts +6 -0
- package/dist/importers/markdown.js +105 -0
- package/dist/importers/mintlify.d.ts +5 -0
- package/dist/importers/mintlify.js +172 -0
- package/dist/importers/notion.d.ts +5 -0
- package/dist/importers/notion.js +174 -0
- package/dist/importers/readme.d.ts +5 -0
- package/dist/importers/readme.js +184 -0
- package/dist/importers/transform.d.ts +90 -0
- package/dist/importers/transform.js +457 -0
- package/dist/importers/types.d.ts +37 -0
- package/dist/importers/types.js +1 -0
- package/dist/plugins/index.js +7 -0
- package/dist/scanner/index.js +37 -24
- package/dist/scanner/python.js +17 -0
- package/dist/template/public/search-index.json +1 -1
- package/dist/template/scripts/build-search-index.mjs +67 -9
- package/dist/template/src/components/mdx/dark-image.tsx +56 -0
- package/dist/template/src/components/mdx/frame.tsx +64 -0
- package/dist/template/src/components/mdx/highlighted-code.tsx +145 -31
- package/dist/template/src/components/mdx/index.tsx +4 -0
- package/dist/template/src/components/mdx/link-preview.tsx +119 -0
- package/dist/template/src/components/mdx/tooltip.tsx +101 -0
- package/dist/template/src/components/syntax-theme-selector.tsx +167 -20
- package/dist/template/src/lib/search-types.ts +4 -1
- package/dist/template/src/lib/search.ts +30 -7
- package/dist/template/src/styles/globals.css +39 -0
- package/dist/utils/files.d.ts +9 -1
- package/dist/utils/files.js +59 -10
- package/dist/utils/validation.js +1 -1
- package/package.json +4 -1
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { readFileSync, existsSync } from 'fs';
|
|
2
|
+
import { join, relative, dirname, basename, extname } from 'path';
|
|
3
|
+
import { findMdxFiles } from '../utils/files.js';
|
|
4
|
+
import { normalizeFrontmatter, getSortWeight } from './transform.js';
|
|
5
|
+
/**
|
|
6
|
+
* Import plain Markdown/MDX directory.
|
|
7
|
+
* Folder structure → groups, files → pages.
|
|
8
|
+
*/
|
|
9
|
+
export function importMarkdown(dir, name) {
|
|
10
|
+
const files = findMdxFiles(dir);
|
|
11
|
+
const result = createEmptyResult('markdown', name);
|
|
12
|
+
const stats = { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 };
|
|
13
|
+
if (files.length === 0) {
|
|
14
|
+
result.warnings.push('No .md or .mdx files found');
|
|
15
|
+
return result;
|
|
16
|
+
}
|
|
17
|
+
// Group files by directory
|
|
18
|
+
const grouped = new Map();
|
|
19
|
+
for (const filePath of files) {
|
|
20
|
+
const rel = relative(dir, filePath);
|
|
21
|
+
const dirName = dirname(rel);
|
|
22
|
+
const group = dirName === '.' ? 'Documentation' : dirName.split('/')[0];
|
|
23
|
+
if (!grouped.has(group))
|
|
24
|
+
grouped.set(group, []);
|
|
25
|
+
grouped.get(group).push({ path: filePath, content: readFileSync(filePath, 'utf-8') });
|
|
26
|
+
}
|
|
27
|
+
// Build navigation
|
|
28
|
+
for (const [groupName, groupFiles] of grouped) {
|
|
29
|
+
const pages = [];
|
|
30
|
+
// Sort: index/README first, then by frontmatter order, then alphabetical
|
|
31
|
+
const sorted = groupFiles.sort((a, b) => {
|
|
32
|
+
const aName = basename(a.path, extname(a.path)).toLowerCase();
|
|
33
|
+
const bName = basename(b.path, extname(b.path)).toLowerCase();
|
|
34
|
+
if (aName === 'index' || aName === 'readme')
|
|
35
|
+
return -1;
|
|
36
|
+
if (bName === 'index' || bName === 'readme')
|
|
37
|
+
return 1;
|
|
38
|
+
const aWeight = getSortWeight(a.content);
|
|
39
|
+
const bWeight = getSortWeight(b.content);
|
|
40
|
+
if (aWeight !== bWeight)
|
|
41
|
+
return aWeight - bWeight;
|
|
42
|
+
return aName.localeCompare(bName);
|
|
43
|
+
});
|
|
44
|
+
for (const file of sorted) {
|
|
45
|
+
const rel = relative(dir, file.path);
|
|
46
|
+
const slug = rel.replace(/\.(md|mdx)$/, '').replace(/\\/g, '/');
|
|
47
|
+
const title = extractTitle(file.content, file.path);
|
|
48
|
+
const transformed = normalizeFrontmatter(file.content, { title });
|
|
49
|
+
const outputPath = `content/docs/${slug}.mdx`;
|
|
50
|
+
pages.push({ title, slug, sourcePath: rel, content: transformed });
|
|
51
|
+
result.files.set(outputPath, transformed);
|
|
52
|
+
// Track images
|
|
53
|
+
const imgMatches = transformed.match(/!\[.*?\]\(([^)]+)\)/g);
|
|
54
|
+
if (imgMatches) {
|
|
55
|
+
for (const imgMatch of imgMatches) {
|
|
56
|
+
const src = imgMatch.match(/!\[.*?\]\(([^)]+)\)/)?.[1];
|
|
57
|
+
if (src && !src.startsWith('http')) {
|
|
58
|
+
const srcPath = join(dir, dirname(rel), src);
|
|
59
|
+
if (existsSync(srcPath)) {
|
|
60
|
+
const destPath = `public/images/${basename(src)}`;
|
|
61
|
+
result.assets.set(destPath, srcPath);
|
|
62
|
+
stats.images++;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
result.navigation.push({ group: titleCase(groupName), pages });
|
|
69
|
+
}
|
|
70
|
+
result.stats = {
|
|
71
|
+
pages: result.files.size,
|
|
72
|
+
groups: result.navigation.length,
|
|
73
|
+
transforms: stats,
|
|
74
|
+
};
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
function extractTitle(content, filePath) {
|
|
78
|
+
// Try frontmatter title
|
|
79
|
+
const fmMatch = content.match(/^---\n[\s\S]*?title:\s*["']?([^"'\n]+)["']?\s*\n[\s\S]*?---/m);
|
|
80
|
+
if (fmMatch)
|
|
81
|
+
return fmMatch[1].trim();
|
|
82
|
+
// Try first H1
|
|
83
|
+
const h1Match = content.match(/^#\s+(.+)$/m);
|
|
84
|
+
if (h1Match)
|
|
85
|
+
return h1Match[1].trim();
|
|
86
|
+
// Fallback to filename
|
|
87
|
+
return titleCase(basename(filePath, extname(filePath)).replace(/[-_]/g, ' '));
|
|
88
|
+
}
|
|
89
|
+
function titleCase(str) {
|
|
90
|
+
return str
|
|
91
|
+
.replace(/[-_]/g, ' ')
|
|
92
|
+
.replace(/\b\w/g, c => c.toUpperCase());
|
|
93
|
+
}
|
|
94
|
+
function createEmptyResult(format, name) {
|
|
95
|
+
return {
|
|
96
|
+
navigation: [],
|
|
97
|
+
name: name || 'Documentation',
|
|
98
|
+
description: 'Imported documentation',
|
|
99
|
+
files: new Map(),
|
|
100
|
+
assets: new Map(),
|
|
101
|
+
warnings: [],
|
|
102
|
+
stats: { pages: 0, groups: 0, transforms: { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 } },
|
|
103
|
+
sourceFormat: format,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { readFileSync, existsSync } from 'fs';
|
|
2
|
+
import { join, relative, basename, extname } from 'path';
|
|
3
|
+
import { findMdxFiles } from '../utils/files.js';
|
|
4
|
+
import { transformMintlifyCallouts, transformMintlifyTabs, normalizeFrontmatter, rewriteImagePaths } from './transform.js';
|
|
5
|
+
/**
|
|
6
|
+
* Import Mintlify documentation.
|
|
7
|
+
*/
|
|
8
|
+
export function importMintlify(dir, name) {
|
|
9
|
+
const result = createEmptyResult(name);
|
|
10
|
+
const stats = { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 };
|
|
11
|
+
// Load config
|
|
12
|
+
const config = loadMintlifyConfig(dir);
|
|
13
|
+
if (!config) {
|
|
14
|
+
result.warnings.push('Could not parse mint.json or docs.json');
|
|
15
|
+
return result;
|
|
16
|
+
}
|
|
17
|
+
result.name = config.name || name || 'Documentation';
|
|
18
|
+
result.description = config.description || 'Imported from Mintlify';
|
|
19
|
+
// Process navigation
|
|
20
|
+
if (config.navigation) {
|
|
21
|
+
for (const navGroup of config.navigation) {
|
|
22
|
+
const pages = [];
|
|
23
|
+
for (const pageRef of navGroup.pages) {
|
|
24
|
+
if (typeof pageRef === 'string') {
|
|
25
|
+
const page = processPage(dir, pageRef, stats, result);
|
|
26
|
+
if (page)
|
|
27
|
+
pages.push(page);
|
|
28
|
+
}
|
|
29
|
+
else if (typeof pageRef === 'object' && pageRef.pages) {
|
|
30
|
+
// Nested group — flatten into pages with prefix
|
|
31
|
+
for (const nestedPage of pageRef.pages) {
|
|
32
|
+
if (typeof nestedPage === 'string') {
|
|
33
|
+
const page = processPage(dir, nestedPage, stats, result);
|
|
34
|
+
if (page)
|
|
35
|
+
pages.push(page);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (pages.length > 0) {
|
|
41
|
+
result.navigation.push({ group: navGroup.group, pages });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
// No navigation config — walk files
|
|
47
|
+
const files = findMdxFiles(dir);
|
|
48
|
+
const pages = [];
|
|
49
|
+
for (const filePath of files) {
|
|
50
|
+
const rel = relative(dir, filePath).replace(/\\/g, '/').replace(/\.(md|mdx)$/, '');
|
|
51
|
+
const page = processPage(dir, rel, stats, result);
|
|
52
|
+
if (page)
|
|
53
|
+
pages.push(page);
|
|
54
|
+
}
|
|
55
|
+
if (pages.length > 0) {
|
|
56
|
+
result.navigation.push({ group: 'Documentation', pages });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// Copy images
|
|
60
|
+
collectImages(dir, result, stats);
|
|
61
|
+
result.stats = {
|
|
62
|
+
pages: result.files.size,
|
|
63
|
+
groups: result.navigation.length,
|
|
64
|
+
transforms: stats,
|
|
65
|
+
};
|
|
66
|
+
return result;
|
|
67
|
+
}
|
|
68
|
+
function processPage(dir, pageRef, stats, result) {
|
|
69
|
+
// Find file (try .mdx first, then .md)
|
|
70
|
+
let filePath = join(dir, `${pageRef}.mdx`);
|
|
71
|
+
if (!existsSync(filePath)) {
|
|
72
|
+
filePath = join(dir, `${pageRef}.md`);
|
|
73
|
+
}
|
|
74
|
+
if (!existsSync(filePath)) {
|
|
75
|
+
result.warnings.push(`Page not found: ${pageRef}`);
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
let content = readFileSync(filePath, 'utf-8');
|
|
79
|
+
// Handle <Snippet file="x.mdx" /> — inline referenced files
|
|
80
|
+
content = content.replace(/<Snippet\s+file="([^"]+)"\s*\/>/g, (_match, snippetPath) => {
|
|
81
|
+
const snippetFile = join(dir, snippetPath);
|
|
82
|
+
// Guard: prevent path traversal outside source directory
|
|
83
|
+
if (!snippetFile.startsWith(dir)) {
|
|
84
|
+
result.warnings.push(`Snippet path traversal blocked: ${snippetPath}`);
|
|
85
|
+
return `<!-- Snippet blocked: ${snippetPath} -->`;
|
|
86
|
+
}
|
|
87
|
+
if (existsSync(snippetFile)) {
|
|
88
|
+
return readFileSync(snippetFile, 'utf-8');
|
|
89
|
+
}
|
|
90
|
+
result.warnings.push(`Snippet not found: ${snippetPath}`);
|
|
91
|
+
return `<!-- Snippet not found: ${snippetPath} -->`;
|
|
92
|
+
});
|
|
93
|
+
// Apply transforms
|
|
94
|
+
const originalContent = content;
|
|
95
|
+
content = transformMintlifyCallouts(content);
|
|
96
|
+
content = transformMintlifyTabs(content);
|
|
97
|
+
content = normalizeFrontmatter(content);
|
|
98
|
+
// Count transforms
|
|
99
|
+
stats.callouts += countDiff(originalContent, content, '<Callout');
|
|
100
|
+
stats.tabs += countDiff(originalContent, content, '<TabPanel');
|
|
101
|
+
const title = extractTitle(content, filePath);
|
|
102
|
+
const slug = pageRef.replace(/\\/g, '/');
|
|
103
|
+
const outputPath = `content/docs/${slug}.mdx`;
|
|
104
|
+
result.files.set(outputPath, content);
|
|
105
|
+
return { title, slug, sourcePath: relative(dir, filePath), content };
|
|
106
|
+
}
|
|
107
|
+
function collectImages(dir, result, stats) {
|
|
108
|
+
// Scan all processed files for image references
|
|
109
|
+
const imageMapping = new Map();
|
|
110
|
+
for (const [, content] of result.files) {
|
|
111
|
+
const imgMatches = content.matchAll(/!\[.*?\]\(([^)]+)\)/g);
|
|
112
|
+
for (const match of imgMatches) {
|
|
113
|
+
const src = match[1];
|
|
114
|
+
if (src && !src.startsWith('http')) {
|
|
115
|
+
const srcPath = join(dir, src);
|
|
116
|
+
if (existsSync(srcPath)) {
|
|
117
|
+
const destPath = `/images/${basename(src)}`;
|
|
118
|
+
result.assets.set(`public${destPath}`, srcPath);
|
|
119
|
+
imageMapping.set(src, destPath);
|
|
120
|
+
stats.images++;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// Rewrite image paths in all files
|
|
126
|
+
if (imageMapping.size > 0) {
|
|
127
|
+
for (const [path, content] of result.files) {
|
|
128
|
+
result.files.set(path, rewriteImagePaths(content, imageMapping));
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
function loadMintlifyConfig(dir) {
|
|
133
|
+
for (const configName of ['mint.json', 'docs.json']) {
|
|
134
|
+
const configPath = join(dir, configName);
|
|
135
|
+
if (existsSync(configPath)) {
|
|
136
|
+
try {
|
|
137
|
+
return JSON.parse(readFileSync(configPath, 'utf-8'));
|
|
138
|
+
}
|
|
139
|
+
catch { /* try next */ }
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
function extractTitle(content, filePath) {
|
|
145
|
+
const fmMatch = content.match(/^---\n[\s\S]*?title:\s*["']?([^"'\n]+)["']?\s*\n[\s\S]*?---/m);
|
|
146
|
+
if (fmMatch)
|
|
147
|
+
return fmMatch[1].trim();
|
|
148
|
+
const h1Match = content.match(/^#\s+(.+)$/m);
|
|
149
|
+
if (h1Match)
|
|
150
|
+
return h1Match[1].trim();
|
|
151
|
+
return basename(filePath, extname(filePath)).replace(/[-_]/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
|
152
|
+
}
|
|
153
|
+
function countDiff(original, transformed, marker) {
|
|
154
|
+
const origCount = (original.match(new RegExp(escapeRegex(marker), 'g')) || []).length;
|
|
155
|
+
const newCount = (transformed.match(new RegExp(escapeRegex(marker), 'g')) || []).length;
|
|
156
|
+
return Math.max(0, newCount - origCount);
|
|
157
|
+
}
|
|
158
|
+
function escapeRegex(str) {
|
|
159
|
+
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
160
|
+
}
|
|
161
|
+
function createEmptyResult(name) {
|
|
162
|
+
return {
|
|
163
|
+
navigation: [],
|
|
164
|
+
name: name || 'Documentation',
|
|
165
|
+
description: 'Imported from Mintlify',
|
|
166
|
+
files: new Map(),
|
|
167
|
+
assets: new Map(),
|
|
168
|
+
warnings: [],
|
|
169
|
+
stats: { pages: 0, groups: 0, transforms: { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 } },
|
|
170
|
+
sourceFormat: 'mintlify',
|
|
171
|
+
};
|
|
172
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { readFileSync, readdirSync, statSync } from 'fs';
|
|
2
|
+
import { join, relative, basename, extname } from 'path';
|
|
3
|
+
import { transformNotionCallouts, transformNotionToggles, normalizeFrontmatter, stripNotionUUIDs, } from './transform.js';
|
|
4
|
+
/**
|
|
5
|
+
* Import Notion export (extracted folder with UUID-named files).
|
|
6
|
+
*/
|
|
7
|
+
export function importNotion(dir, name) {
|
|
8
|
+
const result = createEmptyResult(name);
|
|
9
|
+
const stats = { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 };
|
|
10
|
+
// Walk the export directory
|
|
11
|
+
const groups = discoverGroups(dir);
|
|
12
|
+
for (const group of groups) {
|
|
13
|
+
const pages = [];
|
|
14
|
+
for (const file of group.files) {
|
|
15
|
+
const page = processNotionPage(dir, file, stats, result);
|
|
16
|
+
if (page)
|
|
17
|
+
pages.push(page);
|
|
18
|
+
}
|
|
19
|
+
if (pages.length > 0) {
|
|
20
|
+
result.navigation.push({ group: group.label, pages });
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
// Copy images (strip UUIDs from filenames)
|
|
24
|
+
collectNotionImages(dir, result, stats);
|
|
25
|
+
result.stats = {
|
|
26
|
+
pages: result.files.size,
|
|
27
|
+
groups: result.navigation.length,
|
|
28
|
+
transforms: stats,
|
|
29
|
+
};
|
|
30
|
+
return result;
|
|
31
|
+
}
|
|
32
|
+
function discoverGroups(dir) {
|
|
33
|
+
const groups = [];
|
|
34
|
+
// Root-level markdown files
|
|
35
|
+
const rootFiles = [];
|
|
36
|
+
const entries = readdirSync(dir);
|
|
37
|
+
for (const entry of entries) {
|
|
38
|
+
const fullPath = join(dir, entry);
|
|
39
|
+
try {
|
|
40
|
+
if (statSync(fullPath).isFile() && /\.(md|mdx)$/.test(entry)) {
|
|
41
|
+
rootFiles.push(fullPath);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (rootFiles.length > 0) {
|
|
49
|
+
groups.push({ label: 'Documentation', files: rootFiles });
|
|
50
|
+
}
|
|
51
|
+
// Subdirectories (Notion exports create folders for sub-pages)
|
|
52
|
+
for (const entry of entries) {
|
|
53
|
+
const fullPath = join(dir, entry);
|
|
54
|
+
try {
|
|
55
|
+
if (!statSync(fullPath).isDirectory() || entry.startsWith('.'))
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const mdFiles = findMdFilesRecursive(fullPath);
|
|
62
|
+
if (mdFiles.length === 0)
|
|
63
|
+
continue;
|
|
64
|
+
// Clean UUID from folder name
|
|
65
|
+
const cleanName = stripNotionUUIDs(entry)
|
|
66
|
+
.replace(/[-_]/g, ' ')
|
|
67
|
+
.replace(/\b\w/g, c => c.toUpperCase())
|
|
68
|
+
.trim();
|
|
69
|
+
groups.push({ label: cleanName || entry, files: mdFiles });
|
|
70
|
+
}
|
|
71
|
+
return groups;
|
|
72
|
+
}
|
|
73
|
+
function findMdFilesRecursive(dir) {
|
|
74
|
+
const files = [];
|
|
75
|
+
try {
|
|
76
|
+
for (const entry of readdirSync(dir)) {
|
|
77
|
+
const fullPath = join(dir, entry);
|
|
78
|
+
try {
|
|
79
|
+
const stat = statSync(fullPath);
|
|
80
|
+
if (stat.isDirectory() && !entry.startsWith('.')) {
|
|
81
|
+
files.push(...findMdFilesRecursive(fullPath));
|
|
82
|
+
}
|
|
83
|
+
else if (stat.isFile() && /\.(md|mdx)$/.test(entry)) {
|
|
84
|
+
files.push(fullPath);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
catch { /* skip */ }
|
|
93
|
+
return files;
|
|
94
|
+
}
|
|
95
|
+
function processNotionPage(dir, filePath, stats, result) {
|
|
96
|
+
let content = readFileSync(filePath, 'utf-8');
|
|
97
|
+
const originalContent = content;
|
|
98
|
+
// Apply Notion transforms
|
|
99
|
+
content = transformNotionCallouts(content);
|
|
100
|
+
content = transformNotionToggles(content);
|
|
101
|
+
content = normalizeFrontmatter(content);
|
|
102
|
+
stats.callouts += countNew(originalContent, content, '<Callout');
|
|
103
|
+
stats.accordions += countNew(originalContent, content, '<Accordion');
|
|
104
|
+
const rel = relative(dir, filePath);
|
|
105
|
+
const cleanFilename = stripNotionUUIDs(rel.replace(/\.(md|mdx)$/, '')).replace(/\\/g, '/');
|
|
106
|
+
const slug = cleanFilename.toLowerCase().replace(/[^a-z0-9/]+/g, '-').replace(/^-|-$/g, '');
|
|
107
|
+
const title = extractTitle(content, filePath);
|
|
108
|
+
const outputPath = `content/docs/${slug}.mdx`;
|
|
109
|
+
result.files.set(outputPath, content);
|
|
110
|
+
return { title, slug, sourcePath: rel, content };
|
|
111
|
+
}
|
|
112
|
+
function collectNotionImages(dir, result, stats) {
|
|
113
|
+
const imgExts = new Set(['.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp']);
|
|
114
|
+
function walk(currentDir) {
|
|
115
|
+
try {
|
|
116
|
+
for (const entry of readdirSync(currentDir)) {
|
|
117
|
+
const fullPath = join(currentDir, entry);
|
|
118
|
+
try {
|
|
119
|
+
const stat = statSync(fullPath);
|
|
120
|
+
if (stat.isDirectory()) {
|
|
121
|
+
walk(fullPath);
|
|
122
|
+
}
|
|
123
|
+
else if (imgExts.has(extname(entry).toLowerCase())) {
|
|
124
|
+
const cleanName = stripNotionUUIDs(basename(entry, extname(entry))) + extname(entry);
|
|
125
|
+
const dest = `public/images/${cleanName}`;
|
|
126
|
+
result.assets.set(dest, fullPath);
|
|
127
|
+
// Rewrite paths in content
|
|
128
|
+
const origRef = relative(dir, fullPath);
|
|
129
|
+
for (const [path, content] of result.files) {
|
|
130
|
+
if (content.includes(origRef) || content.includes(entry)) {
|
|
131
|
+
result.files.set(path, content.replaceAll(origRef, `/images/${cleanName}`).replaceAll(entry, `/images/${cleanName}`));
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
stats.images++;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
catch { /* skip */ }
|
|
143
|
+
}
|
|
144
|
+
walk(dir);
|
|
145
|
+
}
|
|
146
|
+
function extractTitle(content, filePath) {
|
|
147
|
+
const fmMatch = content.match(/^---\n[\s\S]*?title:\s*["']?([^"'\n]+)["']?\s*\n[\s\S]*?---/m);
|
|
148
|
+
if (fmMatch)
|
|
149
|
+
return fmMatch[1].trim();
|
|
150
|
+
const h1Match = content.match(/^#\s+(.+)$/m);
|
|
151
|
+
if (h1Match)
|
|
152
|
+
return h1Match[1].trim();
|
|
153
|
+
// Strip UUID from filename for title
|
|
154
|
+
const cleanName = stripNotionUUIDs(basename(filePath, extname(filePath)));
|
|
155
|
+
return cleanName.replace(/[-_]/g, ' ').replace(/\b\w/g, c => c.toUpperCase()).trim();
|
|
156
|
+
}
|
|
157
|
+
function countNew(original, transformed, marker) {
|
|
158
|
+
const escaped = marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
159
|
+
const origCount = (original.match(new RegExp(escaped, 'g')) || []).length;
|
|
160
|
+
const newCount = (transformed.match(new RegExp(escaped, 'g')) || []).length;
|
|
161
|
+
return Math.max(0, newCount - origCount);
|
|
162
|
+
}
|
|
163
|
+
function createEmptyResult(name) {
|
|
164
|
+
return {
|
|
165
|
+
navigation: [],
|
|
166
|
+
name: name || 'Documentation',
|
|
167
|
+
description: 'Imported from Notion',
|
|
168
|
+
files: new Map(),
|
|
169
|
+
assets: new Map(),
|
|
170
|
+
warnings: [],
|
|
171
|
+
stats: { pages: 0, groups: 0, transforms: { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 } },
|
|
172
|
+
sourceFormat: 'notion',
|
|
173
|
+
};
|
|
174
|
+
}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { readFileSync, existsSync, readdirSync, statSync } from 'fs';
|
|
2
|
+
import { join, relative, basename, extname } from 'path';
|
|
3
|
+
import { findMdxFiles } from '../utils/files.js';
|
|
4
|
+
import { transformReadmeCallouts, transformReadmeCodeBlocks, normalizeFrontmatter, } from './transform.js';
|
|
5
|
+
/**
|
|
6
|
+
* Import ReadMe.io documentation export.
|
|
7
|
+
*/
|
|
8
|
+
export function importReadme(dir, name) {
|
|
9
|
+
const result = createEmptyResult(name);
|
|
10
|
+
const stats = { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 };
|
|
11
|
+
// ReadMe organizes by category folders with _order.yaml
|
|
12
|
+
const categories = discoverCategories(dir);
|
|
13
|
+
for (const category of categories) {
|
|
14
|
+
const pages = [];
|
|
15
|
+
for (const file of category.files) {
|
|
16
|
+
const page = processReadmePage(dir, file, stats, result);
|
|
17
|
+
if (page)
|
|
18
|
+
pages.push(page);
|
|
19
|
+
}
|
|
20
|
+
if (pages.length > 0) {
|
|
21
|
+
result.navigation.push({ group: category.label, pages });
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
// Handle CDN images (files.readme.io)
|
|
25
|
+
flagCdnImages(result, stats);
|
|
26
|
+
result.stats = {
|
|
27
|
+
pages: result.files.size,
|
|
28
|
+
groups: result.navigation.length,
|
|
29
|
+
transforms: stats,
|
|
30
|
+
};
|
|
31
|
+
return result;
|
|
32
|
+
}
|
|
33
|
+
function discoverCategories(dir) {
|
|
34
|
+
const categories = [];
|
|
35
|
+
// Check for category subdirectories
|
|
36
|
+
try {
|
|
37
|
+
const entries = readdirSync(dir);
|
|
38
|
+
const hasSubdirs = entries.some(e => {
|
|
39
|
+
try {
|
|
40
|
+
return statSync(join(dir, e)).isDirectory() && !e.startsWith('.');
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
if (hasSubdirs) {
|
|
47
|
+
for (const entry of entries) {
|
|
48
|
+
const fullPath = join(dir, entry);
|
|
49
|
+
try {
|
|
50
|
+
if (!statSync(fullPath).isDirectory() || entry.startsWith('.') || entry === 'node_modules')
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
const files = findMdxFiles(fullPath);
|
|
57
|
+
if (files.length === 0)
|
|
58
|
+
continue;
|
|
59
|
+
// Check _order.yaml
|
|
60
|
+
const orderPath = join(fullPath, '_order.yaml');
|
|
61
|
+
let orderedFiles = files;
|
|
62
|
+
const position = Infinity;
|
|
63
|
+
if (existsSync(orderPath)) {
|
|
64
|
+
try {
|
|
65
|
+
const orderContent = readFileSync(orderPath, 'utf-8');
|
|
66
|
+
const order = orderContent.split('\n').map(l => l.replace(/^-\s*/, '').trim()).filter(Boolean);
|
|
67
|
+
orderedFiles = orderFiles(files, order, fullPath);
|
|
68
|
+
}
|
|
69
|
+
catch { /* skip */ }
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
orderedFiles = sortByFrontmatterOrder(files);
|
|
73
|
+
}
|
|
74
|
+
const label = entry.replace(/[-_]/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
|
75
|
+
categories.push({ label, files: orderedFiles, position });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
// Flat structure — all files in root
|
|
80
|
+
const files = findMdxFiles(dir);
|
|
81
|
+
if (files.length > 0) {
|
|
82
|
+
categories.push({ label: 'Documentation', files: sortByFrontmatterOrder(files), position: 0 });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
catch { /* skip */ }
|
|
87
|
+
return categories.sort((a, b) => a.position - b.position);
|
|
88
|
+
}
|
|
89
|
+
function orderFiles(files, order, _dir) {
|
|
90
|
+
const ordered = [];
|
|
91
|
+
for (const slug of order) {
|
|
92
|
+
const match = files.find(f => {
|
|
93
|
+
const name = basename(f, extname(f));
|
|
94
|
+
return name === slug || name.toLowerCase() === slug.toLowerCase();
|
|
95
|
+
});
|
|
96
|
+
if (match)
|
|
97
|
+
ordered.push(match);
|
|
98
|
+
}
|
|
99
|
+
// Add remaining files not in order
|
|
100
|
+
for (const f of files) {
|
|
101
|
+
if (!ordered.includes(f))
|
|
102
|
+
ordered.push(f);
|
|
103
|
+
}
|
|
104
|
+
return ordered;
|
|
105
|
+
}
|
|
106
|
+
function sortByFrontmatterOrder(files) {
|
|
107
|
+
return [...files].sort((a, b) => {
|
|
108
|
+
try {
|
|
109
|
+
const aContent = readFileSync(a, 'utf-8');
|
|
110
|
+
const bContent = readFileSync(b, 'utf-8');
|
|
111
|
+
const aOrder = extractFmField(aContent, 'order');
|
|
112
|
+
const bOrder = extractFmField(bContent, 'order');
|
|
113
|
+
const aNum = aOrder !== null ? Number(aOrder) : Infinity;
|
|
114
|
+
const bNum = bOrder !== null ? Number(bOrder) : Infinity;
|
|
115
|
+
if (aNum !== bNum)
|
|
116
|
+
return aNum - bNum;
|
|
117
|
+
}
|
|
118
|
+
catch { /* skip */ }
|
|
119
|
+
return basename(a).localeCompare(basename(b));
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
function extractFmField(content, field) {
|
|
123
|
+
const match = content.match(new RegExp(`^---\\n[\\s\\S]*?${field}:\\s*([^\\n]+)\\n[\\s\\S]*?---`, 'm'));
|
|
124
|
+
return match ? match[1].trim() : null;
|
|
125
|
+
}
|
|
126
|
+
function processReadmePage(dir, filePath, stats, result) {
|
|
127
|
+
let content = readFileSync(filePath, 'utf-8');
|
|
128
|
+
const originalContent = content;
|
|
129
|
+
// Apply ReadMe transforms
|
|
130
|
+
content = transformReadmeCallouts(content);
|
|
131
|
+
content = transformReadmeCodeBlocks(content);
|
|
132
|
+
// Map ReadMe-specific frontmatter
|
|
133
|
+
content = content.replace(/^---\n([\s\S]*?)\n---/, (match, fmStr) => {
|
|
134
|
+
const mapped = fmStr.replace(/^excerpt:\s*/m, 'description: ');
|
|
135
|
+
return `---\n${mapped}\n---`;
|
|
136
|
+
});
|
|
137
|
+
content = normalizeFrontmatter(content);
|
|
138
|
+
stats.callouts += countNew(originalContent, content, '<Callout');
|
|
139
|
+
stats.codeGroups += countNew(originalContent, content, '<CodeGroup');
|
|
140
|
+
const title = extractTitle(content, filePath);
|
|
141
|
+
const rel = relative(dir, filePath);
|
|
142
|
+
const slug = rel.replace(/\.(md|mdx)$/, '').replace(/\\/g, '/');
|
|
143
|
+
const outputPath = `content/docs/${slug}.mdx`;
|
|
144
|
+
result.files.set(outputPath, content);
|
|
145
|
+
return { title, slug, sourcePath: rel, content };
|
|
146
|
+
}
|
|
147
|
+
function flagCdnImages(result, stats) {
|
|
148
|
+
for (const [, content] of result.files) {
|
|
149
|
+
const cdnMatches = content.match(/https?:\/\/files\.readme\.io\/[^\s)]+/g);
|
|
150
|
+
if (cdnMatches) {
|
|
151
|
+
for (const url of cdnMatches) {
|
|
152
|
+
result.warnings.push(`CDN image found (manual download needed): ${url}`);
|
|
153
|
+
stats.images++;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
function extractTitle(content, filePath) {
|
|
159
|
+
const fmMatch = content.match(/^---\n[\s\S]*?title:\s*["']?([^"'\n]+)["']?\s*\n[\s\S]*?---/m);
|
|
160
|
+
if (fmMatch)
|
|
161
|
+
return fmMatch[1].trim();
|
|
162
|
+
const h1Match = content.match(/^#\s+(.+)$/m);
|
|
163
|
+
if (h1Match)
|
|
164
|
+
return h1Match[1].trim();
|
|
165
|
+
return basename(filePath, extname(filePath)).replace(/[-_]/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
|
166
|
+
}
|
|
167
|
+
function countNew(original, transformed, marker) {
|
|
168
|
+
const escaped = marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
169
|
+
const origCount = (original.match(new RegExp(escaped, 'g')) || []).length;
|
|
170
|
+
const newCount = (transformed.match(new RegExp(escaped, 'g')) || []).length;
|
|
171
|
+
return Math.max(0, newCount - origCount);
|
|
172
|
+
}
|
|
173
|
+
function createEmptyResult(name) {
|
|
174
|
+
return {
|
|
175
|
+
navigation: [],
|
|
176
|
+
name: name || 'Documentation',
|
|
177
|
+
description: 'Imported from ReadMe',
|
|
178
|
+
files: new Map(),
|
|
179
|
+
assets: new Map(),
|
|
180
|
+
warnings: [],
|
|
181
|
+
stats: { pages: 0, groups: 0, transforms: { callouts: 0, tabs: 0, codeGroups: 0, steps: 0, accordions: 0, images: 0, other: 0 } },
|
|
182
|
+
sourceFormat: 'readme',
|
|
183
|
+
};
|
|
184
|
+
}
|