skrypt-ai 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/index.js +6 -0
- package/dist/binding/binder.d.ts +5 -0
- package/dist/binding/binder.js +63 -0
- package/dist/binding/detector.d.ts +5 -0
- package/dist/binding/detector.js +51 -0
- package/dist/binding/doc-parser.d.ts +9 -0
- package/dist/binding/doc-parser.js +138 -0
- package/dist/binding/extractor.d.ts +14 -0
- package/dist/binding/extractor.js +39 -0
- package/dist/binding/index.d.ts +5 -0
- package/dist/binding/index.js +5 -0
- package/dist/binding/types.d.ts +74 -0
- package/dist/binding/types.js +1 -0
- package/dist/claims/extractor.d.ts +13 -0
- package/dist/claims/extractor.js +138 -0
- package/dist/claims/index.d.ts +4 -0
- package/dist/claims/index.js +4 -0
- package/dist/claims/reporter.d.ts +9 -0
- package/dist/claims/reporter.js +65 -0
- package/dist/claims/store.d.ts +13 -0
- package/dist/claims/store.js +51 -0
- package/dist/claims/types.d.ts +34 -0
- package/dist/claims/types.js +1 -0
- package/dist/cli.js +516 -56
- package/dist/commands/bind.d.ts +2 -0
- package/dist/commands/bind.js +139 -0
- package/dist/commands/claims.d.ts +2 -0
- package/dist/commands/claims.js +84 -0
- package/dist/commands/coverage.d.ts +2 -0
- package/dist/commands/coverage.js +61 -0
- package/dist/commands/generate/index.js +5 -0
- package/dist/commands/generate/scan.js +33 -14
- package/dist/commands/generate/write.d.ts +7 -0
- package/dist/commands/generate/write.js +65 -1
- package/dist/commands/import.js +12 -3
- package/dist/commands/init.js +68 -5
- package/dist/commands/monitor.d.ts +15 -0
- package/dist/commands/monitor.js +2 -2
- package/dist/commands/mutate.d.ts +2 -0
- package/dist/commands/mutate.js +177 -0
- package/dist/config/types.js +2 -2
- package/dist/coverage/calculator.d.ts +7 -0
- package/dist/coverage/calculator.js +86 -0
- package/dist/coverage/index.d.ts +3 -0
- package/dist/coverage/index.js +3 -0
- package/dist/coverage/reporter.d.ts +9 -0
- package/dist/coverage/reporter.js +65 -0
- package/dist/coverage/types.d.ts +36 -0
- package/dist/coverage/types.js +1 -0
- package/dist/generator/generator.d.ts +3 -1
- package/dist/generator/generator.js +137 -23
- package/dist/generator/mdx-serializer.js +3 -2
- package/dist/generator/organizer.d.ts +5 -1
- package/dist/generator/organizer.js +29 -14
- package/dist/generator/types.d.ts +6 -0
- package/dist/generator/writer.js +7 -2
- package/dist/github/org-discovery.js +5 -0
- package/dist/importers/mintlify.js +4 -3
- package/dist/llm/anthropic-client.js +1 -0
- package/dist/llm/index.d.ts +15 -0
- package/dist/llm/index.js +148 -29
- package/dist/llm/openai-client.js +2 -0
- package/dist/mutation/index.d.ts +4 -0
- package/dist/mutation/index.js +4 -0
- package/dist/mutation/mutator.d.ts +5 -0
- package/dist/mutation/mutator.js +101 -0
- package/dist/mutation/reporter.d.ts +14 -0
- package/dist/mutation/reporter.js +66 -0
- package/dist/mutation/runner.d.ts +9 -0
- package/dist/mutation/runner.js +70 -0
- package/dist/mutation/types.d.ts +51 -0
- package/dist/mutation/types.js +1 -0
- package/dist/qa/checks.d.ts +1 -0
- package/dist/qa/checks.js +47 -0
- package/dist/qa/index.js +2 -1
- package/dist/scanner/index.js +78 -11
- package/dist/scanner/typescript.js +42 -31
- package/dist/sentry.d.ts +3 -0
- package/dist/sentry.js +28 -0
- package/dist/template/docs.json +6 -3
- package/dist/template/next.config.mjs +15 -1
- package/dist/template/package.json +4 -3
- package/dist/template/public/docs-schema.json +257 -0
- package/dist/template/sentry.client.config.ts +12 -0
- package/dist/template/sentry.edge.config.ts +7 -0
- package/dist/template/sentry.server.config.ts +7 -0
- package/dist/template/src/app/docs/[...slug]/page.tsx +11 -5
- package/dist/template/src/app/docs/layout.tsx +2 -4
- package/dist/template/src/app/global-error.tsx +60 -0
- package/dist/template/src/app/layout.tsx +7 -16
- package/dist/template/src/app/page.tsx +2 -5
- package/dist/template/src/components/ai-chat-impl.tsx +1 -1
- package/dist/template/src/components/docs-layout.tsx +1 -15
- package/dist/template/src/components/footer.tsx +95 -19
- package/dist/template/src/components/header.tsx +1 -1
- package/dist/template/src/components/search-dialog.tsx +5 -0
- package/dist/template/src/instrumentation.ts +11 -0
- package/dist/template/src/lib/docs-config.ts +235 -0
- package/dist/template/src/lib/fonts.ts +3 -3
- package/dist/testing/runner.js +8 -1
- package/package.json +2 -1
|
@@ -123,28 +123,43 @@ function sortDocsWithinTopic(docs) {
|
|
|
123
123
|
});
|
|
124
124
|
}
|
|
125
125
|
/**
|
|
126
|
-
* Detect cross-references between elements
|
|
126
|
+
* Detect cross-references between elements.
|
|
127
|
+
*
|
|
128
|
+
* Instead of checking every element name against every doc's source text (O(n^2) with
|
|
129
|
+
* string.includes), we build a single regex alternation of all element names and scan
|
|
130
|
+
* each doc's text once. This reduces the inner loop to a single regex pass per doc.
|
|
127
131
|
*/
|
|
128
132
|
export function detectCrossReferences(docs) {
|
|
129
133
|
const refs = [];
|
|
130
134
|
const elementNames = new Set(docs.map(d => d.element.name));
|
|
135
|
+
// Build a single regex matching any element name as a whole word.
|
|
136
|
+
// Escape regex special chars in names, join with alternation, wrap in word boundaries.
|
|
137
|
+
const namesArray = Array.from(elementNames);
|
|
138
|
+
if (namesArray.length === 0)
|
|
139
|
+
return refs;
|
|
140
|
+
const escaped = namesArray.map(n => n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
|
141
|
+
const combinedPattern = new RegExp(`\\b(?:${escaped.join('|')})\\b`, 'g');
|
|
131
142
|
for (const doc of docs) {
|
|
132
143
|
const { element } = doc;
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
refs.push({
|
|
142
|
-
fromElement: element.name,
|
|
143
|
-
toElement: otherName,
|
|
144
|
-
relationship: 'uses'
|
|
145
|
-
});
|
|
144
|
+
const text = (element.sourceContext || '') + '\n' + (element.signature || '');
|
|
145
|
+
// Single-pass regex scan to find all referenced element names
|
|
146
|
+
const referencedNames = new Set();
|
|
147
|
+
let match;
|
|
148
|
+
while ((match = combinedPattern.exec(text)) !== null) {
|
|
149
|
+
const name = match[0];
|
|
150
|
+
if (name !== element.name && elementNames.has(name)) {
|
|
151
|
+
referencedNames.add(name);
|
|
146
152
|
}
|
|
147
153
|
}
|
|
154
|
+
// Reset lastIndex for the next doc since the regex is global
|
|
155
|
+
combinedPattern.lastIndex = 0;
|
|
156
|
+
for (const otherName of referencedNames) {
|
|
157
|
+
refs.push({
|
|
158
|
+
fromElement: element.name,
|
|
159
|
+
toElement: otherName,
|
|
160
|
+
relationship: 'uses'
|
|
161
|
+
});
|
|
162
|
+
}
|
|
148
163
|
// Methods reference their parent class
|
|
149
164
|
if (element.parentClass && elementNames.has(element.parentClass)) {
|
|
150
165
|
refs.push({
|
|
@@ -36,6 +36,12 @@ export interface GenerationOptions {
|
|
|
36
36
|
verify?: boolean;
|
|
37
37
|
/** Error context from previous verification failures, keyed by element name */
|
|
38
38
|
previousErrors?: Map<string, string>;
|
|
39
|
+
/** Max concurrent LLM calls (default: 5) */
|
|
40
|
+
concurrency?: number;
|
|
41
|
+
/** Skip generation cache and always call the LLM */
|
|
42
|
+
noCache?: boolean;
|
|
43
|
+
/** Directory to store cache files (default: project cwd) */
|
|
44
|
+
cacheDir?: string;
|
|
39
45
|
}
|
|
40
46
|
/**
|
|
41
47
|
* Result of generating docs for a file
|
package/dist/generator/writer.js
CHANGED
|
@@ -5,6 +5,7 @@ import { formatAsMarkdown } from './generator.js';
|
|
|
5
5
|
import { organizeByTopic, detectCrossReferences, getCrossRefsForElement } from './organizer.js';
|
|
6
6
|
import { slugify } from '../utils/files.js';
|
|
7
7
|
import { serializeMdxToMarkdown } from './mdx-serializer.js';
|
|
8
|
+
import { stripResponseMarkers } from '../llm/index.js';
|
|
8
9
|
/**
|
|
9
10
|
* Generate llms.txt file (Answer Engine Optimization)
|
|
10
11
|
* Format follows https://llmstxt.org convention
|
|
@@ -124,7 +125,9 @@ export async function writeDocsToDirectory(results, outputDir, sourceDir, option
|
|
|
124
125
|
const title = basename(result.filePath).replace(/\.[^.]+$/, '')
|
|
125
126
|
.replace(/^./, c => c.toUpperCase())
|
|
126
127
|
.replace(/_/g, ' ');
|
|
127
|
-
|
|
128
|
+
let content = formatAsMarkdown(result.docs, title);
|
|
129
|
+
// Defense-in-depth: strip any leaked LLM response markers before writing
|
|
130
|
+
content = stripResponseMarkers(content);
|
|
128
131
|
// Write file
|
|
129
132
|
await writeFile(outputPath, content, 'utf-8');
|
|
130
133
|
filesWritten++;
|
|
@@ -212,7 +215,9 @@ export async function writeDocsByTopic(docs, outputDir) {
|
|
|
212
215
|
// Write each topic as a separate file
|
|
213
216
|
for (const topic of topics) {
|
|
214
217
|
const topicPath = join(outputDir, `${topic.id}.md`);
|
|
215
|
-
|
|
218
|
+
let content = formatTopicMarkdown(topic, crossRefs);
|
|
219
|
+
// Defense-in-depth: strip any leaked LLM response markers before writing
|
|
220
|
+
content = stripResponseMarkers(content);
|
|
216
221
|
await writeFile(topicPath, content, 'utf-8');
|
|
217
222
|
filesWritten++;
|
|
218
223
|
}
|
|
@@ -89,5 +89,10 @@ export async function cloneRepoToTemp(repo, token) {
|
|
|
89
89
|
const stderr = (result.stderr?.toString() || '').replace(/x-access-token:[^@]+@/g, 'x-access-token:***@');
|
|
90
90
|
throw new Error(`Failed to clone ${repo.full_name}: ${stderr}`);
|
|
91
91
|
}
|
|
92
|
+
// Scrub the token from .git/config so it doesn't persist on disk
|
|
93
|
+
try {
|
|
94
|
+
spawnSync('git', ['-C', tempDir, 'remote', 'set-url', 'origin', repo.clone_url], { stdio: 'pipe' });
|
|
95
|
+
}
|
|
96
|
+
catch { /* non-critical — temp dir will be deleted anyway */ }
|
|
92
97
|
return tempDir;
|
|
93
98
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readFileSync, existsSync } from 'fs';
|
|
2
|
-
import { join, relative, basename, extname } from 'path';
|
|
2
|
+
import { join, relative, basename, extname, resolve, sep } from 'path';
|
|
3
3
|
import { findMdxFiles } from '../utils/files.js';
|
|
4
4
|
import { transformMintlifyCallouts, transformMintlifyTabs, normalizeFrontmatter, rewriteImagePaths } from './transform.js';
|
|
5
5
|
/**
|
|
@@ -78,9 +78,10 @@ function processPage(dir, pageRef, stats, result) {
|
|
|
78
78
|
let content = readFileSync(filePath, 'utf-8');
|
|
79
79
|
// Handle <Snippet file="x.mdx" /> — inline referenced files
|
|
80
80
|
content = content.replace(/<Snippet\s+file="([^"]+)"\s*\/>/g, (_match, snippetPath) => {
|
|
81
|
-
const snippetFile = join(dir, snippetPath);
|
|
81
|
+
const snippetFile = resolve(join(dir, snippetPath));
|
|
82
|
+
const resolvedDir = resolve(dir) + sep;
|
|
82
83
|
// Guard: prevent path traversal outside source directory
|
|
83
|
-
if (!snippetFile.startsWith(dir)) {
|
|
84
|
+
if (!snippetFile.startsWith(resolvedDir) && snippetFile !== resolve(dir)) {
|
|
84
85
|
result.warnings.push(`Snippet path traversal blocked: ${snippetPath}`);
|
|
85
86
|
return `<!-- Snippet blocked: ${snippetPath} -->`;
|
|
86
87
|
}
|
package/dist/llm/index.d.ts
CHANGED
|
@@ -55,3 +55,18 @@ export declare function generateDocumentation(client: LLMClient, element: Elemen
|
|
|
55
55
|
verify?: boolean;
|
|
56
56
|
previousError?: string;
|
|
57
57
|
}): Promise<GeneratedDocResult>;
|
|
58
|
+
/**
|
|
59
|
+
* Normalize delimiter variations that LLMs (especially GPT-4o) produce.
|
|
60
|
+
* Models may return delimiters with extra dashes, spaces, backticks, bold markers, or different casing.
|
|
61
|
+
* Examples that should all normalize to ---MARKDOWN---:
|
|
62
|
+
* "--- MARKDOWN ---", "----MARKDOWN----", "**---MARKDOWN---**",
|
|
63
|
+
* "`---MARKDOWN---`", "---Markdown---", "--- markdown ---"
|
|
64
|
+
*/
|
|
65
|
+
export declare function normalizeDelimiters(content: string): string;
|
|
66
|
+
/**
|
|
67
|
+
* Strip all LLM response markers from content.
|
|
68
|
+
* This is a safety net — if parsing fails or partially succeeds, we must never
|
|
69
|
+
* write raw ---CODE---/---END---/---MARKDOWN--- markers to .md files because
|
|
70
|
+
* curly braces inside unfenced code blocks cause MDX compilation errors.
|
|
71
|
+
*/
|
|
72
|
+
export declare function stripResponseMarkers(content: string): string;
|
package/dist/llm/index.js
CHANGED
|
@@ -182,40 +182,159 @@ function buildDocPrompt(element, multiLanguage = false, verify = false) {
|
|
|
182
182
|
}
|
|
183
183
|
return prompt;
|
|
184
184
|
}
|
|
185
|
+
/**
|
|
186
|
+
* Normalize delimiter variations that LLMs (especially GPT-4o) produce.
|
|
187
|
+
* Models may return delimiters with extra dashes, spaces, backticks, bold markers, or different casing.
|
|
188
|
+
* Examples that should all normalize to ---MARKDOWN---:
|
|
189
|
+
* "--- MARKDOWN ---", "----MARKDOWN----", "**---MARKDOWN---**",
|
|
190
|
+
* "`---MARKDOWN---`", "---Markdown---", "--- markdown ---"
|
|
191
|
+
*/
|
|
192
|
+
export function normalizeDelimiters(content) {
|
|
193
|
+
// Match delimiter patterns: optional formatting chars, 2+ dashes, optional spaces,
|
|
194
|
+
// a known keyword, optional spaces, 2+ dashes, optional formatting chars
|
|
195
|
+
// Covers: ---MARKDOWN---, --- MARKDOWN ---, ----MARKDOWN----, **---MARKDOWN---**, `---MARKDOWN---`
|
|
196
|
+
const delimiterPattern = /^[`*]*-{2,}\s*(MARKDOWN|TYPESCRIPT|PYTHON|CODE|END)\s*-{2,}[`*]*$/gim;
|
|
197
|
+
return content.replace(delimiterPattern, (_match, keyword) => `---${keyword.toUpperCase()}---`);
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Strip all LLM response markers from content.
|
|
201
|
+
* This is a safety net — if parsing fails or partially succeeds, we must never
|
|
202
|
+
* write raw ---CODE---/---END---/---MARKDOWN--- markers to .md files because
|
|
203
|
+
* curly braces inside unfenced code blocks cause MDX compilation errors.
|
|
204
|
+
*/
|
|
205
|
+
export function stripResponseMarkers(content) {
|
|
206
|
+
// Strip both exact markers and common LLM variations (extra dashes, spaces, formatting)
|
|
207
|
+
return content.replace(/^[`*]*-{2,}\s*(?:MARKDOWN|TYPESCRIPT|PYTHON|CODE|END)\s*-{2,}[`*]*$/gim, '').trim();
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Strip markdown code fences from a code block.
|
|
211
|
+
* Handles ```language\n...\n```, including nested fences.
|
|
212
|
+
*/
|
|
213
|
+
function stripCodeFences(code) {
|
|
214
|
+
let result = code;
|
|
215
|
+
// Remove repeated opening fences (```typescript, ```python, etc.)
|
|
216
|
+
while (/^```[\w]*\n?/.test(result)) {
|
|
217
|
+
result = result.replace(/^```[\w]*\n?/, '');
|
|
218
|
+
}
|
|
219
|
+
// Remove repeated closing fences
|
|
220
|
+
while (/\n?```\s*$/.test(result)) {
|
|
221
|
+
result = result.replace(/\n?```\s*$/, '');
|
|
222
|
+
}
|
|
223
|
+
return result;
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Fallback parser: extract documentation and code from a response that has no delimiters.
|
|
227
|
+
* Looks for markdown prose followed by code fences.
|
|
228
|
+
*/
|
|
229
|
+
function parseFallbackResponse(content) {
|
|
230
|
+
// Try to split on the first code fence
|
|
231
|
+
const codeFenceMatch = content.match(/([\s\S]*?)```(\w*)\n([\s\S]*?)```([\s\S]*)/);
|
|
232
|
+
if (codeFenceMatch) {
|
|
233
|
+
const markdown = codeFenceMatch[1]?.trim() || content;
|
|
234
|
+
const firstLang = codeFenceMatch[2]?.toLowerCase() || '';
|
|
235
|
+
const firstCode = codeFenceMatch[3]?.trim() || '';
|
|
236
|
+
const rest = codeFenceMatch[4] || '';
|
|
237
|
+
// Check if there's a second code fence (multi-lang)
|
|
238
|
+
const secondFenceMatch = rest.match(/```(\w*)\n([\s\S]*?)```/);
|
|
239
|
+
const secondLang = secondFenceMatch?.[1]?.toLowerCase() || '';
|
|
240
|
+
const secondCode = secondFenceMatch?.[2]?.trim() || '';
|
|
241
|
+
// Assign code blocks by detected language
|
|
242
|
+
const isPythonFirst = firstLang === 'python' || firstLang === 'py';
|
|
243
|
+
const isPythonSecond = secondLang === 'python' || secondLang === 'py';
|
|
244
|
+
const typescriptExample = isPythonFirst ? secondCode : firstCode;
|
|
245
|
+
const pythonExample = isPythonFirst ? firstCode : (isPythonSecond ? secondCode : secondCode);
|
|
246
|
+
return {
|
|
247
|
+
markdown,
|
|
248
|
+
codeExample: typescriptExample || firstCode,
|
|
249
|
+
typescriptExample: typescriptExample || firstCode,
|
|
250
|
+
pythonExample
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
// No code fences found at all — return everything as markdown
|
|
254
|
+
return { markdown: content, codeExample: '', typescriptExample: '' };
|
|
255
|
+
}
|
|
185
256
|
function parseDocResponse(content, elementName) {
|
|
186
|
-
|
|
257
|
+
const normalized = normalizeDelimiters(content);
|
|
258
|
+
// Try to extract with delimiters first
|
|
259
|
+
const markdownMatch = normalized.match(/---MARKDOWN---\s*([\s\S]*?)\s*---CODE---/);
|
|
260
|
+
const codeMatch = normalized.match(/---CODE---\s*([\s\S]*?)\s*---END---/);
|
|
261
|
+
if (markdownMatch && codeMatch) {
|
|
262
|
+
const markdown = markdownMatch[1]?.trim() || '';
|
|
263
|
+
let codeExample = codeMatch[1]?.trim() || '';
|
|
264
|
+
codeExample = stripCodeFences(codeExample);
|
|
265
|
+
return { markdown, codeExample, typescriptExample: codeExample };
|
|
266
|
+
}
|
|
267
|
+
// Delimiter-based parsing failed — try fallback
|
|
268
|
+
const fallback = parseFallbackResponse(content);
|
|
269
|
+
if (!fallback.markdown && !fallback.codeExample) {
|
|
187
270
|
console.warn(` Warning: LLM response missing expected format for ${elementName ?? 'unknown'}`);
|
|
188
271
|
}
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
return { markdown, codeExample, typescriptExample: codeExample };
|
|
272
|
+
const result = fallback.markdown ? fallback : { markdown: content, codeExample: '', typescriptExample: '' };
|
|
273
|
+
// Safety net: strip any leaked markers from the final result
|
|
274
|
+
result.markdown = stripResponseMarkers(result.markdown);
|
|
275
|
+
if (result.codeExample)
|
|
276
|
+
result.codeExample = stripResponseMarkers(result.codeExample);
|
|
277
|
+
if (result.typescriptExample)
|
|
278
|
+
result.typescriptExample = stripResponseMarkers(result.typescriptExample);
|
|
279
|
+
return result;
|
|
198
280
|
}
|
|
199
281
|
function parseMultiLangResponse(content, elementName) {
|
|
200
|
-
|
|
282
|
+
const normalized = normalizeDelimiters(content);
|
|
283
|
+
// Try to extract with delimiters first
|
|
284
|
+
const markdownMatch = normalized.match(/---MARKDOWN---\s*([\s\S]*?)\s*---TYPESCRIPT---/);
|
|
285
|
+
const tsMatch = normalized.match(/---TYPESCRIPT---\s*([\s\S]*?)\s*---PYTHON---/);
|
|
286
|
+
const pyMatch = normalized.match(/---PYTHON---\s*([\s\S]*?)\s*---END---/);
|
|
287
|
+
if (markdownMatch && tsMatch && pyMatch) {
|
|
288
|
+
const markdown = markdownMatch[1]?.trim() || '';
|
|
289
|
+
const typescriptExample = stripCodeFences(tsMatch[1]?.trim() || '');
|
|
290
|
+
const pythonExample = stripCodeFences(pyMatch[1]?.trim() || '');
|
|
291
|
+
return {
|
|
292
|
+
markdown,
|
|
293
|
+
codeExample: typescriptExample,
|
|
294
|
+
typescriptExample,
|
|
295
|
+
pythonExample
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
// Partial delimiter match — try extracting what we can
|
|
299
|
+
if (markdownMatch || normalized.includes('---MARKDOWN---')) {
|
|
300
|
+
let markdown = markdownMatch?.[1]?.trim() || '';
|
|
301
|
+
let typescriptExample = tsMatch?.[1]?.trim() || '';
|
|
302
|
+
typescriptExample = stripCodeFences(typescriptExample);
|
|
303
|
+
let pythonExample = pyMatch?.[1]?.trim() || '';
|
|
304
|
+
pythonExample = stripCodeFences(pythonExample);
|
|
305
|
+
// If we have markdown but no code sections, try extracting code from remaining content
|
|
306
|
+
if (!typescriptExample && !pythonExample) {
|
|
307
|
+
const afterMarkdown = normalized.split(/---MARKDOWN---/)[1] || '';
|
|
308
|
+
const fallbackFromRest = parseFallbackResponse(afterMarkdown);
|
|
309
|
+
typescriptExample = fallbackFromRest.typescriptExample || '';
|
|
310
|
+
pythonExample = fallbackFromRest.pythonExample || '';
|
|
311
|
+
}
|
|
312
|
+
// Safety net: strip any leaked markers
|
|
313
|
+
markdown = stripResponseMarkers(markdown || content);
|
|
314
|
+
typescriptExample = stripResponseMarkers(typescriptExample);
|
|
315
|
+
pythonExample = stripResponseMarkers(pythonExample);
|
|
316
|
+
return {
|
|
317
|
+
markdown,
|
|
318
|
+
codeExample: typescriptExample,
|
|
319
|
+
typescriptExample,
|
|
320
|
+
pythonExample
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
// No delimiters at all — use fallback parser
|
|
324
|
+
const fallback = parseFallbackResponse(content);
|
|
325
|
+
if (!fallback.markdown && !fallback.codeExample) {
|
|
201
326
|
console.warn(` Warning: LLM response missing expected format for ${elementName ?? 'unknown'}`);
|
|
202
327
|
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
//
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
return {
|
|
216
|
-
markdown,
|
|
217
|
-
codeExample: typescriptExample,
|
|
218
|
-
typescriptExample,
|
|
219
|
-
pythonExample
|
|
220
|
-
};
|
|
328
|
+
const result = fallback.markdown
|
|
329
|
+
? fallback
|
|
330
|
+
: { markdown: content, codeExample: '', typescriptExample: '', pythonExample: '' };
|
|
331
|
+
// Safety net: strip any leaked markers from the final result
|
|
332
|
+
result.markdown = stripResponseMarkers(result.markdown);
|
|
333
|
+
if (result.codeExample)
|
|
334
|
+
result.codeExample = stripResponseMarkers(result.codeExample);
|
|
335
|
+
if (result.typescriptExample)
|
|
336
|
+
result.typescriptExample = stripResponseMarkers(result.typescriptExample);
|
|
337
|
+
if (result.pythonExample)
|
|
338
|
+
result.pythonExample = stripResponseMarkers(result.pythonExample);
|
|
339
|
+
return result;
|
|
221
340
|
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { readFileSync } from 'fs';
|
|
2
|
+
import { createHash } from 'crypto';
|
|
3
|
+
/**
|
|
4
|
+
* Generate mutations for a source file
|
|
5
|
+
*/
|
|
6
|
+
export function generateMutants(filePath, maxMutants = 100) {
|
|
7
|
+
const content = readFileSync(filePath, 'utf-8');
|
|
8
|
+
const lines = content.split('\n');
|
|
9
|
+
const mutants = [];
|
|
10
|
+
for (let i = 0; i < lines.length && mutants.length < maxMutants; i++) {
|
|
11
|
+
const line = lines[i];
|
|
12
|
+
const lineMutants = mutateLine(line, i, filePath);
|
|
13
|
+
mutants.push(...lineMutants);
|
|
14
|
+
}
|
|
15
|
+
return mutants.slice(0, maxMutants);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Generate mutations for a single line
|
|
19
|
+
*/
|
|
20
|
+
function mutateLine(line, lineNumber, filePath) {
|
|
21
|
+
const mutants = [];
|
|
22
|
+
// Return value mutations
|
|
23
|
+
const returnTrue = line.match(/return\s+true\b/);
|
|
24
|
+
if (returnTrue) {
|
|
25
|
+
mutants.push(makeMutant(filePath, line, line.replace(/return\s+true\b/, 'return false'), lineNumber, 'return_value', 'Changed return true to return false'));
|
|
26
|
+
}
|
|
27
|
+
const returnFalse = line.match(/return\s+false\b/);
|
|
28
|
+
if (returnFalse) {
|
|
29
|
+
mutants.push(makeMutant(filePath, line, line.replace(/return\s+false\b/, 'return true'), lineNumber, 'return_value', 'Changed return false to return true'));
|
|
30
|
+
}
|
|
31
|
+
const returnZero = line.match(/return\s+0\b/);
|
|
32
|
+
if (returnZero) {
|
|
33
|
+
mutants.push(makeMutant(filePath, line, line.replace(/return\s+0\b/, 'return 1'), lineNumber, 'return_value', 'Changed return 0 to return 1'));
|
|
34
|
+
}
|
|
35
|
+
const returnOne = line.match(/return\s+1\b/);
|
|
36
|
+
if (returnOne) {
|
|
37
|
+
mutants.push(makeMutant(filePath, line, line.replace(/return\s+1\b/, 'return 0'), lineNumber, 'return_value', 'Changed return 1 to return 0'));
|
|
38
|
+
}
|
|
39
|
+
const returnNull = line.match(/return\s+null\b/);
|
|
40
|
+
if (returnNull) {
|
|
41
|
+
mutants.push(makeMutant(filePath, line, line.replace(/return\s+null\b/, 'return undefined'), lineNumber, 'return_value', 'Changed return null to return undefined'));
|
|
42
|
+
}
|
|
43
|
+
const returnEmptyArr = line.match(/return\s+\[\]/);
|
|
44
|
+
if (returnEmptyArr) {
|
|
45
|
+
mutants.push(makeMutant(filePath, line, line.replace(/return\s+\[\]/, 'return [null]'), lineNumber, 'return_value', 'Changed return [] to return [null]'));
|
|
46
|
+
}
|
|
47
|
+
const returnEmptyStr = line.match(/return\s+''|return\s+""/);
|
|
48
|
+
if (returnEmptyStr) {
|
|
49
|
+
mutants.push(makeMutant(filePath, line, line.replace(/return\s+(?:''|"")/, 'return "mutated"'), lineNumber, 'return_value', 'Changed return empty string to "mutated"'));
|
|
50
|
+
}
|
|
51
|
+
// Conditional mutations (avoid JSX, generics, HTML, shift operators)
|
|
52
|
+
if (line.includes(' > ') && !line.includes('=>') && !/<\w/.test(line) && !/\w>/.test(line)) {
|
|
53
|
+
mutants.push(makeMutant(filePath, line, line.replace(' > ', ' <= '), lineNumber, 'conditional', 'Flipped > to <='));
|
|
54
|
+
}
|
|
55
|
+
if (line.includes(' < ') && !line.includes('=>') && !line.includes('<<') && !/<\w/.test(line)) {
|
|
56
|
+
mutants.push(makeMutant(filePath, line, line.replace(' < ', ' >= '), lineNumber, 'conditional', 'Flipped < to >='));
|
|
57
|
+
}
|
|
58
|
+
if (line.includes(' === ')) {
|
|
59
|
+
mutants.push(makeMutant(filePath, line, line.replace(' === ', ' !== '), lineNumber, 'conditional', 'Flipped === to !=='));
|
|
60
|
+
}
|
|
61
|
+
if (line.includes(' !== ')) {
|
|
62
|
+
mutants.push(makeMutant(filePath, line, line.replace(' !== ', ' === '), lineNumber, 'conditional', 'Flipped !== to ==='));
|
|
63
|
+
}
|
|
64
|
+
// Match == but not === (use regex with negative lookahead/lookbehind)
|
|
65
|
+
if (/[^=!] == [^=]/.test(` ${line} `)) {
|
|
66
|
+
mutants.push(makeMutant(filePath, line, line.replace(/ == (?!=)/, ' != '), lineNumber, 'conditional', 'Flipped == to !='));
|
|
67
|
+
}
|
|
68
|
+
// Logical operator mutations
|
|
69
|
+
if (line.includes(' && ')) {
|
|
70
|
+
mutants.push(makeMutant(filePath, line, line.replace(' && ', ' || '), lineNumber, 'logical_operator', 'Changed && to ||'));
|
|
71
|
+
}
|
|
72
|
+
if (line.includes(' || ')) {
|
|
73
|
+
mutants.push(makeMutant(filePath, line, line.replace(' || ', ' && '), lineNumber, 'logical_operator', 'Changed || to &&'));
|
|
74
|
+
}
|
|
75
|
+
// String literal mutations (only for simple cases)
|
|
76
|
+
const stringMatch = line.match(/'([^']{2,20})'|"([^"]{2,20})"/);
|
|
77
|
+
if (stringMatch && !line.includes('import') && !line.includes('require')) {
|
|
78
|
+
const original = stringMatch[0];
|
|
79
|
+
const mutated = original === `'${stringMatch[1]}'`
|
|
80
|
+
? `'MUTATED_${stringMatch[1]}'`
|
|
81
|
+
: `"MUTATED_${stringMatch[2]}"`;
|
|
82
|
+
mutants.push(makeMutant(filePath, line, line.replace(original, mutated), lineNumber, 'string_literal', `Changed string literal ${original}`));
|
|
83
|
+
}
|
|
84
|
+
return mutants;
|
|
85
|
+
}
|
|
86
|
+
function makeMutant(filePath, originalCode, mutatedCode, lineNumber, type, description) {
|
|
87
|
+
const id = createHash('sha256')
|
|
88
|
+
.update(`${filePath}:${lineNumber}:${type}:${description}`)
|
|
89
|
+
.digest('hex')
|
|
90
|
+
.slice(0, 12);
|
|
91
|
+
return {
|
|
92
|
+
id,
|
|
93
|
+
filePath,
|
|
94
|
+
originalCode,
|
|
95
|
+
mutatedCode,
|
|
96
|
+
startLine: lineNumber,
|
|
97
|
+
endLine: lineNumber,
|
|
98
|
+
type,
|
|
99
|
+
description,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { MutantResult, MutationReport } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Print mutation testing report to terminal
|
|
4
|
+
*/
|
|
5
|
+
export declare function printMutationReport(report: MutationReport): void;
|
|
6
|
+
/**
|
|
7
|
+
* Format report as JSON
|
|
8
|
+
*/
|
|
9
|
+
export declare function formatMutationJson(report: MutationReport): string;
|
|
10
|
+
/**
|
|
11
|
+
* Compute mutation score from a list of mutant results.
|
|
12
|
+
* Score = killed / total * 100. Returns 0 when there are no mutants.
|
|
13
|
+
*/
|
|
14
|
+
export declare function computeMutationScore(results: MutantResult[]): number;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
const GREEN = '\x1b[32m';
|
|
2
|
+
const YELLOW = '\x1b[33m';
|
|
3
|
+
const RED = '\x1b[31m';
|
|
4
|
+
const DIM = '\x1b[2m';
|
|
5
|
+
const BOLD = '\x1b[1m';
|
|
6
|
+
const RESET = '\x1b[0m';
|
|
7
|
+
/**
|
|
8
|
+
* Print mutation testing report to terminal
|
|
9
|
+
*/
|
|
10
|
+
export function printMutationReport(report) {
|
|
11
|
+
console.log(`\n${BOLD}Documentation Mutation Testing Report${RESET}\n`);
|
|
12
|
+
// Score bar
|
|
13
|
+
const color = report.mutationScore >= 70 ? GREEN : report.mutationScore >= 40 ? YELLOW : RED;
|
|
14
|
+
const barWidth = 30;
|
|
15
|
+
const filled = Math.round((report.mutationScore / 100) * barWidth);
|
|
16
|
+
const empty = barWidth - filled;
|
|
17
|
+
console.log(` Mutation Score: ${color}[${'█'.repeat(filled)}${'░'.repeat(empty)}] ${report.mutationScore.toFixed(1)}%${RESET}`);
|
|
18
|
+
console.log(` Killed: ${GREEN}${report.killed}${RESET} | Survived: ${RED}${report.survived}${RESET} | Errors: ${DIM}${report.errors}${RESET} | Timeouts: ${DIM}${report.timeouts}${RESET}`);
|
|
19
|
+
console.log(` Total mutants: ${report.totalMutants}`);
|
|
20
|
+
console.log('');
|
|
21
|
+
// Per-file breakdown
|
|
22
|
+
if (report.files.length > 0) {
|
|
23
|
+
console.log(`${BOLD} Per-file Breakdown${RESET}`);
|
|
24
|
+
console.log(` ${'File'.padEnd(45)} ${'Mutants'.padStart(8)} ${'Killed'.padStart(8)} ${'Survived'.padStart(10)} ${'Score'.padStart(8)}`);
|
|
25
|
+
console.log(` ${'─'.repeat(45)} ${'─'.repeat(8)} ${'─'.repeat(8)} ${'─'.repeat(10)} ${'─'.repeat(8)}`);
|
|
26
|
+
for (const file of report.files) {
|
|
27
|
+
const shortPath = file.filePath.length > 44
|
|
28
|
+
? '...' + file.filePath.slice(-41)
|
|
29
|
+
: file.filePath;
|
|
30
|
+
const fileColor = file.score >= 70 ? GREEN : file.score >= 40 ? YELLOW : RED;
|
|
31
|
+
console.log(` ${shortPath.padEnd(45)} ${String(file.mutants).padStart(8)} ${String(file.killed).padStart(8)} ${String(file.survived).padStart(10)} ${fileColor}${(file.score.toFixed(0) + '%').padStart(8)}${RESET}`);
|
|
32
|
+
}
|
|
33
|
+
console.log('');
|
|
34
|
+
}
|
|
35
|
+
// Dangerous survivors
|
|
36
|
+
if (report.survivors.length > 0) {
|
|
37
|
+
const shown = report.survivors.slice(0, 10);
|
|
38
|
+
console.log(`${RED}${BOLD} Dangerous Survivors${RESET} (${report.survivors.length} total)`);
|
|
39
|
+
console.log(` These mutations changed behavior but no doc example caught them:\n`);
|
|
40
|
+
for (const s of shown) {
|
|
41
|
+
console.log(` ${RED}SURVIVED${RESET} ${s.mutant.filePath}:${s.mutant.startLine + 1}`);
|
|
42
|
+
console.log(` ${DIM}${s.mutant.description}${RESET}`);
|
|
43
|
+
console.log(` ${DIM}${s.mutant.originalCode.trim()} → ${s.mutant.mutatedCode.trim()}${RESET}`);
|
|
44
|
+
console.log('');
|
|
45
|
+
}
|
|
46
|
+
if (report.survivors.length > 10) {
|
|
47
|
+
console.log(` ${DIM}... and ${report.survivors.length - 10} more${RESET}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Format report as JSON
|
|
53
|
+
*/
|
|
54
|
+
export function formatMutationJson(report) {
|
|
55
|
+
return JSON.stringify(report, null, 2);
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Compute mutation score from a list of mutant results.
|
|
59
|
+
* Score = killed / total * 100. Returns 0 when there are no mutants.
|
|
60
|
+
*/
|
|
61
|
+
export function computeMutationScore(results) {
|
|
62
|
+
if (results.length === 0)
|
|
63
|
+
return 0;
|
|
64
|
+
const killed = results.filter(r => r.status === 'killed').length;
|
|
65
|
+
return (killed / results.length) * 100;
|
|
66
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Mutant, MutantResult } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Run doc examples against a mutated source file and check if the mutation is caught
|
|
4
|
+
*/
|
|
5
|
+
export declare function runMutant(mutant: Mutant, docExamples: Array<{
|
|
6
|
+
code: string;
|
|
7
|
+
language: string;
|
|
8
|
+
filePath: string;
|
|
9
|
+
}>, timeout?: number): MutantResult;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync, mkdirSync, rmSync } from 'fs';
|
|
2
|
+
import { join, basename, dirname } from 'path';
|
|
3
|
+
import { tmpdir } from 'os';
|
|
4
|
+
import { spawnSync } from 'child_process';
|
|
5
|
+
/**
|
|
6
|
+
* Run doc examples against a mutated source file and check if the mutation is caught
|
|
7
|
+
*/
|
|
8
|
+
export function runMutant(mutant, docExamples, timeout = 10000) {
|
|
9
|
+
// Create temp directory with mutated file
|
|
10
|
+
const tmpDir = join(tmpdir(), `skrypt-mutant-${mutant.id}`);
|
|
11
|
+
try {
|
|
12
|
+
mkdirSync(tmpDir, { recursive: true });
|
|
13
|
+
// Read original file, apply mutation, write to temp
|
|
14
|
+
const originalContent = readFileSync(mutant.filePath, 'utf-8');
|
|
15
|
+
const lines = originalContent.split('\n');
|
|
16
|
+
lines[mutant.startLine] = mutant.mutatedCode;
|
|
17
|
+
const mutatedContent = lines.join('\n');
|
|
18
|
+
const tempFile = join(tmpDir, basename(mutant.filePath));
|
|
19
|
+
writeFileSync(tempFile, mutatedContent);
|
|
20
|
+
// Run each doc example that references symbols in the mutated file
|
|
21
|
+
for (const example of docExamples) {
|
|
22
|
+
const exampleFile = join(tmpDir, `example_${basename(example.filePath)}`);
|
|
23
|
+
writeFileSync(exampleFile, example.code);
|
|
24
|
+
const isTS = example.language === 'typescript' || example.language === 'javascript';
|
|
25
|
+
const isPy = example.language === 'python';
|
|
26
|
+
let result;
|
|
27
|
+
if (isTS) {
|
|
28
|
+
result = spawnSync('node', ['--input-type=module', '-e', example.code], {
|
|
29
|
+
cwd: tmpDir,
|
|
30
|
+
timeout,
|
|
31
|
+
encoding: 'utf-8',
|
|
32
|
+
env: { ...process.env, NODE_PATH: dirname(mutant.filePath) },
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
else if (isPy) {
|
|
36
|
+
result = spawnSync('python3', ['-c', example.code], {
|
|
37
|
+
cwd: tmpDir,
|
|
38
|
+
timeout,
|
|
39
|
+
encoding: 'utf-8',
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
// If example fails, mutation was caught (killed)
|
|
46
|
+
if (result.status !== 0) {
|
|
47
|
+
return {
|
|
48
|
+
mutant,
|
|
49
|
+
status: 'killed',
|
|
50
|
+
failedExample: example.filePath,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
// All examples still pass = mutation survived (bad)
|
|
55
|
+
return { mutant, status: 'survived' };
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
59
|
+
if (message.includes('TIMEOUT') || message.includes('timed out')) {
|
|
60
|
+
return { mutant, status: 'timeout' };
|
|
61
|
+
}
|
|
62
|
+
return { mutant, status: 'error', error: message };
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
try {
|
|
66
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
67
|
+
}
|
|
68
|
+
catch { /* ignore cleanup errors */ }
|
|
69
|
+
}
|
|
70
|
+
}
|