phantom-pr 0.4.19 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +216 -80
- package/dist/core/analysis/astExtractor.d.ts +15 -0
- package/dist/core/analysis/astExtractor.js +354 -0
- package/dist/core/analysis/astExtractor.js.map +1 -0
- package/dist/core/analysis/index.d.ts +3 -0
- package/dist/core/analysis/index.js +3 -0
- package/dist/core/analysis/index.js.map +1 -0
- package/dist/core/analysis/types.d.ts +50 -0
- package/dist/core/analysis/types.js +16 -0
- package/dist/core/analysis/types.js.map +1 -0
- package/dist/core/context/contextSelector.d.ts +29 -0
- package/dist/core/context/contextSelector.js +293 -0
- package/dist/core/context/contextSelector.js.map +1 -0
- package/dist/core/context/index.d.ts +4 -0
- package/dist/core/context/index.js +3 -0
- package/dist/core/context/index.js.map +1 -0
- package/dist/core/generator/enhancedContext.d.ts +34 -0
- package/dist/core/generator/enhancedContext.js +144 -0
- package/dist/core/generator/enhancedContext.js.map +1 -0
- package/dist/core/generator/llmGenerator.js +157 -15
- package/dist/core/generator/llmGenerator.js.map +1 -1
- package/dist/core/generator/postValidation.d.ts +24 -0
- package/dist/core/generator/postValidation.js +57 -0
- package/dist/core/generator/postValidation.js.map +1 -0
- package/dist/core/retry/errorParser.d.ts +12 -0
- package/dist/core/retry/errorParser.js +264 -0
- package/dist/core/retry/errorParser.js.map +1 -0
- package/dist/core/retry/feedbackGenerator.d.ts +12 -0
- package/dist/core/retry/feedbackGenerator.js +164 -0
- package/dist/core/retry/feedbackGenerator.js.map +1 -0
- package/dist/core/retry/index.d.ts +3 -0
- package/dist/core/retry/index.js +3 -0
- package/dist/core/retry/index.js.map +1 -0
- package/dist/core/retry/types.d.ts +40 -0
- package/dist/core/retry/types.js +5 -0
- package/dist/core/retry/types.js.map +1 -0
- package/dist/core/testRunner/retryEnhancer.d.ts +32 -0
- package/dist/core/testRunner/retryEnhancer.js +58 -0
- package/dist/core/testRunner/retryEnhancer.js.map +1 -0
- package/dist/core/validation/coverageVerifier.d.ts +16 -0
- package/dist/core/validation/coverageVerifier.js +226 -0
- package/dist/core/validation/coverageVerifier.js.map +1 -0
- package/dist/core/validation/index.d.ts +2 -0
- package/dist/core/validation/index.js +2 -0
- package/dist/core/validation/index.js.map +1 -0
- package/dist/core/validation/types.d.ts +24 -0
- package/dist/core/validation/types.js +6 -0
- package/dist/core/validation/types.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Feedback generator that creates LLM-friendly retry prompts from parsed errors.
|
|
3
|
+
* Outputs valid prompt text with prioritized, actionable fixes.
|
|
4
|
+
*/
|
|
5
|
+
import type { RetryContext, RetryPrompt } from './types.js';
|
|
6
|
+
/**
|
|
7
|
+
* Generate a retry prompt from the context.
|
|
8
|
+
*
|
|
9
|
+
* @param context - The retry context with errors and previous test
|
|
10
|
+
* @returns A structured retry prompt, or null if no actionable retry possible
|
|
11
|
+
*/
|
|
12
|
+
export declare function generateRetryPrompt(context: RetryContext): RetryPrompt | null;
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Feedback generator that creates LLM-friendly retry prompts from parsed errors.
|
|
3
|
+
* Outputs valid prompt text with prioritized, actionable fixes.
|
|
4
|
+
*/
|
|
5
|
+
/** Maximum characters for generated prompt (roughly 500 tokens). */
|
|
6
|
+
const MAX_PROMPT_CHARS = 2000;
|
|
7
|
+
/**
|
|
8
|
+
* Get priority for an error kind.
|
|
9
|
+
*/
|
|
10
|
+
function getErrorPriority(kind) {
|
|
11
|
+
switch (kind) {
|
|
12
|
+
case 'import_error':
|
|
13
|
+
case 'undefined_variable':
|
|
14
|
+
return 'critical';
|
|
15
|
+
case 'mock_not_called':
|
|
16
|
+
case 'mock_wrong_args':
|
|
17
|
+
case 'type_error':
|
|
18
|
+
return 'high';
|
|
19
|
+
case 'assertion_mismatch':
|
|
20
|
+
case 'timeout':
|
|
21
|
+
return 'medium';
|
|
22
|
+
default:
|
|
23
|
+
return 'medium';
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Get focus area description for an error kind.
|
|
28
|
+
*/
|
|
29
|
+
function getFocusArea(kind) {
|
|
30
|
+
switch (kind) {
|
|
31
|
+
case 'import_error':
|
|
32
|
+
return 'Add missing mock at the top of the file';
|
|
33
|
+
case 'undefined_variable':
|
|
34
|
+
return 'Fix missing import or variable declaration';
|
|
35
|
+
case 'mock_not_called':
|
|
36
|
+
return 'Ensure the code path triggers the expected function call';
|
|
37
|
+
case 'mock_wrong_args':
|
|
38
|
+
return 'Update mock expectations to match actual arguments';
|
|
39
|
+
case 'type_error':
|
|
40
|
+
return 'Fix undefined/null access by properly mocking data';
|
|
41
|
+
case 'assertion_mismatch':
|
|
42
|
+
return 'Update expected value or fix test setup';
|
|
43
|
+
case 'timeout':
|
|
44
|
+
return 'Fix async handling with proper await/act patterns';
|
|
45
|
+
default:
|
|
46
|
+
return 'Review and fix the error based on the message';
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Format error details for the prompt.
|
|
51
|
+
*/
|
|
52
|
+
function formatErrorDetails(error) {
|
|
53
|
+
const parts = [];
|
|
54
|
+
parts.push(`**Error Type:** ${error.kind.replace(/_/g, ' ')}`);
|
|
55
|
+
parts.push(`**Test:** ${error.testName}`);
|
|
56
|
+
if (error.details.expected !== undefined) {
|
|
57
|
+
parts.push(`**Expected:** ${error.details.expected}`);
|
|
58
|
+
}
|
|
59
|
+
if (error.details.received !== undefined) {
|
|
60
|
+
parts.push(`**Received:** ${error.details.received}`);
|
|
61
|
+
}
|
|
62
|
+
if (error.details.callCount !== undefined) {
|
|
63
|
+
parts.push(`**Expected calls:** ${error.details.callCount.expected}`);
|
|
64
|
+
parts.push(`**Received calls:** ${error.details.callCount.received}`);
|
|
65
|
+
}
|
|
66
|
+
if (error.details.missingImport !== undefined) {
|
|
67
|
+
parts.push(`**Missing module:** ${error.details.missingImport}`);
|
|
68
|
+
}
|
|
69
|
+
if (error.details.variableName !== undefined) {
|
|
70
|
+
parts.push(`**Undefined variable:** ${error.details.variableName}`);
|
|
71
|
+
}
|
|
72
|
+
return parts.join('\n');
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Sort errors by priority (most actionable first).
|
|
76
|
+
*/
|
|
77
|
+
function sortByPriority(errors) {
|
|
78
|
+
const priorityOrder = { critical: 0, high: 1, medium: 2 };
|
|
79
|
+
return [...errors].sort((a, b) => {
|
|
80
|
+
const pa = priorityOrder[getErrorPriority(a.kind)];
|
|
81
|
+
const pb = priorityOrder[getErrorPriority(b.kind)];
|
|
82
|
+
return pa - pb;
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Generate a retry prompt from the context.
|
|
87
|
+
*
|
|
88
|
+
* @param context - The retry context with errors and previous test
|
|
89
|
+
* @returns A structured retry prompt, or null if no actionable retry possible
|
|
90
|
+
*/
|
|
91
|
+
export function generateRetryPrompt(context) {
|
|
92
|
+
// Return null if we've exceeded max attempts
|
|
93
|
+
if (context.attempt >= context.maxAttempts) {
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
// Return null if no errors provided
|
|
97
|
+
if (context.errors.length === 0) {
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
// Sort errors by priority and get the most actionable one
|
|
101
|
+
const sortedErrors = sortByPriority(context.errors);
|
|
102
|
+
const primaryError = sortedErrors[0];
|
|
103
|
+
// Return null if all errors are unknown (can't provide actionable feedback)
|
|
104
|
+
if (primaryError === undefined) {
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
if (primaryError.kind === 'unknown' && sortedErrors.every(e => e.kind === 'unknown')) {
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
// If primary is unknown but there are other errors, use the first non-unknown
|
|
111
|
+
const actionableError = primaryError.kind === 'unknown'
|
|
112
|
+
? sortedErrors.find(e => e.kind !== 'unknown') ?? primaryError
|
|
113
|
+
: primaryError;
|
|
114
|
+
const priority = getErrorPriority(actionableError.kind);
|
|
115
|
+
const focusArea = getFocusArea(actionableError.kind);
|
|
116
|
+
// Build the prompt
|
|
117
|
+
const promptParts = [];
|
|
118
|
+
promptParts.push(`## RETRY ATTEMPT ${context.attempt + 1}/${context.maxAttempts}`);
|
|
119
|
+
promptParts.push('');
|
|
120
|
+
promptParts.push('Your previous test failed with the following error:');
|
|
121
|
+
promptParts.push('');
|
|
122
|
+
promptParts.push(formatErrorDetails(actionableError));
|
|
123
|
+
promptParts.push('');
|
|
124
|
+
promptParts.push('## REQUIRED FIX');
|
|
125
|
+
promptParts.push(actionableError.suggestedFix);
|
|
126
|
+
promptParts.push('');
|
|
127
|
+
promptParts.push('## FOCUS AREA');
|
|
128
|
+
promptParts.push(focusArea);
|
|
129
|
+
promptParts.push('');
|
|
130
|
+
// Include other errors as context if there are more
|
|
131
|
+
if (sortedErrors.length > 1) {
|
|
132
|
+
const otherErrors = sortedErrors.slice(1, 4); // Max 3 additional errors
|
|
133
|
+
if (otherErrors.length > 0) {
|
|
134
|
+
promptParts.push('## ADDITIONAL ERRORS (fix after primary)');
|
|
135
|
+
for (const err of otherErrors) {
|
|
136
|
+
promptParts.push(`- ${err.kind.replace(/_/g, ' ')}: ${err.suggestedFix.slice(0, 100)}`);
|
|
137
|
+
}
|
|
138
|
+
promptParts.push('');
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
promptParts.push('## ORIGINAL TEST (with error)');
|
|
142
|
+
promptParts.push('```');
|
|
143
|
+
// Truncate test if needed to stay under limit
|
|
144
|
+
let testContent = context.generatedTest;
|
|
145
|
+
const headerLength = promptParts.join('\n').length + 100; // Buffer for closing
|
|
146
|
+
const maxTestChars = MAX_PROMPT_CHARS - headerLength;
|
|
147
|
+
if (testContent.length > maxTestChars) {
|
|
148
|
+
testContent = testContent.slice(0, maxTestChars - 50) + '\n// ... (truncated)';
|
|
149
|
+
}
|
|
150
|
+
promptParts.push(testContent);
|
|
151
|
+
promptParts.push('```');
|
|
152
|
+
promptParts.push('');
|
|
153
|
+
promptParts.push('## INSTRUCTIONS');
|
|
154
|
+
promptParts.push('1. Fix ONLY the identified error');
|
|
155
|
+
promptParts.push('2. Do NOT change unrelated parts of the test');
|
|
156
|
+
promptParts.push('3. Output the complete corrected test file');
|
|
157
|
+
const prompt = promptParts.join('\n');
|
|
158
|
+
return {
|
|
159
|
+
prompt,
|
|
160
|
+
priority,
|
|
161
|
+
focusArea,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=feedbackGenerator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"feedbackGenerator.js","sourceRoot":"","sources":["../../../src/core/retry/feedbackGenerator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,oEAAoE;AACpE,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAA6B;IACrD,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,cAAc,CAAC;QACpB,KAAK,oBAAoB;YACvB,OAAO,UAAU,CAAC;QACpB,KAAK,iBAAiB,CAAC;QACvB,KAAK,iBAAiB,CAAC;QACvB,KAAK,YAAY;YACf,OAAO,MAAM,CAAC;QAChB,KAAK,oBAAoB,CAAC;QAC1B,KAAK,SAAS;YACZ,OAAO,QAAQ,CAAC;QAClB;YACE,OAAO,QAAQ,CAAC;IACpB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,IAA6B;IACjD,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,cAAc;YACjB,OAAO,yCAAyC,CAAC;QACnD,KAAK,oBAAoB;YACvB,OAAO,4CAA4C,CAAC;QACtD,KAAK,iBAAiB;YACpB,OAAO,0DAA0D,CAAC;QACpE,KAAK,iBAAiB;YACpB,OAAO,oDAAoD,CAAC;QAC9D,KAAK,YAAY;YACf,OAAO,oDAAoD,CAAC;QAC9D,KAAK,oBAAoB;YACvB,OAAO,yCAAyC,CAAC;QACnD,KAAK,SAAS;YACZ,OAAO,mDAAmD,CAAC;QAC7D;YACE,OAAO,+CAA+C,CAAC;IAC3D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,KAAsB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,mBAAmB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/D,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE1C,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,uBAAuB,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtE,KAAK,CAAC,IAAI,CAAC,uBAAuB,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,uBAAuB,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,2BAA2B,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,MAAyB;IAC/C,MAAM,aAAa,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAE1D,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC/B,MAAM,EAAE,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,MAAM,EAAE,GAAG,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,EAAE,GAAG,EAAE,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAqB;IACvD,6CAA6C;IAC7C,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oCAAoC;IACpC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0DAA0D;IAC1D,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAErC,4EAA4E;IAC5E,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,YAAY,CAAC,IAAI,KAAK,SAAS,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC;QACrF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8EAA8E;IAC9E,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,KAAK,SAAS;QACrD,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,YAAY;QAC9D,CAAC,CAAC,YAAY,CAAC;IAEjB,MAAM,QAAQ,GAAG,gBAAgB,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAErD,mBAAmB;IACnB,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,WAAW,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACnF,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrB,WAAW,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;IACxE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrB,WAAW,CAAC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC,CAAC;IACtD,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrB,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACpC,WAAW,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;IAC/C,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrB,WAAW,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAClC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAErB,oDAAoD;IACpD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,0BAA0B;QACxE,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,WAAW,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;YAC7D,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,WAAW,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAC1F,CAAC;YACD,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAClD,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAExB,8CAA8C;IAC9C,IAAI,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC;IACxC,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,qBAAqB;IAC/E,MAAM,YAAY,GAAG,gBAAgB,GAAG,YAAY,CAAC;IAErD,IAAI,WAAW,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;QACtC,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,GAAG,EAAE,CAAC,GAAG,sBAAsB,CAAC;IACjF,CAAC;IAED,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC9B,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxB,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrB,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACpC,WAAW,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IACrD,WAAW,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IACjE,WAAW,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IAE/D,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEtC,OAAO;QACL,MAAM;QACN,QAAQ;QACR,SAAS;KACV,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/retry/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for error parsing and retry feedback generation.
|
|
3
|
+
*/
|
|
4
|
+
/** Kind of test error detected. */
|
|
5
|
+
export type ErrorKind = 'assertion_mismatch' | 'mock_not_called' | 'mock_wrong_args' | 'undefined_variable' | 'import_error' | 'timeout' | 'type_error' | 'unknown';
|
|
6
|
+
/** Details extracted from the error. */
|
|
7
|
+
export type ErrorDetails = {
|
|
8
|
+
expected?: string;
|
|
9
|
+
received?: string;
|
|
10
|
+
callCount?: {
|
|
11
|
+
expected: number;
|
|
12
|
+
received: number;
|
|
13
|
+
};
|
|
14
|
+
missingImport?: string;
|
|
15
|
+
variableName?: string;
|
|
16
|
+
stackTrace?: string;
|
|
17
|
+
};
|
|
18
|
+
/** A parsed test error with actionable information. */
|
|
19
|
+
export type ParsedTestError = {
|
|
20
|
+
kind: ErrorKind;
|
|
21
|
+
testName: string;
|
|
22
|
+
details: ErrorDetails;
|
|
23
|
+
suggestedFix: string;
|
|
24
|
+
};
|
|
25
|
+
/** Priority of a retry prompt. */
|
|
26
|
+
export type RetryPriority = 'critical' | 'high' | 'medium';
|
|
27
|
+
/** Context for generating a retry prompt. */
|
|
28
|
+
export type RetryContext = {
|
|
29
|
+
originalPrompt: string;
|
|
30
|
+
generatedTest: string;
|
|
31
|
+
errors: ParsedTestError[];
|
|
32
|
+
attempt: number;
|
|
33
|
+
maxAttempts: number;
|
|
34
|
+
};
|
|
35
|
+
/** A generated retry prompt. */
|
|
36
|
+
export type RetryPrompt = {
|
|
37
|
+
prompt: string;
|
|
38
|
+
priority: RetryPriority;
|
|
39
|
+
focusArea: string;
|
|
40
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/core/retry/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retry enhancement utilities.
|
|
3
|
+
* Provides smarter retry prompts based on parsed test errors.
|
|
4
|
+
*/
|
|
5
|
+
import { type ParsedTestError, type RetryPrompt } from '../retry/index.js';
|
|
6
|
+
export type RetryEnhanceOptions = {
|
|
7
|
+
originalPrompt: string;
|
|
8
|
+
generatedTest: string;
|
|
9
|
+
testOutput: string;
|
|
10
|
+
attempt: number;
|
|
11
|
+
maxAttempts: number;
|
|
12
|
+
enabled?: boolean;
|
|
13
|
+
};
|
|
14
|
+
export type RetryEnhanceResult = {
|
|
15
|
+
hasActionableErrors: boolean;
|
|
16
|
+
retryPrompt: RetryPrompt | null;
|
|
17
|
+
parsedErrors: ParsedTestError[];
|
|
18
|
+
enabled: boolean;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Enhance retry feedback using parsed test errors.
|
|
22
|
+
* Generates a smarter retry prompt with specific fixes.
|
|
23
|
+
*
|
|
24
|
+
* @param opts - Retry options
|
|
25
|
+
* @returns Enhanced retry result with prompt and parsed errors
|
|
26
|
+
*/
|
|
27
|
+
export declare function enhanceRetryFeedback(opts: RetryEnhanceOptions): RetryEnhanceResult;
|
|
28
|
+
/**
|
|
29
|
+
* Format parsed errors for feedback section in existing prompt format.
|
|
30
|
+
* This provides backward-compatible integration with existing feedback mechanism.
|
|
31
|
+
*/
|
|
32
|
+
export declare function formatErrorsForFeedback(errors: ParsedTestError[]): string;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retry enhancement utilities.
|
|
3
|
+
* Provides smarter retry prompts based on parsed test errors.
|
|
4
|
+
*/
|
|
5
|
+
import { parseTestErrors, generateRetryPrompt } from '../retry/index.js';
|
|
6
|
+
/**
|
|
7
|
+
* Enhance retry feedback using parsed test errors.
|
|
8
|
+
* Generates a smarter retry prompt with specific fixes.
|
|
9
|
+
*
|
|
10
|
+
* @param opts - Retry options
|
|
11
|
+
* @returns Enhanced retry result with prompt and parsed errors
|
|
12
|
+
*/
|
|
13
|
+
export function enhanceRetryFeedback(opts) {
|
|
14
|
+
const enabled = opts.enabled ?? true;
|
|
15
|
+
if (!enabled) {
|
|
16
|
+
return {
|
|
17
|
+
hasActionableErrors: false,
|
|
18
|
+
retryPrompt: null,
|
|
19
|
+
parsedErrors: [],
|
|
20
|
+
enabled: false,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
// Parse errors from test output
|
|
24
|
+
const parsedErrors = parseTestErrors(opts.testOutput);
|
|
25
|
+
// Generate retry prompt if we have actionable errors
|
|
26
|
+
const retryPrompt = generateRetryPrompt({
|
|
27
|
+
originalPrompt: opts.originalPrompt,
|
|
28
|
+
generatedTest: opts.generatedTest,
|
|
29
|
+
errors: parsedErrors,
|
|
30
|
+
attempt: opts.attempt,
|
|
31
|
+
maxAttempts: opts.maxAttempts,
|
|
32
|
+
});
|
|
33
|
+
// Check if we have actionable errors (not all unknown)
|
|
34
|
+
const hasActionableErrors = parsedErrors.some(e => e.kind !== 'unknown');
|
|
35
|
+
return {
|
|
36
|
+
hasActionableErrors,
|
|
37
|
+
retryPrompt,
|
|
38
|
+
parsedErrors,
|
|
39
|
+
enabled: true,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Format parsed errors for feedback section in existing prompt format.
|
|
44
|
+
* This provides backward-compatible integration with existing feedback mechanism.
|
|
45
|
+
*/
|
|
46
|
+
export function formatErrorsForFeedback(errors) {
|
|
47
|
+
if (errors.length === 0) {
|
|
48
|
+
return '';
|
|
49
|
+
}
|
|
50
|
+
const lines = [];
|
|
51
|
+
lines.push('## PARSED ERRORS (prioritized fixes)');
|
|
52
|
+
for (const error of errors.slice(0, 5)) {
|
|
53
|
+
lines.push(`- ${error.kind}: ${error.suggestedFix.slice(0, 100)}`);
|
|
54
|
+
}
|
|
55
|
+
lines.push('');
|
|
56
|
+
return lines.join('\n');
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=retryEnhancer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retryEnhancer.js","sourceRoot":"","sources":["../../../src/core/testRunner/retryEnhancer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAA0C,MAAM,mBAAmB,CAAC;AAkBjH;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAyB;IAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;IAErC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,mBAAmB,EAAE,KAAK;YAC1B,WAAW,EAAE,IAAI;YACjB,YAAY,EAAE,EAAE;YAChB,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;IAED,gCAAgC;IAChC,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEtD,qDAAqD;IACrD,MAAM,WAAW,GAAG,mBAAmB,CAAC;QACtC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM,EAAE,YAAY;QACpB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC,CAAC;IAEH,uDAAuD;IACvD,MAAM,mBAAmB,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAEzE,OAAO;QACL,mBAAmB;QACnB,WAAW;QACX,YAAY;QACZ,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAyB;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IAEnD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Post-generation validator that verifies generated tests cover all identified behaviors.
|
|
3
|
+
* Returns structured validation result with specific rejection codes.
|
|
4
|
+
*
|
|
5
|
+
* Static analysis only - does NOT execute the test.
|
|
6
|
+
*/
|
|
7
|
+
import type { ComponentAnalysis } from '../analysis/types.js';
|
|
8
|
+
import type { ValidationResult } from './types.js';
|
|
9
|
+
/**
|
|
10
|
+
* Verify that generated test adequately covers the analyzed component.
|
|
11
|
+
*
|
|
12
|
+
* @param analysis - Component analysis from AST extraction
|
|
13
|
+
* @param generatedTestContent - The generated test file content
|
|
14
|
+
* @returns ValidationResult with coverage details and any rejections
|
|
15
|
+
*/
|
|
16
|
+
export declare function verifyCoverage(analysis: ComponentAnalysis, generatedTestContent: string): ValidationResult;
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Post-generation validator that verifies generated tests cover all identified behaviors.
|
|
3
|
+
* Returns structured validation result with specific rejection codes.
|
|
4
|
+
*
|
|
5
|
+
* Static analysis only - does NOT execute the test.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Extract test block names from test file content.
|
|
9
|
+
* Matches it('...'), test('...'), describe('...') patterns.
|
|
10
|
+
*/
|
|
11
|
+
function extractTestNames(testContent) {
|
|
12
|
+
const names = [];
|
|
13
|
+
// Match it('name', ...), test('name', ...), describe('name', ...)
|
|
14
|
+
const testBlockRe = /(?:it|test|describe)\s*\(\s*['"`]([^'"`]+)['"`]/g;
|
|
15
|
+
for (const match of testContent.matchAll(testBlockRe)) {
|
|
16
|
+
const name = match[1];
|
|
17
|
+
if (name)
|
|
18
|
+
names.push(name.toLowerCase());
|
|
19
|
+
}
|
|
20
|
+
return names;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Extract jest.mock() and vi.mock() calls from test content.
|
|
24
|
+
*/
|
|
25
|
+
function extractMockedModules(testContent) {
|
|
26
|
+
const mocked = new Set();
|
|
27
|
+
// jest.mock('path') or vi.mock('path')
|
|
28
|
+
const mockRe = /(?:jest|vi)\.mock\s*\(\s*['"`]([^'"`]+)['"`]/g;
|
|
29
|
+
for (const match of testContent.matchAll(mockRe)) {
|
|
30
|
+
const path = match[1];
|
|
31
|
+
if (path)
|
|
32
|
+
mocked.add(path);
|
|
33
|
+
}
|
|
34
|
+
return mocked;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Check if test content appears to test branch conditions.
|
|
38
|
+
* Looks for patterns that suggest conditional testing.
|
|
39
|
+
*/
|
|
40
|
+
function countBranchTests(testContent, branches) {
|
|
41
|
+
const testNames = extractTestNames(testContent);
|
|
42
|
+
const contentLower = testContent.toLowerCase();
|
|
43
|
+
let testedCount = 0;
|
|
44
|
+
for (const branch of branches) {
|
|
45
|
+
const conditionLower = branch.condition.toLowerCase();
|
|
46
|
+
// Check if any test name mentions the condition
|
|
47
|
+
const mentionedInTestName = testNames.some(name => name.includes(conditionLower) ||
|
|
48
|
+
conditionLower.includes(name.split(' ')[0] ?? ''));
|
|
49
|
+
// Check for common branch testing patterns
|
|
50
|
+
const hasConditionTest = contentLower.includes(`when ${conditionLower}`) ||
|
|
51
|
+
contentLower.includes(`if ${conditionLower}`) ||
|
|
52
|
+
contentLower.includes(`${conditionLower} is true`) ||
|
|
53
|
+
contentLower.includes(`${conditionLower} is false`) ||
|
|
54
|
+
contentLower.includes(`should render when ${conditionLower}`) ||
|
|
55
|
+
contentLower.includes(`should not render when`);
|
|
56
|
+
// Check for assertion patterns that might test the branch
|
|
57
|
+
const hasAssertions = contentLower.includes('tobetruthy') ||
|
|
58
|
+
contentLower.includes('tobefalsy') ||
|
|
59
|
+
contentLower.includes('tobeinthedocument') ||
|
|
60
|
+
contentLower.includes('not.tobeinthedocument');
|
|
61
|
+
if (mentionedInTestName || hasConditionTest || hasAssertions) {
|
|
62
|
+
testedCount++;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
// Heuristic: if there are multiple test blocks covering different scenarios,
|
|
66
|
+
// assume branches are being adequately tested (lenient to avoid false rejections)
|
|
67
|
+
if (branches.length > 0 && testNames.length >= 2) {
|
|
68
|
+
// If we have descriptive test names, count them as covering branches
|
|
69
|
+
testedCount = Math.max(testedCount, Math.min(branches.length, testNames.length));
|
|
70
|
+
}
|
|
71
|
+
return testedCount;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Check if test content appears to test React effects.
|
|
75
|
+
*/
|
|
76
|
+
function countEffectTests(testContent, effects) {
|
|
77
|
+
const contentLower = testContent.toLowerCase();
|
|
78
|
+
const testNames = extractTestNames(testContent);
|
|
79
|
+
// Check for effect-related patterns
|
|
80
|
+
const hasEffectTest = contentLower.includes('useeffect') ||
|
|
81
|
+
contentLower.includes('effect') ||
|
|
82
|
+
contentLower.includes('on mount') ||
|
|
83
|
+
contentLower.includes('onmount') ||
|
|
84
|
+
contentLower.includes('lifecycle') ||
|
|
85
|
+
contentLower.includes('side effect');
|
|
86
|
+
// Check for async patterns often used with effects
|
|
87
|
+
const hasAsyncPattern = contentLower.includes('await') ||
|
|
88
|
+
contentLower.includes('waitfor') ||
|
|
89
|
+
contentLower.includes('act(');
|
|
90
|
+
// Check for cleanup patterns (testing cleanup implies testing the effect)
|
|
91
|
+
const hasCleanupPattern = contentLower.includes('unmount') ||
|
|
92
|
+
contentLower.includes('cleanup') ||
|
|
93
|
+
contentLower.includes('clearinterval') ||
|
|
94
|
+
contentLower.includes('cleartimeout');
|
|
95
|
+
// Check for render patterns (basic effect tests render the component)
|
|
96
|
+
const hasRenderPattern = contentLower.includes('render(');
|
|
97
|
+
// If any effect-related pattern is found, assume effects are being tested
|
|
98
|
+
if (hasEffectTest || hasAsyncPattern || hasCleanupPattern) {
|
|
99
|
+
return effects.length;
|
|
100
|
+
}
|
|
101
|
+
// If there are multiple test cases and render is used, be lenient
|
|
102
|
+
if (testNames.length >= 2 && hasRenderPattern) {
|
|
103
|
+
return effects.length;
|
|
104
|
+
}
|
|
105
|
+
return 0;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Check if test content appears to test cleanup behavior.
|
|
109
|
+
*/
|
|
110
|
+
function hasCleanupTest(testContent) {
|
|
111
|
+
const contentLower = testContent.toLowerCase();
|
|
112
|
+
return (contentLower.includes('unmount') ||
|
|
113
|
+
contentLower.includes('cleanup') ||
|
|
114
|
+
contentLower.includes('clearinterval') ||
|
|
115
|
+
contentLower.includes('cleartimeout') ||
|
|
116
|
+
contentLower.includes('removelistener') ||
|
|
117
|
+
contentLower.includes('unsubscribe') ||
|
|
118
|
+
contentLower.includes('dispose'));
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Extract child component names from import path.
|
|
122
|
+
*/
|
|
123
|
+
function normalizeImportToName(importPath) {
|
|
124
|
+
// Get the last segment and strip extension
|
|
125
|
+
const segments = importPath.split('/');
|
|
126
|
+
const last = segments[segments.length - 1] ?? importPath;
|
|
127
|
+
return last.replace(/\.(tsx?|jsx?)$/, '');
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Check which child components are mocked in the test.
|
|
131
|
+
*/
|
|
132
|
+
function findMockedChildren(testContent, children) {
|
|
133
|
+
const mockedModules = extractMockedModules(testContent);
|
|
134
|
+
const mocked = [];
|
|
135
|
+
for (const child of children) {
|
|
136
|
+
// Check if the child's import path is mocked
|
|
137
|
+
const isMocked = mockedModules.has(child.importPath) ||
|
|
138
|
+
[...mockedModules].some(m => m.includes(child.name) ||
|
|
139
|
+
normalizeImportToName(m).toLowerCase() === child.name.toLowerCase());
|
|
140
|
+
if (isMocked) {
|
|
141
|
+
mocked.push(child.name);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
// Also check for inline mocks: jest.fn() assignments that mention component names
|
|
145
|
+
for (const child of children) {
|
|
146
|
+
if (!mocked.includes(child.name)) {
|
|
147
|
+
const inlineMockRe = new RegExp(`${child.name}[\\s\\S]*?=.*?jest\\.fn|vi\\.fn`, 'i');
|
|
148
|
+
if (inlineMockRe.test(testContent)) {
|
|
149
|
+
mocked.push(child.name);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return [...new Set(mocked)].sort();
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Verify that generated test adequately covers the analyzed component.
|
|
157
|
+
*
|
|
158
|
+
* @param analysis - Component analysis from AST extraction
|
|
159
|
+
* @param generatedTestContent - The generated test file content
|
|
160
|
+
* @returns ValidationResult with coverage details and any rejections
|
|
161
|
+
*/
|
|
162
|
+
export function verifyCoverage(analysis, generatedTestContent) {
|
|
163
|
+
const rejections = [];
|
|
164
|
+
const warnings = [];
|
|
165
|
+
// Count branch coverage
|
|
166
|
+
const branchesIdentified = analysis.branches.length;
|
|
167
|
+
const branchesTested = countBranchTests(generatedTestContent, analysis.branches);
|
|
168
|
+
// Count effect coverage
|
|
169
|
+
const effectsIdentified = analysis.effects.length;
|
|
170
|
+
const effectsTested = countEffectTests(generatedTestContent, analysis.effects);
|
|
171
|
+
// Check cleanup requirement
|
|
172
|
+
const cleanupRequired = analysis.effects.some(e => e.hasCleanup);
|
|
173
|
+
const cleanupTested = cleanupRequired ? hasCleanupTest(generatedTestContent) : true;
|
|
174
|
+
// Check child mocking
|
|
175
|
+
const childrenIdentified = analysis.children.map(c => c.name).sort();
|
|
176
|
+
const childrenMocked = findMockedChildren(generatedTestContent, analysis.children);
|
|
177
|
+
// Build coverage report
|
|
178
|
+
const coverage = {
|
|
179
|
+
branchesIdentified,
|
|
180
|
+
branchesTested,
|
|
181
|
+
effectsIdentified,
|
|
182
|
+
effectsTested,
|
|
183
|
+
cleanupRequired,
|
|
184
|
+
cleanupTested,
|
|
185
|
+
childrenIdentified,
|
|
186
|
+
childrenMocked,
|
|
187
|
+
};
|
|
188
|
+
// Apply rejection rules
|
|
189
|
+
// Rule: All branches should be tested
|
|
190
|
+
if (branchesIdentified > 0 && branchesTested < branchesIdentified) {
|
|
191
|
+
// Only reject if significantly under-tested
|
|
192
|
+
const coverage = branchesTested / branchesIdentified;
|
|
193
|
+
if (coverage < 0.5) {
|
|
194
|
+
rejections.push('llm_reject_incomplete_branch_coverage');
|
|
195
|
+
}
|
|
196
|
+
else {
|
|
197
|
+
warnings.push(`Branch coverage: ${branchesTested}/${branchesIdentified} (${Math.round(coverage * 100)}%)`);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
// Rule: Cleanup must be tested if required
|
|
201
|
+
if (cleanupRequired && !cleanupTested) {
|
|
202
|
+
rejections.push('llm_reject_missing_cleanup_test');
|
|
203
|
+
}
|
|
204
|
+
// Rule: All child components should be mocked
|
|
205
|
+
const unmockedChildren = childrenIdentified.filter(c => !childrenMocked.includes(c));
|
|
206
|
+
if (unmockedChildren.length > 0) {
|
|
207
|
+
// Only reject if more than half are unmocked
|
|
208
|
+
if (unmockedChildren.length > childrenIdentified.length / 2) {
|
|
209
|
+
rejections.push('llm_reject_unmocked_child');
|
|
210
|
+
}
|
|
211
|
+
else {
|
|
212
|
+
warnings.push(`Unmocked children: ${unmockedChildren.join(', ')}`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
// Rule: Effects should be tested
|
|
216
|
+
if (effectsIdentified > 0 && effectsTested === 0) {
|
|
217
|
+
rejections.push('llm_reject_missing_effect_test');
|
|
218
|
+
}
|
|
219
|
+
return {
|
|
220
|
+
valid: rejections.length === 0,
|
|
221
|
+
rejections,
|
|
222
|
+
warnings,
|
|
223
|
+
coverage,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
//# sourceMappingURL=coverageVerifier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coverageVerifier.js","sourceRoot":"","sources":["../../../src/core/validation/coverageVerifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH;;;GAGG;AACH,SAAS,gBAAgB,CAAC,WAAmB;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,kEAAkE;IAClE,MAAM,WAAW,GAAG,kDAAkD,CAAC;IACvE,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,WAAmB;IAC/C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IAEjC,uCAAuC;IACvC,MAAM,MAAM,GAAG,+CAA+C,CAAC;IAC/D,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,IAAI;YAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,WAAmB,EAAE,QAAuC;IACpF,MAAM,SAAS,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,YAAY,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAE/C,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;QAEtD,gDAAgD;QAChD,MAAM,mBAAmB,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChD,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC7B,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAClD,CAAC;QAEF,2CAA2C;QAC3C,MAAM,gBAAgB,GACpB,YAAY,CAAC,QAAQ,CAAC,QAAQ,cAAc,EAAE,CAAC;YAC/C,YAAY,CAAC,QAAQ,CAAC,MAAM,cAAc,EAAE,CAAC;YAC7C,YAAY,CAAC,QAAQ,CAAC,GAAG,cAAc,UAAU,CAAC;YAClD,YAAY,CAAC,QAAQ,CAAC,GAAG,cAAc,WAAW,CAAC;YACnD,YAAY,CAAC,QAAQ,CAAC,sBAAsB,cAAc,EAAE,CAAC;YAC7D,YAAY,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;QAElD,0DAA0D;QAC1D,MAAM,aAAa,GACjB,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC;YACnC,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC;YAClC,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC;YAC1C,YAAY,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;QAEjD,IAAI,mBAAmB,IAAI,gBAAgB,IAAI,aAAa,EAAE,CAAC;YAC7D,WAAW,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,kFAAkF;IAClF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACjD,qEAAqE;QACrE,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,WAAmB,EAAE,OAAqC;IAClF,MAAM,YAAY,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAC/C,MAAM,SAAS,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAEhD,oCAAoC;IACpC,MAAM,aAAa,GACjB,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC;QAClC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC/B,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC;QACjC,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC;QAClC,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAEvC,mDAAmD;IACnD,MAAM,eAAe,GACnB,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC9B,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEhC,0EAA0E;IAC1E,MAAM,iBAAiB,GACrB,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,QAAQ,CAAC,eAAe,CAAC;QACtC,YAAY,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IAExC,sEAAsE;IACtE,MAAM,gBAAgB,GAAG,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAE1D,0EAA0E;IAC1E,IAAI,aAAa,IAAI,eAAe,IAAI,iBAAiB,EAAE,CAAC;QAC1D,OAAO,OAAO,CAAC,MAAM,CAAC;IACxB,CAAC;IAED,kEAAkE;IAClE,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,MAAM,CAAC;IACxB,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,WAAmB;IACzC,MAAM,YAAY,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAE/C,OAAO,CACL,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,QAAQ,CAAC,eAAe,CAAC;QACtC,YAAY,CAAC,QAAQ,CAAC,cAAc,CAAC;QACrC,YAAY,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QACvC,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC;QACpC,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,CACjC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,UAAkB;IAC/C,2CAA2C;IAC3C,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,UAAU,CAAC;IACzD,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,WAAmB,EACnB,QAAuC;IAEvC,MAAM,aAAa,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;IACxD,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,6CAA6C;QAC7C,MAAM,QAAQ,GACZ,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC;YACnC,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAC1B,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;gBACtB,qBAAqB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CACpE,CAAC;QAEJ,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,kFAAkF;IAClF,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,IAAI,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,iCAAiC,EAAE,GAAG,CAAC,CAAC;YACrF,IAAI,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACrC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,QAA2B,EAC3B,oBAA4B;IAE5B,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,wBAAwB;IACxB,MAAM,kBAAkB,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;IACpD,MAAM,cAAc,GAAG,gBAAgB,CAAC,oBAAoB,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEjF,wBAAwB;IACxB,MAAM,iBAAiB,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC;IAClD,MAAM,aAAa,GAAG,gBAAgB,CAAC,oBAAoB,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE/E,4BAA4B;IAC5B,MAAM,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG,eAAe,CAAC,CAAC,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpF,sBAAsB;IACtB,MAAM,kBAAkB,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IACrE,MAAM,cAAc,GAAG,kBAAkB,CAAC,oBAAoB,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEnF,wBAAwB;IACxB,MAAM,QAAQ,GAAmB;QAC/B,kBAAkB;QAClB,cAAc;QACd,iBAAiB;QACjB,aAAa;QACb,eAAe;QACf,aAAa;QACb,kBAAkB;QAClB,cAAc;KACf,CAAC;IAEF,wBAAwB;IAExB,sCAAsC;IACtC,IAAI,kBAAkB,GAAG,CAAC,IAAI,cAAc,GAAG,kBAAkB,EAAE,CAAC;QAClE,4CAA4C;QAC5C,MAAM,QAAQ,GAAG,cAAc,GAAG,kBAAkB,CAAC;QACrD,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YACnB,UAAU,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,oBAAoB,cAAc,IAAI,kBAAkB,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7G,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,IAAI,eAAe,IAAI,CAAC,aAAa,EAAE,CAAC;QACtC,UAAU,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,8CAA8C;IAC9C,MAAM,gBAAgB,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,6CAA6C;QAC7C,IAAI,gBAAgB,CAAC,MAAM,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5D,UAAU,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,IAAI,CAAC,sBAAsB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,IAAI,iBAAiB,GAAG,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QACjD,UAAU,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,OAAO;QACL,KAAK,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC;QAC9B,UAAU;QACV,QAAQ;QACR,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/validation/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types for post-generation validation.
|
|
3
|
+
* Verifies generated tests cover all identified behaviors.
|
|
4
|
+
*/
|
|
5
|
+
/** Rejection codes matching existing guardrail pattern (llm_reject_*). */
|
|
6
|
+
export type RejectionCode = 'llm_reject_incomplete_branch_coverage' | 'llm_reject_missing_cleanup_test' | 'llm_reject_unmocked_child' | 'llm_reject_missing_effect_test';
|
|
7
|
+
/** Coverage report showing what was analyzed vs tested. */
|
|
8
|
+
export type CoverageReport = {
|
|
9
|
+
branchesIdentified: number;
|
|
10
|
+
branchesTested: number;
|
|
11
|
+
effectsIdentified: number;
|
|
12
|
+
effectsTested: number;
|
|
13
|
+
cleanupRequired: boolean;
|
|
14
|
+
cleanupTested: boolean;
|
|
15
|
+
childrenIdentified: string[];
|
|
16
|
+
childrenMocked: string[];
|
|
17
|
+
};
|
|
18
|
+
/** Result of coverage verification. */
|
|
19
|
+
export type ValidationResult = {
|
|
20
|
+
valid: boolean;
|
|
21
|
+
rejections: RejectionCode[];
|
|
22
|
+
warnings: string[];
|
|
23
|
+
coverage: CoverageReport;
|
|
24
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/core/validation/types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}
|