wiggum-cli 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +6 -4
  2. package/dist/ai/agents/codebase-analyst.d.ts +3 -0
  3. package/dist/ai/agents/codebase-analyst.d.ts.map +1 -1
  4. package/dist/ai/agents/codebase-analyst.js +3 -0
  5. package/dist/ai/agents/codebase-analyst.js.map +1 -1
  6. package/dist/ai/agents/context-enricher.d.ts +11 -0
  7. package/dist/ai/agents/context-enricher.d.ts.map +1 -0
  8. package/dist/ai/agents/context-enricher.js +163 -0
  9. package/dist/ai/agents/context-enricher.js.map +1 -0
  10. package/dist/ai/agents/evaluator-optimizer.d.ts +13 -0
  11. package/dist/ai/agents/evaluator-optimizer.d.ts.map +1 -0
  12. package/dist/ai/agents/evaluator-optimizer.js +231 -0
  13. package/dist/ai/agents/evaluator-optimizer.js.map +1 -0
  14. package/dist/ai/agents/index.d.ts +21 -3
  15. package/dist/ai/agents/index.d.ts.map +1 -1
  16. package/dist/ai/agents/index.js +151 -82
  17. package/dist/ai/agents/index.js.map +1 -1
  18. package/dist/ai/agents/mcp-detector.d.ts +26 -0
  19. package/dist/ai/agents/mcp-detector.d.ts.map +1 -0
  20. package/dist/ai/agents/mcp-detector.js +186 -0
  21. package/dist/ai/agents/mcp-detector.js.map +1 -0
  22. package/dist/ai/agents/orchestrator.d.ts +3 -0
  23. package/dist/ai/agents/orchestrator.d.ts.map +1 -1
  24. package/dist/ai/agents/orchestrator.js +3 -0
  25. package/dist/ai/agents/orchestrator.js.map +1 -1
  26. package/dist/ai/agents/planning-orchestrator.d.ts +12 -0
  27. package/dist/ai/agents/planning-orchestrator.d.ts.map +1 -0
  28. package/dist/ai/agents/planning-orchestrator.js +133 -0
  29. package/dist/ai/agents/planning-orchestrator.js.map +1 -0
  30. package/dist/ai/agents/stack-researcher.d.ts +3 -0
  31. package/dist/ai/agents/stack-researcher.d.ts.map +1 -1
  32. package/dist/ai/agents/stack-researcher.js +3 -0
  33. package/dist/ai/agents/stack-researcher.js.map +1 -1
  34. package/dist/ai/agents/stack-utils.d.ts +11 -0
  35. package/dist/ai/agents/stack-utils.d.ts.map +1 -0
  36. package/dist/ai/agents/stack-utils.js +27 -0
  37. package/dist/ai/agents/stack-utils.js.map +1 -0
  38. package/dist/ai/agents/synthesis-agent.d.ts +11 -0
  39. package/dist/ai/agents/synthesis-agent.d.ts.map +1 -0
  40. package/dist/ai/agents/synthesis-agent.js +202 -0
  41. package/dist/ai/agents/synthesis-agent.js.map +1 -0
  42. package/dist/ai/agents/tech-researcher.d.ts +16 -0
  43. package/dist/ai/agents/tech-researcher.d.ts.map +1 -0
  44. package/dist/ai/agents/tech-researcher.js +208 -0
  45. package/dist/ai/agents/tech-researcher.js.map +1 -0
  46. package/dist/ai/agents/types.d.ts +121 -0
  47. package/dist/ai/agents/types.d.ts.map +1 -1
  48. package/dist/ai/agents/types.js +6 -0
  49. package/dist/ai/agents/types.js.map +1 -1
  50. package/dist/ai/index.d.ts +1 -1
  51. package/dist/ai/index.d.ts.map +1 -1
  52. package/dist/ai/index.js +14 -2
  53. package/dist/ai/index.js.map +1 -1
  54. package/dist/commands/init.d.ts.map +1 -1
  55. package/dist/commands/init.js +9 -2
  56. package/dist/commands/init.js.map +1 -1
  57. package/dist/utils/tracing.d.ts +5 -0
  58. package/dist/utils/tracing.d.ts.map +1 -1
  59. package/dist/utils/tracing.js +40 -1
  60. package/dist/utils/tracing.js.map +1 -1
  61. package/package.json +5 -2
  62. package/src/ai/agents/codebase-analyst.ts +3 -0
  63. package/src/ai/agents/context-enricher.ts +189 -0
  64. package/src/ai/agents/evaluator-optimizer.ts +277 -0
  65. package/src/ai/agents/index.ts +197 -104
  66. package/src/ai/agents/mcp-detector.test.ts +290 -0
  67. package/src/ai/agents/mcp-detector.ts +210 -0
  68. package/src/ai/agents/orchestrator.ts +3 -0
  69. package/src/ai/agents/planning-orchestrator.ts +140 -0
  70. package/src/ai/agents/stack-researcher.ts +3 -0
  71. package/src/ai/agents/stack-utils.ts +34 -0
  72. package/src/ai/agents/synthesis-agent.ts +240 -0
  73. package/src/ai/agents/tech-researcher.ts +262 -0
  74. package/src/ai/agents/types.ts +153 -0
  75. package/src/ai/index.ts +26 -5
  76. package/src/commands/init.ts +10 -2
  77. package/src/utils/tracing.ts +44 -1
  78. package/tsconfig.json +1 -1
  79. package/vitest.config.ts +7 -0
@@ -1 +1 @@
1
- {"version":3,"file":"tracing.js","sourceRoot":"","sources":["../../src/utils/tracing.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAEzB,6BAA6B;AAC7B,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7D;;GAEG;AACH,IAAI,iBAAiB,GAAG,KAAK,CAAC;AAE9B,MAAM,UAAU,WAAW;IACzB,IAAI,iBAAiB;QAAE,OAAO;IAE9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,sCAAsC;QACtC,OAAO;IACT,CAAC;IAED,IAAI,CAAC;QACH,UAAU,CAAC;YACT,MAAM;YACN,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,YAAY;SACjE,CAAC,CAAC;QACH,iBAAiB,GAAG,IAAI,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,gDAAgD;IAClD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC9B,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;AAC1C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW;IACzB,WAAW,EAAE,CAAC;IAEd,IAAI,gBAAgB,EAAE,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC,EAAE,CAAC,CAAC;IACvB,CAAC;IAED,0DAA0D;IAC1D,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,EAAK,EACL,UAA4C,EAAE;IAE9C,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAC7C,OAAO,UAAU,CAAC,EAAE,EAAE;QACpB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,UAAU;QAChC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,IAAI,WAAW;KAC7C,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"tracing.js","sourceRoot":"","sources":["../../src/utils/tracing.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,IAAI,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7E,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAEzB,6BAA6B;AAC7B,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7D;;GAEG;AACH,IAAI,iBAAiB,GAAG,KAAK,CAAC;AAC9B,IAAI,sBAAsB,GAAG,KAAK,CAAC;AAEnC,MAAM,UAAU,WAAW;IACzB,IAAI,iBAAiB;QAAE,OAAO;IAE9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,sCAAsC;QACtC,OAAO;IACT,CAAC;IAED,IAAI,CAAC;QACH,UAAU,CAAC;YACT,MAAM;YACN,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,YAAY;SACjE,CAAC,CAAC;QACH,iBAAiB,GAAG,IAAI,CAAC;QAEzB,6DAA6D;QAC7D,oBAAoB,EAAE,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,gDAAgD;IAClD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB;IAC3B,IAAI,sBAAsB;QAAE,OAAO;IACnC,sBAAsB,GAAG,IAAI,CAAC;IAE9B,uBAAuB;IACvB,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE;QAClC,MAAM,YAAY,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,2BAA2B;IAC3B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;QAC9B,MAAM,YAAY,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,mBAAmB;IACnB,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;QAC/B,MAAM,YAAY,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,IAAI,CAAC,iBAAiB;QAAE,OAAO;IAE/B,IAAI,CAAC;QACH,MAAM,eAAe,EAAE,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC9B,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;AAC1C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW;IACzB,WAAW,EAAE,CAAC;IAEd,IAAI,gBAAgB,EAAE,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC,EAAE,CAAC,CAAC;IACvB,CAAC;IAED,0DAA0D;IAC1D,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,EAAK,EACL,UAA4C,EAAE;IAE9C,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAC7C,OAAO,UAAU,CAAC,EAAE,EAAE;QACpB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,UAAU;QAChC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,IAAI,WAAW;KAC7C,CAAC,CAAC;AACL,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "wiggum-cli",
3
- "version": "0.3.2",
3
+ "version": "0.4.0",
4
4
  "description": "AI-powered feature development loop CLI",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -12,6 +12,8 @@
12
12
  "copy-templates": "cp -r src/templates dist/",
13
13
  "dev": "tsc --watch",
14
14
  "start": "node bin/ralph.js",
15
+ "test": "vitest run",
16
+ "test:watch": "vitest",
15
17
  "prepublishOnly": "npm run build"
16
18
  },
17
19
  "keywords": [
@@ -39,7 +41,8 @@
39
41
  },
40
42
  "devDependencies": {
41
43
  "@types/node": "^20.10.0",
42
- "typescript": "^5.3.0"
44
+ "typescript": "^5.3.0",
45
+ "vitest": "^4.0.17"
43
46
  },
44
47
  "engines": {
45
48
  "node": ">=18.0.0"
@@ -1,6 +1,9 @@
1
1
  /**
2
2
  * Codebase Analyst Agent
3
3
  * Explores the codebase to understand its structure and patterns
4
+ *
5
+ * @deprecated Use runContextEnricher from context-enricher.ts instead.
6
+ * This agent is kept for backward compatibility.
4
7
  */
5
8
 
6
9
  import { stepCountIs, type LanguageModel } from 'ai';
@@ -0,0 +1,189 @@
1
+ /**
2
+ * Context Enricher Worker (Phase 2)
3
+ * Explores the codebase to gather enriched context based on the analysis plan
4
+ */
5
+
6
+ import { stepCountIs, type LanguageModel } from 'ai';
7
+ import type { ContextEnricherInput, EnrichedContext } from './types.js';
8
+ import { createExplorationTools } from '../tools.js';
9
+ import { isReasoningModel } from '../providers.js';
10
+ import { logger } from '../../utils/logger.js';
11
+ import { parseJsonSafe } from '../../utils/json-repair.js';
12
+ import { getTracedAI } from '../../utils/tracing.js';
13
+ import { detectProjectType } from './stack-utils.js';
14
+
15
+ /**
16
+ * System prompt for the Context Enricher worker
17
+ */
18
+ const CONTEXT_ENRICHER_SYSTEM_PROMPT = `You are a Context Enricher worker. Your job is to explore specific areas of a codebase and answer specific questions.
19
+
20
+ ## Your Mission
21
+ Based on the analysis plan, explore the codebase to:
22
+ 1. Identify entry points and key files
23
+ 2. Understand directory structure and purposes
24
+ 3. Detect naming conventions
25
+ 4. Find available commands (from package.json)
26
+ 5. Answer the specific questions provided
27
+
28
+ ## Tools Available
29
+ - searchCode: Search using ripgrep patterns
30
+ - readFile: Read file contents
31
+ - listDirectory: List directory structure
32
+ - getPackageInfo: Get package.json info
33
+
34
+ ## Exploration Strategy
35
+ 1. List the areas specified in the plan
36
+ 2. Read package.json to understand scripts and dependencies
37
+ 3. Search for patterns to answer the specific questions
38
+ 4. Identify the project type based on structure
39
+
40
+ ## Project Types
41
+ - MCP Server: Has @modelcontextprotocol dependencies
42
+ - REST API: Express/Fastify/Hono with route handlers
43
+ - React SPA: React with components, no server-side rendering
44
+ - Next.js App: Next.js with app or pages directory
45
+ - CLI Tool: Has bin entry in package.json
46
+ - Library: Published package without app entry
47
+
48
+ ## Output Format
49
+ After exploration, output ONLY valid JSON:
50
+ {
51
+ "entryPoints": ["src/index.ts"],
52
+ "keyDirectories": {"src/routes": "API routes", "src/components": "UI components"},
53
+ "namingConventions": "camelCase files, PascalCase components",
54
+ "commands": {"test": "npm test", "build": "npm run build"},
55
+ "answeredQuestions": {"What is the auth strategy?": "NextAuth with JWT"},
56
+ "projectType": "Next.js App"
57
+ }`;
58
+
59
+ /**
60
+ * Run the Context Enricher worker
61
+ */
62
+ export async function runContextEnricher(
63
+ model: LanguageModel,
64
+ modelId: string,
65
+ input: ContextEnricherInput,
66
+ verbose: boolean = false
67
+ ): Promise<EnrichedContext> {
68
+ const tools = createExplorationTools(input.scanResult.projectRoot);
69
+
70
+ const prompt = `Explore this codebase and gather enriched context.
71
+
72
+ Project: ${input.scanResult.projectRoot}
73
+
74
+ ## Areas to Explore
75
+ ${input.areasToExplore.map(a => `- ${a}`).join('\n')}
76
+
77
+ ## Questions to Answer
78
+ ${input.questionsToAnswer.map(q => `- ${q}`).join('\n')}
79
+
80
+ Start by exploring the specified areas, then answer the questions and produce your analysis as JSON.`;
81
+
82
+ try {
83
+ const { generateText } = getTracedAI();
84
+
85
+ const result = await generateText({
86
+ model,
87
+ system: CONTEXT_ENRICHER_SYSTEM_PROMPT,
88
+ prompt,
89
+ tools,
90
+ stopWhen: stepCountIs(5),
91
+ maxOutputTokens: 3000,
92
+ ...(isReasoningModel(modelId) ? {} : { temperature: 0.3 }),
93
+ experimental_telemetry: {
94
+ isEnabled: true,
95
+ metadata: {
96
+ agent: 'context-enricher',
97
+ projectRoot: input.scanResult.projectRoot,
98
+ areasCount: input.areasToExplore.length,
99
+ questionsCount: input.questionsToAnswer.length,
100
+ },
101
+ },
102
+ });
103
+
104
+ // Parse the response
105
+ const context = parseEnrichedContext(result.text, result.steps, verbose);
106
+
107
+ if (verbose) {
108
+ logger.info(`Context Enricher: Found ${context.entryPoints.length} entry points, answered ${Object.keys(context.answeredQuestions).length} questions`);
109
+ }
110
+
111
+ return context;
112
+ } catch (error) {
113
+ if (verbose) {
114
+ logger.error(`Context Enricher error: ${error instanceof Error ? error.message : String(error)}`);
115
+ }
116
+
117
+ return getDefaultEnrichedContext(input);
118
+ }
119
+ }
120
+
121
+ /**
122
+ * Parse the enriched context from agent response
123
+ */
124
+ function parseEnrichedContext(
125
+ text: string,
126
+ steps: Array<{ text?: string }> | undefined,
127
+ verbose: boolean
128
+ ): EnrichedContext {
129
+ // Try to get text from the result or steps
130
+ let textToParse = text;
131
+
132
+ if (!textToParse || textToParse.trim() === '') {
133
+ const stepsList = steps || [];
134
+ for (let i = stepsList.length - 1; i >= 0; i--) {
135
+ const step = stepsList[i];
136
+ if (step.text && step.text.trim() !== '') {
137
+ textToParse = step.text;
138
+ break;
139
+ }
140
+ }
141
+ }
142
+
143
+ if (!textToParse || textToParse.trim() === '') {
144
+ if (verbose) {
145
+ logger.warn('Context Enricher: No text output found');
146
+ }
147
+ return getDefaultEnrichedContext();
148
+ }
149
+
150
+ // Use safe JSON parser with repair capabilities
151
+ const parsed = parseJsonSafe<Partial<EnrichedContext>>(textToParse);
152
+
153
+ if (!parsed) {
154
+ if (verbose) {
155
+ logger.warn('Context Enricher: Failed to parse JSON response');
156
+ }
157
+ return getDefaultEnrichedContext();
158
+ }
159
+
160
+ // Build result with defaults for missing fields
161
+ return {
162
+ entryPoints: parsed.entryPoints || ['src/index.ts'],
163
+ keyDirectories: parsed.keyDirectories || { src: 'Source code' },
164
+ namingConventions: parsed.namingConventions || 'camelCase',
165
+ commands: parsed.commands || { build: 'npm run build' },
166
+ answeredQuestions: parsed.answeredQuestions || {},
167
+ projectType: parsed.projectType || 'Unknown',
168
+ };
169
+ }
170
+
171
+ /**
172
+ * Get default enriched context when parsing fails
173
+ */
174
+ function getDefaultEnrichedContext(input?: ContextEnricherInput): EnrichedContext {
175
+ const projectType = detectProjectType(input?.scanResult.stack);
176
+
177
+ return {
178
+ entryPoints: ['src/index.ts'],
179
+ keyDirectories: { src: 'Source code' },
180
+ namingConventions: 'camelCase',
181
+ commands: {
182
+ test: 'npm test',
183
+ build: 'npm run build',
184
+ dev: 'npm run dev',
185
+ },
186
+ answeredQuestions: {},
187
+ projectType,
188
+ };
189
+ }
@@ -0,0 +1,277 @@
1
+ /**
2
+ * Evaluator-Optimizer Agent (Phase 4)
3
+ * QA loop that validates and improves the analysis result
4
+ * Max 2 iterations to ensure quality without endless loops
5
+ */
6
+
7
+ import { type LanguageModel } from 'ai';
8
+ import { z } from 'zod';
9
+ import type { ScanResult } from '../../scanner/types.js';
10
+ import type { MultiAgentAnalysis, EvaluationResult } from './types.js';
11
+ import { isReasoningModel } from '../providers.js';
12
+ import { logger } from '../../utils/logger.js';
13
+ import { getTracedAI } from '../../utils/tracing.js';
14
+
15
+ /**
16
+ * Quality threshold for passing evaluation (1-10 scale)
17
+ * Results with score >= this value skip optimization
18
+ */
19
+ const QUALITY_THRESHOLD = 7;
20
+
21
+ /**
22
+ * Schema for evaluation output
23
+ */
24
+ const evaluationSchema = z.object({
25
+ qualityScore: z.number().min(1).max(10).describe('Overall quality score from 1-10'),
26
+ hasEntryPoints: z.boolean().describe('Whether entry points are identified'),
27
+ hasImplementationGuidelines: z.boolean().describe('Whether implementation guidelines are provided'),
28
+ hasRelevantMcpServers: z.boolean().describe('Whether relevant MCP servers are recommended'),
29
+ specificIssues: z.array(z.string()).describe('Specific issues found in the analysis'),
30
+ improvementSuggestions: z.array(z.string()).describe('Suggestions for improving the analysis'),
31
+ });
32
+
33
+ /**
34
+ * Schema for optimizer output
35
+ */
36
+ const optimizerOutputSchema = z.object({
37
+ improvedGuidelines: z.array(z.string()).describe('Improved implementation guidelines'),
38
+ additionalEntryPoints: z.array(z.string()).optional().describe('Additional entry points to add'),
39
+ additionalMcpServers: z.array(z.string()).optional().describe('Additional MCP servers to recommend'),
40
+ });
41
+
42
+ /**
43
+ * System prompt for the Evaluator
44
+ */
45
+ const EVALUATOR_SYSTEM_PROMPT = `You are a QA Evaluator for AI-generated codebase analysis.
46
+
47
+ ## Your Mission
48
+ Evaluate the analysis result for:
49
+ 1. Completeness - Are all important areas covered?
50
+ 2. Accuracy - Do the recommendations match the detected stack?
51
+ 3. Actionability - Are the guidelines specific and useful?
52
+ 4. MCP relevance - Are the right MCP servers recommended?
53
+
54
+ ## Scoring Guidelines
55
+ - 9-10: Excellent, comprehensive, highly actionable
56
+ - 7-8: Good, covers main areas, useful guidelines
57
+ - 5-6: Adequate but missing some important details
58
+ - 3-4: Incomplete, vague, or partially incorrect
59
+ - 1-2: Poor, missing critical information
60
+
61
+ ## Quality Checks
62
+ - Entry points should identify actual files, not just "src/"
63
+ - Guidelines should be specific (e.g., "Run npm test" not "Test your code")
64
+ - MCP servers should match the detected database and deployment
65
+
66
+ Be constructive but honest. If it's good, say so. If it needs work, explain why.`;
67
+
68
+ /**
69
+ * System prompt for the Optimizer
70
+ */
71
+ const OPTIMIZER_SYSTEM_PROMPT = `You are an Optimizer that improves AI-generated codebase analysis based on evaluation feedback.
72
+
73
+ ## Your Mission
74
+ Based on the evaluation feedback, improve:
75
+ 1. Implementation guidelines - make them more specific and actionable
76
+ 2. Entry points - add any obvious ones that were missed
77
+ 3. MCP servers - add any that would be useful
78
+
79
+ ## Guidelines for Improvement
80
+ - Keep guidelines to 5-10 words
81
+ - Start with action verbs
82
+ - Be specific to the detected stack
83
+ - Don't remove good content, only add or improve`;
84
+
85
+ /**
86
+ * Run the Evaluator-Optimizer QA loop
87
+ */
88
+ export async function runEvaluatorOptimizer(
89
+ model: LanguageModel,
90
+ modelId: string,
91
+ result: MultiAgentAnalysis,
92
+ scanResult: ScanResult,
93
+ maxIterations: number = 2,
94
+ verbose: boolean = false
95
+ ): Promise<MultiAgentAnalysis> {
96
+ let currentResult = result;
97
+ let iterations = 0;
98
+
99
+ while (iterations < maxIterations) {
100
+ // Evaluate current result
101
+ const evaluation = await evaluateResult(model, modelId, currentResult, scanResult, verbose);
102
+
103
+ if (verbose) {
104
+ logger.info(`Evaluator (iteration ${iterations + 1}): Score ${evaluation.qualityScore}/10`);
105
+ if (evaluation.specificIssues.length > 0) {
106
+ logger.info(`Issues: ${evaluation.specificIssues.join(', ')}`);
107
+ }
108
+ }
109
+
110
+ // Check if quality meets threshold
111
+ if (
112
+ evaluation.qualityScore >= QUALITY_THRESHOLD &&
113
+ evaluation.hasEntryPoints &&
114
+ evaluation.hasImplementationGuidelines
115
+ ) {
116
+ if (verbose) {
117
+ logger.info('Evaluator: Quality threshold met, skipping optimization');
118
+ }
119
+ break;
120
+ }
121
+
122
+ // Optimize based on feedback
123
+ if (verbose) {
124
+ logger.info('Evaluator: Running optimizer to improve result');
125
+ }
126
+
127
+ currentResult = await optimizeResult(model, modelId, currentResult, evaluation, verbose);
128
+ iterations++;
129
+ }
130
+
131
+ return currentResult;
132
+ }
133
+
134
+ /**
135
+ * Evaluate the analysis result
136
+ */
137
+ async function evaluateResult(
138
+ model: LanguageModel,
139
+ modelId: string,
140
+ result: MultiAgentAnalysis,
141
+ scanResult: ScanResult,
142
+ verbose: boolean
143
+ ): Promise<EvaluationResult> {
144
+ const prompt = `Evaluate this codebase analysis:
145
+
146
+ ## Analysis Result
147
+ Project Type: ${result.codebaseAnalysis.projectContext.projectType}
148
+ Entry Points: ${result.codebaseAnalysis.projectContext.entryPoints.join(', ') || 'None'}
149
+ Key Directories: ${Object.keys(result.codebaseAnalysis.projectContext.keyDirectories).join(', ') || 'None'}
150
+ Guidelines: ${result.codebaseAnalysis.implementationGuidelines.length} items
151
+ - ${result.codebaseAnalysis.implementationGuidelines.slice(0, 3).join('\n- ')}
152
+ MCP Essential: ${result.mcpServers.essential.join(', ')}
153
+ MCP Recommended: ${result.mcpServers.recommended.join(', ') || 'None'}
154
+
155
+ ## Original Project Context
156
+ Framework: ${scanResult.stack.framework?.name || 'Unknown'}
157
+ Database: ${scanResult.stack.database?.name || 'None detected'}
158
+ Testing: ${scanResult.stack.testing?.unit?.name || 'None detected'}
159
+
160
+ Evaluate the quality and completeness of this analysis.`;
161
+
162
+ try {
163
+ const { generateObject } = getTracedAI();
164
+
165
+ const { object: evaluation } = await generateObject({
166
+ model,
167
+ schema: evaluationSchema,
168
+ system: EVALUATOR_SYSTEM_PROMPT,
169
+ prompt,
170
+ ...(isReasoningModel(modelId) ? {} : { temperature: 0.2 }),
171
+ experimental_telemetry: {
172
+ isEnabled: true,
173
+ metadata: {
174
+ agent: 'evaluator',
175
+ projectType: result.codebaseAnalysis.projectContext.projectType,
176
+ },
177
+ },
178
+ });
179
+
180
+ return evaluation;
181
+ } catch (error) {
182
+ if (verbose) {
183
+ logger.error(`Evaluator error: ${error instanceof Error ? error.message : String(error)}`);
184
+ }
185
+
186
+ // Return a passing evaluation on error to avoid blocking
187
+ return {
188
+ qualityScore: 7,
189
+ hasEntryPoints: result.codebaseAnalysis.projectContext.entryPoints.length > 0,
190
+ hasImplementationGuidelines: result.codebaseAnalysis.implementationGuidelines.length > 0,
191
+ hasRelevantMcpServers: result.mcpServers.essential.length > 0,
192
+ specificIssues: [],
193
+ improvementSuggestions: [],
194
+ };
195
+ }
196
+ }
197
+
198
+ /**
199
+ * Optimize the result based on evaluation feedback
200
+ */
201
+ async function optimizeResult(
202
+ model: LanguageModel,
203
+ modelId: string,
204
+ result: MultiAgentAnalysis,
205
+ evaluation: EvaluationResult,
206
+ verbose: boolean
207
+ ): Promise<MultiAgentAnalysis> {
208
+ const prompt = `Improve this codebase analysis based on the evaluation feedback.
209
+
210
+ ## Current Analysis
211
+ Project Type: ${result.codebaseAnalysis.projectContext.projectType}
212
+ Entry Points: ${result.codebaseAnalysis.projectContext.entryPoints.join(', ')}
213
+ Current Guidelines:
214
+ ${result.codebaseAnalysis.implementationGuidelines.map(g => `- ${g}`).join('\n')}
215
+
216
+ ## Evaluation Feedback
217
+ Score: ${evaluation.qualityScore}/10
218
+ Issues: ${evaluation.specificIssues.join(', ') || 'None'}
219
+ Suggestions: ${evaluation.improvementSuggestions.join(', ') || 'None'}
220
+
221
+ Provide improved guidelines and any additional entry points or MCP servers.`;
222
+
223
+ try {
224
+ const { generateObject } = getTracedAI();
225
+
226
+ const { object: improvements } = await generateObject({
227
+ model,
228
+ schema: optimizerOutputSchema,
229
+ system: OPTIMIZER_SYSTEM_PROMPT,
230
+ prompt,
231
+ ...(isReasoningModel(modelId) ? {} : { temperature: 0.3 }),
232
+ experimental_telemetry: {
233
+ isEnabled: true,
234
+ metadata: {
235
+ agent: 'optimizer',
236
+ previousScore: evaluation.qualityScore,
237
+ },
238
+ },
239
+ });
240
+
241
+ // Apply improvements to the result
242
+ const improved: MultiAgentAnalysis = {
243
+ ...result,
244
+ codebaseAnalysis: {
245
+ ...result.codebaseAnalysis,
246
+ implementationGuidelines: improvements.improvedGuidelines.length > 0
247
+ ? improvements.improvedGuidelines
248
+ : result.codebaseAnalysis.implementationGuidelines,
249
+ projectContext: {
250
+ ...result.codebaseAnalysis.projectContext,
251
+ entryPoints: improvements.additionalEntryPoints
252
+ ? [...new Set([...result.codebaseAnalysis.projectContext.entryPoints, ...improvements.additionalEntryPoints])]
253
+ : result.codebaseAnalysis.projectContext.entryPoints,
254
+ },
255
+ },
256
+ mcpServers: {
257
+ ...result.mcpServers,
258
+ recommended: improvements.additionalMcpServers
259
+ ? [...new Set([...result.mcpServers.recommended, ...improvements.additionalMcpServers])]
260
+ : result.mcpServers.recommended,
261
+ },
262
+ };
263
+
264
+ if (verbose) {
265
+ logger.info(`Optimizer: Updated ${improvements.improvedGuidelines.length} guidelines`);
266
+ }
267
+
268
+ return improved;
269
+ } catch (error) {
270
+ if (verbose) {
271
+ logger.error(`Optimizer error: ${error instanceof Error ? error.message : String(error)}`);
272
+ }
273
+
274
+ // Return original result on error
275
+ return result;
276
+ }
277
+ }