task-o-matic 0.0.8 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -0
- package/dist/commands/benchmark.d.ts.map +1 -1
- package/dist/commands/benchmark.js +342 -0
- package/dist/commands/tasks/execute-loop.d.ts +3 -0
- package/dist/commands/tasks/execute-loop.d.ts.map +1 -0
- package/dist/commands/tasks/execute-loop.js +118 -0
- package/dist/commands/tasks/index.d.ts +1 -0
- package/dist/commands/tasks/index.d.ts.map +1 -1
- package/dist/commands/tasks/index.js +1 -0
- package/dist/commands/tasks.d.ts.map +1 -1
- package/dist/commands/tasks.js +1 -0
- package/dist/commands/workflow.js +39 -0
- package/dist/lib/benchmark/registry.d.ts.map +1 -1
- package/dist/lib/benchmark/registry.js +11 -0
- package/dist/lib/benchmark/types.d.ts +50 -0
- package/dist/lib/benchmark/types.d.ts.map +1 -1
- package/dist/lib/task-loop-execution.d.ts +25 -0
- package/dist/lib/task-loop-execution.d.ts.map +1 -0
- package/dist/lib/task-loop-execution.js +473 -0
- package/dist/services/prd.d.ts.map +1 -1
- package/dist/services/prd.js +36 -1
- package/dist/services/tasks.d.ts.map +1 -1
- package/dist/services/tasks.js +90 -3
- package/dist/services/workflow-benchmark.d.ts +34 -0
- package/dist/services/workflow-benchmark.d.ts.map +1 -0
- package/dist/services/workflow-benchmark.js +317 -0
- package/dist/services/workflow.d.ts.map +1 -1
- package/dist/services/workflow.js +120 -7
- package/dist/test/task-loop-git.test.d.ts +2 -0
- package/dist/test/task-loop-git.test.d.ts.map +1 -0
- package/dist/test/task-loop-git.test.js +62 -0
- package/dist/types/index.d.ts +50 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/results.d.ts +29 -1
- package/dist/types/results.d.ts.map +1 -1
- package/dist/types/workflow-results.d.ts +27 -0
- package/dist/types/workflow-results.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/services/tasks.js
CHANGED
|
@@ -328,7 +328,33 @@ class TaskService {
|
|
|
328
328
|
message: "Calling AI for enhancement...",
|
|
329
329
|
type: "progress",
|
|
330
330
|
});
|
|
331
|
-
|
|
331
|
+
// Capture metrics
|
|
332
|
+
let tokenUsage;
|
|
333
|
+
let timeToFirstToken;
|
|
334
|
+
const aiStartTime = Date.now();
|
|
335
|
+
// Wrap streaming options to capture metrics
|
|
336
|
+
const metricsStreamingOptions = {
|
|
337
|
+
...streamingOptions,
|
|
338
|
+
onFinish: async (result) => {
|
|
339
|
+
if (result.usage) {
|
|
340
|
+
tokenUsage = {
|
|
341
|
+
prompt: result.usage.inputTokens || result.usage.promptTokens || 0,
|
|
342
|
+
completion: result.usage.outputTokens || result.usage.completionTokens || 0,
|
|
343
|
+
total: result.usage.totalTokens || 0,
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
// Call original onFinish if provided
|
|
347
|
+
await streamingOptions?.onFinish?.(result);
|
|
348
|
+
},
|
|
349
|
+
onChunk: (chunk) => {
|
|
350
|
+
if (chunk && !timeToFirstToken) {
|
|
351
|
+
timeToFirstToken = Date.now() - aiStartTime;
|
|
352
|
+
}
|
|
353
|
+
// Call original onChunk if provided
|
|
354
|
+
streamingOptions?.onChunk?.(chunk);
|
|
355
|
+
},
|
|
356
|
+
};
|
|
357
|
+
const enhancedContent = await (0, ai_service_factory_1.getAIOperations)().enhanceTaskWithDocumentation(task.id, task.title, task.description ?? "", stackInfo, metricsStreamingOptions, undefined, enhancementAIConfig, context.existingResearch);
|
|
332
358
|
hooks_1.hooks.emit("task:progress", {
|
|
333
359
|
message: "Saving enhanced content...",
|
|
334
360
|
type: "progress",
|
|
@@ -369,6 +395,9 @@ class TaskService {
|
|
|
369
395
|
originalLength,
|
|
370
396
|
enhancedLength: enhancedContent.length,
|
|
371
397
|
duration,
|
|
398
|
+
tokenUsage,
|
|
399
|
+
timeToFirstToken,
|
|
400
|
+
cost: undefined, // Cost calculation can be added later
|
|
372
401
|
},
|
|
373
402
|
metadata: {
|
|
374
403
|
aiProvider: aiConfig.provider,
|
|
@@ -406,8 +435,34 @@ class TaskService {
|
|
|
406
435
|
message: "Calling AI to break down task...",
|
|
407
436
|
type: "progress",
|
|
408
437
|
});
|
|
438
|
+
// Capture metrics
|
|
439
|
+
let tokenUsage;
|
|
440
|
+
let timeToFirstToken;
|
|
441
|
+
const aiStartTime = Date.now();
|
|
442
|
+
// Wrap streaming options to capture metrics
|
|
443
|
+
const metricsStreamingOptions = {
|
|
444
|
+
...streamingOptions,
|
|
445
|
+
onFinish: async (result) => {
|
|
446
|
+
if (result.usage) {
|
|
447
|
+
tokenUsage = {
|
|
448
|
+
prompt: result.usage.inputTokens || result.usage.promptTokens || 0,
|
|
449
|
+
completion: result.usage.outputTokens || result.usage.completionTokens || 0,
|
|
450
|
+
total: result.usage.totalTokens || 0,
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
// Call original onFinish if provided
|
|
454
|
+
await streamingOptions?.onFinish?.(result);
|
|
455
|
+
},
|
|
456
|
+
onChunk: (chunk) => {
|
|
457
|
+
if (chunk && !timeToFirstToken) {
|
|
458
|
+
timeToFirstToken = Date.now() - aiStartTime;
|
|
459
|
+
}
|
|
460
|
+
// Call original onChunk if provided
|
|
461
|
+
streamingOptions?.onChunk?.(chunk);
|
|
462
|
+
},
|
|
463
|
+
};
|
|
409
464
|
// Use AI service to break down the task with enhanced context
|
|
410
|
-
const subtaskData = await (0, ai_service_factory_1.getAIOperations)().breakdownTask(task, breakdownAIConfig, promptOverride, messageOverride,
|
|
465
|
+
const subtaskData = await (0, ai_service_factory_1.getAIOperations)().breakdownTask(task, breakdownAIConfig, promptOverride, messageOverride, metricsStreamingOptions, undefined, fullContent, stackInfo, existingSubtasks, enableFilesystemTools);
|
|
411
466
|
hooks_1.hooks.emit("task:progress", {
|
|
412
467
|
message: `Creating ${subtaskData.length} subtasks...`,
|
|
413
468
|
type: "progress",
|
|
@@ -453,6 +508,9 @@ class TaskService {
|
|
|
453
508
|
stats: {
|
|
454
509
|
subtasksCreated: createdSubtasks.length,
|
|
455
510
|
duration,
|
|
511
|
+
tokenUsage,
|
|
512
|
+
timeToFirstToken,
|
|
513
|
+
cost: undefined, // Cost calculation can be added later
|
|
456
514
|
},
|
|
457
515
|
metadata: {
|
|
458
516
|
aiProvider: aiConfig.provider,
|
|
@@ -604,7 +662,33 @@ class TaskService {
|
|
|
604
662
|
message: "Calling AI to create plan...",
|
|
605
663
|
type: "progress",
|
|
606
664
|
});
|
|
607
|
-
|
|
665
|
+
// Capture metrics
|
|
666
|
+
let tokenUsage;
|
|
667
|
+
let timeToFirstToken;
|
|
668
|
+
const aiStartTime = Date.now();
|
|
669
|
+
// Wrap streaming options to capture metrics
|
|
670
|
+
const metricsStreamingOptions = {
|
|
671
|
+
...streamingOptions,
|
|
672
|
+
onFinish: async (result) => {
|
|
673
|
+
if (result.usage) {
|
|
674
|
+
tokenUsage = {
|
|
675
|
+
prompt: result.usage.inputTokens || result.usage.promptTokens || 0,
|
|
676
|
+
completion: result.usage.outputTokens || result.usage.completionTokens || 0,
|
|
677
|
+
total: result.usage.totalTokens || 0,
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
// Call original onFinish if provided
|
|
681
|
+
await streamingOptions?.onFinish?.(result);
|
|
682
|
+
},
|
|
683
|
+
onChunk: (chunk) => {
|
|
684
|
+
if (chunk && !timeToFirstToken) {
|
|
685
|
+
timeToFirstToken = Date.now() - aiStartTime;
|
|
686
|
+
}
|
|
687
|
+
// Call original onChunk if provided
|
|
688
|
+
streamingOptions?.onChunk?.(chunk);
|
|
689
|
+
},
|
|
690
|
+
};
|
|
691
|
+
const plan = await aiService.planTask(taskContext, taskDetails, planAIConfig, undefined, undefined, metricsStreamingOptions);
|
|
608
692
|
hooks_1.hooks.emit("task:progress", {
|
|
609
693
|
message: "Saving plan...",
|
|
610
694
|
type: "progress",
|
|
@@ -623,6 +707,9 @@ class TaskService {
|
|
|
623
707
|
plan,
|
|
624
708
|
stats: {
|
|
625
709
|
duration,
|
|
710
|
+
tokenUsage,
|
|
711
|
+
timeToFirstToken,
|
|
712
|
+
cost: undefined, // Cost calculation can be added later
|
|
626
713
|
},
|
|
627
714
|
metadata: {
|
|
628
715
|
aiProvider: aiConfig.provider,
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { AIOptions } from "../utils/ai-config-builder";
|
|
2
|
+
import { StreamingOptions } from "../types";
|
|
3
|
+
import { WorkflowBenchmarkInput, WorkflowBenchmarkResult } from "../lib/benchmark/types";
|
|
4
|
+
/**
|
|
5
|
+
* WorkflowBenchmarkService - Executes complete workflows for benchmarking
|
|
6
|
+
* Creates isolated environments for each model to ensure fair comparison
|
|
7
|
+
*/
|
|
8
|
+
export declare class WorkflowBenchmarkService {
|
|
9
|
+
/**
|
|
10
|
+
* Execute a complete workflow for benchmarking purposes
|
|
11
|
+
*/
|
|
12
|
+
executeWorkflow(input: WorkflowBenchmarkInput, aiOptions: AIOptions, streamingOptions?: StreamingOptions): Promise<WorkflowBenchmarkResult["output"]>;
|
|
13
|
+
/**
|
|
14
|
+
* Create a temporary project directory for benchmarking
|
|
15
|
+
*/
|
|
16
|
+
private createTempProjectDir;
|
|
17
|
+
/**
|
|
18
|
+
* Clean up temporary project directory
|
|
19
|
+
*/
|
|
20
|
+
private cleanupTempProjectDir;
|
|
21
|
+
/**
|
|
22
|
+
* Apply the results from a selected benchmark to the actual project
|
|
23
|
+
*/
|
|
24
|
+
applyBenchmarkResult(selectedResult: WorkflowBenchmarkResult, targetProjectDir: string, originalResponses: WorkflowBenchmarkInput["collectedResponses"]): Promise<{
|
|
25
|
+
success: boolean;
|
|
26
|
+
message: string;
|
|
27
|
+
}>;
|
|
28
|
+
/**
|
|
29
|
+
* Validate workflow benchmark input
|
|
30
|
+
*/
|
|
31
|
+
validateInput(input: any): input is WorkflowBenchmarkInput;
|
|
32
|
+
}
|
|
33
|
+
export declare const workflowBenchmarkService: WorkflowBenchmarkService;
|
|
34
|
+
//# sourceMappingURL=workflow-benchmark.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workflow-benchmark.d.ts","sourceRoot":"","sources":["../../src/services/workflow-benchmark.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAQ,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAGzF;;;GAGG;AACH,qBAAa,wBAAwB;IACnC;;OAEG;IACG,eAAe,CACnB,KAAK,EAAE,sBAAsB,EAC7B,SAAS,EAAE,SAAS,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,GAClC,OAAO,CAAC,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAiL7C;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAW7B;;OAEG;IACG,oBAAoB,CACxB,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,sBAAsB,CAAC,oBAAoB,CAAC,GAC9D,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAyFjD;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,GAAG,GAAG,KAAK,IAAI,sBAAsB;CAU3D;AAED,eAAO,MAAM,wBAAwB,0BAAiC,CAAC"}
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.workflowBenchmarkService = exports.WorkflowBenchmarkService = void 0;
|
|
37
|
+
const path_1 = require("path");
|
|
38
|
+
const fs_1 = require("fs");
|
|
39
|
+
const workflow_1 = require("./workflow");
|
|
40
|
+
/**
|
|
41
|
+
* WorkflowBenchmarkService - Executes complete workflows for benchmarking
|
|
42
|
+
* Creates isolated environments for each model to ensure fair comparison
|
|
43
|
+
*/
|
|
44
|
+
class WorkflowBenchmarkService {
|
|
45
|
+
/**
|
|
46
|
+
* Execute a complete workflow for benchmarking purposes
|
|
47
|
+
*/
|
|
48
|
+
async executeWorkflow(input, aiOptions, streamingOptions) {
|
|
49
|
+
const startTime = Date.now();
|
|
50
|
+
// Create temporary project directory for this benchmark run
|
|
51
|
+
const tempProjectDir = this.createTempProjectDir(input.tempDirBase || "/tmp", input.collectedResponses.projectName, aiOptions.aiProvider || "unknown", aiOptions.aiModel || "unknown");
|
|
52
|
+
const stats = {
|
|
53
|
+
initDuration: 0,
|
|
54
|
+
prdGenerationDuration: 0,
|
|
55
|
+
prdRefinementDuration: 0,
|
|
56
|
+
taskGenerationDuration: 0,
|
|
57
|
+
taskSplittingDuration: 0,
|
|
58
|
+
totalTasks: 0,
|
|
59
|
+
tasksWithSubtasks: 0,
|
|
60
|
+
avgTaskComplexity: 0,
|
|
61
|
+
prdSize: 0,
|
|
62
|
+
totalSteps: 0,
|
|
63
|
+
successfulSteps: 0,
|
|
64
|
+
};
|
|
65
|
+
let projectDir = tempProjectDir;
|
|
66
|
+
let prdFile = "";
|
|
67
|
+
let prdContent = "";
|
|
68
|
+
let tasks = [];
|
|
69
|
+
try {
|
|
70
|
+
// Step 1: Initialize Project
|
|
71
|
+
const stepStart = Date.now();
|
|
72
|
+
stats.totalSteps++;
|
|
73
|
+
const initResult = await workflow_1.workflowService.initializeProject({
|
|
74
|
+
projectName: input.collectedResponses.projectName,
|
|
75
|
+
projectDir: tempProjectDir,
|
|
76
|
+
initMethod: input.collectedResponses.initMethod,
|
|
77
|
+
projectDescription: input.collectedResponses.projectDescription,
|
|
78
|
+
aiOptions,
|
|
79
|
+
stackConfig: input.collectedResponses.stackConfig,
|
|
80
|
+
bootstrap: true, // Always bootstrap for benchmark unless explicitly disabled
|
|
81
|
+
streamingOptions,
|
|
82
|
+
callbacks: {
|
|
83
|
+
onProgress: () => { }, // Silent for benchmarking
|
|
84
|
+
onError: () => { },
|
|
85
|
+
},
|
|
86
|
+
});
|
|
87
|
+
stats.initDuration = Date.now() - stepStart;
|
|
88
|
+
stats.successfulSteps++;
|
|
89
|
+
projectDir = initResult.projectDir;
|
|
90
|
+
// Step 2: Define PRD
|
|
91
|
+
if (input.collectedResponses.prdMethod !== "skip") {
|
|
92
|
+
const prdStart = Date.now();
|
|
93
|
+
stats.totalSteps++;
|
|
94
|
+
const prdResult = await workflow_1.workflowService.definePRD({
|
|
95
|
+
method: input.collectedResponses.prdMethod,
|
|
96
|
+
prdFile: input.collectedResponses.prdFile,
|
|
97
|
+
prdDescription: input.collectedResponses.prdDescription,
|
|
98
|
+
prdContent: input.collectedResponses.prdContent,
|
|
99
|
+
projectDir,
|
|
100
|
+
aiOptions,
|
|
101
|
+
streamingOptions,
|
|
102
|
+
callbacks: {
|
|
103
|
+
onProgress: () => { },
|
|
104
|
+
onError: () => { },
|
|
105
|
+
},
|
|
106
|
+
});
|
|
107
|
+
stats.prdGenerationDuration = Date.now() - prdStart;
|
|
108
|
+
stats.successfulSteps++;
|
|
109
|
+
prdFile = prdResult.prdFile;
|
|
110
|
+
prdContent = prdResult.prdContent;
|
|
111
|
+
stats.prdSize = prdContent.length;
|
|
112
|
+
// Step 3: Refine PRD (if requested)
|
|
113
|
+
if (input.collectedResponses.refinePrd && input.collectedResponses.refineFeedback) {
|
|
114
|
+
const refineStart = Date.now();
|
|
115
|
+
stats.totalSteps++;
|
|
116
|
+
const refineResult = await workflow_1.workflowService.refinePRD({
|
|
117
|
+
method: "ai",
|
|
118
|
+
prdFile,
|
|
119
|
+
feedback: input.collectedResponses.refineFeedback,
|
|
120
|
+
projectDir,
|
|
121
|
+
aiOptions,
|
|
122
|
+
streamingOptions,
|
|
123
|
+
callbacks: {
|
|
124
|
+
onProgress: () => { },
|
|
125
|
+
onError: () => { },
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
stats.prdRefinementDuration = Date.now() - refineStart;
|
|
129
|
+
stats.successfulSteps++;
|
|
130
|
+
prdContent = refineResult.prdContent;
|
|
131
|
+
stats.prdSize = prdContent.length;
|
|
132
|
+
}
|
|
133
|
+
// Step 4: Generate Tasks
|
|
134
|
+
if (input.collectedResponses.generateTasks !== false && prdFile) {
|
|
135
|
+
const tasksStart = Date.now();
|
|
136
|
+
stats.totalSteps++;
|
|
137
|
+
const tasksResult = await workflow_1.workflowService.generateTasks({
|
|
138
|
+
prdFile,
|
|
139
|
+
method: input.collectedResponses.customInstructions ? "ai" : "standard",
|
|
140
|
+
customInstructions: input.collectedResponses.customInstructions,
|
|
141
|
+
projectDir,
|
|
142
|
+
aiOptions,
|
|
143
|
+
streamingOptions,
|
|
144
|
+
callbacks: {
|
|
145
|
+
onProgress: () => { },
|
|
146
|
+
onError: () => { },
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
stats.taskGenerationDuration = Date.now() - tasksStart;
|
|
150
|
+
stats.successfulSteps++;
|
|
151
|
+
tasks = tasksResult.tasks;
|
|
152
|
+
stats.totalTasks = tasks.length;
|
|
153
|
+
// Step 5: Split Tasks (if requested)
|
|
154
|
+
if (input.collectedResponses.splitTasks && tasks.length > 0) {
|
|
155
|
+
const splitStart = Date.now();
|
|
156
|
+
stats.totalSteps++;
|
|
157
|
+
const tasksToSplit = input.collectedResponses.tasksToSplit?.length
|
|
158
|
+
? input.collectedResponses.tasksToSplit
|
|
159
|
+
: tasks.slice(0, Math.min(3, tasks.length)).map(t => t.id); // Split first 3 tasks by default
|
|
160
|
+
const splitResult = await workflow_1.workflowService.splitTasks({
|
|
161
|
+
taskIds: tasksToSplit,
|
|
162
|
+
splitMethod: input.collectedResponses.splitInstructions ? "custom" : "standard",
|
|
163
|
+
customInstructions: input.collectedResponses.splitInstructions,
|
|
164
|
+
aiOptions,
|
|
165
|
+
streamingOptions,
|
|
166
|
+
callbacks: {
|
|
167
|
+
onProgress: () => { },
|
|
168
|
+
onError: () => { },
|
|
169
|
+
},
|
|
170
|
+
});
|
|
171
|
+
stats.taskSplittingDuration = Date.now() - splitStart;
|
|
172
|
+
stats.successfulSteps++;
|
|
173
|
+
stats.tasksWithSubtasks = splitResult.results.filter(r => !r.error && r.subtasks.length > 0).length;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
// Calculate complexity metrics
|
|
178
|
+
if (tasks.length > 0) {
|
|
179
|
+
const totalComplexity = tasks.reduce((sum, task) => {
|
|
180
|
+
const contentLength = (task.description || "").length;
|
|
181
|
+
return sum + (contentLength > 200 ? 3 : contentLength > 100 ? 2 : 1);
|
|
182
|
+
}, 0);
|
|
183
|
+
stats.avgTaskComplexity = totalComplexity / tasks.length;
|
|
184
|
+
}
|
|
185
|
+
return {
|
|
186
|
+
projectDir,
|
|
187
|
+
prdFile,
|
|
188
|
+
prdContent,
|
|
189
|
+
tasks,
|
|
190
|
+
stats,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
finally {
|
|
194
|
+
// Clean up temporary directory
|
|
195
|
+
this.cleanupTempProjectDir(tempProjectDir);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Create a temporary project directory for benchmarking
|
|
200
|
+
*/
|
|
201
|
+
createTempProjectDir(tempBase, projectName, provider, model) {
|
|
202
|
+
const sanitizedModel = model.replace(/[^a-zA-Z0-9-]/g, "-");
|
|
203
|
+
const dirName = `benchmark-${projectName}-${provider}-${sanitizedModel}-${Date.now()}`;
|
|
204
|
+
const tempDir = (0, path_1.join)(tempBase, "task-o-matic-benchmark", dirName);
|
|
205
|
+
if (!(0, fs_1.existsSync)(tempDir)) {
|
|
206
|
+
(0, fs_1.mkdirSync)(tempDir, { recursive: true });
|
|
207
|
+
}
|
|
208
|
+
return tempDir;
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Clean up temporary project directory
|
|
212
|
+
*/
|
|
213
|
+
cleanupTempProjectDir(projectDir) {
|
|
214
|
+
try {
|
|
215
|
+
if ((0, fs_1.existsSync)(projectDir) && projectDir.includes("task-o-matic-benchmark")) {
|
|
216
|
+
(0, fs_1.rmSync)(projectDir, { recursive: true, force: true });
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
catch (error) {
|
|
220
|
+
// Ignore cleanup errors
|
|
221
|
+
console.warn(`Warning: Could not clean up temp directory ${projectDir}`);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Apply the results from a selected benchmark to the actual project
|
|
226
|
+
*/
|
|
227
|
+
async applyBenchmarkResult(selectedResult, targetProjectDir, originalResponses) {
|
|
228
|
+
try {
|
|
229
|
+
const { workflowService } = await Promise.resolve().then(() => __importStar(require("./workflow")));
|
|
230
|
+
const { writeFileSync, existsSync, mkdirSync } = await Promise.resolve().then(() => __importStar(require("fs")));
|
|
231
|
+
const { join } = await Promise.resolve().then(() => __importStar(require("path")));
|
|
232
|
+
// Ensure target directory exists
|
|
233
|
+
if (!existsSync(targetProjectDir)) {
|
|
234
|
+
mkdirSync(targetProjectDir, { recursive: true });
|
|
235
|
+
}
|
|
236
|
+
// Extract model configuration from modelId (e.g., "openai:gpt-4o")
|
|
237
|
+
const [provider, model] = selectedResult.modelId.split(":").slice(0, 2);
|
|
238
|
+
// Step 1: Initialize actual project with selected model
|
|
239
|
+
const actualResult = await workflowService.initializeProject({
|
|
240
|
+
projectName: originalResponses.projectName,
|
|
241
|
+
projectDir: targetProjectDir,
|
|
242
|
+
initMethod: originalResponses.initMethod,
|
|
243
|
+
projectDescription: originalResponses.projectDescription,
|
|
244
|
+
aiOptions: {
|
|
245
|
+
aiProvider: provider,
|
|
246
|
+
aiModel: model,
|
|
247
|
+
},
|
|
248
|
+
stackConfig: originalResponses.stackConfig,
|
|
249
|
+
bootstrap: true,
|
|
250
|
+
streamingOptions: {},
|
|
251
|
+
callbacks: {
|
|
252
|
+
onProgress: (msg) => {
|
|
253
|
+
const message = typeof msg === 'string' ? msg :
|
|
254
|
+
'message' in msg ? msg.message :
|
|
255
|
+
'text' in msg ? msg.text : 'Progress update';
|
|
256
|
+
console.log(` ${message}`);
|
|
257
|
+
},
|
|
258
|
+
onError: (err) => console.error(` Error: ${err.message || err}`),
|
|
259
|
+
},
|
|
260
|
+
});
|
|
261
|
+
// Step 2: Copy PRD content if available
|
|
262
|
+
if (selectedResult.output.prdContent && selectedResult.output.prdFile) {
|
|
263
|
+
const taskOMaticDir = join(targetProjectDir, ".task-o-matic", "prd");
|
|
264
|
+
if (!existsSync(taskOMaticDir)) {
|
|
265
|
+
mkdirSync(taskOMaticDir, { recursive: true });
|
|
266
|
+
}
|
|
267
|
+
const targetPrdFile = join(taskOMaticDir, "prd.md");
|
|
268
|
+
writeFileSync(targetPrdFile, selectedResult.output.prdContent);
|
|
269
|
+
console.log(` ✓ PRD copied to ${targetPrdFile}`);
|
|
270
|
+
}
|
|
271
|
+
// Step 3: Import tasks if available
|
|
272
|
+
if (selectedResult.output.tasks && selectedResult.output.tasks.length > 0) {
|
|
273
|
+
const { getStorage } = await Promise.resolve().then(() => __importStar(require("../utils/ai-service-factory")));
|
|
274
|
+
// Switch to target directory context
|
|
275
|
+
process.chdir(targetProjectDir);
|
|
276
|
+
for (const task of selectedResult.output.tasks) {
|
|
277
|
+
try {
|
|
278
|
+
await getStorage().createTask({
|
|
279
|
+
title: task.title,
|
|
280
|
+
description: task.description || "",
|
|
281
|
+
content: task.description || "",
|
|
282
|
+
parentId: task.parentId,
|
|
283
|
+
estimatedEffort: task.estimatedEffort,
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
console.warn(` Warning: Could not import task "${task.title}"`);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
console.log(` ✓ Imported ${selectedResult.output.tasks.length} tasks`);
|
|
291
|
+
}
|
|
292
|
+
return {
|
|
293
|
+
success: true,
|
|
294
|
+
message: `Successfully applied results from ${selectedResult.modelId} to ${targetProjectDir}`,
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
catch (error) {
|
|
298
|
+
return {
|
|
299
|
+
success: false,
|
|
300
|
+
message: `Failed to apply benchmark results: ${error instanceof Error ? error.message : String(error)}`,
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* Validate workflow benchmark input
|
|
306
|
+
*/
|
|
307
|
+
validateInput(input) {
|
|
308
|
+
return (input &&
|
|
309
|
+
input.collectedResponses &&
|
|
310
|
+
typeof input.collectedResponses.projectName === "string" &&
|
|
311
|
+
input.collectedResponses.projectName.length > 0 &&
|
|
312
|
+
typeof input.collectedResponses.initMethod === "string" &&
|
|
313
|
+
["quick", "custom", "ai"].includes(input.collectedResponses.initMethod));
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
exports.WorkflowBenchmarkService = WorkflowBenchmarkService;
|
|
317
|
+
exports.workflowBenchmarkService = new WorkflowBenchmarkService();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../../src/services/workflow.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAQ,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,mBAAmB,EACnB,gBAAgB,EACjB,MAAM,2BAA2B,CAAC;AAEnC;;;GAGG;AACH,qBAAa,eAAe;IAC1B;;;OAGG;IACG,iBAAiB,CAAC,KAAK,EAAE;QAC7B,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,OAAO,GAAG,QAAQ,GAAG,IAAI,CAAC;QACvC,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,SAAS,CAAC,EAAE,SAAS,CAAC;QACtB,WAAW,CAAC,EAAE;YACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,IAAI,CAAC,EAAE,OAAO,CAAC;SAChB,CAAC;QACF,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;QACpC,SAAS,CAAC,EAAE,gBAAgB,CAAC;KAC9B,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkN7B;;;OAGG;IACG,SAAS,CAAC,KAAK,EAAE;QACrB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,IAAI,GAAG,MAAM,CAAC;QAC5C,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,SAAS,CAAC;QACtB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;QACpC,SAAS,CAAC,EAAE,gBAAgB,CAAC;KAC9B,GAAG,OAAO,CAAC,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../../src/services/workflow.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAQ,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,mBAAmB,EACnB,gBAAgB,EACjB,MAAM,2BAA2B,CAAC;AAEnC;;;GAGG;AACH,qBAAa,eAAe;IAC1B;;;OAGG;IACG,iBAAiB,CAAC,KAAK,EAAE;QAC7B,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,OAAO,GAAG,QAAQ,GAAG,IAAI,CAAC;QACvC,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,SAAS,CAAC,EAAE,SAAS,CAAC;QACtB,WAAW,CAAC,EAAE;YACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,IAAI,CAAC,EAAE,OAAO,CAAC;SAChB,CAAC;QACF,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;QACpC,SAAS,CAAC,EAAE,gBAAgB,CAAC;KAC9B,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkN7B;;;OAGG;IACG,SAAS,CAAC,KAAK,EAAE;QACrB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,IAAI,GAAG,MAAM,CAAC;QAC5C,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,SAAS,CAAC;QACtB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;QACpC,SAAS,CAAC,EAAE,gBAAgB,CAAC;KAC9B,GAAG,OAAO,CAAC,eAAe,CAAC;IA2G5B;;;OAGG;IACG,SAAS,CAAC,KAAK,EAAE;QACrB,MAAM,EAAE,QAAQ,GAAG,IAAI,GAAG,MAAM,CAAC;QACjC,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,SAAS,CAAC;QACtB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;QACpC,SAAS,CAAC,EAAE,gBAAgB,CAAC;KAC9B,GAAG,OAAO,CAAC,eAAe,CAAC;IA4F5B;;;OAGG;IACG,aAAa,CAAC,KAAK,EAAE;QACzB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;QAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,SAAS,CAAC;QACtB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;QACpC,SAAS,CAAC,EAAE,gBAAgB,CAAC;KAC9B,GAAG,OAAO,CAAC,mBAAmB,CAAC;IA+DhC;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE;QACtB,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,WAAW,EAAE,aAAa,GAAG,UAAU,GAAG,QAAQ,CAAC;QACnD,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,SAAS,CAAC,EAAE,SAAS,CAAC;QACtB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;QACpC,SAAS,CAAC,EAAE,gBAAgB,CAAC;KAC9B,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAkD9B;AAGD,eAAO,MAAM,eAAe,iBAAwB,CAAC"}
|