task-o-matic-core 0.1.4 → 0.1.5-beta.2

package/dist/index.d.ts

@@ -15,10 +15,8 @@ export * from "./lib/task-loop-execution";
 export * from "./services/tasks";
 export * from "./services/workflow";
 export * from "./services/prd";
-export * from "./services/benchmark";
 export * from "./services/project-analysis";
-export * from "./lib/benchmark/registry";
-export * from "./lib/benchmark/types";
+export * from "./lib/benchmark";
 export * from "./utils/ai-service-factory";
 export * from "./utils/task-o-matic-error";
 export * from "./utils/stack-detector";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,0BAA0B,CAAC;AACzC,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,kCAAkC,CAAC;AACjD,cAAc,iCAAiC,CAAC;AAChD,cAAc,2CAA2C,CAAC;AAC1D,cAAc,kCAAkC,CAAC;AACjD,cAAc,sBAAsB,CAAC;AACrC,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAE1C,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,0BAA0B,CAAC;AACzC,cAAc,uBAAuB,CAAC;AAEtC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,4BAA4B,CAAC;AAC3C,cAAc,wBAAwB,CAAC;AAEvC,cAAc,aAAa,CAAC;AAO5B,cAAc,+BAA+B,CAAC;AAC9C,cAAc,sBAAsB,CAAC;AAErC,cAAc,WAAW,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,0BAA0B,CAAC;AACzC,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,kCAAkC,CAAC;AACjD,cAAc,iCAAiC,CAAC;AAChD,cAAc,2CAA2C,CAAC;AAC1D,cAAc,kCAAkC,CAAC;AACjD,cAAc,sBAAsB,CAAC;AACrC,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAE1C,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,iBAAiB,CAAC;AAEhC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,4BAA4B,CAAC;AAC3C,cAAc,wBAAwB,CAAC;AAEvC,cAAc,aAAa,CAAC;AAO5B,cAAc,+BAA+B,CAAC;AAC9C,cAAc,sBAAsB,CAAC;AAErC,cAAc,WAAW,CAAC"}

package/dist/index.js

@@ -31,10 +31,8 @@ __exportStar(require("./lib/task-loop-execution"), exports);
 __exportStar(require("./services/tasks"), exports);
 __exportStar(require("./services/workflow"), exports);
 __exportStar(require("./services/prd"), exports);
-__exportStar(require("./services/benchmark"), exports);
 __exportStar(require("./services/project-analysis"), exports);
-__exportStar(require("./lib/benchmark/registry"), exports);
-__exportStar(require("./lib/benchmark/types"), exports);
+__exportStar(require("./lib/benchmark"), exports);
 __exportStar(require("./utils/ai-service-factory"), exports);
 __exportStar(require("./utils/task-o-matic-error"), exports);
 __exportStar(require("./utils/stack-detector"), exports);

package/dist/lib/ai-service/prd-operations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prd-operations.d.ts","sourceRoot":"","sources":["../../../src/lib/ai-service/prd-operations.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,QAAQ,EAER,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EAIX,SAAS,EACV,MAAM,aAAa,CAAC;AAerB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAMnD,qBAAa,aAAc,SAAQ,cAAc;IACzC,QAAQ,CACZ,UAAU,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,gBAAgB,CAAC,EAAE,MAAM,EACzB,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC,gBAAgB,CAAC;IA6MtB,SAAS,CACb,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,gBAAgB,CAAC,EAAE,MAAM,EACzB,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC,MAAM,CAAC;IA+GZ,oBAAoB,CACxB,UAAU,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,gBAAgB,CAAC,EAAE,MAAM,EACzB,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC,MAAM,EAAE,CAAC;IA6Hd,kBAAkB,CACtB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EAAE,EACnB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,WAAW,CAAC,EAAE;QACZ,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,EACD,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACjC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IA8F5B,WAAW,CACf,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACjC,OAAO,CAAC,MAAM,CAAC;IAqBZ,WAAW,CACf,IAAI,EAAE,MAAM,EAAE,EACd,mBAAmB,EAAE,MAAM,EAC3B,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACjC,OAAO,CAAC,MAAM,CAAC;IA6BlB;;OAEG;IACG,YAAY,CAChB,UAAU,EAAE,MAAM,EAClB,WAAW,CAAC,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,gBAAgB,CAAC,EAAE,MAAM,EACzB,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAiJpD;;;OAGG;IACG,uBAAuB,CAC3B,eAAe,EAAE;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;KACvB,EACD,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC,MAAM,CAAC;IA0FlB;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAqB5B"}
+{"version":3,"file":"prd-operations.d.ts","sourceRoot":"","sources":["../../../src/lib/ai-service/prd-operations.ts"],"names":[],"mappings":"AACA,OAAO,EACL,QAAQ,EAER,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EAIX,SAAS,EACV,MAAM,aAAa,CAAC;AAYrB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,qBAAa,aAAc,SAAQ,cAAc;IACzC,QAAQ,CACZ,UAAU,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,gBAAgB,CAAC,EAAE,MAAM,EACzB,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC,gBAAgB,CAAC;IA6MtB,SAAS,CACb,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,gBAAgB,CAAC,EAAE,MAAM,EACzB,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC,MAAM,CAAC;IA+GZ,oBAAoB,CACxB,UAAU,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,gBAAgB,CAAC,EAAE,MAAM,EACzB,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC,MAAM,EAAE,CAAC;IA2Hd,kBAAkB,CACtB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EAAE,EACnB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,WAAW,CAAC,EAAE;QACZ,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,EACD,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACjC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IA8F5B,WAAW,CACf,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACjC,OAAO,CAAC,MAAM,CAAC;IAqBZ,WAAW,CACf,IAAI,EAAE,MAAM,EAAE,EACd,mBAAmB,EAAE,MAAM,EAC3B,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACjC,OAAO,CAAC,MAAM,CAAC;IA6BlB;;OAEG;IACG,YAAY,CAChB,UAAU,EAAE,MAAM,EAClB,WAAW,CAAC,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,gBAAgB,CAAC,EAAE,MAAM,EACzB,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAiJpD;;;OAGG;IACG,uBAAuB,CAC3B,eAAe,EAAE;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;KACvB,EACD,MAAM,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,EAC1B,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,WAAW,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,EAClC,qBAAqB,CAAC,EAAE,OAAO,GAC9B,OAAO,CAAC,MAAM,CAAC;IA0FlB;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAqB5B"}

package/dist/lib/ai-service/prd-operations.js

@@ -1,37 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PRDOperations = void 0;
 const ai_1 = require("ai");
@@ -40,6 +7,7 @@ const prompts_1 = require("../../prompts");
 const filesystem_tools_1 = require("./filesystem-tools");
 const base_operations_1 = require("./base-operations");
 const task_o_matic_error_1 = require("../../utils/task-o-matic-error");
+const prompts_2 = require("../../prompts");
 class PRDOperations extends base_operations_1.BaseOperations {
     async parsePRD(prdContent, config, promptOverride, userMessage, streamingOptions, retryConfig, workingDirectory, enableFilesystemTools) {
         // console.log(
@@ -321,7 +289,6 @@ Use these tools to understand the current project structure, existing code patte
                 }
                 prompt = promptResult.prompt;
             }
-            const { PRD_QUESTION_SYSTEM_PROMPT } = await Promise.resolve().then(() => __importStar(require("../../prompts")));
             let response;
             if (enableFilesystemTools) {
                 const model = this.modelProvider.getModel({
@@ -332,7 +299,7 @@ Use these tools to understand the current project structure, existing code patte
                 const result = await (0, ai_1.streamText)({
                     model,
                     tools: allTools,
-                    system: PRD_QUESTION_SYSTEM_PROMPT +
+                    system: prompts_2.PRD_QUESTION_SYSTEM_PROMPT +
                         `\n\nYou have access to filesystem tools to check existing code/structure if needed.`,
                     messages: [{ role: "user", content: userMessage || prompt }],
                     maxRetries: 0,
@@ -360,7 +327,7 @@ Use these tools to understand the current project structure, existing code patte
                 response = await result.text;
             }
             else {
-                response = await this.streamText("", config, PRD_QUESTION_SYSTEM_PROMPT, userMessage || prompt, streamingOptions, { maxAttempts: 1 });
+                response = await this.streamText("", config, prompts_2.PRD_QUESTION_SYSTEM_PROMPT, userMessage || prompt, streamingOptions, { maxAttempts: 1 });
             }
             const parseResult = this.jsonParser.parseJSONFromResponse(response);
             if (!parseResult.success) {

package/dist/lib/benchmark/executor.d.ts

@@ -0,0 +1,93 @@
+/**
+ * BenchmarkExecutor - Per-worktree execution logic for benchmarks
+ *
+ * This class handles executing benchmark operations in isolated worktrees.
+ * It supports operations, single task execution, task loops, and full workflows.
+ * Each execution captures timing, token metrics, and code changes.
+ */
+import type { Worktree } from "./worktree-manager";
+import { MetricsCollector } from "./metrics-collector";
+import type { BenchmarkModelConfig, BenchmarkModelResult, OperationBenchmarkInput, ExecutionBenchmarkInput, ExecuteLoopBenchmarkInput, WorkflowBenchmarkInput, BenchmarkableOperation } from "./types";
+/**
+ * BenchmarkExecutor runs benchmark operations in isolated worktrees
+ * and collects comprehensive metrics for comparison.
+ */
+export declare class BenchmarkExecutor {
+    private metricsCollector;
+    private operationRegistry;
+    constructor(metricsCollector?: MetricsCollector, operationRegistry?: Map<string, BenchmarkableOperation>);
+    /**
+     * Register an operation for benchmarking
+     */
+    registerOperation(operation: BenchmarkableOperation): void;
+    /**
+     * Get a registered operation by ID
+     */
+    getOperation(id: string): BenchmarkableOperation | undefined;
+    /**
+     * List all registered operations
+     */
+    listOperations(): BenchmarkableOperation[];
+    /**
+     * Execute a registered operation in a worktree
+     *
+     * @param worktree - The worktree to execute in
+     * @param model - Model configuration to use
+     * @param input - Operation input parameters
+     * @param baseCommit - Base commit for metrics comparison
+     * @returns Benchmark result with metrics
+     */
+    executeOperation(worktree: Worktree, model: BenchmarkModelConfig, input: OperationBenchmarkInput, baseCommit: string): Promise<BenchmarkModelResult>;
+    /**
+     * Execute a single task in a worktree
+     *
+     * @param worktree - The worktree to execute in
+     * @param model - Model configuration to use
+     * @param input - Task execution input
+     * @param baseCommit - Base commit for metrics comparison
+     * @returns Benchmark result with metrics
+     */
+    executeTask(worktree: Worktree, model: BenchmarkModelConfig, input: ExecutionBenchmarkInput, baseCommit: string): Promise<BenchmarkModelResult>;
+    /**
+     * Execute a task loop in a worktree
+     *
+     * @param worktree - The worktree to execute in
+     * @param model - Model configuration to use
+     * @param input - Execute loop input
+     * @param baseCommit - Base commit for metrics comparison
+     * @returns Benchmark result with metrics
+     */
+    executeLoop(worktree: Worktree, model: BenchmarkModelConfig, input: ExecuteLoopBenchmarkInput, baseCommit: string): Promise<BenchmarkModelResult>;
+    /**
+     * Execute a full workflow in a worktree
+     *
+     * For workflow benchmarks, we execute a series of steps:
+     * 1. Initialize project (if needed)
+     * 2. Parse PRD to generate tasks
+     * 3. Execute tasks with the given model
+     *
+     * @param worktree - The worktree to execute in
+     * @param model - Model configuration to use
+     * @param input - Workflow input
+     * @param baseCommit - Base commit for metrics comparison
+     * @returns Benchmark result with metrics
+     */
+    executeWorkflow(worktree: Worktree, model: BenchmarkModelConfig, input: WorkflowBenchmarkInput, baseCommit: string): Promise<BenchmarkModelResult>;
+    /**
+     * Build AI options for a model configuration
+     */
+    private buildModelAIOptions;
+    /**
+     * Collect all metrics for a completed execution
+     */
+    private collectMetrics;
+    /**
+     * Create a success result
+     */
+    private createSuccessResult;
+    /**
+     * Create an error result
+     */
+    private createErrorResult;
+}
+//# sourceMappingURL=executor.d.ts.map

package/dist/lib/benchmark/executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/lib/benchmark/executor.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EACV,oBAAoB,EACpB,oBAAoB,EAEpB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,sBAAsB,EAGtB,sBAAsB,EACvB,MAAM,SAAS,CAAC;AAqBjB;;;GAGG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,iBAAiB,CAAsC;gBAG7D,gBAAgB,CAAC,EAAE,gBAAgB,EACnC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,sBAAsB,CAAC;IAMzD;;OAEG;IACH,iBAAiB,CAAC,SAAS,EAAE,sBAAsB,GAAG,IAAI;IAI1D;;OAEG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,sBAAsB,GAAG,SAAS;IAI5D;;OAEG;IACH,cAAc,IAAI,sBAAsB,EAAE;IAI1C;;;;;;;;OAQG;IACG,gBAAgB,CACpB,QAAQ,EAAE,QAAQ,EAClB,KAAK,EAAE,oBAAoB,EAC3B,KAAK,EAAE,uBAAuB,EAC9B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;IAuFhC;;;;;;;;OAQG;IACG,WAAW,CACf,QAAQ,EAAE,QAAQ,EAClB,KAAK,EAAE,oBAAoB,EAC3B,KAAK,EAAE,uBAAuB,EAC9B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;IA0EhC;;;;;;;;OAQG;IACG,WAAW,CACf,QAAQ,EAAE,QAAQ,EAClB,KAAK,EAAE,oBAAoB,EAC3B,KAAK,EAAE,yBAAyB,EAChC,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;IAmEhC;;;;;;;;;;;;;OAaG;IACG,eAAe,CACnB,QAAQ,EAAE,QAAQ,EAClB,KAAK,EAAE,oBAAoB,EAC3B,KAAK,EAAE,sBAAsB,EAC7B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;IA8GhC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAsB3B;;OAEG;YACW,cAAc;IAkC5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAkB3B;;OAEG;IACH,OAAO,CAAC,iBAAiB;CA6B1B"}

package/dist/lib/benchmark/executor.js

@@ -0,0 +1,395 @@
+"use strict";
+/**
+ * BenchmarkExecutor - Per-worktree execution logic for benchmarks
+ *
+ * This class handles executing benchmark operations in isolated worktrees.
+ * It supports operations, single task execution, task loops, and full workflows.
+ * Each execution captures timing, token metrics, and code changes.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BenchmarkExecutor = void 0;
+const metrics_collector_1 = require("./metrics-collector");
+const logger_1 = require("../logger");
+const config_1 = require("../config");
+const task_execution_core_1 = require("../task-execution-core");
+const task_loop_execution_1 = require("../task-loop-execution");
+const workflow_1 = require("../../services/workflow");
+const prd_1 = require("../../services/prd");
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const task_o_matic_error_1 = require("../../utils/task-o-matic-error");
+/**
+ * BenchmarkExecutor runs benchmark operations in isolated worktrees
+ * and collects comprehensive metrics for comparison.
+ */
+class BenchmarkExecutor {
+    metricsCollector;
+    operationRegistry;
+    constructor(metricsCollector, operationRegistry) {
+        this.metricsCollector = metricsCollector ?? new metrics_collector_1.MetricsCollector();
+        this.operationRegistry = operationRegistry ?? new Map();
+    }
+    /**
+     * Register an operation for benchmarking
+     */
+    registerOperation(operation) {
+        this.operationRegistry.set(operation.id, operation);
+    }
+    /**
+     * Get a registered operation by ID
+     */
+    getOperation(id) {
+        return this.operationRegistry.get(id);
+    }
+    /**
+     * List all registered operations
+     */
+    listOperations() {
+        return Array.from(this.operationRegistry.values());
+    }
+    /**
+     * Execute a registered operation in a worktree
+     *
+     * @param worktree - The worktree to execute in
+     * @param model - Model configuration to use
+     * @param input - Operation input parameters
+     * @param baseCommit - Base commit for metrics comparison
+     * @returns Benchmark result with metrics
+     */
+    async executeOperation(worktree, model, input, baseCommit) {
+        const modelId = `${model.provider}:${model.model}`;
+        const startedAt = Date.now();
+        let timeToFirstOutput;
+        logger_1.logger.info(`Executing operation ${input.operationId} with ${modelId}`);
+        try {
+            // Get the operation from registry
+            const operation = this.operationRegistry.get(input.operationId);
+            if (!operation) {
+                throw (0, task_o_matic_error_1.createStandardError)(task_o_matic_error_1.TaskOMaticErrorCodes.INVALID_INPUT, `Operation not found: ${input.operationId}`, {
+                    context: `The requested operation "${input.operationId}" is not registered.`,
+                    suggestions: ["Check the operation ID", "List available operations"],
+                });
+            }
+            // Validate input
+            if (!operation.validateInput(input.params)) {
+                throw (0, task_o_matic_error_1.createStandardError)(task_o_matic_error_1.TaskOMaticErrorCodes.INVALID_INPUT, `Invalid input for operation ${input.operationId}`, {
+                    context: `The provided parameters do not match the requirements for "${input.operationId}".`,
+                });
+            }
+            // Setup working directory to the worktree
+            await (0, config_1.setupWorkingDirectory)(worktree.path);
+            // Build AI options for this model
+            const aiOptions = this.buildModelAIOptions(model);
+            // Token tracking
+            const tokenTracker = { prompt: 0, completion: 0 };
+            // Create streaming options with token tracking
+            const streamingOptions = {
+                enabled: true,
+                onChunk: () => {
+                    if (!timeToFirstOutput) {
+                        timeToFirstOutput = Date.now() - startedAt;
+                    }
+                },
+                onFinish: (result) => {
+                    if (result.usage) {
+                        tokenTracker.prompt += result.usage.promptTokens ?? 0;
+                        tokenTracker.completion += result.usage.completionTokens ?? 0;
+                    }
+                },
+            };
+            // Execute the operation
+            const output = await operation.execute(input.params, aiOptions, streamingOptions);
+            const completedAt = Date.now();
+            // Collect metrics
+            const metrics = await this.collectMetrics(worktree.path, baseCommit, startedAt, completedAt, timeToFirstOutput, tokenTracker);
+            return this.createSuccessResult(worktree, modelId, output, metrics, completedAt - startedAt);
+        }
+        catch (error) {
+            const completedAt = Date.now();
+            return this.createErrorResult(worktree, modelId, error, completedAt - startedAt, startedAt, completedAt, timeToFirstOutput);
+        }
+    }
+    /**
+     * Execute a single task in a worktree
+     *
+     * @param worktree - The worktree to execute in
+     * @param model - Model configuration to use
+     * @param input - Task execution input
+     * @param baseCommit - Base commit for metrics comparison
+     * @returns Benchmark result with metrics
+     */
+    async executeTask(worktree, model, input, baseCommit) {
+        const modelId = `${model.provider}:${model.model}`;
+        const startedAt = Date.now();
+        let timeToFirstOutput;
+        logger_1.logger.info(`Executing task ${input.taskId} with ${modelId}`);
+        try {
+            // Setup working directory to the worktree
+            await (0, config_1.setupWorkingDirectory)(worktree.path);
+            // Token tracking
+            const tokenTracker = { prompt: 0, completion: 0 };
+            // Build execution config with model override
+            // The executeTaskCore takes taskId and a TaskExecutionConfig object
+            const config = {
+                tool: "opencode", // Default tool
+                enableRetry: (input.maxRetries ?? 1) > 1,
+                maxRetries: input.maxRetries ?? 1,
+                verificationCommands: input.verificationCommands ?? [],
+                executorConfig: {
+                    model: `${model.provider}:${model.model}`,
+                },
+            };
+            // Execute the task
+            const result = await (0, task_execution_core_1.executeTaskCore)(input.taskId, config);
+            const completedAt = Date.now();
+            if (!timeToFirstOutput) {
+                timeToFirstOutput = completedAt - startedAt;
+            }
+            // Collect metrics
+            const metrics = await this.collectMetrics(worktree.path, baseCommit, startedAt, completedAt, timeToFirstOutput, tokenTracker, input.verificationCommands);
+            // TaskExecutionResult has success and attempts, extract error from last attempt if failed
+            const lastAttempt = result.attempts[result.attempts.length - 1];
+            const errorMsg = result.success ? undefined : lastAttempt?.error;
+            const status = result.success ? "success" : "failed";
+            return {
+                modelId,
+                worktree,
+                status,
+                duration: completedAt - startedAt,
+                output: result,
+                error: errorMsg,
+                metrics,
+                timestamp: completedAt,
+            };
+        }
+        catch (error) {
+            const completedAt = Date.now();
+            return this.createErrorResult(worktree, modelId, error, completedAt - startedAt, startedAt, completedAt, timeToFirstOutput);
+        }
+    }
+    /**
+     * Execute a task loop in a worktree
+     *
+     * @param worktree - The worktree to execute in
+     * @param model - Model configuration to use
+     * @param input - Execute loop input
+     * @param baseCommit - Base commit for metrics comparison
+     * @returns Benchmark result with metrics
+     */
+    async executeLoop(worktree, model, input, baseCommit) {
+        const modelId = `${model.provider}:${model.model}`;
+        const startedAt = Date.now();
+        let timeToFirstOutput;
+        logger_1.logger.info(`Executing task loop with ${modelId}`);
+        try {
+            // Setup working directory to the worktree
+            await (0, config_1.setupWorkingDirectory)(worktree.path);
+            // Token tracking
+            const tokenTracker = { prompt: 0, completion: 0 };
+            // Override model in loop options
+            const loopOptions = {
+                ...input.loopOptions,
+                config: {
+                    ...input.loopOptions.config,
+                    model: `${model.provider}:${model.model}`,
+                },
+            };
+            // Execute the loop
+            const result = await (0, task_loop_execution_1.executeTaskLoop)(loopOptions);
+            const completedAt = Date.now();
+            if (!timeToFirstOutput) {
+                timeToFirstOutput = completedAt - startedAt;
+            }
+            // Collect metrics
+            const metrics = await this.collectMetrics(worktree.path, baseCommit, startedAt, completedAt, timeToFirstOutput, tokenTracker, input.loopOptions.config?.verificationCommands);
+            const hasFailures = result.failedTasks > 0;
+            return {
+                modelId,
+                worktree,
+                status: hasFailures ? "failed" : "success",
+                duration: completedAt - startedAt,
+                output: result,
+                error: hasFailures ? `${result.failedTasks} tasks failed` : undefined,
+                metrics,
+                timestamp: completedAt,
+            };
+        }
+        catch (error) {
+            const completedAt = Date.now();
+            return this.createErrorResult(worktree, modelId, error, completedAt - startedAt, startedAt, completedAt, timeToFirstOutput);
+        }
+    }
+    /**
+     * Execute a full workflow in a worktree
+     *
+     * For workflow benchmarks, we execute a series of steps:
+     * 1. Initialize project (if needed)
+     * 2. Parse PRD to generate tasks
+     * 3. Execute tasks with the given model
+     *
+     * @param worktree - The worktree to execute in
+     * @param model - Model configuration to use
+     * @param input - Workflow input
+     * @param baseCommit - Base commit for metrics comparison
+     * @returns Benchmark result with metrics
+     */
+    async executeWorkflow(worktree, model, input, baseCommit) {
+        const modelId = `${model.provider}:${model.model}`;
+        const startedAt = Date.now();
+        let timeToFirstOutput;
+        logger_1.logger.info(`Executing workflow with ${modelId}`);
+        try {
+            // Setup working directory to the worktree
+            const projectDir = input.projectDir ?? worktree.path;
+            await (0, config_1.setupWorkingDirectory)(projectDir);
+            // Token tracking
+            const tokenTracker = { prompt: 0, completion: 0 };
+            // Create workflow service
+            const workflowService = new workflow_1.WorkflowService();
+            // Build AI options for the model
+            const aiOptions = {
+                aiProvider: model.provider,
+                aiModel: model.model,
+                aiReasoning: model.reasoningTokens?.toString(),
+            };
+            // Execute workflow steps based on collected responses
+            const results = {};
+            // Step 1: Initialize project if needed
+            if (input.collectedResponses.projectName) {
+                results.init = await workflowService.initializeProject({
+                    projectName: input.collectedResponses.projectName,
+                    projectDir,
+                    initMethod: input.collectedResponses.initMethod,
+                    projectDescription: input.collectedResponses.projectDescription,
+                    aiOptions,
+                    stackConfig: input.collectedResponses.stackConfig,
+                });
+            }
+            // Step 2: Parse PRD if content is provided
+            if (input.collectedResponses.prdContent || input.collectedResponses.prdFile) {
+                const prdService = new prd_1.PRDService();
+                if (input.collectedResponses.prdContent) {
+                    // If we have PRD content, save it first
+                    const prdDir = (0, node_path_1.join)(projectDir, ".task-o-matic", "prd");
+                    (0, node_fs_1.mkdirSync)(prdDir, { recursive: true });
+                    const prdPath = (0, node_path_1.join)(prdDir, "benchmark-prd.md");
+                    (0, node_fs_1.writeFileSync)(prdPath, input.collectedResponses.prdContent);
+                    results.prdParse = await prdService.parsePRD({
+                        file: prdPath,
+                        aiOptions,
+                    });
+                }
+                else if (input.collectedResponses.prdFile) {
+                    results.prdParse = await prdService.parsePRD({
+                        file: input.collectedResponses.prdFile,
+                        aiOptions,
+                    });
+                }
+            }
+            // Step 3: Execute tasks if requested
+            if (input.collectedResponses.generateTasks) {
+                const loopOptions = {
+                    filters: { status: "todo" },
+                    tool: (input.workflowOptions.executeTool ?? "opencode"),
+                    config: {
+                        maxRetries: input.workflowOptions.executeMaxRetries ?? 3,
+                        verificationCommands: input.workflowOptions.verificationCommands ?? [],
+                        model: modelId,
+                    },
+                };
+                results.execution = await (0, task_loop_execution_1.executeTaskLoop)(loopOptions);
+            }
+            const completedAt = Date.now();
+            if (!timeToFirstOutput) {
+                timeToFirstOutput = completedAt - startedAt;
+            }
+            // Collect metrics
+            const verificationCommands = input.workflowOptions.verificationCommands ?? [];
+            const metrics = await this.collectMetrics(projectDir, baseCommit, startedAt, completedAt, timeToFirstOutput, tokenTracker, verificationCommands);
+            return this.createSuccessResult(worktree, modelId, results, metrics, completedAt - startedAt);
+        }
+        catch (error) {
+            const completedAt = Date.now();
+            return this.createErrorResult(worktree, modelId, error, completedAt - startedAt, startedAt, completedAt, timeToFirstOutput);
+        }
+    }
+    /**
+     * Build AI options for a model configuration
+     */
+    buildModelAIOptions(model) {
+        // Get API key from environment based on provider
+        const envKeyMap = {
+            anthropic: "ANTHROPIC_API_KEY",
+            openai: "OPENAI_API_KEY",
+            openrouter: "OPENROUTER_API_KEY",
+            google: "GOOGLE_API_KEY",
+            gemini: "GEMINI_API_KEY",
+            zai: "ZAI_API_KEY",
+        };
+        const envKey = envKeyMap[model.provider] ?? `${model.provider.toUpperCase()}_API_KEY`;
+        const apiKey = process.env[envKey];
+        return {
+            aiProvider: model.provider,
+            aiModel: model.model,
+            aiKey: apiKey,
+            aiReasoning: model.reasoningTokens?.toString(),
+        };
+    }
+    /**
+     * Collect all metrics for a completed execution
+     */
+    async collectMetrics(worktreePath, baseCommit, startedAt, completedAt, timeToFirstOutput, tokenTracker, verificationCommands) {
+        const timing = {
+            startedAt,
+            completedAt,
+            duration: completedAt - startedAt,
+            timeToFirstOutput,
+        };
+        const tokens = tokenTracker.prompt > 0 || tokenTracker.completion > 0
+            ? {
+                prompt: tokenTracker.prompt,
+                completion: tokenTracker.completion,
+                total: tokenTracker.prompt + tokenTracker.completion,
+            }
+            : undefined;
+        return this.metricsCollector.collectAll(worktreePath, baseCommit, timing, tokens, verificationCommands);
+    }
+    /**
+     * Create a success result
+     */
+    createSuccessResult(worktree, modelId, output, metrics, duration) {
+        return {
+            modelId,
+            worktree,
+            status: "success",
+            duration,
+            output,
+            metrics,
+            timestamp: Date.now(),
+        };
+    }
+    /**
+     * Create an error result
+     */
+    createErrorResult(worktree, modelId, error, duration, startedAt, completedAt, timeToFirstOutput) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        logger_1.logger.error(`Execution failed for ${modelId}: ${errorMessage}`);
+        return {
+            modelId,
+            worktree,
+            status: "error",
+            duration,
+            error: errorMessage,
+            metrics: {
+                timing: {
+                    startedAt,
+                    completedAt,
+                    duration,
+                    timeToFirstOutput,
+                },
+            },
+            timestamp: completedAt,
+        };
+    }
+}
+exports.BenchmarkExecutor = BenchmarkExecutor;

package/dist/lib/benchmark/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Benchmark System
+ *
+ * A unified system for benchmarking AI models across various operations.
+ * Supports parallel execution using git worktrees, persistent storage,
+ * and comprehensive metrics collection.
+ */
+export * from "./types";
+export * from "./worktree-manager";
+export * from "./worktree-pool";
+export * from "./store";
+export { MetricsCollector, type VerificationOptions } from "./metrics-collector";
+export * from "./executor";
+export * from "./orchestrator";
+export * from "./operations";
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/benchmark/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/benchmark/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,cAAc,SAAS,CAAC;AACxB,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAGjF,cAAc,YAAY,CAAC;AAC3B,cAAc,gBAAgB,CAAC;AAG/B,cAAc,cAAc,CAAC"}

package/dist/lib/benchmark/index.js

@@ -0,0 +1,36 @@
+"use strict";
+/**
+ * Benchmark System
+ *
+ * A unified system for benchmarking AI models across various operations.
+ * Supports parallel execution using git worktrees, persistent storage,
+ * and comprehensive metrics collection.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MetricsCollector = void 0;
+// Infrastructure
+__exportStar(require("./types"), exports);
+__exportStar(require("./worktree-manager"), exports);
+__exportStar(require("./worktree-pool"), exports);
+__exportStar(require("./store"), exports);
+var metrics_collector_1 = require("./metrics-collector");
+Object.defineProperty(exports, "MetricsCollector", { enumerable: true, get: function () { return metrics_collector_1.MetricsCollector; } });
+// Execution & Coordination
+__exportStar(require("./executor"), exports);
+__exportStar(require("./orchestrator"), exports);
+// Operations
+__exportStar(require("./operations"), exports);