task-o-matic-core 0.1.0

package/src/lib/benchmark/registry.ts

@@ -0,0 +1,307 @@
+import { BenchmarkableOperation, WorkflowBenchmarkInput } from "./types";
+import { prdService } from "../../services/prd";
+import { taskService } from "../../services/tasks";
+import { workflowBenchmarkService } from "../../services/workflow-benchmark";
+import { AIOptions } from "../../utils/ai-config-builder";
+import { StreamingOptions } from "../../types";
+
+export class BenchmarkRegistry {
+  private operations: Map<string, BenchmarkableOperation> = new Map();
+
+  constructor() {
+    this.registerDefaults();
+  }
+
+  register(op: BenchmarkableOperation) {
+    this.operations.set(op.id, op);
+  }
+
+  get(id: string): BenchmarkableOperation | undefined {
+    return this.operations.get(id);
+  }
+
+  list(): BenchmarkableOperation[] {
+    return Array.from(this.operations.values());
+  }
+
+  private registerDefaults() {
+    // PRD Parsing Adapter
+    this.register({
+      id: "prd-parse",
+      name: "PRD Parsing",
+      description: "Parse a PRD file into tasks",
+      validateInput: (input: any) =>
+        typeof input.file === "string" && input.file.length > 0,
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await prdService.parsePRD({
+          file: input.file,
+          workingDirectory: input.workingDirectory,
+          enableFilesystemTools: input.tools,
+          promptOverride: input.prompt,
+          messageOverride: input.message,
+          aiOptions,
+          streamingOptions,
+          callbacks: {},
+        });
+      },
+    });
+
+    // PRD Rework Adapter
+    this.register({
+      id: "prd-rework",
+      name: "PRD Rework",
+      description: "Rework a PRD based on feedback",
+      validateInput: (input: any) =>
+        typeof input.file === "string" && typeof input.feedback === "string",
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await prdService.reworkPRD({
+          file: input.file,
+          feedback: input.feedback,
+          workingDirectory: input.workingDirectory,
+          enableFilesystemTools: input.tools,
+          promptOverride: input.prompt,
+          messageOverride: input.message,
+          aiOptions,
+          streamingOptions,
+          callbacks: {},
+        });
+      },
+    });
+
+    // Task Breakdown Adapter
+    this.register({
+      id: "task-breakdown",
+      name: "Task Breakdown",
+      description: "Break down a task into subtasks",
+      validateInput: (input: any) => typeof input.taskId === "string",
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await taskService.splitTask(
+          input.taskId,
+          aiOptions,
+          input.prompt,
+          input.message,
+          streamingOptions, // streaming options
+          input.tools
+        );
+      },
+    });
+
+    // Workflow Full Execution
+    this.register({
+      id: "workflow-full",
+      name: "Complete Workflow",
+      description: "Execute the complete workflow: initialization, PRD creation, task generation, and splitting",
+      validateInput: (input: any) => workflowBenchmarkService.validateInput(input),
+      execute: async (
+        input: WorkflowBenchmarkInput,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await workflowBenchmarkService.executeWorkflow(
+          input,
+          aiOptions,
+          streamingOptions
+        );
+      },
+    });
+
+    // Task Create with AI Enhancement
+    this.register({
+      id: "task-create",
+      name: "Task Creation (AI-Enhanced)",
+      description: "Create a new task with AI enhancement using Context7 documentation",
+      validateInput: (input: any) =>
+        typeof input.title === "string" && input.title.length > 0,
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await taskService.createTask({
+          title: input.title,
+          content: input.content,
+          parentId: input.parentId,
+          effort: input.effort,
+          aiEnhance: true, // Always use AI enhancement for benchmarking
+          aiOptions,
+          streamingOptions,
+        });
+      },
+    });
+
+    // Task Enhancement
+    this.register({
+      id: "task-enhance",
+      name: "Task Enhancement",
+      description: "Enhance an existing task with AI using Context7 documentation",
+      validateInput: (input: any) => typeof input.taskId === "string",
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await taskService.enhanceTask(
+          input.taskId,
+          aiOptions,
+          streamingOptions
+        );
+      },
+    });
+
+    // Task Planning
+    this.register({
+      id: "task-plan",
+      name: "Task Planning",
+      description: "Create a detailed implementation plan for a task",
+      validateInput: (input: any) => typeof input.taskId === "string",
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await taskService.planTask(
+          input.taskId,
+          aiOptions,
+          streamingOptions
+        );
+      },
+    });
+
+    // Task Documentation
+    this.register({
+      id: "task-document",
+      name: "Task Documentation",
+      description: "Analyze and generate documentation for a task",
+      validateInput: (input: any) => typeof input.taskId === "string",
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await taskService.documentTask(
+          input.taskId,
+          input.force || false,
+          aiOptions,
+          streamingOptions
+        );
+      },
+    });
+
+    // PRD Creation
+    this.register({
+      id: "prd-create",
+      name: "PRD Creation",
+      description: "Generate a Product Requirements Document from a description",
+      validateInput: (input: any) =>
+        typeof input.description === "string" && input.description.length > 0,
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await prdService.generatePRD({
+          description: input.description,
+          outputDir: input.outputDir,
+          filename: input.filename,
+          aiOptions,
+          streamingOptions,
+          callbacks: {},
+        });
+      },
+    });
+
+    // PRD Combine
+    this.register({
+      id: "prd-combine",
+      name: "PRD Combination",
+      description: "Combine multiple PRD files into a single master PRD",
+      validateInput: (input: any) =>
+        Array.isArray(input.prds) &&
+        input.prds.length > 0 &&
+        typeof input.originalDescription === "string",
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await prdService.combinePRDs({
+          prds: input.prds,
+          originalDescription: input.originalDescription,
+          outputDir: input.outputDir,
+          filename: input.filename,
+          aiOptions,
+          streamingOptions,
+          callbacks: {},
+        });
+      },
+    });
+
+    // PRD Question Generation
+    this.register({
+      id: "prd-question",
+      name: "PRD Question Generation",
+      description: "Generate clarifying questions for a PRD",
+      validateInput: (input: any) =>
+        typeof input.file === "string" && input.file.length > 0,
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await prdService.generateQuestions({
+          file: input.file,
+          workingDirectory: input.workingDirectory,
+          enableFilesystemTools: input.tools,
+          promptOverride: input.prompt,
+          messageOverride: input.message,
+          aiOptions,
+          streamingOptions,
+          callbacks: {},
+        });
+      },
+    });
+
+    // PRD Refinement
+    this.register({
+      id: "prd-refine",
+      name: "PRD Refinement",
+      description: "Refine a PRD by generating questions and incorporating answers",
+      validateInput: (input: any) =>
+        typeof input.file === "string" &&
+        input.file.length > 0 &&
+        typeof input.questionMode === "string" &&
+        (input.questionMode === "user" || input.questionMode === "ai"),
+      execute: async (
+        input: any,
+        aiOptions: AIOptions,
+        streamingOptions?: StreamingOptions
+      ) => {
+        return await prdService.refinePRDWithQuestions({
+          file: input.file,
+          questionMode: input.questionMode,
+          answers: input.answers,
+          questionAIOptions: input.questionAIOptions,
+          workingDirectory: input.workingDirectory,
+          enableFilesystemTools: input.tools,
+          aiOptions,
+          streamingOptions,
+          callbacks: {},
+        });
+      },
+    });
+  }
+}
+
+export const benchmarkRegistry = new BenchmarkRegistry();

package/src/lib/benchmark/runner.ts

@@ -0,0 +1,190 @@
+import {
+  BenchmarkConfig,
+  BenchmarkResult,
+  BenchmarkRun,
+  BenchmarkProgressEvent,
+} from "./types";
+import { benchmarkRegistry } from "./registry";
+import { benchmarkStorage } from "./storage";
+import { AIConfig } from "../../types";
+import { configManager } from "../config";
+
+export class BenchmarkRunner {
+  async run(
+    operationId: string,
+    input: any,
+    config: BenchmarkConfig,
+    onProgress?: (event: BenchmarkProgressEvent) => void
+  ): Promise<BenchmarkRun> {
+    const operation = benchmarkRegistry.get(operationId);
+    if (!operation) {
+      throw new Error(`Operation ${operationId} not found`);
+    }
+
+    if (!operation.validateInput(input)) {
+      throw new Error(`Invalid input for operation ${operationId}`);
+    }
+
+    const runId = `run-${Date.now()}`;
+    const results: BenchmarkResult[] = [];
+
+    // Create a queue of models to process
+    const queue = [...config.models];
+    const activePromises: Promise<void>[] = [];
+
+    // Helper to process a single model
+    const processModel = async (modelConfig: any) => {
+      const modelId = `${modelConfig.provider}:${modelConfig.model}${
+        modelConfig.reasoningTokens
+          ? `:reasoning=${modelConfig.reasoningTokens}`
+          : ""
+      }`;
+
+      const startTime = Date.now();
+      let output: any;
+      let error: string | undefined;
+      let tokenUsage:
+        | { prompt: number; completion: number; total: number }
+        | undefined;
+      let responseSize = 0;
+      let firstTokenTime: number | undefined;
+
+      // Emit start event
+      onProgress?.({ type: "start", modelId });
+
+      try {
+        // Construct AI options for this specific run
+        const aiOptions: any = {
+          aiProvider: modelConfig.provider,
+          aiModel: modelConfig.model,
+          aiReasoning: modelConfig.reasoningTokens,
+        };
+
+        // Setup streaming options to capture metrics
+        const streamingOptions = {
+          onFinish: async (result: any) => {
+            if (result.usage) {
+              tokenUsage = {
+                prompt:
+                  result.usage.inputTokens || result.usage.promptTokens || 0,
+                completion:
+                  result.usage.outputTokens ||
+                  result.usage.completionTokens ||
+                  0,
+                total: result.usage.totalTokens || 0,
+              };
+            }
+            // Estimate response size from text length if available, or JSON stringify
+            if (result.text) {
+              responseSize = Buffer.byteLength(result.text, "utf8");
+            }
+          },
+          onChunk: (chunk: string) => {
+            if (chunk) {
+              if (!firstTokenTime) {
+                firstTokenTime = Date.now();
+              }
+              const chunkSize = Buffer.byteLength(chunk, "utf8");
+              responseSize += chunkSize;
+
+              const currentDuration = Date.now() - startTime;
+              const currentBps =
+                currentDuration > 0
+                  ? Math.round(responseSize / (currentDuration / 1000))
+                  : 0;
+
+              onProgress?.({
+                type: "progress",
+                modelId,
+                currentSize: responseSize,
+                currentBps,
+                chunk: chunk,
+                duration: currentDuration,
+              });
+            }
+          },
+        };
+
+        // Execute operation
+        output = await operation.execute(input, aiOptions, streamingOptions);
+
+        // If responseSize wasn't captured via streaming (e.g. non-streaming response), calculate from output
+        if (responseSize === 0 && output) {
+          const outputStr =
+            typeof output === "string" ? output : JSON.stringify(output);
+          responseSize = Buffer.byteLength(outputStr, "utf8");
+        }
+      } catch (e: any) {
+        error = e.message || String(e);
+        onProgress?.({ type: "error", modelId, error });
+      }
+
+      const duration = Date.now() - startTime;
+      const bps =
+        duration > 0 && responseSize > 0
+          ? Math.round(responseSize / (duration / 1000))
+          : 0;
+      const tps =
+        duration > 0 && tokenUsage?.completion
+          ? Math.round(tokenUsage.completion / (duration / 1000))
+          : 0;
+
+      results.push({
+        modelId,
+        output,
+        duration,
+        error,
+        timestamp: Date.now(),
+        tokenUsage,
+        responseSize,
+        bps,
+        tps,
+        timeToFirstToken: firstTokenTime
+          ? firstTokenTime - startTime
+          : undefined,
+      });
+
+      // Emit complete event
+      onProgress?.({ type: "complete", modelId, duration });
+    };
+
+    // Process queue with concurrency limit
+    while (queue.length > 0 || activePromises.length > 0) {
+      // Fill active promises up to concurrency limit
+      while (queue.length > 0 && activePromises.length < config.concurrency) {
+        const modelConfig = queue.shift();
+        if (modelConfig) {
+          const promise = processModel(modelConfig).then(() => {
+            // Remove self from active promises
+            activePromises.splice(activePromises.indexOf(promise), 1);
+          });
+          activePromises.push(promise);
+
+          // Add delay if configured and there are more items
+          if (config.delay > 0 && queue.length > 0) {
+            await new Promise((resolve) => setTimeout(resolve, config.delay));
+          }
+        }
+      }
+
+      // Wait for at least one promise to complete if we're at capacity or queue is empty
+      if (activePromises.length > 0) {
+        await Promise.race(activePromises);
+      }
+    }
+
+    const run: BenchmarkRun = {
+      id: runId,
+      timestamp: Date.now(),
+      command: operationId,
+      input,
+      config,
+      results,
+    };
+
+    benchmarkStorage.saveRun(run);
+    return run;
+  }
+}
+
+export const benchmarkRunner = new BenchmarkRunner();

package/src/lib/benchmark/storage.ts

@@ -0,0 +1,140 @@
+import { join } from "path";
+import {
+  existsSync,
+  mkdirSync,
+  writeFileSync,
+  readFileSync,
+  readdirSync,
+} from "fs";
+import { configManager } from "../config";
+import { BenchmarkRun } from "./types";
+import { logger } from "../logger";
+
+export class BenchmarkStorage {
+  private getBenchmarkDir(): string {
+    const taskOMaticDir = configManager.getTaskOMaticDir();
+    const benchmarkDir = join(taskOMaticDir, "benchmarks");
+    if (!existsSync(benchmarkDir)) {
+      mkdirSync(benchmarkDir, { recursive: true });
+    }
+    return benchmarkDir;
+  }
+
+  saveRun(run: BenchmarkRun): string {
+    const dir = this.getBenchmarkDir();
+    const runDir = join(dir, run.id);
+
+    if (!existsSync(runDir)) {
+      mkdirSync(runDir, { recursive: true });
+    }
+
+    // Save metadata
+    writeFileSync(
+      join(runDir, "metadata.json"),
+      JSON.stringify(
+        {
+          id: run.id,
+          timestamp: run.timestamp,
+          command: run.command,
+          config: run.config,
+        },
+        null,
+        2
+      )
+    );
+
+    // Save input
+    writeFileSync(
+      join(runDir, "input.json"),
+      JSON.stringify(run.input, null, 2)
+    );
+
+    // Save results
+    const resultsDir = join(runDir, "results");
+    if (!existsSync(resultsDir)) {
+      mkdirSync(resultsDir);
+    }
+
+    run.results.forEach((result) => {
+      // Sanitize filename: replace : and / with -
+      const filename = result.modelId.replace(/[:\/]/g, "-") + ".json";
+      writeFileSync(
+        join(resultsDir, filename),
+        JSON.stringify(result, null, 2)
+      );
+    });
+
+    return runDir;
+  }
+
+  getRun(id: string): BenchmarkRun | null {
+    const dir = this.getBenchmarkDir();
+    const runDir = join(dir, id);
+
+    if (!existsSync(runDir)) {
+      return null;
+    }
+
+    try {
+      const metadata = JSON.parse(
+        readFileSync(join(runDir, "metadata.json"), "utf-8")
+      );
+      const input = JSON.parse(
+        readFileSync(join(runDir, "input.json"), "utf-8")
+      );
+
+      const resultsDir = join(runDir, "results");
+      const results = [];
+
+      if (existsSync(resultsDir)) {
+        const files = readdirSync(resultsDir).filter((f) =>
+          f.endsWith(".json")
+        );
+        for (const file of files) {
+          results.push(
+            JSON.parse(readFileSync(join(resultsDir, file), "utf-8"))
+          );
+        }
+      }
+
+      return {
+        ...metadata,
+        input,
+        results,
+      };
+    } catch (error) {
+      logger.error(`Failed to load benchmark run ${id}: ${error}`);
+      return null;
+    }
+  }
+
+  listRuns(): Array<{ id: string; timestamp: number; command: string }> {
+    const dir = this.getBenchmarkDir();
+    if (!existsSync(dir)) return [];
+
+    const runs = [];
+    const entries = readdirSync(dir, { withFileTypes: true });
+
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        try {
+          const metadataPath = join(dir, entry.name, "metadata.json");
+          if (existsSync(metadataPath)) {
+            const metadata = JSON.parse(readFileSync(metadataPath, "utf-8"));
+            runs.push({
+              id: metadata.id,
+              timestamp: metadata.timestamp,
+              command: metadata.command,
+            });
+          }
+        } catch (e) {
+          // Skip invalid runs
+        }
+      }
+    }
+
+    return runs.sort((a, b) => b.timestamp - a.timestamp);
+  }
+}
+
+export const benchmarkStorage = new BenchmarkStorage();