task-o-matic 0.0.7 → 0.0.8

package/README.md

@@ -6,6 +6,8 @@ AI-powered task management for CLI, TUI, and web applications. Parse PRDs, enhan
 
 - 🤖 **AI-Powered**: Parse PRDs and enhance tasks using multiple AI providers
 - 🎭 **Interactive Workflow**: Guided setup from project init to task generation with AI assistance
+- ❓ **PRD Question/Refine**: AI generates clarifying questions and can answer them automatically
+- 🧠 **AI Reasoning Support**: Enable advanced reasoning for better PRD refinement
 - 📦 **Multi-Purpose Package**: Use as CLI tool, library, or MCP server
 - 📁 **Project-Local Storage**: All data stored locally in `.task-o-matic/` directory
 - 🎯 **Task Management**: Full CRUD operations with AI enhancement
@@ -14,6 +16,7 @@ AI-powered task management for CLI, TUI, and web applications. Parse PRDs, enhan
 - 🔧 **Multi-Provider AI**: Support for OpenAI, Anthropic, OpenRouter, and custom providers
 - 📊 **Smart Breakdown**: AI-powered task decomposition into subtasks
 - 🌊 **Real-time Streaming**: Watch AI responses generate live with streaming output
+- 📊 **Model Benchmarking**: Compare performance and quality across different AI models
 - 🏠 **Single-Project Focus**: Self-contained within each project directory
 - 💻 **Framework-Agnostic**: Easily integrate into TUI, web apps, or any Node.js project
 
@@ -55,14 +58,14 @@ task-o-matic/
 ├── dist/              # Compiled output (published)
 │   ├── lib/           # Library entry point + core exports
 │   ├── cli/           # CLI binary
-│   ├── services/      # Business logic layer
+│   ├── services/      # Business logic layer (WorkflowService, PRDService, TaskService)
 │   ├── commands/      # CLI commands
 │   ├── mcp/           # MCP server
 │   └── types/         # TypeScript definitions
 ├── src/
 │   ├── lib/           # Core library (Storage, Config, AI, etc.)
 │   │   └── index.ts   # Main library exports
-│   ├── services/      # TaskService, PRDService (framework-agnostic)
+│   ├── services/      # WorkflowService, PRDService, TaskService (framework-agnostic)
 │   ├── cli/           # CLI-specific logic
 │   │   └── bin.ts     # CLI binary entry point
 │   ├── commands/      # Commander.js command implementations
@@ -75,8 +78,8 @@ task-o-matic/
 
 ### Core Components
 
-- **Service Layer** (`TaskService`, `PRDService`): Framework-agnostic business logic
-- **AI Service**: Uses Vercel AI SDK for multi-provider support
+- **Service Layer** (`WorkflowService`, `PRDService`, `TaskService`): Framework-agnostic business logic
+- **AI Service**: Uses Vercel AI SDK for multi-provider support with reasoning capabilities
 - **Local Storage**: JSON-based file storage in `.task-o-matic/` directory
 - **Configuration**: Project-local config with AI provider settings
 - **Prompt Templates**: Structured AI prompts for consistent results
@@ -104,24 +107,45 @@ npm install task-o-matic
 
 ```typescript
 import {
+  WorkflowService,
   TaskService,
   PRDService,
   type Task,
   type AIConfig,
 } from "task-o-matic";
 
-// Initialize the service
+// Use the workflow service for complete project setup
+const workflowService = new WorkflowService();
+
+const result = await workflowService.initializeProject({
+  projectName: "my-app",
+  initMethod: "quick",
+  bootstrap: true,
+  aiOptions: {
+    aiProvider: "anthropic",
+    aiModel: "claude-3-5-sonnet",
+    aiKey: process.env.ANTHROPIC_API_KEY,
+  },
+  callbacks: {
+    onProgress: (event) => {
+      console.log(`Progress: ${event.message}`);
+    },
+  },
+});
+
+console.log("Project initialized:", result.projectName);
+
+// Or use task service directly
 const taskService = new TaskService();
 
-// Create a task with AI enhancement
-const result = await taskService.createTask({
+const taskResult = await taskService.createTask({
   title: "Implement user authentication",
   content: "Add login and signup functionality",
   aiEnhance: true,
   aiOptions: {
-    provider: "anthropic",
-    model: "claude-3-5-sonnet",
-    apiKey: process.env.ANTHROPIC_API_KEY,
+    aiProvider: "anthropic",
+    aiModel: "claude-3-5-sonnet",
+    aiKey: process.env.ANTHROPIC_API_KEY,
   },
   callbacks: {
     onProgress: (event) => {
@@ -130,7 +154,7 @@ const result = await taskService.createTask({
   },
 });
 
-console.log("Task created:", result.task);
+console.log("Task created:", taskResult.task);
 ```
 
 #### TUI Integration Example
@@ -167,20 +191,28 @@ const result = await taskService.createTask({
 });
 ```
 
-#### PRD Parsing Example
+#### PRD Question/Refine Example
 
 ```typescript
 import { PRDService } from "task-o-matic";
 
 const prdService = new PRDService();
 
-const result = await prdService.parsePRD({
+// Generate questions and refine PRD with AI answering
+const result = await prdService.refinePRDWithQuestions({
   file: "./requirements.md",
+  questionMode: "ai", // or "user" for interactive
+  questionAIOptions: {
+    // Optional: use a different AI for answering
+    aiProvider: "openrouter",
+    aiModel: "anthropic/claude-3-opus",
+    aiReasoning: "enabled", // Enable reasoning for better answers
+  },
   workingDirectory: process.cwd(),
   aiOptions: {
-    provider: "openrouter",
-    model: "anthropic/claude-3.5-sonnet",
-    apiKey: process.env.OPENROUTER_API_KEY,
+    aiProvider: "anthropic",
+    aiModel: "claude-3-5-sonnet",
+    aiKey: process.env.ANTHROPIC_API_KEY,
   },
   callbacks: {
     onProgress: (event) => {
@@ -189,9 +221,10 @@ const result = await prdService.parsePRD({
   },
 });
 
-console.log(`Created ${result.tasks.length} tasks from PRD`);
-result.tasks.forEach((task) => {
-  console.log(`- ${task.title}`);
+console.log(`Refined PRD with ${result.questions.length} questions`);
+result.questions.forEach((q, i) => {
+  console.log(`Q${i + 1}: ${q}`);
+  console.log(`A${i + 1}: ${result.answers[q]}`);
 });
 ```
 
@@ -227,6 +260,13 @@ import type {
   CreateTaskOptions,
   PRDParseResult,
   TaskAIMetadata,
+  // Workflow types
+  WorkflowService,
+  InitializeResult,
+  DefinePRDResult,
+  RefinePRDResult,
+  GenerateTasksResult,
+  SplitTasksResult,
 } from "task-o-matic";
 ```
 
@@ -316,9 +356,14 @@ task-o-matic workflow --stream
 
 1. **Project Initialization** - Choose quick start, custom, or AI-assisted configuration
 2. **PRD Definition** - Upload file, write manually, or use AI to generate from description
-3. **PRD Refinement** - Optional AI-assisted improvements
-4. **Task Generation** - Parse PRD into actionable tasks
-5. **Task Splitting** - Break down complex tasks into subtasks
+3. **PRD Question/Refine** (NEW) - AI generates clarifying questions and refines PRD
+   - User can answer questions interactively
+   - OR AI can answer with PRD + stack context
+   - Optional: Use different AI model for answering (e.g., smarter model)
+   - Optional: Enable reasoning for better answers
+4. **PRD Manual Refinement** - Optional additional AI-assisted improvements
+5. **Task Generation** - Parse PRD into actionable tasks
+6. **Task Splitting** - Break down complex tasks into subtasks
 
 **AI Assistance at Every Step:**
 
@@ -395,7 +440,25 @@ task-o-matic tasks create --title "Add payment system" --ai-enhance --stream
 task-o-matic tasks split --task-id <task-id>
 ```
 
-### Workflow 3: Project Bootstrapping
+### Workflow 3: Benchmarking Models
+
+Compare different AI models for performance, cost, and quality.
+
+```bash
+# 1. Run a benchmark for PRD parsing
+task-o-matic benchmark run prd-parse \
+  --file requirements.md \
+  --models "openai:gpt-4o,openrouter:anthropic/claude-3.5-sonnet" \
+  --concurrency 5
+
+# 2. Compare results
+task-o-matic benchmark compare <run-id>
+
+# 3. View detailed metrics (Tokens, BPS, Size)
+task-o-matic benchmark show <run-id>
+```
+
+### Workflow 4: Project Bootstrapping
 
 ```bash
 # Option 1: One-step setup (recommended)

package/dist/commands/benchmark.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare const benchmarkCommand: Command;
+//# sourceMappingURL=benchmark.d.ts.map

package/dist/commands/benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/commands/benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,eAAO,MAAM,gBAAgB,SAE5B,CAAC"}

package/dist/commands/benchmark.js

@@ -0,0 +1,227 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.benchmarkCommand = void 0;
+const commander_1 = require("commander");
+const chalk_1 = __importDefault(require("chalk"));
+const benchmark_1 = require("../services/benchmark");
+exports.benchmarkCommand = new commander_1.Command("benchmark").description("Run and manage AI benchmarks");
+// Helper to parse model string
+// Format: provider:model[:reasoning=<tokens>]
+function parseModelString(modelStr) {
+    const parts = modelStr.split(":");
+    if (parts.length < 2) {
+        throw new Error(`Invalid model format: ${modelStr}. Expected provider:model[:reasoning=<tokens>]`);
+    }
+    const provider = parts[0];
+    const model = parts[1];
+    let reasoningTokens;
+    if (parts.length > 2) {
+        const extra = parts[2];
+        if (extra.startsWith("reasoning=")) {
+            reasoningTokens = parseInt(extra.split("=")[1], 10);
+        }
+    }
+    return { provider, model, reasoningTokens };
+}
+exports.benchmarkCommand
+    .command("run")
+    .description("Run a benchmark operation")
+    .argument("<operation>", "Operation to benchmark (e.g., prd-parse, task-breakdown)")
+    .requiredOption("--models <list>", "Comma-separated list of models (provider:model[:reasoning=<tokens>])")
+    .option("--file <path>", "Input file path (for PRD ops)")
+    .option("--task-id <id>", "Task ID (for Task ops)")
+    .option("--concurrency <number>", "Max concurrent requests", "5")
+    .option("--delay <number>", "Delay between requests in ms", "250")
+    .option("--prompt <prompt>", "Override prompt")
+    .option("--message <message>", "User message")
+    .option("--tools", "Enable filesystem tools")
+    .option("--feedback <feedback>", "Feedback (for rework)")
+    .action(async (operation, options) => {
+    try {
+        const modelStrings = options.models.split(",");
+        const models = modelStrings.map((s) => parseModelString(s.trim()));
+        const config = {
+            models,
+            concurrency: parseInt(options.concurrency, 10),
+            delay: parseInt(options.delay, 10),
+        };
+        console.log(chalk_1.default.blue(`Starting benchmark for ${operation}...`));
+        console.log(chalk_1.default.dim(`Models: ${models.length}, Concurrency: ${config.concurrency}, Delay: ${config.delay}ms`));
+        // Construct input object with all potential options
+        const input = {
+            file: options.file,
+            taskId: options.taskId,
+            prompt: options.prompt,
+            message: options.message,
+            tools: options.tools,
+            feedback: options.feedback,
+            workingDirectory: process.cwd(), // Always pass current working directory
+        };
+        // Prepare dashboard
+        console.log(chalk_1.default.bold("\nBenchmark Progress:"));
+        const modelMap = new Map();
+        const modelStatus = new Map();
+        // Print initial lines and map indices
+        models.forEach((m, i) => {
+            const id = `${m.provider}:${m.model}${m.reasoningTokens ? `:reasoning=${m.reasoningTokens}` : ""}`;
+            modelMap.set(id, i);
+            modelStatus.set(id, "Waiting...");
+            console.log(chalk_1.default.dim(`- ${id}: Waiting...`));
+        });
+        const totalModels = models.length;
+        const run = await benchmark_1.benchmarkService.runBenchmark(operation, input, config, (event) => {
+            const index = modelMap.get(event.modelId);
+            if (index === undefined)
+                return;
+            // Update status in memory
+            let statusStr = "";
+            if (event.type === "start") {
+                statusStr = chalk_1.default.yellow("Starting...");
+            }
+            else if (event.type === "progress") {
+                const bps = event.currentBps ? `${event.currentBps} B/s` : "0 B/s";
+                const size = event.currentSize ? `${event.currentSize} B` : "0 B";
+                statusStr = `${chalk_1.default.blue("Running")} - Size: ${size}, Speed: ${bps}`;
+            }
+            else if (event.type === "complete") {
+                statusStr = chalk_1.default.green(`Completed (${event.duration}ms)`);
+            }
+            else if (event.type === "error") {
+                statusStr = chalk_1.default.red(`Failed: ${event.error}`);
+            }
+            modelStatus.set(event.modelId, statusStr);
+            // Update display
+            // Move cursor up to the specific line
+            // Distance from bottom = totalModels - index
+            const up = totalModels - index;
+            process.stdout.write(`\x1B[${up}A`); // Move up
+            process.stdout.write(`\x1B[2K`); // Clear line
+            process.stdout.write(`- ${chalk_1.default.bold(event.modelId)}: ${statusStr}\r`);
+            process.stdout.write(`\x1B[${up}B`); // Move down
+        });
+        console.log(chalk_1.default.green(`\n✓ Benchmark completed! Run ID: ${run.id}`));
+        console.log(chalk_1.default.bold(`\n${"Model".padEnd(40)} | ${"Duration".padEnd(10)} | ${"TTFT".padEnd(8)} | ${"Tokens".padEnd(10)} | ${"TPS".padEnd(8)} | ${"BPS".padEnd(8)} | ${"Size".padEnd(10)} | ${"Cost".padEnd(10)}`));
+        console.log("-".repeat(130)); // Adjusted line length for new columns
+        run.results.forEach((r) => {
+            const duration = `${r.duration}ms`.padEnd(10);
+            const ttft = r.timeToFirstToken
+                ? `${r.timeToFirstToken}ms`.padEnd(8)
+                : "-".padEnd(8);
+            const tokens = r.tokenUsage
+                ? `${r.tokenUsage.total}`.padEnd(10)
+                : "-".padEnd(10);
+            const tps = r.tps ? `${r.tps}`.padEnd(8) : "-".padEnd(8);
+            const bps = r.bps ? `${r.bps}`.padEnd(8) : "-".padEnd(8);
+            const size = r.responseSize
+                ? `${r.responseSize}`.padEnd(10)
+                : "-".padEnd(10);
+            const cost = r.cost
+                ? `$${r.cost.toFixed(6)}`.padEnd(10)
+                : "-".padEnd(10);
+            console.log(`${r.modelId.padEnd(40)} | ${duration} | ${ttft} | ${tokens} | ${tps} | ${bps} | ${size} | ${cost}`);
+            if (r.error) {
+                console.log(chalk_1.default.red(`  Error: ${r.error}`));
+            }
+        });
+    }
+    catch (error) {
+        console.error(chalk_1.default.red("Benchmark failed:"), error.message);
+        process.exit(1);
+    }
+});
+exports.benchmarkCommand
+    .command("list")
+    .description("List past benchmark runs")
+    .action(() => {
+    const runs = benchmark_1.benchmarkService.listRuns();
+    if (runs.length === 0) {
+        console.log(chalk_1.default.yellow("No benchmark runs found."));
+        return;
+    }
+    console.log(chalk_1.default.bold("Benchmark Runs:"));
+    runs.forEach((run) => {
+        const date = new Date(run.timestamp).toLocaleString();
+        console.log(`- ${chalk_1.default.cyan(run.id)} (${date}) - ${run.command}`);
+    });
+});
+exports.benchmarkCommand
+    .command("show")
+    .description("Show details of a benchmark run")
+    .argument("<id>", "Run ID")
+    .action((id) => {
+    const run = benchmark_1.benchmarkService.getRun(id);
+    if (!run) {
+        console.error(chalk_1.default.red(`Run ${id} not found`));
+        process.exit(1);
+    }
+    console.log(chalk_1.default.bold(`Run: ${run.id}`));
+    console.log(`Date: ${new Date(run.timestamp).toLocaleString()}`);
+    console.log(`Command: ${run.command}`);
+    console.log(`Input: ${JSON.stringify(run.input, null, 2)}`); // Might be large
+    console.log(chalk_1.default.bold("\nConfiguration:"));
+    console.log(`Concurrency: ${run.config.concurrency}`);
+    console.log(`Delay: ${run.config.delay}ms`);
+    console.log(chalk_1.default.bold("\nResults:"));
+    const results = run.results;
+    results.forEach((result) => {
+        console.log(chalk_1.default.bold(`\n[${result.modelId}]`));
+        console.log(`Duration: ${result.duration}ms`);
+        if (result.timeToFirstToken) {
+            console.log(`TTFT: ${result.timeToFirstToken}ms`);
+        }
+        if (result.tokenUsage) {
+            console.log(`Tokens: ${result.tokenUsage.total} (Prompt: ${result.tokenUsage.prompt}, Completion: ${result.tokenUsage.completion})`);
+        }
+        if (result.bps) {
+            console.log(`Throughput: ${result.bps} B/s`);
+        }
+        if (result.responseSize) {
+            console.log(`Size: ${result.responseSize} bytes`);
+        }
+        if (result.cost) {
+            console.log(`Estimated Cost: $${result.cost.toFixed(6)}`);
+        }
+        if (result.error) {
+            console.log(chalk_1.default.red(`Error: ${result.error}`));
+        }
+        else {
+            const outputStr = typeof result.output === "string"
+                ? result.output
+                : JSON.stringify(result.output, null, 2);
+            const preview = outputStr.length > 500
+                ? outputStr.substring(0, 500) + "..."
+                : outputStr;
+            console.log(`Output: ${preview}`);
+        }
+    });
+});
+exports.benchmarkCommand
+    .command("compare")
+    .description("Compare results of a benchmark run")
+    .argument("<id>", "Run ID")
+    .action((id) => {
+    const run = benchmark_1.benchmarkService.getRun(id);
+    if (!run) {
+        console.error(chalk_1.default.red(`Run ${id} not found`));
+        process.exit(1);
+    }
+    console.log(chalk_1.default.bold(`Comparison for Run: ${run.id}`));
+    // Simple comparison: Duration and Success/Fail
+    // In future could add diffing of outputs
+    const table = run.results.map((res) => ({
+        Model: res.modelId,
+        Status: res.error ? "FAILED" : "SUCCESS",
+        Duration: `${res.duration}ms`,
+        Tokens: res.tokenUsage ? res.tokenUsage.total : "?",
+        BPS: res.bps ? res.bps : "?",
+        Size: res.responseSize
+            ? res.responseSize
+            : res.output
+                ? JSON.stringify(res.output).length
+                : 0,
+    }));
+    console.table(table);
+});

package/dist/commands/prd.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prd.d.ts","sourceRoot":"","sources":["../../src/commands/prd.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,eAAO,MAAM,UAAU,SAAmE,CAAC"}
+{"version":3,"file":"prd.d.ts","sourceRoot":"","sources":["../../src/commands/prd.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,eAAO,MAAM,UAAU,SAEtB,CAAC"}