universal-agent-memory 0.6.2 → 0.7.0

package/dist/benchmarks/improved-benchmark.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Improved Benchmark Runner for UAM
+ *
+ * Integrates all improvements:
+ * - Dynamic memory retrieval
+ * - Task classification and routing
+ * - Multi-turn agent loop
+ * - Hierarchical prompting
+ * - Execution verification
+ */
+import { type TaskClassification } from '../memory/task-classifier.js';
+import { type VerificationResult } from './execution-verifier.js';
+interface ModelConfig {
+    id: string;
+    name: string;
+    apiModel: string;
+}
+interface BenchmarkTask {
+    id: string;
+    name: string;
+    description: string;
+    prompt: string;
+    difficulty: 'easy' | 'medium' | 'hard';
+    category: string;
+    expectedPatterns: string[];
+}
+interface TaskResult {
+    taskId: string;
+    modelId: string;
+    success: boolean;
+    latencyMs: number;
+    turnsUsed: number;
+    memoryUsed: boolean;
+    classification: TaskClassification;
+    verification: VerificationResult;
+    errors: string[];
+}
+interface ModelResult {
+    modelId: string;
+    modelName: string;
+    tasksRun: number;
+    tasksSucceeded: number;
+    successRate: number;
+    avgLatencyMs: number;
+    avgTurns: number;
+    results: TaskResult[];
+}
+interface BenchmarkReport {
+    timestamp: string;
+    config: {
+        maxTurns: number;
+        useMemory: boolean;
+        useHierarchicalPrompting: boolean;
+    };
+    models: ModelResult[];
+    comparison: {
+        bestOverall: string;
+        fastestModel: string;
+        byDifficulty: Record<string, {
+            model: string;
+            successRate: number;
+        }>;
+        byCategory: Record<string, {
+            model: string;
+            successRate: number;
+        }>;
+    };
+    memoryImpact?: {
+        withMemory: ModelResult[];
+        withoutMemory: ModelResult[];
+        improvement: Record<string, {
+            successDelta: number;
+            speedup: number;
+        }>;
+    };
+}
+declare const MODELS: ModelConfig[];
+declare const BENCHMARK_TASKS: BenchmarkTask[];
+export declare function runImprovedBenchmark(options?: {
+    apiKey?: string;
+    models?: string[];
+    compareMemory?: boolean;
+    maxTurns?: number;
+    useHierarchicalPrompting?: boolean;
+    verbose?: boolean;
+}): Promise<BenchmarkReport>;
+export { MODELS, BENCHMARK_TASKS };
+//# sourceMappingURL=improved-benchmark.d.ts.map

package/dist/benchmarks/improved-benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"improved-benchmark.d.ts","sourceRoot":"","sources":["../../src/benchmarks/improved-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAOH,OAAO,EAAgB,KAAK,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAIrF,OAAO,EAAuB,KAAK,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAUvF,UAAU,WAAW;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,aAAa;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,UAAU,UAAU;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,OAAO,CAAC;IACpB,cAAc,EAAE,kBAAkB,CAAC;IACnC,YAAY,EAAE,kBAAkB,CAAC;IACjC,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,UAAU,eAAe;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,OAAO,CAAC;QACnB,wBAAwB,EAAE,OAAO,CAAC;KACnC,CAAC;IACF,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,UAAU,EAAE;QACV,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACrE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACpE,CAAC;IACF,YAAY,CAAC,EAAE;QACb,UAAU,EAAE,WAAW,EAAE,CAAC;QAC1B,aAAa,EAAE,WAAW,EAAE,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,YAAY,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACxE,CAAC;CACH;AAMD,QAAA,MAAM,MAAM,EAAE,WAAW,EAIxB,CAAC;AAEF,QAAA,MAAM,eAAe,EAAE,aAAa,EA0GnC,CAAC;AAoVF,wBAAsB,oBAAoB,CAAC,OAAO,GAAE;IAClD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,GAAG,OAAO,CAAC,eAAe,CAAC,CAmHhC;AAuBD,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC"}

package/dist/benchmarks/improved-benchmark.js

@@ -0,0 +1,533 @@
+/**
+ * Improved Benchmark Runner for UAM
+ *
+ * Integrates all improvements:
+ * - Dynamic memory retrieval
+ * - Task classification and routing
+ * - Multi-turn agent loop
+ * - Hierarchical prompting
+ * - Execution verification
+ */
+import { writeFileSync, existsSync, readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { execSync } from 'child_process';
+import { classifyTask } from '../memory/task-classifier.js';
+import { retrieveDynamicMemoryContext } from '../memory/dynamic-retrieval.js';
+import { executeWithMultiTurn } from './multi-turn-agent.js';
+import { buildHierarchicalPrompt } from './hierarchical-prompting.js';
+import { verifyBenchmarkTask } from './execution-verifier.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const PROJECT_ROOT = join(__dirname, '../..');
+// ============================================================================
+// Configuration
+// ============================================================================
+const MODELS = [
+    { id: 'opus-4.5', name: 'Claude Opus 4.5', apiModel: 'claude-opus-4-5-20251101' },
+    { id: 'glm-4.7', name: 'GLM 4.7', apiModel: 'glm-4.7' },
+    { id: 'gpt-5.2-codex', name: 'GPT 5.2 Codex', apiModel: 'gpt-5.2-codex' },
+];
+const BENCHMARK_TASKS = [
+    {
+        id: 'task-001-code-generation',
+        name: 'TypeScript Function Generation',
+        description: 'Generate a well-typed TypeScript function',
+        prompt: `Write a TypeScript function called 'calculateAverage' that:
+1. Takes an array of numbers as input
+2. Returns the arithmetic mean
+3. Handles empty arrays (return 0)
+4. Has proper type annotations
+
+Return ONLY the function code, no explanations.`,
+        difficulty: 'easy',
+        category: 'coding',
+        expectedPatterns: ['function calculateAverage', 'number[]', ': number', 'length', 'return'],
+    },
+    {
+        id: 'task-002-bug-fix',
+        name: 'Bug Detection and Fix',
+        description: 'Identify and fix a bug in code',
+        prompt: `Find and fix the bug in this TypeScript code:
+
+function sumPositive(nums: number[]): number {
+  let sum = 0;
+  for (let i = 0; i <= nums.length; i++) {
+    if (nums[i] > 0) {
+      sum += nums[i];
+    }
+  }
+  return sum;
+}
+
+Return ONLY the corrected function code.`,
+        difficulty: 'easy',
+        category: 'debugging',
+        expectedPatterns: ['i < nums.length', 'function sumPositive', 'return sum'],
+    },
+    {
+        id: 'task-003-pattern-implementation',
+        name: 'Design Pattern Implementation',
+        description: 'Implement a singleton pattern',
+        prompt: `Implement a TypeScript singleton class called 'ConfigManager' that:
+1. Has a private constructor
+2. Has a static getInstance() method
+3. Has get(key: string) and set(key: string, value: any) methods
+4. Stores configuration in a private Map
+
+Return ONLY the class code.`,
+        difficulty: 'medium',
+        category: 'coding',
+        expectedPatterns: ['class ConfigManager', 'private constructor', 'static getInstance', 'Map'],
+    },
+    {
+        id: 'task-004-refactoring',
+        name: 'Code Refactoring',
+        description: 'Refactor code for better maintainability',
+        prompt: `Refactor this code to use the Strategy pattern:
+
+function processOrder(order: any) {
+  if (order.type === 'digital') {
+    order.status = 'delivered';
+  } else if (order.type === 'physical') {
+    order.status = 'shipped';
+  } else if (order.type === 'subscription') {
+    order.status = 'active';
+  }
+  return order;
+}
+
+Return the refactored TypeScript code with interfaces and classes.`,
+        difficulty: 'medium',
+        category: 'coding',
+        expectedPatterns: ['interface', 'class', 'implements', 'process'],
+    },
+    {
+        id: 'task-005-algorithm',
+        name: 'Algorithm Implementation',
+        description: 'Implement Dijkstra\'s algorithm',
+        prompt: `Implement a TypeScript function 'findShortestPath' using Dijkstra's algorithm:
+
+1. Input: weighted graph as adjacency list Map<string, Map<string, number>>
+2. Input: start node (string), end node (string)
+3. Output: { path: string[], distance: number } or null if no path
+4. Handle disconnected nodes
+
+Return ONLY the function code with type definitions.`,
+        difficulty: 'hard',
+        category: 'coding',
+        expectedPatterns: ['function findShortestPath', 'Map<string', 'distance', 'path', 'while'],
+    },
+    {
+        id: 'task-006-error-handling',
+        name: 'Comprehensive Error Handling',
+        description: 'Implement robust error handling',
+        prompt: `Create a TypeScript async function 'fetchWithRetry' that:
+
+1. Takes url: string, retryConfig?: { maxRetries: number; backoffMs: number; }
+2. Implements exponential backoff retry logic
+3. Handles network errors and HTTP errors
+4. Returns Promise<Response> or throws custom error
+
+Return ONLY the function code with types.`,
+        difficulty: 'hard',
+        category: 'coding',
+        expectedPatterns: ['async function fetchWithRetry', 'retry', 'catch', 'throw'],
+    },
+];
+// ============================================================================
+// Benchmark Runner
+// ============================================================================
+async function runTaskForModel(task, model, config) {
+    const startTime = Date.now();
+    // Step 1: Classify the task
+    const classification = classifyTask(task.prompt);
+    if (config.verbose) {
+        console.log(`    Category: ${classification.category} (${(classification.confidence * 100).toFixed(0)}% confidence)`);
+    }
+    // Step 2: Get memory context if enabled
+    let memoryContext = '';
+    if (config.useMemory) {
+        try {
+            const dynamicContext = await retrieveDynamicMemoryContext(task.prompt, PROJECT_ROOT);
+            memoryContext = dynamicContext.formattedContext;
+            if (config.verbose) {
+                console.log(`    Memory: ${dynamicContext.relevantMemories.length} memories retrieved`);
+            }
+        }
+        catch (error) {
+            if (config.verbose) {
+                console.log(`    Memory: Failed to retrieve (${error})`);
+            }
+        }
+    }
+    // Step 3: Build prompt
+    let finalPrompt;
+    if (config.useHierarchicalPrompting) {
+        finalPrompt = buildHierarchicalPrompt(task.prompt, classification, memoryContext);
+    }
+    else {
+        finalPrompt = memoryContext ? memoryContext + '\n\n' + task.prompt : task.prompt;
+    }
+    // Step 4: Execute with multi-turn if needed
+    let success = false;
+    let response = '';
+    let turnsUsed = 1;
+    let verification;
+    const errors = [];
+    if (config.maxTurns > 1) {
+        // Use multi-turn agent
+        const multiTurnResult = await executeWithMultiTurn(task.id, task.prompt, {
+            maxTurns: config.maxTurns,
+            model: model.apiModel,
+            apiKey: config.apiKey,
+            useMemory: config.useMemory,
+            projectRoot: PROJECT_ROOT,
+            verbose: config.verbose,
+        });
+        success = multiTurnResult.success;
+        response = multiTurnResult.finalResponse;
+        turnsUsed = multiTurnResult.totalTurns;
+        verification = multiTurnResult.turns[multiTurnResult.turns.length - 1]?.verification || {
+            success: false,
+            executionSucceeded: false,
+            testsRun: 0,
+            testsPassed: 0,
+            errors: ['No verification data'],
+            output: '',
+            executionTimeMs: 0,
+        };
+        for (const turn of multiTurnResult.turns) {
+            errors.push(...turn.verification.errors);
+        }
+    }
+    else {
+        // Single-shot execution
+        try {
+            response = await executeSingleShot(finalPrompt, model.apiModel, config.apiKey);
+            verification = await verifyBenchmarkTask(task.id, response);
+            success = verification.success;
+            errors.push(...verification.errors);
+        }
+        catch (error) {
+            verification = {
+                success: false,
+                executionSucceeded: false,
+                testsRun: 0,
+                testsPassed: 0,
+                errors: [`Execution failed: ${error}`],
+                output: '',
+                executionTimeMs: 0,
+            };
+            errors.push(`Execution failed: ${error}`);
+        }
+    }
+    const latencyMs = Date.now() - startTime;
+    return {
+        taskId: task.id,
+        modelId: model.id,
+        success,
+        latencyMs,
+        turnsUsed,
+        memoryUsed: config.useMemory,
+        classification,
+        verification,
+        errors: [...new Set(errors)].slice(0, 5),
+    };
+}
+async function executeSingleShot(prompt, model, apiKey) {
+    const tmpDir = '/tmp/uam-benchmark';
+    const promptFile = join(tmpDir, `prompt-${Date.now()}.txt`);
+    if (!existsSync(tmpDir)) {
+        execSync(`mkdir -p ${tmpDir}`, { encoding: 'utf-8' });
+    }
+    writeFileSync(promptFile, prompt, 'utf-8');
+    try {
+        const result = execSync(`FACTORY_API_KEY="${apiKey}" droid exec --model "${model}" --auto medium -f "${promptFile}"`, {
+            encoding: 'utf-8',
+            timeout: 300000,
+            maxBuffer: 10 * 1024 * 1024,
+            env: { ...process.env, FACTORY_API_KEY: apiKey },
+        });
+        execSync(`rm "${promptFile}"`, { encoding: 'utf-8' });
+        return result.trim();
+    }
+    catch (error) {
+        try {
+            execSync(`rm "${promptFile}"`, { encoding: 'utf-8' });
+        }
+        catch { }
+        throw error;
+    }
+}
+async function runBenchmarkForModel(model, tasks, config) {
+    const memoryLabel = config.useMemory ? 'with UAM' : 'without UAM';
+    console.log(`\n${'='.repeat(60)}`);
+    console.log(`Running: ${model.name} (${memoryLabel})`);
+    console.log(`${'='.repeat(60)}`);
+    const results = [];
+    for (const task of tasks) {
+        console.log(`  [${task.difficulty.toUpperCase()}] ${task.name}...`);
+        const result = await runTaskForModel(task, model, config);
+        results.push(result);
+        if (result.success) {
+            console.log(`    ✓ Success (${result.latencyMs}ms, ${result.turnsUsed} turn${result.turnsUsed > 1 ? 's' : ''})`);
+        }
+        else {
+            console.log(`    ✗ Failed: ${result.errors[0] || 'Unknown error'}`);
+        }
+        // Delay between tasks
+        await new Promise(r => setTimeout(r, 1000));
+    }
+    const succeeded = results.filter(r => r.success).length;
+    const avgLatency = results.reduce((sum, r) => sum + r.latencyMs, 0) / results.length;
+    const avgTurns = results.reduce((sum, r) => sum + r.turnsUsed, 0) / results.length;
+    return {
+        modelId: model.id,
+        modelName: model.name,
+        tasksRun: tasks.length,
+        tasksSucceeded: succeeded,
+        successRate: (succeeded / tasks.length) * 100,
+        avgLatencyMs: Math.round(avgLatency),
+        avgTurns: Math.round(avgTurns * 10) / 10,
+        results,
+    };
+}
+function generateComparison(modelResults) {
+    const sorted = [...modelResults].sort((a, b) => b.successRate - a.successRate);
+    const fastest = [...modelResults].sort((a, b) => a.avgLatencyMs - b.avgLatencyMs);
+    const byDifficulty = {};
+    const byCategory = {};
+    for (const diff of ['easy', 'medium', 'hard']) {
+        let bestModel = '';
+        let bestRate = 0;
+        for (const modelResult of modelResults) {
+            const diffTasks = modelResult.results.filter(r => {
+                const task = BENCHMARK_TASKS.find(t => t.id === r.taskId);
+                return task?.difficulty === diff;
+            });
+            if (diffTasks.length > 0) {
+                const rate = (diffTasks.filter(t => t.success).length / diffTasks.length) * 100;
+                if (rate > bestRate) {
+                    bestRate = rate;
+                    bestModel = modelResult.modelName;
+                }
+            }
+        }
+        byDifficulty[diff] = { model: bestModel, successRate: bestRate };
+    }
+    // By category
+    const categories = [...new Set(BENCHMARK_TASKS.map(t => t.category))];
+    for (const cat of categories) {
+        let bestModel = '';
+        let bestRate = 0;
+        for (const modelResult of modelResults) {
+            const catTasks = modelResult.results.filter(r => {
+                const task = BENCHMARK_TASKS.find(t => t.id === r.taskId);
+                return task?.category === cat;
+            });
+            if (catTasks.length > 0) {
+                const rate = (catTasks.filter(t => t.success).length / catTasks.length) * 100;
+                if (rate > bestRate) {
+                    bestRate = rate;
+                    bestModel = modelResult.modelName;
+                }
+            }
+        }
+        byCategory[cat] = { model: bestModel, successRate: bestRate };
+    }
+    return {
+        bestOverall: sorted[0]?.modelName || 'N/A',
+        fastestModel: fastest[0]?.modelName || 'N/A',
+        byDifficulty,
+        byCategory,
+    };
+}
+function generateMarkdownReport(report) {
+    const lines = [
+        '# Improved UAM Benchmark Results',
+        '',
+        `**Generated:** ${report.timestamp}`,
+        `**Configuration:**`,
+        `- Max Turns: ${report.config.maxTurns}`,
+        `- Memory Enabled: ${report.config.useMemory}`,
+        `- Hierarchical Prompting: ${report.config.useHierarchicalPrompting}`,
+        '',
+        '---',
+        '',
+        '## Executive Summary',
+        '',
+        '| Model | Success Rate | Avg Latency | Avg Turns |',
+        '|-------|--------------|-------------|-----------|',
+    ];
+    for (const model of report.models) {
+        lines.push(`| ${model.modelName} | ${model.successRate.toFixed(1)}% | ${model.avgLatencyMs}ms | ${model.avgTurns} |`);
+    }
+    lines.push('', '---', '', '## Comparison', '');
+    lines.push(`- **Best Overall:** ${report.comparison.bestOverall}`);
+    lines.push(`- **Fastest Model:** ${report.comparison.fastestModel}`);
+    lines.push('', '### By Difficulty', '');
+    lines.push('| Difficulty | Best Model | Success Rate |');
+    lines.push('|------------|------------|--------------|');
+    for (const [diff, data] of Object.entries(report.comparison.byDifficulty)) {
+        lines.push(`| ${diff} | ${data.model} | ${data.successRate.toFixed(1)}% |`);
+    }
+    lines.push('', '### By Category', '');
+    lines.push('| Category | Best Model | Success Rate |');
+    lines.push('|----------|------------|--------------|');
+    for (const [cat, data] of Object.entries(report.comparison.byCategory)) {
+        lines.push(`| ${cat} | ${data.model} | ${data.successRate.toFixed(1)}% |`);
+    }
+    if (report.memoryImpact) {
+        lines.push('', '---', '', '## UAM Memory Impact', '');
+        lines.push('| Model | Without UAM | With UAM | Improvement |');
+        lines.push('|-------|-------------|----------|-------------|');
+        for (const withMem of report.memoryImpact.withMemory) {
+            const without = report.memoryImpact.withoutMemory.find(r => r.modelId === withMem.modelId);
+            const imp = report.memoryImpact.improvement[withMem.modelId];
+            if (without && imp) {
+                const sign = imp.successDelta >= 0 ? '+' : '';
+                lines.push(`| ${withMem.modelName} | ${without.successRate.toFixed(1)}% | ${withMem.successRate.toFixed(1)}% | ${sign}${imp.successDelta.toFixed(1)}% |`);
+            }
+        }
+    }
+    lines.push('', '---', '', '## Detailed Results', '');
+    for (const model of report.models) {
+        lines.push(`### ${model.modelName}`, '');
+        lines.push('| Task | Difficulty | Success | Latency | Turns | Category |');
+        lines.push('|------|------------|---------|---------|-------|----------|');
+        for (const result of model.results) {
+            const task = BENCHMARK_TASKS.find(t => t.id === result.taskId);
+            const status = result.success ? '✓' : '✗';
+            lines.push(`| ${task?.name || result.taskId} | ${task?.difficulty} | ${status} | ${result.latencyMs}ms | ${result.turnsUsed} | ${result.classification.category} |`);
+        }
+        lines.push('');
+    }
+    lines.push('---', '', '**Report Generated by UAM Improved Benchmark**');
+    return lines.join('\n');
+}
+// ============================================================================
+// Main Entry Point
+// ============================================================================
+export async function runImprovedBenchmark(options = {}) {
+    const apiKey = options.apiKey || process.env.FACTORY_API_KEY || process.env.DROID_API_KEY;
+    if (!apiKey) {
+        throw new Error('API key required (FACTORY_API_KEY or DROID_API_KEY)');
+    }
+    const modelsToTest = options.models
+        ? MODELS.filter(m => options.models.includes(m.id))
+        : MODELS;
+    const compareMemory = options.compareMemory ?? true;
+    const maxTurns = options.maxTurns ?? 2;
+    const useHierarchicalPrompting = options.useHierarchicalPrompting ?? true;
+    const verbose = options.verbose ?? false;
+    console.log('\n' + '█'.repeat(60));
+    console.log('   UAM IMPROVED BENCHMARK');
+    console.log('█'.repeat(60));
+    console.log(`\nModels: ${modelsToTest.map(m => m.name).join(', ')}`);
+    console.log(`Tasks: ${BENCHMARK_TASKS.length}`);
+    console.log(`Max Turns: ${maxTurns}`);
+    console.log(`Memory Comparison: ${compareMemory}`);
+    console.log(`Hierarchical Prompting: ${useHierarchicalPrompting}`);
+    const withoutMemoryResults = [];
+    const withMemoryResults = [];
+    // Run without memory first (if comparing)
+    if (compareMemory) {
+        console.log('\n' + '█'.repeat(60));
+        console.log('   PHASE 1: WITHOUT UAM MEMORY');
+        console.log('█'.repeat(60));
+        for (const model of modelsToTest) {
+            const result = await runBenchmarkForModel(model, BENCHMARK_TASKS, {
+                useMemory: false,
+                useHierarchicalPrompting: false,
+                maxTurns: 1,
+                apiKey,
+                verbose,
+            });
+            withoutMemoryResults.push(result);
+        }
+    }
+    // Run with memory (and all improvements)
+    console.log('\n' + '█'.repeat(60));
+    console.log('   PHASE 2: WITH UAM IMPROVEMENTS');
+    console.log('█'.repeat(60));
+    for (const model of modelsToTest) {
+        const result = await runBenchmarkForModel(model, BENCHMARK_TASKS, {
+            useMemory: true,
+            useHierarchicalPrompting,
+            maxTurns,
+            apiKey,
+            verbose,
+        });
+        withMemoryResults.push(result);
+    }
+    // Calculate improvement
+    const improvement = {};
+    if (compareMemory) {
+        for (const model of modelsToTest) {
+            const without = withoutMemoryResults.find(r => r.modelId === model.id);
+            const withMem = withMemoryResults.find(r => r.modelId === model.id);
+            if (without && withMem) {
+                improvement[model.id] = {
+                    successDelta: withMem.successRate - without.successRate,
+                    speedup: without.avgLatencyMs > 0 ? without.avgLatencyMs / withMem.avgLatencyMs : 1,
+                };
+            }
+        }
+    }
+    const report = {
+        timestamp: new Date().toISOString(),
+        config: {
+            maxTurns,
+            useMemory: true,
+            useHierarchicalPrompting,
+        },
+        models: withMemoryResults,
+        comparison: generateComparison(withMemoryResults),
+        memoryImpact: compareMemory ? {
+            withMemory: withMemoryResults,
+            withoutMemory: withoutMemoryResults,
+            improvement,
+        } : undefined,
+    };
+    // Generate and save report
+    const markdown = generateMarkdownReport(report);
+    const reportPath = join(PROJECT_ROOT, 'IMPROVED_BENCHMARK_RESULTS.md');
+    writeFileSync(reportPath, markdown);
+    console.log(`\nReport saved to: ${reportPath}`);
+    // Print summary
+    console.log('\n' + '='.repeat(60));
+    console.log('   BENCHMARK COMPLETE');
+    console.log('='.repeat(60));
+    if (compareMemory) {
+        console.log('\n--- Memory Impact ---');
+        for (const [modelId, imp] of Object.entries(improvement)) {
+            const model = modelsToTest.find(m => m.id === modelId);
+            const sign = imp.successDelta >= 0 ? '+' : '';
+            console.log(`  ${model?.name}: ${sign}${imp.successDelta.toFixed(1)}% success`);
+        }
+    }
+    console.log(`\nBest Overall: ${report.comparison.bestOverall}`);
+    return report;
+}
+// CLI entry
+if (process.argv[1]?.includes('improved-benchmark')) {
+    const envPath = join(PROJECT_ROOT, '.env');
+    if (existsSync(envPath)) {
+        const envContent = readFileSync(envPath, 'utf-8');
+        for (const line of envContent.split('\n')) {
+            const [key, ...valueParts] = line.split('=');
+            if (key && valueParts.length > 0) {
+                process.env[key.trim()] = valueParts.join('=').trim();
+            }
+        }
+    }
+    runImprovedBenchmark({ verbose: true })
+        .then(() => process.exit(0))
+        .catch(err => {
+        console.error('Benchmark failed:', err);
+        process.exit(1);
+    });
+}
+export { MODELS, BENCHMARK_TASKS };
+//# sourceMappingURL=improved-benchmark.js.map

package/dist/benchmarks/improved-benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"improved-benchmark.js","sourceRoot":"","sources":["../../src/benchmarks/improved-benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAC7D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,YAAY,EAA2B,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAA2B,MAAM,yBAAyB,CAAC;AAEvF,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AACtC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;AAkE9C,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E,MAAM,MAAM,GAAkB;IAC5B,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,0BAA0B,EAAE;IACjF,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE;IACvD,EAAE,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,eAAe,EAAE;CAC1E,CAAC;AAEF,MAAM,eAAe,GAAoB;IACvC;QACE,EAAE,EAAE,0BAA0B;QAC9B,IAAI,EAAE,gCAAgC;QACtC,WAAW,EAAE,2CAA2C;QACxD,MAAM,EAAE;;;;;;gDAMoC;QAC5C,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,CAAC,2BAA2B,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC;KAC5F;IACD;QACE,EAAE,EAAE,kBAAkB;QACtB,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,gCAAgC;QAC7C,MAAM,EAAE;;;;;;;;;;;;yCAY6B;QACrC,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,CAAC,iBAAiB,EAAE,sBAAsB,EAAE,YAAY,CAAC;KAC5E;IACD;QACE,EAAE,EAAE,iCAAiC;QACrC,IAAI,EAAE,+BAA+B;QACrC,WAAW,EAAE,+BAA+B;QAC5C,MAAM,EAAE;;;;;;4BAMgB;QACxB,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,CAAC,qBAAqB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,KAAK,CAAC;KAC9F;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,IAAI,EAAE,kBAAkB;QACxB,WAAW,EAAE,0CAA0C;QACvD,MAAM,EAAE;;;;;;;;;;;;;mEAauD;QAC/D,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,SAAS,CAAC;KAClE;IACD;QACE,EAAE,EAAE,oBAAoB;QACxB,IAAI,EAAE,0BAA0B;QAChC,WAAW,EAAE,iCAAiC;QAC9C,MAAM,EAAE;;;;;;;qDAOyC;QACjD,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,CAAC,2BAA2B,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC;KAC3F;IACD;QACE,EAAE,EAAE,yBAAyB;QAC7B,IAAI,EAAE,8BAA8B;QACpC,WAAW,EAAE,iCAAiC;QAC9C,MAAM,EAAE;;;;;;;0CAO8B;QACtC,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,QAAQ;QAClB,gBAAgB,EAAE,CAAC,+BAA+B,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;KAC/E;CACF,CAAC;AAEF,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,KAAK,UAAU,eAAe,CAC5B,IAAmB,EACnB,KAAkB,EAClB,MAMC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,4BAA4B;IAC5B,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,iBAAiB,cAAc,CAAC,QAAQ,KAAK,CAAC,cAAc,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IACxH,CAAC;IAED,wCAAwC;IACxC,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,MAAM,4BAA4B,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YACrF,aAAa,GAAG,cAAc,CAAC,gBAAgB,CAAC;YAEhD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,OAAO,CAAC,GAAG,CAAC,eAAe,cAAc,CAAC,gBAAgB,CAAC,MAAM,qBAAqB,CAAC,CAAC;YAC1F,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,OAAO,CAAC,GAAG,CAAC,mCAAmC,KAAK,GAAG,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,IAAI,WAAmB,CAAC;IACxB,IAAI,MAAM,CAAC,wBAAwB,EAAE,CAAC;QACpC,WAAW,GAAG,uBAAuB,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;IACpF,CAAC;SAAM,CAAC;QACN,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IACnF,CAAC;IAED,4CAA4C;IAC5C,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAgC,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QACxB,uBAAuB;QACvB,MAAM,eAAe,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE;YACvE,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,KAAK,EAAE,KAAK,CAAC,QAAQ;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,WAAW,EAAE,YAAY;YACzB,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC,CAAC;QAEH,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC;QAClC,QAAQ,GAAG,eAAe,CAAC,aAAa,CAAC;QACzC,SAAS,GAAG,eAAe,CAAC,UAAU,CAAC;QACvC,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,YAAY,IAAI;YACtF,OAAO,EAAE,KAAK;YACd,kBAAkB,EAAE,KAAK;YACzB,QAAQ,EAAE,CAAC;YACX,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,CAAC,sBAAsB,CAAC;YAChC,MAAM,EAAE,EAAE;YACV,eAAe,EAAE,CAAC;SACnB,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,KAAK,EAAE,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;SAAM,CAAC;QACN,wBAAwB;QACxB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/E,YAAY,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YAC5D,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,GAAG;gBACb,OAAO,EAAE,KAAK;gBACd,kBAAkB,EAAE,KAAK;gBACzB,QAAQ,EAAE,CAAC;gBACX,WAAW,EAAE,CAAC;gBACd,MAAM,EAAE,CAAC,qBAAqB,KAAK,EAAE,CAAC;gBACtC,MAAM,EAAE,EAAE;gBACV,eAAe,EAAE,CAAC;aACnB,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAEzC,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,OAAO;QACP,SAAS;QACT,SAAS;QACT,UAAU,EAAE,MAAM,CAAC,SAAS;QAC5B,cAAc;QACd,YAAY;QACZ,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;KACzC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,KAAa,EAAE,MAAc;IAC5E,MAAM,MAAM,GAAG,oBAAoB,CAAC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAE5D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACxB,QAAQ,CAAC,YAAY,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAE3C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CACrB,oBAAoB,MAAM,yBAAyB,KAAK,uBAAuB,UAAU,GAAG,EAC5F;YACE,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;YAC3B,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,eAAe,EAAE,MAAM,EAAE;SACjD,CACF,CAAC;QAEF,QAAQ,CAAC,OAAO,UAAU,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QACtD,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC;YAAC,QAAQ,CAAC,OAAO,UAAU,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,CAAC;QACxE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,KAAkB,EAClB,KAAsB,EACtB,MAMC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,IAAI,KAAK,WAAW,GAAG,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAEjC,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC;QAEpE,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAErB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,SAAS,OAAO,MAAM,CAAC,SAAS,QAAQ,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACnH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,eAAe,EAAE,CAAC,CAAC;QACtE,CAAC;QAED,sBAAsB;QACtB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACxD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACrF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAEnF,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,SAAS,EAAE,KAAK,CAAC,IAAI;QACrB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,cAAc,EAAE,SAAS;QACzB,WAAW,EAAE,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG;QAC7C,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;QACpC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,EAAE;QACxC,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,YAA2B;IACrD,MAAM,MAAM,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;IAElF,MAAM,YAAY,GAA2D,EAAE,CAAC;IAChF,MAAM,UAAU,GAA2D,EAAE,CAAC;IAE9E,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;QAC9C,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;gBAC/C,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;gBAC1D,OAAO,IAAI,EAAE,UAAU,KAAK,IAAI,CAAC;YACnC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;gBAChF,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;oBACpB,QAAQ,GAAG,IAAI,CAAC;oBAChB,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC;gBACpC,CAAC;YACH,CAAC;QACH,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IACnE,CAAC;IAED,cAAc;IACd,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACtE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;gBAC9C,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;gBAC1D,OAAO,IAAI,EAAE,QAAQ,KAAK,GAAG,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;gBAC9E,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;oBACpB,QAAQ,GAAG,IAAI,CAAC;oBAChB,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC;gBACpC,CAAC;YACH,CAAC;QACH,CAAC;QAED,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IAChE,CAAC;IAED,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK;QAC1C,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK;QAC5C,YAAY;QACZ,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,KAAK,GAAa;QACtB,kCAAkC;QAClC,EAAE;QACF,kBAAkB,MAAM,CAAC,SAAS,EAAE;QACpC,oBAAoB;QACpB,gBAAgB,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE;QACxC,qBAAqB,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE;QAC9C,6BAA6B,MAAM,CAAC,MAAM,CAAC,wBAAwB,EAAE;QACrE,EAAE;QACF,KAAK;QACL,EAAE;QACF,sBAAsB;QACtB,EAAE;QACF,oDAAoD;QACpD,oDAAoD;KACrD,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,SAAS,MAAM,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,YAAY,QAAQ,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IACxH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;IAC/C,KAAK,CAAC,IAAI,CAAC,uBAAuB,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;IACnE,KAAK,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC;IAErE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,mBAAmB,EAAE,EAAE,CAAC,CAAC;IACxC,KAAK,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IACzD,KAAK,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IACzD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC1E,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,MAAM,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,iBAAiB,EAAE,EAAE,CAAC,CAAC;IACtC,KAAK,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IACvD,KAAK,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IACvD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACvE,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,sBAAsB,EAAE,EAAE,CAAC,CAAC;QACtD,KAAK,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QAC/D,KAAK,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QAE/D,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC;YACrD,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;YAC3F,MAAM,GAAG,GAAG,MAAM,CAAC,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC7D,IAAI,OAAO,IAAI,GAAG,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9C,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,SAAS,MAAM,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5J,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAErD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;QAC3E,KAAK,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;QAE3E,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,MAAM,MAAM,IAAI,EAAE,UAAU,MAAM,MAAM,MAAM,MAAM,CAAC,SAAS,QAAQ,MAAM,CAAC,SAAS,MAAM,MAAM,CAAC,cAAc,CAAC,QAAQ,IAAI,CAAC,CAAC;QACvK,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,EAAE,gDAAgD,CAAC,CAAC;IAExE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,UAOvC,EAAE;IACJ,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IAC1F,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM;QACjC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,MAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpD,CAAC,CAAC,MAAM,CAAC;IAEX,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;IACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;IACvC,MAAM,wBAAwB,GAAG,OAAO,CAAC,wBAAwB,IAAI,IAAI,CAAC;IAC1E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAEzC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,aAAa,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,UAAU,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,sBAAsB,aAAa,EAAE,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,2BAA2B,wBAAwB,EAAE,CAAC,CAAC;IAEnE,MAAM,oBAAoB,GAAkB,EAAE,CAAC;IAC/C,MAAM,iBAAiB,GAAkB,EAAE,CAAC;IAE5C,0CAA0C;IAC1C,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAE5B,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,eAAe,EAAE;gBAChE,SAAS,EAAE,KAAK;gBAChB,wBAAwB,EAAE,KAAK;gBAC/B,QAAQ,EAAE,CAAC;gBACX,MAAM;gBACN,OAAO;aACR,CAAC,CAAC;YACH,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,eAAe,EAAE;YAChE,SAAS,EAAE,IAAI;YACf,wBAAwB;YACxB,QAAQ;YACR,MAAM;YACN,OAAO;SACR,CAAC,CAAC;QACH,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAED,wBAAwB;IACxB,MAAM,WAAW,GAA8D,EAAE,CAAC;IAClF,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,EAAE,CAAC,CAAC;YACvE,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,EAAE,CAAC,CAAC;YACpE,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;gBACvB,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG;oBACtB,YAAY,EAAE,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW;oBACvD,OAAO,EAAE,OAAO,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;iBACpF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAoB;QAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE;YACN,QAAQ;YACR,SAAS,EAAE,IAAI;YACf,wBAAwB;SACzB;QACD,MAAM,EAAE,iBAAiB;QACzB,UAAU,EAAE,kBAAkB,CAAC,iBAAiB,CAAC;QACjD,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC;YAC5B,UAAU,EAAE,iBAAiB;YAC7B,aAAa,EAAE,oBAAoB;YACnC,WAAW;SACZ,CAAC,CAAC,CAAC,SAAS;KACd,CAAC;IAEF,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,+BAA+B,CAAC,CAAC;IACvE,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;IAEhD,gBAAgB;IAChB,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;QACvC,KAAK,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YACzD,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;YACvD,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,IAAI,KAAK,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;IAEhE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,YAAY;AACZ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC3C,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,MAAM,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IAED,oBAAoB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;SACpC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,GAAG,CAAC,EAAE;QACX,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC"}

package/dist/benchmarks/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Terminal-Bench Adapter for UAM - Index
+ *
+ * Exports everything needed for benchmarking
+ */
+export * from './benchmark.js';
+export * from './agents/naive-agent.js';
+export * from './agents/uam-agent.js';
+export * from './tasks.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/benchmarks/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/benchmarks/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,YAAY,CAAC"}