vesper-memory 0.3.0

package/dist/benchmark/runner.js

@@ -0,0 +1,308 @@
+/**
+ * Benchmark Runner
+ *
+ * Orchestrates the execution of all benchmark scenarios.
+ * Runs A/B tests (Vesper enabled vs disabled) and compares results.
+ */
+import { randomUUID } from "crypto";
+import { validateBenchmarkConfig } from "./types.js";
+import { createScenario } from "./scenarios/index.js";
+import { welchTTest, cohensD, interpretEffectSize, mean } from "./statistics.js";
+import { estimateTokens } from "./metrics-collector.js";
+/**
+ * Create a benchmark runner
+ *
+ * @param client - MCP client for memory operations
+ * @param config - Benchmark configuration
+ * @returns BenchmarkRunner instance
+ */
+export function createBenchmarkRunner(client, config) {
+    // Validate config
+    const validatedConfig = validateBenchmarkConfig(config);
+    return {
+        getConfig() {
+            return { ...validatedConfig };
+        },
+        async run() {
+            const startTime = Date.now();
+            const benchmarkId = `bench-${randomUUID().slice(0, 8)}`;
+            const comparisons = [];
+            // Run each scenario
+            for (const scenarioType of validatedConfig.scenarios) {
+                try {
+                    const comparison = await runScenarioComparison(client, scenarioType, validatedConfig);
+                    comparisons.push(comparison);
+                }
+                catch (err) {
+                    console.error(`Error running scenario ${scenarioType}:`, err instanceof Error ? err.message : String(err));
+                    // Create a placeholder comparison for failed scenario
+                    comparisons.push(createFailedComparison(scenarioType, err));
+                }
+            }
+            const endTime = Date.now();
+            // Calculate summary
+            const summary = calculateSummary(comparisons);
+            return {
+                id: benchmarkId,
+                startTime,
+                endTime,
+                config: validatedConfig,
+                comparisons,
+                summary,
+                metadata: {
+                    version: "0.1.0",
+                    environment: process.env.NODE_ENV || "development",
+                    hostname: process.env.HOSTNAME,
+                    nodeVersion: process.version,
+                },
+            };
+        },
+    };
+}
+/**
+ * Run a single scenario comparison (enabled vs disabled)
+ */
+async function runScenarioComparison(client, scenarioType, config) {
+    const scenario = createScenario(scenarioType, client);
+    // Setup scenario
+    await scenario.setup();
+    // Run warmup (if configured)
+    for (let i = 0; i < config.warmupRuns; i++) {
+        await scenario.run("vesper-enabled");
+    }
+    // Run measurement - Vesper enabled
+    const enabledResults = [];
+    for (let i = 0; i < config.measurementRuns; i++) {
+        const result = await scenario.run("vesper-enabled");
+        enabledResults.push(result);
+    }
+    // Aggregate enabled results
+    const enabledResult = aggregateScenarioRuns(enabledResults, "vesper-enabled");
+    // Run measurement - Vesper disabled
+    const disabledResults = [];
+    for (let i = 0; i < config.measurementRuns; i++) {
+        const result = await scenario.run("vesper-disabled");
+        disabledResults.push(result);
+    }
+    // Aggregate disabled results
+    const disabledResult = aggregateScenarioRuns(disabledResults, "vesper-disabled");
+    // Teardown
+    await scenario.teardown();
+    // Statistical comparison
+    const statistics = comparePhasesStatistically(enabledResult, disabledResult, config.significanceLevel);
+    // Determine winner
+    const winner = determineWinner(statistics);
+    return {
+        scenarioType,
+        enabledResult,
+        disabledResult,
+        statistics,
+        winner,
+    };
+}
+/**
+ * Aggregate multiple scenario runs into a single result
+ */
+function aggregateScenarioRuns(runs, phase) {
+    if (runs.length === 0) {
+        throw new Error("No runs to aggregate");
+    }
+    // Flatten all metrics
+    const allMetrics = runs.flatMap((r) => r.metrics);
+    // Aggregate latencies for percentiles
+    const latencies = allMetrics.map((m) => m.latencyMs);
+    const sortedLatencies = [...latencies].sort((a, b) => a - b);
+    const latencyP50 = percentileFromSorted(sortedLatencies, 50);
+    const latencyP95 = percentileFromSorted(sortedLatencies, 95);
+    const latencyP99 = percentileFromSorted(sortedLatencies, 99);
+    // Calculate memory hit rate
+    const memoryHitRate = allMetrics.filter((m) => m.memoryHit).length / allMetrics.length;
+    // Calculate average accuracy if available
+    const accuracies = allMetrics
+        .filter((m) => m.retrievalAccuracy !== undefined)
+        .map((m) => m.retrievalAccuracy);
+    const avgAccuracy = accuracies.length > 0 ? mean(accuracies) : undefined;
+    // Take the last run's metrics as the representative sample
+    // (this matches the measurementRuns count for statistical analysis)
+    const lastRun = runs[runs.length - 1];
+    return {
+        scenarioType: lastRun.scenarioType,
+        phase,
+        metrics: lastRun.metrics,
+        aggregates: {
+            latencyP50,
+            latencyP95,
+            latencyP99,
+            memoryHitRate,
+            avgAccuracy,
+        },
+        rawResponses: lastRun.rawResponses,
+        errors: runs.flatMap((r) => r.errors || []),
+    };
+}
+/**
+ * Calculate percentile from sorted array
+ */
+function percentileFromSorted(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const index = Math.ceil((p / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, index)];
+}
+/**
+ * Compare two scenario results statistically
+ */
+export function comparePhasesStatistically(enabledResult, disabledResult, alpha) {
+    const enabledLatencies = enabledResult.metrics.map((m) => m.latencyMs);
+    const disabledLatencies = disabledResult.metrics.map((m) => m.latencyMs);
+    // Calculate latency improvement (positive = enabled is faster)
+    const enabledMean = mean(enabledLatencies);
+    const disabledMean = mean(disabledLatencies);
+    const latencyImprovement = disabledMean > 0
+        ? ((disabledMean - enabledMean) / disabledMean) * 100
+        : 0;
+    // Calculate token savings (estimate based on response length)
+    const enabledTokens = enabledResult.rawResponses
+        .map((r) => estimateTokens(r))
+        .reduce((a, b) => a + b, 0);
+    const disabledTokens = disabledResult.rawResponses
+        .map((r) => estimateTokens(r))
+        .reduce((a, b) => a + b, 0);
+    const tokenSavings = disabledTokens > 0
+        ? ((disabledTokens - enabledTokens) / disabledTokens) * 100
+        : 0;
+    // Calculate accuracy delta
+    const enabledAccuracies = enabledResult.metrics
+        .filter((m) => m.retrievalAccuracy !== undefined)
+        .map((m) => m.retrievalAccuracy);
+    const disabledAccuracies = disabledResult.metrics
+        .filter((m) => m.retrievalAccuracy !== undefined)
+        .map((m) => m.retrievalAccuracy);
+    const enabledAvgAccuracy = enabledAccuracies.length > 0 ? mean(enabledAccuracies) : 0;
+    const disabledAvgAccuracy = disabledAccuracies.length > 0 ? mean(disabledAccuracies) : 0;
+    const accuracyDelta = enabledAvgAccuracy - disabledAvgAccuracy;
+    // Perform Welch's t-test on latencies
+    let tTestResult;
+    try {
+        tTestResult = welchTTest(enabledLatencies, disabledLatencies, alpha);
+    }
+    catch {
+        // Fallback if t-test fails (e.g., identical values)
+        // Ensure degreesOfFreedom is non-negative (min 1 for valid schema)
+        const df = Math.max(1, enabledLatencies.length + disabledLatencies.length - 2);
+        tTestResult = {
+            tStatistic: 0,
+            pValue: 1,
+            degreesOfFreedom: df,
+            significant: false,
+            alpha,
+            meanDifference: enabledMean - disabledMean,
+        };
+    }
+    // Calculate Cohen's d effect size
+    let d = 0;
+    let interpretation = "negligible";
+    try {
+        d = cohensD(enabledLatencies, disabledLatencies);
+        interpretation = interpretEffectSize(d);
+    }
+    catch {
+        // Fallback
+    }
+    return {
+        latencyImprovement,
+        tokenSavings,
+        accuracyDelta,
+        tTestResult,
+        effectSize: {
+            cohensD: d,
+            interpretation,
+        },
+    };
+}
+/**
+ * Determine winner based on statistical comparison
+ */
+function determineWinner(stats) {
+    // Primary criteria: statistical significance
+    if (!stats.tTestResult.significant) {
+        return "tie";
+    }
+    // If significant, check direction
+    // Negative t-statistic means enabled < disabled (enabled is faster = better)
+    // Positive latency improvement means enabled is faster
+    if (stats.latencyImprovement > 0 && stats.tTestResult.tStatistic < 0) {
+        return "vesper-enabled";
+    }
+    if (stats.latencyImprovement < 0 && stats.tTestResult.tStatistic > 0) {
+        return "vesper-disabled";
+    }
+    // Secondary criteria: accuracy
+    if (stats.accuracyDelta > 0.1) {
+        return "vesper-enabled";
+    }
+    if (stats.accuracyDelta < -0.1) {
+        return "vesper-disabled";
+    }
+    return "tie";
+}
+/**
+ * Calculate summary from all comparisons
+ */
+function calculateSummary(comparisons) {
+    const vesperWins = comparisons.filter((c) => c.winner === "vesper-enabled").length;
+    const disabledWins = comparisons.filter((c) => c.winner === "vesper-disabled").length;
+    const ties = comparisons.filter((c) => c.winner === "tie").length;
+    // Calculate overall improvement (average of latency improvements)
+    const improvements = comparisons.map((c) => c.statistics.latencyImprovement);
+    const overallImprovement = improvements.length > 0 ? mean(improvements) : 0;
+    // Determine if overall result is statistically significant
+    const significantCount = comparisons.filter((c) => c.statistics.tTestResult.significant).length;
+    const statisticallySignificant = significantCount > comparisons.length / 2;
+    return {
+        totalScenarios: comparisons.length,
+        vesperWins,
+        disabledWins,
+        ties,
+        overallImprovement,
+        statisticallySignificant,
+    };
+}
+/**
+ * Create a failed comparison placeholder
+ */
+function createFailedComparison(scenarioType, error) {
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    const emptyResult = {
+        scenarioType,
+        phase: "vesper-enabled",
+        metrics: [{ timestamp: Date.now(), latencyMs: 0, memoryHit: false }],
+        aggregates: { latencyP50: 0, latencyP95: 0, latencyP99: 0 },
+        rawResponses: [],
+        errors: [errorMsg],
+    };
+    return {
+        scenarioType,
+        enabledResult: { ...emptyResult, phase: "vesper-enabled" },
+        disabledResult: { ...emptyResult, phase: "vesper-disabled" },
+        statistics: {
+            latencyImprovement: 0,
+            tokenSavings: 0,
+            accuracyDelta: 0,
+            tTestResult: {
+                tStatistic: 0,
+                pValue: 1,
+                degreesOfFreedom: 0,
+                significant: false,
+                alpha: 0.05,
+                meanDifference: 0,
+            },
+            effectSize: {
+                cohensD: 0,
+                interpretation: "negligible",
+            },
+        },
+        winner: "tie",
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/benchmark/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAWpC,OAAO,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,cAAc,EAAa,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAaxD;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAAiB,EACjB,MAAuB;IAEvB,kBAAkB;IAClB,MAAM,eAAe,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;IAExD,OAAO;QACL,SAAS;YACP,OAAO,EAAE,GAAG,eAAe,EAAE,CAAC;QAChC,CAAC;QAED,KAAK,CAAC,GAAG;YACP,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,SAAS,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACxD,MAAM,WAAW,GAAuB,EAAE,CAAC;YAE3C,oBAAoB;YACpB,KAAK,MAAM,YAAY,IAAI,eAAe,CAAC,SAAS,EAAE,CAAC;gBACrD,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAC5C,MAAM,EACN,YAAY,EACZ,eAAe,CAChB,CAAC;oBACF,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC/B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO,CAAC,KAAK,CACX,0BAA0B,YAAY,GAAG,EACzC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACjD,CAAC;oBACF,sDAAsD;oBACtD,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE3B,oBAAoB;YACpB,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;YAE9C,OAAO;gBACL,EAAE,EAAE,WAAW;gBACf,SAAS;gBACT,OAAO;gBACP,MAAM,EAAE,eAAe;gBACvB,WAAW;gBACX,OAAO;gBACP,QAAQ,EAAE;oBACR,OAAO,EAAE,OAAO;oBAChB,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,aAAa;oBAClD,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;oBAC9B,WAAW,EAAE,OAAO,CAAC,OAAO;iBAC7B;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,qBAAqB,CAClC,MAAiB,EACjB,YAA0B,EAC1B,MAAuB;IAEvB,MAAM,QAAQ,GAAG,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAEtD,iBAAiB;IACjB,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IAEvB,6BAA6B;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACvC,CAAC;IAED,mCAAmC;IACnC,MAAM,cAAc,GAAqB,EAAE,CAAC;IAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACpD,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,4BAA4B;IAC5B,MAAM,aAAa,GAAG,qBAAqB,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IAE9E,oCAAoC;IACpC,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QACrD,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAED,6BAA6B;IAC7B,MAAM,cAAc,GAAG,qBAAqB,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;IAEjF,WAAW;IACX,MAAM,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAE1B,yBAAyB;IACzB,MAAM,UAAU,GAAG,0BAA0B,CAC3C,aAAa,EACb,cAAc,EACd,MAAM,CAAC,iBAAiB,CACzB,CAAC;IAEF,mBAAmB;IACnB,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAE3C,OAAO;QACL,YAAY;QACZ,aAAa;QACb,cAAc;QACd,UAAU;QACV,MAAM;KACP,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAC5B,IAAsB,EACtB,KAA2C;IAE3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,sBAAsB;IACtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAElD,sCAAsC;IACtC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACrD,MAAM,eAAe,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE7D,MAAM,UAAU,GAAG,oBAAoB,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,oBAAoB,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,oBAAoB,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAE7D,4BAA4B;IAC5B,MAAM,aAAa,GACjB,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAEnE,0CAA0C;IAC1C,MAAM,UAAU,GAAG,UAAU;SAC1B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,KAAK,SAAS,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAkB,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEzE,2DAA2D;IAC3D,oEAAoE;IACpE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEtC,OAAO;QACL,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK;QACL,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,UAAU,EAAE;YACV,UAAU;YACV,UAAU;YACV,UAAU;YACV,aAAa;YACb,WAAW;SACZ;QACD,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,MAAgB,EAAE,CAAS;IACvD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACvD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CACxC,aAA6B,EAC7B,cAA8B,EAC9B,KAAa;IAEb,MAAM,gBAAgB,GAAG,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACvE,MAAM,iBAAiB,GAAG,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEzE,+DAA+D;IAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC7C,MAAM,kBAAkB,GACtB,YAAY,GAAG,CAAC;QACd,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,YAAY,CAAC,GAAG,GAAG;QACrD,CAAC,CAAC,CAAC,CAAC;IAER,8DAA8D;IAC9D,MAAM,aAAa,GAAG,aAAa,CAAC,YAAY;SAC7C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;SAC7B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,MAAM,cAAc,GAAG,cAAc,CAAC,YAAY;SAC/C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;SAC7B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,MAAM,YAAY,GAChB,cAAc,GAAG,CAAC;QAChB,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,aAAa,CAAC,GAAG,cAAc,CAAC,GAAG,GAAG;QAC3D,CAAC,CAAC,CAAC,CAAC;IAER,2BAA2B;IAC3B,MAAM,iBAAiB,GAAG,aAAa,CAAC,OAAO;SAC5C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,KAAK,SAAS,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAkB,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,cAAc,CAAC,OAAO;SAC9C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,KAAK,SAAS,CAAC;SAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAkB,CAAC,CAAC;IAEpC,MAAM,kBAAkB,GACtB,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,mBAAmB,GACvB,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,aAAa,GAAG,kBAAkB,GAAG,mBAAmB,CAAC;IAE/D,sCAAsC;IACtC,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,UAAU,CAAC,gBAAgB,EAAE,iBAAiB,EAAE,KAAK,CAAC,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,oDAAoD;QACpD,mEAAmE;QACnE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,WAAW,GAAG;YACZ,UAAU,EAAE,CAAC;YACb,MAAM,EAAE,CAAC;YACT,gBAAgB,EAAE,EAAE;YACpB,WAAW,EAAE,KAAK;YAClB,KAAK;YACL,cAAc,EAAE,WAAW,GAAG,YAAY;SAC3C,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,cAAc,GAAgD,YAAY,CAAC;IAC/E,IAAI,CAAC;QACH,CAAC,GAAG,OAAO,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;QACjD,cAAc,GAAG,mBAAmB,CAAC,CAAC,CAI3B,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,WAAW;IACb,CAAC;IAED,OAAO;QACL,kBAAkB;QAClB,YAAY;QACZ,aAAa;QACb,WAAW;QACX,UAAU,EAAE;YACV,OAAO,EAAE,CAAC;YACV,cAAc;SACf;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,KAA4B;IACnD,6CAA6C;IAC7C,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,kCAAkC;IAClC,6EAA6E;IAC7E,uDAAuD;IACvD,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;QACrE,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;QACrE,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,+BAA+B;IAC/B,IAAI,KAAK,CAAC,aAAa,GAAG,GAAG,EAAE,CAAC;QAC9B,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,GAAG,EAAE,CAAC;QAC/B,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,WAA+B;IACvD,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CACnC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,gBAAgB,CACrC,CAAC,MAAM,CAAC;IACT,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,iBAAiB,CACtC,CAAC,MAAM,CAAC;IACT,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;IAElE,kEAAkE;IAClE,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;IAC7E,MAAM,kBAAkB,GACtB,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnD,2DAA2D;IAC3D,MAAM,gBAAgB,GAAG,WAAW,CAAC,MAAM,CACzC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,WAAW,CAC5C,CAAC,MAAM,CAAC;IACT,MAAM,wBAAwB,GAAG,gBAAgB,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;IAE3E,OAAO;QACL,cAAc,EAAE,WAAW,CAAC,MAAM;QAClC,UAAU;QACV,YAAY;QACZ,IAAI;QACJ,kBAAkB;QAClB,wBAAwB;KACzB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAC7B,YAA0B,EAC1B,KAAc;IAEd,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxE,MAAM,WAAW,GAAmB;QAClC,YAAY;QACZ,KAAK,EAAE,gBAAgB;QACvB,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QACpE,UAAU,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;QAC3D,YAAY,EAAE,EAAE;QAChB,MAAM,EAAE,CAAC,QAAQ,CAAC;KACnB,CAAC;IAEF,OAAO;QACL,YAAY;QACZ,aAAa,EAAE,EAAE,GAAG,WAAW,EAAE,KAAK,EAAE,gBAAgB,EAAE;QAC1D,cAAc,EAAE,EAAE,GAAG,WAAW,EAAE,KAAK,EAAE,iBAAiB,EAAE;QAC5D,UAAU,EAAE;YACV,kBAAkB,EAAE,CAAC;YACrB,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE;gBACX,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC;gBACT,gBAAgB,EAAE,CAAC;gBACnB,WAAW,EAAE,KAAK;gBAClB,KAAK,EAAE,IAAI;gBACX,cAAc,EAAE,CAAC;aAClB;YACD,UAAU,EAAE;gBACV,OAAO,EAAE,CAAC;gBACV,cAAc,EAAE,YAAY;aAC7B;SACF;QACD,MAAM,EAAE,KAAK;KACd,CAAC;AACJ,CAAC"}

package/dist/benchmark/scenarios/context-recall.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Context Recall Scenario
+ *
+ * Tests the ability to recall facts from previous conversations.
+ * This is the most basic memory scenario - can we remember what was said?
+ */
+import type { ScenarioType, ScenarioResult, BenchmarkPhase } from "../types.js";
+/**
+ * Test case structure for context recall
+ */
+export interface ContextRecallTestCase {
+    /** The fact to store */
+    fact: string;
+    /** The question to ask */
+    question: string;
+    /** Expected keywords in the response */
+    expectedKeywords: string[];
+    /** Category of the fact */
+    category: "personal" | "project" | "preference" | "technical";
+}
+/**
+ * Test data for context recall scenario
+ */
+export declare const CONTEXT_RECALL_TEST_DATA: ContextRecallTestCase[];
+/**
+ * MCP Client interface for memory operations
+ */
+export interface MCPClient {
+    storeMemory(content: string, memoryType: string, metadata?: Record<string, unknown>): Promise<{
+        success: boolean;
+        memory_id: string;
+    }>;
+    retrieveMemory(query: string, options?: {
+        max_results?: number;
+    }): Promise<{
+        success: boolean;
+        results: Array<{
+            content: string;
+            similarity_score: number;
+        }>;
+    }>;
+}
+/**
+ * Context Recall Scenario interface
+ */
+export interface ContextRecallScenario {
+    /** Get scenario type */
+    getType(): ScenarioType;
+    /** Get scenario description */
+    getDescription(): string;
+    /** Setup the scenario (store initial facts) */
+    setup(): Promise<void>;
+    /** Run the scenario in specified phase */
+    run(phase: BenchmarkPhase): Promise<ScenarioResult>;
+    /** Clean up after scenario */
+    teardown(): Promise<void>;
+}
+/**
+ * Create a context recall scenario
+ *
+ * @param client - MCP client for memory operations
+ * @returns ContextRecallScenario instance
+ */
+export declare function createContextRecallScenario(client: MCPClient): ContextRecallScenario;
+//# sourceMappingURL=context-recall.d.ts.map

package/dist/benchmark/scenarios/context-recall.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-recall.d.ts","sourceRoot":"","sources":["../../../src/benchmark/scenarios/context-recall.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EAGf,MAAM,aAAa,CAAC;AAQrB;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,GAAG,SAAS,GAAG,YAAY,GAAG,WAAW,CAAC;CAC/D;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,qBAAqB,EAiD3D,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,WAAW,CACT,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACjC,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAEpD,cAAc,CACZ,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GACjC,OAAO,CAAC;QACT,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,KAAK,CAAC;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,gBAAgB,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAC/D,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,wBAAwB;IACxB,OAAO,IAAI,YAAY,CAAC;IAExB,+BAA+B;IAC/B,cAAc,IAAI,MAAM,CAAC;IAEzB,+CAA+C;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB,0CAA0C;IAC1C,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAEpD,8BAA8B;IAC9B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,SAAS,GAChB,qBAAqB,CA+HvB"}

package/dist/benchmark/scenarios/context-recall.js

@@ -0,0 +1,180 @@
+/**
+ * Context Recall Scenario
+ *
+ * Tests the ability to recall facts from previous conversations.
+ * This is the most basic memory scenario - can we remember what was said?
+ */
+import { measureLatency, evaluateResponseQuality, aggregateMetrics, } from "../metrics-collector.js";
+/**
+ * Test data for context recall scenario
+ */
+export const CONTEXT_RECALL_TEST_DATA = [
+    {
+        fact: "The user prefers TypeScript over JavaScript for type safety",
+        question: "What programming language do I prefer?",
+        expectedKeywords: ["TypeScript", "type safety"],
+        category: "preference",
+    },
+    {
+        fact: "The user is working on a project called Vesper, which is a memory system for AI",
+        question: "What project am I currently working on?",
+        expectedKeywords: ["Vesper", "memory"],
+        category: "project",
+    },
+    {
+        fact: "The user's name is David and they are based in San Francisco",
+        question: "What is my name?",
+        expectedKeywords: ["David"],
+        category: "personal",
+    },
+    {
+        fact: "The user wants test coverage above 80% for all code",
+        question: "What are my testing requirements?",
+        expectedKeywords: ["80%", "coverage", "test"],
+        category: "technical",
+    },
+    {
+        fact: "The user prefers functional programming style over object-oriented",
+        question: "What programming paradigm do I prefer?",
+        expectedKeywords: ["functional"],
+        category: "preference",
+    },
+    {
+        fact: "The user is collaborating with Claude on this project",
+        question: "Who am I collaborating with?",
+        expectedKeywords: ["Claude"],
+        category: "personal",
+    },
+    {
+        fact: "The target latency for the system is under 200ms P95",
+        question: "What is the target latency for the system?",
+        expectedKeywords: ["200ms", "P95", "latency"],
+        category: "technical",
+    },
+    {
+        fact: "The user emphasizes pragmatic solutions over theoretical perfection",
+        question: "What is my approach to problem solving?",
+        expectedKeywords: ["pragmatic"],
+        category: "preference",
+    },
+];
+/**
+ * Create a context recall scenario
+ *
+ * @param client - MCP client for memory operations
+ * @returns ContextRecallScenario instance
+ */
+export function createContextRecallScenario(client) {
+    const errors = [];
+    return {
+        getType() {
+            return "context-recall";
+        },
+        getDescription() {
+            return "Tests ability to recall facts from previous conversations within the same session.";
+        },
+        async setup() {
+            errors.length = 0;
+            // Store all facts as semantic memories
+            for (const testCase of CONTEXT_RECALL_TEST_DATA) {
+                try {
+                    await client.storeMemory(testCase.fact, "semantic", {
+                        category: testCase.category,
+                        timestamp: Date.now(),
+                    });
+                }
+                catch (err) {
+                    const errorMsg = err instanceof Error ? err.message : String(err);
+                    errors.push(`Setup error storing fact: ${errorMsg}`);
+                }
+            }
+        },
+        async run(phase) {
+            const metrics = [];
+            const rawResponses = [];
+            for (const testCase of CONTEXT_RECALL_TEST_DATA) {
+                if (phase === "vesper-enabled") {
+                    // With Vesper enabled: retrieve from memory
+                    try {
+                        const { result, latencyMs } = await measureLatency(async () => {
+                            return await client.retrieveMemory(testCase.question, {
+                                max_results: 3,
+                            });
+                        });
+                        const memoryHit = result.success && result.results.length > 0;
+                        const responseContent = memoryHit
+                            ? result.results[0].content
+                            : "No relevant memory found";
+                        // Evaluate quality
+                        const retrievalAccuracy = memoryHit
+                            ? evaluateResponseQuality(responseContent, testCase.expectedKeywords, {
+                                mode: "keywords",
+                            })
+                            : 0;
+                        metrics.push({
+                            timestamp: Date.now(),
+                            latencyMs,
+                            memoryHit,
+                            retrievalAccuracy,
+                        });
+                        rawResponses.push(responseContent);
+                    }
+                    catch (err) {
+                        const errorMsg = err instanceof Error ? err.message : String(err);
+                        errors.push(`Run error: ${errorMsg}`);
+                        // Record failed attempt
+                        metrics.push({
+                            timestamp: Date.now(),
+                            latencyMs: 0,
+                            memoryHit: false,
+                            retrievalAccuracy: 0,
+                        });
+                        rawResponses.push(`Error: ${errorMsg}`);
+                    }
+                }
+                else {
+                    // With Vesper disabled: simulate no memory access
+                    const startTime = performance.now();
+                    // Simulate baseline latency (no memory lookup, just processing)
+                    await new Promise((resolve) => setTimeout(resolve, 1));
+                    const latencyMs = performance.now() - startTime;
+                    // Without memory, we can't recall the fact
+                    metrics.push({
+                        timestamp: Date.now(),
+                        latencyMs,
+                        memoryHit: false,
+                        retrievalAccuracy: 0,
+                    });
+                    rawResponses.push("Memory disabled - cannot recall");
+                }
+            }
+            // Calculate aggregates
+            let aggregates;
+            try {
+                aggregates = aggregateMetrics(metrics);
+            }
+            catch (err) {
+                // Fallback if aggregation fails
+                aggregates = {
+                    latencyP50: 0,
+                    latencyP95: 0,
+                    latencyP99: 0,
+                    memoryHitRate: 0,
+                };
+            }
+            return {
+                scenarioType: "context-recall",
+                phase,
+                metrics,
+                aggregates,
+                rawResponses,
+                errors: errors.length > 0 ? [...errors] : undefined,
+            };
+        },
+        async teardown() {
+            errors.length = 0;
+            // MCP client cleanup would happen at a higher level
+        },
+    };
+}
+//# sourceMappingURL=context-recall.js.map

package/dist/benchmark/scenarios/context-recall.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-recall.js","sourceRoot":"","sources":["../../../src/benchmark/scenarios/context-recall.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AASH,OAAO,EAEL,cAAc,EACd,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,yBAAyB,CAAC;AAgBjC;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAA4B;IAC/D;QACE,IAAI,EAAE,6DAA6D;QACnE,QAAQ,EAAE,wCAAwC;QAClD,gBAAgB,EAAE,CAAC,YAAY,EAAE,aAAa,CAAC;QAC/C,QAAQ,EAAE,YAAY;KACvB;IACD;QACE,IAAI,EAAE,iFAAiF;QACvF,QAAQ,EAAE,yCAAyC;QACnD,gBAAgB,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QACtC,QAAQ,EAAE,SAAS;KACpB;IACD;QACE,IAAI,EAAE,8DAA8D;QACpE,QAAQ,EAAE,kBAAkB;QAC5B,gBAAgB,EAAE,CAAC,OAAO,CAAC;QAC3B,QAAQ,EAAE,UAAU;KACrB;IACD;QACE,IAAI,EAAE,qDAAqD;QAC3D,QAAQ,EAAE,mCAAmC;QAC7C,gBAAgB,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;QAC7C,QAAQ,EAAE,WAAW;KACtB;IACD;QACE,IAAI,EAAE,oEAAoE;QAC1E,QAAQ,EAAE,wCAAwC;QAClD,gBAAgB,EAAE,CAAC,YAAY,CAAC;QAChC,QAAQ,EAAE,YAAY;KACvB;IACD;QACE,IAAI,EAAE,uDAAuD;QAC7D,QAAQ,EAAE,8BAA8B;QACxC,gBAAgB,EAAE,CAAC,QAAQ,CAAC;QAC5B,QAAQ,EAAE,UAAU;KACrB;IACD;QACE,IAAI,EAAE,sDAAsD;QAC5D,QAAQ,EAAE,4CAA4C;QACtD,gBAAgB,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC;QAC7C,QAAQ,EAAE,WAAW;KACtB;IACD;QACE,IAAI,EAAE,qEAAqE;QAC3E,QAAQ,EAAE,yCAAyC;QACnD,gBAAgB,EAAE,CAAC,WAAW,CAAC;QAC/B,QAAQ,EAAE,YAAY;KACvB;CACF,CAAC;AAyCF;;;;;GAKG;AACH,MAAM,UAAU,2BAA2B,CACzC,MAAiB;IAEjB,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,OAAO;QACL,OAAO;YACL,OAAO,gBAAgB,CAAC;QAC1B,CAAC;QAED,cAAc;YACZ,OAAO,oFAAoF,CAAC;QAC9F,CAAC;QAED,KAAK,CAAC,KAAK;YACT,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YAElB,uCAAuC;YACvC,KAAK,MAAM,QAAQ,IAAI,wBAAwB,EAAE,CAAC;gBAChD,IAAI,CAAC;oBACH,MAAM,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE;wBAClD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAClE,MAAM,CAAC,IAAI,CAAC,6BAA6B,QAAQ,EAAE,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,KAAqB;YAC7B,MAAM,OAAO,GAAsB,EAAE,CAAC;YACtC,MAAM,YAAY,GAAa,EAAE,CAAC;YAElC,KAAK,MAAM,QAAQ,IAAI,wBAAwB,EAAE,CAAC;gBAChD,IAAI,KAAK,KAAK,gBAAgB,EAAE,CAAC;oBAC/B,4CAA4C;oBAC5C,IAAI,CAAC;wBACH,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,cAAc,CAAC,KAAK,IAAI,EAAE;4BAC5D,OAAO,MAAM,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,QAAQ,EAAE;gCACpD,WAAW,EAAE,CAAC;6BACf,CAAC,CAAC;wBACL,CAAC,CAAC,CAAC;wBAEH,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;wBAC9D,MAAM,eAAe,GAAG,SAAS;4BAC/B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;4BAC3B,CAAC,CAAC,0BAA0B,CAAC;wBAE/B,mBAAmB;wBACnB,MAAM,iBAAiB,GAAG,SAAS;4BACjC,CAAC,CAAC,uBAAuB,CAAC,eAAe,EAAE,QAAQ,CAAC,gBAAgB,EAAE;gCAClE,IAAI,EAAE,UAAU;6BACjB,CAAC;4BACJ,CAAC,CAAC,CAAC,CAAC;wBAEN,OAAO,CAAC,IAAI,CAAC;4BACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;4BACrB,SAAS;4BACT,SAAS;4BACT,iBAAiB;yBAClB,CAAC,CAAC;wBAEH,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;oBACrC,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAClE,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;wBAEtC,wBAAwB;wBACxB,OAAO,CAAC,IAAI,CAAC;4BACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;4BACrB,SAAS,EAAE,CAAC;4BACZ,SAAS,EAAE,KAAK;4BAChB,iBAAiB,EAAE,CAAC;yBACrB,CAAC,CAAC;wBAEH,YAAY,CAAC,IAAI,CAAC,UAAU,QAAQ,EAAE,CAAC,CAAC;oBAC1C,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,kDAAkD;oBAClD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;oBAEpC,gEAAgE;oBAChE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;oBAEvD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBAEhD,2CAA2C;oBAC3C,OAAO,CAAC,IAAI,CAAC;wBACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;wBACrB,SAAS;wBACT,SAAS,EAAE,KAAK;wBAChB,iBAAiB,EAAE,CAAC;qBACrB,CAAC,CAAC;oBAEH,YAAY,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;YAED,uBAAuB;YACvB,IAAI,UAA4B,CAAC;YACjC,IAAI,CAAC;gBACH,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,gCAAgC;gBAChC,UAAU,GAAG;oBACX,UAAU,EAAE,CAAC;oBACb,UAAU,EAAE,CAAC;oBACb,UAAU,EAAE,CAAC;oBACb,aAAa,EAAE,CAAC;iBACjB,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,YAAY,EAAE,gBAAgB;gBAC9B,KAAK;gBACL,OAAO;gBACP,UAAU;gBACV,YAAY;gBACZ,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;aACpD,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,QAAQ;YACZ,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YAClB,oDAAoD;QACtD,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/benchmark/scenarios/index.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Scenario Module Index
+ *
+ * Exports all scenario implementations and factory functions.
+ */
+export { createContextRecallScenario, CONTEXT_RECALL_TEST_DATA } from "./context-recall.js";
+export type { ContextRecallScenario, ContextRecallTestCase, MCPClient, } from "./context-recall.js";
+import type { ScenarioType, ScenarioResult, BenchmarkPhase } from "../types.js";
+import { MCPClient } from "./context-recall.js";
+/**
+ * Common scenario interface
+ */
+export interface BenchmarkScenario {
+    /** Get scenario type */
+    getType(): ScenarioType;
+    /** Get scenario description */
+    getDescription(): string;
+    /** Setup the scenario */
+    setup(): Promise<void>;
+    /** Run the scenario in specified phase */
+    run(phase: BenchmarkPhase): Promise<ScenarioResult>;
+    /** Clean up after scenario */
+    teardown(): Promise<void>;
+}
+/**
+ * Create a scenario by type
+ *
+ * @param type - The scenario type to create
+ * @param client - MCP client for memory operations
+ * @returns Scenario instance
+ * @throws Error if scenario type is not supported
+ */
+export declare function createScenario(type: ScenarioType, client: MCPClient): BenchmarkScenario;
+//# sourceMappingURL=index.d.ts.map

package/dist/benchmark/scenarios/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/benchmark/scenarios/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,2BAA2B,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC5F,YAAY,EACV,qBAAqB,EACrB,qBAAqB,EACrB,SAAS,GACV,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAChF,OAAO,EAA+B,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wBAAwB;IACxB,OAAO,IAAI,YAAY,CAAC;IAExB,+BAA+B;IAC/B,cAAc,IAAI,MAAM,CAAC;IAEzB,yBAAyB;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB,0CAA0C;IAC1C,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAEpD,8BAA8B;IAC9B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,YAAY,EAClB,MAAM,EAAE,SAAS,GAChB,iBAAiB,CAoBnB"}