vesper-memory 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +28 -0
- package/CHANGELOG.md +137 -0
- package/LICENSE +21 -0
- package/README.md +665 -0
- package/config/sqlite-schema.sql +413 -0
- package/dist/benchmark/index.d.ts +18 -0
- package/dist/benchmark/index.d.ts.map +1 -0
- package/dist/benchmark/index.js +19 -0
- package/dist/benchmark/index.js.map +1 -0
- package/dist/benchmark/metrics-collector.d.ts +90 -0
- package/dist/benchmark/metrics-collector.d.ts.map +1 -0
- package/dist/benchmark/metrics-collector.js +196 -0
- package/dist/benchmark/metrics-collector.js.map +1 -0
- package/dist/benchmark/report-generator.d.ts +35 -0
- package/dist/benchmark/report-generator.d.ts.map +1 -0
- package/dist/benchmark/report-generator.js +221 -0
- package/dist/benchmark/report-generator.js.map +1 -0
- package/dist/benchmark/runner.d.ts +30 -0
- package/dist/benchmark/runner.d.ts.map +1 -0
- package/dist/benchmark/runner.js +308 -0
- package/dist/benchmark/runner.js.map +1 -0
- package/dist/benchmark/scenarios/context-recall.d.ts +65 -0
- package/dist/benchmark/scenarios/context-recall.d.ts.map +1 -0
- package/dist/benchmark/scenarios/context-recall.js +180 -0
- package/dist/benchmark/scenarios/context-recall.js.map +1 -0
- package/dist/benchmark/scenarios/index.d.ts +34 -0
- package/dist/benchmark/scenarios/index.d.ts.map +1 -0
- package/dist/benchmark/scenarios/index.js +230 -0
- package/dist/benchmark/scenarios/index.js.map +1 -0
- package/dist/benchmark/statistics.d.ts +125 -0
- package/dist/benchmark/statistics.d.ts.map +1 -0
- package/dist/benchmark/statistics.js +520 -0
- package/dist/benchmark/statistics.js.map +1 -0
- package/dist/benchmark/types.d.ts +458 -0
- package/dist/benchmark/types.d.ts.map +1 -0
- package/dist/benchmark/types.js +297 -0
- package/dist/benchmark/types.js.map +1 -0
- package/dist/cli 2.js +583 -0
- package/dist/cli.d 2.ts +8 -0
- package/dist/cli.d.ts +8 -0
- package/dist/cli.d.ts 2.map +1 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +583 -0
- package/dist/cli.js 2.map +1 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/rate-limits.d.ts +76 -0
- package/dist/config/rate-limits.d.ts.map +1 -0
- package/dist/config/rate-limits.js +161 -0
- package/dist/config/rate-limits.js.map +1 -0
- package/dist/config 2/rate-limits.d.ts +76 -0
- package/dist/config 2/rate-limits.d.ts.map +1 -0
- package/dist/config 2/rate-limits.js +161 -0
- package/dist/config 2/rate-limits.js.map +1 -0
- package/dist/consolidation/pipeline.d.ts +52 -0
- package/dist/consolidation/pipeline.d.ts.map +1 -0
- package/dist/consolidation/pipeline.js +193 -0
- package/dist/consolidation/pipeline.js.map +1 -0
- package/dist/consolidation 2/pipeline.d.ts +52 -0
- package/dist/consolidation 2/pipeline.d.ts.map +1 -0
- package/dist/consolidation 2/pipeline.js +193 -0
- package/dist/consolidation 2/pipeline.js.map +1 -0
- package/dist/embeddings/client.d.ts +95 -0
- package/dist/embeddings/client.d.ts.map +1 -0
- package/dist/embeddings/client.js +162 -0
- package/dist/embeddings/client.js.map +1 -0
- package/dist/embeddings 2/client.d.ts +94 -0
- package/dist/embeddings 2/client.d.ts.map +1 -0
- package/dist/embeddings 2/client.js +144 -0
- package/dist/embeddings 2/client.js.map +1 -0
- package/dist/memory-layers/semantic-memory.d.ts +76 -0
- package/dist/memory-layers/semantic-memory.d.ts.map +1 -0
- package/dist/memory-layers/semantic-memory.js +231 -0
- package/dist/memory-layers/semantic-memory.js.map +1 -0
- package/dist/memory-layers/skill-library.d.ts +25 -0
- package/dist/memory-layers/skill-library.d.ts.map +1 -0
- package/dist/memory-layers/skill-library.js +54 -0
- package/dist/memory-layers/skill-library.js.map +1 -0
- package/dist/memory-layers/working-memory.d.ts +57 -0
- package/dist/memory-layers/working-memory.d.ts.map +1 -0
- package/dist/memory-layers/working-memory.js +153 -0
- package/dist/memory-layers/working-memory.js.map +1 -0
- package/dist/memory-layers 2/semantic-memory.d.ts +33 -0
- package/dist/memory-layers 2/semantic-memory.d.ts.map +1 -0
- package/dist/memory-layers 2/semantic-memory.js +114 -0
- package/dist/memory-layers 2/semantic-memory.js.map +1 -0
- package/dist/memory-layers 2/skill-library.d.ts +25 -0
- package/dist/memory-layers 2/skill-library.d.ts.map +1 -0
- package/dist/memory-layers 2/skill-library.js +54 -0
- package/dist/memory-layers 2/skill-library.js.map +1 -0
- package/dist/memory-layers 2/working-memory.d.ts +57 -0
- package/dist/memory-layers 2/working-memory.d.ts.map +1 -0
- package/dist/memory-layers 2/working-memory.js +153 -0
- package/dist/memory-layers 2/working-memory.js.map +1 -0
- package/dist/middleware/auth.d.ts +287 -0
- package/dist/middleware/auth.d.ts.map +1 -0
- package/dist/middleware/auth.js +610 -0
- package/dist/middleware/auth.js.map +1 -0
- package/dist/middleware 2/auth.d.ts +287 -0
- package/dist/middleware 2/auth.d.ts.map +1 -0
- package/dist/middleware 2/auth.js +610 -0
- package/dist/middleware 2/auth.js.map +1 -0
- package/dist/monitoring/health.d.ts +241 -0
- package/dist/monitoring/health.d.ts.map +1 -0
- package/dist/monitoring/health.js +649 -0
- package/dist/monitoring/health.js.map +1 -0
- package/dist/monitoring/metrics.d.ts +153 -0
- package/dist/monitoring/metrics.d.ts.map +1 -0
- package/dist/monitoring/metrics.js +349 -0
- package/dist/monitoring/metrics.js.map +1 -0
- package/dist/monitoring 2/health.d.ts +241 -0
- package/dist/monitoring 2/health.d.ts.map +1 -0
- package/dist/monitoring 2/health.js +649 -0
- package/dist/monitoring 2/health.js.map +1 -0
- package/dist/monitoring 2/metrics.d.ts +153 -0
- package/dist/monitoring 2/metrics.d.ts.map +1 -0
- package/dist/monitoring 2/metrics.js +349 -0
- package/dist/monitoring 2/metrics.js.map +1 -0
- package/dist/retrieval/hybrid-search.d.ts +186 -0
- package/dist/retrieval/hybrid-search.d.ts.map +1 -0
- package/dist/retrieval/hybrid-search.js +330 -0
- package/dist/retrieval/hybrid-search.js.map +1 -0
- package/dist/retrieval 2/hybrid-search.d.ts +186 -0
- package/dist/retrieval 2/hybrid-search.d.ts.map +1 -0
- package/dist/retrieval 2/hybrid-search.js +327 -0
- package/dist/retrieval 2/hybrid-search.js.map +1 -0
- package/dist/router/smart-router.d.ts +157 -0
- package/dist/router/smart-router.d.ts.map +1 -0
- package/dist/router/smart-router.js +524 -0
- package/dist/router/smart-router.js.map +1 -0
- package/dist/router 2/smart-router.d.ts +111 -0
- package/dist/router 2/smart-router.d.ts.map +1 -0
- package/dist/router 2/smart-router.js +383 -0
- package/dist/router 2/smart-router.js.map +1 -0
- package/dist/security/rate-limit-middleware.d.ts +119 -0
- package/dist/security/rate-limit-middleware.d.ts.map +1 -0
- package/dist/security/rate-limit-middleware.js +193 -0
- package/dist/security/rate-limit-middleware.js.map +1 -0
- package/dist/security 2/rate-limit-middleware.d.ts +119 -0
- package/dist/security 2/rate-limit-middleware.d.ts.map +1 -0
- package/dist/security 2/rate-limit-middleware.js +193 -0
- package/dist/security 2/rate-limit-middleware.js.map +1 -0
- package/dist/server 2.js +861 -0
- package/dist/server-wrapper 2.js +170 -0
- package/dist/server-wrapper.d 2.ts +12 -0
- package/dist/server-wrapper.d.ts +12 -0
- package/dist/server-wrapper.d.ts 2.map +1 -0
- package/dist/server-wrapper.d.ts.map +1 -0
- package/dist/server-wrapper.js +170 -0
- package/dist/server-wrapper.js 2.map +1 -0
- package/dist/server-wrapper.js.map +1 -0
- package/dist/server.d 2.ts +14 -0
- package/dist/server.d.ts +14 -0
- package/dist/server.d.ts 2.map +1 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +656 -0
- package/dist/server.js 2.map +1 -0
- package/dist/server.js.map +1 -0
- package/dist/synthesis/conflict-detector.d.ts +43 -0
- package/dist/synthesis/conflict-detector.d.ts.map +1 -0
- package/dist/synthesis/conflict-detector.js +125 -0
- package/dist/synthesis/conflict-detector.js.map +1 -0
- package/dist/synthesis 2/conflict-detector.d.ts +43 -0
- package/dist/synthesis 2/conflict-detector.d.ts.map +1 -0
- package/dist/synthesis 2/conflict-detector.js +125 -0
- package/dist/synthesis 2/conflict-detector.js.map +1 -0
- package/dist/utils/monitoring.d.ts +25 -0
- package/dist/utils/monitoring.d.ts.map +1 -0
- package/dist/utils/monitoring.js +41 -0
- package/dist/utils/monitoring.js.map +1 -0
- package/dist/utils/rate-limiter.d.ts +97 -0
- package/dist/utils/rate-limiter.d.ts.map +1 -0
- package/dist/utils/rate-limiter.js +166 -0
- package/dist/utils/rate-limiter.js.map +1 -0
- package/dist/utils/validation.d.ts +139 -0
- package/dist/utils/validation.d.ts.map +1 -0
- package/dist/utils/validation.js +195 -0
- package/dist/utils/validation.js.map +1 -0
- package/dist/utils 2/monitoring.d.ts +25 -0
- package/dist/utils 2/monitoring.d.ts.map +1 -0
- package/dist/utils 2/monitoring.js +41 -0
- package/dist/utils 2/monitoring.js.map +1 -0
- package/dist/utils 2/rate-limiter.d.ts +97 -0
- package/dist/utils 2/rate-limiter.d.ts.map +1 -0
- package/dist/utils 2/rate-limiter.js +166 -0
- package/dist/utils 2/rate-limiter.js.map +1 -0
- package/dist/utils 2/validation.d.ts +139 -0
- package/dist/utils 2/validation.d.ts.map +1 -0
- package/dist/utils 2/validation.js +195 -0
- package/dist/utils 2/validation.js.map +1 -0
- package/docker-compose.yml +65 -0
- package/embedding-service/Dockerfile +29 -0
- package/embedding-service/README.md +68 -0
- package/embedding-service/requirements.txt +6 -0
- package/embedding-service/server.py +110 -0
- package/package.json +92 -0
- package/scripts/FRESH_INSTALL.md +280 -0
- package/scripts/README.md +101 -0
- package/scripts/ensure-infrastructure.sh +38 -0
- package/scripts/fresh-install-auto.sh +96 -0
- package/scripts/fresh-install.sh +167 -0
- package/scripts/health-check.sh +72 -0
- package/scripts/init-qdrant.ts +201 -0
- package/scripts/postinstall.js +94 -0
- package/scripts/run-accuracy-benchmark.ts +1237 -0
- package/scripts/run-real-benchmark.ts +581 -0
- package/scripts/test-mcp-server.sh +38 -0
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Type Definitions for Scientific Benchmarks
|
|
3
|
+
*
|
|
4
|
+
* Comprehensive type system with runtime validation using Zod.
|
|
5
|
+
* Ensures type safety at compile time and runtime.
|
|
6
|
+
*/
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
// =============================================================================
|
|
9
|
+
// Scenario Types
|
|
10
|
+
// =============================================================================
|
|
11
|
+
/**
|
|
12
|
+
* Available benchmark scenario types
|
|
13
|
+
*/
|
|
14
|
+
export const ScenarioTypeEnum = z.enum([
|
|
15
|
+
"context-recall",
|
|
16
|
+
"cross-session",
|
|
17
|
+
"skill-retrieval",
|
|
18
|
+
"token-efficiency",
|
|
19
|
+
"semantic-accuracy",
|
|
20
|
+
]);
|
|
21
|
+
/**
|
|
22
|
+
* Benchmark phases for A/B comparison
|
|
23
|
+
*/
|
|
24
|
+
export const BenchmarkPhaseEnum = z.enum(["vesper-enabled", "vesper-disabled"]);
|
|
25
|
+
/**
|
|
26
|
+
* Winner determination
|
|
27
|
+
*/
|
|
28
|
+
export const WinnerEnum = z.enum([
|
|
29
|
+
"vesper-enabled",
|
|
30
|
+
"vesper-disabled",
|
|
31
|
+
"tie",
|
|
32
|
+
]);
|
|
33
|
+
// =============================================================================
|
|
34
|
+
// Benchmark Configuration
|
|
35
|
+
// =============================================================================
|
|
36
|
+
/**
|
|
37
|
+
* Configuration for running benchmarks
|
|
38
|
+
*/
|
|
39
|
+
export const BenchmarkConfigSchema = z.object({
|
|
40
|
+
/** Number of warmup runs before measurement (to prime caches) */
|
|
41
|
+
warmupRuns: z.number().int().min(0).max(100).default(3),
|
|
42
|
+
/** Number of measurement runs (min 2 for statistical significance) */
|
|
43
|
+
measurementRuns: z.number().int().min(2).max(1000).default(10),
|
|
44
|
+
/** Timeout per scenario run in milliseconds (max 10 minutes) */
|
|
45
|
+
timeoutMs: z.number().int().positive().max(600000).default(30000),
|
|
46
|
+
/** Scenarios to run */
|
|
47
|
+
scenarios: z.array(ScenarioTypeEnum).min(1),
|
|
48
|
+
/** Output directory for reports */
|
|
49
|
+
outputDirectory: z.string().min(1).default("./benchmark-results"),
|
|
50
|
+
/** Statistical significance level (alpha) */
|
|
51
|
+
significanceLevel: z.number().gt(0).lt(1).default(0.05),
|
|
52
|
+
});
|
|
53
|
+
// =============================================================================
|
|
54
|
+
// Metrics Snapshot
|
|
55
|
+
// =============================================================================
|
|
56
|
+
/**
|
|
57
|
+
* A single metrics measurement point
|
|
58
|
+
*/
|
|
59
|
+
export const MetricsSnapshotSchema = z.object({
|
|
60
|
+
/** Timestamp of measurement */
|
|
61
|
+
timestamp: z.number().int().positive(),
|
|
62
|
+
/** Latency in milliseconds */
|
|
63
|
+
latencyMs: z.number().nonnegative(),
|
|
64
|
+
/** Number of tokens used (optional) */
|
|
65
|
+
tokensUsed: z.number().int().nonnegative().optional(),
|
|
66
|
+
/** Retrieval accuracy 0-1 (optional) */
|
|
67
|
+
retrievalAccuracy: z.number().min(0).max(1).optional(),
|
|
68
|
+
/** Whether memory was hit */
|
|
69
|
+
memoryHit: z.boolean(),
|
|
70
|
+
/** Response quality score 0-1 (optional) */
|
|
71
|
+
responseQuality: z.number().min(0).max(1).optional(),
|
|
72
|
+
/** Additional metadata (optional) */
|
|
73
|
+
metadata: z.record(z.string(), z.unknown()).optional(),
|
|
74
|
+
});
|
|
75
|
+
// =============================================================================
|
|
76
|
+
// Aggregated Metrics
|
|
77
|
+
// =============================================================================
|
|
78
|
+
/**
|
|
79
|
+
* Aggregated metrics from multiple runs
|
|
80
|
+
*/
|
|
81
|
+
export const AggregateMetricsSchema = z.object({
|
|
82
|
+
/** 50th percentile latency */
|
|
83
|
+
latencyP50: z.number().nonnegative(),
|
|
84
|
+
/** 95th percentile latency */
|
|
85
|
+
latencyP95: z.number().nonnegative(),
|
|
86
|
+
/** 99th percentile latency */
|
|
87
|
+
latencyP99: z.number().nonnegative(),
|
|
88
|
+
/** Average tokens used (optional) */
|
|
89
|
+
avgTokens: z.number().nonnegative().optional(),
|
|
90
|
+
/** Average retrieval accuracy (optional) */
|
|
91
|
+
avgAccuracy: z.number().min(0).max(1).optional(),
|
|
92
|
+
/** Memory hit rate (optional) */
|
|
93
|
+
memoryHitRate: z.number().min(0).max(1).optional(),
|
|
94
|
+
/** Mean latency (optional) */
|
|
95
|
+
latencyMean: z.number().nonnegative().optional(),
|
|
96
|
+
/** Standard deviation (optional) */
|
|
97
|
+
latencyStdDev: z.number().nonnegative().optional(),
|
|
98
|
+
});
|
|
99
|
+
// =============================================================================
|
|
100
|
+
// Scenario Result
|
|
101
|
+
// =============================================================================
|
|
102
|
+
/**
|
|
103
|
+
* Result from running a single scenario in one phase
|
|
104
|
+
*/
|
|
105
|
+
export const ScenarioResultSchema = z.object({
|
|
106
|
+
/** Scenario type */
|
|
107
|
+
scenarioType: ScenarioTypeEnum,
|
|
108
|
+
/** Which phase this result is from */
|
|
109
|
+
phase: BenchmarkPhaseEnum,
|
|
110
|
+
/** Individual metric snapshots (at least 1) */
|
|
111
|
+
metrics: z.array(MetricsSnapshotSchema).min(1),
|
|
112
|
+
/** Aggregated statistics */
|
|
113
|
+
aggregates: AggregateMetricsSchema,
|
|
114
|
+
/** Raw responses for quality analysis */
|
|
115
|
+
rawResponses: z.array(z.string()),
|
|
116
|
+
/** Error messages if any */
|
|
117
|
+
errors: z.array(z.string()).optional(),
|
|
118
|
+
});
|
|
119
|
+
// =============================================================================
|
|
120
|
+
// Statistical Results
|
|
121
|
+
// =============================================================================
|
|
122
|
+
/**
|
|
123
|
+
* T-test result structure
|
|
124
|
+
*/
|
|
125
|
+
export const TTestResultSchema = z.object({
|
|
126
|
+
tStatistic: z.number(),
|
|
127
|
+
pValue: z.number().min(0).max(1),
|
|
128
|
+
degreesOfFreedom: z.number().nonnegative(),
|
|
129
|
+
significant: z.boolean(),
|
|
130
|
+
alpha: z.number().gt(0).lt(1),
|
|
131
|
+
meanDifference: z.number(),
|
|
132
|
+
});
|
|
133
|
+
/**
|
|
134
|
+
* Effect size result
|
|
135
|
+
*/
|
|
136
|
+
export const EffectSizeSchema = z.object({
|
|
137
|
+
cohensD: z.number(),
|
|
138
|
+
interpretation: z.enum(["negligible", "small", "medium", "large"]),
|
|
139
|
+
});
|
|
140
|
+
/**
|
|
141
|
+
* Statistical comparison between enabled and disabled
|
|
142
|
+
*/
|
|
143
|
+
export const StatisticalComparisonSchema = z.object({
|
|
144
|
+
/** Latency improvement percentage (positive = vesper is faster) */
|
|
145
|
+
latencyImprovement: z.number(),
|
|
146
|
+
/** Token savings percentage (positive = vesper uses fewer) */
|
|
147
|
+
tokenSavings: z.number(),
|
|
148
|
+
/** Accuracy difference (positive = vesper is more accurate) */
|
|
149
|
+
accuracyDelta: z.number(),
|
|
150
|
+
/** Welch's t-test result */
|
|
151
|
+
tTestResult: TTestResultSchema,
|
|
152
|
+
/** Cohen's d effect size */
|
|
153
|
+
effectSize: EffectSizeSchema,
|
|
154
|
+
});
|
|
155
|
+
// =============================================================================
|
|
156
|
+
// Comparison Result
|
|
157
|
+
// =============================================================================
|
|
158
|
+
/**
|
|
159
|
+
* Comparison between Vesper-enabled and Vesper-disabled for a scenario
|
|
160
|
+
*/
|
|
161
|
+
export const ComparisonResultSchema = z.object({
|
|
162
|
+
/** Scenario type */
|
|
163
|
+
scenarioType: ScenarioTypeEnum,
|
|
164
|
+
/** Results with Vesper enabled */
|
|
165
|
+
enabledResult: ScenarioResultSchema,
|
|
166
|
+
/** Results with Vesper disabled */
|
|
167
|
+
disabledResult: ScenarioResultSchema,
|
|
168
|
+
/** Statistical comparison */
|
|
169
|
+
statistics: StatisticalComparisonSchema,
|
|
170
|
+
/** Winner of this scenario */
|
|
171
|
+
winner: WinnerEnum,
|
|
172
|
+
});
|
|
173
|
+
// =============================================================================
|
|
174
|
+
// Benchmark Summary
|
|
175
|
+
// =============================================================================
|
|
176
|
+
/**
|
|
177
|
+
* Summary of all benchmark results
|
|
178
|
+
*/
|
|
179
|
+
export const BenchmarkSummarySchema = z
|
|
180
|
+
.object({
|
|
181
|
+
/** Total number of scenarios run */
|
|
182
|
+
totalScenarios: z.number().int().nonnegative(),
|
|
183
|
+
/** Scenarios where Vesper won */
|
|
184
|
+
vesperWins: z.number().int().nonnegative(),
|
|
185
|
+
/** Scenarios where disabled won */
|
|
186
|
+
disabledWins: z.number().int().nonnegative(),
|
|
187
|
+
/** Scenarios that were ties */
|
|
188
|
+
ties: z.number().int().nonnegative(),
|
|
189
|
+
/** Overall improvement percentage */
|
|
190
|
+
overallImprovement: z.number(),
|
|
191
|
+
/** Whether results are statistically significant */
|
|
192
|
+
statisticallySignificant: z.boolean(),
|
|
193
|
+
})
|
|
194
|
+
.refine((data) => data.vesperWins + data.disabledWins + data.ties === data.totalScenarios, {
|
|
195
|
+
message: "Wins and ties must add up to total scenarios",
|
|
196
|
+
});
|
|
197
|
+
// =============================================================================
|
|
198
|
+
// Full Benchmark Result
|
|
199
|
+
// =============================================================================
|
|
200
|
+
/**
|
|
201
|
+
* Complete benchmark result with all data
|
|
202
|
+
*/
|
|
203
|
+
export const BenchmarkResultSchema = z
|
|
204
|
+
.object({
|
|
205
|
+
/** Unique identifier for this benchmark run */
|
|
206
|
+
id: z.string().min(1),
|
|
207
|
+
/** Start timestamp */
|
|
208
|
+
startTime: z.number().int().positive(),
|
|
209
|
+
/** End timestamp */
|
|
210
|
+
endTime: z.number().int().positive(),
|
|
211
|
+
/** Configuration used */
|
|
212
|
+
config: BenchmarkConfigSchema,
|
|
213
|
+
/** Per-scenario comparisons */
|
|
214
|
+
comparisons: z.array(ComparisonResultSchema),
|
|
215
|
+
/** Summary statistics */
|
|
216
|
+
summary: BenchmarkSummarySchema,
|
|
217
|
+
/** Metadata */
|
|
218
|
+
metadata: z.object({
|
|
219
|
+
version: z.string(),
|
|
220
|
+
environment: z.string(),
|
|
221
|
+
hostname: z.string().optional(),
|
|
222
|
+
nodeVersion: z.string().optional(),
|
|
223
|
+
}),
|
|
224
|
+
})
|
|
225
|
+
.refine((data) => data.endTime >= data.startTime, {
|
|
226
|
+
message: "End time must be after or equal to start time",
|
|
227
|
+
});
|
|
228
|
+
// =============================================================================
|
|
229
|
+
// Helper Functions
|
|
230
|
+
// =============================================================================
|
|
231
|
+
/**
|
|
232
|
+
* Create a default benchmark configuration
|
|
233
|
+
*/
|
|
234
|
+
export function createDefaultConfig(overrides) {
|
|
235
|
+
const defaults = {
|
|
236
|
+
warmupRuns: 3,
|
|
237
|
+
measurementRuns: 10,
|
|
238
|
+
timeoutMs: 30000,
|
|
239
|
+
scenarios: [
|
|
240
|
+
"context-recall",
|
|
241
|
+
"cross-session",
|
|
242
|
+
"skill-retrieval",
|
|
243
|
+
"token-efficiency",
|
|
244
|
+
"semantic-accuracy",
|
|
245
|
+
],
|
|
246
|
+
outputDirectory: "./benchmark-results",
|
|
247
|
+
significanceLevel: 0.05,
|
|
248
|
+
};
|
|
249
|
+
if (!overrides) {
|
|
250
|
+
return defaults;
|
|
251
|
+
}
|
|
252
|
+
return {
|
|
253
|
+
...defaults,
|
|
254
|
+
...overrides,
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Validate and parse a benchmark configuration
|
|
259
|
+
* @throws ZodError if validation fails
|
|
260
|
+
*/
|
|
261
|
+
export function validateBenchmarkConfig(input) {
|
|
262
|
+
return BenchmarkConfigSchema.parse(input);
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Validate and parse a benchmark result
|
|
266
|
+
* @throws ZodError if validation fails
|
|
267
|
+
*/
|
|
268
|
+
export function validateBenchmarkResult(input) {
|
|
269
|
+
return BenchmarkResultSchema.parse(input);
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Validate and parse a metrics snapshot
|
|
273
|
+
* @throws ZodError if validation fails
|
|
274
|
+
*/
|
|
275
|
+
export function validateMetricsSnapshot(input) {
|
|
276
|
+
return MetricsSnapshotSchema.parse(input);
|
|
277
|
+
}
|
|
278
|
+
// =============================================================================
|
|
279
|
+
// Scenario Descriptions (for documentation and reports)
|
|
280
|
+
// =============================================================================
|
|
281
|
+
/**
|
|
282
|
+
* Human-readable descriptions of each scenario
|
|
283
|
+
*/
|
|
284
|
+
export const SCENARIO_DESCRIPTIONS = {
|
|
285
|
+
"context-recall": "Tests ability to recall facts from previous conversations within the same session.",
|
|
286
|
+
"cross-session": "Tests persistence of context across multiple conversation sessions.",
|
|
287
|
+
"skill-retrieval": "Tests retrieval of learned procedural knowledge (how to do things).",
|
|
288
|
+
"token-efficiency": "Measures token usage reduction from avoiding repeated context.",
|
|
289
|
+
"semantic-accuracy": "Tests semantic search accuracy for finding relevant memories.",
|
|
290
|
+
};
|
|
291
|
+
/**
|
|
292
|
+
* Get description for a scenario
|
|
293
|
+
*/
|
|
294
|
+
export function getScenarioDescription(scenario) {
|
|
295
|
+
return SCENARIO_DESCRIPTIONS[scenario];
|
|
296
|
+
}
|
|
297
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/benchmark/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,gFAAgF;AAChF,iBAAiB;AACjB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,IAAI,CAAC;IACrC,gBAAgB;IAChB,eAAe;IACf,iBAAiB;IACjB,kBAAkB;IAClB,mBAAmB;CACpB,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC,CAAC;AAIhF;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC;IAC/B,gBAAgB;IAChB,iBAAiB;IACjB,KAAK;CACN,CAAC,CAAC;AAIH,gFAAgF;AAChF,0BAA0B;AAC1B,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,iEAAiE;IACjE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAEvD,sEAAsE;IACtE,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAE9D,gEAAgE;IAChE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAEjE,uBAAuB;IACvB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE3C,mCAAmC;IACnC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;IAEjE,6CAA6C;IAC7C,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;CACxD,CAAC,CAAC;AAIH,gFAAgF;AAChF,mBAAmB;AACnB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,+BAA+B;IAC/B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IAEtC,8BAA8B;IAC9B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IAEnC,uCAAuC;IACvC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAErD,wCAAwC;IACxC,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAEtD,6BAA6B;IAC7B,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;IAEtB,4CAA4C;IAC5C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAEpD,qCAAqC;IACrC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAIH,gFAAgF;AAChF,qBAAqB;AACrB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,8BAA8B;IAC9B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IAEpC,8BAA8B;IAC9B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IAEpC,8BAA8B;IAC9B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IAEpC,qCAAqC;IACrC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAE9C,4CAA4C;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAEhD,iCAAiC;IACjC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAElD,8BAA8B;IAC9B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAEhD,oCAAoC;IACpC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC;AAIH,gFAAgF;AAChF,kBAAkB;AAClB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,oBAAoB;IACpB,YAAY,EAAE,gBAAgB;IAE9B,sCAAsC;IACtC,KAAK,EAAE,kBAAkB;IAEzB,+CAA+C;IAC/C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE9C,4BAA4B;IAC5B,UAAU,EAAE,sBAAsB;IAElC,yCAAyC;IACzC,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAEjC,4BAA4B;IAC5B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvC,CAAC,CAAC;AAIH,gFAAgF;AAChF,sBAAsB;AACtB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IAC1C,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE;IACxB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;CAC3B,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;CACnE,CAAC,CAAC;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,mEAAmE;IACnE,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;IAE9B,8DAA8D;IAC9D,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IAExB,+DAA+D;IAC/D,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;IAEzB,4BAA4B;IAC5B,WAAW,EAAE,iBAAiB;IAE9B,4BAA4B;IAC5B,UAAU,EAAE,gBAAgB;CAC7B,CAAC,CAAC;AAIH,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,oBAAoB;IACpB,YAAY,EAAE,gBAAgB;IAE9B,kCAAkC;IAClC,aAAa,EAAE,oBAAoB;IAEnC,mCAAmC;IACnC,cAAc,EAAE,oBAAoB;IAEpC,6BAA6B;IAC7B,UAAU,EAAE,2BAA2B;IAEvC,8BAA8B;IAC9B,MAAM,EAAE,UAAU;CACnB,CAAC,CAAC;AAIH,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,oCAAoC;IACpC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAE9C,iCAAiC;IACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAE1C,mCAAmC;IACnC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAE5C,+BAA+B;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAEpC,qCAAqC;IACrC,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;IAE9B,oDAAoD;IACpD,wBAAwB,EAAE,CAAC,CAAC,OAAO,EAAE;CACtC,CAAC;KACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,cAAc,EAAE;IACzF,OAAO,EAAE,8CAA8C;CACxD,CAAC,CAAC;AAIL,gFAAgF;AAChF,wBAAwB;AACxB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC;KACnC,MAAM,CAAC;IACN,+CAA+C;IAC/C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAErB,sBAAsB;IACtB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IAEtC,oBAAoB;IACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IAEpC,yBAAyB;IACzB,MAAM,EAAE,qBAAqB;IAE7B,+BAA+B;IAC/B,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC;IAE5C,yBAAyB;IACzB,OAAO,EAAE,sBAAsB;IAE/B,eAAe;IACf,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;QACjB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KACnC,CAAC;CACH,CAAC;KACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE;IAChD,OAAO,EAAE,+CAA+C;CACzD,CAAC,CAAC;AAIL,gFAAgF;AAChF,mBAAmB;AACnB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,SAAoC;IAEpC,MAAM,QAAQ,GAAoB;QAChC,UAAU,EAAE,CAAC;QACb,eAAe,EAAE,EAAE;QACnB,SAAS,EAAE,KAAK;QAChB,SAAS,EAAE;YACT,gBAAgB;YAChB,eAAe;YACf,iBAAiB;YACjB,kBAAkB;YAClB,mBAAmB;SACpB;QACD,eAAe,EAAE,qBAAqB;QACtC,iBAAiB,EAAE,IAAI;KACxB,CAAC;IAEF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO;QACL,GAAG,QAAQ;QACX,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACpD,OAAO,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACpD,OAAO,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACpD,OAAO,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED,gFAAgF;AAChF,wDAAwD;AACxD,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAiC;IACjE,gBAAgB,EACd,oFAAoF;IACtF,eAAe,EACb,qEAAqE;IACvE,iBAAiB,EACf,qEAAqE;IACvE,kBAAkB,EAChB,gEAAgE;IAClE,mBAAmB,EACjB,+DAA+D;CAClE,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAsB;IAC3D,OAAO,qBAAqB,CAAC,QAAQ,CAAC,CAAC;AACzC,CAAC"}
|