zouroboros-memory 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/capture.d.ts +57 -0
- package/dist/capture.js +181 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +91 -0
- package/dist/conflict-resolver.d.ts +55 -0
- package/dist/conflict-resolver.js +221 -0
- package/dist/context-budget.d.ts +94 -0
- package/dist/context-budget.js +272 -0
- package/dist/cross-persona.d.ts +31 -0
- package/dist/cross-persona.js +188 -0
- package/dist/database.d.ts +35 -0
- package/dist/database.js +189 -0
- package/dist/embedding-benchmark.d.ts +12 -0
- package/dist/embedding-benchmark.js +224 -0
- package/dist/embeddings.d.ts +79 -0
- package/dist/embeddings.js +233 -0
- package/dist/episode-summarizer.d.ts +51 -0
- package/dist/episode-summarizer.js +285 -0
- package/dist/episodes.d.ts +41 -0
- package/dist/episodes.js +141 -0
- package/dist/facts.d.ts +60 -0
- package/dist/facts.js +263 -0
- package/dist/graph-traversal.d.ts +38 -0
- package/dist/graph-traversal.js +297 -0
- package/dist/graph.d.ts +51 -0
- package/dist/graph.js +221 -0
- package/dist/import-pipeline.d.ts +17 -0
- package/dist/import-pipeline.js +324 -0
- package/dist/index.d.ts +55 -0
- package/dist/index.js +62 -0
- package/dist/mcp-server.d.ts +31 -0
- package/dist/mcp-server.js +285 -0
- package/dist/metrics.d.ts +63 -0
- package/dist/metrics.js +243 -0
- package/dist/multi-hop.d.ts +30 -0
- package/dist/multi-hop.js +238 -0
- package/dist/profiles.d.ts +51 -0
- package/dist/profiles.js +149 -0
- package/package.json +52 -0
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector embeddings for semantic search
|
|
3
|
+
*
|
|
4
|
+
* ECC-010: Memory Explosion Throttling
|
|
5
|
+
* - Rate limiting: max MAX_EMBEDDINGS_PER_MINUTE per conversation (sliding window)
|
|
6
|
+
* - Dedup: same content hash within DEDUP_COOLDOWN_MS returns cached embedding
|
|
7
|
+
* - Tail sampling: when rate limited, return last cached embedding for the conversation
|
|
8
|
+
* - Metrics: throttleCount / dedupCount exported for observability
|
|
9
|
+
*/
|
|
10
|
+
import { createHash } from 'node:crypto';
|
|
11
|
+
// ─── ECC-010 Constants ────────────────────────────────────────────────────────
|
|
12
|
+
const MAX_EMBEDDINGS_PER_MINUTE = 20;
|
|
13
|
+
const DEDUP_COOLDOWN_MS = 300_000; // 5 minutes
|
|
14
|
+
const TAIL_SAMPLE_K = 10; // keep last K embeddings per conversation when limited
|
|
15
|
+
/** Per-conversation sliding rate window. Key: conversationId. */
|
|
16
|
+
const _rateWindows = new Map();
|
|
17
|
+
/** Per-content dedup cache. Key: SHA-256 hex of text. */
|
|
18
|
+
const _dedupCache = new Map();
|
|
19
|
+
/** Exported metrics counters — reset only on process restart. */
|
|
20
|
+
export const throttleMetrics = {
|
|
21
|
+
throttleCount: 0,
|
|
22
|
+
dedupCount: 0,
|
|
23
|
+
};
|
|
24
|
+
function contentHash(text) {
|
|
25
|
+
return createHash('sha256').update(text).digest('hex');
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* ECC-010: Check rate limit for a conversation window.
|
|
29
|
+
* Returns the tail-sampled embedding if over limit, or null if under limit.
|
|
30
|
+
*/
|
|
31
|
+
function checkRateLimit(conversationId) {
|
|
32
|
+
const now = Date.now();
|
|
33
|
+
let window = _rateWindows.get(conversationId);
|
|
34
|
+
if (!window || now - window.windowStart > 60_000) {
|
|
35
|
+
window = { count: 0, windowStart: now, tail: [] };
|
|
36
|
+
_rateWindows.set(conversationId, window);
|
|
37
|
+
}
|
|
38
|
+
if (window.count >= MAX_EMBEDDINGS_PER_MINUTE) {
|
|
39
|
+
// Return tail sample (last produced in this window) or empty fallback
|
|
40
|
+
const sample = window.tail[window.tail.length - 1] ?? [];
|
|
41
|
+
throttleMetrics.throttleCount++;
|
|
42
|
+
return sample;
|
|
43
|
+
}
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
/** ECC-010: Record a produced embedding in the rate window tail buffer. */
|
|
47
|
+
function recordInWindow(conversationId, embedding) {
|
|
48
|
+
const window = _rateWindows.get(conversationId);
|
|
49
|
+
if (!window)
|
|
50
|
+
return;
|
|
51
|
+
window.count++;
|
|
52
|
+
window.tail.push(embedding);
|
|
53
|
+
if (window.tail.length > TAIL_SAMPLE_K) {
|
|
54
|
+
window.tail.shift();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** ECC-010: Reset throttle state (for testing). */
|
|
58
|
+
export function resetThrottleState() {
|
|
59
|
+
_rateWindows.clear();
|
|
60
|
+
_dedupCache.clear();
|
|
61
|
+
throttleMetrics.throttleCount = 0;
|
|
62
|
+
throttleMetrics.dedupCount = 0;
|
|
63
|
+
}
|
|
64
|
+
// ─── Internal Ollama call ─────────────────────────────────────────────────────
|
|
65
|
+
async function _generateEmbeddingFromOllama(text, config) {
|
|
66
|
+
const response = await fetch(`${config.ollamaUrl}/api/embeddings`, {
|
|
67
|
+
method: 'POST',
|
|
68
|
+
headers: { 'Content-Type': 'application/json' },
|
|
69
|
+
body: JSON.stringify({ model: config.ollamaModel, prompt: text }),
|
|
70
|
+
});
|
|
71
|
+
if (!response.ok) {
|
|
72
|
+
throw new Error(`Ollama error: ${response.status} ${response.statusText}`);
|
|
73
|
+
}
|
|
74
|
+
const data = await response.json();
|
|
75
|
+
return data.embedding;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Generate embeddings for text using Ollama.
|
|
79
|
+
*
|
|
80
|
+
* ECC-010: Throttling applied when conversationId is provided:
|
|
81
|
+
* 1. Dedup check — returns cached embedding if same content seen within 5 min
|
|
82
|
+
* 2. Rate limit check — returns tail-sampled embedding if > 20/min per conversation
|
|
83
|
+
* 3. Ollama call — only reached if dedup and rate limit both pass
|
|
84
|
+
*
|
|
85
|
+
* @param conversationId Optional. When provided, enables per-conversation throttling.
|
|
86
|
+
*/
|
|
87
|
+
export async function generateEmbedding(text, config, conversationId) {
|
|
88
|
+
if (!config.vectorEnabled) {
|
|
89
|
+
throw new Error('Vector search is disabled in configuration');
|
|
90
|
+
}
|
|
91
|
+
if (conversationId) {
|
|
92
|
+
// Layer 1: Dedup check
|
|
93
|
+
const hash = contentHash(text);
|
|
94
|
+
const cached = _dedupCache.get(hash);
|
|
95
|
+
if (cached && Date.now() < cached.expiresAt) {
|
|
96
|
+
throttleMetrics.dedupCount++;
|
|
97
|
+
return cached.embedding;
|
|
98
|
+
}
|
|
99
|
+
// Layer 2: Rate limit check (tail sampling on overflow)
|
|
100
|
+
const sampled = checkRateLimit(conversationId);
|
|
101
|
+
if (sampled !== null) {
|
|
102
|
+
return sampled;
|
|
103
|
+
}
|
|
104
|
+
// Layer 3: Produce embedding and cache it
|
|
105
|
+
const embedding = await _generateEmbeddingFromOllama(text, config);
|
|
106
|
+
_dedupCache.set(hash, { embedding, expiresAt: Date.now() + DEDUP_COOLDOWN_MS });
|
|
107
|
+
recordInWindow(conversationId, embedding);
|
|
108
|
+
return embedding;
|
|
109
|
+
}
|
|
110
|
+
// No conversationId: bypass throttling (internal/system calls)
|
|
111
|
+
return _generateEmbeddingFromOllama(text, config);
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Generate a hypothetical answer using Ollama's generate endpoint.
|
|
115
|
+
* Used by HyDE to create an ideal document for embedding.
|
|
116
|
+
*/
|
|
117
|
+
export async function generateHypotheticalAnswer(query, config, options = {}) {
|
|
118
|
+
const model = options.model ?? 'llama3';
|
|
119
|
+
const prompt = `Answer the following question concisely in 2-3 sentences as if you had perfect knowledge. Do not hedge or say "I don't know".\n\nQuestion: ${query}\n\nAnswer:`;
|
|
120
|
+
const response = await fetch(`${config.ollamaUrl}/api/generate`, {
|
|
121
|
+
method: 'POST',
|
|
122
|
+
headers: { 'Content-Type': 'application/json' },
|
|
123
|
+
body: JSON.stringify({
|
|
124
|
+
model,
|
|
125
|
+
prompt,
|
|
126
|
+
stream: false,
|
|
127
|
+
options: { num_predict: options.maxTokens ?? 150 },
|
|
128
|
+
}),
|
|
129
|
+
});
|
|
130
|
+
if (!response.ok) {
|
|
131
|
+
throw new Error(`Ollama generate error: ${response.status} ${response.statusText}`);
|
|
132
|
+
}
|
|
133
|
+
const data = await response.json();
|
|
134
|
+
return data.response.trim();
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Generate HyDE (Hypothetical Document Expansion) embeddings.
|
|
138
|
+
*
|
|
139
|
+
* 1. Embeds the original query.
|
|
140
|
+
* 2. Uses an LLM to generate a hypothetical ideal answer.
|
|
141
|
+
* 3. Embeds the hypothetical answer.
|
|
142
|
+
* 4. Returns both embeddings so the caller can blend them.
|
|
143
|
+
*
|
|
144
|
+
* Falls back to duplicating the original embedding if generation fails.
|
|
145
|
+
*/
|
|
146
|
+
export async function generateHyDEExpansion(query, config, options = {}) {
|
|
147
|
+
const original = await generateEmbedding(query, config);
|
|
148
|
+
let hypothetical;
|
|
149
|
+
try {
|
|
150
|
+
hypothetical = await generateHypotheticalAnswer(query, config, {
|
|
151
|
+
model: options.generationModel,
|
|
152
|
+
maxTokens: options.maxTokens,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
return { original, expanded: original, hypothetical: query };
|
|
157
|
+
}
|
|
158
|
+
const expanded = await generateEmbedding(hypothetical, config);
|
|
159
|
+
return { original, expanded, hypothetical };
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Blend two embeddings by weighted average.
|
|
163
|
+
* Default: 40% original query, 60% hypothetical answer (HyDE sweet spot).
|
|
164
|
+
*/
|
|
165
|
+
export function blendEmbeddings(a, b, weightA = 0.4) {
|
|
166
|
+
if (a.length !== b.length) {
|
|
167
|
+
throw new Error('Embeddings must have the same dimension');
|
|
168
|
+
}
|
|
169
|
+
const weightB = 1 - weightA;
|
|
170
|
+
return a.map((val, i) => val * weightA + b[i] * weightB);
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Calculate cosine similarity between two vectors
|
|
174
|
+
*/
|
|
175
|
+
export function cosineSimilarity(a, b) {
|
|
176
|
+
if (a.length !== b.length) {
|
|
177
|
+
throw new Error('Vectors must have the same length');
|
|
178
|
+
}
|
|
179
|
+
let dotProduct = 0;
|
|
180
|
+
let normA = 0;
|
|
181
|
+
let normB = 0;
|
|
182
|
+
for (let i = 0; i < a.length; i++) {
|
|
183
|
+
dotProduct += a[i] * b[i];
|
|
184
|
+
normA += a[i] * a[i];
|
|
185
|
+
normB += b[i] * b[i];
|
|
186
|
+
}
|
|
187
|
+
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Serialize embedding for SQLite storage
|
|
191
|
+
*/
|
|
192
|
+
export function serializeEmbedding(embedding) {
|
|
193
|
+
// Convert to Float32Array and then to Buffer
|
|
194
|
+
const floatArray = new Float32Array(embedding);
|
|
195
|
+
return Buffer.from(floatArray.buffer);
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Deserialize embedding from SQLite storage
|
|
199
|
+
*/
|
|
200
|
+
export function deserializeEmbedding(buffer) {
|
|
201
|
+
const floatArray = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / 4);
|
|
202
|
+
return Array.from(floatArray);
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Check if Ollama is available
|
|
206
|
+
*/
|
|
207
|
+
export async function checkOllamaHealth(config) {
|
|
208
|
+
try {
|
|
209
|
+
const response = await fetch(`${config.ollamaUrl}/api/tags`, {
|
|
210
|
+
method: 'GET',
|
|
211
|
+
signal: AbortSignal.timeout(5000),
|
|
212
|
+
});
|
|
213
|
+
return response.ok;
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
return false;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* List available models from Ollama
|
|
221
|
+
*/
|
|
222
|
+
export async function listAvailableModels(config) {
|
|
223
|
+
try {
|
|
224
|
+
const response = await fetch(`${config.ollamaUrl}/api/tags`);
|
|
225
|
+
if (!response.ok)
|
|
226
|
+
return [];
|
|
227
|
+
const data = await response.json();
|
|
228
|
+
return data.models?.map(m => m.name) || [];
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
return [];
|
|
232
|
+
}
|
|
233
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* episode-summarizer.ts — Recursive Episode Summarization for Long Conversations
|
|
4
|
+
* MEM-002: Recursive Episode Summarization
|
|
5
|
+
*
|
|
6
|
+
* When episode count exceeds threshold, oldest messages are summarized into
|
|
7
|
+
* compressed episode records — preserving gist without token explosion.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* bun episode-summarizer.ts summarize --episodeIds <id1,id2,...> --outputTokens 800
|
|
11
|
+
* bun episode-summarizer.ts should-summarize --epCount 25 --threshold 20
|
|
12
|
+
* bun episode-summarizer.ts list-summaries
|
|
13
|
+
*/
|
|
14
|
+
import { Database } from "bun:sqlite";
|
|
15
|
+
export interface CompressedEpisode {
|
|
16
|
+
id: string;
|
|
17
|
+
summary: string;
|
|
18
|
+
compressedFrom: string[];
|
|
19
|
+
compressionRatio: number;
|
|
20
|
+
originalTokenEstimate: number;
|
|
21
|
+
compressedTokenEstimate: number;
|
|
22
|
+
keyDecisions: string[];
|
|
23
|
+
keyOutcomes: string[];
|
|
24
|
+
createdAt: number;
|
|
25
|
+
}
|
|
26
|
+
export interface SummarizationResult {
|
|
27
|
+
compressedEpisode: CompressedEpisode;
|
|
28
|
+
originalCount: number;
|
|
29
|
+
originalTokens: number;
|
|
30
|
+
compressedTokens: number;
|
|
31
|
+
compressionRatio: number;
|
|
32
|
+
}
|
|
33
|
+
export interface EpisodeForCompression {
|
|
34
|
+
id: string;
|
|
35
|
+
summary: string;
|
|
36
|
+
outcome: string;
|
|
37
|
+
happenedAt: number;
|
|
38
|
+
durationMs?: number;
|
|
39
|
+
entities: string[];
|
|
40
|
+
metadata?: Record<string, unknown>;
|
|
41
|
+
}
|
|
42
|
+
export declare function compressEpisodes(episodeIds: string[], db?: Database): Promise<SummarizationResult>;
|
|
43
|
+
export declare function getCompressedEpisode(compressedId: string, db?: Database): CompressedEpisode | null;
|
|
44
|
+
export declare function listCompressedEpisodes(limit?: number, db?: Database): CompressedEpisode[];
|
|
45
|
+
export interface ShouldSummarizeResult {
|
|
46
|
+
shouldCompress: boolean;
|
|
47
|
+
reason: string;
|
|
48
|
+
oldestEpisodes?: string[];
|
|
49
|
+
oldestTimestamp?: number;
|
|
50
|
+
}
|
|
51
|
+
export declare function shouldSummarize(count: number, threshold?: number, db?: Database): ShouldSummarizeResult;
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* episode-summarizer.ts — Recursive Episode Summarization for Long Conversations
|
|
4
|
+
* MEM-002: Recursive Episode Summarization
|
|
5
|
+
*
|
|
6
|
+
* When episode count exceeds threshold, oldest messages are summarized into
|
|
7
|
+
* compressed episode records — preserving gist without token explosion.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* bun episode-summarizer.ts summarize --episodeIds <id1,id2,...> --outputTokens 800
|
|
11
|
+
* bun episode-summarizer.ts should-summarize --epCount 25 --threshold 20
|
|
12
|
+
* bun episode-summarizer.ts list-summaries
|
|
13
|
+
*/
|
|
14
|
+
import { Database } from "bun:sqlite";
|
|
15
|
+
import { randomUUID } from "crypto";
|
|
16
|
+
const DB_PATH = process.env.ZO_MEMORY_DB || "/home/workspace/.zo/memory/shared-facts.db";
|
|
17
|
+
const OLLAMA_URL = process.env.OLLAMA_URL || "http://localhost:11434";
|
|
18
|
+
const SUMMARIZE_MODEL = process.env.ZO_SUMMARIZE_MODEL || "qwen2.5:7b";
|
|
19
|
+
const EPISODE_WINDOW = 14 * 24 * 3600; // 14 days in seconds
|
|
20
|
+
// ─── Database ─────────────────────────────────────────────────────────────────
|
|
21
|
+
function getDb() {
|
|
22
|
+
const db = new Database(DB_PATH);
|
|
23
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
24
|
+
db.exec(`
|
|
25
|
+
CREATE TABLE IF NOT EXISTS compressed_episodes (
|
|
26
|
+
id TEXT PRIMARY KEY,
|
|
27
|
+
summary TEXT NOT NULL,
|
|
28
|
+
compressed_from TEXT NOT NULL,
|
|
29
|
+
key_decisions TEXT NOT NULL DEFAULT '[]',
|
|
30
|
+
key_outcomes TEXT NOT NULL DEFAULT '[]',
|
|
31
|
+
original_token_estimate INTEGER NOT NULL DEFAULT 0,
|
|
32
|
+
compressed_token_estimate INTEGER NOT NULL DEFAULT 0,
|
|
33
|
+
compression_ratio REAL NOT NULL DEFAULT 0,
|
|
34
|
+
created_at INTEGER DEFAULT (strftime('%s','now'))
|
|
35
|
+
);
|
|
36
|
+
CREATE INDEX IF NOT EXISTS idx_compressed_episodes_created ON compressed_episodes(created_at);
|
|
37
|
+
`);
|
|
38
|
+
return db;
|
|
39
|
+
}
|
|
40
|
+
// ─── Ollama summarization ─────────────────────────────────────────────────────
|
|
41
|
+
async function generateSummary(episodes) {
|
|
42
|
+
const prompt = `You are compressing a sequence of conversation episodes into a single summary.
|
|
43
|
+
|
|
44
|
+
EPISODES TO COMPRESS (oldest first):
|
|
45
|
+
${episodes.map((e, i) => `[${i + 1}] (${e.outcome}) ${e.summary}`).join("\n")}
|
|
46
|
+
|
|
47
|
+
Generate a concise summary that preserves:
|
|
48
|
+
1. The overall arc and flow
|
|
49
|
+
2. Key decisions made and their rationale
|
|
50
|
+
3. Final outcomes
|
|
51
|
+
4. Any unresolved threads
|
|
52
|
+
|
|
53
|
+
Respond ONLY with valid JSON (no markdown, no explanation):
|
|
54
|
+
{
|
|
55
|
+
"summary": "2-3 sentence narrative of the full sequence",
|
|
56
|
+
"keyDecisions": ["decision 1", "decision 2"],
|
|
57
|
+
"keyOutcomes": ["outcome 1", "outcome 2"]
|
|
58
|
+
}`;
|
|
59
|
+
const resp = await fetch(`${OLLAMA_URL}/api/generate`, {
|
|
60
|
+
method: "POST",
|
|
61
|
+
headers: { "Content-Type": "application/json" },
|
|
62
|
+
body: JSON.stringify({
|
|
63
|
+
model: SUMMARIZE_MODEL,
|
|
64
|
+
prompt,
|
|
65
|
+
stream: false,
|
|
66
|
+
keep_alive: "1h",
|
|
67
|
+
options: { temperature: 0.3, num_predict: 400 },
|
|
68
|
+
}),
|
|
69
|
+
});
|
|
70
|
+
if (!resp.ok)
|
|
71
|
+
throw new Error(`Ollama error: ${resp.status}`);
|
|
72
|
+
const data = await resp.json();
|
|
73
|
+
const raw = data.response.trim();
|
|
74
|
+
const jsonMatch = raw.match(/\{[\s\S]*\}/);
|
|
75
|
+
if (!jsonMatch)
|
|
76
|
+
throw new Error(`Failed to parse summary JSON: ${raw.slice(0, 200)}`);
|
|
77
|
+
return JSON.parse(jsonMatch[0]);
|
|
78
|
+
}
|
|
79
|
+
function estimateTokens(text) {
|
|
80
|
+
return Math.ceil(text.length / 4);
|
|
81
|
+
}
|
|
82
|
+
// ─── Core summarization logic ─────────────────────────────────────────────────
|
|
83
|
+
export async function compressEpisodes(episodeIds, db) {
|
|
84
|
+
const _db = db || getDb();
|
|
85
|
+
// Load episodes
|
|
86
|
+
const episodes = [];
|
|
87
|
+
for (const id of episodeIds) {
|
|
88
|
+
const row = _db.prepare("SELECT * FROM episodes WHERE id = ?").get(id);
|
|
89
|
+
if (!row)
|
|
90
|
+
continue;
|
|
91
|
+
const entities = _db.prepare("SELECT entity FROM episode_entities WHERE episode_id = ?").all(id).map(e => e.entity);
|
|
92
|
+
episodes.push({
|
|
93
|
+
id: row.id,
|
|
94
|
+
summary: row.summary,
|
|
95
|
+
outcome: row.outcome,
|
|
96
|
+
happenedAt: row.happened_at,
|
|
97
|
+
durationMs: row.duration_ms,
|
|
98
|
+
entities,
|
|
99
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
if (episodes.length === 0)
|
|
103
|
+
throw new Error("No valid episodes found for compression");
|
|
104
|
+
// Sort oldest first
|
|
105
|
+
episodes.sort((a, b) => a.happenedAt - b.happenedAt);
|
|
106
|
+
// Generate compressed summary via Ollama
|
|
107
|
+
const { summary, keyDecisions, keyOutcomes } = await generateSummary(episodes);
|
|
108
|
+
// Compute compression stats
|
|
109
|
+
const originalTokens = episodes.reduce((sum, e) => sum + estimateTokens(e.summary), 0);
|
|
110
|
+
const compressedTokens = estimateTokens(summary);
|
|
111
|
+
const compressionRatio = originalTokens > 0 ? Math.max(0, 1 - (compressedTokens / originalTokens)) : 0;
|
|
112
|
+
// Store compressed episode
|
|
113
|
+
const compressedId = `cmpe-${randomUUID().slice(0, 8)}`;
|
|
114
|
+
const nowSec = Math.floor(Date.now() / 1000);
|
|
115
|
+
_db.prepare(`
|
|
116
|
+
INSERT INTO compressed_episodes
|
|
117
|
+
(id, summary, compressed_from, key_decisions, key_outcomes, original_token_estimate, compressed_token_estimate, compression_ratio, created_at)
|
|
118
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
119
|
+
`).run(compressedId, summary, JSON.stringify(episodeIds), JSON.stringify(keyDecisions), JSON.stringify(keyOutcomes), originalTokens, compressedTokens, compressionRatio, nowSec);
|
|
120
|
+
const compressedEpisode = {
|
|
121
|
+
id: compressedId,
|
|
122
|
+
summary,
|
|
123
|
+
compressedFrom: episodeIds,
|
|
124
|
+
compressionRatio,
|
|
125
|
+
originalTokenEstimate: originalTokens,
|
|
126
|
+
compressedTokenEstimate: compressedTokens,
|
|
127
|
+
keyDecisions,
|
|
128
|
+
keyOutcomes,
|
|
129
|
+
createdAt: nowSec,
|
|
130
|
+
};
|
|
131
|
+
return {
|
|
132
|
+
compressedEpisode,
|
|
133
|
+
originalCount: episodes.length,
|
|
134
|
+
originalTokens,
|
|
135
|
+
compressedTokens,
|
|
136
|
+
compressionRatio,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
export function getCompressedEpisode(compressedId, db) {
|
|
140
|
+
const _db = db || getDb();
|
|
141
|
+
const row = _db.prepare("SELECT * FROM compressed_episodes WHERE id = ?").get(compressedId);
|
|
142
|
+
if (!row)
|
|
143
|
+
return null;
|
|
144
|
+
return {
|
|
145
|
+
id: row.id,
|
|
146
|
+
summary: row.summary,
|
|
147
|
+
compressedFrom: JSON.parse(row.compressed_from),
|
|
148
|
+
compressionRatio: row.compression_ratio,
|
|
149
|
+
originalTokenEstimate: row.original_token_estimate,
|
|
150
|
+
compressedTokenEstimate: row.compressed_token_estimate,
|
|
151
|
+
keyDecisions: JSON.parse(row.key_decisions),
|
|
152
|
+
keyOutcomes: JSON.parse(row.key_outcomes),
|
|
153
|
+
createdAt: row.created_at,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
export function listCompressedEpisodes(limit = 20, db) {
|
|
157
|
+
const _db = db || getDb();
|
|
158
|
+
const rows = _db.prepare("SELECT * FROM compressed_episodes ORDER BY created_at DESC LIMIT ?").all(limit);
|
|
159
|
+
return rows.map(row => ({
|
|
160
|
+
id: row.id,
|
|
161
|
+
summary: row.summary,
|
|
162
|
+
compressedFrom: JSON.parse(row.compressed_from),
|
|
163
|
+
compressionRatio: row.compression_ratio,
|
|
164
|
+
originalTokenEstimate: row.original_token_estimate,
|
|
165
|
+
compressedTokenEstimate: row.compressed_token_estimate,
|
|
166
|
+
keyDecisions: JSON.parse(row.key_decisions),
|
|
167
|
+
keyOutcomes: JSON.parse(row.key_outcomes),
|
|
168
|
+
createdAt: row.created_at,
|
|
169
|
+
}));
|
|
170
|
+
}
|
|
171
|
+
export function shouldSummarize(count, threshold = 20, db) {
|
|
172
|
+
if (count <= threshold) {
|
|
173
|
+
return { shouldCompress: false, reason: `Episode count ${count} is below threshold ${threshold}` };
|
|
174
|
+
}
|
|
175
|
+
const _db = db || getDb();
|
|
176
|
+
const cutoff = Math.floor(Date.now() / 1000) - EPISODE_WINDOW;
|
|
177
|
+
const oldEps = _db.prepare("SELECT id, happened_at FROM episodes WHERE happened_at < ? ORDER BY happened_at ASC LIMIT ?").all(cutoff, Math.min(count - threshold + 1, 50));
|
|
178
|
+
if (oldEps.length === 0) {
|
|
179
|
+
return { shouldCompress: false, reason: "No episodes older than 14 days to compress" };
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
shouldCompress: true,
|
|
183
|
+
reason: `${oldEps.length} episodes exceed window threshold`,
|
|
184
|
+
oldestEpisodes: oldEps.map(e => e.id),
|
|
185
|
+
oldestTimestamp: oldEps[0]?.happened_at,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
// ─── CLI ─────────────────────────────────────────────────────────────────────
|
|
189
|
+
async function main() {
|
|
190
|
+
const args = process.argv.slice(2);
|
|
191
|
+
if (args.length === 0) {
|
|
192
|
+
console.log(`Episode Summarizer CLI - v1.0
|
|
193
|
+
|
|
194
|
+
Commands:
|
|
195
|
+
should-summarize --count <n> --threshold <n>
|
|
196
|
+
summarize --episodeIds <id1,id2,...> --outputTokens <n>
|
|
197
|
+
list-summaries
|
|
198
|
+
show-summary <compressedId>
|
|
199
|
+
`);
|
|
200
|
+
process.exit(0);
|
|
201
|
+
}
|
|
202
|
+
const command = args[0];
|
|
203
|
+
switch (command) {
|
|
204
|
+
case "should-summarize": {
|
|
205
|
+
let count = 0, threshold = 20;
|
|
206
|
+
for (let i = 1; i < args.length; i++) {
|
|
207
|
+
if (args[i] === "--count" || args[i] === "-c")
|
|
208
|
+
count = parseInt(args[i + 1] || "0");
|
|
209
|
+
if (args[i] === "--threshold" || args[i] === "-t")
|
|
210
|
+
threshold = parseInt(args[i + 1] || "20");
|
|
211
|
+
}
|
|
212
|
+
const result = shouldSummarize(count, threshold);
|
|
213
|
+
console.log(`Should compress: ${result.shouldCompress}`);
|
|
214
|
+
console.log(`Reason: ${result.reason}`);
|
|
215
|
+
if (result.oldestEpisodes?.length)
|
|
216
|
+
console.log(`Episode candidates: ${result.oldestEpisodes.length}`);
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
case "summarize": {
|
|
220
|
+
let episodeIds = [];
|
|
221
|
+
for (let i = 1; i < args.length; i++) {
|
|
222
|
+
if (args[i] === "--episodeIds" || args[i] === "-e") {
|
|
223
|
+
episodeIds = (args[i + 1] || "").split(",").map(s => s.trim()).filter(Boolean);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (episodeIds.length === 0) {
|
|
227
|
+
console.error("No episode IDs provided");
|
|
228
|
+
process.exit(1);
|
|
229
|
+
}
|
|
230
|
+
console.log(`Compressing ${episodeIds.length} episodes...`);
|
|
231
|
+
const result = await compressEpisodes(episodeIds);
|
|
232
|
+
console.log(`\nCompressed Episode: ${result.compressedEpisode.id}`);
|
|
233
|
+
console.log(`Summary: ${result.compressedEpisode.summary}`);
|
|
234
|
+
console.log(`Decisions: ${result.compressedEpisode.keyDecisions.join(", ") || "(none)"}`);
|
|
235
|
+
console.log(`Outcomes: ${result.compressedEpisode.keyOutcomes.join(", ") || "(none)"}`);
|
|
236
|
+
console.log(`Compression: ${result.originalTokens}t -> ${result.compressedTokens}t (${Math.round(result.compressionRatio * 100)}% saved)`);
|
|
237
|
+
break;
|
|
238
|
+
}
|
|
239
|
+
case "list-summaries": {
|
|
240
|
+
const summaries = listCompressedEpisodes();
|
|
241
|
+
if (summaries.length === 0) {
|
|
242
|
+
console.log("No compressed episodes.");
|
|
243
|
+
break;
|
|
244
|
+
}
|
|
245
|
+
console.log(`Compressed Episodes (${summaries.length})\n`);
|
|
246
|
+
for (const s of summaries) {
|
|
247
|
+
console.log(`[${s.id}] ${s.compressionRatio.toFixed(1)}% saved | ${s.compressedFrom.length} -> 1 episodes`);
|
|
248
|
+
console.log(` ${s.summary.slice(0, 120)}${s.summary.length > 120 ? "..." : ""}`);
|
|
249
|
+
if (s.keyDecisions.length > 0)
|
|
250
|
+
console.log(` Decisions: ${s.keyDecisions.slice(0, 3).join(" | ")}`);
|
|
251
|
+
if (s.keyOutcomes.length > 0)
|
|
252
|
+
console.log(` Outcomes: ${s.keyOutcomes.slice(0, 3).join(" | ")}`);
|
|
253
|
+
console.log();
|
|
254
|
+
}
|
|
255
|
+
break;
|
|
256
|
+
}
|
|
257
|
+
case "show-summary": {
|
|
258
|
+
const id = args[1];
|
|
259
|
+
if (!id) {
|
|
260
|
+
console.error("Provide a compressed episode ID");
|
|
261
|
+
process.exit(1);
|
|
262
|
+
}
|
|
263
|
+
const s = getCompressedEpisode(id);
|
|
264
|
+
if (!s) {
|
|
265
|
+
console.error(`Compressed episode not found: ${id}`);
|
|
266
|
+
process.exit(1);
|
|
267
|
+
}
|
|
268
|
+
console.log(`Compressed Episode: ${s.id}`);
|
|
269
|
+
console.log(`Created: ${new Date(s.createdAt * 1000).toISOString()}`);
|
|
270
|
+
console.log(`Compression: ${s.originalTokenEstimate}t -> ${s.compressedTokenEstimate}t (${Math.round(s.compressionRatio * 100)}% saved)`);
|
|
271
|
+
console.log(`\nSummary:\n${s.summary}`);
|
|
272
|
+
if (s.keyDecisions.length > 0)
|
|
273
|
+
console.log(`\nKey Decisions: ${s.keyDecisions.join(", ")}`);
|
|
274
|
+
if (s.keyOutcomes.length > 0)
|
|
275
|
+
console.log(`Key Outcomes: ${s.keyOutcomes.join(", ")}`);
|
|
276
|
+
console.log(`\nCompressed from ${s.compressedFrom.length} episodes: ${s.compressedFrom.join(", ")}`);
|
|
277
|
+
break;
|
|
278
|
+
}
|
|
279
|
+
default:
|
|
280
|
+
console.error(`Unknown command: ${command}`);
|
|
281
|
+
process.exit(1);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
if (import.meta.main)
|
|
285
|
+
main();
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Episodic memory for event-based storage
|
|
3
|
+
*/
|
|
4
|
+
import type { EpisodicMemory, TemporalQuery } from 'zouroboros-core';
|
|
5
|
+
type Outcome = 'success' | 'failure' | 'resolved' | 'ongoing';
|
|
6
|
+
interface CreateEpisodeInput {
|
|
7
|
+
summary: string;
|
|
8
|
+
outcome: Outcome;
|
|
9
|
+
entities: string[];
|
|
10
|
+
happenedAt?: Date;
|
|
11
|
+
durationMs?: number;
|
|
12
|
+
procedureId?: string;
|
|
13
|
+
metadata?: Record<string, unknown>;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Create a new episode
|
|
17
|
+
*/
|
|
18
|
+
export declare function createEpisode(input: CreateEpisodeInput): EpisodicMemory;
|
|
19
|
+
/**
|
|
20
|
+
* Search episodes with temporal filters
|
|
21
|
+
*/
|
|
22
|
+
export declare function searchEpisodes(query: TemporalQuery): EpisodicMemory[];
|
|
23
|
+
/**
|
|
24
|
+
* Get episodes for a specific entity
|
|
25
|
+
*/
|
|
26
|
+
export declare function getEntityEpisodes(entity: string, options?: {
|
|
27
|
+
limit?: number;
|
|
28
|
+
outcome?: Outcome;
|
|
29
|
+
}): EpisodicMemory[];
|
|
30
|
+
/**
|
|
31
|
+
* Update episode outcome
|
|
32
|
+
*/
|
|
33
|
+
export declare function updateEpisodeOutcome(id: string, outcome: Outcome): boolean;
|
|
34
|
+
/**
|
|
35
|
+
* Get episode statistics
|
|
36
|
+
*/
|
|
37
|
+
export declare function getEpisodeStats(): {
|
|
38
|
+
total: number;
|
|
39
|
+
byOutcome: Record<Outcome, number>;
|
|
40
|
+
};
|
|
41
|
+
export {};
|