vesper-memory 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +28 -0
- package/CHANGELOG.md +137 -0
- package/LICENSE +21 -0
- package/README.md +665 -0
- package/config/sqlite-schema.sql +413 -0
- package/dist/benchmark/index.d.ts +18 -0
- package/dist/benchmark/index.d.ts.map +1 -0
- package/dist/benchmark/index.js +19 -0
- package/dist/benchmark/index.js.map +1 -0
- package/dist/benchmark/metrics-collector.d.ts +90 -0
- package/dist/benchmark/metrics-collector.d.ts.map +1 -0
- package/dist/benchmark/metrics-collector.js +196 -0
- package/dist/benchmark/metrics-collector.js.map +1 -0
- package/dist/benchmark/report-generator.d.ts +35 -0
- package/dist/benchmark/report-generator.d.ts.map +1 -0
- package/dist/benchmark/report-generator.js +221 -0
- package/dist/benchmark/report-generator.js.map +1 -0
- package/dist/benchmark/runner.d.ts +30 -0
- package/dist/benchmark/runner.d.ts.map +1 -0
- package/dist/benchmark/runner.js +308 -0
- package/dist/benchmark/runner.js.map +1 -0
- package/dist/benchmark/scenarios/context-recall.d.ts +65 -0
- package/dist/benchmark/scenarios/context-recall.d.ts.map +1 -0
- package/dist/benchmark/scenarios/context-recall.js +180 -0
- package/dist/benchmark/scenarios/context-recall.js.map +1 -0
- package/dist/benchmark/scenarios/index.d.ts +34 -0
- package/dist/benchmark/scenarios/index.d.ts.map +1 -0
- package/dist/benchmark/scenarios/index.js +230 -0
- package/dist/benchmark/scenarios/index.js.map +1 -0
- package/dist/benchmark/statistics.d.ts +125 -0
- package/dist/benchmark/statistics.d.ts.map +1 -0
- package/dist/benchmark/statistics.js +520 -0
- package/dist/benchmark/statistics.js.map +1 -0
- package/dist/benchmark/types.d.ts +458 -0
- package/dist/benchmark/types.d.ts.map +1 -0
- package/dist/benchmark/types.js +297 -0
- package/dist/benchmark/types.js.map +1 -0
- package/dist/cli 2.js +583 -0
- package/dist/cli.d 2.ts +8 -0
- package/dist/cli.d.ts +8 -0
- package/dist/cli.d.ts 2.map +1 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +583 -0
- package/dist/cli.js 2.map +1 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/rate-limits.d.ts +76 -0
- package/dist/config/rate-limits.d.ts.map +1 -0
- package/dist/config/rate-limits.js +161 -0
- package/dist/config/rate-limits.js.map +1 -0
- package/dist/config 2/rate-limits.d.ts +76 -0
- package/dist/config 2/rate-limits.d.ts.map +1 -0
- package/dist/config 2/rate-limits.js +161 -0
- package/dist/config 2/rate-limits.js.map +1 -0
- package/dist/consolidation/pipeline.d.ts +52 -0
- package/dist/consolidation/pipeline.d.ts.map +1 -0
- package/dist/consolidation/pipeline.js +193 -0
- package/dist/consolidation/pipeline.js.map +1 -0
- package/dist/consolidation 2/pipeline.d.ts +52 -0
- package/dist/consolidation 2/pipeline.d.ts.map +1 -0
- package/dist/consolidation 2/pipeline.js +193 -0
- package/dist/consolidation 2/pipeline.js.map +1 -0
- package/dist/embeddings/client.d.ts +95 -0
- package/dist/embeddings/client.d.ts.map +1 -0
- package/dist/embeddings/client.js +162 -0
- package/dist/embeddings/client.js.map +1 -0
- package/dist/embeddings 2/client.d.ts +94 -0
- package/dist/embeddings 2/client.d.ts.map +1 -0
- package/dist/embeddings 2/client.js +144 -0
- package/dist/embeddings 2/client.js.map +1 -0
- package/dist/memory-layers/semantic-memory.d.ts +76 -0
- package/dist/memory-layers/semantic-memory.d.ts.map +1 -0
- package/dist/memory-layers/semantic-memory.js +231 -0
- package/dist/memory-layers/semantic-memory.js.map +1 -0
- package/dist/memory-layers/skill-library.d.ts +25 -0
- package/dist/memory-layers/skill-library.d.ts.map +1 -0
- package/dist/memory-layers/skill-library.js +54 -0
- package/dist/memory-layers/skill-library.js.map +1 -0
- package/dist/memory-layers/working-memory.d.ts +57 -0
- package/dist/memory-layers/working-memory.d.ts.map +1 -0
- package/dist/memory-layers/working-memory.js +153 -0
- package/dist/memory-layers/working-memory.js.map +1 -0
- package/dist/memory-layers 2/semantic-memory.d.ts +33 -0
- package/dist/memory-layers 2/semantic-memory.d.ts.map +1 -0
- package/dist/memory-layers 2/semantic-memory.js +114 -0
- package/dist/memory-layers 2/semantic-memory.js.map +1 -0
- package/dist/memory-layers 2/skill-library.d.ts +25 -0
- package/dist/memory-layers 2/skill-library.d.ts.map +1 -0
- package/dist/memory-layers 2/skill-library.js +54 -0
- package/dist/memory-layers 2/skill-library.js.map +1 -0
- package/dist/memory-layers 2/working-memory.d.ts +57 -0
- package/dist/memory-layers 2/working-memory.d.ts.map +1 -0
- package/dist/memory-layers 2/working-memory.js +153 -0
- package/dist/memory-layers 2/working-memory.js.map +1 -0
- package/dist/middleware/auth.d.ts +287 -0
- package/dist/middleware/auth.d.ts.map +1 -0
- package/dist/middleware/auth.js +610 -0
- package/dist/middleware/auth.js.map +1 -0
- package/dist/middleware 2/auth.d.ts +287 -0
- package/dist/middleware 2/auth.d.ts.map +1 -0
- package/dist/middleware 2/auth.js +610 -0
- package/dist/middleware 2/auth.js.map +1 -0
- package/dist/monitoring/health.d.ts +241 -0
- package/dist/monitoring/health.d.ts.map +1 -0
- package/dist/monitoring/health.js +649 -0
- package/dist/monitoring/health.js.map +1 -0
- package/dist/monitoring/metrics.d.ts +153 -0
- package/dist/monitoring/metrics.d.ts.map +1 -0
- package/dist/monitoring/metrics.js +349 -0
- package/dist/monitoring/metrics.js.map +1 -0
- package/dist/monitoring 2/health.d.ts +241 -0
- package/dist/monitoring 2/health.d.ts.map +1 -0
- package/dist/monitoring 2/health.js +649 -0
- package/dist/monitoring 2/health.js.map +1 -0
- package/dist/monitoring 2/metrics.d.ts +153 -0
- package/dist/monitoring 2/metrics.d.ts.map +1 -0
- package/dist/monitoring 2/metrics.js +349 -0
- package/dist/monitoring 2/metrics.js.map +1 -0
- package/dist/retrieval/hybrid-search.d.ts +186 -0
- package/dist/retrieval/hybrid-search.d.ts.map +1 -0
- package/dist/retrieval/hybrid-search.js +330 -0
- package/dist/retrieval/hybrid-search.js.map +1 -0
- package/dist/retrieval 2/hybrid-search.d.ts +186 -0
- package/dist/retrieval 2/hybrid-search.d.ts.map +1 -0
- package/dist/retrieval 2/hybrid-search.js +327 -0
- package/dist/retrieval 2/hybrid-search.js.map +1 -0
- package/dist/router/smart-router.d.ts +157 -0
- package/dist/router/smart-router.d.ts.map +1 -0
- package/dist/router/smart-router.js +524 -0
- package/dist/router/smart-router.js.map +1 -0
- package/dist/router 2/smart-router.d.ts +111 -0
- package/dist/router 2/smart-router.d.ts.map +1 -0
- package/dist/router 2/smart-router.js +383 -0
- package/dist/router 2/smart-router.js.map +1 -0
- package/dist/security/rate-limit-middleware.d.ts +119 -0
- package/dist/security/rate-limit-middleware.d.ts.map +1 -0
- package/dist/security/rate-limit-middleware.js +193 -0
- package/dist/security/rate-limit-middleware.js.map +1 -0
- package/dist/security 2/rate-limit-middleware.d.ts +119 -0
- package/dist/security 2/rate-limit-middleware.d.ts.map +1 -0
- package/dist/security 2/rate-limit-middleware.js +193 -0
- package/dist/security 2/rate-limit-middleware.js.map +1 -0
- package/dist/server 2.js +861 -0
- package/dist/server-wrapper 2.js +170 -0
- package/dist/server-wrapper.d 2.ts +12 -0
- package/dist/server-wrapper.d.ts +12 -0
- package/dist/server-wrapper.d.ts 2.map +1 -0
- package/dist/server-wrapper.d.ts.map +1 -0
- package/dist/server-wrapper.js +170 -0
- package/dist/server-wrapper.js 2.map +1 -0
- package/dist/server-wrapper.js.map +1 -0
- package/dist/server.d 2.ts +14 -0
- package/dist/server.d.ts +14 -0
- package/dist/server.d.ts 2.map +1 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +656 -0
- package/dist/server.js 2.map +1 -0
- package/dist/server.js.map +1 -0
- package/dist/synthesis/conflict-detector.d.ts +43 -0
- package/dist/synthesis/conflict-detector.d.ts.map +1 -0
- package/dist/synthesis/conflict-detector.js +125 -0
- package/dist/synthesis/conflict-detector.js.map +1 -0
- package/dist/synthesis 2/conflict-detector.d.ts +43 -0
- package/dist/synthesis 2/conflict-detector.d.ts.map +1 -0
- package/dist/synthesis 2/conflict-detector.js +125 -0
- package/dist/synthesis 2/conflict-detector.js.map +1 -0
- package/dist/utils/monitoring.d.ts +25 -0
- package/dist/utils/monitoring.d.ts.map +1 -0
- package/dist/utils/monitoring.js +41 -0
- package/dist/utils/monitoring.js.map +1 -0
- package/dist/utils/rate-limiter.d.ts +97 -0
- package/dist/utils/rate-limiter.d.ts.map +1 -0
- package/dist/utils/rate-limiter.js +166 -0
- package/dist/utils/rate-limiter.js.map +1 -0
- package/dist/utils/validation.d.ts +139 -0
- package/dist/utils/validation.d.ts.map +1 -0
- package/dist/utils/validation.js +195 -0
- package/dist/utils/validation.js.map +1 -0
- package/dist/utils 2/monitoring.d.ts +25 -0
- package/dist/utils 2/monitoring.d.ts.map +1 -0
- package/dist/utils 2/monitoring.js +41 -0
- package/dist/utils 2/monitoring.js.map +1 -0
- package/dist/utils 2/rate-limiter.d.ts +97 -0
- package/dist/utils 2/rate-limiter.d.ts.map +1 -0
- package/dist/utils 2/rate-limiter.js +166 -0
- package/dist/utils 2/rate-limiter.js.map +1 -0
- package/dist/utils 2/validation.d.ts +139 -0
- package/dist/utils 2/validation.d.ts.map +1 -0
- package/dist/utils 2/validation.js +195 -0
- package/dist/utils 2/validation.js.map +1 -0
- package/docker-compose.yml +65 -0
- package/embedding-service/Dockerfile +29 -0
- package/embedding-service/README.md +68 -0
- package/embedding-service/requirements.txt +6 -0
- package/embedding-service/server.py +110 -0
- package/package.json +92 -0
- package/scripts/FRESH_INSTALL.md +280 -0
- package/scripts/README.md +101 -0
- package/scripts/ensure-infrastructure.sh +38 -0
- package/scripts/fresh-install-auto.sh +96 -0
- package/scripts/fresh-install.sh +167 -0
- package/scripts/health-check.sh +72 -0
- package/scripts/init-qdrant.ts +201 -0
- package/scripts/postinstall.js +94 -0
- package/scripts/run-accuracy-benchmark.ts +1237 -0
- package/scripts/run-real-benchmark.ts +581 -0
- package/scripts/test-mcp-server.sh +38 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consolidation Pipeline - v3.0 Implementation
|
|
3
|
+
*
|
|
4
|
+
* Nightly process: working → semantic memory transformation.
|
|
5
|
+
* Runs at 3 AM, extracts knowledge, applies decay, detects conflicts.
|
|
6
|
+
*/
|
|
7
|
+
export class ConsolidationPipeline {
|
|
8
|
+
workingMemory;
|
|
9
|
+
semanticMemory;
|
|
10
|
+
skillLibrary;
|
|
11
|
+
db;
|
|
12
|
+
constructor(workingMemory, semanticMemory, skillLibrary, db) {
|
|
13
|
+
this.workingMemory = workingMemory;
|
|
14
|
+
this.semanticMemory = semanticMemory;
|
|
15
|
+
this.skillLibrary = skillLibrary;
|
|
16
|
+
this.db = db;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Run full consolidation cycle
|
|
20
|
+
*/
|
|
21
|
+
async consolidate() {
|
|
22
|
+
const startTime = Date.now();
|
|
23
|
+
console.error('[CONSOLIDATION] Starting consolidation cycle...');
|
|
24
|
+
const stats = {
|
|
25
|
+
memoriesProcessed: 0,
|
|
26
|
+
entitiesExtracted: 0,
|
|
27
|
+
relationshipsCreated: 0,
|
|
28
|
+
conflictsDetected: 0,
|
|
29
|
+
memoriesPruned: 0,
|
|
30
|
+
skillsExtracted: 0,
|
|
31
|
+
duration: 0,
|
|
32
|
+
};
|
|
33
|
+
try {
|
|
34
|
+
// Step 1: Extract from working memory
|
|
35
|
+
const recentMemories = await this.workingMemory.getRecent(100);
|
|
36
|
+
stats.memoriesProcessed = recentMemories.length;
|
|
37
|
+
for (const memory of recentMemories) {
|
|
38
|
+
// Extract entities and relationships
|
|
39
|
+
const extracted = this.extractKnowledge(memory);
|
|
40
|
+
for (const entity of extracted.entities) {
|
|
41
|
+
this.semanticMemory.upsertEntity(entity);
|
|
42
|
+
stats.entitiesExtracted++;
|
|
43
|
+
}
|
|
44
|
+
for (const rel of extracted.relationships) {
|
|
45
|
+
this.semanticMemory.upsertRelationship(rel);
|
|
46
|
+
stats.relationshipsCreated++;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
// Step 2: Apply temporal decay
|
|
50
|
+
this.semanticMemory.applyTemporalDecay();
|
|
51
|
+
// Step 3: Detect conflicts
|
|
52
|
+
stats.conflictsDetected = this.detectConflicts();
|
|
53
|
+
// Step 4: Prune low-value memories
|
|
54
|
+
stats.memoriesPruned = this.pruneMemories();
|
|
55
|
+
// Step 5: Extract skills (simplified)
|
|
56
|
+
stats.skillsExtracted = this.extractSkills(recentMemories);
|
|
57
|
+
// Step 6: Create backup
|
|
58
|
+
this.createBackup();
|
|
59
|
+
stats.duration = Date.now() - startTime;
|
|
60
|
+
console.error(`[CONSOLIDATION] Complete! Processed ${stats.memoriesProcessed} memories in ${stats.duration}ms`);
|
|
61
|
+
console.error(`[CONSOLIDATION] Extracted ${stats.entitiesExtracted} entities, ${stats.relationshipsCreated} relationships`);
|
|
62
|
+
console.error(`[CONSOLIDATION] Detected ${stats.conflictsDetected} conflicts, pruned ${stats.memoriesPruned} memories`);
|
|
63
|
+
return stats;
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
console.error('[CONSOLIDATION] Error:', error);
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Extract entities and relationships from memory
|
|
72
|
+
* Simple keyword-based extraction (will enhance with LLM later)
|
|
73
|
+
*/
|
|
74
|
+
extractKnowledge(memory) {
|
|
75
|
+
const entities = [];
|
|
76
|
+
const relationships = [];
|
|
77
|
+
// Use pre-extracted entities if available
|
|
78
|
+
if (memory.keyEntities && memory.keyEntities.length > 0) {
|
|
79
|
+
for (const entityName of memory.keyEntities) {
|
|
80
|
+
entities.push({
|
|
81
|
+
name: entityName,
|
|
82
|
+
type: 'concept',
|
|
83
|
+
confidence: 0.8,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Extract preferences
|
|
88
|
+
const prefPatterns = [
|
|
89
|
+
/prefer (\w+)/gi,
|
|
90
|
+
/like (\w+)/gi,
|
|
91
|
+
/favorite is (\w+)/gi,
|
|
92
|
+
];
|
|
93
|
+
for (const pattern of prefPatterns) {
|
|
94
|
+
const matches = memory.fullText.matchAll(pattern);
|
|
95
|
+
for (const match of matches) {
|
|
96
|
+
entities.push({
|
|
97
|
+
name: match[1],
|
|
98
|
+
type: 'preference',
|
|
99
|
+
confidence: 0.9,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return { entities, relationships };
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Detect simple conflicts (temporal impossibilities, contradictions)
|
|
107
|
+
*/
|
|
108
|
+
detectConflicts() {
|
|
109
|
+
let detected = 0;
|
|
110
|
+
// Temporal conflicts: same property, overlapping time ranges, different values
|
|
111
|
+
const conflicts = this.db.prepare(`
|
|
112
|
+
SELECT f1.id as id1, f2.id as id2, f1.property, f1.value as val1, f2.value as val2
|
|
113
|
+
FROM facts f1
|
|
114
|
+
JOIN facts f2 ON f1.entity_id = f2.entity_id AND f1.property = f2.property
|
|
115
|
+
WHERE f1.id < f2.id
|
|
116
|
+
AND f1.value != f2.value
|
|
117
|
+
AND f1.valid_until IS NULL
|
|
118
|
+
AND f2.valid_until IS NULL
|
|
119
|
+
`).all();
|
|
120
|
+
for (const conflict of conflicts) {
|
|
121
|
+
// Check if already flagged
|
|
122
|
+
const existing = this.db.prepare('SELECT * FROM conflicts WHERE fact_id_1 = ? AND fact_id_2 = ?').get(conflict.id1, conflict.id2);
|
|
123
|
+
if (!existing) {
|
|
124
|
+
const conflictId = 'conflict_' + Date.now() + '_' + Math.random().toString(36).substr(2, 9);
|
|
125
|
+
this.db.prepare(`
|
|
126
|
+
INSERT INTO conflicts (id, fact_id_1, fact_id_2, conflict_type, description, severity, resolution_status)
|
|
127
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
128
|
+
`).run(conflictId, conflict.id1, conflict.id2, 'contradiction', `Conflicting values for ${conflict.property}: "${conflict.val1}" vs "${conflict.val2}"`, 'medium', 'flagged');
|
|
129
|
+
// Lower confidence on both facts
|
|
130
|
+
this.db.prepare('UPDATE facts SET confidence = 0.5 WHERE id = ?').run(conflict.id1);
|
|
131
|
+
this.db.prepare('UPDATE facts SET confidence = 0.5 WHERE id = ?').run(conflict.id2);
|
|
132
|
+
detected++;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return detected;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Prune low-value memories
|
|
139
|
+
*/
|
|
140
|
+
pruneMemories() {
|
|
141
|
+
// Find relationships to prune (weak relationships with low-access entities)
|
|
142
|
+
const toPrune = this.db.prepare(`
|
|
143
|
+
SELECT r.id
|
|
144
|
+
FROM relationships r
|
|
145
|
+
INNER JOIN entities source ON r.source_id = source.id
|
|
146
|
+
INNER JOIN entities target ON r.target_id = target.id
|
|
147
|
+
WHERE r.strength < 0.05
|
|
148
|
+
AND source.access_count < 3
|
|
149
|
+
AND target.access_count < 3
|
|
150
|
+
`).all();
|
|
151
|
+
// Delete them
|
|
152
|
+
let deleted = 0;
|
|
153
|
+
for (const row of toPrune) {
|
|
154
|
+
this.db.prepare('DELETE FROM relationships WHERE id = ?').run(row.id);
|
|
155
|
+
deleted++;
|
|
156
|
+
}
|
|
157
|
+
return deleted;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Extract skills from positive interactions
|
|
161
|
+
*/
|
|
162
|
+
extractSkills(memories) {
|
|
163
|
+
let extracted = 0;
|
|
164
|
+
for (const memory of memories) {
|
|
165
|
+
// Look for repeated patterns
|
|
166
|
+
if (memory.topics && memory.topics.includes('analysis')) {
|
|
167
|
+
// Check if skill already exists
|
|
168
|
+
const existing = this.skillLibrary.search('data analysis', 1);
|
|
169
|
+
if (existing.length === 0) {
|
|
170
|
+
this.skillLibrary.addSkill({
|
|
171
|
+
name: 'Data Analysis',
|
|
172
|
+
description: 'Analyze datasets and provide insights',
|
|
173
|
+
category: 'analysis',
|
|
174
|
+
triggers: ['analyze', 'data', 'insights'],
|
|
175
|
+
});
|
|
176
|
+
extracted++;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return extracted;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Create backup for rollback capability
|
|
184
|
+
*/
|
|
185
|
+
createBackup() {
|
|
186
|
+
const backupId = 'backup_' + Date.now() + '_' + Math.random().toString(36).substr(2, 9);
|
|
187
|
+
this.db.prepare(`
|
|
188
|
+
INSERT INTO backup_metadata (id, backup_type, backup_path, created_at, expires_at)
|
|
189
|
+
VALUES (?, ?, ?, ?, ?)
|
|
190
|
+
`).run(backupId, 'consolidation', './backups/' + backupId + '.db', new Date().toISOString(), new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString());
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/consolidation/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiBH,MAAM,OAAO,qBAAqB;IACxB,aAAa,CAAqB;IAClC,cAAc,CAAsB;IACpC,YAAY,CAAe;IAC3B,EAAE,CAAoB;IAE9B,YACE,aAAiC,EACjC,cAAmC,EACnC,YAA0B,EAC1B,EAAqB;QAErB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAEjE,MAAM,KAAK,GAAuB;YAChC,iBAAiB,EAAE,CAAC;YACpB,iBAAiB,EAAE,CAAC;YACpB,oBAAoB,EAAE,CAAC;YACvB,iBAAiB,EAAE,CAAC;YACpB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,QAAQ,EAAE,CAAC;SACZ,CAAC;QAEF,IAAI,CAAC;YACH,sCAAsC;YACtC,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC/D,KAAK,CAAC,iBAAiB,GAAG,cAAc,CAAC,MAAM,CAAC;YAEhD,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,qCAAqC;gBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;gBAEhD,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;oBACxC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;oBACzC,KAAK,CAAC,iBAAiB,EAAE,CAAC;gBAC5B,CAAC;gBAED,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;oBAC1C,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;oBAC5C,KAAK,CAAC,oBAAoB,EAAE,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,IAAI,CAAC,cAAc,CAAC,kBAAkB,EAAE,CAAC;YAEzC,2BAA2B;YAC3B,KAAK,CAAC,iBAAiB,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YAEjD,mCAAmC;YACnC,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAE5C,sCAAsC;YACtC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;YAE3D,wBAAwB;YACxB,IAAI,CAAC,YAAY,EAAE,CAAC;YAEpB,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAExC,OAAO,CAAC,KAAK,CAAC,uCAAuC,KAAK,CAAC,iBAAiB,gBAAgB,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;YAChH,OAAO,CAAC,KAAK,CAAC,6BAA6B,KAAK,CAAC,iBAAiB,cAAc,KAAK,CAAC,oBAAoB,gBAAgB,CAAC,CAAC;YAC5H,OAAO,CAAC,KAAK,CAAC,4BAA4B,KAAK,CAAC,iBAAiB,sBAAsB,KAAK,CAAC,cAAc,WAAW,CAAC,CAAC;YAExH,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,gBAAgB,CAAC,MAAW;QAIlC,MAAM,QAAQ,GAAuG,EAAE,CAAC;QACxH,MAAM,aAAa,GAAU,EAAE,CAAC;QAEhC,0CAA0C;QAC1C,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC5C,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,SAAkB;oBACxB,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,sBAAsB;QACtB,MAAM,YAAY,GAAG;YACnB,gBAAgB;YAChB,cAAc;YACd,qBAAqB;SACtB,CAAC;QAEF,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAClD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;oBACd,IAAI,EAAE,YAAqB;oBAC3B,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACK,eAAe;QACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,+EAA+E;QAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;KAQjC,CAAC,CAAC,GAAG,EAAW,CAAC;QAElB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,2BAA2B;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC9B,+DAA+D,CAChE,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YAElC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,UAAU,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAE5F,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;SAGf,CAAC,CAAC,GAAG,CACJ,UAAU,EACV,QAAQ,CAAC,GAAG,EACZ,QAAQ,CAAC,GAAG,EACZ,eAAe,EACf,0BAA0B,QAAQ,CAAC,QAAQ,MAAM,QAAQ,CAAC,IAAI,SAAS,QAAQ,CAAC,IAAI,GAAG,EACvF,QAAQ,EACR,SAAS,CACV,CAAC;gBAEF,iCAAiC;gBACjC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACpF,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBAEpF,QAAQ,EAAE,CAAC;YACb,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,4EAA4E;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;KAQ/B,CAAC,CAAC,GAAG,EAAW,CAAC;QAElB,cAAc;QACd,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,QAAe;QACnC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,6BAA6B;YAC7B,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACxD,gCAAgC;gBAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;gBAE9D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC;wBACzB,IAAI,EAAE,eAAe;wBACrB,WAAW,EAAE,uCAAuC;wBACpD,QAAQ,EAAE,UAAU;wBACpB,QAAQ,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC;qBAC1C,CAAC,CAAC;oBACH,SAAS,EAAE,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAExF,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAGf,CAAC,CAAC,GAAG,CACJ,QAAQ,EACR,eAAe,EACf,YAAY,GAAG,QAAQ,GAAG,KAAK,EAC/B,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACxB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAC7D,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consolidation Pipeline - v3.0 Implementation
|
|
3
|
+
*
|
|
4
|
+
* Nightly process: working → semantic memory transformation.
|
|
5
|
+
* Runs at 3 AM, extracts knowledge, applies decay, detects conflicts.
|
|
6
|
+
*/
|
|
7
|
+
import { WorkingMemoryLayer } from '../memory-layers/working-memory.js';
|
|
8
|
+
import { SemanticMemoryLayer } from '../memory-layers/semantic-memory.js';
|
|
9
|
+
import { SkillLibrary } from '../memory-layers/skill-library.js';
|
|
10
|
+
import Database from 'better-sqlite3';
|
|
11
|
+
export interface ConsolidationStats {
|
|
12
|
+
memoriesProcessed: number;
|
|
13
|
+
entitiesExtracted: number;
|
|
14
|
+
relationshipsCreated: number;
|
|
15
|
+
conflictsDetected: number;
|
|
16
|
+
memoriesPruned: number;
|
|
17
|
+
skillsExtracted: number;
|
|
18
|
+
duration: number;
|
|
19
|
+
}
|
|
20
|
+
export declare class ConsolidationPipeline {
|
|
21
|
+
private workingMemory;
|
|
22
|
+
private semanticMemory;
|
|
23
|
+
private skillLibrary;
|
|
24
|
+
private db;
|
|
25
|
+
constructor(workingMemory: WorkingMemoryLayer, semanticMemory: SemanticMemoryLayer, skillLibrary: SkillLibrary, db: Database.Database);
|
|
26
|
+
/**
|
|
27
|
+
* Run full consolidation cycle
|
|
28
|
+
*/
|
|
29
|
+
consolidate(): Promise<ConsolidationStats>;
|
|
30
|
+
/**
|
|
31
|
+
* Extract entities and relationships from memory
|
|
32
|
+
* Simple keyword-based extraction (will enhance with LLM later)
|
|
33
|
+
*/
|
|
34
|
+
private extractKnowledge;
|
|
35
|
+
/**
|
|
36
|
+
* Detect simple conflicts (temporal impossibilities, contradictions)
|
|
37
|
+
*/
|
|
38
|
+
private detectConflicts;
|
|
39
|
+
/**
|
|
40
|
+
* Prune low-value memories
|
|
41
|
+
*/
|
|
42
|
+
private pruneMemories;
|
|
43
|
+
/**
|
|
44
|
+
* Extract skills from positive interactions
|
|
45
|
+
*/
|
|
46
|
+
private extractSkills;
|
|
47
|
+
/**
|
|
48
|
+
* Create backup for rollback capability
|
|
49
|
+
*/
|
|
50
|
+
private createBackup;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/consolidation/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAEtC,MAAM,WAAW,kBAAkB;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,qBAAqB;IAChC,OAAO,CAAC,aAAa,CAAqB;IAC1C,OAAO,CAAC,cAAc,CAAsB;IAC5C,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,EAAE,CAAoB;gBAG5B,aAAa,EAAE,kBAAkB,EACjC,cAAc,EAAE,mBAAmB,EACnC,YAAY,EAAE,YAAY,EAC1B,EAAE,EAAE,QAAQ,CAAC,QAAQ;IAQvB;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,kBAAkB,CAAC;IA8DhD;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAuCxB;;OAEG;IACH,OAAO,CAAC,eAAe;IA+CvB;;OAEG;IACH,OAAO,CAAC,aAAa;IAsBrB;;OAEG;IACH,OAAO,CAAC,aAAa;IAwBrB;;OAEG;IACH,OAAO,CAAC,YAAY;CAcrB"}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consolidation Pipeline - v3.0 Implementation
|
|
3
|
+
*
|
|
4
|
+
* Nightly process: working → semantic memory transformation.
|
|
5
|
+
* Runs at 3 AM, extracts knowledge, applies decay, detects conflicts.
|
|
6
|
+
*/
|
|
7
|
+
export class ConsolidationPipeline {
|
|
8
|
+
workingMemory;
|
|
9
|
+
semanticMemory;
|
|
10
|
+
skillLibrary;
|
|
11
|
+
db;
|
|
12
|
+
constructor(workingMemory, semanticMemory, skillLibrary, db) {
|
|
13
|
+
this.workingMemory = workingMemory;
|
|
14
|
+
this.semanticMemory = semanticMemory;
|
|
15
|
+
this.skillLibrary = skillLibrary;
|
|
16
|
+
this.db = db;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Run full consolidation cycle
|
|
20
|
+
*/
|
|
21
|
+
async consolidate() {
|
|
22
|
+
const startTime = Date.now();
|
|
23
|
+
console.error('[CONSOLIDATION] Starting consolidation cycle...');
|
|
24
|
+
const stats = {
|
|
25
|
+
memoriesProcessed: 0,
|
|
26
|
+
entitiesExtracted: 0,
|
|
27
|
+
relationshipsCreated: 0,
|
|
28
|
+
conflictsDetected: 0,
|
|
29
|
+
memoriesPruned: 0,
|
|
30
|
+
skillsExtracted: 0,
|
|
31
|
+
duration: 0,
|
|
32
|
+
};
|
|
33
|
+
try {
|
|
34
|
+
// Step 1: Extract from working memory
|
|
35
|
+
const recentMemories = await this.workingMemory.getRecent(100);
|
|
36
|
+
stats.memoriesProcessed = recentMemories.length;
|
|
37
|
+
for (const memory of recentMemories) {
|
|
38
|
+
// Extract entities and relationships
|
|
39
|
+
const extracted = this.extractKnowledge(memory);
|
|
40
|
+
for (const entity of extracted.entities) {
|
|
41
|
+
this.semanticMemory.upsertEntity(entity);
|
|
42
|
+
stats.entitiesExtracted++;
|
|
43
|
+
}
|
|
44
|
+
for (const rel of extracted.relationships) {
|
|
45
|
+
this.semanticMemory.upsertRelationship(rel);
|
|
46
|
+
stats.relationshipsCreated++;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
// Step 2: Apply temporal decay
|
|
50
|
+
this.semanticMemory.applyTemporalDecay();
|
|
51
|
+
// Step 3: Detect conflicts
|
|
52
|
+
stats.conflictsDetected = this.detectConflicts();
|
|
53
|
+
// Step 4: Prune low-value memories
|
|
54
|
+
stats.memoriesPruned = this.pruneMemories();
|
|
55
|
+
// Step 5: Extract skills (simplified)
|
|
56
|
+
stats.skillsExtracted = this.extractSkills(recentMemories);
|
|
57
|
+
// Step 6: Create backup
|
|
58
|
+
this.createBackup();
|
|
59
|
+
stats.duration = Date.now() - startTime;
|
|
60
|
+
console.error(`[CONSOLIDATION] Complete! Processed ${stats.memoriesProcessed} memories in ${stats.duration}ms`);
|
|
61
|
+
console.error(`[CONSOLIDATION] Extracted ${stats.entitiesExtracted} entities, ${stats.relationshipsCreated} relationships`);
|
|
62
|
+
console.error(`[CONSOLIDATION] Detected ${stats.conflictsDetected} conflicts, pruned ${stats.memoriesPruned} memories`);
|
|
63
|
+
return stats;
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
console.error('[CONSOLIDATION] Error:', error);
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Extract entities and relationships from memory
|
|
72
|
+
* Simple keyword-based extraction (will enhance with LLM later)
|
|
73
|
+
*/
|
|
74
|
+
extractKnowledge(memory) {
|
|
75
|
+
const entities = [];
|
|
76
|
+
const relationships = [];
|
|
77
|
+
// Use pre-extracted entities if available
|
|
78
|
+
if (memory.keyEntities && memory.keyEntities.length > 0) {
|
|
79
|
+
for (const entityName of memory.keyEntities) {
|
|
80
|
+
entities.push({
|
|
81
|
+
name: entityName,
|
|
82
|
+
type: 'concept',
|
|
83
|
+
confidence: 0.8,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Extract preferences
|
|
88
|
+
const prefPatterns = [
|
|
89
|
+
/prefer (\w+)/gi,
|
|
90
|
+
/like (\w+)/gi,
|
|
91
|
+
/favorite is (\w+)/gi,
|
|
92
|
+
];
|
|
93
|
+
for (const pattern of prefPatterns) {
|
|
94
|
+
const matches = memory.fullText.matchAll(pattern);
|
|
95
|
+
for (const match of matches) {
|
|
96
|
+
entities.push({
|
|
97
|
+
name: match[1],
|
|
98
|
+
type: 'preference',
|
|
99
|
+
confidence: 0.9,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return { entities, relationships };
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Detect simple conflicts (temporal impossibilities, contradictions)
|
|
107
|
+
*/
|
|
108
|
+
detectConflicts() {
|
|
109
|
+
let detected = 0;
|
|
110
|
+
// Temporal conflicts: same property, overlapping time ranges, different values
|
|
111
|
+
const conflicts = this.db.prepare(`
|
|
112
|
+
SELECT f1.id as id1, f2.id as id2, f1.property, f1.value as val1, f2.value as val2
|
|
113
|
+
FROM facts f1
|
|
114
|
+
JOIN facts f2 ON f1.entity_id = f2.entity_id AND f1.property = f2.property
|
|
115
|
+
WHERE f1.id < f2.id
|
|
116
|
+
AND f1.value != f2.value
|
|
117
|
+
AND f1.valid_until IS NULL
|
|
118
|
+
AND f2.valid_until IS NULL
|
|
119
|
+
`).all();
|
|
120
|
+
for (const conflict of conflicts) {
|
|
121
|
+
// Check if already flagged
|
|
122
|
+
const existing = this.db.prepare('SELECT * FROM conflicts WHERE fact_id_1 = ? AND fact_id_2 = ?').get(conflict.id1, conflict.id2);
|
|
123
|
+
if (!existing) {
|
|
124
|
+
const conflictId = 'conflict_' + Date.now() + '_' + Math.random().toString(36).substr(2, 9);
|
|
125
|
+
this.db.prepare(`
|
|
126
|
+
INSERT INTO conflicts (id, fact_id_1, fact_id_2, conflict_type, description, severity, resolution_status)
|
|
127
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
128
|
+
`).run(conflictId, conflict.id1, conflict.id2, 'contradiction', `Conflicting values for ${conflict.property}: "${conflict.val1}" vs "${conflict.val2}"`, 'medium', 'flagged');
|
|
129
|
+
// Lower confidence on both facts
|
|
130
|
+
this.db.prepare('UPDATE facts SET confidence = 0.5 WHERE id = ?').run(conflict.id1);
|
|
131
|
+
this.db.prepare('UPDATE facts SET confidence = 0.5 WHERE id = ?').run(conflict.id2);
|
|
132
|
+
detected++;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return detected;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Prune low-value memories
|
|
139
|
+
*/
|
|
140
|
+
pruneMemories() {
|
|
141
|
+
// Find relationships to prune (weak relationships with low-access entities)
|
|
142
|
+
const toPrune = this.db.prepare(`
|
|
143
|
+
SELECT r.id
|
|
144
|
+
FROM relationships r
|
|
145
|
+
INNER JOIN entities source ON r.source_id = source.id
|
|
146
|
+
INNER JOIN entities target ON r.target_id = target.id
|
|
147
|
+
WHERE r.strength < 0.05
|
|
148
|
+
AND source.access_count < 3
|
|
149
|
+
AND target.access_count < 3
|
|
150
|
+
`).all();
|
|
151
|
+
// Delete them
|
|
152
|
+
let deleted = 0;
|
|
153
|
+
for (const row of toPrune) {
|
|
154
|
+
this.db.prepare('DELETE FROM relationships WHERE id = ?').run(row.id);
|
|
155
|
+
deleted++;
|
|
156
|
+
}
|
|
157
|
+
return deleted;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Extract skills from positive interactions
|
|
161
|
+
*/
|
|
162
|
+
extractSkills(memories) {
|
|
163
|
+
let extracted = 0;
|
|
164
|
+
for (const memory of memories) {
|
|
165
|
+
// Look for repeated patterns
|
|
166
|
+
if (memory.topics && memory.topics.includes('analysis')) {
|
|
167
|
+
// Check if skill already exists
|
|
168
|
+
const existing = this.skillLibrary.search('data analysis', 1);
|
|
169
|
+
if (existing.length === 0) {
|
|
170
|
+
this.skillLibrary.addSkill({
|
|
171
|
+
name: 'Data Analysis',
|
|
172
|
+
description: 'Analyze datasets and provide insights',
|
|
173
|
+
category: 'analysis',
|
|
174
|
+
triggers: ['analyze', 'data', 'insights'],
|
|
175
|
+
});
|
|
176
|
+
extracted++;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return extracted;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Create backup for rollback capability
|
|
184
|
+
*/
|
|
185
|
+
createBackup() {
|
|
186
|
+
const backupId = 'backup_' + Date.now() + '_' + Math.random().toString(36).substr(2, 9);
|
|
187
|
+
this.db.prepare(`
|
|
188
|
+
INSERT INTO backup_metadata (id, backup_type, backup_path, created_at, expires_at)
|
|
189
|
+
VALUES (?, ?, ?, ?, ?)
|
|
190
|
+
`).run(backupId, 'consolidation', './backups/' + backupId + '.db', new Date().toISOString(), new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString());
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/consolidation/pipeline.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiBH,MAAM,OAAO,qBAAqB;IACxB,aAAa,CAAqB;IAClC,cAAc,CAAsB;IACpC,YAAY,CAAe;IAC3B,EAAE,CAAoB;IAE9B,YACE,aAAiC,EACjC,cAAmC,EACnC,YAA0B,EAC1B,EAAqB;QAErB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAEjE,MAAM,KAAK,GAAuB;YAChC,iBAAiB,EAAE,CAAC;YACpB,iBAAiB,EAAE,CAAC;YACpB,oBAAoB,EAAE,CAAC;YACvB,iBAAiB,EAAE,CAAC;YACpB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,QAAQ,EAAE,CAAC;SACZ,CAAC;QAEF,IAAI,CAAC;YACH,sCAAsC;YACtC,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC/D,KAAK,CAAC,iBAAiB,GAAG,cAAc,CAAC,MAAM,CAAC;YAEhD,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBACpC,qCAAqC;gBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;gBAEhD,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;oBACxC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;oBACzC,KAAK,CAAC,iBAAiB,EAAE,CAAC;gBAC5B,CAAC;gBAED,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;oBAC1C,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;oBAC5C,KAAK,CAAC,oBAAoB,EAAE,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,IAAI,CAAC,cAAc,CAAC,kBAAkB,EAAE,CAAC;YAEzC,2BAA2B;YAC3B,KAAK,CAAC,iBAAiB,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YAEjD,mCAAmC;YACnC,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAE5C,sCAAsC;YACtC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;YAE3D,wBAAwB;YACxB,IAAI,CAAC,YAAY,EAAE,CAAC;YAEpB,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAExC,OAAO,CAAC,KAAK,CAAC,uCAAuC,KAAK,CAAC,iBAAiB,gBAAgB,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;YAChH,OAAO,CAAC,KAAK,CAAC,6BAA6B,KAAK,CAAC,iBAAiB,cAAc,KAAK,CAAC,oBAAoB,gBAAgB,CAAC,CAAC;YAC5H,OAAO,CAAC,KAAK,CAAC,4BAA4B,KAAK,CAAC,iBAAiB,sBAAsB,KAAK,CAAC,cAAc,WAAW,CAAC,CAAC;YAExH,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,gBAAgB,CAAC,MAAW;QAIlC,MAAM,QAAQ,GAAuG,EAAE,CAAC;QACxH,MAAM,aAAa,GAAU,EAAE,CAAC;QAEhC,0CAA0C;QAC1C,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC5C,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,SAAkB;oBACxB,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,sBAAsB;QACtB,MAAM,YAAY,GAAG;YACnB,gBAAgB;YAChB,cAAc;YACd,qBAAqB;SACtB,CAAC;QAEF,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAClD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;oBACd,IAAI,EAAE,YAAqB;oBAC3B,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACK,eAAe;QACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,+EAA+E;QAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;KAQjC,CAAC,CAAC,GAAG,EAAW,CAAC;QAElB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,2BAA2B;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC9B,+DAA+D,CAChE,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YAElC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,UAAU,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAE5F,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;SAGf,CAAC,CAAC,GAAG,CACJ,UAAU,EACV,QAAQ,CAAC,GAAG,EACZ,QAAQ,CAAC,GAAG,EACZ,eAAe,EACf,0BAA0B,QAAQ,CAAC,QAAQ,MAAM,QAAQ,CAAC,IAAI,SAAS,QAAQ,CAAC,IAAI,GAAG,EACvF,QAAQ,EACR,SAAS,CACV,CAAC;gBAEF,iCAAiC;gBACjC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACpF,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBAEpF,QAAQ,EAAE,CAAC;YACb,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,4EAA4E;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;KAQ/B,CAAC,CAAC,GAAG,EAAW,CAAC;QAElB,cAAc;QACd,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,QAAe;QACnC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,6BAA6B;YAC7B,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACxD,gCAAgC;gBAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;gBAE9D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC;wBACzB,IAAI,EAAE,eAAe;wBACrB,WAAW,EAAE,uCAAuC;wBACpD,QAAQ,EAAE,UAAU;wBACpB,QAAQ,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC;qBAC1C,CAAC,CAAC;oBACH,SAAS,EAAE,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAExF,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAGf,CAAC,CAAC,GAAG,CACJ,QAAQ,EACR,eAAe,EACf,YAAY,GAAG,QAAQ,GAAG,KAAK,EAC/B,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACxB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAC7D,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BGE-large Embedding Client
|
|
3
|
+
*
|
|
4
|
+
* TypeScript client for the BGE-large embedding service.
|
|
5
|
+
* Provides methods for generating 1024-dimensional dense embeddings.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Embedding service response
|
|
9
|
+
*/
|
|
10
|
+
export interface EmbeddingResponse {
|
|
11
|
+
embeddings: number[][];
|
|
12
|
+
dimensions: number;
|
|
13
|
+
count: number;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Health check response
|
|
17
|
+
*/
|
|
18
|
+
export interface HealthResponse {
|
|
19
|
+
status: string;
|
|
20
|
+
model: string;
|
|
21
|
+
dimensions: number;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* BGE-large embedding client configuration
|
|
25
|
+
*/
|
|
26
|
+
export interface EmbeddingClientConfig {
|
|
27
|
+
/** Embedding service URL (default: http://localhost:8000) */
|
|
28
|
+
serviceUrl?: string;
|
|
29
|
+
/** Request timeout in milliseconds (default: 30000) */
|
|
30
|
+
timeout?: number;
|
|
31
|
+
/** Maximum retries on failure (default: 3) */
|
|
32
|
+
maxRetries?: number;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* BGE-large Embedding Client
|
|
36
|
+
*
|
|
37
|
+
* Communicates with the Python embedding service to generate
|
|
38
|
+
* 1024-dimensional dense embeddings using BGE-large-en-v1.5.
|
|
39
|
+
*/
|
|
40
|
+
export declare class EmbeddingClient {
|
|
41
|
+
private serviceUrl;
|
|
42
|
+
private timeout;
|
|
43
|
+
private maxRetries;
|
|
44
|
+
private embeddingCache;
|
|
45
|
+
constructor(config?: EmbeddingClientConfig);
|
|
46
|
+
/**
|
|
47
|
+
* Check if the embedding service is healthy
|
|
48
|
+
*
|
|
49
|
+
* @returns Health status and model information
|
|
50
|
+
*/
|
|
51
|
+
health(): Promise<HealthResponse>;
|
|
52
|
+
/**
|
|
53
|
+
* Generate embedding for a single text
|
|
54
|
+
*
|
|
55
|
+
* @param text - Text to embed
|
|
56
|
+
* @param normalize - Whether to normalize the embedding (default: true)
|
|
57
|
+
* @returns 1024-dimensional embedding vector
|
|
58
|
+
*/
|
|
59
|
+
embed(text: string, normalize?: boolean): Promise<number[]>;
|
|
60
|
+
/**
|
|
61
|
+
* Generate embeddings for multiple texts
|
|
62
|
+
*
|
|
63
|
+
* @param texts - Array of texts to embed
|
|
64
|
+
* @param normalize - Whether to normalize embeddings (default: true)
|
|
65
|
+
* @returns Array of 1024-dimensional embedding vectors
|
|
66
|
+
*/
|
|
67
|
+
embedBatch(texts: string[], normalize?: boolean): Promise<EmbeddingResponse>;
|
|
68
|
+
/**
|
|
69
|
+
* Generate embeddings with optimized batching
|
|
70
|
+
*
|
|
71
|
+
* Automatically splits large batches to avoid memory issues
|
|
72
|
+
* and timeouts on the embedding service.
|
|
73
|
+
*
|
|
74
|
+
* @param texts - Array of texts to embed
|
|
75
|
+
* @param batchSize - Number of texts per batch (default: 32)
|
|
76
|
+
* @returns Array of 1024-dimensional embedding vectors
|
|
77
|
+
*/
|
|
78
|
+
embedLargeBatch(texts: string[], batchSize?: number): Promise<number[][]>;
|
|
79
|
+
/**
|
|
80
|
+
* Fetch with automatic retry on failure
|
|
81
|
+
*
|
|
82
|
+
* @param url - URL to fetch
|
|
83
|
+
* @param options - Fetch options
|
|
84
|
+
* @returns Response
|
|
85
|
+
*/
|
|
86
|
+
private fetchWithRetry;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Factory function to create an embedding client
|
|
90
|
+
*
|
|
91
|
+
* @param config - Optional configuration
|
|
92
|
+
* @returns Embedding client instance
|
|
93
|
+
*/
|
|
94
|
+
export declare function createEmbeddingClient(config?: EmbeddingClientConfig): EmbeddingClient;
|
|
95
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/embeddings/client.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,EAAE,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAA6B;gBAEvC,MAAM,GAAE,qBAA0B;IAY9C;;;;OAIG;IACG,MAAM,IAAI,OAAO,CAAC,cAAc,CAAC;IAYvC;;;;;;OAMG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,OAAc,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAoBvE;;;;;;OAMG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,GAAE,OAAc,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA4CxF;;;;;;;;;OASG;IACG,eAAe,CACnB,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,GAAE,MAAW,GACrB,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAYtB;;;;;;OAMG;YACW,cAAc;CA6B7B;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,CAAC,EAAE,qBAAqB,GAC7B,eAAe,CAEjB"}
|