total-recall-brain 3.2.3 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/extension/background.js +14 -2
- package/extension/content-script.js +18 -1
- package/extension/lib/brain-client.js +86 -52
- package/extension/lib/preconfigured.js +5 -0
- package/extension/manifest.json +1 -1
- package/extension/options/options.html +1 -0
- package/extension/options/options.js +12 -6
- package/extension/popup/popup.html +1 -0
- package/extension/popup/popup.js +29 -4
- package/extension/sidepanel/sidepanel.html +1 -0
- package/extension/sidepanel/sidepanel.js +24 -6
- package/frontend/index.html +8 -0
- package/frontend/src/api.ts +1 -2
- package/frontend/src/pages/ChatPage.tsx +9 -6
- package/frontend/src/pages/IntegrationsPage.tsx +144 -3
- package/package.json +2 -2
- package/scaffold/.agent/skills/total-recall/SKILL.md +55 -6
- package/scaffold/.agent/skills/total-recall/memory-vault/invariants/operating-instructions.md +2 -2
- package/scaffold/.agent/skills/total-recall/skills/cli-agents/agents.yml +1 -6
- package/scaffold/.agent/skills/total-recall/skills/ssss/SKILL.md +19 -0
- package/scaffold/.agent/skills/total-recall/skills/ssss/references/ssss-spec.md +33 -5
- package/src/cli/forget.mjs +18 -12
- package/src/cli/ingest/index.mjs +3 -2
- package/src/cli/init.mjs +30 -0
- package/src/cli/map.mjs +2 -2
- package/src/cli/rebuild.mjs +1 -1
- package/src/cli/remember.mjs +21 -11
- package/src/cli/setup.mjs +10 -1
- package/src/cli/share.mjs +7 -2
- package/src/core/append-log.mjs +131 -0
- package/src/core/clarity-rewriter.mjs +10 -6
- package/src/core/conclusion-writer.mjs +3 -2
- package/src/core/config.mjs +6 -0
- package/src/core/conflict-detector.mjs +4 -2
- package/src/core/context-compiler.mjs +412 -0
- package/src/core/daemon-loop.mjs +15 -0
- package/src/core/dream.mjs +4 -4
- package/src/core/drift-detector.mjs +3 -3
- package/src/core/embeddings.mjs +60 -10
- package/src/core/fact-seeker.mjs +70 -44
- package/src/core/import-rules.mjs +3 -2
- package/src/core/inference-engine.mjs +5 -4
- package/src/core/migrate.mjs +4 -3
- package/src/core/optimizer.mjs +6 -5
- package/src/core/parallel-context.mjs +288 -0
- package/src/core/post-mortem.mjs +4 -3
- package/src/core/research-queue.mjs +4 -3
- package/src/core/runtime.mjs +2 -2
- package/src/core/sandbox.mjs +56 -26
- package/src/core/scheduler.mjs +6 -5
- package/src/core/schema.mjs +2 -2
- package/src/core/search.mjs +167 -101
- package/src/core/surface.mjs +203 -53
- package/src/core/validated-write.mjs +100 -0
- package/src/core/vault-cache.mjs +46 -18
- package/src/core/vault.mjs +17 -29
- package/src/core/vector-field.mjs +659 -0
- package/src/core/vector-store.mjs +85 -0
- package/src/server/api.mjs +4 -3
- package/src/server/index.mjs +13 -61
- package/src/server/rest.mjs +238 -29
- package/src/server/route-manifest.json +344 -77
- package/src/server/routes/memory.mjs +85 -39
- package/src/server/routes/sessions.mjs +6 -2
- package/src/server/routes/share.mjs +9 -2
|
@@ -0,0 +1,659 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/core/vector-field.mjs
|
|
3
|
+
*
|
|
4
|
+
* The Infinite Vector Field
|
|
5
|
+
*
|
|
6
|
+
* Every node is a measurement point in continuous embedding space.
|
|
7
|
+
* Each point carries:
|
|
8
|
+
* - position: embedding[768] — where in concept space
|
|
9
|
+
* - half_life: λ (decay constant) — how fast relevance decays
|
|
10
|
+
* - trajectory: d²R/dt² — is decay accelerating or decelerating
|
|
11
|
+
* - velocity: dR/dt — rate of relevance change
|
|
12
|
+
* - novelty: information density — has model seen this before?
|
|
13
|
+
* - coupling: covariance[i][j] — entanglement with every other point
|
|
14
|
+
* - dependencies: slug[] — nodes that must co-occur
|
|
15
|
+
*
|
|
16
|
+
* The field exists everywhere. Nodes are where we've sampled it.
|
|
17
|
+
* Queries sample new points. The field interpolates.
|
|
18
|
+
* The compiled field replaces all LLM calls for known query regions.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import fs from 'node:fs';
|
|
22
|
+
import path from 'node:path';
|
|
23
|
+
import { getNodes } from './vault-cache.mjs';
|
|
24
|
+
import { getEmbedding, cosineSimilarity, nodeToEmbedText, loadEmbeddingsIndex, saveEmbeddingToIndex } from './embeddings.mjs';
|
|
25
|
+
import { inferMemoryLayer } from './memory-layers.mjs';
|
|
26
|
+
import { logger } from './logger.mjs';
|
|
27
|
+
import { brainDir } from './config.mjs';
|
|
28
|
+
|
|
29
|
+
const DERIVED_DIR = path.join(brainDir, 'memory-derived');
|
|
30
|
+
|
|
31
|
+
function fieldPath(derivedDir = DERIVED_DIR) {
|
|
32
|
+
return path.join(derivedDir, 'vector-field.json');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function queryLogPath(derivedDir = DERIVED_DIR) {
|
|
36
|
+
return path.join(derivedDir, 'query-log.jsonl');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const LN2 = Math.log(2);
|
|
40
|
+
|
|
41
|
+
// ─── Half-Life Constants (milliseconds) ─────────────────────────────────────
|
|
42
|
+
|
|
43
|
+
const HALF_LIFE_TABLE = {
|
|
44
|
+
// Category-based half-lives
|
|
45
|
+
invariant: Infinity, // never decays
|
|
46
|
+
invariants: Infinity, // never decays
|
|
47
|
+
correction: 60 * 24 * 60 * 60 * 1000, // 60 days
|
|
48
|
+
corrections: 60 * 24 * 60 * 60 * 1000, // 60 days
|
|
49
|
+
decision: 90 * 24 * 60 * 60 * 1000, // 90 days
|
|
50
|
+
decisions: 90 * 24 * 60 * 60 * 1000, // 90 days
|
|
51
|
+
preference: 180 * 24 * 60 * 60 * 1000, // 180 days
|
|
52
|
+
preferences: 180 * 24 * 60 * 60 * 1000, // 180 days
|
|
53
|
+
fact: 30 * 24 * 60 * 60 * 1000, // 30 days (post-cutoff facts)
|
|
54
|
+
facts: 30 * 24 * 60 * 60 * 1000, // 30 days (post-cutoff facts)
|
|
55
|
+
concept: 120 * 24 * 60 * 60 * 1000, // 120 days
|
|
56
|
+
concepts: 120 * 24 * 60 * 60 * 1000, // 120 days
|
|
57
|
+
pattern: 90 * 24 * 60 * 60 * 1000, // 90 days
|
|
58
|
+
patterns: 90 * 24 * 60 * 60 * 1000, // 90 days
|
|
59
|
+
'anti-pattern': 60 * 24 * 60 * 60 * 1000, // 60 days
|
|
60
|
+
'anti-patterns': 60 * 24 * 60 * 60 * 1000, // 60 days
|
|
61
|
+
lore: 365 * 24 * 60 * 60 * 1000, // 1 year
|
|
62
|
+
// Priority overrides
|
|
63
|
+
_absolute: Infinity, // absolute priority = infinite half-life
|
|
64
|
+
_high: 180 * 24 * 60 * 60 * 1000, // 180 days minimum
|
|
65
|
+
// Ephemeral types
|
|
66
|
+
session: 2 * 60 * 60 * 1000, // 2 hours
|
|
67
|
+
task: 30 * 60 * 1000, // 30 minutes
|
|
68
|
+
grounding: 30 * 60 * 1000, // 30 minutes
|
|
69
|
+
research: 14 * 24 * 60 * 60 * 1000, // 14 days
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Determine the half-life for a node based on its category and priority.
|
|
74
|
+
*/
|
|
75
|
+
function computeHalfLife(node) {
|
|
76
|
+
// Priority overrides category
|
|
77
|
+
if (node.priority === 'absolute') return Infinity;
|
|
78
|
+
if (node.priority === 'high') {
|
|
79
|
+
const categoryHL = HALF_LIFE_TABLE[node.category] || HALF_LIFE_TABLE.fact;
|
|
80
|
+
return Math.max(categoryHL, HALF_LIFE_TABLE._high);
|
|
81
|
+
}
|
|
82
|
+
return HALF_LIFE_TABLE[node.category] || HALF_LIFE_TABLE.fact;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Compute decay constant λ = ln(2) / half_life
|
|
87
|
+
*/
|
|
88
|
+
function decayConstant(halfLife) {
|
|
89
|
+
if (!isFinite(halfLife)) return 0; // no decay
|
|
90
|
+
return LN2 / halfLife;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Compute current effective relevance given peak relevance and time elapsed.
|
|
95
|
+
* R(t) = R₀ × e^(-λt) × (1 + trajectory × t_normalized)
|
|
96
|
+
*/
|
|
97
|
+
function effectiveRelevance(peakRelevance, lambda, elapsedMs, trajectory = 0) {
|
|
98
|
+
if (lambda === 0) return peakRelevance; // infinite half-life
|
|
99
|
+
const decayed = peakRelevance * Math.exp(-lambda * elapsedMs);
|
|
100
|
+
// Trajectory modulates: positive trajectory slows decay, negative accelerates
|
|
101
|
+
const tNorm = Math.min(elapsedMs / (7 * 24 * 60 * 60 * 1000), 1); // normalize to 1 week
|
|
102
|
+
const trajectoryMod = 1 + (trajectory * tNorm * 0.3); // ±30% max adjustment
|
|
103
|
+
return Math.max(0, decayed * trajectoryMod);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ─── Novelty Scoring ────────────────────────────────────────────────────────
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Compute novelty score for a node.
|
|
110
|
+
* Novelty = inverse of how many times this node has been injected into context.
|
|
111
|
+
* First time = 1.0, after 100 times = ~0.1
|
|
112
|
+
*/
|
|
113
|
+
function computeNovelty(accessCount) {
|
|
114
|
+
if (accessCount <= 0) return 1.0; // never seen = maximum novelty
|
|
115
|
+
return 1 / (1 + Math.log1p(accessCount)); // logarithmic decay of novelty
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ─── Compilation ────────────────────────────────────────────────────────────
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Compile the full vector field from the vault.
|
|
122
|
+
*
|
|
123
|
+
* 1. Embed every active node → position vectors
|
|
124
|
+
* 2. Compute half-life, trajectory, novelty per point
|
|
125
|
+
* 3. Compute full pairwise covariance → coupling matrix
|
|
126
|
+
* 4. Extract dependency graph
|
|
127
|
+
* 5. Persist to disk
|
|
128
|
+
*/
|
|
129
|
+
export async function compileField({ vaultDir, derivedDir = DERIVED_DIR } = {}) {
|
|
130
|
+
const startTime = Date.now();
|
|
131
|
+
const allNodes = getNodes(vaultDir).filter(n => n.status === 'active');
|
|
132
|
+
const totalNodes = allNodes.length;
|
|
133
|
+
const now = Date.now();
|
|
134
|
+
|
|
135
|
+
logger.info('vector-field', `Compiling field from ${totalNodes} nodes...`);
|
|
136
|
+
|
|
137
|
+
// ── Load query log for trajectory computation ──
|
|
138
|
+
const queryHistory = loadQueryHistory(derivedDir);
|
|
139
|
+
|
|
140
|
+
// ── Phase 1: Embed all nodes + compute physics ──
|
|
141
|
+
const existingIndex = loadEmbeddingsIndex(derivedDir);
|
|
142
|
+
const points = [];
|
|
143
|
+
const newlyEmbedded = new Map();
|
|
144
|
+
let embedded = 0;
|
|
145
|
+
let cached = 0;
|
|
146
|
+
|
|
147
|
+
const EMBED_CONCURRENCY = 5;
|
|
148
|
+
let cursor = 0;
|
|
149
|
+
|
|
150
|
+
async function embedWorker() {
|
|
151
|
+
while (cursor < allNodes.length) {
|
|
152
|
+
const i = cursor++;
|
|
153
|
+
const node = allNodes[i];
|
|
154
|
+
const text = nodeToEmbedText(node);
|
|
155
|
+
|
|
156
|
+
let embedding;
|
|
157
|
+
if (existingIndex[node.slug]?.embedding) {
|
|
158
|
+
embedding = existingIndex[node.slug].embedding;
|
|
159
|
+
cached++;
|
|
160
|
+
} else {
|
|
161
|
+
try {
|
|
162
|
+
embedding = await getEmbedding(text);
|
|
163
|
+
newlyEmbedded.set(node.slug, embedding);
|
|
164
|
+
embedded++;
|
|
165
|
+
} catch (err) {
|
|
166
|
+
logger.error('vector-field', `Failed to embed ${node.slug}: ${err.message}`);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// ── Physics computation ──
|
|
172
|
+
const decay = node.decay || {};
|
|
173
|
+
const accessCount = decay.access_count || 0;
|
|
174
|
+
const lastAccessed = node.last_accessed ? new Date(node.last_accessed).getTime() : 0;
|
|
175
|
+
const createdAt = node.created ? new Date(node.created).getTime() : now;
|
|
176
|
+
const ageMs = now - createdAt;
|
|
177
|
+
|
|
178
|
+
// Half-life
|
|
179
|
+
const halfLife = computeHalfLife(node);
|
|
180
|
+
const lambda = decayConstant(halfLife);
|
|
181
|
+
|
|
182
|
+
// Velocity: dR/dt from query log
|
|
183
|
+
const { velocity, trajectory } = computeTrajectory(node.slug, queryHistory, now);
|
|
184
|
+
|
|
185
|
+
// Novelty
|
|
186
|
+
const novelty = computeNovelty(accessCount);
|
|
187
|
+
|
|
188
|
+
// Peak relevance (baseline from importance + priority)
|
|
189
|
+
const importanceWeight = (node.importance || 3) / 5;
|
|
190
|
+
const priorityWeight = node.priority === 'absolute' ? 1.0 : node.priority === 'high' ? 0.8 : node.priority === 'normal' ? 0.5 : 0.3;
|
|
191
|
+
const peakRelevance = Math.max(importanceWeight, priorityWeight);
|
|
192
|
+
|
|
193
|
+
// Time since last access (for decay computation at sample time)
|
|
194
|
+
const timeSinceAccess = lastAccessed > 0 ? now - lastAccessed : ageMs;
|
|
195
|
+
|
|
196
|
+
// Dependencies: extract from node.related field
|
|
197
|
+
const dependencies = node.related ? (Array.isArray(node.related) ? node.related : String(node.related).split(',').map(s => s.trim())) : [];
|
|
198
|
+
|
|
199
|
+
// Precompute L2 norm
|
|
200
|
+
let mag = 0;
|
|
201
|
+
for (let j = 0; j < embedding.length; j++) mag += embedding[j] * embedding[j];
|
|
202
|
+
mag = Math.sqrt(mag);
|
|
203
|
+
|
|
204
|
+
points.push({
|
|
205
|
+
slug: node.slug,
|
|
206
|
+
title: node.title || node.slug,
|
|
207
|
+
category: node.category || 'unknown',
|
|
208
|
+
layer: inferMemoryLayer(node),
|
|
209
|
+
embedding,
|
|
210
|
+
magnitude: mag,
|
|
211
|
+
// ── Physics ──
|
|
212
|
+
half_life: isFinite(halfLife) ? halfLife : null, // null = infinite
|
|
213
|
+
lambda, // decay constant
|
|
214
|
+
peak_relevance: peakRelevance, // R₀
|
|
215
|
+
velocity, // dR/dt
|
|
216
|
+
trajectory, // d²R/dt² (acceleration)
|
|
217
|
+
novelty, // information density
|
|
218
|
+
// ── Metadata ──
|
|
219
|
+
access_count: accessCount,
|
|
220
|
+
last_accessed: lastAccessed,
|
|
221
|
+
created_at: createdAt,
|
|
222
|
+
time_since_access: timeSinceAccess,
|
|
223
|
+
importance: node.importance || 3,
|
|
224
|
+
priority: node.priority || 'normal',
|
|
225
|
+
dependencies,
|
|
226
|
+
body_length: (node.body || node.content || '').length,
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
const workers = Array.from(
|
|
232
|
+
{ length: Math.min(EMBED_CONCURRENCY, allNodes.length) },
|
|
233
|
+
() => embedWorker()
|
|
234
|
+
);
|
|
235
|
+
await Promise.all(workers);
|
|
236
|
+
|
|
237
|
+
for (const [slug, embedding] of newlyEmbedded) {
|
|
238
|
+
saveEmbeddingToIndex(derivedDir, slug, embedding);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const embedMs = Date.now() - startTime;
|
|
242
|
+
logger.info('vector-field', `Embedded ${embedded} new + ${cached} cached in ${embedMs}ms`);
|
|
243
|
+
|
|
244
|
+
// ── Phase 2: Pairwise covariance matrix ──
|
|
245
|
+
const N = points.length;
|
|
246
|
+
const slugIndex = points.map(p => p.slug);
|
|
247
|
+
const covariance = new Array(N);
|
|
248
|
+
|
|
249
|
+
for (let i = 0; i < N; i++) {
|
|
250
|
+
covariance[i] = new Float32Array(N);
|
|
251
|
+
for (let j = i; j < N; j++) {
|
|
252
|
+
if (i === j) {
|
|
253
|
+
covariance[i][j] = 1.0;
|
|
254
|
+
} else {
|
|
255
|
+
const sim = fastCosine(points[i].embedding, points[j].embedding, points[i].magnitude, points[j].magnitude);
|
|
256
|
+
covariance[i][j] = sim;
|
|
257
|
+
if (!covariance[j]) covariance[j] = new Float32Array(N);
|
|
258
|
+
covariance[j][i] = sim;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const covMs = Date.now() - startTime - embedMs;
|
|
264
|
+
|
|
265
|
+
// ── Phase 3: Compile and persist ──
|
|
266
|
+
const covarianceSerializable = covariance.map(row => Array.from(row));
|
|
267
|
+
|
|
268
|
+
const field = {
|
|
269
|
+
points,
|
|
270
|
+
covariance: covarianceSerializable,
|
|
271
|
+
slugIndex,
|
|
272
|
+
compiled_at: now,
|
|
273
|
+
dimensions: points[0]?.embedding?.length || 768,
|
|
274
|
+
meta: {
|
|
275
|
+
total_nodes: totalNodes,
|
|
276
|
+
points_compiled: N,
|
|
277
|
+
embedded_new: embedded,
|
|
278
|
+
embedded_cached: cached,
|
|
279
|
+
embed_ms: embedMs,
|
|
280
|
+
covariance_ms: covMs,
|
|
281
|
+
compile_ms: Date.now() - startTime,
|
|
282
|
+
matrix_size: `${N}×${N}`,
|
|
283
|
+
matrix_entries: N * N,
|
|
284
|
+
infinite_halflife: points.filter(p => p.lambda === 0).length,
|
|
285
|
+
with_dependencies: points.filter(p => p.dependencies.length > 0).length,
|
|
286
|
+
avg_novelty: points.length > 0 ? Math.round((points.reduce((s, p) => s + p.novelty, 0) / points.length) * 1000) / 1000 : 0,
|
|
287
|
+
},
|
|
288
|
+
};
|
|
289
|
+
|
|
290
|
+
const outPath = fieldPath(derivedDir);
|
|
291
|
+
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
|
292
|
+
fs.writeFileSync(outPath, JSON.stringify(field));
|
|
293
|
+
|
|
294
|
+
logger.info('vector-field', `Field compiled: ${N} points, ${N * N} couplings, ${field.meta.infinite_halflife} immortal, ${field.meta.with_dependencies} with deps, ${field.meta.compile_ms}ms`);
|
|
295
|
+
|
|
296
|
+
return field;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// ─── Trajectory Computation ─────────────────────────────────────────────────
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Compute velocity (dR/dt) and trajectory (d²R/dt²) from query log.
|
|
303
|
+
* Uses 3 time windows to detect acceleration/deceleration.
|
|
304
|
+
*/
|
|
305
|
+
function computeTrajectory(slug, queryHistory, now) {
|
|
306
|
+
const WINDOW = 24 * 60 * 60 * 1000; // 1 day
|
|
307
|
+
|
|
308
|
+
// Count hits in 3 windows: [0-24h], [24-48h], [48-72h]
|
|
309
|
+
let w0 = 0, w1 = 0, w2 = 0;
|
|
310
|
+
for (const entry of queryHistory) {
|
|
311
|
+
if (!entry.hits?.includes(slug)) continue;
|
|
312
|
+
const age = now - entry.t;
|
|
313
|
+
if (age < WINDOW) w0++;
|
|
314
|
+
else if (age < 2 * WINDOW) w1++;
|
|
315
|
+
else if (age < 3 * WINDOW) w2++;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// Velocity = first derivative (recent vs previous)
|
|
319
|
+
const velocity = (w0 - w1) * 0.1;
|
|
320
|
+
|
|
321
|
+
// Trajectory = second derivative (acceleration of velocity)
|
|
322
|
+
const v_prev = (w1 - w2) * 0.1;
|
|
323
|
+
const trajectory = velocity - v_prev; // positive = accelerating, negative = decelerating
|
|
324
|
+
|
|
325
|
+
return {
|
|
326
|
+
velocity: Math.max(-1, Math.min(1, velocity)),
|
|
327
|
+
trajectory: Math.max(-1, Math.min(1, trajectory)),
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function loadQueryHistory(derivedDir = DERIVED_DIR) {
|
|
332
|
+
try {
|
|
333
|
+
const logPath = queryLogPath(derivedDir);
|
|
334
|
+
if (!fs.existsSync(logPath)) return [];
|
|
335
|
+
const lines = fs.readFileSync(logPath, 'utf8').trim().split('\n');
|
|
336
|
+
const cutoff = Date.now() - (3 * 24 * 60 * 60 * 1000); // only last 3 days
|
|
337
|
+
return lines.map(l => { try { return JSON.parse(l); } catch { return null; } })
|
|
338
|
+
.filter(e => e && e.t > cutoff);
|
|
339
|
+
} catch { return []; }
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// ─── Fast Cosine ────────────────────────────────────────────────────────────
|
|
343
|
+
|
|
344
|
+
function fastCosine(a, b, magA, magB) {
|
|
345
|
+
if (!a || !b || a.length !== b.length) return 0;
|
|
346
|
+
const denom = magA * magB;
|
|
347
|
+
if (denom === 0) return 0;
|
|
348
|
+
let dot = 0;
|
|
349
|
+
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
350
|
+
return dot / denom;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// ─── Field Sampling ─────────────────────────────────────────────────────────
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Sample the vector field at a query point.
|
|
357
|
+
*
|
|
358
|
+
* Pipeline:
|
|
359
|
+
* 1. Embed query → position in field
|
|
360
|
+
* 2. Direct similarity to all points
|
|
361
|
+
* 3. Apply half-life decay: R(t) = R₀ × e^(-λΔt) × trajectory_mod
|
|
362
|
+
* 4. Entanglement propagation through covariance
|
|
363
|
+
* 5. Novelty weighting (unseen info > repeated info)
|
|
364
|
+
* 6. Dependency resolution (pull in required co-nodes)
|
|
365
|
+
* 7. Interference detection (flag contradictions)
|
|
366
|
+
* 8. Return ranked results
|
|
367
|
+
*/
|
|
368
|
+
export async function sampleField({
|
|
369
|
+
query,
|
|
370
|
+
field = null,
|
|
371
|
+
topK = 20,
|
|
372
|
+
entanglementBoost = 0.15,
|
|
373
|
+
noveltyWeight = 0.1,
|
|
374
|
+
derivedDir = DERIVED_DIR,
|
|
375
|
+
} = {}) {
|
|
376
|
+
const startTime = Date.now();
|
|
377
|
+
const now = Date.now();
|
|
378
|
+
|
|
379
|
+
if (!field) {
|
|
380
|
+
field = loadField(derivedDir);
|
|
381
|
+
if (!field) throw new Error('Vector field not compiled. Run compileField() first.');
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
const { points, covariance, slugIndex } = field;
|
|
385
|
+
const N = points.length;
|
|
386
|
+
if (N === 0) return { results: [], query_embedding: [], sample_ms: 0, interference: [] };
|
|
387
|
+
|
|
388
|
+
// ── 1. Embed query ──
|
|
389
|
+
const queryEmbedding = await getEmbedding(query);
|
|
390
|
+
let queryMag = 0;
|
|
391
|
+
for (let i = 0; i < queryEmbedding.length; i++) queryMag += queryEmbedding[i] * queryEmbedding[i];
|
|
392
|
+
queryMag = Math.sqrt(queryMag);
|
|
393
|
+
|
|
394
|
+
// ── 2. Direct similarity ──
|
|
395
|
+
const directScores = new Float64Array(N);
|
|
396
|
+
for (let i = 0; i < N; i++) {
|
|
397
|
+
directScores[i] = fastCosine(queryEmbedding, points[i].embedding, queryMag, points[i].magnitude);
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
// ── 3. Half-life decay ──
|
|
401
|
+
const decayedScores = new Float64Array(N);
|
|
402
|
+
for (let i = 0; i < N; i++) {
|
|
403
|
+
const p = points[i];
|
|
404
|
+
const elapsed = p.last_accessed > 0 ? now - p.last_accessed : p.time_since_access || 0;
|
|
405
|
+
const decayedRelevance = effectiveRelevance(p.peak_relevance, p.lambda, elapsed, p.trajectory);
|
|
406
|
+
// Blend: 70% similarity, 30% decayed relevance
|
|
407
|
+
decayedScores[i] = (directScores[i] * 0.7) + (decayedRelevance * 0.3);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
// ── 4. Entanglement propagation ──
|
|
411
|
+
const propagatedScores = new Float64Array(N);
|
|
412
|
+
for (let i = 0; i < N; i++) {
|
|
413
|
+
let entSum = 0;
|
|
414
|
+
for (let j = 0; j < N; j++) {
|
|
415
|
+
if (i === j) continue;
|
|
416
|
+
entSum += covariance[i][j] * decayedScores[j];
|
|
417
|
+
}
|
|
418
|
+
const avgEnt = N > 1 ? entSum / (N - 1) : 0;
|
|
419
|
+
propagatedScores[i] = decayedScores[i] + (entanglementBoost * avgEnt);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// ── 5. Novelty weighting ──
|
|
423
|
+
const finalScores = new Float64Array(N);
|
|
424
|
+
for (let i = 0; i < N; i++) {
|
|
425
|
+
finalScores[i] = propagatedScores[i] + (points[i].novelty * noveltyWeight);
|
|
426
|
+
// Absolute priority floor
|
|
427
|
+
if (points[i].priority === 'absolute') {
|
|
428
|
+
finalScores[i] = Math.max(finalScores[i], 0.8);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// ── 6. Rank ──
|
|
433
|
+
const indexed = Array.from(finalScores).map((score, i) => ({ index: i, score }));
|
|
434
|
+
indexed.sort((a, b) => b.score - a.score);
|
|
435
|
+
let topResults = indexed.slice(0, topK);
|
|
436
|
+
|
|
437
|
+
// ── 7. Dependency resolution — pull in required nodes ──
|
|
438
|
+
const includedSlugs = new Set(topResults.map(r => points[r.index].slug));
|
|
439
|
+
const depsPulledIn = [];
|
|
440
|
+
for (const r of [...topResults]) {
|
|
441
|
+
const p = points[r.index];
|
|
442
|
+
for (const dep of p.dependencies) {
|
|
443
|
+
if (!includedSlugs.has(dep)) {
|
|
444
|
+
const depIdx = points.findIndex(pt => pt.slug === dep);
|
|
445
|
+
if (depIdx >= 0) {
|
|
446
|
+
topResults.push({ index: depIdx, score: finalScores[depIdx] });
|
|
447
|
+
includedSlugs.add(dep);
|
|
448
|
+
depsPulledIn.push(dep);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
// Re-sort after dependency injection
|
|
454
|
+
topResults.sort((a, b) => b.score - a.score);
|
|
455
|
+
topResults = topResults.slice(0, topK + depsPulledIn.length); // allow deps to exceed topK
|
|
456
|
+
|
|
457
|
+
// ── 8. Interference detection ──
|
|
458
|
+
const interference = detectInterference(topResults, points, covariance);
|
|
459
|
+
|
|
460
|
+
// ── Format results ──
|
|
461
|
+
const results = topResults.map(({ index, score }) => {
|
|
462
|
+
const p = points[index];
|
|
463
|
+
const elapsed = p.last_accessed > 0 ? now - p.last_accessed : p.time_since_access || 0;
|
|
464
|
+
const remainingHL = p.lambda > 0 ? Math.round((LN2 / p.lambda) / (24 * 60 * 60 * 1000)) : null;
|
|
465
|
+
return {
|
|
466
|
+
slug: p.slug,
|
|
467
|
+
title: p.title,
|
|
468
|
+
category: p.category,
|
|
469
|
+
layer: p.layer,
|
|
470
|
+
score: round4(score),
|
|
471
|
+
direct_similarity: round4(directScores[index]),
|
|
472
|
+
decayed_relevance: round4(decayedScores[index]),
|
|
473
|
+
entanglement_boost: round4(propagatedScores[index] - decayedScores[index]),
|
|
474
|
+
novelty: round4(p.novelty),
|
|
475
|
+
velocity: round4(p.velocity),
|
|
476
|
+
trajectory: round4(p.trajectory),
|
|
477
|
+
half_life_days: remainingHL,
|
|
478
|
+
time_since_access_hours: Math.round(elapsed / (60 * 60 * 1000) * 10) / 10,
|
|
479
|
+
importance: p.importance,
|
|
480
|
+
priority: p.priority,
|
|
481
|
+
access_count: p.access_count,
|
|
482
|
+
dependencies: p.dependencies,
|
|
483
|
+
pulled_by_dependency: depsPulledIn.includes(p.slug),
|
|
484
|
+
};
|
|
485
|
+
});
|
|
486
|
+
|
|
487
|
+
const sampleMs = Date.now() - startTime;
|
|
488
|
+
logQuery(query, results.map(r => r.slug), derivedDir);
|
|
489
|
+
|
|
490
|
+
logger.info('vector-field', `Sampled in ${sampleMs}ms: ${results.length} results, ${depsPulledIn.length} deps pulled, ${interference.length} interference pairs`);
|
|
491
|
+
|
|
492
|
+
return { results, query_embedding: queryEmbedding, sample_ms: sampleMs, interference, deps_pulled: depsPulledIn };
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// ─── Interference Detection ─────────────────────────────────────────────────
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Detect constructive and destructive interference between included nodes.
|
|
499
|
+
* Destructive: nodes with opposing modalities (must vs must_not) or
|
|
500
|
+
* semantically similar but from conflicting categories (correction vs pattern).
|
|
501
|
+
* Constructive: nodes that reinforce each other (high covariance, same direction).
|
|
502
|
+
*/
|
|
503
|
+
function detectInterference(topResults, points, covariance) {
|
|
504
|
+
const pairs = [];
|
|
505
|
+
for (let i = 0; i < topResults.length; i++) {
|
|
506
|
+
for (let j = i + 1; j < topResults.length; j++) {
|
|
507
|
+
const pi = points[topResults[i].index];
|
|
508
|
+
const pj = points[topResults[j].index];
|
|
509
|
+
const cov = covariance[topResults[i].index]?.[topResults[j].index] || 0;
|
|
510
|
+
|
|
511
|
+
// High similarity + opposing categories = potential destructive interference
|
|
512
|
+
if (cov > 0.8) {
|
|
513
|
+
const opposing =
|
|
514
|
+
(pi.category === 'correction' && pj.category === 'pattern') ||
|
|
515
|
+
(pi.category === 'pattern' && pj.category === 'correction') ||
|
|
516
|
+
(pi.category === 'anti-pattern' && pj.category === 'pattern') ||
|
|
517
|
+
(pi.category === 'pattern' && pj.category === 'anti-pattern');
|
|
518
|
+
|
|
519
|
+
if (opposing) {
|
|
520
|
+
pairs.push({
|
|
521
|
+
type: 'destructive',
|
|
522
|
+
a: pi.slug,
|
|
523
|
+
b: pj.slug,
|
|
524
|
+
coupling: round4(cov),
|
|
525
|
+
reason: `${pi.category} ↔ ${pj.category} (opposing categories, high similarity)`,
|
|
526
|
+
});
|
|
527
|
+
} else {
|
|
528
|
+
pairs.push({
|
|
529
|
+
type: 'constructive',
|
|
530
|
+
a: pi.slug,
|
|
531
|
+
b: pj.slug,
|
|
532
|
+
coupling: round4(cov),
|
|
533
|
+
reason: 'reinforcing (high similarity, compatible categories)',
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// Opposing velocities on similar topics = divergence signal
|
|
539
|
+
if (cov > 0.7 && pi.velocity * pj.velocity < 0) {
|
|
540
|
+
pairs.push({
|
|
541
|
+
type: 'divergence',
|
|
542
|
+
a: pi.slug,
|
|
543
|
+
b: pj.slug,
|
|
544
|
+
coupling: round4(cov),
|
|
545
|
+
reason: `velocity divergence: ${pi.slug} ${pi.velocity > 0 ? '↑' : '↓'} vs ${pj.slug} ${pj.velocity > 0 ? '↑' : '↓'}`,
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
return pairs;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// ─── Helpers ────────────────────────────────────────────────────────────────
|
|
554
|
+
|
|
555
|
+
function round4(n) { return Math.round(n * 10000) / 10000; }
|
|
556
|
+
|
|
557
|
+
// ─── Field Loading ──────────────────────────────────────────────────────────
|
|
558
|
+
|
|
559
|
+
let _cachedField = null;
|
|
560
|
+
let _cachedFieldMtime = 0;
|
|
561
|
+
|
|
562
|
+
export function loadField(derivedDir = DERIVED_DIR) {
|
|
563
|
+
try {
|
|
564
|
+
const fp = fieldPath(derivedDir);
|
|
565
|
+
if (!fs.existsSync(fp)) return null;
|
|
566
|
+
const stat = fs.statSync(fp);
|
|
567
|
+
if (_cachedField && stat.mtimeMs === _cachedFieldMtime) return _cachedField;
|
|
568
|
+
const raw = JSON.parse(fs.readFileSync(fp, 'utf8'));
|
|
569
|
+
_cachedField = raw;
|
|
570
|
+
_cachedFieldMtime = stat.mtimeMs;
|
|
571
|
+
return raw;
|
|
572
|
+
} catch { return null; }
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// ─── Query Logging ──────────────────────────────────────────────────────────
|
|
576
|
+
|
|
577
|
+
function logQuery(query, hitSlugs, derivedDir = DERIVED_DIR) {
|
|
578
|
+
try {
|
|
579
|
+
const logPath = queryLogPath(derivedDir);
|
|
580
|
+
fs.mkdirSync(path.dirname(logPath), { recursive: true });
|
|
581
|
+
fs.appendFileSync(logPath, JSON.stringify({ t: Date.now(), q: query.slice(0, 200), hits: hitSlugs.slice(0, 20) }) + '\n');
|
|
582
|
+
} catch { /* fire and forget */ }
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// ─── Velocity Recomputation ─────────────────────────────────────────────────
|
|
586
|
+
|
|
587
|
+
/**
|
|
588
|
+
* Recompute velocity + trajectory vectors from query log.
|
|
589
|
+
* Call periodically (dream cycle) to update the field in-place.
|
|
590
|
+
*/
|
|
591
|
+
export function recomputeVelocities(field, derivedDir = DERIVED_DIR) {
|
|
592
|
+
if (!field) return field;
|
|
593
|
+
const now = Date.now();
|
|
594
|
+
const history = loadQueryHistory(derivedDir);
|
|
595
|
+
|
|
596
|
+
for (const point of field.points) {
|
|
597
|
+
const { velocity, trajectory } = computeTrajectory(point.slug, history, now);
|
|
598
|
+
point.velocity = velocity;
|
|
599
|
+
point.trajectory = trajectory;
|
|
600
|
+
point.novelty = computeNovelty(point.access_count);
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
return field;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// ─── Field Statistics ───────────────────────────────────────────────────────
|
|
607
|
+
|
|
608
|
+
export function fieldStats(derivedDir = DERIVED_DIR) {
|
|
609
|
+
const field = loadField(derivedDir);
|
|
610
|
+
if (!field) return { compiled: false };
|
|
611
|
+
|
|
612
|
+
const points = field.points || [];
|
|
613
|
+
const ascending = points.filter(p => p.velocity > 0).length;
|
|
614
|
+
const descending = points.filter(p => p.velocity < 0).length;
|
|
615
|
+
const stable = points.filter(p => p.velocity === 0).length;
|
|
616
|
+
const immortal = points.filter(p => p.lambda === 0).length;
|
|
617
|
+
const accelerating = points.filter(p => p.trajectory > 0).length;
|
|
618
|
+
const decelerating = points.filter(p => p.trajectory < 0).length;
|
|
619
|
+
|
|
620
|
+
// Half-life distribution
|
|
621
|
+
const hlBuckets = { immortal: 0, hours: 0, days: 0, weeks: 0, months: 0, years: 0 };
|
|
622
|
+
for (const p of points) {
|
|
623
|
+
if (p.lambda === 0) { hlBuckets.immortal++; continue; }
|
|
624
|
+
const hlDays = p.half_life ? p.half_life / (24 * 60 * 60 * 1000) : 30;
|
|
625
|
+
if (hlDays < 1) hlBuckets.hours++;
|
|
626
|
+
else if (hlDays < 7) hlBuckets.days++;
|
|
627
|
+
else if (hlDays < 30) hlBuckets.weeks++;
|
|
628
|
+
else if (hlDays < 365) hlBuckets.months++;
|
|
629
|
+
else hlBuckets.years++;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
// Strongest couplings
|
|
633
|
+
const couplings = [];
|
|
634
|
+
const N = field.covariance?.length || 0;
|
|
635
|
+
for (let i = 0; i < N; i++) {
|
|
636
|
+
for (let j = i + 1; j < N; j++) {
|
|
637
|
+
const v = field.covariance[i][j];
|
|
638
|
+
if (Math.abs(v) > 0.7) {
|
|
639
|
+
couplings.push({ a: field.slugIndex[i], b: field.slugIndex[j], coupling: round4(v) });
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
couplings.sort((a, b) => Math.abs(b.coupling) - Math.abs(a.coupling));
|
|
644
|
+
|
|
645
|
+
return {
|
|
646
|
+
compiled: true,
|
|
647
|
+
compiled_at: new Date(field.compiled_at).toISOString(),
|
|
648
|
+
dimensions: field.dimensions,
|
|
649
|
+
points: points.length,
|
|
650
|
+
matrix: `${N}×${N}`,
|
|
651
|
+
velocities: { ascending, descending, stable },
|
|
652
|
+
trajectories: { accelerating, decelerating, coasting: points.length - accelerating - decelerating },
|
|
653
|
+
half_life_distribution: hlBuckets,
|
|
654
|
+
immortal_nodes: immortal,
|
|
655
|
+
avg_novelty: points.length > 0 ? round4(points.reduce((s, p) => s + p.novelty, 0) / points.length) : 0,
|
|
656
|
+
strong_couplings: couplings.slice(0, 10),
|
|
657
|
+
meta: field.meta,
|
|
658
|
+
};
|
|
659
|
+
}
|