total-recall-brain 3.2.3 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +1 -1
  2. package/extension/background.js +14 -2
  3. package/extension/content-script.js +18 -1
  4. package/extension/lib/brain-client.js +86 -52
  5. package/extension/lib/preconfigured.js +5 -0
  6. package/extension/manifest.json +1 -1
  7. package/extension/options/options.html +1 -0
  8. package/extension/options/options.js +12 -6
  9. package/extension/popup/popup.html +1 -0
  10. package/extension/popup/popup.js +29 -4
  11. package/extension/sidepanel/sidepanel.html +1 -0
  12. package/extension/sidepanel/sidepanel.js +24 -6
  13. package/frontend/index.html +8 -0
  14. package/frontend/src/api.ts +1 -2
  15. package/frontend/src/pages/ChatPage.tsx +9 -6
  16. package/frontend/src/pages/IntegrationsPage.tsx +144 -3
  17. package/package.json +2 -2
  18. package/scaffold/.agent/skills/total-recall/SKILL.md +55 -6
  19. package/scaffold/.agent/skills/total-recall/memory-vault/invariants/operating-instructions.md +2 -2
  20. package/scaffold/.agent/skills/total-recall/skills/cli-agents/agents.yml +1 -6
  21. package/scaffold/.agent/skills/total-recall/skills/ssss/SKILL.md +19 -0
  22. package/scaffold/.agent/skills/total-recall/skills/ssss/references/ssss-spec.md +33 -5
  23. package/src/cli/forget.mjs +18 -12
  24. package/src/cli/ingest/index.mjs +3 -2
  25. package/src/cli/init.mjs +30 -0
  26. package/src/cli/map.mjs +2 -2
  27. package/src/cli/rebuild.mjs +1 -1
  28. package/src/cli/remember.mjs +21 -11
  29. package/src/cli/setup.mjs +10 -1
  30. package/src/cli/share.mjs +7 -2
  31. package/src/core/append-log.mjs +131 -0
  32. package/src/core/clarity-rewriter.mjs +10 -6
  33. package/src/core/conclusion-writer.mjs +3 -2
  34. package/src/core/config.mjs +6 -0
  35. package/src/core/conflict-detector.mjs +4 -2
  36. package/src/core/context-compiler.mjs +412 -0
  37. package/src/core/daemon-loop.mjs +15 -0
  38. package/src/core/dream.mjs +4 -4
  39. package/src/core/drift-detector.mjs +3 -3
  40. package/src/core/embeddings.mjs +60 -10
  41. package/src/core/fact-seeker.mjs +70 -44
  42. package/src/core/import-rules.mjs +3 -2
  43. package/src/core/inference-engine.mjs +5 -4
  44. package/src/core/migrate.mjs +4 -3
  45. package/src/core/optimizer.mjs +6 -5
  46. package/src/core/parallel-context.mjs +288 -0
  47. package/src/core/post-mortem.mjs +4 -3
  48. package/src/core/research-queue.mjs +4 -3
  49. package/src/core/runtime.mjs +2 -2
  50. package/src/core/sandbox.mjs +56 -26
  51. package/src/core/scheduler.mjs +6 -5
  52. package/src/core/schema.mjs +2 -2
  53. package/src/core/search.mjs +167 -101
  54. package/src/core/surface.mjs +203 -53
  55. package/src/core/validated-write.mjs +100 -0
  56. package/src/core/vault-cache.mjs +46 -18
  57. package/src/core/vault.mjs +17 -29
  58. package/src/core/vector-field.mjs +659 -0
  59. package/src/core/vector-store.mjs +85 -0
  60. package/src/server/api.mjs +4 -3
  61. package/src/server/index.mjs +13 -61
  62. package/src/server/rest.mjs +238 -29
  63. package/src/server/route-manifest.json +344 -77
  64. package/src/server/routes/memory.mjs +85 -39
  65. package/src/server/routes/sessions.mjs +6 -2
  66. package/src/server/routes/share.mjs +9 -2
@@ -0,0 +1,659 @@
1
+ /**
2
+ * src/core/vector-field.mjs
3
+ *
4
+ * The Infinite Vector Field
5
+ *
6
+ * Every node is a measurement point in continuous embedding space.
7
+ * Each point carries:
8
+ * - position: embedding[768] — where in concept space
9
+ * - half_life: λ (decay constant) — how fast relevance decays
10
+ * - trajectory: d²R/dt² — is decay accelerating or decelerating
11
+ * - velocity: dR/dt — rate of relevance change
12
+ * - novelty: information density — has model seen this before?
13
+ * - coupling: covariance[i][j] — entanglement with every other point
14
+ * - dependencies: slug[] — nodes that must co-occur
15
+ *
16
+ * The field exists everywhere. Nodes are where we've sampled it.
17
+ * Queries sample new points. The field interpolates.
18
+ * The compiled field replaces all LLM calls for known query regions.
19
+ */
20
+
21
+ import fs from 'node:fs';
22
+ import path from 'node:path';
23
+ import { getNodes } from './vault-cache.mjs';
24
+ import { getEmbedding, cosineSimilarity, nodeToEmbedText, loadEmbeddingsIndex, saveEmbeddingToIndex } from './embeddings.mjs';
25
+ import { inferMemoryLayer } from './memory-layers.mjs';
26
+ import { logger } from './logger.mjs';
27
+ import { brainDir } from './config.mjs';
28
+
29
+ const DERIVED_DIR = path.join(brainDir, 'memory-derived');
30
+
31
+ function fieldPath(derivedDir = DERIVED_DIR) {
32
+ return path.join(derivedDir, 'vector-field.json');
33
+ }
34
+
35
+ function queryLogPath(derivedDir = DERIVED_DIR) {
36
+ return path.join(derivedDir, 'query-log.jsonl');
37
+ }
38
+
39
+ const LN2 = Math.log(2);
40
+
41
+ // ─── Half-Life Constants (milliseconds) ─────────────────────────────────────
42
+
43
+ const HALF_LIFE_TABLE = {
44
+ // Category-based half-lives
45
+ invariant: Infinity, // never decays
46
+ invariants: Infinity, // never decays
47
+ correction: 60 * 24 * 60 * 60 * 1000, // 60 days
48
+ corrections: 60 * 24 * 60 * 60 * 1000, // 60 days
49
+ decision: 90 * 24 * 60 * 60 * 1000, // 90 days
50
+ decisions: 90 * 24 * 60 * 60 * 1000, // 90 days
51
+ preference: 180 * 24 * 60 * 60 * 1000, // 180 days
52
+ preferences: 180 * 24 * 60 * 60 * 1000, // 180 days
53
+ fact: 30 * 24 * 60 * 60 * 1000, // 30 days (post-cutoff facts)
54
+ facts: 30 * 24 * 60 * 60 * 1000, // 30 days (post-cutoff facts)
55
+ concept: 120 * 24 * 60 * 60 * 1000, // 120 days
56
+ concepts: 120 * 24 * 60 * 60 * 1000, // 120 days
57
+ pattern: 90 * 24 * 60 * 60 * 1000, // 90 days
58
+ patterns: 90 * 24 * 60 * 60 * 1000, // 90 days
59
+ 'anti-pattern': 60 * 24 * 60 * 60 * 1000, // 60 days
60
+ 'anti-patterns': 60 * 24 * 60 * 60 * 1000, // 60 days
61
+ lore: 365 * 24 * 60 * 60 * 1000, // 1 year
62
+ // Priority overrides
63
+ _absolute: Infinity, // absolute priority = infinite half-life
64
+ _high: 180 * 24 * 60 * 60 * 1000, // 180 days minimum
65
+ // Ephemeral types
66
+ session: 2 * 60 * 60 * 1000, // 2 hours
67
+ task: 30 * 60 * 1000, // 30 minutes
68
+ grounding: 30 * 60 * 1000, // 30 minutes
69
+ research: 14 * 24 * 60 * 60 * 1000, // 14 days
70
+ };
71
+
72
+ /**
73
+ * Determine the half-life for a node based on its category and priority.
74
+ */
75
+ function computeHalfLife(node) {
76
+ // Priority overrides category
77
+ if (node.priority === 'absolute') return Infinity;
78
+ if (node.priority === 'high') {
79
+ const categoryHL = HALF_LIFE_TABLE[node.category] || HALF_LIFE_TABLE.fact;
80
+ return Math.max(categoryHL, HALF_LIFE_TABLE._high);
81
+ }
82
+ return HALF_LIFE_TABLE[node.category] || HALF_LIFE_TABLE.fact;
83
+ }
84
+
85
+ /**
86
+ * Compute decay constant λ = ln(2) / half_life
87
+ */
88
+ function decayConstant(halfLife) {
89
+ if (!isFinite(halfLife)) return 0; // no decay
90
+ return LN2 / halfLife;
91
+ }
92
+
93
+ /**
94
+ * Compute current effective relevance given peak relevance and time elapsed.
95
+ * R(t) = R₀ × e^(-λt) × (1 + trajectory × t_normalized)
96
+ */
97
+ function effectiveRelevance(peakRelevance, lambda, elapsedMs, trajectory = 0) {
98
+ if (lambda === 0) return peakRelevance; // infinite half-life
99
+ const decayed = peakRelevance * Math.exp(-lambda * elapsedMs);
100
+ // Trajectory modulates: positive trajectory slows decay, negative accelerates
101
+ const tNorm = Math.min(elapsedMs / (7 * 24 * 60 * 60 * 1000), 1); // normalize to 1 week
102
+ const trajectoryMod = 1 + (trajectory * tNorm * 0.3); // ±30% max adjustment
103
+ return Math.max(0, decayed * trajectoryMod);
104
+ }
105
+
106
+ // ─── Novelty Scoring ────────────────────────────────────────────────────────
107
+
108
+ /**
109
+ * Compute novelty score for a node.
110
+ * Novelty = inverse of how many times this node has been injected into context.
111
+ * First time = 1.0, after 100 times = ~0.1
112
+ */
113
+ function computeNovelty(accessCount) {
114
+ if (accessCount <= 0) return 1.0; // never seen = maximum novelty
115
+ return 1 / (1 + Math.log1p(accessCount)); // logarithmic decay of novelty
116
+ }
117
+
118
+ // ─── Compilation ────────────────────────────────────────────────────────────
119
+
120
+ /**
121
+ * Compile the full vector field from the vault.
122
+ *
123
+ * 1. Embed every active node → position vectors
124
+ * 2. Compute half-life, trajectory, novelty per point
125
+ * 3. Compute full pairwise covariance → coupling matrix
126
+ * 4. Extract dependency graph
127
+ * 5. Persist to disk
128
+ */
129
+ export async function compileField({ vaultDir, derivedDir = DERIVED_DIR } = {}) {
130
+ const startTime = Date.now();
131
+ const allNodes = getNodes(vaultDir).filter(n => n.status === 'active');
132
+ const totalNodes = allNodes.length;
133
+ const now = Date.now();
134
+
135
+ logger.info('vector-field', `Compiling field from ${totalNodes} nodes...`);
136
+
137
+ // ── Load query log for trajectory computation ──
138
+ const queryHistory = loadQueryHistory(derivedDir);
139
+
140
+ // ── Phase 1: Embed all nodes + compute physics ──
141
+ const existingIndex = loadEmbeddingsIndex(derivedDir);
142
+ const points = [];
143
+ const newlyEmbedded = new Map();
144
+ let embedded = 0;
145
+ let cached = 0;
146
+
147
+ const EMBED_CONCURRENCY = 5;
148
+ let cursor = 0;
149
+
150
+ async function embedWorker() {
151
+ while (cursor < allNodes.length) {
152
+ const i = cursor++;
153
+ const node = allNodes[i];
154
+ const text = nodeToEmbedText(node);
155
+
156
+ let embedding;
157
+ if (existingIndex[node.slug]?.embedding) {
158
+ embedding = existingIndex[node.slug].embedding;
159
+ cached++;
160
+ } else {
161
+ try {
162
+ embedding = await getEmbedding(text);
163
+ newlyEmbedded.set(node.slug, embedding);
164
+ embedded++;
165
+ } catch (err) {
166
+ logger.error('vector-field', `Failed to embed ${node.slug}: ${err.message}`);
167
+ continue;
168
+ }
169
+ }
170
+
171
+ // ── Physics computation ──
172
+ const decay = node.decay || {};
173
+ const accessCount = decay.access_count || 0;
174
+ const lastAccessed = node.last_accessed ? new Date(node.last_accessed).getTime() : 0;
175
+ const createdAt = node.created ? new Date(node.created).getTime() : now;
176
+ const ageMs = now - createdAt;
177
+
178
+ // Half-life
179
+ const halfLife = computeHalfLife(node);
180
+ const lambda = decayConstant(halfLife);
181
+
182
+ // Velocity: dR/dt from query log
183
+ const { velocity, trajectory } = computeTrajectory(node.slug, queryHistory, now);
184
+
185
+ // Novelty
186
+ const novelty = computeNovelty(accessCount);
187
+
188
+ // Peak relevance (baseline from importance + priority)
189
+ const importanceWeight = (node.importance || 3) / 5;
190
+ const priorityWeight = node.priority === 'absolute' ? 1.0 : node.priority === 'high' ? 0.8 : node.priority === 'normal' ? 0.5 : 0.3;
191
+ const peakRelevance = Math.max(importanceWeight, priorityWeight);
192
+
193
+ // Time since last access (for decay computation at sample time)
194
+ const timeSinceAccess = lastAccessed > 0 ? now - lastAccessed : ageMs;
195
+
196
+ // Dependencies: extract from node.related field
197
+ const dependencies = node.related ? (Array.isArray(node.related) ? node.related : String(node.related).split(',').map(s => s.trim())) : [];
198
+
199
+ // Precompute L2 norm
200
+ let mag = 0;
201
+ for (let j = 0; j < embedding.length; j++) mag += embedding[j] * embedding[j];
202
+ mag = Math.sqrt(mag);
203
+
204
+ points.push({
205
+ slug: node.slug,
206
+ title: node.title || node.slug,
207
+ category: node.category || 'unknown',
208
+ layer: inferMemoryLayer(node),
209
+ embedding,
210
+ magnitude: mag,
211
+ // ── Physics ──
212
+ half_life: isFinite(halfLife) ? halfLife : null, // null = infinite
213
+ lambda, // decay constant
214
+ peak_relevance: peakRelevance, // R₀
215
+ velocity, // dR/dt
216
+ trajectory, // d²R/dt² (acceleration)
217
+ novelty, // information density
218
+ // ── Metadata ──
219
+ access_count: accessCount,
220
+ last_accessed: lastAccessed,
221
+ created_at: createdAt,
222
+ time_since_access: timeSinceAccess,
223
+ importance: node.importance || 3,
224
+ priority: node.priority || 'normal',
225
+ dependencies,
226
+ body_length: (node.body || node.content || '').length,
227
+ });
228
+ }
229
+ }
230
+
231
+ const workers = Array.from(
232
+ { length: Math.min(EMBED_CONCURRENCY, allNodes.length) },
233
+ () => embedWorker()
234
+ );
235
+ await Promise.all(workers);
236
+
237
+ for (const [slug, embedding] of newlyEmbedded) {
238
+ saveEmbeddingToIndex(derivedDir, slug, embedding);
239
+ }
240
+
241
+ const embedMs = Date.now() - startTime;
242
+ logger.info('vector-field', `Embedded ${embedded} new + ${cached} cached in ${embedMs}ms`);
243
+
244
+ // ── Phase 2: Pairwise covariance matrix ──
245
+ const N = points.length;
246
+ const slugIndex = points.map(p => p.slug);
247
+ const covariance = new Array(N);
248
+
249
+ for (let i = 0; i < N; i++) {
250
+ covariance[i] = new Float32Array(N);
251
+ for (let j = i; j < N; j++) {
252
+ if (i === j) {
253
+ covariance[i][j] = 1.0;
254
+ } else {
255
+ const sim = fastCosine(points[i].embedding, points[j].embedding, points[i].magnitude, points[j].magnitude);
256
+ covariance[i][j] = sim;
257
+ if (!covariance[j]) covariance[j] = new Float32Array(N);
258
+ covariance[j][i] = sim;
259
+ }
260
+ }
261
+ }
262
+
263
+ const covMs = Date.now() - startTime - embedMs;
264
+
265
+ // ── Phase 3: Compile and persist ──
266
+ const covarianceSerializable = covariance.map(row => Array.from(row));
267
+
268
+ const field = {
269
+ points,
270
+ covariance: covarianceSerializable,
271
+ slugIndex,
272
+ compiled_at: now,
273
+ dimensions: points[0]?.embedding?.length || 768,
274
+ meta: {
275
+ total_nodes: totalNodes,
276
+ points_compiled: N,
277
+ embedded_new: embedded,
278
+ embedded_cached: cached,
279
+ embed_ms: embedMs,
280
+ covariance_ms: covMs,
281
+ compile_ms: Date.now() - startTime,
282
+ matrix_size: `${N}×${N}`,
283
+ matrix_entries: N * N,
284
+ infinite_halflife: points.filter(p => p.lambda === 0).length,
285
+ with_dependencies: points.filter(p => p.dependencies.length > 0).length,
286
+ avg_novelty: points.length > 0 ? Math.round((points.reduce((s, p) => s + p.novelty, 0) / points.length) * 1000) / 1000 : 0,
287
+ },
288
+ };
289
+
290
+ const outPath = fieldPath(derivedDir);
291
+ fs.mkdirSync(path.dirname(outPath), { recursive: true });
292
+ fs.writeFileSync(outPath, JSON.stringify(field));
293
+
294
+ logger.info('vector-field', `Field compiled: ${N} points, ${N * N} couplings, ${field.meta.infinite_halflife} immortal, ${field.meta.with_dependencies} with deps, ${field.meta.compile_ms}ms`);
295
+
296
+ return field;
297
+ }
298
+
299
+ // ─── Trajectory Computation ─────────────────────────────────────────────────
300
+
301
+ /**
302
+ * Compute velocity (dR/dt) and trajectory (d²R/dt²) from query log.
303
+ * Uses 3 time windows to detect acceleration/deceleration.
304
+ */
305
+ function computeTrajectory(slug, queryHistory, now) {
306
+ const WINDOW = 24 * 60 * 60 * 1000; // 1 day
307
+
308
+ // Count hits in 3 windows: [0-24h], [24-48h], [48-72h]
309
+ let w0 = 0, w1 = 0, w2 = 0;
310
+ for (const entry of queryHistory) {
311
+ if (!entry.hits?.includes(slug)) continue;
312
+ const age = now - entry.t;
313
+ if (age < WINDOW) w0++;
314
+ else if (age < 2 * WINDOW) w1++;
315
+ else if (age < 3 * WINDOW) w2++;
316
+ }
317
+
318
+ // Velocity = first derivative (recent vs previous)
319
+ const velocity = (w0 - w1) * 0.1;
320
+
321
+ // Trajectory = second derivative (acceleration of velocity)
322
+ const v_prev = (w1 - w2) * 0.1;
323
+ const trajectory = velocity - v_prev; // positive = accelerating, negative = decelerating
324
+
325
+ return {
326
+ velocity: Math.max(-1, Math.min(1, velocity)),
327
+ trajectory: Math.max(-1, Math.min(1, trajectory)),
328
+ };
329
+ }
330
+
331
+ function loadQueryHistory(derivedDir = DERIVED_DIR) {
332
+ try {
333
+ const logPath = queryLogPath(derivedDir);
334
+ if (!fs.existsSync(logPath)) return [];
335
+ const lines = fs.readFileSync(logPath, 'utf8').trim().split('\n');
336
+ const cutoff = Date.now() - (3 * 24 * 60 * 60 * 1000); // only last 3 days
337
+ return lines.map(l => { try { return JSON.parse(l); } catch { return null; } })
338
+ .filter(e => e && e.t > cutoff);
339
+ } catch { return []; }
340
+ }
341
+
342
+ // ─── Fast Cosine ────────────────────────────────────────────────────────────
343
+
344
+ function fastCosine(a, b, magA, magB) {
345
+ if (!a || !b || a.length !== b.length) return 0;
346
+ const denom = magA * magB;
347
+ if (denom === 0) return 0;
348
+ let dot = 0;
349
+ for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
350
+ return dot / denom;
351
+ }
352
+
353
+ // ─── Field Sampling ─────────────────────────────────────────────────────────
354
+
355
+ /**
356
+ * Sample the vector field at a query point.
357
+ *
358
+ * Pipeline:
359
+ * 1. Embed query → position in field
360
+ * 2. Direct similarity to all points
361
+ * 3. Apply half-life decay: R(t) = R₀ × e^(-λΔt) × trajectory_mod
362
+ * 4. Entanglement propagation through covariance
363
+ * 5. Novelty weighting (unseen info > repeated info)
364
+ * 6. Dependency resolution (pull in required co-nodes)
365
+ * 7. Interference detection (flag contradictions)
366
+ * 8. Return ranked results
367
+ */
368
+ export async function sampleField({
369
+ query,
370
+ field = null,
371
+ topK = 20,
372
+ entanglementBoost = 0.15,
373
+ noveltyWeight = 0.1,
374
+ derivedDir = DERIVED_DIR,
375
+ } = {}) {
376
+ const startTime = Date.now();
377
+ const now = Date.now();
378
+
379
+ if (!field) {
380
+ field = loadField(derivedDir);
381
+ if (!field) throw new Error('Vector field not compiled. Run compileField() first.');
382
+ }
383
+
384
+ const { points, covariance, slugIndex } = field;
385
+ const N = points.length;
386
+ if (N === 0) return { results: [], query_embedding: [], sample_ms: 0, interference: [] };
387
+
388
+ // ── 1. Embed query ──
389
+ const queryEmbedding = await getEmbedding(query);
390
+ let queryMag = 0;
391
+ for (let i = 0; i < queryEmbedding.length; i++) queryMag += queryEmbedding[i] * queryEmbedding[i];
392
+ queryMag = Math.sqrt(queryMag);
393
+
394
+ // ── 2. Direct similarity ──
395
+ const directScores = new Float64Array(N);
396
+ for (let i = 0; i < N; i++) {
397
+ directScores[i] = fastCosine(queryEmbedding, points[i].embedding, queryMag, points[i].magnitude);
398
+ }
399
+
400
+ // ── 3. Half-life decay ──
401
+ const decayedScores = new Float64Array(N);
402
+ for (let i = 0; i < N; i++) {
403
+ const p = points[i];
404
+ const elapsed = p.last_accessed > 0 ? now - p.last_accessed : p.time_since_access || 0;
405
+ const decayedRelevance = effectiveRelevance(p.peak_relevance, p.lambda, elapsed, p.trajectory);
406
+ // Blend: 70% similarity, 30% decayed relevance
407
+ decayedScores[i] = (directScores[i] * 0.7) + (decayedRelevance * 0.3);
408
+ }
409
+
410
+ // ── 4. Entanglement propagation ──
411
+ const propagatedScores = new Float64Array(N);
412
+ for (let i = 0; i < N; i++) {
413
+ let entSum = 0;
414
+ for (let j = 0; j < N; j++) {
415
+ if (i === j) continue;
416
+ entSum += covariance[i][j] * decayedScores[j];
417
+ }
418
+ const avgEnt = N > 1 ? entSum / (N - 1) : 0;
419
+ propagatedScores[i] = decayedScores[i] + (entanglementBoost * avgEnt);
420
+ }
421
+
422
+ // ── 5. Novelty weighting ──
423
+ const finalScores = new Float64Array(N);
424
+ for (let i = 0; i < N; i++) {
425
+ finalScores[i] = propagatedScores[i] + (points[i].novelty * noveltyWeight);
426
+ // Absolute priority floor
427
+ if (points[i].priority === 'absolute') {
428
+ finalScores[i] = Math.max(finalScores[i], 0.8);
429
+ }
430
+ }
431
+
432
+ // ── 6. Rank ──
433
+ const indexed = Array.from(finalScores).map((score, i) => ({ index: i, score }));
434
+ indexed.sort((a, b) => b.score - a.score);
435
+ let topResults = indexed.slice(0, topK);
436
+
437
+ // ── 7. Dependency resolution — pull in required nodes ──
438
+ const includedSlugs = new Set(topResults.map(r => points[r.index].slug));
439
+ const depsPulledIn = [];
440
+ for (const r of [...topResults]) {
441
+ const p = points[r.index];
442
+ for (const dep of p.dependencies) {
443
+ if (!includedSlugs.has(dep)) {
444
+ const depIdx = points.findIndex(pt => pt.slug === dep);
445
+ if (depIdx >= 0) {
446
+ topResults.push({ index: depIdx, score: finalScores[depIdx] });
447
+ includedSlugs.add(dep);
448
+ depsPulledIn.push(dep);
449
+ }
450
+ }
451
+ }
452
+ }
453
+ // Re-sort after dependency injection
454
+ topResults.sort((a, b) => b.score - a.score);
455
+ topResults = topResults.slice(0, topK + depsPulledIn.length); // allow deps to exceed topK
456
+
457
+ // ── 8. Interference detection ──
458
+ const interference = detectInterference(topResults, points, covariance);
459
+
460
+ // ── Format results ──
461
+ const results = topResults.map(({ index, score }) => {
462
+ const p = points[index];
463
+ const elapsed = p.last_accessed > 0 ? now - p.last_accessed : p.time_since_access || 0;
464
+ const remainingHL = p.lambda > 0 ? Math.round((LN2 / p.lambda) / (24 * 60 * 60 * 1000)) : null;
465
+ return {
466
+ slug: p.slug,
467
+ title: p.title,
468
+ category: p.category,
469
+ layer: p.layer,
470
+ score: round4(score),
471
+ direct_similarity: round4(directScores[index]),
472
+ decayed_relevance: round4(decayedScores[index]),
473
+ entanglement_boost: round4(propagatedScores[index] - decayedScores[index]),
474
+ novelty: round4(p.novelty),
475
+ velocity: round4(p.velocity),
476
+ trajectory: round4(p.trajectory),
477
+ half_life_days: remainingHL,
478
+ time_since_access_hours: Math.round(elapsed / (60 * 60 * 1000) * 10) / 10,
479
+ importance: p.importance,
480
+ priority: p.priority,
481
+ access_count: p.access_count,
482
+ dependencies: p.dependencies,
483
+ pulled_by_dependency: depsPulledIn.includes(p.slug),
484
+ };
485
+ });
486
+
487
+ const sampleMs = Date.now() - startTime;
488
+ logQuery(query, results.map(r => r.slug), derivedDir);
489
+
490
+ logger.info('vector-field', `Sampled in ${sampleMs}ms: ${results.length} results, ${depsPulledIn.length} deps pulled, ${interference.length} interference pairs`);
491
+
492
+ return { results, query_embedding: queryEmbedding, sample_ms: sampleMs, interference, deps_pulled: depsPulledIn };
493
+ }
494
+
495
+ // ─── Interference Detection ─────────────────────────────────────────────────
496
+
497
+ /**
498
+ * Detect constructive and destructive interference between included nodes.
499
+ * Destructive: nodes with opposing modalities (must vs must_not) or
500
+ * semantically similar but from conflicting categories (correction vs pattern).
501
+ * Constructive: nodes that reinforce each other (high covariance, same direction).
502
+ */
503
+ function detectInterference(topResults, points, covariance) {
504
+ const pairs = [];
505
+ for (let i = 0; i < topResults.length; i++) {
506
+ for (let j = i + 1; j < topResults.length; j++) {
507
+ const pi = points[topResults[i].index];
508
+ const pj = points[topResults[j].index];
509
+ const cov = covariance[topResults[i].index]?.[topResults[j].index] || 0;
510
+
511
+ // High similarity + opposing categories = potential destructive interference
512
+ if (cov > 0.8) {
513
+ const opposing =
514
+ (pi.category === 'correction' && pj.category === 'pattern') ||
515
+ (pi.category === 'pattern' && pj.category === 'correction') ||
516
+ (pi.category === 'anti-pattern' && pj.category === 'pattern') ||
517
+ (pi.category === 'pattern' && pj.category === 'anti-pattern');
518
+
519
+ if (opposing) {
520
+ pairs.push({
521
+ type: 'destructive',
522
+ a: pi.slug,
523
+ b: pj.slug,
524
+ coupling: round4(cov),
525
+ reason: `${pi.category} ↔ ${pj.category} (opposing categories, high similarity)`,
526
+ });
527
+ } else {
528
+ pairs.push({
529
+ type: 'constructive',
530
+ a: pi.slug,
531
+ b: pj.slug,
532
+ coupling: round4(cov),
533
+ reason: 'reinforcing (high similarity, compatible categories)',
534
+ });
535
+ }
536
+ }
537
+
538
+ // Opposing velocities on similar topics = divergence signal
539
+ if (cov > 0.7 && pi.velocity * pj.velocity < 0) {
540
+ pairs.push({
541
+ type: 'divergence',
542
+ a: pi.slug,
543
+ b: pj.slug,
544
+ coupling: round4(cov),
545
+ reason: `velocity divergence: ${pi.slug} ${pi.velocity > 0 ? '↑' : '↓'} vs ${pj.slug} ${pj.velocity > 0 ? '↑' : '↓'}`,
546
+ });
547
+ }
548
+ }
549
+ }
550
+ return pairs;
551
+ }
552
+
553
+ // ─── Helpers ────────────────────────────────────────────────────────────────
554
+
555
+ function round4(n) { return Math.round(n * 10000) / 10000; }
556
+
557
+ // ─── Field Loading ──────────────────────────────────────────────────────────
558
+
559
+ let _cachedField = null;
560
+ let _cachedFieldMtime = 0;
561
+
562
+ export function loadField(derivedDir = DERIVED_DIR) {
563
+ try {
564
+ const fp = fieldPath(derivedDir);
565
+ if (!fs.existsSync(fp)) return null;
566
+ const stat = fs.statSync(fp);
567
+ if (_cachedField && stat.mtimeMs === _cachedFieldMtime) return _cachedField;
568
+ const raw = JSON.parse(fs.readFileSync(fp, 'utf8'));
569
+ _cachedField = raw;
570
+ _cachedFieldMtime = stat.mtimeMs;
571
+ return raw;
572
+ } catch { return null; }
573
+ }
574
+
575
+ // ─── Query Logging ──────────────────────────────────────────────────────────
576
+
577
+ function logQuery(query, hitSlugs, derivedDir = DERIVED_DIR) {
578
+ try {
579
+ const logPath = queryLogPath(derivedDir);
580
+ fs.mkdirSync(path.dirname(logPath), { recursive: true });
581
+ fs.appendFileSync(logPath, JSON.stringify({ t: Date.now(), q: query.slice(0, 200), hits: hitSlugs.slice(0, 20) }) + '\n');
582
+ } catch { /* fire and forget */ }
583
+ }
584
+
585
+ // ─── Velocity Recomputation ─────────────────────────────────────────────────
586
+
587
+ /**
588
+ * Recompute velocity + trajectory vectors from query log.
589
+ * Call periodically (dream cycle) to update the field in-place.
590
+ */
591
+ export function recomputeVelocities(field, derivedDir = DERIVED_DIR) {
592
+ if (!field) return field;
593
+ const now = Date.now();
594
+ const history = loadQueryHistory(derivedDir);
595
+
596
+ for (const point of field.points) {
597
+ const { velocity, trajectory } = computeTrajectory(point.slug, history, now);
598
+ point.velocity = velocity;
599
+ point.trajectory = trajectory;
600
+ point.novelty = computeNovelty(point.access_count);
601
+ }
602
+
603
+ return field;
604
+ }
605
+
606
+ // ─── Field Statistics ───────────────────────────────────────────────────────
607
+
608
+ export function fieldStats(derivedDir = DERIVED_DIR) {
609
+ const field = loadField(derivedDir);
610
+ if (!field) return { compiled: false };
611
+
612
+ const points = field.points || [];
613
+ const ascending = points.filter(p => p.velocity > 0).length;
614
+ const descending = points.filter(p => p.velocity < 0).length;
615
+ const stable = points.filter(p => p.velocity === 0).length;
616
+ const immortal = points.filter(p => p.lambda === 0).length;
617
+ const accelerating = points.filter(p => p.trajectory > 0).length;
618
+ const decelerating = points.filter(p => p.trajectory < 0).length;
619
+
620
+ // Half-life distribution
621
+ const hlBuckets = { immortal: 0, hours: 0, days: 0, weeks: 0, months: 0, years: 0 };
622
+ for (const p of points) {
623
+ if (p.lambda === 0) { hlBuckets.immortal++; continue; }
624
+ const hlDays = p.half_life ? p.half_life / (24 * 60 * 60 * 1000) : 30;
625
+ if (hlDays < 1) hlBuckets.hours++;
626
+ else if (hlDays < 7) hlBuckets.days++;
627
+ else if (hlDays < 30) hlBuckets.weeks++;
628
+ else if (hlDays < 365) hlBuckets.months++;
629
+ else hlBuckets.years++;
630
+ }
631
+
632
+ // Strongest couplings
633
+ const couplings = [];
634
+ const N = field.covariance?.length || 0;
635
+ for (let i = 0; i < N; i++) {
636
+ for (let j = i + 1; j < N; j++) {
637
+ const v = field.covariance[i][j];
638
+ if (Math.abs(v) > 0.7) {
639
+ couplings.push({ a: field.slugIndex[i], b: field.slugIndex[j], coupling: round4(v) });
640
+ }
641
+ }
642
+ }
643
+ couplings.sort((a, b) => Math.abs(b.coupling) - Math.abs(a.coupling));
644
+
645
+ return {
646
+ compiled: true,
647
+ compiled_at: new Date(field.compiled_at).toISOString(),
648
+ dimensions: field.dimensions,
649
+ points: points.length,
650
+ matrix: `${N}×${N}`,
651
+ velocities: { ascending, descending, stable },
652
+ trajectories: { accelerating, decelerating, coasting: points.length - accelerating - decelerating },
653
+ half_life_distribution: hlBuckets,
654
+ immortal_nodes: immortal,
655
+ avg_novelty: points.length > 0 ? round4(points.reduce((s, p) => s + p.novelty, 0) / points.length) : 0,
656
+ strong_couplings: couplings.slice(0, 10),
657
+ meta: field.meta,
658
+ };
659
+ }