mem-os-python 1.6.27__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mem_os_python-1.6.27.dist-info/METADATA +1032 -0
- mem_os_python-1.6.27.dist-info/RECORD +253 -0
- mem_os_python-1.6.27.dist-info/WHEEL +4 -0
- mem_os_python-1.6.27.dist-info/entry_points.txt +2 -0
- mem_os_python-1.6.27.dist-info/licenses/LICENSE +21 -0
- memos/__init__.py +16 -0
- memos/_js/adapters/anthropic.d.ts +279 -0
- memos/_js/adapters/anthropic.d.ts.map +1 -0
- memos/_js/adapters/anthropic.js +348 -0
- memos/_js/adapters/anthropic.js.map +1 -0
- memos/_js/adapters/base.d.ts +10 -0
- memos/_js/adapters/base.d.ts.map +1 -0
- memos/_js/adapters/base.js +2 -0
- memos/_js/adapters/base.js.map +1 -0
- memos/_js/adapters/openai.d.ts +264 -0
- memos/_js/adapters/openai.d.ts.map +1 -0
- memos/_js/adapters/openai.js +342 -0
- memos/_js/adapters/openai.js.map +1 -0
- memos/_js/apps/explorer.d.ts +244 -0
- memos/_js/apps/explorer.d.ts.map +1 -0
- memos/_js/apps/explorer.js +738 -0
- memos/_js/apps/explorer.js.map +1 -0
- memos/_js/citations.d.ts +65 -0
- memos/_js/citations.d.ts.map +1 -0
- memos/_js/citations.js +143 -0
- memos/_js/citations.js.map +1 -0
- memos/_js/cli-browse.d.ts +3 -0
- memos/_js/cli-browse.d.ts.map +1 -0
- memos/_js/cli-browse.js +129 -0
- memos/_js/cli-browse.js.map +1 -0
- memos/_js/cli-completion.d.ts +31 -0
- memos/_js/cli-completion.d.ts.map +1 -0
- memos/_js/cli-completion.js +340 -0
- memos/_js/cli-completion.js.map +1 -0
- memos/_js/cli-help.d.ts +18 -0
- memos/_js/cli-help.d.ts.map +1 -0
- memos/_js/cli-help.js +357 -0
- memos/_js/cli-help.js.map +1 -0
- memos/_js/cli.d.ts +28 -0
- memos/_js/cli.d.ts.map +1 -0
- memos/_js/cli.js +2880 -0
- memos/_js/cli.js.map +1 -0
- memos/_js/compression.d.ts +25 -0
- memos/_js/compression.d.ts.map +1 -0
- memos/_js/compression.js +319 -0
- memos/_js/compression.js.map +1 -0
- memos/_js/confidence-machine.d.ts +99 -0
- memos/_js/confidence-machine.d.ts.map +1 -0
- memos/_js/confidence-machine.js +222 -0
- memos/_js/confidence-machine.js.map +1 -0
- memos/_js/context-pack.d.ts +506 -0
- memos/_js/context-pack.d.ts.map +1 -0
- memos/_js/context-pack.js +1435 -0
- memos/_js/context-pack.js.map +1 -0
- memos/_js/contradictions.d.ts +160 -0
- memos/_js/contradictions.d.ts.map +1 -0
- memos/_js/contradictions.js +255 -0
- memos/_js/contradictions.js.map +1 -0
- memos/_js/dashboard.d.ts +30 -0
- memos/_js/dashboard.d.ts.map +1 -0
- memos/_js/dashboard.js +302 -0
- memos/_js/dashboard.js.map +1 -0
- memos/_js/ebbinghaus.d.ts +29 -0
- memos/_js/ebbinghaus.d.ts.map +1 -0
- memos/_js/ebbinghaus.js +32 -0
- memos/_js/ebbinghaus.js.map +1 -0
- memos/_js/embedding-queue.d.ts +190 -0
- memos/_js/embedding-queue.d.ts.map +1 -0
- memos/_js/embedding-queue.js +316 -0
- memos/_js/embedding-queue.js.map +1 -0
- memos/_js/embeddings.d.ts +187 -0
- memos/_js/embeddings.d.ts.map +1 -0
- memos/_js/embeddings.js +633 -0
- memos/_js/embeddings.js.map +1 -0
- memos/_js/entity-extraction.d.ts +54 -0
- memos/_js/entity-extraction.d.ts.map +1 -0
- memos/_js/entity-extraction.js +325 -0
- memos/_js/entity-extraction.js.map +1 -0
- memos/_js/entity-index.d.ts +93 -0
- memos/_js/entity-index.d.ts.map +1 -0
- memos/_js/entity-index.js +165 -0
- memos/_js/entity-index.js.map +1 -0
- memos/_js/event-memory.d.ts +147 -0
- memos/_js/event-memory.d.ts.map +1 -0
- memos/_js/event-memory.js +290 -0
- memos/_js/event-memory.js.map +1 -0
- memos/_js/external-import.d.ts +98 -0
- memos/_js/external-import.d.ts.map +1 -0
- memos/_js/external-import.js +669 -0
- memos/_js/external-import.js.map +1 -0
- memos/_js/fidelity.d.ts +367 -0
- memos/_js/fidelity.d.ts.map +1 -0
- memos/_js/fidelity.js +911 -0
- memos/_js/fidelity.js.map +1 -0
- memos/_js/graph-expansion.d.ts +119 -0
- memos/_js/graph-expansion.d.ts.map +1 -0
- memos/_js/graph-expansion.js +213 -0
- memos/_js/graph-expansion.js.map +1 -0
- memos/_js/graph-mermaid.d.ts +13 -0
- memos/_js/graph-mermaid.d.ts.map +1 -0
- memos/_js/graph-mermaid.js +32 -0
- memos/_js/graph-mermaid.js.map +1 -0
- memos/_js/graph.d.ts +170 -0
- memos/_js/graph.d.ts.map +1 -0
- memos/_js/graph.js +482 -0
- memos/_js/graph.js.map +1 -0
- memos/_js/harness.d.ts +63 -0
- memos/_js/harness.d.ts.map +1 -0
- memos/_js/harness.js +93 -0
- memos/_js/harness.js.map +1 -0
- memos/_js/importance.d.ts +118 -0
- memos/_js/importance.d.ts.map +1 -0
- memos/_js/importance.js +144 -0
- memos/_js/importance.js.map +1 -0
- memos/_js/index.d.ts +51 -0
- memos/_js/index.d.ts.map +1 -0
- memos/_js/index.js +53 -0
- memos/_js/index.js.map +1 -0
- memos/_js/mcp.d.ts +58 -0
- memos/_js/mcp.d.ts.map +1 -0
- memos/_js/mcp.js +1853 -0
- memos/_js/mcp.js.map +1 -0
- memos/_js/memory.d.ts +1762 -0
- memos/_js/memory.d.ts.map +1 -0
- memos/_js/memory.js +6283 -0
- memos/_js/memory.js.map +1 -0
- memos/_js/procedural.d.ts +74 -0
- memos/_js/procedural.d.ts.map +1 -0
- memos/_js/procedural.js +125 -0
- memos/_js/procedural.js.map +1 -0
- memos/_js/project-config.d.ts +68 -0
- memos/_js/project-config.d.ts.map +1 -0
- memos/_js/project-config.js +59 -0
- memos/_js/project-config.js.map +1 -0
- memos/_js/proof-of-forgetting.d.ts +57 -0
- memos/_js/proof-of-forgetting.d.ts.map +1 -0
- memos/_js/proof-of-forgetting.js +80 -0
- memos/_js/proof-of-forgetting.js.map +1 -0
- memos/_js/provenance.d.ts +122 -0
- memos/_js/provenance.d.ts.map +1 -0
- memos/_js/provenance.js +182 -0
- memos/_js/provenance.js.map +1 -0
- memos/_js/quarantine-commands.d.ts +23 -0
- memos/_js/quarantine-commands.d.ts.map +1 -0
- memos/_js/quarantine-commands.js +42 -0
- memos/_js/quarantine-commands.js.map +1 -0
- memos/_js/quarantine.d.ts +120 -0
- memos/_js/quarantine.d.ts.map +1 -0
- memos/_js/quarantine.js +873 -0
- memos/_js/quarantine.js.map +1 -0
- memos/_js/redact.d.ts +12 -0
- memos/_js/redact.d.ts.map +1 -0
- memos/_js/redact.js +26 -0
- memos/_js/redact.js.map +1 -0
- memos/_js/repl.d.ts +57 -0
- memos/_js/repl.d.ts.map +1 -0
- memos/_js/repl.js +295 -0
- memos/_js/repl.js.map +1 -0
- memos/_js/rerank-local.d.ts +61 -0
- memos/_js/rerank-local.d.ts.map +1 -0
- memos/_js/rerank-local.js +210 -0
- memos/_js/rerank-local.js.map +1 -0
- memos/_js/retain-filter.d.ts +98 -0
- memos/_js/retain-filter.d.ts.map +1 -0
- memos/_js/retain-filter.js +232 -0
- memos/_js/retain-filter.js.map +1 -0
- memos/_js/retrieval.d.ts +265 -0
- memos/_js/retrieval.d.ts.map +1 -0
- memos/_js/retrieval.js +1018 -0
- memos/_js/retrieval.js.map +1 -0
- memos/_js/revert.d.ts +77 -0
- memos/_js/revert.d.ts.map +1 -0
- memos/_js/revert.js +252 -0
- memos/_js/revert.js.map +1 -0
- memos/_js/scope.d.ts +30 -0
- memos/_js/scope.d.ts.map +1 -0
- memos/_js/scope.js +71 -0
- memos/_js/scope.js.map +1 -0
- memos/_js/semantic-screen.d.ts +152 -0
- memos/_js/semantic-screen.d.ts.map +1 -0
- memos/_js/semantic-screen.js +290 -0
- memos/_js/semantic-screen.js.map +1 -0
- memos/_js/session-hooks.d.ts +127 -0
- memos/_js/session-hooks.d.ts.map +1 -0
- memos/_js/session-hooks.js +482 -0
- memos/_js/session-hooks.js.map +1 -0
- memos/_js/storage/sqlite.d.ts +427 -0
- memos/_js/storage/sqlite.d.ts.map +1 -0
- memos/_js/storage/sqlite.js +2428 -0
- memos/_js/storage/sqlite.js.map +1 -0
- memos/_js/sync.d.ts +213 -0
- memos/_js/sync.d.ts.map +1 -0
- memos/_js/sync.js +389 -0
- memos/_js/sync.js.map +1 -0
- memos/_js/tamper-log.d.ts +98 -0
- memos/_js/tamper-log.d.ts.map +1 -0
- memos/_js/tamper-log.js +327 -0
- memos/_js/tamper-log.js.map +1 -0
- memos/_js/temporal.d.ts +92 -0
- memos/_js/temporal.d.ts.map +1 -0
- memos/_js/temporal.js +536 -0
- memos/_js/temporal.js.map +1 -0
- memos/_js/time-travel.d.ts +38 -0
- memos/_js/time-travel.d.ts.map +1 -0
- memos/_js/time-travel.js +46 -0
- memos/_js/time-travel.js.map +1 -0
- memos/_js/timestamp-format.d.ts +18 -0
- memos/_js/timestamp-format.d.ts.map +1 -0
- memos/_js/timestamp-format.js +52 -0
- memos/_js/timestamp-format.js.map +1 -0
- memos/_js/triple-extraction.d.ts +31 -0
- memos/_js/triple-extraction.d.ts.map +1 -0
- memos/_js/triple-extraction.js +116 -0
- memos/_js/triple-extraction.js.map +1 -0
- memos/_js/types.d.ts +2474 -0
- memos/_js/types.d.ts.map +1 -0
- memos/_js/types.js +20 -0
- memos/_js/types.js.map +1 -0
- memos/_js/updater.d.ts +95 -0
- memos/_js/updater.d.ts.map +1 -0
- memos/_js/updater.js +268 -0
- memos/_js/updater.js.map +1 -0
- memos/_js/version.d.ts +17 -0
- memos/_js/version.d.ts.map +1 -0
- memos/_js/version.js +34 -0
- memos/_js/version.js.map +1 -0
- memos/_js/vision/clip-query-cache.d.ts +58 -0
- memos/_js/vision/clip-query-cache.d.ts.map +1 -0
- memos/_js/vision/clip-query-cache.js +79 -0
- memos/_js/vision/clip-query-cache.js.map +1 -0
- memos/_js/vision/clip.d.ts +113 -0
- memos/_js/vision/clip.d.ts.map +1 -0
- memos/_js/vision/clip.js +224 -0
- memos/_js/vision/clip.js.map +1 -0
- memos/_js/vision/fake-clip.d.ts +55 -0
- memos/_js/vision/fake-clip.d.ts.map +1 -0
- memos/_js/vision/fake-clip.js +122 -0
- memos/_js/vision/fake-clip.js.map +1 -0
- memos/_js/vision/index.d.ts +15 -0
- memos/_js/vision/index.d.ts.map +1 -0
- memos/_js/vision/index.js +13 -0
- memos/_js/vision/index.js.map +1 -0
- memos/_js/vision/store.d.ts +137 -0
- memos/_js/vision/store.d.ts.map +1 -0
- memos/_js/vision/store.js +383 -0
- memos/_js/vision/store.js.map +1 -0
- memos/adapters/__init__.py +0 -0
- memos/adapters/crewai.py +385 -0
- memos/adapters/langchain.py +301 -0
- memos/adapters/ollama.py +396 -0
- memos/server/__init__.py +0 -0
- memos/server/main.py +547 -0
- memos/server/routes.py +360 -0
|
@@ -0,0 +1,2428 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite-backed storage adapter for MemOS.
|
|
3
|
+
*
|
|
4
|
+
* Uses `better-sqlite3` for synchronous, high-performance local persistence.
|
|
5
|
+
* Full-text search is powered by SQLite FTS5.
|
|
6
|
+
*
|
|
7
|
+
* @module @memos/storage/sqlite
|
|
8
|
+
*/
|
|
9
|
+
import Database from "better-sqlite3";
|
|
10
|
+
import { createRequire } from "node:module";
|
|
11
|
+
import { randomUUID } from "node:crypto";
|
|
12
|
+
import { homedir } from "node:os";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { escapeLikePrefix } from "../scope.js";
|
|
15
|
+
import { clampLessonScore } from "../procedural.js";
|
|
16
|
+
import { isProvenanceTier } from "../provenance.js";
|
|
17
|
+
import { backfillEntityIndex, ensureEntityIndexTable, extractContentEntities, searchByEntities as searchEntityIndex, writeEntityIndexRows, } from "../entity-index.js";
|
|
18
|
+
import { GENESIS_PREV, entryHash as tamperEntryHash } from "../tamper-log.js";
|
|
19
|
+
import { cosineSimilarity } from "../embeddings.js";
|
|
20
|
+
/**
|
|
21
|
+
* Default database path: `~/.memos/memos.db`.
|
|
22
|
+
*/
|
|
23
|
+
export function defaultDbPath() {
|
|
24
|
+
const home = homedir() || process.env.USERPROFILE || process.env.HOME || ".";
|
|
25
|
+
return join(home, ".memos", "memos.db");
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Streaming bounded top-K selector.
|
|
29
|
+
*
|
|
30
|
+
* A binary min-heap of capacity `k` that keeps the largest `k` scores
|
|
31
|
+
* seen so far, so the caller never has to materialise an O(N) pair array
|
|
32
|
+
* and re-heapify it afterwards (O(N log K) either way, but without the
|
|
33
|
+
* allocation churn — and with the heap floor available *while* scanning,
|
|
34
|
+
* which is what the early-exit bound needs).
|
|
35
|
+
*
|
|
36
|
+
* Retention is identical to `takeTopK(pairs, k)` / `pairs.sort(desc).slice(0, k)`:
|
|
37
|
+
* same elements, same descending order, and the same strict-improvement
|
|
38
|
+
* rule — a candidate must beat the current floor strictly to displace it,
|
|
39
|
+
* so among equal scores the earlier-scanned one wins.
|
|
40
|
+
*/
|
|
41
|
+
class TopKSelector {
|
|
42
|
+
heap = [];
|
|
43
|
+
capacity;
|
|
44
|
+
constructor(capacity) {
|
|
45
|
+
this.capacity = capacity > 0 ? capacity : 0;
|
|
46
|
+
}
|
|
47
|
+
/** Current K-th best score, or `-Infinity` while the heap is filling. */
|
|
48
|
+
get floor() {
|
|
49
|
+
return this.heap.length > 0 ? this.heap[0].score : -Infinity;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* True once the heap holds `capacity` items. The early-exit bound in
|
|
53
|
+
* the caller is only sound at this point: a full heap can never admit
|
|
54
|
+
* another candidate without a strict improvement over its floor,
|
|
55
|
+
* whereas a partially-filled heap still has slots that equally-scored
|
|
56
|
+
* (tied) candidates must be allowed to take.
|
|
57
|
+
*/
|
|
58
|
+
get full() {
|
|
59
|
+
return this.heap.length >= this.capacity;
|
|
60
|
+
}
|
|
61
|
+
/** Offer a candidate; returns true when it was retained. */
|
|
62
|
+
push(item) {
|
|
63
|
+
if (this.capacity === 0)
|
|
64
|
+
return false;
|
|
65
|
+
if (this.heap.length < this.capacity) {
|
|
66
|
+
this.heap.push(item);
|
|
67
|
+
this.siftDown(0);
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
70
|
+
if (item.score > this.heap[0].score) {
|
|
71
|
+
this.heap[0] = item;
|
|
72
|
+
this.siftDown(0);
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
/** Retained items, best first. */
|
|
78
|
+
result() {
|
|
79
|
+
return this.heap.sort((a, b) => b.score - a.score);
|
|
80
|
+
}
|
|
81
|
+
siftDown(start) {
|
|
82
|
+
let i = start;
|
|
83
|
+
for (;;) {
|
|
84
|
+
const left = 2 * i + 1;
|
|
85
|
+
const right = left + 1;
|
|
86
|
+
let smallest = i;
|
|
87
|
+
if (left < this.heap.length &&
|
|
88
|
+
this.heap[left].score < this.heap[smallest].score)
|
|
89
|
+
smallest = left;
|
|
90
|
+
if (right < this.heap.length &&
|
|
91
|
+
this.heap[right].score < this.heap[smallest].score)
|
|
92
|
+
smallest = right;
|
|
93
|
+
if (smallest === i)
|
|
94
|
+
break;
|
|
95
|
+
const tmp = this.heap[i];
|
|
96
|
+
this.heap[i] = this.heap[smallest];
|
|
97
|
+
this.heap[smallest] = tmp;
|
|
98
|
+
i = smallest;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Squared L2 norm, accumulated in index order.
|
|
104
|
+
*
|
|
105
|
+
* Kept as a standalone helper (rather than inlined into
|
|
106
|
+
* `cosineSimilarity`) so the semantic scan can cache it per stored row:
|
|
107
|
+
* the value only changes when the row is written, and every write goes
|
|
108
|
+
* through `SQLiteStorage.vectorCacheSet`, so the cache can never drift.
|
|
109
|
+
*/
|
|
110
|
+
function sumSquares(vector) {
|
|
111
|
+
let total = 0;
|
|
112
|
+
for (let index = 0; index < vector.length; index += 1) {
|
|
113
|
+
const value = vector[index];
|
|
114
|
+
total += value * value;
|
|
115
|
+
}
|
|
116
|
+
return total;
|
|
117
|
+
}
|
|
118
|
+
/** Stride of the coarse prefix projection used by the prefilter mode. */
|
|
119
|
+
const PREFILTER_STRIDE = 8;
|
|
120
|
+
/** Floor for the prefilter's exact-rescore window. */
|
|
121
|
+
const PREFILTER_MIN_KEEP = 32;
|
|
122
|
+
/**
|
|
123
|
+
* Minimal function-word stoplist for FTS5 query tokenization.
|
|
124
|
+
*
|
|
125
|
+
* Deliberately SMALL: only grammatical glue words. Content words
|
|
126
|
+
* ("development", "language", "preferences", ...) must survive —
|
|
127
|
+
* filtering them out destroys the retrieval signal. Words like
|
|
128
|
+
* "user" are intentionally NOT listed: FTS5's bm25 ranking already
|
|
129
|
+
* down-weights terms that appear in (nearly) every document.
|
|
130
|
+
*/
|
|
131
|
+
const FTS_STOPWORDS = new Set([
|
|
132
|
+
"the",
|
|
133
|
+
"a",
|
|
134
|
+
"an",
|
|
135
|
+
"is",
|
|
136
|
+
"are",
|
|
137
|
+
"was",
|
|
138
|
+
"were",
|
|
139
|
+
"be",
|
|
140
|
+
"been",
|
|
141
|
+
"being",
|
|
142
|
+
"does",
|
|
143
|
+
"did",
|
|
144
|
+
"do",
|
|
145
|
+
"has",
|
|
146
|
+
"have",
|
|
147
|
+
"had",
|
|
148
|
+
"of",
|
|
149
|
+
"in",
|
|
150
|
+
"on",
|
|
151
|
+
"at",
|
|
152
|
+
"to",
|
|
153
|
+
"for",
|
|
154
|
+
"and",
|
|
155
|
+
"or",
|
|
156
|
+
"not",
|
|
157
|
+
"no",
|
|
158
|
+
"with",
|
|
159
|
+
"by",
|
|
160
|
+
"from",
|
|
161
|
+
"as",
|
|
162
|
+
"it",
|
|
163
|
+
"its",
|
|
164
|
+
"this",
|
|
165
|
+
"that",
|
|
166
|
+
"these",
|
|
167
|
+
"those",
|
|
168
|
+
"what",
|
|
169
|
+
"when",
|
|
170
|
+
"where",
|
|
171
|
+
"who",
|
|
172
|
+
"whom",
|
|
173
|
+
"which",
|
|
174
|
+
"how",
|
|
175
|
+
"why",
|
|
176
|
+
"can",
|
|
177
|
+
"could",
|
|
178
|
+
"will",
|
|
179
|
+
"would",
|
|
180
|
+
"should",
|
|
181
|
+
"shall",
|
|
182
|
+
"may",
|
|
183
|
+
"might",
|
|
184
|
+
"must",
|
|
185
|
+
"about",
|
|
186
|
+
"into",
|
|
187
|
+
"over",
|
|
188
|
+
"after",
|
|
189
|
+
"before",
|
|
190
|
+
"they",
|
|
191
|
+
"them",
|
|
192
|
+
"their",
|
|
193
|
+
"theirs",
|
|
194
|
+
"he",
|
|
195
|
+
"she",
|
|
196
|
+
"his",
|
|
197
|
+
"her",
|
|
198
|
+
"hers",
|
|
199
|
+
"we",
|
|
200
|
+
"us",
|
|
201
|
+
"our",
|
|
202
|
+
"ours",
|
|
203
|
+
"you",
|
|
204
|
+
"your",
|
|
205
|
+
"yours",
|
|
206
|
+
"i",
|
|
207
|
+
"me",
|
|
208
|
+
"my",
|
|
209
|
+
"mine",
|
|
210
|
+
"if",
|
|
211
|
+
"then",
|
|
212
|
+
"than",
|
|
213
|
+
"so",
|
|
214
|
+
"there",
|
|
215
|
+
"here",
|
|
216
|
+
"any",
|
|
217
|
+
"all",
|
|
218
|
+
"some",
|
|
219
|
+
"very",
|
|
220
|
+
"too",
|
|
221
|
+
"also",
|
|
222
|
+
]);
|
|
223
|
+
/**
|
|
224
|
+
* Tokenize a natural-language query into individually quoted FTS5
|
|
225
|
+
* terms.
|
|
226
|
+
*
|
|
227
|
+
* The query is split on non-alphanumeric boundaries, lowercased, and
|
|
228
|
+
* function words are dropped. Each surviving term is double-quoted
|
|
229
|
+
* (escaping any embedded quote) so FTS5 special characters cannot
|
|
230
|
+
* inject syntax.
|
|
231
|
+
*
|
|
232
|
+
* If stopword removal empties the query (e.g. "what is it"), the raw
|
|
233
|
+
* tokens are used instead so the search never degrades to a
|
|
234
|
+
* match-everything no-op.
|
|
235
|
+
*
|
|
236
|
+
* @param query — Raw user query text.
|
|
237
|
+
* @returns Quoted FTS5 terms, e.g. `['"dark"', '"mode"']`.
|
|
238
|
+
*/
|
|
239
|
+
export function buildFtsTerms(query) {
|
|
240
|
+
const tokens = query
|
|
241
|
+
.toLowerCase()
|
|
242
|
+
.split(/[^a-z0-9]+/)
|
|
243
|
+
.filter((t) => t.length > 1);
|
|
244
|
+
const terms = tokens.filter((t) => !FTS_STOPWORDS.has(t));
|
|
245
|
+
const effective = terms.length > 0 ? terms : tokens;
|
|
246
|
+
if (effective.length === 0) {
|
|
247
|
+
// Nothing tokenizable (e.g. punctuation-only). Quote the raw
|
|
248
|
+
// string so FTS5 treats it as a literal (likely zero-result)
|
|
249
|
+
// phrase rather than a syntax error.
|
|
250
|
+
return [`"${query.replace(/"/g, '""')}"`];
|
|
251
|
+
}
|
|
252
|
+
return effective.map((t) => `"${t.replace(/"/g, '""')}"`);
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Build an FTS5 MATCH expression from a natural-language query.
|
|
256
|
+
*
|
|
257
|
+
* Terms are joined with `operator` — `" AND "` for precision (every
|
|
258
|
+
* term must appear) or `" OR "` for recall (any term may appear).
|
|
259
|
+
*
|
|
260
|
+
* @param query — Raw user query text.
|
|
261
|
+
* @param operator — Join operator between terms. Default `" AND "`.
|
|
262
|
+
* @returns An FTS5 MATCH expression, e.g. `"dark" AND "mode"`.
|
|
263
|
+
*/
|
|
264
|
+
export function buildFtsQuery(query, operator = " AND ") {
|
|
265
|
+
return buildFtsTerms(query).join(operator);
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* SQLite storage implementation.
|
|
269
|
+
*/
|
|
270
|
+
export class SQLiteStorage {
|
|
271
|
+
db;
|
|
272
|
+
cipherKey;
|
|
273
|
+
path;
|
|
274
|
+
wal;
|
|
275
|
+
/**
|
|
276
|
+
* Per-id access count delta buffer. Batched into a single UPDATE per
|
|
277
|
+
* id on a 500 ms debounce. Removes the read-modify-write write that
|
|
278
|
+
* was previously amplified to a `SET access_count = access_count + 1`
|
|
279
|
+
* on every `getNode` call. See {@link flushAccessCounts}.
|
|
280
|
+
*/
|
|
281
|
+
accessBuffer = new Map();
|
|
282
|
+
accessFlushTimer = null;
|
|
283
|
+
accessFlushIntervalMs;
|
|
284
|
+
/**
|
|
285
|
+
* Create a new SQLiteStorage instance.
|
|
286
|
+
*
|
|
287
|
+
* @param path — Filesystem path to the `.db` file.
|
|
288
|
+
* @param wal — Enable WAL journal mode (recommended).
|
|
289
|
+
*/
|
|
290
|
+
constructor(path, wal = true, options = {}) {
|
|
291
|
+
this.path = path;
|
|
292
|
+
this.wal = wal;
|
|
293
|
+
// 500 ms is the sweet spot — long enough to coalesce rapid reads,
|
|
294
|
+
// short enough that the persisted value is never far behind the
|
|
295
|
+
// in-memory truth.
|
|
296
|
+
this.accessFlushIntervalMs = 500;
|
|
297
|
+
this.vectorCacheMaxEntries = Math.max(100, options.vectorCacheEntries ?? 25_000);
|
|
298
|
+
this.semanticScanMode = options.semanticScan ?? "exact";
|
|
299
|
+
this.cipherKey = options.cipherKey;
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Open the database file, switching to the cipher-capable driver and
|
|
303
|
+
* applying `PRAGMA key` when encryption is configured. The key MUST be
|
|
304
|
+
* set before any other statement on an encrypted database.
|
|
305
|
+
*/
|
|
306
|
+
openDatabase(path, cipherKey) {
|
|
307
|
+
if (!cipherKey || path === ":memory:" || path.startsWith("file::memory:")) {
|
|
308
|
+
// In-memory databases cannot be encrypted (the driver rejects
|
|
309
|
+
// PRAGMA key) — and there is nothing on disk to protect.
|
|
310
|
+
return new Database(path);
|
|
311
|
+
}
|
|
312
|
+
const CipherDatabase = loadCipherDriver();
|
|
313
|
+
const db = new CipherDatabase(path);
|
|
314
|
+
// Escape single quotes for the pragma literal; a mismatched key makes
|
|
315
|
+
// the first real statement fail with SQLITE_NOTADB, which callers see
|
|
316
|
+
// as a clear decryption error.
|
|
317
|
+
const escaped = cipherKey.replace(/'/g, "''");
|
|
318
|
+
db.pragma(`key = '${escaped}'`);
|
|
319
|
+
return db;
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Open the database and create tables / indices if they do not exist.
|
|
323
|
+
*/
|
|
324
|
+
async init() {
|
|
325
|
+
const fs = await import("fs");
|
|
326
|
+
const { dirname } = await import("path");
|
|
327
|
+
const dir = dirname(this.path);
|
|
328
|
+
if (dir && !fs.existsSync(dir)) {
|
|
329
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
330
|
+
}
|
|
331
|
+
this.db = this.openDatabase(this.path, this.cipherKey);
|
|
332
|
+
if (this.wal) {
|
|
333
|
+
this.db.pragma("journal_mode = WAL");
|
|
334
|
+
}
|
|
335
|
+
this.db.pragma("foreign_keys = ON");
|
|
336
|
+
// Cross-OS performance pragmas. `synchronous = NORMAL` is safe with WAL
|
|
337
|
+
// (the WAL file itself is still fsynced) and gives ~10x faster writes
|
|
338
|
+
// compared to FULL on macOS / Windows where fsync is expensive. The
|
|
339
|
+
// 32 MiB page cache and 256 MiB mmap cut read latency for medium
|
|
340
|
+
// stores without bumping RSS materially. Temp tables in memory keep
|
|
341
|
+
// ORDER BY and GROUP BY on heavy queries from spilling to disk.
|
|
342
|
+
this.db.pragma("synchronous = NORMAL");
|
|
343
|
+
this.db.pragma("cache_size = -32000");
|
|
344
|
+
this.db.pragma("temp_store = MEMORY");
|
|
345
|
+
this.db.pragma("wal_autocheckpoint = 1000");
|
|
346
|
+
this.db.pragma("mmap_size = 268435456");
|
|
347
|
+
this.db.exec(`
|
|
348
|
+
CREATE TABLE IF NOT EXISTS nodes (
|
|
349
|
+
id TEXT PRIMARY KEY,
|
|
350
|
+
content TEXT NOT NULL,
|
|
351
|
+
summary TEXT NOT NULL DEFAULT '',
|
|
352
|
+
type TEXT NOT NULL DEFAULT 'fact',
|
|
353
|
+
metadata TEXT NOT NULL DEFAULT '{}',
|
|
354
|
+
importance REAL NOT NULL DEFAULT 0.5,
|
|
355
|
+
created_at INTEGER NOT NULL,
|
|
356
|
+
updated_at INTEGER NOT NULL,
|
|
357
|
+
access_count INTEGER NOT NULL DEFAULT 0,
|
|
358
|
+
last_accessed INTEGER NOT NULL DEFAULT 0
|
|
359
|
+
);
|
|
360
|
+
|
|
361
|
+
CREATE TABLE IF NOT EXISTS edges (
|
|
362
|
+
id TEXT PRIMARY KEY,
|
|
363
|
+
source_id TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
|
|
364
|
+
target_id TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
|
|
365
|
+
relation TEXT NOT NULL DEFAULT 'relates_to',
|
|
366
|
+
weight REAL NOT NULL DEFAULT 0.5,
|
|
367
|
+
metadata TEXT NOT NULL DEFAULT '{}',
|
|
368
|
+
-- Bitemporal transaction time: when MemOS learned the fact.
|
|
369
|
+
-- Never renamed/repurposed; event-time validity lives in
|
|
370
|
+
-- valid_from / valid_to (added by migration below).
|
|
371
|
+
created_at INTEGER NOT NULL,
|
|
372
|
+
UNIQUE(source_id, target_id, relation)
|
|
373
|
+
);
|
|
374
|
+
|
|
375
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_type ON nodes(type);
|
|
376
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_importance ON nodes(importance);
|
|
377
|
+
CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source_id);
|
|
378
|
+
CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target_id);
|
|
379
|
+
|
|
380
|
+
-- Store vectors as float32 BLOBs instead of JSON to keep local DBs smaller
|
|
381
|
+
-- and avoid OS-specific JSON number formatting differences.
|
|
382
|
+
CREATE TABLE IF NOT EXISTS embeddings (
|
|
383
|
+
node_id TEXT PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
|
|
384
|
+
vector BLOB NOT NULL,
|
|
385
|
+
dimensions INTEGER NOT NULL,
|
|
386
|
+
model TEXT NOT NULL,
|
|
387
|
+
updated_at INTEGER NOT NULL
|
|
388
|
+
);
|
|
389
|
+
|
|
390
|
+
-- Covering index for the semantic id-select: every query filters on
|
|
391
|
+
-- (dimensions, model) before scoring, so without this SQLite walks
|
|
392
|
+
-- the whole table even when the caller needs no node-side filters.
|
|
393
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_model_dims ON embeddings(dimensions, model);
|
|
394
|
+
|
|
395
|
+
-- Image (CLIP) vectors live in their OWN space: cosine similarity
|
|
396
|
+
-- between a 512-d CLIP vector and a 384-d bge text vector is
|
|
397
|
+
-- meaningless, so the two must never share a table (every existing
|
|
398
|
+
-- query would have to reason about which space a row is in). Same
|
|
399
|
+
-- float32-LE serialization as embeddings; ON DELETE CASCADE
|
|
400
|
+
-- does the cleanup, so forget() needs no image-side change.
|
|
401
|
+
CREATE TABLE IF NOT EXISTS image_embeddings (
|
|
402
|
+
node_id TEXT PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
|
|
403
|
+
vector BLOB NOT NULL,
|
|
404
|
+
dimensions INTEGER NOT NULL,
|
|
405
|
+
model TEXT NOT NULL,
|
|
406
|
+
updated_at INTEGER NOT NULL
|
|
407
|
+
);
|
|
408
|
+
|
|
409
|
+
CREATE INDEX IF NOT EXISTS idx_image_embeddings_model_dims
|
|
410
|
+
ON image_embeddings(dimensions, model);
|
|
411
|
+
|
|
412
|
+
-- Contradiction pairs (accuracy item 4): write-time rule-based
|
|
413
|
+
-- detection records node pairs that disagree (similarity in the
|
|
414
|
+
-- 0.5–0.85 band + entity overlap + shared subject). Add-only is
|
|
415
|
+
-- preserved — both versions are kept; read-time resolution demotes
|
|
416
|
+
-- the older member instead of deleting anything. node_a/node_b are
|
|
417
|
+
-- stored in canonical (sorted) order so the UNIQUE constraint
|
|
418
|
+
-- dedups pairs regardless of insertion order.
|
|
419
|
+
CREATE TABLE IF NOT EXISTS contradictions (
|
|
420
|
+
id TEXT PRIMARY KEY,
|
|
421
|
+
node_a TEXT NOT NULL,
|
|
422
|
+
node_b TEXT NOT NULL,
|
|
423
|
+
detected_at INTEGER NOT NULL,
|
|
424
|
+
status TEXT NOT NULL DEFAULT 'unresolved',
|
|
425
|
+
UNIQUE(node_a, node_b)
|
|
426
|
+
);
|
|
427
|
+
|
|
428
|
+
CREATE INDEX IF NOT EXISTS idx_contradictions_nodes ON contradictions(node_a, node_b);
|
|
429
|
+
|
|
430
|
+
-- Procedural lessons (self-editing instructions): behavioral
|
|
431
|
+
-- guidance rather than facts. Scores are adjusted by outcome
|
|
432
|
+
-- feedback (success reinforces, failure demotes) with a mild
|
|
433
|
+
-- read-time decay; all scoring math lives in src/procedural.ts.
|
|
434
|
+
CREATE TABLE IF NOT EXISTS procedural_lessons (
|
|
435
|
+
id TEXT PRIMARY KEY,
|
|
436
|
+
lesson TEXT NOT NULL,
|
|
437
|
+
context TEXT NOT NULL DEFAULT '',
|
|
438
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
439
|
+
score REAL NOT NULL DEFAULT 0.5,
|
|
440
|
+
success_count INTEGER NOT NULL DEFAULT 0,
|
|
441
|
+
failure_count INTEGER NOT NULL DEFAULT 0,
|
|
442
|
+
use_count INTEGER NOT NULL DEFAULT 0,
|
|
443
|
+
namespace TEXT NOT NULL DEFAULT 'default',
|
|
444
|
+
created_at INTEGER NOT NULL,
|
|
445
|
+
updated_at INTEGER NOT NULL,
|
|
446
|
+
last_used_at INTEGER NOT NULL DEFAULT 0
|
|
447
|
+
);
|
|
448
|
+
|
|
449
|
+
CREATE INDEX IF NOT EXISTS idx_procedural_lessons_namespace ON procedural_lessons(namespace);
|
|
450
|
+
CREATE INDEX IF NOT EXISTS idx_procedural_lessons_score ON procedural_lessons(score);
|
|
451
|
+
|
|
452
|
+
-- Tamper-evident mutation log: append-only hash chain over every
|
|
453
|
+
-- node mutation (store / import / update / forget). The chain
|
|
454
|
+
-- itself is checked by verifyTamperLog(); the tip hash can be
|
|
455
|
+
-- anchored externally via the memos log checkpoint command.
|
|
456
|
+
CREATE TABLE IF NOT EXISTS tamper_log (
|
|
457
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
458
|
+
ts INTEGER NOT NULL,
|
|
459
|
+
op TEXT NOT NULL,
|
|
460
|
+
node_id TEXT NOT NULL,
|
|
461
|
+
node_hash TEXT NOT NULL,
|
|
462
|
+
prev_hash TEXT NOT NULL,
|
|
463
|
+
entry_hash TEXT NOT NULL
|
|
464
|
+
);
|
|
465
|
+
|
|
466
|
+
CREATE INDEX IF NOT EXISTS idx_tamper_log_node ON tamper_log(node_id);
|
|
467
|
+
`);
|
|
468
|
+
// Migration from the first-day schema, which stored a content-only
|
|
469
|
+
// `content_hash`. Carry old rows into `node_hash`; verifyTamperLog
|
|
470
|
+
// keeps a legacy v1 check for those rows (chain linkage + content
|
|
471
|
+
// comparison) and reports them as legacyEntries.
|
|
472
|
+
const tamperCols = this.db
|
|
473
|
+
.prepare(`PRAGMA table_info(tamper_log)`)
|
|
474
|
+
.all();
|
|
475
|
+
const tamperNames = new Set(tamperCols.map((c) => c.name));
|
|
476
|
+
if (tamperNames.has("content_hash") && !tamperNames.has("node_hash")) {
|
|
477
|
+
this.db.exec(`ALTER TABLE tamper_log ADD COLUMN node_hash TEXT NOT NULL DEFAULT ''`);
|
|
478
|
+
this.db.exec(`UPDATE tamper_log SET node_hash = content_hash WHERE node_hash = ''`);
|
|
479
|
+
// The old column is fully redundant after the backfill, and its
|
|
480
|
+
// NOT NULL constraint would reject new inserts — drop it.
|
|
481
|
+
this.db.exec(`ALTER TABLE tamper_log DROP COLUMN content_hash`);
|
|
482
|
+
}
|
|
483
|
+
// Migration: add expires_at column if missing
|
|
484
|
+
this.migrateAddColumn("nodes", "expires_at", "INTEGER DEFAULT NULL");
|
|
485
|
+
// Migration: add tags column if missing
|
|
486
|
+
this.migrateAddColumn("nodes", "tags", "TEXT NOT NULL DEFAULT '[]'");
|
|
487
|
+
// Migration: add namespace column if missing
|
|
488
|
+
this.migrateAddColumn("nodes", "namespace", "TEXT NOT NULL DEFAULT 'default'");
|
|
489
|
+
// Migration: temporal validity columns
|
|
490
|
+
this.migrateAddColumn("nodes", "valid_from", "INTEGER DEFAULT NULL");
|
|
491
|
+
this.migrateAddColumn("nodes", "valid_to", "INTEGER DEFAULT NULL");
|
|
492
|
+
// Migration: provenance & trust columns
|
|
493
|
+
this.migrateAddColumn("nodes", "source", "TEXT NOT NULL DEFAULT 'user_input'");
|
|
494
|
+
this.migrateAddColumn("nodes", "trust_score", "REAL NOT NULL DEFAULT 1.0");
|
|
495
|
+
// Migration: multi-granularity retrieval pool (event | note | procedure).
|
|
496
|
+
// Every pre-pool memory reads as "event", matching the legacy semantics.
|
|
497
|
+
this.migrateAddColumn("nodes", "pool", "TEXT NOT NULL DEFAULT 'event'");
|
|
498
|
+
// Migration: provenance-trust layer. Every pre-layer memory reads as
|
|
499
|
+
// the "user" tier (ordinary statements) with no quarantine history —
|
|
500
|
+
// the exclusion default below is a no-op for legacy rows, so recall
|
|
501
|
+
// behaviour on existing databases is unchanged.
|
|
502
|
+
this.migrateAddColumn("nodes", "provenance", "TEXT NOT NULL DEFAULT 'user'");
|
|
503
|
+
this.migrateAddColumn("nodes", "quarantined", "INTEGER NOT NULL DEFAULT 0");
|
|
504
|
+
this.migrateAddColumn("nodes", "quarantined_at", "INTEGER DEFAULT NULL");
|
|
505
|
+
this.migrateAddColumn("nodes", "quarantine_reason", "TEXT DEFAULT NULL");
|
|
506
|
+
// Migration: harness attribution ("one memory, every harness").
|
|
507
|
+
// Every pre-attribution row reads as "unknown", so scoped recall
|
|
508
|
+
// treats legacy data as unscoped and `harness list` shows it under
|
|
509
|
+
// "unknown".
|
|
510
|
+
this.migrateAddColumn("nodes", "harness", "TEXT NOT NULL DEFAULT 'unknown'");
|
|
511
|
+
// Migration: bitemporal event-time validity on edges. `created_at`
|
|
512
|
+
// remains the transaction/ingest time — it is NOT renamed; it is the
|
|
513
|
+
// transaction-time axis of the bitemporal pair.
|
|
514
|
+
this.migrateAddColumn("edges", "valid_from", "INTEGER DEFAULT NULL");
|
|
515
|
+
this.migrateAddColumn("edges", "valid_to", "INTEGER DEFAULT NULL");
|
|
516
|
+
// Index for edge validity-interval queries (as-of / time-travel reads).
|
|
517
|
+
// Created here rather than in the schema block above because the
|
|
518
|
+
// columns only exist after the migrations run on pre-existing DBs.
|
|
519
|
+
this.db.exec(`
|
|
520
|
+
CREATE INDEX IF NOT EXISTS idx_edges_valid_from ON edges(valid_from);
|
|
521
|
+
CREATE INDEX IF NOT EXISTS idx_edges_valid_to ON edges(valid_to);
|
|
522
|
+
`);
|
|
523
|
+
// Index for TTL sweep
|
|
524
|
+
this.db.exec(`
|
|
525
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_expires_at ON nodes(expires_at);
|
|
526
|
+
`);
|
|
527
|
+
// Index for temporal queries
|
|
528
|
+
this.db.exec(`
|
|
529
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_valid_from ON nodes(valid_from);
|
|
530
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_valid_to ON nodes(valid_to);
|
|
531
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_trust_score ON nodes(trust_score);
|
|
532
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_source ON nodes(source);
|
|
533
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_pool ON nodes(pool);
|
|
534
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_harness ON nodes(harness);
|
|
535
|
+
`);
|
|
536
|
+
// FTS5 virtual table for full-text search. The `porter unicode61`
|
|
537
|
+
// tokenizer applies Porter stemming, so word variants match
|
|
538
|
+
// ("research" finds "researching", "studies" finds "study").
|
|
539
|
+
// Note: `IF NOT EXISTS` means pre-existing databases keep their
|
|
540
|
+
// original tokenizer; new databases get stemming.
|
|
541
|
+
this.db.exec(`
|
|
542
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
|
|
543
|
+
content,
|
|
544
|
+
summary,
|
|
545
|
+
content='nodes',
|
|
546
|
+
content_rowid='rowid',
|
|
547
|
+
tokenize='porter unicode61'
|
|
548
|
+
);
|
|
549
|
+
`);
|
|
550
|
+
// Triggers to keep FTS index in sync
|
|
551
|
+
this.db.exec(`
|
|
552
|
+
CREATE TRIGGER IF NOT EXISTS nodes_ai AFTER INSERT ON nodes BEGIN
|
|
553
|
+
INSERT INTO nodes_fts(rowid, content, summary)
|
|
554
|
+
VALUES (new.rowid, new.content, new.summary);
|
|
555
|
+
END;
|
|
556
|
+
|
|
557
|
+
CREATE TRIGGER IF NOT EXISTS nodes_ad AFTER DELETE ON nodes BEGIN
|
|
558
|
+
INSERT INTO nodes_fts(nodes_fts, rowid, content, summary)
|
|
559
|
+
VALUES ('delete', old.rowid, old.content, old.summary);
|
|
560
|
+
END;
|
|
561
|
+
|
|
562
|
+
CREATE TRIGGER IF NOT EXISTS nodes_au AFTER UPDATE ON nodes BEGIN
|
|
563
|
+
INSERT INTO nodes_fts(nodes_fts, rowid, content, summary)
|
|
564
|
+
VALUES ('delete', old.rowid, old.content, old.summary);
|
|
565
|
+
INSERT INTO nodes_fts(rowid, content, summary)
|
|
566
|
+
VALUES (new.rowid, new.content, new.summary);
|
|
567
|
+
END;
|
|
568
|
+
`);
|
|
569
|
+
// Tag join table. Replaces the JSON-LIKE scan on every tag query with
|
|
570
|
+
// a real B-tree index. The (node_id, tag) primary key makes writes
|
|
571
|
+
// O(tags) and the (tag) secondary index makes reverse lookups O(1).
|
|
572
|
+
this.db.exec(`
|
|
573
|
+
CREATE TABLE IF NOT EXISTS node_tags (
|
|
574
|
+
node_id TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
|
|
575
|
+
tag TEXT NOT NULL,
|
|
576
|
+
PRIMARY KEY (node_id, tag)
|
|
577
|
+
);
|
|
578
|
+
CREATE INDEX IF NOT EXISTS idx_node_tags_tag ON node_tags(tag);
|
|
579
|
+
`);
|
|
580
|
+
// Migration: back-fill node_tags from the JSON column for any pre-
|
|
581
|
+
// existing rows. Safe to run on every init — INSERT OR IGNORE means
|
|
582
|
+
// already-synced rows are no-ops.
|
|
583
|
+
this.migrateBackfillNodeTags();
|
|
584
|
+
// ── item2: entity leg ── entity→memories inverted index (first-class
|
|
585
|
+
// entity channel). Mirrors the node_tags pattern: join table with a
|
|
586
|
+
// (entity) lookup index, backfilled from existing rows on init.
|
|
587
|
+
ensureEntityIndexTable(this.db);
|
|
588
|
+
this.migrateBackfillEntityIndex();
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* One-time backfill: for every node, read the JSON `tags` column and
|
|
592
|
+
* ensure each tag exists in `node_tags`. Idempotent thanks to
|
|
593
|
+
* `INSERT OR IGNORE`. Runs inside a single transaction so a 100k-node
|
|
594
|
+
* database finishes in well under a second on a warm cache.
|
|
595
|
+
*/
|
|
596
|
+
migrateBackfillNodeTags() {
|
|
597
|
+
const rows = this.db.prepare("SELECT id, tags FROM nodes").all();
|
|
598
|
+
if (rows.length === 0)
|
|
599
|
+
return;
|
|
600
|
+
const insert = this.db.prepare("INSERT OR IGNORE INTO node_tags (node_id, tag) VALUES (?, ?)");
|
|
601
|
+
const seen = new Map();
|
|
602
|
+
const tx = this.db.transaction(() => {
|
|
603
|
+
for (const row of rows) {
|
|
604
|
+
let tags;
|
|
605
|
+
try {
|
|
606
|
+
tags = JSON.parse(row.tags);
|
|
607
|
+
}
|
|
608
|
+
catch {
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
if (!Array.isArray(tags))
|
|
612
|
+
continue;
|
|
613
|
+
let ids = seen.get(row.id);
|
|
614
|
+
if (!ids) {
|
|
615
|
+
ids = new Set();
|
|
616
|
+
seen.set(row.id, ids);
|
|
617
|
+
}
|
|
618
|
+
for (const tag of tags) {
|
|
619
|
+
if (typeof tag !== "string" || tag.length === 0)
|
|
620
|
+
continue;
|
|
621
|
+
if (ids.has(tag))
|
|
622
|
+
continue;
|
|
623
|
+
ids.add(tag);
|
|
624
|
+
insert.run(row.id, tag);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
});
|
|
628
|
+
tx();
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* One-time backfill of the `entity_index` inverted table for pre-existing
|
|
632
|
+
* rows (databases created before the table existed). Idempotent — nodes
|
|
633
|
+
* already present in the index are skipped. Delegates to
|
|
634
|
+
* `src/entity-index.ts`, which prefers each row's stored
|
|
635
|
+
* `metadata.entities` (canonicalized with the deployment's aliases at
|
|
636
|
+
* write time) and falls back to extracting from content.
|
|
637
|
+
*/
|
|
638
|
+
migrateBackfillEntityIndex() {
|
|
639
|
+
backfillEntityIndex(this.db);
|
|
640
|
+
}
|
|
641
|
+
/**
|
|
642
|
+
* Replace the `node_tags` rows for a single node. Used after
|
|
643
|
+
* `saveNode` / `updateNode` to keep the join table in lockstep with
|
|
644
|
+
* the JSON column.
|
|
645
|
+
*
|
|
646
|
+
* Runs inside the caller's transaction (saveNode/updateNode wrap their
|
|
647
|
+
* write + this call in one `db.transaction`) and reuses cached prepared
|
|
648
|
+
* statements — previously each call paid two uncached `db.prepare()`
|
|
649
|
+
* parses plus its own transaction commit.
|
|
650
|
+
*/
|
|
651
|
+
writeNodeTags(nodeId, tags) {
|
|
652
|
+
const deleteStmt = this.getPreparedStatement("writeNodeTags::delete", "DELETE FROM node_tags WHERE node_id = ?");
|
|
653
|
+
deleteStmt.run(nodeId);
|
|
654
|
+
if (tags.length === 0)
|
|
655
|
+
return;
|
|
656
|
+
const insert = this.getPreparedStatement("writeNodeTags::insert", "INSERT OR IGNORE INTO node_tags (node_id, tag) VALUES (?, ?)");
|
|
657
|
+
const seen = new Set();
|
|
658
|
+
for (const tag of tags) {
|
|
659
|
+
if (typeof tag !== "string" || tag.length === 0)
|
|
660
|
+
continue;
|
|
661
|
+
if (seen.has(tag))
|
|
662
|
+
continue;
|
|
663
|
+
seen.add(tag);
|
|
664
|
+
insert.run(nodeId, tag);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
/**
|
|
668
|
+
* ── item2: entity leg ── mirror a node's entities into the
|
|
669
|
+
* `entity_index` inverted table, keeping it in lockstep with the node
|
|
670
|
+
* row (the entity analogue of `writeNodeTags`). Runs inside the
|
|
671
|
+
* caller's transaction (`saveNode` / `updateNode`).
|
|
672
|
+
*
|
|
673
|
+
* Prefers the write-time canonicalized `metadata.entities` — `MemOS.store`
|
|
674
|
+
* canonicalizes with the deployment's `entityAliases`, so reusing it
|
|
675
|
+
* keeps the index consistent with query-time canonicalization. Falls
|
|
676
|
+
* back to extracting from content (built-in aliases) for rows written
|
|
677
|
+
* straight through storage.
|
|
678
|
+
*/
|
|
679
|
+
writeNodeEntities(node) {
|
|
680
|
+
const stored = node.metadata
|
|
681
|
+
?.entities;
|
|
682
|
+
const entities = Array.isArray(stored)
|
|
683
|
+
? stored
|
|
684
|
+
: extractContentEntities(node.content);
|
|
685
|
+
writeEntityIndexRows(this.db, node.id, entities);
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* Migration-safe column addition. Checks if column exists before ALTER TABLE.
|
|
689
|
+
* SQLite does not support ALTER TABLE IF NOT EXISTS, so we check pragmatically.
|
|
690
|
+
*/
|
|
691
|
+
migrateAddColumn(table, column, definition) {
|
|
692
|
+
const cols = this.db.prepare(`PRAGMA table_info(${table})`).all();
|
|
693
|
+
const exists = cols.some((c) => c.name === column);
|
|
694
|
+
if (!exists) {
|
|
695
|
+
this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
// -----------------------------------------------------------------------
|
|
699
|
+
// Nodes
|
|
700
|
+
// -----------------------------------------------------------------------
|
|
701
|
+
async saveNode(node) {
|
|
702
|
+
const stmt = this.getPreparedStatement("saveNode", `INSERT INTO nodes (id, content, summary, type, metadata, importance, created_at, updated_at, access_count, last_accessed, expires_at, tags, namespace, valid_from, valid_to, source, trust_score, pool, provenance, quarantined, quarantined_at, quarantine_reason, harness)
|
|
703
|
+
VALUES (@id, @content, @summary, @type, @metadata, @importance, @createdAt, @updatedAt, @accessCount, @lastAccessed, @expiresAt, @tags, @namespace, @validFrom, @validTo, @source, @trustScore, @pool, @provenance, @quarantined, @quarantinedAt, @quarantineReason, @harness)`);
|
|
704
|
+
// One transaction for the row INSERT plus the tag-join mirror:
|
|
705
|
+
// previously each ran in its own autocommit transaction (three commits
|
|
706
|
+
// per store), and writeNodeTags also paid two uncached prepares.
|
|
707
|
+
const tx = this.db.transaction((params) => {
|
|
708
|
+
stmt.run(params);
|
|
709
|
+
// Mirror to the join table for index-backed tag lookups.
|
|
710
|
+
this.writeNodeTags(node.id, node.tags);
|
|
711
|
+
// ── item2: entity leg ── mirror the node's entities into the
|
|
712
|
+
// entity→memories inverted index (same transaction). Prefer the
|
|
713
|
+
// write-time canonicalized `metadata.entities` (deployment aliases
|
|
714
|
+
// applied in `MemOS.store`); fall back to content extraction for
|
|
715
|
+
// rows written straight through storage.
|
|
716
|
+
this.writeNodeEntities(node);
|
|
717
|
+
});
|
|
718
|
+
tx({
|
|
719
|
+
id: node.id,
|
|
720
|
+
content: node.content,
|
|
721
|
+
summary: node.summary,
|
|
722
|
+
type: node.type,
|
|
723
|
+
metadata: JSON.stringify({
|
|
724
|
+
...node.metadata,
|
|
725
|
+
confidence: node.confidence,
|
|
726
|
+
evidenceCount: node.evidenceCount,
|
|
727
|
+
}),
|
|
728
|
+
importance: node.importance,
|
|
729
|
+
createdAt: node.createdAt,
|
|
730
|
+
updatedAt: node.updatedAt,
|
|
731
|
+
accessCount: node.accessCount,
|
|
732
|
+
lastAccessed: node.lastAccessed,
|
|
733
|
+
expiresAt: node.expiresAt,
|
|
734
|
+
tags: JSON.stringify(node.tags),
|
|
735
|
+
namespace: node.namespace,
|
|
736
|
+
validFrom: node.validFrom,
|
|
737
|
+
validTo: node.validTo,
|
|
738
|
+
source: node.source,
|
|
739
|
+
trustScore: node.trustScore,
|
|
740
|
+
pool: node.pool ?? "event",
|
|
741
|
+
// Provenance-trust columns: default at the storage boundary so
|
|
742
|
+
// node literals built before the layer (or by custom adapters)
|
|
743
|
+
// still save — the columns stay NOT NULL in the schema.
|
|
744
|
+
provenance: node.provenance ?? "user",
|
|
745
|
+
quarantined: node.quarantined ? 1 : 0,
|
|
746
|
+
quarantinedAt: node.quarantinedAt ?? null,
|
|
747
|
+
quarantineReason: node.quarantineReason ?? null,
|
|
748
|
+
// Harness attribution: same boundary defaulting — a missing tag
|
|
749
|
+
// (legacy literals, custom adapters) stores as "unknown".
|
|
750
|
+
harness: node.harness ?? "unknown",
|
|
751
|
+
});
|
|
752
|
+
return node;
|
|
753
|
+
}
|
|
754
|
+
async getNode(id) {
|
|
755
|
+
const stmt = this.getPreparedStatement("getNode", "SELECT * FROM nodes WHERE id = ?");
|
|
756
|
+
const row = stmt.get(id);
|
|
757
|
+
if (!row)
|
|
758
|
+
return null;
|
|
759
|
+
// Buffer the access stats instead of writing immediately. Coalesces
|
|
760
|
+
// bursts of reads (e.g. a search that re-fetches the same node) into
|
|
761
|
+
// a single UPDATE per id.
|
|
762
|
+
this.bufferAccess(id, Date.now());
|
|
763
|
+
return this.rowToNode(row);
|
|
764
|
+
}
|
|
765
|
+
/**
|
|
766
|
+
* Record a read access. Routes through the same debounced
|
|
767
|
+
* `accessBuffer` as `getNode`, so a search that touches the top N
|
|
768
|
+
* results costs one coalesced UPDATE per id instead of a
|
|
769
|
+
* read-modify-write per hit — and, critically, does NOT stamp
|
|
770
|
+
* `updated_at`.
|
|
771
|
+
*/
|
|
772
|
+
recordAccess(id) {
|
|
773
|
+
this.bufferAccess(id, Date.now());
|
|
774
|
+
}
|
|
775
|
+
bufferAccess(id, now) {
|
|
776
|
+
const existing = this.accessBuffer.get(id);
|
|
777
|
+
if (existing) {
|
|
778
|
+
existing.count += 1;
|
|
779
|
+
existing.lastAccessed = now;
|
|
780
|
+
}
|
|
781
|
+
else {
|
|
782
|
+
this.accessBuffer.set(id, { count: 1, lastAccessed: now });
|
|
783
|
+
}
|
|
784
|
+
this.scheduleAccessFlush();
|
|
785
|
+
}
|
|
786
|
+
scheduleAccessFlush() {
|
|
787
|
+
if (this.accessFlushTimer !== null)
|
|
788
|
+
return;
|
|
789
|
+
this.accessFlushTimer = setTimeout(() => {
|
|
790
|
+
this.accessFlushTimer = null;
|
|
791
|
+
try {
|
|
792
|
+
this.flushAccessCounts();
|
|
793
|
+
}
|
|
794
|
+
catch {
|
|
795
|
+
// Flush errors are non-fatal; we'll retry on the next buffer hit.
|
|
796
|
+
}
|
|
797
|
+
}, this.accessFlushIntervalMs);
|
|
798
|
+
// Don't keep the process alive for a flush.
|
|
799
|
+
if (this.accessFlushTimer &&
|
|
800
|
+
typeof this.accessFlushTimer.unref === "function") {
|
|
801
|
+
this.accessFlushTimer.unref();
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
/** Drain the buffered access counters to SQLite. */
|
|
805
|
+
flushAccessCounts() {
|
|
806
|
+
if (this.accessBuffer.size === 0)
|
|
807
|
+
return;
|
|
808
|
+
const updates = this.db.prepare("UPDATE nodes SET access_count = access_count + ?, last_accessed = MAX(last_accessed, ?) WHERE id = ?");
|
|
809
|
+
const tx = this.db.transaction(() => {
|
|
810
|
+
for (const [id, { count, lastAccessed }] of this.accessBuffer) {
|
|
811
|
+
updates.run(count, lastAccessed, id);
|
|
812
|
+
}
|
|
813
|
+
this.accessBuffer.clear();
|
|
814
|
+
});
|
|
815
|
+
tx();
|
|
816
|
+
}
|
|
817
|
+
async peekNode(id) {
|
|
818
|
+
const stmt = this.getPreparedStatement("peekNode", "SELECT * FROM nodes WHERE id = ?");
|
|
819
|
+
const row = stmt.get(id);
|
|
820
|
+
return row ? this.rowToNode(row) : null;
|
|
821
|
+
}
|
|
822
|
+
async evictLeastImportant() {
|
|
823
|
+
// Pick the least important + oldest-accessed node in a single SQL
|
|
824
|
+
// statement. Avoids loading the whole table into memory.
|
|
825
|
+
const row = this.db
|
|
826
|
+
.prepare("SELECT id FROM nodes ORDER BY importance ASC, last_accessed ASC, created_at ASC LIMIT 1")
|
|
827
|
+
.get();
|
|
828
|
+
if (!row)
|
|
829
|
+
return null;
|
|
830
|
+
const id = row.id;
|
|
831
|
+
const result = this.db.prepare("DELETE FROM nodes WHERE id = ?").run(id);
|
|
832
|
+
if (result.changes > 0)
|
|
833
|
+
return id;
|
|
834
|
+
return null;
|
|
835
|
+
}
|
|
836
|
+
async updateNode(id, input) {
|
|
837
|
+
const existing = await this.getNode(id);
|
|
838
|
+
if (!existing)
|
|
839
|
+
return null;
|
|
840
|
+
const updated = {
|
|
841
|
+
...existing,
|
|
842
|
+
...input,
|
|
843
|
+
metadata: input.metadata ?? existing.metadata,
|
|
844
|
+
tags: input.tags ?? existing.tags,
|
|
845
|
+
namespace: input.namespace ?? existing.namespace,
|
|
846
|
+
validFrom: input.validFrom !== undefined ? input.validFrom : existing.validFrom,
|
|
847
|
+
validTo: input.validTo !== undefined ? input.validTo : existing.validTo,
|
|
848
|
+
source: input.source ?? existing.source,
|
|
849
|
+
trustScore: input.trustScore !== undefined ? input.trustScore : existing.trustScore,
|
|
850
|
+
provenance: input.provenance ?? existing.provenance,
|
|
851
|
+
quarantined: input.quarantined !== undefined
|
|
852
|
+
? input.quarantined
|
|
853
|
+
: existing.quarantined,
|
|
854
|
+
quarantinedAt: input.quarantinedAt !== undefined
|
|
855
|
+
? input.quarantinedAt
|
|
856
|
+
: existing.quarantinedAt,
|
|
857
|
+
quarantineReason: input.quarantineReason !== undefined
|
|
858
|
+
? input.quarantineReason
|
|
859
|
+
: existing.quarantineReason,
|
|
860
|
+
updatedAt: Date.now(),
|
|
861
|
+
};
|
|
862
|
+
const updateStmt = this.getPreparedStatement("updateNode", `UPDATE nodes SET content = @content, summary = @summary, type = @type,
|
|
863
|
+
metadata = @metadata, importance = @importance, updated_at = @updatedAt,
|
|
864
|
+
tags = @tags, namespace = @namespace,
|
|
865
|
+
valid_from = @validFrom, valid_to = @validTo, source = @source, trust_score = @trustScore,
|
|
866
|
+
provenance = @provenance, quarantined = @quarantined, quarantined_at = @quarantinedAt, quarantine_reason = @quarantineReason
|
|
867
|
+
WHERE id = @id`);
|
|
868
|
+
const params = {
|
|
869
|
+
id: updated.id,
|
|
870
|
+
content: updated.content,
|
|
871
|
+
summary: updated.summary,
|
|
872
|
+
type: updated.type,
|
|
873
|
+
metadata: JSON.stringify({
|
|
874
|
+
...updated.metadata,
|
|
875
|
+
// Keep the node-level evidence state in sync with the persisted
|
|
876
|
+
// metadata blob — rowToNode reads confidence/evidenceCount back
|
|
877
|
+
// from here, so omitting this silently drops reinforcement.
|
|
878
|
+
confidence: updated.confidence,
|
|
879
|
+
evidenceCount: updated.evidenceCount,
|
|
880
|
+
}),
|
|
881
|
+
importance: updated.importance,
|
|
882
|
+
updatedAt: updated.updatedAt,
|
|
883
|
+
tags: JSON.stringify(updated.tags),
|
|
884
|
+
namespace: updated.namespace,
|
|
885
|
+
validFrom: updated.validFrom,
|
|
886
|
+
validTo: updated.validTo,
|
|
887
|
+
source: updated.source,
|
|
888
|
+
trustScore: updated.trustScore,
|
|
889
|
+
provenance: updated.provenance,
|
|
890
|
+
quarantined: updated.quarantined ? 1 : 0,
|
|
891
|
+
quarantinedAt: updated.quarantinedAt,
|
|
892
|
+
quarantineReason: updated.quarantineReason,
|
|
893
|
+
};
|
|
894
|
+
// Keep the tag join table in sync when the tag set changed — in the
|
|
895
|
+
// same transaction as the row UPDATE (previously two commits).
|
|
896
|
+
const syncTags = input.tags !== undefined;
|
|
897
|
+
// ── item2: entity leg ── the entity signal may have changed when the
|
|
898
|
+
// content or metadata was updated; re-sync the inverted index rows.
|
|
899
|
+
const syncEntities = input.content !== undefined || input.metadata !== undefined;
|
|
900
|
+
const tx = this.db.transaction(() => {
|
|
901
|
+
updateStmt.run(params);
|
|
902
|
+
if (syncTags)
|
|
903
|
+
this.writeNodeTags(updated.id, updated.tags);
|
|
904
|
+
if (syncEntities)
|
|
905
|
+
this.writeNodeEntities(updated);
|
|
906
|
+
});
|
|
907
|
+
tx();
|
|
908
|
+
return updated;
|
|
909
|
+
}
|
|
910
|
+
async deleteNode(id) {
|
|
911
|
+
// The `image_embeddings` row goes with the node via ON DELETE
|
|
912
|
+
// CASCADE; the in-memory cache entry has to go explicitly (a node
|
|
913
|
+
// deleted between two queries must not be served from cache).
|
|
914
|
+
this.vectorCache.delete(id);
|
|
915
|
+
const result = this.db.prepare("DELETE FROM nodes WHERE id = ?").run(id);
|
|
916
|
+
return result.changes > 0;
|
|
917
|
+
}
|
|
918
|
+
// -----------------------------------------------------------------------
|
|
919
|
+
// Tamper-evident mutation log
|
|
920
|
+
// -----------------------------------------------------------------------
|
|
921
|
+
async appendTamperEntry(entry) {
|
|
922
|
+
// Atomic chain-position assignment: tip lookup, seq allocation,
|
|
923
|
+
// entry-hash computation, and insert happen in ONE transaction, so
|
|
924
|
+
// concurrent writers can never fork the chain with the same prevHash.
|
|
925
|
+
// BEGIN IMMEDIATE takes the write lock up front, which also keeps
|
|
926
|
+
// two MemOS instances on the same DB file from interleaving.
|
|
927
|
+
this.db.prepare("BEGIN IMMEDIATE").run();
|
|
928
|
+
try {
|
|
929
|
+
const tip = this.db
|
|
930
|
+
.prepare(`SELECT entry_hash FROM tamper_log ORDER BY seq DESC LIMIT 1`)
|
|
931
|
+
.get();
|
|
932
|
+
const prevHash = tip?.entry_hash ?? GENESIS_PREV;
|
|
933
|
+
// AUTOINCREMENT assigns seq; lastInsertRowid gives it back inside
|
|
934
|
+
// the same transaction, so the hash commits the true position.
|
|
935
|
+
const seq = Number(this.db
|
|
936
|
+
.prepare(`INSERT INTO tamper_log (ts, op, node_id, node_hash, prev_hash, entry_hash)
|
|
937
|
+
VALUES (?, ?, ?, ?, ?, '')`)
|
|
938
|
+
.run(entry.ts, entry.op, entry.nodeId, entry.nodeHash, prevHash)
|
|
939
|
+
.lastInsertRowid);
|
|
940
|
+
const entryHash = tamperEntryHash({
|
|
941
|
+
seq,
|
|
942
|
+
ts: entry.ts,
|
|
943
|
+
op: entry.op,
|
|
944
|
+
nodeId: entry.nodeId,
|
|
945
|
+
nodeHash: entry.nodeHash,
|
|
946
|
+
prevHash,
|
|
947
|
+
});
|
|
948
|
+
this.db
|
|
949
|
+
.prepare(`UPDATE tamper_log SET entry_hash = ? WHERE seq = ?`)
|
|
950
|
+
.run(entryHash, seq);
|
|
951
|
+
this.db.prepare("COMMIT").run();
|
|
952
|
+
return {
|
|
953
|
+
seq,
|
|
954
|
+
ts: entry.ts,
|
|
955
|
+
op: entry.op,
|
|
956
|
+
nodeId: entry.nodeId,
|
|
957
|
+
nodeHash: entry.nodeHash,
|
|
958
|
+
prevHash,
|
|
959
|
+
entryHash,
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
catch (err) {
|
|
963
|
+
this.db.prepare("ROLLBACK").run();
|
|
964
|
+
throw err;
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
async readTamperLog() {
|
|
968
|
+
const rows = this.db
|
|
969
|
+
.prepare(`SELECT seq, ts, op, node_id, node_hash, prev_hash, entry_hash
|
|
970
|
+
FROM tamper_log ORDER BY seq ASC`)
|
|
971
|
+
.all();
|
|
972
|
+
return rows.map((r) => ({
|
|
973
|
+
seq: r.seq,
|
|
974
|
+
ts: r.ts,
|
|
975
|
+
op: r.op,
|
|
976
|
+
nodeId: r.node_id,
|
|
977
|
+
nodeHash: r.node_hash,
|
|
978
|
+
prevHash: r.prev_hash,
|
|
979
|
+
entryHash: r.entry_hash,
|
|
980
|
+
}));
|
|
981
|
+
}
|
|
982
|
+
// -----------------------------------------------------------------------
|
|
983
|
+
// Edges
|
|
984
|
+
// -----------------------------------------------------------------------
|
|
985
|
+
async saveEdge(edge) {
|
|
986
|
+
const stmt = this.getPreparedStatement("saveEdge", `INSERT INTO edges (id, source_id, target_id, relation, weight, metadata, created_at, valid_from, valid_to)
|
|
987
|
+
VALUES (@id, @sourceId, @targetId, @relation, @weight, @metadata, @createdAt, @validFrom, @validTo)`);
|
|
988
|
+
stmt.run({
|
|
989
|
+
id: edge.id,
|
|
990
|
+
sourceId: edge.sourceId,
|
|
991
|
+
targetId: edge.targetId,
|
|
992
|
+
relation: edge.relation,
|
|
993
|
+
weight: edge.weight,
|
|
994
|
+
metadata: JSON.stringify(edge.metadata),
|
|
995
|
+
createdAt: edge.createdAt,
|
|
996
|
+
validFrom: edge.validFrom ?? null,
|
|
997
|
+
validTo: edge.validTo ?? null,
|
|
998
|
+
});
|
|
999
|
+
return edge;
|
|
1000
|
+
}
|
|
1001
|
+
/**
|
|
1002
|
+
* Batch-insert multiple edges in a single transaction.
|
|
1003
|
+
* Uses a cached prepared statement — ~10x faster than calling
|
|
1004
|
+
* saveEdge() in a loop for 10+ edges.
|
|
1005
|
+
*/
|
|
1006
|
+
async saveEdgesBatch(edges) {
|
|
1007
|
+
if (edges.length === 0)
|
|
1008
|
+
return [];
|
|
1009
|
+
const stmt = this.getPreparedStatement("saveEdge", `INSERT INTO edges (id, source_id, target_id, relation, weight, metadata, created_at, valid_from, valid_to)
|
|
1010
|
+
VALUES (@id, @sourceId, @targetId, @relation, @weight, @metadata, @createdAt, @validFrom, @validTo)`);
|
|
1011
|
+
const insert = (edge) => stmt.run({
|
|
1012
|
+
id: edge.id,
|
|
1013
|
+
sourceId: edge.sourceId,
|
|
1014
|
+
targetId: edge.targetId,
|
|
1015
|
+
relation: edge.relation,
|
|
1016
|
+
weight: edge.weight,
|
|
1017
|
+
metadata: JSON.stringify(edge.metadata),
|
|
1018
|
+
createdAt: edge.createdAt,
|
|
1019
|
+
validFrom: edge.validFrom ?? null,
|
|
1020
|
+
validTo: edge.validTo ?? null,
|
|
1021
|
+
});
|
|
1022
|
+
const tx = this.db.transaction(() => {
|
|
1023
|
+
for (const edge of edges)
|
|
1024
|
+
insert(edge);
|
|
1025
|
+
});
|
|
1026
|
+
tx();
|
|
1027
|
+
return edges;
|
|
1028
|
+
}
|
|
1029
|
+
/**
|
|
1030
|
+
* Prepared statement cache. better-sqlite3 caches at the C++ level,
|
|
1031
|
+
* but the JS Statement wrapper is still recreated on each prepare()
|
|
1032
|
+
* call. Caching the JS object avoids that overhead on hot paths.
|
|
1033
|
+
*/
|
|
1034
|
+
stmtCache = new Map();
|
|
1035
|
+
getPreparedStatement(key, sql) {
|
|
1036
|
+
let stmt = this.stmtCache.get(key);
|
|
1037
|
+
if (!stmt) {
|
|
1038
|
+
stmt = this.db.prepare(sql);
|
|
1039
|
+
this.stmtCache.set(key, stmt);
|
|
1040
|
+
}
|
|
1041
|
+
return stmt;
|
|
1042
|
+
}
|
|
1043
|
+
async getEdge(id) {
|
|
1044
|
+
const row = this.db.prepare("SELECT * FROM edges WHERE id = ?").get(id);
|
|
1045
|
+
if (!row)
|
|
1046
|
+
return null;
|
|
1047
|
+
return this.rowToEdge(row);
|
|
1048
|
+
}
|
|
1049
|
+
async deleteEdge(id) {
|
|
1050
|
+
const result = this.db.prepare("DELETE FROM edges WHERE id = ?").run(id);
|
|
1051
|
+
return result.changes > 0;
|
|
1052
|
+
}
|
|
1053
|
+
/**
|
|
1054
|
+
* Close (invalidate) the currently-valid edges incident to a node by
|
|
1055
|
+
* stamping `valid_to`. Rows are updated, never deleted — as-of reads at
|
|
1056
|
+
* earlier timestamps still return them. Returns the number of edges closed.
|
|
1057
|
+
*/
|
|
1058
|
+
async closeEdgesForNode(nodeId, validTo) {
|
|
1059
|
+
const result = this.db
|
|
1060
|
+
.prepare(`UPDATE edges SET valid_to = ?
|
|
1061
|
+
WHERE (source_id = ? OR target_id = ?)
|
|
1062
|
+
AND (valid_to IS NULL OR valid_to > ?)`)
|
|
1063
|
+
.run(validTo, nodeId, nodeId, validTo);
|
|
1064
|
+
return result.changes;
|
|
1065
|
+
}
|
|
1066
|
+
// -----------------------------------------------------------------------
|
|
1067
|
+
// Queries
|
|
1068
|
+
// -----------------------------------------------------------------------
|
|
1069
|
+
async queryNodes(filter = {}) {
|
|
1070
|
+
let rows;
|
|
1071
|
+
// Build the extra filters that apply to both FTS and structured paths.
|
|
1072
|
+
const extraConds = [];
|
|
1073
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
1074
|
+
const extraParams = [];
|
|
1075
|
+
if (filter.source) {
|
|
1076
|
+
extraConds.push("n.source = ?");
|
|
1077
|
+
extraParams.push(filter.source);
|
|
1078
|
+
}
|
|
1079
|
+
// Scoped-space namespace constraint: exact (`namespace`), multi
|
|
1080
|
+
// (`namespaces`), or hierarchical (`namespacePrefix`). When
|
|
1081
|
+
// `includeSharedScope` is set, the shared `default` namespace is
|
|
1082
|
+
// OR'd in — a scoped read always sees shared memories.
|
|
1083
|
+
const nsCond = this.buildNamespaceCondition(filter, "n");
|
|
1084
|
+
if (nsCond) {
|
|
1085
|
+
extraConds.push(nsCond.sql);
|
|
1086
|
+
extraParams.push(...nsCond.params);
|
|
1087
|
+
}
|
|
1088
|
+
if (filter.minTrustScore !== undefined) {
|
|
1089
|
+
extraConds.push("n.trust_score >= ?");
|
|
1090
|
+
extraParams.push(filter.minTrustScore);
|
|
1091
|
+
}
|
|
1092
|
+
if (filter.provenance) {
|
|
1093
|
+
extraConds.push("n.provenance = ?");
|
|
1094
|
+
extraParams.push(filter.provenance);
|
|
1095
|
+
}
|
|
1096
|
+
// Harness attribution: "all" (or unset) is the behavior-preserving
|
|
1097
|
+
// default — the filter applies to both the FTS and the structured
|
|
1098
|
+
// path via extraConds.
|
|
1099
|
+
if (filter.harness && filter.harness !== "all") {
|
|
1100
|
+
extraConds.push("n.harness = ?");
|
|
1101
|
+
extraParams.push(filter.harness);
|
|
1102
|
+
}
|
|
1103
|
+
// Provenance-trust: quarantine visibility. Quarantined memories are
|
|
1104
|
+
// excluded from recall by default (the write-gate guarantee);
|
|
1105
|
+
// `includeQuarantined: true` shows everything, `quarantinedOnly: true`
|
|
1106
|
+
// shows only quarantined rows (the review queue).
|
|
1107
|
+
if (filter.quarantinedOnly === true) {
|
|
1108
|
+
extraConds.push("n.quarantined = 1");
|
|
1109
|
+
}
|
|
1110
|
+
else if (filter.includeQuarantined !== true) {
|
|
1111
|
+
extraConds.push("n.quarantined = 0");
|
|
1112
|
+
}
|
|
1113
|
+
// Temporal: exclude historical (validTo in the past) unless
|
|
1114
|
+
// includeHistorical is explicitly true. Also support validAt.
|
|
1115
|
+
const now = Date.now();
|
|
1116
|
+
if (filter.validAt !== undefined) {
|
|
1117
|
+
extraConds.push("(n.valid_from IS NULL OR n.valid_from <= ?)");
|
|
1118
|
+
extraParams.push(filter.validAt);
|
|
1119
|
+
extraConds.push("(n.valid_to IS NULL OR n.valid_to >= ?)");
|
|
1120
|
+
extraParams.push(filter.validAt);
|
|
1121
|
+
}
|
|
1122
|
+
else if (filter.includeHistorical !== true) {
|
|
1123
|
+
// Default and explicit false: exclude historical memories.
|
|
1124
|
+
// Strict `>` (not `>=`): a memory superseded in the same millisecond
|
|
1125
|
+
// as this query (validTo === now) is already historical and must be
|
|
1126
|
+
// excluded. `>=` would keep it. validAt below intentionally stays `>=`.
|
|
1127
|
+
extraConds.push("(n.valid_to IS NULL OR n.valid_to > ?)");
|
|
1128
|
+
extraParams.push(now);
|
|
1129
|
+
}
|
|
1130
|
+
// Type and importance filters apply to the FTS path too — without
|
|
1131
|
+
// these, `search({ query, type })` leaked keyword hits of any type
|
|
1132
|
+
// into hybrid results (the structured branch already filtered them).
|
|
1133
|
+
if (filter.type) {
|
|
1134
|
+
extraConds.push("n.type = ?");
|
|
1135
|
+
extraParams.push(filter.type);
|
|
1136
|
+
}
|
|
1137
|
+
if (filter.minImportance !== undefined) {
|
|
1138
|
+
extraConds.push("n.importance >= ?");
|
|
1139
|
+
extraParams.push(filter.minImportance);
|
|
1140
|
+
}
|
|
1141
|
+
if (filter.maxImportance !== undefined) {
|
|
1142
|
+
extraConds.push("n.importance <= ?");
|
|
1143
|
+
extraParams.push(filter.maxImportance);
|
|
1144
|
+
}
|
|
1145
|
+
// Multi-granularity pool filter (OR within a list). Pre-pool
|
|
1146
|
+
// memories carry the 'event' default from the migration.
|
|
1147
|
+
if (filter.pool !== undefined) {
|
|
1148
|
+
const pools = Array.isArray(filter.pool) ? filter.pool : [filter.pool];
|
|
1149
|
+
if (pools.length > 0) {
|
|
1150
|
+
extraConds.push(`n.pool IN (${pools.map(() => "?").join(", ")})`);
|
|
1151
|
+
extraParams.push(...pools);
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
if (filter.query) {
|
|
1155
|
+
// Full-text search via FTS5. The query is tokenized into
|
|
1156
|
+
// individually quoted terms (see `buildFtsTerms`) instead of
|
|
1157
|
+
// being wrapped as a single phrase: a phrase match requires the
|
|
1158
|
+
// exact question text to appear verbatim in stored content,
|
|
1159
|
+
// which never happens for natural-language queries — so phrase
|
|
1160
|
+
// mode silently returned zero keyword results for every query.
|
|
1161
|
+
//
|
|
1162
|
+
// Matching strategy: AND first (every term must appear — keeps
|
|
1163
|
+
// multi-word queries precise), then fall back to OR when the
|
|
1164
|
+
// AND query matches nothing (any term may appear — keeps
|
|
1165
|
+
// paraphrased queries from returning empty). bm25 ranking
|
|
1166
|
+
// (`ORDER BY rank`) puts documents matching more terms first.
|
|
1167
|
+
const ftsTerms = buildFtsTerms(filter.query);
|
|
1168
|
+
const tagFilter = this.buildTagFilter(filter.tags, "n");
|
|
1169
|
+
const metadata = this.buildMetadataFilter(filter.metadata, "n");
|
|
1170
|
+
const whereParts = [
|
|
1171
|
+
tagFilter,
|
|
1172
|
+
...metadata.conditions,
|
|
1173
|
+
...extraConds,
|
|
1174
|
+
].filter(Boolean);
|
|
1175
|
+
const whereExtra = whereParts.length
|
|
1176
|
+
? ` AND ${whereParts.join(" AND ")}`
|
|
1177
|
+
: "";
|
|
1178
|
+
const ftsSql = `SELECT n.*, rank
|
|
1179
|
+
FROM nodes n
|
|
1180
|
+
INNER JOIN nodes_fts fts ON fts.rowid = n.rowid
|
|
1181
|
+
WHERE nodes_fts MATCH ?${whereExtra}
|
|
1182
|
+
ORDER BY rank
|
|
1183
|
+
LIMIT ? OFFSET ?`;
|
|
1184
|
+
// Statement cache keyed by the built SQL text: `whereExtra` /
|
|
1185
|
+
// `namespaceFilter` vary with the caller's filter combination, and
|
|
1186
|
+
// better-sqlite3 charges ~20-60µs of parse/plan time per
|
|
1187
|
+
// `db.prepare()` call. Reusing the prepared object for repeated
|
|
1188
|
+
// identical shapes removes that cost from the hot search path.
|
|
1189
|
+
const runFts = (ftsQuery) => {
|
|
1190
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
1191
|
+
const params = [ftsQuery];
|
|
1192
|
+
if (filter.tags && filter.tags.length > 0) {
|
|
1193
|
+
for (const tag of filter.tags) {
|
|
1194
|
+
params.push(tag);
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
params.push(...metadata.params);
|
|
1198
|
+
params.push(...extraParams);
|
|
1199
|
+
params.push(filter.limit ?? 50, filter.offset ?? 0);
|
|
1200
|
+
return this.getPreparedStatement(`fts::${ftsSql}`, ftsSql).all(...params);
|
|
1201
|
+
};
|
|
1202
|
+
const operator = filter.ftsOperator ?? "AUTO";
|
|
1203
|
+
const andRows = runFts(ftsTerms.join(" AND "));
|
|
1204
|
+
if (operator === "OR") {
|
|
1205
|
+
rows = runFts(ftsTerms.join(" OR "));
|
|
1206
|
+
}
|
|
1207
|
+
else if (operator === "AUTO" &&
|
|
1208
|
+
andRows.length === 0 &&
|
|
1209
|
+
ftsTerms.length > 1) {
|
|
1210
|
+
// AUTO (default): precision AND first; only when it matches
|
|
1211
|
+
// NOTHING relax to OR so partial-term matches still surface
|
|
1212
|
+
// instead of an empty result set. `ftsOperator: "OR"` forces the
|
|
1213
|
+
// recall-first query (benchmark ablation for multi-evidence work).
|
|
1214
|
+
rows = runFts(ftsTerms.join(" OR "));
|
|
1215
|
+
}
|
|
1216
|
+
else {
|
|
1217
|
+
rows = andRows;
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
else {
|
|
1221
|
+
// Structured query
|
|
1222
|
+
const conditions = [];
|
|
1223
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
1224
|
+
const params = [];
|
|
1225
|
+
if (filter.type) {
|
|
1226
|
+
conditions.push("type = ?");
|
|
1227
|
+
params.push(filter.type);
|
|
1228
|
+
}
|
|
1229
|
+
if (filter.minImportance !== undefined) {
|
|
1230
|
+
conditions.push("importance >= ?");
|
|
1231
|
+
params.push(filter.minImportance);
|
|
1232
|
+
}
|
|
1233
|
+
if (filter.maxImportance !== undefined) {
|
|
1234
|
+
conditions.push("importance <= ?");
|
|
1235
|
+
params.push(filter.maxImportance);
|
|
1236
|
+
}
|
|
1237
|
+
// Scoped-space namespace constraint (same semantics as the FTS
|
|
1238
|
+
// path: exact / multi / hierarchical, optional shared-union).
|
|
1239
|
+
const nsCond = this.buildNamespaceCondition(filter, "");
|
|
1240
|
+
if (nsCond) {
|
|
1241
|
+
conditions.push(nsCond.sql);
|
|
1242
|
+
params.push(...nsCond.params);
|
|
1243
|
+
}
|
|
1244
|
+
if (filter.source) {
|
|
1245
|
+
conditions.push("source = ?");
|
|
1246
|
+
params.push(filter.source);
|
|
1247
|
+
}
|
|
1248
|
+
if (filter.provenance) {
|
|
1249
|
+
conditions.push("provenance = ?");
|
|
1250
|
+
params.push(filter.provenance);
|
|
1251
|
+
}
|
|
1252
|
+
// Harness attribution (structured path; the FTS path applies the
|
|
1253
|
+
// same rule via extraConds above).
|
|
1254
|
+
if (filter.harness && filter.harness !== "all") {
|
|
1255
|
+
conditions.push("harness = ?");
|
|
1256
|
+
params.push(filter.harness);
|
|
1257
|
+
}
|
|
1258
|
+
// Provenance-trust: quarantine visibility (see the FTS branch above
|
|
1259
|
+
// for the rationale — excluded from recall by default).
|
|
1260
|
+
if (filter.quarantinedOnly === true) {
|
|
1261
|
+
conditions.push("quarantined = 1");
|
|
1262
|
+
}
|
|
1263
|
+
else if (filter.includeQuarantined !== true) {
|
|
1264
|
+
conditions.push("quarantined = 0");
|
|
1265
|
+
}
|
|
1266
|
+
if (filter.minTrustScore !== undefined) {
|
|
1267
|
+
conditions.push("trust_score >= ?");
|
|
1268
|
+
params.push(filter.minTrustScore);
|
|
1269
|
+
}
|
|
1270
|
+
// Temporal filters (structured path)
|
|
1271
|
+
if (filter.validAt !== undefined) {
|
|
1272
|
+
conditions.push("(valid_from IS NULL OR valid_from <= ?)");
|
|
1273
|
+
params.push(filter.validAt);
|
|
1274
|
+
conditions.push("(valid_to IS NULL OR valid_to >= ?)");
|
|
1275
|
+
params.push(filter.validAt);
|
|
1276
|
+
}
|
|
1277
|
+
else if (filter.includeHistorical !== true) {
|
|
1278
|
+
// Strict `>`: a memory superseded in the same millisecond as the
|
|
1279
|
+
// query (validTo === now) is historical and must be excluded.
|
|
1280
|
+
conditions.push("(valid_to IS NULL OR valid_to > ?)");
|
|
1281
|
+
params.push(now);
|
|
1282
|
+
}
|
|
1283
|
+
const metadata = this.buildMetadataFilter(filter.metadata);
|
|
1284
|
+
conditions.push(...metadata.conditions);
|
|
1285
|
+
params.push(...metadata.params);
|
|
1286
|
+
// Tag filter: node must contain ALL specified tags
|
|
1287
|
+
if (filter.tags && filter.tags.length > 0) {
|
|
1288
|
+
for (const tag of filter.tags) {
|
|
1289
|
+
conditions.push("tags LIKE ?");
|
|
1290
|
+
params.push(`%${JSON.stringify(tag)}%`);
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
const where = conditions.length
|
|
1294
|
+
? "WHERE " + conditions.join(" AND ")
|
|
1295
|
+
: "";
|
|
1296
|
+
const sortField = this.mapSortField(filter.sortBy);
|
|
1297
|
+
// Runtime allowlist: filter.sortOrder is typed "asc" | "desc" but
|
|
1298
|
+
// arrives as unchecked runtime data (e.g. MCP JSON input), and it is
|
|
1299
|
+
// interpolated into the SQL below — so it must be validated here.
|
|
1300
|
+
const order = filter.sortOrder === "asc" ? "asc" : "desc";
|
|
1301
|
+
// Cache the prepared statement by its built SQL — the shape varies
|
|
1302
|
+
// with the filter combination, so the key must include the text.
|
|
1303
|
+
const structuredSql = `SELECT * FROM nodes ${where} ORDER BY ${sortField} ${order} LIMIT ? OFFSET ?`;
|
|
1304
|
+
rows = this.getPreparedStatement(`structured::${structuredSql}`, structuredSql).all(...params, filter.limit ?? 50, filter.offset ?? 0);
|
|
1305
|
+
}
|
|
1306
|
+
return rows.map((row) => ({
|
|
1307
|
+
node: this.rowToNode(row),
|
|
1308
|
+
score: typeof row.rank === "number" ? row.rank : 1,
|
|
1309
|
+
}));
|
|
1310
|
+
}
|
|
1311
|
+
async saveEmbedding(nodeId, vector, model) {
|
|
1312
|
+
const normalized = this.serializeEmbedding(vector);
|
|
1313
|
+
this.db
|
|
1314
|
+
.prepare(`INSERT INTO embeddings (node_id, vector, dimensions, model, updated_at)
|
|
1315
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1316
|
+
ON CONFLICT(node_id) DO UPDATE SET
|
|
1317
|
+
vector = excluded.vector,
|
|
1318
|
+
dimensions = excluded.dimensions,
|
|
1319
|
+
model = excluded.model,
|
|
1320
|
+
updated_at = excluded.updated_at`)
|
|
1321
|
+
.run(nodeId, normalized, vector.length, model, Date.now());
|
|
1322
|
+
this.vectorCacheSet(nodeId, this.parseEmbeddingF32(normalized), "text");
|
|
1323
|
+
}
|
|
1324
|
+
async getEmbeddingInfo(nodeId) {
|
|
1325
|
+
const row = this.getPreparedStatement("getEmbeddingInfo", "SELECT model, dimensions, updated_at FROM embeddings WHERE node_id = ?").get(nodeId);
|
|
1326
|
+
if (!row)
|
|
1327
|
+
return null;
|
|
1328
|
+
return {
|
|
1329
|
+
model: row.model,
|
|
1330
|
+
dimensions: row.dimensions,
|
|
1331
|
+
updatedAt: row.updated_at,
|
|
1332
|
+
};
|
|
1333
|
+
}
|
|
1334
|
+
/**
|
|
1335
|
+
* Bulk-fetch stored embedding vectors for the given node ids —
|
|
1336
|
+
* the write-time contradiction scan's bounded neighbor lookup.
|
|
1337
|
+
* Cache hits are zero-cost; misses are fetched in chunks of 500
|
|
1338
|
+
* (SQLite bound-parameter limits). Returns only ids with a stored
|
|
1339
|
+
* vector.
|
|
1340
|
+
*/
|
|
1341
|
+
async getEmbeddingVectors(nodeIds) {
|
|
1342
|
+
const result = new Map();
|
|
1343
|
+
const missing = [];
|
|
1344
|
+
for (const id of nodeIds) {
|
|
1345
|
+
const cached = this.vectorCacheGet(id, "text");
|
|
1346
|
+
if (cached) {
|
|
1347
|
+
result.set(id, Array.from(cached.vector));
|
|
1348
|
+
}
|
|
1349
|
+
else if (!missing.includes(id)) {
|
|
1350
|
+
missing.push(id);
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
for (let start = 0; start < missing.length; start += 500) {
|
|
1354
|
+
const chunk = missing.slice(start, start + 500);
|
|
1355
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1356
|
+
const rows = this.getPreparedStatement(`getEmbeddingVectors::${chunk.length}`, `SELECT node_id, vector FROM embeddings WHERE node_id IN (${placeholders})`).all(...chunk);
|
|
1357
|
+
for (const row of rows) {
|
|
1358
|
+
if (Buffer.isBuffer(row.vector) || row.vector instanceof Uint8Array) {
|
|
1359
|
+
this.vectorCacheSet(row.node_id, this.parseEmbeddingF32(row.vector), "text");
|
|
1360
|
+
result.set(row.node_id, this.parseEmbedding(row.vector));
|
|
1361
|
+
}
|
|
1362
|
+
else {
|
|
1363
|
+
const parsed = this.parseEmbedding(row.vector);
|
|
1364
|
+
if (parsed.length > 0)
|
|
1365
|
+
result.set(row.node_id, parsed);
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
return result;
|
|
1370
|
+
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Record a contradiction pair. Node ids are canonicalized (sorted)
|
|
1373
|
+
* so the UNIQUE(node_a, node_b) constraint dedups reversed pairs;
|
|
1374
|
+
* INSERT OR IGNORE makes repeat detection idempotent.
|
|
1375
|
+
*/
|
|
1376
|
+
async addContradiction(nodeA, nodeB, status = "unresolved") {
|
|
1377
|
+
const [a, b] = nodeA < nodeB ? [nodeA, nodeB] : [nodeB, nodeA];
|
|
1378
|
+
const existing = this.db
|
|
1379
|
+
.prepare("SELECT * FROM contradictions WHERE node_a = ? AND node_b = ?")
|
|
1380
|
+
.get(a, b);
|
|
1381
|
+
if (existing)
|
|
1382
|
+
return this.rowToContradiction(existing);
|
|
1383
|
+
const record = {
|
|
1384
|
+
id: randomUUID(),
|
|
1385
|
+
nodeA: a,
|
|
1386
|
+
nodeB: b,
|
|
1387
|
+
detectedAt: Date.now(),
|
|
1388
|
+
status,
|
|
1389
|
+
};
|
|
1390
|
+
this.db
|
|
1391
|
+
.prepare(`INSERT OR IGNORE INTO contradictions (id, node_a, node_b, detected_at, status)
|
|
1392
|
+
VALUES (?, ?, ?, ?, ?)`)
|
|
1393
|
+
.run(record.id, a, b, record.detectedAt, status);
|
|
1394
|
+
return record;
|
|
1395
|
+
}
|
|
1396
|
+
/**
|
|
1397
|
+
* Bounded read-time lookup: recorded contradiction pairs where
|
|
1398
|
+
* node_a OR node_b is in `nodeIds`. Empty input returns [] without
|
|
1399
|
+
* touching the database.
|
|
1400
|
+
*/
|
|
1401
|
+
async getContradictionPairsFor(nodeIds) {
|
|
1402
|
+
if (nodeIds.length === 0)
|
|
1403
|
+
return [];
|
|
1404
|
+
const placeholders = nodeIds.map(() => "?").join(",");
|
|
1405
|
+
const rows = this.getPreparedStatement(`getContradictionPairsFor::${nodeIds.length}`, `SELECT * FROM contradictions WHERE node_a IN (${placeholders}) OR node_b IN (${placeholders})`).all(...nodeIds, ...nodeIds);
|
|
1406
|
+
return rows.map((row) => this.rowToContradiction(row));
|
|
1407
|
+
}
|
|
1408
|
+
async updateContradictionStatus(id, status) {
|
|
1409
|
+
this.db
|
|
1410
|
+
.prepare("UPDATE contradictions SET status = ? WHERE id = ?")
|
|
1411
|
+
.run(status, id);
|
|
1412
|
+
}
|
|
1413
|
+
rowToContradiction(row) {
|
|
1414
|
+
return {
|
|
1415
|
+
id: row.id,
|
|
1416
|
+
nodeA: row.node_a,
|
|
1417
|
+
nodeB: row.node_b,
|
|
1418
|
+
detectedAt: row.detected_at,
|
|
1419
|
+
status: row.status,
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1422
|
+
// -----------------------------------------------------------------------
|
|
1423
|
+
// Procedural lessons (self-editing instructions)
|
|
1424
|
+
// -----------------------------------------------------------------------
|
|
1425
|
+
async saveProceduralLesson(input) {
|
|
1426
|
+
const now = Date.now();
|
|
1427
|
+
const score = clampLessonScore(input.initialScore ?? 0.5);
|
|
1428
|
+
const lesson = {
|
|
1429
|
+
id: randomUUID(),
|
|
1430
|
+
lesson: input.lesson,
|
|
1431
|
+
context: input.context ?? "",
|
|
1432
|
+
tags: input.tags ?? [],
|
|
1433
|
+
score,
|
|
1434
|
+
successCount: 0,
|
|
1435
|
+
failureCount: 0,
|
|
1436
|
+
useCount: 0,
|
|
1437
|
+
namespace: input.namespace ?? "default",
|
|
1438
|
+
createdAt: now,
|
|
1439
|
+
updatedAt: now,
|
|
1440
|
+
lastUsedAt: 0,
|
|
1441
|
+
};
|
|
1442
|
+
this.db
|
|
1443
|
+
.prepare(`INSERT INTO procedural_lessons
|
|
1444
|
+
(id, lesson, context, tags, score, success_count, failure_count,
|
|
1445
|
+
use_count, namespace, created_at, updated_at, last_used_at)
|
|
1446
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
1447
|
+
.run(lesson.id, lesson.lesson, lesson.context, JSON.stringify(lesson.tags), lesson.score, lesson.successCount, lesson.failureCount, lesson.useCount, lesson.namespace, lesson.createdAt, lesson.updatedAt, lesson.lastUsedAt);
|
|
1448
|
+
return lesson;
|
|
1449
|
+
}
|
|
1450
|
+
async getProceduralLesson(id) {
|
|
1451
|
+
const row = this.getPreparedStatement("getProceduralLesson", "SELECT * FROM procedural_lessons WHERE id = ?").get(id);
|
|
1452
|
+
return row ? this.rowToProceduralLesson(row) : null;
|
|
1453
|
+
}
|
|
1454
|
+
async updateProceduralLesson(id, patch) {
|
|
1455
|
+
const current = await this.getProceduralLesson(id);
|
|
1456
|
+
if (!current)
|
|
1457
|
+
return null;
|
|
1458
|
+
const next = { ...current, ...patch, id: current.id };
|
|
1459
|
+
this.db
|
|
1460
|
+
.prepare(`UPDATE procedural_lessons
|
|
1461
|
+
SET lesson = ?, context = ?, tags = ?, score = ?,
|
|
1462
|
+
success_count = ?, failure_count = ?, use_count = ?,
|
|
1463
|
+
namespace = ?, updated_at = ?, last_used_at = ?
|
|
1464
|
+
WHERE id = ?`)
|
|
1465
|
+
.run(next.lesson, next.context, JSON.stringify(next.tags), next.score, next.successCount, next.failureCount, next.useCount, next.namespace, next.updatedAt, next.lastUsedAt, id);
|
|
1466
|
+
return next;
|
|
1467
|
+
}
|
|
1468
|
+
async listProceduralLessons(namespace) {
|
|
1469
|
+
const rows = (namespace
|
|
1470
|
+
? this.getPreparedStatement("listProceduralLessons::ns", "SELECT * FROM procedural_lessons WHERE namespace = ? ORDER BY score DESC, updated_at DESC").all(namespace)
|
|
1471
|
+
: this.getPreparedStatement("listProceduralLessons::all", "SELECT * FROM procedural_lessons ORDER BY score DESC, updated_at DESC").all());
|
|
1472
|
+
return rows.map((row) => this.rowToProceduralLesson(row));
|
|
1473
|
+
}
|
|
1474
|
+
rowToProceduralLesson(row) {
|
|
1475
|
+
let tags = [];
|
|
1476
|
+
try {
|
|
1477
|
+
const parsed = JSON.parse(row.tags ?? "[]");
|
|
1478
|
+
if (Array.isArray(parsed))
|
|
1479
|
+
tags = parsed.filter((t) => typeof t === "string");
|
|
1480
|
+
}
|
|
1481
|
+
catch {
|
|
1482
|
+
tags = [];
|
|
1483
|
+
}
|
|
1484
|
+
return {
|
|
1485
|
+
id: row.id,
|
|
1486
|
+
lesson: row.lesson,
|
|
1487
|
+
context: row.context ?? "",
|
|
1488
|
+
tags,
|
|
1489
|
+
score: row.score,
|
|
1490
|
+
successCount: row.success_count,
|
|
1491
|
+
failureCount: row.failure_count,
|
|
1492
|
+
useCount: row.use_count,
|
|
1493
|
+
namespace: row.namespace ?? "default",
|
|
1494
|
+
createdAt: row.created_at,
|
|
1495
|
+
updatedAt: row.updated_at,
|
|
1496
|
+
lastUsedAt: row.last_used_at ?? 0,
|
|
1497
|
+
};
|
|
1498
|
+
}
|
|
1499
|
+
/**
|
|
1500
|
+
* Bounded id-prefix lookup backing citation resolution. Dashes in
|
|
1501
|
+
* stored ids are ignored so `[mem:a3f9]` matches `a3f9c2e1-…`.
|
|
1502
|
+
*/
|
|
1503
|
+
async findNodesByIdPrefix(prefix) {
|
|
1504
|
+
const clean = prefix.toLowerCase().replace(/[^0-9a-f]/g, "");
|
|
1505
|
+
if (clean.length < 2)
|
|
1506
|
+
return [];
|
|
1507
|
+
// `clean` is hex-only after the replace, so interpolation is safe —
|
|
1508
|
+
// LIKE wildcards cannot appear in it.
|
|
1509
|
+
const rows = this.db
|
|
1510
|
+
.prepare(`SELECT * FROM nodes
|
|
1511
|
+
WHERE replace(id, '-', '') LIKE '${clean}%'
|
|
1512
|
+
ORDER BY updated_at DESC
|
|
1513
|
+
LIMIT 25`)
|
|
1514
|
+
.all();
|
|
1515
|
+
return rows.map((row) => this.rowToNode(row));
|
|
1516
|
+
}
|
|
1517
|
+
async getAllEmbeddingInfos() {
|
|
1518
|
+
const rows = this.getPreparedStatement("getAllEmbeddingInfos", "SELECT node_id, model, dimensions, updated_at FROM embeddings").all();
|
|
1519
|
+
return rows.map((row) => ({
|
|
1520
|
+
nodeId: row.node_id,
|
|
1521
|
+
model: row.model,
|
|
1522
|
+
dimensions: row.dimensions,
|
|
1523
|
+
updatedAt: row.updated_at,
|
|
1524
|
+
}));
|
|
1525
|
+
}
|
|
1526
|
+
async getAllEmbeddings() {
|
|
1527
|
+
const rows = this.db
|
|
1528
|
+
.prepare("SELECT node_id, vector, model, dimensions FROM embeddings")
|
|
1529
|
+
.all();
|
|
1530
|
+
return rows.map((r) => ({
|
|
1531
|
+
nodeId: r.node_id,
|
|
1532
|
+
model: r.model,
|
|
1533
|
+
vector: this.parseEmbedding(r.vector),
|
|
1534
|
+
}));
|
|
1535
|
+
}
|
|
1536
|
+
async setNodeNamespace(id, namespace) {
|
|
1537
|
+
this.db
|
|
1538
|
+
.prepare("UPDATE nodes SET namespace = ? WHERE id = ?")
|
|
1539
|
+
.run(namespace, id);
|
|
1540
|
+
}
|
|
1541
|
+
async querySimilarEmbeddings(vector, filter = {}, limit, threshold = 0, model) {
|
|
1542
|
+
return this.scanSimilar("text", vector, filter, limit, threshold, model);
|
|
1543
|
+
}
|
|
1544
|
+
/**
|
|
1545
|
+
* Query image memories by CLIP vector similarity.
|
|
1546
|
+
*
|
|
1547
|
+
* Identical scoring to `querySimilarEmbeddings` — the scan loop is
|
|
1548
|
+
* table-agnostic — but runs against `image_embeddings` and filters on
|
|
1549
|
+
* model + dimensions, so a text-space vector can never be scored
|
|
1550
|
+
* against a CLIP query (or vice versa).
|
|
1551
|
+
*
|
|
1552
|
+
* The hook for image→image search is already here: pass an
|
|
1553
|
+
* *image-encoded* query and the same call returns the nearest
|
|
1554
|
+
* neighbouring images ("more like this screenshot").
|
|
1555
|
+
*/
|
|
1556
|
+
async querySimilarImages(vector, filter = {}, limit, threshold = 0, model) {
|
|
1557
|
+
return this.scanSimilar("image", vector, filter, limit, threshold, model);
|
|
1558
|
+
}
|
|
1559
|
+
/**
|
|
1560
|
+
* The shared semantic scan. `space` selects the vector table
|
|
1561
|
+
* (`embeddings` vs `image_embeddings`) AND the vectorCache namespace,
|
|
1562
|
+
* so the two spaces can never cross even when one nodeId carries both.
|
|
1563
|
+
*/
|
|
1564
|
+
async scanSimilar(space, vector, filter, limit, threshold, model) {
|
|
1565
|
+
const table = space === "image" ? "image_embeddings" : "embeddings";
|
|
1566
|
+
const indexName = space === "image"
|
|
1567
|
+
? "idx_image_embeddings_model_dims"
|
|
1568
|
+
: "idx_embeddings_model_dims";
|
|
1569
|
+
const conditions = [];
|
|
1570
|
+
const params = [vector.length];
|
|
1571
|
+
// True as soon as any node-side (`n.*`) condition is added — only then
|
|
1572
|
+
// do we need the JOIN. The common case (no type/namespace/pool/tags/
|
|
1573
|
+
// importance/metadata filter) selects ids straight from the table,
|
|
1574
|
+
// which the model/dims index serves without a scan.
|
|
1575
|
+
let needsNodeJoin = false;
|
|
1576
|
+
// Only compare vectors produced by compatible models/dimensions; cosine
|
|
1577
|
+
// similarity is meaningless when the stored vector length differs, and
|
|
1578
|
+
// two models can share a dimensionality while producing incomparable
|
|
1579
|
+
// vector spaces — hence the explicit model check when provided.
|
|
1580
|
+
conditions.push("e.dimensions = ?");
|
|
1581
|
+
if (model) {
|
|
1582
|
+
conditions.push("e.model = ?");
|
|
1583
|
+
params.push(model);
|
|
1584
|
+
}
|
|
1585
|
+
if (filter.type) {
|
|
1586
|
+
conditions.push("n.type = ?");
|
|
1587
|
+
params.push(filter.type);
|
|
1588
|
+
needsNodeJoin = true;
|
|
1589
|
+
}
|
|
1590
|
+
if (filter.minImportance !== undefined) {
|
|
1591
|
+
conditions.push("n.importance >= ?");
|
|
1592
|
+
params.push(filter.minImportance);
|
|
1593
|
+
needsNodeJoin = true;
|
|
1594
|
+
}
|
|
1595
|
+
if (filter.maxImportance !== undefined) {
|
|
1596
|
+
conditions.push("n.importance <= ?");
|
|
1597
|
+
params.push(filter.maxImportance);
|
|
1598
|
+
needsNodeJoin = true;
|
|
1599
|
+
}
|
|
1600
|
+
if (filter.namespace) {
|
|
1601
|
+
conditions.push("n.namespace = ?");
|
|
1602
|
+
params.push(filter.namespace);
|
|
1603
|
+
needsNodeJoin = true;
|
|
1604
|
+
}
|
|
1605
|
+
if (filter.namespacePrefix) {
|
|
1606
|
+
conditions.push("n.namespace LIKE ? ESCAPE '\\'");
|
|
1607
|
+
params.push(escapeLikePrefix(filter.namespacePrefix) + "%");
|
|
1608
|
+
needsNodeJoin = true;
|
|
1609
|
+
}
|
|
1610
|
+
if (filter.pool !== undefined) {
|
|
1611
|
+
const pools = Array.isArray(filter.pool) ? filter.pool : [filter.pool];
|
|
1612
|
+
if (pools.length > 0) {
|
|
1613
|
+
conditions.push(`n.pool IN (${pools.map(() => "?").join(", ")})`);
|
|
1614
|
+
params.push(...pools);
|
|
1615
|
+
needsNodeJoin = true;
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
const metadata = this.buildMetadataFilter(filter.metadata, "n");
|
|
1619
|
+
conditions.push(...metadata.conditions);
|
|
1620
|
+
params.push(...metadata.params);
|
|
1621
|
+
if (metadata.conditions.length > 0)
|
|
1622
|
+
needsNodeJoin = true;
|
|
1623
|
+
if (filter.tags && filter.tags.length > 0) {
|
|
1624
|
+
for (const tag of filter.tags) {
|
|
1625
|
+
conditions.push("n.tags LIKE ?");
|
|
1626
|
+
params.push(`%${JSON.stringify(tag)}%`);
|
|
1627
|
+
}
|
|
1628
|
+
needsNodeJoin = true;
|
|
1629
|
+
}
|
|
1630
|
+
// Harness attribution on the semantic leg: needs the node join.
|
|
1631
|
+
// "all" (or unset) keeps the default cross-harness behavior.
|
|
1632
|
+
if (filter.harness && filter.harness !== "all") {
|
|
1633
|
+
conditions.push("n.harness = ?");
|
|
1634
|
+
params.push(filter.harness);
|
|
1635
|
+
needsNodeJoin = true;
|
|
1636
|
+
}
|
|
1637
|
+
// Provenance-trust: quarantine visibility (see `queryNodes` — excluded
|
|
1638
|
+
// from recall by default; the entity-leg and graph-expansion paths in
|
|
1639
|
+
// MemOS apply the same rule via `quarantineVisible`).
|
|
1640
|
+
if (filter.quarantinedOnly === true) {
|
|
1641
|
+
conditions.push("n.quarantined = 1");
|
|
1642
|
+
needsNodeJoin = true;
|
|
1643
|
+
}
|
|
1644
|
+
else if (filter.includeQuarantined !== true) {
|
|
1645
|
+
conditions.push("n.quarantined = 0");
|
|
1646
|
+
needsNodeJoin = true;
|
|
1647
|
+
}
|
|
1648
|
+
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
1649
|
+
const fromClause = needsNodeJoin
|
|
1650
|
+
? `FROM ${table} e JOIN nodes n ON n.id = e.node_id`
|
|
1651
|
+
: `FROM ${table} e`;
|
|
1652
|
+
// Lazy hydration: score every candidate vector, but fully hydrate
|
|
1653
|
+
// (JSON.parse metadata + tags + timestamps) ONLY the top-`limit` rows.
|
|
1654
|
+
// The previous implementation called `rowToNode()` for every row —
|
|
1655
|
+
// O(N) JSON parses per query, of which all but `limit` were thrown
|
|
1656
|
+
// away. Cosine scoring still touches every vector (unavoidable
|
|
1657
|
+
// without a vector index), but the expensive per-row hydration now
|
|
1658
|
+
// runs at most `limit` times instead of N times.
|
|
1659
|
+
//
|
|
1660
|
+
// Statement cache key includes the built WHERE clause AND the join
|
|
1661
|
+
// shape: the filter composition above varies the SQL text, and reusing
|
|
1662
|
+
// one prepared statement across different SQL would return wrong
|
|
1663
|
+
// results.
|
|
1664
|
+
// Two-phase retrieval: ids are cheap to select; vectors come from the
|
|
1665
|
+
// in-memory cache when warm, so repeat queries skip re-fetching every
|
|
1666
|
+
// BLOB (tens of MB per query on large stores). Statement cache keys
|
|
1667
|
+
// include the WHERE clause — filter composition varies the SQL.
|
|
1668
|
+
const idRows = this.getPreparedStatement(`querySimilarEmbeddingsIds::${space}::${needsNodeJoin ? "jn" : "noj"}::${where}`, `SELECT e.node_id
|
|
1669
|
+
${fromClause}
|
|
1670
|
+
${where}`).all(...params);
|
|
1671
|
+
const candidateIds = idRows.map((r) => r.node_id);
|
|
1672
|
+
const missing = candidateIds.filter((id) => !this.vectorCacheGet(id, space));
|
|
1673
|
+
// Chunked fetch for cache misses (SQLite bound-parameter limits).
|
|
1674
|
+
// A miss also materialises the cached squared norm (and, in prefilter
|
|
1675
|
+
// mode, the coarse projection) — once per stored row, not once per
|
|
1676
|
+
// query. That is the count the scan reports as `normComputations`.
|
|
1677
|
+
let normsComputed = 0;
|
|
1678
|
+
for (let start = 0; start < missing.length; start += 500) {
|
|
1679
|
+
const chunk = missing.slice(start, start + 500);
|
|
1680
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1681
|
+
const blobRows = this.getPreparedStatement(`querySimilarEmbeddingsBlobs::${space}::${placeholders.length}`, `SELECT node_id, vector FROM ${table} WHERE node_id IN (${placeholders})`).all(...chunk);
|
|
1682
|
+
for (const row of blobRows) {
|
|
1683
|
+
if (Buffer.isBuffer(row.vector) || row.vector instanceof Uint8Array) {
|
|
1684
|
+
this.vectorCacheSet(row.node_id, this.parseEmbeddingF32(row.vector), space);
|
|
1685
|
+
normsComputed += 1;
|
|
1686
|
+
}
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
// Score — (nodeId, score) pairs only, no node hydration. Warm-cache
|
|
1690
|
+
// hits reuse the parsed Float32Array AND its cached squared norm;
|
|
1691
|
+
// legacy JSON-stored rows (never cached) fall back to `parseEmbedding`
|
|
1692
|
+
// per lookup.
|
|
1693
|
+
//
|
|
1694
|
+
// Every candidate is still visited — without an ANN index there is no
|
|
1695
|
+
// way to prove a candidate irrelevant without scoring it — but the
|
|
1696
|
+
// PER-CANDIDATE cost is what actually dominated: `cosineSimilarity`
|
|
1697
|
+
// recomputed the stored row's L2 norm on every query, paying 3·d
|
|
1698
|
+
// multiply-adds per candidate where d suffice. The norm now lives in
|
|
1699
|
+
// the cache entry next to the vector it came from, computed once per
|
|
1700
|
+
// stored row.
|
|
1701
|
+
//
|
|
1702
|
+
// Scores are bit-identical to `cosineSimilarity(query, stored)`: the
|
|
1703
|
+
// dot product accumulates in the same index order, and the divisor is
|
|
1704
|
+
// `Math.sqrt(queryNormSq · storedNormSq)` with the same operand order,
|
|
1705
|
+
// so rankings and reported scores do not move.
|
|
1706
|
+
const queryF32 = vector instanceof Float32Array ? vector : Float32Array.from(vector);
|
|
1707
|
+
const queryNormSq = sumSquares(queryF32);
|
|
1708
|
+
// Opt-in coarse prefilter (`storageOptions.semanticScan`): rank the
|
|
1709
|
+
// candidates by a strided prefix projection and exactly rescore only
|
|
1710
|
+
// the best `keep`. Bounded work on large corpora, approximate: a
|
|
1711
|
+
// candidate the projection ranks below the cutoff is never seen
|
|
1712
|
+
// again. Off by default, so the exact scan above is the shipped
|
|
1713
|
+
// behavior.
|
|
1714
|
+
let scanIds = candidateIds;
|
|
1715
|
+
let prefilterSkipped = 0;
|
|
1716
|
+
if (this.semanticScanMode === "prefilter" &&
|
|
1717
|
+
candidateIds.length > PREFILTER_MIN_KEEP) {
|
|
1718
|
+
const keep = Math.max(PREFILTER_MIN_KEEP, Math.min(candidateIds.length, limit * 4));
|
|
1719
|
+
const ranked = this.rankByProjection(candidateIds, queryF32, space);
|
|
1720
|
+
scanIds = ranked.slice(0, keep);
|
|
1721
|
+
prefilterSkipped = candidateIds.length - scanIds.length;
|
|
1722
|
+
}
|
|
1723
|
+
// Streaming bounded top-K: the selector is fed while scoring instead
|
|
1724
|
+
// of materialising an O(N) pair array and re-heapifying it, and its
|
|
1725
|
+
// floor doubles as the early-exit bound. Its retention rule is
|
|
1726
|
+
// identical to the previous `takeTopK(scoredPairs, limit)` — same
|
|
1727
|
+
// elements, same descending order, same strict-improvement tie
|
|
1728
|
+
// break — so rankings do not move.
|
|
1729
|
+
const top = new TopKSelector(limit);
|
|
1730
|
+
let cacheHits = 0;
|
|
1731
|
+
let legacyRows = 0;
|
|
1732
|
+
let earlyExit = false;
|
|
1733
|
+
for (const nodeId of scanIds) {
|
|
1734
|
+
const cached = this.vectorCacheGet(nodeId, space);
|
|
1735
|
+
let score;
|
|
1736
|
+
if (cached && cached.vector.length === queryF32.length) {
|
|
1737
|
+
cacheHits += 1;
|
|
1738
|
+
if (cached.normSq === 0 || queryNormSq === 0) {
|
|
1739
|
+
// cosineSimilarity's zero-norm guard: an all-zero row scores 0.
|
|
1740
|
+
score = 0;
|
|
1741
|
+
}
|
|
1742
|
+
else {
|
|
1743
|
+
let dot = 0;
|
|
1744
|
+
const stored = cached.vector;
|
|
1745
|
+
for (let i = 0; i < stored.length; i += 1) {
|
|
1746
|
+
dot += queryF32[i] * stored[i];
|
|
1747
|
+
}
|
|
1748
|
+
score = dot / Math.sqrt(queryNormSq * cached.normSq);
|
|
1749
|
+
}
|
|
1750
|
+
}
|
|
1751
|
+
else {
|
|
1752
|
+
legacyRows += 1;
|
|
1753
|
+
score = cosineSimilarity(vector, this.parseEmbedding(this.lookupStoredVector(nodeId, space)));
|
|
1754
|
+
}
|
|
1755
|
+
if (score < threshold)
|
|
1756
|
+
continue;
|
|
1757
|
+
top.push({ nodeId, score });
|
|
1758
|
+
// Cosine similarity is bounded by 1 and retention requires a
|
|
1759
|
+
// STRICT improvement, so once the selector is FULL and its floor
|
|
1760
|
+
// is 1.0 no remaining candidate can enter the top-K: stop
|
|
1761
|
+
// scanning. Fires on exact-duplicate / verbatim-needle queries.
|
|
1762
|
+
// The fullness check is what makes this correct: a full floor of
|
|
1763
|
+
// 1.0 cannot be beaten, but a HALF-full heap at floor 1.0 can
|
|
1764
|
+
// still admit the remaining equally-scored rows (ties keep the
|
|
1765
|
+
// earlier-scanned one but must still fill the requested K), so
|
|
1766
|
+
// stopping there would silently drop recall.
|
|
1767
|
+
if (top.full && top.floor >= 1) {
|
|
1768
|
+
earlyExit = true;
|
|
1769
|
+
break;
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
this.semanticScanTelemetry = {
|
|
1773
|
+
candidates: candidateIds.length,
|
|
1774
|
+
cacheHits,
|
|
1775
|
+
normComputations: normsComputed,
|
|
1776
|
+
legacyRows,
|
|
1777
|
+
prefilterSkipped,
|
|
1778
|
+
rescored: scanIds.length,
|
|
1779
|
+
earlyExit,
|
|
1780
|
+
};
|
|
1781
|
+
// Descending score; equal scores keep scan order (the selector only
|
|
1782
|
+
// displaces the floor on a strict improvement).
|
|
1783
|
+
const survivors = top.result();
|
|
1784
|
+
// Hydrate only the survivors, preserving score order.
|
|
1785
|
+
const results = [];
|
|
1786
|
+
for (const { nodeId, score } of survivors) {
|
|
1787
|
+
const node = await this.getNode(nodeId);
|
|
1788
|
+
if (!node)
|
|
1789
|
+
continue; // node deleted between the two queries
|
|
1790
|
+
results.push({
|
|
1791
|
+
node,
|
|
1792
|
+
score,
|
|
1793
|
+
scores: { semantic: score },
|
|
1794
|
+
});
|
|
1795
|
+
}
|
|
1796
|
+
return results;
|
|
1797
|
+
}
|
|
1798
|
+
/**
|
|
1799
|
+
* Rank candidate ids by cosine against a strided prefix projection of
|
|
1800
|
+
* each stored vector (opt-in prefilter path). The projection is built
|
|
1801
|
+
* once per stored row and cached with it, so repeat queries pay only
|
|
1802
|
+
* the `dims / PREFILTER_STRIDE` dot product per candidate.
|
|
1803
|
+
*
|
|
1804
|
+
* Approximate by design: it orders candidates, it does not certify
|
|
1805
|
+
* them. Callers must rescore the survivors exactly.
|
|
1806
|
+
*/
|
|
1807
|
+
rankByProjection(candidateIds, queryF32, space) {
|
|
1808
|
+
const projQuery = this.project(queryF32);
|
|
1809
|
+
const projQueryNormSq = sumSquares(projQuery);
|
|
1810
|
+
if (projQueryNormSq === 0)
|
|
1811
|
+
return candidateIds;
|
|
1812
|
+
const queryNorm = Math.sqrt(projQueryNormSq);
|
|
1813
|
+
const scored = [];
|
|
1814
|
+
for (const nodeId of candidateIds) {
|
|
1815
|
+
const cached = this.vectorCacheGet(nodeId, space);
|
|
1816
|
+
const proj = cached?.proj;
|
|
1817
|
+
const projNormSq = cached?.projNormSq;
|
|
1818
|
+
if (proj === undefined || projNormSq === undefined || projNormSq === 0) {
|
|
1819
|
+
continue;
|
|
1820
|
+
}
|
|
1821
|
+
let dot = 0;
|
|
1822
|
+
const length = Math.min(proj.length, projQuery.length);
|
|
1823
|
+
for (let i = 0; i < length; i += 1) {
|
|
1824
|
+
dot += projQuery[i] * proj[i];
|
|
1825
|
+
}
|
|
1826
|
+
scored.push({ nodeId, score: dot / (queryNorm * Math.sqrt(projNormSq)) });
|
|
1827
|
+
}
|
|
1828
|
+
scored.sort((a, b) => b.score - a.score);
|
|
1829
|
+
// Candidates the cache never hydrated (or whose projection vanished)
|
|
1830
|
+
// are appended after the ranked ones rather than dropped, so a cold
|
|
1831
|
+
// cache degrades to "projection-ranked first, everything else after"
|
|
1832
|
+
// instead of silently losing rows.
|
|
1833
|
+
const ranked = new Set(scored.map((s) => s.nodeId));
|
|
1834
|
+
return [
|
|
1835
|
+
...scored.map((s) => s.nodeId),
|
|
1836
|
+
...candidateIds.filter((id) => !ranked.has(id)),
|
|
1837
|
+
];
|
|
1838
|
+
}
|
|
1839
|
+
/**
|
|
1840
|
+
* Strided prefix projection used by the prefilter. Built lazily on the
|
|
1841
|
+
* first prefilter query so the exact path never pays for it.
|
|
1842
|
+
*/
|
|
1843
|
+
project(vector) {
|
|
1844
|
+
const length = Math.ceil(vector.length / PREFILTER_STRIDE);
|
|
1845
|
+
const out = new Float32Array(length);
|
|
1846
|
+
for (let i = 0; i < length; i += 1) {
|
|
1847
|
+
out[i] = vector[i * PREFILTER_STRIDE] ?? 0;
|
|
1848
|
+
}
|
|
1849
|
+
return out;
|
|
1850
|
+
}
|
|
1851
|
+
async getEmbeddingModelCounts() {
|
|
1852
|
+
const rows = this.db
|
|
1853
|
+
.prepare(`SELECT model, COUNT(*) as count FROM embeddings GROUP BY model ORDER BY count DESC`)
|
|
1854
|
+
.all();
|
|
1855
|
+
return rows;
|
|
1856
|
+
}
|
|
1857
|
+
async deleteEmbeddingsByModel(model) {
|
|
1858
|
+
const info = this.db
|
|
1859
|
+
.prepare(`DELETE FROM embeddings WHERE model = ?`)
|
|
1860
|
+
.run(model);
|
|
1861
|
+
this.vectorCache.clear();
|
|
1862
|
+
return info.changes;
|
|
1863
|
+
}
|
|
1864
|
+
// -----------------------------------------------------------------------
|
|
1865
|
+
// Image (CLIP) embeddings — the second vector space
|
|
1866
|
+
// -----------------------------------------------------------------------
|
|
1867
|
+
async saveImageEmbedding(nodeId, vector, model) {
|
|
1868
|
+
const normalized = this.serializeEmbedding(vector);
|
|
1869
|
+
this.db
|
|
1870
|
+
.prepare(`INSERT INTO image_embeddings (node_id, vector, dimensions, model, updated_at)
|
|
1871
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1872
|
+
ON CONFLICT(node_id) DO UPDATE SET
|
|
1873
|
+
vector = excluded.vector,
|
|
1874
|
+
dimensions = excluded.dimensions,
|
|
1875
|
+
model = excluded.model,
|
|
1876
|
+
updated_at = excluded.updated_at`)
|
|
1877
|
+
.run(nodeId, normalized, vector.length, model, Date.now());
|
|
1878
|
+
this.vectorCacheSet(nodeId, this.parseEmbeddingF32(normalized), "image");
|
|
1879
|
+
}
|
|
1880
|
+
async getImageEmbeddingInfo(nodeId) {
|
|
1881
|
+
const row = this.getPreparedStatement("getImageEmbeddingInfo", "SELECT model, dimensions, updated_at FROM image_embeddings WHERE node_id = ?").get(nodeId);
|
|
1882
|
+
if (!row)
|
|
1883
|
+
return null;
|
|
1884
|
+
return {
|
|
1885
|
+
model: row.model,
|
|
1886
|
+
dimensions: row.dimensions,
|
|
1887
|
+
updatedAt: row.updated_at,
|
|
1888
|
+
};
|
|
1889
|
+
}
|
|
1890
|
+
async getAllImageEmbeddings() {
|
|
1891
|
+
const rows = this.db
|
|
1892
|
+
.prepare("SELECT node_id, vector, model, dimensions FROM image_embeddings")
|
|
1893
|
+
.all();
|
|
1894
|
+
return rows.map((r) => ({
|
|
1895
|
+
nodeId: r.node_id,
|
|
1896
|
+
model: r.model,
|
|
1897
|
+
vector: this.parseEmbedding(r.vector),
|
|
1898
|
+
}));
|
|
1899
|
+
}
|
|
1900
|
+
async getImageEmbeddingModelCounts() {
|
|
1901
|
+
return this.db
|
|
1902
|
+
.prepare(`SELECT model, COUNT(*) as count FROM image_embeddings
|
|
1903
|
+
GROUP BY model ORDER BY count DESC`)
|
|
1904
|
+
.all();
|
|
1905
|
+
}
|
|
1906
|
+
async deleteImageEmbedding(nodeId) {
|
|
1907
|
+
const info = this.db
|
|
1908
|
+
.prepare("DELETE FROM image_embeddings WHERE node_id = ?")
|
|
1909
|
+
.run(nodeId);
|
|
1910
|
+
const entry = this.vectorCache.get(nodeId);
|
|
1911
|
+
if (entry && entry.space === "image") {
|
|
1912
|
+
this.vectorCache.delete(nodeId);
|
|
1913
|
+
}
|
|
1914
|
+
return info.changes > 0;
|
|
1915
|
+
}
|
|
1916
|
+
async deleteImageEmbeddingsByModel(model) {
|
|
1917
|
+
const info = this.db
|
|
1918
|
+
.prepare("DELETE FROM image_embeddings WHERE model = ?")
|
|
1919
|
+
.run(model);
|
|
1920
|
+
this.vectorCache.clear();
|
|
1921
|
+
return info.changes;
|
|
1922
|
+
}
|
|
1923
|
+
async queryEdges(filter = {}) {
|
|
1924
|
+
const conditions = [];
|
|
1925
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
1926
|
+
const params = [];
|
|
1927
|
+
if (filter.sourceId) {
|
|
1928
|
+
conditions.push("source_id = ?");
|
|
1929
|
+
params.push(filter.sourceId);
|
|
1930
|
+
}
|
|
1931
|
+
if (filter.targetId) {
|
|
1932
|
+
conditions.push("target_id = ?");
|
|
1933
|
+
params.push(filter.targetId);
|
|
1934
|
+
}
|
|
1935
|
+
if (filter.relation) {
|
|
1936
|
+
conditions.push("relation = ?");
|
|
1937
|
+
params.push(filter.relation);
|
|
1938
|
+
}
|
|
1939
|
+
if (filter.validAt !== undefined) {
|
|
1940
|
+
// Bitemporal as-of read: the event-time validity interval must cover
|
|
1941
|
+
// the timestamp. NULL bounds are open-ended.
|
|
1942
|
+
conditions.push("(valid_from IS NULL OR valid_from <= ?)");
|
|
1943
|
+
params.push(filter.validAt);
|
|
1944
|
+
conditions.push("(valid_to IS NULL OR valid_to > ?)");
|
|
1945
|
+
params.push(filter.validAt);
|
|
1946
|
+
}
|
|
1947
|
+
const where = conditions.length ? "WHERE " + conditions.join(" AND ") : "";
|
|
1948
|
+
const rows = this.db
|
|
1949
|
+
.prepare(`SELECT * FROM edges ${where}`)
|
|
1950
|
+
.all(...params);
|
|
1951
|
+
return rows.map((r) => this.rowToEdge(r));
|
|
1952
|
+
}
|
|
1953
|
+
async getGraph() {
|
|
1954
|
+
const nodes = this.db.prepare("SELECT * FROM nodes").all();
|
|
1955
|
+
const edges = this.db.prepare("SELECT * FROM edges").all();
|
|
1956
|
+
return {
|
|
1957
|
+
nodes: nodes.map((r) => this.rowToNode(r)),
|
|
1958
|
+
edges: edges.map((r) => this.rowToEdge(r)),
|
|
1959
|
+
};
|
|
1960
|
+
}
|
|
1961
|
+
/**
|
|
1962
|
+
* Bitemporal as-of graph read: nodes and edges whose validity interval
|
|
1963
|
+
* covers `atTime` (unix ms). Predicates mirror the existing as-of read
|
|
1964
|
+
* paths exactly — nodes use `valid_to >= ?` (see the `validAt` search
|
|
1965
|
+
* filter), edges use strict `valid_to > ?` (see `queryEdges`).
|
|
1966
|
+
*/
|
|
1967
|
+
async getGraphAtTime(atTime) {
|
|
1968
|
+
const nodes = this.db
|
|
1969
|
+
.prepare(`SELECT * FROM nodes
|
|
1970
|
+
WHERE (valid_from IS NULL OR valid_from <= ?)
|
|
1971
|
+
AND (valid_to IS NULL OR valid_to >= ?)`)
|
|
1972
|
+
.all(atTime, atTime);
|
|
1973
|
+
const edges = this.db
|
|
1974
|
+
.prepare(`SELECT * FROM edges
|
|
1975
|
+
WHERE (valid_from IS NULL OR valid_from <= ?)
|
|
1976
|
+
AND (valid_to IS NULL OR valid_to > ?)`)
|
|
1977
|
+
.all(atTime, atTime);
|
|
1978
|
+
return {
|
|
1979
|
+
nodes: nodes.map((r) => this.rowToNode(r)),
|
|
1980
|
+
edges: edges.map((r) => this.rowToEdge(r)),
|
|
1981
|
+
};
|
|
1982
|
+
}
|
|
1983
|
+
// -----------------------------------------------------------------------
|
|
1984
|
+
// TTL
|
|
1985
|
+
// -----------------------------------------------------------------------
|
|
1986
|
+
async setTTL(id, seconds) {
|
|
1987
|
+
const expiresAt = Math.floor(Date.now() / 1000) + seconds;
|
|
1988
|
+
this.db
|
|
1989
|
+
.prepare("UPDATE nodes SET expires_at = ? WHERE id = ?")
|
|
1990
|
+
.run(expiresAt, id);
|
|
1991
|
+
}
|
|
1992
|
+
async clearTTL(id) {
|
|
1993
|
+
this.db.prepare("UPDATE nodes SET expires_at = NULL WHERE id = ?").run(id);
|
|
1994
|
+
}
|
|
1995
|
+
async sweepExpired() {
|
|
1996
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1997
|
+
const result = this.db
|
|
1998
|
+
.prepare("DELETE FROM nodes WHERE expires_at IS NOT NULL AND expires_at <= ?")
|
|
1999
|
+
.run(now);
|
|
2000
|
+
return result.changes;
|
|
2001
|
+
}
|
|
2002
|
+
// -----------------------------------------------------------------------
|
|
2003
|
+
// Tags
|
|
2004
|
+
// -----------------------------------------------------------------------
|
|
2005
|
+
async queryNodesByTag(tag) {
|
|
2006
|
+
// Uses the (tag) secondary index on node_tags — O(log n) lookup, no
|
|
2007
|
+
// JSON scan.
|
|
2008
|
+
const rows = this.db
|
|
2009
|
+
.prepare(`SELECT n.* FROM nodes n
|
|
2010
|
+
INNER JOIN node_tags nt ON nt.node_id = n.id
|
|
2011
|
+
WHERE nt.tag = ?`)
|
|
2012
|
+
.all(tag);
|
|
2013
|
+
return rows.map((r) => this.rowToNode(r));
|
|
2014
|
+
}
|
|
2015
|
+
/**
|
|
2016
|
+
* ── item2: entity leg ── candidate generation for hybrid search: node
|
|
2017
|
+
* ids whose indexed entities intersect `entities`, ranked by number of
|
|
2018
|
+
* matched entities (desc), node id tiebreak. Backed by the
|
|
2019
|
+
* `entity_index` inverted table — O(indexed entities), no content scan.
|
|
2020
|
+
*/
|
|
2021
|
+
async searchByEntities(entities, limit) {
|
|
2022
|
+
return searchEntityIndex(this.db, entities, limit);
|
|
2023
|
+
}
|
|
2024
|
+
// -----------------------------------------------------------------------
|
|
2025
|
+
// Harness attribution ("one memory, every harness")
|
|
2026
|
+
// -----------------------------------------------------------------------
|
|
2027
|
+
/**
|
|
2028
|
+
* Per-harness memory counts for `memos harness list`. Legacy and
|
|
2029
|
+
* unattributed rows read as `"unknown"` via the column default.
|
|
2030
|
+
*/
|
|
2031
|
+
async getHarnessCounts() {
|
|
2032
|
+
const rows = this.db
|
|
2033
|
+
.prepare(`SELECT harness, COUNT(*) AS count FROM nodes
|
|
2034
|
+
GROUP BY harness ORDER BY count DESC, harness ASC`)
|
|
2035
|
+
.all();
|
|
2036
|
+
return rows.map((r) => ({
|
|
2037
|
+
harness: r.harness || "unknown",
|
|
2038
|
+
count: Number(r.count),
|
|
2039
|
+
}));
|
|
2040
|
+
}
|
|
2041
|
+
/**
|
|
2042
|
+
* Full node snapshot for harness merges (`memos harness merge
|
|
2043
|
+
* --from`). Returns EVERYTHING — including historical (superseded)
|
|
2044
|
+
* and quarantined rows — so the merge preserves the bitemporal
|
|
2045
|
+
* version ledger verbatim. Read-only; the merge writes through the
|
|
2046
|
+
* target storage's normal save path.
|
|
2047
|
+
*/
|
|
2048
|
+
async listAllNodes() {
|
|
2049
|
+
const rows = this.db.prepare("SELECT * FROM nodes").all();
|
|
2050
|
+
return rows.map((r) => this.rowToNode(r));
|
|
2051
|
+
}
|
|
2052
|
+
/**
|
|
2053
|
+
* Full edge snapshot for harness merges. Endpoints are remapped by
|
|
2054
|
+
* the merge when node ids collide in the target.
|
|
2055
|
+
*/
|
|
2056
|
+
async listAllEdges() {
|
|
2057
|
+
const rows = this.db.prepare("SELECT * FROM edges").all();
|
|
2058
|
+
return rows.map((r) => this.rowToEdge(r));
|
|
2059
|
+
}
|
|
2060
|
+
// -----------------------------------------------------------------------
|
|
2061
|
+
// Bulk Operations
|
|
2062
|
+
// -----------------------------------------------------------------------
|
|
2063
|
+
async deleteAllNodes() {
|
|
2064
|
+
this.vectorCache.clear();
|
|
2065
|
+
this.db.exec("DELETE FROM embeddings");
|
|
2066
|
+
this.db.exec("DELETE FROM image_embeddings");
|
|
2067
|
+
this.db.exec("DELETE FROM edges");
|
|
2068
|
+
this.db.exec("DELETE FROM nodes");
|
|
2069
|
+
// Rebuild FTS index
|
|
2070
|
+
this.db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')");
|
|
2071
|
+
}
|
|
2072
|
+
// -----------------------------------------------------------------------
|
|
2073
|
+
// Lifecycle
|
|
2074
|
+
// -----------------------------------------------------------------------
|
|
2075
|
+
async close() {
|
|
2076
|
+
if (this.accessFlushTimer !== null) {
|
|
2077
|
+
clearTimeout(this.accessFlushTimer);
|
|
2078
|
+
this.accessFlushTimer = null;
|
|
2079
|
+
}
|
|
2080
|
+
this.flushAccessCounts();
|
|
2081
|
+
if (this.db) {
|
|
2082
|
+
this.db.close();
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
// -----------------------------------------------------------------------
|
|
2086
|
+
// Helpers
|
|
2087
|
+
// -----------------------------------------------------------------------
|
|
2088
|
+
rowToNode(row) {
|
|
2089
|
+
const metadata = JSON.parse(row.metadata || "{}");
|
|
2090
|
+
return {
|
|
2091
|
+
id: row.id,
|
|
2092
|
+
content: row.content,
|
|
2093
|
+
summary: row.summary,
|
|
2094
|
+
type: row.type,
|
|
2095
|
+
metadata,
|
|
2096
|
+
importance: row.importance,
|
|
2097
|
+
createdAt: row.created_at,
|
|
2098
|
+
updatedAt: row.updated_at,
|
|
2099
|
+
accessCount: row.access_count,
|
|
2100
|
+
lastAccessed: row.last_accessed,
|
|
2101
|
+
tags: JSON.parse(row.tags || "[]"),
|
|
2102
|
+
expiresAt: row.expires_at ?? null,
|
|
2103
|
+
namespace: row.namespace || "default",
|
|
2104
|
+
validFrom: row.valid_from ?? null,
|
|
2105
|
+
validTo: row.valid_to ?? null,
|
|
2106
|
+
source: row.source || "user_input",
|
|
2107
|
+
trustScore: row.trust_score ?? 1.0,
|
|
2108
|
+
pool: (row.pool || "event"),
|
|
2109
|
+
provenance: isProvenanceTier(row.provenance) ? row.provenance : "user",
|
|
2110
|
+
quarantined: row.quarantined === 1,
|
|
2111
|
+
quarantinedAt: row.quarantined_at ?? null,
|
|
2112
|
+
quarantineReason: row.quarantine_reason ?? null,
|
|
2113
|
+
// Harness attribution reads as "unknown" for legacy rows (and for
|
|
2114
|
+
// rows written by custom adapters that never set the tag).
|
|
2115
|
+
harness: row.harness || "unknown",
|
|
2116
|
+
// Load confidence state machine values from metadata
|
|
2117
|
+
confidence: metadata.confidence,
|
|
2118
|
+
evidenceCount: metadata.evidenceCount,
|
|
2119
|
+
};
|
|
2120
|
+
}
|
|
2121
|
+
rowToEdge(row) {
|
|
2122
|
+
return {
|
|
2123
|
+
id: row.id,
|
|
2124
|
+
sourceId: row.source_id,
|
|
2125
|
+
targetId: row.target_id,
|
|
2126
|
+
relation: row.relation,
|
|
2127
|
+
weight: row.weight,
|
|
2128
|
+
metadata: JSON.parse(row.metadata || "{}"),
|
|
2129
|
+
createdAt: row.created_at,
|
|
2130
|
+
validFrom: row.valid_from ?? null,
|
|
2131
|
+
validTo: row.valid_to ?? null,
|
|
2132
|
+
};
|
|
2133
|
+
}
|
|
2134
|
+
/**
|
|
2135
|
+
* In-process parsed-vector cache (nodeId -> vector + derived norms).
|
|
2136
|
+
* The semantic leg otherwise re-fetches EVERY stored vector BLOB from
|
|
2137
|
+
* SQLite on every query (tens of MB of memcpy per query on large
|
|
2138
|
+
* stores). Writes go through this class, so invalidation is exact;
|
|
2139
|
+
* external writers are not tracked (single-process deployments are the
|
|
2140
|
+
* supported shape — the Python bridge runs one Node process).
|
|
2141
|
+
*
|
|
2142
|
+
* The entry bundles the squared L2 norm with the vector: it is
|
|
2143
|
+
* computed once per stored row instead of once per query, and it is
|
|
2144
|
+
* invalidated by the same `set`/eviction/clear that invalidates the
|
|
2145
|
+
* vector itself, so it can never disagree with the stored row.
|
|
2146
|
+
*/
|
|
2147
|
+
vectorCache = new Map();
|
|
2148
|
+
vectorCacheMaxEntries = 25_000;
|
|
2149
|
+
semanticScanMode = "exact";
|
|
2150
|
+
vectorCacheGet(nodeId, space) {
|
|
2151
|
+
const entry = this.vectorCache.get(nodeId);
|
|
2152
|
+
// A hit from the WRONG space is a miss, not a stale read: the two
|
|
2153
|
+
// spaces share a nodeId (an image node has both), and serving one's
|
|
2154
|
+
// vector to the other's query is the silent-corruption bug this
|
|
2155
|
+
// namespace exists to prevent.
|
|
2156
|
+
if (entry && entry.space !== space)
|
|
2157
|
+
return undefined;
|
|
2158
|
+
return entry;
|
|
2159
|
+
}
|
|
2160
|
+
vectorCacheSet(nodeId, vector, space) {
|
|
2161
|
+
if (this.vectorCache.size >= this.vectorCacheMaxEntries &&
|
|
2162
|
+
!this.vectorCache.has(nodeId)) {
|
|
2163
|
+
// FIFO eviction (Map preserves insertion order). The norm rides in
|
|
2164
|
+
// the same entry, so deleting the entry drops both.
|
|
2165
|
+
const oldest = this.vectorCache.keys().next().value;
|
|
2166
|
+
if (oldest !== undefined)
|
|
2167
|
+
this.vectorCache.delete(oldest);
|
|
2168
|
+
}
|
|
2169
|
+
const entry = { vector, space, normSq: sumSquares(vector) };
|
|
2170
|
+
// The coarse projection only exists in prefilter mode, so the exact
|
|
2171
|
+
// path never pays to build it. Same funnel as the vector itself —
|
|
2172
|
+
// every write re-derives it, so it can never go stale.
|
|
2173
|
+
if (this.semanticScanMode === "prefilter") {
|
|
2174
|
+
entry.proj = this.project(vector);
|
|
2175
|
+
entry.projNormSq = sumSquares(entry.proj);
|
|
2176
|
+
}
|
|
2177
|
+
this.vectorCache.set(nodeId, entry);
|
|
2178
|
+
}
|
|
2179
|
+
/**
|
|
2180
|
+
* Telemetry for the last `querySimilarEmbeddings` call. Read-only: the
|
|
2181
|
+
* scan fills it in, nothing else mutates it.
|
|
2182
|
+
*/
|
|
2183
|
+
semanticScanTelemetry = {
|
|
2184
|
+
candidates: 0,
|
|
2185
|
+
cacheHits: 0,
|
|
2186
|
+
normComputations: 0,
|
|
2187
|
+
legacyRows: 0,
|
|
2188
|
+
prefilterSkipped: 0,
|
|
2189
|
+
rescored: 0,
|
|
2190
|
+
earlyExit: false,
|
|
2191
|
+
};
|
|
2192
|
+
/** Scan telemetry for the most recent semantic query. */
|
|
2193
|
+
get semanticScanStats() {
|
|
2194
|
+
return this.semanticScanTelemetry;
|
|
2195
|
+
}
|
|
2196
|
+
lookupStoredVector(nodeId, space) {
|
|
2197
|
+
const table = space === "image" ? "image_embeddings" : "embeddings";
|
|
2198
|
+
const row = this.getPreparedStatement(`querySimilarEmbeddingsOneVector::${space}`, `SELECT vector FROM ${table} WHERE node_id = ?`).get(nodeId);
|
|
2199
|
+
return row?.vector;
|
|
2200
|
+
}
|
|
2201
|
+
/**
|
|
2202
|
+
* Zero-copy Float32Array view over a float32 BLOB. Byte-order and values
|
|
2203
|
+
* are identical to `parseEmbedding`'s BLOB branch; alignment is guarded
|
|
2204
|
+
* (Node Buffers are 8-byte aligned from the pool, but a sliced view may
|
|
2205
|
+
* not be 4-byte aligned — those fall back to `parseEmbedding`).
|
|
2206
|
+
*/
|
|
2207
|
+
parseEmbeddingF32(value) {
|
|
2208
|
+
const byteOffset = value.byteOffset;
|
|
2209
|
+
const byteLength = value.byteLength;
|
|
2210
|
+
if (byteOffset % 4 === 0 && byteLength % 4 === 0) {
|
|
2211
|
+
return new Float32Array(value.buffer, byteOffset, byteLength / 4);
|
|
2212
|
+
}
|
|
2213
|
+
const aligned = Buffer.from(value);
|
|
2214
|
+
return new Float32Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 4);
|
|
2215
|
+
}
|
|
2216
|
+
parseEmbedding(value) {
|
|
2217
|
+
// Current databases store compact float32 blobs. The JSON branch keeps old
|
|
2218
|
+
// databases readable so users do not need a manual migration.
|
|
2219
|
+
if (Buffer.isBuffer(value) || value instanceof Uint8Array) {
|
|
2220
|
+
const buffer = Buffer.from(value);
|
|
2221
|
+
const vector = [];
|
|
2222
|
+
for (let offset = 0; offset + 3 < buffer.length; offset += 4) {
|
|
2223
|
+
vector.push(buffer.readFloatLE(offset));
|
|
2224
|
+
}
|
|
2225
|
+
return vector;
|
|
2226
|
+
}
|
|
2227
|
+
if (typeof value !== "string")
|
|
2228
|
+
return [];
|
|
2229
|
+
const parsed = JSON.parse(value);
|
|
2230
|
+
return Array.isArray(parsed)
|
|
2231
|
+
? parsed.filter((item) => typeof item === "number")
|
|
2232
|
+
: [];
|
|
2233
|
+
}
|
|
2234
|
+
serializeEmbedding(vector) {
|
|
2235
|
+
// Float32 precision is enough for retrieval ranking and cuts storage size
|
|
2236
|
+
// roughly in half compared with float64 arrays.
|
|
2237
|
+
const buffer = Buffer.allocUnsafe(vector.length * 4);
|
|
2238
|
+
for (let index = 0; index < vector.length; index += 1) {
|
|
2239
|
+
buffer.writeFloatLE(vector[index], index * 4);
|
|
2240
|
+
}
|
|
2241
|
+
return buffer;
|
|
2242
|
+
}
|
|
2243
|
+
mapSortField(field) {
|
|
2244
|
+
switch (field) {
|
|
2245
|
+
case "importance":
|
|
2246
|
+
return "importance";
|
|
2247
|
+
case "createdAt":
|
|
2248
|
+
return "created_at";
|
|
2249
|
+
case "updatedAt":
|
|
2250
|
+
return "updated_at";
|
|
2251
|
+
case "accessCount":
|
|
2252
|
+
return "access_count";
|
|
2253
|
+
case "trustScore":
|
|
2254
|
+
return "trust_score";
|
|
2255
|
+
case "relevance":
|
|
2256
|
+
default:
|
|
2257
|
+
return "updated_at";
|
|
2258
|
+
}
|
|
2259
|
+
}
|
|
2260
|
+
buildTagFilter(tags, tableAlias) {
|
|
2261
|
+
if (!tags || tags.length === 0)
|
|
2262
|
+
return "";
|
|
2263
|
+
const prefix = tableAlias ? `${tableAlias}.` : "";
|
|
2264
|
+
// AND across tags via EXISTS subqueries on the indexed node_tags
|
|
2265
|
+
// table. The planner uses the (tag) index for each subquery, which
|
|
2266
|
+
// is dramatically faster than the legacy `tags LIKE '%"tag"%'`.
|
|
2267
|
+
return tags
|
|
2268
|
+
.map(() => `EXISTS (SELECT 1 FROM node_tags nt ` +
|
|
2269
|
+
`WHERE nt.node_id = ${prefix}id AND nt.tag = ?)`)
|
|
2270
|
+
.join(" AND ");
|
|
2271
|
+
}
|
|
2272
|
+
buildMetadataFilter(metadata, tableAlias) {
|
|
2273
|
+
if (!metadata || Object.keys(metadata).length === 0) {
|
|
2274
|
+
return { conditions: [], params: [] };
|
|
2275
|
+
}
|
|
2276
|
+
const prefix = tableAlias ? `${tableAlias}.` : "";
|
|
2277
|
+
const conditions = [];
|
|
2278
|
+
const params = [];
|
|
2279
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
2280
|
+
if (!/^[A-Za-z0-9_-]+$/.test(key))
|
|
2281
|
+
continue;
|
|
2282
|
+
conditions.push(`json_extract(${prefix}metadata, ?) = ?`);
|
|
2283
|
+
params.push(`$.${key}`, typeof value === "object" ? JSON.stringify(value) : value);
|
|
2284
|
+
}
|
|
2285
|
+
return { conditions, params };
|
|
2286
|
+
}
|
|
2287
|
+
/**
|
|
2288
|
+
* Namespace/scope predicate shared by the FTS and structured query
|
|
2289
|
+
* paths. Covers exact (`namespace`), multi (`namespaces`), and
|
|
2290
|
+
* hierarchical (`namespacePrefix`) constraints. When
|
|
2291
|
+
* `includeSharedScope` is set, the shared `default` namespace is OR'd
|
|
2292
|
+
* in — a scoped read always sees shared memories (company-brain
|
|
2293
|
+
* container semantics). Returns null when no namespace constraint is
|
|
2294
|
+
* present (unscoped searches are untouched).
|
|
2295
|
+
*/
|
|
2296
|
+
buildNamespaceCondition(filter, tableAlias) {
|
|
2297
|
+
const col = tableAlias ? `${tableAlias}.namespace` : "namespace";
|
|
2298
|
+
const parts = [];
|
|
2299
|
+
const params = [];
|
|
2300
|
+
if (filter.namespace) {
|
|
2301
|
+
parts.push(`${col} = ?`);
|
|
2302
|
+
params.push(filter.namespace);
|
|
2303
|
+
}
|
|
2304
|
+
if (filter.namespaces && filter.namespaces.length > 0) {
|
|
2305
|
+
parts.push(`${col} IN (${filter.namespaces.map(() => "?").join(", ")})`);
|
|
2306
|
+
params.push(...filter.namespaces);
|
|
2307
|
+
}
|
|
2308
|
+
if (filter.namespacePrefix) {
|
|
2309
|
+
// Hierarchical scope match: escaped LIKE prefix so user values
|
|
2310
|
+
// cannot inject % or _ wildcards.
|
|
2311
|
+
parts.push(`${col} LIKE ? ESCAPE '\\'`);
|
|
2312
|
+
params.push(escapeLikePrefix(filter.namespacePrefix) + "%");
|
|
2313
|
+
}
|
|
2314
|
+
if (parts.length === 0)
|
|
2315
|
+
return null;
|
|
2316
|
+
const base = parts.length === 1 ? parts[0] : `(${parts.join(" OR ")})`;
|
|
2317
|
+
if (filter.includeSharedScope) {
|
|
2318
|
+
return { sql: `(${base} OR ${col} = 'default')`, params };
|
|
2319
|
+
}
|
|
2320
|
+
return { sql: base, params };
|
|
2321
|
+
}
|
|
2322
|
+
/**
|
|
2323
|
+
* Scoped-space inventory: every namespace with its live memory count.
|
|
2324
|
+
* Powers `memos scope list`.
|
|
2325
|
+
*/
|
|
2326
|
+
async listNamespaces() {
|
|
2327
|
+
const rows = this.db
|
|
2328
|
+
.prepare("SELECT namespace, COUNT(*) AS count FROM nodes GROUP BY namespace ORDER BY count DESC, namespace ASC")
|
|
2329
|
+
.all();
|
|
2330
|
+
return rows.map((r) => ({ namespace: r.namespace, count: r.count }));
|
|
2331
|
+
}
|
|
2332
|
+
}
|
|
2333
|
+
/**
|
|
2334
|
+
* Load the cipher-capable driver (an API-compatible better-sqlite3 fork
|
|
2335
|
+
* with multiple-cipher support). Kept optional: installs without it keep
|
|
2336
|
+
* the default plaintext driver and never load this module. Exported for
|
|
2337
|
+
* the `memos encrypt` / `memos decrypt` CLI commands.
|
|
2338
|
+
*/
|
|
2339
|
+
export function loadCipherDriver() {
|
|
2340
|
+
const require = createRequire(import.meta.url);
|
|
2341
|
+
try {
|
|
2342
|
+
const mod = require("better-sqlite3-multiple-ciphers");
|
|
2343
|
+
return (mod.default ?? mod);
|
|
2344
|
+
}
|
|
2345
|
+
catch {
|
|
2346
|
+
throw new Error("Encryption requires the optional driver better-sqlite3-multiple-ciphers.\n" +
|
|
2347
|
+
"Install it with: npm i better-sqlite3-multiple-ciphers");
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
/**
|
|
2351
|
+
* Encrypt a plaintext database in place. The original file is preserved
|
|
2352
|
+
* as `<path>.plaintext.bak`. Uses the standard sqlcipher_export recipe.
|
|
2353
|
+
* Returns the backup path.
|
|
2354
|
+
*/
|
|
2355
|
+
export async function encryptDatabase(dbPath, key) {
|
|
2356
|
+
return convertDatabase(dbPath, key, "encrypt");
|
|
2357
|
+
}
|
|
2358
|
+
export async function decryptDatabase(dbPath, key) {
|
|
2359
|
+
return convertDatabase(dbPath, key, "decrypt");
|
|
2360
|
+
}
|
|
2361
|
+
/**
|
|
2362
|
+
* Convert a database between plaintext and encrypted forms by copying
|
|
2363
|
+
* schema and data through an attached target (the standard
|
|
2364
|
+
* better-sqlite3-multiple-ciphers migration path — sqlcipher_export is
|
|
2365
|
+
* not available in this driver). The original file is preserved as a
|
|
2366
|
+
* `.plaintext.bak` / `.encrypted.bak` sibling.
|
|
2367
|
+
*/
|
|
2368
|
+
async function convertDatabase(dbPath, key, mode) {
|
|
2369
|
+
const Cipher = loadCipherDriver();
|
|
2370
|
+
const tmp = mode === "encrypt" ? `${dbPath}.encrypted.tmp` : `${dbPath}.decrypted.tmp`;
|
|
2371
|
+
const escapedKey = key.replace(/'/g, "''");
|
|
2372
|
+
const escapedSrc = dbPath.replace(/'/g, "''");
|
|
2373
|
+
const target = new Cipher(tmp);
|
|
2374
|
+
try {
|
|
2375
|
+
if (mode === "encrypt") {
|
|
2376
|
+
target.pragma(`key = '${escapedKey}'`);
|
|
2377
|
+
}
|
|
2378
|
+
// The SOURCE attaches with a key only when it is encrypted (decrypt);
|
|
2379
|
+
// a plaintext ATTACH must omit the KEY clause entirely — an explicit
|
|
2380
|
+
// empty key makes the driver expect an encrypted file.
|
|
2381
|
+
const sourceKey = mode === "decrypt" ? ` KEY '${escapedKey}'` : " KEY ''";
|
|
2382
|
+
target.exec(`ATTACH DATABASE '${escapedSrc}' AS src${sourceKey}`);
|
|
2383
|
+
const objects = target
|
|
2384
|
+
.prepare("SELECT type, name, sql FROM src.sqlite_master " +
|
|
2385
|
+
"WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' " +
|
|
2386
|
+
// Exclude only the FTS5 SHADOW tables — the virtual
|
|
2387
|
+
// table itself must be recreated (then rebuilt).
|
|
2388
|
+
"AND name NOT LIKE 'nodes\\_fts\\_%' ESCAPE '\\'")
|
|
2389
|
+
.all();
|
|
2390
|
+
const tables = objects.filter((o) => o.type === "table");
|
|
2391
|
+
const indexes = objects.filter((o) => o.type === "index");
|
|
2392
|
+
const triggers = objects.filter((o) => o.type === "trigger");
|
|
2393
|
+
// Tables first (the FTS virtual table is created empty), then data,
|
|
2394
|
+
// then an FTS rebuild, then indexes and triggers last so nothing
|
|
2395
|
+
// fires mid-copy.
|
|
2396
|
+
for (const table of tables)
|
|
2397
|
+
target.exec(table.sql);
|
|
2398
|
+
for (const table of tables) {
|
|
2399
|
+
if (/CREATE VIRTUAL TABLE/i.test(table.sql))
|
|
2400
|
+
continue;
|
|
2401
|
+
target.exec(`INSERT INTO main."${table.name}" SELECT * FROM src."${table.name}"`);
|
|
2402
|
+
}
|
|
2403
|
+
try {
|
|
2404
|
+
// NOTE: unqualified — FTS5's special insert syntax does not
|
|
2405
|
+
// accept schema-qualified table names.
|
|
2406
|
+
target.exec(`INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')`);
|
|
2407
|
+
}
|
|
2408
|
+
catch (error) {
|
|
2409
|
+
if (!error.message.includes("no such table")) {
|
|
2410
|
+
console.error("[convert] FTS rebuild failed:", error.message);
|
|
2411
|
+
}
|
|
2412
|
+
}
|
|
2413
|
+
for (const index of indexes)
|
|
2414
|
+
target.exec(index.sql);
|
|
2415
|
+
for (const trigger of triggers)
|
|
2416
|
+
target.exec(trigger.sql);
|
|
2417
|
+
target.exec("DETACH DATABASE src");
|
|
2418
|
+
}
|
|
2419
|
+
finally {
|
|
2420
|
+
target.close();
|
|
2421
|
+
}
|
|
2422
|
+
const fs = await import("node:fs");
|
|
2423
|
+
const backup = mode === "encrypt" ? `${dbPath}.plaintext.bak` : `${dbPath}.encrypted.bak`;
|
|
2424
|
+
fs.renameSync(dbPath, backup);
|
|
2425
|
+
fs.renameSync(tmp, dbPath);
|
|
2426
|
+
return backup;
|
|
2427
|
+
}
|
|
2428
|
+
//# sourceMappingURL=sqlite.js.map
|