renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,201 +0,0 @@
1
- "use strict";
2
- // Vault-staleness guard (D1). The production indexer once served a vault
3
- // checkout that was 330 commits / 12 days stale and nothing alarmed — the fix
4
- // is to make staleness a loud, machine-readable signal at startup instead of a
5
- // silent property of a mis-configured VAULT_PATH.
6
- //
7
- // LOCAL-ONLY by design: this module never runs `git fetch` or any network call.
8
- // It compares HEAD against the ALREADY-FETCHED upstream ref (refs/remotes/*)
9
- // and HEAD's own commit age. A stale *fetch* is a deployment concern (a cron
10
- // `git fetch` upstream); a stale *index* at read time is the failure D1
11
- // shipped, and both signals catch that class.
12
- var __importDefault = (this && this.__importDefault) || function (mod) {
13
- return (mod && mod.__esModule) ? mod : { "default": mod };
14
- };
15
- Object.defineProperty(exports, "__esModule", { value: true });
16
- exports.checkVaultStaleness = checkVaultStaleness;
17
- exports.stalenessMode = stalenessMode;
18
- exports.stalenessArmed = stalenessArmed;
19
- exports.stalenessJsonField = stalenessJsonField;
20
- exports.stalenessGate = stalenessGate;
21
- const child_process_1 = require("child_process");
22
- const fs_1 = __importDefault(require("fs"));
23
- const path_1 = __importDefault(require("path"));
24
- const DAY_MS = 24 * 60 * 60 * 1000;
25
- function git(cwd, args) {
26
- try {
27
- return (0, child_process_1.execFileSync)('git', args, { cwd, stdio: ['ignore', 'pipe', 'ignore'] })
28
- .toString()
29
- .trim();
30
- }
31
- catch {
32
- return null;
33
- }
34
- }
35
- /**
36
- * Resolve the upstream ref of the current branch WITHOUT any network I/O:
37
- * branch.<name>.remote + branch.<name>.merge → refs/remotes/<remote>/<branch>.
38
- * Detached HEAD or no upstream → null (age signal still applies).
39
- */
40
- function upstreamRef(cwd) {
41
- const branch = git(cwd, ['rev-parse', '--abbrev-ref', 'HEAD']);
42
- if (!branch || branch === 'HEAD')
43
- return null;
44
- const remote = git(cwd, ['config', '--get', `branch.${branch}.remote`]);
45
- const merge = git(cwd, ['config', '--get', `branch.${branch}.merge`]);
46
- if (!remote || !merge)
47
- return null;
48
- const ref = merge.replace('refs/heads/', `refs/remotes/${remote}/`);
49
- // The ref must EXIST locally (fetched at some point). A ref that only
50
- // exists on the remote is invisible here — that is by design.
51
- if (!git(cwd, ['rev-parse', '--verify', '--quiet', ref]))
52
- return null;
53
- return ref;
54
- }
55
- /**
56
- * Evaluate vault staleness for the directory the indexer is about to serve.
57
- * Zero model calls, zero network. Pure filesystem + local git metadata.
58
- */
59
- function checkVaultStaleness(vaultPath, opts = {}) {
60
- const maxHeadAgeDays = opts.maxHeadAgeDays ?? 3;
61
- // WHICH checkout is this verdict about? The vault path itself when it is a work-tree root
62
- // (`vaultPath/.git` exists — a directory, or a file for a linked worktree or submodule),
63
- // otherwise the ENCLOSING work tree that contains it. A vault that is a subdirectory of a
64
- // project repo is an ordinary layout, and answering "plain, never alarm" for it would
65
- // silently serve exactly the stale checkout this guard exists to catch: measured on a
66
- // 28-day-old repository, its root reported stale while `<root>/notes` reported
67
- // `{stale:false, kind:'plain'}`.
68
- //
69
- // A nested vault is judged by the SAME checkout-level signals as a root one — behind the
70
- // already-fetched upstream ref, and HEAD commit age — so two vaults inside one repository
71
- // cannot mean two different things.
72
- //
73
- // The enclosing root is often the PLATFORM repository, and that is stated here rather than
74
- // rediscovered: `engine/` is a subtree of the platform repo, so `git rev-parse
75
- // --show-toplevel` run from inside `engine/` returns the platform root — which is exactly why
76
- // `engine/scripts/ci/local.sh` (lines 25-28) and `engine/scripts/eval/run-cycle.sh`
77
- // (lines 3-6) resolve their own root from `$0` instead of asking git. So a vault under this
78
- // checkout is judged against the platform checkout, and `repoRoot` records that.
79
- const isWorkTreeRoot = fs_1.default.existsSync(path_1.default.join(vaultPath, '.git'));
80
- const repoRoot = isWorkTreeRoot ? vaultPath : git(vaultPath, ['rev-parse', '--show-toplevel']);
81
- if (!repoRoot) {
82
- // In no work tree at all: nothing to judge, never alarm on a plain folder.
83
- return { stale: false, reasons: [], kind: 'plain' };
84
- }
85
- const reasons = [];
86
- const out = { stale: false, reasons: [], kind: 'git', repoRoot };
87
- // A nested vault is judged against the ENCLOSING checkout, so every reason it produces names
88
- // that root: a refusal whose banner read only "HEAD is 3 commit(s) behind" would not tell the
89
- // operator WHICH repository is behind, and `fatal` exits before anything else is printed.
90
- const scope = repoRoot === vaultPath ? '' : ` (enclosing checkout ${repoRoot})`;
91
- const head = git(repoRoot, ['rev-parse', 'HEAD']);
92
- out.head = head ?? undefined;
93
- if (!head)
94
- return out; // unreadable repo — do not guess, do not alarm
95
- const upstream = upstreamRef(repoRoot);
96
- out.upstream = upstream;
97
- if (upstream) {
98
- const aheadBehind = git(repoRoot, ['rev-list', '--left-right', '--count', `${upstream}...${head}`]);
99
- if (aheadBehind) {
100
- const [behindStr] = aheadBehind.split(/\s+/);
101
- const behind = parseInt(behindStr, 10);
102
- out.behind = behind;
103
- if (behind > 0) {
104
- reasons.push(`HEAD is ${behind} commit(s) behind local upstream ref ${upstream}${scope}`);
105
- }
106
- }
107
- }
108
- const ts = git(repoRoot, ['log', '-1', '--format=%ct', 'HEAD']);
109
- if (ts) {
110
- const ageDays = Math.max(0, (Date.now() - parseInt(ts, 10) * 1000) / DAY_MS);
111
- out.headAgeDays = Math.round(ageDays * 10) / 10;
112
- if (ageDays > maxHeadAgeDays) {
113
- reasons.push(`HEAD commit is ${out.headAgeDays}d old (>${maxHeadAgeDays}d) — the checkout may not be receiving updates${scope}`);
114
- }
115
- }
116
- out.stale = reasons.length > 0;
117
- out.reasons = reasons;
118
- return out;
119
- }
120
- /** RENOR_VAULT_STALENESS=off|warn|fatal (default warn; empty/unset = warn). */
121
- function stalenessMode() {
122
- const raw = (process.env.RENOR_VAULT_STALENESS ?? '').trim().toLowerCase();
123
- if (raw === 'off' || raw === 'warn' || raw === 'fatal')
124
- return raw;
125
- return 'warn';
126
- }
127
- /**
128
- * Is the guard EXPLICITLY armed?
129
- *
130
- * `RENOR_VAULT_STALENESS` unset means "the operator asked for nothing", which is not the
131
- * same as asking for `warn`: `stalenessMode()` defaults to `warn` so the startup banner is
132
- * loud by default, but a default the operator never typed must never change a response
133
- * shape. Only the two modes that name a behaviour explicitly count as armed.
134
- */
135
- function stalenessArmed(env = process.env) {
136
- const raw = (env.RENOR_VAULT_STALENESS ?? '').trim().toLowerCase();
137
- return raw === 'warn' || raw === 'fatal';
138
- }
139
- /**
140
- * The `search --json` field for a staleness verdict, or nothing.
141
- *
142
- * The machine-readable transport is a FROZEN public contract: its own test
143
- * (`tests/context/product-read-context-contract.test.ts`, "adds nothing to the
144
- * response when the flag is unset") requires the un-armed response to be
145
- * byte-identical to the shipped one — no new top-level key.
146
- *
147
- * That is why the verdict is OPT-IN, and why the earlier "stale-only" rule was not enough.
148
- * Stale-only kept the key off a healthy vault, but the shipped contract test's fixture is a
149
- * plain, non-git folder (`kind: 'plain'`, never stale), so it could not see the stale path:
150
- * a stale vault with `RENOR_VAULT_STALENESS` UNSET still returned `['hits','vaultStaleness']`
151
- * and the repo's own assertion failed on exactly that vault
152
- * (`expected [ 'hits', 'vaultStaleness' ] to deeply equal [ 'hits' ]`). Measured, not reasoned.
153
- *
154
- * So the field is attached only when the guard is EXPLICITLY armed (see `stalenessArmed`) AND
155
- * the vault is stale. An operator who types `RENOR_VAULT_STALENESS=warn` gets the signal on
156
- * the client surface as well as on stderr; an operator who types nothing gets a response
157
- * byte-identical to the shipped one, stale or not. `off` mutes the field as it mutes the banner.
158
- */
159
- function stalenessJsonField(staleness, opts = {}) {
160
- const mode = opts.mode ?? stalenessMode();
161
- const armed = opts.armed ?? stalenessArmed();
162
- if (!armed || mode === 'off' || !staleness.stale)
163
- return {};
164
- return { vaultStaleness: staleness };
165
- }
166
- /**
167
- * Gate helper for cli.ts setup(): evaluate + render. Pure — no process.exit,
168
- * no console. The caller prints stderrLines and may refuse to continue.
169
- *
170
- * WHICH SIGNAL IS FATAL, stated because the two are different kinds of evidence:
171
- *
172
- * - HARD, fatal-capable: `behind > 0` — HEAD is behind the ALREADY-FETCHED upstream ref.
173
- * The local checkout provably does not have commits upstream already has, so refusing to
174
- * serve is refusing to serve content that is known to be old.
175
- * - ADVISORY, never fatal: `headAgeDays > maxHeadAgeDays` — the checkout has received no
176
- * commits for N days. It exists because `behind` cannot see a stale FETCH (nothing fetched
177
- * means nothing to be behind of), but age alone is a heuristic about activity, not proof
178
- * that the content is outdated: a quiet vault whose HEAD still matches its upstream would
179
- * otherwise be taken offline by `fatal` with no knob to tune the threshold.
180
- *
181
- * So `fatal` fires only on the hard signal. An age-only verdict still renders its banner and
182
- * still reaches the JSON field when the guard is armed; it just does not stop serving.
183
- */
184
- function stalenessGate(vaultPath, opts = {}) {
185
- const mode = opts.mode ?? stalenessMode();
186
- const staleness = checkVaultStaleness(vaultPath, opts);
187
- const hardEvidence = (staleness.behind ?? 0) > 0;
188
- const stderrLines = [];
189
- if (staleness.stale && mode !== 'off') {
190
- for (const r of staleness.reasons) {
191
- stderrLines.push(`[staleness] VAULT STALE: ${r}`);
192
- }
193
- // The hint must not contradict the narrowed rule: telling an operator who is ALREADY in
194
- // `fatal` mode to "set =fatal to fail startup" is noise, and calling an age-only verdict
195
- // "out-of-date" asserts more than age establishes.
196
- stderrLines.push(hardEvidence
197
- ? '[staleness] serving an out-of-date vault silently caused defect D1; set RENOR_VAULT_STALENESS=fatal to fail startup, or off to mute'
198
- : '[staleness] the age signal is ADVISORY and never refuses service; only HEAD behind its already-fetched upstream is fatal. Set RENOR_VAULT_STALENESS=off to mute');
199
- }
200
- return { stderrLines, fatal: mode === 'fatal' && hardEvidence, staleness };
201
- }
@@ -1,77 +0,0 @@
1
- "use strict";
2
- /**
3
- * The engine's token counter, loaded on first use rather than at import time.
4
- *
5
- * WHY THIS IS LAZY, with the measurement rather than the intuition: requiring
6
- * `gpt-tokenizer` cost ~344 ms on Node 22, of which ~205 ms is the o200k_base
7
- * BPE ranks table — the single heaviest module on a cold start of
8
- * dist/mvp/cli.js, ahead of express (97 ms) and the MCP SDK (205 ms). Both of
9
- * the engine's entry paths need a count eventually (the chunker when it cuts a
10
- * note, the context assembler when it budgets a reply), so this is not a module
11
- * that can be dropped; it is one that a wake does not need yet.
12
- *
13
- * WHY IT IS ./o200k AND NOT gpt-tokenizer ANY MORE: gpt-tokenizer builds a Map of
14
- * 200,000 strings, which is ~45 MiB of resident memory in every running
15
- * workspace for ever after its first count (~60 MiB under load). ./o200k counts
16
- * the same tokens from ~4 MiB of typed arrays; tests/indexer/o200k-counter.test.ts
17
- * holds its counts equal to gpt-tokenizer's. gpt-tokenizer stays a dependency for
18
- * the evaluation harness and for those tests, which are the oracle.
19
- * docs/performance/MEMORY-PER-WORKSPACE.md has the measurement.
20
- *
21
- * WHY IT IS SAFE: a count is a pure function of its input. There is no
22
- * construction cost, no ordering requirement, no ambient state to initialise and
23
- * nothing that observes whether it was imported eagerly. The only observable
24
- * difference is WHEN the vocabulary is built, and this file is the single place
25
- * that decides that.
26
- */
27
- Object.defineProperty(exports, "__esModule", { value: true });
28
- exports.encodeTokens = encodeTokens;
29
- exports.tokenizerLoaded = tokenizerLoaded;
30
- const o200k_1 = require("./o200k");
31
- /**
32
- * A run of one kind of character (all whitespace, or none) this long is not encoded. The encoder's merge loop costs
33
- * time quadratic in the length of such a run: 60,000 characters of `x` take ~5 s, where prose of the same size takes
34
- * ~3 ms. Real text has no runs this long, so only a pathological or pasted blob reaches it.
35
- */
36
- const LONG_RUN = 1_000;
37
- const RUN_RE = /\S+|\s+/g;
38
- /**
39
- * Characters per token for a long run, measured on this encoder (o200k): 60,000 `x` are 7,500 tokens and 60,000
40
- * spaces are 471. A blob of base64 or minified code is nearer the first, so the estimate is a middle figure for
41
- * non-space runs, and a generous one for whitespace.
42
- */
43
- const CHARS_PER_TOKEN = { text: 4, space: 128 };
44
- /**
45
- * The token count of `text`, with the counter loaded on first call.
46
- *
47
- * Special token syntax (`<|im_start|>`, `<|endoftext|>`, …) is counted as plain
48
- * text, not refused: it can legitimately appear inside note CONTENT, and
49
- * gpt-tokenizer rejects such text as chat protocol by default. Counting it as
50
- * plain text is what keeps indexing from failing closed on an imported ShareGPT
51
- * history.
52
- *
53
- * A run of at least LONG_RUN characters is estimated (see CHARS_PER_TOKEN) and taken out of the text the encoder
54
- * sees, so the estimate is not its exact count. Text without such a run is counted exactly, as before.
55
- */
56
- function encodeTokens(text) {
57
- let estimated = 0;
58
- let rest = text;
59
- if (text.length >= LONG_RUN) {
60
- const runs = text.match(RUN_RE) ?? [];
61
- if (runs.some(run => run.length >= LONG_RUN)) {
62
- rest = runs
63
- .map(run => {
64
- if (run.length < LONG_RUN)
65
- return run;
66
- estimated += Math.ceil(run.length / (/\s/.test(run) ? CHARS_PER_TOKEN.space : CHARS_PER_TOKEN.text));
67
- return ' ';
68
- })
69
- .join('');
70
- }
71
- }
72
- return (0, o200k_1.countO200k)(rest) + estimated;
73
- }
74
- /** True once the counter module has actually been loaded. Diagnostics and tests. */
75
- function tokenizerLoaded() {
76
- return (0, o200k_1.o200kLoaded)();
77
- }
@@ -1,170 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.walkVault = walkVault;
7
- exports.walkVaultStats = walkVaultStats;
8
- exports.statVaultFiles = statVaultFiles;
9
- exports.vaultIgnores = vaultIgnores;
10
- exports.isIgnoredByVault = isIgnoredByVault;
11
- const promises_1 = __importDefault(require("fs/promises"));
12
- const fs_1 = __importDefault(require("fs"));
13
- const buffer_1 = require("buffer");
14
- const path_1 = __importDefault(require("path"));
15
- const DEFAULT_IGNORE = ['.obsidian/', '.trash/', '.semantic-memory/', '.memory-index/', 'node_modules/'];
16
- /**
17
- * The vault's ignore test, built once per walk instead of rebuilt for every entry: a relative path is
18
- * ignored when a pattern names it, a directory above it, a directory anywhere inside it, or its last segment.
19
- */
20
- function compileIgnore(patterns) {
21
- const rules = patterns.map(p => {
22
- const clean = p.replace(/\/$/, '');
23
- return { clean, dir: clean + '/', inside: '/' + clean + '/', tail: '/' + clean };
24
- });
25
- return rel => rules.some(r => rel === r.clean || rel.startsWith(r.dir) || rel.includes(r.inside) || rel.endsWith(r.tail));
26
- }
27
- /** The default ignore patterns, then `extra`, then the lines of the vault's .semanticignore (blank and `#` lines skipped). */
28
- async function vaultIgnorePatterns(vaultPath, extra = []) {
29
- const patterns = [...DEFAULT_IGNORE, ...extra];
30
- try {
31
- const raw = await promises_1.default.readFile(path_1.default.join(vaultPath, '.semanticignore'), 'utf8');
32
- patterns.push(...raw.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#')));
33
- }
34
- catch { /* no .semanticignore */ }
35
- return patterns;
36
- }
37
- async function walkVault(vaultPath, options = {}) {
38
- const ignored = compileIgnore(await vaultIgnorePatterns(vaultPath, options.ignorePatterns));
39
- const results = [];
40
- /**
41
- * `relDir` is the folder's path below the vault. `raw` is the folder as bytes and `shown` its path below the
42
- * vault as a message writes it; both are set once a name on the way down is not valid UTF-8, because `dir`
43
- * then names nothing real.
44
- */
45
- async function walk(dir, relDir, raw, shown) {
46
- // Names are read as bytes only to report the ones that are not UTF-8; otherwise as strings, which Node decodes
47
- // the same way. Every save walks the vault, so this and the join below are paid per file per save.
48
- const entries = options.onNonUtf8Markdown
49
- ? await promises_1.default.readdir(raw ?? dir, { withFileTypes: true, encoding: 'buffer' })
50
- : await promises_1.default.readdir(dir, { withFileTypes: true });
51
- // `path.join(dir, name)` for a name read from a folder (one segment, never `.` or `..`): the folder as join
52
- // writes it, joined once per folder rather than once per entry.
53
- const folder = path_1.default.join(dir, 'x').slice(0, -1);
54
- for (const entry of entries) {
55
- const bytes = typeof entry.name === 'string' ? null : entry.name;
56
- const name = bytes ? bytes.toString('utf8') : entry.name;
57
- const full = folder + name;
58
- const rel = relDir ? relDir + path_1.default.sep + name : name;
59
- if (ignored(rel))
60
- continue;
61
- if (options.onNonUtf8Markdown && bytes && (raw || !(0, buffer_1.isUtf8)(bytes))) {
62
- const prefix = shown ?? relDir.split(path_1.default.sep).filter(Boolean).map(showName).join('/');
63
- const label = prefix ? `${prefix}/${showName(bytes)}` : showName(bytes);
64
- if (entry.isDirectory())
65
- await walk(full, rel, Buffer.concat([raw ?? Buffer.from(dir), Buffer.from(path_1.default.sep), bytes]), label);
66
- else if (entry.isFile() && name.endsWith('.md'))
67
- options.onNonUtf8Markdown(label);
68
- continue;
69
- }
70
- if (entry.isDirectory())
71
- await walk(full, rel);
72
- else if (entry.isFile() && name.endsWith('.md'))
73
- results.push(full);
74
- }
75
- }
76
- await walk(vaultPath, '');
77
- return results;
78
- }
79
- /**
80
- * A name as a message writes it: valid UTF-8 as it is, every other byte as `\xNN`, and a
81
- * backslash or control character the same way (the backslash doubled), so the result is
82
- * unambiguous, one line, and cannot carry a terminal escape sequence.
83
- */
84
- function showName(name) {
85
- const bytes = typeof name === 'string' ? Buffer.from(name) : name;
86
- let shown = '';
87
- for (let i = 0; i < bytes.length;) {
88
- const lead = bytes[i];
89
- const length = lead < 0xc2 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : lead < 0xf5 ? 4 : 0;
90
- const isC1Control = lead === 0xc2 && bytes[i + 1] < 0xa0; // U+0080-U+009F
91
- if (lead >= 0x80 && length > 0 && i + length <= bytes.length && (0, buffer_1.isUtf8)(bytes.subarray(i, i + length)) && !isC1Control) {
92
- shown += bytes.toString('utf8', i, i + length);
93
- i += length;
94
- continue;
95
- }
96
- if (lead === 0x5c)
97
- shown += '\\\\';
98
- else if (lead >= 0x80 || lead < 0x20 || lead === 0x7f)
99
- shown += `\\x${lead.toString(16).padStart(2, '0')}`;
100
- else
101
- shown += String.fromCharCode(lead);
102
- i += 1;
103
- }
104
- return shown;
105
- }
106
- const STAT_CHUNK = 512;
107
- /**
108
- * The files walkVault lists, each with the stat fields above, for a caller that wants to know what changed
109
- * without reading anything. Stats are synchronous (a few microseconds each, against tens for an asynchronous
110
- * call that fans out over the thread pool) in chunks, with a turn of the event loop between them. A file that
111
- * vanishes between the listing and its stat is left out; one whose stat fails for another reason is listed with
112
- * a stamp that never matches, so whoever reads it reports the error as a full pass would.
113
- */
114
- async function walkVaultStats(vaultPath, options = {}) {
115
- const files = await walkVault(vaultPath, options);
116
- const stats = [];
117
- for (let i = 0; i < files.length; i += STAT_CHUNK) {
118
- for (const file of files.slice(i, i + STAT_CHUNK)) {
119
- const s = statVaultFile(file);
120
- if (s)
121
- stats.push(s);
122
- }
123
- if (i + STAT_CHUNK < files.length)
124
- await new Promise(resolve => setImmediate(resolve));
125
- }
126
- return stats;
127
- }
128
- /** One file's stat (see walkVaultStats): undefined when it is gone, a stamp that never matches when its stat fails. */
129
- function statVaultFile(file) {
130
- try {
131
- const s = fs_1.default.statSync(file, { throwIfNoEntry: false });
132
- return s ? { file, mtimeMs: s.mtimeMs, ctimeMs: s.ctimeMs, size: s.size, ino: s.ino } : undefined;
133
- }
134
- catch {
135
- return { file, mtimeMs: NaN, ctimeMs: NaN, size: -1, ino: -1 };
136
- }
137
- }
138
- /**
139
- * The stat of each of the listed files that is there, for a caller that names the files it wants to check (a save's)
140
- * instead of walking the vault. A file that is gone is left out, as walkVaultStats leaves it out.
141
- */
142
- function statVaultFiles(files) {
143
- const stats = [];
144
- for (const file of files) {
145
- const s = statVaultFile(file);
146
- if (s)
147
- stats.push(s);
148
- }
149
- return stats;
150
- }
151
- /**
152
- * The vault's ignore rules as walkVault applies them, for a caller that names files instead of listing them: true when the
153
- * walk would never reach the file, which it prunes at an ignored folder as well as skipping an ignored file. Reads
154
- * .semanticignore once, when asked.
155
- */
156
- async function vaultIgnores(vaultPath) {
157
- const ignored = compileIgnore(await vaultIgnorePatterns(vaultPath));
158
- return (file) => {
159
- const parts = path_1.default.relative(vaultPath, file).split(path_1.default.sep);
160
- return parts.some((_, i) => ignored(parts.slice(0, i + 1).join(path_1.default.sep)));
161
- };
162
- }
163
- /**
164
- * Whether the vault's own ignore rules (the defaults and .semanticignore) leave out a vault-relative path. The tools that
165
- * address a note by id (read_memory, retire_memory) ask it, so a file the owner excluded from memory cannot be read or
166
- * moved through its path any more than it can be found by search.
167
- */
168
- async function isIgnoredByVault(vaultPath, rel) {
169
- return compileIgnore(await vaultIgnorePatterns(vaultPath))(rel.split('/').join(path_1.default.sep));
170
- }
@@ -1,113 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.run = run;
7
- const node_fs_1 = __importDefault(require("node:fs"));
8
- const settings_1 = require("./settings");
9
- const store_1 = require("./store");
10
- /**
11
- * Manage one memory folder's learning log: `status`, `export`, `prune` and `delete-all`. They work whether or not recording is
12
- * on, so a log written before it was switched off can still be read or deleted, and they never create a log that does not exist.
13
- *
14
- * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts status
15
- * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts export [--out <file>] (the file is written owner-only)
16
- * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts prune
17
- * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts delete-all --yes
18
- *
19
- * Stop a running server before deleting, as the data-deletion runbook does; a running server would otherwise keep its open
20
- * handle to the file. The counts and the export never contain a key secret or a memory's text.
21
- */
22
- const USAGE = 'usage: learning-cli <status | export [--out <file>] | prune | delete-all --yes>';
23
- function print(value) {
24
- process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
25
- }
26
- function run(argv, env = process.env) {
27
- const memoryDir = (env.MVP_MEMORY_DIR ?? '').trim();
28
- if (memoryDir === '')
29
- throw new Error('MVP_MEMORY_DIR is required: the memory folder whose learning log this manages');
30
- const [command, ...rest] = argv;
31
- const file = (0, settings_1.learningDbPath)(memoryDir);
32
- const open = () => {
33
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
34
- if (log === null)
35
- throw new Error(`no learning log at ${file}: nothing has been recorded there`);
36
- return log;
37
- };
38
- switch (command) {
39
- case 'status': {
40
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
41
- if (log === null) {
42
- print({ present: false, file });
43
- return;
44
- }
45
- try {
46
- print({ present: true, file, retentionDays: (0, settings_1.learningRetentionDays)(env), counts: log.counts() });
47
- }
48
- finally {
49
- log.close();
50
- }
51
- return;
52
- }
53
- case 'export': {
54
- const out = rest[0] === '--out' ? rest[1] : undefined;
55
- if (rest[0] === '--out' && (out === undefined || out === ''))
56
- throw new Error(USAGE);
57
- const log = open();
58
- let text;
59
- try {
60
- text = `${JSON.stringify(log.exportLog(), null, 2)}\n`;
61
- }
62
- finally {
63
- log.close();
64
- }
65
- if (out === undefined) {
66
- process.stdout.write(text);
67
- }
68
- else {
69
- node_fs_1.default.writeFileSync(out, text, { mode: 0o600 });
70
- process.stderr.write(`wrote ${out}\n`);
71
- }
72
- return;
73
- }
74
- case 'prune': {
75
- const log = open();
76
- try {
77
- print(log.prune());
78
- }
79
- finally {
80
- log.close();
81
- }
82
- return;
83
- }
84
- case 'delete-all': {
85
- if (rest[0] !== '--yes')
86
- throw new Error('delete-all removes every recorded row and needs --yes');
87
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
88
- if (log === null) {
89
- print({ deleted: false, file, reason: 'no learning log' });
90
- return;
91
- }
92
- try {
93
- log.deleteAll();
94
- print({ deleted: true, file, counts: log.counts() });
95
- }
96
- finally {
97
- log.close();
98
- }
99
- return;
100
- }
101
- default:
102
- throw new Error(USAGE);
103
- }
104
- }
105
- if (require.main === module) {
106
- try {
107
- run(process.argv.slice(2));
108
- }
109
- catch (err) {
110
- process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
111
- process.exitCode = 1;
112
- }
113
- }