renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,201 +0,0 @@
1
- "use strict";
2
- // Vault-staleness guard (D1). The production indexer once served a vault
3
- // checkout that was 330 commits / 12 days stale and nothing alarmed — the fix
4
- // is to make staleness a loud, machine-readable signal at startup instead of a
5
- // silent property of a mis-configured VAULT_PATH.
6
- //
7
- // LOCAL-ONLY by design: this module never runs `git fetch` or any network call.
8
- // It compares HEAD against the ALREADY-FETCHED upstream ref (refs/remotes/*)
9
- // and HEAD's own commit age. A stale *fetch* is a deployment concern (a cron
10
- // `git fetch` upstream); a stale *index* at read time is the failure D1
11
- // shipped, and both signals catch that class.
12
- var __importDefault = (this && this.__importDefault) || function (mod) {
13
- return (mod && mod.__esModule) ? mod : { "default": mod };
14
- };
15
- Object.defineProperty(exports, "__esModule", { value: true });
16
- exports.checkVaultStaleness = checkVaultStaleness;
17
- exports.stalenessMode = stalenessMode;
18
- exports.stalenessArmed = stalenessArmed;
19
- exports.stalenessJsonField = stalenessJsonField;
20
- exports.stalenessGate = stalenessGate;
21
- const child_process_1 = require("child_process");
22
- const fs_1 = __importDefault(require("fs"));
23
- const path_1 = __importDefault(require("path"));
24
- const DAY_MS = 24 * 60 * 60 * 1000;
25
- function git(cwd, args) {
26
- try {
27
- return (0, child_process_1.execFileSync)('git', args, { cwd, stdio: ['ignore', 'pipe', 'ignore'] })
28
- .toString()
29
- .trim();
30
- }
31
- catch {
32
- return null;
33
- }
34
- }
35
- /**
36
- * Resolve the upstream ref of the current branch WITHOUT any network I/O:
37
- * branch.<name>.remote + branch.<name>.merge → refs/remotes/<remote>/<branch>.
38
- * Detached HEAD or no upstream → null (age signal still applies).
39
- */
40
- function upstreamRef(cwd) {
41
- const branch = git(cwd, ['rev-parse', '--abbrev-ref', 'HEAD']);
42
- if (!branch || branch === 'HEAD')
43
- return null;
44
- const remote = git(cwd, ['config', '--get', `branch.${branch}.remote`]);
45
- const merge = git(cwd, ['config', '--get', `branch.${branch}.merge`]);
46
- if (!remote || !merge)
47
- return null;
48
- const ref = merge.replace('refs/heads/', `refs/remotes/${remote}/`);
49
- // The ref must EXIST locally (fetched at some point). A ref that only
50
- // exists on the remote is invisible here — that is by design.
51
- if (!git(cwd, ['rev-parse', '--verify', '--quiet', ref]))
52
- return null;
53
- return ref;
54
- }
55
- /**
56
- * Evaluate vault staleness for the directory the indexer is about to serve.
57
- * Zero model calls, zero network. Pure filesystem + local git metadata.
58
- */
59
- function checkVaultStaleness(vaultPath, opts = {}) {
60
- const maxHeadAgeDays = opts.maxHeadAgeDays ?? 3;
61
- // WHICH checkout is this verdict about? The vault path itself when it is a work-tree root
62
- // (`vaultPath/.git` exists — a directory, or a file for a linked worktree or submodule),
63
- // otherwise the ENCLOSING work tree that contains it. A vault that is a subdirectory of a
64
- // project repo is an ordinary layout, and answering "plain, never alarm" for it would
65
- // silently serve exactly the stale checkout this guard exists to catch: measured on a
66
- // 28-day-old repository, its root reported stale while `<root>/notes` reported
67
- // `{stale:false, kind:'plain'}`.
68
- //
69
- // A nested vault is judged by the SAME checkout-level signals as a root one — behind the
70
- // already-fetched upstream ref, and HEAD commit age — so two vaults inside one repository
71
- // cannot mean two different things.
72
- //
73
- // The enclosing root is often the PLATFORM repository, and that is stated here rather than
74
- // rediscovered: `engine/` is a subtree of the platform repo, so `git rev-parse
75
- // --show-toplevel` run from inside `engine/` returns the platform root — which is exactly why
76
- // `engine/scripts/ci/local.sh` (lines 25-28) and `engine/scripts/eval/run-cycle.sh`
77
- // (lines 3-6) resolve their own root from `$0` instead of asking git. So a vault under this
78
- // checkout is judged against the platform checkout, and `repoRoot` records that.
79
- const isWorkTreeRoot = fs_1.default.existsSync(path_1.default.join(vaultPath, '.git'));
80
- const repoRoot = isWorkTreeRoot ? vaultPath : git(vaultPath, ['rev-parse', '--show-toplevel']);
81
- if (!repoRoot) {
82
- // In no work tree at all: nothing to judge, never alarm on a plain folder.
83
- return { stale: false, reasons: [], kind: 'plain' };
84
- }
85
- const reasons = [];
86
- const out = { stale: false, reasons: [], kind: 'git', repoRoot };
87
- // A nested vault is judged against the ENCLOSING checkout, so every reason it produces names
88
- // that root: a refusal whose banner read only "HEAD is 3 commit(s) behind" would not tell the
89
- // operator WHICH repository is behind, and `fatal` exits before anything else is printed.
90
- const scope = repoRoot === vaultPath ? '' : ` (enclosing checkout ${repoRoot})`;
91
- const head = git(repoRoot, ['rev-parse', 'HEAD']);
92
- out.head = head ?? undefined;
93
- if (!head)
94
- return out; // unreadable repo — do not guess, do not alarm
95
- const upstream = upstreamRef(repoRoot);
96
- out.upstream = upstream;
97
- if (upstream) {
98
- const aheadBehind = git(repoRoot, ['rev-list', '--left-right', '--count', `${upstream}...${head}`]);
99
- if (aheadBehind) {
100
- const [behindStr] = aheadBehind.split(/\s+/);
101
- const behind = parseInt(behindStr, 10);
102
- out.behind = behind;
103
- if (behind > 0) {
104
- reasons.push(`HEAD is ${behind} commit(s) behind local upstream ref ${upstream}${scope}`);
105
- }
106
- }
107
- }
108
- const ts = git(repoRoot, ['log', '-1', '--format=%ct', 'HEAD']);
109
- if (ts) {
110
- const ageDays = Math.max(0, (Date.now() - parseInt(ts, 10) * 1000) / DAY_MS);
111
- out.headAgeDays = Math.round(ageDays * 10) / 10;
112
- if (ageDays > maxHeadAgeDays) {
113
- reasons.push(`HEAD commit is ${out.headAgeDays}d old (>${maxHeadAgeDays}d) — the checkout may not be receiving updates${scope}`);
114
- }
115
- }
116
- out.stale = reasons.length > 0;
117
- out.reasons = reasons;
118
- return out;
119
- }
120
- /** RENOR_VAULT_STALENESS=off|warn|fatal (default warn; empty/unset = warn). */
121
- function stalenessMode() {
122
- const raw = (process.env.RENOR_VAULT_STALENESS ?? '').trim().toLowerCase();
123
- if (raw === 'off' || raw === 'warn' || raw === 'fatal')
124
- return raw;
125
- return 'warn';
126
- }
127
- /**
128
- * Is the guard EXPLICITLY armed?
129
- *
130
- * `RENOR_VAULT_STALENESS` unset means "the operator asked for nothing", which is not the
131
- * same as asking for `warn`: `stalenessMode()` defaults to `warn` so the startup banner is
132
- * loud by default, but a default the operator never typed must never change a response
133
- * shape. Only the two modes that name a behaviour explicitly count as armed.
134
- */
135
- function stalenessArmed(env = process.env) {
136
- const raw = (env.RENOR_VAULT_STALENESS ?? '').trim().toLowerCase();
137
- return raw === 'warn' || raw === 'fatal';
138
- }
139
- /**
140
- * The `search --json` field for a staleness verdict, or nothing.
141
- *
142
- * The machine-readable transport is a FROZEN public contract: its own test
143
- * (`tests/context/product-read-context-contract.test.ts`, "adds nothing to the
144
- * response when the flag is unset") requires the un-armed response to be
145
- * byte-identical to the shipped one — no new top-level key.
146
- *
147
- * That is why the verdict is OPT-IN, and why the earlier "stale-only" rule was not enough.
148
- * Stale-only kept the key off a healthy vault, but the shipped contract test's fixture is a
149
- * plain, non-git folder (`kind: 'plain'`, never stale), so it could not see the stale path:
150
- * a stale vault with `RENOR_VAULT_STALENESS` UNSET still returned `['hits','vaultStaleness']`
151
- * and the repo's own assertion failed on exactly that vault
152
- * (`expected [ 'hits', 'vaultStaleness' ] to deeply equal [ 'hits' ]`). Measured, not reasoned.
153
- *
154
- * So the field is attached only when the guard is EXPLICITLY armed (see `stalenessArmed`) AND
155
- * the vault is stale. An operator who types `RENOR_VAULT_STALENESS=warn` gets the signal on
156
- * the client surface as well as on stderr; an operator who types nothing gets a response
157
- * byte-identical to the shipped one, stale or not. `off` mutes the field as it mutes the banner.
158
- */
159
- function stalenessJsonField(staleness, opts = {}) {
160
- const mode = opts.mode ?? stalenessMode();
161
- const armed = opts.armed ?? stalenessArmed();
162
- if (!armed || mode === 'off' || !staleness.stale)
163
- return {};
164
- return { vaultStaleness: staleness };
165
- }
166
- /**
167
- * Gate helper for cli.ts setup(): evaluate + render. Pure — no process.exit,
168
- * no console. The caller prints stderrLines and may refuse to continue.
169
- *
170
- * WHICH SIGNAL IS FATAL, stated because the two are different kinds of evidence:
171
- *
172
- * - HARD, fatal-capable: `behind > 0` — HEAD is behind the ALREADY-FETCHED upstream ref.
173
- * The local checkout provably does not have commits upstream already has, so refusing to
174
- * serve is refusing to serve content that is known to be old.
175
- * - ADVISORY, never fatal: `headAgeDays > maxHeadAgeDays` — the checkout has received no
176
- * commits for N days. It exists because `behind` cannot see a stale FETCH (nothing fetched
177
- * means nothing to be behind of), but age alone is a heuristic about activity, not proof
178
- * that the content is outdated: a quiet vault whose HEAD still matches its upstream would
179
- * otherwise be taken offline by `fatal` with no knob to tune the threshold.
180
- *
181
- * So `fatal` fires only on the hard signal. An age-only verdict still renders its banner and
182
- * still reaches the JSON field when the guard is armed; it just does not stop serving.
183
- */
184
- function stalenessGate(vaultPath, opts = {}) {
185
- const mode = opts.mode ?? stalenessMode();
186
- const staleness = checkVaultStaleness(vaultPath, opts);
187
- const hardEvidence = (staleness.behind ?? 0) > 0;
188
- const stderrLines = [];
189
- if (staleness.stale && mode !== 'off') {
190
- for (const r of staleness.reasons) {
191
- stderrLines.push(`[staleness] VAULT STALE: ${r}`);
192
- }
193
- // The hint must not contradict the narrowed rule: telling an operator who is ALREADY in
194
- // `fatal` mode to "set =fatal to fail startup" is noise, and calling an age-only verdict
195
- // "out-of-date" asserts more than age establishes.
196
- stderrLines.push(hardEvidence
197
- ? '[staleness] serving an out-of-date vault silently caused defect D1; set RENOR_VAULT_STALENESS=fatal to fail startup, or off to mute'
198
- : '[staleness] the age signal is ADVISORY and never refuses service; only HEAD behind its already-fetched upstream is fatal. Set RENOR_VAULT_STALENESS=off to mute');
199
- }
200
- return { stderrLines, fatal: mode === 'fatal' && hardEvidence, staleness };
201
- }
@@ -1,77 +0,0 @@
1
- "use strict";
2
- /**
3
- * The engine's token counter, loaded on first use rather than at import time.
4
- *
5
- * WHY THIS IS LAZY, with the measurement rather than the intuition: requiring
6
- * `gpt-tokenizer` cost ~344 ms on Node 22, of which ~205 ms is the o200k_base
7
- * BPE ranks table — the single heaviest module on a cold start of
8
- * dist/mvp/cli.js, ahead of express (97 ms) and the MCP SDK (205 ms). Both of
9
- * the engine's entry paths need a count eventually (the chunker when it cuts a
10
- * note, the context assembler when it budgets a reply), so this is not a module
11
- * that can be dropped; it is one that a wake does not need yet.
12
- *
13
- * WHY IT IS ./o200k AND NOT gpt-tokenizer ANY MORE: gpt-tokenizer builds a Map of
14
- * 200,000 strings, which is ~45 MiB of resident memory in every running
15
- * workspace for ever after its first count (~60 MiB under load). ./o200k counts
16
- * the same tokens from ~4 MiB of typed arrays; tests/indexer/o200k-counter.test.ts
17
- * holds its counts equal to gpt-tokenizer's. gpt-tokenizer stays a dependency for
18
- * the evaluation harness and for those tests, which are the oracle.
19
- * docs/performance/MEMORY-PER-WORKSPACE.md has the measurement.
20
- *
21
- * WHY IT IS SAFE: a count is a pure function of its input. There is no
22
- * construction cost, no ordering requirement, no ambient state to initialise and
23
- * nothing that observes whether it was imported eagerly. The only observable
24
- * difference is WHEN the vocabulary is built, and this file is the single place
25
- * that decides that.
26
- */
27
- Object.defineProperty(exports, "__esModule", { value: true });
28
- exports.encodeTokens = encodeTokens;
29
- exports.tokenizerLoaded = tokenizerLoaded;
30
- const o200k_1 = require("./o200k");
31
- /**
32
- * A run of one kind of character (all whitespace, or none) this long is not encoded. The encoder's merge loop costs
33
- * time quadratic in the length of such a run: 60,000 characters of `x` take ~5 s, where prose of the same size takes
34
- * ~3 ms. Real text has no runs this long, so only a pathological or pasted blob reaches it.
35
- */
36
- const LONG_RUN = 1_000;
37
- const RUN_RE = /\S+|\s+/g;
38
- /**
39
- * Characters per token for a long run, measured on this encoder (o200k): 60,000 `x` are 7,500 tokens and 60,000
40
- * spaces are 471. A blob of base64 or minified code is nearer the first, so the estimate is a middle figure for
41
- * non-space runs, and a generous one for whitespace.
42
- */
43
- const CHARS_PER_TOKEN = { text: 4, space: 128 };
44
- /**
45
- * The token count of `text`, with the counter loaded on first call.
46
- *
47
- * Special token syntax (`<|im_start|>`, `<|endoftext|>`, …) is counted as plain
48
- * text, not refused: it can legitimately appear inside note CONTENT, and
49
- * gpt-tokenizer rejects such text as chat protocol by default. Counting it as
50
- * plain text is what keeps indexing from failing closed on an imported ShareGPT
51
- * history.
52
- *
53
- * A run of at least LONG_RUN characters is estimated (see CHARS_PER_TOKEN) and taken out of the text the encoder
54
- * sees, so the estimate is not its exact count. Text without such a run is counted exactly, as before.
55
- */
56
- function encodeTokens(text) {
57
- let estimated = 0;
58
- let rest = text;
59
- if (text.length >= LONG_RUN) {
60
- const runs = text.match(RUN_RE) ?? [];
61
- if (runs.some(run => run.length >= LONG_RUN)) {
62
- rest = runs
63
- .map(run => {
64
- if (run.length < LONG_RUN)
65
- return run;
66
- estimated += Math.ceil(run.length / (/\s/.test(run) ? CHARS_PER_TOKEN.space : CHARS_PER_TOKEN.text));
67
- return ' ';
68
- })
69
- .join('');
70
- }
71
- }
72
- return (0, o200k_1.countO200k)(rest) + estimated;
73
- }
74
- /** True once the counter module has actually been loaded. Diagnostics and tests. */
75
- function tokenizerLoaded() {
76
- return (0, o200k_1.o200kLoaded)();
77
- }
@@ -1,170 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.walkVault = walkVault;
7
- exports.walkVaultStats = walkVaultStats;
8
- exports.statVaultFiles = statVaultFiles;
9
- exports.vaultIgnores = vaultIgnores;
10
- exports.isIgnoredByVault = isIgnoredByVault;
11
- const promises_1 = __importDefault(require("fs/promises"));
12
- const fs_1 = __importDefault(require("fs"));
13
- const buffer_1 = require("buffer");
14
- const path_1 = __importDefault(require("path"));
15
- const DEFAULT_IGNORE = ['.obsidian/', '.trash/', '.semantic-memory/', '.memory-index/', 'node_modules/'];
16
- /**
17
- * The vault's ignore test, built once per walk instead of rebuilt for every entry: a relative path is
18
- * ignored when a pattern names it, a directory above it, a directory anywhere inside it, or its last segment.
19
- */
20
- function compileIgnore(patterns) {
21
- const rules = patterns.map(p => {
22
- const clean = p.replace(/\/$/, '');
23
- return { clean, dir: clean + '/', inside: '/' + clean + '/', tail: '/' + clean };
24
- });
25
- return rel => rules.some(r => rel === r.clean || rel.startsWith(r.dir) || rel.includes(r.inside) || rel.endsWith(r.tail));
26
- }
27
- /** The default ignore patterns, then `extra`, then the lines of the vault's .semanticignore (blank and `#` lines skipped). */
28
- async function vaultIgnorePatterns(vaultPath, extra = []) {
29
- const patterns = [...DEFAULT_IGNORE, ...extra];
30
- try {
31
- const raw = await promises_1.default.readFile(path_1.default.join(vaultPath, '.semanticignore'), 'utf8');
32
- patterns.push(...raw.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#')));
33
- }
34
- catch { /* no .semanticignore */ }
35
- return patterns;
36
- }
37
- async function walkVault(vaultPath, options = {}) {
38
- const ignored = compileIgnore(await vaultIgnorePatterns(vaultPath, options.ignorePatterns));
39
- const results = [];
40
- /**
41
- * `relDir` is the folder's path below the vault. `raw` is the folder as bytes and `shown` its path below the
42
- * vault as a message writes it; both are set once a name on the way down is not valid UTF-8, because `dir`
43
- * then names nothing real.
44
- */
45
- async function walk(dir, relDir, raw, shown) {
46
- // Names are read as bytes only to report the ones that are not UTF-8; otherwise as strings, which Node decodes
47
- // the same way. Every save walks the vault, so this and the join below are paid per file per save.
48
- const entries = options.onNonUtf8Markdown
49
- ? await promises_1.default.readdir(raw ?? dir, { withFileTypes: true, encoding: 'buffer' })
50
- : await promises_1.default.readdir(dir, { withFileTypes: true });
51
- // `path.join(dir, name)` for a name read from a folder (one segment, never `.` or `..`): the folder as join
52
- // writes it, joined once per folder rather than once per entry.
53
- const folder = path_1.default.join(dir, 'x').slice(0, -1);
54
- for (const entry of entries) {
55
- const bytes = typeof entry.name === 'string' ? null : entry.name;
56
- const name = bytes ? bytes.toString('utf8') : entry.name;
57
- const full = folder + name;
58
- const rel = relDir ? relDir + path_1.default.sep + name : name;
59
- if (ignored(rel))
60
- continue;
61
- if (options.onNonUtf8Markdown && bytes && (raw || !(0, buffer_1.isUtf8)(bytes))) {
62
- const prefix = shown ?? relDir.split(path_1.default.sep).filter(Boolean).map(showName).join('/');
63
- const label = prefix ? `${prefix}/${showName(bytes)}` : showName(bytes);
64
- if (entry.isDirectory())
65
- await walk(full, rel, Buffer.concat([raw ?? Buffer.from(dir), Buffer.from(path_1.default.sep), bytes]), label);
66
- else if (entry.isFile() && name.endsWith('.md'))
67
- options.onNonUtf8Markdown(label);
68
- continue;
69
- }
70
- if (entry.isDirectory())
71
- await walk(full, rel);
72
- else if (entry.isFile() && name.endsWith('.md'))
73
- results.push(full);
74
- }
75
- }
76
- await walk(vaultPath, '');
77
- return results;
78
- }
79
- /**
80
- * A name as a message writes it: valid UTF-8 as it is, every other byte as `\xNN`, and a
81
- * backslash or control character the same way (the backslash doubled), so the result is
82
- * unambiguous, one line, and cannot carry a terminal escape sequence.
83
- */
84
- function showName(name) {
85
- const bytes = typeof name === 'string' ? Buffer.from(name) : name;
86
- let shown = '';
87
- for (let i = 0; i < bytes.length;) {
88
- const lead = bytes[i];
89
- const length = lead < 0xc2 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : lead < 0xf5 ? 4 : 0;
90
- const isC1Control = lead === 0xc2 && bytes[i + 1] < 0xa0; // U+0080-U+009F
91
- if (lead >= 0x80 && length > 0 && i + length <= bytes.length && (0, buffer_1.isUtf8)(bytes.subarray(i, i + length)) && !isC1Control) {
92
- shown += bytes.toString('utf8', i, i + length);
93
- i += length;
94
- continue;
95
- }
96
- if (lead === 0x5c)
97
- shown += '\\\\';
98
- else if (lead >= 0x80 || lead < 0x20 || lead === 0x7f)
99
- shown += `\\x${lead.toString(16).padStart(2, '0')}`;
100
- else
101
- shown += String.fromCharCode(lead);
102
- i += 1;
103
- }
104
- return shown;
105
- }
106
- const STAT_CHUNK = 512;
107
- /**
108
- * The files walkVault lists, each with the stat fields above, for a caller that wants to know what changed
109
- * without reading anything. Stats are synchronous (a few microseconds each, against tens for an asynchronous
110
- * call that fans out over the thread pool) in chunks, with a turn of the event loop between them. A file that
111
- * vanishes between the listing and its stat is left out; one whose stat fails for another reason is listed with
112
- * a stamp that never matches, so whoever reads it reports the error as a full pass would.
113
- */
114
- async function walkVaultStats(vaultPath, options = {}) {
115
- const files = await walkVault(vaultPath, options);
116
- const stats = [];
117
- for (let i = 0; i < files.length; i += STAT_CHUNK) {
118
- for (const file of files.slice(i, i + STAT_CHUNK)) {
119
- const s = statVaultFile(file);
120
- if (s)
121
- stats.push(s);
122
- }
123
- if (i + STAT_CHUNK < files.length)
124
- await new Promise(resolve => setImmediate(resolve));
125
- }
126
- return stats;
127
- }
128
- /** One file's stat (see walkVaultStats): undefined when it is gone, a stamp that never matches when its stat fails. */
129
- function statVaultFile(file) {
130
- try {
131
- const s = fs_1.default.statSync(file, { throwIfNoEntry: false });
132
- return s ? { file, mtimeMs: s.mtimeMs, ctimeMs: s.ctimeMs, size: s.size, ino: s.ino } : undefined;
133
- }
134
- catch {
135
- return { file, mtimeMs: NaN, ctimeMs: NaN, size: -1, ino: -1 };
136
- }
137
- }
138
- /**
139
- * The stat of each of the listed files that is there, for a caller that names the files it wants to check (a save's)
140
- * instead of walking the vault. A file that is gone is left out, as walkVaultStats leaves it out.
141
- */
142
- function statVaultFiles(files) {
143
- const stats = [];
144
- for (const file of files) {
145
- const s = statVaultFile(file);
146
- if (s)
147
- stats.push(s);
148
- }
149
- return stats;
150
- }
151
- /**
152
- * The vault's ignore rules as walkVault applies them, for a caller that names files instead of listing them: true when the
153
- * walk would never reach the file, which it prunes at an ignored folder as well as skipping an ignored file. Reads
154
- * .semanticignore once, when asked.
155
- */
156
- async function vaultIgnores(vaultPath) {
157
- const ignored = compileIgnore(await vaultIgnorePatterns(vaultPath));
158
- return (file) => {
159
- const parts = path_1.default.relative(vaultPath, file).split(path_1.default.sep);
160
- return parts.some((_, i) => ignored(parts.slice(0, i + 1).join(path_1.default.sep)));
161
- };
162
- }
163
- /**
164
- * Whether the vault's own ignore rules (the defaults and .semanticignore) leave out a vault-relative path. The tools that
165
- * address a note by id (read_memory, retire_memory) ask it, so a file the owner excluded from memory cannot be read or
166
- * moved through its path any more than it can be found by search.
167
- */
168
- async function isIgnoredByVault(vaultPath, rel) {
169
- return compileIgnore(await vaultIgnorePatterns(vaultPath))(rel.split('/').join(path_1.default.sep));
170
- }
@@ -1,113 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.run = run;
7
- const node_fs_1 = __importDefault(require("node:fs"));
8
- const settings_1 = require("./settings");
9
- const store_1 = require("./store");
10
- /**
11
- * Manage one memory folder's learning log: `status`, `export`, `prune` and `delete-all`. They work whether or not recording is
12
- * on, so a log written before it was switched off can still be read or deleted, and they never create a log that does not exist.
13
- *
14
- * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts status
15
- * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts export [--out <file>] (the file is written owner-only)
16
- * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts prune
17
- * MVP_MEMORY_DIR=<memory folder> npx tsx src/learning/cli.ts delete-all --yes
18
- *
19
- * Stop a running server before deleting, as the data-deletion runbook does; a running server would otherwise keep its open
20
- * handle to the file. The counts and the export never contain a key secret or a memory's text.
21
- */
22
- const USAGE = 'usage: learning-cli <status | export [--out <file>] | prune | delete-all --yes>';
23
- function print(value) {
24
- process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
25
- }
26
- function run(argv, env = process.env) {
27
- const memoryDir = (env.MVP_MEMORY_DIR ?? '').trim();
28
- if (memoryDir === '')
29
- throw new Error('MVP_MEMORY_DIR is required: the memory folder whose learning log this manages');
30
- const [command, ...rest] = argv;
31
- const file = (0, settings_1.learningDbPath)(memoryDir);
32
- const open = () => {
33
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
34
- if (log === null)
35
- throw new Error(`no learning log at ${file}: nothing has been recorded there`);
36
- return log;
37
- };
38
- switch (command) {
39
- case 'status': {
40
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
41
- if (log === null) {
42
- print({ present: false, file });
43
- return;
44
- }
45
- try {
46
- print({ present: true, file, retentionDays: (0, settings_1.learningRetentionDays)(env), counts: log.counts() });
47
- }
48
- finally {
49
- log.close();
50
- }
51
- return;
52
- }
53
- case 'export': {
54
- const out = rest[0] === '--out' ? rest[1] : undefined;
55
- if (rest[0] === '--out' && (out === undefined || out === ''))
56
- throw new Error(USAGE);
57
- const log = open();
58
- let text;
59
- try {
60
- text = `${JSON.stringify(log.exportLog(), null, 2)}\n`;
61
- }
62
- finally {
63
- log.close();
64
- }
65
- if (out === undefined) {
66
- process.stdout.write(text);
67
- }
68
- else {
69
- node_fs_1.default.writeFileSync(out, text, { mode: 0o600 });
70
- process.stderr.write(`wrote ${out}\n`);
71
- }
72
- return;
73
- }
74
- case 'prune': {
75
- const log = open();
76
- try {
77
- print(log.prune());
78
- }
79
- finally {
80
- log.close();
81
- }
82
- return;
83
- }
84
- case 'delete-all': {
85
- if (rest[0] !== '--yes')
86
- throw new Error('delete-all removes every recorded row and needs --yes');
87
- const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
88
- if (log === null) {
89
- print({ deleted: false, file, reason: 'no learning log' });
90
- return;
91
- }
92
- try {
93
- log.deleteAll();
94
- print({ deleted: true, file, counts: log.counts() });
95
- }
96
- finally {
97
- log.close();
98
- }
99
- return;
100
- }
101
- default:
102
- throw new Error(USAGE);
103
- }
104
- }
105
- if (require.main === module) {
106
- try {
107
- run(process.argv.slice(2));
108
- }
109
- catch (err) {
110
- process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
111
- process.exitCode = 1;
112
- }
113
- }