renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,236 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.NEAR_DUP_POOL = exports.NEAR_DUP_JACCARD = exports.ENTITY_LANE_DF_SHARE = exports.ENTITY_LANE_MIN_DF_LIMIT = exports.ENTITY_LANE_MAX_QUERY_TERMS = exports.ENTITY_LANE_MAX_TERMS = exports.ENTITY_LANE_BOOST = void 0;
4
- exports.entityLaneEnabled = entityLaneEnabled;
5
- exports.queryRewriteEnabled = queryRewriteEnabled;
6
- exports.resolveNearDupCap = resolveNearDupCap;
7
- exports.contentTerms = contentTerms;
8
- exports.rewriteConversationalQuery = rewriteConversationalQuery;
9
- exports.entityLaneTerms = entityLaneTerms;
10
- exports.entityLaneChunkIds = entityLaneChunkIds;
11
- exports.laneTermShare = laneTermShare;
12
- exports.jaccard = jaccard;
13
- exports.applyNearDupCap = applyNearDupCap;
14
- const statements_1 = require("../db/statements");
15
- const keyword_1 = require("./keyword");
16
- const ledgerPresentation_1 = require("./ledgerPresentation");
17
- const queryHelpers_1 = require("./queryHelpers");
18
- /**
19
- * Plain-wording retrieval and crowding: three switches, each default OFF, each a kill switch of its own.
20
- *
21
- * Why (the owner's real use, 2026-10-21): where Renor held the memory, the owner's own words as the query returned it
22
- * 5 of 24 times against 19 of 24 for a hand-aimed topic query, and of 183 agent queries that once returned a memory the
23
- * agent acted on, 40% no longer return it as the workspace grows. None of the three calls a model.
24
- *
25
- * - RENOR_ENTITY_LANE: the rare words a query names (a subject such as "jev", a service, a project) are looked up in
26
- * the keyword index; the chunks that carry them join the semantic candidates at their own cosine, and every candidate
27
- * that carries them is lifted by up to ENTITY_LANE_BOOST (the lift `+entity` already gives an entity's note).
28
- * - RENOR_QUERY_REWRITE: the conversational wrapper of a query ("hey, do you remember what we said about ...") is taken
29
- * off before it is embedded, so the words that are embedded are the ones stored memories use.
30
- * - RENOR_NEAR_DUP_CAP=<n>: at most `n` near-copies (word-set Jaccard >= NEAR_DUP_JACCARD) in one window, the best
31
- * scored kept; with it the candidate pool is read deeper (NEAR_DUP_POOL), so the distinct memories that newer
32
- * near-copies pushed below the old pool depth can take the freed places.
33
- */
34
- function on(value) {
35
- const v = (value ?? '').trim().toLowerCase();
36
- return v === '1' || v === 'true' || v === 'yes' || v === 'on';
37
- }
38
- function entityLaneEnabled(env = process.env) {
39
- return on(env.RENOR_ENTITY_LANE);
40
- }
41
- function queryRewriteEnabled(env = process.env) {
42
- return on(env.RENOR_QUERY_REWRITE);
43
- }
44
- /** The near-copy cap, or 0 (off) when unset, empty, junk or below 1. */
45
- function resolveNearDupCap(env = process.env) {
46
- const raw = (env.RENOR_NEAR_DUP_CAP ?? '').trim();
47
- if (raw === '')
48
- return 0;
49
- const cap = Number(raw);
50
- return Number.isFinite(cap) && cap >= 1 ? Math.floor(cap) : 0;
51
- }
52
- /** Words that never name what a memory is about: function words and the words of asking and remembering. */
53
- const PLAIN_STOP = new Set([
54
- 'the', 'and', 'for', 'are', 'was', 'were', 'what', 'which', 'who', 'whom', 'whose', 'where', 'when', 'why', 'how',
55
- 'does', 'did', 'done', 'with', 'about', 'this', 'that', 'these', 'those', 'there', 'here', 'then', 'than', 'from',
56
- 'into', 'onto', 'over', 'your', 'yours', 'our', 'ours', 'their', 'theirs', 'his', 'her', 'hers', 'its', 'you', 'they',
57
- 'them', 'she', 'him', 'can', 'could', 'would', 'should', 'will', 'shall', 'might', 'must', 'may', 'have', 'has', 'had',
58
- 'been', 'being', 'not', 'any', 'all', 'some', 'just', 'also', 'very', 'really', 'again', 'still', 'now', 'ever',
59
- 'remember', 'remind', 'recall', 'tell', 'told', 'say', 'said', 'know', 'think', 'thought', 'mean', 'meant', 'please',
60
- 'thanks', 'hey', 'okay', 'yeah', 'yes', 'like', 'thing', 'things', 'stuff', 'something', 'anything', 'everything',
61
- 'one', 'ones', 'get', 'got', 'let', 'lets', 'need', 'want', 'look', 'check', 'find', 'give', 'show', 'saved', 'save',
62
- 'memory', 'memories', 'renor', 'claude', 'myself', 'yourself', 'we', 'us', 'me', 'my', 'it', 'is', 'in', 'on', 'of',
63
- 'to', 'do', 'be', 'or', 'an', 'at', 'as', 'by', 'if', 'so', 'up', 'am', 'last', 'time', 'before', 'earlier', 'after',
64
- 'since', 'until', 'during', 'while', 'because', 'between', 'other', 'another', 'first', 'next', 'every', 'each', 'many',
65
- 'much', 'more', 'most', 'less', 'same', 'such', 'only', 'well', 'today', 'yesterday', 'ago', 'way', 'kind', 'sort',
66
- ]);
67
- /** The words of `query` that could name its subject: three characters or more (a hyphenated name is one word), no stop word. */
68
- function contentTerms(query) {
69
- const words = (query ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}][\p{L}\p{N}\p{M}_-]*/gu) ?? [];
70
- const out = [];
71
- for (const raw of words) {
72
- const word = raw.replace(/[-_]+$/, '').replace(/'s$/, '');
73
- if ([...word].length < 3 || PLAIN_STOP.has(word) || out.includes(word))
74
- continue;
75
- out.push(word);
76
- }
77
- return out;
78
- }
79
- /**
80
- * Wrappers a person puts around a question when they talk to an agent. Taken off the front of the query, repeatedly,
81
- * and only there: inside a query the same words can be what it is about.
82
- */
83
- const LEADING_WRAPPERS = [
84
- /^(?:hey|hi|hello|ok|okay|so|um|uh|well|right|and|but|also|btw|by the way|quick question|question)\b[\s,:;.!-]*/i,
85
- /^(?:claude|renor|codex|cursor)\b[\s,:;.!-]*/i,
86
- /^(?:please|pls|can you|could you|would you|will you|do you|did you|are you able to)\b\s*/i,
87
- /^(?:(?:still )?remember|recall|remind me(?: of)?|tell me|look up|find|check|pull up|bring up|show me)\b\s*/i,
88
- /^(?:what (?:(?:did|do|have) )?(?:i|we|you) (?:say|said|tell you|told you|decide|decided|note|noted|save|saved|write|wrote|agree|agreed|settle|settled)(?: on)?)\b\s*/i,
89
- /^(?:what (?:was|were|is|are) (?:it|that|the thing|those|these))\b\s*/i,
90
- /^(?:the (?:thing|stuff|notes?)|our (?:notes?|discussion|conversation))\b\s*/i,
91
- /^(?:about|on|regarding|re|for|of|with|that)\b[\s,:]*/i,
92
- ];
93
- /**
94
- * The query without its conversational wrapper ("hey, do you remember what we said about the jev measurements?" ->
95
- * "the jev measurements"). The question's own words are kept as written; a trailing "again", "?" or "please" goes too.
96
- * When the rewrite would leave fewer than two content words, or nothing changed, the query is returned as it came.
97
- */
98
- function rewriteConversationalQuery(query) {
99
- const original = (query ?? '').trim();
100
- let q = original;
101
- for (let guard = 0; guard < 12; guard++) {
102
- const before = q;
103
- for (const wrapper of LEADING_WRAPPERS)
104
- q = q.replace(wrapper, '');
105
- q = q.trim();
106
- if (q === before)
107
- break;
108
- }
109
- q = q.replace(/[\s,]*(?:again|please|for me|real quick|if you can|do you remember|remember)?[\s?.!]*$/i, '').trim();
110
- const kept = contentTerms(q).length;
111
- if (q === original.replace(/[\s?.!]+$/, '') || kept === 0 || kept < Math.min(2, contentTerms(original).length))
112
- return original;
113
- return q;
114
- }
115
- /** The lift a candidate that carries every lane word gets: the lift `+entity` gives an entity's note (orchestrator.ts). */
116
- exports.ENTITY_LANE_BOOST = 0.2;
117
- /** At most this many lane words per query: the rarest. */
118
- exports.ENTITY_LANE_MAX_TERMS = 3;
119
- /** The lane reads queries of at most this many content words. */
120
- exports.ENTITY_LANE_MAX_QUERY_TERMS = 12;
121
- /** A word is rare enough to name a subject when at most this many chunks carry it, or ENTITY_LANE_DF_SHARE of all chunks. */
122
- exports.ENTITY_LANE_MIN_DF_LIMIT = 5;
123
- exports.ENTITY_LANE_DF_SHARE = 0.02;
124
- /**
125
- * The lane words of `query`: its content words that name a registered entity, or that few enough chunks carry to name a
126
- * subject, rarest first (a named entity before any other word). Two bounded lookups per word; none for a query of more than
127
- * ENTITY_LANE_MAX_QUERY_TERMS content words.
128
- */
129
- function entityLaneTerms(db, query) {
130
- const terms = contentTerms(query);
131
- // A query of many words is a passage, not a named subject: its rarest word is as likely a heading's (measured: a pasted
132
- // section lost its own chunk to a "Parking lot" section of another note). The semantic lane already matches a passage.
133
- if (terms.length === 0 || terms.length > exports.ENTITY_LANE_MAX_QUERY_TERMS)
134
- return [];
135
- const chunks = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM chunks').get().n;
136
- const limit = Math.max(exports.ENTITY_LANE_MIN_DF_LIMIT, Math.floor(chunks * exports.ENTITY_LANE_DF_SHARE));
137
- const counted = [];
138
- for (const term of terms) {
139
- const match = (0, queryHelpers_1.quoteFtsQuery)(term);
140
- if (!match)
141
- continue;
142
- let df;
143
- try {
144
- df = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM (SELECT 1 FROM chunks_fts WHERE chunks_fts MATCH ? LIMIT ?)').get(match, limit + 1).n;
145
- }
146
- catch {
147
- continue; // a word FTS5 cannot read is no lane word
148
- }
149
- if (df === 0)
150
- continue;
151
- const named = (0, statements_1.prepareOnce)(db, `SELECT 1 FROM entities WHERE canonical_name = ? COLLATE NOCASE
152
- UNION ALL SELECT 1 FROM entities, json_each(entities.aliases_json) AS alias WHERE lower(alias.value) = ? LIMIT 1`).get(term, term) !== undefined;
153
- if (named || df <= limit)
154
- counted.push({ term, df, named });
155
- }
156
- return counted
157
- .sort((a, b) => Number(b.named) - Number(a.named) || a.df - b.df)
158
- .slice(0, exports.ENTITY_LANE_MAX_TERMS)
159
- .map(c => c.term);
160
- }
161
- /**
162
- * The chunks that carry any lane word, best keyword match first, at most `perTerm` for each word. Read as every keyword
163
- * reader reads (shownPage): the ledger's header and the events the word is not in are judged away before the page is cut.
164
- */
165
- function entityLaneChunkIds(db, terms, perTerm, isLedgerNote) {
166
- const ids = [];
167
- for (const term of terms) {
168
- const match = (0, queryHelpers_1.quoteFtsQuery)(term);
169
- if (!match)
170
- continue;
171
- const page = (0, queryHelpers_1.shownPage)(rows => (0, keyword_1.keywordSearch)(db, match, rows), rows => (0, ledgerPresentation_1.keywordLedgerHits)(rows, isLedgerNote, term), perTerm);
172
- for (const hit of page)
173
- if (!ids.includes(hit.chunkId))
174
- ids.push(hit.chunkId);
175
- }
176
- return ids;
177
- }
178
- function escapeRegExp(s) {
179
- return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
180
- }
181
- /** The share of `terms` that `text` carries as whole words (a hyphen or underscore in a lane word matches any separator). */
182
- function laneTermShare(text, terms) {
183
- if (terms.length === 0)
184
- return 0;
185
- const lower = (text ?? '').toLowerCase().normalize('NFC');
186
- let carried = 0;
187
- for (const term of terms) {
188
- const pattern = term.split(/[-_]+/).map(escapeRegExp).join('[\\s_-]+');
189
- if (new RegExp(`(?<![\\p{L}\\p{N}])${pattern}(?![\\p{L}\\p{N}])`, 'u').test(lower))
190
- carried++;
191
- }
192
- return carried / terms.length;
193
- }
194
- /** Two texts whose word sets share at least this share of their union are near-copies. */
195
- exports.NEAR_DUP_JACCARD = 0.6;
196
- /** How deep the candidate pool is read while the near-copy cap is on (the default reads min(40, topK * 4)). */
197
- exports.NEAR_DUP_POOL = { perTopK: 8, max: 160 };
198
- function wordSet(text) {
199
- return new Set((text ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}]+/gu) ?? []);
200
- }
201
- function jaccard(a, b) {
202
- if (a.size === 0 && b.size === 0)
203
- return 1;
204
- let shared = 0;
205
- for (const word of a)
206
- if (b.has(word))
207
- shared++;
208
- return shared / (a.size + b.size - shared);
209
- }
210
- /**
211
- * At most `cap` near-copies in the window: in rank order, a hit is kept while fewer than `cap` kept hits are near-copies
212
- * of it. A fact row (a negative chunkId) is left alone. `cap < 1` returns the hits as they came. `textOf` is the text the
213
- * copies are compared on (the orchestrator passes the visible text, without markdown and speaker labels).
214
- */
215
- function applyNearDupCap(hits, cap, textOf = t => t) {
216
- if (cap < 1)
217
- return hits;
218
- const kept = [];
219
- const keptWords = [];
220
- for (const hit of hits) {
221
- if (hit.chunkId < 0) {
222
- kept.push(hit);
223
- continue;
224
- }
225
- const words = wordSet(textOf(hit.text));
226
- let copies = 0;
227
- for (const other of keptWords)
228
- if (jaccard(words, other) >= exports.NEAR_DUP_JACCARD && ++copies >= cap)
229
- break;
230
- if (copies >= cap)
231
- continue;
232
- kept.push(hit);
233
- keptWords.push(words);
234
- }
235
- return kept;
236
- }
@@ -1,142 +0,0 @@
1
- "use strict";
2
- /**
3
- * Agent-facing query helpers: FTS5 quoting and identity aliases.
4
- * Keep this free of I/O so tests stay in-process.
5
- */
6
- Object.defineProperty(exports, "__esModule", { value: true });
7
- exports.ownerNames = ownerNames;
8
- exports.identityKey = identityKey;
9
- exports.identityNames = identityNames;
10
- exports.quoteFtsQuery = quoteFtsQuery;
11
- exports.shownPage = shownPage;
12
- exports.pickTopK = pickTopK;
13
- exports.pickEntityName = pickEntityName;
14
- /**
15
- * The workspace owner's names, from RENOR_OWNER_NAMES: a comma-separated list, the first name canonical, every
16
- * other name an alias of it. Unset means no alias group: no name is built in, so the same name is one identity
17
- * only by case and Unicode form. Read per call (cheap) so tests and hosted config can change it.
18
- */
19
- function ownerNames() {
20
- return parsedOwnerNames().slice();
21
- }
22
- // Identity is on the replay hot path (one call per fact), so parse the variable once per value, not per call.
23
- let ownerSource;
24
- let ownerParsed = [];
25
- function parsedOwnerNames() {
26
- const raw = process.env.RENOR_OWNER_NAMES ?? '';
27
- if (raw !== ownerSource) {
28
- ownerSource = raw;
29
- ownerParsed = raw.split(',').map(n => n.trim()).filter(Boolean);
30
- }
31
- return ownerParsed;
32
- }
33
- /** The alias groups the identity primitive folds together: the owner's names, when configured. */
34
- function aliasGroups() {
35
- const owner = parsedOwnerNames();
36
- return owner.length ? [owner] : [];
37
- }
38
- /**
39
- * Conservative Unicode fold: NFC + plain `toLowerCase()`, marks preserved.
40
- *
41
- * Deliberately NOT `normDedupeValue` (context/rowStore.ts), which is NFKD and
42
- * strips Latin combining marks: that fold merges `José`/`Jose`, `İstanbul`/`Istanbul`
43
- * and `file`/`file`, which conflates distinct names. This fold closes only the
44
- * case axis and the Unicode normalization-form axis (NFC vs NFD), so
45
- * `José`/`JOSÉ`/NFD-`José` share one identity while `José`/`Jose`,
46
- * `Αθήνα`/`Αθηνα` (Greek tonos), `Пётр`/`Петр` (ё/е) and `Straße`/`STRASSE` stay
47
- * distinct. `toLocaleLowerCase('tr')` is never used: it would map `İ`→`i` and
48
- * collapse a real distinction. Re-normalizing after `toLowerCase()` matters,
49
- * because lowercasing can decompose (`İ` → `i` + U+0307).
50
- */
51
- function fold(s) {
52
- return s.normalize('NFC').trim().toLowerCase().normalize('NFC');
53
- }
54
- /**
55
- * One identity key for a subject name: the repo's single subject-identity primitive.
56
- *
57
- * The import planner and the lifecycle layer must never answer "is this the same
58
- * subject?" differently, so both call this. It is I/O-free and DB-free, so the
59
- * planner path stays pure. Explicit aliases collapse to one key
60
- * (the owner's names from RENOR_OWNER_NAMES); nothing else merges
61
- * except case and Unicode normalization form.
62
- *
63
- * Empty-fold guard: a key that normalizes to nothing (only reachable for an empty /
64
- * whitespace-only name, since no character is deleted) falls back to the raw name so
65
- * distinct inputs can never collide on `''`.
66
- */
67
- function identityKey(name) {
68
- const folded = fold(name);
69
- if (folded === '')
70
- return `\u0000raw:${name}`;
71
- for (const names of aliasGroups()) {
72
- if (names.some(n => fold(n) === folded))
73
- return fold(names[0]);
74
- }
75
- return folded;
76
- }
77
- /** All names that should share one lifecycle identity with `name`. */
78
- function identityNames(name) {
79
- const key = fold(name);
80
- for (const names of aliasGroups()) {
81
- if (names.some(n => fold(n) === key)) {
82
- const extra = names.includes(name) ? names : [name, ...names];
83
- return [...new Set(extra)];
84
- }
85
- }
86
- return [name];
87
- }
88
- /**
89
- * Quote a user string for FTS5 MATCH so hyphens/colons cannot be parsed as
90
- * operators (`tensaku-capture` must not become `capture` as a column).
91
- * Empty after trim → empty string (caller should not MATCH).
92
- */
93
- function quoteFtsQuery(raw) {
94
- const q = (raw ?? '').trim();
95
- if (!q)
96
- return '';
97
- if (q.length > 200)
98
- return '';
99
- const escaped = q.replace(/"/g, '""');
100
- return `"${escaped}"`;
101
- }
102
- /**
103
- * A page of a ranked read, cut after its rows have been judged and never before.
104
- *
105
- * `read(n)` is the best `n` rows (fewer only when there are no more). `show` judges the rows it is given, each on its
106
- * own, and returns the ones that are shown, in the order given. Cutting the read at `limit` rows and judging them
107
- * afterwards lets the rows that are turned away take the places of the ones that are kept: when enough of the best
108
- * rows are turned away, the page comes up short, or empty, with what belonged on it a little further down. So when the
109
- * rows read do not fill the page the read is made longer and judged again, until the page is full or every row has been
110
- * read. The first read is of `first` rows (never fewer than `limit`): a reader that has always read a wider window
111
- * than its page keeps reading it, so a search whose window holds a full page of rows that are shown, or every row there
112
- * is, makes that one read of the same rows, as it always did.
113
- *
114
- * The rows are judged from the best, a few at a time, and no row below the one that fills the page is judged: the page
115
- * is the first `limit` rows that are shown, and `show` judges each row on its own, so judging a prefix is judging the
116
- * whole read, as far as the page goes. A keyword search reads a window of 60 rows to show 6: the other 54 were judged
117
- * for nothing.
118
- */
119
- function shownPage(read, show, limit, first = limit) {
120
- if (!(limit > 0))
121
- return [];
122
- const batch = Math.max(limit, 8);
123
- for (let rows = first > limit ? first : limit;; rows *= 4) {
124
- const ranked = read(rows);
125
- const shown = [];
126
- for (let at = 0; at < ranked.length && shown.length < limit; at += batch)
127
- shown.push(...show(ranked.slice(at, at + batch)));
128
- if (shown.length >= limit || ranked.length < rows)
129
- return shown.slice(0, limit);
130
- }
131
- }
132
- function pickTopK(args, fallback = 20, max = 100) {
133
- const raw = args.topK ?? args.top_k ?? fallback;
134
- const n = Number(raw);
135
- if (!Number.isFinite(n))
136
- return fallback;
137
- return Math.min(max, Math.max(1, Math.floor(n)));
138
- }
139
- function pickEntityName(args) {
140
- const v = args.entityName ?? args.name ?? args.subject;
141
- return typeof v === 'string' ? v.trim() : '';
142
- }