renor-engine 0.0.0-stage → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +75 -0
  3. package/README.md +105 -2
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/BUILD.json +6 -0
  6. package/dist/api/apiAuth.js +43 -0
  7. package/dist/api/server.js +174 -0
  8. package/dist/buildIdentity.js +149 -0
  9. package/dist/callPhases.js +82 -0
  10. package/dist/cli-extractors.js +46 -0
  11. package/dist/cli.js +500 -0
  12. package/dist/config.js +65 -0
  13. package/dist/context/composer.js +320 -0
  14. package/dist/context/readHonesty.js +478 -0
  15. package/dist/context/rowStore.js +238 -0
  16. package/dist/conversation/formatter.js +81 -0
  17. package/dist/conversation/parser.js +193 -0
  18. package/dist/conversation/types.js +2 -0
  19. package/dist/conversation/writer.js +42 -0
  20. package/dist/db/client.js +149 -0
  21. package/dist/db/handoffTable.js +31 -0
  22. package/dist/db/indexSalvage.js +101 -0
  23. package/dist/db/migrations.js +670 -0
  24. package/dist/db/readCache.js +109 -0
  25. package/dist/db/schema.js +5 -0
  26. package/dist/db/sqlite.js +396 -0
  27. package/dist/db/statements.js +89 -0
  28. package/dist/digests/digestCache.js +146 -0
  29. package/dist/digests/digests.js +597 -0
  30. package/dist/digests/worker.js +350 -0
  31. package/dist/embeddings/caching.js +337 -0
  32. package/dist/embeddings/coalescing.js +177 -0
  33. package/dist/embeddings/interactive.js +103 -0
  34. package/dist/embeddings/local.js +70 -0
  35. package/dist/embeddings/openai.js +106 -0
  36. package/dist/embeddings/rebuild.js +166 -0
  37. package/dist/embeddings/simdScan.js +260 -0
  38. package/dist/embeddings/space.js +21 -0
  39. package/dist/embeddings/types.js +14 -0
  40. package/dist/embeddings/vectorIndex.js +674 -0
  41. package/dist/embeddings/vertex.js +334 -0
  42. package/dist/eval/artifacts.js +42 -0
  43. package/dist/eval/cacheReport.js +146 -0
  44. package/dist/eval/comparison.js +182 -0
  45. package/dist/eval/context700-candidate.js +145 -0
  46. package/dist/eval/evalCache.js +568 -0
  47. package/dist/eval/hashEmbedding.js +35 -0
  48. package/dist/eval/indexSnapshots.js +548 -0
  49. package/dist/eval/longmemeval.js +56 -0
  50. package/dist/eval/metrics.js +181 -0
  51. package/dist/eval/pageDump.js +67 -0
  52. package/dist/eval/renorbenchBaseline.js +486 -0
  53. package/dist/eval/renorbenchDocuments.js +134 -0
  54. package/dist/eval/renorbenchProduct.js +182 -0
  55. package/dist/eval/runLedger.js +279 -0
  56. package/dist/eval/runManifest.js +485 -0
  57. package/dist/eval/scoring.js +197 -0
  58. package/dist/eval/textTokens.js +17 -0
  59. package/dist/evidence/attach.js +264 -0
  60. package/dist/evidence/background.js +147 -0
  61. package/dist/evidence/evidenceCache.js +182 -0
  62. package/dist/evidence/gate.js +37 -0
  63. package/dist/evidence/keyPool.js +85 -0
  64. package/dist/evidence/ollamaChat.js +84 -0
  65. package/dist/evidence/optin.js +165 -0
  66. package/dist/evidence/options.js +151 -0
  67. package/dist/evidence/secondPass.js +294 -0
  68. package/dist/extraction/classifyCapture.js +34 -0
  69. package/dist/extraction/decision-model.js +174 -0
  70. package/dist/extraction/eventExtractor.js +113 -0
  71. package/dist/extraction/extractionClient.js +122 -0
  72. package/dist/extraction/factExtractor.js +46 -0
  73. package/dist/extraction/factsWritePath.js +179 -0
  74. package/dist/extraction/preferenceExtractor.js +150 -0
  75. package/dist/extraction/secretGuard.js +176 -0
  76. package/dist/extraction/secretScanner.js +665 -0
  77. package/dist/extraction/typedFacts.js +224 -0
  78. package/dist/extraction/writePathAdjudication.js +110 -0
  79. package/dist/importing/apply.js +672 -0
  80. package/dist/importing/bundle-write.js +80 -0
  81. package/dist/importing/cli.js +408 -0
  82. package/dist/importing/existing.js +51 -0
  83. package/dist/importing/facts.js +121 -0
  84. package/dist/importing/forgetImported.js +79 -0
  85. package/dist/importing/hosted.js +467 -0
  86. package/dist/importing/journalLog.js +96 -0
  87. package/dist/importing/markdown.js +98 -0
  88. package/dist/importing/plan.js +201 -0
  89. package/dist/importing/secretScreen.js +113 -0
  90. package/dist/importing/serialize.js +188 -0
  91. package/dist/importing/source.js +463 -0
  92. package/dist/importing/types.js +6 -0
  93. package/dist/indexer/chunker.js +319 -0
  94. package/dist/indexer/dailyEntries.js +63 -0
  95. package/dist/indexer/hasher.js +10 -0
  96. package/dist/indexer/laneRuns.js +48 -0
  97. package/dist/indexer/o200k.js +481 -0
  98. package/dist/indexer/o200k.json +8 -0
  99. package/dist/indexer/outsideRoot.js +77 -0
  100. package/dist/indexer/parser.js +170 -0
  101. package/dist/indexer/pipeline.js +988 -0
  102. package/dist/indexer/staleness.js +201 -0
  103. package/dist/indexer/tokenizer.js +77 -0
  104. package/dist/indexer/walker.js +170 -0
  105. package/dist/mcp/hive.js +88 -0
  106. package/dist/mcp/requestErrors.js +47 -0
  107. package/dist/mcp/server.js +426 -0
  108. package/dist/memory/agentLease.js +43 -0
  109. package/dist/memory/confinedWrite.js +315 -0
  110. package/dist/memory/controls.js +73 -0
  111. package/dist/memory/dailyNote.js +265 -0
  112. package/dist/memory/durableAppend.js +57 -0
  113. package/dist/memory/factConflicts.js +203 -0
  114. package/dist/memory/factTerms.js +321 -0
  115. package/dist/memory/handoff.js +363 -0
  116. package/dist/memory/hiveNameHeader.cjs +10 -0
  117. package/dist/memory/inventory.js +55 -0
  118. package/dist/memory/invisibleText.js +20 -0
  119. package/dist/memory/lifecycle.js +1991 -0
  120. package/dist/memory/limits.js +13 -0
  121. package/dist/memory/requests.js +183 -0
  122. package/dist/memory/sourceName.js +276 -0
  123. package/dist/memory/statusFacts.js +61 -0
  124. package/dist/memory/whatsNew.js +158 -0
  125. package/dist/memory/writer.js +175 -0
  126. package/dist/memory/writerConflicts.js +85 -0
  127. package/dist/memorymap/memoryMap.js +1594 -0
  128. package/dist/memorymap/replacedJournal.js +58 -0
  129. package/dist/memorymap/replacements.js +410 -0
  130. package/dist/mvp/activity.js +86 -0
  131. package/dist/mvp/agentMemory.js +1800 -0
  132. package/dist/mvp/answerCheck.js +211 -0
  133. package/dist/mvp/auth.js +531 -0
  134. package/dist/mvp/cli.js +75 -0
  135. package/dist/mvp/client-store.js +462 -0
  136. package/dist/mvp/compileCache.js +242 -0
  137. package/dist/mvp/compileSeed.js +33 -0
  138. package/dist/mvp/config.js +288 -0
  139. package/dist/mvp/connectionStatus.js +80 -0
  140. package/dist/mvp/contextDistinct.js +53 -0
  141. package/dist/mvp/contextProfile.js +87 -0
  142. package/dist/mvp/contextRelevance.js +413 -0
  143. package/dist/mvp/directorySurface.js +68 -0
  144. package/dist/mvp/engine.js +1078 -0
  145. package/dist/mvp/exportAll.js +374 -0
  146. package/dist/mvp/factLanePack.js +77 -0
  147. package/dist/mvp/factMarkers.js +65 -0
  148. package/dist/mvp/factsLane.js +203 -0
  149. package/dist/mvp/hiddenSpans.js +585 -0
  150. package/dist/mvp/history.js +414 -0
  151. package/dist/mvp/hitDates.js +31 -0
  152. package/dist/mvp/hitFloor.js +36 -0
  153. package/dist/mvp/identity.js +18 -0
  154. package/dist/mvp/idle.js +113 -0
  155. package/dist/mvp/log.js +118 -0
  156. package/dist/mvp/memoryDir.js +98 -0
  157. package/dist/mvp/posthogMcp.js +210 -0
  158. package/dist/mvp/readScope.js +104 -0
  159. package/dist/mvp/readUsefulness.js +222 -0
  160. package/dist/mvp/sanitize.js +48 -0
  161. package/dist/mvp/schemaRejections.js +91 -0
  162. package/dist/mvp/scope.js +127 -0
  163. package/dist/mvp/server.js +2604 -0
  164. package/dist/mvp/serverCard.js +38 -0
  165. package/dist/mvp/storedSize.js +139 -0
  166. package/dist/mvp/telemetry.js +474 -0
  167. package/dist/net/poolFetch.js +174 -0
  168. package/dist/recall/actionRecall.js +282 -0
  169. package/dist/recall/autoRecall.js +447 -0
  170. package/dist/recall/searchRecall.js +189 -0
  171. package/dist/retrieval/adviceLines.js +195 -0
  172. package/dist/retrieval/adviceQuestion.js +44 -0
  173. package/dist/retrieval/conversationShape.js +44 -0
  174. package/dist/retrieval/dailyEntryTime.js +55 -0
  175. package/dist/retrieval/entityLookup.js +162 -0
  176. package/dist/retrieval/factsFirst.js +188 -0
  177. package/dist/retrieval/keyword.js +84 -0
  178. package/dist/retrieval/ledgerEventTime.js +51 -0
  179. package/dist/retrieval/ledgerLiveness.js +83 -0
  180. package/dist/retrieval/ledgerPresentation.js +161 -0
  181. package/dist/retrieval/noteIdentity.js +57 -0
  182. package/dist/retrieval/orchestrator.js +1171 -0
  183. package/dist/retrieval/plainWording.js +236 -0
  184. package/dist/retrieval/queryHelpers.js +142 -0
  185. package/dist/retrieval/questionSpan.js +475 -0
  186. package/dist/retrieval/readerEvents.js +690 -0
  187. package/dist/retrieval/readerLayout.js +422 -0
  188. package/dist/retrieval/relativeTimeHints.js +113 -0
  189. package/dist/retrieval/semantic.js +67 -0
  190. package/dist/router/routerCache.js +106 -0
  191. package/dist/router/sessionRouter.js +726 -0
  192. package/dist/toolRefusal.js +120 -0
  193. package/dist/watcher/watcher.js +77 -0
  194. package/package.json +118 -4
@@ -0,0 +1,236 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.NEAR_DUP_POOL = exports.NEAR_DUP_JACCARD = exports.ENTITY_LANE_DF_SHARE = exports.ENTITY_LANE_MIN_DF_LIMIT = exports.ENTITY_LANE_MAX_QUERY_TERMS = exports.ENTITY_LANE_MAX_TERMS = exports.ENTITY_LANE_BOOST = void 0;
4
+ exports.entityLaneEnabled = entityLaneEnabled;
5
+ exports.queryRewriteEnabled = queryRewriteEnabled;
6
+ exports.resolveNearDupCap = resolveNearDupCap;
7
+ exports.contentTerms = contentTerms;
8
+ exports.rewriteConversationalQuery = rewriteConversationalQuery;
9
+ exports.entityLaneTerms = entityLaneTerms;
10
+ exports.entityLaneChunkIds = entityLaneChunkIds;
11
+ exports.laneTermShare = laneTermShare;
12
+ exports.jaccard = jaccard;
13
+ exports.applyNearDupCap = applyNearDupCap;
14
+ const statements_1 = require("../db/statements");
15
+ const keyword_1 = require("./keyword");
16
+ const ledgerPresentation_1 = require("./ledgerPresentation");
17
+ const queryHelpers_1 = require("./queryHelpers");
18
+ /**
19
+ * Plain-wording retrieval and crowding: three switches, each default OFF, each a kill switch of its own.
20
+ *
21
+ * Why (the owner's real use, 2026-10-21): where Renor held the memory, the owner's own words as the query returned it
22
+ * 5 of 24 times against 19 of 24 for a hand-aimed topic query, and of 183 agent queries that once returned a memory the
23
+ * agent acted on, 40% no longer return it as the workspace grows. None of the three calls a model.
24
+ *
25
+ * - RENOR_ENTITY_LANE: the rare words a query names (a subject such as "jev", a service, a project) are looked up in
26
+ * the keyword index; the chunks that carry them join the semantic candidates at their own cosine, and every candidate
27
+ * that carries them is lifted by up to ENTITY_LANE_BOOST (the lift `+entity` already gives an entity's note).
28
+ * - RENOR_QUERY_REWRITE: the conversational wrapper of a query ("hey, do you remember what we said about ...") is taken
29
+ * off before it is embedded, so the words that are embedded are the ones stored memories use.
30
+ * - RENOR_NEAR_DUP_CAP=<n>: at most `n` near-copies (word-set Jaccard >= NEAR_DUP_JACCARD) in one window, the best
31
+ * scored kept; with it the candidate pool is read deeper (NEAR_DUP_POOL), so the distinct memories that newer
32
+ * near-copies pushed below the old pool depth can take the freed places.
33
+ */
34
+ function on(value) {
35
+ const v = (value ?? '').trim().toLowerCase();
36
+ return v === '1' || v === 'true' || v === 'yes' || v === 'on';
37
+ }
38
+ function entityLaneEnabled(env = process.env) {
39
+ return on(env.RENOR_ENTITY_LANE);
40
+ }
41
+ function queryRewriteEnabled(env = process.env) {
42
+ return on(env.RENOR_QUERY_REWRITE);
43
+ }
44
+ /** The near-copy cap, or 0 (off) when unset, empty, junk or below 1. */
45
+ function resolveNearDupCap(env = process.env) {
46
+ const raw = (env.RENOR_NEAR_DUP_CAP ?? '').trim();
47
+ if (raw === '')
48
+ return 0;
49
+ const cap = Number(raw);
50
+ return Number.isFinite(cap) && cap >= 1 ? Math.floor(cap) : 0;
51
+ }
52
+ /** Words that never name what a memory is about: function words and the words of asking and remembering. */
53
+ const PLAIN_STOP = new Set([
54
+ 'the', 'and', 'for', 'are', 'was', 'were', 'what', 'which', 'who', 'whom', 'whose', 'where', 'when', 'why', 'how',
55
+ 'does', 'did', 'done', 'with', 'about', 'this', 'that', 'these', 'those', 'there', 'here', 'then', 'than', 'from',
56
+ 'into', 'onto', 'over', 'your', 'yours', 'our', 'ours', 'their', 'theirs', 'his', 'her', 'hers', 'its', 'you', 'they',
57
+ 'them', 'she', 'him', 'can', 'could', 'would', 'should', 'will', 'shall', 'might', 'must', 'may', 'have', 'has', 'had',
58
+ 'been', 'being', 'not', 'any', 'all', 'some', 'just', 'also', 'very', 'really', 'again', 'still', 'now', 'ever',
59
+ 'remember', 'remind', 'recall', 'tell', 'told', 'say', 'said', 'know', 'think', 'thought', 'mean', 'meant', 'please',
60
+ 'thanks', 'hey', 'okay', 'yeah', 'yes', 'like', 'thing', 'things', 'stuff', 'something', 'anything', 'everything',
61
+ 'one', 'ones', 'get', 'got', 'let', 'lets', 'need', 'want', 'look', 'check', 'find', 'give', 'show', 'saved', 'save',
62
+ 'memory', 'memories', 'renor', 'claude', 'myself', 'yourself', 'we', 'us', 'me', 'my', 'it', 'is', 'in', 'on', 'of',
63
+ 'to', 'do', 'be', 'or', 'an', 'at', 'as', 'by', 'if', 'so', 'up', 'am', 'last', 'time', 'before', 'earlier', 'after',
64
+ 'since', 'until', 'during', 'while', 'because', 'between', 'other', 'another', 'first', 'next', 'every', 'each', 'many',
65
+ 'much', 'more', 'most', 'less', 'same', 'such', 'only', 'well', 'today', 'yesterday', 'ago', 'way', 'kind', 'sort',
66
+ ]);
67
+ /** The words of `query` that could name its subject: three characters or more (a hyphenated name is one word), no stop word. */
68
+ function contentTerms(query) {
69
+ const words = (query ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}][\p{L}\p{N}\p{M}_-]*/gu) ?? [];
70
+ const out = [];
71
+ for (const raw of words) {
72
+ const word = raw.replace(/[-_]+$/, '').replace(/'s$/, '');
73
+ if ([...word].length < 3 || PLAIN_STOP.has(word) || out.includes(word))
74
+ continue;
75
+ out.push(word);
76
+ }
77
+ return out;
78
+ }
79
+ /**
80
+ * Wrappers a person puts around a question when they talk to an agent. Taken off the front of the query, repeatedly,
81
+ * and only there: inside a query the same words can be what it is about.
82
+ */
83
+ const LEADING_WRAPPERS = [
84
+ /^(?:hey|hi|hello|ok|okay|so|um|uh|well|right|and|but|also|btw|by the way|quick question|question)\b[\s,:;.!-]*/i,
85
+ /^(?:claude|renor|codex|cursor)\b[\s,:;.!-]*/i,
86
+ /^(?:please|pls|can you|could you|would you|will you|do you|did you|are you able to)\b\s*/i,
87
+ /^(?:(?:still )?remember|recall|remind me(?: of)?|tell me|look up|find|check|pull up|bring up|show me)\b\s*/i,
88
+ /^(?:what (?:(?:did|do|have) )?(?:i|we|you) (?:say|said|tell you|told you|decide|decided|note|noted|save|saved|write|wrote|agree|agreed|settle|settled)(?: on)?)\b\s*/i,
89
+ /^(?:what (?:was|were|is|are) (?:it|that|the thing|those|these))\b\s*/i,
90
+ /^(?:the (?:thing|stuff|notes?)|our (?:notes?|discussion|conversation))\b\s*/i,
91
+ /^(?:about|on|regarding|re|for|of|with|that)\b[\s,:]*/i,
92
+ ];
93
+ /**
94
+ * The query without its conversational wrapper ("hey, do you remember what we said about the jev measurements?" ->
95
+ * "the jev measurements"). The question's own words are kept as written; a trailing "again", "?" or "please" goes too.
96
+ * When the rewrite would leave fewer than two content words, or nothing changed, the query is returned as it came.
97
+ */
98
+ function rewriteConversationalQuery(query) {
99
+ const original = (query ?? '').trim();
100
+ let q = original;
101
+ for (let guard = 0; guard < 12; guard++) {
102
+ const before = q;
103
+ for (const wrapper of LEADING_WRAPPERS)
104
+ q = q.replace(wrapper, '');
105
+ q = q.trim();
106
+ if (q === before)
107
+ break;
108
+ }
109
+ q = q.replace(/[\s,]*(?:again|please|for me|real quick|if you can|do you remember|remember)?[\s?.!]*$/i, '').trim();
110
+ const kept = contentTerms(q).length;
111
+ if (q === original.replace(/[\s?.!]+$/, '') || kept === 0 || kept < Math.min(2, contentTerms(original).length))
112
+ return original;
113
+ return q;
114
+ }
115
+ /** The lift a candidate that carries every lane word gets: the lift `+entity` gives an entity's note (orchestrator.ts). */
116
+ exports.ENTITY_LANE_BOOST = 0.2;
117
+ /** At most this many lane words per query: the rarest. */
118
+ exports.ENTITY_LANE_MAX_TERMS = 3;
119
+ /** The lane reads queries of at most this many content words. */
120
+ exports.ENTITY_LANE_MAX_QUERY_TERMS = 12;
121
+ /** A word is rare enough to name a subject when at most this many chunks carry it, or ENTITY_LANE_DF_SHARE of all chunks. */
122
+ exports.ENTITY_LANE_MIN_DF_LIMIT = 5;
123
+ exports.ENTITY_LANE_DF_SHARE = 0.02;
124
+ /**
125
+ * The lane words of `query`: its content words that name a registered entity, or that few enough chunks carry to name a
126
+ * subject, rarest first (a named entity before any other word). Two bounded lookups per word; none for a query of more than
127
+ * ENTITY_LANE_MAX_QUERY_TERMS content words.
128
+ */
129
+ function entityLaneTerms(db, query) {
130
+ const terms = contentTerms(query);
131
+ // A query of many words is a passage, not a named subject: its rarest word is as likely a heading's (measured: a pasted
132
+ // section lost its own chunk to a "Parking lot" section of another note). The semantic lane already matches a passage.
133
+ if (terms.length === 0 || terms.length > exports.ENTITY_LANE_MAX_QUERY_TERMS)
134
+ return [];
135
+ const chunks = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM chunks').get().n;
136
+ const limit = Math.max(exports.ENTITY_LANE_MIN_DF_LIMIT, Math.floor(chunks * exports.ENTITY_LANE_DF_SHARE));
137
+ const counted = [];
138
+ for (const term of terms) {
139
+ const match = (0, queryHelpers_1.quoteFtsQuery)(term);
140
+ if (!match)
141
+ continue;
142
+ let df;
143
+ try {
144
+ df = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM (SELECT 1 FROM chunks_fts WHERE chunks_fts MATCH ? LIMIT ?)').get(match, limit + 1).n;
145
+ }
146
+ catch {
147
+ continue; // a word FTS5 cannot read is no lane word
148
+ }
149
+ if (df === 0)
150
+ continue;
151
+ const named = (0, statements_1.prepareOnce)(db, `SELECT 1 FROM entities WHERE canonical_name = ? COLLATE NOCASE
152
+ UNION ALL SELECT 1 FROM entities, json_each(entities.aliases_json) AS alias WHERE lower(alias.value) = ? LIMIT 1`).get(term, term) !== undefined;
153
+ if (named || df <= limit)
154
+ counted.push({ term, df, named });
155
+ }
156
+ return counted
157
+ .sort((a, b) => Number(b.named) - Number(a.named) || a.df - b.df)
158
+ .slice(0, exports.ENTITY_LANE_MAX_TERMS)
159
+ .map(c => c.term);
160
+ }
161
+ /**
162
+ * The chunks that carry any lane word, best keyword match first, at most `perTerm` for each word. Read as every keyword
163
+ * reader reads (shownPage): the ledger's header and the events the word is not in are judged away before the page is cut.
164
+ */
165
+ function entityLaneChunkIds(db, terms, perTerm, isLedgerNote) {
166
+ const ids = [];
167
+ for (const term of terms) {
168
+ const match = (0, queryHelpers_1.quoteFtsQuery)(term);
169
+ if (!match)
170
+ continue;
171
+ const page = (0, queryHelpers_1.shownPage)(rows => (0, keyword_1.keywordSearch)(db, match, rows), rows => (0, ledgerPresentation_1.keywordLedgerHits)(rows, isLedgerNote, term), perTerm);
172
+ for (const hit of page)
173
+ if (!ids.includes(hit.chunkId))
174
+ ids.push(hit.chunkId);
175
+ }
176
+ return ids;
177
+ }
178
+ function escapeRegExp(s) {
179
+ return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
180
+ }
181
+ /** The share of `terms` that `text` carries as whole words (a hyphen or underscore in a lane word matches any separator). */
182
+ function laneTermShare(text, terms) {
183
+ if (terms.length === 0)
184
+ return 0;
185
+ const lower = (text ?? '').toLowerCase().normalize('NFC');
186
+ let carried = 0;
187
+ for (const term of terms) {
188
+ const pattern = term.split(/[-_]+/).map(escapeRegExp).join('[\\s_-]+');
189
+ if (new RegExp(`(?<![\\p{L}\\p{N}])${pattern}(?![\\p{L}\\p{N}])`, 'u').test(lower))
190
+ carried++;
191
+ }
192
+ return carried / terms.length;
193
+ }
194
+ /** Two texts whose word sets share at least this share of their union are near-copies. */
195
+ exports.NEAR_DUP_JACCARD = 0.6;
196
+ /** How deep the candidate pool is read while the near-copy cap is on (the default reads min(40, topK * 4)). */
197
+ exports.NEAR_DUP_POOL = { perTopK: 8, max: 160 };
198
+ function wordSet(text) {
199
+ return new Set((text ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}]+/gu) ?? []);
200
+ }
201
+ function jaccard(a, b) {
202
+ if (a.size === 0 && b.size === 0)
203
+ return 1;
204
+ let shared = 0;
205
+ for (const word of a)
206
+ if (b.has(word))
207
+ shared++;
208
+ return shared / (a.size + b.size - shared);
209
+ }
210
+ /**
211
+ * At most `cap` near-copies in the window: in rank order, a hit is kept while fewer than `cap` kept hits are near-copies
212
+ * of it. A fact row (a negative chunkId) is left alone. `cap < 1` returns the hits as they came. `textOf` is the text the
213
+ * copies are compared on (the orchestrator passes the visible text, without markdown and speaker labels).
214
+ */
215
+ function applyNearDupCap(hits, cap, textOf = t => t) {
216
+ if (cap < 1)
217
+ return hits;
218
+ const kept = [];
219
+ const keptWords = [];
220
+ for (const hit of hits) {
221
+ if (hit.chunkId < 0) {
222
+ kept.push(hit);
223
+ continue;
224
+ }
225
+ const words = wordSet(textOf(hit.text));
226
+ let copies = 0;
227
+ for (const other of keptWords)
228
+ if (jaccard(words, other) >= exports.NEAR_DUP_JACCARD && ++copies >= cap)
229
+ break;
230
+ if (copies >= cap)
231
+ continue;
232
+ kept.push(hit);
233
+ keptWords.push(words);
234
+ }
235
+ return kept;
236
+ }
@@ -0,0 +1,142 @@
1
+ "use strict";
2
+ /**
3
+ * Agent-facing query helpers: FTS5 quoting and identity aliases.
4
+ * Keep this free of I/O so tests stay in-process.
5
+ */
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ exports.ownerNames = ownerNames;
8
+ exports.identityKey = identityKey;
9
+ exports.identityNames = identityNames;
10
+ exports.quoteFtsQuery = quoteFtsQuery;
11
+ exports.shownPage = shownPage;
12
+ exports.pickTopK = pickTopK;
13
+ exports.pickEntityName = pickEntityName;
14
+ /**
15
+ * The workspace owner's names, from RENOR_OWNER_NAMES: a comma-separated list, the first name canonical, every
16
+ * other name an alias of it. Unset means no alias group: no name is built in, so the same name is one identity
17
+ * only by case and Unicode form. Read per call (cheap) so tests and hosted config can change it.
18
+ */
19
+ function ownerNames() {
20
+ return parsedOwnerNames().slice();
21
+ }
22
+ // Identity is on the replay hot path (one call per fact), so parse the variable once per value, not per call.
23
+ let ownerSource;
24
+ let ownerParsed = [];
25
+ function parsedOwnerNames() {
26
+ const raw = process.env.RENOR_OWNER_NAMES ?? '';
27
+ if (raw !== ownerSource) {
28
+ ownerSource = raw;
29
+ ownerParsed = raw.split(',').map(n => n.trim()).filter(Boolean);
30
+ }
31
+ return ownerParsed;
32
+ }
33
+ /** The alias groups the identity primitive folds together: the owner's names, when configured. */
34
+ function aliasGroups() {
35
+ const owner = parsedOwnerNames();
36
+ return owner.length ? [owner] : [];
37
+ }
38
+ /**
39
+ * Conservative Unicode fold: NFC + plain `toLowerCase()`, marks preserved.
40
+ *
41
+ * Deliberately NOT `normDedupeValue` (context/rowStore.ts), which is NFKD and
42
+ * strips Latin combining marks: that fold merges `José`/`Jose`, `İstanbul`/`Istanbul`
43
+ * and `file`/`file`, which conflates distinct names. This fold closes only the
44
+ * case axis and the Unicode normalization-form axis (NFC vs NFD), so
45
+ * `José`/`JOSÉ`/NFD-`José` share one identity while `José`/`Jose`,
46
+ * `Αθήνα`/`Αθηνα` (Greek tonos), `Пётр`/`Петр` (ё/е) and `Straße`/`STRASSE` stay
47
+ * distinct. `toLocaleLowerCase('tr')` is never used: it would map `İ`→`i` and
48
+ * collapse a real distinction. Re-normalizing after `toLowerCase()` matters,
49
+ * because lowercasing can decompose (`İ` → `i` + U+0307).
50
+ */
51
+ function fold(s) {
52
+ return s.normalize('NFC').trim().toLowerCase().normalize('NFC');
53
+ }
54
+ /**
55
+ * One identity key for a subject name: the repo's single subject-identity primitive.
56
+ *
57
+ * The import planner and the lifecycle layer must never answer "is this the same
58
+ * subject?" differently, so both call this. It is I/O-free and DB-free, so the
59
+ * planner path stays pure. Explicit aliases collapse to one key
60
+ * (the owner's names from RENOR_OWNER_NAMES); nothing else merges
61
+ * except case and Unicode normalization form.
62
+ *
63
+ * Empty-fold guard: a key that normalizes to nothing (only reachable for an empty /
64
+ * whitespace-only name, since no character is deleted) falls back to the raw name so
65
+ * distinct inputs can never collide on `''`.
66
+ */
67
+ function identityKey(name) {
68
+ const folded = fold(name);
69
+ if (folded === '')
70
+ return `\u0000raw:${name}`;
71
+ for (const names of aliasGroups()) {
72
+ if (names.some(n => fold(n) === folded))
73
+ return fold(names[0]);
74
+ }
75
+ return folded;
76
+ }
77
+ /** All names that should share one lifecycle identity with `name`. */
78
+ function identityNames(name) {
79
+ const key = fold(name);
80
+ for (const names of aliasGroups()) {
81
+ if (names.some(n => fold(n) === key)) {
82
+ const extra = names.includes(name) ? names : [name, ...names];
83
+ return [...new Set(extra)];
84
+ }
85
+ }
86
+ return [name];
87
+ }
88
+ /**
89
+ * Quote a user string for FTS5 MATCH so hyphens/colons cannot be parsed as
90
+ * operators (`tensaku-capture` must not become `capture` as a column).
91
+ * Empty after trim → empty string (caller should not MATCH).
92
+ */
93
+ function quoteFtsQuery(raw) {
94
+ const q = (raw ?? '').trim();
95
+ if (!q)
96
+ return '';
97
+ if (q.length > 200)
98
+ return '';
99
+ const escaped = q.replace(/"/g, '""');
100
+ return `"${escaped}"`;
101
+ }
102
+ /**
103
+ * A page of a ranked read, cut after its rows have been judged and never before.
104
+ *
105
+ * `read(n)` is the best `n` rows (fewer only when there are no more). `show` judges the rows it is given, each on its
106
+ * own, and returns the ones that are shown, in the order given. Cutting the read at `limit` rows and judging them
107
+ * afterwards lets the rows that are turned away take the places of the ones that are kept: when enough of the best
108
+ * rows are turned away, the page comes up short, or empty, with what belonged on it a little further down. So when the
109
+ * rows read do not fill the page the read is made longer and judged again, until the page is full or every row has been
110
+ * read. The first read is of `first` rows (never fewer than `limit`): a reader that has always read a wider window
111
+ * than its page keeps reading it, so a search whose window holds a full page of rows that are shown, or every row there
112
+ * is, makes that one read of the same rows, as it always did.
113
+ *
114
+ * The rows are judged from the best, a few at a time, and no row below the one that fills the page is judged: the page
115
+ * is the first `limit` rows that are shown, and `show` judges each row on its own, so judging a prefix is judging the
116
+ * whole read, as far as the page goes. A keyword search reads a window of 60 rows to show 6: the other 54 were judged
117
+ * for nothing.
118
+ */
119
+ function shownPage(read, show, limit, first = limit) {
120
+ if (!(limit > 0))
121
+ return [];
122
+ const batch = Math.max(limit, 8);
123
+ for (let rows = first > limit ? first : limit;; rows *= 4) {
124
+ const ranked = read(rows);
125
+ const shown = [];
126
+ for (let at = 0; at < ranked.length && shown.length < limit; at += batch)
127
+ shown.push(...show(ranked.slice(at, at + batch)));
128
+ if (shown.length >= limit || ranked.length < rows)
129
+ return shown.slice(0, limit);
130
+ }
131
+ }
132
+ function pickTopK(args, fallback = 20, max = 100) {
133
+ const raw = args.topK ?? args.top_k ?? fallback;
134
+ const n = Number(raw);
135
+ if (!Number.isFinite(n))
136
+ return fallback;
137
+ return Math.min(max, Math.max(1, Math.floor(n)));
138
+ }
139
+ function pickEntityName(args) {
140
+ const v = args.entityName ?? args.name ?? args.subject;
141
+ return typeof v === 'string' ? v.trim() : '';
142
+ }