renor-engine 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/LICENSE +202 -661
  2. package/NOTICE +8 -6
  3. package/README.md +6 -6
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/BUILD.json +2 -2
  6. package/dist/api/server.js +10 -8
  7. package/dist/buildIdentity.js +18 -1
  8. package/dist/callPhases.js +82 -0
  9. package/dist/cli.js +10 -1
  10. package/dist/context/rowStore.js +1 -1
  11. package/dist/conversation/parser.js +40 -36
  12. package/dist/db/client.js +6 -4
  13. package/dist/db/handoffTable.js +31 -0
  14. package/dist/db/indexSalvage.js +24 -4
  15. package/dist/db/migrations.js +30 -2
  16. package/dist/db/sqlite.js +396 -0
  17. package/dist/digests/digestCache.js +6 -2
  18. package/dist/digests/digests.js +3 -3
  19. package/dist/digests/worker.js +3 -1
  20. package/dist/embeddings/caching.js +5 -5
  21. package/dist/embeddings/local.js +5 -3
  22. package/dist/embeddings/openai.js +4 -3
  23. package/dist/embeddings/simdScan.js +3 -1
  24. package/dist/embeddings/vectorIndex.js +0 -1
  25. package/dist/embeddings/vertex.js +123 -13
  26. package/dist/eval/cacheReport.js +7 -0
  27. package/dist/eval/context700-candidate.js +5 -5
  28. package/dist/eval/evalCache.js +46 -3
  29. package/dist/eval/indexSnapshots.js +548 -0
  30. package/dist/eval/pageDump.js +67 -0
  31. package/dist/eval/runManifest.js +6 -0
  32. package/dist/eval/textTokens.js +17 -0
  33. package/dist/evidence/attach.js +0 -3
  34. package/dist/evidence/evidenceCache.js +3 -3
  35. package/dist/evidence/options.js +46 -25
  36. package/dist/evidence/secondPass.js +8 -170
  37. package/dist/extraction/factExtractor.js +1 -1
  38. package/dist/extraction/preferenceExtractor.js +1 -4
  39. package/dist/extraction/typedFacts.js +1 -1
  40. package/dist/importing/apply.js +42 -14
  41. package/dist/importing/cli.js +9 -8
  42. package/dist/importing/existing.js +2 -0
  43. package/dist/importing/facts.js +0 -2
  44. package/dist/importing/plan.js +0 -5
  45. package/dist/importing/source.js +1 -1
  46. package/dist/indexer/parser.js +4 -1
  47. package/dist/indexer/pipeline.js +99 -279
  48. package/dist/indexer/walker.js +11 -25
  49. package/dist/mcp/server.js +36 -6
  50. package/dist/memory/handoff.js +363 -0
  51. package/dist/memory/lifecycle.js +310 -75
  52. package/dist/memory/sourceName.js +77 -1
  53. package/dist/memory/statusFacts.js +61 -0
  54. package/dist/memorymap/memoryMap.js +39 -13
  55. package/dist/memorymap/replacements.js +8 -3
  56. package/dist/mvp/agentMemory.js +106 -50
  57. package/dist/mvp/answerCheck.js +211 -0
  58. package/dist/mvp/cli.js +3 -1
  59. package/dist/mvp/compileCache.js +3 -10
  60. package/dist/mvp/compileSeed.js +9 -3
  61. package/dist/mvp/contextDistinct.js +53 -0
  62. package/dist/mvp/contextProfile.js +87 -0
  63. package/dist/mvp/directorySurface.js +68 -0
  64. package/dist/mvp/engine.js +39 -26
  65. package/dist/mvp/exportAll.js +17 -2
  66. package/dist/mvp/factMarkers.js +18 -2
  67. package/dist/mvp/history.js +5 -4
  68. package/dist/mvp/log.js +118 -0
  69. package/dist/mvp/server.js +642 -127
  70. package/dist/mvp/serverCard.js +12 -3
  71. package/dist/mvp/telemetry.js +35 -2
  72. package/dist/net/poolFetch.js +174 -0
  73. package/dist/recall/actionRecall.js +282 -0
  74. package/dist/recall/autoRecall.js +447 -0
  75. package/dist/recall/searchRecall.js +189 -0
  76. package/dist/retrieval/entityLookup.js +53 -4
  77. package/dist/retrieval/orchestrator.js +93 -8
  78. package/dist/retrieval/plainWording.js +236 -0
  79. package/dist/retrieval/readerLayout.js +143 -224
  80. package/dist/router/routerCache.js +2 -2
  81. package/dist/watcher/watcher.js +5 -3
  82. package/package.json +5 -7
  83. package/dist/importing/index.js +0 -29
@@ -43,9 +43,34 @@ function liveEntityFilter() {
43
43
  function tieOrder(table = '') {
44
44
  return `(${table}source_note = '${LIFECYCLE_NOTE_PATH}') DESC, ${table}source_note, ${table}id`;
45
45
  }
46
+ /**
47
+ * The units that occur in `column` of any entity, folded the way SQLite's `lower` folds (A to Z only: a build with ICU would
48
+ * fold more, and `lowerIsAscii` says whether this one does). Null when a value is not text.
49
+ */
50
+ function unitsOf(db, column) {
51
+ const units = new Uint32Array(0x10000 / 32);
52
+ for (const value of (0, statements_1.prepareOnce)(db, `SELECT ${column} FROM entities`).pluck().all()) {
53
+ if (value === null)
54
+ continue; // instr(NULL, x) is NULL: such a row never matches
55
+ if (typeof value !== 'string')
56
+ return null;
57
+ for (let i = 0; i < value.length; i++) {
58
+ const unit = value.charCodeAt(i);
59
+ const folded = unit >= 65 && unit <= 90 ? unit + 32 : unit;
60
+ units[folded >> 5] |= 1 << (folded & 31);
61
+ }
62
+ }
63
+ return units;
64
+ }
65
+ /** Whether `lower` leaves everything but A to Z alone, which is what `unitsOf` assumes. */
66
+ function lowerIsAscii(db) {
67
+ const probe = 'ÀÉÎÕÜÇΩДЖİß';
68
+ return (0, statements_1.prepareOnce)(db, 'SELECT lower(?)').pluck().get(`${probe}ABC`) === `${probe}abc`;
69
+ }
46
70
  /**
47
71
  * The longest canonical name, alias and source path, in SQLite characters (one code point, not a JavaScript UTF-16
48
- * unit). Read once until the next write. An alias list that cannot be read counts as unbounded, so that query still runs.
72
+ * unit), and the units that occur in the names and in the paths. Read once until the next write. An alias list that cannot
73
+ * be read counts as unbounded, so that query still runs.
49
74
  */
50
75
  function entityNameBounds(db) {
51
76
  return (0, readCache_1.readUntilWrite)(db, 'entityLookupBounds', () => {
@@ -55,9 +80,33 @@ function entityNameBounds(db) {
55
80
  alias = (0, statements_1.prepareOnce)(db, `SELECT COALESCE(MAX(length(alias.value)), 0) AS n FROM entities AS e, json_each(e.aliases_json) AS alias`).pluck().get();
56
81
  }
57
82
  catch { /* malformed aliases: do not skip the alias query */ }
58
- return { name: Number(row.name), alias: Number(alias), note: Number(row.note) };
83
+ const folded = lowerIsAscii(db);
84
+ return {
85
+ name: Number(row.name),
86
+ alias: Number(alias),
87
+ note: Number(row.note),
88
+ names: folded ? unitsOf(db, 'canonical_name') : null,
89
+ notes: folded ? unitsOf(db, 'source_note') : null,
90
+ };
59
91
  });
60
92
  }
93
+ /**
94
+ * False when `name` cannot be a substring of any value whose units are `units`: it has a unit (capitals folded) that none
95
+ * of them has. A name with a half of a pair, or a NUL, is left to SQLite.
96
+ */
97
+ function mayBeWithin(units, name) {
98
+ if (units === null)
99
+ return true;
100
+ for (let i = 0; i < name.length; i++) {
101
+ const unit = name.charCodeAt(i);
102
+ if (unit === 0 || (unit >= 0xd800 && unit <= 0xdfff))
103
+ return true;
104
+ const folded = unit >= 65 && unit <= 90 ? unit + 32 : unit;
105
+ if ((units[folded >> 5] & (1 << (folded & 31))) === 0)
106
+ return false;
107
+ }
108
+ return true;
109
+ }
61
110
  /**
62
111
  * Resolve an entity by name or alias, then by substring of a canonical name,
63
112
  * then by substring of its source note path.
@@ -93,14 +142,14 @@ function lookupEntity(db, name, filter) {
93
142
  return toRow(byAlias);
94
143
  }
95
144
  // Partial match: canonical_name contains the search term (e.g. "User" → "USER.md — About User")
96
- if (fits(bounds.name)) {
145
+ if (fits(bounds.name) && mayBeWithin(bounds.names, name)) {
97
146
  const byPartial = (0, statements_1.prepareOnce)(db, `SELECT * FROM entities WHERE instr(lower(canonical_name), lower(?)) > 0${where} ORDER BY length(canonical_name) ASC, ${tieOrder()} LIMIT 1`)
98
147
  .get(name, ...params);
99
148
  if (byPartial)
100
149
  return toRow(byPartial);
101
150
  }
102
151
  // Path match: source note path contains the search term (e.g. "Cora" → notes in .../Cora/...)
103
- if (fits(bounds.note)) {
152
+ if (fits(bounds.note) && mayBeWithin(bounds.notes, name)) {
104
153
  const byPath = (0, statements_1.prepareOnce)(db, `SELECT * FROM entities WHERE instr(lower(source_note), lower(?)) > 0${where} ORDER BY length(canonical_name) ASC, ${tieOrder()} LIMIT 1`)
105
154
  .get(name, ...params);
106
155
  if (byPath)
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
3
+ exports.ENTITY_LANE_RESERVED = exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
4
4
  exports.searchBestFirstEnabled = searchBestFirstEnabled;
5
5
  exports.parseQueryDateRef = parseQueryDateRef;
6
6
  exports.queryDateRefMs = queryDateRefMs;
@@ -22,6 +22,7 @@ exports.isSpeakerStub = isSpeakerStub;
22
22
  exports.resolveSessionDiversityCap = resolveSessionDiversityCap;
23
23
  exports.resolveSessionExpand = resolveSessionExpand;
24
24
  exports.resolveRankedExpansion = resolveRankedExpansion;
25
+ exports.rankedStatementTokens = rankedStatementTokens;
25
26
  exports.calibratedModelKey = calibratedModelKey;
26
27
  exports.retrieveContext = retrieveContext;
27
28
  const ledgerLiveness_1 = require("./ledgerLiveness");
@@ -39,6 +40,7 @@ const contextRelevance_1 = require("../mvp/contextRelevance");
39
40
  const keyword_1 = require("./keyword");
40
41
  const queryHelpers_1 = require("./queryHelpers");
41
42
  const ledgerPresentation_2 = require("./ledgerPresentation");
43
+ const plainWording_1 = require("./plainWording");
42
44
  function envFlagOn(name) {
43
45
  const v = (process.env[name] ?? '').trim().toLowerCase();
44
46
  return v === '1' || v === 'true' || v === 'yes' || v === 'on';
@@ -460,7 +462,7 @@ function resolveSessionDiversityCap(env = process.env, unsetCap = exports.SESSIO
460
462
  function countsTowardNoteCap(h) {
461
463
  return h.chunkId > 0 && (0, ledgerLiveness_1.ledgerEventIds)(h).length === 0 && (0, dailyNote_1.dailyNoteDayName)(h.notePath) === null;
462
464
  }
463
- function applySessionDiversity(hits, cap, topK) {
465
+ function applySessionDiversity(hits, cap, _topK) {
464
466
  // `cap < 1` is the flag-OFF short-circuit and must stay: 0 restores the window exactly as it was
465
467
  // before the default existed. The former `|| hits.length <= topK` here was a bypass, not an optimization — it
466
468
  // skipped the configured cap entirely whenever the pool was no deeper than the window.
@@ -603,10 +605,27 @@ function resolveRankedExpansion(env = process.env) {
603
605
  return {
604
606
  notes: rawNotes === '' ? Number.POSITIVE_INFINITY : legacy.notes,
605
607
  tokenBudget: rawBudget === '' ? exports.SESSION_EXPAND_RANKED_TOKENS : legacy.tokenBudget,
606
- statementTokens: legacy.statementTokens,
608
+ statementTokens: rankedStatementTokens(env) ?? legacy.statementTokens,
607
609
  documentStatementTokens: legacy.documentStatementTokens,
608
610
  };
609
611
  }
612
+ /**
613
+ * RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS (default unset, which is off): the longest conversation statement, in tokens,
614
+ * that the RANKED expansion alone may add. The former expansion keeps RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120), and an
615
+ * unset, empty, non-numeric or below-1 value changes nothing anywhere, byte for byte. Set, it wins over
616
+ * RENOR_SESSION_EXPAND_STATEMENT_TOKENS in the ranked expansion only.
617
+ *
618
+ * Why it exists: the session expansion skips a conversation statement over its limit, whole, and the long user turn that
619
+ * answers a question is often just over 120 tokens (501 to 700 characters: missing from the page 5 times in 21, against 6 in
620
+ * 777 for shorter ones, on the Haiku default run of 2026-10-23). The chunker splits a section of over 400 tokens into
621
+ * paragraphs, which this limit never takes, so 400 is the largest value that changes anything. Measured as 400 beside
622
+ * RENOR_EMBEDDING_SERVED_BY=vertex (the ranked expansion a hosted workspace runs) on the official LongMemEval-S instrument:
623
+ * engine/docs/evals/results/ranked-statement-limit-20261024/ and LEDGER section BE. A research switch until a default flip lands.
624
+ */
625
+ function rankedStatementTokens(env = process.env) {
626
+ const value = Number((env.RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS ?? '').trim());
627
+ return Number.isFinite(value) && value >= 1 ? Math.floor(value) : null;
628
+ }
610
629
  /**
611
630
  * The calibrated key (CALIBRATED_BARS) of the embedding model behind a provider, or null when there is none.
612
631
  *
@@ -758,9 +777,13 @@ async function retrieveContext(db, vectorIndex, provider, query, topK = 20, prio
758
777
  contentfulHeld.clear(); // what another read worked out is not this read's
759
778
  const factHits = (0, factsFirst_1.factsFirstHits)(db, query, Math.min(3, topK), opts.scope?.shows);
760
779
  const queryProbe = opts.probe ?? {}; // the query vector, for the expansion's ranking
761
- const candidateLimit = Math.min(40, topK * 4);
780
+ // RENOR_NEAR_DUP_CAP (default off): the pool is read deeper, so the places near-copies leave are taken (plainWording.ts).
781
+ const nearDupCap = (0, plainWording_1.resolveNearDupCap)();
782
+ const candidateLimit = nearDupCap > 0 ? Math.min(plainWording_1.NEAR_DUP_POOL.max, topK * plainWording_1.NEAR_DUP_POOL.perTopK) : Math.min(40, topK * 4);
783
+ // RENOR_QUERY_REWRITE (default off): what is embedded is the query without its conversational wrapper.
784
+ const embeddedQuery = (0, plainWording_1.queryRewriteEnabled)() ? (0, plainWording_1.rewriteConversationalQuery)(query) : query;
762
785
  const semanticHits = vectorIndex && provider
763
- ? await (0, semantic_1.semanticSearch)(db, vectorIndex, provider, query, candidateLimit, queryProbe, opts.scope)
786
+ ? await (0, semantic_1.semanticSearch)(db, vectorIndex, provider, embeddedQuery, candidateLimit, queryProbe, opts.scope)
764
787
  : (() => {
765
788
  const match = (0, queryHelpers_1.quoteFtsQuery)(query);
766
789
  if (!match)
@@ -772,13 +795,67 @@ async function retrieveContext(db, vectorIndex, provider, query, topK = 20, prio
772
795
  return opts.scope ? live.filter(hit => opts.scope.shows(hit)) : live;
773
796
  }, candidateLimit);
774
797
  })();
798
+ // RENOR_ENTITY_LANE (default off): the chunks that carry the rare words the query names join the pool at their own cosine.
799
+ const laneTerms = (0, plainWording_1.entityLaneEnabled)() && vectorIndex && queryProbe.queryVector
800
+ ? (0, readCache_1.inReadSnapshot)(db, () => entityLanePool(db, vectorIndex, queryProbe.queryVector, query, topK, semanticHits, opts))
801
+ : [];
775
802
  // The window is ranked without waiting for anything, so it is read in one transaction (inReadSnapshot).
776
- const window = (0, readCache_1.inReadSnapshot)(db, () => rankWindow({ db, query, topK, priorityPaths, clock, opts, factHits, semanticHits }));
803
+ const window = (0, readCache_1.inReadSnapshot)(db, () => rankWindow({ db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap }));
777
804
  return finishContext({ db, vectorIndex, provider, query, queryProbe, window, bestFirst: opts.bestFirst === true });
778
805
  }
806
+ /**
807
+ * The entity lane (RENOR_ENTITY_LANE): the lane words of the query (plainWording.ts entityLaneTerms) and, appended to
808
+ * `pool`, the chunks that carry them and are not in it yet, each at its own cosine to the query vector (one
809
+ * `VectorIndex.search` among their ids: no embedding, no model call). A scoped read judges them as it judges the pool.
810
+ * Returns the lane words, which the window lifts every hit that carries them by.
811
+ */
812
+ function entityLanePool(db, vectorIndex, queryVector, query, topK, pool, opts) {
813
+ const terms = (0, plainWording_1.entityLaneTerms)(db, query);
814
+ if (terms.length === 0)
815
+ return [];
816
+ const inPool = new Set(pool.map(h => h.chunkId));
817
+ const ids = (0, plainWording_1.entityLaneChunkIds)(db, terms, topK, opts.isLedgerNote).filter(id => !inPool.has(id));
818
+ if (ids.length === 0)
819
+ return terms;
820
+ const cosine = new Map(vectorIndex.search(queryVector, ids.length, ids).map(h => [h.chunkId, h.score]));
821
+ const rows = (0, statements_1.prepareOnce)(db, 'SELECT id AS chunkId, note_path AS notePath, heading_path AS headingPath, text FROM chunks WHERE id IN (SELECT value FROM json_each(?))').all(JSON.stringify(ids));
822
+ for (const row of rows) {
823
+ if (!cosine.has(row.chunkId))
824
+ continue; // no vector yet: the semantic lane could not have found it either
825
+ if (opts.scope && !opts.scope.shows(row))
826
+ continue;
827
+ pool.push({ ...row, score: cosine.get(row.chunkId) });
828
+ }
829
+ return terms;
830
+ }
831
+ /** How many places of a window the entity lane keeps for the best hits that carry every lane word. */
832
+ exports.ENTITY_LANE_RESERVED = 3;
833
+ /**
834
+ * The first `topK` of `ranked`, but with its best ENTITY_LANE_RESERVED hits that carry every lane word in it: each one
835
+ * the cut would leave out takes the place of the last hit that carries none. A lift alone depends on the embedding's
836
+ * scale (a cosine of 0.8 against 0.1 on the lexical stub, a few hundredths on a real model); a place does not. Without
837
+ * lane words, the first `topK`.
838
+ */
839
+ function reserveLanePlaces(ranked, topK, laneTerms) {
840
+ const window = ranked.slice(0, topK);
841
+ if (laneTerms.length === 0)
842
+ return window;
843
+ const carries = (h) => (0, plainWording_1.laneTermShare)(h.text, laneTerms) === 1;
844
+ const missing = ranked.filter(carries).slice(0, exports.ENTITY_LANE_RESERVED).filter(h => !window.includes(h));
845
+ for (const hit of missing) {
846
+ let at = window.length - 1;
847
+ while (at >= 0 && (0, plainWording_1.laneTermShare)(window[at].text, laneTerms) > 0)
848
+ at--;
849
+ if (at < 0)
850
+ break;
851
+ window.splice(at, 1);
852
+ window.push(hit);
853
+ }
854
+ return window;
855
+ }
779
856
  /** The window of hits of a read: scored, with the graph of its notes, their facts and dates, live and capped per note. Waits for nothing. */
780
857
  function rankWindow(read) {
781
- const { db, query, topK, priorityPaths, clock, opts, factHits, semanticHits } = read;
858
+ const { db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap } = read;
782
859
  // Raw cosines, kept only for a caller that asked to see them (get_context's relevance signal).
783
860
  const cosineByChunk = opts.probe ? new Map(semanticHits.map(h => [h.chunkId, h.score])) : null;
784
861
  const entityHit = (0, entityLookup_1.lookupEntity)(db, query.trim());
@@ -819,6 +896,13 @@ function rankWindow(read) {
819
896
  score += 0.2;
820
897
  reason += '+entity';
821
898
  }
899
+ if (laneTerms.length > 0) {
900
+ const share = (0, plainWording_1.laneTermShare)(h.text, laneTerms);
901
+ if (share > 0) {
902
+ score += plainWording_1.ENTITY_LANE_BOOST * share;
903
+ reason += '+entity-lane';
904
+ }
905
+ }
822
906
  // Time-decay recency: +0.1 for today, linearly decaying to 0 over 30 days.
823
907
  // Skipped for explicit past-date (as-of) queries: recency answers "now",
824
908
  // the as-of query asks about "then".
@@ -1027,7 +1111,8 @@ function rankWindow(read) {
1027
1111
  // The ledger's own header is no memory: it comes off with the closed events, before the page is cut.
1028
1112
  const mixed = (0, ledgerPresentation_1.dropLedgerPreamble)((0, ledgerLiveness_1.resolveLedgerHits)([...datedFactHits, ...contentful.filter(h => !seen.has(h.text))], (0, ledgerLiveness_1.closedLedgerEvents)(db), opts.includeHistory === true), opts.isLedgerNote);
1029
1113
  const diversityCap = resolveSessionDiversityCap(process.env, opts.noteCapWhenUnset);
1030
- let hits = applySessionDiversity(mixed, diversityCap, topK).slice(0, topK).map(h => ({
1114
+ // RENOR_NEAR_DUP_CAP (default off): near-copies are capped before the per-note cap, on the visible text.
1115
+ let hits = reserveLanePlaces(applySessionDiversity((0, plainWording_1.applyNearDupCap)(mixed, nearDupCap, contentfulText), diversityCap, topK), topK, laneTerms).map(h => ({
1031
1116
  notePath: h.notePath,
1032
1117
  chunkId: h.chunkId,
1033
1118
  headingPath: h.headingPath,
@@ -0,0 +1,236 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.NEAR_DUP_POOL = exports.NEAR_DUP_JACCARD = exports.ENTITY_LANE_DF_SHARE = exports.ENTITY_LANE_MIN_DF_LIMIT = exports.ENTITY_LANE_MAX_QUERY_TERMS = exports.ENTITY_LANE_MAX_TERMS = exports.ENTITY_LANE_BOOST = void 0;
4
+ exports.entityLaneEnabled = entityLaneEnabled;
5
+ exports.queryRewriteEnabled = queryRewriteEnabled;
6
+ exports.resolveNearDupCap = resolveNearDupCap;
7
+ exports.contentTerms = contentTerms;
8
+ exports.rewriteConversationalQuery = rewriteConversationalQuery;
9
+ exports.entityLaneTerms = entityLaneTerms;
10
+ exports.entityLaneChunkIds = entityLaneChunkIds;
11
+ exports.laneTermShare = laneTermShare;
12
+ exports.jaccard = jaccard;
13
+ exports.applyNearDupCap = applyNearDupCap;
14
+ const statements_1 = require("../db/statements");
15
+ const keyword_1 = require("./keyword");
16
+ const ledgerPresentation_1 = require("./ledgerPresentation");
17
+ const queryHelpers_1 = require("./queryHelpers");
18
+ /**
19
+ * Plain-wording retrieval and crowding: three switches, each default OFF, each a kill switch of its own.
20
+ *
21
+ * Why (the owner's real use, 2026-10-21): where Renor held the memory, the owner's own words as the query returned it
22
+ * 5 of 24 times against 19 of 24 for a hand-aimed topic query, and of 183 agent queries that once returned a memory the
23
+ * agent acted on, 40% no longer return it as the workspace grows. None of the three calls a model.
24
+ *
25
+ * - RENOR_ENTITY_LANE: the rare words a query names (a subject such as "jev", a service, a project) are looked up in
26
+ * the keyword index; the chunks that carry them join the semantic candidates at their own cosine, and every candidate
27
+ * that carries them is lifted by up to ENTITY_LANE_BOOST (the lift `+entity` already gives an entity's note).
28
+ * - RENOR_QUERY_REWRITE: the conversational wrapper of a query ("hey, do you remember what we said about ...") is taken
29
+ * off before it is embedded, so the words that are embedded are the ones stored memories use.
30
+ * - RENOR_NEAR_DUP_CAP=<n>: at most `n` near-copies (word-set Jaccard >= NEAR_DUP_JACCARD) in one window, the best
31
+ * scored kept; with it the candidate pool is read deeper (NEAR_DUP_POOL), so the distinct memories that newer
32
+ * near-copies pushed below the old pool depth can take the freed places.
33
+ */
34
+ function on(value) {
35
+ const v = (value ?? '').trim().toLowerCase();
36
+ return v === '1' || v === 'true' || v === 'yes' || v === 'on';
37
+ }
38
+ function entityLaneEnabled(env = process.env) {
39
+ return on(env.RENOR_ENTITY_LANE);
40
+ }
41
+ function queryRewriteEnabled(env = process.env) {
42
+ return on(env.RENOR_QUERY_REWRITE);
43
+ }
44
+ /** The near-copy cap, or 0 (off) when unset, empty, junk or below 1. */
45
+ function resolveNearDupCap(env = process.env) {
46
+ const raw = (env.RENOR_NEAR_DUP_CAP ?? '').trim();
47
+ if (raw === '')
48
+ return 0;
49
+ const cap = Number(raw);
50
+ return Number.isFinite(cap) && cap >= 1 ? Math.floor(cap) : 0;
51
+ }
52
+ /** Words that never name what a memory is about: function words and the words of asking and remembering. */
53
+ const PLAIN_STOP = new Set([
54
+ 'the', 'and', 'for', 'are', 'was', 'were', 'what', 'which', 'who', 'whom', 'whose', 'where', 'when', 'why', 'how',
55
+ 'does', 'did', 'done', 'with', 'about', 'this', 'that', 'these', 'those', 'there', 'here', 'then', 'than', 'from',
56
+ 'into', 'onto', 'over', 'your', 'yours', 'our', 'ours', 'their', 'theirs', 'his', 'her', 'hers', 'its', 'you', 'they',
57
+ 'them', 'she', 'him', 'can', 'could', 'would', 'should', 'will', 'shall', 'might', 'must', 'may', 'have', 'has', 'had',
58
+ 'been', 'being', 'not', 'any', 'all', 'some', 'just', 'also', 'very', 'really', 'again', 'still', 'now', 'ever',
59
+ 'remember', 'remind', 'recall', 'tell', 'told', 'say', 'said', 'know', 'think', 'thought', 'mean', 'meant', 'please',
60
+ 'thanks', 'hey', 'okay', 'yeah', 'yes', 'like', 'thing', 'things', 'stuff', 'something', 'anything', 'everything',
61
+ 'one', 'ones', 'get', 'got', 'let', 'lets', 'need', 'want', 'look', 'check', 'find', 'give', 'show', 'saved', 'save',
62
+ 'memory', 'memories', 'renor', 'claude', 'myself', 'yourself', 'we', 'us', 'me', 'my', 'it', 'is', 'in', 'on', 'of',
63
+ 'to', 'do', 'be', 'or', 'an', 'at', 'as', 'by', 'if', 'so', 'up', 'am', 'last', 'time', 'before', 'earlier', 'after',
64
+ 'since', 'until', 'during', 'while', 'because', 'between', 'other', 'another', 'first', 'next', 'every', 'each', 'many',
65
+ 'much', 'more', 'most', 'less', 'same', 'such', 'only', 'well', 'today', 'yesterday', 'ago', 'way', 'kind', 'sort',
66
+ ]);
67
+ /** The words of `query` that could name its subject: three characters or more (a hyphenated name is one word), no stop word. */
68
+ function contentTerms(query) {
69
+ const words = (query ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}][\p{L}\p{N}\p{M}_-]*/gu) ?? [];
70
+ const out = [];
71
+ for (const raw of words) {
72
+ const word = raw.replace(/[-_]+$/, '').replace(/'s$/, '');
73
+ if ([...word].length < 3 || PLAIN_STOP.has(word) || out.includes(word))
74
+ continue;
75
+ out.push(word);
76
+ }
77
+ return out;
78
+ }
79
+ /**
80
+ * Wrappers a person puts around a question when they talk to an agent. Taken off the front of the query, repeatedly,
81
+ * and only there: inside a query the same words can be what it is about.
82
+ */
83
+ const LEADING_WRAPPERS = [
84
+ /^(?:hey|hi|hello|ok|okay|so|um|uh|well|right|and|but|also|btw|by the way|quick question|question)\b[\s,:;.!-]*/i,
85
+ /^(?:claude|renor|codex|cursor)\b[\s,:;.!-]*/i,
86
+ /^(?:please|pls|can you|could you|would you|will you|do you|did you|are you able to)\b\s*/i,
87
+ /^(?:(?:still )?remember|recall|remind me(?: of)?|tell me|look up|find|check|pull up|bring up|show me)\b\s*/i,
88
+ /^(?:what (?:(?:did|do|have) )?(?:i|we|you) (?:say|said|tell you|told you|decide|decided|note|noted|save|saved|write|wrote|agree|agreed|settle|settled)(?: on)?)\b\s*/i,
89
+ /^(?:what (?:was|were|is|are) (?:it|that|the thing|those|these))\b\s*/i,
90
+ /^(?:the (?:thing|stuff|notes?)|our (?:notes?|discussion|conversation))\b\s*/i,
91
+ /^(?:about|on|regarding|re|for|of|with|that)\b[\s,:]*/i,
92
+ ];
93
+ /**
94
+ * The query without its conversational wrapper ("hey, do you remember what we said about the jev measurements?" ->
95
+ * "the jev measurements"). The question's own words are kept as written; a trailing "again", "?" or "please" goes too.
96
+ * When the rewrite would leave fewer than two content words, or nothing changed, the query is returned as it came.
97
+ */
98
+ function rewriteConversationalQuery(query) {
99
+ const original = (query ?? '').trim();
100
+ let q = original;
101
+ for (let guard = 0; guard < 12; guard++) {
102
+ const before = q;
103
+ for (const wrapper of LEADING_WRAPPERS)
104
+ q = q.replace(wrapper, '');
105
+ q = q.trim();
106
+ if (q === before)
107
+ break;
108
+ }
109
+ q = q.replace(/[\s,]*(?:again|please|for me|real quick|if you can|do you remember|remember)?[\s?.!]*$/i, '').trim();
110
+ const kept = contentTerms(q).length;
111
+ if (q === original.replace(/[\s?.!]+$/, '') || kept === 0 || kept < Math.min(2, contentTerms(original).length))
112
+ return original;
113
+ return q;
114
+ }
115
+ /** The lift a candidate that carries every lane word gets: the lift `+entity` gives an entity's note (orchestrator.ts). */
116
+ exports.ENTITY_LANE_BOOST = 0.2;
117
+ /** At most this many lane words per query: the rarest. */
118
+ exports.ENTITY_LANE_MAX_TERMS = 3;
119
+ /** The lane reads queries of at most this many content words. */
120
+ exports.ENTITY_LANE_MAX_QUERY_TERMS = 12;
121
+ /** A word is rare enough to name a subject when at most this many chunks carry it, or ENTITY_LANE_DF_SHARE of all chunks. */
122
+ exports.ENTITY_LANE_MIN_DF_LIMIT = 5;
123
+ exports.ENTITY_LANE_DF_SHARE = 0.02;
124
+ /**
125
+ * The lane words of `query`: its content words that name a registered entity, or that few enough chunks carry to name a
126
+ * subject, rarest first (a named entity before any other word). Two bounded lookups per word; none for a query of more than
127
+ * ENTITY_LANE_MAX_QUERY_TERMS content words.
128
+ */
129
+ function entityLaneTerms(db, query) {
130
+ const terms = contentTerms(query);
131
+ // A query of many words is a passage, not a named subject: its rarest word is as likely a heading's (measured: a pasted
132
+ // section lost its own chunk to a "Parking lot" section of another note). The semantic lane already matches a passage.
133
+ if (terms.length === 0 || terms.length > exports.ENTITY_LANE_MAX_QUERY_TERMS)
134
+ return [];
135
+ const chunks = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM chunks').get().n;
136
+ const limit = Math.max(exports.ENTITY_LANE_MIN_DF_LIMIT, Math.floor(chunks * exports.ENTITY_LANE_DF_SHARE));
137
+ const counted = [];
138
+ for (const term of terms) {
139
+ const match = (0, queryHelpers_1.quoteFtsQuery)(term);
140
+ if (!match)
141
+ continue;
142
+ let df;
143
+ try {
144
+ df = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM (SELECT 1 FROM chunks_fts WHERE chunks_fts MATCH ? LIMIT ?)').get(match, limit + 1).n;
145
+ }
146
+ catch {
147
+ continue; // a word FTS5 cannot read is no lane word
148
+ }
149
+ if (df === 0)
150
+ continue;
151
+ const named = (0, statements_1.prepareOnce)(db, `SELECT 1 FROM entities WHERE canonical_name = ? COLLATE NOCASE
152
+ UNION ALL SELECT 1 FROM entities, json_each(entities.aliases_json) AS alias WHERE lower(alias.value) = ? LIMIT 1`).get(term, term) !== undefined;
153
+ if (named || df <= limit)
154
+ counted.push({ term, df, named });
155
+ }
156
+ return counted
157
+ .sort((a, b) => Number(b.named) - Number(a.named) || a.df - b.df)
158
+ .slice(0, exports.ENTITY_LANE_MAX_TERMS)
159
+ .map(c => c.term);
160
+ }
161
+ /**
162
+ * The chunks that carry any lane word, best keyword match first, at most `perTerm` for each word. Read as every keyword
163
+ * reader reads (shownPage): the ledger's header and the events the word is not in are judged away before the page is cut.
164
+ */
165
+ function entityLaneChunkIds(db, terms, perTerm, isLedgerNote) {
166
+ const ids = [];
167
+ for (const term of terms) {
168
+ const match = (0, queryHelpers_1.quoteFtsQuery)(term);
169
+ if (!match)
170
+ continue;
171
+ const page = (0, queryHelpers_1.shownPage)(rows => (0, keyword_1.keywordSearch)(db, match, rows), rows => (0, ledgerPresentation_1.keywordLedgerHits)(rows, isLedgerNote, term), perTerm);
172
+ for (const hit of page)
173
+ if (!ids.includes(hit.chunkId))
174
+ ids.push(hit.chunkId);
175
+ }
176
+ return ids;
177
+ }
178
+ function escapeRegExp(s) {
179
+ return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
180
+ }
181
+ /** The share of `terms` that `text` carries as whole words (a hyphen or underscore in a lane word matches any separator). */
182
+ function laneTermShare(text, terms) {
183
+ if (terms.length === 0)
184
+ return 0;
185
+ const lower = (text ?? '').toLowerCase().normalize('NFC');
186
+ let carried = 0;
187
+ for (const term of terms) {
188
+ const pattern = term.split(/[-_]+/).map(escapeRegExp).join('[\\s_-]+');
189
+ if (new RegExp(`(?<![\\p{L}\\p{N}])${pattern}(?![\\p{L}\\p{N}])`, 'u').test(lower))
190
+ carried++;
191
+ }
192
+ return carried / terms.length;
193
+ }
194
+ /** Two texts whose word sets share at least this share of their union are near-copies. */
195
+ exports.NEAR_DUP_JACCARD = 0.6;
196
+ /** How deep the candidate pool is read while the near-copy cap is on (the default reads min(40, topK * 4)). */
197
+ exports.NEAR_DUP_POOL = { perTopK: 8, max: 160 };
198
+ function wordSet(text) {
199
+ return new Set((text ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}]+/gu) ?? []);
200
+ }
201
+ function jaccard(a, b) {
202
+ if (a.size === 0 && b.size === 0)
203
+ return 1;
204
+ let shared = 0;
205
+ for (const word of a)
206
+ if (b.has(word))
207
+ shared++;
208
+ return shared / (a.size + b.size - shared);
209
+ }
210
+ /**
211
+ * At most `cap` near-copies in the window: in rank order, a hit is kept while fewer than `cap` kept hits are near-copies
212
+ * of it. A fact row (a negative chunkId) is left alone. `cap < 1` returns the hits as they came. `textOf` is the text the
213
+ * copies are compared on (the orchestrator passes the visible text, without markdown and speaker labels).
214
+ */
215
+ function applyNearDupCap(hits, cap, textOf = t => t) {
216
+ if (cap < 1)
217
+ return hits;
218
+ const kept = [];
219
+ const keptWords = [];
220
+ for (const hit of hits) {
221
+ if (hit.chunkId < 0) {
222
+ kept.push(hit);
223
+ continue;
224
+ }
225
+ const words = wordSet(textOf(hit.text));
226
+ let copies = 0;
227
+ for (const other of keptWords)
228
+ if (jaccard(words, other) >= exports.NEAR_DUP_JACCARD && ++copies >= cap)
229
+ break;
230
+ if (copies >= cap)
231
+ continue;
232
+ kept.push(hit);
233
+ keptWords.push(words);
234
+ }
235
+ return kept;
236
+ }