renor-engine 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/LICENSE +202 -661
  2. package/NOTICE +8 -6
  3. package/README.md +13 -8
  4. package/dist/.tsbuildinfo +1 -0
  5. package/dist/api/server.js +10 -8
  6. package/dist/buildIdentity.js +18 -1
  7. package/dist/callPhases.js +82 -0
  8. package/dist/cli.js +36 -1
  9. package/dist/context/rowStore.js +1 -1
  10. package/dist/conversation/parser.js +40 -36
  11. package/dist/db/client.js +6 -4
  12. package/dist/db/handoffTable.js +31 -0
  13. package/dist/db/indexSalvage.js +24 -4
  14. package/dist/db/migrations.js +30 -2
  15. package/dist/db/sqlite.js +396 -0
  16. package/dist/digests/digestCache.js +6 -2
  17. package/dist/digests/digests.js +3 -3
  18. package/dist/digests/worker.js +3 -1
  19. package/dist/embeddings/caching.js +5 -5
  20. package/dist/embeddings/local.js +5 -3
  21. package/dist/embeddings/openai.js +4 -3
  22. package/dist/embeddings/simdScan.js +3 -1
  23. package/dist/embeddings/vectorIndex.js +0 -1
  24. package/dist/embeddings/vertex.js +123 -13
  25. package/dist/eval/cacheReport.js +7 -0
  26. package/dist/eval/context700-candidate.js +5 -5
  27. package/dist/eval/evalCache.js +46 -3
  28. package/dist/eval/indexSnapshots.js +549 -0
  29. package/dist/eval/pageDump.js +67 -0
  30. package/dist/eval/runManifest.js +6 -0
  31. package/dist/eval/textTokens.js +17 -0
  32. package/dist/evidence/attach.js +0 -3
  33. package/dist/evidence/evidenceCache.js +3 -3
  34. package/dist/evidence/ollamaChat.js +1 -1
  35. package/dist/evidence/options.js +46 -25
  36. package/dist/evidence/secondPass.js +8 -170
  37. package/dist/extraction/factExtractor.js +1 -1
  38. package/dist/extraction/jevDecisionModel.js +307 -0
  39. package/dist/extraction/preferenceExtractor.js +1 -4
  40. package/dist/extraction/secretGuard.js +7 -3
  41. package/dist/extraction/typedFacts.js +1 -1
  42. package/dist/importing/apply.js +42 -14
  43. package/dist/importing/cli.js +9 -8
  44. package/dist/importing/existing.js +2 -0
  45. package/dist/importing/facts.js +0 -2
  46. package/dist/importing/plan.js +0 -5
  47. package/dist/importing/source.js +1 -1
  48. package/dist/indexer/parser.js +4 -1
  49. package/dist/indexer/pipeline.js +99 -279
  50. package/dist/indexer/walker.js +11 -25
  51. package/dist/learning/cli.js +113 -0
  52. package/dist/learning/schema.js +144 -0
  53. package/dist/learning/settings.js +45 -0
  54. package/dist/learning/store.js +267 -0
  55. package/dist/learning/tap.js +401 -0
  56. package/dist/mcp/server.js +36 -6
  57. package/dist/memory/dreamer/adjudicate.js +215 -0
  58. package/dist/memory/dreamer/apply.js +120 -0
  59. package/dist/memory/dreamer/budget.js +64 -0
  60. package/dist/memory/dreamer/candidates.js +149 -0
  61. package/dist/memory/dreamer/collection.js +104 -0
  62. package/dist/memory/dreamer/dreamer.js +352 -0
  63. package/dist/memory/dreamer/review.js +111 -0
  64. package/dist/memory/factConflicts.js +2 -0
  65. package/dist/memory/handoff.js +363 -0
  66. package/dist/memory/lifecycle.js +338 -80
  67. package/dist/memory/sourceName.js +77 -1
  68. package/dist/memory/statusFacts.js +61 -0
  69. package/dist/memory/writer.js +3 -2
  70. package/dist/memorymap/memoryMap.js +39 -13
  71. package/dist/memorymap/replacements.js +8 -3
  72. package/dist/mvp/agentMemory.js +109 -50
  73. package/dist/mvp/answerCheck.js +211 -0
  74. package/dist/mvp/cli.js +14 -1
  75. package/dist/mvp/compileCache.js +3 -10
  76. package/dist/mvp/compileSeed.js +9 -3
  77. package/dist/mvp/config.js +4 -0
  78. package/dist/mvp/contextDistinct.js +53 -0
  79. package/dist/mvp/contextProfile.js +87 -0
  80. package/dist/mvp/directorySurface.js +68 -0
  81. package/dist/mvp/engine.js +39 -26
  82. package/dist/mvp/exportAll.js +36 -2
  83. package/dist/mvp/factMarkers.js +18 -2
  84. package/dist/mvp/history.js +5 -4
  85. package/dist/mvp/log.js +118 -0
  86. package/dist/mvp/server.js +759 -136
  87. package/dist/mvp/serverCard.js +12 -3
  88. package/dist/mvp/telemetry.js +35 -2
  89. package/dist/net/poolFetch.js +174 -0
  90. package/dist/recall/actionRecall.js +282 -0
  91. package/dist/recall/autoRecall.js +447 -0
  92. package/dist/recall/searchRecall.js +189 -0
  93. package/dist/retrieval/entityLookup.js +53 -4
  94. package/dist/retrieval/orchestrator.js +148 -10
  95. package/dist/retrieval/plainWording.js +236 -0
  96. package/dist/retrieval/rankingTrace.js +184 -0
  97. package/dist/retrieval/readerLayout.js +143 -224
  98. package/dist/router/routerCache.js +2 -2
  99. package/dist/sharing/cli.js +181 -0
  100. package/dist/sharing/examples.js +136 -0
  101. package/dist/sharing/fields.js +225 -0
  102. package/dist/sharing/loop.js +80 -0
  103. package/dist/sharing/project.js +327 -0
  104. package/dist/sharing/send.js +466 -0
  105. package/dist/sharing/state.js +222 -0
  106. package/dist/sharing/wording.js +48 -0
  107. package/dist/watcher/watcher.js +5 -3
  108. package/package.json +5 -7
  109. package/dist/BUILD.json +0 -6
  110. package/dist/importing/index.js +0 -29
@@ -43,9 +43,34 @@ function liveEntityFilter() {
43
43
  function tieOrder(table = '') {
44
44
  return `(${table}source_note = '${LIFECYCLE_NOTE_PATH}') DESC, ${table}source_note, ${table}id`;
45
45
  }
46
+ /**
47
+ * The units that occur in `column` of any entity, folded the way SQLite's `lower` folds (A to Z only: a build with ICU would
48
+ * fold more, and `lowerIsAscii` says whether this one does). Null when a value is not text.
49
+ */
50
+ function unitsOf(db, column) {
51
+ const units = new Uint32Array(0x10000 / 32);
52
+ for (const value of (0, statements_1.prepareOnce)(db, `SELECT ${column} FROM entities`).pluck().all()) {
53
+ if (value === null)
54
+ continue; // instr(NULL, x) is NULL: such a row never matches
55
+ if (typeof value !== 'string')
56
+ return null;
57
+ for (let i = 0; i < value.length; i++) {
58
+ const unit = value.charCodeAt(i);
59
+ const folded = unit >= 65 && unit <= 90 ? unit + 32 : unit;
60
+ units[folded >> 5] |= 1 << (folded & 31);
61
+ }
62
+ }
63
+ return units;
64
+ }
65
+ /** Whether `lower` leaves everything but A to Z alone, which is what `unitsOf` assumes. */
66
+ function lowerIsAscii(db) {
67
+ const probe = 'ÀÉÎÕÜÇΩДЖİß';
68
+ return (0, statements_1.prepareOnce)(db, 'SELECT lower(?)').pluck().get(`${probe}ABC`) === `${probe}abc`;
69
+ }
46
70
  /**
47
71
  * The longest canonical name, alias and source path, in SQLite characters (one code point, not a JavaScript UTF-16
48
- * unit). Read once until the next write. An alias list that cannot be read counts as unbounded, so that query still runs.
72
+ * unit), and the units that occur in the names and in the paths. Read once until the next write. An alias list that cannot
73
+ * be read counts as unbounded, so that query still runs.
49
74
  */
50
75
  function entityNameBounds(db) {
51
76
  return (0, readCache_1.readUntilWrite)(db, 'entityLookupBounds', () => {
@@ -55,9 +80,33 @@ function entityNameBounds(db) {
55
80
  alias = (0, statements_1.prepareOnce)(db, `SELECT COALESCE(MAX(length(alias.value)), 0) AS n FROM entities AS e, json_each(e.aliases_json) AS alias`).pluck().get();
56
81
  }
57
82
  catch { /* malformed aliases: do not skip the alias query */ }
58
- return { name: Number(row.name), alias: Number(alias), note: Number(row.note) };
83
+ const folded = lowerIsAscii(db);
84
+ return {
85
+ name: Number(row.name),
86
+ alias: Number(alias),
87
+ note: Number(row.note),
88
+ names: folded ? unitsOf(db, 'canonical_name') : null,
89
+ notes: folded ? unitsOf(db, 'source_note') : null,
90
+ };
59
91
  });
60
92
  }
93
+ /**
94
+ * False when `name` cannot be a substring of any value whose units are `units`: it has a unit (capitals folded) that none
95
+ * of them has. A name with a half of a pair, or a NUL, is left to SQLite.
96
+ */
97
+ function mayBeWithin(units, name) {
98
+ if (units === null)
99
+ return true;
100
+ for (let i = 0; i < name.length; i++) {
101
+ const unit = name.charCodeAt(i);
102
+ if (unit === 0 || (unit >= 0xd800 && unit <= 0xdfff))
103
+ return true;
104
+ const folded = unit >= 65 && unit <= 90 ? unit + 32 : unit;
105
+ if ((units[folded >> 5] & (1 << (folded & 31))) === 0)
106
+ return false;
107
+ }
108
+ return true;
109
+ }
61
110
  /**
62
111
  * Resolve an entity by name or alias, then by substring of a canonical name,
63
112
  * then by substring of its source note path.
@@ -93,14 +142,14 @@ function lookupEntity(db, name, filter) {
93
142
  return toRow(byAlias);
94
143
  }
95
144
  // Partial match: canonical_name contains the search term (e.g. "User" → "USER.md — About User")
96
- if (fits(bounds.name)) {
145
+ if (fits(bounds.name) && mayBeWithin(bounds.names, name)) {
97
146
  const byPartial = (0, statements_1.prepareOnce)(db, `SELECT * FROM entities WHERE instr(lower(canonical_name), lower(?)) > 0${where} ORDER BY length(canonical_name) ASC, ${tieOrder()} LIMIT 1`)
98
147
  .get(name, ...params);
99
148
  if (byPartial)
100
149
  return toRow(byPartial);
101
150
  }
102
151
  // Path match: source note path contains the search term (e.g. "Cora" → notes in .../Cora/...)
103
- if (fits(bounds.note)) {
152
+ if (fits(bounds.note) && mayBeWithin(bounds.notes, name)) {
104
153
  const byPath = (0, statements_1.prepareOnce)(db, `SELECT * FROM entities WHERE instr(lower(source_note), lower(?)) > 0${where} ORDER BY length(canonical_name) ASC, ${tieOrder()} LIMIT 1`)
105
154
  .get(name, ...params);
106
155
  if (byPath)
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
3
+ exports.ENTITY_LANE_RESERVED = exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
4
4
  exports.searchBestFirstEnabled = searchBestFirstEnabled;
5
5
  exports.parseQueryDateRef = parseQueryDateRef;
6
6
  exports.queryDateRefMs = queryDateRefMs;
@@ -22,6 +22,7 @@ exports.isSpeakerStub = isSpeakerStub;
22
22
  exports.resolveSessionDiversityCap = resolveSessionDiversityCap;
23
23
  exports.resolveSessionExpand = resolveSessionExpand;
24
24
  exports.resolveRankedExpansion = resolveRankedExpansion;
25
+ exports.rankedStatementTokens = rankedStatementTokens;
25
26
  exports.calibratedModelKey = calibratedModelKey;
26
27
  exports.retrieveContext = retrieveContext;
27
28
  const ledgerLiveness_1 = require("./ledgerLiveness");
@@ -39,6 +40,8 @@ const contextRelevance_1 = require("../mvp/contextRelevance");
39
40
  const keyword_1 = require("./keyword");
40
41
  const queryHelpers_1 = require("./queryHelpers");
41
42
  const ledgerPresentation_2 = require("./ledgerPresentation");
43
+ const plainWording_1 = require("./plainWording");
44
+ const rankingTrace_1 = require("./rankingTrace");
42
45
  function envFlagOn(name) {
43
46
  const v = (process.env[name] ?? '').trim().toLowerCase();
44
47
  return v === '1' || v === 'true' || v === 'yes' || v === 'on';
@@ -460,7 +463,7 @@ function resolveSessionDiversityCap(env = process.env, unsetCap = exports.SESSIO
460
463
  function countsTowardNoteCap(h) {
461
464
  return h.chunkId > 0 && (0, ledgerLiveness_1.ledgerEventIds)(h).length === 0 && (0, dailyNote_1.dailyNoteDayName)(h.notePath) === null;
462
465
  }
463
- function applySessionDiversity(hits, cap, topK) {
466
+ function applySessionDiversity(hits, cap, _topK) {
464
467
  // `cap < 1` is the flag-OFF short-circuit and must stay: 0 restores the window exactly as it was
465
468
  // before the default existed. The former `|| hits.length <= topK` here was a bypass, not an optimization — it
466
469
  // skipped the configured cap entirely whenever the pool was no deeper than the window.
@@ -603,10 +606,38 @@ function resolveRankedExpansion(env = process.env) {
603
606
  return {
604
607
  notes: rawNotes === '' ? Number.POSITIVE_INFINITY : legacy.notes,
605
608
  tokenBudget: rawBudget === '' ? exports.SESSION_EXPAND_RANKED_TOKENS : legacy.tokenBudget,
606
- statementTokens: legacy.statementTokens,
609
+ statementTokens: rankedStatementTokens(env) ?? legacy.statementTokens,
607
610
  documentStatementTokens: legacy.documentStatementTokens,
608
611
  };
609
612
  }
613
+ /**
614
+ * The longest conversation statement the RANKED expansion adds when nothing says otherwise: 400 tokens. The former expansion
615
+ * (a command line that does not declare a calibrated model) keeps RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120).
616
+ */
617
+ exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = 400;
618
+ /**
619
+ * RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS (default 400 since the flip of 2026-10-25): the longest conversation statement,
620
+ * in tokens, that the RANKED expansion alone may add. Unset or empty: 400, unless RENOR_SESSION_EXPAND_STATEMENT_TOKENS is set,
621
+ * which an operator's explicit value keeps winning as it always did. A number of 1 or more: that limit, over the general one.
622
+ * The off switch: `0`, a negative number or anything that is not a number returns null, and the ranked expansion then takes
623
+ * RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120 unset), as it did before the flip. The former expansion and the documents are
624
+ * never touched by it.
625
+ *
626
+ * Why 400: the session expansion skips a conversation statement over its limit, whole, and the long user turn that answers a
627
+ * question is often just over 120 tokens (501 to 700 characters: missing from the page 5 times in 21, against 6 in 777 for
628
+ * shorter ones, on the Haiku default run of 2026-10-23). The chunker leaves no section over 400 tokens as one chunk, so 400 is
629
+ * the largest value that changes anything. On the official LongMemEval-S instrument with a Haiku 5.5 reader the ranked
630
+ * expansion with 400 scored 433 of 500 and with 120 (the control) 431, against 418, 417 and 423 for the command-line default;
631
+ * the four pages only the limit completes are right 4 of 4 with it and 0 of 4 without, and the page is 5.5% smaller:
632
+ * engine/docs/evals/results/ranked-statement-limit-20261024/ and LEDGER sections BE and BF.
633
+ */
634
+ function rankedStatementTokens(env = process.env) {
635
+ const raw = (env.RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS ?? '').trim();
636
+ if (raw === '')
637
+ return (env.RENOR_SESSION_EXPAND_STATEMENT_TOKENS ?? '').trim() === '' ? exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS : null;
638
+ const value = Number(raw);
639
+ return Number.isFinite(value) && value >= 1 ? Math.floor(value) : null;
640
+ }
610
641
  /**
611
642
  * The calibrated key (CALIBRATED_BARS) of the embedding model behind a provider, or null when there is none.
612
643
  *
@@ -758,9 +789,13 @@ async function retrieveContext(db, vectorIndex, provider, query, topK = 20, prio
758
789
  contentfulHeld.clear(); // what another read worked out is not this read's
759
790
  const factHits = (0, factsFirst_1.factsFirstHits)(db, query, Math.min(3, topK), opts.scope?.shows);
760
791
  const queryProbe = opts.probe ?? {}; // the query vector, for the expansion's ranking
761
- const candidateLimit = Math.min(40, topK * 4);
792
+ // RENOR_NEAR_DUP_CAP (default off): the pool is read deeper, so the places near-copies leave are taken (plainWording.ts).
793
+ const nearDupCap = (0, plainWording_1.resolveNearDupCap)();
794
+ const candidateLimit = nearDupCap > 0 ? Math.min(plainWording_1.NEAR_DUP_POOL.max, topK * plainWording_1.NEAR_DUP_POOL.perTopK) : Math.min(40, topK * 4);
795
+ // RENOR_QUERY_REWRITE (default off): what is embedded is the query without its conversational wrapper.
796
+ const embeddedQuery = (0, plainWording_1.queryRewriteEnabled)() ? (0, plainWording_1.rewriteConversationalQuery)(query) : query;
762
797
  const semanticHits = vectorIndex && provider
763
- ? await (0, semantic_1.semanticSearch)(db, vectorIndex, provider, query, candidateLimit, queryProbe, opts.scope)
798
+ ? await (0, semantic_1.semanticSearch)(db, vectorIndex, provider, embeddedQuery, candidateLimit, queryProbe, opts.scope)
764
799
  : (() => {
765
800
  const match = (0, queryHelpers_1.quoteFtsQuery)(query);
766
801
  if (!match)
@@ -772,13 +807,69 @@ async function retrieveContext(db, vectorIndex, provider, query, topK = 20, prio
772
807
  return opts.scope ? live.filter(hit => opts.scope.shows(hit)) : live;
773
808
  }, candidateLimit);
774
809
  })();
810
+ // RENOR_ENTITY_LANE (default off): the chunks that carry the rare words the query names join the pool at their own cosine.
811
+ const laneTerms = (0, plainWording_1.entityLaneEnabled)() && vectorIndex && queryProbe.queryVector
812
+ ? (0, readCache_1.inReadSnapshot)(db, () => entityLanePool(db, vectorIndex, queryProbe.queryVector, query, topK, semanticHits, opts))
813
+ : [];
775
814
  // The window is ranked without waiting for anything, so it is read in one transaction (inReadSnapshot).
776
- const window = (0, readCache_1.inReadSnapshot)(db, () => rankWindow({ db, query, topK, priorityPaths, clock, opts, factHits, semanticHits }));
815
+ const window = (0, readCache_1.inReadSnapshot)(db, () => rankWindow({ db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap }));
777
816
  return finishContext({ db, vectorIndex, provider, query, queryProbe, window, bestFirst: opts.bestFirst === true });
778
817
  }
818
+ /**
819
+ * The entity lane (RENOR_ENTITY_LANE): the lane words of the query (plainWording.ts entityLaneTerms) and, appended to
820
+ * `pool`, the chunks that carry them and are not in it yet, each at its own cosine to the query vector (one
821
+ * `VectorIndex.search` among their ids: no embedding, no model call). A scoped read judges them as it judges the pool.
822
+ * Returns the lane words, which the window lifts every hit that carries them by.
823
+ */
824
+ function entityLanePool(db, vectorIndex, queryVector, query, topK, pool, opts) {
825
+ const terms = (0, plainWording_1.entityLaneTerms)(db, query);
826
+ if (terms.length === 0)
827
+ return [];
828
+ const inPool = new Set(pool.map(h => h.chunkId));
829
+ const ids = (0, plainWording_1.entityLaneChunkIds)(db, terms, topK, opts.isLedgerNote).filter(id => !inPool.has(id));
830
+ if (ids.length === 0)
831
+ return terms;
832
+ const cosine = new Map(vectorIndex.search(queryVector, ids.length, ids).map(h => [h.chunkId, h.score]));
833
+ const rows = (0, statements_1.prepareOnce)(db, 'SELECT id AS chunkId, note_path AS notePath, heading_path AS headingPath, text FROM chunks WHERE id IN (SELECT value FROM json_each(?))').all(JSON.stringify(ids));
834
+ for (const row of rows) {
835
+ if (!cosine.has(row.chunkId))
836
+ continue; // no vector yet: the semantic lane could not have found it either
837
+ if (opts.scope && !opts.scope.shows(row))
838
+ continue;
839
+ pool.push({ ...row, score: cosine.get(row.chunkId) });
840
+ }
841
+ return terms;
842
+ }
843
+ /** How many places of a window the entity lane keeps for the best hits that carry every lane word. */
844
+ exports.ENTITY_LANE_RESERVED = 3;
845
+ /**
846
+ * The first `topK` of `ranked`, but with its best ENTITY_LANE_RESERVED hits that carry every lane word in it: each one
847
+ * the cut would leave out takes the place of the last hit that carries none. A lift alone depends on the embedding's
848
+ * scale (a cosine of 0.8 against 0.1 on the lexical stub, a few hundredths on a real model); a place does not. Without
849
+ * lane words, the first `topK`.
850
+ */
851
+ function reserveLanePlaces(ranked, topK, laneTerms) {
852
+ const window = ranked.slice(0, topK);
853
+ if (laneTerms.length === 0)
854
+ return window;
855
+ const carries = (h) => (0, plainWording_1.laneTermShare)(h.text, laneTerms) === 1;
856
+ const missing = ranked.filter(carries).slice(0, exports.ENTITY_LANE_RESERVED).filter(h => !window.includes(h));
857
+ for (const hit of missing) {
858
+ let at = window.length - 1;
859
+ while (at >= 0 && (0, plainWording_1.laneTermShare)(window[at].text, laneTerms) > 0)
860
+ at--;
861
+ if (at < 0)
862
+ break;
863
+ window.splice(at, 1);
864
+ window.push(hit);
865
+ }
866
+ return window;
867
+ }
779
868
  /** The window of hits of a read: scored, with the graph of its notes, their facts and dates, live and capped per note. Waits for nothing. */
780
869
  function rankWindow(read) {
781
- const { db, query, topK, priorityPaths, clock, opts, factHits, semanticHits } = read;
870
+ const { db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap } = read;
871
+ // Set only while the learning log collects the ranker's terms (rankingTrace.ts); otherwise every use below is skipped.
872
+ const trace = (0, rankingTrace_1.currentRankingTrace)();
782
873
  // Raw cosines, kept only for a caller that asked to see them (get_context's relevance signal).
783
874
  const cosineByChunk = opts.probe ? new Map(semanticHits.map(h => [h.chunkId, h.score])) : null;
784
875
  const entityHit = (0, entityLookup_1.lookupEntity)(db, query.trim());
@@ -800,8 +891,11 @@ function rankWindow(read) {
800
891
  const scored = semanticHits.map(h => {
801
892
  let score = h.score;
802
893
  let reason = 'semantic';
894
+ const terms = trace ? (0, rankingTrace_1.newRankingTerms)() : null;
803
895
  if (isSpeakerStub(h.text)) {
804
896
  score -= 0.5;
897
+ if (terms)
898
+ terms[(0, rankingTrace_1.rankingTermIndex)('stub')] -= 0.5;
805
899
  reason = 'semantic+stub';
806
900
  }
807
901
  else {
@@ -809,15 +903,31 @@ function rankWindow(read) {
809
903
  if (boost > 0) {
810
904
  score += boost;
811
905
  reason += '+content';
906
+ if (terms)
907
+ terms[(0, rankingTrace_1.rankingTermIndex)('content')] += boost;
812
908
  }
813
909
  }
814
910
  if (priorityPaths.some(p => h.notePath.includes('/' + p) || h.notePath.startsWith(p))) {
815
911
  score += 0.1;
816
912
  reason += '+priority';
913
+ if (terms)
914
+ terms[(0, rankingTrace_1.rankingTermIndex)('priority')] += 0.1;
817
915
  }
818
916
  if (entityHit?.sourceNote === h.notePath) {
819
917
  score += 0.2;
820
918
  reason += '+entity';
919
+ if (terms)
920
+ terms[(0, rankingTrace_1.rankingTermIndex)('entity')] += 0.2;
921
+ }
922
+ if (laneTerms.length > 0) {
923
+ const share = (0, plainWording_1.laneTermShare)(h.text, laneTerms);
924
+ if (share > 0) {
925
+ const boost = plainWording_1.ENTITY_LANE_BOOST * share;
926
+ score += boost;
927
+ reason += '+entity-lane';
928
+ if (terms)
929
+ terms[(0, rankingTrace_1.rankingTermIndex)('entity_lane')] += boost;
930
+ }
821
931
  }
822
932
  // Time-decay recency: +0.1 for today, linearly decaying to 0 over 30 days.
823
933
  // Skipped for explicit past-date (as-of) queries: recency answers "now",
@@ -830,9 +940,12 @@ function rankWindow(read) {
830
940
  if (recencyBoost > 0.001) {
831
941
  score += recencyBoost;
832
942
  reason += '+recency';
943
+ if (terms)
944
+ terms[(0, rankingTrace_1.rankingTermIndex)('recency')] += recencyBoost;
833
945
  }
834
946
  }
835
- return { ...h, score, reason, session_date };
947
+ const out = { ...h, score, reason, session_date };
948
+ return terms ? (0, rankingTrace_1.attachRankingTerms)(out, terms) : out;
836
949
  });
837
950
  // Graph expansion
838
951
  const hitNotePaths = [...new Set(semanticHits.map(h => h.notePath))];
@@ -933,6 +1046,8 @@ function rankWindow(read) {
933
1046
  (f.predicate ?? '').toLowerCase().includes(w)));
934
1047
  if (factMatch) {
935
1048
  hit.score += 0.15;
1049
+ if (trace)
1050
+ (0, rankingTrace_1.addRankingTerm)(hit, 'fact', 0.15);
936
1051
  hit.reason += '+fact';
937
1052
  }
938
1053
  // As-of lane: the query names a past date, so hits carrying the same
@@ -943,11 +1058,15 @@ function rankWindow(read) {
943
1058
  const lowerText = hit.text.toLowerCase();
944
1059
  if (lowerText.includes(yearToken)) {
945
1060
  hit.score += 0.08;
1061
+ if (trace)
1062
+ (0, rankingTrace_1.addRankingTerm)(hit, 'asof_year', 0.08);
946
1063
  hit.reason += '+asof-year';
947
1064
  if (asofRef.month !== undefined) {
948
1065
  const monthName = Object.keys(ASOF_MONTHS).find(m => ASOF_MONTHS[m] === asofRef.month);
949
1066
  if (monthName && lowerText.includes(monthName)) {
950
1067
  hit.score += 0.04;
1068
+ if (trace)
1069
+ (0, rankingTrace_1.addRankingTerm)(hit, 'asof_month', 0.04);
951
1070
  hit.reason += '+asof-month';
952
1071
  }
953
1072
  }
@@ -970,6 +1089,8 @@ function rankWindow(read) {
970
1089
  // Must clear recency (+0.1 max) + fact boost (+0.15) the current note
971
1090
  // typically carries, or past-intent queries can never win.
972
1091
  hit.score += 0.3;
1092
+ if (trace)
1093
+ (0, rankingTrace_1.addRankingTerm)(hit, 'historical_intent', 0.3);
973
1094
  hit.reason += '+historical-intent';
974
1095
  }
975
1096
  }
@@ -982,10 +1103,14 @@ function rankWindow(read) {
982
1103
  if (stale.length >= 3 && text.includes(stale)) {
983
1104
  if (pastIntent) {
984
1105
  hit.score += 0.1;
1106
+ if (trace)
1107
+ (0, rankingTrace_1.addRankingTerm)(hit, 'past_fact', 0.1);
985
1108
  hit.reason += '+historical-intent';
986
1109
  }
987
1110
  else {
988
1111
  hit.score -= 0.15;
1112
+ if (trace)
1113
+ (0, rankingTrace_1.addRankingTerm)(hit, 'stale_fact', -0.15);
989
1114
  hit.reason += '+stale-fact';
990
1115
  }
991
1116
  break; // one demotion per hit; stale signal already applied
@@ -1002,6 +1127,8 @@ function rankWindow(read) {
1002
1127
  const cur = (f.object_text ?? '').trim().toLowerCase();
1003
1128
  if (cur.length >= 3 && text.includes(cur)) {
1004
1129
  hit.score += 0.1;
1130
+ if (trace)
1131
+ (0, rankingTrace_1.addRankingTerm)(hit, 'current_fact', 0.1);
1005
1132
  hit.reason += '+current-fact';
1006
1133
  break;
1007
1134
  }
@@ -1010,6 +1137,8 @@ function rankWindow(read) {
1010
1137
  }
1011
1138
  }
1012
1139
  const ranked = dedupeHits(scored.sort(semantic_1.byScoreThenPlace));
1140
+ if (trace)
1141
+ ranked.forEach((hit, index) => trace.pool(hit, index + 1));
1013
1142
  // Fail-open floor. The stub filter must never empty the window: a zero-hit
1014
1143
  // retrieval is a hard abort on the product path, because the benchmark client
1015
1144
  // refuses to score a case with no hits ("Retrieval precondition failed: public
@@ -1027,7 +1156,13 @@ function rankWindow(read) {
1027
1156
  // The ledger's own header is no memory: it comes off with the closed events, before the page is cut.
1028
1157
  const mixed = (0, ledgerPresentation_1.dropLedgerPreamble)((0, ledgerLiveness_1.resolveLedgerHits)([...datedFactHits, ...contentful.filter(h => !seen.has(h.text))], (0, ledgerLiveness_1.closedLedgerEvents)(db), opts.includeHistory === true), opts.isLedgerNote);
1029
1158
  const diversityCap = resolveSessionDiversityCap(process.env, opts.noteCapWhenUnset);
1030
- let hits = applySessionDiversity(mixed, diversityCap, topK).slice(0, topK).map(h => ({
1159
+ // RENOR_NEAR_DUP_CAP (default off): near-copies are capped before the per-note cap, on the visible text.
1160
+ const windowed = reserveLanePlaces(applySessionDiversity((0, plainWording_1.applyNearDupCap)(mixed, nearDupCap, contentfulText), diversityCap, topK), topK, laneTerms);
1161
+ // The window's candidates are recorded before they are copied into the plain hits below, which carry no terms.
1162
+ if (trace)
1163
+ for (const hit of windowed)
1164
+ trace.touch(hit);
1165
+ let hits = windowed.map(h => ({
1031
1166
  notePath: h.notePath,
1032
1167
  chunkId: h.chunkId,
1033
1168
  headingPath: h.headingPath,
@@ -1082,5 +1217,8 @@ async function finishContext(read) {
1082
1217
  const newestFirst = !bestFirst && newestFirstEnabled() && !(temporal && pastIntent) && !asofPast;
1083
1218
  const rankedBy = newestFirst ? 'session_date' : 'score';
1084
1219
  contentfulHeld.clear(); // nothing outlives the read it was worked out for
1085
- return { query, hits: orderHits(hits, rankedBy), rankedBy };
1220
+ // The order the reply presents its hits in, so a candidate's place in it is recorded for the learning log (rankingTrace.ts).
1221
+ const presented = orderHits(hits, rankedBy);
1222
+ (0, rankingTrace_1.currentRankingTrace)()?.presented(presented);
1223
+ return { query, hits: presented, rankedBy };
1086
1224
  }
@@ -0,0 +1,236 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.NEAR_DUP_POOL = exports.NEAR_DUP_JACCARD = exports.ENTITY_LANE_DF_SHARE = exports.ENTITY_LANE_MIN_DF_LIMIT = exports.ENTITY_LANE_MAX_QUERY_TERMS = exports.ENTITY_LANE_MAX_TERMS = exports.ENTITY_LANE_BOOST = void 0;
4
+ exports.entityLaneEnabled = entityLaneEnabled;
5
+ exports.queryRewriteEnabled = queryRewriteEnabled;
6
+ exports.resolveNearDupCap = resolveNearDupCap;
7
+ exports.contentTerms = contentTerms;
8
+ exports.rewriteConversationalQuery = rewriteConversationalQuery;
9
+ exports.entityLaneTerms = entityLaneTerms;
10
+ exports.entityLaneChunkIds = entityLaneChunkIds;
11
+ exports.laneTermShare = laneTermShare;
12
+ exports.jaccard = jaccard;
13
+ exports.applyNearDupCap = applyNearDupCap;
14
+ const statements_1 = require("../db/statements");
15
+ const keyword_1 = require("./keyword");
16
+ const ledgerPresentation_1 = require("./ledgerPresentation");
17
+ const queryHelpers_1 = require("./queryHelpers");
18
+ /**
19
+ * Plain-wording retrieval and crowding: three switches, each default OFF, each a kill switch of its own.
20
+ *
21
+ * Why (the owner's real use, 2026-10-21): where Renor held the memory, the owner's own words as the query returned it
22
+ * 5 of 24 times against 19 of 24 for a hand-aimed topic query, and of 183 agent queries that once returned a memory the
23
+ * agent acted on, 40% no longer return it as the workspace grows. None of the three calls a model.
24
+ *
25
+ * - RENOR_ENTITY_LANE: the rare words a query names (a subject such as "jev", a service, a project) are looked up in
26
+ * the keyword index; the chunks that carry them join the semantic candidates at their own cosine, and every candidate
27
+ * that carries them is lifted by up to ENTITY_LANE_BOOST (the lift `+entity` already gives an entity's note).
28
+ * - RENOR_QUERY_REWRITE: the conversational wrapper of a query ("hey, do you remember what we said about ...") is taken
29
+ * off before it is embedded, so the words that are embedded are the ones stored memories use.
30
+ * - RENOR_NEAR_DUP_CAP=<n>: at most `n` near-copies (word-set Jaccard >= NEAR_DUP_JACCARD) in one window, the best
31
+ * scored kept; with it the candidate pool is read deeper (NEAR_DUP_POOL), so the distinct memories that newer
32
+ * near-copies pushed below the old pool depth can take the freed places.
33
+ */
34
+ function on(value) {
35
+ const v = (value ?? '').trim().toLowerCase();
36
+ return v === '1' || v === 'true' || v === 'yes' || v === 'on';
37
+ }
38
+ function entityLaneEnabled(env = process.env) {
39
+ return on(env.RENOR_ENTITY_LANE);
40
+ }
41
+ function queryRewriteEnabled(env = process.env) {
42
+ return on(env.RENOR_QUERY_REWRITE);
43
+ }
44
+ /** The near-copy cap, or 0 (off) when unset, empty, junk or below 1. */
45
+ function resolveNearDupCap(env = process.env) {
46
+ const raw = (env.RENOR_NEAR_DUP_CAP ?? '').trim();
47
+ if (raw === '')
48
+ return 0;
49
+ const cap = Number(raw);
50
+ return Number.isFinite(cap) && cap >= 1 ? Math.floor(cap) : 0;
51
+ }
52
+ /** Words that never name what a memory is about: function words and the words of asking and remembering. */
53
+ const PLAIN_STOP = new Set([
54
+ 'the', 'and', 'for', 'are', 'was', 'were', 'what', 'which', 'who', 'whom', 'whose', 'where', 'when', 'why', 'how',
55
+ 'does', 'did', 'done', 'with', 'about', 'this', 'that', 'these', 'those', 'there', 'here', 'then', 'than', 'from',
56
+ 'into', 'onto', 'over', 'your', 'yours', 'our', 'ours', 'their', 'theirs', 'his', 'her', 'hers', 'its', 'you', 'they',
57
+ 'them', 'she', 'him', 'can', 'could', 'would', 'should', 'will', 'shall', 'might', 'must', 'may', 'have', 'has', 'had',
58
+ 'been', 'being', 'not', 'any', 'all', 'some', 'just', 'also', 'very', 'really', 'again', 'still', 'now', 'ever',
59
+ 'remember', 'remind', 'recall', 'tell', 'told', 'say', 'said', 'know', 'think', 'thought', 'mean', 'meant', 'please',
60
+ 'thanks', 'hey', 'okay', 'yeah', 'yes', 'like', 'thing', 'things', 'stuff', 'something', 'anything', 'everything',
61
+ 'one', 'ones', 'get', 'got', 'let', 'lets', 'need', 'want', 'look', 'check', 'find', 'give', 'show', 'saved', 'save',
62
+ 'memory', 'memories', 'renor', 'claude', 'myself', 'yourself', 'we', 'us', 'me', 'my', 'it', 'is', 'in', 'on', 'of',
63
+ 'to', 'do', 'be', 'or', 'an', 'at', 'as', 'by', 'if', 'so', 'up', 'am', 'last', 'time', 'before', 'earlier', 'after',
64
+ 'since', 'until', 'during', 'while', 'because', 'between', 'other', 'another', 'first', 'next', 'every', 'each', 'many',
65
+ 'much', 'more', 'most', 'less', 'same', 'such', 'only', 'well', 'today', 'yesterday', 'ago', 'way', 'kind', 'sort',
66
+ ]);
67
+ /** The words of `query` that could name its subject: three characters or more (a hyphenated name is one word), no stop word. */
68
+ function contentTerms(query) {
69
+ const words = (query ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}][\p{L}\p{N}\p{M}_-]*/gu) ?? [];
70
+ const out = [];
71
+ for (const raw of words) {
72
+ const word = raw.replace(/[-_]+$/, '').replace(/'s$/, '');
73
+ if ([...word].length < 3 || PLAIN_STOP.has(word) || out.includes(word))
74
+ continue;
75
+ out.push(word);
76
+ }
77
+ return out;
78
+ }
79
+ /**
80
+ * Wrappers a person puts around a question when they talk to an agent. Taken off the front of the query, repeatedly,
81
+ * and only there: inside a query the same words can be what it is about.
82
+ */
83
+ const LEADING_WRAPPERS = [
84
+ /^(?:hey|hi|hello|ok|okay|so|um|uh|well|right|and|but|also|btw|by the way|quick question|question)\b[\s,:;.!-]*/i,
85
+ /^(?:claude|renor|codex|cursor)\b[\s,:;.!-]*/i,
86
+ /^(?:please|pls|can you|could you|would you|will you|do you|did you|are you able to)\b\s*/i,
87
+ /^(?:(?:still )?remember|recall|remind me(?: of)?|tell me|look up|find|check|pull up|bring up|show me)\b\s*/i,
88
+ /^(?:what (?:(?:did|do|have) )?(?:i|we|you) (?:say|said|tell you|told you|decide|decided|note|noted|save|saved|write|wrote|agree|agreed|settle|settled)(?: on)?)\b\s*/i,
89
+ /^(?:what (?:was|were|is|are) (?:it|that|the thing|those|these))\b\s*/i,
90
+ /^(?:the (?:thing|stuff|notes?)|our (?:notes?|discussion|conversation))\b\s*/i,
91
+ /^(?:about|on|regarding|re|for|of|with|that)\b[\s,:]*/i,
92
+ ];
93
+ /**
94
+ * The query without its conversational wrapper ("hey, do you remember what we said about the jev measurements?" ->
95
+ * "the jev measurements"). The question's own words are kept as written; a trailing "again", "?" or "please" goes too.
96
+ * When the rewrite would leave fewer than two content words, or nothing changed, the query is returned as it came.
97
+ */
98
+ function rewriteConversationalQuery(query) {
99
+ const original = (query ?? '').trim();
100
+ let q = original;
101
+ for (let guard = 0; guard < 12; guard++) {
102
+ const before = q;
103
+ for (const wrapper of LEADING_WRAPPERS)
104
+ q = q.replace(wrapper, '');
105
+ q = q.trim();
106
+ if (q === before)
107
+ break;
108
+ }
109
+ q = q.replace(/[\s,]*(?:again|please|for me|real quick|if you can|do you remember|remember)?[\s?.!]*$/i, '').trim();
110
+ const kept = contentTerms(q).length;
111
+ if (q === original.replace(/[\s?.!]+$/, '') || kept === 0 || kept < Math.min(2, contentTerms(original).length))
112
+ return original;
113
+ return q;
114
+ }
115
+ /** The lift a candidate that carries every lane word gets: the lift `+entity` gives an entity's note (orchestrator.ts). */
116
+ exports.ENTITY_LANE_BOOST = 0.2;
117
+ /** At most this many lane words per query: the rarest. */
118
+ exports.ENTITY_LANE_MAX_TERMS = 3;
119
+ /** The lane reads queries of at most this many content words. */
120
+ exports.ENTITY_LANE_MAX_QUERY_TERMS = 12;
121
+ /** A word is rare enough to name a subject when at most this many chunks carry it, or ENTITY_LANE_DF_SHARE of all chunks. */
122
+ exports.ENTITY_LANE_MIN_DF_LIMIT = 5;
123
+ exports.ENTITY_LANE_DF_SHARE = 0.02;
124
+ /**
125
+ * The lane words of `query`: its content words that name a registered entity, or that few enough chunks carry to name a
126
+ * subject, rarest first (a named entity before any other word). Two bounded lookups per word; none for a query of more than
127
+ * ENTITY_LANE_MAX_QUERY_TERMS content words.
128
+ */
129
+ function entityLaneTerms(db, query) {
130
+ const terms = contentTerms(query);
131
+ // A query of many words is a passage, not a named subject: its rarest word is as likely a heading's (measured: a pasted
132
+ // section lost its own chunk to a "Parking lot" section of another note). The semantic lane already matches a passage.
133
+ if (terms.length === 0 || terms.length > exports.ENTITY_LANE_MAX_QUERY_TERMS)
134
+ return [];
135
+ const chunks = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM chunks').get().n;
136
+ const limit = Math.max(exports.ENTITY_LANE_MIN_DF_LIMIT, Math.floor(chunks * exports.ENTITY_LANE_DF_SHARE));
137
+ const counted = [];
138
+ for (const term of terms) {
139
+ const match = (0, queryHelpers_1.quoteFtsQuery)(term);
140
+ if (!match)
141
+ continue;
142
+ let df;
143
+ try {
144
+ df = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM (SELECT 1 FROM chunks_fts WHERE chunks_fts MATCH ? LIMIT ?)').get(match, limit + 1).n;
145
+ }
146
+ catch {
147
+ continue; // a word FTS5 cannot read is no lane word
148
+ }
149
+ if (df === 0)
150
+ continue;
151
+ const named = (0, statements_1.prepareOnce)(db, `SELECT 1 FROM entities WHERE canonical_name = ? COLLATE NOCASE
152
+ UNION ALL SELECT 1 FROM entities, json_each(entities.aliases_json) AS alias WHERE lower(alias.value) = ? LIMIT 1`).get(term, term) !== undefined;
153
+ if (named || df <= limit)
154
+ counted.push({ term, df, named });
155
+ }
156
+ return counted
157
+ .sort((a, b) => Number(b.named) - Number(a.named) || a.df - b.df)
158
+ .slice(0, exports.ENTITY_LANE_MAX_TERMS)
159
+ .map(c => c.term);
160
+ }
161
+ /**
162
+ * The chunks that carry any lane word, best keyword match first, at most `perTerm` for each word. Read as every keyword
163
+ * reader reads (shownPage): the ledger's header and the events the word is not in are judged away before the page is cut.
164
+ */
165
+ function entityLaneChunkIds(db, terms, perTerm, isLedgerNote) {
166
+ const ids = [];
167
+ for (const term of terms) {
168
+ const match = (0, queryHelpers_1.quoteFtsQuery)(term);
169
+ if (!match)
170
+ continue;
171
+ const page = (0, queryHelpers_1.shownPage)(rows => (0, keyword_1.keywordSearch)(db, match, rows), rows => (0, ledgerPresentation_1.keywordLedgerHits)(rows, isLedgerNote, term), perTerm);
172
+ for (const hit of page)
173
+ if (!ids.includes(hit.chunkId))
174
+ ids.push(hit.chunkId);
175
+ }
176
+ return ids;
177
+ }
178
+ function escapeRegExp(s) {
179
+ return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
180
+ }
181
+ /** The share of `terms` that `text` carries as whole words (a hyphen or underscore in a lane word matches any separator). */
182
+ function laneTermShare(text, terms) {
183
+ if (terms.length === 0)
184
+ return 0;
185
+ const lower = (text ?? '').toLowerCase().normalize('NFC');
186
+ let carried = 0;
187
+ for (const term of terms) {
188
+ const pattern = term.split(/[-_]+/).map(escapeRegExp).join('[\\s_-]+');
189
+ if (new RegExp(`(?<![\\p{L}\\p{N}])${pattern}(?![\\p{L}\\p{N}])`, 'u').test(lower))
190
+ carried++;
191
+ }
192
+ return carried / terms.length;
193
+ }
194
+ /** Two texts whose word sets share at least this share of their union are near-copies. */
195
+ exports.NEAR_DUP_JACCARD = 0.6;
196
+ /** How deep the candidate pool is read while the near-copy cap is on (the default reads min(40, topK * 4)). */
197
+ exports.NEAR_DUP_POOL = { perTopK: 8, max: 160 };
198
+ function wordSet(text) {
199
+ return new Set((text ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}]+/gu) ?? []);
200
+ }
201
+ function jaccard(a, b) {
202
+ if (a.size === 0 && b.size === 0)
203
+ return 1;
204
+ let shared = 0;
205
+ for (const word of a)
206
+ if (b.has(word))
207
+ shared++;
208
+ return shared / (a.size + b.size - shared);
209
+ }
210
+ /**
211
+ * At most `cap` near-copies in the window: in rank order, a hit is kept while fewer than `cap` kept hits are near-copies
212
+ * of it. A fact row (a negative chunkId) is left alone. `cap < 1` returns the hits as they came. `textOf` is the text the
213
+ * copies are compared on (the orchestrator passes the visible text, without markdown and speaker labels).
214
+ */
215
+ function applyNearDupCap(hits, cap, textOf = t => t) {
216
+ if (cap < 1)
217
+ return hits;
218
+ const kept = [];
219
+ const keptWords = [];
220
+ for (const hit of hits) {
221
+ if (hit.chunkId < 0) {
222
+ kept.push(hit);
223
+ continue;
224
+ }
225
+ const words = wordSet(textOf(hit.text));
226
+ let copies = 0;
227
+ for (const other of keptWords)
228
+ if (jaccard(words, other) >= exports.NEAR_DUP_JACCARD && ++copies >= cap)
229
+ break;
230
+ if (copies >= cap)
231
+ continue;
232
+ kept.push(hit);
233
+ keptWords.push(words);
234
+ }
235
+ return kept;
236
+ }