renor-engine 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -661
- package/NOTICE +8 -6
- package/README.md +13 -8
- package/dist/.tsbuildinfo +1 -0
- package/dist/api/server.js +10 -8
- package/dist/buildIdentity.js +18 -1
- package/dist/callPhases.js +82 -0
- package/dist/cli.js +36 -1
- package/dist/context/rowStore.js +1 -1
- package/dist/conversation/parser.js +40 -36
- package/dist/db/client.js +6 -4
- package/dist/db/handoffTable.js +31 -0
- package/dist/db/indexSalvage.js +24 -4
- package/dist/db/migrations.js +30 -2
- package/dist/db/sqlite.js +396 -0
- package/dist/digests/digestCache.js +6 -2
- package/dist/digests/digests.js +3 -3
- package/dist/digests/worker.js +3 -1
- package/dist/embeddings/caching.js +5 -5
- package/dist/embeddings/local.js +5 -3
- package/dist/embeddings/openai.js +4 -3
- package/dist/embeddings/simdScan.js +3 -1
- package/dist/embeddings/vectorIndex.js +0 -1
- package/dist/embeddings/vertex.js +123 -13
- package/dist/eval/cacheReport.js +7 -0
- package/dist/eval/context700-candidate.js +5 -5
- package/dist/eval/evalCache.js +46 -3
- package/dist/eval/indexSnapshots.js +549 -0
- package/dist/eval/pageDump.js +67 -0
- package/dist/eval/runManifest.js +6 -0
- package/dist/eval/textTokens.js +17 -0
- package/dist/evidence/attach.js +0 -3
- package/dist/evidence/evidenceCache.js +3 -3
- package/dist/evidence/ollamaChat.js +1 -1
- package/dist/evidence/options.js +46 -25
- package/dist/evidence/secondPass.js +8 -170
- package/dist/extraction/factExtractor.js +1 -1
- package/dist/extraction/jevDecisionModel.js +307 -0
- package/dist/extraction/preferenceExtractor.js +1 -4
- package/dist/extraction/secretGuard.js +7 -3
- package/dist/extraction/typedFacts.js +1 -1
- package/dist/importing/apply.js +42 -14
- package/dist/importing/cli.js +9 -8
- package/dist/importing/existing.js +2 -0
- package/dist/importing/facts.js +0 -2
- package/dist/importing/plan.js +0 -5
- package/dist/importing/source.js +1 -1
- package/dist/indexer/parser.js +4 -1
- package/dist/indexer/pipeline.js +99 -279
- package/dist/indexer/walker.js +11 -25
- package/dist/learning/cli.js +113 -0
- package/dist/learning/schema.js +144 -0
- package/dist/learning/settings.js +45 -0
- package/dist/learning/store.js +267 -0
- package/dist/learning/tap.js +401 -0
- package/dist/mcp/server.js +36 -6
- package/dist/memory/dreamer/adjudicate.js +215 -0
- package/dist/memory/dreamer/apply.js +120 -0
- package/dist/memory/dreamer/budget.js +64 -0
- package/dist/memory/dreamer/candidates.js +149 -0
- package/dist/memory/dreamer/collection.js +104 -0
- package/dist/memory/dreamer/dreamer.js +352 -0
- package/dist/memory/dreamer/review.js +111 -0
- package/dist/memory/factConflicts.js +2 -0
- package/dist/memory/handoff.js +363 -0
- package/dist/memory/lifecycle.js +338 -80
- package/dist/memory/sourceName.js +77 -1
- package/dist/memory/statusFacts.js +61 -0
- package/dist/memory/writer.js +3 -2
- package/dist/memorymap/memoryMap.js +39 -13
- package/dist/memorymap/replacements.js +8 -3
- package/dist/mvp/agentMemory.js +109 -50
- package/dist/mvp/answerCheck.js +211 -0
- package/dist/mvp/cli.js +14 -1
- package/dist/mvp/compileCache.js +3 -10
- package/dist/mvp/compileSeed.js +9 -3
- package/dist/mvp/config.js +4 -0
- package/dist/mvp/contextDistinct.js +53 -0
- package/dist/mvp/contextProfile.js +87 -0
- package/dist/mvp/directorySurface.js +68 -0
- package/dist/mvp/engine.js +39 -26
- package/dist/mvp/exportAll.js +36 -2
- package/dist/mvp/factMarkers.js +18 -2
- package/dist/mvp/history.js +5 -4
- package/dist/mvp/log.js +118 -0
- package/dist/mvp/server.js +759 -136
- package/dist/mvp/serverCard.js +12 -3
- package/dist/mvp/telemetry.js +35 -2
- package/dist/net/poolFetch.js +174 -0
- package/dist/recall/actionRecall.js +282 -0
- package/dist/recall/autoRecall.js +447 -0
- package/dist/recall/searchRecall.js +189 -0
- package/dist/retrieval/entityLookup.js +53 -4
- package/dist/retrieval/orchestrator.js +148 -10
- package/dist/retrieval/plainWording.js +236 -0
- package/dist/retrieval/rankingTrace.js +184 -0
- package/dist/retrieval/readerLayout.js +143 -224
- package/dist/router/routerCache.js +2 -2
- package/dist/sharing/cli.js +181 -0
- package/dist/sharing/examples.js +136 -0
- package/dist/sharing/fields.js +225 -0
- package/dist/sharing/loop.js +80 -0
- package/dist/sharing/project.js +327 -0
- package/dist/sharing/send.js +466 -0
- package/dist/sharing/state.js +222 -0
- package/dist/sharing/wording.js +48 -0
- package/dist/watcher/watcher.js +5 -3
- package/package.json +5 -7
- package/dist/BUILD.json +0 -6
- package/dist/importing/index.js +0 -29
|
@@ -43,9 +43,34 @@ function liveEntityFilter() {
|
|
|
43
43
|
function tieOrder(table = '') {
|
|
44
44
|
return `(${table}source_note = '${LIFECYCLE_NOTE_PATH}') DESC, ${table}source_note, ${table}id`;
|
|
45
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* The units that occur in `column` of any entity, folded the way SQLite's `lower` folds (A to Z only: a build with ICU would
|
|
48
|
+
* fold more, and `lowerIsAscii` says whether this one does). Null when a value is not text.
|
|
49
|
+
*/
|
|
50
|
+
function unitsOf(db, column) {
|
|
51
|
+
const units = new Uint32Array(0x10000 / 32);
|
|
52
|
+
for (const value of (0, statements_1.prepareOnce)(db, `SELECT ${column} FROM entities`).pluck().all()) {
|
|
53
|
+
if (value === null)
|
|
54
|
+
continue; // instr(NULL, x) is NULL: such a row never matches
|
|
55
|
+
if (typeof value !== 'string')
|
|
56
|
+
return null;
|
|
57
|
+
for (let i = 0; i < value.length; i++) {
|
|
58
|
+
const unit = value.charCodeAt(i);
|
|
59
|
+
const folded = unit >= 65 && unit <= 90 ? unit + 32 : unit;
|
|
60
|
+
units[folded >> 5] |= 1 << (folded & 31);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return units;
|
|
64
|
+
}
|
|
65
|
+
/** Whether `lower` leaves everything but A to Z alone, which is what `unitsOf` assumes. */
|
|
66
|
+
function lowerIsAscii(db) {
|
|
67
|
+
const probe = 'ÀÉÎÕÜÇΩДЖİß';
|
|
68
|
+
return (0, statements_1.prepareOnce)(db, 'SELECT lower(?)').pluck().get(`${probe}ABC`) === `${probe}abc`;
|
|
69
|
+
}
|
|
46
70
|
/**
|
|
47
71
|
* The longest canonical name, alias and source path, in SQLite characters (one code point, not a JavaScript UTF-16
|
|
48
|
-
* unit). Read once until the next write. An alias list that cannot
|
|
72
|
+
* unit), and the units that occur in the names and in the paths. Read once until the next write. An alias list that cannot
|
|
73
|
+
* be read counts as unbounded, so that query still runs.
|
|
49
74
|
*/
|
|
50
75
|
function entityNameBounds(db) {
|
|
51
76
|
return (0, readCache_1.readUntilWrite)(db, 'entityLookupBounds', () => {
|
|
@@ -55,9 +80,33 @@ function entityNameBounds(db) {
|
|
|
55
80
|
alias = (0, statements_1.prepareOnce)(db, `SELECT COALESCE(MAX(length(alias.value)), 0) AS n FROM entities AS e, json_each(e.aliases_json) AS alias`).pluck().get();
|
|
56
81
|
}
|
|
57
82
|
catch { /* malformed aliases: do not skip the alias query */ }
|
|
58
|
-
|
|
83
|
+
const folded = lowerIsAscii(db);
|
|
84
|
+
return {
|
|
85
|
+
name: Number(row.name),
|
|
86
|
+
alias: Number(alias),
|
|
87
|
+
note: Number(row.note),
|
|
88
|
+
names: folded ? unitsOf(db, 'canonical_name') : null,
|
|
89
|
+
notes: folded ? unitsOf(db, 'source_note') : null,
|
|
90
|
+
};
|
|
59
91
|
});
|
|
60
92
|
}
|
|
93
|
+
/**
|
|
94
|
+
* False when `name` cannot be a substring of any value whose units are `units`: it has a unit (capitals folded) that none
|
|
95
|
+
* of them has. A name with a half of a pair, or a NUL, is left to SQLite.
|
|
96
|
+
*/
|
|
97
|
+
function mayBeWithin(units, name) {
|
|
98
|
+
if (units === null)
|
|
99
|
+
return true;
|
|
100
|
+
for (let i = 0; i < name.length; i++) {
|
|
101
|
+
const unit = name.charCodeAt(i);
|
|
102
|
+
if (unit === 0 || (unit >= 0xd800 && unit <= 0xdfff))
|
|
103
|
+
return true;
|
|
104
|
+
const folded = unit >= 65 && unit <= 90 ? unit + 32 : unit;
|
|
105
|
+
if ((units[folded >> 5] & (1 << (folded & 31))) === 0)
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
61
110
|
/**
|
|
62
111
|
* Resolve an entity by name or alias, then by substring of a canonical name,
|
|
63
112
|
* then by substring of its source note path.
|
|
@@ -93,14 +142,14 @@ function lookupEntity(db, name, filter) {
|
|
|
93
142
|
return toRow(byAlias);
|
|
94
143
|
}
|
|
95
144
|
// Partial match: canonical_name contains the search term (e.g. "User" → "USER.md — About User")
|
|
96
|
-
if (fits(bounds.name)) {
|
|
145
|
+
if (fits(bounds.name) && mayBeWithin(bounds.names, name)) {
|
|
97
146
|
const byPartial = (0, statements_1.prepareOnce)(db, `SELECT * FROM entities WHERE instr(lower(canonical_name), lower(?)) > 0${where} ORDER BY length(canonical_name) ASC, ${tieOrder()} LIMIT 1`)
|
|
98
147
|
.get(name, ...params);
|
|
99
148
|
if (byPartial)
|
|
100
149
|
return toRow(byPartial);
|
|
101
150
|
}
|
|
102
151
|
// Path match: source note path contains the search term (e.g. "Cora" → notes in .../Cora/...)
|
|
103
|
-
if (fits(bounds.note)) {
|
|
152
|
+
if (fits(bounds.note) && mayBeWithin(bounds.notes, name)) {
|
|
104
153
|
const byPath = (0, statements_1.prepareOnce)(db, `SELECT * FROM entities WHERE instr(lower(source_note), lower(?)) > 0${where} ORDER BY length(canonical_name) ASC, ${tieOrder()} LIMIT 1`)
|
|
105
154
|
.get(name, ...params);
|
|
106
155
|
if (byPath)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
|
|
3
|
+
exports.ENTITY_LANE_RESERVED = exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
|
|
4
4
|
exports.searchBestFirstEnabled = searchBestFirstEnabled;
|
|
5
5
|
exports.parseQueryDateRef = parseQueryDateRef;
|
|
6
6
|
exports.queryDateRefMs = queryDateRefMs;
|
|
@@ -22,6 +22,7 @@ exports.isSpeakerStub = isSpeakerStub;
|
|
|
22
22
|
exports.resolveSessionDiversityCap = resolveSessionDiversityCap;
|
|
23
23
|
exports.resolveSessionExpand = resolveSessionExpand;
|
|
24
24
|
exports.resolveRankedExpansion = resolveRankedExpansion;
|
|
25
|
+
exports.rankedStatementTokens = rankedStatementTokens;
|
|
25
26
|
exports.calibratedModelKey = calibratedModelKey;
|
|
26
27
|
exports.retrieveContext = retrieveContext;
|
|
27
28
|
const ledgerLiveness_1 = require("./ledgerLiveness");
|
|
@@ -39,6 +40,8 @@ const contextRelevance_1 = require("../mvp/contextRelevance");
|
|
|
39
40
|
const keyword_1 = require("./keyword");
|
|
40
41
|
const queryHelpers_1 = require("./queryHelpers");
|
|
41
42
|
const ledgerPresentation_2 = require("./ledgerPresentation");
|
|
43
|
+
const plainWording_1 = require("./plainWording");
|
|
44
|
+
const rankingTrace_1 = require("./rankingTrace");
|
|
42
45
|
function envFlagOn(name) {
|
|
43
46
|
const v = (process.env[name] ?? '').trim().toLowerCase();
|
|
44
47
|
return v === '1' || v === 'true' || v === 'yes' || v === 'on';
|
|
@@ -460,7 +463,7 @@ function resolveSessionDiversityCap(env = process.env, unsetCap = exports.SESSIO
|
|
|
460
463
|
function countsTowardNoteCap(h) {
|
|
461
464
|
return h.chunkId > 0 && (0, ledgerLiveness_1.ledgerEventIds)(h).length === 0 && (0, dailyNote_1.dailyNoteDayName)(h.notePath) === null;
|
|
462
465
|
}
|
|
463
|
-
function applySessionDiversity(hits, cap,
|
|
466
|
+
function applySessionDiversity(hits, cap, _topK) {
|
|
464
467
|
// `cap < 1` is the flag-OFF short-circuit and must stay: 0 restores the window exactly as it was
|
|
465
468
|
// before the default existed. The former `|| hits.length <= topK` here was a bypass, not an optimization — it
|
|
466
469
|
// skipped the configured cap entirely whenever the pool was no deeper than the window.
|
|
@@ -603,10 +606,38 @@ function resolveRankedExpansion(env = process.env) {
|
|
|
603
606
|
return {
|
|
604
607
|
notes: rawNotes === '' ? Number.POSITIVE_INFINITY : legacy.notes,
|
|
605
608
|
tokenBudget: rawBudget === '' ? exports.SESSION_EXPAND_RANKED_TOKENS : legacy.tokenBudget,
|
|
606
|
-
statementTokens: legacy.statementTokens,
|
|
609
|
+
statementTokens: rankedStatementTokens(env) ?? legacy.statementTokens,
|
|
607
610
|
documentStatementTokens: legacy.documentStatementTokens,
|
|
608
611
|
};
|
|
609
612
|
}
|
|
613
|
+
/**
|
|
614
|
+
* The longest conversation statement the RANKED expansion adds when nothing says otherwise: 400 tokens. The former expansion
|
|
615
|
+
* (a command line that does not declare a calibrated model) keeps RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120).
|
|
616
|
+
*/
|
|
617
|
+
exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = 400;
|
|
618
|
+
/**
|
|
619
|
+
* RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS (default 400 since the flip of 2026-10-25): the longest conversation statement,
|
|
620
|
+
* in tokens, that the RANKED expansion alone may add. Unset or empty: 400, unless RENOR_SESSION_EXPAND_STATEMENT_TOKENS is set,
|
|
621
|
+
* which an operator's explicit value keeps winning as it always did. A number of 1 or more: that limit, over the general one.
|
|
622
|
+
* The off switch: `0`, a negative number or anything that is not a number returns null, and the ranked expansion then takes
|
|
623
|
+
* RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120 unset), as it did before the flip. The former expansion and the documents are
|
|
624
|
+
* never touched by it.
|
|
625
|
+
*
|
|
626
|
+
* Why 400: the session expansion skips a conversation statement over its limit, whole, and the long user turn that answers a
|
|
627
|
+
* question is often just over 120 tokens (501 to 700 characters: missing from the page 5 times in 21, against 6 in 777 for
|
|
628
|
+
* shorter ones, on the Haiku default run of 2026-10-23). The chunker leaves no section over 400 tokens as one chunk, so 400 is
|
|
629
|
+
* the largest value that changes anything. On the official LongMemEval-S instrument with a Haiku 5.5 reader the ranked
|
|
630
|
+
* expansion with 400 scored 433 of 500 and with 120 (the control) 431, against 418, 417 and 423 for the command-line default;
|
|
631
|
+
* the four pages only the limit completes are right 4 of 4 with it and 0 of 4 without, and the page is 5.5% smaller:
|
|
632
|
+
* engine/docs/evals/results/ranked-statement-limit-20261024/ and LEDGER sections BE and BF.
|
|
633
|
+
*/
|
|
634
|
+
function rankedStatementTokens(env = process.env) {
|
|
635
|
+
const raw = (env.RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS ?? '').trim();
|
|
636
|
+
if (raw === '')
|
|
637
|
+
return (env.RENOR_SESSION_EXPAND_STATEMENT_TOKENS ?? '').trim() === '' ? exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS : null;
|
|
638
|
+
const value = Number(raw);
|
|
639
|
+
return Number.isFinite(value) && value >= 1 ? Math.floor(value) : null;
|
|
640
|
+
}
|
|
610
641
|
/**
|
|
611
642
|
* The calibrated key (CALIBRATED_BARS) of the embedding model behind a provider, or null when there is none.
|
|
612
643
|
*
|
|
@@ -758,9 +789,13 @@ async function retrieveContext(db, vectorIndex, provider, query, topK = 20, prio
|
|
|
758
789
|
contentfulHeld.clear(); // what another read worked out is not this read's
|
|
759
790
|
const factHits = (0, factsFirst_1.factsFirstHits)(db, query, Math.min(3, topK), opts.scope?.shows);
|
|
760
791
|
const queryProbe = opts.probe ?? {}; // the query vector, for the expansion's ranking
|
|
761
|
-
|
|
792
|
+
// RENOR_NEAR_DUP_CAP (default off): the pool is read deeper, so the places near-copies leave are taken (plainWording.ts).
|
|
793
|
+
const nearDupCap = (0, plainWording_1.resolveNearDupCap)();
|
|
794
|
+
const candidateLimit = nearDupCap > 0 ? Math.min(plainWording_1.NEAR_DUP_POOL.max, topK * plainWording_1.NEAR_DUP_POOL.perTopK) : Math.min(40, topK * 4);
|
|
795
|
+
// RENOR_QUERY_REWRITE (default off): what is embedded is the query without its conversational wrapper.
|
|
796
|
+
const embeddedQuery = (0, plainWording_1.queryRewriteEnabled)() ? (0, plainWording_1.rewriteConversationalQuery)(query) : query;
|
|
762
797
|
const semanticHits = vectorIndex && provider
|
|
763
|
-
? await (0, semantic_1.semanticSearch)(db, vectorIndex, provider,
|
|
798
|
+
? await (0, semantic_1.semanticSearch)(db, vectorIndex, provider, embeddedQuery, candidateLimit, queryProbe, opts.scope)
|
|
764
799
|
: (() => {
|
|
765
800
|
const match = (0, queryHelpers_1.quoteFtsQuery)(query);
|
|
766
801
|
if (!match)
|
|
@@ -772,13 +807,69 @@ async function retrieveContext(db, vectorIndex, provider, query, topK = 20, prio
|
|
|
772
807
|
return opts.scope ? live.filter(hit => opts.scope.shows(hit)) : live;
|
|
773
808
|
}, candidateLimit);
|
|
774
809
|
})();
|
|
810
|
+
// RENOR_ENTITY_LANE (default off): the chunks that carry the rare words the query names join the pool at their own cosine.
|
|
811
|
+
const laneTerms = (0, plainWording_1.entityLaneEnabled)() && vectorIndex && queryProbe.queryVector
|
|
812
|
+
? (0, readCache_1.inReadSnapshot)(db, () => entityLanePool(db, vectorIndex, queryProbe.queryVector, query, topK, semanticHits, opts))
|
|
813
|
+
: [];
|
|
775
814
|
// The window is ranked without waiting for anything, so it is read in one transaction (inReadSnapshot).
|
|
776
|
-
const window = (0, readCache_1.inReadSnapshot)(db, () => rankWindow({ db, query, topK, priorityPaths, clock, opts, factHits, semanticHits }));
|
|
815
|
+
const window = (0, readCache_1.inReadSnapshot)(db, () => rankWindow({ db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap }));
|
|
777
816
|
return finishContext({ db, vectorIndex, provider, query, queryProbe, window, bestFirst: opts.bestFirst === true });
|
|
778
817
|
}
|
|
818
|
+
/**
|
|
819
|
+
* The entity lane (RENOR_ENTITY_LANE): the lane words of the query (plainWording.ts entityLaneTerms) and, appended to
|
|
820
|
+
* `pool`, the chunks that carry them and are not in it yet, each at its own cosine to the query vector (one
|
|
821
|
+
* `VectorIndex.search` among their ids: no embedding, no model call). A scoped read judges them as it judges the pool.
|
|
822
|
+
* Returns the lane words, which the window lifts every hit that carries them by.
|
|
823
|
+
*/
|
|
824
|
+
function entityLanePool(db, vectorIndex, queryVector, query, topK, pool, opts) {
|
|
825
|
+
const terms = (0, plainWording_1.entityLaneTerms)(db, query);
|
|
826
|
+
if (terms.length === 0)
|
|
827
|
+
return [];
|
|
828
|
+
const inPool = new Set(pool.map(h => h.chunkId));
|
|
829
|
+
const ids = (0, plainWording_1.entityLaneChunkIds)(db, terms, topK, opts.isLedgerNote).filter(id => !inPool.has(id));
|
|
830
|
+
if (ids.length === 0)
|
|
831
|
+
return terms;
|
|
832
|
+
const cosine = new Map(vectorIndex.search(queryVector, ids.length, ids).map(h => [h.chunkId, h.score]));
|
|
833
|
+
const rows = (0, statements_1.prepareOnce)(db, 'SELECT id AS chunkId, note_path AS notePath, heading_path AS headingPath, text FROM chunks WHERE id IN (SELECT value FROM json_each(?))').all(JSON.stringify(ids));
|
|
834
|
+
for (const row of rows) {
|
|
835
|
+
if (!cosine.has(row.chunkId))
|
|
836
|
+
continue; // no vector yet: the semantic lane could not have found it either
|
|
837
|
+
if (opts.scope && !opts.scope.shows(row))
|
|
838
|
+
continue;
|
|
839
|
+
pool.push({ ...row, score: cosine.get(row.chunkId) });
|
|
840
|
+
}
|
|
841
|
+
return terms;
|
|
842
|
+
}
|
|
843
|
+
/** How many places of a window the entity lane keeps for the best hits that carry every lane word. */
|
|
844
|
+
exports.ENTITY_LANE_RESERVED = 3;
|
|
845
|
+
/**
|
|
846
|
+
* The first `topK` of `ranked`, but with its best ENTITY_LANE_RESERVED hits that carry every lane word in it: each one
|
|
847
|
+
* the cut would leave out takes the place of the last hit that carries none. A lift alone depends on the embedding's
|
|
848
|
+
* scale (a cosine of 0.8 against 0.1 on the lexical stub, a few hundredths on a real model); a place does not. Without
|
|
849
|
+
* lane words, the first `topK`.
|
|
850
|
+
*/
|
|
851
|
+
function reserveLanePlaces(ranked, topK, laneTerms) {
|
|
852
|
+
const window = ranked.slice(0, topK);
|
|
853
|
+
if (laneTerms.length === 0)
|
|
854
|
+
return window;
|
|
855
|
+
const carries = (h) => (0, plainWording_1.laneTermShare)(h.text, laneTerms) === 1;
|
|
856
|
+
const missing = ranked.filter(carries).slice(0, exports.ENTITY_LANE_RESERVED).filter(h => !window.includes(h));
|
|
857
|
+
for (const hit of missing) {
|
|
858
|
+
let at = window.length - 1;
|
|
859
|
+
while (at >= 0 && (0, plainWording_1.laneTermShare)(window[at].text, laneTerms) > 0)
|
|
860
|
+
at--;
|
|
861
|
+
if (at < 0)
|
|
862
|
+
break;
|
|
863
|
+
window.splice(at, 1);
|
|
864
|
+
window.push(hit);
|
|
865
|
+
}
|
|
866
|
+
return window;
|
|
867
|
+
}
|
|
779
868
|
/** The window of hits of a read: scored, with the graph of its notes, their facts and dates, live and capped per note. Waits for nothing. */
|
|
780
869
|
function rankWindow(read) {
|
|
781
|
-
const { db, query, topK, priorityPaths, clock, opts, factHits, semanticHits } = read;
|
|
870
|
+
const { db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap } = read;
|
|
871
|
+
// Set only while the learning log collects the ranker's terms (rankingTrace.ts); otherwise every use below is skipped.
|
|
872
|
+
const trace = (0, rankingTrace_1.currentRankingTrace)();
|
|
782
873
|
// Raw cosines, kept only for a caller that asked to see them (get_context's relevance signal).
|
|
783
874
|
const cosineByChunk = opts.probe ? new Map(semanticHits.map(h => [h.chunkId, h.score])) : null;
|
|
784
875
|
const entityHit = (0, entityLookup_1.lookupEntity)(db, query.trim());
|
|
@@ -800,8 +891,11 @@ function rankWindow(read) {
|
|
|
800
891
|
const scored = semanticHits.map(h => {
|
|
801
892
|
let score = h.score;
|
|
802
893
|
let reason = 'semantic';
|
|
894
|
+
const terms = trace ? (0, rankingTrace_1.newRankingTerms)() : null;
|
|
803
895
|
if (isSpeakerStub(h.text)) {
|
|
804
896
|
score -= 0.5;
|
|
897
|
+
if (terms)
|
|
898
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('stub')] -= 0.5;
|
|
805
899
|
reason = 'semantic+stub';
|
|
806
900
|
}
|
|
807
901
|
else {
|
|
@@ -809,15 +903,31 @@ function rankWindow(read) {
|
|
|
809
903
|
if (boost > 0) {
|
|
810
904
|
score += boost;
|
|
811
905
|
reason += '+content';
|
|
906
|
+
if (terms)
|
|
907
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('content')] += boost;
|
|
812
908
|
}
|
|
813
909
|
}
|
|
814
910
|
if (priorityPaths.some(p => h.notePath.includes('/' + p) || h.notePath.startsWith(p))) {
|
|
815
911
|
score += 0.1;
|
|
816
912
|
reason += '+priority';
|
|
913
|
+
if (terms)
|
|
914
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('priority')] += 0.1;
|
|
817
915
|
}
|
|
818
916
|
if (entityHit?.sourceNote === h.notePath) {
|
|
819
917
|
score += 0.2;
|
|
820
918
|
reason += '+entity';
|
|
919
|
+
if (terms)
|
|
920
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('entity')] += 0.2;
|
|
921
|
+
}
|
|
922
|
+
if (laneTerms.length > 0) {
|
|
923
|
+
const share = (0, plainWording_1.laneTermShare)(h.text, laneTerms);
|
|
924
|
+
if (share > 0) {
|
|
925
|
+
const boost = plainWording_1.ENTITY_LANE_BOOST * share;
|
|
926
|
+
score += boost;
|
|
927
|
+
reason += '+entity-lane';
|
|
928
|
+
if (terms)
|
|
929
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('entity_lane')] += boost;
|
|
930
|
+
}
|
|
821
931
|
}
|
|
822
932
|
// Time-decay recency: +0.1 for today, linearly decaying to 0 over 30 days.
|
|
823
933
|
// Skipped for explicit past-date (as-of) queries: recency answers "now",
|
|
@@ -830,9 +940,12 @@ function rankWindow(read) {
|
|
|
830
940
|
if (recencyBoost > 0.001) {
|
|
831
941
|
score += recencyBoost;
|
|
832
942
|
reason += '+recency';
|
|
943
|
+
if (terms)
|
|
944
|
+
terms[(0, rankingTrace_1.rankingTermIndex)('recency')] += recencyBoost;
|
|
833
945
|
}
|
|
834
946
|
}
|
|
835
|
-
|
|
947
|
+
const out = { ...h, score, reason, session_date };
|
|
948
|
+
return terms ? (0, rankingTrace_1.attachRankingTerms)(out, terms) : out;
|
|
836
949
|
});
|
|
837
950
|
// Graph expansion
|
|
838
951
|
const hitNotePaths = [...new Set(semanticHits.map(h => h.notePath))];
|
|
@@ -933,6 +1046,8 @@ function rankWindow(read) {
|
|
|
933
1046
|
(f.predicate ?? '').toLowerCase().includes(w)));
|
|
934
1047
|
if (factMatch) {
|
|
935
1048
|
hit.score += 0.15;
|
|
1049
|
+
if (trace)
|
|
1050
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'fact', 0.15);
|
|
936
1051
|
hit.reason += '+fact';
|
|
937
1052
|
}
|
|
938
1053
|
// As-of lane: the query names a past date, so hits carrying the same
|
|
@@ -943,11 +1058,15 @@ function rankWindow(read) {
|
|
|
943
1058
|
const lowerText = hit.text.toLowerCase();
|
|
944
1059
|
if (lowerText.includes(yearToken)) {
|
|
945
1060
|
hit.score += 0.08;
|
|
1061
|
+
if (trace)
|
|
1062
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'asof_year', 0.08);
|
|
946
1063
|
hit.reason += '+asof-year';
|
|
947
1064
|
if (asofRef.month !== undefined) {
|
|
948
1065
|
const monthName = Object.keys(ASOF_MONTHS).find(m => ASOF_MONTHS[m] === asofRef.month);
|
|
949
1066
|
if (monthName && lowerText.includes(monthName)) {
|
|
950
1067
|
hit.score += 0.04;
|
|
1068
|
+
if (trace)
|
|
1069
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'asof_month', 0.04);
|
|
951
1070
|
hit.reason += '+asof-month';
|
|
952
1071
|
}
|
|
953
1072
|
}
|
|
@@ -970,6 +1089,8 @@ function rankWindow(read) {
|
|
|
970
1089
|
// Must clear recency (+0.1 max) + fact boost (+0.15) the current note
|
|
971
1090
|
// typically carries, or past-intent queries can never win.
|
|
972
1091
|
hit.score += 0.3;
|
|
1092
|
+
if (trace)
|
|
1093
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'historical_intent', 0.3);
|
|
973
1094
|
hit.reason += '+historical-intent';
|
|
974
1095
|
}
|
|
975
1096
|
}
|
|
@@ -982,10 +1103,14 @@ function rankWindow(read) {
|
|
|
982
1103
|
if (stale.length >= 3 && text.includes(stale)) {
|
|
983
1104
|
if (pastIntent) {
|
|
984
1105
|
hit.score += 0.1;
|
|
1106
|
+
if (trace)
|
|
1107
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'past_fact', 0.1);
|
|
985
1108
|
hit.reason += '+historical-intent';
|
|
986
1109
|
}
|
|
987
1110
|
else {
|
|
988
1111
|
hit.score -= 0.15;
|
|
1112
|
+
if (trace)
|
|
1113
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'stale_fact', -0.15);
|
|
989
1114
|
hit.reason += '+stale-fact';
|
|
990
1115
|
}
|
|
991
1116
|
break; // one demotion per hit; stale signal already applied
|
|
@@ -1002,6 +1127,8 @@ function rankWindow(read) {
|
|
|
1002
1127
|
const cur = (f.object_text ?? '').trim().toLowerCase();
|
|
1003
1128
|
if (cur.length >= 3 && text.includes(cur)) {
|
|
1004
1129
|
hit.score += 0.1;
|
|
1130
|
+
if (trace)
|
|
1131
|
+
(0, rankingTrace_1.addRankingTerm)(hit, 'current_fact', 0.1);
|
|
1005
1132
|
hit.reason += '+current-fact';
|
|
1006
1133
|
break;
|
|
1007
1134
|
}
|
|
@@ -1010,6 +1137,8 @@ function rankWindow(read) {
|
|
|
1010
1137
|
}
|
|
1011
1138
|
}
|
|
1012
1139
|
const ranked = dedupeHits(scored.sort(semantic_1.byScoreThenPlace));
|
|
1140
|
+
if (trace)
|
|
1141
|
+
ranked.forEach((hit, index) => trace.pool(hit, index + 1));
|
|
1013
1142
|
// Fail-open floor. The stub filter must never empty the window: a zero-hit
|
|
1014
1143
|
// retrieval is a hard abort on the product path, because the benchmark client
|
|
1015
1144
|
// refuses to score a case with no hits ("Retrieval precondition failed: public
|
|
@@ -1027,7 +1156,13 @@ function rankWindow(read) {
|
|
|
1027
1156
|
// The ledger's own header is no memory: it comes off with the closed events, before the page is cut.
|
|
1028
1157
|
const mixed = (0, ledgerPresentation_1.dropLedgerPreamble)((0, ledgerLiveness_1.resolveLedgerHits)([...datedFactHits, ...contentful.filter(h => !seen.has(h.text))], (0, ledgerLiveness_1.closedLedgerEvents)(db), opts.includeHistory === true), opts.isLedgerNote);
|
|
1029
1158
|
const diversityCap = resolveSessionDiversityCap(process.env, opts.noteCapWhenUnset);
|
|
1030
|
-
|
|
1159
|
+
// RENOR_NEAR_DUP_CAP (default off): near-copies are capped before the per-note cap, on the visible text.
|
|
1160
|
+
const windowed = reserveLanePlaces(applySessionDiversity((0, plainWording_1.applyNearDupCap)(mixed, nearDupCap, contentfulText), diversityCap, topK), topK, laneTerms);
|
|
1161
|
+
// The window's candidates are recorded before they are copied into the plain hits below, which carry no terms.
|
|
1162
|
+
if (trace)
|
|
1163
|
+
for (const hit of windowed)
|
|
1164
|
+
trace.touch(hit);
|
|
1165
|
+
let hits = windowed.map(h => ({
|
|
1031
1166
|
notePath: h.notePath,
|
|
1032
1167
|
chunkId: h.chunkId,
|
|
1033
1168
|
headingPath: h.headingPath,
|
|
@@ -1082,5 +1217,8 @@ async function finishContext(read) {
|
|
|
1082
1217
|
const newestFirst = !bestFirst && newestFirstEnabled() && !(temporal && pastIntent) && !asofPast;
|
|
1083
1218
|
const rankedBy = newestFirst ? 'session_date' : 'score';
|
|
1084
1219
|
contentfulHeld.clear(); // nothing outlives the read it was worked out for
|
|
1085
|
-
|
|
1220
|
+
// The order the reply presents its hits in, so a candidate's place in it is recorded for the learning log (rankingTrace.ts).
|
|
1221
|
+
const presented = orderHits(hits, rankedBy);
|
|
1222
|
+
(0, rankingTrace_1.currentRankingTrace)()?.presented(presented);
|
|
1223
|
+
return { query, hits: presented, rankedBy };
|
|
1086
1224
|
}
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NEAR_DUP_POOL = exports.NEAR_DUP_JACCARD = exports.ENTITY_LANE_DF_SHARE = exports.ENTITY_LANE_MIN_DF_LIMIT = exports.ENTITY_LANE_MAX_QUERY_TERMS = exports.ENTITY_LANE_MAX_TERMS = exports.ENTITY_LANE_BOOST = void 0;
|
|
4
|
+
exports.entityLaneEnabled = entityLaneEnabled;
|
|
5
|
+
exports.queryRewriteEnabled = queryRewriteEnabled;
|
|
6
|
+
exports.resolveNearDupCap = resolveNearDupCap;
|
|
7
|
+
exports.contentTerms = contentTerms;
|
|
8
|
+
exports.rewriteConversationalQuery = rewriteConversationalQuery;
|
|
9
|
+
exports.entityLaneTerms = entityLaneTerms;
|
|
10
|
+
exports.entityLaneChunkIds = entityLaneChunkIds;
|
|
11
|
+
exports.laneTermShare = laneTermShare;
|
|
12
|
+
exports.jaccard = jaccard;
|
|
13
|
+
exports.applyNearDupCap = applyNearDupCap;
|
|
14
|
+
const statements_1 = require("../db/statements");
|
|
15
|
+
const keyword_1 = require("./keyword");
|
|
16
|
+
const ledgerPresentation_1 = require("./ledgerPresentation");
|
|
17
|
+
const queryHelpers_1 = require("./queryHelpers");
|
|
18
|
+
/**
|
|
19
|
+
* Plain-wording retrieval and crowding: three switches, each default OFF, each a kill switch of its own.
|
|
20
|
+
*
|
|
21
|
+
* Why (the owner's real use, 2026-10-21): where Renor held the memory, the owner's own words as the query returned it
|
|
22
|
+
* 5 of 24 times against 19 of 24 for a hand-aimed topic query, and of 183 agent queries that once returned a memory the
|
|
23
|
+
* agent acted on, 40% no longer return it as the workspace grows. None of the three calls a model.
|
|
24
|
+
*
|
|
25
|
+
* - RENOR_ENTITY_LANE: the rare words a query names (a subject such as "jev", a service, a project) are looked up in
|
|
26
|
+
* the keyword index; the chunks that carry them join the semantic candidates at their own cosine, and every candidate
|
|
27
|
+
* that carries them is lifted by up to ENTITY_LANE_BOOST (the lift `+entity` already gives an entity's note).
|
|
28
|
+
* - RENOR_QUERY_REWRITE: the conversational wrapper of a query ("hey, do you remember what we said about ...") is taken
|
|
29
|
+
* off before it is embedded, so the words that are embedded are the ones stored memories use.
|
|
30
|
+
* - RENOR_NEAR_DUP_CAP=<n>: at most `n` near-copies (word-set Jaccard >= NEAR_DUP_JACCARD) in one window, the best
|
|
31
|
+
* scored kept; with it the candidate pool is read deeper (NEAR_DUP_POOL), so the distinct memories that newer
|
|
32
|
+
* near-copies pushed below the old pool depth can take the freed places.
|
|
33
|
+
*/
|
|
34
|
+
function on(value) {
|
|
35
|
+
const v = (value ?? '').trim().toLowerCase();
|
|
36
|
+
return v === '1' || v === 'true' || v === 'yes' || v === 'on';
|
|
37
|
+
}
|
|
38
|
+
function entityLaneEnabled(env = process.env) {
|
|
39
|
+
return on(env.RENOR_ENTITY_LANE);
|
|
40
|
+
}
|
|
41
|
+
function queryRewriteEnabled(env = process.env) {
|
|
42
|
+
return on(env.RENOR_QUERY_REWRITE);
|
|
43
|
+
}
|
|
44
|
+
/** The near-copy cap, or 0 (off) when unset, empty, junk or below 1. */
|
|
45
|
+
function resolveNearDupCap(env = process.env) {
|
|
46
|
+
const raw = (env.RENOR_NEAR_DUP_CAP ?? '').trim();
|
|
47
|
+
if (raw === '')
|
|
48
|
+
return 0;
|
|
49
|
+
const cap = Number(raw);
|
|
50
|
+
return Number.isFinite(cap) && cap >= 1 ? Math.floor(cap) : 0;
|
|
51
|
+
}
|
|
52
|
+
/** Words that never name what a memory is about: function words and the words of asking and remembering. */
|
|
53
|
+
const PLAIN_STOP = new Set([
|
|
54
|
+
'the', 'and', 'for', 'are', 'was', 'were', 'what', 'which', 'who', 'whom', 'whose', 'where', 'when', 'why', 'how',
|
|
55
|
+
'does', 'did', 'done', 'with', 'about', 'this', 'that', 'these', 'those', 'there', 'here', 'then', 'than', 'from',
|
|
56
|
+
'into', 'onto', 'over', 'your', 'yours', 'our', 'ours', 'their', 'theirs', 'his', 'her', 'hers', 'its', 'you', 'they',
|
|
57
|
+
'them', 'she', 'him', 'can', 'could', 'would', 'should', 'will', 'shall', 'might', 'must', 'may', 'have', 'has', 'had',
|
|
58
|
+
'been', 'being', 'not', 'any', 'all', 'some', 'just', 'also', 'very', 'really', 'again', 'still', 'now', 'ever',
|
|
59
|
+
'remember', 'remind', 'recall', 'tell', 'told', 'say', 'said', 'know', 'think', 'thought', 'mean', 'meant', 'please',
|
|
60
|
+
'thanks', 'hey', 'okay', 'yeah', 'yes', 'like', 'thing', 'things', 'stuff', 'something', 'anything', 'everything',
|
|
61
|
+
'one', 'ones', 'get', 'got', 'let', 'lets', 'need', 'want', 'look', 'check', 'find', 'give', 'show', 'saved', 'save',
|
|
62
|
+
'memory', 'memories', 'renor', 'claude', 'myself', 'yourself', 'we', 'us', 'me', 'my', 'it', 'is', 'in', 'on', 'of',
|
|
63
|
+
'to', 'do', 'be', 'or', 'an', 'at', 'as', 'by', 'if', 'so', 'up', 'am', 'last', 'time', 'before', 'earlier', 'after',
|
|
64
|
+
'since', 'until', 'during', 'while', 'because', 'between', 'other', 'another', 'first', 'next', 'every', 'each', 'many',
|
|
65
|
+
'much', 'more', 'most', 'less', 'same', 'such', 'only', 'well', 'today', 'yesterday', 'ago', 'way', 'kind', 'sort',
|
|
66
|
+
]);
|
|
67
|
+
/** The words of `query` that could name its subject: three characters or more (a hyphenated name is one word), no stop word. */
|
|
68
|
+
function contentTerms(query) {
|
|
69
|
+
const words = (query ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}][\p{L}\p{N}\p{M}_-]*/gu) ?? [];
|
|
70
|
+
const out = [];
|
|
71
|
+
for (const raw of words) {
|
|
72
|
+
const word = raw.replace(/[-_]+$/, '').replace(/'s$/, '');
|
|
73
|
+
if ([...word].length < 3 || PLAIN_STOP.has(word) || out.includes(word))
|
|
74
|
+
continue;
|
|
75
|
+
out.push(word);
|
|
76
|
+
}
|
|
77
|
+
return out;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Wrappers a person puts around a question when they talk to an agent. Taken off the front of the query, repeatedly,
|
|
81
|
+
* and only there: inside a query the same words can be what it is about.
|
|
82
|
+
*/
|
|
83
|
+
const LEADING_WRAPPERS = [
|
|
84
|
+
/^(?:hey|hi|hello|ok|okay|so|um|uh|well|right|and|but|also|btw|by the way|quick question|question)\b[\s,:;.!-]*/i,
|
|
85
|
+
/^(?:claude|renor|codex|cursor)\b[\s,:;.!-]*/i,
|
|
86
|
+
/^(?:please|pls|can you|could you|would you|will you|do you|did you|are you able to)\b\s*/i,
|
|
87
|
+
/^(?:(?:still )?remember|recall|remind me(?: of)?|tell me|look up|find|check|pull up|bring up|show me)\b\s*/i,
|
|
88
|
+
/^(?:what (?:(?:did|do|have) )?(?:i|we|you) (?:say|said|tell you|told you|decide|decided|note|noted|save|saved|write|wrote|agree|agreed|settle|settled)(?: on)?)\b\s*/i,
|
|
89
|
+
/^(?:what (?:was|were|is|are) (?:it|that|the thing|those|these))\b\s*/i,
|
|
90
|
+
/^(?:the (?:thing|stuff|notes?)|our (?:notes?|discussion|conversation))\b\s*/i,
|
|
91
|
+
/^(?:about|on|regarding|re|for|of|with|that)\b[\s,:]*/i,
|
|
92
|
+
];
|
|
93
|
+
/**
|
|
94
|
+
* The query without its conversational wrapper ("hey, do you remember what we said about the jev measurements?" ->
|
|
95
|
+
* "the jev measurements"). The question's own words are kept as written; a trailing "again", "?" or "please" goes too.
|
|
96
|
+
* When the rewrite would leave fewer than two content words, or nothing changed, the query is returned as it came.
|
|
97
|
+
*/
|
|
98
|
+
function rewriteConversationalQuery(query) {
|
|
99
|
+
const original = (query ?? '').trim();
|
|
100
|
+
let q = original;
|
|
101
|
+
for (let guard = 0; guard < 12; guard++) {
|
|
102
|
+
const before = q;
|
|
103
|
+
for (const wrapper of LEADING_WRAPPERS)
|
|
104
|
+
q = q.replace(wrapper, '');
|
|
105
|
+
q = q.trim();
|
|
106
|
+
if (q === before)
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
q = q.replace(/[\s,]*(?:again|please|for me|real quick|if you can|do you remember|remember)?[\s?.!]*$/i, '').trim();
|
|
110
|
+
const kept = contentTerms(q).length;
|
|
111
|
+
if (q === original.replace(/[\s?.!]+$/, '') || kept === 0 || kept < Math.min(2, contentTerms(original).length))
|
|
112
|
+
return original;
|
|
113
|
+
return q;
|
|
114
|
+
}
|
|
115
|
+
/** The lift a candidate that carries every lane word gets: the lift `+entity` gives an entity's note (orchestrator.ts). */
|
|
116
|
+
exports.ENTITY_LANE_BOOST = 0.2;
|
|
117
|
+
/** At most this many lane words per query: the rarest. */
|
|
118
|
+
exports.ENTITY_LANE_MAX_TERMS = 3;
|
|
119
|
+
/** The lane reads queries of at most this many content words. */
|
|
120
|
+
exports.ENTITY_LANE_MAX_QUERY_TERMS = 12;
|
|
121
|
+
/** A word is rare enough to name a subject when at most this many chunks carry it, or ENTITY_LANE_DF_SHARE of all chunks. */
|
|
122
|
+
exports.ENTITY_LANE_MIN_DF_LIMIT = 5;
|
|
123
|
+
exports.ENTITY_LANE_DF_SHARE = 0.02;
|
|
124
|
+
/**
|
|
125
|
+
* The lane words of `query`: its content words that name a registered entity, or that few enough chunks carry to name a
|
|
126
|
+
* subject, rarest first (a named entity before any other word). Two bounded lookups per word; none for a query of more than
|
|
127
|
+
* ENTITY_LANE_MAX_QUERY_TERMS content words.
|
|
128
|
+
*/
|
|
129
|
+
function entityLaneTerms(db, query) {
|
|
130
|
+
const terms = contentTerms(query);
|
|
131
|
+
// A query of many words is a passage, not a named subject: its rarest word is as likely a heading's (measured: a pasted
|
|
132
|
+
// section lost its own chunk to a "Parking lot" section of another note). The semantic lane already matches a passage.
|
|
133
|
+
if (terms.length === 0 || terms.length > exports.ENTITY_LANE_MAX_QUERY_TERMS)
|
|
134
|
+
return [];
|
|
135
|
+
const chunks = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM chunks').get().n;
|
|
136
|
+
const limit = Math.max(exports.ENTITY_LANE_MIN_DF_LIMIT, Math.floor(chunks * exports.ENTITY_LANE_DF_SHARE));
|
|
137
|
+
const counted = [];
|
|
138
|
+
for (const term of terms) {
|
|
139
|
+
const match = (0, queryHelpers_1.quoteFtsQuery)(term);
|
|
140
|
+
if (!match)
|
|
141
|
+
continue;
|
|
142
|
+
let df;
|
|
143
|
+
try {
|
|
144
|
+
df = (0, statements_1.prepareOnce)(db, 'SELECT COUNT(*) AS n FROM (SELECT 1 FROM chunks_fts WHERE chunks_fts MATCH ? LIMIT ?)').get(match, limit + 1).n;
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
continue; // a word FTS5 cannot read is no lane word
|
|
148
|
+
}
|
|
149
|
+
if (df === 0)
|
|
150
|
+
continue;
|
|
151
|
+
const named = (0, statements_1.prepareOnce)(db, `SELECT 1 FROM entities WHERE canonical_name = ? COLLATE NOCASE
|
|
152
|
+
UNION ALL SELECT 1 FROM entities, json_each(entities.aliases_json) AS alias WHERE lower(alias.value) = ? LIMIT 1`).get(term, term) !== undefined;
|
|
153
|
+
if (named || df <= limit)
|
|
154
|
+
counted.push({ term, df, named });
|
|
155
|
+
}
|
|
156
|
+
return counted
|
|
157
|
+
.sort((a, b) => Number(b.named) - Number(a.named) || a.df - b.df)
|
|
158
|
+
.slice(0, exports.ENTITY_LANE_MAX_TERMS)
|
|
159
|
+
.map(c => c.term);
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* The chunks that carry any lane word, best keyword match first, at most `perTerm` for each word. Read as every keyword
|
|
163
|
+
* reader reads (shownPage): the ledger's header and the events the word is not in are judged away before the page is cut.
|
|
164
|
+
*/
|
|
165
|
+
function entityLaneChunkIds(db, terms, perTerm, isLedgerNote) {
|
|
166
|
+
const ids = [];
|
|
167
|
+
for (const term of terms) {
|
|
168
|
+
const match = (0, queryHelpers_1.quoteFtsQuery)(term);
|
|
169
|
+
if (!match)
|
|
170
|
+
continue;
|
|
171
|
+
const page = (0, queryHelpers_1.shownPage)(rows => (0, keyword_1.keywordSearch)(db, match, rows), rows => (0, ledgerPresentation_1.keywordLedgerHits)(rows, isLedgerNote, term), perTerm);
|
|
172
|
+
for (const hit of page)
|
|
173
|
+
if (!ids.includes(hit.chunkId))
|
|
174
|
+
ids.push(hit.chunkId);
|
|
175
|
+
}
|
|
176
|
+
return ids;
|
|
177
|
+
}
|
|
178
|
+
function escapeRegExp(s) {
|
|
179
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
180
|
+
}
|
|
181
|
+
/** The share of `terms` that `text` carries as whole words (a hyphen or underscore in a lane word matches any separator). */
|
|
182
|
+
function laneTermShare(text, terms) {
|
|
183
|
+
if (terms.length === 0)
|
|
184
|
+
return 0;
|
|
185
|
+
const lower = (text ?? '').toLowerCase().normalize('NFC');
|
|
186
|
+
let carried = 0;
|
|
187
|
+
for (const term of terms) {
|
|
188
|
+
const pattern = term.split(/[-_]+/).map(escapeRegExp).join('[\\s_-]+');
|
|
189
|
+
if (new RegExp(`(?<![\\p{L}\\p{N}])${pattern}(?![\\p{L}\\p{N}])`, 'u').test(lower))
|
|
190
|
+
carried++;
|
|
191
|
+
}
|
|
192
|
+
return carried / terms.length;
|
|
193
|
+
}
|
|
194
|
+
/** Two texts whose word sets share at least this share of their union are near-copies. */
|
|
195
|
+
exports.NEAR_DUP_JACCARD = 0.6;
|
|
196
|
+
/** How deep the candidate pool is read while the near-copy cap is on (the default reads min(40, topK * 4)). */
|
|
197
|
+
exports.NEAR_DUP_POOL = { perTopK: 8, max: 160 };
|
|
198
|
+
function wordSet(text) {
|
|
199
|
+
return new Set((text ?? '').toLowerCase().normalize('NFC').match(/[\p{L}\p{N}]+/gu) ?? []);
|
|
200
|
+
}
|
|
201
|
+
function jaccard(a, b) {
|
|
202
|
+
if (a.size === 0 && b.size === 0)
|
|
203
|
+
return 1;
|
|
204
|
+
let shared = 0;
|
|
205
|
+
for (const word of a)
|
|
206
|
+
if (b.has(word))
|
|
207
|
+
shared++;
|
|
208
|
+
return shared / (a.size + b.size - shared);
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* At most `cap` near-copies in the window: in rank order, a hit is kept while fewer than `cap` kept hits are near-copies
|
|
212
|
+
* of it. A fact row (a negative chunkId) is left alone. `cap < 1` returns the hits as they came. `textOf` is the text the
|
|
213
|
+
* copies are compared on (the orchestrator passes the visible text, without markdown and speaker labels).
|
|
214
|
+
*/
|
|
215
|
+
function applyNearDupCap(hits, cap, textOf = t => t) {
|
|
216
|
+
if (cap < 1)
|
|
217
|
+
return hits;
|
|
218
|
+
const kept = [];
|
|
219
|
+
const keptWords = [];
|
|
220
|
+
for (const hit of hits) {
|
|
221
|
+
if (hit.chunkId < 0) {
|
|
222
|
+
kept.push(hit);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
const words = wordSet(textOf(hit.text));
|
|
226
|
+
let copies = 0;
|
|
227
|
+
for (const other of keptWords)
|
|
228
|
+
if (jaccard(words, other) >= exports.NEAR_DUP_JACCARD && ++copies >= cap)
|
|
229
|
+
break;
|
|
230
|
+
if (copies >= cap)
|
|
231
|
+
continue;
|
|
232
|
+
kept.push(hit);
|
|
233
|
+
keptWords.push(words);
|
|
234
|
+
}
|
|
235
|
+
return kept;
|
|
236
|
+
}
|