pi-braid 0.43.8 → 0.43.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/build-info.json +4 -4
- package/dist/lib/config/braid-settings.d.ts +22 -0
- package/dist/lib/config/braid-settings.js +26 -0
- package/dist/lib/memory/constants.d.ts +38 -0
- package/dist/lib/memory/constants.js +196 -0
- package/dist/lib/memory/index.js +49 -10
- package/dist/lib/memory/lifecycle.d.ts +8 -0
- package/dist/lib/memory/lifecycle.js +43 -18
- package/docs/LOCAL_DEV.md +1 -1
- package/docs/SETUP.md +1 -0
- package/package.json +1 -1
- package/src/lib/config/braid-settings.ts +51 -0
- package/src/lib/memory/constants.ts +202 -0
- package/src/lib/memory/index.ts +59 -8
- package/src/lib/memory/lifecycle.ts +50 -16
package/CHANGELOG.md
CHANGED
|
@@ -53,6 +53,39 @@ record; the tag scheme above applies unchanged.
|
|
|
53
53
|
|
|
54
54
|
---
|
|
55
55
|
|
|
56
|
+
## Core — 0.43.9 (2026-10-31: a query is read as tokens, and the store can be told what to keep)
|
|
57
|
+
|
|
58
|
+
- **A question could not match a word it contained.** Recall split a query on whitespace alone, so a
|
|
59
|
+
question ending in `APIs?` carried the token `apis?` and never matched the stored word `apis`: the
|
|
60
|
+
content was invisible to the matcher, and no amount of weighting could reach it, because a token the
|
|
61
|
+
matcher cannot see cannot be weighted. Measured on a 260-entry store, that single defect kept a correct
|
|
62
|
+
answer out of a five-item result list entirely; it now returns at rank one. The query is tokenized on
|
|
63
|
+
non-alphanumeric characters as well as whitespace, and the test plants the revert: restoring the old
|
|
64
|
+
splitter reddens exactly the punctuation case and nothing else.
|
|
65
|
+
- **Function words stop counting as evidence, and rare words start.** Every query word longer than two
|
|
66
|
+
characters scored the same, so `which`, `should` and `when` each counted as a hit and a long entry
|
|
67
|
+
outranked a short precise one. A generic English stop list and `idf(w) = ln(1 + N/df)` now weight the
|
|
68
|
+
terms, and the effect was measured against a control of five ordinary entries chosen by margin rather
|
|
69
|
+
than by taste: all five hold rank one, and the leader-to-runner-up margins widen from 1.10–1.92× to
|
|
70
|
+
2.06–3.55×. The honest attribution is in the report: the stop list and the weights alone moved no rank,
|
|
71
|
+
because tokenization was the defect; they sharpen what tokenization made reachable.
|
|
72
|
+
- **What the store keeps can be configured, and by default nothing changes.** `pruneMemories` ordered
|
|
73
|
+
every entry by utility alone, so a backfilled provenance entry scored 0.53 against a floor of 0.605 and
|
|
74
|
+
a prune one hour after the write kept four of sixty. A reserved partition — entries whose `source`
|
|
75
|
+
starts with a configured prefix may occupy up to a configured number of slots — ships **off**, and with
|
|
76
|
+
nothing configured the prune returns an element-identical id sequence to the previous build at one
|
|
77
|
+
minute, one hour and one day after the write. Configured at twenty slots it keeps twenty of sixty, and
|
|
78
|
+
the cost is stated rather than hidden: twenty ordinary entries leave the store, the reserve is a hard
|
|
79
|
+
cap rather than a home for all sixty, and which twenty survive among utility ties is the shipped
|
|
80
|
+
comparator's order, not anything about the entry.
|
|
81
|
+
- **The retrieval goal it was aimed at is one third reached, and the reason is not lexical.** Ten questions
|
|
82
|
+
were pre-registered; a presence scan then showed only three of them name an answer that both exists in
|
|
83
|
+
the corpus and survives the classifier, so the ceiling for any reordering was three. One is now at rank
|
|
84
|
+
one. The second cannot be reached by any scoring change: the winner and the correct ruling match exactly
|
|
85
|
+
the same two query terms, and the 1.5× gap is utility (0.815 against 0.670) plus a project boost the
|
|
86
|
+
ruling cannot earn plus an access boost its rival has. That is a policy question about provenance
|
|
87
|
+
entries, not a ranking question, and it is named here so the next change starts from the measurement.
|
|
88
|
+
|
|
56
89
|
## Core — 0.43.8 (2026-10-31: the harness stops accusing work that happened, and starts answering from the archive)
|
|
57
90
|
|
|
58
91
|
- **An unterminated page cannot delete the evidence tail any more.** A redaction of a detected HTML page
|
package/build-info.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-braid",
|
|
3
|
-
"version": "0.43.
|
|
4
|
-
"sha": "
|
|
5
|
-
"shaShort": "
|
|
3
|
+
"version": "0.43.9",
|
|
4
|
+
"sha": "bce25581c535c3a318787d6c3a12b6beb76ff7ca",
|
|
5
|
+
"shaShort": "bce25581",
|
|
6
6
|
"branch": "detached",
|
|
7
7
|
"dirty": false,
|
|
8
|
-
"packedAt": "2026-10-
|
|
8
|
+
"packedAt": "2026-10-31T15:50:23.130Z"
|
|
9
9
|
}
|
|
@@ -140,6 +140,28 @@ interface RawWriteGateBlock {
|
|
|
140
140
|
* is unit-testable without touching live settings.json.
|
|
141
141
|
*/
|
|
142
142
|
export declare function resolveMemoryWriteGateConfig(raw?: RawWriteGateBlock): MemoryWriteGateConfig;
|
|
143
|
+
/** A reserved partition: which source prefix is protected, and for how many slots. */
|
|
144
|
+
export interface ReservedProvenanceConfig {
|
|
145
|
+
prefix: string;
|
|
146
|
+
slots: number;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Pure resolver for the braid.memory.reservedProvenance block. Absent or
|
|
150
|
+
* malformed fields fall back to RESERVED_PROVENANCE_DEFAULTS (partition off),
|
|
151
|
+
* so a typo cannot start holding slots for a class nobody named. `slots` is
|
|
152
|
+
* clamped to MAX_ENTRIES: a reservation at or above the cap would leave the
|
|
153
|
+
* general pool no room at all. Kept pure so the merge is testable without
|
|
154
|
+
* touching live settings.json.
|
|
155
|
+
*/
|
|
156
|
+
export declare function resolveReservedProvenance(raw?: {
|
|
157
|
+
prefix?: unknown;
|
|
158
|
+
slots?: unknown;
|
|
159
|
+
}): ReservedProvenanceConfig;
|
|
160
|
+
/**
|
|
161
|
+
* Read braid.memory.reservedProvenance from settings.json, fresh on every call
|
|
162
|
+
* (no caching), like the other readers. Default: no partition.
|
|
163
|
+
*/
|
|
164
|
+
export declare function getMemoryReservedProvenance(): ReservedProvenanceConfig;
|
|
143
165
|
/**
|
|
144
166
|
* The owner time zone for all braid-facing time rendering (lib/utils/local-time.ts).
|
|
145
167
|
* braid.timezone in settings.json when set to a non-empty string; otherwise
|
|
@@ -9,6 +9,7 @@ import { existsSync, readFileSync } from "node:fs";
|
|
|
9
9
|
import { createLogger } from "../core/logger.js";
|
|
10
10
|
import { USAGE_COMPACT_MEDIUM, USAGE_WHISPER_CRITICAL, USAGE_WHISPER_WARNING, } from "../detection/context-thresholds.js";
|
|
11
11
|
import { MEMORY_WRITE_GATE_DEFAULTS, WRITE_GATE_SITES, } from "../memory/write-quality.js";
|
|
12
|
+
import { MAX_ENTRIES, RESERVED_PROVENANCE_DEFAULTS } from "../memory/constants.js";
|
|
12
13
|
import { getSettingsPath } from "./per-tool-settings.js";
|
|
13
14
|
const _log = createLogger("braid-settings");
|
|
14
15
|
/** Read and parse settings.json. Not cached - always reads fresh to respect runtime changes. */
|
|
@@ -185,6 +186,31 @@ function normalizeWriteGateSites(raw) {
|
|
|
185
186
|
}
|
|
186
187
|
return sites;
|
|
187
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* Pure resolver for the braid.memory.reservedProvenance block. Absent or
|
|
191
|
+
* malformed fields fall back to RESERVED_PROVENANCE_DEFAULTS (partition off),
|
|
192
|
+
* so a typo cannot start holding slots for a class nobody named. `slots` is
|
|
193
|
+
* clamped to MAX_ENTRIES: a reservation at or above the cap would leave the
|
|
194
|
+
* general pool no room at all. Kept pure so the merge is testable without
|
|
195
|
+
* touching live settings.json.
|
|
196
|
+
*/
|
|
197
|
+
export function resolveReservedProvenance(raw) {
|
|
198
|
+
const prefixValue = typeof raw?.prefix === "string" ? raw.prefix.trim() : "";
|
|
199
|
+
const prefix = prefixValue || RESERVED_PROVENANCE_DEFAULTS.prefix;
|
|
200
|
+
const slotsValue = raw?.slots;
|
|
201
|
+
const slots = typeof slotsValue === "number" && Number.isInteger(slotsValue) && slotsValue >= 0
|
|
202
|
+
? Math.min(slotsValue, MAX_ENTRIES)
|
|
203
|
+
: RESERVED_PROVENANCE_DEFAULTS.slots;
|
|
204
|
+
return { prefix, slots };
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Read braid.memory.reservedProvenance from settings.json, fresh on every call
|
|
208
|
+
* (no caching), like the other readers. Default: no partition.
|
|
209
|
+
*/
|
|
210
|
+
export function getMemoryReservedProvenance() {
|
|
211
|
+
const settings = readSettings();
|
|
212
|
+
return resolveReservedProvenance(settings.braid?.memory?.reservedProvenance);
|
|
213
|
+
}
|
|
188
214
|
/**
|
|
189
215
|
* The owner time zone for all braid-facing time rendering (lib/utils/local-time.ts).
|
|
190
216
|
* braid.timezone in settings.json when set to a non-empty string; otherwise
|
|
@@ -7,5 +7,43 @@ export declare const MEMORY_FILE: string;
|
|
|
7
7
|
export declare const MAX_ENTRIES = 200;
|
|
8
8
|
export declare const MIN_UTILITY_THRESHOLD = 0.15;
|
|
9
9
|
export declare const CONSOLIDATION_INTERVAL = 5;
|
|
10
|
+
/**
|
|
11
|
+
* Shortest token the matcher looks at. One- and two-character tokens carry no
|
|
12
|
+
* topic in any language this store sees, and matching them turns a query into
|
|
13
|
+
* a scan of nearly every entry.
|
|
14
|
+
*/
|
|
15
|
+
export declare const QUERY_MIN_TOKEN_LENGTH = 3;
|
|
16
|
+
/**
|
|
17
|
+
* English function words, dropped from a query before matching. They name no
|
|
18
|
+
* topic, and counting each as a full word match is what let a long entry
|
|
19
|
+
* outrank a short quote that shared the question's actual subject.
|
|
20
|
+
*
|
|
21
|
+
* Generic vocabulary only. A word that names anything about this deployment,
|
|
22
|
+
* its files, or its roles belongs to that owner, not in a shared matcher.
|
|
23
|
+
*/
|
|
24
|
+
export declare const QUERY_STOP_WORDS: ReadonlySet<string>;
|
|
25
|
+
/**
|
|
26
|
+
* What a query-named tag is worth, in matched-word units: two ordinary terms.
|
|
27
|
+
* The shipped matcher counted a word as 1 and a tag as 2, so 2 keeps the
|
|
28
|
+
* shipped meaning now that a word's weight is a rarity weight instead.
|
|
29
|
+
*/
|
|
30
|
+
export declare const MEMORY_TAG_BONUS_WORDS = 2;
|
|
31
|
+
/**
|
|
32
|
+
* Document-frequency fraction that fixes the "ordinary term" the weights are
|
|
33
|
+
* expressed in: a token carried by this share of the store. At 0.5 the
|
|
34
|
+
* reference weight is ln(1 + 1 / 0.5) = ln 3, and a tag is worth
|
|
35
|
+
* MEMORY_TAG_BONUS_WORDS of those.
|
|
36
|
+
*/
|
|
37
|
+
export declare const MEMORY_REFERENCE_DF_FRACTION = 0.5;
|
|
38
|
+
/**
|
|
39
|
+
* Defaults for `braid.memory.reservedProvenance`. The partition ships OFF:
|
|
40
|
+
* which entries carry a provenance class is a property of the writer that
|
|
41
|
+
* filed them, so the operator names it with a source prefix. An empty prefix
|
|
42
|
+
* and a zero slot count both mean "no partition", leaving the prune as it was.
|
|
43
|
+
*/
|
|
44
|
+
export declare const RESERVED_PROVENANCE_DEFAULTS: {
|
|
45
|
+
readonly prefix: "";
|
|
46
|
+
readonly slots: 0;
|
|
47
|
+
};
|
|
10
48
|
export declare const CATEGORY_BASE_UTILITY: Record<CognitiveMemoryType, number>;
|
|
11
49
|
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -16,6 +16,202 @@ export const MEMORY_FILE = SEMANTIC_MEMORY_FILE;
|
|
|
16
16
|
export const MAX_ENTRIES = 200;
|
|
17
17
|
export const MIN_UTILITY_THRESHOLD = 0.15;
|
|
18
18
|
export const CONSOLIDATION_INTERVAL = 5;
|
|
19
|
+
// ── Recall scoring ───────────────────────────────────────────────────────────
|
|
20
|
+
/**
|
|
21
|
+
* Shortest token the matcher looks at. One- and two-character tokens carry no
|
|
22
|
+
* topic in any language this store sees, and matching them turns a query into
|
|
23
|
+
* a scan of nearly every entry.
|
|
24
|
+
*/
|
|
25
|
+
export const QUERY_MIN_TOKEN_LENGTH = 3;
|
|
26
|
+
/**
|
|
27
|
+
* English function words, dropped from a query before matching. They name no
|
|
28
|
+
* topic, and counting each as a full word match is what let a long entry
|
|
29
|
+
* outrank a short quote that shared the question's actual subject.
|
|
30
|
+
*
|
|
31
|
+
* Generic vocabulary only. A word that names anything about this deployment,
|
|
32
|
+
* its files, or its roles belongs to that owner, not in a shared matcher.
|
|
33
|
+
*/
|
|
34
|
+
export const QUERY_STOP_WORDS = new Set([
|
|
35
|
+
"a",
|
|
36
|
+
"about",
|
|
37
|
+
"above",
|
|
38
|
+
"across",
|
|
39
|
+
"after",
|
|
40
|
+
"against",
|
|
41
|
+
"all",
|
|
42
|
+
"along",
|
|
43
|
+
"also",
|
|
44
|
+
"am",
|
|
45
|
+
"among",
|
|
46
|
+
"an",
|
|
47
|
+
"and",
|
|
48
|
+
"another",
|
|
49
|
+
"any",
|
|
50
|
+
"are",
|
|
51
|
+
"aren't",
|
|
52
|
+
"around",
|
|
53
|
+
"as",
|
|
54
|
+
"at",
|
|
55
|
+
"be",
|
|
56
|
+
"because",
|
|
57
|
+
"been",
|
|
58
|
+
"before",
|
|
59
|
+
"behind",
|
|
60
|
+
"being",
|
|
61
|
+
"below",
|
|
62
|
+
"beneath",
|
|
63
|
+
"beside",
|
|
64
|
+
"between",
|
|
65
|
+
"beyond",
|
|
66
|
+
"both",
|
|
67
|
+
"but",
|
|
68
|
+
"by",
|
|
69
|
+
"can",
|
|
70
|
+
"could",
|
|
71
|
+
"did",
|
|
72
|
+
"do",
|
|
73
|
+
"does",
|
|
74
|
+
"doesn't",
|
|
75
|
+
"doing",
|
|
76
|
+
"done",
|
|
77
|
+
"don't",
|
|
78
|
+
"during",
|
|
79
|
+
"each",
|
|
80
|
+
"either",
|
|
81
|
+
"every",
|
|
82
|
+
"except",
|
|
83
|
+
"few",
|
|
84
|
+
"for",
|
|
85
|
+
"from",
|
|
86
|
+
"had",
|
|
87
|
+
"has",
|
|
88
|
+
"have",
|
|
89
|
+
"having",
|
|
90
|
+
"he",
|
|
91
|
+
"her",
|
|
92
|
+
"hers",
|
|
93
|
+
"him",
|
|
94
|
+
"his",
|
|
95
|
+
"how",
|
|
96
|
+
"however",
|
|
97
|
+
"i",
|
|
98
|
+
"if",
|
|
99
|
+
"in",
|
|
100
|
+
"inside",
|
|
101
|
+
"into",
|
|
102
|
+
"is",
|
|
103
|
+
"isn't",
|
|
104
|
+
"it",
|
|
105
|
+
"its",
|
|
106
|
+
"just",
|
|
107
|
+
"many",
|
|
108
|
+
"may",
|
|
109
|
+
"me",
|
|
110
|
+
"might",
|
|
111
|
+
"mine",
|
|
112
|
+
"more",
|
|
113
|
+
"most",
|
|
114
|
+
"much",
|
|
115
|
+
"must",
|
|
116
|
+
"my",
|
|
117
|
+
"near",
|
|
118
|
+
"neither",
|
|
119
|
+
"no",
|
|
120
|
+
"nor",
|
|
121
|
+
"not",
|
|
122
|
+
"of",
|
|
123
|
+
"off",
|
|
124
|
+
"on",
|
|
125
|
+
"one",
|
|
126
|
+
"only",
|
|
127
|
+
"onto",
|
|
128
|
+
"or",
|
|
129
|
+
"other",
|
|
130
|
+
"our",
|
|
131
|
+
"ours",
|
|
132
|
+
"out",
|
|
133
|
+
"outside",
|
|
134
|
+
"over",
|
|
135
|
+
"own",
|
|
136
|
+
"per",
|
|
137
|
+
"same",
|
|
138
|
+
"shall",
|
|
139
|
+
"she",
|
|
140
|
+
"should",
|
|
141
|
+
"since",
|
|
142
|
+
"so",
|
|
143
|
+
"some",
|
|
144
|
+
"such",
|
|
145
|
+
"than",
|
|
146
|
+
"that",
|
|
147
|
+
"the",
|
|
148
|
+
"their",
|
|
149
|
+
"theirs",
|
|
150
|
+
"them",
|
|
151
|
+
"then",
|
|
152
|
+
"there",
|
|
153
|
+
"these",
|
|
154
|
+
"they",
|
|
155
|
+
"this",
|
|
156
|
+
"those",
|
|
157
|
+
"through",
|
|
158
|
+
"throughout",
|
|
159
|
+
"till",
|
|
160
|
+
"to",
|
|
161
|
+
"too",
|
|
162
|
+
"toward",
|
|
163
|
+
"two",
|
|
164
|
+
"under",
|
|
165
|
+
"until",
|
|
166
|
+
"up",
|
|
167
|
+
"upon",
|
|
168
|
+
"us",
|
|
169
|
+
"very",
|
|
170
|
+
"via",
|
|
171
|
+
"was",
|
|
172
|
+
"wasn't",
|
|
173
|
+
"we",
|
|
174
|
+
"were",
|
|
175
|
+
"weren't",
|
|
176
|
+
"what",
|
|
177
|
+
"when",
|
|
178
|
+
"where",
|
|
179
|
+
"which",
|
|
180
|
+
"while",
|
|
181
|
+
"who",
|
|
182
|
+
"whom",
|
|
183
|
+
"whose",
|
|
184
|
+
"why",
|
|
185
|
+
"will",
|
|
186
|
+
"with",
|
|
187
|
+
"within",
|
|
188
|
+
"without",
|
|
189
|
+
"would",
|
|
190
|
+
"yet",
|
|
191
|
+
"you",
|
|
192
|
+
"your",
|
|
193
|
+
"yours",
|
|
194
|
+
]);
|
|
195
|
+
/**
|
|
196
|
+
* What a query-named tag is worth, in matched-word units: two ordinary terms.
|
|
197
|
+
* The shipped matcher counted a word as 1 and a tag as 2, so 2 keeps the
|
|
198
|
+
* shipped meaning now that a word's weight is a rarity weight instead.
|
|
199
|
+
*/
|
|
200
|
+
export const MEMORY_TAG_BONUS_WORDS = 2;
|
|
201
|
+
/**
|
|
202
|
+
* Document-frequency fraction that fixes the "ordinary term" the weights are
|
|
203
|
+
* expressed in: a token carried by this share of the store. At 0.5 the
|
|
204
|
+
* reference weight is ln(1 + 1 / 0.5) = ln 3, and a tag is worth
|
|
205
|
+
* MEMORY_TAG_BONUS_WORDS of those.
|
|
206
|
+
*/
|
|
207
|
+
export const MEMORY_REFERENCE_DF_FRACTION = 0.5;
|
|
208
|
+
/**
|
|
209
|
+
* Defaults for `braid.memory.reservedProvenance`. The partition ships OFF:
|
|
210
|
+
* which entries carry a provenance class is a property of the writer that
|
|
211
|
+
* filed them, so the operator names it with a source prefix. An empty prefix
|
|
212
|
+
* and a zero slot count both mean "no partition", leaving the prune as it was.
|
|
213
|
+
*/
|
|
214
|
+
export const RESERVED_PROVENANCE_DEFAULTS = { prefix: "", slots: 0 };
|
|
19
215
|
export const CATEGORY_BASE_UTILITY = {
|
|
20
216
|
goal: 0.9,
|
|
21
217
|
error: 0.8,
|
package/dist/lib/memory/index.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* helpers are in sibling modules (types.ts, constants.ts, storage.ts, utils.ts).
|
|
6
6
|
*/
|
|
7
7
|
import { createLogger } from "../core/logger.js";
|
|
8
|
-
import { MAX_ENTRIES, CONSOLIDATION_INTERVAL } from "./constants.js";
|
|
8
|
+
import { MAX_ENTRIES, CONSOLIDATION_INTERVAL, QUERY_MIN_TOKEN_LENGTH, QUERY_STOP_WORDS, MEMORY_TAG_BONUS_WORDS, MEMORY_REFERENCE_DF_FRACTION, } from "./constants.js";
|
|
9
9
|
import { loadMemory, saveMemory } from "./storage.js";
|
|
10
10
|
import { generateAutoTags, generateId } from "./utils.js";
|
|
11
11
|
// MAX_ENTRIES, MIN_UTILITY_THRESHOLD, CATEGORY_BASE_UTILITY are
|
|
@@ -17,6 +17,17 @@ export { formatMemoryTimestamp, formatMemoryEntry, parseStructuredContent, gener
|
|
|
17
17
|
// constants (MAX_ENTRIES, MIN_UTILITY_THRESHOLD, CATEGORY_BASE_UTILITY)
|
|
18
18
|
// are now in ./lifecycle.ts. The class below uses them.
|
|
19
19
|
import { shouldStore, calculateUtility, pruneMemories, consolidate } from "./lifecycle.js";
|
|
20
|
+
/**
|
|
21
|
+
* The tokens a matcher can see: lowercase, split on everything except letters,
|
|
22
|
+
* digits and inner hyphens, and long enough to name something. Splitting the
|
|
23
|
+
* query this way is what lets a question ending in "APIs?" reach the stored
|
|
24
|
+
* token "apis" instead of carrying "apis?" to a substring test that can never
|
|
25
|
+
* match it. The same split runs over content, so a phrase test compares tokens
|
|
26
|
+
* with tokens instead of punctuation with punctuation.
|
|
27
|
+
*/
|
|
28
|
+
function matchTokens(text) {
|
|
29
|
+
return (text.toLowerCase().match(/[a-z0-9-]+/g) ?? []).filter((token) => token.length >= QUERY_MIN_TOKEN_LENGTH);
|
|
30
|
+
}
|
|
20
31
|
// INTENTIONAL: CognitiveMemory is a cross-session knowledge base (memories
|
|
21
32
|
// from past sessions inform new sessions via memory-recall). The singleton
|
|
22
33
|
// pattern here means all callers share the same in-memory store, which is
|
|
@@ -245,10 +256,33 @@ export class CognitiveMemory {
|
|
|
245
256
|
}
|
|
246
257
|
async recall(query, options) {
|
|
247
258
|
const lowerQuery = query.toLowerCase();
|
|
248
|
-
const queryWords = lowerQuery
|
|
259
|
+
const queryWords = matchTokens(lowerQuery).filter((word) => !QUERY_STOP_WORDS.has(word));
|
|
260
|
+
const phrase = queryWords.join(" ");
|
|
249
261
|
const limit = options?.limit ?? 5;
|
|
262
|
+
// Rarity is measured over the live store, once per call: df(w) is the
|
|
263
|
+
// number of entries carrying w, and a matched word weighs ln(1 + N/df). A
|
|
264
|
+
// word no entry carries (df 0) earns the maximum weight, which is what a
|
|
265
|
+
// word the store has never seen should earn. What this buys: a store's
|
|
266
|
+
// common words stop deciding by count, so four of them no longer outweigh
|
|
267
|
+
// the one rare word a question is about.
|
|
268
|
+
const live = this.filterExpired(this.memoryStore.memories);
|
|
269
|
+
const contentTokens = new Map();
|
|
270
|
+
const documentFrequency = new Map();
|
|
271
|
+
for (const memory of live) {
|
|
272
|
+
const tokens = matchTokens(memory.content);
|
|
273
|
+
contentTokens.set(memory.id, tokens);
|
|
274
|
+
for (const token of new Set(tokens)) {
|
|
275
|
+
documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
const idf = (word) => Math.log(1 + live.length / Math.max(1, documentFrequency.get(word) ?? 1));
|
|
279
|
+
const phraseWeight = queryWords.reduce((sum, word) => sum + idf(word), 0);
|
|
280
|
+
// A tag stays worth MEMORY_TAG_BONUS_WORDS ordinary terms, measured at the
|
|
281
|
+
// reference document frequency, so the bonus keeps its shipped meaning on
|
|
282
|
+
// the rarity scale.
|
|
283
|
+
const tagBonus = MEMORY_TAG_BONUS_WORDS * Math.log(1 + 1 / MEMORY_REFERENCE_DF_FRACTION);
|
|
250
284
|
const results = [];
|
|
251
|
-
for (const memory of
|
|
285
|
+
for (const memory of live) {
|
|
252
286
|
if (options?.type && memory.type !== options.type)
|
|
253
287
|
continue;
|
|
254
288
|
if (options?.tags?.length && !options.tags.some((t) => memory.tags.includes(t)))
|
|
@@ -261,17 +295,22 @@ export class CognitiveMemory {
|
|
|
261
295
|
memory.projectId !== options.currentProject) {
|
|
262
296
|
continue;
|
|
263
297
|
}
|
|
264
|
-
const
|
|
298
|
+
const haystack = (contentTokens.get(memory.id) ?? []).join(" ");
|
|
265
299
|
let matches = 0;
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
300
|
+
if (phrase && haystack.includes(phrase)) {
|
|
301
|
+
// The whole normalized query, in order: the strongest lexical signal
|
|
302
|
+
// the store can offer, so it is scored as every term matched.
|
|
303
|
+
matches = phraseWeight;
|
|
304
|
+
}
|
|
305
|
+
else {
|
|
306
|
+
for (const word of queryWords) {
|
|
307
|
+
if (haystack.includes(word))
|
|
308
|
+
matches += idf(word);
|
|
309
|
+
}
|
|
269
310
|
}
|
|
270
|
-
if (contentLower.includes(lowerQuery))
|
|
271
|
-
matches += queryWords.length;
|
|
272
311
|
for (const tag of memory.tags) {
|
|
273
312
|
if (lowerQuery.includes(tag.toLowerCase()))
|
|
274
|
-
matches +=
|
|
313
|
+
matches += tagBonus;
|
|
275
314
|
}
|
|
276
315
|
const relevance = matches * (0.5 + memory.utility * 0.5);
|
|
277
316
|
if (relevance > 0) {
|
|
@@ -37,6 +37,14 @@ export declare function calculateUtility(type: CognitiveMemoryType, content: str
|
|
|
37
37
|
* Prune memories down to MAX_ENTRIES, keeping recent-access and
|
|
38
38
|
* high-utility entries. Recent (last 5 min) entries are always kept.
|
|
39
39
|
* Other entries are sorted by utility, then timestamp, and trimmed.
|
|
40
|
+
*
|
|
41
|
+
* A configured partition (`braid.memory.reservedProvenance`) takes its slots
|
|
42
|
+
* first: entries whose `source` starts with the configured prefix may occupy
|
|
43
|
+
* up to `slots` of the cap, and everything else, including class entries that
|
|
44
|
+
* did not win a slot, competes for what remains. The class arrives as
|
|
45
|
+
* configuration because which entries carry provenance is a property of the
|
|
46
|
+
* writer that filed them; the store only reads the prefix as an opaque
|
|
47
|
+
* string. Absent or zero, this function is the prune it always was.
|
|
40
48
|
*/
|
|
41
49
|
export declare function pruneMemories(memories: CognitiveMemoryEntry[]): CognitiveMemoryEntry[];
|
|
42
50
|
/**
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* a new CognitiveMemoryStore with the consolidated state.
|
|
14
14
|
*/
|
|
15
15
|
import { jaccardSimilarity } from "./utils.js";
|
|
16
|
+
import { getMemoryReservedProvenance } from "../config/braid-settings.js";
|
|
16
17
|
// ── Constants ────────────────────────────────────────────────────────────────
|
|
17
18
|
const GENERIC_CONTENT_PATTERNS = [/auto-captured memory from enforcement/i, /auto-captured/i];
|
|
18
19
|
// Re-exported for the CognitiveMemory class which also gates on these.
|
|
@@ -59,32 +60,56 @@ export function calculateUtility(type, content, importance) {
|
|
|
59
60
|
return base * 0.7 + importance * 0.3;
|
|
60
61
|
}
|
|
61
62
|
/**
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
63
|
+
* The order the prune applies: a utility difference larger than 0.1 decides,
|
|
64
|
+
* and inside that tolerance the newer entry wins. Extracted so the reserved
|
|
65
|
+
* partition ranks its own population exactly as the general pool is ranked.
|
|
65
66
|
*/
|
|
66
|
-
|
|
67
|
+
function byUtilityThenRecency(a, b) {
|
|
68
|
+
if (Math.abs(a.utility - b.utility) > 0.1)
|
|
69
|
+
return b.utility - a.utility;
|
|
70
|
+
return b.timestamp - a.timestamp;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Keep the top `cap` entries: everything touched in the last five minutes is
|
|
74
|
+
* kept outside the competition for slots, and the rest is ranked by
|
|
75
|
+
* byUtilityThenRecency.
|
|
76
|
+
*/
|
|
77
|
+
function pruneToCap(memories, cap) {
|
|
67
78
|
const now = Date.now();
|
|
68
79
|
const RECENT_ACCESS_MS = 5 * 60 * 1000;
|
|
69
80
|
const recent = memories.filter((m) => now - m.lastAccessed < RECENT_ACCESS_MS);
|
|
70
81
|
const others = memories.filter((m) => now - m.lastAccessed >= RECENT_ACCESS_MS);
|
|
71
|
-
const sorted = [...others].sort(
|
|
72
|
-
|
|
73
|
-
return b.utility - a.utility;
|
|
74
|
-
return b.timestamp - a.timestamp;
|
|
75
|
-
});
|
|
76
|
-
const remainingSlots = Math.max(0, MAX_ENTRIES - recent.length);
|
|
82
|
+
const sorted = [...others].sort(byUtilityThenRecency);
|
|
83
|
+
const remainingSlots = Math.max(0, cap - recent.length);
|
|
77
84
|
const pruned = sorted
|
|
78
85
|
.filter((m, i) => i < remainingSlots || m.utility >= MIN_UTILITY_THRESHOLD)
|
|
79
86
|
.slice(0, remainingSlots);
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
87
|
+
return [...recent, ...pruned].sort(byUtilityThenRecency).slice(0, cap);
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Prune memories down to MAX_ENTRIES, keeping recent-access and
|
|
91
|
+
* high-utility entries. Recent (last 5 min) entries are always kept.
|
|
92
|
+
* Other entries are sorted by utility, then timestamp, and trimmed.
|
|
93
|
+
*
|
|
94
|
+
* A configured partition (`braid.memory.reservedProvenance`) takes its slots
|
|
95
|
+
* first: entries whose `source` starts with the configured prefix may occupy
|
|
96
|
+
* up to `slots` of the cap, and everything else, including class entries that
|
|
97
|
+
* did not win a slot, competes for what remains. The class arrives as
|
|
98
|
+
* configuration because which entries carry provenance is a property of the
|
|
99
|
+
* writer that filed them; the store only reads the prefix as an opaque
|
|
100
|
+
* string. Absent or zero, this function is the prune it always was.
|
|
101
|
+
*/
|
|
102
|
+
export function pruneMemories(memories) {
|
|
103
|
+
const { prefix, slots } = getMemoryReservedProvenance();
|
|
104
|
+
const classEntries = prefix && slots > 0
|
|
105
|
+
? memories.filter((memory) => typeof memory.source === "string" && memory.source.startsWith(prefix))
|
|
106
|
+
: [];
|
|
107
|
+
if (classEntries.length === 0)
|
|
108
|
+
return pruneToCap(memories, MAX_ENTRIES);
|
|
109
|
+
const reserved = [...classEntries].sort(byUtilityThenRecency).slice(0, slots);
|
|
110
|
+
const reservedIds = new Set(reserved.map((memory) => memory.id));
|
|
111
|
+
const general = pruneToCap(memories.filter((memory) => !reservedIds.has(memory.id)), Math.max(0, MAX_ENTRIES - reserved.length));
|
|
112
|
+
return [...reserved, ...general];
|
|
88
113
|
}
|
|
89
114
|
/**
|
|
90
115
|
* Consolidate a memory store: merge near-duplicates, decay old utility.
|
package/docs/LOCAL_DEV.md
CHANGED
|
@@ -138,7 +138,7 @@ before and after a full run. A non-zero delta is the leak this rule prevents.
|
|
|
138
138
|
### New fixtures: use `node:test`
|
|
139
139
|
|
|
140
140
|
A new fixture is written on `node:test`. The runner does not care which style a
|
|
141
|
-
file uses -
|
|
141
|
+
file uses - 94 of the core suite's 329 fixtures already use `node:test` - so the
|
|
142
142
|
reason is the failure the other style produces. A hand-rolled helper that calls
|
|
143
143
|
`fn()` without awaiting it discards the case's promise, so an `async` case runs
|
|
144
144
|
detached and the file can print its green tally before that case settles, with
|
package/docs/SETUP.md
CHANGED
|
@@ -305,6 +305,7 @@ The user-global file always exists; the project one is optional and belongs to p
|
|
|
305
305
|
| `braid.timezone` | IANA name | `Europe/Moscow` | Time zone for all braid-facing time rendering. |
|
|
306
306
|
| `braid.enforcement.nudgeConfigs.<type>` | object | built-in per-type defaults | Per-nudge-type overrides with leaf shape `braid.enforcement.nudgeConfigs.<type>.<key>`; accepted fields are `baseCooldownMs` (number), `escalateAfter` (number), and `deduplicate` (boolean). Types: `validation`, `recordFailure`, `malformed`, `memory`, `scratchpad`, `context`, `sessionSignals`. Invalid entries are logged and dropped. |
|
|
307
307
|
| `braid.memory.writeGate` | object | gate ON, 20-min cooldown, `enforce` | Review-before-write gate on episode writes; `mode: "warn"` logs decisions and writes anyway; `sites` opts individual writers out. |
|
|
308
|
+
| `braid.memory.reservedProvenance` | object | OFF (`prefix: ""`, `slots: 0`) | Reserved partition for the semantic store's prune: entries whose `source` starts with `prefix` may occupy up to `slots` of the 200-entry cap, and everything else competes for what remains. The prefix is the operator's, because which entries carry provenance belongs to the writer that filed them. `slots` is an integer clamped to 200; an empty prefix or `slots: 0` leaves the prune exactly as shipped. |
|
|
308
309
|
| `braid.timers.maxWakeAttempts` | integer 1..10 | `3` | Redelivery cap for a failed timer wake before it is dropped with a visible notice. |
|
|
309
310
|
| `braid.session.routeBreaker` | object | ON, 3 failures, 1h floor | Route-failure latch: after `threshold` provider failures of any class except a context-window overflow or an abort since the session's last successful call, self-started turns park - timer wakes, watchdog escalations and explicitly-waking whispers are delivered non-waking, the content still reaches the session and the out-of-band line still fires. Released by a successful provider call, a clean wake turn, a route change, an owner recovery or a session start; while open it still releases one turn per `floorMs`. `enabled: false` makes it inert. |
|
|
310
311
|
| `braid.timers.wakeAlert` | object | ON, `outcomeTimeouts: 2` | Dead-wake page: repeated outcome timeouts / exhausted redeliveries feed the watchdog's wake-delivery condition, which pages the owner out of band (telegram) once per episode plus one recovery line. `mode: "off"` disables the feed (delivery itself is unchanged); `outcomeTimeouts` is an integer 2..10 (1 would page on every normal long turn). |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-braid",
|
|
3
|
-
"version": "0.43.
|
|
3
|
+
"version": "0.43.9",
|
|
4
4
|
"description": "🥨 Braid - Metacognitive extension suite for pi coding agent: Adds memory, plan tracking, structured cognitive signals and completion checks on top of pi core loop, so long sessions keep their context and stop claiming unfinished work.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
type MemoryWriteGateMode,
|
|
22
22
|
type WriteGateSite,
|
|
23
23
|
} from "../memory/write-quality.js";
|
|
24
|
+
import { MAX_ENTRIES, RESERVED_PROVENANCE_DEFAULTS } from "../memory/constants.js";
|
|
24
25
|
import { getSettingsPath } from "./per-tool-settings.js";
|
|
25
26
|
|
|
26
27
|
const _log = createLogger("braid-settings");
|
|
@@ -126,6 +127,19 @@ interface BraidConfig {
|
|
|
126
127
|
mode?: MemoryWriteGateMode;
|
|
127
128
|
sites?: Partial<Record<WriteGateSite, boolean>>;
|
|
128
129
|
};
|
|
130
|
+
/**
|
|
131
|
+
* Reserved partition for the semantic store's prune: entries whose
|
|
132
|
+
* `source` starts with `prefix` may occupy up to `slots` of the entry cap,
|
|
133
|
+
* and everything else competes for what remains. Unset, or with an empty
|
|
134
|
+
* prefix or `slots: 0`, the prune is unchanged.
|
|
135
|
+
*
|
|
136
|
+
* Example:
|
|
137
|
+
* "braid": { "memory": { "reservedProvenance": { "prefix": "decision: ", "slots": 20 } } }
|
|
138
|
+
*/
|
|
139
|
+
reservedProvenance?: {
|
|
140
|
+
prefix?: unknown;
|
|
141
|
+
slots?: unknown;
|
|
142
|
+
};
|
|
129
143
|
};
|
|
130
144
|
/**
|
|
131
145
|
* Context-layer config. decouples braid's effective context window
|
|
@@ -579,6 +593,43 @@ function normalizeWriteGateSites(raw: unknown): Partial<Record<WriteGateSite, bo
|
|
|
579
593
|
return sites;
|
|
580
594
|
}
|
|
581
595
|
|
|
596
|
+
/** A reserved partition: which source prefix is protected, and for how many slots. */
|
|
597
|
+
export interface ReservedProvenanceConfig {
|
|
598
|
+
prefix: string;
|
|
599
|
+
slots: number;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* Pure resolver for the braid.memory.reservedProvenance block. Absent or
|
|
604
|
+
* malformed fields fall back to RESERVED_PROVENANCE_DEFAULTS (partition off),
|
|
605
|
+
* so a typo cannot start holding slots for a class nobody named. `slots` is
|
|
606
|
+
* clamped to MAX_ENTRIES: a reservation at or above the cap would leave the
|
|
607
|
+
* general pool no room at all. Kept pure so the merge is testable without
|
|
608
|
+
* touching live settings.json.
|
|
609
|
+
*/
|
|
610
|
+
export function resolveReservedProvenance(raw?: {
|
|
611
|
+
prefix?: unknown;
|
|
612
|
+
slots?: unknown;
|
|
613
|
+
}): ReservedProvenanceConfig {
|
|
614
|
+
const prefixValue = typeof raw?.prefix === "string" ? raw.prefix.trim() : "";
|
|
615
|
+
const prefix = prefixValue || RESERVED_PROVENANCE_DEFAULTS.prefix;
|
|
616
|
+
const slotsValue = raw?.slots;
|
|
617
|
+
const slots =
|
|
618
|
+
typeof slotsValue === "number" && Number.isInteger(slotsValue) && slotsValue >= 0
|
|
619
|
+
? Math.min(slotsValue, MAX_ENTRIES)
|
|
620
|
+
: RESERVED_PROVENANCE_DEFAULTS.slots;
|
|
621
|
+
return { prefix, slots };
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* Read braid.memory.reservedProvenance from settings.json, fresh on every call
|
|
626
|
+
* (no caching), like the other readers. Default: no partition.
|
|
627
|
+
*/
|
|
628
|
+
export function getMemoryReservedProvenance(): ReservedProvenanceConfig {
|
|
629
|
+
const settings = readSettings();
|
|
630
|
+
return resolveReservedProvenance(settings.braid?.memory?.reservedProvenance);
|
|
631
|
+
}
|
|
632
|
+
|
|
582
633
|
/**
|
|
583
634
|
* The owner time zone for all braid-facing time rendering (lib/utils/local-time.ts).
|
|
584
635
|
* braid.timezone in settings.json when set to a non-empty string; otherwise
|
|
@@ -22,6 +22,208 @@ export const MAX_ENTRIES = 200;
|
|
|
22
22
|
export const MIN_UTILITY_THRESHOLD = 0.15;
|
|
23
23
|
export const CONSOLIDATION_INTERVAL = 5;
|
|
24
24
|
|
|
25
|
+
// ── Recall scoring ───────────────────────────────────────────────────────────
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Shortest token the matcher looks at. One- and two-character tokens carry no
|
|
29
|
+
* topic in any language this store sees, and matching them turns a query into
|
|
30
|
+
* a scan of nearly every entry.
|
|
31
|
+
*/
|
|
32
|
+
export const QUERY_MIN_TOKEN_LENGTH = 3;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* English function words, dropped from a query before matching. They name no
|
|
36
|
+
* topic, and counting each as a full word match is what let a long entry
|
|
37
|
+
* outrank a short quote that shared the question's actual subject.
|
|
38
|
+
*
|
|
39
|
+
* Generic vocabulary only. A word that names anything about this deployment,
|
|
40
|
+
* its files, or its roles belongs to that owner, not in a shared matcher.
|
|
41
|
+
*/
|
|
42
|
+
export const QUERY_STOP_WORDS: ReadonlySet<string> = new Set([
|
|
43
|
+
"a",
|
|
44
|
+
"about",
|
|
45
|
+
"above",
|
|
46
|
+
"across",
|
|
47
|
+
"after",
|
|
48
|
+
"against",
|
|
49
|
+
"all",
|
|
50
|
+
"along",
|
|
51
|
+
"also",
|
|
52
|
+
"am",
|
|
53
|
+
"among",
|
|
54
|
+
"an",
|
|
55
|
+
"and",
|
|
56
|
+
"another",
|
|
57
|
+
"any",
|
|
58
|
+
"are",
|
|
59
|
+
"aren't",
|
|
60
|
+
"around",
|
|
61
|
+
"as",
|
|
62
|
+
"at",
|
|
63
|
+
"be",
|
|
64
|
+
"because",
|
|
65
|
+
"been",
|
|
66
|
+
"before",
|
|
67
|
+
"behind",
|
|
68
|
+
"being",
|
|
69
|
+
"below",
|
|
70
|
+
"beneath",
|
|
71
|
+
"beside",
|
|
72
|
+
"between",
|
|
73
|
+
"beyond",
|
|
74
|
+
"both",
|
|
75
|
+
"but",
|
|
76
|
+
"by",
|
|
77
|
+
"can",
|
|
78
|
+
"could",
|
|
79
|
+
"did",
|
|
80
|
+
"do",
|
|
81
|
+
"does",
|
|
82
|
+
"doesn't",
|
|
83
|
+
"doing",
|
|
84
|
+
"done",
|
|
85
|
+
"don't",
|
|
86
|
+
"during",
|
|
87
|
+
"each",
|
|
88
|
+
"either",
|
|
89
|
+
"every",
|
|
90
|
+
"except",
|
|
91
|
+
"few",
|
|
92
|
+
"for",
|
|
93
|
+
"from",
|
|
94
|
+
"had",
|
|
95
|
+
"has",
|
|
96
|
+
"have",
|
|
97
|
+
"having",
|
|
98
|
+
"he",
|
|
99
|
+
"her",
|
|
100
|
+
"hers",
|
|
101
|
+
"him",
|
|
102
|
+
"his",
|
|
103
|
+
"how",
|
|
104
|
+
"however",
|
|
105
|
+
"i",
|
|
106
|
+
"if",
|
|
107
|
+
"in",
|
|
108
|
+
"inside",
|
|
109
|
+
"into",
|
|
110
|
+
"is",
|
|
111
|
+
"isn't",
|
|
112
|
+
"it",
|
|
113
|
+
"its",
|
|
114
|
+
"just",
|
|
115
|
+
"many",
|
|
116
|
+
"may",
|
|
117
|
+
"me",
|
|
118
|
+
"might",
|
|
119
|
+
"mine",
|
|
120
|
+
"more",
|
|
121
|
+
"most",
|
|
122
|
+
"much",
|
|
123
|
+
"must",
|
|
124
|
+
"my",
|
|
125
|
+
"near",
|
|
126
|
+
"neither",
|
|
127
|
+
"no",
|
|
128
|
+
"nor",
|
|
129
|
+
"not",
|
|
130
|
+
"of",
|
|
131
|
+
"off",
|
|
132
|
+
"on",
|
|
133
|
+
"one",
|
|
134
|
+
"only",
|
|
135
|
+
"onto",
|
|
136
|
+
"or",
|
|
137
|
+
"other",
|
|
138
|
+
"our",
|
|
139
|
+
"ours",
|
|
140
|
+
"out",
|
|
141
|
+
"outside",
|
|
142
|
+
"over",
|
|
143
|
+
"own",
|
|
144
|
+
"per",
|
|
145
|
+
"same",
|
|
146
|
+
"shall",
|
|
147
|
+
"she",
|
|
148
|
+
"should",
|
|
149
|
+
"since",
|
|
150
|
+
"so",
|
|
151
|
+
"some",
|
|
152
|
+
"such",
|
|
153
|
+
"than",
|
|
154
|
+
"that",
|
|
155
|
+
"the",
|
|
156
|
+
"their",
|
|
157
|
+
"theirs",
|
|
158
|
+
"them",
|
|
159
|
+
"then",
|
|
160
|
+
"there",
|
|
161
|
+
"these",
|
|
162
|
+
"they",
|
|
163
|
+
"this",
|
|
164
|
+
"those",
|
|
165
|
+
"through",
|
|
166
|
+
"throughout",
|
|
167
|
+
"till",
|
|
168
|
+
"to",
|
|
169
|
+
"too",
|
|
170
|
+
"toward",
|
|
171
|
+
"two",
|
|
172
|
+
"under",
|
|
173
|
+
"until",
|
|
174
|
+
"up",
|
|
175
|
+
"upon",
|
|
176
|
+
"us",
|
|
177
|
+
"very",
|
|
178
|
+
"via",
|
|
179
|
+
"was",
|
|
180
|
+
"wasn't",
|
|
181
|
+
"we",
|
|
182
|
+
"were",
|
|
183
|
+
"weren't",
|
|
184
|
+
"what",
|
|
185
|
+
"when",
|
|
186
|
+
"where",
|
|
187
|
+
"which",
|
|
188
|
+
"while",
|
|
189
|
+
"who",
|
|
190
|
+
"whom",
|
|
191
|
+
"whose",
|
|
192
|
+
"why",
|
|
193
|
+
"will",
|
|
194
|
+
"with",
|
|
195
|
+
"within",
|
|
196
|
+
"without",
|
|
197
|
+
"would",
|
|
198
|
+
"yet",
|
|
199
|
+
"you",
|
|
200
|
+
"your",
|
|
201
|
+
"yours",
|
|
202
|
+
]);
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* What a query-named tag is worth, in matched-word units: two ordinary terms.
|
|
206
|
+
* The shipped matcher counted a word as 1 and a tag as 2, so 2 keeps the
|
|
207
|
+
* shipped meaning now that a word's weight is a rarity weight instead.
|
|
208
|
+
*/
|
|
209
|
+
export const MEMORY_TAG_BONUS_WORDS = 2;
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Document-frequency fraction that fixes the "ordinary term" the weights are
|
|
213
|
+
* expressed in: a token carried by this share of the store. At 0.5 the
|
|
214
|
+
* reference weight is ln(1 + 1 / 0.5) = ln 3, and a tag is worth
|
|
215
|
+
* MEMORY_TAG_BONUS_WORDS of those.
|
|
216
|
+
*/
|
|
217
|
+
export const MEMORY_REFERENCE_DF_FRACTION = 0.5;
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Defaults for `braid.memory.reservedProvenance`. The partition ships OFF:
|
|
221
|
+
* which entries carry a provenance class is a property of the writer that
|
|
222
|
+
* filed them, so the operator names it with a source prefix. An empty prefix
|
|
223
|
+
* and a zero slot count both mean "no partition", leaving the prune as it was.
|
|
224
|
+
*/
|
|
225
|
+
export const RESERVED_PROVENANCE_DEFAULTS = { prefix: "", slots: 0 } as const;
|
|
226
|
+
|
|
25
227
|
export const CATEGORY_BASE_UTILITY: Record<CognitiveMemoryType, number> = {
|
|
26
228
|
goal: 0.9,
|
|
27
229
|
error: 0.8,
|
package/src/lib/memory/index.ts
CHANGED
|
@@ -13,7 +13,14 @@ import type {
|
|
|
13
13
|
MemoryContext,
|
|
14
14
|
MemoryLink,
|
|
15
15
|
} from "./types.js";
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
MAX_ENTRIES,
|
|
18
|
+
CONSOLIDATION_INTERVAL,
|
|
19
|
+
QUERY_MIN_TOKEN_LENGTH,
|
|
20
|
+
QUERY_STOP_WORDS,
|
|
21
|
+
MEMORY_TAG_BONUS_WORDS,
|
|
22
|
+
MEMORY_REFERENCE_DF_FRACTION,
|
|
23
|
+
} from "./constants.js";
|
|
17
24
|
import { loadMemory, saveMemory } from "./storage.js";
|
|
18
25
|
import { generateAutoTags, generateId } from "./utils.js";
|
|
19
26
|
|
|
@@ -43,6 +50,20 @@ export {
|
|
|
43
50
|
// are now in ./lifecycle.ts. The class below uses them.
|
|
44
51
|
import { shouldStore, calculateUtility, pruneMemories, consolidate } from "./lifecycle.js";
|
|
45
52
|
|
|
53
|
+
/**
|
|
54
|
+
* The tokens a matcher can see: lowercase, split on everything except letters,
|
|
55
|
+
* digits and inner hyphens, and long enough to name something. Splitting the
|
|
56
|
+
* query this way is what lets a question ending in "APIs?" reach the stored
|
|
57
|
+
* token "apis" instead of carrying "apis?" to a substring test that can never
|
|
58
|
+
* match it. The same split runs over content, so a phrase test compares tokens
|
|
59
|
+
* with tokens instead of punctuation with punctuation.
|
|
60
|
+
*/
|
|
61
|
+
function matchTokens(text: string): string[] {
|
|
62
|
+
return (text.toLowerCase().match(/[a-z0-9-]+/g) ?? []).filter(
|
|
63
|
+
(token) => token.length >= QUERY_MIN_TOKEN_LENGTH,
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
46
67
|
// INTENTIONAL: CognitiveMemory is a cross-session knowledge base (memories
|
|
47
68
|
// from past sessions inform new sessions via memory-recall). The singleton
|
|
48
69
|
// pattern here means all callers share the same in-memory store, which is
|
|
@@ -331,12 +352,37 @@ export class CognitiveMemory {
|
|
|
331
352
|
},
|
|
332
353
|
): Promise<Array<{ memory: CognitiveMemoryEntry; relevance: number }>> {
|
|
333
354
|
const lowerQuery = query.toLowerCase();
|
|
334
|
-
const queryWords = lowerQuery
|
|
355
|
+
const queryWords = matchTokens(lowerQuery).filter((word) => !QUERY_STOP_WORDS.has(word));
|
|
356
|
+
const phrase = queryWords.join(" ");
|
|
335
357
|
const limit = options?.limit ?? 5;
|
|
336
358
|
|
|
359
|
+
// Rarity is measured over the live store, once per call: df(w) is the
|
|
360
|
+
// number of entries carrying w, and a matched word weighs ln(1 + N/df). A
|
|
361
|
+
// word no entry carries (df 0) earns the maximum weight, which is what a
|
|
362
|
+
// word the store has never seen should earn. What this buys: a store's
|
|
363
|
+
// common words stop deciding by count, so four of them no longer outweigh
|
|
364
|
+
// the one rare word a question is about.
|
|
365
|
+
const live = this.filterExpired(this.memoryStore.memories);
|
|
366
|
+
const contentTokens = new Map<string, string[]>();
|
|
367
|
+
const documentFrequency = new Map<string, number>();
|
|
368
|
+
for (const memory of live) {
|
|
369
|
+
const tokens = matchTokens(memory.content);
|
|
370
|
+
contentTokens.set(memory.id, tokens);
|
|
371
|
+
for (const token of new Set(tokens)) {
|
|
372
|
+
documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
const idf = (word: string): number =>
|
|
376
|
+
Math.log(1 + live.length / Math.max(1, documentFrequency.get(word) ?? 1));
|
|
377
|
+
const phraseWeight = queryWords.reduce((sum, word) => sum + idf(word), 0);
|
|
378
|
+
// A tag stays worth MEMORY_TAG_BONUS_WORDS ordinary terms, measured at the
|
|
379
|
+
// reference document frequency, so the bonus keeps its shipped meaning on
|
|
380
|
+
// the rarity scale.
|
|
381
|
+
const tagBonus = MEMORY_TAG_BONUS_WORDS * Math.log(1 + 1 / MEMORY_REFERENCE_DF_FRACTION);
|
|
382
|
+
|
|
337
383
|
const results: Array<{ memory: CognitiveMemoryEntry; relevance: number }> = [];
|
|
338
384
|
|
|
339
|
-
for (const memory of
|
|
385
|
+
for (const memory of live) {
|
|
340
386
|
if (options?.type && memory.type !== options.type) continue;
|
|
341
387
|
if (options?.tags?.length && !options.tags.some((t) => memory.tags.includes(t))) continue;
|
|
342
388
|
|
|
@@ -351,14 +397,19 @@ export class CognitiveMemory {
|
|
|
351
397
|
continue;
|
|
352
398
|
}
|
|
353
399
|
|
|
354
|
-
const
|
|
400
|
+
const haystack = (contentTokens.get(memory.id) ?? []).join(" ");
|
|
355
401
|
let matches = 0;
|
|
356
|
-
|
|
357
|
-
|
|
402
|
+
if (phrase && haystack.includes(phrase)) {
|
|
403
|
+
// The whole normalized query, in order: the strongest lexical signal
|
|
404
|
+
// the store can offer, so it is scored as every term matched.
|
|
405
|
+
matches = phraseWeight;
|
|
406
|
+
} else {
|
|
407
|
+
for (const word of queryWords) {
|
|
408
|
+
if (haystack.includes(word)) matches += idf(word);
|
|
409
|
+
}
|
|
358
410
|
}
|
|
359
|
-
if (contentLower.includes(lowerQuery)) matches += queryWords.length;
|
|
360
411
|
for (const tag of memory.tags) {
|
|
361
|
-
if (lowerQuery.includes(tag.toLowerCase())) matches +=
|
|
412
|
+
if (lowerQuery.includes(tag.toLowerCase())) matches += tagBonus;
|
|
362
413
|
}
|
|
363
414
|
|
|
364
415
|
const relevance = matches * (0.5 + memory.utility * 0.5);
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
import type { CognitiveMemoryEntry, CognitiveMemoryType, CognitiveMemoryStore } from "./types.js";
|
|
17
17
|
import { jaccardSimilarity } from "./utils.js";
|
|
18
|
+
import { getMemoryReservedProvenance } from "../config/braid-settings.js";
|
|
18
19
|
|
|
19
20
|
// ── Constants ────────────────────────────────────────────────────────────────
|
|
20
21
|
|
|
@@ -72,34 +73,67 @@ export function calculateUtility(
|
|
|
72
73
|
}
|
|
73
74
|
|
|
74
75
|
/**
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
76
|
+
* The order the prune applies: a utility difference larger than 0.1 decides,
|
|
77
|
+
* and inside that tolerance the newer entry wins. Extracted so the reserved
|
|
78
|
+
* partition ranks its own population exactly as the general pool is ranked.
|
|
78
79
|
*/
|
|
79
|
-
|
|
80
|
+
function byUtilityThenRecency(a: CognitiveMemoryEntry, b: CognitiveMemoryEntry): number {
|
|
81
|
+
if (Math.abs(a.utility - b.utility) > 0.1) return b.utility - a.utility;
|
|
82
|
+
return b.timestamp - a.timestamp;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Keep the top `cap` entries: everything touched in the last five minutes is
|
|
87
|
+
* kept outside the competition for slots, and the rest is ranked by
|
|
88
|
+
* byUtilityThenRecency.
|
|
89
|
+
*/
|
|
90
|
+
function pruneToCap(memories: CognitiveMemoryEntry[], cap: number): CognitiveMemoryEntry[] {
|
|
80
91
|
const now = Date.now();
|
|
81
92
|
const RECENT_ACCESS_MS = 5 * 60 * 1000;
|
|
82
93
|
|
|
83
94
|
const recent = memories.filter((m) => now - m.lastAccessed < RECENT_ACCESS_MS);
|
|
84
95
|
const others = memories.filter((m) => now - m.lastAccessed >= RECENT_ACCESS_MS);
|
|
85
96
|
|
|
86
|
-
const sorted = [...others].sort(
|
|
87
|
-
if (Math.abs(a.utility - b.utility) > 0.1) return b.utility - a.utility;
|
|
88
|
-
return b.timestamp - a.timestamp;
|
|
89
|
-
});
|
|
97
|
+
const sorted = [...others].sort(byUtilityThenRecency);
|
|
90
98
|
|
|
91
|
-
const remainingSlots = Math.max(0,
|
|
99
|
+
const remainingSlots = Math.max(0, cap - recent.length);
|
|
92
100
|
const pruned = sorted
|
|
93
101
|
.filter((m, i) => i < remainingSlots || m.utility >= MIN_UTILITY_THRESHOLD)
|
|
94
102
|
.slice(0, remainingSlots);
|
|
95
103
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
104
|
+
return [...recent, ...pruned].sort(byUtilityThenRecency).slice(0, cap);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Prune memories down to MAX_ENTRIES, keeping recent-access and
|
|
109
|
+
* high-utility entries. Recent (last 5 min) entries are always kept.
|
|
110
|
+
* Other entries are sorted by utility, then timestamp, and trimmed.
|
|
111
|
+
*
|
|
112
|
+
* A configured partition (`braid.memory.reservedProvenance`) takes its slots
|
|
113
|
+
* first: entries whose `source` starts with the configured prefix may occupy
|
|
114
|
+
* up to `slots` of the cap, and everything else, including class entries that
|
|
115
|
+
* did not win a slot, competes for what remains. The class arrives as
|
|
116
|
+
* configuration because which entries carry provenance is a property of the
|
|
117
|
+
* writer that filed them; the store only reads the prefix as an opaque
|
|
118
|
+
* string. Absent or zero, this function is the prune it always was.
|
|
119
|
+
*/
|
|
120
|
+
export function pruneMemories(memories: CognitiveMemoryEntry[]): CognitiveMemoryEntry[] {
|
|
121
|
+
const { prefix, slots } = getMemoryReservedProvenance();
|
|
122
|
+
const classEntries =
|
|
123
|
+
prefix && slots > 0
|
|
124
|
+
? memories.filter(
|
|
125
|
+
(memory) => typeof memory.source === "string" && memory.source.startsWith(prefix),
|
|
126
|
+
)
|
|
127
|
+
: [];
|
|
128
|
+
if (classEntries.length === 0) return pruneToCap(memories, MAX_ENTRIES);
|
|
129
|
+
|
|
130
|
+
const reserved = [...classEntries].sort(byUtilityThenRecency).slice(0, slots);
|
|
131
|
+
const reservedIds = new Set(reserved.map((memory) => memory.id));
|
|
132
|
+
const general = pruneToCap(
|
|
133
|
+
memories.filter((memory) => !reservedIds.has(memory.id)),
|
|
134
|
+
Math.max(0, MAX_ENTRIES - reserved.length),
|
|
135
|
+
);
|
|
136
|
+
return [...reserved, ...general];
|
|
103
137
|
}
|
|
104
138
|
|
|
105
139
|
/**
|