pi-braid 0.43.9 → 0.43.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,405 @@
1
+ /**
2
+ * Decision grammar: the generic half of the ruling write path.
3
+ *
4
+ * One body of text in, either the standing instruction it carries or `null`.
5
+ * "Generic" is a claim about what the module is allowed to know: sentence
6
+ * spans, the marker table that turns a sentence into a standing instruction,
7
+ * the shapes that are not instructions (a question, a status report, a paste, a
8
+ * hedge, a conversational aside), and the first- and second-person grammar of
9
+ * English. It knows nothing about who wrote the text, which channel carried it,
10
+ * or what the caller's world is made of.
11
+ *
12
+ * The last of those is the one thing a grammar cannot supply, so the caller
13
+ * hands it in as `GrammarVocabulary`: which words name a subject a ruling can be
14
+ * about, which words describe the caller's own machinery, and how an explicit
15
+ * generalization reads. `classifyRulingBody()` applies those three tests at the
16
+ * same two places it applies its own - the turn and the sentence that gets
17
+ * quoted - so a directive aimed at the reader is told apart from a rule about
18
+ * the system by vocabulary this layer never spells.
19
+ *
20
+ * The second half of the module is the store side of the same write path:
21
+ * `deriveTopicTags()` reads a quote and the population it is filed into and
22
+ * returns the words worth carrying as tags. The rule reasons about tokens,
23
+ * document frequency and a generic English stop list, so it belongs next to the
24
+ * grammar rather than in the caller's vocabulary.
25
+ *
26
+ * Pure by construction: no I/O, no clock, no randomness, so its behaviour is a
27
+ * fixture instead of a story.
28
+ */
29
+
30
+ import { QUERY_STOP_WORDS } from "./constants.js";
31
+
32
+ /** The closed set of ruling shapes the classifier can name. */
33
+ export type RulingClass = "prohibition" | "preference" | "requirement" | "policy";
34
+
35
+ /** A classified body: what kind of instruction it is, and the words to keep. */
36
+ export interface RulingShape {
37
+ class: RulingClass;
38
+ /** A contiguous, verbatim slice of the body, capped with a trailing ellipsis. */
39
+ quote: string;
40
+ }
41
+
42
+ /**
43
+ * The three questions a grammar has to ask the caller's world.
44
+ *
45
+ * `namesSubject` is the escape hatch for a bare directive: "Do not read or
46
+ * write files." names no system, so without first person it is an instruction
47
+ * aimed at the reader, not a rule about anything. `talksProcess` refuses a body
48
+ * whose subject is the caller's own machinery rather than the thing a ruling is
49
+ * about, and `generalizes` admits a second-person body that says it speaks to
50
+ * every recipient ("any instance", "generically"), which is a rule rather than
51
+ * a steer.
52
+ */
53
+ export interface GrammarVocabulary {
54
+ namesSubject(text: string): boolean;
55
+ talksProcess(text: string): boolean;
56
+ generalizes(text: string): boolean;
57
+ }
58
+
59
+ /** Longest quote the caller receives; past this the quote is cut, not summarized. */
60
+ export const QUOTE_LIMIT = 400;
61
+
62
+ /**
63
+ * A body longer than this is a pasted brief, a log or a tool dump, not the
64
+ * sentence an owner states a ruling in. The quote is capped at QUOTE_LIMIT
65
+ * anyway, so a long body can only contribute a fragment whose subject was three
66
+ * paragraphs earlier.
67
+ */
68
+ const MAX_TURN_CHARS = 1200;
69
+
70
+ /** Below this a body is an acknowledgement ("ok", "proceed"), never a ruling. */
71
+ const MIN_TURN_WORDS = 3;
72
+
73
+ /** A continuation sentence is kept only while the quote stays short enough to read. */
74
+ const CONTINUATION_LIMIT = 200;
75
+
76
+ /** The owner's own voice: first person, singular or collective. */
77
+ const FIRST_PERSON = /\b(?:i|i'm|i've|i'll|i'd|we|we're|we've|we'll|we'd|my|our|ours|let's)\b/i;
78
+
79
+ /**
80
+ * Second-person address. A directive written to *you* is an instruction for
81
+ * the reader; a rule for the system talks about it, or generalizes explicitly.
82
+ */
83
+ const SECOND_PERSON = /\b(?:you|your|yours|yourself|yourselves)\b/i;
84
+
85
+ /**
86
+ * First-person report voice: the speaker narrating what was seen, changed or
87
+ * disabled. Such a sentence can carry "always" or "no longer" by accident ("I
88
+ * still see only ...", "it no longer indexed"), so it is refused unless it opens
89
+ * with a directive.
90
+ */
91
+ const REPORT_VOICE =
92
+ /\b(?:i|we)\b[^.!?]{0,120}\b(?:noticed|noticing|see|saw|seen|disabled|enabled|changed|updated|merged|pushed|committed|created|deleted|found|think|thought|guess|believe|realised|realized|checked|ran|built|fixed|added|removed|renamed|replaced|read|told|said|mentioned)\b/i;
93
+
94
+ /**
95
+ * Conversational openers that carry no decision: thinking aloud. A marker
96
+ * inside one of these ("doesn't really matter") is not a ruling.
97
+ */
98
+ const CONVERSATIONAL =
99
+ /^\s*(?:or i don'?t know|i don'?t know|not sure|doesn'?t really matter|whatever|never mind|nevermind)\b/i;
100
+
101
+ /** A body that reads as a report of work already done, on its first line. */
102
+ const STATUS_OPENER =
103
+ /^(?:done|finished|merged|shipped|pushed|committed|built|fixed|updated|completed|released|deployed|landed|green|all (?:tests|gates|checks)|tests? (?:pass|green)|gates? (?:pass|green))/i;
104
+
105
+ /**
106
+ * Hedged advice: the shape of a recommendation that was floated and not
107
+ * adopted. `should` and `could` are deliberately absent from the durable
108
+ * marker table below - they are how advice is phrased, not how a ruling is
109
+ * stated - so a hedged sentence has to be refused explicitly.
110
+ */
111
+ const HEDGE =
112
+ /\b(?:maybe|perhaps|might|could|would|possibly|probably|consider|i think|i guess|not sure|unsure|what if|why not|we should probably)\b/i;
113
+
114
+ /**
115
+ * Absolute frames that make a hedged sentence a ruling anyway: "maybe we should
116
+ * never post through an API" has still ruled, while "we could consider APIs"
117
+ * has not.
118
+ */
119
+ const ABSOLUTE =
120
+ /\b(?:never|must|always|only one|required|mandatory|forbidden|banned|not allowed|by default|from now on|going forward)\b/i;
121
+
122
+ /**
123
+ * Opening words that make a sentence a question, used only together with a
124
+ * question mark: the mark alone is enough at the end of a body, but a body that
125
+ * opens with a question and closes with a ruling keeps its ruling.
126
+ */
127
+ const INTERROGATIVE_OPENER = /^\s*(?:how|what|why|when|where|which|who|whose|whether)\b/i;
128
+
129
+ const QUESTION_OPENER =
130
+ /^(?:should|shall|can|could|may|might|do|does|did|is|are|was|were|will|would|have|has|how|what|why|when|where|which|who|whose|am|if)\b/i;
131
+
132
+ /**
133
+ * Decision frames, in class-priority order. A frame is the grammar of a
134
+ * standing instruction rather than of this turn's work: an absolute (`never`,
135
+ * `always`, `must`, `only one`), a comparison between ways of doing something
136
+ * (`prefer`, `instead of`, `rather than`, `better than`), or a prescription
137
+ * stated in the collective voice (`we need to`, `should`). Bare imperatives are
138
+ * deliberately absent: "investigate X" is a task for this turn, and a table
139
+ * built from every imperative is a task list filed as policy.
140
+ */
141
+ const MARKERS: ReadonlyArray<{ class: RulingClass; pattern: RegExp }> = [
142
+ {
143
+ class: "prohibition",
144
+ pattern:
145
+ /\b(?:never|don't|do not|must not|may not|not allowed|forbidden|banned|no longer|stop (?:using|doing|running|adding|writing|posting|editing|treating))\b/i,
146
+ },
147
+ {
148
+ class: "preference",
149
+ pattern: /\b(?:prefer|preferably|instead of|rather than|better than|avoid)\b/i,
150
+ },
151
+ {
152
+ class: "requirement",
153
+ pattern: /\b(?:must|should|always|required|mandatory|only one|need(?:s)? to|have to|has to)\b/i,
154
+ },
155
+ {
156
+ class: "policy",
157
+ pattern:
158
+ /\b(?:by default|default is|from now on|going forward|rule is|policy|standard|convention|principle)\b/i,
159
+ },
160
+ ];
161
+
162
+ /** A sentence span over the original text; every quote is a slice of one. */
163
+ interface Span {
164
+ start: number;
165
+ end: number;
166
+ }
167
+
168
+ /** Abbreviations whose period does not end a sentence. */
169
+ const ABBREVIATION = /(?:^|[\s(])(?:e\.g|i\.e|etc|vs|cf|approx|resp|no|dr|mr|mrs|ms|st)\.$/i;
170
+
171
+ /**
172
+ * Split a body into contiguous sentence spans.
173
+ *
174
+ * The spans are offsets into the original string, so a quote assembled from
175
+ * them is a byte-for-byte slice of the words - the property the pointer check
176
+ * leans on. Newlines end a sentence; a period does not end one inside a known
177
+ * abbreviation, a number (`3.5`), or a path (`decision-grammar.ts`).
178
+ */
179
+ function sentenceSpans(text: string): Span[] {
180
+ const spans: Span[] = [];
181
+ let start = 0;
182
+ const push = (end: number): void => {
183
+ if (end > start) {
184
+ const slice = text.slice(start, end);
185
+ if (slice.trim()) spans.push({ start, end });
186
+ }
187
+ start = end;
188
+ };
189
+ for (let i = 0; i < text.length; i++) {
190
+ const ch = text[i];
191
+ if (ch === "\n") {
192
+ push(i);
193
+ continue;
194
+ }
195
+ if (ch !== "." && ch !== "!" && ch !== "?") continue;
196
+ const next = text[i + 1];
197
+ if (next !== undefined && !/\s/.test(next)) continue;
198
+ if (ch === "." && ABBREVIATION.test(text.slice(Math.max(0, i - 12), i + 1))) continue;
199
+ if (ch === "." && /\d$/.test(text[i - 1] ?? "") && /^\d/.test(next ?? "")) continue;
200
+ push(i + 1);
201
+ }
202
+ push(text.length);
203
+ return spans;
204
+ }
205
+
206
+ /** Words in a string, counted the way a reader would. */
207
+ function wordCount(text: string): number {
208
+ return text.split(/\s+/).filter(Boolean).length;
209
+ }
210
+
211
+ /**
212
+ * Is the body a paste - a shell transcript, a code block, a log or a payload?
213
+ *
214
+ * A pasted command is the single most common false ruling: it carries
215
+ * imperatives ("rm -rf"), digits, and often the word "only". The screens are
216
+ * cheap because a ruling sentence is prose, and prose does not start every line
217
+ * with a prompt, a box, or a stack frame.
218
+ */
219
+ function looksPasted(text: string): boolean {
220
+ if (text.includes("```")) return true;
221
+ if (/^\s*[{[]/.test(text)) return true;
222
+ if (/^\s*(?:\$|%|> |#!)\s/m.test(text)) return true;
223
+ if (/(?:^|\n)\s*(?:[│├└─]|\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2})/m.test(text)) return true;
224
+ if (/Traceback \(most recent call last\)|\bat [\w$.]+ \([\w/.-]+:\d+:\d+\)/.test(text))
225
+ return true;
226
+ const lines = text.split("\n").filter((line) => line.trim());
227
+ const commandLines = lines.filter((line) =>
228
+ /^\s*(?:sudo |git |npm |npx |node |python3? |cd |ls |cat |rg |grep |curl |chmod |brew |make |docker |kubectl |export |rm )/.test(
229
+ line,
230
+ ),
231
+ );
232
+ if (commandLines.length >= 2) return true;
233
+ // A payload of paths and key=value rows, with almost no sentence structure.
234
+ if (lines.length >= 6 && lines.every((line) => line.length < 200)) {
235
+ const payloadish = lines.filter((line) => /^[\w./-]{2,}\s*[:=]\s*\S/.test(line));
236
+ if (payloadish.length >= lines.length / 2) return true;
237
+ }
238
+ return false;
239
+ }
240
+
241
+ /** Does the body end in a question, or open with one and carry nothing after it? */
242
+ function isQuestionShaped(text: string, spans: Span[]): boolean {
243
+ const trimmed = text.trimEnd();
244
+ if (trimmed.endsWith("?")) return true;
245
+ const first = spans[0] ? text.slice(spans[0].start, spans[0].end).trim() : "";
246
+ return QUESTION_OPENER.test(first) && first.includes("?") && spans.length === 1;
247
+ }
248
+
249
+ /**
250
+ * The class a sentence marks, or `null` when it marks no standing instruction.
251
+ * First match in table order wins, so "must never" reads as a prohibition and
252
+ * "always prefer" as a preference.
253
+ */
254
+ function markerClass(sentence: string): RulingClass | null {
255
+ for (const { class: kind, pattern } of MARKERS) {
256
+ if (pattern.test(sentence)) return kind;
257
+ }
258
+ return null;
259
+ }
260
+
261
+ /**
262
+ * Assemble the quote: the sentence that carries the marker, plus its immediate
263
+ * continuation while that stays short. Contiguous, never reordered, never
264
+ * stitched from two places in the body.
265
+ */
266
+ function quoteFrom(text: string, spans: Span[], markerIndex: number): string {
267
+ let end = spans[markerIndex].end;
268
+ const next = spans[markerIndex + 1];
269
+ if (next) {
270
+ const candidate = text.slice(spans[markerIndex].start, next.end);
271
+ if (candidate.length <= CONTINUATION_LIMIT) end = next.end;
272
+ }
273
+ const quote = text.slice(spans[markerIndex].start, end).trim();
274
+ if (quote.length <= QUOTE_LIMIT) return quote;
275
+ return `${quote.slice(0, QUOTE_LIMIT).trimEnd()}...`;
276
+ }
277
+
278
+ /**
279
+ * Classify one body that is already known to be owner speech.
280
+ *
281
+ * Returns `null` for everything that is not a standing instruction: a question,
282
+ * a paste, a status report, a hedged recommendation, an acknowledgement, a
283
+ * second-person directive, or a body too long to cut a self-contained sentence
284
+ * from. Returns the class and the verbatim quote otherwise. The caller owns the
285
+ * provenance fields, because the caller is the one that knows where the text
286
+ * came from.
287
+ */
288
+ export function classifyRulingBody(
289
+ text: string,
290
+ vocabulary: GrammarVocabulary,
291
+ ): RulingShape | null {
292
+ if (typeof text !== "string") return null;
293
+ const body = text.trim();
294
+ if (!body) return null;
295
+ if (wordCount(body) < MIN_TURN_WORDS) return null;
296
+ if (body.length > MAX_TURN_CHARS) return null;
297
+ if (looksPasted(body)) return null;
298
+
299
+ const spans = sentenceSpans(body);
300
+ if (spans.length === 0) return null;
301
+ if (isQuestionShaped(body, spans)) return null;
302
+
303
+ const firstSentence = body.slice(spans[0].start, spans[0].end).trim();
304
+ if (STATUS_OPENER.test(firstSentence)) return null;
305
+ if (INTERROGATIVE_OPENER.test(firstSentence)) return null;
306
+
307
+ const firstPerson = FIRST_PERSON.test(body);
308
+ if (SECOND_PERSON.test(body) && !vocabulary.generalizes(body)) return null;
309
+ if (!firstPerson && !vocabulary.namesSubject(body)) return null;
310
+ if (!firstPerson && vocabulary.talksProcess(body)) return null;
311
+
312
+ for (let index = 0; index < spans.length; index++) {
313
+ const sentence = body.slice(spans[index].start, spans[index].end).trim();
314
+ const kind = markerClass(sentence);
315
+ if (!kind) continue;
316
+ if (STATUS_OPENER.test(sentence)) continue;
317
+ if (CONVERSATIONAL.test(sentence)) continue;
318
+ if (sentence.includes("?")) continue;
319
+ if (SECOND_PERSON.test(sentence) && !vocabulary.generalizes(sentence)) continue;
320
+ // The same voice test the body passed, applied to the sentence that will
321
+ // be quoted: a bare directive lifted out of a longer message is an
322
+ // instruction for the reader, not a decision about the system.
323
+ const sentenceFirstPerson = FIRST_PERSON.test(sentence);
324
+ if (!sentenceFirstPerson && !vocabulary.namesSubject(sentence)) continue;
325
+ if (!sentenceFirstPerson && vocabulary.talksProcess(sentence)) continue;
326
+ if (
327
+ REPORT_VOICE.test(sentence) &&
328
+ !/^\s*(?:never|always|prefer|must not|by default|from now on)/i.test(sentence)
329
+ ) {
330
+ continue;
331
+ }
332
+ if (HEDGE.test(sentence) && !ABSOLUTE.test(sentence)) continue;
333
+ const quote = quoteFrom(body, spans, index);
334
+ if (!quote) continue;
335
+ return { class: kind, quote };
336
+ }
337
+ return null;
338
+ }
339
+
340
+ // ── Tag derivation ──────────────────────────────────────────────────────────
341
+
342
+ /** How many tags a ruling carries. */
343
+ export const TAG_LIMIT = 3;
344
+
345
+ /** Shortest token a tag may be cut from. */
346
+ const TAG_TOKEN_MIN_LENGTH = 4;
347
+
348
+ /**
349
+ * The share of the population past which a token names nothing. A word a
350
+ * quarter of the store already carries is the store's background, not a topic:
351
+ * it would fire on every query and add nothing a reader could act on.
352
+ */
353
+ const TAG_POPULATION_SHARE = 0.25;
354
+
355
+ /** Tokens a tag may be cut from: letters, then letters/digits/hyphens. */
356
+ const TAG_TOKEN = /[a-z][a-z0-9-]{3,}/g;
357
+
358
+ /**
359
+ * The population a quote is filed into, as the frequency test sees it:
360
+ * how many documents it holds, and how many of them carry a token.
361
+ */
362
+ export interface TagPopulation {
363
+ /** Documents in the population. */
364
+ total: number;
365
+ /** Documents carrying `token`. */
366
+ count(token: string): number;
367
+ }
368
+
369
+ /**
370
+ * The tags a quote earns from the population it is filed into.
371
+ *
372
+ * Take the quote's tokens of four characters or more, drop the read path's stop
373
+ * words, drop any token more than a quarter of the population carries, and keep
374
+ * the three the population carries most, ties by first occurrence.
375
+ *
376
+ * The ranking is by frequency and not by rarity, and the reason is the read
377
+ * path's own arithmetic: a tag's bonus is a flat constant however rare the
378
+ * token is, so rarity buys nothing a later query will say. Two tokens of equal
379
+ * rarity are worth the same as tags, and the one a question is likelier to name
380
+ * is the one the population's own prose uses more often. That makes the
381
+ * frequency ranking the findability ranking, and it leaves the 25% cap as the
382
+ * only guard against a tag that names nothing.
383
+ */
384
+ export function deriveTopicTags(
385
+ text: string,
386
+ population: TagPopulation,
387
+ limit: number = TAG_LIMIT,
388
+ ): string[] {
389
+ const ceiling = population.total * TAG_POPULATION_SHARE;
390
+ const seen = new Set<string>();
391
+ const candidates: Array<{ token: string; count: number; first: number }> = [];
392
+ for (const token of String(text ?? "")
393
+ .toLowerCase()
394
+ .match(TAG_TOKEN) ?? []) {
395
+ if (token.length < TAG_TOKEN_MIN_LENGTH) continue;
396
+ if (QUERY_STOP_WORDS.has(token)) continue;
397
+ if (seen.has(token)) continue;
398
+ seen.add(token);
399
+ const count = population.count(token);
400
+ if (count > ceiling) continue;
401
+ candidates.push({ token, count, first: candidates.length });
402
+ }
403
+ candidates.sort((a, b) => b.count - a.count || a.first - b.first);
404
+ return candidates.slice(0, limit).map((candidate) => candidate.token);
405
+ }
@@ -44,6 +44,16 @@ export {
44
44
  parseStructuredContent,
45
45
  generateAutoTags,
46
46
  } from "./utils.js";
47
+ export {
48
+ classifyRulingBody,
49
+ deriveTopicTags,
50
+ QUOTE_LIMIT,
51
+ TAG_LIMIT,
52
+ type GrammarVocabulary,
53
+ type RulingClass,
54
+ type RulingShape,
55
+ type TagPopulation,
56
+ } from "./decision-grammar.js";
47
57
 
48
58
  // shouldStore, calculateUtility, pruneMemories, consolidate, and the
49
59
  // constants (MAX_ENTRIES, MIN_UTILITY_THRESHOLD, CATEGORY_BASE_UTILITY)