pi-braid 0.43.7 → 0.43.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +39 -0
  2. package/TOOLS.md +1 -1
  3. package/build-info.json +4 -4
  4. package/dist/extensions/core/prompt-builder/tool-guidelines.js +1 -1
  5. package/dist/extensions/core/skills-tool/auto-inject.d.ts +21 -0
  6. package/dist/extensions/core/skills-tool/auto-inject.js +105 -1
  7. package/dist/extensions/core/skills-tool/index.js +20 -2
  8. package/dist/extensions/core/skills-tool/loading-guidance.d.ts +4 -1
  9. package/dist/extensions/core/skills-tool/loading-guidance.js +4 -1
  10. package/dist/extensions/core/skills-tool/session-records.d.ts +107 -9
  11. package/dist/extensions/core/skills-tool/session-records.js +141 -5
  12. package/dist/extensions/enforcement/handlers/analysis/dreamer-handler.js +5 -1
  13. package/dist/extensions/enforcement/handlers/register-tool-call.js +5 -2
  14. package/dist/extensions/enforcement/verification-judge.d.ts +7 -0
  15. package/dist/extensions/enforcement/verification-judge.js +28 -21
  16. package/dist/extensions/memory/session-recall/ask.d.ts +95 -0
  17. package/dist/extensions/memory/session-recall/ask.js +376 -0
  18. package/dist/extensions/memory/session-recall/corpus-index.d.ts +200 -0
  19. package/dist/extensions/memory/session-recall/corpus-index.js +480 -0
  20. package/dist/extensions/memory/session-recall/index.js +173 -21
  21. package/dist/extensions/memory/session-recall/search.d.ts +29 -0
  22. package/dist/extensions/memory/session-recall/search.js +42 -9
  23. package/dist/lib/ai/payload-hygiene.d.ts +3 -0
  24. package/dist/lib/ai/payload-hygiene.js +34 -2
  25. package/dist/lib/config/braid-paths.d.ts +4 -0
  26. package/dist/lib/config/braid-paths.js +6 -0
  27. package/dist/lib/config/braid-settings.d.ts +22 -0
  28. package/dist/lib/config/braid-settings.js +15 -0
  29. package/dist/lib/detection/tool-loop-detector.d.ts +95 -46
  30. package/dist/lib/detection/tool-loop-detector.js +288 -180
  31. package/dist/lib/reasoning/dreamer.d.ts +2 -1
  32. package/dist/lib/reasoning/dreamer.js +12 -7
  33. package/docs/LOCAL_DEV.md +1 -1
  34. package/docs/SETUP.md +1 -0
  35. package/package.json +1 -1
  36. package/src/extensions/core/AGENTS.md +3 -2
  37. package/src/extensions/core/prompt-builder/tool-guidelines.ts +1 -1
  38. package/src/extensions/core/skills-tool/auto-inject.ts +118 -1
  39. package/src/extensions/core/skills-tool/index.ts +21 -1
  40. package/src/extensions/core/skills-tool/loading-guidance.ts +4 -1
  41. package/src/extensions/core/skills-tool/session-records.ts +201 -9
  42. package/src/extensions/enforcement/handlers/analysis/dreamer-handler.ts +5 -1
  43. package/src/extensions/enforcement/handlers/register-tool-call.ts +5 -2
  44. package/src/extensions/enforcement/verification-judge.ts +32 -25
  45. package/src/extensions/memory/AGENTS.md +2 -2
  46. package/src/extensions/memory/session-recall/AGENTS.md +92 -18
  47. package/src/extensions/memory/session-recall/ask.ts +435 -0
  48. package/src/extensions/memory/session-recall/corpus-index.ts +609 -0
  49. package/src/extensions/memory/session-recall/index.ts +202 -21
  50. package/src/extensions/memory/session-recall/search.ts +61 -10
  51. package/src/lib/ai/payload-hygiene.ts +47 -3
  52. package/src/lib/config/braid-paths.ts +7 -0
  53. package/src/lib/config/braid-settings.ts +42 -3
  54. package/src/lib/detection/AGENTS.md +8 -7
  55. package/src/lib/detection/tool-loop-detector.ts +339 -195
  56. package/src/lib/reasoning/dreamer.ts +13 -6
@@ -126,6 +126,13 @@ export interface JudgeContext {
126
126
 
127
127
  /**
128
128
  * Build the verification analysis prompt. Exported for tests.
129
+ *
130
+ * Order is load-bearing: the ground-truth sections (git history, the
131
+ * disk-checked deliverables manifest, the code-changes flag) come BEFORE the
132
+ * bounded ones (task evidence, tool log, reasoning, response). The bounded
133
+ * sections are the ones a bound can shorten: a head kept under a size cap, a
134
+ * tail cut by a redaction. A reader that meets the verdict-bearing sections
135
+ * first cannot have them removed by whatever truncated the evidence.
129
136
  */
130
137
  export function buildVerificationPrompt(context: JudgeContext): string {
131
138
  const sections: string[] = [];
@@ -138,31 +145,6 @@ export function buildVerificationPrompt(context: JudgeContext): string {
138
145
  sections.push(context.toolsUsed.length > 0 ? context.toolsUsed.join(", ") : "(none)");
139
146
  sections.push("");
140
147
 
141
- // Task-wide evidence (since last user message, session-store
142
- // backed) comes BEFORE the current-window tool log — it is the surviving
143
- // view when turn breaks cut toolLog down to the tail turn.
144
- if (context.taskEvidence && context.taskEvidence.length > 0) {
145
- sections.push("## TASK EVIDENCE (since last user message, disk/session-surviving)");
146
- sections.push(context.taskEvidence.join("\n"));
147
- sections.push("");
148
- }
149
-
150
- if (context.toolLog.length > 0) {
151
- sections.push("## TOOL EXECUTION LOG");
152
- sections.push(context.toolLog.slice(-50).join("\n")); // Last 50 entries (matches turnHistory cap)
153
- sections.push("");
154
- }
155
-
156
- sections.push(`## CODE CHANGES: ${context.hasCodeChanges ? "YES" : "NO"}`);
157
- // (Verification evidence comes from the TOOL EXECUTION LOG above, not a boolean)
158
- sections.push("");
159
-
160
- if (context.thinking) {
161
- sections.push("## AGENT REASONING (thinking blocks)");
162
- sections.push(context.thinking);
163
- sections.push("");
164
- }
165
-
166
148
  if (context.gitHistory) {
167
149
  sections.push("## GIT HISTORY (recent commits)");
168
150
  sections.push(context.gitHistory);
@@ -188,6 +170,31 @@ export function buildVerificationPrompt(context: JudgeContext): string {
188
170
  sections.push("");
189
171
  }
190
172
 
173
+ sections.push(`## CODE CHANGES: ${context.hasCodeChanges ? "YES" : "NO"}`);
174
+ // (Verification evidence comes from the TOOL EXECUTION LOG, not a boolean)
175
+ sections.push("");
176
+
177
+ // Task-wide evidence (since last user message, session-store backed) comes
178
+ // BEFORE the current-window tool log — it is the surviving view when turn
179
+ // breaks cut toolLog down to the tail turn.
180
+ if (context.taskEvidence && context.taskEvidence.length > 0) {
181
+ sections.push("## TASK EVIDENCE (since last user message, disk/session-surviving)");
182
+ sections.push(context.taskEvidence.join("\n"));
183
+ sections.push("");
184
+ }
185
+
186
+ if (context.toolLog.length > 0) {
187
+ sections.push("## TOOL EXECUTION LOG");
188
+ sections.push(context.toolLog.slice(-50).join("\n")); // Last 50 entries (matches turnHistory cap)
189
+ sections.push("");
190
+ }
191
+
192
+ if (context.thinking) {
193
+ sections.push("## AGENT REASONING (thinking blocks)");
194
+ sections.push(context.thinking);
195
+ sections.push("");
196
+ }
197
+
191
198
  if (context.lastResponse) {
192
199
  sections.push("## AGENT RESPONSE (visible text)");
193
200
  sections.push(context.lastResponse);
@@ -117,8 +117,8 @@ The memory extensions expose 4 LLM-callable tools. The unified `memory` tool cov
117
117
  | -------------- | ------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
118
118
  | `memory_store` | memory-agent-bridge | semantic (unified) | Save insights/patterns for cross-session recall |
119
119
  | `memory` | memory-agent-bridge | all three | Unified access: `recall_search` (semantic search, also cross-searches episodes), `recall_list`, `recall_get`, `episodes_search` (case-based reasoning), `episodes_list`, `episodes_add`, `heuristics_list`, `heuristics_add`, `heuristics_evolve` |
120
- | `session` | session-recall | session log | Search past sessions (action=search) or query one (action=query) |
120
+ | `session` | session-recall | session log | Search past sessions (action=search), ask what the owner said earlier with citations (action=ask), or query one (action=query) |
121
121
 
122
- **Rule of thumb:** start with `memory` action=recall_search for "have I seen this before?". Use `memory` action=episodes_search for "how did a similar task go?". Use `memory` action=heuristics_list when investigating behavior tuning. Use `session` for cross-session history.
122
+ **Rule of thumb:** start with `memory` action=recall_search for "have I seen this before?". Use `memory` action=episodes_search for "how did a similar task go?". Use `memory` action=heuristics_list when investigating behavior tuning. Use `session` for cross-session history, and `session` action=ask when the question is "what did the owner say or decide about X?" - it reads the indexed user turns and cites each hit.
123
123
 
124
124
  The data-layer functions (`addEpisode`, `addPrinciple`, `recordToolCall`, etc.) live in `three-memory-stores/episodes.ts`, `heuristics.ts`, `tool-reliability.ts` and are imported by many extensions (e.g. `finalize.ts` adds episodes on task completion). These are infrastructure and should not be moved out of the directory.
@@ -2,26 +2,43 @@
2
2
 
3
3
  # Session Recall Extension
4
4
 
5
- Search and query past sessions. The agent can call this to find prior
6
- work without leaving the current session.
5
+ Search, ask over, and query past sessions. The agent can call this to find
6
+ prior work - or to find what the owner said - without leaving the current
7
+ session.
7
8
 
8
9
  ## Architecture
9
10
 
10
- This file (`index.ts`) is the extension entry point. Internal
11
- helpers are co-located (no sub-modules) because the surface area is
12
- small (one tool, one event handler).
11
+ `index.ts` is the extension entry point and the tool surface. Helpers are
12
+ co-located by role, not extracted further:
13
+
14
+ | File | Owns |
15
+ | ---------------------- | ------------------------------------------------------------------------- |
16
+ | `search.ts` | live file discovery, the rg/grep/node backends, line hits, project labels |
17
+ | `rank.ts` | the four-signal file ranking used by `search` |
18
+ | `query.ts` | one session's transcript for the LLM paths |
19
+ | `corpus-index.ts` | the user-turn index: extraction, incremental build, fail-closed load |
20
+ | `ask.ts` | BM25 retrieval over that index and the cited answer's rendering |
21
+ | `analyze.ts` | multi-session LLM analysis |
22
+ | `context-aggregate.ts` | the `context` action's aggregation |
23
+ | `stats.ts` | session-file statistics |
13
24
 
14
25
  ## Tools exposed
15
26
 
16
27
  | Tool | Actions | Purpose |
17
28
  | --------- | ------- | -------------------------------------------------------------------- |
18
29
  | `session` | search | Full-text search across session history |
30
+ | `session` | ask | BM25 over the user-turn index, answered with citations |
19
31
  | `session` | query | Structured query (by date, by tool, by tag) |
20
32
  | `session` | analyze | Multi-session LLM analysis over a session list |
21
33
  | `session` | batch | Pattern aggregation across sessions (by project/month) |
22
34
  | `session` | stats | Session-file statistics (count, size, by project/month) |
23
35
  | `session` | context | Cross-session §8 context-adoption aggregate (loss/coverage/adoption) |
24
36
 
37
+ `search`, `batch` and `ask` never return the calling session's own transcript
38
+ (`search.ts: excludeSessionPath`), and none of them reports a bare match count
39
+ as its result: `search` and `batch` render the matched lines with their line
40
+ numbers, ranked and capped; `ask` renders cited turns.
41
+
25
42
  ## `context` action
26
43
 
27
44
  Repeatable cross-session aggregation of the §8 context-management counters -
@@ -43,6 +60,58 @@ Signals are matched to the durable session-file markers:
43
60
  pure aggregation lives in `context-aggregate.ts` (testable against fixture
44
61
  session files); `search.ts` shares the file-discovery helpers.
45
62
 
63
+ ## The corpus index and `ask`
64
+
65
+ The transcript corpus holds gigabytes of agent reasoning and tool output, and
66
+ under 10 MB of the owner's own words. `ask` reads that fraction as a
67
+ population: a derived store of one row per user turn, written by
68
+ `scripts/build-corpus-index.mjs` and queried by `askCorpusIndex`.
69
+
70
+ - **Location**: `<braid root>/corpus-index/`, one shard per project
71
+ (`corpusIndexDir()` in `lib/config/braid-paths.ts`; the shard's file-name
72
+ stem is `shardSlug`, one rule for the writer and the reader). Derived data:
73
+ deleting the directory costs a rebuild, nothing else.
74
+ - **Row**: `{session, session_id, project, ts, line, text}`, plus the shard's
75
+ first line, a manifest carrying format version, project, build time, row
76
+ count, text bytes and the `(path, mtime, size, turns)` of every source file.
77
+ - **Content**: `message` entries with role `user` and text. Tool results
78
+ arrive as user-role messages with no text block and drop out with the text
79
+ filter; assistant turns, `custom_message` injections and durable `custom`
80
+ entries never enter a row, so no key, tool argument or environment value is
81
+ retrievable from the index.
82
+ - **Compaction archives are not indexed**: an `archives/*.jsonl` envelope is a
83
+ second copy of messages the session body carries (`writeOriginalsArchive`
84
+ stores the pre-compaction originals), so indexing both would count the same
85
+ words twice.
86
+ - **Build**: one pass over the session files on a full build; later runs re-parse
87
+ only the files whose `(mtime, size)` moved. A shard that cannot be read is
88
+ rebuilt, never merged - derived data has one authority.
89
+ - **Fail closed**: `loadCorpusIndex` returns `{ok: false, reason}` naming the
90
+ shard and the defect when a manifest does not parse, the format is unknown,
91
+ a row does not parse, or the row count disagrees with the manifest. `ask`
92
+ then says it cannot answer and prints the build command; it never answers
93
+ from half a store.
94
+ - **Scoring**: BM25 (`k1 = 1.2`, `b = 0.75`) over one document per turn, with a
95
+ fixed English function-word list and a symmetric suffix fold, so `renders`
96
+ reaches `rendered` and a hyphen separates words. No embeddings, no vector
97
+ store: at this size the lexical model is cheap, testable and re-derivable by
98
+ eye. What would earn one is a measured recall@3 below the bar on questions
99
+ whose answering passage is in the index.
100
+ - **Freshness**: an answer states the population it read and how many session
101
+ files moved since the build (`corpusIndexFreshness`), so a stale shard is
102
+ visible rather than silent.
103
+
104
+ Rebuild after a day of sessions:
105
+
106
+ ```bash
107
+ npm run build && node packages/core/scripts/build-corpus-index.mjs
108
+ ```
109
+
110
+ Recall over a pre-registered question set is measured on the live corpus, not in
111
+ the suite: `test/manual/ask-corpus-recall.mjs --questions <file.json>` scores
112
+ whether the answering turn appears in the top 3 and prints the rank of every
113
+ expected turn, including the misses.
114
+
46
115
  ## What it indexes
47
116
 
48
117
  - Session title (if set)
@@ -78,22 +147,27 @@ full-corpus backend invocation and an N-term query N.
78
147
 
79
148
  ## Storage
80
149
 
81
- Sessions are stored as JSONL in `BRAID_DATA_DIR/sessions/`. Each
82
- session is a single file. The index is built on first search (no
83
- precomputed index - sessions are small enough that linear scan is
84
- fast enough for the typical use case).
150
+ Sessions are stored as JSONL under pi's session root (one encoded-cwd directory
151
+ per project). `search` and `batch` scan those files live with the rg/grep/node
152
+ backends; `ask` reads the precomputed corpus index described above. The
153
+ extension writes no session file - the index is the one artifact it writes, and
154
+ it is written only by the build script.
85
155
 
86
156
  ## Performance
87
157
 
88
- The corpus measured for this change is ~3,250 session files / 2.3 GB. A
89
- single-term search is one full-corpus pass plus the bounded refinements: ~4.5 s
90
- on the dev machine against that corpus, down from ~10.6 s when the role signal
91
- cost a second full pass. A multi-term query is one pass per term (~8.9 s for
92
- two, down from ~19.3 s). `--max-filesize` caps any one file at 32 MB, so a
93
- runaway session cannot dominate a scan; the five files above the cap (0.15% of
94
- the corpus, all larger than 32 MB) are not searched. A precomputed inverted
95
- index remains future work - this change removes the duplicate pass and bounds
96
- the refinement, it does not make the scan itself cheap.
158
+ The corpus measured when the search cost work landed was ~3,250 session files /
159
+ 2.3 GB. A single-term search is one full-corpus pass plus the bounded
160
+ refinements: ~4.5 s on the dev machine against that corpus, down from ~10.6 s
161
+ when the role signal cost a second full pass. A multi-term query is one pass per
162
+ term (~8.9 s for two, down from ~19.3 s). `--max-filesize` caps any one file at
163
+ 32 MB, so a runaway session cannot dominate a scan; the files above the cap are
164
+ not searched.
165
+
166
+ The corpus measured for the corpus index is 3,832 session files / 3.8 GB. A full
167
+ index build parses every file in 20.6 s and writes 20,477 turns (9.66 MB) across
168
+ 341 project shards; an incremental build re-parsing today's changed sessions
169
+ reports 1.0 s of work. An `ask` pays 0.2 s to load the shards, 0.2 s for the
170
+ freshness walk, and ~0.5 s to tokenize and score the population.
97
171
 
98
172
  ## Local Contracts
99
173
 
@@ -0,0 +1,435 @@
1
+ /**
2
+ * Session corpus ask - lexical retrieval over the user-turn index.
3
+ *
4
+ * BM25 over one document per user turn. At this corpus size the lexical model
5
+ * is the cheap, testable choice: no embedding pass to keep current, no vector
6
+ * store to back up, no key to spend on a query, and a score a reader can
7
+ * re-derive by eye from the terms it names. What would earn a vector store is
8
+ * a measured failure: recall@3 below the bar on questions whose answering
9
+ * passage is in the index (the report's falsifier is that measurement).
10
+ *
11
+ * Every hit carries its citation - project, session path, line, timestamp -
12
+ * because the answer is the owner's own words and a reader must be able to
13
+ * check them against the session file. The refusal is a first-class result: a
14
+ * query whose terms appear nowhere in the population says so and names what it
15
+ * searched, rather than returning the least bad rows.
16
+ */
17
+
18
+ import type { CorpusFreshness, CorpusTurn, LoadedShard } from "./corpus-index.js";
19
+
20
+ /** BM25 term-frequency saturation. */
21
+ export const BM25_K1 = 1.2;
22
+ /** BM25 length normalisation. */
23
+ export const BM25_B = 0.75;
24
+ /** Hits a query returns when the caller names no limit. */
25
+ export const MAX_ASK_HITS = 5;
26
+ /** Characters of a turn the answer prints before it trims. */
27
+ const PASSAGE_CHARS = 360;
28
+
29
+ /**
30
+ * Words that carry no retrieval signal in a question about work. A fixed
31
+ * language table, not a project vocabulary: it holds English function words
32
+ * only, so a query term that names something always survives it.
33
+ */
34
+ const STOPWORDS = new Set([
35
+ "a",
36
+ "about",
37
+ "after",
38
+ "all",
39
+ "also",
40
+ "am",
41
+ "an",
42
+ "and",
43
+ "any",
44
+ "are",
45
+ "as",
46
+ "at",
47
+ "be",
48
+ "been",
49
+ "before",
50
+ "being",
51
+ "both",
52
+ "but",
53
+ "by",
54
+ "can",
55
+ "could",
56
+ "did",
57
+ "do",
58
+ "does",
59
+ "doing",
60
+ "done",
61
+ "for",
62
+ "from",
63
+ "had",
64
+ "has",
65
+ "have",
66
+ "having",
67
+ "he",
68
+ "her",
69
+ "here",
70
+ "him",
71
+ "his",
72
+ "how",
73
+ "i",
74
+ "if",
75
+ "in",
76
+ "into",
77
+ "is",
78
+ "it",
79
+ "its",
80
+ "just",
81
+ "me",
82
+ "my",
83
+ "no",
84
+ "not",
85
+ "of",
86
+ "on",
87
+ "once",
88
+ "only",
89
+ "or",
90
+ "other",
91
+ "our",
92
+ "out",
93
+ "over",
94
+ "own",
95
+ "same",
96
+ "she",
97
+ "should",
98
+ "so",
99
+ "some",
100
+ "such",
101
+ "than",
102
+ "that",
103
+ "the",
104
+ "their",
105
+ "them",
106
+ "then",
107
+ "there",
108
+ "these",
109
+ "they",
110
+ "this",
111
+ "those",
112
+ "to",
113
+ "too",
114
+ "under",
115
+ "up",
116
+ "us",
117
+ "very",
118
+ "was",
119
+ "we",
120
+ "were",
121
+ "what",
122
+ "when",
123
+ "where",
124
+ "which",
125
+ "while",
126
+ "who",
127
+ "why",
128
+ "will",
129
+ "with",
130
+ "would",
131
+ "you",
132
+ "your",
133
+ ]);
134
+
135
+ /**
136
+ * Fold one token to its retrieval stem.
137
+ *
138
+ * Inflected forms of a word are one word for retrieval: a question saying
139
+ * "rendered" must reach a turn saying "renders", and "picks" must reach
140
+ * "pick". The fold is symmetric - the query and the turns pass through the
141
+ * same function - so a fold that is linguistically wrong ("boxes" -> "boxe")
142
+ * is still a fold both sides agree on. The doubled-consonant rule exists for
143
+ * the same reason: "hitting" must reach "hit".
144
+ */
145
+ function foldToken(token: string): string {
146
+ if (token.length <= 3) return token;
147
+ let stem = token;
148
+ if (stem.endsWith("ies") && stem.length > 4) stem = `${stem.slice(0, -3)}y`;
149
+ else if (stem.endsWith("s") && !stem.endsWith("ss")) stem = stem.slice(0, -1);
150
+ if (stem.endsWith("ing") && stem.length > 5) stem = stem.slice(0, -3);
151
+ else if (stem.endsWith("ed") && stem.length > 4) stem = stem.slice(0, -2);
152
+ const last = stem.at(-1);
153
+ if (/[bdfglmnprt]/.test(last ?? "") && stem.at(-2) === last && stem.length > 3) {
154
+ stem = stem.slice(0, -1);
155
+ }
156
+ return stem;
157
+ }
158
+
159
+ /**
160
+ * Words a document is scored on: letter/number runs, lowercased, stemmed.
161
+ *
162
+ * A hyphen separates words: the owner's compound names ("looping-clanker",
163
+ * "anti-crowd") are two words a question may spell either way, and a tokenizer
164
+ * that keeps the compound as one token can never match its own parts.
165
+ */
166
+ export function tokenizeForIndex(text: string): string[] {
167
+ const raw = text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
168
+ const tokens: string[] = [];
169
+ for (const token of raw) {
170
+ if (token.length < 2 && !/\d/.test(token)) continue;
171
+ tokens.push(foldToken(token));
172
+ }
173
+ return tokens;
174
+ }
175
+
176
+ /**
177
+ * Query terms: the document tokens minus function words. Duplicates collapse,
178
+ * so a word typed twice scores once.
179
+ */
180
+ export function askQueryTerms(query: string): string[] {
181
+ const seen = new Set<string>();
182
+ for (const token of tokenizeForIndex(query)) {
183
+ if (STOPWORDS.has(token)) continue;
184
+ seen.add(token);
185
+ }
186
+ return [...seen];
187
+ }
188
+
189
+ export interface AskHit {
190
+ turn: CorpusTurn;
191
+ score: number;
192
+ /** Query terms this turn carries. */
193
+ matched: string[];
194
+ }
195
+
196
+ export interface AskResult {
197
+ query: string;
198
+ /** Query terms after the function-word filter. */
199
+ terms: string[];
200
+ /** Terms that occur in the searched population. */
201
+ matched_terms: string[];
202
+ /** Terms that occur nowhere in the searched population. */
203
+ unmatched_terms: string[];
204
+ hits: AskHit[];
205
+ /** Whether the index could answer at all (at least one hit). */
206
+ answered: boolean;
207
+ population: {
208
+ turns: number;
209
+ sessions: number;
210
+ projects: string[];
211
+ /** Oldest and newest shard build time in the population. */
212
+ built_oldest: string;
213
+ built_newest: string;
214
+ shards: number;
215
+ };
216
+ freshness: CorpusFreshness;
217
+ filters: { project: string | null; since: string | null };
218
+ /** True when the caller's own session was withheld from the results. */
219
+ self_excluded: boolean;
220
+ }
221
+
222
+ export interface AskOptions {
223
+ /** Project label filter; null reads every shard the store loaded. */
224
+ project?: string | null;
225
+ /** Inclusive lower bound on the turn's date (YYYY-MM-DD). */
226
+ since?: string | null;
227
+ limit?: number;
228
+ /** Session file never returned to itself. */
229
+ excludeSession?: string | null;
230
+ freshness: CorpusFreshness;
231
+ }
232
+
233
+ /** BM25 score of one document against the query terms. */
234
+ function scoreDocument(
235
+ termFrequencies: Map<string, number>,
236
+ docLength: number,
237
+ averageLength: number,
238
+ idf: Map<string, number>,
239
+ ): { score: number; matched: string[] } {
240
+ const lengthNorm = averageLength > 0 ? 1 - BM25_B + (BM25_B * docLength) / averageLength : 1;
241
+ let score = 0;
242
+ const matched: string[] = [];
243
+ for (const [term, frequency] of termFrequencies) {
244
+ const weight = idf.get(term);
245
+ if (weight === undefined) continue;
246
+ matched.push(term);
247
+ score += (weight * (frequency * (BM25_K1 + 1))) / (frequency + BM25_K1 * lengthNorm);
248
+ }
249
+ return { score, matched };
250
+ }
251
+
252
+ /**
253
+ * Retrieve the turns that answer a question, with citations.
254
+ *
255
+ * The population searched is stated in the result, so a refusal can show what
256
+ * it looked at. `df` and the average document length are computed over the
257
+ * filtered population: an index narrowed by `--project` or `--since` is the
258
+ * corpus the answer claims, not a sample of a larger one it never read.
259
+ */
260
+ export function askCorpusIndex(shards: LoadedShard[], query: string, opts: AskOptions): AskResult {
261
+ const terms = askQueryTerms(query);
262
+ const allowed = new Set(terms);
263
+ const candidates: CorpusTurn[] = [];
264
+ const projects = new Set<string>();
265
+ const builtAt = new Set<string>();
266
+ for (const shard of shards) {
267
+ projects.add(shard.manifest.project);
268
+ builtAt.add(shard.manifest.built_at);
269
+ for (const turn of shard.turns) {
270
+ if (opts.excludeSession && turn.session === opts.excludeSession) continue;
271
+ if (opts.since && turn.ts.slice(0, 10) < opts.since) continue;
272
+ candidates.push(turn);
273
+ }
274
+ }
275
+
276
+ const sessions = new Set<string>();
277
+ for (const turn of candidates) sessions.add(turn.session);
278
+
279
+ // One pass: term frequencies for the query terms, and the document's length.
280
+ const frequencies: Array<Map<string, number>> = new Array(candidates.length);
281
+ const lengths: number[] = new Array(candidates.length);
282
+ const documentFrequency = new Map<string, number>();
283
+ let totalLength = 0;
284
+ for (let i = 0; i < candidates.length; i++) {
285
+ const tokens = tokenizeForIndex((candidates[i] as CorpusTurn).text);
286
+ lengths[i] = tokens.length;
287
+ totalLength += tokens.length;
288
+ const counts = new Map<string, number>();
289
+ for (const token of tokens) {
290
+ if (!allowed.has(token)) continue;
291
+ counts.set(token, (counts.get(token) ?? 0) + 1);
292
+ }
293
+ frequencies[i] = counts;
294
+ for (const term of counts.keys()) {
295
+ documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1);
296
+ }
297
+ }
298
+
299
+ const total = candidates.length;
300
+ const averageLength = total > 0 ? totalLength / total : 0;
301
+ const idf = new Map<string, number>();
302
+ for (const [term, df] of documentFrequency) {
303
+ idf.set(term, Math.log(1 + (total - df + 0.5) / (df + 0.5)));
304
+ }
305
+
306
+ const limit = Math.max(1, opts.limit ?? MAX_ASK_HITS);
307
+ const scored: AskHit[] = [];
308
+ for (let i = 0; i < candidates.length; i++) {
309
+ const counts = frequencies[i];
310
+ const turn = candidates[i];
311
+ if (!counts || !turn || counts.size === 0) continue;
312
+ const { score, matched } = scoreDocument(counts, lengths[i] ?? 0, averageLength, idf);
313
+ if (score <= 0) continue;
314
+ scored.push({ turn, score, matched });
315
+ }
316
+ scored.sort(
317
+ (a, b) =>
318
+ b.score - a.score ||
319
+ b.turn.ts.localeCompare(a.turn.ts) ||
320
+ a.turn.session.localeCompare(b.turn.session) ||
321
+ a.turn.line - b.turn.line,
322
+ );
323
+
324
+ const matchedTerms = terms.filter((term) => (documentFrequency.get(term) ?? 0) > 0);
325
+ return {
326
+ query,
327
+ terms,
328
+ matched_terms: matchedTerms,
329
+ unmatched_terms: terms.filter((term) => !matchedTerms.includes(term)),
330
+ hits: scored.slice(0, limit),
331
+ answered: scored.length > 0,
332
+ population: {
333
+ turns: total,
334
+ sessions: sessions.size,
335
+ projects: [...projects].sort(),
336
+ built_oldest: [...builtAt].sort()[0] ?? "",
337
+ built_newest: [...builtAt].sort().at(-1) ?? "",
338
+ shards: shards.length,
339
+ },
340
+ freshness: opts.freshness,
341
+ filters: { project: opts.project ?? null, since: opts.since ?? null },
342
+ self_excluded: Boolean(opts.excludeSession),
343
+ };
344
+ }
345
+
346
+ /** One passage, trimmed around the best-matching term. */
347
+ function passage(text: string, matched: string[]): string {
348
+ const flat = text.replace(/\s+/g, " ").trim();
349
+ if (flat.length <= PASSAGE_CHARS) return flat;
350
+ const anchor = matched.find((term) => flat.toLowerCase().includes(term)) ?? "";
351
+ const at = anchor ? flat.toLowerCase().indexOf(anchor) : -1;
352
+ if (at < 0) return `${flat.slice(0, PASSAGE_CHARS)}...`;
353
+ const start = Math.max(0, at - Math.floor(PASSAGE_CHARS / 3));
354
+ const end = Math.min(flat.length, start + PASSAGE_CHARS);
355
+ return `${start > 0 ? "..." : ""}${flat.slice(start, end)}${end < flat.length ? "..." : ""}`;
356
+ }
357
+
358
+ /** Freshness in one phrase, or null when the index is current. */
359
+ function freshnessNote(freshness: CorpusFreshness): string | null {
360
+ const parts: string[] = [];
361
+ if (freshness.changed > 0) parts.push(`${freshness.changed} changed`);
362
+ if (freshness.added > 0) parts.push(`${freshness.added} new`);
363
+ if (freshness.removed > 0) parts.push(`${freshness.removed} gone`);
364
+ if (parts.length === 0) return null;
365
+ const newest = freshness.newest_change ? `, newest ${freshness.newest_change}` : "";
366
+ return `${parts.join(", ")} session file(s) since the build${newest}; rerun the index build for them`;
367
+ }
368
+
369
+ function number(value: number): string {
370
+ return value.toLocaleString("en-US");
371
+ }
372
+
373
+ /**
374
+ * Render the answer: cited passages, or a refusal that names the population it
375
+ * searched and the terms it found.
376
+ */
377
+ export function formatAskResult(result: AskResult): string {
378
+ const population =
379
+ `${number(result.population.turns)} user turns from ` +
380
+ `${number(result.population.sessions)} sessions across ` +
381
+ `${result.population.projects.length} project(s)`;
382
+ const built =
383
+ result.population.built_oldest === result.population.built_newest
384
+ ? `built ${result.population.built_newest}`
385
+ : `shards built ${result.population.built_oldest} to ${result.population.built_newest}`;
386
+ const note = freshnessNote(result.freshness);
387
+ const header = [
388
+ `**Ask:** ${result.query}`,
389
+ `Index: ${population}; ${built}.${note ? `\nStaleness: ${note}.` : ""}`,
390
+ result.filters.project || result.filters.since
391
+ ? `Filters: project=${result.filters.project ?? "all"}, since=${result.filters.since ?? "none"}`
392
+ : "",
393
+ result.self_excluded ? "Withheld: this session's own turns." : "",
394
+ ].filter(Boolean);
395
+
396
+ if (!result.answered) {
397
+ const found =
398
+ result.matched_terms.length > 0
399
+ ? `terms found elsewhere in the population: ${result.matched_terms.join(", ")}`
400
+ : "none of the query terms appear in the population";
401
+ const missing =
402
+ result.unmatched_terms.length > 0
403
+ ? `terms not found: ${result.unmatched_terms.join(", ")}`
404
+ : "every term was found, but no turn carries them together";
405
+ return [
406
+ ...header,
407
+ "",
408
+ "**The index cannot answer this.**",
409
+ `Searched ${population}.`,
410
+ `Result: ${found}; ${missing}.`,
411
+ "Widen the words used, drop the project filter, or rebuild the index if the sessions are newer than the build.",
412
+ ].join("\n");
413
+ }
414
+
415
+ const blocks = result.hits.map((hit, index) => {
416
+ const where =
417
+ `**${index + 1}.** ${hit.turn.project} - ${hit.turn.ts} - line ${hit.turn.line}\n` +
418
+ `Session: \`${hit.turn.session}\``;
419
+ return `${where}\n> ${passage(hit.turn.text, hit.matched)}`;
420
+ });
421
+ const lines = [
422
+ ...header,
423
+ "",
424
+ `${result.hits.length} turn(s) carry the query, ranked by BM25:`,
425
+ "",
426
+ blocks.join("\n\n"),
427
+ ];
428
+ if (result.unmatched_terms.length > 0) {
429
+ lines.push(
430
+ "",
431
+ `Terms found in no turn (the lexical index cannot see them): ${result.unmatched_terms.join(", ")}.`,
432
+ );
433
+ }
434
+ return lines.join("\n");
435
+ }