pi-blackhole 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/index.js +11660 -0
- package/dist/index.js.map +1 -0
- package/example-config.json +1 -1
- package/index.ts +37 -63
- package/package.json +21 -9
- package/src/commands/cleanup.ts +279 -240
- package/src/commands/memory.ts +236 -184
- package/src/commands/pi-vcc.ts +202 -152
- package/src/commands/vcc-recall.ts +126 -95
- package/src/core/brief.ts +167 -33
- package/src/core/build-sections.ts +8 -2
- package/src/core/config-env.ts +117 -0
- package/src/core/content.ts +31 -7
- package/src/core/drill-down.ts +41 -11
- package/src/core/filter-noise.ts +9 -3
- package/src/core/format-recall.ts +15 -6
- package/src/core/format.ts +14 -4
- package/src/core/lineage.ts +9 -3
- package/src/core/load-messages.ts +24 -5
- package/src/core/normalize.ts +38 -14
- package/src/core/recall-scope.ts +11 -3
- package/src/core/render-entries.ts +22 -6
- package/src/core/sanitize.ts +5 -1
- package/src/core/search-entries.ts +111 -19
- package/src/core/settings.ts +1 -3
- package/src/core/summarize.ts +42 -21
- package/src/core/unified-config.ts +549 -411
- package/src/extract/commits.ts +4 -2
- package/src/extract/files.ts +10 -5
- package/src/extract/goals.ts +7 -2
- package/src/hooks/before-compact.ts +210 -88
- package/src/om/agents/dropper/agent.ts +380 -265
- package/src/om/agents/dropper/coverage.ts +102 -82
- package/src/om/agents/observer/agent.ts +242 -206
- package/src/om/agents/reflector/agent.ts +212 -153
- package/src/om/cleanup.ts +239 -218
- package/src/om/clipboard.ts +59 -51
- package/src/om/compaction-trigger.ts +448 -333
- package/src/om/config.ts +13 -6
- package/src/om/configure-overlay.ts +518 -355
- package/src/om/consolidation.ts +1460 -953
- package/src/om/cooldown.ts +75 -65
- package/src/om/debug-log.ts +86 -68
- package/src/om/ids.ts +1 -1
- package/src/om/ledger/fold.ts +89 -78
- package/src/om/ledger/progress.ts +181 -153
- package/src/om/ledger/projection.ts +248 -185
- package/src/om/ledger/recall.ts +247 -196
- package/src/om/ledger/render-summary.ts +79 -50
- package/src/om/ledger/types.ts +146 -117
- package/src/om/model-budget.ts +23 -13
- package/src/om/pending.ts +243 -179
- package/src/om/provider-stream.ts +52 -7
- package/src/om/retryable-error.ts +12 -16
- package/src/om/reverse-recall.ts +97 -91
- package/src/om/runtime.ts +474 -375
- package/src/om/serialize.ts +190 -166
- package/src/om/status-overlay.ts +246 -195
- package/src/om/tokens.ts +28 -21
- package/src/pi-base/blackhole-settings.ts +437 -0
- package/src/pi-base/config-manager.ts +440 -0
- package/src/pi-base/config.ts +469 -0
- package/src/pi-base/env.ts +43 -0
- package/src/pi-base/paths.ts +47 -0
- package/src/pi-base/settings/body.ts +1648 -0
- package/src/pi-base/settings/fields/action.ts +43 -0
- package/src/pi-base/settings/fields/boolean.ts +47 -0
- package/src/pi-base/settings/fields/custom.ts +72 -0
- package/src/pi-base/settings/fields/enum.ts +310 -0
- package/src/pi-base/settings/fields/index.ts +46 -0
- package/src/pi-base/settings/fields/model.ts +452 -0
- package/src/pi-base/settings/fields/string.ts +527 -0
- package/src/pi-base/settings/fields/text.ts +115 -0
- package/src/pi-base/settings/frame.ts +197 -0
- package/src/pi-base/settings/index.ts +77 -0
- package/src/pi-base/settings/inline-edit.ts +313 -0
- package/src/pi-base/settings/modal.ts +152 -0
- package/src/pi-base/settings/types.ts +500 -0
- package/src/pi-base/settings/validate-field.ts +113 -0
- package/src/pi-base/shell.ts +117 -0
- package/src/pi-base/types.ts +6 -0
- package/src/pi-base/ui.ts +32 -0
- package/src/tools/recall.ts +347 -225
- package/src/types.ts +20 -3
- package/tsup.config.ts +23 -0
- package/vitest.config.ts +15 -15
package/src/core/format.ts
CHANGED
|
@@ -17,16 +17,26 @@ const TUI_SAFE_LINE_CHARS = 120;
|
|
|
17
17
|
*/
|
|
18
18
|
function wrapLineWithContinuation(line: string, maxChars: number): string[] {
|
|
19
19
|
const indent = line.match(/^\s*(?:[-*]\s+|\d+\.\s+)?/)?.[0] ?? "";
|
|
20
|
-
const continuationIndent = indent
|
|
20
|
+
const continuationIndent = indent
|
|
21
|
+
? " ".repeat(Math.min(indent.length, 8))
|
|
22
|
+
: "";
|
|
21
23
|
// Wrap at reduced width so prepending continuationIndent doesn't exceed maxChars
|
|
22
|
-
const safeMaxChars = continuationIndent
|
|
24
|
+
const safeMaxChars = continuationIndent
|
|
25
|
+
? maxChars - continuationIndent.length
|
|
26
|
+
: maxChars;
|
|
23
27
|
const wrapped = wrapTextWithAnsi(line, safeMaxChars);
|
|
24
28
|
if (wrapped.length <= 1 || !continuationIndent) return wrapped;
|
|
25
29
|
return [wrapped[0], ...wrapped.slice(1).map((l) => continuationIndent + l)];
|
|
26
30
|
}
|
|
27
31
|
|
|
28
|
-
export const wrapLongLines = (
|
|
29
|
-
text
|
|
32
|
+
export const wrapLongLines = (
|
|
33
|
+
text: string,
|
|
34
|
+
maxChars = TUI_SAFE_LINE_CHARS,
|
|
35
|
+
): string =>
|
|
36
|
+
text
|
|
37
|
+
.split("\n")
|
|
38
|
+
.flatMap((line) => wrapLineWithContinuation(line, maxChars))
|
|
39
|
+
.join("\n");
|
|
30
40
|
|
|
31
41
|
export const capBrief = (text: string): string => {
|
|
32
42
|
const lines = text.split("\n");
|
package/src/core/lineage.ts
CHANGED
|
@@ -7,11 +7,15 @@ export interface LineageSessionManagerLike {
|
|
|
7
7
|
getEntries?: () => LineageEntryLike[];
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
-
export const getActiveLineageEntryIds = (
|
|
10
|
+
export const getActiveLineageEntryIds = (
|
|
11
|
+
sessionManager: LineageSessionManagerLike,
|
|
12
|
+
): Set<string> => {
|
|
11
13
|
try {
|
|
12
14
|
const branch = sessionManager.getBranch() ?? [];
|
|
13
15
|
if (branch.length > 0) {
|
|
14
|
-
return new Set(
|
|
16
|
+
return new Set(
|
|
17
|
+
branch.map((e) => e.id).filter((id): id is string => Boolean(id)),
|
|
18
|
+
);
|
|
15
19
|
}
|
|
16
20
|
} catch {
|
|
17
21
|
// fall through to defensive fallback
|
|
@@ -19,7 +23,9 @@ export const getActiveLineageEntryIds = (sessionManager: LineageSessionManagerLi
|
|
|
19
23
|
|
|
20
24
|
try {
|
|
21
25
|
const all = sessionManager.getEntries?.() ?? [];
|
|
22
|
-
return new Set(
|
|
26
|
+
return new Set(
|
|
27
|
+
all.map((e) => e.id).filter((id): id is string => Boolean(id)),
|
|
28
|
+
);
|
|
23
29
|
} catch {
|
|
24
30
|
return new Set();
|
|
25
31
|
}
|
|
@@ -20,7 +20,11 @@ const MAX_CACHE_SIZE = 3;
|
|
|
20
20
|
const CACHE_TTL_MS = 2_000;
|
|
21
21
|
const cache = new Map<string, CacheEntry>();
|
|
22
22
|
|
|
23
|
-
function cacheKey(
|
|
23
|
+
function cacheKey(
|
|
24
|
+
sessionFile: string,
|
|
25
|
+
full: boolean,
|
|
26
|
+
allowedEntryIds: Set<string> | undefined,
|
|
27
|
+
): string {
|
|
24
28
|
let hash = `${sessionFile}::${full}`;
|
|
25
29
|
if (allowedEntryIds && allowedEntryIds.size > 0) {
|
|
26
30
|
// Include the full set as sorted JSON for collision-free caching
|
|
@@ -29,7 +33,11 @@ function cacheKey(sessionFile: string, full: boolean, allowedEntryIds: Set<strin
|
|
|
29
33
|
return hash;
|
|
30
34
|
}
|
|
31
35
|
|
|
32
|
-
function getCached(
|
|
36
|
+
function getCached(
|
|
37
|
+
sessionFile: string,
|
|
38
|
+
full: boolean,
|
|
39
|
+
allowedEntryIds?: Set<string>,
|
|
40
|
+
): LoadedMessages | undefined {
|
|
33
41
|
const key = cacheKey(sessionFile, full, allowedEntryIds);
|
|
34
42
|
const entry = cache.get(key);
|
|
35
43
|
if (!entry) return undefined;
|
|
@@ -55,7 +63,12 @@ function getCached(sessionFile: string, full: boolean, allowedEntryIds?: Set<str
|
|
|
55
63
|
return entry.result;
|
|
56
64
|
}
|
|
57
65
|
|
|
58
|
-
function setCache(
|
|
66
|
+
function setCache(
|
|
67
|
+
sessionFile: string,
|
|
68
|
+
full: boolean,
|
|
69
|
+
allowedEntryIds: Set<string> | undefined,
|
|
70
|
+
result: LoadedMessages,
|
|
71
|
+
): void {
|
|
59
72
|
// Evict oldest if at capacity
|
|
60
73
|
if (cache.size >= MAX_CACHE_SIZE) {
|
|
61
74
|
const oldest = cache.entries().next();
|
|
@@ -85,10 +98,16 @@ export const loadAllMessages = (
|
|
|
85
98
|
let parseErrors = 0;
|
|
86
99
|
for (const line of content.split("\n")) {
|
|
87
100
|
if (!line.trim()) continue;
|
|
88
|
-
try {
|
|
101
|
+
try {
|
|
102
|
+
entries.push(JSON.parse(line));
|
|
103
|
+
} catch {
|
|
104
|
+
parseErrors++;
|
|
105
|
+
}
|
|
89
106
|
}
|
|
90
107
|
if (parseErrors > 0) {
|
|
91
|
-
console.warn(
|
|
108
|
+
console.warn(
|
|
109
|
+
`blackhole: ${parseErrors} malformed JSONL line(s) in ${sessionFile}`,
|
|
110
|
+
);
|
|
92
111
|
}
|
|
93
112
|
const rendered: RenderedEntry[] = [];
|
|
94
113
|
const rawMessages: Message[] = [];
|
package/src/core/normalize.ts
CHANGED
|
@@ -25,38 +25,64 @@ const normalizeOne = (msg: Message, msgIndex: number): NormalizedBlock[] => {
|
|
|
25
25
|
if (msg.content && typeof msg.content !== "string") {
|
|
26
26
|
for (const part of msg.content) {
|
|
27
27
|
if (part.type === "image") {
|
|
28
|
-
blocks.push({
|
|
28
|
+
blocks.push({
|
|
29
|
+
kind: "user",
|
|
30
|
+
text: `[image: ${part.mimeType}]`,
|
|
31
|
+
sourceIndex: msgIndex,
|
|
32
|
+
});
|
|
29
33
|
}
|
|
30
34
|
}
|
|
31
35
|
}
|
|
32
|
-
return blocks.length > 0
|
|
36
|
+
return blocks.length > 0
|
|
37
|
+
? blocks
|
|
38
|
+
: [{ kind: "user", text: "", sourceIndex: msgIndex }];
|
|
33
39
|
}
|
|
34
40
|
|
|
35
41
|
if ((msg as any).role === "bashExecution") {
|
|
36
42
|
const bashMsg = msg as unknown as LocalBashMessage;
|
|
37
|
-
return [
|
|
43
|
+
return [
|
|
44
|
+
{
|
|
45
|
+
kind: "bash",
|
|
46
|
+
command: bashMsg.command ?? "",
|
|
47
|
+
output: bashMsg.output ?? "",
|
|
48
|
+
exitCode: bashMsg.exitCode,
|
|
49
|
+
sourceIndex: msgIndex,
|
|
50
|
+
},
|
|
51
|
+
];
|
|
38
52
|
}
|
|
39
53
|
|
|
40
54
|
if (msg.role === "toolResult") {
|
|
41
|
-
return [
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
55
|
+
return [
|
|
56
|
+
{
|
|
57
|
+
kind: "tool_result",
|
|
58
|
+
name: msg.toolName,
|
|
59
|
+
text: sanitize(textOf(msg.content)),
|
|
60
|
+
isError: msg.isError,
|
|
61
|
+
sourceIndex: msgIndex,
|
|
62
|
+
},
|
|
63
|
+
];
|
|
48
64
|
}
|
|
49
65
|
|
|
50
66
|
if (msg.role === "assistant") {
|
|
51
67
|
if (!msg.content) return [];
|
|
52
68
|
if (typeof msg.content === "string") {
|
|
53
|
-
return [
|
|
69
|
+
return [
|
|
70
|
+
{
|
|
71
|
+
kind: "assistant",
|
|
72
|
+
text: sanitize(msg.content),
|
|
73
|
+
sourceIndex: msgIndex,
|
|
74
|
+
},
|
|
75
|
+
];
|
|
54
76
|
}
|
|
55
77
|
|
|
56
78
|
const blocks: NormalizedBlock[] = [];
|
|
57
79
|
for (const part of msg.content) {
|
|
58
80
|
if (part.type === "text") {
|
|
59
|
-
blocks.push({
|
|
81
|
+
blocks.push({
|
|
82
|
+
kind: "assistant",
|
|
83
|
+
text: sanitize(part.text),
|
|
84
|
+
sourceIndex: msgIndex,
|
|
85
|
+
});
|
|
60
86
|
} else if (part.type === "thinking") {
|
|
61
87
|
blocks.push({
|
|
62
88
|
kind: "thinking",
|
|
@@ -81,5 +107,3 @@ const normalizeOne = (msg: Message, msgIndex: number): NormalizedBlock[] => {
|
|
|
81
107
|
|
|
82
108
|
export const normalize = (messages: Message[]): NormalizedBlock[] =>
|
|
83
109
|
messages.flatMap((msg, i) => normalizeOne(msg, i));
|
|
84
|
-
|
|
85
|
-
|
package/src/core/recall-scope.ts
CHANGED
|
@@ -7,19 +7,27 @@ const MODE_RE = /\bmode:(hybrid|file|touched)\b/i;
|
|
|
7
7
|
const VALID_MODES = new Set(["hybrid", "file", "touched"]);
|
|
8
8
|
|
|
9
9
|
export const normalizeRecallScope = (scope?: unknown): RecallScope =>
|
|
10
|
-
typeof scope === "string" && scope.toLowerCase() === "all"
|
|
10
|
+
typeof scope === "string" && scope.toLowerCase() === "all"
|
|
11
|
+
? "all"
|
|
12
|
+
: "lineage";
|
|
11
13
|
|
|
12
14
|
export const normalizeRecallMode = (mode?: unknown): RecallMode =>
|
|
13
15
|
typeof mode === "string" && VALID_MODES.has(mode.toLowerCase())
|
|
14
16
|
? (mode.toLowerCase() as RecallMode)
|
|
15
17
|
: "hybrid";
|
|
16
18
|
|
|
17
|
-
export const parseRecallScope = (
|
|
19
|
+
export const parseRecallScope = (
|
|
20
|
+
text: string,
|
|
21
|
+
): { scope: RecallScope; mode: RecallMode; text: string } => {
|
|
18
22
|
const scopeMatch = text.match(SCOPE_RE);
|
|
19
23
|
const modeMatch = text.match(MODE_RE);
|
|
20
24
|
return {
|
|
21
25
|
scope: normalizeRecallScope(scopeMatch?.[1]),
|
|
22
26
|
mode: normalizeRecallMode(modeMatch?.[1]),
|
|
23
|
-
text: text
|
|
27
|
+
text: text
|
|
28
|
+
.replace(SCOPE_RE, "")
|
|
29
|
+
.replace(MODE_RE, "")
|
|
30
|
+
.replace(/\s+/g, " ")
|
|
31
|
+
.trim(),
|
|
24
32
|
};
|
|
25
33
|
};
|
|
@@ -34,15 +34,27 @@ const extractFilesFromContent = (content: Message["content"]): string[] => {
|
|
|
34
34
|
.filter((p): p is string => p !== null);
|
|
35
35
|
};
|
|
36
36
|
|
|
37
|
-
export const renderMessage = (
|
|
37
|
+
export const renderMessage = (
|
|
38
|
+
msg: Message,
|
|
39
|
+
index: number,
|
|
40
|
+
id: string,
|
|
41
|
+
full = false,
|
|
42
|
+
): RenderedEntry => {
|
|
38
43
|
if (msg.role === "user") {
|
|
39
|
-
return {
|
|
44
|
+
return {
|
|
45
|
+
index,
|
|
46
|
+
id,
|
|
47
|
+
role: "user",
|
|
48
|
+
summary: full ? textOf(msg.content) : clip(textOf(msg.content), 300),
|
|
49
|
+
};
|
|
40
50
|
}
|
|
41
51
|
if (msg.role === "toolResult") {
|
|
42
52
|
const prefix = msg.isError ? "ERROR " : "";
|
|
43
53
|
const text = full ? textOf(msg.content) : clip(textOf(msg.content), 200);
|
|
44
54
|
return {
|
|
45
|
-
index,
|
|
55
|
+
index,
|
|
56
|
+
id,
|
|
57
|
+
role: "tool_result",
|
|
46
58
|
summary: `${prefix}[${msg.toolName}] ${text}`,
|
|
47
59
|
};
|
|
48
60
|
}
|
|
@@ -58,7 +70,11 @@ export const renderMessage = (msg: Message, index: number, id: string, full = fa
|
|
|
58
70
|
const tools = toolCalls(msg.content);
|
|
59
71
|
const files = extractFilesFromContent(msg.content);
|
|
60
72
|
const summary = tools ? `${tools}\n${text}` : text;
|
|
61
|
-
return {
|
|
73
|
+
return {
|
|
74
|
+
index,
|
|
75
|
+
id,
|
|
76
|
+
role: "assistant",
|
|
77
|
+
summary,
|
|
78
|
+
...(files.length > 0 && { files }),
|
|
79
|
+
};
|
|
62
80
|
};
|
|
63
|
-
|
|
64
|
-
|
package/src/core/sanitize.ts
CHANGED
|
@@ -2,4 +2,8 @@ const ANSI_RE = /\x1b\[[0-9;]*[A-Za-z]/g;
|
|
|
2
2
|
const CTRL_RE = /[\x00-\x08\x0b\x0c\x0e-\x1f]/g;
|
|
3
3
|
|
|
4
4
|
export const sanitize = (text: string): string =>
|
|
5
|
-
text
|
|
5
|
+
text
|
|
6
|
+
.replace(/\r\n/g, "\n")
|
|
7
|
+
.replace(/\r/g, "\n")
|
|
8
|
+
.replace(ANSI_RE, "")
|
|
9
|
+
.replace(CTRL_RE, "");
|
|
@@ -81,21 +81,100 @@ const snippetRegex = (terms: string[]): RegExp => {
|
|
|
81
81
|
// ── Stopwords for natural language queries ──
|
|
82
82
|
const STOPWORDS = new Set([
|
|
83
83
|
// English
|
|
84
|
-
"the",
|
|
85
|
-
"
|
|
86
|
-
"
|
|
87
|
-
"
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"
|
|
91
|
-
"
|
|
92
|
-
"
|
|
93
|
-
"
|
|
84
|
+
"the",
|
|
85
|
+
"a",
|
|
86
|
+
"an",
|
|
87
|
+
"is",
|
|
88
|
+
"are",
|
|
89
|
+
"was",
|
|
90
|
+
"were",
|
|
91
|
+
"be",
|
|
92
|
+
"been",
|
|
93
|
+
"being",
|
|
94
|
+
"have",
|
|
95
|
+
"has",
|
|
96
|
+
"had",
|
|
97
|
+
"do",
|
|
98
|
+
"does",
|
|
99
|
+
"did",
|
|
100
|
+
"will",
|
|
101
|
+
"would",
|
|
102
|
+
"could",
|
|
103
|
+
"should",
|
|
104
|
+
"may",
|
|
105
|
+
"might",
|
|
106
|
+
"can",
|
|
107
|
+
"shall",
|
|
108
|
+
"of",
|
|
109
|
+
"in",
|
|
110
|
+
"to",
|
|
111
|
+
"for",
|
|
112
|
+
"with",
|
|
113
|
+
"on",
|
|
114
|
+
"at",
|
|
115
|
+
"from",
|
|
116
|
+
"by",
|
|
117
|
+
"as",
|
|
118
|
+
"into",
|
|
119
|
+
"through",
|
|
120
|
+
"during",
|
|
121
|
+
"before",
|
|
122
|
+
"after",
|
|
123
|
+
"above",
|
|
124
|
+
"below",
|
|
125
|
+
"between",
|
|
126
|
+
"out",
|
|
127
|
+
"off",
|
|
128
|
+
"over",
|
|
129
|
+
"under",
|
|
130
|
+
"again",
|
|
131
|
+
"further",
|
|
132
|
+
"then",
|
|
133
|
+
"once",
|
|
134
|
+
"here",
|
|
135
|
+
"there",
|
|
136
|
+
"when",
|
|
137
|
+
"where",
|
|
138
|
+
"why",
|
|
139
|
+
"how",
|
|
140
|
+
"all",
|
|
141
|
+
"both",
|
|
142
|
+
"each",
|
|
143
|
+
"few",
|
|
144
|
+
"more",
|
|
145
|
+
"most",
|
|
146
|
+
"other",
|
|
147
|
+
"some",
|
|
148
|
+
"such",
|
|
149
|
+
"no",
|
|
150
|
+
"nor",
|
|
151
|
+
"not",
|
|
152
|
+
"only",
|
|
153
|
+
"own",
|
|
154
|
+
"same",
|
|
155
|
+
"so",
|
|
156
|
+
"than",
|
|
157
|
+
"too",
|
|
158
|
+
"very",
|
|
159
|
+
"just",
|
|
160
|
+
"about",
|
|
161
|
+
"it",
|
|
162
|
+
"its",
|
|
163
|
+
"that",
|
|
164
|
+
"this",
|
|
165
|
+
"what",
|
|
166
|
+
"which",
|
|
167
|
+
"who",
|
|
168
|
+
"whom",
|
|
169
|
+
"these",
|
|
170
|
+
"those",
|
|
94
171
|
]);
|
|
95
172
|
|
|
96
173
|
/** Remove stopwords, keep meaningful terms. */
|
|
97
174
|
const filterStopwords = (terms: string[]): string[] => {
|
|
98
|
-
const meaningful = terms.filter(
|
|
175
|
+
const meaningful = terms.filter(
|
|
176
|
+
(t) => !STOPWORDS.has(t.toLowerCase()) && t.length > 1,
|
|
177
|
+
);
|
|
99
178
|
// If all terms were stopwords, return original (don't lose everything)
|
|
100
179
|
return meaningful.length > 0 ? meaningful : terms;
|
|
101
180
|
};
|
|
@@ -120,8 +199,8 @@ const termFreq = (text: string, pattern: RegExp): number => {
|
|
|
120
199
|
};
|
|
121
200
|
|
|
122
201
|
interface BM25Context {
|
|
123
|
-
n: number;
|
|
124
|
-
avgDl: number;
|
|
202
|
+
n: number; // total docs
|
|
203
|
+
avgDl: number; // average doc length (words)
|
|
125
204
|
df: Map<string, number>; // term -> number of docs containing it
|
|
126
205
|
}
|
|
127
206
|
|
|
@@ -156,7 +235,9 @@ const bm25Score = (doc: string, terms: string[], ctx: BM25Context): number => {
|
|
|
156
235
|
// IDF: log((N - df + 0.5) / (df + 0.5) + 1)
|
|
157
236
|
const idf = Math.log((ctx.n - docFreq + 0.5) / (docFreq + 0.5) + 1);
|
|
158
237
|
// TF saturation with length normalization
|
|
159
|
-
const tfNorm =
|
|
238
|
+
const tfNorm =
|
|
239
|
+
(tf * (BM25_K + 1)) /
|
|
240
|
+
(tf + BM25_K * (1 - BM25_B + (BM25_B * dl) / ctx.avgDl));
|
|
160
241
|
score += idf * tfNorm;
|
|
161
242
|
}
|
|
162
243
|
|
|
@@ -164,7 +245,11 @@ const bm25Score = (doc: string, terms: string[], ctx: BM25Context): number => {
|
|
|
164
245
|
};
|
|
165
246
|
|
|
166
247
|
/** Line-based snippet: ±contextLines around first regex match. */
|
|
167
|
-
const lineSnippet = (
|
|
248
|
+
const lineSnippet = (
|
|
249
|
+
text: string,
|
|
250
|
+
regex: RegExp,
|
|
251
|
+
contextLines = 2,
|
|
252
|
+
): string | undefined => {
|
|
168
253
|
const lines = text.split("\n");
|
|
169
254
|
let matchIdx = -1;
|
|
170
255
|
for (let i = 0; i < lines.length; i++) {
|
|
@@ -216,8 +301,10 @@ function extractToolCallText(args: Record<string, unknown>): string {
|
|
|
216
301
|
}
|
|
217
302
|
}
|
|
218
303
|
}
|
|
219
|
-
if (typeof args.oldText === "string" && !Array.isArray(args.edits))
|
|
220
|
-
|
|
304
|
+
if (typeof args.oldText === "string" && !Array.isArray(args.edits))
|
|
305
|
+
text += args.oldText + "\n";
|
|
306
|
+
if (typeof args.newText === "string" && !Array.isArray(args.edits))
|
|
307
|
+
text += args.newText + "\n";
|
|
221
308
|
return text;
|
|
222
309
|
}
|
|
223
310
|
|
|
@@ -247,12 +334,17 @@ export function getFileIndicators(msg: Message): FileMatch[] {
|
|
|
247
334
|
return fileMatches;
|
|
248
335
|
}
|
|
249
336
|
|
|
250
|
-
function computeFileMatches(
|
|
337
|
+
function computeFileMatches(
|
|
338
|
+
msg: Message | undefined,
|
|
339
|
+
query: string,
|
|
340
|
+
): FileMatch[] {
|
|
251
341
|
if (!msg?.content || typeof msg.content === "string") return [];
|
|
252
342
|
const rawQuery = query.trim();
|
|
253
343
|
const hasQuery = rawQuery.length > 0;
|
|
254
344
|
if (!hasQuery) return getFileIndicators(msg as Message);
|
|
255
|
-
const regex = looksLikeRegex(rawQuery)
|
|
345
|
+
const regex = looksLikeRegex(rawQuery)
|
|
346
|
+
? safeRegex(rawQuery)
|
|
347
|
+
: snippetRegex(rawQuery.split(/\s+/));
|
|
256
348
|
const fileMatches: FileMatch[] = [];
|
|
257
349
|
|
|
258
350
|
for (const part of msg.content) {
|
package/src/core/settings.ts
CHANGED
package/src/core/summarize.ts
CHANGED
|
@@ -17,7 +17,13 @@ export interface CompileInput {
|
|
|
17
17
|
fileOps?: FileOps;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
const HEADER_NAMES = [
|
|
20
|
+
const HEADER_NAMES = [
|
|
21
|
+
"Session Goal",
|
|
22
|
+
"Files And Changes",
|
|
23
|
+
"Commits",
|
|
24
|
+
"Outstanding Context",
|
|
25
|
+
"User Preferences",
|
|
26
|
+
];
|
|
21
27
|
|
|
22
28
|
const SEPARATOR = "\n\n---\n\n";
|
|
23
29
|
|
|
@@ -28,8 +34,7 @@ const sectionOf = (text: string, header: string): string => {
|
|
|
28
34
|
if (start < 0) return "";
|
|
29
35
|
const after = text.slice(start);
|
|
30
36
|
// Find next section header (must start at line boundary to avoid matching in content)
|
|
31
|
-
const nextSection = HEADER_NAMES
|
|
32
|
-
.filter((h) => h !== header)
|
|
37
|
+
const nextSection = HEADER_NAMES.filter((h) => h !== header)
|
|
33
38
|
.map((h) => {
|
|
34
39
|
// Escape the header name for regex safety
|
|
35
40
|
const escaped = h.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
@@ -41,7 +46,9 @@ const sectionOf = (text: string, header: string): string => {
|
|
|
41
46
|
})
|
|
42
47
|
.filter((n) => n >= 0);
|
|
43
48
|
const nextSep = after.indexOf("\n\n---\n\n");
|
|
44
|
-
const candidates = [...nextSection, ...(nextSep > 0 ? [nextSep] : [])].sort(
|
|
49
|
+
const candidates = [...nextSection, ...(nextSep > 0 ? [nextSep] : [])].sort(
|
|
50
|
+
(a, b) => a - b,
|
|
51
|
+
);
|
|
45
52
|
const end = candidates[0];
|
|
46
53
|
return (end ? after.slice(0, end) : after).trim();
|
|
47
54
|
};
|
|
@@ -54,7 +61,11 @@ const briefOf = (text: string): string => {
|
|
|
54
61
|
};
|
|
55
62
|
|
|
56
63
|
/** Merge a header section */
|
|
57
|
-
const mergeHeaderSection = (
|
|
64
|
+
const mergeHeaderSection = (
|
|
65
|
+
header: string,
|
|
66
|
+
prev: string,
|
|
67
|
+
fresh: string,
|
|
68
|
+
): string => {
|
|
58
69
|
// Outstanding Context is volatile -- always use fresh only
|
|
59
70
|
if (header === "Outstanding Context") return fresh;
|
|
60
71
|
if (!prev) return fresh;
|
|
@@ -66,16 +77,20 @@ const mergeHeaderSection = (header: string, prev: string, fresh: string): string
|
|
|
66
77
|
}
|
|
67
78
|
|
|
68
79
|
// Session Goal, User Preferences: line-level dedup, cap
|
|
69
|
-
const isClean = (l: string) =>
|
|
80
|
+
const isClean = (l: string) =>
|
|
81
|
+
l.startsWith("- ") && !l.includes("<skill") && !l.includes("</skill");
|
|
70
82
|
const prevLines = prev.split("\n").filter(isClean);
|
|
71
83
|
const freshLines = fresh.split("\n").filter(isClean);
|
|
72
84
|
const combined = [...new Set([...prevLines, ...freshLines])];
|
|
73
85
|
const CAP = header === "Session Goal" ? 8 : header === "Commits" ? 8 : 15;
|
|
74
86
|
// Session Goal: keep first items so the original first message persists
|
|
75
87
|
// Other sections: keep last items (fresh overrides stale)
|
|
76
|
-
const capped =
|
|
77
|
-
|
|
78
|
-
|
|
88
|
+
const capped =
|
|
89
|
+
combined.length > CAP
|
|
90
|
+
? header === "Session Goal"
|
|
91
|
+
? combined.slice(0, CAP)
|
|
92
|
+
: combined.slice(-CAP)
|
|
93
|
+
: combined;
|
|
79
94
|
if (capped.length === 0) return "";
|
|
80
95
|
return `[${header}]\n${capped.join("\n")}`;
|
|
81
96
|
};
|
|
@@ -115,8 +130,10 @@ const mergeFileLines = (prev: string, fresh: string): string => {
|
|
|
115
130
|
};
|
|
116
131
|
|
|
117
132
|
const lines: string[] = [];
|
|
118
|
-
if (merged.Modified.size > 0)
|
|
119
|
-
|
|
133
|
+
if (merged.Modified.size > 0)
|
|
134
|
+
lines.push(`- Modified: ${cap(merged.Modified, 10)}`);
|
|
135
|
+
if (merged.Created.size > 0)
|
|
136
|
+
lines.push(`- Created: ${cap(merged.Created, 10)}`);
|
|
120
137
|
if (merged.Read.size > 0) lines.push(`- Read: ${cap(merged.Read, 10)}`);
|
|
121
138
|
if (lines.length === 0) return "";
|
|
122
139
|
return `[Files And Changes]\n${lines.join("\n")}`;
|
|
@@ -130,13 +147,11 @@ const mergeBriefTranscript = (prev: string, fresh: string): string => {
|
|
|
130
147
|
|
|
131
148
|
const mergePrevious = (prev: string, fresh: string): string => {
|
|
132
149
|
// Merge header sections
|
|
133
|
-
const headers = HEADER_NAMES
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
})
|
|
139
|
-
.filter(Boolean);
|
|
150
|
+
const headers = HEADER_NAMES.map((header) => {
|
|
151
|
+
const freshSec = sectionOf(fresh, header);
|
|
152
|
+
const prevSec = sectionOf(prev, header);
|
|
153
|
+
return mergeHeaderSection(header, prevSec, freshSec);
|
|
154
|
+
}).filter(Boolean);
|
|
140
155
|
|
|
141
156
|
// Merge brief transcript
|
|
142
157
|
const prevBrief = briefOf(prev);
|
|
@@ -189,7 +204,9 @@ export const compile = (input: CompileInput): string => {
|
|
|
189
204
|
*/
|
|
190
205
|
const stripRecallNotes = (text: string): string => {
|
|
191
206
|
const paragraphs = text.split(/\n\n+/);
|
|
192
|
-
const kept = paragraphs.filter(
|
|
207
|
+
const kept = paragraphs.filter(
|
|
208
|
+
(p) => !p.includes("The conversation before this point has been compacted"),
|
|
209
|
+
);
|
|
193
210
|
return kept.join("\n\n");
|
|
194
211
|
};
|
|
195
212
|
|
|
@@ -204,12 +221,16 @@ const stripOMContent = (text: string): string => {
|
|
|
204
221
|
const obsIdx = obsMatch ? obsMatch.index! : -1;
|
|
205
222
|
|
|
206
223
|
// Also detect the basic recall-guidance footer (no observation preamble)
|
|
207
|
-
const basicFooterIdx = text.indexOf(
|
|
224
|
+
const basicFooterIdx = text.indexOf(
|
|
225
|
+
"Use `recall` with an id to retrieve original context, or `#N:path` drill-down",
|
|
226
|
+
);
|
|
208
227
|
|
|
209
228
|
// Find the start of OM content: either the instructions preamble or the first section header
|
|
210
229
|
let stripFrom = -1;
|
|
211
230
|
if (reflIdx >= 0 || obsIdx >= 0) {
|
|
212
|
-
const preambleIdx = text.indexOf(
|
|
231
|
+
const preambleIdx = text.indexOf(
|
|
232
|
+
"These are condensed memories from earlier in this session.",
|
|
233
|
+
);
|
|
213
234
|
const minSectionIdx = Math.min(
|
|
214
235
|
reflIdx >= 0 ? reflIdx : Infinity,
|
|
215
236
|
obsIdx >= 0 ? obsIdx : Infinity,
|