pi-continuous-learning 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis-event-log.d.ts +50 -0
- package/dist/analysis-event-log.d.ts.map +1 -0
- package/dist/analysis-event-log.js +120 -0
- package/dist/analysis-event-log.js.map +1 -0
- package/dist/analysis-notification.d.ts +20 -0
- package/dist/analysis-notification.d.ts.map +1 -0
- package/dist/analysis-notification.js +63 -0
- package/dist/analysis-notification.js.map +1 -0
- package/dist/cli/analyze-single-shot.d.ts +12 -0
- package/dist/cli/analyze-single-shot.d.ts.map +1 -1
- package/dist/cli/analyze-single-shot.js +84 -2
- package/dist/cli/analyze-single-shot.js.map +1 -1
- package/dist/cli/analyze.js +112 -8
- package/dist/cli/analyze.js.map +1 -1
- package/dist/confidence.d.ts +12 -1
- package/dist/confidence.d.ts.map +1 -1
- package/dist/confidence.js +35 -8
- package/dist/confidence.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/instinct-parser.d.ts.map +1 -1
- package/dist/instinct-parser.js +6 -0
- package/dist/instinct-parser.js.map +1 -1
- package/dist/observation-signal.d.ts +34 -0
- package/dist/observation-signal.d.ts.map +1 -0
- package/dist/observation-signal.js +66 -0
- package/dist/observation-signal.js.map +1 -0
- package/dist/prompts/analyzer-system-single-shot.d.ts.map +1 -1
- package/dist/prompts/analyzer-system-single-shot.js +41 -2
- package/dist/prompts/analyzer-system-single-shot.js.map +1 -1
- package/dist/prompts/analyzer-user-single-shot.d.ts.map +1 -1
- package/dist/prompts/analyzer-user-single-shot.js +4 -2
- package/dist/prompts/analyzer-user-single-shot.js.map +1 -1
- package/dist/types.d.ts +1 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/analysis-event-log.ts +171 -0
- package/src/analysis-notification.ts +79 -0
- package/src/cli/analyze-single-shot.ts +98 -2
- package/src/cli/analyze.ts +138 -7
- package/src/confidence.ts +33 -7
- package/src/index.ts +2 -0
- package/src/instinct-parser.ts +6 -0
- package/src/observation-signal.ts +80 -0
- package/src/prompts/analyzer-system-single-shot.ts +41 -2
- package/src/prompts/analyzer-user-single-shot.ts +5 -2
- package/src/types.ts +1 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append-only analysis event log with atomic rename for safe consumption.
|
|
3
|
+
*
|
|
4
|
+
* The background analyzer appends events to `analysis-events.jsonl`.
|
|
5
|
+
* The extension consumes events by atomically renaming the file to
|
|
6
|
+
* `.consumed`, reading it, then deleting it. On POSIX, rename is atomic -
|
|
7
|
+
* any in-flight appends follow the inode to the renamed file.
|
|
8
|
+
*
|
|
9
|
+
* Multiple analyzer runs can append before the extension reads. No events
|
|
10
|
+
* are lost because each run only appends; the file is never truncated by
|
|
11
|
+
* the analyzer.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
appendFileSync,
|
|
16
|
+
existsSync,
|
|
17
|
+
mkdirSync,
|
|
18
|
+
readFileSync,
|
|
19
|
+
renameSync,
|
|
20
|
+
unlinkSync,
|
|
21
|
+
} from "node:fs";
|
|
22
|
+
import { dirname, join } from "node:path";
|
|
23
|
+
import { getProjectDir } from "./storage.js";
|
|
24
|
+
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Constants
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
const EVENTS_FILENAME = "analysis-events.jsonl";
|
|
30
|
+
const CONSUMED_FILENAME = "analysis-events.consumed";
|
|
31
|
+
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
// Types
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
|
|
36
|
+
export interface InstinctChangeSummary {
|
|
37
|
+
readonly id: string;
|
|
38
|
+
readonly title: string;
|
|
39
|
+
readonly scope: "project" | "global";
|
|
40
|
+
readonly trigger?: string;
|
|
41
|
+
readonly action?: string;
|
|
42
|
+
readonly confidence_delta?: number;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface AnalysisEvent {
|
|
46
|
+
readonly timestamp: string;
|
|
47
|
+
readonly project_id: string;
|
|
48
|
+
readonly project_name: string;
|
|
49
|
+
readonly created: readonly InstinctChangeSummary[];
|
|
50
|
+
readonly updated: readonly InstinctChangeSummary[];
|
|
51
|
+
readonly deleted: readonly InstinctChangeSummary[];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// ---------------------------------------------------------------------------
|
|
55
|
+
// Paths
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
export function getEventsPath(projectId: string, baseDir?: string): string {
|
|
59
|
+
return join(getProjectDir(projectId, baseDir), EVENTS_FILENAME);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function getConsumedPath(projectId: string, baseDir?: string): string {
|
|
63
|
+
return join(getProjectDir(projectId, baseDir), CONSUMED_FILENAME);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// ---------------------------------------------------------------------------
|
|
67
|
+
// Write (analyzer side)
|
|
68
|
+
// ---------------------------------------------------------------------------
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Appends an analysis event to the project's event log.
|
|
72
|
+
* Skips writing if nothing changed (all arrays empty).
|
|
73
|
+
* Creates the parent directory if needed.
|
|
74
|
+
*/
|
|
75
|
+
export function appendAnalysisEvent(event: AnalysisEvent, baseDir?: string): void {
|
|
76
|
+
if (
|
|
77
|
+
event.created.length === 0 &&
|
|
78
|
+
event.updated.length === 0 &&
|
|
79
|
+
event.deleted.length === 0
|
|
80
|
+
) {
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const eventsPath = getEventsPath(event.project_id, baseDir);
|
|
85
|
+
mkdirSync(dirname(eventsPath), { recursive: true });
|
|
86
|
+
appendFileSync(eventsPath, JSON.stringify(event) + "\n", "utf-8");
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// ---------------------------------------------------------------------------
|
|
90
|
+
// Read and clear (extension side)
|
|
91
|
+
// ---------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Atomically consumes all pending analysis events for a project.
|
|
95
|
+
*
|
|
96
|
+
* Strategy:
|
|
97
|
+
* 1. Check for orphaned `.consumed` file from a prior crash - read it first
|
|
98
|
+
* 2. Rename `analysis-events.jsonl` to `.consumed` (atomic on POSIX)
|
|
99
|
+
* 3. Read and parse all lines from `.consumed`
|
|
100
|
+
* 4. Delete `.consumed`
|
|
101
|
+
*
|
|
102
|
+
* Returns an empty array if no events exist or rename fails (e.g. file
|
|
103
|
+
* doesn't exist, or another consumer raced us).
|
|
104
|
+
*/
|
|
105
|
+
export function consumeAnalysisEvents(
|
|
106
|
+
projectId: string,
|
|
107
|
+
baseDir?: string
|
|
108
|
+
): readonly AnalysisEvent[] {
|
|
109
|
+
const eventsPath = getEventsPath(projectId, baseDir);
|
|
110
|
+
const consumedPath = getConsumedPath(projectId, baseDir);
|
|
111
|
+
|
|
112
|
+
const allEvents: AnalysisEvent[] = [];
|
|
113
|
+
|
|
114
|
+
// Step 1: recover orphaned consumed file from prior crash
|
|
115
|
+
if (existsSync(consumedPath)) {
|
|
116
|
+
allEvents.push(...parseEventsFile(consumedPath));
|
|
117
|
+
safeUnlink(consumedPath);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Step 2: atomically rename the events file
|
|
121
|
+
if (existsSync(eventsPath)) {
|
|
122
|
+
try {
|
|
123
|
+
renameSync(eventsPath, consumedPath);
|
|
124
|
+
} catch {
|
|
125
|
+
// Rename failed (race with another consumer, or OS issue).
|
|
126
|
+
// Return whatever we recovered from step 1.
|
|
127
|
+
return allEvents;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Step 3: read the renamed file
|
|
131
|
+
allEvents.push(...parseEventsFile(consumedPath));
|
|
132
|
+
|
|
133
|
+
// Step 4: delete consumed file
|
|
134
|
+
safeUnlink(consumedPath);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
return allEvents;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// ---------------------------------------------------------------------------
|
|
141
|
+
// Helpers
|
|
142
|
+
// ---------------------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
function parseEventsFile(filePath: string): AnalysisEvent[] {
|
|
145
|
+
const events: AnalysisEvent[] = [];
|
|
146
|
+
|
|
147
|
+
try {
|
|
148
|
+
const content = readFileSync(filePath, "utf-8");
|
|
149
|
+
const lines = content.split("\n").filter((line) => line.trim().length > 0);
|
|
150
|
+
|
|
151
|
+
for (const line of lines) {
|
|
152
|
+
try {
|
|
153
|
+
events.push(JSON.parse(line) as AnalysisEvent);
|
|
154
|
+
} catch {
|
|
155
|
+
// Skip malformed lines - don't lose other events
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
} catch {
|
|
159
|
+
// File read failed - return empty
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return events;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function safeUnlink(filePath: string): void {
|
|
166
|
+
try {
|
|
167
|
+
if (existsSync(filePath)) unlinkSync(filePath);
|
|
168
|
+
} catch {
|
|
169
|
+
// Best effort cleanup
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extension-side notification for analysis events.
|
|
3
|
+
*
|
|
4
|
+
* On `before_agent_start`, consumes pending analysis events and shows
|
|
5
|
+
* a brief one-line notification summarizing instinct changes since the
|
|
6
|
+
* last session interaction.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
|
|
10
|
+
import {
|
|
11
|
+
consumeAnalysisEvents,
|
|
12
|
+
type AnalysisEvent,
|
|
13
|
+
} from "./analysis-event-log.js";
|
|
14
|
+
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
// Formatting
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Aggregates multiple analysis events into a single summary line.
|
|
21
|
+
* Returns null when no changes occurred.
|
|
22
|
+
*/
|
|
23
|
+
export function formatNotification(events: readonly AnalysisEvent[]): string | null {
|
|
24
|
+
if (events.length === 0) return null;
|
|
25
|
+
|
|
26
|
+
let created = 0;
|
|
27
|
+
let updated = 0;
|
|
28
|
+
let deleted = 0;
|
|
29
|
+
const createdIds: string[] = [];
|
|
30
|
+
|
|
31
|
+
for (const event of events) {
|
|
32
|
+
created += event.created.length;
|
|
33
|
+
updated += event.updated.length;
|
|
34
|
+
deleted += event.deleted.length;
|
|
35
|
+
for (const c of event.created) {
|
|
36
|
+
createdIds.push(c.id);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (created === 0 && updated === 0 && deleted === 0) return null;
|
|
41
|
+
|
|
42
|
+
const parts: string[] = [];
|
|
43
|
+
if (created > 0) {
|
|
44
|
+
const idList = createdIds.slice(0, 3).join(", ");
|
|
45
|
+
const suffix = createdIds.length > 3 ? ", ..." : "";
|
|
46
|
+
parts.push(`+${created} new (${idList}${suffix})`);
|
|
47
|
+
}
|
|
48
|
+
if (updated > 0) {
|
|
49
|
+
parts.push(`${updated} updated`);
|
|
50
|
+
}
|
|
51
|
+
if (deleted > 0) {
|
|
52
|
+
parts.push(`${deleted} deleted`);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return `[instincts] Background analysis: ${parts.join(", ")}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
// Handler
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Checks for pending analysis events and shows a notification if any exist.
|
|
64
|
+
* Safe to call on every `before_agent_start` - no-ops when there's nothing.
|
|
65
|
+
*/
|
|
66
|
+
export function checkAnalysisNotifications(
|
|
67
|
+
ctx: ExtensionContext,
|
|
68
|
+
projectId: string | null,
|
|
69
|
+
baseDir?: string
|
|
70
|
+
): void {
|
|
71
|
+
if (!projectId) return;
|
|
72
|
+
|
|
73
|
+
const events = consumeAnalysisEvents(projectId, baseDir);
|
|
74
|
+
const message = formatNotification(events);
|
|
75
|
+
|
|
76
|
+
if (message) {
|
|
77
|
+
ctx.ui.notify(message, "info");
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -9,7 +9,11 @@ import type { AssistantMessage, Context } from "@mariozechner/pi-ai";
|
|
|
9
9
|
import { complete } from "@mariozechner/pi-ai";
|
|
10
10
|
import type { Instinct } from "../types.js";
|
|
11
11
|
import { serializeInstinct } from "../instinct-parser.js";
|
|
12
|
+
|
|
13
|
+
/** Chars-per-token heuristic for prompt size estimation. */
|
|
14
|
+
const CHARS_PER_TOKEN = 4;
|
|
12
15
|
import { validateInstinct, findSimilarInstinct } from "../instinct-validator.js";
|
|
16
|
+
import { confirmationDelta } from "../confidence.js";
|
|
13
17
|
|
|
14
18
|
export interface InstinctChangePayload {
|
|
15
19
|
id: string;
|
|
@@ -24,6 +28,7 @@ export interface InstinctChangePayload {
|
|
|
24
28
|
contradicted_count?: number;
|
|
25
29
|
inactive_count?: number;
|
|
26
30
|
evidence?: string[];
|
|
31
|
+
last_confirmed_session?: string;
|
|
27
32
|
}
|
|
28
33
|
|
|
29
34
|
export interface InstinctChange {
|
|
@@ -117,12 +122,57 @@ export function buildInstinctFromChange(
|
|
|
117
122
|
|
|
118
123
|
const now = new Date().toISOString();
|
|
119
124
|
|
|
125
|
+
// For updates, recompute confidence client-side to enforce:
|
|
126
|
+
// 1. Per-session deduplication: only one confirmation per unique session_id
|
|
127
|
+
// 2. Diminishing returns: each additional confirmation yields a smaller delta
|
|
128
|
+
let resolvedConfidence: number;
|
|
129
|
+
let resolvedConfirmedCount = payload.confirmed_count ?? existing?.confirmed_count ?? 0;
|
|
130
|
+
let resolvedLastConfirmedSession = payload.last_confirmed_session ?? existing?.last_confirmed_session;
|
|
131
|
+
|
|
132
|
+
if (change.action === "update" && existing !== null) {
|
|
133
|
+
const prevConfirmedCount = existing.confirmed_count;
|
|
134
|
+
const newConfirmedCount = payload.confirmed_count ?? prevConfirmedCount;
|
|
135
|
+
const contradictionsAdded = Math.max(
|
|
136
|
+
0,
|
|
137
|
+
(payload.contradicted_count ?? 0) - existing.contradicted_count,
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
// Detect whether the LLM intends to add a confirmation
|
|
141
|
+
const wantsToConfirm = newConfirmedCount > prevConfirmedCount;
|
|
142
|
+
|
|
143
|
+
// Session dedup: reject the confirmation if the confirming session is the
|
|
144
|
+
// same as the one that last confirmed this instinct.
|
|
145
|
+
const sessionDuplicate =
|
|
146
|
+
wantsToConfirm &&
|
|
147
|
+
resolvedLastConfirmedSession !== undefined &&
|
|
148
|
+
payload.last_confirmed_session !== undefined &&
|
|
149
|
+
payload.last_confirmed_session === existing.last_confirmed_session;
|
|
150
|
+
|
|
151
|
+
if (sessionDuplicate) {
|
|
152
|
+
// Revert to existing count - this session already confirmed the instinct
|
|
153
|
+
resolvedConfirmedCount = prevConfirmedCount;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Recompute confidence from existing + explicit deltas (don't trust LLM arithmetic)
|
|
157
|
+
resolvedConfidence = existing.confidence;
|
|
158
|
+
if (wantsToConfirm && !sessionDuplicate) {
|
|
159
|
+
resolvedConfidence += confirmationDelta(prevConfirmedCount);
|
|
160
|
+
}
|
|
161
|
+
if (contradictionsAdded > 0) {
|
|
162
|
+
resolvedConfidence -= 0.15 * contradictionsAdded;
|
|
163
|
+
}
|
|
164
|
+
resolvedConfidence = Math.max(0.1, Math.min(0.9, resolvedConfidence));
|
|
165
|
+
} else {
|
|
166
|
+
// For creates, trust the LLM's initial confidence (no prior state to base delta on)
|
|
167
|
+
resolvedConfidence = Math.max(0.1, Math.min(0.9, payload.confidence));
|
|
168
|
+
}
|
|
169
|
+
|
|
120
170
|
return {
|
|
121
171
|
id: payload.id,
|
|
122
172
|
title: payload.title,
|
|
123
173
|
trigger: payload.trigger,
|
|
124
174
|
action: payload.action,
|
|
125
|
-
confidence:
|
|
175
|
+
confidence: resolvedConfidence,
|
|
126
176
|
domain: payload.domain,
|
|
127
177
|
scope: payload.scope,
|
|
128
178
|
source: "personal",
|
|
@@ -130,15 +180,61 @@ export function buildInstinctFromChange(
|
|
|
130
180
|
created_at: existing?.created_at ?? now,
|
|
131
181
|
updated_at: now,
|
|
132
182
|
observation_count: payload.observation_count ?? 1,
|
|
133
|
-
confirmed_count:
|
|
183
|
+
confirmed_count: resolvedConfirmedCount,
|
|
134
184
|
contradicted_count: payload.contradicted_count ?? 0,
|
|
135
185
|
inactive_count: payload.inactive_count ?? 0,
|
|
136
186
|
...(payload.evidence !== undefined ? { evidence: payload.evidence } : {}),
|
|
187
|
+
...(resolvedLastConfirmedSession !== undefined
|
|
188
|
+
? { last_confirmed_session: resolvedLastConfirmedSession }
|
|
189
|
+
: {}),
|
|
137
190
|
};
|
|
138
191
|
}
|
|
139
192
|
|
|
193
|
+
/**
|
|
194
|
+
* Returns days elapsed since the given ISO 8601 date string.
|
|
195
|
+
*/
|
|
196
|
+
function daysSince(dateStr: string): number {
|
|
197
|
+
const ms = Date.now() - new Date(dateStr).getTime();
|
|
198
|
+
return Math.max(0, Math.floor(ms / (1000 * 60 * 60 * 24)));
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Formats existing instincts as a compact JSON array for inline context.
|
|
203
|
+
* Reduces token usage by ~70% compared to full YAML+markdown serialization.
|
|
204
|
+
* Includes only the fields the analyzer needs to make decisions.
|
|
205
|
+
*/
|
|
206
|
+
export function formatInstinctsCompact(instincts: Instinct[]): string {
|
|
207
|
+
if (instincts.length === 0) {
|
|
208
|
+
return "[]";
|
|
209
|
+
}
|
|
210
|
+
const summaries = instincts.map((i) => ({
|
|
211
|
+
id: i.id,
|
|
212
|
+
trigger: i.trigger,
|
|
213
|
+
action: i.action,
|
|
214
|
+
confidence: i.confidence,
|
|
215
|
+
domain: i.domain,
|
|
216
|
+
scope: i.scope,
|
|
217
|
+
confirmed: i.confirmed_count,
|
|
218
|
+
contradicted: i.contradicted_count,
|
|
219
|
+
inactive: i.inactive_count,
|
|
220
|
+
age_days: daysSince(i.created_at),
|
|
221
|
+
...(i.last_confirmed_session !== undefined
|
|
222
|
+
? { last_confirmed_session: i.last_confirmed_session }
|
|
223
|
+
: {}),
|
|
224
|
+
}));
|
|
225
|
+
return JSON.stringify(summaries);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Estimates the token count of a text string using a chars/token heuristic.
|
|
230
|
+
*/
|
|
231
|
+
export function estimateTokens(text: string): number {
|
|
232
|
+
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
|
233
|
+
}
|
|
234
|
+
|
|
140
235
|
/**
|
|
141
236
|
* Formats existing instincts as serialized markdown blocks for inline context.
|
|
237
|
+
* @deprecated Use formatInstinctsCompact for lower token usage.
|
|
142
238
|
*/
|
|
143
239
|
export function formatInstinctsForPrompt(instincts: Instinct[]): string {
|
|
144
240
|
if (instincts.length === 0) {
|
package/src/cli/analyze.ts
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
writeFileSync,
|
|
7
7
|
unlinkSync,
|
|
8
8
|
} from "node:fs";
|
|
9
|
+
import { createHash } from "node:crypto";
|
|
9
10
|
import { join } from "node:path";
|
|
10
11
|
import { AuthStorage } from "@mariozechner/pi-coding-agent";
|
|
11
12
|
import { getModel } from "@mariozechner/pi-ai";
|
|
@@ -29,7 +30,14 @@ import { buildSingleShotUserPrompt } from "../prompts/analyzer-user-single-shot.
|
|
|
29
30
|
import {
|
|
30
31
|
runSingleShot,
|
|
31
32
|
buildInstinctFromChange,
|
|
33
|
+
estimateTokens,
|
|
32
34
|
} from "./analyze-single-shot.js";
|
|
35
|
+
import { isLowSignalBatch } from "../observation-signal.js";
|
|
36
|
+
import {
|
|
37
|
+
appendAnalysisEvent,
|
|
38
|
+
type InstinctChangeSummary,
|
|
39
|
+
type AnalysisEvent,
|
|
40
|
+
} from "../analysis-event-log.js";
|
|
33
41
|
import {
|
|
34
42
|
loadProjectInstincts,
|
|
35
43
|
loadGlobalInstincts,
|
|
@@ -107,9 +115,31 @@ function startGlobalTimeout(timeoutMs: number, logger: AnalyzeLogger): void {
|
|
|
107
115
|
// Per-project analysis
|
|
108
116
|
// ---------------------------------------------------------------------------
|
|
109
117
|
|
|
118
|
+
/** Max estimated tokens before fallback strategies are applied. */
|
|
119
|
+
const PROMPT_TOKEN_BUDGET = 40_000;
|
|
120
|
+
|
|
110
121
|
interface ProjectMeta {
|
|
111
122
|
last_analyzed_at?: string;
|
|
112
123
|
last_observation_line_count?: number;
|
|
124
|
+
/** SHA-256 hash of the last AGENTS.md content sent for this project (project-level file). */
|
|
125
|
+
agents_md_project_hash?: string;
|
|
126
|
+
/** SHA-256 hash of the last AGENTS.md content sent (global file). */
|
|
127
|
+
agents_md_global_hash?: string;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function hashContent(content: string): string {
|
|
131
|
+
return createHash("sha256").update(content).digest("hex");
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Truncates AGENTS.md content to section headers only (lines starting with #).
|
|
136
|
+
* Used as a fallback when the prompt is over the token budget.
|
|
137
|
+
*/
|
|
138
|
+
function truncateAgentsMdToHeaders(content: string): string {
|
|
139
|
+
return content
|
|
140
|
+
.split("\n")
|
|
141
|
+
.filter((line) => line.startsWith("#"))
|
|
142
|
+
.join("\n");
|
|
113
143
|
}
|
|
114
144
|
|
|
115
145
|
function loadProjectsRegistry(baseDir: string): Record<string, ProjectEntry> {
|
|
@@ -179,6 +209,10 @@ async function analyzeProject(
|
|
|
179
209
|
return { ran: false, skippedReason: "no new observation lines after preprocessing" };
|
|
180
210
|
}
|
|
181
211
|
|
|
212
|
+
if (isLowSignalBatch(newObsLines)) {
|
|
213
|
+
return { ran: false, skippedReason: "low-signal batch (no errors, corrections, or user redirections)" };
|
|
214
|
+
}
|
|
215
|
+
|
|
182
216
|
const obsCount = countObservations(project.id, baseDir);
|
|
183
217
|
if (obsCount < config.min_observations_to_analyze) {
|
|
184
218
|
return { ran: false, skippedReason: `below threshold (${obsCount}/${config.min_observations_to_analyze})` };
|
|
@@ -195,8 +229,21 @@ async function analyzeProject(
|
|
|
195
229
|
const globalInstincts = loadGlobalInstincts(baseDir);
|
|
196
230
|
const allInstincts = [...projectInstincts, ...globalInstincts];
|
|
197
231
|
|
|
198
|
-
|
|
199
|
-
const
|
|
232
|
+
// Load AGENTS.md, skipping if content hash is unchanged since last run.
|
|
233
|
+
const rawAgentsMdProject = readAgentsMd(join(project.root, "AGENTS.md"));
|
|
234
|
+
const rawAgentsMdGlobal = readAgentsMd(join(homedir(), ".pi", "agent", "AGENTS.md"));
|
|
235
|
+
|
|
236
|
+
const projectMdHash = rawAgentsMdProject ? hashContent(rawAgentsMdProject) : null;
|
|
237
|
+
const globalMdHash = rawAgentsMdGlobal ? hashContent(rawAgentsMdGlobal) : null;
|
|
238
|
+
|
|
239
|
+
const agentsMdProject =
|
|
240
|
+
rawAgentsMdProject && projectMdHash !== meta.agents_md_project_hash
|
|
241
|
+
? rawAgentsMdProject
|
|
242
|
+
: null;
|
|
243
|
+
const agentsMdGlobal =
|
|
244
|
+
rawAgentsMdGlobal && globalMdHash !== meta.agents_md_global_hash
|
|
245
|
+
? rawAgentsMdGlobal
|
|
246
|
+
: null;
|
|
200
247
|
|
|
201
248
|
let installedSkills: InstalledSkill[] = [];
|
|
202
249
|
try {
|
|
@@ -210,9 +257,51 @@ async function analyzeProject(
|
|
|
210
257
|
// Skills loading is best-effort - continue without them
|
|
211
258
|
}
|
|
212
259
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
260
|
+
let promptObsLines = newObsLines;
|
|
261
|
+
let promptAgentsMdProject = agentsMdProject;
|
|
262
|
+
let promptAgentsMdGlobal = agentsMdGlobal;
|
|
263
|
+
|
|
264
|
+
const userPrompt = buildSingleShotUserPrompt(project, allInstincts, promptObsLines, {
|
|
265
|
+
agentsMdProject: promptAgentsMdProject,
|
|
266
|
+
agentsMdGlobal: promptAgentsMdGlobal,
|
|
267
|
+
installedSkills,
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
// Estimate token budget and apply fallbacks if over limit.
|
|
271
|
+
const systemPromptTokens = estimateTokens(buildSingleShotSystemPrompt());
|
|
272
|
+
let estimatedTotal = systemPromptTokens + estimateTokens(userPrompt);
|
|
273
|
+
|
|
274
|
+
if (estimatedTotal > PROMPT_TOKEN_BUDGET) {
|
|
275
|
+
logger.warn(
|
|
276
|
+
`Prompt over budget (${estimatedTotal} est. tokens > ${PROMPT_TOKEN_BUDGET}). Applying fallbacks.`
|
|
277
|
+
);
|
|
278
|
+
|
|
279
|
+
// Fallback 1: truncate AGENTS.md to headers only.
|
|
280
|
+
if (promptAgentsMdProject) {
|
|
281
|
+
promptAgentsMdProject = truncateAgentsMdToHeaders(promptAgentsMdProject);
|
|
282
|
+
}
|
|
283
|
+
if (promptAgentsMdGlobal) {
|
|
284
|
+
promptAgentsMdGlobal = truncateAgentsMdToHeaders(promptAgentsMdGlobal);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Fallback 2: reduce observation lines to fit budget.
|
|
288
|
+
// Use binary-search-like reduction: keep halving until under budget.
|
|
289
|
+
while (promptObsLines.length > 1) {
|
|
290
|
+
const trimmedPrompt = buildSingleShotUserPrompt(
|
|
291
|
+
project,
|
|
292
|
+
allInstincts,
|
|
293
|
+
promptObsLines,
|
|
294
|
+
{ agentsMdProject: promptAgentsMdProject, agentsMdGlobal: promptAgentsMdGlobal, installedSkills }
|
|
295
|
+
);
|
|
296
|
+
estimatedTotal = systemPromptTokens + estimateTokens(trimmedPrompt);
|
|
297
|
+
if (estimatedTotal <= PROMPT_TOKEN_BUDGET) break;
|
|
298
|
+
promptObsLines = promptObsLines.slice(Math.floor(promptObsLines.length / 2));
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const finalUserPrompt = buildSingleShotUserPrompt(project, allInstincts, promptObsLines, {
|
|
303
|
+
agentsMdProject: promptAgentsMdProject,
|
|
304
|
+
agentsMdGlobal: promptAgentsMdGlobal,
|
|
216
305
|
installedSkills,
|
|
217
306
|
});
|
|
218
307
|
|
|
@@ -228,7 +317,7 @@ async function analyzeProject(
|
|
|
228
317
|
const context = {
|
|
229
318
|
systemPrompt: buildSingleShotSystemPrompt(),
|
|
230
319
|
messages: [
|
|
231
|
-
{ role: "user" as const, content:
|
|
320
|
+
{ role: "user" as const, content: finalUserPrompt, timestamp: Date.now() },
|
|
232
321
|
],
|
|
233
322
|
};
|
|
234
323
|
|
|
@@ -237,6 +326,9 @@ async function analyzeProject(
|
|
|
237
326
|
const timeoutHandle = setTimeout(() => abortController.abort(), timeoutMs);
|
|
238
327
|
|
|
239
328
|
const instinctCounts = { created: 0, updated: 0, deleted: 0 };
|
|
329
|
+
const createdSummaries: InstinctChangeSummary[] = [];
|
|
330
|
+
const updatedSummaries: InstinctChangeSummary[] = [];
|
|
331
|
+
const deletedSummaries: InstinctChangeSummary[] = [];
|
|
240
332
|
const projectInstinctsDir = getProjectInstinctsDir(project.id, "personal", baseDir);
|
|
241
333
|
const globalInstinctsDir = getGlobalInstinctsDir("personal", baseDir);
|
|
242
334
|
|
|
@@ -258,6 +350,11 @@ async function analyzeProject(
|
|
|
258
350
|
if (existsSync(filePath)) {
|
|
259
351
|
unlinkSync(filePath);
|
|
260
352
|
instinctCounts.deleted++;
|
|
353
|
+
deletedSummaries.push({
|
|
354
|
+
id,
|
|
355
|
+
title: id,
|
|
356
|
+
scope: change.scope ?? "project",
|
|
357
|
+
});
|
|
261
358
|
}
|
|
262
359
|
} else if (change.action === "create") {
|
|
263
360
|
if (createsRemaining <= 0) continue; // rate limit reached
|
|
@@ -269,6 +366,13 @@ async function analyzeProject(
|
|
|
269
366
|
saveInstinct(instinct, dir);
|
|
270
367
|
instinctCounts.created++;
|
|
271
368
|
createsRemaining--;
|
|
369
|
+
createdSummaries.push({
|
|
370
|
+
id: instinct.id,
|
|
371
|
+
title: instinct.title,
|
|
372
|
+
scope: instinct.scope,
|
|
373
|
+
trigger: instinct.trigger,
|
|
374
|
+
action: instinct.action,
|
|
375
|
+
});
|
|
272
376
|
} else {
|
|
273
377
|
// update
|
|
274
378
|
const existing = allInstincts.find((i) => i.id === change.instinct?.id) ?? null;
|
|
@@ -278,6 +382,15 @@ async function analyzeProject(
|
|
|
278
382
|
const dir = instinct.scope === "global" ? globalInstinctsDir : projectInstinctsDir;
|
|
279
383
|
saveInstinct(instinct, dir);
|
|
280
384
|
instinctCounts.updated++;
|
|
385
|
+
const delta = existing
|
|
386
|
+
? instinct.confidence - existing.confidence
|
|
387
|
+
: undefined;
|
|
388
|
+
updatedSummaries.push({
|
|
389
|
+
id: instinct.id,
|
|
390
|
+
title: instinct.title,
|
|
391
|
+
scope: instinct.scope,
|
|
392
|
+
...(delta !== undefined ? { confidence_delta: delta } : {}),
|
|
393
|
+
});
|
|
281
394
|
}
|
|
282
395
|
}
|
|
283
396
|
} finally {
|
|
@@ -307,9 +420,27 @@ async function analyzeProject(
|
|
|
307
420
|
|
|
308
421
|
logger.projectComplete(stats);
|
|
309
422
|
|
|
423
|
+
// Write analysis event for extension notification
|
|
424
|
+
const analysisEvent: AnalysisEvent = {
|
|
425
|
+
timestamp: new Date().toISOString(),
|
|
426
|
+
project_id: project.id,
|
|
427
|
+
project_name: project.name,
|
|
428
|
+
created: createdSummaries,
|
|
429
|
+
updated: updatedSummaries,
|
|
430
|
+
deleted: deletedSummaries,
|
|
431
|
+
};
|
|
432
|
+
appendAnalysisEvent(analysisEvent, baseDir);
|
|
433
|
+
|
|
310
434
|
saveProjectMeta(
|
|
311
435
|
project.id,
|
|
312
|
-
{
|
|
436
|
+
{
|
|
437
|
+
...meta,
|
|
438
|
+
last_analyzed_at: new Date().toISOString(),
|
|
439
|
+
last_observation_line_count: totalLineCount,
|
|
440
|
+
// Update AGENTS.md hashes only when the content was actually sent.
|
|
441
|
+
...(agentsMdProject && projectMdHash ? { agents_md_project_hash: projectMdHash } : {}),
|
|
442
|
+
...(agentsMdGlobal && globalMdHash ? { agents_md_global_hash: globalMdHash } : {}),
|
|
443
|
+
},
|
|
313
444
|
baseDir
|
|
314
445
|
);
|
|
315
446
|
|
package/src/confidence.ts
CHANGED
|
@@ -21,10 +21,16 @@ const OBS_BRACKET_MED_MAX = 5;
|
|
|
21
21
|
const OBS_BRACKET_HIGH_MAX = 10;
|
|
22
22
|
|
|
23
23
|
// adjustConfidence deltas
|
|
24
|
-
|
|
24
|
+
// Confirmation uses diminishing returns to prevent runaway confidence on trivially easy-to-confirm instincts.
|
|
25
|
+
const DELTA_CONFIRMED_TIER1 = 0.05; // 1st–3rd confirmation
|
|
26
|
+
const DELTA_CONFIRMED_TIER2 = 0.03; // 4th–6th confirmation
|
|
27
|
+
const DELTA_CONFIRMED_TIER3 = 0.01; // 7th+ confirmation
|
|
25
28
|
const DELTA_CONTRADICTED = -0.15;
|
|
26
29
|
const DELTA_INACTIVE = 0;
|
|
27
30
|
|
|
31
|
+
const CONFIRMED_TIER1_MAX = 3;
|
|
32
|
+
const CONFIRMED_TIER2_MAX = 6;
|
|
33
|
+
|
|
28
34
|
// applyPassiveDecay
|
|
29
35
|
// Increased from 0.02 to 0.05: at 0.5 confidence, reaches 0.1 in ~8 weeks instead of 20.
|
|
30
36
|
const DECAY_PER_WEEK = 0.05;
|
|
@@ -36,6 +42,16 @@ const MS_PER_WEEK = 7 * 24 * 60 * 60 * 1000;
|
|
|
36
42
|
|
|
37
43
|
export type FeedbackOutcome = "confirmed" | "contradicted" | "inactive";
|
|
38
44
|
|
|
45
|
+
/**
|
|
46
|
+
* Returns the confirmation confidence delta using diminishing returns.
|
|
47
|
+
* Higher confirmed_count yields smaller increments to prevent runaway scores.
|
|
48
|
+
*/
|
|
49
|
+
export function confirmationDelta(confirmedCount: number): number {
|
|
50
|
+
if (confirmedCount <= CONFIRMED_TIER1_MAX) return DELTA_CONFIRMED_TIER1;
|
|
51
|
+
if (confirmedCount <= CONFIRMED_TIER2_MAX) return DELTA_CONFIRMED_TIER2;
|
|
52
|
+
return DELTA_CONFIRMED_TIER3;
|
|
53
|
+
}
|
|
54
|
+
|
|
39
55
|
export interface ConfidenceResult {
|
|
40
56
|
confidence: number;
|
|
41
57
|
flaggedForRemoval: boolean;
|
|
@@ -71,18 +87,28 @@ export function initialConfidence(observationCount: number): number {
|
|
|
71
87
|
|
|
72
88
|
/**
|
|
73
89
|
* Adjusts confidence based on a feedback outcome from the observer loop.
|
|
90
|
+
* For "confirmed" outcomes, applies diminishing returns based on how many
|
|
91
|
+
* times the instinct has already been confirmed (higher count = smaller delta).
|
|
74
92
|
* Returns the clamped confidence and a flag indicating if removal is warranted.
|
|
93
|
+
*
|
|
94
|
+
* @param current - Current confidence value
|
|
95
|
+
* @param outcome - Feedback outcome type
|
|
96
|
+
* @param confirmedCount - Current confirmed_count (used for diminishing returns on confirmations)
|
|
75
97
|
*/
|
|
76
98
|
export function adjustConfidence(
|
|
77
99
|
current: number,
|
|
78
100
|
outcome: FeedbackOutcome,
|
|
101
|
+
confirmedCount = 0,
|
|
79
102
|
): ConfidenceResult {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
103
|
+
let delta: number;
|
|
104
|
+
if (outcome === "confirmed") {
|
|
105
|
+
delta = confirmationDelta(confirmedCount);
|
|
106
|
+
} else if (outcome === "contradicted") {
|
|
107
|
+
delta = DELTA_CONTRADICTED;
|
|
108
|
+
} else {
|
|
109
|
+
delta = DELTA_INACTIVE;
|
|
110
|
+
}
|
|
111
|
+
const raw = current + delta;
|
|
86
112
|
return toResult(raw);
|
|
87
113
|
}
|
|
88
114
|
|