pi-ghost-autocomplete 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/LICENSE +561 -0
  2. package/README.md +464 -331
  3. package/dist/bench-cli.js +1 -123
  4. package/dist/bench.d.ts +7 -3
  5. package/dist/bench.js +1 -522
  6. package/dist/config.d.ts +4 -1
  7. package/dist/context-aggregator.d.ts +4 -1
  8. package/dist/cycle-manager.d.ts +2 -21
  9. package/dist/entropy-stop.d.ts +4 -1
  10. package/dist/eval-dataset.d.ts +2 -13
  11. package/dist/fuzzy-matcher.d.ts +4 -1
  12. package/dist/ghost-editor.d.ts +11 -2
  13. package/dist/index.d.ts +4 -1
  14. package/dist/index.js +1 -289
  15. package/dist/intent.d.ts +179 -0
  16. package/dist/metrics.d.ts +4 -1
  17. package/dist/path-index.d.ts +4 -1
  18. package/dist/path-matcher.d.ts +4 -1
  19. package/dist/profile-store.d.ts +4 -1
  20. package/dist/profile.d.ts +2 -20
  21. package/dist/providers/cloud-provider.d.ts +4 -1
  22. package/dist/providers/local-provider.d.ts +4 -1
  23. package/dist/providers/openai-compat-provider.d.ts +24 -1
  24. package/dist/providers/race.d.ts +4 -1
  25. package/dist/providers/types.d.ts +13 -1
  26. package/dist/recent-paths.d.ts +4 -1
  27. package/dist/replay-cli.js +1 -166
  28. package/dist/replay.d.ts +2 -7
  29. package/package.json +90 -80
  30. package/dist/bench-cli.d.ts +0 -3
  31. package/dist/bench-cli.d.ts.map +0 -1
  32. package/dist/bench-cli.js.map +0 -1
  33. package/dist/bench.d.ts.map +0 -1
  34. package/dist/bench.js.map +0 -1
  35. package/dist/cache.d.ts +0 -124
  36. package/dist/cache.d.ts.map +0 -1
  37. package/dist/cache.js +0 -173
  38. package/dist/cache.js.map +0 -1
  39. package/dist/candidate-rerank.d.ts +0 -57
  40. package/dist/candidate-rerank.d.ts.map +0 -1
  41. package/dist/candidate-rerank.js +0 -78
  42. package/dist/candidate-rerank.js.map +0 -1
  43. package/dist/config.d.ts.map +0 -1
  44. package/dist/config.js +0 -317
  45. package/dist/config.js.map +0 -1
  46. package/dist/context-aggregator.d.ts.map +0 -1
  47. package/dist/context-aggregator.js +0 -87
  48. package/dist/context-aggregator.js.map +0 -1
  49. package/dist/context-selector.d.ts +0 -128
  50. package/dist/context-selector.d.ts.map +0 -1
  51. package/dist/context-selector.js +0 -300
  52. package/dist/context-selector.js.map +0 -1
  53. package/dist/cycle-manager.d.ts.map +0 -1
  54. package/dist/cycle-manager.js +0 -168
  55. package/dist/cycle-manager.js.map +0 -1
  56. package/dist/edit-history-ring.d.ts +0 -148
  57. package/dist/edit-history-ring.d.ts.map +0 -1
  58. package/dist/edit-history-ring.js +0 -262
  59. package/dist/edit-history-ring.js.map +0 -1
  60. package/dist/entropy-stop.d.ts.map +0 -1
  61. package/dist/entropy-stop.js +0 -141
  62. package/dist/entropy-stop.js.map +0 -1
  63. package/dist/eval-dataset.d.ts.map +0 -1
  64. package/dist/eval-dataset.js +0 -127
  65. package/dist/eval-dataset.js.map +0 -1
  66. package/dist/fast-path-engine.d.ts +0 -286
  67. package/dist/fast-path-engine.d.ts.map +0 -1
  68. package/dist/fast-path-engine.js +0 -341
  69. package/dist/fast-path-engine.js.map +0 -1
  70. package/dist/fuzzy-matcher.d.ts.map +0 -1
  71. package/dist/fuzzy-matcher.js +0 -162
  72. package/dist/fuzzy-matcher.js.map +0 -1
  73. package/dist/ghost-editor.d.ts.map +0 -1
  74. package/dist/ghost-editor.js +0 -1491
  75. package/dist/ghost-editor.js.map +0 -1
  76. package/dist/index.d.ts.map +0 -1
  77. package/dist/index.js.map +0 -1
  78. package/dist/keystroke-timer.d.ts +0 -59
  79. package/dist/keystroke-timer.d.ts.map +0 -1
  80. package/dist/keystroke-timer.js +0 -94
  81. package/dist/keystroke-timer.js.map +0 -1
  82. package/dist/metrics-builder.d.ts +0 -71
  83. package/dist/metrics-builder.d.ts.map +0 -1
  84. package/dist/metrics-builder.js +0 -71
  85. package/dist/metrics-builder.js.map +0 -1
  86. package/dist/metrics.d.ts.map +0 -1
  87. package/dist/metrics.js +0 -212
  88. package/dist/metrics.js.map +0 -1
  89. package/dist/path-index.d.ts.map +0 -1
  90. package/dist/path-index.js +0 -148
  91. package/dist/path-index.js.map +0 -1
  92. package/dist/path-matcher.d.ts.map +0 -1
  93. package/dist/path-matcher.js +0 -88
  94. package/dist/path-matcher.js.map +0 -1
  95. package/dist/profile-store.d.ts.map +0 -1
  96. package/dist/profile-store.js +0 -199
  97. package/dist/profile-store.js.map +0 -1
  98. package/dist/profile.d.ts.map +0 -1
  99. package/dist/profile.js +0 -434
  100. package/dist/profile.js.map +0 -1
  101. package/dist/providers/cloud-provider.d.ts.map +0 -1
  102. package/dist/providers/cloud-provider.js +0 -109
  103. package/dist/providers/cloud-provider.js.map +0 -1
  104. package/dist/providers/local-provider.d.ts.map +0 -1
  105. package/dist/providers/local-provider.js +0 -48
  106. package/dist/providers/local-provider.js.map +0 -1
  107. package/dist/providers/openai-compat-provider.d.ts.map +0 -1
  108. package/dist/providers/openai-compat-provider.js +0 -215
  109. package/dist/providers/openai-compat-provider.js.map +0 -1
  110. package/dist/providers/openai-completions-runner.d.ts +0 -52
  111. package/dist/providers/openai-completions-runner.d.ts.map +0 -1
  112. package/dist/providers/openai-completions-runner.js +0 -141
  113. package/dist/providers/openai-completions-runner.js.map +0 -1
  114. package/dist/providers/race.d.ts.map +0 -1
  115. package/dist/providers/race.js +0 -441
  116. package/dist/providers/race.js.map +0 -1
  117. package/dist/providers/truncate.d.ts +0 -7
  118. package/dist/providers/truncate.d.ts.map +0 -1
  119. package/dist/providers/truncate.js +0 -57
  120. package/dist/providers/truncate.js.map +0 -1
  121. package/dist/providers/types.d.ts.map +0 -1
  122. package/dist/providers/types.js +0 -60
  123. package/dist/providers/types.js.map +0 -1
  124. package/dist/provisional-manager.d.ts +0 -93
  125. package/dist/provisional-manager.d.ts.map +0 -1
  126. package/dist/provisional-manager.js +0 -98
  127. package/dist/provisional-manager.js.map +0 -1
  128. package/dist/provisional.d.ts +0 -150
  129. package/dist/provisional.d.ts.map +0 -1
  130. package/dist/provisional.js +0 -238
  131. package/dist/provisional.js.map +0 -1
  132. package/dist/recent-paths.d.ts.map +0 -1
  133. package/dist/recent-paths.js +0 -114
  134. package/dist/recent-paths.js.map +0 -1
  135. package/dist/regret-tracker.d.ts +0 -84
  136. package/dist/regret-tracker.d.ts.map +0 -1
  137. package/dist/regret-tracker.js +0 -145
  138. package/dist/regret-tracker.js.map +0 -1
  139. package/dist/replay-cli.d.ts +0 -13
  140. package/dist/replay-cli.d.ts.map +0 -1
  141. package/dist/replay-cli.js.map +0 -1
  142. package/dist/replay.d.ts.map +0 -1
  143. package/dist/replay.js +0 -225
  144. package/dist/replay.js.map +0 -1
  145. package/dist/reranker.d.ts +0 -181
  146. package/dist/reranker.d.ts.map +0 -1
  147. package/dist/reranker.js +0 -324
  148. package/dist/reranker.js.map +0 -1
  149. package/dist/session-trie.d.ts +0 -103
  150. package/dist/session-trie.d.ts.map +0 -1
  151. package/dist/session-trie.js +0 -280
  152. package/dist/session-trie.js.map +0 -1
  153. package/dist/speculative.d.ts +0 -74
  154. package/dist/speculative.d.ts.map +0 -1
  155. package/dist/speculative.js +0 -107
  156. package/dist/speculative.js.map +0 -1
  157. package/dist/tool-summarize.d.ts +0 -40
  158. package/dist/tool-summarize.d.ts.map +0 -1
  159. package/dist/tool-summarize.js +0 -245
  160. package/dist/tool-summarize.js.map +0 -1
  161. package/dist/trie-sanitize.d.ts +0 -22
  162. package/dist/trie-sanitize.d.ts.map +0 -1
  163. package/dist/trie-sanitize.js +0 -142
  164. package/dist/trie-sanitize.js.map +0 -1
  165. package/dist/versions.d.ts +0 -23
  166. package/dist/versions.d.ts.map +0 -1
  167. package/dist/versions.js +0 -31
  168. package/dist/versions.js.map +0 -1
@@ -1,262 +0,0 @@
1
- /**
2
- * Edit-history ring + aggregated index. Enhancement #1
3
- * (Next-Edit-Prediction motivated). See `docs/edit-history-signals-spec.md`.
4
- *
5
- * Records the most recent K edits (default 32) made in-session, then
6
- * exposes:
7
- * - a `snapshot()` of newest-first events with age filtering, and
8
- * - an `index()` of decayed token frequencies + per-URI line proximity.
9
- *
10
- * The reranker reads `index()` to compute `editProximityBoost`; the M6
11
- * context selector reads it (opt-in) for a token-overlap locality bit.
12
- *
13
- * Privacy: events are pre-sanitized at capture (`containsSensitive` on
14
- * the raw added/removed text — caller is responsible for invoking it
15
- * before calling `record`). Only tokens + scalars + URIs survive. No
16
- * raw text is ever stored on the ring. The ring lives in-process only —
17
- * no persistence in v0.5.
18
- */
19
- const TOKEN_RE = /\b[\w][\w-]*\b/g;
20
- export const TOKEN_CAP_PER_EVENT = 16;
21
- export const DEFAULT_CAPACITY = 32;
22
- export const DEFAULT_MAX_AGE_MS = 10 * 60 * 1000;
23
- export const DEFAULT_HALF_LIFE_MS = 2 * 60 * 1000;
24
- /**
25
- * Tokenize text using the same regex as `profile.ts` (`\b[\w][\w-]*\b`).
26
- * Code identifiers are case-sensitive — we do NOT lowercase. Result is
27
- * capped at `TOKEN_CAP_PER_EVENT` to bound memory per event.
28
- */
29
- export function tokenize(text) {
30
- if (!text)
31
- return [];
32
- const out = [];
33
- for (const m of text.matchAll(TOKEN_RE)) {
34
- if (m[0])
35
- out.push(m[0]);
36
- if (out.length >= TOKEN_CAP_PER_EVENT)
37
- break;
38
- }
39
- return out;
40
- }
41
- /**
42
- * Exponential time-decay weight: 1.0 at `ageMs=0`, halves every
43
- * `halfLifeMs`. Pure helper for tests + index().
44
- */
45
- export function timeDecayWeight(ageMs, halfLifeMs = DEFAULT_HALF_LIFE_MS) {
46
- if (ageMs <= 0)
47
- return 1;
48
- if (halfLifeMs <= 0)
49
- return 0;
50
- return Math.exp((-ageMs * Math.LN2) / halfLifeMs);
51
- }
52
- export class EditHistoryRing {
53
- events = [];
54
- capacity;
55
- maxAgeMs;
56
- halfLifeMs;
57
- constructor(opts = {}) {
58
- this.capacity = Math.max(1, Math.floor(opts.capacity ?? DEFAULT_CAPACITY));
59
- this.maxAgeMs = Math.max(0, opts.maxAgeMs ?? DEFAULT_MAX_AGE_MS);
60
- this.halfLifeMs = Math.max(1, opts.halfLifeMs ?? DEFAULT_HALF_LIFE_MS);
61
- }
62
- /**
63
- * Append an event. `ts` defaults to `Date.now()`. Caller is responsible
64
- * for sanitization — the ring does not re-validate. Overflow drops the
65
- * oldest event.
66
- */
67
- record(ev) {
68
- const stored = {
69
- ts: ev.ts ?? Date.now(),
70
- uri: ev.uri,
71
- rangeStartLine: ev.rangeStartLine,
72
- addedTokens: ev.addedTokens.slice(0, TOKEN_CAP_PER_EVENT),
73
- removedTokens: ev.removedTokens.slice(0, TOKEN_CAP_PER_EVENT),
74
- netCharDelta: ev.netCharDelta,
75
- };
76
- this.events.push(stored);
77
- while (this.events.length > this.capacity)
78
- this.events.shift();
79
- }
80
- /**
81
- * Newest-first list of events not older than `maxAgeMs`. Returns a
82
- * copy — safe for the caller to retain.
83
- *
84
- * Scans without early-break: we cannot assume `record()` was called in
85
- * ts-monotonic order (a backfill or test clock could insert older
86
- * events after newer ones). At capacity=32 the scan cost is trivial.
87
- */
88
- snapshot(now = Date.now()) {
89
- const cutoff = now - this.maxAgeMs;
90
- const out = [];
91
- for (let i = this.events.length - 1; i >= 0; i--) {
92
- const ev = this.events[i];
93
- if (ev.ts >= cutoff)
94
- out.push(ev);
95
- }
96
- return out;
97
- }
98
- /**
99
- * Aggregated index of recent edit signals, with exponential time-decay
100
- * applied to token frequencies. URIs touched within `maxAgeMs` are
101
- * collected; the *most recent* edit line per URI is kept (used by the
102
- * reranker for line-proximity scoring).
103
- */
104
- index(now = Date.now()) {
105
- const addedFreq = new Map();
106
- const removedFreq = new Map();
107
- const recentUris = new Set();
108
- const lastEditLineByUri = new Map();
109
- let totalEvents = 0;
110
- const events = this.snapshot(now);
111
- for (const ev of events) {
112
- const w = timeDecayWeight(now - ev.ts, this.halfLifeMs);
113
- for (const t of ev.addedTokens)
114
- addedFreq.set(t, (addedFreq.get(t) ?? 0) + w);
115
- for (const t of ev.removedTokens)
116
- removedFreq.set(t, (removedFreq.get(t) ?? 0) + w);
117
- if (ev.uri) {
118
- recentUris.add(ev.uri);
119
- if (!lastEditLineByUri.has(ev.uri)) {
120
- lastEditLineByUri.set(ev.uri, ev.rangeStartLine);
121
- }
122
- }
123
- totalEvents++;
124
- }
125
- return { addedFreq, removedFreq, recentUris, lastEditLineByUri, totalEvents };
126
- }
127
- clear() {
128
- this.events.length = 0;
129
- }
130
- get size() {
131
- return this.events.length;
132
- }
133
- }
134
- /** Default per-entry size cap. 1 MB — large enough for any realistic
135
- * source file, small enough that 16 × 1MB = 16MB worst-case is bounded. */
136
- export const DEFAULT_PRE_EDIT_MAX_BYTES = 1024 * 1024;
137
- /**
138
- * Per-document pre-edit text cache. Holds the most recent `capacity`
139
- * document texts so we can derive the *removed* slice of an edit before
140
- * the on-disk text mutates. LRU-bounded to avoid pinning every project
141
- * file in RAM. v0.5 stores transiently in process only — never persisted.
142
- *
143
- * Per-entry size is capped at `maxBytesPerEntry` (default 1 MB). Texts
144
- * exceeding the cap are stored as empty string — the next edit on that
145
- * document will derive empty `removedTokens`, which is the same
146
- * degradation as a fresh LRU miss. This bounds worst-case memory at
147
- * `capacity * maxBytesPerEntry` regardless of project file sizes.
148
- */
149
- export class PreEditTextCache {
150
- map = new Map();
151
- capacity;
152
- maxBytesPerEntry;
153
- constructor(capacity = 16, maxBytesPerEntry = DEFAULT_PRE_EDIT_MAX_BYTES) {
154
- this.capacity = Math.max(1, Math.floor(capacity));
155
- this.maxBytesPerEntry = Math.max(0, Math.floor(maxBytesPerEntry));
156
- }
157
- /** Returns the snapshot for `uri` (or `""` if absent) without touching LRU order. */
158
- peek(uri) {
159
- return this.map.get(uri) ?? "";
160
- }
161
- /**
162
- * Update the snapshot. Move-to-front semantics; oldest is evicted when
163
- * capacity is exceeded. Texts exceeding `maxBytesPerEntry` are stored
164
- * as empty string (the next edit on this URI gets no `removedTokens`,
165
- * matching the LRU-miss degradation path).
166
- *
167
- * Byte length uses UTF-16 code units (`text.length`), not UTF-8 bytes —
168
- * the cap is approximate but stable across machines.
169
- */
170
- update(uri, text) {
171
- const safe = text.length <= this.maxBytesPerEntry ? text : "";
172
- if (this.map.has(uri))
173
- this.map.delete(uri);
174
- this.map.set(uri, safe);
175
- while (this.map.size > this.capacity) {
176
- const oldest = this.map.keys().next().value;
177
- if (oldest === undefined)
178
- break;
179
- this.map.delete(oldest);
180
- }
181
- }
182
- clear() {
183
- this.map.clear();
184
- }
185
- get size() {
186
- return this.map.size;
187
- }
188
- }
189
- /**
190
- * Parse positive-integer env. Returns fallback when unset or invalid.
191
- * Bounded by `[1, max]`.
192
- */
193
- function parsePosIntEnv(raw, fallback, max) {
194
- if (raw === undefined)
195
- return fallback;
196
- const v = Number(raw);
197
- if (!Number.isFinite(v) || v < 1 || v > max)
198
- return fallback;
199
- return Math.floor(v);
200
- }
201
- /**
202
- * Spec §5.1 — compute the raw `editProximityScore` for a candidate. Pure
203
- * function. Returns 0 when the ring is empty (zero events). Output is
204
- * un-normalized: the reranker's min-max normalization re-maps the value
205
- * across the candidate set so absolute scale doesn't matter — only the
206
- * relative ordering does. The same un-normalized value is reported on
207
- * the `MetricRecord` for observability.
208
- *
209
- * `currentUri` / `currentLine` describe the user's cursor at predict
210
- * time. The prompt-buffer editor uses a static `"prompt"` URI sentinel
211
- * and the current cursor line.
212
- */
213
- export function editProximityScore(text, index, currentUri, currentLine) {
214
- if (index.totalEvents === 0)
215
- return 0;
216
- const tokens = tokenize(text);
217
- let added = 0;
218
- let removed = 0;
219
- for (const t of tokens) {
220
- added += index.addedFreq.get(t) ?? 0;
221
- removed += index.removedFreq.get(t) ?? 0;
222
- }
223
- const tokenSignal = Math.log(1 + added) - 0.5 * Math.log(1 + removed);
224
- const uriBoost = index.recentUris.has(currentUri) ? 1 : 0;
225
- const lastLine = index.lastEditLineByUri.get(currentUri);
226
- const lineProximity = lastLine === undefined ? 0 : Math.exp(-Math.abs(lastLine - currentLine) / 8);
227
- return tokenSignal + 0.5 * uriBoost + 0.3 * lineProximity;
228
- }
229
- /**
230
- * Pure character-diff helper. Returns the minimal `{ added, removed }`
231
- * slice between two strings by stripping the common prefix + suffix.
232
- * Caller is responsible for `containsSensitive` checks before tokenizing
233
- * the result. Returns empty strings for both sides when texts are equal.
234
- */
235
- export function diffStrings(prev, next) {
236
- if (prev === next)
237
- return { added: "", removed: "" };
238
- let p = 0;
239
- const maxP = Math.min(prev.length, next.length);
240
- while (p < maxP && prev.charCodeAt(p) === next.charCodeAt(p))
241
- p++;
242
- let s = 0;
243
- while (s < maxP - p &&
244
- prev.charCodeAt(prev.length - 1 - s) === next.charCodeAt(next.length - 1 - s)) {
245
- s++;
246
- }
247
- return {
248
- added: next.slice(p, next.length - s),
249
- removed: prev.slice(p, prev.length - s),
250
- };
251
- }
252
- export function readEditHistoryEnv(env = process.env) {
253
- const enabledRaw = env.PI_GHOST_EDIT_HISTORY;
254
- const enabled = enabledRaw === undefined || enabledRaw === "1";
255
- return {
256
- enabled,
257
- capacity: parsePosIntEnv(env.PI_GHOST_EDIT_HISTORY_CAP, DEFAULT_CAPACITY, 1024),
258
- maxAgeMs: parsePosIntEnv(env.PI_GHOST_EDIT_HISTORY_MAX_AGE_MS, DEFAULT_MAX_AGE_MS, 24 * 60 * 60 * 1000),
259
- halfLifeMs: parsePosIntEnv(env.PI_GHOST_EDIT_HISTORY_HALF_LIFE_MS, DEFAULT_HALF_LIFE_MS, 24 * 60 * 60 * 1000),
260
- };
261
- }
262
- //# sourceMappingURL=edit-history-ring.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"edit-history-ring.js","sourceRoot":"","sources":["../src/edit-history-ring.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,MAAM,QAAQ,GAAG,iBAAiB,CAAC;AACnC,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AACtC,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,CAAC;AACnC,MAAM,CAAC,MAAM,kBAAkB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACjD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAyBlD;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,CAAC,CAAC,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,GAAG,CAAC,MAAM,IAAI,mBAAmB;YAAE,MAAM;IAC9C,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa,EAAE,aAAqB,oBAAoB;IACvF,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,IAAI,UAAU,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,OAAO,eAAe;IACV,MAAM,GAAgB,EAAE,CAAC;IACjC,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,UAAU,CAAS;IAE5B,YAAY,OAAwB,EAAE;QACrC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,IAAI,gBAAgB,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,kBAAkB,CAAC,CAAC;QACjE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,IAAI,oBAAoB,CAAC,CAAC;IACxE,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,EAA2C;QACjD,MAAM,MAAM,GAAc;YACzB,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE;YACvB,GAAG,EAAE,EAAE,CAAC,GAAG;YACX,cAAc,EAAE,EAAE,CAAC,cAAc;YACjC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC;YACzD,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC;YAC7D,YAAY,EAAE,EAAE,CAAC,YAAY;SAC7B,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAChE,CAAC;IAED;;;;;;;OAOG;IACH,QAAQ,CAAC,MAAc,IAAI,CAAC,GAAG,EAAE;QAChC,MAAM,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QACnC,MAAM,GAAG,GAAgB,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;YAC3B,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM;gBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,GAAG,CAAC;IACZ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAc,IAAI,CAAC,GAAG,EAAE;QAC7B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC9C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QACrC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACpD,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAClC,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,eAAe,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACxD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW;gBAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9E,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,aAAa;gBAAE,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACpF,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC;gBACZ,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;gBACvB,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;oBACpC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC;gBAClD,CAAC;YACF,CAAC;YACD,WAAW,EAAE,CAAC;QACf,CAAC;QACD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,iBAAiB,EAAE,WAAW,EAAE,CAAC;IAC/E,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC3B,CAAC;CACD;AAED;4EAC4E;AAC5E,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,CAAC;AAEtD;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,gBAAgB;IACX,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,QAAQ,CAAS;IACjB,gBAAgB,CAAS;IAElC,YAAY,QAAQ,GAAG,EAAE,EAAE,mBAA2B,0BAA0B;QAC/E,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,qFAAqF;IACrF,IAAI,CAAC,GAAW;QACf,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,GAAW,EAAE,IAAY;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAC5C,IAAI,MAAM,KAAK,SAAS;gBAAE,MAAM;YAChC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;IACF,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;IACtB,CAAC;CACD;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAuB,EAAE,QAAgB,EAAE,GAAW;IAC7E,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,kBAAkB,CACjC,IAAY,EACZ,KAAuB,EACvB,UAAkB,EAClB,WAAmB;IAEnB,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACxB,KAAK,IAAI,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACzD,MAAM,aAAa,GAClB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9E,OAAO,WAAW,GAAG,GAAG,GAAG,QAAQ,GAAG,GAAG,GAAG,aAAa,CAAC;AAC3D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,IAAY;IACrD,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACrD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IAClE,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OACC,CAAC,GAAG,IAAI,GAAG,CAAC;QACZ,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,EAC5E,CAAC;QACF,CAAC,EAAE,CAAC;IACL,CAAC;IACD,OAAO;QACN,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACrC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;KACvC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,MAA0C,OAAO,CAAC,GAAG;IAErD,MAAM,UAAU,GAAG,GAAG,CAAC,qBAAqB,CAAC;IAC7C,MAAM,OAAO,GAAG,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,GAAG,CAAC;IAC/D,OAAO;QACN,OAAO;QACP,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC,yBAAyB,EAAE,gBAAgB,EAAE,IAAI,CAAC;QAC/E,QAAQ,EAAE,cAAc,CACvB,GAAG,CAAC,gCAAgC,EACpC,kBAAkB,EAClB,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CACnB;QACD,UAAU,EAAE,cAAc,CACzB,GAAG,CAAC,kCAAkC,EACtC,oBAAoB,EACpB,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CACnB;KACD,CAAC;AACH,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"entropy-stop.d.ts","sourceRoot":"","sources":["../src/entropy-stop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,eAAO,MAAM,qBAAqB,IAAI,CAAC;AACvC,eAAO,MAAM,kBAAkB,MAAM,CAAC;AACtC,eAAO,MAAM,kBAAkB,KAAK,CAAC;AAKrC,MAAM,WAAW,kBAAkB;IAClC,0DAA0D;IAC1D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC5B,iDAAiD;IACjD,IAAI,EAAE,OAAO,CAAC;IACd;;;;OAIG;IACH,MAAM,CAAC,EAAE,eAAe,GAAG,YAAY,CAAC;IACxC;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAiCD,qBAAa,cAAc;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,UAAU,CAAK;gBAEX,IAAI,GAAE,kBAAuB;IAMzC;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,YAAY;IAoBxD;;;;OAIG;IACH,QAAQ,IAAI;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE;CAO3E;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAOxD;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAC1B,MAAM,EAAE,QAAQ,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC,EACvD,IAAI,GAAE,kBAAuB,GAC3B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,eAAe,GAAG,YAAY,CAAA;CAAE,CAS7E"}
@@ -1,141 +0,0 @@
1
- /**
2
- * Entropy-based early stop. PRD §M7.
3
- *
4
- * Streams `(token, -logprob)` pairs; cuts when a token's `-logprob` spikes
5
- * above a learned baseline AND the token is whitespace/punctuation
6
- * (natural cut points). Returns the text up to (not including) the
7
- * cutting token.
8
- *
9
- * Pure state machine — no I/O, no provider coupling. Caller drives the
10
- * stream and acts on the returned `StopDecision`.
11
- *
12
- * Capability gate (PRD §M7):
13
- * Mercury Edit FIM: no logprobs → caller skips this module
14
- * OpenAI chat: supported
15
- * Groq/Ollama: model-dependent
16
- *
17
- * Disabled when fewer than `warmup` tokens have been seen — no statistical
18
- * baseline yet. Hard cap at `maxTokens` regardless.
19
- */
20
- export const DEFAULT_WARMUP_TOKENS = 5;
21
- export const DEFAULT_SIGMA_MULT = 1.5;
22
- export const DEFAULT_MAX_TOKENS = 64;
23
- /** Characters at which an entropy spike is allowed to terminate. */
24
- const NATURAL_CUT_PATTERN = /[\s.,;:!?)\]}>'"`]/;
25
- /**
26
- * Simple incremental mean+std (Welford's algorithm). One-pass over the
27
- * warmup window, stable for the small N=5 case the PRD specifies.
28
- */
29
- class RunningStats {
30
- n = 0;
31
- mean = 0;
32
- m2 = 0;
33
- push(x) {
34
- this.n++;
35
- const delta = x - this.mean;
36
- this.mean += delta / this.n;
37
- const delta2 = x - this.mean;
38
- this.m2 += delta * delta2;
39
- }
40
- get count() {
41
- return this.n;
42
- }
43
- get currentMean() {
44
- return this.mean;
45
- }
46
- get currentStd() {
47
- if (this.n < 2)
48
- return 0;
49
- return Math.sqrt(this.m2 / (this.n - 1));
50
- }
51
- }
52
- export class EntropyStopper {
53
- warmup;
54
- sigma;
55
- cap;
56
- stats = new RunningStats();
57
- tokensSeen = 0;
58
- constructor(opts = {}) {
59
- this.warmup = opts.warmupTokens ?? DEFAULT_WARMUP_TOKENS;
60
- this.sigma = opts.sigmaMult ?? DEFAULT_SIGMA_MULT;
61
- this.cap = opts.maxTokens ?? DEFAULT_MAX_TOKENS;
62
- }
63
- /**
64
- * Feed one streaming token + its `-logprob` (higher = more
65
- * surprising / less confident). Returns whether the caller should
66
- * stop the stream BEFORE this token. The caller's accumulated text
67
- * up to (not including) this token is the "accept" boundary.
68
- *
69
- * Order of evaluation:
70
- * 1. Hard cap: token index >= cap → stop
71
- * 2. Warmup: token index < warmup → feed stats, no stop
72
- * 3. Spike: -logprob > mean + sigma*std AND token is natural cut
73
- * → stop
74
- * 4. Else: feed stats, no stop
75
- */
76
- observe(token, neglogprob) {
77
- const idx = this.tokensSeen;
78
- this.tokensSeen++;
79
- if (idx >= this.cap) {
80
- return { stop: true, reason: "max-tokens" };
81
- }
82
- if (idx < this.warmup) {
83
- this.stats.push(neglogprob);
84
- return { stop: false };
85
- }
86
- const mean = this.stats.currentMean;
87
- const std = this.stats.currentStd;
88
- const threshold = mean + this.sigma * std;
89
- if (neglogprob > threshold && isNaturalCutToken(token)) {
90
- return { stop: true, reason: "entropy-spike", mean, std };
91
- }
92
- this.stats.push(neglogprob);
93
- return { stop: false, mean, std };
94
- }
95
- /**
96
- * Diagnostic snapshot — count + current mean/std. Returns `null`
97
- * mean/std until the warmup window has filled enough samples for a
98
- * meaningful estimate.
99
- */
100
- snapshot() {
101
- return {
102
- tokensSeen: this.tokensSeen,
103
- mean: this.stats.count >= 1 ? this.stats.currentMean : null,
104
- std: this.stats.count >= 2 ? this.stats.currentStd : null,
105
- };
106
- }
107
- }
108
- /**
109
- * True iff `token` is a natural cut point — whitespace, end-of-clause
110
- * punctuation, or a closing bracket. PRD §M7 limits entropy-driven
111
- * cutoffs to these to avoid mid-identifier truncation.
112
- */
113
- export function isNaturalCutToken(token) {
114
- if (!token)
115
- return false;
116
- // Token may be multiple characters; check the FIRST char so the
117
- // algorithm cuts at the natural boundary inside the token rather
118
- // than waiting for a pure-punctuation token.
119
- const ch = token[0];
120
- return NATURAL_CUT_PATTERN.test(ch);
121
- }
122
- /**
123
- * Run a stream through an entropy stopper. Returns the prefix of joined
124
- * tokens up to (but not including) the stop point. When the stream ends
125
- * naturally without a stop, returns the full concatenation.
126
- *
127
- * Exposed as a convenience for tests + future provider wiring; editor
128
- * integration is deferred until pi-ai exposes per-token logprob events.
129
- */
130
- export function runWithStop(tokens, opts = {}) {
131
- const stopper = new EntropyStopper(opts);
132
- let text = "";
133
- for (const { token, neglogprob } of tokens) {
134
- const d = stopper.observe(token, neglogprob);
135
- if (d.stop)
136
- return { text, stopped: true, reason: d.reason };
137
- text += token;
138
- }
139
- return { text, stopped: false };
140
- }
141
- //# sourceMappingURL=entropy-stop.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"entropy-stop.js","sourceRoot":"","sources":["../src/entropy-stop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AACvC,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AACtC,MAAM,CAAC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAErC,oEAAoE;AACpE,MAAM,mBAAmB,GAAG,oBAAoB,CAAC;AA6BjD;;;GAGG;AACH,MAAM,YAAY;IACT,CAAC,GAAG,CAAC,CAAC;IACN,IAAI,GAAG,CAAC,CAAC;IACT,EAAE,GAAG,CAAC,CAAC;IAEf,IAAI,CAAC,CAAS;QACb,IAAI,CAAC,CAAC,EAAE,CAAC;QACT,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;QAC7B,IAAI,CAAC,EAAE,IAAI,KAAK,GAAG,MAAM,CAAC;IAC3B,CAAC;IAED,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,CAAC,CAAC;IACf,CAAC;IAED,IAAI,WAAW;QACd,OAAO,IAAI,CAAC,IAAI,CAAC;IAClB,CAAC;IAED,IAAI,UAAU;QACb,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;CACD;AAED,MAAM,OAAO,cAAc;IACT,MAAM,CAAS;IACf,KAAK,CAAS;IACd,GAAG,CAAS;IACZ,KAAK,GAAG,IAAI,YAAY,EAAE,CAAC;IACpC,UAAU,GAAG,CAAC,CAAC;IAEvB,YAAY,OAA2B,EAAE;QACxC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,qBAAqB,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAClD,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,KAAa,EAAE,UAAkB;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC;QAC5B,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACrB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;QAC7C,CAAC;QACD,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACxB,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QACpC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;QAC1C,IAAI,UAAU,GAAG,SAAS,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACP,OAAO;YACN,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI;YAC3D,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;SACzD,CAAC;IACH,CAAC;CACD;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC9C,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,gEAAgE;IAChE,iEAAiE;IACjE,6CAA6C;IAC7C,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;IACrB,OAAO,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACrC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CAC1B,MAAuD,EACvD,OAA2B,EAAE;IAE7B,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,MAAM,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC7C,IAAI,CAAC,CAAC,IAAI;YAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QAC7D,IAAI,IAAI,KAAK,CAAC;IACf,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AACjC,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"eval-dataset.d.ts","sourceRoot":"","sources":["../src/eval-dataset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,eAAO,MAAM,QAAQ,sCAAsC,CAAC;AAE5D,wEAAwE;AACxE,eAAO,MAAM,uBAAuB,EAAE,SAAS,MAAM,EAAsB,CAAC;AAC5E,qEAAqE;AACrE,eAAO,MAAM,eAAe,KAAK,CAAC;AAClC,eAAO,MAAM,cAAc,KAAK,CAAC;AAEjC,MAAM,WAAW,WAAW;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,CAAC;IACjB,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,KAAK,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAehH;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACtC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE;IACL,OAAO,EAAE,MAAM,CAAC;IAChB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;CACrB,EACD,EAAE,GAAE,MAAmB,GACrB,WAAW,EAAE,CAef;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAUxF;AAED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAStE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,CAwBzD;AAED,wEAAwE;AACxE,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAEpE"}
@@ -1,127 +0,0 @@
1
- /**
2
- * Offline-eval dataset — capture + load. PRD §M5c.
3
- *
4
- * One JSONL example per committed prompt sample. The shape captures
5
- * everything a replay run needs to reproduce the original ghost call:
6
- * `(context, prefix, expected continuation, provider fingerprint)`.
7
- *
8
- * Privacy: examples DO contain raw prompt text. Capture is gated on the
9
- * `PI_GHOST_EVAL_CAPTURE=1` env (not a default behavior — opt-in only).
10
- * Sanitization still applies through `containsSensitive` so messages
11
- * with API keys are dropped at capture time. Downstream tooling (CLI
12
- * replay) ships a `--redact` flag that emits aggregate metrics without
13
- * raw examples for publication.
14
- */
15
- import { appendFile, mkdir, readFile } from "node:fs/promises";
16
- import { dirname, join } from "node:path";
17
- import { containsSensitive } from "./trie-sanitize.js";
18
- export const EVAL_REL = ".pi/ghost-autocomplete/eval.jsonl";
19
- /** Char positions at which to sample prefixes from a committed line. */
20
- export const PREFIX_SAMPLE_POSITIONS = [3, 5, 8, 12, 20];
21
- /** Beyond `LONG_LINE_START`, sample every `LONG_LINE_STEP` chars. */
22
- export const LONG_LINE_START = 20;
23
- export const LONG_LINE_STEP = 10;
24
- /**
25
- * Generate sample (prefix, expected) splits from a committed line. PRD
26
- * §M5c: positions `[3, 5, 8, 12, 20]` plus every 10 chars beyond 20.
27
- * Returns empty when the committed text is too short for any sample.
28
- */
29
- export function samplePrefixes(committed) {
30
- if (!committed)
31
- return [];
32
- const out = [];
33
- const seen = new Set();
34
- const len = committed.length;
35
- for (const pos of PREFIX_SAMPLE_POSITIONS) {
36
- if (pos >= len)
37
- break;
38
- seen.add(pos);
39
- out.push({ prefixLen: pos, prefix: committed.slice(0, pos), expected: committed.slice(pos) });
40
- }
41
- for (let pos = LONG_LINE_START + LONG_LINE_STEP; pos < len; pos += LONG_LINE_STEP) {
42
- if (seen.has(pos))
43
- continue;
44
- out.push({ prefixLen: pos, prefix: committed.slice(0, pos), expected: committed.slice(pos) });
45
- }
46
- return out;
47
- }
48
- /**
49
- * Build eval examples for a single commit. Drops sensitive messages
50
- * entirely. `context` should already be rendered by the caller (the
51
- * provider's recent-messages block joined into a single string) — keeps
52
- * this module decoupled from provider context shape.
53
- */
54
- export function buildExamplesFromCommit(committed, meta, ts = Date.now()) {
55
- if (!committed)
56
- return [];
57
- // Skip multi-line commits — sampling assumes a single source line.
58
- // First-line-only would distort expected continuations across newlines.
59
- if (committed.includes("\n"))
60
- return [];
61
- if (containsSensitive(committed))
62
- return [];
63
- if (containsSensitive(meta.context))
64
- return [];
65
- return samplePrefixes(committed).map((s) => ({
66
- ts,
67
- committed,
68
- prefixLen: s.prefixLen,
69
- prefix: s.prefix,
70
- expected: s.expected,
71
- ...meta,
72
- }));
73
- }
74
- export async function appendExamples(cwd, examples) {
75
- if (examples.length === 0)
76
- return;
77
- const path = join(cwd, EVAL_REL);
78
- try {
79
- await mkdir(dirname(path), { recursive: true });
80
- const body = examples.map((e) => JSON.stringify(e)).join("\n") + "\n";
81
- await appendFile(path, body, "utf8");
82
- }
83
- catch {
84
- // best-effort, never break completions
85
- }
86
- }
87
- export async function loadExamples(cwd) {
88
- const path = join(cwd, EVAL_REL);
89
- let body;
90
- try {
91
- body = await readFile(path, "utf8");
92
- }
93
- catch {
94
- return [];
95
- }
96
- return parseExamples(body);
97
- }
98
- export function parseExamples(body) {
99
- const out = [];
100
- for (const line of body.split("\n")) {
101
- if (!line)
102
- continue;
103
- try {
104
- const e = JSON.parse(line);
105
- if (typeof e.ts === "number" &&
106
- typeof e.committed === "string" &&
107
- typeof e.prefixLen === "number" &&
108
- typeof e.prefix === "string" &&
109
- typeof e.expected === "string" &&
110
- typeof e.context === "string" &&
111
- typeof e.providerFingerprint === "string" &&
112
- typeof e.versionFingerprint === "string" &&
113
- typeof e.providerName === "string") {
114
- out.push(e);
115
- }
116
- }
117
- catch {
118
- // partial trailing write — ignore
119
- }
120
- }
121
- return out;
122
- }
123
- /** Parse `PI_GHOST_EVAL_CAPTURE`. Returns true only when set to "1". */
124
- export function parseEvalCaptureEnv(raw) {
125
- return raw === "1";
126
- }
127
- //# sourceMappingURL=eval-dataset.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"eval-dataset.js","sourceRoot":"","sources":["../src/eval-dataset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,CAAC,MAAM,QAAQ,GAAG,mCAAmC,CAAC;AAE5D,wEAAwE;AACxE,MAAM,CAAC,MAAM,uBAAuB,GAAsB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;AAC5E,qEAAqE;AACrE,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAC;AAClC,MAAM,CAAC,MAAM,cAAc,GAAG,EAAE,CAAC;AAqBjC;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC/C,IAAI,CAAC,SAAS;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,GAAG,GAAmE,EAAE,CAAC;IAC/E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,uBAAuB,EAAE,CAAC;QAC3C,IAAI,GAAG,IAAI,GAAG;YAAE,MAAM;QACtB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,KAAK,IAAI,GAAG,GAAG,eAAe,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,IAAI,cAAc,EAAE,CAAC;QACnF,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CACtC,SAAiB,EACjB,IAKC,EACD,KAAa,IAAI,CAAC,GAAG,EAAE;IAEvB,IAAI,CAAC,SAAS;QAAE,OAAO,EAAE,CAAC;IAC1B,mEAAmE;IACnE,wEAAwE;IACxE,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,IAAI,iBAAiB,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IAC5C,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/C,OAAO,cAAc,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,EAAE;QACF,SAAS;QACT,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,GAAG,IAAI;KACP,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,QAAuB;IACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACjC,IAAI,CAAC;QACJ,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QACtE,MAAM,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACR,uCAAuC;IACxC,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW;IAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACjC,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,EAAE,CAAC;IACX,CAAC;IACD,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACzC,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,CAAC;YACJ,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAgB,CAAC;YAC1C,IACC,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ;gBACxB,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ;gBAC/B,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ;gBAC/B,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ;gBAC5B,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;gBAC9B,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ;gBAC7B,OAAO,CAAC,CAAC,mBAAmB,KAAK,QAAQ;gBACzC,OAAO,CAAC,CAAC,kBAAkB,KAAK,QAAQ;gBACxC,OAAO,CAAC,CAAC,YAAY,KAAK,QAAQ,EACjC,CAAC;gBACF,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACb,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,kCAAkC;QACnC,CAAC;IACF,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,mBAAmB,CAAC,GAAuB;IAC1D,OAAO,GAAG,KAAK,GAAG,CAAC;AACpB,CAAC"}