pi-context-engine 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -5
- package/package.json +1 -1
- package/src/config.ts +8 -0
- package/src/engine.ts +54 -7
- package/src/extension.ts +121 -0
- package/src/indexer/pipeline.ts +1 -2
- package/src/indexer/watcher.ts +195 -0
- package/src/retrieval/reranker.ts +15 -1
package/README.md
CHANGED
|
@@ -37,7 +37,7 @@ runs fixed queries. Run it yourself; nothing is cherry-picked.
|
|
|
37
37
|
|
|
38
38
|
## What you get
|
|
39
39
|
|
|
40
|
-
Ten tools the agent can call, plus
|
|
40
|
+
Ten tools the agent can call, plus four slash commands.
|
|
41
41
|
|
|
42
42
|
**Search**
|
|
43
43
|
- **`context_search`**: hybrid vector + BM25 retrieval with confidence scores.
|
|
@@ -56,7 +56,8 @@ Ten tools the agent can call, plus three slash commands.
|
|
|
56
56
|
- **`index_status`**: chunk count and measured savings.
|
|
57
57
|
- **`reindex`**: refresh a path or the whole project.
|
|
58
58
|
|
|
59
|
-
Commands: `/context-index [--full]`, `/context-status`, `/context-savings
|
|
59
|
+
Commands: `/context-index [--full]`, `/context-status`, `/context-savings`,
|
|
60
|
+
`/context-warmup`.
|
|
60
61
|
|
|
61
62
|
**Shipped guidance.** The package also injects a ~900-token guidance section
|
|
62
63
|
into the system prompt while `context_search` is active: how to formulate
|
|
@@ -95,12 +96,30 @@ unchanged repo took 237 ms.
|
|
|
95
96
|
it. You get a notification only when something changed.
|
|
96
97
|
- **Lazy fallback**: if a search still lands on an empty index, indexing kicks
|
|
97
98
|
off on the spot and the tool asks you to retry in a few seconds.
|
|
98
|
-
- **
|
|
99
|
-
|
|
99
|
+
- **Live reindex while you work**: a recursive file watcher re-indexes on
|
|
100
|
+
edits, saves and branch switches (debounced, so a `git checkout` collapses
|
|
101
|
+
into one run). The agent's own edits are searchable right away instead of
|
|
102
|
+
after the next session.
|
|
103
|
+
- **Switches**: `indexer.auto_index: false` disables the session-start and
|
|
104
|
+
lazy kicks; `indexer.watch: false` disables the file watcher. Use
|
|
105
|
+
`/context-index` for manual builds.
|
|
100
106
|
- **Why it matters**: incremental runs are cheap (a few hundred ms on an
|
|
101
107
|
unchanged repo), so this removes the classic cold-start failure of searching
|
|
102
108
|
an empty index, without ever blocking the UI.
|
|
103
109
|
|
|
110
|
+
### Model warmup
|
|
111
|
+
|
|
112
|
+
- **`/context-warmup`** downloads the embedding model up front, no project or
|
|
113
|
+
index needed. With `retrieval.rerank: local` it also fetches the reranker
|
|
114
|
+
model; with Ollama as the backend it forces the model pull through the same
|
|
115
|
+
command.
|
|
116
|
+
- **Why it matters**: the embedder initializes lazily, only when a run has
|
|
117
|
+
chunks to embed. An empty repo or an already up-to-date index never touches
|
|
118
|
+
the network, so on a fresh machine or an ephemeral container the download
|
|
119
|
+
lands inside the first search. Warming turns it into an explicit
|
|
120
|
+
provisioning step (a devcontainer `postCreateCommand`, a CI cache setup),
|
|
121
|
+
keeping first searches fast.
|
|
122
|
+
|
|
104
123
|
## Requirements
|
|
105
124
|
|
|
106
125
|
- **Node 22+** on macOS, Linux or Windows.
|
|
@@ -167,6 +186,7 @@ indexer:
|
|
|
167
186
|
chunking: legacy # legacy (default) | ast-density — cAST-style budgeted split-merge (experimental)
|
|
168
187
|
chunk_budget_chars: 1200 # non-whitespace budget per ast-density chunk
|
|
169
188
|
auto_index: true # incremental background reindex shortly after session start
|
|
189
|
+
watch: true # reindex on file changes while the session runs (debounced, incremental)
|
|
170
190
|
memory:
|
|
171
191
|
redact_pii: true
|
|
172
192
|
pricing:
|
|
@@ -247,7 +267,7 @@ bootstrapping for non-pi editors.
|
|
|
247
267
|
|
|
248
268
|
```bash
|
|
249
269
|
npm install
|
|
250
|
-
npm run check # typecheck +
|
|
270
|
+
npm run check # typecheck + 251 unit tests
|
|
251
271
|
npm run smoke:env # tree-sitter + sqlite-vec capability check
|
|
252
272
|
npm run smoke:loader # load the extension through pi's jiti loader
|
|
253
273
|
npm run smoke:real-world # index this repo with the local embedder
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-context-engine",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Semantic code context for pi: tree-sitter index + hybrid vector/BM25 retrieval + cross-session memory, all local. Port of code-context-engine (MIT, elara-labs) as a native pi extension.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package",
|
package/src/config.ts
CHANGED
|
@@ -95,6 +95,12 @@ export interface Config {
|
|
|
95
95
|
indexerContextHeaders: boolean;
|
|
96
96
|
/** Run an incremental reindex in the background shortly after session start. */
|
|
97
97
|
indexerAutoIndex: boolean;
|
|
98
|
+
/**
|
|
99
|
+
* Watch the project tree while a session runs and reindex on changes
|
|
100
|
+
* (debounced, incremental). Independent of `indexerAutoIndex`, which only
|
|
101
|
+
* gates the session-start kick.
|
|
102
|
+
*/
|
|
103
|
+
indexerWatch: boolean;
|
|
98
104
|
indexerRedactSecrets: boolean;
|
|
99
105
|
memoryRedactPii: boolean;
|
|
100
106
|
pricingModel: string;
|
|
@@ -128,6 +134,7 @@ export function defaultConfig(): Config {
|
|
|
128
134
|
indexerIgnore: [...DEFAULT_IGNORE],
|
|
129
135
|
indexerContextHeaders: true,
|
|
130
136
|
indexerAutoIndex: true,
|
|
137
|
+
indexerWatch: true,
|
|
131
138
|
indexerRedactSecrets: true,
|
|
132
139
|
memoryRedactPii: true,
|
|
133
140
|
pricingModel: "opus",
|
|
@@ -162,6 +169,7 @@ const MAPPING: Record<string, keyof Config> = {
|
|
|
162
169
|
"indexer.ignore": "indexerIgnore",
|
|
163
170
|
"indexer.context_headers": "indexerContextHeaders",
|
|
164
171
|
"indexer.auto_index": "indexerAutoIndex",
|
|
172
|
+
"indexer.watch": "indexerWatch",
|
|
165
173
|
"indexer.redact_secrets": "indexerRedactSecrets",
|
|
166
174
|
"memory.redact_pii": "memoryRedactPii",
|
|
167
175
|
"storage.path": "storagePath",
|
package/src/engine.ts
CHANGED
|
@@ -125,8 +125,19 @@ export interface SessionStats {
|
|
|
125
125
|
export interface EngineOptions {
|
|
126
126
|
projectDir: string;
|
|
127
127
|
config?: Config;
|
|
128
|
-
/** Test seam: reuse an existing backend/embedder instead of auto-detecting. */
|
|
129
|
-
runtime?: { backend: LocalBackend; embedder: Embedder };
|
|
128
|
+
/** Test seam: reuse an existing backend/embedder/reranker instead of auto-detecting. */
|
|
129
|
+
runtime?: { backend: LocalBackend; embedder: Embedder; reranker?: LocalReranker };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Issue #11 (upstream warmup): what /context-warmup resolved + cached. */
|
|
133
|
+
export interface WarmupReport {
|
|
134
|
+
backend: string;
|
|
135
|
+
model: string;
|
|
136
|
+
dimension: number;
|
|
137
|
+
/** Present only when retrieval.rerank is "local". */
|
|
138
|
+
reranker?: string;
|
|
139
|
+
rerankerReady?: boolean;
|
|
140
|
+
rerankerError?: string;
|
|
130
141
|
}
|
|
131
142
|
|
|
132
143
|
/** R13: cheap keyword extraction for decision tags (no LLM): distinctive
|
|
@@ -150,10 +161,13 @@ export class ContextEngine {
|
|
|
150
161
|
readonly projectName: string;
|
|
151
162
|
readonly storageBase: string;
|
|
152
163
|
config: Config;
|
|
153
|
-
private readonly injectedRuntime:
|
|
164
|
+
private readonly injectedRuntime:
|
|
165
|
+
| { backend: LocalBackend; embedder: Embedder; reranker?: LocalReranker }
|
|
166
|
+
| undefined;
|
|
154
167
|
private backendInstance: LocalBackend | null = null;
|
|
155
168
|
private embedderInstance: Embedder | null = null;
|
|
156
169
|
private retrieverInstance: HybridRetriever | null = null;
|
|
170
|
+
private rerankerInstance: LocalReranker | null = null;
|
|
157
171
|
private compressorInstance: Compressor | null = null;
|
|
158
172
|
private memoryInstance: MemoryDb | null = null;
|
|
159
173
|
private memoryFailed = false;
|
|
@@ -210,15 +224,48 @@ export class ContextEngine {
|
|
|
210
224
|
this.retrieverInstance = new HybridRetriever(this.backend, await this.getEmbedder(), {
|
|
211
225
|
expandContext: this.config.retrievalExpandContext,
|
|
212
226
|
autoElaborate: this.config.retrievalAutoElaborate,
|
|
213
|
-
reranker:
|
|
214
|
-
this.config.retrievalRerank === "local"
|
|
215
|
-
? new LocalReranker(this.config.retrievalRerankModel)
|
|
216
|
-
: null,
|
|
227
|
+
reranker: this.getReranker(),
|
|
217
228
|
});
|
|
218
229
|
}
|
|
219
230
|
return this.retrieverInstance;
|
|
220
231
|
}
|
|
221
232
|
|
|
233
|
+
/**
|
|
234
|
+
* The opt-in local reranker, or null when reranking is off. Memoized so
|
|
235
|
+
* /context-warmup loads the SAME instance the retriever later searches
|
|
236
|
+
* with — the model stays warm across the command boundary.
|
|
237
|
+
*/
|
|
238
|
+
private getReranker(): LocalReranker | null {
|
|
239
|
+
if (this.config.retrievalRerank !== "local") return null;
|
|
240
|
+
this.rerankerInstance ??= this.injectedRuntime?.reranker ?? new LocalReranker(this.config.retrievalRerankModel);
|
|
241
|
+
return this.rerankerInstance;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Warm the model caches (issue #11, upstream elara-labs#168): resolve +
|
|
246
|
+
* download + dimension-probe the embedding backend, and — when the opt-in
|
|
247
|
+
* local reranker is configured — its cross-encoder too. Unlike a reindex
|
|
248
|
+
* on an empty or fully up-to-date repo (which never initializes the
|
|
249
|
+
* embedder and therefore never touches the network), this guarantees the
|
|
250
|
+
* models are cached, so the first search or index doesn't pay the fetch
|
|
251
|
+
* inline. Ollama pulls ride the existing ensureModel() path.
|
|
252
|
+
*/
|
|
253
|
+
async warmup(): Promise<WarmupReport> {
|
|
254
|
+
const embedder = await this.getEmbedder();
|
|
255
|
+
const report: WarmupReport = {
|
|
256
|
+
backend: embedder.backendName,
|
|
257
|
+
model: embedder.modelName,
|
|
258
|
+
dimension: embedder.dimension,
|
|
259
|
+
};
|
|
260
|
+
const reranker = this.getReranker();
|
|
261
|
+
if (reranker) {
|
|
262
|
+
report.reranker = reranker.model;
|
|
263
|
+
report.rerankerReady = await reranker.warm();
|
|
264
|
+
if (!report.rerankerReady) report.rerankerError = reranker.loadError ?? "unknown error";
|
|
265
|
+
}
|
|
266
|
+
return report;
|
|
267
|
+
}
|
|
268
|
+
|
|
222
269
|
async getCompressor(): Promise<Compressor> {
|
|
223
270
|
if (!this.compressorInstance) {
|
|
224
271
|
this.compressorInstance = new Compressor(
|
package/src/extension.ts
CHANGED
|
@@ -12,6 +12,7 @@ import fs from "node:fs";
|
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { ContextEngine, isPceConfig } from "./engine.js";
|
|
14
14
|
import { applyContextSearchGuidance } from "./guidance.js";
|
|
15
|
+
import { ProjectWatcher } from "./indexer/watcher.js";
|
|
15
16
|
|
|
16
17
|
/** Tools whose payloads are worth capturing for session timelines. */
|
|
17
18
|
const CAPTURED_TOOLS = new Set(["bash", "read", "edit", "write", "grep", "find", "ls"]);
|
|
@@ -32,6 +33,100 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
|
|
|
32
33
|
let autoIndexTimer: NodeJS.Timeout | null = null;
|
|
33
34
|
let indexInFlight = false;
|
|
34
35
|
|
|
36
|
+
/**
|
|
37
|
+
* File watcher (config `indexer.watch`, default true): a debounced change
|
|
38
|
+
* signal over the project tree. Unlike the session-start auto-index this
|
|
39
|
+
* is re-kickable — one kick per settled change burst, for the whole
|
|
40
|
+
* session. Owned by this closure, like `autoIndexTimer`: engine instances
|
|
41
|
+
* are replaced per `session_start` without a shutdown event.
|
|
42
|
+
*/
|
|
43
|
+
let watcher: ProjectWatcher | null = null;
|
|
44
|
+
let watcherKickPending = false;
|
|
45
|
+
let watcherNotify: ((msg: string, level: "info" | "warning") => void) | undefined;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Watcher-triggered incremental reindex. Unlike `kickBackgroundIndex` this
|
|
49
|
+
* calls `reindex()` directly — the watcher must re-fire all session, not
|
|
50
|
+
* consume a one-shot. Shares `indexInFlight` with the other kicks so runs
|
|
51
|
+
* never stack; a change that arrives mid-run coalesces into exactly one
|
|
52
|
+
* trailing kick, so edits made *while indexing* still land. Successful
|
|
53
|
+
* runs stay silent (the user's own edit caused them); failures — both
|
|
54
|
+
* rejections AND per-file errors resolved in `result.errors` — notify.
|
|
55
|
+
*/
|
|
56
|
+
const kickWatcherIndex = (eng: ContextEngine, notify?: (msg: string, level: "info" | "warning") => void) => {
|
|
57
|
+
if (engine !== eng) return; // session moved on
|
|
58
|
+
if (indexInFlight) {
|
|
59
|
+
watcherKickPending = true;
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
indexInFlight = true;
|
|
63
|
+
void eng
|
|
64
|
+
.reindex({})
|
|
65
|
+
.then((result) => {
|
|
66
|
+
// Per-file chunking/read failures RESOLVE in result.errors — mirroring
|
|
67
|
+
// kickBackgroundIndex, surface them or affected files go stale silently.
|
|
68
|
+
if (engine !== eng || !notify) return; // session moved on
|
|
69
|
+
if (result.errors.length > 0) {
|
|
70
|
+
notify(`Context index: watcher reindex hit ${result.errors.length} error(s)`, "warning");
|
|
71
|
+
}
|
|
72
|
+
})
|
|
73
|
+
.catch((err: unknown) => {
|
|
74
|
+
if (engine === eng && notify) {
|
|
75
|
+
const detail = (err instanceof Error ? err.message : String(err)).slice(0, 120);
|
|
76
|
+
notify(`Context index: watcher reindex failed (${detail})`, "warning");
|
|
77
|
+
}
|
|
78
|
+
})
|
|
79
|
+
.finally(() => {
|
|
80
|
+
indexInFlight = false;
|
|
81
|
+
drainWatcherKick();
|
|
82
|
+
});
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
/** Deliver a watcher change that arrived while a background run was in flight. */
|
|
86
|
+
const drainWatcherKick = () => {
|
|
87
|
+
if (!watcherKickPending) return;
|
|
88
|
+
watcherKickPending = false;
|
|
89
|
+
if (engine && watcher) kickWatcherIndex(engine, watcherNotify);
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const stopWatcher = () => {
|
|
93
|
+
watcher?.stop();
|
|
94
|
+
watcher = null;
|
|
95
|
+
watcherKickPending = false;
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* (Re)point the file watcher at the live engine's project dir. Idempotent
|
|
100
|
+
* per dir (called from every `getEngine`), so a mid-session project switch
|
|
101
|
+
* re-arms the watcher exactly like the auto-index timer re-points its kick.
|
|
102
|
+
*/
|
|
103
|
+
const attachWatcher = (eng: ContextEngine) => {
|
|
104
|
+
if (watcher && watcher.dir === eng.projectDir) return;
|
|
105
|
+
stopWatcher();
|
|
106
|
+
if (!eng.config.indexerWatch) return;
|
|
107
|
+
// An in-project storage dir (storage.path inside the checkout) is written
|
|
108
|
+
// on EVERY reindex run (manifest.save()), so its events must not be
|
|
109
|
+
// accepted — otherwise each kick schedules the next one forever.
|
|
110
|
+
const storageRel = path.relative(eng.projectDir, eng.storageBase);
|
|
111
|
+
const storageInsideProject = storageRel !== "" && !storageRel.startsWith("..") && !path.isAbsolute(storageRel);
|
|
112
|
+
const w = new ProjectWatcher(eng.projectDir, {
|
|
113
|
+
// Same sets the walker enforces: any-depth + root-only semantics,
|
|
114
|
+
// user additions to indexer.ignore included, case-sensitive.
|
|
115
|
+
ignoreSegments: eng.config.indexerIgnore,
|
|
116
|
+
ignorePaths: storageInsideProject ? [eng.storageBase] : [],
|
|
117
|
+
onChange: () => {
|
|
118
|
+
if (engine) kickWatcherIndex(engine, watcherNotify);
|
|
119
|
+
},
|
|
120
|
+
onError: (err) => {
|
|
121
|
+
// E.g. Linux inotify exhaustion: degrade to session-start/lazy
|
|
122
|
+
// indexing and say so once. Never crash the session.
|
|
123
|
+
stopWatcher();
|
|
124
|
+
watcherNotify?.(`Context index: file watcher disabled (${err.message.slice(0, 100)}) — back to session-start indexing only`, "warning");
|
|
125
|
+
},
|
|
126
|
+
});
|
|
127
|
+
if (w.start()) watcher = w;
|
|
128
|
+
};
|
|
129
|
+
|
|
35
130
|
/**
|
|
36
131
|
* Kick the one-shot background index (session-start auto-index and the
|
|
37
132
|
* first-search lazy path share this). Never stacks: `lazyIndexStarted`
|
|
@@ -69,6 +164,7 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
|
|
|
69
164
|
})
|
|
70
165
|
.finally(() => {
|
|
71
166
|
indexInFlight = false;
|
|
167
|
+
drainWatcherKick(); // a watcher change may have landed mid-run
|
|
72
168
|
});
|
|
73
169
|
};
|
|
74
170
|
|
|
@@ -82,6 +178,7 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
|
|
|
82
178
|
// Always track the live engine so `session_shutdown` closes it even when
|
|
83
179
|
// a tool ran before `session_start` (tests, mid-reload edge cases).
|
|
84
180
|
engine = new ContextEngine({ projectDir: dir });
|
|
181
|
+
attachWatcher(engine);
|
|
85
182
|
return engine;
|
|
86
183
|
};
|
|
87
184
|
|
|
@@ -364,6 +461,22 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
|
|
|
364
461
|
},
|
|
365
462
|
});
|
|
366
463
|
|
|
464
|
+
pi.registerCommand("context-warmup", {
|
|
465
|
+
description: "Pre-download the embedding (and opt-in reranker) model so the first search doesn't fetch inline",
|
|
466
|
+
handler: async (_args, ctx) => {
|
|
467
|
+
ctx.ui.notify("Warming up embedding model…", "info");
|
|
468
|
+
const report = await getEngine(ctx.cwd).warmup();
|
|
469
|
+
const parts = [`${report.backend} ${report.model} (${report.dimension} dims)`];
|
|
470
|
+
if (report.reranker) {
|
|
471
|
+
parts.push(report.rerankerReady ? `reranker ${report.reranker} ready` : `reranker ${report.reranker} FAILED: ${report.rerankerError}`);
|
|
472
|
+
}
|
|
473
|
+
ctx.ui.notify(
|
|
474
|
+
`✓ Warmup complete: ${parts.join(" · ")}`,
|
|
475
|
+
report.reranker && !report.rerankerReady ? "warning" : "info",
|
|
476
|
+
);
|
|
477
|
+
},
|
|
478
|
+
});
|
|
479
|
+
|
|
367
480
|
/* ── Lifecycle hooks (replace upstream's curl-based hooks) ───────────── */
|
|
368
481
|
|
|
369
482
|
pi.on("session_start", async (_event, ctx) => {
|
|
@@ -372,6 +485,12 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
|
|
|
372
485
|
engine.startBackgroundWork();
|
|
373
486
|
lazyIndexStarted = false;
|
|
374
487
|
bootstrapSent = false;
|
|
488
|
+
watcherNotify = ctx.hasUI ? (msg, level) => ctx.ui.notify(msg, level) : undefined;
|
|
489
|
+
// (Re)arm the file watcher for this session: fresh notify channel, and
|
|
490
|
+
// config may have changed since the last session. No-op when
|
|
491
|
+
// `indexer.watch` is off.
|
|
492
|
+
stopWatcher();
|
|
493
|
+
attachWatcher(engine);
|
|
375
494
|
if (autoIndexTimer) {
|
|
376
495
|
clearTimeout(autoIndexTimer);
|
|
377
496
|
autoIndexTimer = null;
|
|
@@ -455,6 +574,8 @@ export default function contextEngineExtension(pi: ExtensionAPI) {
|
|
|
455
574
|
clearTimeout(autoIndexTimer);
|
|
456
575
|
autoIndexTimer = null;
|
|
457
576
|
}
|
|
577
|
+
stopWatcher();
|
|
578
|
+
watcherNotify = undefined;
|
|
458
579
|
await engine?.shutdown().catch(() => undefined);
|
|
459
580
|
engine = null;
|
|
460
581
|
});
|
package/src/indexer/pipeline.ts
CHANGED
|
@@ -368,8 +368,7 @@ export async function indexCommits(
|
|
|
368
368
|
|
|
369
369
|
/**
|
|
370
370
|
* Per-storage-dir indexing lock so concurrent reindex runs (auto-index kick,
|
|
371
|
-
* lazy search kick, manual /context-index
|
|
372
|
-
* can't interleave.
|
|
371
|
+
* lazy search kick, file watcher, manual /context-index) can't interleave.
|
|
373
372
|
*/
|
|
374
373
|
const pipelineLocks = new Map<string, Promise<unknown>>();
|
|
375
374
|
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Real-time reindex trigger — issue #12. A thin debounced wrapper around
|
|
3
|
+
* Node's built-in `fs.watch(dir, { recursive: true })` (one OS handle on
|
|
4
|
+
* Windows/macOS, inotify tree on Linux since Node 19.1).
|
|
5
|
+
*
|
|
6
|
+
* The watcher only produces a *change signal*: the incremental pipeline
|
|
7
|
+
* re-walks and re-hashes anyway, so false positives cost one cheap walk and
|
|
8
|
+
* only missed events matter. That is why the built-in suffices and neither
|
|
9
|
+
* chokidar (per-file stat-confirmed events, Node >= 22.22) nor
|
|
10
|
+
* @parcel/watcher (native addon, snapshot diffs) are needed.
|
|
11
|
+
*
|
|
12
|
+
* Filter semantics deliberately MIRROR the pipeline's inclusion policy
|
|
13
|
+
* (`targetExcluded` / `iterProjectFiles` in pipeline.ts), because a dropped
|
|
14
|
+
* event is the only unrecoverable failure mode: a wrongly accepted event just
|
|
15
|
+
* costs one cheap incremental run.
|
|
16
|
+
*/
|
|
17
|
+
import fs from "node:fs";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import { DEFAULT_IGNORE, ROOT_ONLY_IGNORE } from "../config.js";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Trailing-edge debounce for watcher events: long enough that a burst
|
|
23
|
+
* (editor save, `git checkout` rewriting many files) collapses into one
|
|
24
|
+
* kick, short enough that retrieval catches up quickly after an edit.
|
|
25
|
+
*/
|
|
26
|
+
export const WATCH_DEBOUNCE_MS = 1500;
|
|
27
|
+
|
|
28
|
+
const ROOT_ONLY_SET = new Set(ROOT_ONLY_IGNORE);
|
|
29
|
+
|
|
30
|
+
export interface ProjectWatcherOptions {
|
|
31
|
+
/** Called once per debounced change burst. Keep it cheap and sync. */
|
|
32
|
+
onChange: () => void;
|
|
33
|
+
/** Fatal watcher problem (e.g. Linux ENOSPC inotify exhaustion). The watcher closes itself; the caller falls back to session-start/lazy indexing. */
|
|
34
|
+
onError?: (err: Error) => void;
|
|
35
|
+
/** Defaults to {@link WATCH_DEBOUNCE_MS}. */
|
|
36
|
+
debounceMs?: number;
|
|
37
|
+
/**
|
|
38
|
+
* Segment names ignored at ANY depth — defaults to `DEFAULT_IGNORE`; the
|
|
39
|
+
* extension passes `config.indexerIgnore` so user additions apply too
|
|
40
|
+
* (same set the walker checks via `ignoreSet.has(seg)`).
|
|
41
|
+
*/
|
|
42
|
+
ignoreSegments?: readonly string[];
|
|
43
|
+
/**
|
|
44
|
+
* Absolute directories whose events are dropped entirely. The extension
|
|
45
|
+
* passes the engine's storage dir when it points INSIDE the checkout:
|
|
46
|
+
* every reindex rewrites `manifest.json` there unconditionally, so
|
|
47
|
+
* without this exclusion a watcher kick would re-trigger itself forever
|
|
48
|
+
* after source edits stop (PR #15 review, #12).
|
|
49
|
+
*/
|
|
50
|
+
ignorePaths?: readonly string[];
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export class ProjectWatcher {
|
|
54
|
+
private watcher: fs.FSWatcher | null = null;
|
|
55
|
+
private debounceTimer: NodeJS.Timeout | null = null;
|
|
56
|
+
private stopped = false;
|
|
57
|
+
private watchRoot: string | null = null;
|
|
58
|
+
private readonly ignoreSet: Set<string>;
|
|
59
|
+
private ignorePaths: string[] = [];
|
|
60
|
+
|
|
61
|
+
constructor(
|
|
62
|
+
readonly dir: string,
|
|
63
|
+
private readonly options: ProjectWatcherOptions,
|
|
64
|
+
) {
|
|
65
|
+
this.ignoreSet = new Set(options.ignoreSegments ?? DEFAULT_IGNORE);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** True while an OS watch handle is held. */
|
|
69
|
+
get watching(): boolean {
|
|
70
|
+
return this.watcher !== null;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Start watching. Returns false (after reporting via `onError`) when the watch could not be established — never throws. */
|
|
74
|
+
start(): boolean {
|
|
75
|
+
if (this.watcher) return true;
|
|
76
|
+
let watcher: fs.FSWatcher;
|
|
77
|
+
try {
|
|
78
|
+
// Canonicalize 8.3 short names first: on Windows,
|
|
79
|
+
// fs.watch({recursive}) hits an uncatchable libuv assertion
|
|
80
|
+
// (fs-event.c: `filename` vs watched dir prefix mismatch) that ABORTS
|
|
81
|
+
// the whole host process when the watched path has short-name
|
|
82
|
+
// components (common for os.tmpdir() and some tool-set cwds) and an
|
|
83
|
+
// event then reports the long name. realpathSync.native returns the
|
|
84
|
+
// canonical long path and avoids the mismatch entirely.
|
|
85
|
+
const dir = fs.realpathSync.native(this.dir);
|
|
86
|
+
this.watchRoot = dir;
|
|
87
|
+
this.ignorePaths = (this.options.ignorePaths ?? []).map((p) => {
|
|
88
|
+
try {
|
|
89
|
+
return fs.realpathSync.native(p);
|
|
90
|
+
} catch {
|
|
91
|
+
return path.resolve(p);
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
watcher = fs.watch(dir, { recursive: true }, (_event, filename) => this.onEvent(filename));
|
|
95
|
+
} catch (err) {
|
|
96
|
+
this.options.onError?.(err as Error);
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
this.watcher = watcher;
|
|
100
|
+
watcher.on("error", (err) => {
|
|
101
|
+
// E.g. inotify watch exhaustion on huge Linux trees. Close and let
|
|
102
|
+
// the caller degrade to session-start/lazy indexing — a watcher
|
|
103
|
+
// problem must never crash or stall the session.
|
|
104
|
+
this.closeWatcher();
|
|
105
|
+
if (!this.stopped) this.options.onError?.(err);
|
|
106
|
+
});
|
|
107
|
+
// Never keep the host process alive just for indexing.
|
|
108
|
+
watcher.unref?.();
|
|
109
|
+
return true;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
private onEvent(filename: string | Buffer | null): void {
|
|
113
|
+
if (this.stopped || !this.watcher) return;
|
|
114
|
+
// No filename (happens, e.g. Linux inotify): cannot filter — a missed
|
|
115
|
+
// event is the unrecoverable failure mode, so accept and debounce it.
|
|
116
|
+
if (filename === null) {
|
|
117
|
+
this.scheduleKick();
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
const name = Buffer.isBuffer(filename) ? filename.toString("utf-8") : filename;
|
|
121
|
+
if (name !== "" && this.isNoise(name)) return;
|
|
122
|
+
this.scheduleKick();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Mirrors the walker's inclusion policy (see header): case-sensitive
|
|
127
|
+
* segment match at any depth, root-only names only for the FIRST segment,
|
|
128
|
+
* and explicit storage-dir path exclusion. Returns "noise" = drop event.
|
|
129
|
+
*/
|
|
130
|
+
private isNoise(filename: string): boolean {
|
|
131
|
+
const rel = this.relativeToRoot(filename);
|
|
132
|
+
if (rel === null) return false; // absolute path outside the watch root: be conservative, kick
|
|
133
|
+
const segments = rel.split(/[\\/]/).filter((s) => s !== "");
|
|
134
|
+
if (segments.length === 0) return false;
|
|
135
|
+
// Root-only names (`storage`, `logs`, `tmp`, ...) count only at the
|
|
136
|
+
// project root — nested `src/storage/` is real source the pipeline
|
|
137
|
+
// indexes, so its events must kick (PR #15 review).
|
|
138
|
+
if (ROOT_ONLY_SET.has(segments[0]!)) return true;
|
|
139
|
+
// Case-SENSITIVE like `ignoreSet.has(seg)`: the pipeline indexes
|
|
140
|
+
// `src/Build/`, so its events must not be dropped (PR #15 review).
|
|
141
|
+
if (segments.some((s) => this.ignoreSet.has(s))) return true;
|
|
142
|
+
return this.isUnderIgnoredPath(rel);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Relative path from the watch root, or null when outside/unknown. */
|
|
146
|
+
private relativeToRoot(filename: string): string | null {
|
|
147
|
+
if (!path.isAbsolute(filename)) return filename;
|
|
148
|
+
const rel = path.relative(this.watchRoot ?? this.dir, filename);
|
|
149
|
+
if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) return null;
|
|
150
|
+
return rel;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** True when the event's resolved path lives under an ignored dir (e.g. an in-project storage dir). */
|
|
154
|
+
private isUnderIgnoredPath(rel: string): boolean {
|
|
155
|
+
if (this.ignorePaths.length === 0 || this.watchRoot === null) return false;
|
|
156
|
+
const abs = path.resolve(this.watchRoot, rel);
|
|
157
|
+
const fold = (p: string) => (process.platform === "win32" ? p.toLowerCase() : p);
|
|
158
|
+
const absFolded = fold(abs);
|
|
159
|
+
return this.ignorePaths.some((p) => {
|
|
160
|
+
const ignored = fold(p);
|
|
161
|
+
return absFolded === ignored || absFolded.startsWith(ignored + path.sep);
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private scheduleKick(): void {
|
|
166
|
+
if (this.debounceTimer) clearTimeout(this.debounceTimer);
|
|
167
|
+
this.debounceTimer = setTimeout(() => {
|
|
168
|
+
this.debounceTimer = null;
|
|
169
|
+
// `stopped` mid-debounce, or the watcher died (error path) while
|
|
170
|
+
// the timer was pending: no kick.
|
|
171
|
+
if (this.stopped || !this.watcher) return;
|
|
172
|
+
this.options.onChange();
|
|
173
|
+
}, this.options.debounceMs ?? WATCH_DEBOUNCE_MS);
|
|
174
|
+
this.debounceTimer.unref?.();
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Stop watching and cancel any pending debounced kick. Idempotent. */
|
|
178
|
+
stop(): void {
|
|
179
|
+
this.stopped = true;
|
|
180
|
+
if (this.debounceTimer) {
|
|
181
|
+
clearTimeout(this.debounceTimer);
|
|
182
|
+
this.debounceTimer = null;
|
|
183
|
+
}
|
|
184
|
+
this.closeWatcher();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
private closeWatcher(): void {
|
|
188
|
+
try {
|
|
189
|
+
this.watcher?.close();
|
|
190
|
+
} catch {
|
|
191
|
+
// already closed by the runtime
|
|
192
|
+
}
|
|
193
|
+
this.watcher = null;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
@@ -56,6 +56,8 @@ export class LocalReranker implements RerankerLike {
|
|
|
56
56
|
private loading: Promise<LoadedReranker | null> | null = null;
|
|
57
57
|
/** Set after a load failure; every later call short-circuits to no-op. */
|
|
58
58
|
failed = false;
|
|
59
|
+
/** Why the load failed (null before any attempt). Surfaced by /context-warmup. */
|
|
60
|
+
loadError: string | null = null;
|
|
59
61
|
|
|
60
62
|
constructor(
|
|
61
63
|
readonly model: string,
|
|
@@ -88,6 +90,17 @@ export class LocalReranker implements RerankerLike {
|
|
|
88
90
|
}
|
|
89
91
|
}
|
|
90
92
|
|
|
93
|
+
/**
|
|
94
|
+
* Force the model load now (issue #11, upstream warmup): downloads and
|
|
95
|
+
* initializes the cross-encoder without needing a query. Returns false
|
|
96
|
+
* when the load failed — the reranker then stays a silent no-op, per its
|
|
97
|
+
* degrade-never-break design; `loadError` carries the reason.
|
|
98
|
+
*/
|
|
99
|
+
async warm(): Promise<boolean> {
|
|
100
|
+
if (this.failed) return false;
|
|
101
|
+
return (await this.ensureLoaded()) !== null;
|
|
102
|
+
}
|
|
103
|
+
|
|
91
104
|
private ensureLoaded(): Promise<LoadedReranker | null> {
|
|
92
105
|
if (this.loaded) return Promise.resolve(this.loaded);
|
|
93
106
|
if (!this.loading) {
|
|
@@ -102,8 +115,9 @@ export class LocalReranker implements RerankerLike {
|
|
|
102
115
|
});
|
|
103
116
|
this.loaded = { tokenizer, model };
|
|
104
117
|
return this.loaded;
|
|
105
|
-
})().catch(() => {
|
|
118
|
+
})().catch((err: unknown) => {
|
|
106
119
|
this.failed = true;
|
|
120
|
+
this.loadError = err instanceof Error ? err.message : String(err);
|
|
107
121
|
return null;
|
|
108
122
|
});
|
|
109
123
|
}
|