pi-doc-injector 0.5.3 → 0.6.0

package/README.md

@@ -112,7 +112,8 @@ Create `.pi/doc-injector.json` in your project root to customize behavior:
 ```json
 {
   "docsPath": "./docs",
-  "matchThreshold": 1,
+  "matchThreshold": 2,
+  "streamWindowSize": 500,
   "contextThreshold": 80,
   "recursive": true,
   "autoKeywords": true,
@@ -124,7 +125,8 @@ Create `.pi/doc-injector.json` in your project root to customize behavior:
 | Option             | Default    | Description                                              |
 | ------------------ | ---------- | -------------------------------------------------------- |
 | `docsPath`         | `"./docs"` | Path to docs folder (relative to project root)           |
-| `matchThreshold`   | `1`        | Minimum keyword matches required to inject a doc         |
+| `matchThreshold`   | `2`        | Minimum distinct keyword matches required to inject a doc |
+| `streamWindowSize` | `500`      | Rolling tail window (chars) scanned per streaming chunk; matches accumulate across chunks. `0` scans the full buffer |
 | `contextThreshold` | `80`       | Skip injection when context usage exceeds this % (0–100) |
 | `recursive`        | `true`     | Scan docs subdirectories recursively                     |
 | `autoKeywords`     | `true`     | Generate keywords heuristically when frontmatter is missing |
@@ -243,7 +245,6 @@ the registry mid-injection).
 The `injected` flag is per-session: it's reset on `session_start` and can
 be manually cleared with `/doc-inject reset`.
 
-For the full source-level verification, see the JSDoc block in `index.ts`.
 For the full source-level verification, see the JSDoc block in `index.ts`.
 
 ## Development

package/commands.ts

@@ -1,7 +1,7 @@
 /**
  * Slash commands for the Doc Injector extension.
  */
-import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
 import type { DocRegistry } from "./registry";
 import type { DocInjectorConfig } from "./types";
 
@@ -43,7 +43,7 @@ export function registerCommands(pi: ExtensionAPI, deps: CommandDeps): void {
     } else if (a === "reset") {
       const reg = deps.getRegistry();
       if (reg) {
-        reg.reset();
+        reg.markAllNotInjected();
         ctx.ui.notify("📄 Injection state reset", "info");
       } else {
         ctx.ui.notify("📄 No registry loaded", "warning");

package/config.ts

@@ -32,7 +32,6 @@ function clampInt(
     return intVal;
 }
 
-/**
 /**
  * Validate a glob pattern array.
  * Rejects non-array or entries that aren't strings. Returns default on error.
@@ -57,12 +56,6 @@ function validateGlobArray(
     return result.length > 0 ? result : [...defaultVal];
 }
 
-/**
- * Load config from `.pi/doc-injector.json` relative to the given cwd.
- * Now async — uses readFile from fs/promises.
- * Validates and clamps all numeric fields. Falls back to DEFAULT_CONFIG
- * if file doesn't exist or is invalid.
- */
 /**
  * Load config from `.pi/doc-injector.json` relative to the given cwd.
  * Async — uses readFile from fs/promises. Validates and clamps all numeric
@@ -79,6 +72,7 @@ export async function loadConfig(cwd: string, notifier: Notifier): Promise<DocIn
         return {
             docsPath: parsed.docsPath ?? DEFAULT_CONFIG.docsPath,
             matchThreshold: clampInt(parsed.matchThreshold, DEFAULT_CONFIG.matchThreshold, 1, Infinity, "matchThreshold", notifier),
+            streamWindowSize: clampInt(parsed.streamWindowSize, DEFAULT_CONFIG.streamWindowSize, 0, Infinity, "streamWindowSize", notifier),
             contextThreshold: clampInt(parsed.contextThreshold, DEFAULT_CONFIG.contextThreshold, 0, 100, "contextThreshold", notifier),
             recursive: parsed.recursive ?? DEFAULT_CONFIG.recursive,
             include: validateGlobArray(parsed.include, DEFAULT_CONFIG.include, notifier),

package/index.ts

@@ -63,8 +63,8 @@
  * is cleared after injection, and `markInjected()` operates on the registry's
  * current entries, not the stale array.
  */
-import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
-import { Type } from "@sinclair/typebox";
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
 import { resolve } from "node:path";
 import { loadCache, saveCache } from "./cache";
 import { loadConfig } from "./config";
@@ -73,7 +73,7 @@ import { buildKeywordGenPrompt } from "./keyword-llm";
 import { extractText, KeywordMatcher } from "./matcher";
 import { ExtensionNotifier, type Notifier } from "./notifier";
 import { DocRegistry } from "./registry";
-import { DEFAULT_MATCHER_OPTIONS, LLM_CACHE_SENTINEL, type DocEntry, type MatchResult, type KeywordCache, type CacheEntry } from "./types";
+import { LLM_CACHE_SENTINEL, type DocEntry, type KeywordCache, type CacheEntry, type MatcherOptions } from "./types";
 import { registerCommands } from "./commands";
 
 export default async function docInjectorExtension(pi: ExtensionAPI) {
@@ -88,7 +88,12 @@ export default async function docInjectorExtension(pi: ExtensionAPI) {
   let initRegistryPromise: Promise<void> | null = null;
   let enabled = true;
   let textBuffer = "";
-  let pendingMatches = new Map<string, string[]>(); // filePath → matchedKeywords
+  let pendingMatches = new Map<string, string[]>(); // filePath → matchedKeywords (docs that met the threshold)
+  // filePath → distinct keywords seen so far in the CURRENT streaming message.
+  // The streaming matcher scans only a rolling tail window each chunk, so a
+  // keyword can scroll out of the window before another appears. Accumulating
+  // the union here lets matchThreshold be met across chunks; reset at message_end.
+  let streamHits = new Map<string, Set<string>>();
   let abortingForInjection = false; // guard against cascading aborts
 
   // P5.4b — Guard flags for LLM keyword generation
@@ -134,15 +139,13 @@ export default async function docInjectorExtension(pi: ExtensionAPI) {
     if (Object.keys(dirty).length > 0) {
       await safeSaveCache(cwd, dirty);
     }
-
-    const count = registry.getEntries().length;
   };
 
-  const buildMatcher = (): KeywordMatcher | null => {
+  const buildMatcher = (options?: Partial<MatcherOptions>): KeywordMatcher | null => {
     if (!registry) return null;
     return new KeywordMatcher(
       registry.getNonInjectedEntries(),
-      { matchThreshold: config.matchThreshold },
+      { matchThreshold: config.matchThreshold, ...options },
     );
   };
 
@@ -303,17 +306,27 @@ export default async function docInjectorExtension(pi: ExtensionAPI) {
     textBuffer = extractText(content);
     if (!textBuffer) return;
 
-    const matcher = buildMatcher();
+    // Discovery matcher: threshold 1 finds any keyword present in the rolling
+    // tail window. The real matchThreshold is applied below against the
+    // keywords accumulated across all chunks of this message.
+    const matcher = buildMatcher({ matchThreshold: 1, windowSize: config.streamWindowSize });
     if (!matcher) return;
 
     const results = matcher.match(textBuffer);
 
     let hasNew = false;
     for (const result of results) {
-      if (!pendingMatches.has(result.entry.filePath)) {
+      const filePath = result.entry.filePath;
+      const seen = streamHits.get(filePath) ?? new Set<string>();
+      for (const kw of result.matchedKeywords) seen.add(kw);
+      streamHits.set(filePath, seen);
+
+      // Promote to a pending injection once enough distinct keywords have been
+      // seen across the message. Only the first crossing counts as "new".
+      if (seen.size >= config.matchThreshold && !pendingMatches.has(filePath)) {
+        pendingMatches.set(filePath, [...seen]);
         hasNew = true;
       }
-      pendingMatches.set(result.entry.filePath, result.matchedKeywords);
     }
 
     if (hasNew && !ctx.isIdle() && !abortingForInjection) {
@@ -325,11 +338,15 @@ export default async function docInjectorExtension(pi: ExtensionAPI) {
   // ---- Event: message_end (finalize matches) ----
   // Notification moved to before_agent_start so it fires for both user-triggered
   // and auto-abort-triggered injections. message_end now just resets state.
+  // State resets run BEFORE the role/registry guards so a non-assistant
+  // message_end (e.g. tool result, user message) can never leak a stale
+  // textBuffer or streamHits into the next assistant turn.
   pi.on("message_end", async (event, _ctx) => {
+    textBuffer = "";
+    streamHits.clear();
     if (!enabled || !registry) return;
     const msg = event.message;
     if (msg.role !== "assistant") return;
-    textBuffer = "";
   });
 
   // ---- Event: before_agent_start (inject as CustomMessage) ----

package/matcher.ts

@@ -28,34 +28,78 @@ export function extractText(content: unknown): string {
   return parts.join("\n");
 }
 
+/** Characters treated as part of a "word" for boundary detection. */
+const WORD_CHAR = /[A-Za-z0-9_]/;
+
+/** Escape a string so it can be embedded literally in a RegExp. */
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Compile a keyword into a regex that matches it literally with smart word
+ * boundaries.
+ *
+ * Boundaries are only asserted on an edge when the adjacent keyword character
+ * is a word character ([A-Za-z0-9_]). This gives word-boundary semantics for
+ * normal keywords ("test" won't match "latest" or "testing") while still
+ * matching symbol keywords literally ("$", "C++", "func()", "[test]", "a|b").
+ */
+function buildKeywordRegex(keyword: string, caseSensitive: boolean): RegExp {
+  const escaped = escapeRegExp(keyword);
+  const left = WORD_CHAR.test(keyword[0]) ? "(?<![A-Za-z0-9_])" : "";
+  const right = WORD_CHAR.test(keyword[keyword.length - 1]) ? "(?![A-Za-z0-9_])" : "";
+  return new RegExp(`${left}${escaped}${right}`, caseSensitive ? "" : "i");
+}
+
+interface CompiledEntry {
+  entry: DocEntry;
+  patterns: Array<{ keyword: string; regex: RegExp }>;
+}
+
 export class KeywordMatcher {
   private options: MatcherOptions;
+  // Regexes are precompiled once per matcher so streaming re-matches (which run
+  // on every chunk) don't recompile patterns on each call.
+  private compiled: CompiledEntry[];
 
   /**
    * @param entries - The document entries to match against
    * @param options - Optional matcher settings (merged with defaults)
    */
-  constructor(private entries: DocEntry[], options?: Partial<MatcherOptions>) {
+  constructor(entries: DocEntry[], options?: Partial<MatcherOptions>) {
     this.options = { ...DEFAULT_MATCHER_OPTIONS, ...options };
+    this.compiled = entries.map((entry) => ({
+      entry,
+      patterns: (entry.keywords ?? [])
+        // Skip empty/whitespace keywords — they'd match everything.
+        .filter((kw) => kw && kw.trim().length > 0)
+        .map((keyword) => ({ keyword, regex: buildKeywordRegex(keyword, this.options.caseSensitive) })),
+    }));
   }
 
   /** Match text against keyword index. Returns matching docs with hit details. */
   match(text: string): MatchResult[] {
-    if (!text || this.entries.length === 0) return [];
+    if (!text || this.compiled.length === 0) return [];
+
+    // Rolling tail window: only scan the last `windowSize` chars. Keeps the
+    // per-chunk cost bounded on long streaming messages. Callers that need the
+    // full text (one-shot user input) pass windowSize 0. Matches still
+    // accumulate across chunks in the caller, so keywords that scroll past the
+    // window aren't lost.
+    const { windowSize } = this.options;
+    const scanText =
+      windowSize > 0 && text.length > windowSize ? text.slice(-windowSize) : text;
 
     const results: MatchResult[] = [];
 
-    for (const entry of this.entries) {
+    for (const { entry, patterns } of this.compiled) {
       if (entry.injected) continue;
-
-      // Skip entries with no keywords (empty array or falsy)
-      if (!entry.keywords || entry.keywords.length === 0) continue;
+      if (patterns.length === 0) continue;
 
       const matchedKeywords: string[] = [];
-      for (const keyword of entry.keywords) {
-        // Skip empty keywords — they'd match everything with word boundaries
-        if (!keyword || keyword.trim().length === 0) continue;
-        if (this.keywordMatches(text, keyword)) {
+      for (const { keyword, regex } of patterns) {
+        if (regex.test(scanText)) {
           matchedKeywords.push(keyword);
         }
       }
@@ -71,14 +115,4 @@ export class KeywordMatcher {
 
     return results;
   }
-
-  /**
-   * Check if a single keyword matches the given text.
-   * Uses simple substring inclusion (case-insensitive by default).
-   */
-  private keywordMatches(text: string, keyword: string): boolean {
-    const search = this.options.caseSensitive ? text : text.toLowerCase();
-    const kw = this.options.caseSensitive ? keyword : keyword.toLowerCase();
-    return search.includes(kw);
-  }
 }

package/notifier.ts

@@ -16,7 +16,7 @@
  * satisfying the `Notifier` interface (or a `vi.fn()` spy) — no real
  * extension context is needed.
  */
-import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
 
 export type NotifierLevel = "info" | "warning" | "error";
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-doc-injector",
-  "version": "0.5.3",
+  "version": "0.6.0",
   "description": "Auto-inject relevant project documentation into Pi's LLM context based on keyword matching",
   "type": "module",
   "main": "./index.ts",
@@ -15,7 +15,8 @@
   ],
   "scripts": {
     "test": "vitest run",
-    "test:watch": "vitest"
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
   },
   "keywords": [
     "pi-package",
@@ -35,10 +36,11 @@
     ]
   },
   "dependencies": {
-    "picomatch": "^4.0.2"
+    "picomatch": "^4.0.2",
+    "typebox": "^1.1.38"
   },
   "peerDependencies": {
-    "@mariozechner/pi-coding-agent": "*"
+    "@earendil-works/pi-coding-agent": "^0.79.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

package/registry.ts

@@ -325,8 +325,9 @@ export class DocRegistry {
    *   4. Skip (no frontmatter, no cache, autoKeywords disabled)
    *
    * LLM-generated keywords populate the cache via the `_doc_injector_keywords`
-   * tool, so they surface as `keywordSource: "cache"` on the next rebuild
-   * (their `mtimeMs` is set to the file's current mtime when written).
+   * tool with `mtimeMs` set to LLM_CACHE_SENTINEL, so they surface as
+   * `keywordSource: "llm"` on the next rebuild (the sentinel never matches a
+   * real file mtime, keeping them distinct from heuristic cache hits).
    */
   private async processFile(
     { filePath, relativePath, fileName }: ScanResult,

package/types.ts

@@ -29,6 +29,12 @@ export interface DocEntry {
 export interface MatcherOptions {
   matchThreshold: number;
   caseSensitive: boolean;
+  /**
+   * If > 0, only the last `windowSize` characters of the text are scanned.
+   * Used by the streaming path to bound per-chunk scan cost; 0 disables
+   * windowing (scan the full text), which is the default for one-shot matches.
+   */
+  windowSize: number;
 }
 
 /** Result from a keyword match. */
@@ -68,6 +74,13 @@ export interface DocInjectorConfig {
   docsPath: string;
   /** Minimum keyword matches to trigger injection */
   matchThreshold: number;
+  /**
+   * Size (in chars) of the rolling tail window scanned on each streaming chunk.
+   * Bounds per-chunk scan cost; matches still accumulate across chunks so the
+   * threshold can be met even when keywords scroll out of the window. 0 = scan
+   * the full accumulated buffer.
+   */
+  streamWindowSize: number;
   /** Skip injection if context usage exceeds this % (0-100) */
   contextThreshold: number;
   /** Whether to scan subdirectories */
@@ -91,7 +104,8 @@ export interface DocInjectorConfig {
 /** Default configuration values. */
 export const DEFAULT_CONFIG: DocInjectorConfig = {
   docsPath: "./docs",
-  matchThreshold: 1,
+  matchThreshold: 2,
+  streamWindowSize: 500,
   contextThreshold: 80,
   recursive: true,
   include: ["**/*.md", "**/*.txt"],
@@ -107,6 +121,8 @@ export const DEFAULT_CONFIG: DocInjectorConfig = {
 export const DEFAULT_MATCHER_OPTIONS: MatcherOptions = {
   matchThreshold: DEFAULT_CONFIG.matchThreshold,
   caseSensitive: false,
+  // Default to no windowing; the streaming caller opts in via streamWindowSize.
+  windowSize: 0,
 };
 
 /**