pi-doc-injector 0.2.1 → 0.3.1

package/cache.ts

@@ -0,0 +1,79 @@
+/**
+ * Keyword cache persistence — load/save the `.pi/doc-injector-cache.json` file.
+ *
+ * Cache format:
+ *   { version: 1, files: { [relativePath]: { mtimeMs: number, keywords: string[] } } }
+ *
+ * Invalid files (wrong version, bad JSON, ENOENT) result in an empty cache.
+ */
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import type { KeywordCache } from "./types";
+
+const CACHE_FILENAME = ".pi/doc-injector-cache.json";
+const CACHE_VERSION = 1;
+
+/**
+ * Load the keyword cache from disk.
+ * Returns an empty cache (version 1, no files) if the file doesn't exist,
+ * has wrong version, or is corrupted.
+ */
+export async function loadCache(cwd: string): Promise<KeywordCache> {
+  const cachePath = join(cwd, CACHE_FILENAME);
+
+  try {
+    const raw = await readFile(cachePath, "utf-8");
+    const parsed: unknown = JSON.parse(raw);
+
+    if (!isValidCache(parsed)) {
+      console.warn(
+        `[doc-injector] Invalid cache format or version at ${cachePath}, resetting.`,
+      );
+      return emptyCache();
+    }
+
+    return parsed;
+  } catch (err) {
+    // ENOENT = no cache file yet, that's fine
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+      console.warn(
+        `[doc-injector] Failed to read cache at ${cachePath}:`,
+        err instanceof Error ? err.message : String(err),
+      );
+    }
+    return emptyCache();
+  }
+}
+
+/**
+ * Save the keyword cache to disk.
+ * Creates parent directories if needed.
+ */
+export async function saveCache(
+  cwd: string,
+  cache: KeywordCache,
+): Promise<void> {
+  const cachePath = join(cwd, CACHE_FILENAME);
+
+  try {
+    await mkdir(dirname(cachePath), { recursive: true });
+  } catch {
+    // Ignore — directory may already exist
+  }
+
+  await writeFile(cachePath, JSON.stringify(cache, null, 2), "utf-8");
+}
+
+/** Check that a parsed value matches the KeywordCache shape. */
+function isValidCache(value: unknown): value is KeywordCache {
+  if (!value || typeof value !== "object") return false;
+  const c = value as Record<string, unknown>;
+  if (c.version !== CACHE_VERSION) return false;
+  if (!c.files || typeof c.files !== "object") return false;
+  return true;
+}
+
+/** Return a fresh empty cache. */
+function emptyCache(): KeywordCache {
+  return { version: CACHE_VERSION, files: {} };
+}

package/commands.ts

@@ -3,14 +3,26 @@
  */
 import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
 import type { DocRegistry } from "./registry";
+import type { DocInjectorConfig } from "./types";
 
+/** Dependencies injected into the command registrar. */
 export interface CommandDeps {
   getRegistry: () => DocRegistry | null;
   getEnabled: () => boolean;
   setEnabled: (v: boolean) => void;
   reloadRegistry: () => Promise<number>;
+  getConfig: () => DocInjectorConfig;
+  generateKeywordsLLM: (files: Array<{ path: string; snippet: string; existingKeywords: string[] }>) => Promise<void>;
 }
 
+/**
+ * Register all doc-injector slash commands on the given ExtensionAPI.
+ *
+ * Commands:
+ * - `/doc-inject [on|off|toggle|list|reset|status]` — manage injection state
+ * - `/doc-reload` — re-scan docs folder
+ * - `/doc-keywords-gen [path]` — generate LLM keywords for keyword-less files
+ */
 export function registerCommands(pi: ExtensionAPI, deps: CommandDeps): void {
   const cmd = (name: string, desc: string, handler: (args: string, ctx: ExtensionContext) => Promise<void>) => {
     pi.registerCommand(name, { description: desc, handler });
@@ -49,7 +61,8 @@ export function registerCommands(pi: ExtensionAPI, deps: CommandDeps): void {
       }
       const lines = entries.map((e) => {
         const status = e.injected ? "✅" : "⬜";
-        return `${status} ${e.relativePath}: "${e.title}" — keywords: [${e.keywords.join(", ")}]`;
+        const sourceTag = `[${e.keywordSource}]`;
+        return `${status} ${sourceTag} ${e.relativePath}: "${e.title}" — keywords: [${e.keywords.join(", ")}]`;
       });
       ctx.ui.notify(`📄 Registered docs:\n${lines.join("\n")}`, "info");
     } else {
@@ -81,4 +94,58 @@ export function registerCommands(pi: ExtensionAPI, deps: CommandDeps): void {
       ctx.ui.notify(`📄 Reload failed: ${err instanceof Error ? err.message : String(err)}`, "error");
     }
   });
+
+  cmd("doc-keywords-gen", "Generate LLM keywords: /doc-keywords-gen [path] — no arg = all keyword-less files", async (args, ctx) => {
+    const reg = deps.getRegistry();
+    if (!reg) {
+      ctx.ui.notify("📄 No registry loaded", "warning");
+      return;
+    }
+
+    const config = deps.getConfig();
+    if (!config.llmKeywords) {
+      ctx.ui.notify("📄 LLM keyword generation is disabled (llmKeywords: false in config)", "warning");
+      return;
+    }
+
+    const targetPath = args.trim();
+
+    // Filter to keyword-less entries (keywordSource !== "frontmatter", "cache", or "llm")
+    let candidates = reg.getEntries().filter((e) => {
+      if (e.keywordSource === "frontmatter") return false;
+      if (e.keywordSource === "cache") return false;
+      if (e.keywordSource === "llm") return false; // already LLM-generated
+      return true;
+    });
+
+    if (targetPath) {
+      candidates = candidates.filter((e) => e.relativePath.includes(targetPath));
+      if (candidates.length === 0) {
+        ctx.ui.notify(`📄 No keyword-less files matching "${targetPath}"`, "info");
+        return;
+      }
+    }
+
+    if (candidates.length === 0) {
+      ctx.ui.notify("📄 All files already have keywords", "info");
+      return;
+    }
+
+    const batchSize = config.llmBatchSize;
+    const batches: Array<Array<{ path: string; snippet: string; existingKeywords: string[] }>> = [];
+    for (let i = 0; i < candidates.length; i += batchSize) {
+      const batch = candidates.slice(i, i + batchSize).map((e) => ({
+        path: e.relativePath,
+        snippet: e.content.slice(0, 500),
+        existingKeywords: e.keywords,
+      }));
+      batches.push(batch);
+    }
+
+    ctx.ui.notify(`📄 Sending ${batches.length} keyword-generation batch(es) for ${candidates.length} file(s)...`, "info");
+
+    for (const batch of batches) {
+      await deps.generateKeywordsLLM(batch);
+    }
+  });
 }

package/config.ts

@@ -2,50 +2,85 @@
  * Configuration loader for the Doc Injector extension.
  * Reads from `.pi/doc-injector.json` with fallback to defaults.
  */
-import { existsSync, readFileSync } from "node:fs";
+import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { DEFAULT_CONFIG, type DocInjectorConfig } from "./types";
 
 /**
- * Load config from `.pi/doc-injector.json` relative to the given cwd.
- * Falls back to DEFAULT_CONFIG if file doesn't exist or is invalid.
+ * Clamp an integer value to [min, max] range.
+ * Warns and clamps if out of range. Returns the default if not a number.
  */
-export function loadConfig(cwd: string): DocInjectorConfig {
-  const configPath = join(cwd, ".pi", "doc-injector.json");
+function clampInt(
+  value: unknown,
+  defaultVal: number,
+  min: number,
+  max: number,
+  fieldName: string,
+): number {
+  if (typeof value !== "number" || Number.isNaN(value)) {
+    return defaultVal;
+  }
+  const intVal = Math.trunc(value);
+  if (intVal < min || intVal > max) {
+    const clamped = Math.max(min, Math.min(max, intVal));
+    console.warn(`[doc-injector] ${fieldName} must be ${min}-${max}, got ${intVal}. Clamping to ${clamped}.`);
+    return clamped;
+  }
+  return intVal;
+}
 
-  if (!existsSync(configPath)) {
-    return { ...DEFAULT_CONFIG };
+/**
+ * Validate a glob pattern array.
+ * Rejects non-array or entries that aren't strings. Returns default on error.
+ */
+function validateGlobArray(value: unknown, defaultVal: string[]): string[] {
+  if (!Array.isArray(value)) {
+    return [...defaultVal];
   }
+  const result: string[] = [];
+  for (const item of value) {
+    if (typeof item === "string") {
+      result.push(item);
+    } else {
+      console.warn(`[doc-injector] Non-string entry in glob array ignored: ${String(item)}`);
+    }
+  }
+  return result.length > 0 ? result : [...defaultVal];
+}
+
+/**
+ * Load config from `.pi/doc-injector.json` relative to the given cwd.
+ * Now async — uses readFile from fs/promises.
+ * Validates and clamps all numeric fields. Falls back to DEFAULT_CONFIG
+ * if file doesn't exist or is invalid.
+ */
+export async function loadConfig(cwd: string): Promise<DocInjectorConfig> {
+  const configPath = join(cwd, ".pi", "doc-injector.json");
 
   try {
-    const raw = readFileSync(configPath, "utf-8");
+    const raw = await readFile(configPath, "utf-8");
     const parsed = JSON.parse(raw) as Partial<DocInjectorConfig>;
 
-    // Clamp contextThreshold to 0-100 range
-    let contextThreshold = parsed.contextThreshold ?? DEFAULT_CONFIG.contextThreshold;
-    if (typeof contextThreshold === "number" && (contextThreshold < 0 || contextThreshold > 100)) {
-      console.warn(`[doc-injector] contextThreshold must be 0-100, got ${contextThreshold}. Clamping.`);
-      contextThreshold = Math.max(0, Math.min(100, contextThreshold));
-    }
-
-    // Clamp matchThreshold to positive integers
-    let matchThreshold = parsed.matchThreshold ?? DEFAULT_CONFIG.matchThreshold;
-    if (typeof matchThreshold === "number" && matchThreshold < 1) {
-      console.warn(`[doc-injector] matchThreshold must be >= 1, got ${matchThreshold}. Using 1.`);
-      matchThreshold = 1;
-    }
-
     return {
       docsPath: parsed.docsPath ?? DEFAULT_CONFIG.docsPath,
-      matchThreshold,
-      contextThreshold,
+      matchThreshold: clampInt(parsed.matchThreshold, DEFAULT_CONFIG.matchThreshold, 1, Infinity, "matchThreshold"),
+      contextThreshold: clampInt(parsed.contextThreshold, DEFAULT_CONFIG.contextThreshold, 0, 100, "contextThreshold"),
       recursive: parsed.recursive ?? DEFAULT_CONFIG.recursive,
+      include: validateGlobArray(parsed.include, DEFAULT_CONFIG.include),
+      exclude: validateGlobArray(parsed.exclude, DEFAULT_CONFIG.exclude),
+      maxFileSize: clampInt(parsed.maxFileSize, DEFAULT_CONFIG.maxFileSize, 1024, 10 * 1024 * 1024, "maxFileSize"),
+      autoKeywords: parsed.autoKeywords ?? DEFAULT_CONFIG.autoKeywords,
+      llmKeywords: parsed.llmKeywords ?? DEFAULT_CONFIG.llmKeywords,
+      maxConcurrent: clampInt(parsed.maxConcurrent, DEFAULT_CONFIG.maxConcurrent, 1, 100, "maxConcurrent"),
+      llmBatchSize: clampInt(parsed.llmBatchSize, DEFAULT_CONFIG.llmBatchSize, 1, 100, "llmBatchSize"),
     };
   } catch (err) {
-    console.warn(
-      `[doc-injector] Failed to parse config at ${configPath}:`,
-      err instanceof Error ? err.message : String(err),
-    );
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+      console.warn(
+        `[doc-injector] Failed to parse config at ${configPath}:`,
+        err instanceof Error ? err.message : String(err),
+      );
+    }
     return { ...DEFAULT_CONFIG };
   }
 }

package/docs/async-subagent-bug.md

@@ -0,0 +1,199 @@
+# Doc-Injector & Async Subagents Bug Analysis
+
+## Overview
+
+The doc-injector extension causes spinner hangs when used alongside async subagent workflows (e.g., `subagent_isolated`, `subagent_with_context` with `async: true`).
+
+## Root Cause
+
+### The Flow
+
+1. Parent spawns async subagents with `async: true, notifyOnComplete: "inject"`
+2. Parent streams output (e.g., "I've launched 5 agents...")
+3. Doc-injector's `message_update` fires, detects keyword match
+4. Doc-injector calls `ctx.abort()` to abort current turn
+5. `agent_end` fires → extension injects `"continue"` via `setTimeout()`
+6. Parent receives `"continue"` → immediately calls `get_subagent_result x5`
+7. Subagents still running → results not available → **spinner stuck**
+
+### Why This Happens
+
+The current doc-injector design assumes it can abort the current turn and restart immediately with injected docs. This works for simple single-turn workflows but breaks async subagent workflows because:
+
+1. **The injected `"continue"` disrupts the expected flow** - parent should wait for `notifyOnComplete: "inject"` messages, not immediately poll for results
+2. **`get_subagent_result` blocks** - when called before subagents finish, it hangs the parent thread
+3. **The abort signal is shared** - in some subagent implementations (e.g., the `subagent` tool in pi-subagentura), async subagents explicitly don't inherit the parent abort signal. But the injected `"continue"` still breaks the flow
+
+### Key Code Path (doc-injector/index.ts)
+
+```typescript
+// message_update handler - triggers abort on keyword match
+pi.on("message_update", async (event, ctx) => {
+  if (hasNew && !ctx.isIdle() && !abortingForInjection) {
+    abortingForInjection = true;
+    ctx.abort();  // ← PROBLEM: aborts current turn
+  }
+});
+
+// agent_end handler - injects "continue" after abort
+pi.on("agent_end", async (event, ctx) => {
+  if (abortingForInjection) {
+    abortingForInjection = false;
+    setTimeout(() => {
+      pi.sendUserMessage("continue");  // ← PROBLEM: disrupts async flow
+    }, 0);
+  }
+});
+```
+
+## Subagentura System Context
+
+### How Async Subagents Work
+
+The subagentura system (`/Users/applesucks/dev/pi-agents/subagent.ts`) spawns async subagents that:
+
+1. Run in the same process (not subprocess)
+2. Have independent session/context
+3. Signal via `notifyOnComplete: "inject"` - results injected as user messages when complete
+4. Use `get_subagent_result` to poll/block for results
+
+### Key Line (subagent.ts line 541, 744)
+
+```typescript
+signal: undefined, // async: don't inherit parent signal (would abort subagent when tool returns)
+```
+
+### Option A: Remove ctx.abort() Entirely (Recommended)
+
+**Change:** Never call `ctx.abort()` in `message_update`. Just record matches and inject on next turn.
+
+**Pros:**
+- ✅ Simplest fix
+- ✅ Works with ANY async workflow automatically
+- ✅ No special-casing for subagents
+
+**Cons:**
+- ❌ Loses "immediate injection during streaming" optimization
+- ❌ Docs injected on next turn instead of immediately
+
+**Implementation:**
+```typescript
+pi.on("message_update", async (event, ctx) => {
+  if (keywordGenInFlight) return;
+  if (!enabled || !registry) return;
+  if (msg.role !== "assistant") return;
+
+  const content = (msg as unknown as { content: unknown }).content;
+  textBuffer = extractText(content);
+  if (!textBuffer) return;
+
+  const matcher = buildMatcher();
+  if (!matcher) return;
+
+  const results = matcher.match(textBuffer);
+  for (const result of results) {
+    if (!pendingMatches.has(result.entry.filePath)) {
+      hasNew = true;  // Just record - don't abort
+    }
+    pendingMatches.set(result.entry.filePath, result.matchedKeywords);
+  }
+
+  // REMOVE: ctx.abort() entirely
+  // Just let the turn complete naturally
+});
+```
+
+### Option B: Detect Async Subagents Before Aborting
+
+**Change:** Check if assistant just spawned async subagents, skip abort if so.
+
+**Pros:**
+- ✅ Preserves immediate injection behavior
+- ✅ Subagents continue running
+
+**Cons:**
+- ❌ Fragile - depends on detecting specific tool call patterns
+- ❌ Doesn't scale to other async workflows
+
+### Option C: Only Inject on User Input
+
+**Change:** Only trigger immediate injection on `input` event (user messages), not on assistant streaming.
+
+**Pros:**
+- ✅ Clean separation of concerns
+- ✅ User intent gets immediate response
+
+**Cons:**
+- ❌ Loses streaming injection optimization
+- ❌ Still may not handle all async edge cases
+
+## Recommendation
+
+**Use Option A:** Remove `ctx.abort()` entirely.
+
+The streaming injection optimization is not worth the complexity and the risk of breaking async workflows. Most keyword matches from assistant streaming are not time-critical - they can wait for the next turn.
+
+## Additional Notes
+
+### The notifyOnComplete:"inject" Flow
+
+1. Parent spawns `async: true, notifyOnComplete: "inject"`
+2. Parent continues immediately (doesn't wait)
+3. Each subagent runs independently
+4. When subagent completes, result is injected as user message via `pi.sendUserMessage()`
+5. Parent receives the injected result and continues naturally
+
+This flow is designed to work without any intervention from the parent. Breaking it with `ctx.abort()` + `"continue"` defeats the purpose.
+
+### Session Context
+
+Each subagent runs in its **own session** with its own doc-injector instance. Doc injection in subagent sessions is fine - they inject their own docs into their own context. The problem is only in the **parent session**.
+
+## References
+
+- doc-injector: `/Users/applesucks/dev/pi-docs/index.ts`
+- subagentura: `/Users/applesucks/dev/pi-agents/subagent.ts`
+- subagentura helpers: `/Users/applesucks/dev/pi-agents/helpers.ts`
+
+## Web Research Summary
+
+### Official Documentation
+
+From `pi.dev/docs/latest/extensions`, the key lifecycle events are:
+
+- `before_agent_start` - Fires after user submits prompt, **before** agent loop. Can inject a message and/or modify system prompt.
+- `agent_start` / `agent_end` - Fires once per user prompt
+- `message_update` - Fires during streaming output
+- `input` - Fires on user input
+
+**Important note from docs:** "For critical events (before_agent_start, context, tool_call), if all handlers fail, the system continues with default behavior."
+
+### Key Extension API Points
+
+1. **`ctx.abort()`** - "Request a graceful shutdown of pi. Interactive mode: Deferred until the agent becomes idle (after processing all queued steering and follow-up messages)."
+2. **`ctx.signal`** - Available for extensions to forward cancellation into nested model calls, fetch(), and other abort-aware work.
+3. **`before_agent_start`** - The recommended injection point for context, as it runs before the agent loop starts.
+
+### Related GitHub Issues
+
+- [Issue #624](https://github.com/earendil-works/pi/issues/624): "before_agent_start event does not properly inject message" - indicates there have been historical issues with injection timing
+- [Issue #2660](https://github.com/earendil-works/pi/issues/2660): "Expose abort signal on ExtensionContext" - led to `ctx.signal` being added
+
+### Subagent Extensions Found
+
+1. **tintinweb/pi-subagents** (`pi.dev/packages/pi-subagents`)
+   - Claude Code-style autonomous subagents
+   - Spawns agents in isolated sessions
+   - Run in foreground or background
+   - Similar to subagentura
+
+2. **nicobailon/pi-subagents** (GitHub)
+   - Async subagent delegation with truncation, artifacts, session sharing
+
+### Conclusion from Web Research
+
+No documented solutions exist for the specific conflict between streaming injection (via `ctx.abort()`) and async subagent workflows. The recommended approach based on official docs is:
+
+1. Use `before_agent_start` for context injection (runs before agent loop)
+2. Avoid `ctx.abort()` during streaming - it disrupts any workflow that expects continuous streaming
+3. The `input` event is the cleanest injection point for user-driven context since it represents new user intent

package/globber.ts

@@ -0,0 +1,48 @@
+/**
+ * Glob filter for include/exclude pattern matching.
+ * Uses picomatch (0 deps, ~18 KB) to compile patterns once for O(1) matching.
+ */
+import picomatch from "picomatch";
+import type { GlobFilter } from "./types";
+
+/**
+ * Create a glob filter from include and exclude patterns.
+ *
+ * A path matches if it matches at least one `include` pattern AND
+ * does not match any `exclude` pattern.
+ *
+ * When `include` is empty, all files are considered included
+ * (subject to exclude filtering).
+ *
+ * @param include - Glob patterns for files to include
+ * @param exclude - Glob patterns for files/dirs to exclude
+ * @returns A GlobFilter with a `match` method
+ */
+export function createGlobFilter(
+  include: string[],
+  exclude: string[],
+): GlobFilter {
+  const includeMatcher =
+    include.length > 0
+      ? picomatch(include, { dot: true })
+      : null;
+
+  const excludeMatcher =
+    exclude.length > 0
+      ? picomatch(exclude, { dot: true })
+      : null;
+
+  return {
+    match(relativePath: string): boolean {
+      // If include patterns are specified, path must match at least one
+      if (includeMatcher && !includeMatcher(relativePath)) {
+        return false;
+      }
+      // Path must not match any exclude pattern
+      if (excludeMatcher && excludeMatcher(relativePath)) {
+        return false;
+      }
+      return true;
+    },
+  };
+}