pi-doc-injector 0.2.0 → 0.3.0

package/keyword-gen.ts

@@ -0,0 +1,142 @@
+/**
+ * Local keyword generation — extracts keywords from filenames and content
+ * when no frontmatter is available.
+ *
+ * Extraction sources:
+ *   1. Filename parts (split on -, _, .)
+ *   2. Markdown headings (# Title, ## Title, etc.)
+ *   3. Code symbols (function, class, const, interface, type, enum)
+ *
+ * All keywords are lowercased, deduplicated, and filtered through a stop-word list.
+ * Output is capped at 20 keywords.
+ */
+
+const STOP_WORDS = new Set<string>([
+  // Articles
+  "a", "an", "the",
+  // Pronouns
+  "i", "you", "he", "she", "it", "we", "they",
+  "me", "him", "her", "us", "them",
+  "my", "your", "his", "its", "our", "their",
+  "this", "that", "these", "those",
+  "who", "whom", "whose", "which", "what",
+  // Prepositions
+  "in", "on", "at", "by", "for", "with", "about",
+  "to", "from", "of", "into", "onto", "upon",
+  "over", "under", "between", "among", "through",
+  "during", "before", "after", "above", "below",
+  "up", "down", "out", "off",
+  // Conjunctions
+  "and", "but", "or", "nor", "so", "yet", "for",
+  "if", "then", "than", "as", "when", "while",
+  "because", "since", "although", "though",
+  // Auxiliary/modal verbs
+  "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "having",
+  "do", "does", "did", "doing",
+  "will", "would", "shall", "should", "can", "could",
+  "may", "might", "must",
+  // Common adverbs
+  "not", "no", "yes",
+  "just", "only", "also", "too", "very", "now", "then",
+  "here", "there", "where", "how", "why",
+  "all", "each", "every", "both", "few", "more", "most",
+  "some", "any", "other", "another", "such",
+  "much", "many", "little", "less",
+  // Common content-less words
+  "get", "set", "put", "use", "make", "see", "need",
+  "one", "two", "three", "first", "second", "third",
+  "using", "used", "into", "onto", "new",
+  "note", "notes", "example", "examples", "todo",
+]);
+
+/**
+ * Generate up to 20 keywords from a file's name and content.
+ *
+ * Sources (in order, each adds keywords until cap is reached):
+ *   1. Filename parts — split on `-`, `_`, and `.`, keep segments ≥ 3 chars
+ *   2. Markdown headings — text after `#` markers
+ *   3. Code symbols — function/class/const/interface/type/enum declarations
+ *
+ * Each candidate is lowercased, filtered through a stop-word list, deduplicated,
+ * and limited to words with ≥ 3 characters.
+ *
+ * @param fileName - The basename of the file (e.g. "api-authentication.md")
+ * @param content  - The full file content
+ * @returns Up to 20 deduplicated keyword strings
+ */
+export function generateKeywords(
+  fileName: string,
+  content: string,
+): string[] {
+  const keywords: string[] = [];
+
+  // Source 1: Filename parts
+  addFromFilename(fileName, keywords);
+
+  // Source 2: Markdown headings
+  addFromHeadings(content, keywords);
+
+  // Source 3: Code symbols
+  addFromCodeSymbols(content, keywords);
+
+  // Deduplicate while preserving order
+  const seen = new Set<string>();
+  const result: string[] = [];
+  for (const kw of keywords) {
+    const lower = kw.toLowerCase();
+    if (seen.has(lower)) continue;
+    seen.add(lower);
+    result.push(kw);
+  }
+
+  return result.slice(0, 20);
+}
+
+/** Extract keyword candidates from filename parts. */
+function addFromFilename(fileName: string, out: string[]): void {
+  // Strip extension(s)
+  const nameWithoutExt = fileName.replace(/\.[^.]+$/, "");
+
+  // Split on common delimiters
+  const parts = nameWithoutExt.split(/[-_.\s]+/);
+
+  for (const part of parts) {
+    const cleaned = part.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
+    if (cleaned.length >= 3 && !STOP_WORDS.has(cleaned)) {
+      out.push(cleaned);
+    }
+  }
+}
+
+/** Extract keyword candidates from markdown headings (#, ##, ###, etc.). */
+function addFromHeadings(content: string, out: string[]): void {
+  const headingRegex = /^#{1,6}\s+(.+)$/gm;
+  let match: RegExpExecArray | null;
+  while ((match = headingRegex.exec(content)) !== null) {
+    const headingText = match[1].trim();
+    // Split heading into words
+    const words = headingText.split(/\s+/);
+    for (const word of words) {
+      const cleaned = word.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
+      if (cleaned.length >= 3 && !STOP_WORDS.has(cleaned)) {
+        out.push(cleaned);
+      }
+    }
+  }
+}
+
+/** Extract keyword candidates from code symbol declarations. */
+function addFromCodeSymbols(content: string, out: string[]): void {
+  // Match: function name, class name, const name, interface name, type name, enum name
+  // Also: export function, export class, export const, etc.
+  const symbolRegex = /(?:export\s+)?(?:async\s+)?(?:function|class|const|interface|type|enum)\s+(\w+)/gm;
+  let match: RegExpExecArray | null;
+  while ((match = symbolRegex.exec(content)) !== null) {
+    const name = match[1];
+    const cleaned = name.toLowerCase();
+    if (cleaned.length >= 3 && !STOP_WORDS.has(cleaned)) {
+      out.push(cleaned);
+    }
+  }
+}

package/keyword-llm.ts

@@ -0,0 +1,57 @@
+/**
+ * LLM Keyword Generation — builds prompts for the LLM to generate keywords
+ * for documentation files via the _doc_injector_keywords tool.
+ */
+
+/** Input for a single file in a keyword generation batch. */
+export interface FileInput {
+  /** Path relative to cwd (e.g. "docs/api.md") */
+  path: string;
+  /** First ~500 chars of the file content as context */
+  snippet: string;
+  /** Existing keywords (from frontmatter/heuristic), so LLM augments not replaces */
+  existingKeywords: string[];
+}
+
+/**
+ * Build a user message prompt instructing the LLM to generate keywords
+ * for a batch of documentation files by calling the _doc_injector_keywords tool.
+ *
+ * The prompt asks the LLM to read each file's snippet and produce 3-10 concise,
+ * searchable keywords per file, incorporating any existing keywords.
+ */
+export function buildKeywordGenPrompt(files: FileInput[]): string {
+  if (files.length === 0) return "";
+
+  const fileDescriptions = files.map((f, i) => {
+    const existing = f.existingKeywords.length > 0
+      ? `  Existing keywords: ${f.existingKeywords.join(", ")}`
+      : "";
+    // Escape markdown special chars in path to prevent prompt injection
+    const safePath = f.path.replace(/[*`\[\]]/g, "\\$&");
+    // Escape backticks in snippet to prevent breaking code fences
+    const safeSnippet = f.snippet.replace(/```/g, "'''");
+    return `${i + 1}. **${safePath}**\n${existing}\n   Snippet:\n\`\`\`\n${safeSnippet}\n\`\`\``;
+  }).join("\n\n");
+
+  const expectedOutput = files.map((f) => {
+    const safePath = f.path.replace(/[*`\[\]]/g, "\\$&");
+    return `  - "${safePath}": keywords array incorporating relevant existing keywords [${f.existingKeywords.slice(0, 5).map(k => `"${k}"`).join(", ")}${f.existingKeywords.length > 5 ? ", ..." : ""}]`;
+  }).join("\n");
+
+  return `Generate documentation keywords for the following ${files.length} file(s). For each file, read the snippet and produce 3-10 concise, searchable keywords that someone might type when looking for this documentation.
+
+Rules:
+- Keywords should be lowercase, 3+ characters, no stop-words
+- Incorporate any existing keywords that are still relevant
+- Focus on the document's core topic, not generic terms
+- Prefer specific technical terms over vague ones
+
+Files:
+${fileDescriptions}
+
+After analysis, call the \`_doc_injector_keywords\` tool with a \`keywords\` array like:
+${expectedOutput}
+
+Do not output any other text — just call the tool with the keywords.`;
+}

package/matcher.ts

@@ -31,6 +31,10 @@ export function extractText(content: unknown): string {
 export class KeywordMatcher {
   private options: MatcherOptions;
 
+  /**
+   * @param entries - The document entries to match against
+   * @param options - Optional matcher settings (merged with defaults)
+   */
   constructor(private entries: DocEntry[], options?: Partial<MatcherOptions>) {
     this.options = { ...DEFAULT_MATCHER_OPTIONS, ...options };
   }
@@ -44,8 +48,13 @@ export class KeywordMatcher {
     for (const entry of this.entries) {
       if (entry.injected) continue;
 
+      // Skip entries with no keywords (empty array or falsy)
+      if (!entry.keywords || entry.keywords.length === 0) continue;
+
       const matchedKeywords: string[] = [];
       for (const keyword of entry.keywords) {
+        // Skip empty keywords — they'd match everything with word boundaries
+        if (!keyword || keyword.trim().length === 0) continue;
         if (this.keywordMatches(text, keyword)) {
           matchedKeywords.push(keyword);
         }
@@ -63,18 +72,13 @@ export class KeywordMatcher {
     return results;
   }
 
+  /**
+   * Check if a single keyword matches the given text.
+   * Uses simple substring inclusion (case-insensitive by default).
+   */
   private keywordMatches(text: string, keyword: string): boolean {
     const search = this.options.caseSensitive ? text : text.toLowerCase();
     const kw = this.options.caseSensitive ? keyword : keyword.toLowerCase();
-
-    if (this.options.wordBoundary) {
-      // Escape special regex chars in keyword, then apply word boundary
-      const escaped = kw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-      const flags = this.options.caseSensitive ? "" : "i";
-      const regex = new RegExp(`\\b${escaped}\\b`, flags);
-      return regex.test(search);
-    }
-
     return search.includes(kw);
   }
-}
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-doc-injector",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Auto-inject relevant project documentation into Pi's LLM context based on keyword matching",
   "type": "module",
   "main": "./index.ts",
@@ -9,6 +9,7 @@
   },
   "files": [
     "*.ts",
+    "*.d.ts",
     "docs/**/*.md",
     "README.md"
   ],
@@ -33,6 +34,9 @@
       "./index.ts"
     ]
   },
+  "dependencies": {
+    "picomatch": "^4.0.2"
+  },
   "peerDependencies": {
     "@mariozechner/pi-coding-agent": "*"
   },

package/picomatch.d.ts

@@ -0,0 +1,11 @@
+declare module "picomatch" {
+  interface PicomatchOptions {
+    dot?: boolean;
+  }
+
+  type Matcher = (input: string) => boolean;
+
+  function picomatch(patterns: string | string[], options?: PicomatchOptions): Matcher;
+
+  export default picomatch;
+}