pi-para 0.5.2

package/src/store.ts

@@ -0,0 +1,288 @@
+/**
+ * qmd store lifecycle — manages the qmd-engine store instance.
+ *
+ * Handles store creation/opening, search with scope filtering,
+ * re-indexing after wiki changes, and embedding lifecycle.
+ */
+
+import { readFileSync, existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { parse as parseYaml } from "yaml";
+import { createStore, extractSnippet } from "qmd-engine";
+import type { QMDStore, CollectionConfig } from "qmd-engine";
+import { parseFrontmatter } from "./frontmatter.js";
+import { matchesScope } from "./scope.js";
+import type { PageFrontmatter, PageRef, ParaCategory } from "./wiki.js";
+import type { ProjectScope } from "./scope.js";
+
+// Re-export QMDStore for convenience
+export type { QMDStore } from "qmd-engine";
+
+// -- Types ------------------------------------------------------------------
+
+export interface WikiSearchOptions {
+  scope?: ProjectScope;
+  category?: ParaCategory;
+  limit?: number;
+  includeArchives?: boolean; // default false
+  graphBoost?: boolean; // default true — 1-hop wikilink expansion
+}
+
+export interface WikiSearchResult {
+  page: PageRef;
+  score: number;
+  snippet: string;
+  frontmatter: PageFrontmatter;
+}
+
+// -- PARA context descriptions -----------------------------------------------
+
+const PARA_CONTEXTS: Record<string, string> = {
+  "projects/": "Active projects with defined goals and end dates",
+  "areas/": "Ongoing responsibilities and standards",
+  "resources/": "Reference material and how-to guides",
+  "archives/": "Completed or deprecated items",
+};
+
+const RAW_CONTEXT = "Immutable source material: articles, documents, notes. Not synthesized.";
+
+// -- Pending embed tracking --------------------------------------------------
+
+const pendingEmbeds = new WeakMap<QMDStore, Promise<void>>();
+
+// -- qmd config loading ------------------------------------------------------
+
+/**
+ * Read providers from ~/.config/qmd/index.yml.
+ * Returns undefined if no config file or no providers configured.
+ */
+function loadQmdProviders(): Record<string, unknown> | undefined {
+  const configPath = join(homedir(), ".config", "qmd", "index.yml");
+  if (!existsSync(configPath)) return undefined;
+
+  try {
+    const content = readFileSync(configPath, "utf-8");
+    const config = parseYaml(content) as { providers?: Record<string, unknown> };
+    return config?.providers;
+  } catch {
+    return undefined;
+  }
+}
+
+// -- Functions ---------------------------------------------------------------
+
+/**
+ * Open and configure a qmd store for the wiki directory.
+ *
+ * Creates the store with two collections:
+ * - wiki: all .md files except raw/
+ * - raw: .md files under raw/, not included in default queries
+ *
+ * Adds PARA category contexts, runs update() for BM25,
+ * and schedules embed() in the background.
+ */
+export async function openStore(wikiDir: string): Promise<QMDStore> {
+  // Read providers from ~/.config/qmd/index.yml so external API providers
+  // (embed, chat, rerank) are used instead of local GGUF models.
+  const providers = loadQmdProviders();
+
+  const store = await createStore({
+    dbPath: join(wikiDir, ".qmd.sqlite"),
+    config: {
+      collections: {
+        wiki: {
+          path: wikiDir,
+          pattern: "**/*.md",
+          ignore: ["raw/**"],
+        },
+        raw: {
+          path: join(wikiDir, "raw"),
+          pattern: "**/*.md",
+          includeByDefault: false,
+        },
+      },
+      ...(providers ? { providers: providers as CollectionConfig["providers"] } : {}),
+    },
+  });
+
+  // Add PARA category contexts to the wiki collection
+  for (const [prefix, description] of Object.entries(PARA_CONTEXTS)) {
+    await store.addContext("wiki", prefix, description);
+  }
+
+  // Add context for the raw collection root
+  await store.addContext("raw", "/", RAW_CONTEXT);
+
+  // Sync filesystem -> BM25 index (fast, search ready immediately)
+  await store.update();
+
+  // Schedule embedding in background (non-blocking).
+  // BM25 search works immediately; hybrid search improves once embed completes.
+  // Suppress console.error from qmd's internal embed failures.
+  const origError = console.error;
+  const embedPromise = store.embed().then(() => {
+    pendingEmbeds.delete(store);
+  }).catch(() => {
+    // Embedding failures are non-fatal — hybrid search degrades to BM25
+    pendingEmbeds.delete(store);
+  }).finally(() => {
+    console.error = origError;
+  });
+  // Temporarily suppress embed error noise during background embed
+  console.error = (...args: unknown[]) => {
+    const msg = String(args[0] ?? "");
+    if (msg.includes("embed error") || msg.includes("fetch failed")) return;
+    origError(...args);
+  };
+  pendingEmbeds.set(store, embedPromise);
+
+  return store;
+}
+
+/**
+ * Close the store. Awaits any pending embed operation (with timeout) first.
+ */
+export async function closeStore(store: QMDStore): Promise<void> {
+  const pending = pendingEmbeds.get(store);
+  if (pending) {
+    // Wait for pending embed with a short timeout — don't hang forever
+    const timeout = new Promise<void>((resolve) => setTimeout(resolve, 5_000));
+    await Promise.race([pending, timeout]);
+    pendingEmbeds.delete(store);
+  }
+  await store.close();
+}
+
+/**
+ * Search the wiki with scope and category filtering.
+ *
+ * Uses BM25 search (searchLex) which requires no LLM. Each result's
+ * frontmatter is parsed to apply scope/category/archive filters.
+ */
+export async function searchWiki(
+  store: QMDStore,
+  query: string,
+  opts: WikiSearchOptions,
+): Promise<WikiSearchResult[]> {
+  const limit = opts.limit ?? 10;
+
+  // Build metadata filter for SQL-level filtering (qmd v2.2.0+)
+  // Note: we can only use metadata.category here, NOT scope.
+  // Scope filtering must remain as post-filter because:
+  // - Pages with scope:["global"] must match ANY project scope
+  // - Pages in the scope.include list need OR matching
+  // - The exclude list needs post-filter negation
+  // SQL-level scope filter would miss global pages.
+  const metadata: Record<string, string> = {};
+  if (opts.category) {
+    metadata.category = opts.category;
+  }
+
+  const fetchLimit = opts.scope ? limit * 3 : limit;
+
+  const rawResults = await store.searchLex(query, {
+    limit: fetchLimit,
+    collection: "wiki",
+    metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+    graphBoost: opts.graphBoost ?? true,
+  });
+
+  const results: WikiSearchResult[] = [];
+
+  for (const result of rawResults) {
+    // Parse category and slug from the display path (e.g. "projects/auth-refactor.md")
+    const parsed = parseDisplayPath(result.displayPath);
+    if (!parsed) continue;
+
+    const { category, slug } = parsed;
+
+    // Exclude archives unless explicitly requested (post-filter fallback
+    // for pages without frontmatter metadata)
+    if (category === "archives" && !opts.includeArchives) {
+      continue;
+    }
+
+    // Parse frontmatter from body
+    const body = result.body ?? "";
+    const { frontmatter } = parseFrontmatter(body);
+
+    // Scope post-filter fallback for pages without metadata
+    if (opts.scope && !matchesScope(frontmatter.scope, opts.scope)) {
+      continue;
+    }
+
+    // Extract a snippet for display
+    const snippetResult = extractSnippet(body, query);
+    const snippet = snippetResult.snippet;
+
+    results.push({
+      page: {
+        category,
+        slug,
+        title: frontmatter.title || result.title,
+        path: `${category}/${slug}.md`,
+      },
+      score: result.score,
+      snippet,
+      frontmatter,
+    });
+
+    if (results.length >= limit) break;
+  }
+
+  return results;
+}
+
+/**
+ * Re-index the store (fast filesystem scan). Makes new/changed
+ * content BM25-searchable immediately. Call after wiki writes.
+ */
+export async function reindex(store: QMDStore): Promise<void> {
+  await store.update();
+}
+
+/**
+ * Generate vector embeddings for new/changed pages.
+ * Slow — call only at startup (via openStore) or shutdown.
+ */
+export async function embedIfNeeded(store: QMDStore): Promise<void> {
+  await store.embed();
+}
+
+// -- Helpers -----------------------------------------------------------------
+
+/** Valid PARA categories for path parsing. */
+const VALID_CATEGORIES = new Set<string>([
+  "projects",
+  "areas",
+  "resources",
+  "archives",
+]);
+
+/**
+ * Parse a qmd display path into category + slug.
+ * Display paths from searchLex include the collection prefix:
+ *   "wiki/resources/ssl-certs.md" or "wiki/schema.md"
+ * Returns null for root-level files (schema.md, index.md, etc.) — they
+ * still appear in search results but aren't PageRef-able.
+ */
+function parseDisplayPath(
+  displayPath: string,
+): { category: ParaCategory; slug: string } | null {
+  const parts = displayPath.split("/");
+
+  // Display paths are: collection/category/slug.md (3+ parts)
+  // or collection/file.md (2 parts, root-level files like schema.md)
+  if (parts.length < 3) return null;
+
+  // Skip collection prefix (parts[0] = "wiki")
+  const category = parts[1];
+  if (!VALID_CATEGORIES.has(category)) return null;
+
+  const filename = parts.slice(2).join("/");
+  if (!filename.endsWith(".md")) return null;
+
+  const slug = filename.slice(0, -3);
+  return { category: category as ParaCategory, slug };
+}

package/src/summarize.ts

@@ -0,0 +1,273 @@
+/**
+ * Wiki summarization — adapts pi's compaction approach for wiki knowledge.
+ *
+ * Produces structured wiki summaries in the standard format:
+ * Topic, Key Facts, Insights, Connections, Open Questions, Sources.
+ *
+ * Supports ingest, session, page, and iterative summarization modes.
+ *
+ * Functions here generate prompt strings — they do NOT call the LLM.
+ * The caller (capture.ts, tools.ts) passes the prompt to a standalone Agent.
+ */
+
+import type { WikiPage } from "./wiki.js";
+import type { ParaCategory } from "./wiki.js";
+import type { ProjectScope } from "./scope.js";
+import {
+  convertToLlm,
+  serializeConversation,
+} from "@mariozechner/pi-coding-agent";
+import {
+  SUMMARIZE_SYSTEM_PROMPT,
+  ITERATIVE_UPDATE_PROMPT,
+  OVERVIEW_PROMPT,
+  CAPTURE_PROMPT,
+} from "./templates/prompts.js";
+
+// -- Types ------------------------------------------------------------------
+
+export interface SummarizeOptions {
+  mode: "ingest" | "session" | "page" | "iterative";
+  existingContent?: string; // for iterative mode
+  scope: ProjectScope;
+  category?: ParaCategory;
+}
+
+// -- Constants ---------------------------------------------------------------
+
+/** Maximum characters for a tool result in serialized output. */
+const TOOL_RESULT_MAX_CHARS = 2000;
+
+// -- Fallback serialization --------------------------------------------------
+
+/**
+ * Truncate text to a maximum character length for summarization.
+ * Keeps the beginning and appends a truncation marker.
+ */
+function truncateForSummary(text: string, maxChars: number): string {
+  if (text.length <= maxChars) return text;
+  const truncatedChars = text.length - maxChars;
+  return `${text.slice(0, maxChars)}\n\n[... ${truncatedChars} more characters truncated]`;
+}
+
+/**
+ * Extract text content from a message content field.
+ * Handles both string and array-of-blocks shapes.
+ */
+function extractTextContent(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .filter(
+      (c: unknown): c is { type: "text"; text: string } =>
+        typeof c === "object" &&
+        c !== null &&
+        "type" in c &&
+        (c as { type: string }).type === "text" &&
+        "text" in c,
+    )
+    .map((c) => c.text)
+    .join("");
+}
+
+/**
+ * Fallback serialization that mirrors pi's serializeConversation format.
+ * Used when @mariozechner/pi-coding-agent is not available.
+ *
+ * Formats: [User]:, [Assistant]:, [Assistant thinking]:,
+ *          [Assistant tool calls]:, [Tool result]:
+ */
+function serializeMessagesFallback(messages: unknown[]): string {
+  const parts: string[] = [];
+
+  for (const msg of messages) {
+    if (typeof msg !== "object" || msg === null || !("role" in msg)) continue;
+    const m = msg as Record<string, unknown>;
+
+    if (m.role === "user") {
+      const text = extractTextContent(m.content);
+      if (text) parts.push(`[User]: ${text}`);
+    } else if (m.role === "assistant") {
+      const content = m.content;
+      if (!Array.isArray(content)) continue;
+
+      const textParts: string[] = [];
+      const thinkingParts: string[] = [];
+      const toolCalls: string[] = [];
+
+      for (const block of content) {
+        if (typeof block !== "object" || block === null || !("type" in block)) continue;
+        const b = block as Record<string, unknown>;
+
+        if (b.type === "text" && typeof b.text === "string") {
+          textParts.push(b.text);
+        } else if (b.type === "thinking" && typeof b.thinking === "string") {
+          thinkingParts.push(b.thinking);
+        } else if (b.type === "toolCall" && typeof b.name === "string") {
+          const args = (b.arguments ?? {}) as Record<string, unknown>;
+          const argsStr = Object.entries(args)
+            .map(([k, v]) => `${k}=${JSON.stringify(v)}`)
+            .join(", ");
+          toolCalls.push(`${b.name}(${argsStr})`);
+        }
+      }
+
+      if (thinkingParts.length > 0) {
+        parts.push(`[Assistant thinking]: ${thinkingParts.join("\n")}`);
+      }
+      if (textParts.length > 0) {
+        parts.push(`[Assistant]: ${textParts.join("\n")}`);
+      }
+      if (toolCalls.length > 0) {
+        parts.push(`[Assistant tool calls]: ${toolCalls.join("; ")}`);
+      }
+    } else if (m.role === "toolResult") {
+      const text = extractTextContent(m.content);
+      if (text) {
+        parts.push(`[Tool result]: ${truncateForSummary(text, TOOL_RESULT_MAX_CHARS)}`);
+      }
+    }
+  }
+
+  return parts.join("\n\n");
+}
+
+// -- Public API --------------------------------------------------------------
+
+/**
+ * Serialize session messages for wiki capture.
+ *
+ * Converts AgentMessage[] to a text representation that prevents the LLM
+ * from treating it as a conversation to continue. Uses pi's convertToLlm +
+ * serializeConversation when the messages contain pi-specific types
+ * (bashExecution, compactionSummary, branchSummary, etc.). Falls back to
+ * a compatible reimplementation for plain user/assistant/toolResult messages.
+ *
+ * @param messages - AgentMessage[] from @mariozechner/pi-agent-core
+ * @returns Serialized text in [User]: / [Assistant]: / [Tool result]: format
+ */
+export function serializeSessionForWiki(
+  messages: unknown[], // AgentMessage[] — typed as unknown[] to avoid hard dep
+): string {
+  // Use pi's convertToLlm to handle custom message types (bashExecution,
+  // compactionSummary, branchSummary, etc.), then serialize the resulting
+  // standard LLM messages.
+  try {
+    const llmMessages = convertToLlm(messages as Parameters<typeof convertToLlm>[0]);
+    return serializeConversation(llmMessages);
+  } catch {
+    // If convertToLlm fails (e.g., unexpected message shapes), fall back
+    // to our reimplementation that handles the base roles.
+    return serializeMessagesFallback(messages);
+  }
+}
+
+/**
+ * Generate a wiki-format summary prompt from raw content.
+ *
+ * Returns the prompt string to send to the LLM. The caller passes this
+ * to a standalone Agent or includes it in a tool result for the session agent.
+ *
+ * @param content - Source text, serialized conversation, or page content
+ * @param options - Summarization mode and context
+ * @returns Prompt string for the LLM
+ */
+export function generateSummary(
+  content: string,
+  options: SummarizeOptions,
+): string {
+  const scopeInfo = `Current scope: ${options.scope.name} (tags: ${options.scope.include.join(", ")})`;
+  const categoryHint = options.category ? `\nSuggested PARA category: ${options.category}` : "";
+
+  switch (options.mode) {
+    case "ingest":
+      return [
+        SUMMARIZE_SYSTEM_PROMPT,
+        "",
+        scopeInfo,
+        categoryHint,
+        "",
+        "Produce a wiki page from the following source material:",
+        "",
+        "<source-content>",
+        content,
+        "</source-content>",
+      ].join("\n");
+
+    case "session":
+      return [
+        CAPTURE_PROMPT,
+        content,
+        "</session-conversation>",
+      ].join("\n");
+
+    case "page":
+      return [
+        SUMMARIZE_SYSTEM_PROMPT,
+        "",
+        scopeInfo,
+        "",
+        "Summarize the following wiki page into a concise overview:",
+        "",
+        "<page-content>",
+        content,
+        "</page-content>",
+      ].join("\n");
+
+    case "iterative":
+      if (!options.existingContent) {
+        // No existing content — treat as a fresh page
+        return generateSummary(content, { ...options, mode: "ingest" });
+      }
+      return [
+        ITERATIVE_UPDATE_PROMPT,
+        "",
+        scopeInfo,
+        categoryHint,
+        "",
+        "<existing-page>",
+        options.existingContent,
+        "</existing-page>",
+        "",
+        "<new-content>",
+        content,
+        "</new-content>",
+      ].join("\n");
+  }
+}
+
+/**
+ * Generate an overview prompt from multiple wiki pages.
+ *
+ * Used by /wiki-summarize and wiki_summarize tool to produce high-level
+ * overviews of a category or the entire wiki.
+ *
+ * @param pages - Wiki pages to summarize
+ * @param scope - Current project scope for context
+ * @returns Prompt string for the LLM
+ */
+export function generateOverviewPrompt(
+  pages: WikiPage[],
+  scope: ProjectScope,
+): string {
+  const scopeInfo = `Current scope: ${scope.name} (tags: ${scope.include.join(", ")})`;
+
+  const pageEntries = pages.map((page) => {
+    const fm = page.frontmatter;
+    return [
+      `### ${fm.title} (${page.category}/${page.slug})`,
+      `PARA: ${fm.para} | Scope: ${fm.scope.join(", ")} | Tags: ${fm.tags.join(", ")}`,
+      "",
+      page.body,
+    ].join("\n");
+  });
+
+  return [
+    OVERVIEW_PROMPT,
+    "",
+    scopeInfo,
+    `Total pages: ${pages.length}`,
+    "",
+    ...pageEntries,
+  ].join("\n");
+}