pi-para 0.5.2

package/src/python/rlm_capture.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""
+RLM-based session knowledge capture for pi-para.
+
+Uses dspy.RLM with MiniMaxLM to extract knowledge from arbitrarily
+large pi sessions. RLM handles chunking itself via sandboxed Python REPL.
+
+Usage:
+    python3 rlm_capture.py <session_jsonl> <wiki_dir> <scope_name> [--already-captured slug1,slug2]
+"""
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+import dspy
+import yaml
+
+sys.path.insert(0, str(Path(__file__).parent))
+from minimax_lm import MiniMaxLM
+
+
+def load_session(session_path: str) -> str:
+    """Load and serialize a session JSONL file."""
+    entries = []
+    with open(session_path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entries.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+
+    parts = []
+    for entry in entries:
+        if entry.get("type") != "message":
+            if entry.get("type") == "compaction" and entry.get("summary"):
+                parts.append(f"[Compaction Summary]:\n{entry['summary']}")
+            continue
+
+        msg = entry.get("message", {})
+        role = msg.get("role", "")
+        content = msg.get("content", "")
+        if role not in ("user", "assistant", "toolResult"):
+            continue
+
+        if isinstance(content, str):
+            text = content
+        elif isinstance(content, list):
+            text_parts = []
+            tool_calls = []
+            for block in content:
+                if not isinstance(block, dict):
+                    continue
+                if block.get("type") == "text" and block.get("text"):
+                    text_parts.append(block["text"])
+                elif block.get("type") == "toolCall":
+                    name = block.get("name", "?")
+                    args_str = json.dumps(block.get("arguments", {}))
+                    if len(args_str) > 300:
+                        args_str = args_str[:300] + "..."
+                    tool_calls.append(f"{name}({args_str})")
+            text = "\n".join(text_parts)
+            if tool_calls:
+                text += f"\n[Tools]: {'; '.join(tool_calls)}"
+        else:
+            continue
+
+        if not text.strip():
+            continue
+        if role == "toolResult" and len(text) > 1000:
+            text = text[:1000] + "\n[truncated]"
+
+        label = {"user": "User", "assistant": "Assistant", "toolResult": "Tool"}.get(role, role)
+        parts.append(f"[{label}]: {text}")
+
+    return "\n\n".join(parts)
+
+
+def load_wiki_pages(wiki_dir: str, slugs: list[str]) -> str:
+    if not slugs:
+        return ""
+    pages = []
+    for slug in slugs:
+        for cat in ["projects", "areas", "resources", "archives"]:
+            path = Path(wiki_dir) / cat / f"{slug}.md"
+            if path.exists():
+                content = path.read_text()
+                if content.startswith("---"):
+                    end = content.find("---", 3)
+                    if end > 0:
+                        content = content[end + 3:].strip()
+                pages.append(f"[[{slug}]]: {content[:2000]}")
+                break
+    return "\n\n".join(pages) if pages else ""
+
+
+def setup_lm() -> MiniMaxLM:
+    api_key = os.environ.get("MINIMAX_CN_API_KEY")
+    if not api_key:
+        config_path = Path.home() / ".config" / "qmd" / "index.yml"
+        if config_path.exists():
+            with open(config_path) as f:
+                cfg = yaml.safe_load(f) or {}
+            chat = cfg.get("providers", {}).get("chat", {})
+            if chat.get("key") and "minimaxi.com" in chat.get("url", ""):
+                api_key = chat["key"]
+    if not api_key:
+        print("Error: No MiniMax API key.", file=sys.stderr)
+        sys.exit(1)
+
+    os.environ["MINIMAX_CN_API_KEY"] = api_key
+    return MiniMaxLM(model="MiniMax-M2.7-highspeed", china=True, max_tokens=8192)
+
+
+def run_capture(session_text: str, wiki_context: str, scope: str, lm: MiniMaxLM) -> dict:
+    dspy.configure(lm=lm)
+
+    rlm = dspy.RLM(
+        "context, query -> answer",
+        max_iterations=50,
+        max_llm_calls=100,
+        max_output_chars=30_000,
+        sub_lm=lm,
+        verbose=True,
+    )
+
+    query = f"""You have a large coding session transcript in the `context` variable ({len(session_text)} chars).
+It's too large to process in one llm_query() call. Use Python to work with it efficiently:
+
+1. Use Python string slicing to examine sections: context[:5000], context[5000:10000], etc.
+2. Use Python's `in` operator or string methods to search: "keyword" in context, context.find("...")
+3. For each interesting section found, use llm_query() on SMALL excerpts (under 20000 chars)
+4. Accumulate findings in a Python list
+
+{f"Already captured (skip these topics): {wiki_context}" if wiki_context else ""}
+
+Extract ALL valuable knowledge:
+- Architecture decisions, debugging solutions, server details
+- Build/deploy procedures, tool configs, package names+versions
+- File paths, conventions, operational knowledge
+
+When done, SUBMIT a JSON array:
+[{{"category":"projects|areas|resources|archives","slug":"name","title":"Title","scope":["{scope}"],"tags":["t1"],"body":"## Topic\\n...\\n## Key Facts\\n- ..."}}]
+
+If nothing valuable found, SUBMIT: []"""
+
+    try:
+        result = rlm(context=session_text, query=query)
+        answer = result.answer.strip()
+    except Exception as e:
+        return {"pages": [], "error": str(e)}
+
+    # Extract JSON
+    if "```" in answer:
+        lines = answer.split("\n")
+        json_lines = []
+        in_block = False
+        for line in lines:
+            if line.strip().startswith("```"):
+                in_block = not in_block
+                continue
+            if in_block:
+                json_lines.append(line)
+        if json_lines:
+            answer = "\n".join(json_lines)
+
+    start = answer.find("[")
+    end = answer.rfind("]")
+    if start >= 0 and end > start:
+        answer = answer[start:end + 1]
+
+    try:
+        pages = json.loads(answer)
+        return {"pages": pages, "error": None} if isinstance(pages, list) else {"pages": [], "error": "not array"}
+    except json.JSONDecodeError as e:
+        return {"pages": [], "error": f"JSON: {e}\nRaw: {answer[:500]}"}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("session_jsonl")
+    parser.add_argument("wiki_dir")
+    parser.add_argument("scope_name")
+    parser.add_argument("--already-captured", default="")
+    args = parser.parse_args()
+
+    lm = setup_lm()
+
+    print(f"Loading: {args.session_jsonl}", file=sys.stderr)
+    session_text = load_session(args.session_jsonl)
+    print(f"Serialized: {len(session_text)} chars", file=sys.stderr)
+
+    if not session_text.strip():
+        json.dump({"pages": [], "error": "empty"}, sys.stdout)
+        return
+
+    already = [s.strip() for s in args.already_captured.split(",") if s.strip()]
+    wiki_context = load_wiki_pages(args.wiki_dir, already)
+
+    print("Running RLM capture...", file=sys.stderr)
+    result = run_capture(session_text, wiki_context, args.scope_name, lm)
+
+    if result.get("error"):
+        print(f"Error: {result['error']}", file=sys.stderr)
+    print(f"Result: {len(result.get('pages', []))} pages", file=sys.stderr)
+    json.dump(result, sys.stdout, indent=2)
+
+
+if __name__ == "__main__":
+    main()

package/src/query.ts

@@ -0,0 +1,148 @@
+/**
+ * Query pipeline — searches the wiki and returns relevant results.
+ *
+ * Performs scoped search via qmd and returns results for the LLM
+ * to synthesize answers from. This is the reusable library function;
+ * the wiki_query tool wraps it with LLM prompt formatting.
+ */
+
+import type { QMDStore } from "qmd-engine";
+import type { ParaCategory } from "./wiki.js";
+import { searchWiki } from "./store.js";
+import type { WikiSearchResult } from "./store.js";
+import type { ProjectScope } from "./scope.js";
+
+// -- Types ------------------------------------------------------------------
+
+export interface QueryOptions {
+  /** Natural language search query */
+  query: string;
+  /** Override auto-detected scope tags */
+  scope?: string[];
+  /** Search all scopes (ignore project filter) */
+  global?: boolean;
+  /** Restrict to a PARA category */
+  category?: ParaCategory;
+  /** Maximum results (default 10) */
+  limit?: number;
+  /** Include archives in results (default false) */
+  includeArchives?: boolean;
+  /** Also search the raw/ collection (default false) */
+  includeRaw?: boolean;
+  /** Enable graph-boosted search via 1-hop wikilink expansion (default true) */
+  graphBoost?: boolean;
+}
+
+export interface QueryResult {
+  /** Search results with page metadata and snippets */
+  results: WikiSearchResult[];
+  /** The scope that was used for filtering */
+  scopeUsed: ProjectScope;
+  /** Whether the search was global (no scope filtering) */
+  wasGlobal: boolean;
+}
+
+// -- Main pipeline -----------------------------------------------------------
+
+/**
+ * Search the wiki with scope and category filtering.
+ *
+ * This is the reusable library function. It:
+ * 1. Applies scope filtering (unless global flag is set)
+ * 2. Filters by PARA category if specified
+ * 3. Excludes archives unless explicitly requested
+ * 4. Returns results sorted by relevance score
+ *
+ * The caller (tool or standalone agent) can format the results
+ * into an LLM prompt or use them directly.
+ */
+export async function queryWiki(
+  store: QMDStore,
+  options: QueryOptions,
+  scope: ProjectScope,
+): Promise<QueryResult> {
+  const limit = options.limit ?? 10;
+  const isGlobal = options.global ?? false;
+  const includeArchives = options.includeArchives ?? (options.category === "archives");
+
+  // Build scope override if explicit scope tags provided
+  let effectiveScope: ProjectScope | undefined;
+  if (isGlobal) {
+    effectiveScope = undefined; // no scope filtering
+  } else if (options.scope && options.scope.length > 0) {
+    effectiveScope = {
+      ...scope,
+      name: scope.name,
+      include: options.scope,
+    };
+  } else {
+    effectiveScope = scope;
+  }
+
+  const results = await searchWiki(store, options.query, {
+    scope: effectiveScope,
+    category: options.category,
+    limit,
+    includeArchives,
+    graphBoost: options.graphBoost,
+  });
+
+  return {
+    results,
+    scopeUsed: effectiveScope ?? scope,
+    wasGlobal: isGlobal,
+  };
+}
+
+/**
+ * Format query results as a readable text block.
+ * Useful for building LLM prompts or display.
+ *
+ * Includes freshness indicators so the LLM knows which pages to trust
+ * vs. which need verification against current code/state.
+ */
+export function formatQueryResults(results: WikiSearchResult[]): string {
+  if (results.length === 0) return "No results found.";
+
+  const now = Date.now();
+
+  return results
+    .map((r, i) => {
+      const fm = r.frontmatter;
+      const freshness = formatFreshness(fm.updated, now);
+      return [
+        `### Result ${i + 1}: ${fm.title} (${r.page.path})`,
+        `PARA: ${fm.para} | Scope: ${fm.scope.join(", ")} | Tags: ${fm.tags.join(", ")}`,
+        `Updated: ${fm.updated} | ${freshness}`,
+        `Score: ${r.score.toFixed(3)}`,
+        "",
+        r.snippet,
+      ].join("\n");
+    })
+    .join("\n\n");
+}
+
+// -- Freshness helpers -------------------------------------------------------
+
+/**
+ * Compute a human-readable freshness indicator from the page's `updated` date.
+ *
+ * - < 7 days  → "✅ FRESH"
+ * - 7-14 days → "✅ Recent — N days old"
+ * - 14-30 days → "⚠️ AGING — N days old — verify claims about code/configs"
+ * - 30-90 days → "⚠️ STALE — N days old — verify before trusting"
+ * - > 90 days  → "🚨 VERY STALE — N days old — likely outdated, verify everything"
+ */
+export function formatFreshness(updatedISO: string, nowMs: number): string {
+  const updatedMs = Date.parse(updatedISO);
+  if (isNaN(updatedMs)) return "❓ Unknown freshness (no valid date)";
+
+  const ageDays = Math.floor((nowMs - updatedMs) / (1000 * 60 * 60 * 24));
+
+  if (ageDays < 0) return "✅ FRESH";
+  if (ageDays < 7) return "✅ FRESH";
+  if (ageDays < 14) return `✅ Recent — ${ageDays} days old`;
+  if (ageDays < 30) return `⚠️ AGING — ${ageDays} days old — verify claims about code/configs`;
+  if (ageDays < 90) return `⚠️ STALE — ${ageDays} days old — verify before trusting`;
+  return `🚨 VERY STALE — ${ageDays} days old — likely outdated, verify everything`;
+}

package/src/raw.ts

@@ -0,0 +1,276 @@
+/**
+ * Raw source vault and session digests.
+ *
+ * Manages immutable source material in ~/.pi/wiki/raw/ and
+ * session digest entries in sessions.md.
+ */
+
+import { readFile, writeFile, mkdir, access } from "node:fs/promises";
+import { join, basename, extname } from "node:path";
+
+// -- Types ------------------------------------------------------------------
+
+export interface RawSource {
+  type: "url" | "file" | "text";
+  originalPath: string;
+  savedPath: string; // path within raw/ directory
+  ingestedAt: string; // ISO date
+  wikiPages: string[]; // wiki pages derived from this source
+}
+
+export interface SessionDigest {
+  date: string;
+  project: string;
+  sessionFile: string;
+  scope: string;
+  capturedPages: string[];
+  summary: string;
+}
+
+// -- Helpers -----------------------------------------------------------------
+
+const RAW_SUBDIRS: Record<"url" | "file" | "text", string> = {
+  url: "articles",
+  file: "docs",
+  text: "notes",
+};
+
+/**
+ * Slugify a string for use as a filename. Lowercase, hyphens, no special
+ * characters. Collapses multiple hyphens. Trims leading/trailing hyphens.
+ */
+function slugify(input: string): string {
+  return input
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "");
+}
+
+/**
+ * Derive a filename slug from the original path based on source type.
+ *
+ * - URL: extract domain + path components
+ * - file: use the filename without extension
+ * - text: use a timestamp-based name
+ */
+function deriveSlug(type: "url" | "file" | "text", originalPath: string): string {
+  if (type === "url") {
+    try {
+      const url = new URL(originalPath);
+      const host = url.hostname.replace(/^www\./, "");
+      const pathParts = url.pathname
+        .split("/")
+        .filter((p) => p.length > 0)
+        .slice(0, 3);
+      const raw = [host, ...pathParts].join("-");
+      return slugify(raw) || "untitled";
+    } catch {
+      return slugify(originalPath) || "untitled";
+    }
+  }
+
+  if (type === "file") {
+    const name = basename(originalPath, extname(originalPath));
+    return slugify(name) || "untitled";
+  }
+
+  // text: timestamp-based
+  const ts = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
+  return `note-${ts}`;
+}
+
+async function fileExists(path: string): Promise<boolean> {
+  try {
+    await access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Ensure a unique filename by appending -2, -3, etc. if the slug already exists.
+ */
+async function uniquePath(dir: string, slug: string, ext: string): Promise<string> {
+  let candidate = join(dir, `${slug}${ext}`);
+  if (!(await fileExists(candidate))) return candidate;
+
+  let i = 2;
+  while (await fileExists(join(dir, `${slug}-${i}${ext}`))) {
+    i++;
+  }
+  return join(dir, `${slug}-${i}${ext}`);
+}
+
+// -- Functions ---------------------------------------------------------------
+
+/**
+ * Save a raw source to the vault. URL sources go to raw/articles/,
+ * file sources to raw/docs/, text sources to raw/notes/.
+ *
+ * The file is saved as markdown with a metadata header.
+ * Returns the path relative to wikiDir (e.g. "raw/articles/example-com-page.md").
+ */
+export async function saveRawSource(
+  wikiDir: string,
+  source: {
+    type: "url" | "file" | "text";
+    content: string;
+    originalPath: string;
+  },
+): Promise<string> {
+  const subdir = RAW_SUBDIRS[source.type];
+  const dir = join(wikiDir, "raw", subdir);
+  await mkdir(dir, { recursive: true });
+
+  const slug = deriveSlug(source.type, source.originalPath);
+  const filePath = await uniquePath(dir, slug, ".md");
+  const fileName = basename(filePath);
+
+  const header = [
+    "---",
+    `source_type: ${source.type}`,
+    `original_path: "${source.originalPath}"`,
+    `ingested_at: "${new Date().toISOString()}"`,
+    "---",
+    "",
+  ].join("\n");
+
+  await writeFile(filePath, header + source.content, "utf-8");
+
+  return `raw/${subdir}/${fileName}`;
+}
+
+/**
+ * Append a session digest entry to sessions.md in the documented format:
+ *
+ * ```
+ * ## [2026-04-27] pi-mono | SSL certificate debugging
+ * - **Session**: `~/.pi/agent/sessions/.../file.jsonl`
+ * - **Scope**: pi-mono
+ * - **Captured**: [[ssl-cert-gotchas]]
+ * - **Summary**: One-paragraph summary...
+ * ```
+ *
+ * Creates sessions.md if it does not exist.
+ */
+export async function appendSessionDigest(
+  wikiDir: string,
+  digest: SessionDigest,
+): Promise<void> {
+  const sessionsPath = join(wikiDir, "sessions.md");
+
+  let existing: string;
+  try {
+    existing = await readFile(sessionsPath, "utf-8");
+  } catch (err: unknown) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      existing = "# Session Digests\n";
+    } else {
+      throw err;
+    }
+  }
+
+  const captured = digest.capturedPages.map((p) => `[[${p}]]`).join(", ");
+  const heading = digest.summary.split(/[.!?\n]/)[0].trim() || "Session capture";
+
+  const entry = [
+    "",
+    `## [${digest.date}] ${digest.project} | ${heading}`,
+    `- **Session**: \`${digest.sessionFile}\``,
+    `- **Scope**: ${digest.scope}`,
+    `- **Captured**: ${captured || "none"}`,
+    `- **Summary**: ${digest.summary}`,
+    "",
+  ].join("\n");
+
+  await writeFile(sessionsPath, existing + entry, "utf-8");
+}
+
+/**
+ * Parse sessions.md and return session digest entries.
+ * Optionally filter by scope and/or limit the number of results.
+ * Results are returned in file order (oldest first).
+ */
+export async function readSessionDigests(
+  wikiDir: string,
+  options?: { limit?: number; scope?: string },
+): Promise<SessionDigest[]> {
+  const sessionsPath = join(wikiDir, "sessions.md");
+
+  let content: string;
+  try {
+    content = await readFile(sessionsPath, "utf-8");
+  } catch (err: unknown) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return [];
+    }
+    throw err;
+  }
+
+  const digests: SessionDigest[] = [];
+
+  // Split on ## headings (session entries)
+  const entryPattern = /^## \[([^\]]+)\] ([^|]+)\| (.+)$/gm;
+  let match: RegExpExecArray | null;
+
+  while ((match = entryPattern.exec(content)) !== null) {
+    const date = match[1].trim();
+    const project = match[2].trim();
+    // Extract the block after this heading until the next heading or end
+    const startIdx = match.index + match[0].length;
+    const nextHeading = content.indexOf("\n## ", startIdx);
+    const block = nextHeading === -1
+      ? content.slice(startIdx)
+      : content.slice(startIdx, nextHeading);
+
+    const sessionFile = extractField(block, "Session");
+    const scope = extractField(block, "Scope");
+    const capturedRaw = extractField(block, "Captured");
+    const summary = extractField(block, "Summary");
+
+    // Parse captured wikilinks: [[slug1]], [[slug2]]
+    const capturedPages: string[] = [];
+    const linkPattern = /\[\[([^\]]+)\]\]/g;
+    let linkMatch: RegExpExecArray | null;
+    while ((linkMatch = linkPattern.exec(capturedRaw)) !== null) {
+      capturedPages.push(linkMatch[1]);
+    }
+
+    digests.push({
+      date,
+      project,
+      sessionFile,
+      scope,
+      capturedPages,
+      summary,
+    });
+  }
+
+  // Apply scope filter
+  let filtered = digests;
+  if (options?.scope) {
+    const scopeFilter = options.scope;
+    filtered = filtered.filter((d) => d.scope === scopeFilter);
+  }
+
+  // Apply limit (return the last N entries — most recent)
+  if (options?.limit !== undefined && options.limit < filtered.length) {
+    filtered = filtered.slice(filtered.length - options.limit);
+  }
+
+  return filtered;
+}
+
+/**
+ * Extract a field value from a session digest block.
+ * Matches lines like: `- **Field**: value` or `- **Field**: \`value\``
+ */
+function extractField(block: string, field: string): string {
+  const pattern = new RegExp(`^- \\*\\*${field}\\*\\*:\\s*(.+)$`, "m");
+  const match = pattern.exec(block);
+  if (!match) return "";
+  // Strip backticks wrapping the value
+  return match[1].trim().replace(/^`|`$/g, "");
+}