pi-para 0.5.2 → 0.6.0

package/src/status.ts

@@ -0,0 +1,115 @@
+import { existsSync } from "node:fs";
+import { loadParaConfig } from "./config.js";
+import { getParaPaths } from "./paths.js";
+import { listPages, type ParaCategory } from "./wiki.js";
+import { SchedulerStateDB, type QueueStatus } from "./scheduler/state.js";
+
+export interface PiParaStatusOptions {
+  homeDir?: string;
+}
+
+export interface PiParaStatusResult {
+  configPath: string;
+  wikiDir: string;
+  configSource: "canonical" | "migrated";
+  schedulerDbPath: string;
+  qmdDbPath: string;
+  pages: {
+    total: number;
+    byCategory: Record<ParaCategory, number>;
+  };
+  scheduler: Record<QueueStatus, number>;
+  qmdDbExists: boolean;
+  warnings: string[];
+}
+
+const QUEUE_STATUSES: QueueStatus[] = ["queued", "running", "done", "failed"];
+
+export async function getPiParaStatus(options: PiParaStatusOptions = {}): Promise<PiParaStatusResult> {
+  const loaded = await loadParaConfig({ homeDir: options.homeDir, migrate: true });
+  const paths = getParaPaths({ homeDir: options.homeDir, wikiDir: loaded.config.wiki.dir });
+  const warnings: string[] = [];
+
+  const pages = await getPageCounts(paths.wikiDir, warnings);
+  const scheduler = getSchedulerCounts(paths.schedulerDbPath, warnings);
+
+  if (!existsSync(paths.wikiDir)) warnings.push(`wiki directory not found: ${paths.wikiDir}`);
+  if (!existsSync(paths.schedulerDbPath)) warnings.push("scheduler DB not created yet; it will be initialized when Pi starts");
+  if (!existsSync(paths.qmdDbPath)) warnings.push("QMD DB not created yet; search index is initialized on first store open");
+
+  return {
+    configPath: paths.userConfigPath,
+    wikiDir: paths.wikiDir,
+    configSource: loaded.migratedFromLegacy ? "migrated" : "canonical",
+    schedulerDbPath: paths.schedulerDbPath,
+    qmdDbPath: paths.qmdDbPath,
+    pages,
+    scheduler,
+    qmdDbExists: existsSync(paths.qmdDbPath),
+    warnings,
+  };
+}
+
+export function formatPiParaStatus(status: PiParaStatusResult): string {
+  const lines = [
+    "pi-para status",
+    `  Config: ${status.configPath} (${status.configSource})`,
+    `  Wiki: ${status.wikiDir}`,
+    `  Pages: ${status.pages.total} total (${formatCategoryCounts(status.pages.byCategory)})`,
+    `  Scheduler: ${status.scheduler.queued} queued, ${status.scheduler.running} running, ${status.scheduler.failed} failed`,
+    `  QMD SDK DB: ${status.qmdDbExists ? status.qmdDbPath : "not initialized"}`,
+  ];
+
+  if (status.warnings.length > 0) {
+    lines.push("", "Warnings:");
+    for (const warning of status.warnings) lines.push(`  ⚠ ${warning}`);
+  }
+
+  lines.push("", "Next: use 'pi-para doctor' for diagnostics or '/wiki-scheduler status' inside Pi.");
+  return lines.join("\n");
+}
+
+async function getPageCounts(wikiDir: string, warnings: string[]): Promise<PiParaStatusResult["pages"]> {
+  const byCategory: Record<ParaCategory, number> = {
+    projects: 0,
+    areas: 0,
+    resources: 0,
+    archives: 0,
+  };
+
+  try {
+    if (!existsSync(wikiDir)) return { total: 0, byCategory };
+    const pages = await listPages(wikiDir);
+    for (const page of pages) byCategory[page.category]++;
+    return { total: pages.length, byCategory };
+  } catch (err) {
+    warnings.push(`could not count wiki pages: ${err instanceof Error ? err.message : String(err)}`);
+    return { total: 0, byCategory };
+  }
+}
+
+function getSchedulerCounts(dbPath: string, warnings: string[]): Record<QueueStatus, number> {
+  const counts: Record<QueueStatus, number> = {
+    queued: 0,
+    running: 0,
+    done: 0,
+    failed: 0,
+  };
+
+  try {
+    const db = new SchedulerStateDB(dbPath);
+    try {
+      for (const status of QUEUE_STATUSES) counts[status] = db.list(status).length;
+    } finally {
+      db.close();
+    }
+  } catch (err) {
+    warnings.push(`could not read scheduler DB: ${err instanceof Error ? err.message : String(err)}`);
+  }
+
+  return counts;
+}
+
+function formatCategoryCounts(counts: Record<ParaCategory, number>): string {
+  return `projects ${counts.projects}, areas ${counts.areas}, resources ${counts.resources}, archives ${counts.archives}`;
+}

package/src/store.ts

@@ -12,6 +12,8 @@ import { parse as parseYaml } from "yaml";
 import { createStore, extractSnippet } from "qmd-engine";
 import type { QMDStore, CollectionConfig } from "qmd-engine";
 import { parseFrontmatter } from "./frontmatter.js";
+import type { ParaUserConfig } from "./config.js";
+import { buildQmdProvidersFromParaConfig, type QmdProvidersConfig } from "./qmd-providers.js";
 import { matchesScope } from "./scope.js";
 import type { PageFrontmatter, PageRef, ParaCategory } from "./wiki.js";
 import type { ProjectScope } from "./scope.js";
@@ -21,6 +23,18 @@ export type { QMDStore } from "qmd-engine";
 
 // -- Types ------------------------------------------------------------------
 
+export interface OpenStoreOptions {
+  paraConfig?: ParaUserConfig;
+  secretsPath?: string;
+  authStorage?: { getApiKey(provider: string): Promise<string | undefined> };
+  /**
+   * Start slow vector embedding in the background after BM25 indexing.
+   * Pi session startup disables this so single-shot `pi -p` processes can exit
+   * promptly; callers can still invoke embedIfNeeded() explicitly.
+   */
+  backgroundEmbed?: boolean;
+}
+
 export interface WikiSearchOptions {
   scope?: ProjectScope;
   category?: ParaCategory;
@@ -36,6 +50,14 @@ export interface WikiSearchResult {
   frontmatter: PageFrontmatter;
 }
 
+interface RawQmdSearchResult {
+  displayPath: string;
+  score: number;
+  body?: string;
+  title?: string;
+  [key: string]: unknown;
+}
+
 // -- PARA context descriptions -----------------------------------------------
 
 const PARA_CONTEXTS: Record<string, string> = {
@@ -51,6 +73,22 @@ const RAW_CONTEXT = "Immutable source material: articles, documents, notes. Not
 
 const pendingEmbeds = new WeakMap<QMDStore, Promise<void>>();
 
+// -- qmd provider config -----------------------------------------------------
+
+/**
+ * qmd-engine currently falls back to node-llama-cpp when no API provider is
+ * configured. pi-para never uses local LLMs, so provide an inert API endpoint
+ * whenever neither pi-para profiles nor legacy provider config exists. BM25
+ * search works without calling this endpoint; accidental vector/chat calls fail
+ * fast instead of downloading/building local models.
+ */
+const NO_LOCAL_PROVIDERS: QmdProvidersConfig = {
+  embed: {
+    url: "http://127.0.0.1:9",
+    model: "disabled-local-llm",
+  },
+};
+
 // -- qmd config loading ------------------------------------------------------
 
 /**
@@ -79,13 +117,21 @@ function loadQmdProviders(): Record<string, unknown> | undefined {
  * - wiki: all .md files except raw/
  * - raw: .md files under raw/, not included in default queries
  *
- * Adds PARA category contexts, runs update() for BM25,
- * and schedules embed() in the background.
+ * Adds PARA category contexts and runs update() for BM25. When requested,
+ * also schedules embed() in the background.
  */
-export async function openStore(wikiDir: string): Promise<QMDStore> {
-  // Read providers from ~/.config/qmd/index.yml so external API providers
-  // (embed, chat, rerank) are used instead of local GGUF models.
-  const providers = loadQmdProviders();
+export async function openStore(wikiDir: string, opts: OpenStoreOptions = {}): Promise<QMDStore> {
+  // Prefer pi-para's in-memory provider profiles. Legacy QMD YAML remains a
+  // fallback for migrated installs and tests that do not pass paraConfig.
+  const configuredProviders = opts.paraConfig
+    ? opts.paraConfig.qmd.providerConfig === "legacy-qmd-compatible"
+      ? loadQmdProviders()
+      : await buildQmdProvidersFromParaConfig(opts.paraConfig, {
+        secretsPath: opts.secretsPath,
+        authStorage: opts.authStorage,
+      })
+    : loadQmdProviders();
+  const providers = configuredProviders ?? NO_LOCAL_PROVIDERS;
 
   const store = await createStore({
     dbPath: join(wikiDir, ".qmd.sqlite"),
@@ -102,7 +148,7 @@ export async function openStore(wikiDir: string): Promise<QMDStore> {
           includeByDefault: false,
         },
       },
-      ...(providers ? { providers: providers as CollectionConfig["providers"] } : {}),
+      providers: providers as CollectionConfig["providers"],
     },
   });
 
@@ -117,25 +163,27 @@ export async function openStore(wikiDir: string): Promise<QMDStore> {
   // Sync filesystem -> BM25 index (fast, search ready immediately)
   await store.update();
 
-  // Schedule embedding in background (non-blocking).
-  // BM25 search works immediately; hybrid search improves once embed completes.
-  // Suppress console.error from qmd's internal embed failures.
-  const origError = console.error;
-  const embedPromise = store.embed().then(() => {
-    pendingEmbeds.delete(store);
-  }).catch(() => {
-    // Embedding failures are non-fatal — hybrid search degrades to BM25
-    pendingEmbeds.delete(store);
-  }).finally(() => {
-    console.error = origError;
-  });
-  // Temporarily suppress embed error noise during background embed
-  console.error = (...args: unknown[]) => {
-    const msg = String(args[0] ?? "");
-    if (msg.includes("embed error") || msg.includes("fetch failed")) return;
-    origError(...args);
-  };
-  pendingEmbeds.set(store, embedPromise);
+  if (opts.backgroundEmbed !== false && opts.paraConfig?.qmd.embedEnabled !== false) {
+    // Schedule embedding in background (non-blocking).
+    // BM25 search works immediately; hybrid search improves once embed completes.
+    // Suppress console.error from qmd's internal embed failures.
+    const origError = console.error;
+    const embedPromise = store.embed().then(() => {
+      pendingEmbeds.delete(store);
+    }).catch(() => {
+      // Embedding failures are non-fatal — hybrid search degrades to BM25
+      pendingEmbeds.delete(store);
+    }).finally(() => {
+      console.error = origError;
+    });
+    // Temporarily suppress embed error noise during background embed
+    console.error = (...args: unknown[]) => {
+      const msg = String(args[0] ?? "");
+      if (msg.includes("embed error") || msg.includes("fetch failed")) return;
+      origError(...args);
+    };
+    pendingEmbeds.set(store, embedPromise);
+  }
 
   return store;
 }
@@ -147,7 +195,10 @@ export async function closeStore(store: QMDStore): Promise<void> {
   const pending = pendingEmbeds.get(store);
   if (pending) {
     // Wait for pending embed with a short timeout — don't hang forever
-    const timeout = new Promise<void>((resolve) => setTimeout(resolve, 5_000));
+    const timeout = new Promise<void>((resolve) => {
+      const timer = setTimeout(resolve, 5_000);
+      (timer as { unref?: () => void }).unref?.();
+    });
     await Promise.race([pending, timeout]);
     pendingEmbeds.delete(store);
   }
@@ -157,8 +208,9 @@ export async function closeStore(store: QMDStore): Promise<void> {
 /**
  * Search the wiki with scope and category filtering.
  *
- * Uses BM25 search (searchLex) which requires no LLM. Each result's
- * frontmatter is parsed to apply scope/category/archive filters.
+ * Uses vector search when an embedding index is available, merged with BM25
+ * lexical search for exact keyword matches. Each result's frontmatter is parsed
+ * to apply scope/category/archive filters.
  */
 export async function searchWiki(
   store: QMDStore,
@@ -181,13 +233,14 @@ export async function searchWiki(
 
   const fetchLimit = opts.scope ? limit * 3 : limit;
 
-  const rawResults = await store.searchLex(query, {
+  const lexResults = await store.searchLex(query, {
     limit: fetchLimit,
     collection: "wiki",
     metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
     graphBoost: opts.graphBoost ?? true,
   });
 
+  const rawResults = await mergeLexAndVectorResults(store, query, lexResults, fetchLimit);
   const results: WikiSearchResult[] = [];
 
   for (const result of rawResults) {
@@ -220,7 +273,7 @@ export async function searchWiki(
       page: {
         category,
         slug,
-        title: frontmatter.title || result.title,
+        title: frontmatter.title || result.title || slug,
         path: `${category}/${slug}.md`,
       },
       score: result.score,
@@ -234,6 +287,38 @@ export async function searchWiki(
   return results;
 }
 
+async function mergeLexAndVectorResults(
+  store: QMDStore,
+  query: string,
+  lexResults: RawQmdSearchResult[],
+  limit: number,
+): Promise<RawQmdSearchResult[]> {
+  let vecResults: RawQmdSearchResult[] = [];
+  try {
+    const status = await store.getStatus();
+    const llm = (store.internal as unknown as { llm?: unknown }).llm;
+    const embedModel = (llm as { embedModelName?: string } | undefined)?.embedModelName;
+    if (status.hasVectorIndex && llm && embedModel) {
+      vecResults = await store.internal.searchVec(query, embedModel, limit, "wiki", llm as any) as RawQmdSearchResult[];
+    }
+  } catch {
+    // Semantic search is best-effort; lexical BM25 remains the reliable fallback.
+  }
+
+  const merged = new Map<string, RawQmdSearchResult>();
+  for (const result of lexResults) {
+    merged.set(result.displayPath, { ...result, score: result.score * 0.85 });
+  }
+  for (const result of vecResults) {
+    const existing = merged.get(result.displayPath);
+    if (!existing || result.score > existing.score) {
+      merged.set(result.displayPath, result);
+    }
+  }
+
+  return [...merged.values()].sort((a, b) => b.score - a.score).slice(0, limit * 2);
+}
+
 /**
  * Re-index the store (fast filesystem scan). Makes new/changed
  * content BM25-searchable immediately. Call after wiki writes.

package/src/summarize.ts

@@ -16,7 +16,7 @@ import type { ProjectScope } from "./scope.js";
 import {
   convertToLlm,
   serializeConversation,
-} from "@mariozechner/pi-coding-agent";
+} from "@earendil-works/pi-coding-agent";
 import {
   SUMMARIZE_SYSTEM_PROMPT,
   ITERATIVE_UPDATE_PROMPT,
@@ -72,7 +72,7 @@ function extractTextContent(content: unknown): string {
 
 /**
  * Fallback serialization that mirrors pi's serializeConversation format.
- * Used when @mariozechner/pi-coding-agent is not available.
+ * Used when @earendil-works/pi-coding-agent is not available.
  *
  * Formats: [User]:, [Assistant]:, [Assistant thinking]:,
  *          [Assistant tool calls]:, [Tool result]:
@@ -143,7 +143,7 @@ function serializeMessagesFallback(messages: unknown[]): string {
  * (bashExecution, compactionSummary, branchSummary, etc.). Falls back to
  * a compatible reimplementation for plain user/assistant/toolResult messages.
  *
- * @param messages - AgentMessage[] from @mariozechner/pi-agent-core
+ * @param messages - AgentMessage[] from @earendil-works/pi-agent-core
  * @returns Serialized text in [User]: / [Assistant]: / [Tool result]: format
  */
 export function serializeSessionForWiki(
@@ -246,28 +246,84 @@ export function generateSummary(
  * @param scope - Current project scope for context
  * @returns Prompt string for the LLM
  */
+export interface OverviewPromptOptions {
+  depth?: "brief" | "detailed";
+  maxChars?: number;
+}
+
+const DEFAULT_BRIEF_OVERVIEW_MAX_CHARS = 28_000;
+const DEFAULT_DETAILED_OVERVIEW_MAX_CHARS = 80_000;
+const BRIEF_EXCERPT_CHARS = 180;
+const DETAILED_BODY_CHARS = 2_000;
+
 export function generateOverviewPrompt(
   pages: WikiPage[],
   scope: ProjectScope,
+  options: OverviewPromptOptions = {},
 ): string {
+  const depth = options.depth ?? "brief";
+  const maxChars = options.maxChars ?? (depth === "detailed" ? DEFAULT_DETAILED_OVERVIEW_MAX_CHARS : DEFAULT_BRIEF_OVERVIEW_MAX_CHARS);
   const scopeInfo = `Current scope: ${scope.name} (tags: ${scope.include.join(", ")})`;
-
-  const pageEntries = pages.map((page) => {
-    const fm = page.frontmatter;
-    return [
-      `### ${fm.title} (${page.category}/${page.slug})`,
-      `PARA: ${fm.para} | Scope: ${fm.scope.join(", ")} | Tags: ${fm.tags.join(", ")}`,
-      "",
-      page.body,
-    ].join("\n");
-  });
-
-  return [
+  const header = [
     OVERVIEW_PROMPT,
     "",
     scopeInfo,
     `Total pages: ${pages.length}`,
+    `Input mode: ${depth === "detailed" ? "bounded detailed excerpts" : "compact metadata/excerpts"}`,
+    "",
+  ].join("\n");
+
+  const entries: string[] = [];
+  let usedChars = header.length;
+  let omitted = 0;
+
+  for (const page of pages) {
+    const entry = depth === "detailed" ? detailedPageEntry(page) : briefPageEntry(page);
+    if (usedChars + entry.length + 2 > maxChars) {
+      omitted++;
+      continue;
+    }
+    entries.push(entry);
+    usedChars += entry.length + 2;
+  }
+
+  const omittedNote = omitted > 0
+    ? `\n\n[${omitted} page(s) omitted to keep wiki_summarize within the context budget. Use a category, page path, or depth=\"detailed\" on a narrower target for more detail.]`
+    : "";
+
+  return `${header}${entries.join("\n\n")}${omittedNote}`;
+}
+
+function briefPageEntry(page: WikiPage): string {
+  const fm = page.frontmatter;
+  const excerpt = firstUsefulExcerpt(page.body, BRIEF_EXCERPT_CHARS);
+  return [
+    `- ${fm.title} (${page.category}/${page.slug})`,
+    `  PARA: ${fm.para} | Scope: ${fm.scope.join(", ")} | Tags: ${fm.tags.join(", ") || "-"} | Updated: ${fm.updated ?? "-"}`,
+    excerpt ? `  Excerpt: ${excerpt}` : undefined,
+    fm.links?.length ? `  Links: ${fm.links.slice(0, 8).map((link) => `[[${link}]]`).join(", ")}` : undefined,
+  ].filter(Boolean).join("\n");
+}
+
+function detailedPageEntry(page: WikiPage): string {
+  const fm = page.frontmatter;
+  return [
+    `### ${fm.title} (${page.category}/${page.slug})`,
+    `PARA: ${fm.para} | Scope: ${fm.scope.join(", ")} | Tags: ${fm.tags.join(", ")}`,
+    `Updated: ${fm.updated ?? "-"} | Links: ${(fm.links ?? []).slice(0, 12).map((link) => `[[${link}]]`).join(", ") || "-"}`,
     "",
-    ...pageEntries,
+    truncateForSummary(page.body, DETAILED_BODY_CHARS),
   ].join("\n");
 }
+
+function firstUsefulExcerpt(body: string, maxChars: number): string {
+  const cleaned = body
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line && !line.startsWith("#") && !/^[-*]\s*\[?\[?/.test(line))
+    .join(" ")
+    .replace(/\s+/g, " ")
+    .trim();
+  if (!cleaned) return "";
+  return cleaned.length <= maxChars ? cleaned : `${cleaned.slice(0, maxChars - 1)}…`;
+}

package/src/templates/prompts.ts

@@ -134,7 +134,7 @@ Instructions:
 FRESHNESS VERIFICATION (critical):
 - Each result shows an "Updated" timestamp and an age indicator (e.g. "⚠️ STALE — 45 days old"). Treat pages not updated in >14 days with healthy skepticism.
 - When a wiki page makes claims about code, configs, file paths, ports, API endpoints, or architecture — and the page is >14 days old — VERIFY by checking the actual source (read the file, run a command) before trusting the wiki.
-- When you discover a wiki page contains outdated or incorrect information, IMMEDIATELY fix it with wiki_write(mode: "edit") so future queries get correct answers. This is the self-healing loop.
+- When you discover a wiki page contains outdated or incorrect information, IMMEDIATELY fix it with wiki_edit so future queries get correct answers. This is the self-healing loop.
 - If you cannot verify a claim (e.g. no file access, external service), tell the user: "Wiki says X (last updated DATE) — I could not verify this is still current."
 - Never silently pass along stale wiki content as fact. Either verify it or flag it.`;