pi-para 0.5.2 → 0.6.0

package/src/qmd-providers.ts

@@ -0,0 +1,63 @@
+import type { CollectionConfig } from "qmd-engine";
+import type { ParaUserConfig, ProviderCredentialRef } from "./config.js";
+import { resolveCredentialRef, type CredentialRef } from "./credentials.js";
+
+export type QmdProvidersConfig = NonNullable<CollectionConfig["providers"]>;
+
+export interface BuildQmdProvidersOptions {
+  secretsPath?: string;
+  authStorage?: { getApiKey(provider: string): Promise<string | undefined> };
+}
+
+export async function buildQmdProvidersFromParaConfig(
+  config: ParaUserConfig,
+  opts: BuildQmdProvidersOptions = {},
+): Promise<QmdProvidersConfig | undefined> {
+  if (config.qmd.providerConfig !== "pi-para-profiles") return undefined;
+
+  const providers: QmdProvidersConfig = {};
+
+  if (config.qmd.embedEnabled && config.qmd.embedding) {
+    providers.embed = await toQmdEndpoint(config.qmd.embedding, "embed", opts);
+  }
+
+  if (config.qmd.rerank) {
+    providers.rerank = await toQmdEndpoint(config.qmd.rerank, "rerank", opts);
+  }
+
+  return Object.keys(providers).length > 0 ? providers : undefined;
+}
+
+async function toQmdEndpoint(
+  profile: ProviderCredentialRef,
+  role: "embed" | "rerank",
+  opts: BuildQmdProvidersOptions,
+): Promise<NonNullable<QmdProvidersConfig[typeof role]>> {
+  const resolved = await resolveCredentialRef(profile.credentialRef as CredentialRef, opts);
+  const endpoint: NonNullable<QmdProvidersConfig[typeof role]> = {
+    url: profile.baseUrl || defaultBaseUrl(profile.provider, role),
+    model: profile.model,
+  };
+
+  if (profile.apiFormat === "openai" || profile.apiFormat === "anthropic") {
+    endpoint.api = profile.apiFormat;
+  }
+  if (resolved.ok && resolved.value) {
+    endpoint.key = resolved.value;
+  }
+  if (role === "embed" && profile.dims) {
+    (endpoint as NonNullable<QmdProvidersConfig["embed"]>).dims = profile.dims;
+  }
+
+  return endpoint;
+}
+
+export function defaultBaseUrl(provider: string, role: "embed" | "rerank"): string | undefined {
+  const normalized = provider.toLowerCase();
+  if (normalized === "openai") return "https://api.openai.com/v1";
+  if (normalized === "openrouter") return "https://openrouter.ai/api/v1";
+  if (normalized === "jina" && role === "rerank") return "https://api.jina.ai/v1";
+  if (normalized === "voyage") return "https://api.voyageai.com/v1";
+  if (normalized === "cohere") return "https://api.cohere.com/v2";
+  return undefined;
+}

package/src/query.ts

@@ -79,7 +79,7 @@ export async function queryWiki(
     effectiveScope = scope;
   }
 
-  const results = await searchWiki(store, options.query, {
+  let results = await searchWiki(store, options.query, {
     scope: effectiveScope,
     category: options.category,
     limit,
@@ -87,6 +87,20 @@ export async function queryWiki(
     graphBoost: options.graphBoost,
   });
 
+  // If QMD's BM25 index is stale, freshly written/readable pages can return
+  // as zero search results. On a miss, refresh the filesystem index once and
+  // retry before telling the agent the wiki lacks matching content.
+  if (results.length === 0) {
+    await store.update();
+    results = await searchWiki(store, options.query, {
+      scope: effectiveScope,
+      category: options.category,
+      limit,
+      includeArchives,
+      graphBoost: options.graphBoost,
+    });
+  }
+
   return {
     results,
     scopeUsed: effectiveScope ?? scope,

package/src/repair.ts

@@ -0,0 +1,49 @@
+import { existsSync, chmodSync, readFileSync, statSync } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+export const GENERATED_STATE_GITIGNORE_PATTERNS = [
+  ".qmd.sqlite*",
+  ".daemon.sqlite*",
+  ".pi-para.sqlite*",
+  "gepa/input/",
+  "gepa/output/",
+];
+
+export interface GitignoreRepairResult {
+  path: string;
+  changed: boolean;
+  added: string[];
+}
+
+export function missingGeneratedStateGitignorePatterns(wikiDir: string): string[] {
+  const path = join(wikiDir, ".gitignore");
+  const existing = existsSync(path) ? readFileSync(path, "utf-8") : "";
+  const lines = existing.split(/\r?\n/);
+  return GENERATED_STATE_GITIGNORE_PATTERNS.filter((pattern) => !lines.includes(pattern));
+}
+
+export async function ensureGeneratedStateGitignore(wikiDir: string): Promise<GitignoreRepairResult> {
+  const path = join(wikiDir, ".gitignore");
+  await mkdir(dirname(path), { recursive: true });
+  const existing = existsSync(path) ? readFileSync(path, "utf-8") : "";
+  const added = missingGeneratedStateGitignorePatterns(wikiDir);
+  if (added.length === 0) return { path, changed: false, added: [] };
+
+  const prefix = existing.trimEnd().length > 0 ? `${existing.trimEnd()}\n` : "";
+  const block = `${prefix}${prefix ? "\n" : ""}# pi-para generated state (do not version)\n${added.join("\n")}\n`;
+  await writeFile(path, block, "utf-8");
+  return { path, changed: true, added };
+}
+
+export function fixSecretPermissions(secretsPath: string): boolean {
+  if (!existsSync(secretsPath)) return false;
+  const before = readMode(secretsPath);
+  if ((before & 0o077) === 0) return false;
+  chmodSync(secretsPath, 0o600);
+  return true;
+}
+
+export function readMode(path: string): number {
+  return existsSync(path) ? (statSync(path).mode & 0o777) : 0;
+}

package/src/scheduler/controls.ts

@@ -0,0 +1,125 @@
+import { getParaPaths } from "../paths.js";
+import { StateDB } from "../state.js";
+import { WikiScheduler } from "./index.js";
+import { SchedulerStateDB, type QueueItem, type QueueStatus, type SchedulerHistoryItem } from "./state.js";
+import { enqueueCompletedSessions, readCompletedSessionRegistry } from "./session-capture.js";
+
+export interface QueueCaptureRecentOptions {
+  wikiDir: string;
+  dbPath?: string;
+  hours?: number;
+  now?: Date;
+}
+
+export interface QueueCaptureRecentResult {
+  scanned: number;
+  eligible: number;
+  queuedIds: number[];
+}
+
+export async function queueCaptureRecent(options: QueueCaptureRecentOptions): Promise<QueueCaptureRecentResult> {
+  const now = options.now ?? new Date();
+  const hours = options.hours ?? 24;
+  const cutoff = now.getTime() - hours * 60 * 60 * 1000;
+  const entries = await readCompletedSessionRegistry(options.wikiDir);
+  const eligible = entries.filter((entry) => {
+    const ts = Date.parse(entry.timestamp);
+    return Number.isFinite(ts) && ts >= cutoff;
+  });
+
+  const dbPath = options.dbPath ?? getParaPaths({ wikiDir: options.wikiDir }).schedulerDbPath;
+  const scheduler = new WikiScheduler({ wikiDir: options.wikiDir, dbPath, enabled: true });
+  const stateDb = new StateDB(options.wikiDir);
+  try {
+    const queuedIds = enqueueCompletedSessions(scheduler, eligible, { stateDb });
+    return { scanned: entries.length, eligible: eligible.length, queuedIds };
+  } finally {
+    stateDb.close();
+    scheduler.stop();
+  }
+}
+
+export function listSchedulerTasks(
+  wikiDir: string,
+  opts: { dbPath?: string; status?: QueueStatus } = {},
+): QueueItem[] {
+  const db = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath);
+  try {
+    return db.list(opts.status);
+  } finally {
+    db.close();
+  }
+}
+
+export function retryFailedSchedulerTasks(
+  wikiDir: string,
+  opts: { dbPath?: string; taskName?: string; now?: Date } = {},
+): number {
+  const db = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath);
+  try {
+    return db.retryFailed(opts.taskName, opts.now);
+  } finally {
+    db.close();
+  }
+}
+
+export function getSchedulerTask(
+  wikiDir: string,
+  id: number,
+  opts: { dbPath?: string } = {},
+): QueueItem | null {
+  const db = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath);
+  try {
+    return db.get(id);
+  } finally {
+    db.close();
+  }
+}
+
+export function listSchedulerHistory(
+  wikiDir: string,
+  opts: { dbPath?: string; taskName?: string; limit?: number } = {},
+): SchedulerHistoryItem[] {
+  const db = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath);
+  try {
+    return db.listHistory({ taskName: opts.taskName, limit: opts.limit });
+  } finally {
+    db.close();
+  }
+}
+
+export function formatQueueItems(items: QueueItem[]): string {
+  if (items.length === 0) return "No scheduler tasks queued.";
+  return items
+    .map((item) => `${item.id} ${item.status} ${item.taskName} attempts=${item.attempts} dedupe=${item.dedupeKey ?? "-"}`)
+    .join("\n");
+}
+
+export function formatQueueCaptureRecentResult(result: QueueCaptureRecentResult): string {
+  return [
+    `Scanned ${result.scanned} completed-session entr${result.scanned === 1 ? "y" : "ies"}.`,
+    `Eligible within window: ${result.eligible}.`,
+    `Queued ${result.queuedIds.length} capture task${result.queuedIds.length === 1 ? "" : "s"}.`,
+  ].join("\n");
+}
+
+export function formatQueueItem(item: QueueItem | null): string {
+  if (!item) return "Scheduler task not found.";
+  return [
+    `Task ${item.id}: ${item.taskName}`,
+    `Status: ${item.status}`,
+    `Attempts: ${item.attempts}`,
+    `Available: ${item.availableAt}`,
+    `Dedupe: ${item.dedupeKey ?? "-"}`,
+    `Payload: ${JSON.stringify(item.payload, null, 2)}`,
+  ].join("\n");
+}
+
+export function formatSchedulerHistory(items: SchedulerHistoryItem[]): string {
+  if (items.length === 0) return "No scheduler history.";
+  return items.map((item) => {
+    const duration = item.durationMs === null ? "-" : `${item.durationMs}ms`;
+    const error = item.error ? ` error=${item.error.slice(0, 80)}` : "";
+    return `${item.id} ${item.status} ${item.taskName} duration=${duration}${error}`;
+  }).join("\n");
+}

package/src/scheduler/index.ts

@@ -0,0 +1,210 @@
+import type { QMDStore } from "qmd-engine";
+import { randomUUID } from "node:crypto";
+import { gitCommit, rebuildIndex } from "../wiki.js";
+import { reindex } from "../store.js";
+import { getParaPaths } from "../paths.js";
+import { acquireLease, releaseLease } from "./leases.js";
+import { SchedulerStateDB, type QueueItem } from "./state.js";
+
+export type SchedulerTaskHandler = (item: QueueItem, scheduler: WikiScheduler) => Promise<void>;
+
+export interface WikiSchedulerOptions {
+  wikiDir: string;
+  dbPath?: string;
+  intervalMs?: number;
+  enabled?: boolean;
+  storeProvider?: () => QMDStore | null;
+  markDirty?: () => void;
+  handlers?: Record<string, SchedulerTaskHandler>;
+}
+
+export class WikiScheduler {
+  readonly wikiDir: string;
+  readonly holderId: string;
+  readonly state: SchedulerStateDB;
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private running = false;
+  private stopped = false;
+  private closed = false;
+  private readonly intervalMs: number;
+  private readonly enabled: boolean;
+  private readonly storeProvider: () => QMDStore | null;
+  private readonly markDirty: () => void;
+  private handlers: Record<string, SchedulerTaskHandler>;
+
+  constructor(opts: WikiSchedulerOptions) {
+    this.wikiDir = opts.wikiDir;
+    this.holderId = `scheduler:${process.pid}:${randomUUID()}`;
+    this.state = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir: opts.wikiDir }).schedulerDbPath);
+    this.intervalMs = opts.intervalMs ?? 15 * 60_000;
+    this.enabled = opts.enabled ?? true;
+    this.storeProvider = opts.storeProvider ?? (() => null);
+    this.markDirty = opts.markDirty ?? (() => {});
+    this.handlers = {
+      "wiki-maintenance": async () => this.runWikiMaintenance(),
+      ...(opts.handlers ?? {}),
+    };
+  }
+
+  start(): void {
+    if (!this.enabled || this.timer || this.closed) return;
+    this.stopped = false;
+    this.state.requeueRunning();
+    void this.tick();
+    this.timer = setInterval(() => void this.tick(), this.intervalMs);
+    (this.timer as { unref?: () => void }).unref?.();
+  }
+
+  stop(): void {
+    this.stopped = true;
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+    if (!this.running) this.closeState();
+  }
+
+  private closeState(): void {
+    if (this.closed) return;
+    this.closed = true;
+    this.state.close();
+  }
+
+  enqueue(taskName: string, payload: unknown = {}, opts: { priority?: number; dedupeKey?: string; availableAt?: Date } = {}): number {
+    if (this.closed) throw new Error("Wiki scheduler is stopped.");
+    return this.state.enqueue(taskName, payload, opts);
+  }
+
+  registerHandlers(handlers: Record<string, SchedulerTaskHandler>): void {
+    this.handlers = { ...this.handlers, ...handlers };
+  }
+
+  async tick(): Promise<number> {
+    if (!this.enabled || this.running || this.stopped || this.closed) return 0;
+    this.running = true;
+    let processed = 0;
+    try {
+      for (;;) {
+        if (this.stopped) break;
+        const item = this.state.claimNext();
+        if (!item) break;
+        await this.runItem(item);
+        processed++;
+      }
+    } finally {
+      this.running = false;
+      if (this.stopped) this.closeState();
+    }
+    return processed;
+  }
+
+  private async runItem(item: QueueItem): Promise<void> {
+    const handler = this.handlers[item.taskName];
+    const started = new Date();
+    if (!handler) {
+      this.state.fail(item.id, `Unknown scheduler task: ${item.taskName}`, { now: started, maxAttempts: 1 });
+      return;
+    }
+
+    try {
+      await handler(item, this);
+      if (this.stopped || this.closed) return;
+      const finished = new Date();
+      this.state.complete(item.id, finished);
+      this.state.recordHistory(item.taskName, item.payload, "done", started, finished, finished.getTime() - started.getTime(), null);
+    } catch (err) {
+      if (this.stopped || this.closed) return;
+      const retryAt = new Date(Date.now() + Math.min(60_000, 1_000 * 2 ** item.attempts));
+      this.state.fail(item.id, err instanceof Error ? err.message : String(err), { retryAt, now: new Date() });
+    }
+  }
+
+  private async runWikiMaintenance(): Promise<void> {
+    const leaseKey = `wiki-maintenance:${this.wikiDir}`;
+    if (!acquireLease(this.state.db, leaseKey, this.holderId, { ttlMs: 5 * 60_000 })) return;
+    try {
+      await rebuildIndex(this.wikiDir);
+      const store = this.storeProvider();
+      if (store) await reindex(store);
+      this.markDirty();
+      await gitCommit(this.wikiDir, "wiki: rebuild index and refresh search");
+    } finally {
+      releaseLease(this.state.db, leaseKey, this.holderId);
+    }
+  }
+}
+
+const schedulers = new Map<string, WikiScheduler>();
+const debounceTimers = new Map<string, ReturnType<typeof setTimeout>>();
+
+export function getWikiScheduler(opts: WikiSchedulerOptions): WikiScheduler {
+  const key = opts.dbPath ?? getParaPaths({ wikiDir: opts.wikiDir }).schedulerDbPath;
+  const existing = schedulers.get(key);
+  if (existing) {
+    if (opts.handlers) existing.registerHandlers(opts.handlers);
+    return existing;
+  }
+  const scheduler = new WikiScheduler(opts);
+  schedulers.set(key, scheduler);
+  return scheduler;
+}
+
+export function startWikiScheduler(opts: WikiSchedulerOptions): WikiScheduler {
+  const scheduler = getWikiScheduler(opts);
+  scheduler.start();
+  return scheduler;
+}
+
+export function stopWikiScheduler(wikiDir: string, dbPath?: string): void {
+  const key = dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath;
+  const timer = debounceTimers.get(key);
+  if (timer) {
+    clearTimeout(timer);
+    debounceTimers.delete(key);
+  }
+  const scheduler = schedulers.get(key);
+  if (!scheduler) return;
+  scheduler.stop();
+  schedulers.delete(key);
+}
+
+export function enqueueWikiMaintenance(
+  wikiDir: string,
+  store: QMDStore,
+  markDirty: () => void,
+  opts: { delayMs?: number; dbPath?: string } = {},
+): number | null {
+  const key = opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath;
+  const existingTimer = debounceTimers.get(key);
+  if (existingTimer) clearTimeout(existingTimer);
+
+  const scheduler = getWikiScheduler({
+    wikiDir,
+    dbPath: opts.dbPath,
+    storeProvider: () => store,
+    markDirty,
+  });
+
+  const delayMs = opts.delayMs ?? 2_000;
+  if (delayMs <= 0) {
+    const id = scheduler.enqueue("wiki-maintenance", {}, { dedupeKey: "wiki-maintenance", priority: 10 });
+    void scheduler.tick();
+    return id;
+  }
+
+  const timer = setTimeout(() => {
+    debounceTimers.delete(key);
+    scheduler.enqueue("wiki-maintenance", {}, { dedupeKey: "wiki-maintenance", priority: 10 });
+    void scheduler.tick();
+  }, delayMs);
+  (timer as { unref?: () => void }).unref?.();
+  debounceTimers.set(key, timer);
+  return null;
+}
+
+export function resetSchedulersForTests(): void {
+  for (const timer of debounceTimers.values()) clearTimeout(timer);
+  debounceTimers.clear();
+  for (const [key, scheduler] of schedulers) {
+    scheduler.stop();
+    schedulers.delete(key);
+  }
+}

package/src/scheduler/leases.ts

@@ -0,0 +1,98 @@
+import Database from "better-sqlite3";
+import { randomUUID } from "node:crypto";
+
+export interface LeaseRecord {
+  leaseKey: string;
+  holderId: string;
+  expiresAt: string;
+  heartbeatAt: string;
+  metadataJson: string | null;
+}
+
+export function ensureLeaseSchema(db: Database.Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS scheduler_leases (
+      lease_key TEXT PRIMARY KEY,
+      holder_id TEXT NOT NULL,
+      expires_at TEXT NOT NULL,
+      heartbeat_at TEXT NOT NULL,
+      metadata_json TEXT
+    );
+  `);
+}
+
+export function createLeaseHolder(prefix = "pi-para"): string {
+  return `${prefix}:${process.pid}:${randomUUID()}`;
+}
+
+export function acquireLease(
+  db: Database.Database,
+  leaseKey: string,
+  holderId: string,
+  opts: { ttlMs?: number; now?: Date; metadata?: unknown } = {},
+): boolean {
+  ensureLeaseSchema(db);
+  const now = opts.now ?? new Date();
+  const ttlMs = opts.ttlMs ?? 2 * 60_000;
+  const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
+  const nowIso = now.toISOString();
+  const metadataJson = opts.metadata === undefined ? null : JSON.stringify(opts.metadata);
+
+  db.exec("BEGIN IMMEDIATE");
+  try {
+    const current = db.prepare("SELECT holder_id, expires_at FROM scheduler_leases WHERE lease_key = ?").get(leaseKey) as
+      | { holder_id: string; expires_at: string }
+      | undefined;
+
+    const expired = !current || Date.parse(current.expires_at) <= now.getTime();
+    const sameHolder = current?.holder_id === holderId;
+    if (!expired && !sameHolder) {
+      db.exec("ROLLBACK");
+      return false;
+    }
+
+    db.prepare(`
+      INSERT OR REPLACE INTO scheduler_leases
+      (lease_key, holder_id, expires_at, heartbeat_at, metadata_json)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(leaseKey, holderId, expiresAt, nowIso, metadataJson);
+    db.exec("COMMIT");
+    return true;
+  } catch (err) {
+    try { db.exec("ROLLBACK"); } catch {}
+    throw err;
+  }
+}
+
+export function renewLease(
+  db: Database.Database,
+  leaseKey: string,
+  holderId: string,
+  opts: { ttlMs?: number; now?: Date } = {},
+): boolean {
+  ensureLeaseSchema(db);
+  const now = opts.now ?? new Date();
+  const expiresAt = new Date(now.getTime() + (opts.ttlMs ?? 2 * 60_000)).toISOString();
+  const result = db.prepare(`
+    UPDATE scheduler_leases
+    SET expires_at = ?, heartbeat_at = ?
+    WHERE lease_key = ? AND holder_id = ?
+  `).run(expiresAt, now.toISOString(), leaseKey, holderId);
+  return result.changes === 1;
+}
+
+export function releaseLease(db: Database.Database, leaseKey: string, holderId: string): boolean {
+  ensureLeaseSchema(db);
+  const result = db.prepare("DELETE FROM scheduler_leases WHERE lease_key = ? AND holder_id = ?").run(leaseKey, holderId);
+  return result.changes === 1;
+}
+
+export function getLease(db: Database.Database, leaseKey: string): LeaseRecord | null {
+  ensureLeaseSchema(db);
+  const row = db.prepare(`
+    SELECT lease_key as leaseKey, holder_id as holderId, expires_at as expiresAt,
+           heartbeat_at as heartbeatAt, metadata_json as metadataJson
+    FROM scheduler_leases WHERE lease_key = ?
+  `).get(leaseKey) as LeaseRecord | undefined;
+  return row ?? null;
+}