sphica 0.6.0 → 0.6.2

package/dist/mcp.js

@@ -46233,34 +46233,81 @@ async function checkFindings(db, projectId2, files, raw) {
 }
 
 // server/src/search.ts
-var POOL = 200;
+var UNIT_PAGE = 200;
+var UNIT_SCAN_MAX = 2000;
+var SOURCE_PAGE = 50;
+var SOURCE_SCAN_MAX = 600;
+var SOURCE_SCAN_BYTES = 64 * 1024 * 1024;
 var LIFE_ORDER = { active: 0, candidate: 1, superseded: 2, withdrawn: 3 };
 var strong = (matched, of) => matched * 2 > of;
 async function searchUnits(db, projectId2, q) {
   const wanted = queryTerms(q.question);
   const match = ftsQuery(q.question);
   if (!match)
-    return { hits: [], weaker: 0, terms: wanted };
-  let query = db.selectFrom(sql`(select unit_fts.rowid as rowid, bm25(unit_fts, 3, 2, 1) as rank from unit_fts join unit on unit.id = unit_fts.rowid where unit_fts match ${match} and unit.project_id = ${projectId2})`.as("f")).innerJoin("unit as u", "u.id", "f.rowid").where("u.project_id", "=", projectId2).where("u.extraction", "=", "supported").select([
-    "u.id",
-    "u.key",
-    "u.kind",
-    "u.stance",
-    "u.lifecycle",
-    "u.text",
-    "u.why",
-    "u.scope_note",
-    "u.revisit_when",
-    "u.content_hash",
-    "f.rank"
-  ]);
+    return { hits: [], weaker: 0, terms: wanted, stopped: false, read: 0 };
+  let query = db.selectFrom(sql`(select unit_fts.rowid as rowid, bm25(unit_fts, 3, 2, 1) as rank from unit_fts join unit on unit.id = unit_fts.rowid where unit_fts match ${match} and unit.project_id = ${projectId2})`.as("f")).innerJoin("unit as u", "u.id", "f.rowid").where("u.project_id", "=", projectId2).where("u.extraction", "=", "supported");
   if (q.kinds?.length)
     query = query.where("u.kind", "in", q.kinds);
   if (q.lifecycles?.length)
     query = query.where("u.lifecycle", "in", q.lifecycles);
   if (q.path)
     query = query.where(({ exists, selectFrom }) => exists(selectFrom("unit_anchor as a").select("a.id").whereRef("a.unit_id", "=", "u.id").where("a.path", "=", q.path ?? "").where("a.retired_at", "is", null)));
-  const rows = await query.orderBy("f.rank").limit(POOL).execute();
+  const ranked = await query.select(["u.id", "f.rank"]).orderBy("f.rank").orderBy("u.id").limit(UNIT_SCAN_MAX + 1).execute();
+  const stopped = ranked.length > UNIT_SCAN_MAX;
+  let weaker = 0;
+  let read = 0;
+  const hits = [];
+  for (let at = 0;at < Math.min(ranked.length, UNIT_SCAN_MAX); at += UNIT_PAGE) {
+    const part2 = ranked.slice(at, Math.min(at + UNIT_PAGE, UNIT_SCAN_MAX));
+    const order = new Map(part2.map((r, i) => [r.id, i]));
+    const rows = (await query.select([
+      "u.id",
+      "u.key",
+      "u.kind",
+      "u.stance",
+      "u.lifecycle",
+      "u.text",
+      "u.why",
+      "u.scope_note",
+      "u.revisit_when",
+      "u.content_hash",
+      "f.rank"
+    ]).where("u.id", "in", part2.map((r) => r.id)).execute()).sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
+    read += rows.length;
+    weaker += await judgeUnits(db, rows, wanted, hits);
+  }
+  const shown = new Set(hits.map((h) => h.id));
+  for (const h of [...hits].filter((x) => x.lifecycle === "superseded")) {
+    let successors = db.selectFrom("unit_link as l").innerJoin("unit as n", "n.id", "l.from_unit").where("l.to_unit", "=", h.id).where("l.kind", "=", "supersedes").where("n.extraction", "=", "supported");
+    if (q.kinds?.length)
+      successors = successors.where("n.kind", "in", q.kinds);
+    if (q.lifecycles?.length)
+      successors = successors.where("n.lifecycle", "in", q.lifecycles);
+    const next = await successors.select(["n.id", "n.key", "n.kind", "n.stance", "n.lifecycle", "n.text", "n.why", "n.revisit_when"]).execute();
+    for (const n of next)
+      if (!shown.has(n.id)) {
+        shown.add(n.id);
+        hits.push({
+          ...n,
+          options: [],
+          anchors: [],
+          matched: [],
+          aliasOnly: false,
+          successorOf: h.key,
+          rank: h.rank
+        });
+      }
+  }
+  hits.sort((a, b) => (LIFE_ORDER[a.lifecycle] ?? 9) - (LIFE_ORDER[b.lifecycle] ?? 9) || b.matched.length - a.matched.length || a.rank - b.rank);
+  return {
+    hits: hits.slice(0, q.limit).map(({ rank: _rank, ...h }) => h),
+    weaker,
+    terms: wanted,
+    stopped,
+    read
+  };
+}
+async function judgeUnits(db, rows, wanted, hits) {
   const ids = rows.map((r) => r.id);
   const [options, anchors, aliases] = ids.length ? await Promise.all([
     db.selectFrom("unit_option").select(["unit_id", "text", "outcome", "why"]).where("unit_id", "in", ids).orderBy("position").execute(),
@@ -46268,7 +46315,6 @@ async function searchUnits(db, projectId2, q) {
     db.selectFrom("unit_alias").select(["unit_id", "terms", "content_hash"]).where("unit_id", "in", ids).orderBy("id", "desc").execute()
   ]) : [[], [], []];
   let weaker = 0;
-  const hits = [];
   for (const r of rows) {
     const opts = options.filter((o) => o.unit_id === r.id);
     const anch = anchors.filter((a) => a.unit_id === r.id);
@@ -46305,69 +46351,63 @@ async function searchUnits(db, projectId2, q) {
       rank: r.rank
     });
   }
-  const shown = new Set(hits.map((h) => h.id));
-  for (const h of [...hits].filter((x) => x.lifecycle === "superseded")) {
-    let successors = db.selectFrom("unit_link as l").innerJoin("unit as n", "n.id", "l.from_unit").where("l.to_unit", "=", h.id).where("l.kind", "=", "supersedes").where("n.extraction", "=", "supported");
-    if (q.kinds?.length)
-      successors = successors.where("n.kind", "in", q.kinds);
-    if (q.lifecycles?.length)
-      successors = successors.where("n.lifecycle", "in", q.lifecycles);
-    const next = await successors.select(["n.id", "n.key", "n.kind", "n.stance", "n.lifecycle", "n.text", "n.why", "n.revisit_when"]).execute();
-    for (const n of next)
-      if (!shown.has(n.id)) {
-        shown.add(n.id);
-        hits.push({
-          ...n,
-          options: [],
-          anchors: [],
-          matched: [],
-          aliasOnly: false,
-          successorOf: h.key,
-          rank: h.rank
-        });
-      }
-  }
-  hits.sort((a, b) => (LIFE_ORDER[a.lifecycle] ?? 9) - (LIFE_ORDER[b.lifecycle] ?? 9) || b.matched.length - a.matched.length || a.rank - b.rank);
-  return { hits: hits.slice(0, q.limit).map(({ rank: _rank, ...h }) => h), weaker, terms: wanted };
+  return weaker;
 }
 async function searchSources(db, projectId2, question, limit) {
   const wanted = queryTerms(question);
   const match = ftsQuery(question);
   if (!match)
-    return { hits: [], weaker: 0, terms: wanted };
-  const rows = await db.selectFrom(sql`(select source_fts.rowid as rowid, bm25(source_fts) as rank from source_fts join source on source.id = source_fts.rowid where source_fts match ${match} and source.project_id = ${projectId2})`.as("f")).innerJoin("source as s", "s.id", "f.rowid").where("s.project_id", "=", projectId2).select([
-    "s.id",
-    "s.kind",
-    "s.artifact",
-    "s.author_kind",
-    "s.author_login",
-    "s.created_at",
-    "s.text",
-    "f.rank"
-  ]).orderBy("f.rank").limit(POOL).execute();
+    return { hits: [], weaker: 0, terms: wanted, stopped: false, read: 0 };
+  const query = db.selectFrom(sql`(select source_fts.rowid as rowid, bm25(source_fts) as rank from source_fts join source on source.id = source_fts.rowid where source_fts match ${match} and source.project_id = ${projectId2})`.as("f")).innerJoin("source as s", "s.id", "f.rowid").where("s.project_id", "=", projectId2);
+  const ranked = await query.select(["s.id", "f.rank"]).orderBy("f.rank").orderBy("s.id").limit(SOURCE_SCAN_MAX + 1).execute();
   let weaker = 0;
+  let stopped = false;
   const hits = [];
-  for (const r of rows) {
-    const own2 = new Set(terms(r.text));
-    const matched = wanted.filter((w) => own2.has(w));
-    if (!strong(matched.length, wanted.length)) {
-      weaker++;
-      continue;
+  let read = 0;
+  let bytes2 = 0;
+  scan:
+    for (let at = 0;at < ranked.length; at += SOURCE_PAGE) {
+      const ids = ranked.slice(at, at + SOURCE_PAGE).map((r) => r.id);
+      const byId = new Map((await db.selectFrom("source").select(["id", "kind", "artifact", "author_kind", "author_login", "created_at", "text"]).where("id", "in", ids).execute()).map((r) => [r.id, r]));
+      for (const id of ids) {
+        if (read >= SOURCE_SCAN_MAX || bytes2 >= SOURCE_SCAN_BYTES) {
+          stopped = true;
+          break scan;
+        }
+        const r = byId.get(id);
+        if (!r)
+          continue;
+        read++;
+        bytes2 += Buffer.byteLength(r.text);
+        const own2 = new Set(terms(r.text));
+        const matched = wanted.filter((w) => own2.has(w));
+        if (!strong(matched.length, wanted.length)) {
+          weaker++;
+          continue;
+        }
+        hits.push({
+          id: r.id,
+          kind: r.kind,
+          artifact: r.artifact,
+          author: r.author_login ?? r.author_kind,
+          created_at: r.created_at,
+          text: r.text,
+          matched
+        });
+        if (hits.length >= limit)
+          break scan;
+      }
     }
-    hits.push({
-      id: r.id,
-      kind: r.kind,
-      artifact: r.artifact,
-      author: r.author_login ?? r.author_kind,
-      created_at: r.created_at,
-      text: r.text,
-      matched
-    });
-  }
-  return { hits: hits.slice(0, limit), weaker, terms: wanted };
+  if (!stopped && hits.length < limit && ranked.length > SOURCE_SCAN_MAX)
+    stopped = true;
+  return { hits, weaker, terms: wanted, stopped, read };
 }
 
 // server/src/status.ts
+async function pendingCount(db, projectId2) {
+  const r = await db.selectFrom("session as s").where("s.project_id", "=", projectId2).where((eb) => eb.exists(eb.selectFrom("source as m").whereRef("m.session_id", "=", "s.id").where("m.author_kind", "=", "owner").where((eb2) => eb2.not(eb2.exists(eb2.selectFrom("source_processing as p").whereRef("p.source_id", "=", "m.id").select(sql`1`.as("x"))))).select(sql`1`.as("x")))).select((eb) => eb.fn.countAll().as("n")).executeTakeFirst();
+  return Number(r?.n ?? 0);
+}
 async function coverage(db, projectId2) {
   const count = async (q) => Number((await q)?.n ?? 0);
   const units = (where) => count(where(unitBase()).select((eb) => eb.fn.countAll().as("n")).executeTakeFirst());
@@ -46375,7 +46415,7 @@ async function coverage(db, projectId2) {
   const processed = (outcome) => db.selectFrom("source_processing as p").innerJoin("source as m", "m.id", "p.source_id").where("m.project_id", "=", projectId2).where("p.outcome", "=", outcome).where("m.session_id", "is not", null).select("m.session_id");
   const [sessions, pendingSessions, emptySessions, sources, active, candidates, quarantined, work] = await Promise.all([
     count(db.selectFrom("session").where("project_id", "=", projectId2).select((eb) => eb.fn.countAll().as("n")).executeTakeFirst()),
-    count(db.selectFrom("session as s").where("s.project_id", "=", projectId2).where((eb) => eb.exists(eb.selectFrom("source as m").whereRef("m.session_id", "=", "s.id").where("m.author_kind", "=", "owner").where((eb2) => eb2.not(eb2.exists(eb2.selectFrom("source_processing as p").whereRef("p.source_id", "=", "m.id").select(sql`1`.as("x"))))).select(sql`1`.as("x")))).select((eb) => eb.fn.countAll().as("n")).executeTakeFirst()),
+    pendingCount(db, projectId2),
     count(db.selectFrom("session").where("id", "in", processed("no_unit")).where("id", "not in", processed("units")).select((eb) => eb.fn.countAll().as("n")).executeTakeFirst()),
     count(db.selectFrom("source").where("project_id", "=", projectId2).select((eb) => eb.fn.countAll().as("n")).executeTakeFirst()),
     units((q) => q.where("lifecycle", "=", "active")),
@@ -46418,6 +46458,10 @@ async function projectOf(cwd) {
     return `${head(inline(place.name), 200)} is not registered with Sphica (run \`sphica init\` there).`;
   return { id, root: place.root };
 }
+var among = (r) => r.stopped ? `among the first ${r.read} candidates by rank ` : "";
+var stoppedAfter = (r) => r.stopped ? `
+
+Stopped after ${r.read} candidates by rank; more may match.` : "";
 var hitText = (h) => [
   `## ${h.key} (u${h.id}): ${h.kind}${h.stance ? ` ${h.stance}` : ""}, ${h.lifecycle}`,
   head(h.text, 600),
@@ -46484,12 +46528,12 @@ server.registerTool("search", {
     if (a.sources) {
       const r2 = await searchSources(db, p.id, a.query, limit);
       if (!r2.hits.length)
-        return text(`No source holds most of: ${r2.terms.join(", ") || "(no searchable words)"}. ${r2.weaker} weaker matches left out.`);
+        return text(`No source ${among(r2)}holds most of: ${r2.terms.join(", ") || "(no searchable words)"}. ${r2.weaker} weaker matches left out.${r2.stopped ? " Search with more specific words." : ""}`);
       return text(framed(r2.hits.map((h) => `## s${h.id}: ${h.kind} ${h.artifact}, ${h.author}, ${h.created_at}
 ${head(h.text, 800)}
 Matched: ${h.matched.join(", ")}`).join(`
 
-`)));
+`).concat(stoppedAfter(r2))));
     }
     const r = await searchUnits(db, p.id, {
       question: a.query,
@@ -46499,10 +46543,10 @@ Matched: ${h.matched.join(", ")}`).join(`
       limit
     });
     if (!r.hits.length)
-      return text(`No record holds most of: ${r.terms.join(", ") || "(no searchable words)"}. ${r.weaker} weaker matches left out. ` + "Search again with other words or the other language, or search sources; status says whether sessions are still untraced.");
+      return text(`No record ${among(r)}holds most of: ${r.terms.join(", ") || "(no searchable words)"}. ${r.weaker} weaker matches left out. ` + (r.stopped ? "Search with more specific words. " : "") + "Search again with other words or the other language, or search sources; status says whether sessions are still untraced.");
     return text(framed(`${r.hits.map(hitText).join(`
 
-`)}
+`)}${stoppedAfter(r)}
 
 Read a record by its key before relying on it.`));
   } catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sphica",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "Records Claude Code and Codex sessions on your machine and keeps past implementation and decisions, with their sources, for your agent to find.",
   "license": "MIT",
   "type": "module",