sphica 0.5.7 → 0.6.1

package/dist/cli.js

@@ -14187,7 +14187,7 @@ import os from "node:os";
 import path2 from "node:path";
 import { constants as C, DatabaseSync } from "node:sqlite";
 var SCHEMA_GENERATION = 2;
-var SCHEMA_REVISION = 1;
+var SCHEMA_REVISION = 2;
 var sphicaHome = () => process.env.SPHICA_HOME || path2.join(os.homedir(), ".sphica");
 var dbFile = () => process.env.SPHICA_DB || path2.join(sphicaHome(), "sphica.db");
 function requireRuntime() {
@@ -14211,7 +14211,7 @@ function prepare(raw, check) {
   const got = raw.prepare("pragma user_version").get()?.user_version;
   if (got === SCHEMA_REVISION)
     return;
-  throw new Error(`The database schema is revision ${got}, but this Sphica expects revision ${SCHEMA_REVISION}. ` + ((got ?? 0) < SCHEMA_REVISION ? "Move it aside (it is left unchanged), then run `sphica init`." : "Update sphica."));
+  throw new Error(`The database schema is revision ${got}, but this Sphica expects revision ${SCHEMA_REVISION}. ` + ((got ?? 0) < SCHEMA_REVISION ? "Update the sphica CLI (`npm i -g sphica`), then run `sphica init` to migrate it (records are kept)." : "Update sphica."));
 }
 function generationOf(raw) {
   const has = raw.prepare("select 1 from sqlite_schema where type = 'table' and name = 'sphica_generation'").get();
@@ -14383,22 +14383,54 @@ function captureAuthorizer(own, action, p1, p2, triggerOrView) {
   return C2.SQLITE_DENY;
 }
 var writes = (action) => action === C2.SQLITE_INSERT || action === C2.SQLITE_UPDATE || action === C2.SQLITE_DELETE;
+var FORGET_ONLY = new Set(["forget_batch", "source_forgotten"]);
+var KEPT_BY_INGEST = new Set(["source", "unit_evidence", "unit_adoption"]);
 function ingestAuthorizer(action, p1, p2) {
   if (DDL().has(action))
     return C2.SQLITE_DENY;
-  if (writes(action) && p1 === "owner_identity")
+  if (writes(action) && (p1 === "owner_identity" || FORGET_ONLY.has(p1 ?? "")))
+    return C2.SQLITE_DENY;
+  if (action === C2.SQLITE_DELETE && KEPT_BY_INGEST.has(p1 ?? ""))
     return C2.SQLITE_DENY;
   if (action === C2.SQLITE_PRAGMA)
     return readsDataVersion(p1, p2) ? C2.SQLITE_OK : C2.SQLITE_DENY;
   return C2.SQLITE_OK;
 }
+var FORGET_WRITES = {
+  [C2.SQLITE_INSERT]: new Set(["forget_batch", "source_forgotten", "unit_state", "source_fts"]),
+  [C2.SQLITE_DELETE]: new Set([
+    "source",
+    "external_reference",
+    "unit_evidence",
+    "unit_adoption",
+    "source_processing",
+    "source_fts"
+  ])
+};
+var FORGET_UPDATES = {
+  unit: new Set(["lifecycle", "revision"]),
+  unit_state: new Set(["source_id"])
+};
+function forgetAuthorizer(action, p1, p2) {
+  if (SHADOW.test(p1 ?? "") && writes(action))
+    return C2.SQLITE_OK;
+  if (action === C2.SQLITE_UPDATE)
+    return FORGET_UPDATES[p1 ?? ""]?.has(p2 ?? "") ? C2.SQLITE_OK : C2.SQLITE_DENY;
+  if (action === C2.SQLITE_INSERT || action === C2.SQLITE_DELETE)
+    return FORGET_WRITES[action]?.has(p1 ?? "") ? C2.SQLITE_OK : C2.SQLITE_DENY;
+  if (action === C2.SQLITE_PRAGMA)
+    return readsDataVersion(p1, p2) || p1 === "secure_delete" && ["on", "1", "true"].includes((p2 ?? "").toLowerCase()) || p1 === "wal_checkpoint" ? C2.SQLITE_OK : C2.SQLITE_DENY;
+  if (action === C2.SQLITE_READ || action === C2.SQLITE_SELECT || action === C2.SQLITE_FUNCTION || action === C2.SQLITE_TRANSACTION || action === C2.SQLITE_SAVEPOINT)
+    return C2.SQLITE_OK;
+  return C2.SQLITE_DENY;
+}
 function connectWriter(role, file = dbFile(), create = false) {
   requireRuntime();
   if (!create)
     requireFile(file);
   const raw = new DatabaseSync2(file);
   try {
-    prepare(raw, create ? "none" : role === "ingest" ? "revision" : "generation");
+    prepare(raw, create ? "none" : role === "ingest" || role === "forget" ? "revision" : "generation");
     raw.function("sphica_terms", { deterministic: true }, (text) => terms(String(text ?? "")).join(" "));
   } catch (e) {
     raw.close();
@@ -14406,6 +14438,8 @@ function connectWriter(role, file = dbFile(), create = false) {
   }
   if (role === "ingest")
     raw.setAuthorizer(ingestAuthorizer);
+  else if (role === "forget")
+    raw.setAuthorizer(forgetAuthorizer);
   else if (role === "capture") {
     let own = false;
     raw.setAuthorizer((action, p1, p2, _db, triggerOrView) => captureAuthorizer(own, action, p1, p2, triggerOrView));
@@ -14431,6 +14465,7 @@ function openWriter(role, file = dbFile()) {
 // server/src/admin.ts
 var say = (text) => console.log(indent(text));
 var SCHEMA = () => path3.join(dbDir(), "schema.sql");
+var MIGRATION = (revision) => path3.join(dbDir(), "migrations", `${String(revision).padStart(4, "0")}.sql`);
 var versionOf = (raw) => raw.prepare("pragma user_version").get().user_version;
 function immediate(raw, fn) {
   raw.exec("begin immediate");
@@ -14451,6 +14486,32 @@ function withOwner(file, fn, create = false) {
     raw.close();
   }
 }
+function migrate(file = dbFile()) {
+  return withOwner(file, (raw) => {
+    const from = versionOf(raw);
+    for (let r2 = from + 1;r2 <= SCHEMA_REVISION; r2++) {
+      const script = MIGRATION(r2);
+      if (!fs3.existsSync(script))
+        throw new Error(`No migration from revision ${r2 - 1}. Move the database aside, then run \`sphica init\`.`);
+      raw.exec("pragma foreign_keys = off");
+      try {
+        immediate(raw, () => {
+          if (versionOf(raw) !== r2 - 1)
+            return;
+          raw.exec(fs3.readFileSync(script, "utf8"));
+          const broken = raw.prepare("pragma foreign_key_check").all().length;
+          if (broken)
+            throw new Error(`Migration to revision ${r2} left ${plural(broken, "broken reference")}; nothing was changed`);
+          if (versionOf(raw) !== r2)
+            throw new Error(`${path3.basename(script)} did not set revision ${r2}`);
+        });
+      } finally {
+        raw.exec("pragma foreign_keys = on");
+      }
+    }
+    return from;
+  });
+}
 function dbInit(file = dbFile()) {
   if (fs3.existsSync(file)) {
     const look = new DatabaseSync3(file, { readOnly: true });
@@ -14465,8 +14526,12 @@ function dbInit(file = dbFile()) {
     const got = withOwner(file, versionOf);
     if (got === SCHEMA_REVISION)
       say(`Already exists: ${file} (revision ${got})`);
-    else
-      say(`Already exists: ${file} (revision ${got}; this Sphica expects ${SCHEMA_REVISION}. Move it aside, then run \`sphica init\`.)`);
+    else if (got > SCHEMA_REVISION)
+      say(`Already exists: ${file} (revision ${got}, made by a newer Sphica; this one expects ${SCHEMA_REVISION}. Update sphica.)`);
+    else {
+      migrate(file);
+      say(`Migrated: ${file} (revision ${got} → ${SCHEMA_REVISION})`);
+    }
     return;
   }
   fs3.mkdirSync(path3.dirname(file), { recursive: true, mode: 448 });
@@ -15067,6 +15132,7 @@ function lock() {
   return null;
 }
 var BATCH = 500;
+var FLUSH_BUDGET_MS = 30000;
 var ROWS = 1000;
 var chunks = (xs, n2 = ROWS) => Array.from({ length: Math.ceil(xs.length / n2) }, (_, i2) => xs.slice(i2 * n2, (i2 + 1) * n2));
 async function write(db, batch, projects) {
@@ -15142,98 +15208,135 @@ async function write(db, batch, projects) {
 }
 var REJECTED = new Set([18, 19, 20, 25]);
 var rejected = (e) => REJECTED.has(sqliteCode(e) ?? -1);
-async function flush(file = dbFile()) {
-  const unlock = lock();
-  if (!unlock)
-    return { sent: 0, deferred: 0, rejected: 0, busy: true };
-  const dir = spoolDir();
-  let client = null;
+var queued = (from) => {
   try {
-    const held = unregisteredDir();
-    const list = (from) => {
+    return fs6.readdirSync(from).filter((f) => f.endsWith(".json") && !f.startsWith(".")).sort().map((name) => ({ name, from }));
+  } catch {
+    return [];
+  }
+};
+async function sendBatch(db, names) {
+  const held = unregisteredDir();
+  const records = [];
+  for (const { name, from } of names) {
+    let r2;
+    try {
+      r2 = current(JSON.parse(fs6.readFileSync(path7.join(from, name), "utf8")));
+    } catch {
+      fs6.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
+      fs6.renameSync(path7.join(from, name), path7.join(rejectedDir(), name));
+      continue;
+    }
+    if (r2)
+      records.push({ name, from, r: r2 });
+    else
+      fs6.rmSync(path7.join(from, name), { force: true });
+  }
+  const projects = new Map((await db.selectFrom("project").select(["id", "key", "name"]).where("key", "in", [...new Set(records.map((x) => x.r.project))]).execute()).map((p) => [p.key, { id: p.id, name: p.name }]));
+  const known = records.filter((x) => projects.has(x.r.project));
+  const strayed = records.filter((x) => !projects.has(x.r.project));
+  let sent = 0;
+  const bad = [];
+  try {
+    sent = await write(db, known.map((x) => x.r), projects);
+  } catch (e) {
+    if (!rejected(e))
+      throw e;
+    for (const x of known) {
       try {
-        return fs6.readdirSync(from).filter((f) => f.endsWith(".json") && !f.startsWith(".")).map((name) => ({ name, from }));
-      } catch {
-        return [];
+        sent += await write(db, [x.r], projects);
+      } catch (e2) {
+        if (!rejected(e2))
+          throw e2;
+        bad.push(x);
       }
-    };
-    const names = [...list(dir), ...list(held)].sort((a2, b) => a2.name < b.name ? -1 : a2.name > b.name ? 1 : 0).slice(0, BATCH);
-    if (names.length === 0)
-      return { sent: 0, deferred: 0, rejected: 0 };
-    const records = [];
-    for (const { name, from } of names) {
-      let r2;
+    }
+  }
+  if (bad.length) {
+    fs6.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
+    for (const x of bad) {
       try {
-        r2 = current(JSON.parse(fs6.readFileSync(path7.join(from, name), "utf8")));
-      } catch {
-        fs6.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
-        fs6.renameSync(path7.join(from, name), path7.join(rejectedDir(), name));
-        continue;
-      }
-      if (r2)
-        records.push({ name, from, r: r2 });
-      else
-        fs6.rmSync(path7.join(from, name), { force: true });
-    }
-    const db = openWriter("capture", file);
-    client = db;
-    const projects = new Map((await db.selectFrom("project").select(["id", "key", "name"]).where("key", "in", [...new Set(records.map((x) => x.r.project))]).execute()).map((p) => [p.key, { id: p.id, name: p.name }]));
-    const known = records.filter((x) => projects.has(x.r.project));
-    const strayed = records.filter((x) => !projects.has(x.r.project));
-    let sent = 0;
-    const bad = [];
-    try {
-      sent = await write(db, known.map((x) => x.r), projects);
-    } catch (e) {
-      if (!rejected(e))
-        throw e;
-      for (const x of known) {
-        try {
-          sent += await write(db, [x.r], projects);
-        } catch (e2) {
-          if (!rejected(e2))
-            throw e2;
-          bad.push(x);
-        }
+        fs6.renameSync(path7.join(x.from, x.name), path7.join(rejectedDir(), x.name));
+      } catch (e) {
+        if (e.code !== "ENOENT")
+          throw e;
       }
     }
-    if (bad.length) {
-      fs6.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
-      for (const x of bad) {
-        try {
-          fs6.renameSync(path7.join(x.from, x.name), path7.join(rejectedDir(), x.name));
-        } catch (e) {
-          if (e.code !== "ENOENT")
-            throw e;
-        }
+  }
+  if (strayed.length) {
+    fs6.mkdirSync(held, { recursive: true, mode: 448 });
+    for (const x of strayed) {
+      if (x.from === held)
+        continue;
+      try {
+        fs6.renameSync(path7.join(x.from, x.name), path7.join(held, x.name));
+      } catch (e) {
+        if (e.code !== "ENOENT")
+          throw e;
       }
     }
-    if (strayed.length) {
-      fs6.mkdirSync(held, { recursive: true, mode: 448 });
-      for (const x of strayed) {
-        if (x.from === held)
-          continue;
-        try {
-          fs6.renameSync(path7.join(x.from, x.name), path7.join(held, x.name));
-        } catch (e) {
-          if (e.code !== "ENOENT")
-            throw e;
+  }
+  const moved = new Set([...bad, ...strayed].map((x) => x.name));
+  for (const x of records)
+    if (!moved.has(x.name))
+      fs6.rmSync(path7.join(x.from, x.name), { force: true });
+  return { sent, rejected: bad.length };
+}
+async function flush(file = dbFile(), budgetMs = FLUSH_BUDGET_MS) {
+  const deadline = Date.now() + budgetMs;
+  const dir = spoolDir();
+  const held = unregisteredDir();
+  const total = { sent: 0, deferred: 0, rejected: 0 };
+  let batches = 0;
+  let queueBatches = 0;
+  for (let first = true;; first = false) {
+    let unlock = lock();
+    while (!unlock && Date.now() < deadline) {
+      await new Promise((resolve) => setTimeout(resolve, Math.min(100, deadline - Date.now())));
+      if (Date.now() < deadline)
+        unlock = lock();
+    }
+    if (!unlock)
+      return first ? { ...total, busy: true } : total;
+    let client = null;
+    const send = async (names) => {
+      client ??= openWriter("capture", file);
+      const r2 = await sendBatch(client, names);
+      total.sent += r2.sent;
+      total.rejected += r2.rejected;
+      batches++;
+    };
+    const late = () => batches > 0 && Date.now() >= deadline;
+    const sentBefore = batches;
+    try {
+      if (first) {
+        prune(held);
+        for (const part of chunks(queued(held), BATCH)) {
+          if (late())
+            break;
+          await send(part);
         }
       }
+      while (queueBatches === 0 || !late()) {
+        const part = queued(dir).slice(0, BATCH);
+        if (part.length === 0)
+          break;
+        await send(part);
+        queueBatches++;
+      }
+      prune(held);
+      total.deferred = queued(held).length;
+      if (batches > sentBefore)
+        writeState({ flushedAt: new Date().toISOString(), error: null, deferred: total.deferred });
+    } catch (e) {
+      writeState({ flushedAt: new Date().toISOString(), error: reason(e).slice(0, 300) });
+      throw e;
+    } finally {
+      await client?.destroy().catch(() => {});
+      unlock();
     }
-    const moved = new Set([...bad, ...strayed].map((x) => x.name));
-    for (const x of records)
-      if (!moved.has(x.name))
-        fs6.rmSync(path7.join(x.from, x.name), { force: true });
-    prune(held);
-    writeState({ flushedAt: new Date().toISOString(), error: null, deferred: strayed.length });
-    return { sent, deferred: strayed.length, rejected: bad.length };
-  } catch (e) {
-    writeState({ flushedAt: new Date().toISOString(), error: reason(e).slice(0, 300) });
-    throw e;
-  } finally {
-    await client?.destroy().catch(() => {});
-    unlock();
+    if (Date.now() >= deadline || queued(dir).length === 0)
+      return total;
   }
 }
 async function readInput(stream) {
@@ -15736,7 +15839,7 @@ async function doctor(cwd) {
       const x = inspect(file);
       usable2 = x.revision === SCHEMA_REVISION;
       say2("ok", "DB", `${file} (${(x.bytes / 1024 / 1024).toFixed(1)} MB)`);
-      say2(usable2 ? "ok" : "fail", "Schema version", usable2 ? `revision ${x.revision}` : `revision ${x.revision}, this Sphica expects ${SCHEMA_REVISION} (${x.revision < SCHEMA_REVISION ? "made by an older Sphica: move it aside, then run sphica init" : "update sphica"})`);
+      say2(usable2 ? "ok" : "fail", "Schema version", usable2 ? `revision ${x.revision}` : `revision ${x.revision}, this Sphica expects ${SCHEMA_REVISION} (${x.revision < SCHEMA_REVISION ? "made by an older Sphica: run sphica init to migrate it" : "update sphica"})`);
       const broken = Object.entries(x.fts).filter(([, v]) => v !== null);
       say2(broken.length ? "fail" : "ok", "Full-text index", broken.length ? `broken: ${broken.map(([k, v]) => `${k} (${plain(v ?? "")})`).join(" / ")}. Rebuild it with sphica doctor --reindex` : "healthy");
     } catch (e) {