sphica 0.5.7 → 0.6.1

package/dist/deliver.js

@@ -11086,7 +11086,7 @@ import os from "node:os";
 import path2 from "node:path";
 import { constants as C, DatabaseSync } from "node:sqlite";
 var SCHEMA_GENERATION = 2;
-var SCHEMA_REVISION = 1;
+var SCHEMA_REVISION = 2;
 var sphicaHome = () => process.env.SPHICA_HOME || path2.join(os.homedir(), ".sphica");
 var dbFile = () => process.env.SPHICA_DB || path2.join(sphicaHome(), "sphica.db");
 function requireRuntime() {
@@ -11110,7 +11110,7 @@ function prepare(raw, check) {
   const got = raw.prepare("pragma user_version").get()?.user_version;
   if (got === SCHEMA_REVISION)
     return;
-  throw new Error(`The database schema is revision ${got}, but this Sphica expects revision ${SCHEMA_REVISION}. ` + ((got ?? 0) < SCHEMA_REVISION ? "Move it aside (it is left unchanged), then run `sphica init`." : "Update sphica."));
+  throw new Error(`The database schema is revision ${got}, but this Sphica expects revision ${SCHEMA_REVISION}. ` + ((got ?? 0) < SCHEMA_REVISION ? "Update the sphica CLI (`npm i -g sphica`), then run `sphica init` to migrate it (records are kept)." : "Update sphica."));
 }
 function generationOf(raw) {
   const has = raw.prepare("select 1 from sqlite_schema where type = 'table' and name = 'sphica_generation'").get();
@@ -11282,22 +11282,54 @@ function captureAuthorizer(own, action, p1, p2, triggerOrView) {
   return C2.SQLITE_DENY;
 }
 var writes = (action) => action === C2.SQLITE_INSERT || action === C2.SQLITE_UPDATE || action === C2.SQLITE_DELETE;
+var FORGET_ONLY = new Set(["forget_batch", "source_forgotten"]);
+var KEPT_BY_INGEST = new Set(["source", "unit_evidence", "unit_adoption"]);
 function ingestAuthorizer(action, p1, p2) {
   if (DDL().has(action))
     return C2.SQLITE_DENY;
-  if (writes(action) && p1 === "owner_identity")
+  if (writes(action) && (p1 === "owner_identity" || FORGET_ONLY.has(p1 ?? "")))
+    return C2.SQLITE_DENY;
+  if (action === C2.SQLITE_DELETE && KEPT_BY_INGEST.has(p1 ?? ""))
     return C2.SQLITE_DENY;
   if (action === C2.SQLITE_PRAGMA)
     return readsDataVersion(p1, p2) ? C2.SQLITE_OK : C2.SQLITE_DENY;
   return C2.SQLITE_OK;
 }
+var FORGET_WRITES = {
+  [C2.SQLITE_INSERT]: new Set(["forget_batch", "source_forgotten", "unit_state", "source_fts"]),
+  [C2.SQLITE_DELETE]: new Set([
+    "source",
+    "external_reference",
+    "unit_evidence",
+    "unit_adoption",
+    "source_processing",
+    "source_fts"
+  ])
+};
+var FORGET_UPDATES = {
+  unit: new Set(["lifecycle", "revision"]),
+  unit_state: new Set(["source_id"])
+};
+function forgetAuthorizer(action, p1, p2) {
+  if (SHADOW.test(p1 ?? "") && writes(action))
+    return C2.SQLITE_OK;
+  if (action === C2.SQLITE_UPDATE)
+    return FORGET_UPDATES[p1 ?? ""]?.has(p2 ?? "") ? C2.SQLITE_OK : C2.SQLITE_DENY;
+  if (action === C2.SQLITE_INSERT || action === C2.SQLITE_DELETE)
+    return FORGET_WRITES[action]?.has(p1 ?? "") ? C2.SQLITE_OK : C2.SQLITE_DENY;
+  if (action === C2.SQLITE_PRAGMA)
+    return readsDataVersion(p1, p2) || p1 === "secure_delete" && ["on", "1", "true"].includes((p2 ?? "").toLowerCase()) || p1 === "wal_checkpoint" ? C2.SQLITE_OK : C2.SQLITE_DENY;
+  if (action === C2.SQLITE_READ || action === C2.SQLITE_SELECT || action === C2.SQLITE_FUNCTION || action === C2.SQLITE_TRANSACTION || action === C2.SQLITE_SAVEPOINT)
+    return C2.SQLITE_OK;
+  return C2.SQLITE_DENY;
+}
 function connectWriter(role, file = dbFile(), create = false) {
   requireRuntime();
   if (!create)
     requireFile(file);
   const raw = new DatabaseSync2(file);
   try {
-    prepare(raw, create ? "none" : role === "ingest" ? "revision" : "generation");
+    prepare(raw, create ? "none" : role === "ingest" || role === "forget" ? "revision" : "generation");
     raw.function("sphica_terms", { deterministic: true }, (text) => terms(String(text ?? "")).join(" "));
   } catch (e) {
     raw.close();
@@ -11305,6 +11337,8 @@ function connectWriter(role, file = dbFile(), create = false) {
   }
   if (role === "ingest")
     raw.setAuthorizer(ingestAuthorizer);
+  else if (role === "forget")
+    raw.setAuthorizer(forgetAuthorizer);
   else if (role === "capture") {
     let own = false;
     raw.setAuthorizer((action, p1, p2, _db, triggerOrView) => captureAuthorizer(own, action, p1, p2, triggerOrView));
@@ -11813,6 +11847,7 @@ function lock() {
   return null;
 }
 var BATCH = 500;
+var FLUSH_BUDGET_MS = 30000;
 var ROWS = 1000;
 var chunks = (xs, n = ROWS) => Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, (i + 1) * n));
 async function write(db, batch, projects) {
@@ -11888,98 +11923,135 @@ async function write(db, batch, projects) {
 }
 var REJECTED = new Set([18, 19, 20, 25]);
 var rejected = (e) => REJECTED.has(sqliteCode(e) ?? -1);
-async function flush(file = dbFile()) {
-  const unlock = lock();
-  if (!unlock)
-    return { sent: 0, deferred: 0, rejected: 0, busy: true };
-  const dir = spoolDir();
-  let client = null;
+var queued = (from) => {
+  try {
+    return fs4.readdirSync(from).filter((f) => f.endsWith(".json") && !f.startsWith(".")).sort().map((name) => ({ name, from }));
+  } catch {
+    return [];
+  }
+};
+async function sendBatch(db, names) {
+  const held = unregisteredDir();
+  const records = [];
+  for (const { name, from } of names) {
+    let r;
+    try {
+      r = current(JSON.parse(fs4.readFileSync(path5.join(from, name), "utf8")));
+    } catch {
+      fs4.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
+      fs4.renameSync(path5.join(from, name), path5.join(rejectedDir(), name));
+      continue;
+    }
+    if (r)
+      records.push({ name, from, r });
+    else
+      fs4.rmSync(path5.join(from, name), { force: true });
+  }
+  const projects = new Map((await db.selectFrom("project").select(["id", "key", "name"]).where("key", "in", [...new Set(records.map((x) => x.r.project))]).execute()).map((p) => [p.key, { id: p.id, name: p.name }]));
+  const known = records.filter((x) => projects.has(x.r.project));
+  const strayed = records.filter((x) => !projects.has(x.r.project));
+  let sent = 0;
+  const bad = [];
   try {
-    const held = unregisteredDir();
-    const list = (from) => {
+    sent = await write(db, known.map((x) => x.r), projects);
+  } catch (e) {
+    if (!rejected(e))
+      throw e;
+    for (const x of known) {
       try {
-        return fs4.readdirSync(from).filter((f) => f.endsWith(".json") && !f.startsWith(".")).map((name) => ({ name, from }));
-      } catch {
-        return [];
+        sent += await write(db, [x.r], projects);
+      } catch (e2) {
+        if (!rejected(e2))
+          throw e2;
+        bad.push(x);
       }
-    };
-    const names = [...list(dir), ...list(held)].sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0).slice(0, BATCH);
-    if (names.length === 0)
-      return { sent: 0, deferred: 0, rejected: 0 };
-    const records = [];
-    for (const { name, from } of names) {
-      let r;
+    }
+  }
+  if (bad.length) {
+    fs4.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
+    for (const x of bad) {
       try {
-        r = current(JSON.parse(fs4.readFileSync(path5.join(from, name), "utf8")));
-      } catch {
-        fs4.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
-        fs4.renameSync(path5.join(from, name), path5.join(rejectedDir(), name));
-        continue;
-      }
-      if (r)
-        records.push({ name, from, r });
-      else
-        fs4.rmSync(path5.join(from, name), { force: true });
-    }
-    const db = openWriter("capture", file);
-    client = db;
-    const projects = new Map((await db.selectFrom("project").select(["id", "key", "name"]).where("key", "in", [...new Set(records.map((x) => x.r.project))]).execute()).map((p) => [p.key, { id: p.id, name: p.name }]));
-    const known = records.filter((x) => projects.has(x.r.project));
-    const strayed = records.filter((x) => !projects.has(x.r.project));
-    let sent = 0;
-    const bad = [];
-    try {
-      sent = await write(db, known.map((x) => x.r), projects);
-    } catch (e) {
-      if (!rejected(e))
-        throw e;
-      for (const x of known) {
-        try {
-          sent += await write(db, [x.r], projects);
-        } catch (e2) {
-          if (!rejected(e2))
-            throw e2;
-          bad.push(x);
-        }
+        fs4.renameSync(path5.join(x.from, x.name), path5.join(rejectedDir(), x.name));
+      } catch (e) {
+        if (e.code !== "ENOENT")
+          throw e;
       }
     }
-    if (bad.length) {
-      fs4.mkdirSync(rejectedDir(), { recursive: true, mode: 448 });
-      for (const x of bad) {
-        try {
-          fs4.renameSync(path5.join(x.from, x.name), path5.join(rejectedDir(), x.name));
-        } catch (e) {
-          if (e.code !== "ENOENT")
-            throw e;
-        }
+  }
+  if (strayed.length) {
+    fs4.mkdirSync(held, { recursive: true, mode: 448 });
+    for (const x of strayed) {
+      if (x.from === held)
+        continue;
+      try {
+        fs4.renameSync(path5.join(x.from, x.name), path5.join(held, x.name));
+      } catch (e) {
+        if (e.code !== "ENOENT")
+          throw e;
       }
     }
-    if (strayed.length) {
-      fs4.mkdirSync(held, { recursive: true, mode: 448 });
-      for (const x of strayed) {
-        if (x.from === held)
-          continue;
-        try {
-          fs4.renameSync(path5.join(x.from, x.name), path5.join(held, x.name));
-        } catch (e) {
-          if (e.code !== "ENOENT")
-            throw e;
+  }
+  const moved = new Set([...bad, ...strayed].map((x) => x.name));
+  for (const x of records)
+    if (!moved.has(x.name))
+      fs4.rmSync(path5.join(x.from, x.name), { force: true });
+  return { sent, rejected: bad.length };
+}
+async function flush(file = dbFile(), budgetMs = FLUSH_BUDGET_MS) {
+  const deadline = Date.now() + budgetMs;
+  const dir = spoolDir();
+  const held = unregisteredDir();
+  const total = { sent: 0, deferred: 0, rejected: 0 };
+  let batches = 0;
+  let queueBatches = 0;
+  for (let first = true;; first = false) {
+    let unlock = lock();
+    while (!unlock && Date.now() < deadline) {
+      await new Promise((resolve) => setTimeout(resolve, Math.min(100, deadline - Date.now())));
+      if (Date.now() < deadline)
+        unlock = lock();
+    }
+    if (!unlock)
+      return first ? { ...total, busy: true } : total;
+    let client = null;
+    const send = async (names) => {
+      client ??= openWriter("capture", file);
+      const r = await sendBatch(client, names);
+      total.sent += r.sent;
+      total.rejected += r.rejected;
+      batches++;
+    };
+    const late = () => batches > 0 && Date.now() >= deadline;
+    const sentBefore = batches;
+    try {
+      if (first) {
+        prune(held);
+        for (const part of chunks(queued(held), BATCH)) {
+          if (late())
+            break;
+          await send(part);
         }
       }
+      while (queueBatches === 0 || !late()) {
+        const part = queued(dir).slice(0, BATCH);
+        if (part.length === 0)
+          break;
+        await send(part);
+        queueBatches++;
+      }
+      prune(held);
+      total.deferred = queued(held).length;
+      if (batches > sentBefore)
+        writeState({ flushedAt: new Date().toISOString(), error: null, deferred: total.deferred });
+    } catch (e) {
+      writeState({ flushedAt: new Date().toISOString(), error: reason(e).slice(0, 300) });
+      throw e;
+    } finally {
+      await client?.destroy().catch(() => {});
+      unlock();
     }
-    const moved = new Set([...bad, ...strayed].map((x) => x.name));
-    for (const x of records)
-      if (!moved.has(x.name))
-        fs4.rmSync(path5.join(x.from, x.name), { force: true });
-    prune(held);
-    writeState({ flushedAt: new Date().toISOString(), error: null, deferred: strayed.length });
-    return { sent, deferred: strayed.length, rejected: bad.length };
-  } catch (e) {
-    writeState({ flushedAt: new Date().toISOString(), error: reason(e).slice(0, 300) });
-    throw e;
-  } finally {
-    await client?.destroy().catch(() => {});
-    unlock();
+    if (Date.now() >= deadline || queued(dir).length === 0)
+      return total;
   }
 }
 async function readInput(stream) {