sphica 0.5.6 → 0.6.0

package/dist/mcp-record.js

@@ -34719,9 +34719,6 @@ class StdioServerTransport {
   }
 }
 
-// server/src/db-write.ts
-import { constants as C2, DatabaseSync as DatabaseSync2 } from "node:sqlite";
-
 // server/node_modules/kysely/dist/util/object-utils.js
 function isUndefined(obj) {
   return typeof obj === "undefined" || obj === undefined;
@@ -45530,7 +45527,7 @@ import os from "node:os";
 import path from "node:path";
 import { constants as C, DatabaseSync } from "node:sqlite";
 var SCHEMA_GENERATION = 2;
-var SCHEMA_REVISION = 1;
+var SCHEMA_REVISION = 2;
 var sphicaHome = () => process.env.SPHICA_HOME || path.join(os.homedir(), ".sphica");
 var dbFile = () => process.env.SPHICA_DB || path.join(sphicaHome(), "sphica.db");
 function requireRuntime() {
@@ -45554,7 +45551,7 @@ function prepare(raw, check2) {
   const got = raw.prepare("pragma user_version").get()?.user_version;
   if (got === SCHEMA_REVISION)
     return;
-  throw new Error(`The database schema is revision ${got}, but this Sphica expects revision ${SCHEMA_REVISION}. ` + ((got ?? 0) < SCHEMA_REVISION ? "Move it aside (it is left unchanged), then run `sphica init`." : "Update sphica."));
+  throw new Error(`The database schema is revision ${got}, but this Sphica expects revision ${SCHEMA_REVISION}. ` + ((got ?? 0) < SCHEMA_REVISION ? "Update the sphica CLI (`npm i -g sphica`), then run `sphica init` to migrate it (records are kept)." : "Update sphica."));
 }
 function generationOf(raw) {
   const has = raw.prepare("select 1 from sqlite_schema where type = 'table' and name = 'sphica_generation'").get();
@@ -45626,6 +45623,9 @@ function sqliteCode(e) {
   return x?.code === "ERR_SQLITE_ERROR" && typeof x.errcode === "number" ? x.errcode & 255 : null;
 }
 
+// server/src/db-write.ts
+import { constants as C2, DatabaseSync as DatabaseSync2 } from "node:sqlite";
+
 // server/src/text.ts
 import crypto from "node:crypto";
 var segmenter = new Intl.Segmenter("ja", { granularity: "word" });
@@ -45822,27 +45822,80 @@ var MYSQL_PASSWORD = /(\s-p)(?:'[^'\n]*'|"[^"\n]*"|(?=[^\s-])\S+)/;
 var URL_CREDENTIALS = /\b((?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|rediss?|amqps?|https?):\/\/[^:\s/@]*:)[^\s/]*@([^@\s/?#]+)/g;
 var KEY_BEGIN = /-----BEGIN [A-Z ]*PRIVATE KEY-----/g;
 var KEY_END = /-----END [A-Z ]*PRIVATE KEY-----/g;
-function maskPrivateKeys(text) {
+function keyBlocks(text) {
   const ends = [...text.matchAll(KEY_END)].map((m) => [m.index, m.index + m[0].length]);
-  if (ends.length === 0)
-    return text;
-  let out = "";
+  const out = [];
   let last = 0;
   let e = 0;
-  for (const m of text.matchAll(KEY_BEGIN)) {
-    const after = m.index + m[0].length;
-    if (m.index < last)
-      continue;
-    while (e < ends.length && (ends[e]?.[0] ?? 0) < after)
-      e++;
-    const end = ends[e];
-    if (!end)
-      break;
-    out += `${text.slice(last, m.index)}[redacted: private key]`;
-    last = end[1];
+  if (ends.length > 0)
+    for (const m of text.matchAll(KEY_BEGIN)) {
+      const after = m.index + m[0].length;
+      if (m.index < last)
+        continue;
+      while (e < ends.length && (ends[e]?.[0] ?? 0) < after)
+        e++;
+      const end = ends[e];
+      if (!end)
+        break;
+      out.push([m.index, end[1]]);
+      last = end[1];
+    }
+  return out;
+}
+function maskPrivateKeys(text) {
+  let out = "";
+  let last = 0;
+  for (const [a, b] of keyBlocks(text)) {
+    out += `${text.slice(last, a)}[redacted: private key]`;
+    last = b;
   }
   return out + text.slice(last);
 }
+function privateKeyRanges(text) {
+  return byteRanges(text, keyBlocks(text));
+}
+function byteRanges(text, ranges) {
+  const out = [];
+  let at = 0;
+  let pos = 0;
+  for (const [a, b] of ranges) {
+    pos += bytes(text.slice(at, a));
+    const end = pos + bytes(text.slice(a, b));
+    out.push([pos, end]);
+    at = b;
+    pos = end;
+  }
+  return out;
+}
+var PLACEHOLDER = /\[redacted(?:: [^\]\n]{1,40})?\]/g;
+function placeholderRanges(masked) {
+  return [...masked.matchAll(PLACEHOLDER)].map((x) => [x.index, x.index + x[0].length]);
+}
+function starts(hay, needle) {
+  const out = [];
+  for (let at = hay.indexOf(needle);at >= 0; at = hay.indexOf(needle, at + 1))
+    out.push(at);
+  return out;
+}
+function quoteSpan(raw, masked, quote) {
+  const q = Buffer.from(quote, "utf8");
+  if (q.length === 0)
+    return null;
+  const m = Buffer.from(masked, "utf8");
+  if (raw === masked) {
+    const at = m.indexOf(q);
+    return at < 0 ? null : [at, at + q.length];
+  }
+  const holes = byteRanges(masked, placeholderRanges(masked));
+  let h = 0;
+  const kept = starts(m, q).filter((at) => {
+    while ((holes[h]?.[1] ?? Number.POSITIVE_INFINITY) <= at)
+      h++;
+    return !((holes[h]?.[0] ?? Number.POSITIVE_INFINITY) < at + q.length);
+  });
+  const first = kept[0];
+  return first !== undefined && kept.length === starts(Buffer.from(raw, "utf8"), q).length ? [first, first + q.length] : null;
+}
 function mask(text) {
   let out = maskFields(maskPrivateKeys(text).replace(URL_CREDENTIALS, "$1[redacted]@$2").replace(AUTH_HEADER, "$1[redacted]").replace(HEADER_BEARER, "$1[redacted]").replace(ENV_ASSIGN, "$1$2[redacted]")).replace(MYSQL_COMMAND, (command) => command.replace(MYSQL_PASSWORD, "$1[redacted]"));
   for (const [re, what] of SECRETS)
@@ -45940,22 +45993,54 @@ function captureAuthorizer(own2, action, p1, p2, triggerOrView) {
   return C2.SQLITE_DENY;
 }
 var writes = (action) => action === C2.SQLITE_INSERT || action === C2.SQLITE_UPDATE || action === C2.SQLITE_DELETE;
+var FORGET_ONLY = new Set(["forget_batch", "source_forgotten"]);
+var KEPT_BY_INGEST = new Set(["source", "unit_evidence", "unit_adoption"]);
 function ingestAuthorizer(action, p1, p2) {
   if (DDL().has(action))
     return C2.SQLITE_DENY;
-  if (writes(action) && p1 === "owner_identity")
+  if (writes(action) && (p1 === "owner_identity" || FORGET_ONLY.has(p1 ?? "")))
+    return C2.SQLITE_DENY;
+  if (action === C2.SQLITE_DELETE && KEPT_BY_INGEST.has(p1 ?? ""))
     return C2.SQLITE_DENY;
   if (action === C2.SQLITE_PRAGMA)
     return readsDataVersion(p1, p2) ? C2.SQLITE_OK : C2.SQLITE_DENY;
   return C2.SQLITE_OK;
 }
+var FORGET_WRITES = {
+  [C2.SQLITE_INSERT]: new Set(["forget_batch", "source_forgotten", "unit_state", "source_fts"]),
+  [C2.SQLITE_DELETE]: new Set([
+    "source",
+    "external_reference",
+    "unit_evidence",
+    "unit_adoption",
+    "source_processing",
+    "source_fts"
+  ])
+};
+var FORGET_UPDATES = {
+  unit: new Set(["lifecycle", "revision"]),
+  unit_state: new Set(["source_id"])
+};
+function forgetAuthorizer(action, p1, p2) {
+  if (SHADOW.test(p1 ?? "") && writes(action))
+    return C2.SQLITE_OK;
+  if (action === C2.SQLITE_UPDATE)
+    return FORGET_UPDATES[p1 ?? ""]?.has(p2 ?? "") ? C2.SQLITE_OK : C2.SQLITE_DENY;
+  if (action === C2.SQLITE_INSERT || action === C2.SQLITE_DELETE)
+    return FORGET_WRITES[action]?.has(p1 ?? "") ? C2.SQLITE_OK : C2.SQLITE_DENY;
+  if (action === C2.SQLITE_PRAGMA)
+    return readsDataVersion(p1, p2) || p1 === "secure_delete" && ["on", "1", "true"].includes((p2 ?? "").toLowerCase()) || p1 === "wal_checkpoint" ? C2.SQLITE_OK : C2.SQLITE_DENY;
+  if (action === C2.SQLITE_READ || action === C2.SQLITE_SELECT || action === C2.SQLITE_FUNCTION || action === C2.SQLITE_TRANSACTION || action === C2.SQLITE_SAVEPOINT)
+    return C2.SQLITE_OK;
+  return C2.SQLITE_DENY;
+}
 function connectWriter(role, file2 = dbFile(), create = false) {
   requireRuntime();
   if (!create)
     requireFile(file2);
   const raw = new DatabaseSync2(file2);
   try {
-    prepare(raw, create ? "none" : role === "ingest" ? "revision" : "generation");
+    prepare(raw, create ? "none" : role === "ingest" || role === "forget" ? "revision" : "generation");
     raw.function("sphica_terms", { deterministic: true }, (text) => terms(String(text ?? "")).join(" "));
   } catch (e) {
     raw.close();
@@ -45963,6 +46048,8 @@ function connectWriter(role, file2 = dbFile(), create = false) {
   }
   if (role === "ingest")
     raw.setAuthorizer(ingestAuthorizer);
+  else if (role === "forget")
+    raw.setAuthorizer(forgetAuthorizer);
   else if (role === "capture") {
     let own2 = false;
     raw.setAuthorizer((action, p1, p2, _db, triggerOrView) => captureAuthorizer(own2, action, p1, p2, triggerOrView));
@@ -46053,14 +46140,61 @@ function readText(root, rel) {
 function findSymbol(text, symbol2) {
   const re = new RegExp(`(?<![\\w$])${literal3(symbol2)}(?![\\w$])`);
   const lines = text.split(/\r?\n/);
-  const i = lines.findIndex((l) => re.test(l));
-  return i < 0 ? null : { line: i + 1, excerpt: (lines[i] ?? "").trim().slice(0, 200) };
+  return { lines, i: lines.findIndex((l) => re.test(l)) };
+}
+function swallowed(root, rel, symbol2) {
+  if (mask(symbol2) !== symbol2)
+    return true;
+  const text = root ? readText(root, rel) : null;
+  if (text === undefined)
+    return true;
+  if (text === null)
+    return false;
+  const re = new RegExp(`(?<![\\w$])${literal3(symbol2)}(?![\\w$])`, "g");
+  let raw = 0;
+  for (let m = re.exec(text);m; m = re.exec(text))
+    raw++;
+  if (raw === 0)
+    return false;
+  const masked = mask(text);
+  const holes = placeholderRanges(masked);
+  let h = 0;
+  let kept = 0;
+  re.lastIndex = 0;
+  for (let m = re.exec(masked);m; m = re.exec(masked)) {
+    while ((holes[h]?.[1] ?? Number.POSITIVE_INFINITY) < m.index)
+      h++;
+    if (!((holes[h]?.[0] ?? Number.POSITIVE_INFINITY) <= m.index + symbol2.length))
+      kept++;
+  }
+  return kept !== raw;
+}
+function masksSymbol(root, rel, symbol2) {
+  const trimmed = symbol2.trim();
+  return swallowed(root, rel, symbol2) || trimmed !== symbol2 && trimmed !== "" && swallowed(root, rel, trimmed);
 }
 function locate(root, rel, symbol2) {
   if (!root)
     return null;
   const text = readText(root, rel);
-  return typeof text === "string" ? findSymbol(text, symbol2) : null;
+  if (typeof text !== "string")
+    return null;
+  const { lines, i } = findSymbol(text, symbol2);
+  if (i < 0)
+    return null;
+  const line = lines[i] ?? "";
+  const from = bytes(lines.slice(0, i).join(`
+`)) + (i > 0 ? 1 : 0);
+  if (privateKeyRanges(lines.join(`
+`)).some(([a, b]) => a < from + bytes(line) && b > from))
+    return { line: i + 1, excerpt: "[redacted: private key]" };
+  const before = lines.slice(0, i).map((l) => `${l}
+`).join("");
+  const after = lines.slice(i + 1).map((l) => `
+${l}`).join("");
+  const own2 = mask(line);
+  const cut = mask(before + line) !== mask(before) + own2 || mask(line + after) !== own2 + mask(after);
+  return { line: i + 1, excerpt: cut ? "[redacted]" : own2.trim().slice(0, 200) };
 }
 
 // server/src/project.ts
@@ -46940,11 +47074,13 @@ async function storeItems(db, projectId2, items) {
     const kept = fit(it.text);
     const hash2 = sha256(kept.body);
     const latest = await db.selectFrom("source").select(["id", "revision", "content_hash"]).where("project_id", "=", projectId2).where("kind", "=", it.kind).where("external_id", "=", it.externalId).orderBy("revision", "desc").executeTakeFirst();
-    if (latest && Buffer.from(latest.content_hash).equals(hash2)) {
+    const forgotten = await db.selectFrom("source_forgotten").select(["revision", "content_hash"]).where("project_id", "=", projectId2).where("kind", "=", it.kind).where("external_id", "=", it.externalId).execute();
+    const lastForgotten = Math.max(0, ...forgotten.map((f) => f.revision));
+    if (latest && latest.revision > lastForgotten && Buffer.from(latest.content_hash).equals(hash2)) {
       ids.push(latest.id);
       continue;
     }
-    if (!latest && !kept.body.trim()) {
+    if (forgotten.some((f) => Buffer.from(f.content_hash).equals(hash2)) || !latest && !kept.body.trim()) {
       ids.push(null);
       continue;
     }
@@ -46957,7 +47093,7 @@ async function storeItems(db, projectId2, items) {
       kind: it.kind,
       artifact: it.artifact,
       external_id: it.externalId,
-      revision: (latest?.revision ?? 0) + 1,
+      revision: Math.max(latest?.revision ?? 0, lastForgotten) + 1,
       author_kind: kind,
       author_login: it.author?.login ?? null,
       author_external_id: authorId,
@@ -46966,7 +47102,7 @@ async function storeItems(db, projectId2, items) {
       event_kind: it.event,
       url: it.url,
       created_at: created,
-      available_at: latest ? null : created,
+      available_at: latest || lastForgotten ? null : created,
       captured_at: now,
       text: kept.body,
       truncated: kept.truncated ? 1 : 0,
@@ -47002,7 +47138,7 @@ async function pullSources(db, projectId2, number5) {
     `pr:${number5}`,
     ...(await db.selectFrom("artifact_link").select("to_artifact").where("project_id", "=", projectId2).where("from_artifact", "=", `pr:${number5}`).execute()).map((l) => l.to_artifact)
   ];
-  return db.selectFrom("source as s").where("s.project_id", "=", projectId2).where("s.artifact", "in", artifacts).where(({ not, exists, selectFrom }) => not(exists(selectFrom("source as n").select("n.id").whereRef("n.project_id", "=", "s.project_id").whereRef("n.kind", "=", "s.kind").whereRef("n.external_id", "=", "s.external_id").whereRef("n.revision", ">", "s.revision")))).select([
+  return db.selectFrom("source as s").where("s.project_id", "=", projectId2).where("s.artifact", "in", artifacts).where(({ not, exists, selectFrom }) => not(exists(selectFrom("source as n").select("n.id").whereRef("n.project_id", "=", "s.project_id").whereRef("n.kind", "=", "s.kind").whereRef("n.external_id", "=", "s.external_id").whereRef("n.revision", ">", "s.revision")))).where(({ not, exists, selectFrom }) => not(exists(selectFrom("source_forgotten as f").select("f.source_id").whereRef("f.project_id", "=", "s.project_id").whereRef("f.kind", "=", "s.kind").whereRef("f.external_id", "=", "s.external_id").whereRef("f.revision", ">", "s.revision")))).select([
     "s.id",
     "s.kind",
     "s.artifact",
@@ -47212,19 +47348,36 @@ async function checkRecord(db, target, raw) {
       adoption.push({ source: s.id, start: span[0], end: span[1], route });
     }
     const anchors = [];
+    const fallbacks = new Set;
     for (const a of u.anchors) {
       const p = repoPath(a.path);
       if (!p) {
         problems.push(`${key}: anchor path ${JSON.stringify(head(a.path, 80))} is not inside the repository; left out`);
         continue;
       }
+      const symbol2 = a.symbol && masksSymbol(target.root, p, a.symbol) ? undefined : a.symbol;
+      if (a.symbol && !symbol2)
+        problems.push(`${key}: anchor symbol in ${p} is text Sphica masks; the anchor keeps only its path`);
       let commit = a.commit;
       if (commit && !(target.root && commitHolds(target.root, commit, p))) {
         problems.push(`${key}: commit ${commit.slice(0, 12)} does not hold ${p} in the repository${target.root ? "" : " (no working tree to check)"}; the anchor keeps no commit`);
         commit = undefined;
       }
       const observation = a.role === "evidence" && !commit && target.sessionId ? (await db.selectFrom("edit_observation").select("id").where("session_id", "=", target.sessionId).where("path", "=", p).orderBy("id", "desc").executeTakeFirst())?.id ?? null : null;
-      anchors.push({ ...a, commit, path: p, observation });
+      const planned = { ...a, symbol: symbol2, commit, path: p, observation };
+      if (a.symbol && !symbol2)
+        fallbacks.add(planned);
+      anchors.push(planned);
+    }
+    const place = (x) => `${x.path}\x00${x.role}\x00${x.commit}\x00${x.lines ? [x.lines[0], Math.max(...x.lines)] : ""}`;
+    const covered = new Set(anchors.filter((x) => !x.symbol && !fallbacks.has(x)).map(place));
+    for (const x of [...fallbacks]) {
+      if (!covered.has(place(x))) {
+        covered.add(place(x));
+        continue;
+      }
+      problems.push(`${key}: another path-only anchor on ${x.path} already covers it; left out`);
+      anchors.splice(anchors.indexOf(x), 1);
     }
     const aliases = [...new Set(u.aliases.map((a) => a.trim()))];
     const bad = aliases.filter((a) => !a || a.length > 40);
@@ -47346,12 +47499,13 @@ ${checked.errors.map((e) => `  ${e}`).join(`
       }).onConflict((oc) => oc.doNothing()).execute();
     }
     for (const a of p.anchors) {
-      const at = a.lines ? null : a.symbol ? locate(target.root, a.path, a.symbol) : null;
+      const symbol2 = a.symbol && !masksSymbol(target.root, a.path, a.symbol) ? a.symbol : undefined;
+      const at = a.lines ? null : symbol2 ? locate(target.root, a.path, symbol2) : null;
       const lines = a.lines ?? (at ? [at.line, at.line] : null);
       await trx.insertInto("unit_anchor").values({
         unit_id: id,
         path: a.path,
-        symbol: a.symbol ?? null,
+        symbol: symbol2 ?? null,
         commit_sha: a.commit ?? null,
         line_start: lines?.[0] ?? null,
         line_end: lines ? Math.max(lines[0], lines[1]) : null,
@@ -47549,15 +47703,24 @@ function readExcerpt(root, file2) {
   } catch {
     throw new Error(`${p} is not UTF-8 text`);
   }
-  const starts = [0];
+  const starts2 = [0];
   for (let i = 0;i < buf.length; i++)
     if (buf[i] === 10 && i + 1 < buf.length)
-      starts.push(i + 1);
+      starts2.push(i + 1);
   const [a, b] = file2.lines;
-  if (a > b || b > starts.length)
-    throw new Error(`${p} has ${starts.length} lines; lines ${a}-${b} are not in it`);
-  const text2 = buf.subarray(starts[a - 1], b < starts.length ? starts[b] : buf.length).toString("utf8");
-  return { path: p, commit, blob, size, lines: [a, b], text: text2 };
+  if (a > b || b > starts2.length)
+    throw new Error(`${p} has ${starts2.length} lines; lines ${a}-${b} are not in it`);
+  const from = starts2[a - 1] ?? 0;
+  const to = b < starts2.length ? starts2[b] ?? buf.length : buf.length;
+  if (privateKeyRanges(buf.toString("utf8")).some(([s, e]) => s < to && e > from && (s < from || e > to)))
+    throw new Error(`${p} lines ${a}-${b} are inside a private key; cite lines outside it`);
+  const raw = buf.subarray(from, to).toString("utf8");
+  const text2 = mask(raw);
+  const before = buf.subarray(0, from).toString("utf8");
+  const after = buf.subarray(to).toString("utf8");
+  if (mask(before + raw) !== mask(before) + text2 || mask(raw + after) !== text2 + mask(after))
+    throw new Error(`${p} lines ${a}-${b} cut through text Sphica masks; cite more lines around it`);
+  return { path: p, commit, blob, size, lines: [a, b], raw, text: text2 };
 }
 var locate3 = (body, q) => {
   const at = Buffer.from(body, "utf8").indexOf(Buffer.from(q, "utf8"));
@@ -47621,8 +47784,12 @@ async function checkGlean(db, target, raw) {
           if (!target.root)
             throw new Error("the repository is not known");
           excerpt = readExcerpt(target.root, op.file);
-          if (!locate3(excerpt.text, op.quote))
+          if ((await forgottenExcerpt(db, target.projectId, excerpt)).same)
+            errors3.push(`${what}: the owner forgot ${excerpt.path} lines ${op.file.lines.join("-")}; cite something else`);
+          else if (!locate3(excerpt.raw, op.quote))
             errors3.push(`${what}: quote not found in ${excerpt.path} lines ${op.file.lines.join("-")}`);
+          else if (!quoteSpan(excerpt.raw, excerpt.text, op.quote))
+            errors3.push(`${what}: the quote also appears in text Sphica masks; quote a longer or different part`);
         } catch (e) {
           errors3.push(`${what}: ${e.message}`);
         }
@@ -47643,6 +47810,10 @@ async function checkGlean(db, target, raw) {
     }
     if (op.op === "anchor" && !repoPath(op.path))
       errors3.push(`${what}: the path is not inside the repository`);
+    const pinned = op.op === "anchor" ? op : op.op === "replace_anchor" ? op.to : null;
+    const pinnedPath = pinned && repoPath(pinned.path);
+    if (pinned?.symbol && pinnedPath && masksSymbol(target.root, pinnedPath, pinned.symbol))
+      errors3.push(`${what}: the symbol is text Sphica masks; anchor a name, not a key`);
     if (op.op === "anchor" && op.commit) {
       const rel2 = repoPath(op.path);
       if (rel2 && !(target.root && commitHolds(target.root, op.commit, rel2)))
@@ -47730,26 +47901,38 @@ async function checkGlean(db, target, raw) {
   }
   return { errors: errors3, problems, units, ops };
 }
+async function forgottenExcerpt(db, projectId2, x) {
+  const rows = await db.selectFrom("source_forgotten").select(["revision", "content_hash"]).where("project_id", "=", projectId2).where("kind", "=", "file_excerpt").where("external_id", "=", `file:${x.path}@${x.blob}#L${x.lines[0]}-${x.lines[1]}`).execute();
+  const hash2 = sha256(x.text);
+  return {
+    same: rows.some((r) => Buffer.from(r.content_hash).equals(hash2)),
+    last: Math.max(0, ...rows.map((r) => r.revision))
+  };
+}
 async function excerptSource(trx, projectId2, x) {
   const external2 = `file:${x.path}@${x.blob}#L${x.lines[0]}-${x.lines[1]}`;
-  const found = await trx.selectFrom("source").select("id").where("project_id", "=", projectId2).where("kind", "=", "file_excerpt").where("external_id", "=", external2).executeTakeFirst();
-  if (found)
+  const hash2 = sha256(x.text);
+  const found = await trx.selectFrom("source").select(["id", "revision", "content_hash"]).where("project_id", "=", projectId2).where("kind", "=", "file_excerpt").where("external_id", "=", external2).orderBy("revision", "desc").executeTakeFirst();
+  const forgotten = await forgottenExcerpt(trx, projectId2, x);
+  if (found && found.revision > forgotten.last && found.content_hash.equals(hash2))
     return found.id;
+  if (forgotten.same)
+    throw new Error(`the owner forgot ${x.path} lines ${x.lines.join("-")}; cite something else`);
   const now = iso(Date.now());
-  const partial2 = Buffer.byteLength(x.text, "utf8") !== x.size;
   const row = await trx.insertInto("source").values({
     project_id: projectId2,
     kind: "file_excerpt",
     artifact: `file:${x.path}`,
     external_id: external2,
-    revision: 1,
+    revision: Math.max(found?.revision ?? 0, forgotten.last) + 1,
     author_kind: "person",
     created_at: now,
     captured_at: now,
     text: x.text,
-    truncated: partial2 ? 1 : 0,
+    truncated: bytes(x.raw) !== x.size ? 1 : 0,
+    redacted: x.text !== x.raw ? 1 : 0,
     original_bytes: x.size,
-    content_hash: sha256(x.text),
+    content_hash: hash2,
     path: x.path,
     line_start: x.lines[0],
     line_end: x.lines[1],
@@ -47799,7 +47982,10 @@ ${c.errors.map((e) => `  ${e}`).join(`
     touched.set(p.unitId, op.unit);
     if (op.op === "add_evidence") {
       const s = p.excerpt ? { id: await excerptSource(trx, target.projectId, p.excerpt), text: p.excerpt.text } : await trx.selectFrom("source").select(["id", "text"]).where("id", "=", Number(op.source?.slice(1))).executeTakeFirstOrThrow();
-      const at = locate3(s.text, op.quote) ?? [0, 0];
+      const at = (p.excerpt ? quoteSpan(p.excerpt.raw, s.text, op.quote) : locate3(s.text, op.quote)) ?? [
+        0,
+        0
+      ];
       await trx.insertInto("unit_evidence").values({
         unit_id: p.unitId,
         source_id: s.id,
@@ -47826,11 +48012,13 @@ ${c.errors.map((e) => `  ${e}`).join(`
       changed2.push(`${op.unit}: adopted`);
     } else if (op.op === "anchor" || op.op === "replace_anchor") {
       const to = op.op === "anchor" ? op : op.to;
-      const at = to.symbol ? locate(target.root, repoPath(to.path) ?? to.path, to.symbol) : null;
+      const rel = repoPath(to.path) ?? to.path;
+      const symbol2 = to.symbol && !masksSymbol(target.root, rel, to.symbol) ? to.symbol : null;
+      const at = symbol2 ? locate(target.root, rel, symbol2) : null;
       const added = await trx.insertInto("unit_anchor").values({
         unit_id: p.unitId,
-        path: repoPath(to.path) ?? to.path,
-        symbol: to.symbol ?? null,
+        path: rel,
+        symbol: symbol2,
         commit_sha: op.op === "anchor" ? op.commit ?? null : null,
         line_start: at?.line ?? null,
         line_end: at?.line ?? null,
@@ -47988,7 +48176,7 @@ async function beginHarvest(db, projectId2, number5, get) {
   const project = await db.selectFrom("project").select("key").where("id", "=", projectId2).executeTakeFirst();
   const pull = await readPull(get, number5, repoOf(project?.key ?? ""));
   return inTransaction(db, async (trx) => {
-    await storeItems(trx, projectId2, pull.items);
+    const kept = (await storeItems(trx, projectId2, pull.items)).filter((id) => id !== null).length;
     await linkIssues(trx, projectId2, number5, pull.closes);
     const run = newRunId();
     await openRun(trx, {
@@ -47998,7 +48186,7 @@ async function beginHarvest(db, projectId2, number5, get) {
       sessionId: null,
       draftId: run
     });
-    return { run, sources: pull.items.length };
+    return { run, sources: kept };
   });
 }
 async function bound(db, id, projectId2) {
@@ -48149,6 +48337,153 @@ async function saveText(db, id, projectId2, root, record3) {
   });
 }
 
+// server/src/forget.ts
+class Preview extends Error {
+  outcome;
+  constructor(outcome) {
+    super("preview");
+    this.outcome = outcome;
+  }
+}
+var unknown3 = (id) => new Error(`s${id} is not a source of this project`);
+async function forgetIn(trx, projectId2, ids, at) {
+  const unique = [...new Set(ids)];
+  const rows = unique.length ? await trx.selectFrom("source").where("id", "in", unique).where("project_id", "=", projectId2).select([
+    "id",
+    "kind",
+    "artifact",
+    "external_id",
+    "revision",
+    "content_hash",
+    sql`length(cast(text as blob))`.as("bytes")
+  ]).orderBy("id").execute() : [];
+  const gone = unique.length ? await trx.selectFrom("source_forgotten").where("source_id", "in", unique).where("project_id", "=", projectId2).select("source_id").execute() : [];
+  const already = gone.map((g) => g.source_id).sort((a, b) => a - b);
+  for (const id of unique)
+    if (!rows.some((r) => r.id === id) && !already.includes(id))
+      throw unknown3(id);
+  const outcome = {
+    sources: rows.map((r) => ({ id: r.id, kind: r.kind, artifact: r.artifact, bytes: r.bytes })),
+    already,
+    units: [],
+    references: 0
+  };
+  if (!rows.length)
+    return outcome;
+  const targets = rows.map((r) => r.id);
+  const touched = await trx.selectFrom("unit as u").where("u.project_id", "=", projectId2).where((eb) => eb.or(["unit_evidence", "unit_adoption"].flatMap((t) => [
+    eb.exists(eb.selectFrom(t).select(`${t}.id`).whereRef(`${t}.unit_id`, "=", "u.id").where(`${t}.source_id`, "in", targets)),
+    eb.exists(eb.selectFrom(t).select(`${t}.id`).whereRef(`${t}.unit_id`, "=", "u.id").where(`${t}.retraction_source_id`, "in", targets))
+  ]))).select(["u.id", "u.key", "u.lifecycle"]).orderBy("u.id").execute();
+  const count = async (t, unitId) => Number((await trx.selectFrom(t).where("unit_id", "=", unitId).where((eb) => eb.or([eb("source_id", "in", targets), eb("retraction_source_id", "in", targets)])).select((eb) => eb.fn.countAll().as("n")).executeTakeFirstOrThrow()).n);
+  const removed = new Map;
+  for (const u of touched)
+    removed.set(u.id, await count("unit_evidence", u.id) + await count("unit_adoption", u.id));
+  const batch = (await trx.insertInto("forget_batch").values({ project_id: projectId2, at }).returning("id").executeTakeFirstOrThrow()).id;
+  await trx.insertInto("source_forgotten").values(rows.map((r) => ({
+    source_id: r.id,
+    project_id: projectId2,
+    artifact: r.artifact,
+    kind: r.kind,
+    external_id: r.external_id,
+    revision: r.revision,
+    content_hash: r.content_hash,
+    batch_id: batch
+  }))).execute();
+  for (const t of ["unit_evidence", "unit_adoption"])
+    await trx.deleteFrom(t).where("retraction_source_id", "in", targets).where("retracted_at", "is not", null).where("source_id", "not in", targets).execute();
+  outcome.references = Number((await trx.deleteFrom("external_reference").where("owner_source_id", "in", targets).executeTakeFirst()).numDeletedRows);
+  await trx.deleteFrom("source").where("id", "in", targets).execute();
+  const reason2 = `sources ${targets.map((id) => `s${id}`).join(", ")} forgotten by the owner`;
+  for (const u of touched) {
+    let after = u.lifecycle;
+    if (u.lifecycle === "active") {
+      await trx.insertInto("unit_state").values({ unit_id: u.id, from_state: "active", to_state: "candidate", at, reason: reason2, forget_id: batch }).execute();
+      after = "candidate";
+      try {
+        await trx.insertInto("unit_state").values({
+          unit_id: u.id,
+          from_state: "candidate",
+          to_state: "active",
+          at,
+          reason: "support checked again after forgetting sources",
+          forget_id: batch
+        }).execute();
+        after = "active";
+      } catch (e) {
+        if (!ACTIVATION.test(e.message))
+          throw e;
+      }
+    }
+    outcome.units.push({ key: u.key, before: u.lifecycle, after, removed: removed.get(u.id) ?? 0 });
+  }
+  return outcome;
+}
+async function previewForget(file2, projectId2, ids) {
+  const db = openWriter("forget", file2);
+  try {
+    return await db.connection().execute(async (c) => {
+      await sql`begin immediate`.execute(c);
+      try {
+        throw new Preview(await forgetIn(c, projectId2, ids, iso(Date.now())));
+      } catch (e) {
+        await sql`rollback`.execute(c).catch(() => {});
+        if (e instanceof Preview)
+          return e.outcome;
+        throw e;
+      }
+    });
+  } finally {
+    await db.destroy();
+  }
+}
+async function applyForget(file2, projectId2, ids, confirmed, signal) {
+  const db = openWriter("forget", file2);
+  try {
+    return await db.connection().execute(async (c) => {
+      await sql`pragma secure_delete = on`.execute(c);
+      await sql`begin immediate`.execute(c);
+      let outcome;
+      try {
+        outcome = await forgetIn(c, projectId2, ids, iso(Date.now()));
+        if (JSON.stringify(outcome) !== JSON.stringify(confirmed))
+          throw new Error("What these sources support changed after you confirmed. Nothing was forgotten; look at the preview again");
+        if (signal?.aborted)
+          throw new Error("The call was cancelled, so nothing was forgotten");
+        await sql`commit`.execute(c);
+      } catch (e) {
+        await sql`rollback`.execute(c).catch(() => {});
+        throw e;
+      }
+      try {
+        await sql`insert into source_fts (source_fts) values ('optimize')`.execute(c);
+        const checkpoint = await sql`pragma wal_checkpoint(TRUNCATE)`.execute(c);
+        return { outcome, cleanup: checkpoint.rows[0]?.busy === 0 ? "done" : "incomplete" };
+      } catch {
+        return { outcome, cleanup: "incomplete" };
+      }
+    });
+  } finally {
+    await db.destroy();
+  }
+}
+function forgetText(o) {
+  const lines = o.sources.map((s) => `- s${s.id} ${s.kind} in ${inline(s.artifact).slice(0, 120)} (${plural2(s.bytes, "byte")})`);
+  if (o.already.length)
+    lines.push(`- already forgotten: ${o.already.map((id) => `s${id}`).join(", ")} (only the cleanup runs again)`);
+  for (const u of o.units) {
+    const change = u.before === u.after ? `stays ${u.after}` : `${u.before} → ${u.after}`;
+    lines.push(`- record ${inline(u.key).slice(0, 120)}: ${change}, loses ${plural2(u.removed, "citation")}`);
+  }
+  if (o.references)
+    lines.push(`- ${plural2(o.references, "unfetched reference")} the forgotten messages gave`);
+  if (o.units.length)
+    lines.push("Records keep their own text: if one repeats the forgotten words, they stay in it.");
+  lines.push("Copies outside the database (capture's waiting and set-aside files, backups) are not touched.");
+  return lines.join(`
+`);
+}
+
 // server/src/frame.ts
 import crypto3 from "node:crypto";
 function framed(body) {
@@ -48206,7 +48541,7 @@ var tool = (fn) => fn().then((t) => reply(t), (e) => reply(`Sphica: ${head(reaso
 var server = new McpServer({ name: "sphica-record", version: versionAt(ROOT) ?? "unknown" }, {
   capabilities: { experimental: { "codex/sandbox-state-meta": {} } },
   instructions: [
-    "Writes Sphica records for the trace, harvest, and glean Skills. Use these tools only while running one of those Skills.",
+    "Writes Sphica records for the trace, harvest, glean, and forget Skills. Use these tools only while running one of those Skills.",
     "Flow: begin (trace_begin, harvest_begin, or glean_begin) returns a run id; context shows what the run may cite; check the record; save it.",
     "Always pass the repository root as cwd."
   ].join(`
@@ -48287,4 +48622,66 @@ server.registerTool("record_save", {
   const p = await projectOf(a.cwd, extra._meta);
   return saveText(conn(), a.run, p.projectId, p.root, a.record);
 }));
+var SOURCES = exports_external.array(exports_external.string().regex(/^s[1-9][0-9]{0,15}$/)).min(1).max(50).describe("Sources to forget, as search and read show them (s12)");
+var idsOf = (refs) => refs.map((r) => Number(r.slice(1)));
+var ANSWER_MS = 10 * 60 * 1000;
+server.registerTool("forget_preview", {
+  title: "What forgetting sources would do",
+  description: "Shows which sources would be removed and which records would lose citations or leave active. Changes nothing. Never shows the text.",
+  inputSchema: { sources: SOURCES, cwd: CWD },
+  annotations: READ
+}, async (a, extra) => tool(async () => {
+  const p = await projectOf(a.cwd, extra._meta);
+  return forgetText(await previewForget(dbFile(), p.projectId, idsOf(a.sources)));
+}));
+server.registerTool("forget_apply", {
+  title: "Forget sources",
+  description: "Asks the owner in the host to confirm by typing the number of sources, then removes them, their index entries, and the bytes left in the file, and judges the records that cited them again. Nothing is removed without that answer.",
+  inputSchema: { sources: SOURCES, cwd: CWD },
+  annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
+}, async (a, extra) => tool(async () => {
+  const p = await projectOf(a.cwd, extra._meta);
+  const ids = idsOf(a.sources);
+  const seen = await previewForget(dbFile(), p.projectId, ids);
+  const n = seen.sources.length;
+  if (n > 0) {
+    const refused = (why) => new Error(`${why}, so nothing was forgotten`);
+    if (!server.server.getClientCapabilities()?.elicitation)
+      throw refused("This host cannot ask you directly (run /sphica:forget in Claude Code)");
+    let answer;
+    try {
+      answer = await server.server.elicitInput({
+        mode: "form",
+        message: `Forget ${plural2(n, "source")} for good?
+${forgetText(seen)}`,
+        requestedSchema: {
+          type: "object",
+          properties: {
+            confirm: {
+              type: "string",
+              title: "Number of sources",
+              description: `Type ${n} to forget ${plural2(n, "source")}`
+            }
+          },
+          required: ["confirm"]
+        }
+      }, { timeout: ANSWER_MS, signal: extra.signal });
+    } catch (e) {
+      throw refused(`The confirmation did not come back (${head(reason(e), 200)})`);
+    }
+    if (extra.signal.aborted)
+      throw refused("The call was cancelled");
+    if (answer.action !== "accept")
+      throw refused(`You ${answer.action === "decline" ? "declined" : "cancelled"}`);
+    if (String(answer.content?.confirm ?? "").trim() !== String(n))
+      throw refused(`The number typed does not match ${n}`);
+  }
+  const done = await applyForget(dbFile(), p.projectId, ids, seen, extra.signal);
+  return [
+    n ? `Forgot ${plural2(n, "source")}.` : "Nothing new to forget.",
+    forgetText(done.outcome),
+    done.cleanup === "done" ? "The deleted text was cleared from the database file." : "Clearing the deleted text from the database file did not finish (another session may be reading it), so it may stay there until you run forget_apply with the same sources again."
+  ].join(`
+`);
+}));
 await server.connect(new StdioServerTransport);