lethe-delete 0.1.0__py3-none-any.whl

lethe/__init__.py

@@ -0,0 +1,5 @@
+from .core import Lethe
+from .models import Certificate, LayerResult, TagRecord
+from .version import __version__
+
+__all__ = ["__version__", "Lethe", "Certificate", "LayerResult", "TagRecord"]

lethe/audit.py

@@ -0,0 +1,83 @@
+import hashlib
+import hmac
+import json
+
+import psycopg
+
+GENESIS = "0" * 64
+
+AUDIT_SCHEMA = """
+CREATE TABLE IF NOT EXISTS lethe_audit (
+    seq BIGSERIAL PRIMARY KEY,
+    entry JSONB NOT NULL,
+    prev_hash TEXT NOT NULL,
+    entry_hash TEXT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+"""
+
+
+def _entry_hash(prev_hash: str, entry: dict) -> str:
+    data = prev_hash + json.dumps(entry, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(data.encode()).hexdigest()
+
+
+class AuditLog:
+    def __init__(self, conn: psycopg.Connection):
+        self.conn = conn
+
+    def init_schema(self) -> None:
+        with self.conn.cursor() as cur:
+            cur.execute(AUDIT_SCHEMA)
+        self.conn.commit()
+
+    def _last_hash(self) -> str:
+        with self.conn.cursor() as cur:
+            cur.execute("SELECT entry_hash FROM lethe_audit ORDER BY seq DESC LIMIT 1")
+            row = cur.fetchone()
+        return row[0] if row else GENESIS
+
+    def append(self, entry: dict) -> str:
+        prev = self._last_hash()
+        h = _entry_hash(prev, entry)
+        with self.conn.cursor() as cur:
+            cur.execute(
+                "INSERT INTO lethe_audit (entry, prev_hash, entry_hash) VALUES (%s, %s, %s)",
+                (json.dumps(entry), prev, h),
+            )
+        self.conn.commit()
+        return h
+
+    def head(self) -> str:
+        """Current tip hash (GENESIS if the log is empty). Record this
+        out-of-band; later pass it to verify_chain(expected_head=...) to detect
+        tip-truncation (deletion of the most recent entries)."""
+        return self._last_hash()
+
+    def verify_chain(self, expected_head: str | None = None) -> bool:
+        """Verify the hash chain links from genesis to the current tip.
+
+        SECURITY: an internal walk proves no entry was *altered* or removed from
+        the *middle* (that breaks a prev_hash link), but it CANNOT detect tip
+        truncation — deleting the most recent entries leaves a self-consistent,
+        merely shorter chain. To catch an attacker lopping off the tail (erasing
+        evidence of recent erasures), the operator must record the head hash
+        out-of-band and pass it as ``expected_head`` — the same pin-it-externally
+        discipline the certificate uses for the trusted public key. With no
+        entries the head is ``GENESIS``.
+        """
+        with self.conn.cursor() as cur:
+            cur.execute(
+                "SELECT entry, prev_hash, entry_hash FROM lethe_audit ORDER BY seq ASC"
+            )
+            rows = cur.fetchall()
+        prev = GENESIS
+        for entry, prev_hash, entry_hash in rows:
+            if prev_hash != prev:
+                return False
+            if _entry_hash(prev_hash, entry) != entry_hash:
+                return False
+            prev = entry_hash
+        if expected_head is not None and not hmac.compare_digest(prev, expected_head):
+            return False
+        return True

lethe/cert_schema.py

@@ -0,0 +1,82 @@
+"""Machine verification of Lethe certificates: JSON-Schema shape check plus
+key-pinned cryptographic verification with structured, agent-branchable reasons."""
+
+import base64
+import copy
+import hashlib
+import hmac as hmac_mod
+import json
+from functools import lru_cache
+from importlib import resources
+
+import jsonschema
+
+from .certificate import canonical_payload_bytes
+from .signing import verify_signature
+
+
+@lru_cache(maxsize=1)
+def _load_schema() -> dict:
+    text = (
+        resources.files("lethe").joinpath("schemas/certificate-v1.json").read_text()
+    )
+    return json.loads(text)
+
+
+@lru_cache(maxsize=1)
+def _validator() -> jsonschema.Draft202012Validator:
+    return jsonschema.Draft202012Validator(_load_schema())
+
+
+def load_schema() -> dict:
+    """Return the certificate v1 JSON Schema.
+
+    Returns a defensive deep copy: the cached schema dict is shared process-wide,
+    so callers must never be handed the mutable original.
+    """
+    return copy.deepcopy(_load_schema())
+
+
+def schema_errors(data) -> list[str]:
+    return [
+        f"{'/'.join(str(p) for p in e.absolute_path) or '<root>'}: {e.message}"
+        for e in sorted(
+            _validator().iter_errors(data),
+            key=lambda e: [str(x) for x in e.absolute_path],
+        )
+    ]
+
+
+def verify_certificate_json(data, trusted_public_key: str) -> dict:
+    """Verify a certificate received as JSON. The trusted key is REQUIRED:
+    an unpinned check only proves self-consistency, never authenticity.
+    Reasons an agent can branch on, checked in order:
+    SCHEMA_MISMATCH -> KEY_MISMATCH -> PAYLOAD_TAMPERED -> BAD_SIGNATURE."""
+    errors = schema_errors(data)
+    if errors:
+        return {"valid": False, "reasons": ["SCHEMA_MISMATCH"], "detail": errors}
+    try:
+        embedded = base64.b64decode(data["public_key"], validate=True)
+        trusted = base64.b64decode(trusted_public_key, validate=True)
+    except Exception:
+        return {
+            "valid": False, "reasons": ["KEY_MISMATCH"],
+            "detail": ["public key is not valid base64"],
+        }
+    if not hmac_mod.compare_digest(embedded, trusted):
+        return {
+            "valid": False, "reasons": ["KEY_MISMATCH"],
+            "detail": ["certificate's embedded key does not match the trusted key"],
+        }
+    payload_bytes = canonical_payload_bytes(data["payload"])
+    if hashlib.sha256(payload_bytes).hexdigest() != data["payload_hash"]:
+        return {
+            "valid": False, "reasons": ["PAYLOAD_TAMPERED"],
+            "detail": ["payload_hash does not match the payload"],
+        }
+    if not verify_signature(data["public_key"], payload_bytes, data["signature"]):
+        return {
+            "valid": False, "reasons": ["BAD_SIGNATURE"],
+            "detail": ["Ed25519 signature verification failed"],
+        }
+    return {"valid": True, "reasons": [], "detail": []}

lethe/certificate.py

@@ -0,0 +1,126 @@
+import base64
+import hashlib
+import hmac
+import json
+
+from .models import Certificate, LayerResult
+from .signing import Signer, verify_signature
+
+CLAIM = (
+    "Deleted across the listed retrieval layers and verified absent by re-query "
+    "at issue time against the configured endpoint. Not a guarantee of erasure "
+    "from backups, model weights, or systems outside Lethe's configured "
+    "connectors, nor a guarantee against read replicas, caches, or asynchronous "
+    "propagation (e.g. eventually-consistent stores such as Pinecone)."
+)
+
+CERT_SCHEMA_VERSION = "lethe.cert/1"
+
+
+def _canonical_bytes(payload: dict) -> bytes:
+    return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
+
+
+def build_certificate(
+    *,
+    request_id: str,
+    subject_hash: str,
+    layers: list[LayerResult],
+    issued_at: str,
+    version: str,
+    signer: Signer,
+) -> Certificate:
+    ordered = sorted(layers, key=lambda l: (l.store, l.namespace))
+
+    def _is_erasure(l: LayerResult) -> bool:
+        # A layer is a genuine, certifiable erasure only when Lethe was handled
+        # by a real connector, confirmed the records absent, AND actually removed
+        # records this run. deleted_count 0 with verified_absent True means the
+        # data was already gone (another subject's forget, or a prior partial
+        # run) — Lethe did NOT perform this erasure and must not certify it.
+        return l.handled and l.verified_absent and l.deleted_count > 0
+
+    erasures = [_is_erasure(l) for l in ordered]
+
+    # all_verified is the load-bearing claim a regulator relies on. It is True
+    # ONLY if at least one layer was found AND every layer is a genuine erasure.
+    # Empty layers (unknown subject / nothing found) => all([]) would be True;
+    # we require a non-empty layer set so "found nothing" can never read as a
+    # successful, verified erasure.
+    all_verified = bool(ordered) and all(erasures)
+
+    payload = {
+        "schema": CERT_SCHEMA_VERSION,
+        "request_id": request_id,
+        "subject_hash": subject_hash,
+        "issued_at": issued_at,
+        "lethe_version": version,
+        "claim": CLAIM,
+        "all_verified": all_verified,
+        # Honest summary of what actually happened, so a zero-deletion or
+        # unhandled-store outcome cannot be misread off the layer list.
+        "layers_found": len(ordered),
+        "records_deleted": sum(l.deleted_count for l in ordered),
+        "all_layers_handled": all(l.handled for l in ordered) if ordered else True,
+        "layers": [
+            {
+                "store": l.store,
+                "namespace": l.namespace,
+                "deleted_count": l.deleted_count,
+                "verified_absent": l.verified_absent,
+                "requested_count": l.requested_count,
+                "handled": l.handled,
+                "erased": erased,
+            }
+            for l, erased in zip(ordered, erasures)
+        ],
+    }
+    data = _canonical_bytes(payload)
+    return Certificate(
+        payload=payload,
+        payload_hash=hashlib.sha256(data).hexdigest(),
+        signature=signer.sign(data),
+        public_key=signer.public_key_b64(),
+    )
+
+
+def verify_certificate(cert: Certificate, trusted_public_key: str) -> bool:
+    """Verify a deletion certificate. Key pinning is MANDATORY.
+
+    SECURITY: a certificate carries the public key that signed it, so a
+    self-verifying check ("does the embedded key validate the embedded
+    signature?") only proves internal consistency, NOT authenticity — an
+    attacker can mint a fully self-consistent certificate with their own
+    keypair. A certificate can therefore only be verified against the
+    operator's out-of-band-published key (e.g. from /.well-known), passed as
+    ``trusted_public_key``; self-consistency-only checks are not proof and
+    are no longer offered. The certificate's embedded key must match the
+    trusted key before the signature is even checked.
+    """
+    try:
+        embedded = base64.b64decode(cert.public_key, validate=True)
+        trusted = base64.b64decode(trusted_public_key, validate=True)
+    except Exception:
+        return False
+    if not hmac.compare_digest(embedded, trusted):
+        return False
+
+    data = _canonical_bytes(cert.payload)
+    if hashlib.sha256(data).hexdigest() != cert.payload_hash:
+        return False
+    return verify_signature(cert.public_key, data, cert.signature)
+
+
+def canonical_payload_bytes(payload: dict) -> bytes:
+    """Public alias: the exact byte encoding that is hashed and signed."""
+    return _canonical_bytes(payload)
+
+
+def certificate_to_dict(cert: Certificate) -> dict:
+    """The wire/JSON envelope — matches schemas/certificate-v1.json."""
+    return {
+        "payload": cert.payload,
+        "payload_hash": cert.payload_hash,
+        "signature": cert.signature,
+        "public_key": cert.public_key,
+    }

lethe/cli.py

@@ -0,0 +1,119 @@
+import json
+
+import click
+import psycopg
+
+from .audit import AuditLog
+from .certificate import verify_certificate
+from .connectors.pgvector import PgVectorConnector
+from .core import Lethe
+from .ledger import Ledger
+from .models import Certificate
+from .signing import Signer
+
+
+@click.group()
+def cli() -> None:
+    """Lethe — the forget button for AI."""
+
+
+@cli.command("keygen")
+@click.option("--out", "out", required=True, help="Path to write the Ed25519 private key.")
+def keygen(out: str) -> None:
+    signer = Signer.generate()
+    with open(out, "wb") as f:
+        f.write(signer.private_bytes())
+    click.echo(f"Wrote signing key to {out}")
+    click.echo(f"Public key (publish this for verifiers): {signer.public_key_b64()}")
+
+
+@cli.command("init-db")
+@click.option("--database-url", envvar="DATABASE_URL", required=True)
+def init_db(database_url: str) -> None:
+    with psycopg.connect(database_url) as conn:
+        Ledger(conn).init_schema()
+        AuditLog(conn).init_schema()
+    click.echo("Lethe schema initialized.")
+
+
+@cli.command("forget")
+@click.argument("subject_id")
+@click.option("--database-url", envvar="DATABASE_URL", required=True)
+@click.option("--salt", envvar="LETHE_SALT", required=True)
+@click.option("--key-file", envvar="LETHE_KEY_FILE", required=True)
+def forget(subject_id: str, database_url: str, salt: str, key_file: str) -> None:
+    with open(key_file, "rb") as f:
+        signer = Signer.from_private_bytes(f.read())
+    with psycopg.connect(database_url) as conn:
+        lethe = Lethe(
+            ledger=Ledger(conn),
+            audit=AuditLog(conn),
+            signer=signer,
+            connectors={"pgvector": PgVectorConnector(conn)},
+            salt=salt,
+        )
+        cert = lethe.forget(subject_id)
+    click.echo(
+        json.dumps(
+            {
+                "payload": cert.payload,
+                "payload_hash": cert.payload_hash,
+                "signature": cert.signature,
+                "public_key": cert.public_key,
+            },
+            indent=2,
+        )
+    )
+
+
+@cli.command("verify")
+@click.argument("cert_file")
+@click.option(
+    "--public-key",
+    "public_key",
+    required=True,
+    help="The operator's published (trusted) Ed25519 public key, base64. "
+    "Required: an unpinned check only proves self-consistency, not authenticity.",
+)
+def verify(cert_file: str, public_key: str) -> None:
+    with open(cert_file) as f:
+        data = json.load(f)
+    cert = Certificate(
+        payload=data["payload"],
+        payload_hash=data["payload_hash"],
+        signature=data["signature"],
+        public_key=data["public_key"],
+    )
+    ok = verify_certificate(cert, trusted_public_key=public_key)
+    click.echo("VALID" if ok else "INVALID")
+    raise SystemExit(0 if ok else 1)
+
+
+@cli.command("audit-head")
+@click.option("--database-url", envvar="DATABASE_URL", required=True)
+def audit_head(database_url: str) -> None:
+    """Print the current audit-log head hash. Record it out-of-band so you can
+    later detect tip-truncation with `verify-audit --expected-head`."""
+    with psycopg.connect(database_url) as conn:
+        click.echo(AuditLog(conn).head())
+
+
+@cli.command("verify-audit")
+@click.option("--database-url", envvar="DATABASE_URL", required=True)
+@click.option(
+    "--expected-head",
+    default=None,
+    help="The audit head you recorded out-of-band. Without it, tip-truncation "
+    "(deletion of the most recent entries) cannot be detected.",
+)
+def verify_audit(database_url: str, expected_head: str | None) -> None:
+    with psycopg.connect(database_url) as conn:
+        ok = AuditLog(conn).verify_chain(expected_head=expected_head)
+    if expected_head is None:
+        click.echo(
+            "WARNING: no --expected-head given; internal links checked but "
+            "tip-truncation is undetectable.",
+            err=True,
+        )
+    click.echo("VALID" if ok else "INVALID")
+    raise SystemExit(0 if ok else 1)

lethe/connectors/base.py

@@ -0,0 +1,14 @@
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Connector(Protocol):
+    name: str
+
+    def delete(self, namespace: str, record_ids: list[str]) -> int:
+        """Delete the given records from `namespace`; return rows deleted."""
+        ...
+
+    def verify(self, namespace: str, record_ids: list[str]) -> bool:
+        """Return True only if none of `record_ids` remain in `namespace`."""
+        ...

lethe/connectors/pgvector.py

@@ -0,0 +1,33 @@
+import psycopg
+from psycopg import sql
+
+
+class PgVectorConnector:
+    name = "pgvector"
+
+    def __init__(self, conn: psycopg.Connection, id_column: str = "id"):
+        self.conn = conn
+        self.id_column = id_column
+
+    def delete(self, namespace: str, record_ids: list[str]) -> int:
+        if not record_ids:
+            return 0
+        query = sql.SQL("DELETE FROM {tbl} WHERE {col} = ANY(%s)").format(
+            tbl=sql.Identifier(namespace), col=sql.Identifier(self.id_column)
+        )
+        with self.conn.cursor() as cur:
+            cur.execute(query, (record_ids,))
+            n = cur.rowcount
+        self.conn.commit()
+        return n
+
+    def verify(self, namespace: str, record_ids: list[str]) -> bool:
+        if not record_ids:
+            return True
+        query = sql.SQL("SELECT count(*) FROM {tbl} WHERE {col} = ANY(%s)").format(
+            tbl=sql.Identifier(namespace), col=sql.Identifier(self.id_column)
+        )
+        with self.conn.cursor() as cur:
+            cur.execute(query, (record_ids,))
+            (count,) = cur.fetchone()
+        return count == 0

lethe/connectors/pinecone.py

@@ -0,0 +1,59 @@
+"""Pinecone connector: delete records by id and verify their absence.
+
+Wraps a Pinecone ``Index`` object (duck-typed — Lethe does not depend on the
+pinecone client). Lethe's ``namespace`` maps to a Pinecone namespace.
+
+Three Pinecone realities shape this connector:
+  * ``index.delete(ids=...)`` returns no count, so to report an accurate
+    ``deleted_count`` for the certificate, ``delete()`` first ``fetch``es the
+    ids to see how many actually existed, then deletes.
+  * Pinecone upserts on id collision, so ids must be unique per record — the
+    LetheVectorStore wrapper enforces that up front (see its id_key handling).
+  * Pinecone deletes are EVENTUALLY CONSISTENT. ``verify()`` fetches
+    immediately after ``delete()``, so ``verified_absent`` reflects what the
+    queried endpoint returns at issue time — propagation to every replica is
+    not instantaneous. The certificate's claim is scoped to say exactly this;
+    do not read a Pinecone ``verified_absent: true`` as a cross-replica proof.
+"""
+
+# Pinecone caps ids per delete/fetch call; chunk to stay under it.
+_BATCH = 1000
+
+
+def _chunks(seq, n):
+    for i in range(0, len(seq), n):
+        yield seq[i : i + n]
+
+
+def _fetched_ids(resp) -> set:
+    # pinecone-client returns a FetchResponse with `.vectors`, or a dict with
+    # a "vectors" key, mapping id -> vector. Support both shapes.
+    vectors = getattr(resp, "vectors", None)
+    if vectors is None and isinstance(resp, dict):
+        vectors = resp.get("vectors", {})
+    return set((vectors or {}).keys())
+
+
+class PineconeConnector:
+    name = "pinecone"
+
+    def __init__(self, index):
+        self.index = index
+
+    def delete(self, namespace: str, record_ids: list[str]) -> int:
+        if not record_ids:
+            return 0
+        existing: set = set()
+        for chunk in _chunks(record_ids, _BATCH):
+            existing |= _fetched_ids(self.index.fetch(ids=list(chunk), namespace=namespace))
+        for chunk in _chunks(record_ids, _BATCH):
+            self.index.delete(ids=list(chunk), namespace=namespace)
+        return len(existing)
+
+    def verify(self, namespace: str, record_ids: list[str]) -> bool:
+        if not record_ids:
+            return True
+        for chunk in _chunks(record_ids, _BATCH):
+            if _fetched_ids(self.index.fetch(ids=list(chunk), namespace=namespace)):
+                return False
+        return True

lethe/core.py

@@ -0,0 +1,145 @@
+import uuid
+from collections import defaultdict
+from datetime import datetime, timezone
+
+from .audit import AuditLog
+from .certificate import build_certificate
+from .connectors.base import Connector
+from .hashing import hash_subject
+from .ledger import Ledger
+from .models import Certificate, LayerResult, TagRecord
+from .signing import Signer
+from .version import __version__
+
+
+class Lethe:
+    def __init__(
+        self,
+        *,
+        ledger: Ledger,
+        audit: AuditLog,
+        signer: Signer,
+        connectors: dict[str, Connector],
+        salt: str,
+    ):
+        self.ledger = ledger
+        self.audit = audit
+        self.signer = signer
+        self.connectors = connectors
+        self.salt = salt
+
+    def _subject_hash(self, subject_id: str) -> str:
+        return hash_subject(subject_id, self.salt)
+
+    def tag(self, subject_id: str, store: str, namespace: str, record_id: str) -> None:
+        self.ledger.record(
+            TagRecord(
+                subject_hash=self._subject_hash(subject_id),
+                store=store,
+                namespace=namespace,
+                record_id=record_id,
+            )
+        )
+
+    def preview(self, subject_id: str) -> dict:
+        """Read-only blast radius: what forget() WOULD touch, per layer.
+        Feeds the MCP two-step guard; deletes nothing, purges nothing."""
+        subject_hash = self._subject_hash(subject_id)
+        counts: dict[tuple[str, str], int] = defaultdict(int)
+        for row in self.ledger.lookup(subject_hash):
+            counts[(row.store, row.namespace)] += 1
+        return {
+            "subject_hash": subject_hash,
+            "layers": [
+                {"store": store, "namespace": namespace, "count": n}
+                for (store, namespace), n in sorted(counts.items())
+            ],
+        }
+
+    def forget(
+        self,
+        subject_id: str,
+        *,
+        request_id: str | None = None,
+        now: datetime | None = None,
+    ) -> Certificate:
+        subject_hash = self._subject_hash(subject_id)
+        request_id = request_id or str(uuid.uuid4())
+        issued_at = (now or datetime.now(timezone.utc)).isoformat()
+
+        groups: dict[tuple[str, str], list[str]] = defaultdict(list)
+        for row in self.ledger.lookup(subject_hash):
+            groups[(row.store, row.namespace)].append(row.record_id)
+
+        # Pre-flight audit event: a mid-loop connector failure must never
+        # leave real deletions with no audit trace; the started/completed
+        # pair brackets every destructive attempt.
+        self.audit.append(
+            {
+                "event": "forget_started",
+                "request_id": request_id,
+                "subject_hash": subject_hash,
+                "issued_at": issued_at,
+                "requested_layers": len(groups),
+            }
+        )
+
+        layers: list[LayerResult] = []
+        for (store, namespace), ids in groups.items():
+            connector = self.connectors.get(store)
+            if connector is None:
+                # A tagged store with no configured connector must NOT crash the
+                # loop mid-flight (that would leave already-deleted layers with no
+                # certificate). Record it as an unhandled, unverified layer so the
+                # certificate is honest and the ledger is preserved for retry once
+                # the connector is configured.
+                layers.append(
+                    LayerResult(
+                        store,
+                        namespace,
+                        deleted_count=0,
+                        verified_absent=False,
+                        requested_count=len(ids),
+                        handled=False,
+                    )
+                )
+                continue
+            deleted = connector.delete(namespace, ids)
+            verified = connector.verify(namespace, ids)
+            layers.append(
+                LayerResult(
+                    store,
+                    namespace,
+                    deleted_count=deleted,
+                    verified_absent=verified,
+                    requested_count=len(ids),
+                    handled=True,
+                )
+            )
+
+        cert = build_certificate(
+            request_id=request_id,
+            subject_hash=subject_hash,
+            layers=layers,
+            issued_at=issued_at,
+            version=__version__,
+            signer=self.signer,
+        )
+
+        self.audit.append(
+            {
+                "event": "forget",
+                "request_id": request_id,
+                "subject_hash": subject_hash,
+                "issued_at": issued_at,
+                "payload_hash": cert.payload_hash,
+                "all_verified": cert.payload["all_verified"],
+            }
+        )
+
+        # Only purge the provenance map once deletion is fully verified — otherwise
+        # we keep the map so the operation can be retried.
+        if cert.payload["all_verified"]:
+            self.ledger.purge(subject_hash)
+
+        return cert