ledgerchain 0.1.0__py3-none-any.whl

ledgerchain/__init__.py

@@ -0,0 +1,62 @@
+"""ledgerchain: a hash-chained, tamper-evident decision ledger on SQLAlchemy.
+
+Append events inside your own transaction, verify the whole chain by
+recomputation, checkpoint it with Merkle roots, and anchor those roots to
+object storage as an independent witness.
+"""
+
+from .anchor import (
+    AnchorConfig,
+    AnchorResult,
+    AnchorVerifyResult,
+    anchor_checkpoint,
+    anchor_payload,
+    retry_failed_anchors,
+    verify_anchor,
+)
+from .canonical import CanonicalizationError, canonicalize, sha256_hex
+from .checkpoint import CheckpointResult, build_checkpoint, merkle_root
+from .immutability import create_all, drop_immutability, install_immutability
+from .models import Base, DecisionLedgerEntry, LedgerCheckpoint, LedgerHead, metadata
+from .verify import VerifyResult, verify_chain
+from .writer import (
+    CHECKPOINT_EVENT_TYPE,
+    GENESIS_EVENT_TYPE,
+    GENESIS_PREV_HASH,
+    LedgerWriter,
+    compute_entry_hash,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AnchorConfig",
+    "AnchorResult",
+    "AnchorVerifyResult",
+    "Base",
+    "CHECKPOINT_EVENT_TYPE",
+    "CanonicalizationError",
+    "CheckpointResult",
+    "DecisionLedgerEntry",
+    "GENESIS_EVENT_TYPE",
+    "GENESIS_PREV_HASH",
+    "LedgerCheckpoint",
+    "LedgerHead",
+    "LedgerWriter",
+    "VerifyResult",
+    "anchor_checkpoint",
+    "anchor_payload",
+    "build_checkpoint",
+    "canonicalize",
+    "compute_entry_hash",
+    "create_all",
+    "drop_immutability",
+    "install_immutability",
+    "merkle_root",
+    "metadata",
+    "retry_failed_anchors",
+    "sha256_hex",
+    "verify_anchor",
+    "verify_chain",
+    "__version__",
+]

ledgerchain/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

ledgerchain/anchor.py

@@ -0,0 +1,243 @@
+"""External anchoring of checkpoints to S3-compatible object storage.
+
+The hash chain is already tamper-evident. Anchoring defeats a different
+threat: a fully privileged database administrator rewriting the whole
+chain and its checkpoints consistently. A checkpoint's Merkle root written
+to a bucket under Object Lock is an independent witness that the live
+database cannot alter.
+
+Anchoring is eventually consistent by design. A storage outage marks the
+checkpoint ``failed`` for retry and never blocks or fails the job that
+built it.
+
+Configuration comes from :class:`AnchorConfig`, either built explicitly or
+read from the environment:
+
+``LEDGER_ANCHOR_BUCKET``
+    Target bucket. Anchoring is skipped, not failed, when unset.
+``LEDGER_ANCHOR_PREFIX``
+    Key prefix. Default ``ledger-checkpoints/``.
+``LEDGER_ANCHOR_ENDPOINT_URL``
+    S3-compatible endpoint (MinIO and friends). Unset uses AWS defaults.
+``LEDGER_ANCHOR_OBJECT_LOCK_MODE``
+    ``COMPLIANCE`` (default) or ``GOVERNANCE``. An empty string disables
+    Object Lock, for buckets that do not have it enabled.
+
+The S3 client is injectable. Anything with ``put_object(**kwargs)`` and
+``get_object(Bucket=..., Key=...)`` works, which keeps the test suite free
+of network access and credentials. ``boto3`` is only imported when no
+client is supplied (install the ``s3`` extra).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any, Protocol
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from .models import LedgerCheckpoint
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_PREFIX = "ledger-checkpoints/"
+DEFAULT_RETAIN_UNTIL = datetime(2126, 1, 1, tzinfo=timezone.utc)
+"""A fixed far-future retain-until date approximates "keep forever" without
+depending on bucket-default retention being configured."""
+
+
+class ObjectStore(Protocol):
+    """The slice of the boto3 S3 client this module uses."""
+
+    def put_object(self, **kwargs: Any) -> Any: ...
+
+    def get_object(self, **kwargs: Any) -> Any: ...
+
+
+@dataclass(frozen=True)
+class AnchorConfig:
+    bucket: str | None
+    prefix: str = DEFAULT_PREFIX
+    endpoint_url: str | None = None
+    object_lock_mode: str | None = "COMPLIANCE"
+    retain_until: datetime = DEFAULT_RETAIN_UNTIL
+
+    @classmethod
+    def from_env(cls, environ: Mapping[str, str] | None = None) -> AnchorConfig:
+        env = os.environ if environ is None else environ
+        bucket = env.get("LEDGER_ANCHOR_BUCKET", "").strip() or None
+        prefix = env.get("LEDGER_ANCHOR_PREFIX", DEFAULT_PREFIX)
+        endpoint = env.get("LEDGER_ANCHOR_ENDPOINT_URL", "").strip() or None
+        lock_mode = env.get("LEDGER_ANCHOR_OBJECT_LOCK_MODE", "COMPLIANCE").strip().upper() or None
+        return cls(bucket=bucket, prefix=prefix, endpoint_url=endpoint, object_lock_mode=lock_mode)
+
+
+@dataclass(frozen=True)
+class AnchorResult:
+    ok: bool
+    anchor_uri: str | None = None
+    error: str | None = None
+    skipped: bool = False
+
+
+@dataclass(frozen=True)
+class AnchorVerifyResult:
+    """Outcome of :func:`verify_anchor`.
+
+    ``ok`` means the anchored copy was read and matches the checkpoint row.
+    ``matched=False`` with ``ok=False`` means it was read and disagrees,
+    which is the signal that the database copy was rewritten after the
+    anchor was taken. ``error`` is set when the copy could not be read.
+    """
+
+    ok: bool
+    matched: bool | None = None
+    anchored: dict[str, Any] | None = None
+    error: str | None = None
+
+
+def anchor_payload(checkpoint: LedgerCheckpoint) -> dict[str, Any]:
+    """``{from_seq, to_seq, merkle_root, ts, prev_checkpoint_root}``."""
+    return {
+        "from_seq": checkpoint.from_seq,
+        "to_seq": checkpoint.to_seq,
+        "merkle_root": checkpoint.merkle_root,
+        "ts": checkpoint.ts.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
+        "prev_checkpoint_root": checkpoint.prev_checkpoint_root,
+    }
+
+
+def anchor_key(config: AnchorConfig, checkpoint: LedgerCheckpoint) -> str:
+    return f"{config.prefix}{checkpoint.from_seq:020d}-{checkpoint.to_seq:020d}.json"
+
+
+def parse_anchor_uri(uri: str) -> tuple[str, str]:
+    """Split ``s3://bucket/key`` into ``(bucket, key)``."""
+    if not uri.startswith("s3://"):
+        raise ValueError(f"not an s3:// URI: {uri!r}")
+    rest = uri[len("s3://") :]
+    bucket, sep, key = rest.partition("/")
+    if not bucket or not sep or not key:
+        raise ValueError(f"malformed s3:// URI: {uri!r}")
+    return bucket, key
+
+
+def default_client(config: AnchorConfig) -> ObjectStore:
+    """A boto3 S3 client for ``config``. Requires the ``s3`` extra."""
+    import boto3  # imported lazily so the base install has no AWS dependency
+
+    client: ObjectStore = boto3.client("s3", endpoint_url=config.endpoint_url)
+    return client
+
+
+def anchor_checkpoint(
+    session: Session,
+    checkpoint: LedgerCheckpoint,
+    *,
+    config: AnchorConfig | None = None,
+    client: ObjectStore | None = None,
+) -> AnchorResult:
+    """Write ``checkpoint``'s anchor payload to the configured bucket.
+
+    Updates ``anchor_status`` and ``anchor_uri`` in place and commits.
+    Never raises: any failure (bucket unset, credentials missing, network
+    error, Object Lock rejected) is logged and recorded as
+    ``anchor_status='failed'`` for a later retry.
+    """
+    cfg = config if config is not None else AnchorConfig.from_env()
+    if cfg.bucket is None:
+        return AnchorResult(ok=False, skipped=True, error="LEDGER_ANCHOR_BUCKET not configured")
+
+    key = anchor_key(cfg, checkpoint)
+    body = json.dumps(anchor_payload(checkpoint), sort_keys=True).encode("utf-8")
+    put_kwargs: dict[str, Any] = {
+        "Bucket": cfg.bucket,
+        "Key": key,
+        "Body": body,
+        "ContentType": "application/json",
+    }
+    if cfg.object_lock_mode:
+        put_kwargs["ObjectLockMode"] = cfg.object_lock_mode
+        put_kwargs["ObjectLockRetainUntilDate"] = cfg.retain_until
+
+    try:
+        store = client if client is not None else default_client(cfg)
+        store.put_object(**put_kwargs)
+        anchor_uri = f"s3://{cfg.bucket}/{key}"
+        checkpoint.anchor_uri = anchor_uri
+        checkpoint.anchor_status = "anchored"
+        session.commit()
+        return AnchorResult(ok=True, anchor_uri=anchor_uri)
+    except Exception as exc:  # noqa: BLE001 - anchoring is best-effort and retryable
+        logger.error("checkpoint %s anchor failed: %s: %s", checkpoint.id, type(exc).__name__, exc)
+        try:
+            checkpoint.anchor_status = "failed"
+            session.commit()
+        except Exception:  # noqa: BLE001
+            session.rollback()
+        return AnchorResult(ok=False, error=f"{type(exc).__name__}: {exc}")
+
+
+def retry_failed_anchors(
+    session: Session,
+    *,
+    limit: int = 20,
+    config: AnchorConfig | None = None,
+    client: ObjectStore | None = None,
+) -> list[AnchorResult]:
+    """Retry every ``pending`` or ``failed`` checkpoint, oldest first.
+
+    Run this from a periodic job so a transient storage outage self-heals
+    without operator intervention.
+    """
+    rows = (
+        session.execute(
+            select(LedgerCheckpoint)
+            .where(LedgerCheckpoint.anchor_status.in_(["pending", "failed"]))
+            .order_by(LedgerCheckpoint.id.asc())
+            .limit(limit)
+        )
+        .scalars()
+        .all()
+    )
+    return [anchor_checkpoint(session, row, config=config, client=client) for row in rows]
+
+
+def verify_anchor(
+    checkpoint: LedgerCheckpoint,
+    *,
+    config: AnchorConfig | None = None,
+    client: ObjectStore | None = None,
+) -> AnchorVerifyResult:
+    """Read the anchored copy of ``checkpoint`` and compare it to the row.
+
+    This is the independent-witness check: the object in the bucket was
+    written when the checkpoint was taken and cannot be changed under
+    Object Lock, so a mismatch proves the database row was rewritten after
+    anchoring. Never raises.
+    """
+    if checkpoint.anchor_uri is None:
+        return AnchorVerifyResult(ok=False, error="checkpoint has no anchor_uri")
+    cfg = config if config is not None else AnchorConfig.from_env()
+    try:
+        bucket, key = parse_anchor_uri(checkpoint.anchor_uri)
+        store = client if client is not None else default_client(cfg)
+        response = store.get_object(Bucket=bucket, Key=key)
+        raw = response["Body"].read()
+        anchored = json.loads(raw)
+    except Exception as exc:  # noqa: BLE001 - report, never raise
+        return AnchorVerifyResult(ok=False, error=f"{type(exc).__name__}: {exc}")
+
+    matched = (
+        anchored.get("from_seq") == checkpoint.from_seq
+        and anchored.get("to_seq") == checkpoint.to_seq
+        and anchored.get("merkle_root") == checkpoint.merkle_root
+        and anchored.get("prev_checkpoint_root") == checkpoint.prev_checkpoint_root
+    )
+    return AnchorVerifyResult(ok=matched, matched=matched, anchored=anchored)

ledgerchain/canonical.py

@@ -0,0 +1,32 @@
+"""RFC 8785 (JSON Canonicalization Scheme) helpers.
+
+Every hash in this package is computed over the canonical byte form of a
+JSON value: ``payload_sha256`` for ledger entries, and anything a caller
+chooses to sign before appending. Two dicts with the same content but a
+different key order, or the same number written two ways, hash the same.
+
+Canonicalization is delegated to the ``rfc8785`` package rather than
+hand-rolled. ECMAScript shortest-round-trip number formatting and UTF-16
+code-unit key ordering are easy to get subtly wrong, and every hash in the
+chain depends on them being exact.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+import rfc8785
+
+CanonicalizationError = rfc8785.CanonicalizationError
+"""Raised when a value cannot be canonicalized (for example, a ``set``)."""
+
+
+def canonicalize(value: Any) -> bytes:
+    """Serialize ``value`` to canonical JSON bytes per RFC 8785."""
+    return rfc8785.dumps(value)
+
+
+def sha256_hex(value: Any) -> str:
+    """SHA-256 of the canonical JSON serialization, as lowercase hex."""
+    return hashlib.sha256(canonicalize(value)).hexdigest()

ledgerchain/checkpoint.py

@@ -0,0 +1,132 @@
+"""Merkle checkpoints over the ledger.
+
+A checkpoint summarizes a contiguous ``seq`` range with one Merkle root and
+chains to the previous checkpoint's root via ``prev_checkpoint_root``. The
+checkpoint is also appended to the ledger as a ``checkpoint`` entry, so the
+chain carries its own checkpoint history. External anchoring is a separate
+step (:mod:`ledgerchain.anchor`) so a storage outage never blocks
+checkpoint creation.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from .models import DecisionLedgerEntry, LedgerCheckpoint, LedgerHead
+from .writer import CHECKPOINT_EVENT_TYPE, LedgerWriter
+
+CHECKPOINT_ACTOR: dict[str, Any] = {"kind": "system", "sub": "ledger-checkpoint"}
+
+
+def merkle_root(leaf_hexdigests: Sequence[str]) -> str:
+    """Binary Merkle root over hex SHA-256 leaves, returned as hex.
+
+    Pairs are hashed as ``sha256(left || right)`` over the raw 32-byte
+    digests. An odd node at any level is promoted unchanged rather than
+    duplicated, which avoids the known second-preimage weakness of the
+    duplicate-last-node variant. Raises ``ValueError`` on an empty input.
+    """
+    if not leaf_hexdigests:
+        raise ValueError("merkle_root requires at least one leaf")
+
+    level = [bytes.fromhex(h) for h in leaf_hexdigests]
+    while len(level) > 1:
+        next_level = [
+            hashlib.sha256(level[i] + level[i + 1]).digest() for i in range(0, len(level) - 1, 2)
+        ]
+        if len(level) % 2 == 1:
+            next_level.append(level[-1])
+        level = next_level
+    return level[0].hex()
+
+
+@dataclass(frozen=True)
+class CheckpointResult:
+    checkpoint: LedgerCheckpoint | None
+    skipped_reason: str | None = None
+
+
+def build_checkpoint(
+    session: Session,
+    *,
+    tenant_id: str = "default",
+    commit: bool = True,
+) -> CheckpointResult:
+    """Build one checkpoint over every entry since the last checkpoint.
+
+    The range is ``(last_checkpoint.to_seq, head.last_seq]``, or
+    ``[1, head.last_seq]`` when no checkpoint exists yet. Writes the
+    ``LedgerCheckpoint`` row and appends a ``checkpoint`` ledger entry
+    carrying the same ``{from_seq, to_seq, merkle_root,
+    prev_checkpoint_root}``. With ``commit=True`` (the default) both land
+    in one commit; pass ``commit=False`` to fold them into a larger
+    transaction you control.
+
+    Returns ``skipped_reason`` and no checkpoint when there is nothing new
+    to cover: an empty ledger, or a head that has not advanced past the
+    last checkpoint.
+    """
+    head = session.execute(select(LedgerHead)).scalar_one_or_none()
+    if head is None:
+        return CheckpointResult(checkpoint=None, skipped_reason="ledger is empty (no genesis yet)")
+
+    last_checkpoint = (
+        session.execute(select(LedgerCheckpoint).order_by(LedgerCheckpoint.to_seq.desc()))
+        .scalars()
+        .first()
+    )
+    from_seq = 1 if last_checkpoint is None else last_checkpoint.to_seq + 1
+    to_seq = head.last_seq
+
+    if from_seq > to_seq:
+        covered = last_checkpoint.to_seq if last_checkpoint is not None else 0
+        return CheckpointResult(
+            checkpoint=None,
+            skipped_reason=f"nothing new since last checkpoint (to_seq={covered})",
+        )
+
+    hashes = (
+        session.execute(
+            select(DecisionLedgerEntry.entry_hash)
+            .where(DecisionLedgerEntry.seq >= from_seq, DecisionLedgerEntry.seq <= to_seq)
+            .order_by(DecisionLedgerEntry.seq.asc())
+        )
+        .scalars()
+        .all()
+    )
+    root = merkle_root(hashes)
+    prev_root = last_checkpoint.merkle_root if last_checkpoint is not None else None
+
+    checkpoint = LedgerCheckpoint(
+        from_seq=from_seq,
+        to_seq=to_seq,
+        merkle_root=root,
+        prev_checkpoint_root=prev_root,
+        anchor_status="pending",
+    )
+    session.add(checkpoint)
+    session.flush()
+
+    LedgerWriter.append(
+        session,
+        event_type=CHECKPOINT_EVENT_TYPE,
+        payload={
+            "checkpoint_id": checkpoint.id,
+            "from_seq": from_seq,
+            "to_seq": to_seq,
+            "merkle_root": root,
+            "prev_checkpoint_root": prev_root,
+        },
+        actor=CHECKPOINT_ACTOR,
+        tenant_id=tenant_id,
+        subject_uri=f"ledger://checkpoints/{checkpoint.id}",
+    )
+    if commit:
+        session.commit()
+    return CheckpointResult(checkpoint=checkpoint)

ledgerchain/cli.py

@@ -0,0 +1,170 @@
+"""``ledgerchain`` command line.
+
+Built for auditors and operators, not for the application: every
+subcommand opens its own engine from ``--database-url`` (or
+``LEDGER_DATABASE_URL``, then ``DATABASE_URL``), does one thing, prints
+JSON, and exits non-zero on a failed verification so it composes in
+scripts and CI.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from collections.abc import Sequence
+from typing import Any
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from .anchor import AnchorConfig, anchor_checkpoint, retry_failed_anchors, verify_anchor
+from .checkpoint import build_checkpoint
+from .immutability import create_all
+from .models import LedgerCheckpoint
+from .verify import verify_chain
+
+
+def resolve_database_url(explicit: str | None) -> str:
+    url = explicit or os.environ.get("LEDGER_DATABASE_URL") or os.environ.get("DATABASE_URL")
+    if not url:
+        raise SystemExit(
+            "no database URL: pass --database-url or set LEDGER_DATABASE_URL / DATABASE_URL"
+        )
+    return url
+
+
+def _session(url: str) -> tuple[Any, Session]:
+    engine = create_engine(url)
+    return engine, sessionmaker(bind=engine)()
+
+
+def _emit(payload: dict[str, Any]) -> None:
+    print(json.dumps(payload, indent=2, default=str))
+
+
+def cmd_init_db(args: argparse.Namespace) -> int:
+    engine = create_engine(resolve_database_url(args.database_url))
+    try:
+        create_all(engine, immutable=not args.no_triggers)
+    finally:
+        engine.dispose()
+    _emit({"ok": True, "triggers": not args.no_triggers})
+    return 0
+
+
+def cmd_verify(args: argparse.Namespace) -> int:
+    engine, session = _session(resolve_database_url(args.database_url))
+    try:
+        result = verify_chain(session, from_seq=args.from_seq, to_seq=args.to_seq)
+        body = result.to_dict()
+        anchors_ok = True
+        if args.check_anchors:
+            anchor_reports = []
+            for summary in result.checkpoints:
+                if summary["anchor_status"] != "anchored":
+                    continue
+                cp = session.get(LedgerCheckpoint, summary["id"])
+                if cp is None:
+                    continue
+                report = verify_anchor(cp)
+                anchors_ok = anchors_ok and report.ok
+                anchor_reports.append(
+                    {
+                        "id": cp.id,
+                        "anchor_uri": cp.anchor_uri,
+                        "ok": report.ok,
+                        "matched": report.matched,
+                        "error": report.error,
+                    }
+                )
+            body["anchors"] = anchor_reports
+            body["anchors_valid"] = anchors_ok
+    finally:
+        session.close()
+        engine.dispose()
+    _emit(body)
+    return 0 if result.valid and anchors_ok else 1
+
+
+def cmd_checkpoint(args: argparse.Namespace) -> int:
+    engine, session = _session(resolve_database_url(args.database_url))
+    try:
+        result = build_checkpoint(session, tenant_id=args.tenant_id)
+        body: dict[str, Any] = {
+            "checkpoint": result.checkpoint.to_dict() if result.checkpoint else None,
+            "skipped_reason": result.skipped_reason,
+        }
+        if args.anchor and result.checkpoint is not None:
+            anchored = anchor_checkpoint(session, result.checkpoint, config=AnchorConfig.from_env())
+            body["anchor"] = anchored.__dict__
+    finally:
+        session.close()
+        engine.dispose()
+    _emit(body)
+    return 0
+
+
+def cmd_anchor(args: argparse.Namespace) -> int:
+    engine, session = _session(resolve_database_url(args.database_url))
+    try:
+        results = retry_failed_anchors(session, limit=args.limit, config=AnchorConfig.from_env())
+    finally:
+        session.close()
+        engine.dispose()
+    _emit({"attempted": len(results), "results": [r.__dict__ for r in results]})
+    return 0 if all(r.ok or r.skipped for r in results) else 1
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="ledgerchain", description="Tamper-evident decision ledger tools."
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    def add_db(p: argparse.ArgumentParser) -> None:
+        p.add_argument(
+            "--database-url",
+            default=None,
+            help="SQLAlchemy URL. Overrides LEDGER_DATABASE_URL and DATABASE_URL.",
+        )
+
+    p = sub.add_parser("init-db", help="Create the ledger tables and immutability triggers.")
+    add_db(p)
+    p.add_argument("--no-triggers", action="store_true", help="Create tables only.")
+    p.set_defaults(func=cmd_init_db)
+
+    p = sub.add_parser("verify", help="Recompute the chain and report the first bad seq.")
+    add_db(p)
+    p.add_argument("--from-seq", type=int, default=None)
+    p.add_argument("--to-seq", type=int, default=None)
+    p.add_argument(
+        "--check-anchors",
+        action="store_true",
+        help="Also read each anchored checkpoint back from object storage and compare.",
+    )
+    p.set_defaults(func=cmd_verify)
+
+    p = sub.add_parser("checkpoint", help="Build a Merkle checkpoint over new entries.")
+    add_db(p)
+    p.add_argument("--tenant-id", default="default")
+    p.add_argument("--anchor", action="store_true", help="Anchor it immediately.")
+    p.set_defaults(func=cmd_checkpoint)
+
+    p = sub.add_parser("anchor", help="Retry pending and failed anchors.")
+    add_db(p)
+    p.add_argument("--limit", type=int, default=20)
+    p.set_defaults(func=cmd_anchor)
+
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    code: int = args.func(args)
+    return code
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())