superlocalmemory 3.8.13 → 4.0.0

package/src/superlocalmemory/core/transactions/manifest.py

@@ -0,0 +1,255 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterable
+from dataclasses import dataclass
+from enum import StrEnum
+
+MANIFEST_V1: int = 1
+MANIFEST_V2: int = 2
+
+from superlocalmemory.core.transactions.obligations import Obligation
+from superlocalmemory.core.transactions.owners import (
+    ObligationKind,
+    ObligationState,
+    is_terminal_success,
+)
+
+
+class ManifestState(StrEnum):
+    COMPLETE = "COMPLETE"
+    DEGRADED = "DEGRADED"
+    FAILED = "FAILED"
+
+
+@dataclass(frozen=True, slots=True)
+class OwnerEvidence:
+    owner: str
+    kind: ObligationKind
+    state: ObligationState
+    revision: int
+    checksum: str
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "owner": self.owner,
+            "kind": str(self.kind),
+            "state": str(self.state),
+            "revision": self.revision,
+            "checksum": self.checksum,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class CompletionManifest:
+    operation_id: str
+    profile_id: str
+    state: ManifestState
+    all_met: bool
+    obligation_count: int
+    owner_evidence: tuple[OwnerEvidence, ...]
+    manifest_hash: str
+    created_at: float
+    updated_at: float
+
+
+def build_evidence(obligations: Iterable[Obligation]) -> tuple[OwnerEvidence, ...]:
+    evidence = [
+        OwnerEvidence(
+            owner=o.owner,
+            kind=o.kind,
+            state=o.state,
+            revision=o.revision,
+            checksum=o.checksum or "",
+        )
+        for o in obligations
+    ]
+    evidence.sort(key=lambda e: (e.owner, str(e.kind), e.revision))
+    return tuple(evidence)
+
+
+def _sorted_evidence_dicts(evidence: Iterable[OwnerEvidence]) -> list[dict[str, object]]:
+    return sorted(
+        (e.as_dict() for e in evidence),
+        key=lambda d: (d["owner"], d["kind"], d["revision"]),
+    )
+
+
+def evidence_json(evidence: Iterable[OwnerEvidence]) -> str:
+    return json.dumps(
+        _sorted_evidence_dicts(evidence),
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+    )
+
+
+def compute_envelope_hash(
+    *,
+    operation_id: str,
+    profile_id: str,
+    state: ManifestState,
+    all_met: bool,
+    obligation_count: int,
+    evidence: Iterable[OwnerEvidence],
+) -> str:
+    envelope = {
+        "operation_id": operation_id,
+        "profile_id": profile_id,
+        "state": str(state),
+        "all_met": bool(all_met),
+        "obligation_count": obligation_count,
+        "evidence": _sorted_evidence_dicts(evidence),
+    }
+    canonical = json.dumps(
+        envelope, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def hash_envelope_fields(
+    *,
+    operation_id: str,
+    profile_id: str,
+    state: str,
+    all_met: bool,
+    obligation_count: int,
+    evidence_dicts: list[dict[str, object]],
+) -> str:
+    envelope = {
+        "operation_id": operation_id,
+        "profile_id": profile_id,
+        "state": state,
+        "all_met": bool(all_met),
+        "obligation_count": obligation_count,
+        "evidence": evidence_dicts,
+    }
+    canonical = json.dumps(
+        envelope, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _canonical_envelope(
+    *,
+    operation_id: str,
+    profile_id: str,
+    state: str,
+    all_met: bool,
+    obligation_count: int,
+    evidence_dicts: list[dict[str, object]],
+    manifest_version: int | None = None,
+) -> bytes:
+    """Produce the deterministic byte representation of the manifest envelope.
+
+    When ``manifest_version`` is provided (v2+), it is bound into the canonical
+    bytes so a downgrade from v2 → v1 is detected by the MAC mismatch: the v1
+    SHA path does NOT include version, so any v2 HMAC covers a strictly different
+    byte string than any v1 SHA.
+    """
+    envelope: dict[str, object] = {
+        "operation_id": operation_id,
+        "profile_id": profile_id,
+        "state": state,
+        "all_met": bool(all_met),
+        "obligation_count": obligation_count,
+        "evidence": evidence_dicts,
+    }
+    if manifest_version is not None:
+        envelope["manifest_version"] = manifest_version
+    return json.dumps(
+        envelope, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
+    ).encode("utf-8")
+
+
+def compute_envelope_hmac(
+    *,
+    operation_id: str,
+    profile_id: str,
+    state: ManifestState,
+    all_met: bool,
+    obligation_count: int,
+    evidence: Iterable[OwnerEvidence],
+    key: bytes | None = None,
+) -> str:
+    """HMAC-SHA256 keyed manifest hash (v2).  ``key`` is injected in tests;
+    production code passes ``key=None`` to auto-derive from the installation key."""
+    from superlocalmemory.core.transactions.manifest_key import (
+        compute_hmac,
+        derive_manifest_hmac_key,
+    )
+
+    actual_key = key if key is not None else derive_manifest_hmac_key()
+    payload = _canonical_envelope(
+        operation_id=operation_id,
+        profile_id=profile_id,
+        state=str(state),
+        all_met=all_met,
+        obligation_count=obligation_count,
+        evidence_dicts=_sorted_evidence_dicts(evidence),
+        manifest_version=MANIFEST_V2,
+    )
+    return compute_hmac(actual_key, payload)
+
+
+def verify_envelope_hmac(
+    *,
+    operation_id: str,
+    profile_id: str,
+    state: str,
+    all_met: bool,
+    obligation_count: int,
+    evidence_dicts: list[dict[str, object]],
+    stored_mac: str,
+    key: bytes | None = None,
+) -> bool:
+    """Constant-time verify an HMAC-keyed manifest hash (v2)."""
+    from superlocalmemory.core.transactions.manifest_key import (
+        derive_manifest_hmac_key,
+        verify_hmac,
+    )
+
+    actual_key = key if key is not None else derive_manifest_hmac_key()
+    payload = _canonical_envelope(
+        operation_id=operation_id,
+        profile_id=profile_id,
+        state=state,
+        all_met=all_met,
+        obligation_count=obligation_count,
+        evidence_dicts=evidence_dicts,
+        manifest_version=MANIFEST_V2,
+    )
+    return verify_hmac(stored_mac, actual_key, payload)
+
+
+def derive_state(
+    obligations: tuple[Obligation, ...], *, canonical_committed: bool,
+) -> tuple[ManifestState, bool]:
+    if not canonical_committed:
+        return ManifestState.FAILED, False
+    if not obligations:
+        return ManifestState.FAILED, False
+    all_met = all(is_terminal_success(o.state) for o in obligations)
+    if all_met:
+        return ManifestState.COMPLETE, True
+    return ManifestState.DEGRADED, False
+
+
+__all__ = [
+    "MANIFEST_V1",
+    "MANIFEST_V2",
+    "CompletionManifest",
+    "ManifestState",
+    "OwnerEvidence",
+    "build_evidence",
+    "compute_envelope_hash",
+    "compute_envelope_hmac",
+    "derive_state",
+    "evidence_json",
+    "hash_envelope_fields",
+    "verify_envelope_hmac",
+]

package/src/superlocalmemory/core/transactions/manifest_key.py

@@ -0,0 +1,155 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Installation-bound HMAC signing key for completion manifests and erasure receipts.
+
+The signing key is a 32-byte secret persisted at
+``~/.superlocalmemory/.manifest_signing_key`` (mode 0600).  It is generated
+once on first use and read on every subsequent call.  Purpose-specific
+sub-keys are derived via HMAC so a key compromise in one context cannot be
+used to forge a receipt in another.
+
+The key is NEVER logged or included in any error message.
+
+Key file location: resolved via ``state_path(".manifest_signing_key")``.
+In tests, monkeypatch ``_signing_key_path`` and/or ``_ensure_signing_key``
+to return a deterministic path or byte sequence.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac as _hmac
+import os
+import secrets
+import sys
+from pathlib import Path
+
+_KEY_CONTEXT_MANIFEST: bytes = b"slm-manifest-v2"
+_KEY_CONTEXT_RECEIPT: bytes = b"slm-receipt-v2"
+_MANIFEST_VERSION: int = 2
+
+
+def _signing_key_path() -> Path:
+    from superlocalmemory.infra.data_root import state_path
+
+    return state_path(".manifest_signing_key")
+
+
+def _ensure_signing_key() -> bytes:
+    """Read the 32-byte hex signing key, creating it on first call.
+
+    Returns the raw bytes of the key (NOT the hex string).
+
+    Raises RuntimeError if the key cannot be obtained (corrupt, unreadable,
+    permission error).  Ephemeral keys are NEVER returned — they would
+    permanently desync multi-process daemons and silently break all
+    verification of previously-signed manifests.
+    """
+    key_path = _signing_key_path()
+    try:
+        key_path.parent.mkdir(parents=True, exist_ok=True)
+    except OSError as exc:
+        raise RuntimeError(
+            "signing key: cannot create key directory (key never logged)"
+        ) from exc
+
+    try:
+        key_exists = key_path.exists()
+    except OSError as exc:
+        # stat() itself can fail (e.g. an unreadable parent dir). Fail closed
+        # with the same RuntimeError contract — never leak a raw OSError or
+        # fall through to an ephemeral key.
+        raise RuntimeError(
+            "signing key: cannot access key path (key never logged)"
+        ) from exc
+
+    if key_exists:
+        try:
+            raw = key_path.read_text(encoding="utf-8").strip()
+        except OSError as exc:
+            raise RuntimeError(
+                "signing key: cannot read existing key file (key never logged)"
+            ) from exc
+        if len(raw) != 64:
+            raise RuntimeError(
+                "signing key: file has unexpected length (corrupt or truncated) — "
+                "remove the file and restart to generate a fresh key"
+            )
+        return bytes.fromhex(raw)
+
+    # First-use: generate and persist atomically.
+    raw = secrets.token_hex(32)
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        fd = os.open(str(key_path), flags, 0o600)
+        try:
+            os.write(fd, raw.encode("utf-8"))
+        finally:
+            os.close(fd)
+    except FileExistsError:
+        # Another process won the race — read its key (never return our own).
+        try:
+            existing = key_path.read_text(encoding="utf-8").strip()
+        except OSError as exc:
+            raise RuntimeError(
+                "signing key: race winner's key unreadable (key never logged)"
+            ) from exc
+        if len(existing) != 64:
+            raise RuntimeError(
+                "signing key: race winner wrote a corrupt key (key never logged)"
+            )
+        return bytes.fromhex(existing)
+    except OSError as exc:
+        # Permission / symlink error — never return an ephemeral key.
+        raise RuntimeError(
+            "signing key: cannot create key file (key never logged)"
+        ) from exc
+
+    if sys.platform != "win32":
+        try:
+            os.chmod(str(key_path), 0o600)
+        except OSError:
+            pass
+
+    return bytes.fromhex(raw)
+
+
+def derive_manifest_hmac_key() -> bytes:
+    """Return the HMAC sub-key for completion manifests."""
+    master = _ensure_signing_key()
+    return _hmac.new(master, _KEY_CONTEXT_MANIFEST, hashlib.sha256).digest()
+
+
+def derive_receipt_hmac_key() -> bytes:
+    """Return the HMAC sub-key for erasure receipts."""
+    master = _ensure_signing_key()
+    return _hmac.new(master, _KEY_CONTEXT_RECEIPT, hashlib.sha256).digest()
+
+
+def compute_hmac(key: bytes, payload: bytes) -> str:
+    """Return HMAC-SHA256 of ``payload`` under ``key`` as a hex string."""
+    return _hmac.new(key, payload, hashlib.sha256).hexdigest()
+
+
+def verify_hmac(stored_mac: str, key: bytes, payload: bytes) -> bool:
+    """Constant-time verify that ``stored_mac`` matches HMAC of ``payload`` under ``key``."""
+    try:
+        expected = _hmac.new(key, payload, hashlib.sha256).digest()
+        stored_bytes = bytes.fromhex(stored_mac)
+    except (ValueError, TypeError):
+        return False
+    return _hmac.compare_digest(expected, stored_bytes)
+
+
+__all__ = [
+    "MANIFEST_VERSION",
+    "compute_hmac",
+    "derive_manifest_hmac_key",
+    "derive_receipt_hmac_key",
+    "verify_hmac",
+]
+
+MANIFEST_VERSION = _MANIFEST_VERSION

package/src/superlocalmemory/core/transactions/obligations.py

@@ -0,0 +1,272 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sqlite3
+import time
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from superlocalmemory.core.transactions.owners import (
+    ObligationKind,
+    ObligationState,
+    OperationContext,
+)
+
+
+class ObligationConflictError(ValueError):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class Obligation:
+    operation_id: str
+    profile_id: str
+    owner: str
+    kind: ObligationKind
+    subject_id: str
+    revision: int
+    state: ObligationState
+    checksum: str | None
+    detail: Mapping[str, Any] | None
+    attempts: int
+    context_digest: str | None
+    verify_attempts: int
+    last_verified_at: float | None
+    created_at: float
+    updated_at: float
+
+
+_SELECT_COLUMNS = (
+    "operation_id, profile_id, owner, kind, subject_id, revision, state, "
+    "checksum, detail, attempts, context_digest, verify_attempts, "
+    "last_verified_at, created_at, updated_at"
+)
+
+
+def context_digest(
+    context: OperationContext, owner: str, kind: ObligationKind,
+) -> str:
+    payload = "\0".join([
+        context.operation_id,
+        context.profile_id,
+        owner,
+        str(kind),
+    ])
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+class ObligationLedger:
+    def record(
+        self,
+        conn: sqlite3.Connection,
+        context: OperationContext,
+        owner: str,
+        kind: ObligationKind,
+    ) -> None:
+        digest = context_digest(context, owner, kind)
+        existing = conn.execute(
+            "SELECT context_digest, profile_id FROM projection_obligations "
+            "WHERE operation_id = ? AND owner = ? AND kind = ?",
+            (context.operation_id, owner, str(kind)),
+        ).fetchone()
+        if existing is not None:
+            stored, stored_profile = existing[0], existing[1]
+            if stored_profile != context.profile_id:
+                raise ObligationConflictError(
+                    f"obligation {context.operation_id}/{owner}/{kind} "
+                    "already recorded for a different profile"
+                )
+            if stored is None:
+                conn.execute(
+                    "UPDATE projection_obligations SET context_digest = ? "
+                    "WHERE operation_id = ? AND owner = ? AND kind = ?",
+                    (digest, context.operation_id, owner, str(kind)),
+                )
+                return
+            if stored != digest:
+                raise ObligationConflictError(
+                    f"obligation {context.operation_id}/{owner}/{kind} "
+                    "already recorded with a different context"
+                )
+            return
+        now = time.time()
+        conn.execute(
+            "INSERT INTO projection_obligations "
+            "(operation_id, profile_id, owner, kind, subject_id, revision, "
+            "state, checksum, detail, attempts, context_digest, "
+            "verify_attempts, last_verified_at, created_at, updated_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, NULL, NULL, 0, ?, 0, NULL, ?, ?)",
+            (
+                context.operation_id,
+                context.profile_id,
+                owner,
+                str(kind),
+                context.subject_id,
+                context.revision,
+                str(ObligationState.PENDING),
+                digest,
+                now,
+                now,
+            ),
+        )
+
+    def record_many(
+        self,
+        conn: sqlite3.Connection,
+        context: OperationContext,
+        owners: Iterable[str],
+        kind: ObligationKind,
+    ) -> None:
+        for owner in owners:
+            self.record(conn, context, owner, kind)
+
+    def mark(
+        self,
+        conn: sqlite3.Connection,
+        operation_id: str,
+        owner: str,
+        kind: ObligationKind,
+        state: ObligationState,
+        *,
+        checksum: str | None = None,
+        detail: Mapping[str, Any] | None = None,
+        bump_attempts: bool = False,
+        bump_verify_attempts: bool = False,
+        set_verified_at: bool = False,
+    ) -> int:
+        detail_json = None if detail is None else json.dumps(
+            dict(detail), sort_keys=True, separators=(",", ":"), ensure_ascii=False
+        )
+        now = time.time()
+        cursor = conn.execute(
+            "UPDATE projection_obligations SET "
+            "state = ?, "
+            "checksum = COALESCE(?, checksum), "
+            "detail = ?, "
+            "attempts = attempts + ?, "
+            "verify_attempts = verify_attempts + ?, "
+            "last_verified_at = CASE WHEN ? THEN ? ELSE last_verified_at END, "
+            "updated_at = ? "
+            "WHERE operation_id = ? AND owner = ? AND kind = ?",
+            (
+                str(state),
+                checksum,
+                detail_json,
+                1 if bump_attempts else 0,
+                1 if bump_verify_attempts else 0,
+                1 if set_verified_at else 0,
+                now,
+                now,
+                operation_id,
+                owner,
+                str(kind),
+            ),
+        )
+        return cursor.rowcount
+
+    def fetch(
+        self, conn: sqlite3.Connection, operation_id: str,
+    ) -> tuple[Obligation, ...]:
+        rows = conn.execute(
+            f"SELECT {_SELECT_COLUMNS} FROM projection_obligations "
+            "WHERE operation_id = ? ORDER BY owner, kind",
+            (operation_id,),
+        ).fetchall()
+        return tuple(_row_to_obligation(row) for row in rows)
+
+    def fetch_by_state(
+        self,
+        conn: sqlite3.Connection,
+        state: ObligationState,
+        *,
+        limit: int = 100,
+    ) -> tuple[Obligation, ...]:
+        rows = conn.execute(
+            f"SELECT {_SELECT_COLUMNS} FROM projection_obligations "
+            "WHERE state = ? ORDER BY updated_at LIMIT ?",
+            (str(state), limit),
+        ).fetchall()
+        return tuple(_row_to_obligation(row) for row in rows)
+
+    def pending_operation_ids(
+        self,
+        conn: sqlite3.Connection,
+        *,
+        profile_id: str | None = None,
+        limit: int = 100,
+        max_attempts: int = 10,
+    ) -> tuple[str, ...]:
+        where = "state NOT IN (?, ?) AND attempts < ?"
+        params: list[Any] = [
+            str(ObligationState.VERIFIED), str(ObligationState.ERASED), max_attempts,
+        ]
+        if profile_id is not None:
+            where += " AND profile_id = ?"
+            params.append(profile_id)
+        params.append(limit)
+        rows = conn.execute(
+            f"SELECT DISTINCT operation_id FROM projection_obligations "
+            f"WHERE {where} ORDER BY updated_at LIMIT ?",
+            tuple(params),
+        ).fetchall()
+        return tuple(row[0] for row in rows)
+
+    def operations_missing_manifest(
+        self,
+        conn: sqlite3.Connection,
+        *,
+        profile_id: str | None = None,
+        limit: int = 100,
+        max_attempts: int = 10,
+    ) -> tuple[str, ...]:
+        where = "m.operation_id IS NULL AND o.attempts < ?"
+        params: list[Any] = [max_attempts]
+        if profile_id is not None:
+            where += " AND o.profile_id = ?"
+            params.append(profile_id)
+        params.append(limit)
+        rows = conn.execute(
+            f"SELECT DISTINCT o.operation_id FROM projection_obligations o "
+            f"LEFT JOIN completion_manifests m ON m.operation_id = o.operation_id "
+            f"WHERE {where} ORDER BY o.operation_id LIMIT ?",
+            tuple(params),
+        ).fetchall()
+        return tuple(row[0] for row in rows)
+
+
+def _row_to_obligation(row: Any) -> Obligation:
+    detail_raw = row[8]
+    detail: Mapping[str, Any] | None
+    if detail_raw is None:
+        detail = None
+    else:
+        try:
+            parsed = json.loads(detail_raw)
+            detail = parsed if isinstance(parsed, dict) else {"value": parsed}
+        except (TypeError, json.JSONDecodeError):
+            detail = None
+    return Obligation(
+        operation_id=row[0],
+        profile_id=row[1],
+        owner=row[2],
+        kind=ObligationKind(row[3]),
+        subject_id=row[4],
+        revision=int(row[5]),
+        state=ObligationState(row[6]),
+        checksum=row[7],
+        detail=detail,
+        attempts=int(row[9]),
+        context_digest=row[10],
+        verify_attempts=int(row[11]) if row[11] is not None else 0,
+        last_verified_at=float(row[12]) if row[12] is not None else None,
+        created_at=float(row[13]),
+        updated_at=float(row[14]),
+    )
+
+
+__all__ = ["Obligation", "ObligationConflictError", "ObligationLedger", "context_digest"]