cortexm 0.3.0__py3-none-any.whl

cortexm/provenance/vc.py

@@ -0,0 +1,250 @@
+"""W3C Verifiable Credentials 2.0 export for memory ranges.
+
+A memory range (e.g. "all facts about user=alice from 2026-01-01 to
+2026-02-01") can be exported as a W3C Verifiable Credential with an
+eddsa-jcs-2022 Data Integrity proof.
+
+The VC contains:
+  - The BLAKE3 root of the range (the Merkle root over the fact hashes)
+  - The range query (user_id, valid_from, valid_to)
+  - The number of facts in the range
+  - The COSE Sign1 envelope that proves the agent signed it
+
+External verifiers can confirm:
+  (1) The range query is well-formed
+  (2) The agent signed the range (via the COSE Sign1)
+  (3) The BLAKE3 root is correct (by recomputing it from the facts)
+  — without seeing the individual facts themselves.
+
+This implements the W3C VC Data Model 2.0 + the eddsa-jcs-2022
+cryptosuite. JSON Canonicalization Scheme (JCS) is RFC 8785.
+
+Reference:
+  - W3C VC Data Model 2.0: https://www.w3.org/TR/vc-data-model-2.0/
+  - eddsa-jcs-2022 cryptosuite:
+      https://www.w3.org/TR/vc-di-eddsa/
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+from cortexm.provenance.agent import Ed25519AgentKey, get_default_agent
+from cortexm.provenance.cose import CoseSign1Envelope, sign_commit
+
+
+@dataclass
+class VerifiableCredential:
+    """A W3C VC 2.0 with an eddsa-jcs-2022 Data Integrity proof."""
+    context: list[str] = field(default_factory=lambda: [
+        "https://www.w3.org/ns/credentials/v2",
+        "https://context-m.dev/ns/memory/v1",
+    ])
+    id: str = ""
+    type: list[str] = field(default_factory=lambda: [
+        "VerifiableCredential", "MemoryRangeCredential"
+    ])
+    issuer: str = ""           # did:key of the issuing agent
+    valid_from: str = ""       # ISO timestamp
+    credential_subject: dict = field(default_factory=dict)
+    proof: dict = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        return {
+            "@context": self.context,
+            "id": self.id,
+            "type": self.type,
+            "issuer": self.issuer,
+            "validFrom": self.valid_from,
+            "credentialSubject": self.credential_subject,
+            "proof": self.proof,
+        }
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), sort_keys=True, indent=2)
+
+
+def _merkle_root(hashes: list[str]) -> str:
+    """Compute a Merkle root over a list of BLAKE3 fact hashes.
+
+    For <2 hashes: returns the single hash (or empty string if 0).
+    For 2+ hashes: pairwise concat + BLAKE3, repeat until 1 remains.
+    Pads odd levels by repeating the last hash.
+    """
+    if not hashes:
+        return ""
+    if len(hashes) == 1:
+        return hashes[0]
+    level = list(hashes)
+    while len(level) > 1:
+        if len(level) % 2 == 1:
+            level.append(level[-1])  # pad
+        next_level = []
+        for i in range(0, len(level), 2):
+            combined = (level[i] + level[i + 1]).encode("utf-8")
+            try:
+                import blake3
+                h = blake3.blake3(combined).hexdigest()
+            except ImportError:
+                h = hashlib.blake2b(combined, digest_size=32).hexdigest()
+            next_level.append(h)
+        level = next_level
+    return level[0]
+
+
+def _jcs_canonical(obj: dict) -> str:
+    """JSON Canonicalization Scheme (RFC 8785).
+
+    Sorts keys recursively, uses minimal whitespace, escapes per RFC.
+    Python's json.dumps with sort_keys=True + separators=(",", ":")
+    produces output that mostly matches JCS, with one caveat: unicode
+    escapes. JCS requires unescaped non-ASCII, which Python's
+    ensure_ascii=False provides.
+    """
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"),
+                       ensure_ascii=False)
+
+
+def export_memory_range_vc(store, *,
+                            user_id: str | None = None,
+                            valid_from: str | None = None,
+                            valid_to: str | None = None,
+                            include_hypotheses: bool = False,
+                            agent: Ed25519AgentKey | None = None,
+                            issuer_id: str | None = None) -> VerifiableCredential:
+    """Export a memory range as a W3C Verifiable Credential.
+
+    The VC reveals:
+      - The range query (user_id, valid_from, valid_to)
+      - The number of facts in the range
+      - The BLAKE3 Merkle root over the fact hashes
+      - The issuing agent's did:key
+
+    The VC does NOT reveal:
+      - Individual fact values (only the aggregate Merkle root)
+      - The underlying chunks (only metadata + root)
+    """
+    if agent is None:
+        agent = get_default_agent()
+
+    where = "is_active=1 AND quarantined=0"
+    args: list = []
+    if user_id is not None:
+        where += " AND user_id=?"
+        args.append(user_id)
+    if valid_from is not None:
+        where += " AND valid_from >= ?"
+        args.append(valid_from)
+    if valid_to is not None:
+        where += " AND valid_to <= ?"
+        args.append(valid_to)
+    if not include_hypotheses:
+        # exclude derived facts (hypotheses, abstractions, analogies)
+        where += " AND is_derived=0"
+
+    rows = store.conn.execute(
+        f"SELECT id, source_hash FROM facts WHERE {where} "
+        f"ORDER BY valid_from, id", args).fetchall()
+
+    hashes = [r[1] or r[0] for r in rows]  # prefer source_hash, fallback to id
+    root = _merkle_root(hashes)
+
+    credential_subject = {
+        "range": {
+            "user_id": user_id,
+            "valid_from": valid_from,
+            "valid_to": valid_to,
+        },
+        "n_facts": len(rows),
+        "merkle_root": root,
+        "hash_algorithm": "BLAKE3-256",
+        "storage": "context-m-trace",
+    }
+
+    # build the proof: eddsa-jcs-2022 Data Integrity proof
+    ts_created = datetime.now(timezone.utc).isoformat()
+    proof_options = {
+        "type": "DataIntegrityProof",
+        "cryptosuite": "eddsa-jcs-2022",
+        "created": ts_created,
+        "verificationMethod": agent.did,
+        "proofPurpose": "assertionMethod",
+    }
+    # canonicalize the subject + proof options (minus the proofValue)
+    canon_input = {
+        "@context": [
+            "https://www.w3.org/ns/credentials/v2",
+            "https://context-m.dev/ns/memory/v1",
+        ],
+        "type": ["VerifiableCredential", "MemoryRangeCredential"],
+        "issuer": agent.did,
+        "validFrom": ts_created,
+        "credentialSubject": credential_subject,
+    }
+    canon_input.update(proof_options)
+    canon_str = _jcs_canonical(canon_input)
+    signature = agent.sign(canon_str.encode("utf-8"))
+
+    import base64
+    proof = proof_options.copy()
+    proof["proofValue"] = base64.urlsafe_b64encode(signature).decode().rstrip("=")
+
+    vc = VerifiableCredential(
+        id=issuer_id or f"urn:uuid:{hashlib.sha256(canon_str.encode()).hexdigest()[:32]}",
+        issuer=agent.did,
+        valid_from=ts_created,
+        credential_subject=credential_subject,
+        proof=proof,
+    )
+    return vc
+
+
+def verify_vc(vc: VerifiableCredential,
+              agent: Ed25519AgentKey | None = None,
+              expected_root: str | None = None,
+              expected_n_facts: int | None = None) -> bool:
+    """Verify a VerifiableCredential's Data Integrity proof."""
+    if agent is None:
+        agent = get_default_agent()
+        if agent.did != vc.issuer:
+            return False
+
+    if expected_root and vc.credential_subject.get("merkle_root") != expected_root:
+        return False
+    if expected_n_facts is not None and vc.credential_subject.get("n_facts") != expected_n_facts:
+        return False
+
+    proof = vc.proof
+    if proof.get("type") != "DataIntegrityProof":
+        return False
+    if proof.get("cryptosuite") != "eddsa-jcs-2022":
+        return False
+
+    import base64
+    proof_value = proof.get("proofValue", "")
+    if not proof_value:
+        return False
+    signature = base64.urlsafe_b64decode(proof_value + "=" * (-len(proof_value) % 4))
+
+    # reconstruct canonical input (minus the proofValue)
+    proof_options = {k: v for k, v in proof.items() if k != "proofValue"}
+    canon_input = {
+        "@context": vc.context,
+        "type": vc.type,
+        "issuer": vc.issuer,
+        "validFrom": vc.valid_from,
+        "credentialSubject": vc.credential_subject,
+    }
+    canon_input.update(proof_options)
+    canon_str = _jcs_canonical(canon_input)
+    return agent.verify(canon_str.encode("utf-8"), signature)
+
+
+__all__ = [
+    "VerifiableCredential",
+    "export_memory_range_vc", "verify_vc",
+]

cortexm/security/crypto.py

@@ -0,0 +1,162 @@
+"""Encryption at rest — AES-256-GCM envelope encryption.
+
+Threat model: attacker obtains the database file (disk theft, S3 bucket
+leak, backup exposure) but not the master key. Every sensitive column —
+chunk text (raw conversation), vault payloads, API key hashes' pepper —
+is ciphertext in the file; the key never touches disk unless explicitly
+exported to a key file (0600) or supplied via ``CONTEXT_M_MASTER_KEY``.
+
+Design (standard envelope):
+  master key (KEK)  — 32 bytes, from env / key file / generated
+  data key   (DEK)  — 32 bytes, random per database, wrapped by KEK,
+                       stored in the ``kv`` table (``enc:dek``)
+  payloads          — AES-256-GCM, 12-byte random nonce, AAD-bound
+
+The cipher is injectable into TraceStore (field encryption) and the PII
+vault. μ=0 is preserved: all primitives are symmetric, deterministic
+given key material, and make zero network or LLM calls.
+"""
+
+from __future__ import annotations
+
+import base64
+import os
+import secrets
+
+try:  # cryptography >= 3.0 ships AESGCM
+    from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+    _HAVE_CRYPTO = True
+except Exception:  # pragma: no cover — documented degraded mode
+    _HAVE_CRYPTO = False
+
+ENV_MASTER_KEY = "CONTEXT_M_MASTER_KEY"
+
+
+class CryptoUnavailable(RuntimeError):
+    pass
+
+
+def _b64e(b: bytes) -> str:
+    return base64.urlsafe_b64encode(b).decode()
+
+
+def _b64d(s: str) -> bytes:
+    return base64.urlsafe_b64decode(s.encode())
+
+
+class AESGCMCipher:
+    """One cipher object per data key. Encrypt/decrypt strings and bytes."""
+
+    FORMAT = "enc:v1"          # versioned envelope, forward-compatible
+    PREFIX = "«enc:v1:"
+
+    def __init__(self, master_key: bytes, dek: bytes | None = None,
+                 store=None) -> None:
+        if not _HAVE_CRYPTO:
+            raise CryptoUnavailable(
+                "pip install cryptography  # required for encryption at rest")
+        if len(master_key) != 32:
+            raise ValueError("master key must be 32 bytes (256-bit)")
+        self._kek = master_key
+        self._store = store
+        if dek is not None:
+            self._dek = dek
+        elif store is not None:
+            self._dek = self._load_or_create_dek(store)
+        else:
+            self._dek = secrets.token_bytes(32)
+
+    # -------------------------------------------------------------- DEK mgmt
+    def _load_or_create_dek(self, store) -> bytes:
+        raw = store.kv_get("enc:dek")
+        if raw and raw.startswith(f"{self.FORMAT}:"):
+            try:
+                blob = _b64d(raw[len(self.FORMAT) + 1:])
+                nonce, wrapped = blob[:12], blob[12:]
+                return AESGCM(self._kek).decrypt(nonce, wrapped, b"context-m-dek")
+            except Exception:
+                raise CryptoUnavailable(
+                    "cannot unwrap data key — wrong master key for this DB?")
+        dek = secrets.token_bytes(32)
+        nonce = secrets.token_bytes(12)
+        wrapped = AESGCM(self._kek).encrypt(nonce, dek, b"context-m-dek")
+        store.kv_set("enc:dek", f"{self.FORMAT}:{_b64e(nonce + wrapped)}")
+        return dek
+
+    # -------------------------------------------------------------- primitives
+    def encrypt(self, plaintext: bytes, aad: bytes = b"") -> str:
+        nonce = secrets.token_bytes(12)
+        ct = AESGCM(self._dek).encrypt(nonce, plaintext, aad)
+        return f"{self.PREFIX}{_b64e(nonce + ct)}»"
+
+    def decrypt(self, envelope: str, aad: bytes = b"") -> bytes:
+        if not envelope.startswith(self.PREFIX) or not envelope.endswith("»"):
+            raise ValueError("not an encrypted envelope")
+        blob = _b64d(envelope[len(self.PREFIX):-1])
+        nonce, ct = blob[:12], blob[12:]
+        return AESGCM(self._dek).decrypt(nonce, ct, aad)
+
+    def encrypt_str(self, s: str) -> str:
+        return self.encrypt(s.encode())
+
+    def decrypt_str(self, envelope: str) -> str:
+        return self.decrypt(envelope).decode()
+
+    def is_envelope(self, s: str | None) -> bool:
+        return bool(s) and s.startswith(self.PREFIX) and s.endswith("»")
+
+    # -------------------------------------------------------------- rotation
+    def rotate(self, new_master_key: bytes) -> bytes:
+        """Re-wrap the DEK under a new master key. Returns the DEK so the
+        caller can construct a new cipher bound to the same store."""
+        if len(new_master_key) != 32:
+            raise ValueError("master key must be 32 bytes")
+        if self._store is None:
+            raise ValueError("rotation requires a store-bound cipher")
+        nonce = secrets.token_bytes(12)
+        wrapped = AESGCM(new_master_key).encrypt(nonce, self._dek, b"context-m-dek")
+        self._store.kv_set("enc:dek", f"{self.FORMAT}:{_b64e(nonce + wrapped)}")
+        return self._dek
+
+
+# ------------------------------------------------------------------ helpers
+def load_master_key(path: str | None = None,
+                    env: str = ENV_MASTER_KEY) -> bytes | None:
+    """Master key resolution order: explicit path, env var (hex/base64),
+    sidecar ``<db>.key``. Returns None when no key is configured."""
+    if path and os.path.exists(path):
+        data = open(path, "rb").read().strip()
+        return _coerce_key(data)
+    val = os.environ.get(env)
+    if val:
+        return _coerce_key(val.encode().strip())
+    return None
+
+
+def _coerce_key(data: bytes) -> bytes:
+    if len(data) == 32:
+        return data
+    try:
+        raw = _b64d(data.decode())
+        if len(raw) == 32:
+            return raw
+    except Exception:
+        pass
+    try:
+        raw = bytes.fromhex(data.decode())
+        if len(raw) == 32:
+            return raw
+    except Exception:
+        pass
+    # derive a stable key from arbitrary passphrase material (HKDF-lite)
+    import hashlib
+    return hashlib.blake2b(data, digest_size=32).digest()
+
+
+def generate_master_key(path: str) -> bytes:
+    """Generate a new master key and persist it 0600 (operator flow)."""
+    key = secrets.token_bytes(32)
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "wb") as fh:
+        fh.write(_b64e(key).encode())
+    return key

cortexm/security/hashes.py

@@ -0,0 +1,140 @@
+"""Cryptographic primitives: BLAKE3 hashing, Merkle trees, attestations.
+
+Spec requirement (Section 1.1 / InjecMEM defense): every fact carries a
+BLAKE3 hash of its source text; on retrieval the hash is re-verified, and
+Memory-Git commits form a tamper-evident hash chain. BLAKE2b-256 is the
+automatic fallback when the optional ``blake3`` wheel is absent — the
+provider name is always reported so audits stay honest.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import logging
+
+try:  # optional dependency, preferred per spec
+    import blake3 as _blake3
+
+    _HAS_BLAKE3 = True
+except Exception:  # pragma: no cover - environment dependent
+    _blake3 = None
+    _HAS_BLAKE3 = False
+
+_log = logging.getLogger("cortexm.security")
+_DOWNGRADE_WARNED = False
+
+
+def has_blake3() -> bool:
+    return _HAS_BLAKE3
+
+
+def _warn_downgrade() -> None:
+    """Loud, once-per-process warning when BLAKE3 is unavailable.
+
+    Silent capability downgrade is a credibility bug: the docs promise
+    BLAKE3 and the user must know when they are actually getting
+    BLAKE2b. Install with ``pip install context-m[blake3]`` to silence.
+    """
+    global _DOWNGRADE_WARNED
+    if not _DOWNGRADE_WARNED:
+        _DOWNGRADE_WARNED = True
+        _log.warning(
+            "blake3 wheel not installed — downgrading hash provider to "
+            "BLAKE2b-256. Integrity guarantees hold (collision resistance "
+            "is equivalent at 256-bit) but this differs from the documented "
+            "default. Fix: pip install 'context-m[blake3]'. "
+            "The active provider is always reported in stats() and audit "
+            "output as hash_provider."
+        )
+
+
+class HashProvider:
+    """BLAKE3-256 (preferred) or BLAKE2b-256 (fallback). Same interface."""
+
+    def __init__(self, algo: str = "blake3") -> None:
+        if algo == "blake3" and not _HAS_BLAKE3:
+            _warn_downgrade()
+            algo = "blake2b"
+        if algo not in ("blake3", "blake2b"):
+            raise ValueError(f"unsupported hash provider {algo!r}")
+        self.algo = algo
+
+    @property
+    def name(self) -> str:
+        return "blake3-256" if self.algo == "blake3" else "blake2b-256"
+
+    def hash_bytes(self, data: bytes) -> str:
+        if self.algo == "blake3":
+            return _blake3.blake3(data).hexdigest()
+        return hashlib.blake2b(data, digest_size=32).hexdigest()
+
+    def hash_text(self, text: str) -> str:
+        return self.hash_bytes(text.encode("utf-8"))
+
+    def hash_json(self, obj) -> str:
+        payload = json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str)
+        return self.hash_text(payload)
+
+    def short(self, hexdigest: str, n: int = 8) -> str:
+        return hexdigest[:n]
+
+
+# --------------------------------------------------------------------------
+# Merkle trees (binary, duplicate-last-when-odd) — used by ZK-lite proofs.
+# --------------------------------------------------------------------------
+
+def _h_pair(provider: HashProvider, a: str, b: str) -> str:
+    if len(a) == 64 and len(b) == 64:
+        try:
+            return provider.hash_bytes(bytes.fromhex(a) + bytes.fromhex(b))
+        except ValueError:
+            pass
+    return provider.hash_text(a + b)
+
+
+def merkle_root(provider: HashProvider, leaves: list[str]) -> str:
+    if not leaves:
+        return provider.hash_text("empty")
+    level = list(leaves)
+    while len(level) > 1:
+        if len(level) % 2:
+            level.append(level[-1])
+        level = [_h_pair(provider, level[i], level[i + 1]) for i in range(0, len(level), 2)]
+    return level[0]
+
+
+def merkle_proof(provider: HashProvider, leaves: list[str], index: int) -> tuple[str, list[dict]]:
+    """Return (root, path) where path = [{side, hash}, ...] for leaves[index]."""
+    if not leaves or not (0 <= index < len(leaves)):
+        raise IndexError("index out of range")
+    level = list(leaves)
+    idx = index
+    path: list[dict] = []
+    while len(level) > 1:
+        if len(level) % 2:
+            level.append(level[-1])
+        sib = idx ^ 1
+        path.append({"side": "right" if sib > idx else "left", "hash": level[sib]})
+        level = [_h_pair(provider, level[i], level[i + 1]) for i in range(0, len(level), 2)]
+        idx //= 2
+    return level[0], path
+
+
+def merkle_verify(provider: HashProvider, leaf: str, path: list[dict], root: str) -> bool:
+    cur = leaf
+    for step in path:
+        if step["side"] == "right":
+            cur = _h_pair(provider, cur, step["hash"])
+        else:
+            cur = _h_pair(provider, step["hash"], cur)
+    return hmac.compare_digest(cur, root)
+
+
+def attest(provider: HashProvider, key: bytes, message: str) -> str:
+    return hmac.new(key, message.encode("utf-8"), hashlib.sha256).hexdigest()
+
+
+def verify_attest(provider: HashProvider, key: bytes, message: str, tag: str) -> bool:
+    return hmac.compare_digest(attest(provider, key, message), tag)