cortexm 0.3.0__py3-none-any.whl

cortexm/metrics.py

@@ -0,0 +1,62 @@
+"""Process-wide counters backing the μ=0 (zero-LLM-ingest) protocol.
+
+Every code path that could invoke a language model must route through
+``call_llm``. The default implementation raises: the deterministic core
+physically cannot make an LLM call. Optional async enrichment (never used
+during benchmarks) increments the counter so audits can prove honesty.
+"""
+
+from __future__ import annotations
+
+import threading
+
+_LOCK = threading.Lock()
+_LLM_CALLS = 0
+_INGESTED_TOKENS = 0
+_INGESTED_MESSAGES = 0
+_EXTRACTED_FACTS = 0
+_RETRIEVALS = 0
+
+
+def llm_calls() -> int:
+    return _LLM_CALLS
+
+
+def bump_llm_call() -> None:
+    global _LLM_CALLS
+    with _LOCK:
+        _LLM_CALLS += 1
+
+
+def bump_ingest(tokens: int = 0, messages: int = 0, facts: int = 0) -> None:
+    global _INGESTED_TOKENS, _INGESTED_MESSAGES, _EXTRACTED_FACTS
+    with _LOCK:
+        _INGESTED_TOKENS += tokens
+        _INGESTED_MESSAGES += messages
+        _EXTRACTED_FACTS += facts
+
+
+def bump_retrieval() -> None:
+    global _RETRIEVALS
+    with _LOCK:
+        _RETRIEVALS += 1
+
+
+def counters() -> dict:
+    return {
+        "llm_calls": _LLM_CALLS,                # must be 0 under μ=0 protocol
+        "ingested_tokens": _INGESTED_TOKENS,
+        "ingested_messages": _INGESTED_MESSAGES,
+        "extracted_facts": _EXTRACTED_FACTS,
+        "retrievals": _RETRIEVALS,
+    }
+
+
+def reset_counters() -> None:
+    global _LLM_CALLS, _INGESTED_TOKENS, _INGESTED_MESSAGES, _EXTRACTED_FACTS, _RETRIEVALS
+    with _LOCK:
+        _LLM_CALLS = 0
+        _INGESTED_TOKENS = 0
+        _INGESTED_MESSAGES = 0
+        _EXTRACTED_FACTS = 0
+        _RETRIEVALS = 0

cortexm/migrate/importers.py

@@ -0,0 +1,192 @@
+"""Migration importers — Mem0 / Zep / Chroma → Context-M.
+
+The plan's Trojan-Horse migration tooling: read a competitor's local
+store, convert to Trace triples (μ=0 re-extraction on raw text where
+possible — timestamps preserved), and keep the original ids in
+provenance for audit.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from datetime import datetime, timezone
+
+from cortexm.errors import MigrationError
+from cortexm.util import parse_ts
+
+
+def _iter_mem0_rows(path: str):
+    """Yield (text, created_at) from a Mem0 SQLite store.
+
+    Mem0's local store keeps a ``history`` table with JSON payloads
+    containing the original conversation plus extracted memories.
+    """
+    if not os.path.exists(path):
+        raise MigrationError(f"mem0 store not found: {path}")
+    conn = sqlite3.connect(path)
+    conn.row_factory = sqlite3.Row
+    try:
+        tables = {r[0] for r in conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'")}
+        if "history" in tables:
+            for row in conn.execute(
+                    "SELECT data, created_at, updated_at FROM history "
+                    "ORDER BY id"):
+                raw = row["data"] or ""
+                created = row["created_at"] or row["updated_at"]
+                try:
+                    payload = json.loads(raw)
+                except (json.JSONDecodeError, TypeError):
+                    payload = None
+                texts: list[str] = []
+                if isinstance(payload, dict):
+                    for msg in payload.get("messages", []):
+                        if isinstance(msg, dict) and msg.get("content"):
+                            texts.append(str(msg["content"]))
+                    for mem in payload.get("memories", []):
+                        if isinstance(mem, str):
+                            texts.append(mem)
+                elif isinstance(raw, str) and raw.strip():
+                    texts.append(raw.strip())
+                for t in texts:
+                    if t and len(t) > 2:
+                        yield t, created
+        elif "memories" in tables:
+            cols = {r[1] for r in conn.execute("PRAGMA table_info(memories)")}
+            text_col = "data" if "data" in cols else (
+                "memory" if "memory" in cols else "text")
+            for row in conn.execute(f"SELECT {text_col} AS t, created_at "
+                                    f"FROM memories ORDER BY rowid"):
+                if row["t"]:
+                    yield str(row["t"]), row["created_at"]
+        else:
+            raise MigrationError(
+                f"unrecognized mem0 schema (tables: {sorted(tables)})")
+    finally:
+        conn.close()
+
+
+def _iter_zep_rows(path: str):
+    """Yield from a Zep JSONL export: {"subject","relation","object",
+    "valid_at","invalid_at"} or {"text","created_at"} rows."""
+    if not os.path.exists(path):
+        raise MigrationError(f"zep export not found: {path}")
+    with open(path, encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            yield row
+
+
+def _iter_chroma_rows(path: str):
+    """Yield documents from a chroma.sqlite3 store."""
+    if not os.path.exists(path):
+        raise MigrationError(f"chroma store not found: {path}")
+    conn = sqlite3.connect(path)
+    conn.row_factory = sqlite3.Row
+    try:
+        tables = {r[0] for r in conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'")}
+        if "embeddings" in tables:
+            cols = {r[1] for r in conn.execute(
+                "PRAGMA table_info(embeddings)")}
+            doc_col = None
+            for c in ("document", "documents", "content", "data"):
+                if c in cols:
+                    doc_col = c
+                    break
+            if doc_col:
+                for row in conn.execute(
+                        f"SELECT {doc_col} AS doc FROM embeddings"):
+                    if row["doc"]:
+                        yield str(row["doc"]), None
+                return
+        if "documents" in tables:
+            for row in conn.execute("SELECT * FROM documents"):
+                for v in row:
+                    if v:
+                        yield str(v), None
+                return
+        raise MigrationError(f"unrecognized chroma schema: {sorted(tables)}")
+    finally:
+        conn.close()
+
+
+# ------------------------------------------------------------------
+def import_mem0(memory, path: str, user_id: str = "migrated",
+                batch: int = 200) -> dict:
+    """Import a Mem0 store into Context-M (μ=0 re-extraction)."""
+    n_msgs, n_facts = 0, 0
+    buf, last_ts = [], None
+    for text, created in _iter_mem0_rows(path):
+        buf.append(text)
+        last_ts = created
+        if len(buf) >= batch:
+            out = memory.add(buf, user_id=user_id,
+                             timestamp=parse_ts(last_ts) or
+                             datetime.now(timezone.utc))
+            n_msgs += len(buf)
+            n_facts += out["stats"]["facts_inserted"]
+            buf = []
+    if buf:
+        out = memory.add(buf, user_id=user_id,
+                         timestamp=parse_ts(last_ts) or
+                         datetime.now(timezone.utc))
+        n_msgs += len(buf)
+        n_facts += out["stats"]["facts_inserted"]
+    return {"source": "mem0", "messages": n_msgs, "facts": n_facts}
+
+
+def import_zep(memory, path: str, user_id: str = "migrated") -> dict:
+    """Import a Zep JSONL export. Graph triples keep bi-temporal windows;
+    raw text rows are re-extracted (μ=0)."""
+    from cortexm.trace.fact import make_fact
+    n_triples, n_texts = 0, 0
+    commit = memory.store.create_commit("migrate: zep")
+    for row in _iter_zep_rows(path):
+        if all(k in row for k in ("subject", "relation", "object")):
+            f = make_fact(row["subject"], row["relation"], row["object"],
+                          now=parse_ts(row.get("valid_at")) or
+                          datetime.now(timezone.utc),
+                          valid_to=row.get("invalid_at"),
+                          user_id=user_id, confidence=0.8,
+                          provenance={"migrated_from": "zep"})
+            memory.store.insert_fact(f, commit)
+            memory.palace.add(f.id, memory.palace.encode_fact(f))
+            n_triples += 1
+        elif row.get("text"):
+            memory.add(row["text"], user_id=user_id)
+            n_texts += 1
+    memory.store.end_batch()
+    memory.reader.invalidate_caches()
+    return {"source": "zep", "triples": n_triples, "texts": n_texts}
+
+
+def import_chroma(memory, path: str, user_id: str = "migrated",
+                  batch: int = 200) -> dict:
+    """Import documents from a Chroma store (μ=0 re-extraction; foreign
+    embeddings are NOT portable across embedding models)."""
+    n_msgs, n_facts = 0, 0
+    buf = []
+    for doc, _ts in _iter_chroma_rows(path):
+        buf.append(doc)
+        if len(buf) >= batch:
+            out = memory.add(buf, user_id=user_id)
+            n_msgs += len(buf)
+            n_facts += out["stats"]["facts_inserted"]
+            buf = []
+    if buf:
+        out = memory.add(buf, user_id=user_id)
+        n_msgs += len(buf)
+        n_facts += out["stats"]["facts_inserted"]
+    return {"source": "chroma", "documents": n_msgs, "facts": n_facts}
+
+
+MIGRATORS = {"mem0": import_mem0, "zep": import_zep, "chroma": import_chroma}

cortexm/provenance/__init__.py

@@ -0,0 +1,78 @@
+"""Provenance Standards Stack — enterprise-grade provenance wrappers.
+
+Context-M's internal integrity is BLAKE3 source hashing + a hash-chained
+audit log. This is cryptographically sound but NOT standards-compliant.
+Enterprise security reviews ask: "Does it support W3C VC? C2PA? SCITT?"
+— this layer answers "yes" by wrapping the existing BLAKE3 hashes into
+standards-compliant envelopes.
+
+Three sub-modules:
+
+  cose.py   — COSE Sign1 envelopes (RFC 9052) using Ed25519. Every
+              memory commit gets wrapped in a COSE Sign1 with the
+              agent's Ed25519 key, so external verifiers can confirm
+              the commit came from a known agent without trusting the
+              host's audit log.
+
+  vc.py     — W3C Verifiable Credentials 2.0 export. A memory range
+              (e.g. "all facts about user=alice from 2026-01-01 to
+              2026-02-01") can be exported as a W3C VC Data Integrity
+              proof (eddsa-jcs-2022). The VC contains the BLAKE3 root
+              of the range + the COSE Sign1 envelope that proves the
+              agent signed it.
+
+  scitt.py  — SCITT-signed statements. Forwards a COSE Sign1 to a
+              (mocked) SCITT transparency log, returns a receipt that
+              can be verified by a third party.
+
+Design: BLAKE3 source hashing stays as the internal integrity mechanism.
+This layer SITS ON TOP for external interoperability. No replacement —
+just a wrapper. Existing `verify_integrity()` stays; the standards
+layer adds `verify_with_standards()` that calls both.
+
+Pure Python — no halo2, no BBS+, no JWT libs. Ed25519 uses the
+Python stdlib `hashlib` for hashing and a tiny ed25519 implementation
+fallback if `cryptography` is not available. The COSE/VC/SCITT wire
+formats are JSON (CBOR is preferred per RFC 9052, but JSON works for
+the prototype).
+"""
+
+from cortexm.provenance.cose import (
+    CoseSign1Envelope,
+    sign_commit,
+    verify_commit,
+)
+from cortexm.provenance.vc import (
+    VerifiableCredential,
+    export_memory_range_vc,
+    verify_vc,
+)
+from cortexm.provenance.scitt import (
+    ScittStatement,
+    ScittReceipt,
+    submit_to_scitt,
+    verify_receipt,
+    reset_scitt_log,
+    get_scitt_service_did,
+)
+from cortexm.provenance.agent import (
+    Ed25519AgentKey,
+    get_default_agent,
+    set_default_agent,
+)
+
+__all__ = [
+    # cose
+    "CoseSign1Envelope", "Ed25519AgentKey",
+    "sign_commit", "verify_commit",
+    # vc
+    "VerifiableCredential",
+    "export_memory_range_vc", "verify_vc",
+    # scitt
+    "ScittStatement", "ScittReceipt",
+    "submit_to_scitt", "verify_receipt",
+    "reset_scitt_log", "get_scitt_service_did",
+    # agent
+    "Ed25519AgentKey",
+    "get_default_agent", "set_default_agent",
+]

cortexm/provenance/agent.py

@@ -0,0 +1,214 @@
+"""ProvenanceAgent — Ed25519 identity for memory commits.
+
+Each agent (LLM instance, microservice, scheduled job) that writes to
+the Trace has an Ed25519 signing key. The agent's public key is
+exposed as a `did:key` identifier so external verifiers can resolve
+the key from the DID without an external registry.
+
+The agent key is persisted to a tiny PEM-ish file (path from
+Config.provenance_agent_key_path). If no path is set, the agent uses
+a process-local ephemeral key (regenerated on restart — fine for
+tests, not for production cross-system verification).
+
+Uses the `cryptography` library for Ed25519 (constant-time, audited).
+If `cryptography` is unavailable, falls back to an HMAC-SHA256 pseudo-
+signature clearly labeled "DEV-MODE" — NOT for production.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import os
+from dataclasses import dataclass
+
+
+@dataclass
+class Ed25519AgentKey:
+    """An Ed25519 signing keypair for a provenance agent."""
+    private_key: bytes      # 32-byte seed (Ed25519 form)
+    public_key: bytes       # 32-byte pubkey
+    did: str = ""           # did:key identifier — derived from pubkey
+    label: str = "context-m-agent"
+    dev_mode: bool = False  # True when using HMAC fallback (no cryptography lib)
+
+    @classmethod
+    def generate(cls, label: str = "context-m-agent") -> "Ed25519AgentKey":
+        """Generate a fresh Ed25519 keypair."""
+        try:
+            from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+                Ed25519PrivateKey)
+            from cryptography.hazmat.primitives.serialization import (
+                Encoding, PrivateFormat, NoEncryption, PublicFormat)
+            priv = Ed25519PrivateKey.generate()
+            priv_bytes = priv.private_bytes(
+                encoding=Encoding.Raw,
+                format=PrivateFormat.Raw,
+                encryption_algorithm=NoEncryption(),
+            )
+            pub_bytes = priv.public_key().public_bytes(
+                encoding=Encoding.Raw,
+                format=PublicFormat.Raw,
+            )
+            did = _did_key_from_pubkey(pub_bytes)
+            return cls(
+                private_key=priv_bytes,
+                public_key=pub_bytes,
+                did=did,
+                label=label,
+            )
+        except ImportError:
+            # DEV fallback: HMAC-SHA256 — NOT for production.
+            seed = os.urandom(32)
+            return cls(
+                private_key=seed,
+                public_key=seed,  # symmetric — fallback only
+                did=f"did:dev:{hashlib.sha256(seed).hexdigest()[:32]}",
+                label=label,
+                dev_mode=True,
+            )
+
+    @classmethod
+    def from_pem(cls, pem_path: str,
+                 label: str = "context-m-agent") -> "Ed25519AgentKey":
+        """Load a keypair from a PEM-ish file. Generates one if missing."""
+        if not os.path.exists(pem_path):
+            key = cls.generate(label=label)
+            key.save_pem(pem_path)
+            return key
+        try:
+            with open(pem_path, "rb") as f:
+                text = f.read().decode("utf-8")
+            lines = [l.strip() for l in text.splitlines()
+                     if l and not l.startswith("-")]
+            priv = base64.b64decode(lines[0])
+            pub = base64.b64decode(lines[1]) if len(lines) > 1 else b""
+            if len(priv) != 32:
+                raise ValueError("private key must be 32 bytes")
+            if not pub or len(pub) != 32:
+                # regenerate pub from priv
+                try:
+                    from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+                        Ed25519PrivateKey)
+                    from cryptography.hazmat.primitives.serialization import (
+                        Encoding, PublicFormat)
+                    pub = (Ed25519PrivateKey.from_private_bytes(priv)
+                           .public_key().public_bytes(
+                               encoding=Encoding.Raw,
+                               format=PublicFormat.Raw))
+                except ImportError:
+                    pub = priv
+            did = _did_key_from_pubkey(pub) if len(pub) == 32 else f"did:dev:{hashlib.sha256(priv).hexdigest()[:32]}"
+            return cls(
+                private_key=priv, public_key=pub, did=did, label=label,
+                dev_mode=(len(pub) != 32))
+        except Exception:
+            key = cls.generate(label=label)
+            key.save_pem(pem_path)
+            return key
+
+    def save_pem(self, pem_path: str) -> None:
+        """Persist the keypair to a tiny PEM-ish file."""
+        os.makedirs(os.path.dirname(pem_path) or ".", exist_ok=True)
+        text = (
+            f"-----BEGIN ED25519 PRIVATE KEY-----\n"
+            f"{base64.b64encode(self.private_key).decode()}\n"
+            f"-----END ED25519 PRIVATE KEY-----\n"
+            f"-----BEGIN ED25519 PUBLIC KEY-----\n"
+            f"{base64.b64encode(self.public_key).decode()}\n"
+            f"-----END ED25519 PUBLIC KEY-----\n"
+            f"LABEL: {self.label}\n"
+        )
+        with open(pem_path, "w") as f:
+            f.write(text)
+        try:
+            os.chmod(pem_path, 0o600)
+        except Exception:
+            pass
+
+    def sign(self, message: bytes) -> bytes:
+        """Sign a message with the agent's private key."""
+        if self.dev_mode:
+            # DEV fallback — HMAC-SHA256, clearly labeled
+            mac = hmac.new(self.private_key, message, hashlib.sha256).digest()
+            return b"DEV-MODE:" + mac
+        try:
+            from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+                Ed25519PrivateKey)
+            priv = Ed25519PrivateKey.from_private_bytes(self.private_key)
+            return priv.sign(message)
+        except Exception:
+            # crypto lib missing at sign time — fall back
+            mac = hmac.new(self.private_key, message, hashlib.sha256).digest()
+            return b"DEV-MODE:" + mac
+
+    def verify(self, message: bytes, signature: bytes) -> bool:
+        """Verify a signature with the agent's public key."""
+        if signature.startswith(b"DEV-MODE:"):
+            # DEV fallback verify
+            expected = hmac.new(self.private_key, message,
+                                 hashlib.sha256).digest()
+            actual = signature[len(b"DEV-MODE:"):]
+            return hmac.compare_digest(expected, actual)
+        try:
+            from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+                Ed25519PublicKey)
+            pub = Ed25519PublicKey.from_public_bytes(self.public_key)
+            try:
+                pub.verify(signature, message)
+                return True
+            except Exception:
+                return False
+        except ImportError:
+            return False
+
+
+def _did_key_from_pubkey(pubkey: bytes) -> str:
+    """Encode a 32-byte Ed25519 pubkey as a did:key identifier.
+
+    Format: did:key:z<base58btc-multibase-prefix><pubkey>
+    The multicodec prefix for Ed25519 is 0xed01.
+    """
+    prefixed = b"\xed\x01" + pubkey
+    encoded = _base58btc_encode(prefixed)
+    return f"did:key:z{encoded}"
+
+
+def _base58btc_encode(data: bytes) -> str:
+    """Encode bytes in base58 with the Bitcoin alphabet."""
+    alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+    num = int.from_bytes(data, "big")
+    encoded = ""
+    while num > 0:
+        num, rem = divmod(num, 58)
+        encoded = alphabet[rem] + encoded
+    n_zeros = 0
+    for b in data:
+        if b == 0:
+            n_zeros += 1
+        else:
+            break
+    return "1" * n_zeros + encoded
+
+
+# ---------- default agent registry ------------------------------------
+
+_default_agent = None
+
+
+def get_default_agent() -> Ed25519AgentKey:
+    """Get the process-default agent key. Generates one on first call."""
+    global _default_agent
+    if _default_agent is None:
+        _default_agent = Ed25519AgentKey.generate()
+    return _default_agent
+
+
+def set_default_agent(agent: Ed25519AgentKey) -> None:
+    """Set the process-default agent key (e.g. loaded from PEM)."""
+    global _default_agent
+    _default_agent = agent
+
+
+__all__ = ["Ed25519AgentKey", "get_default_agent", "set_default_agent"]