agmem 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

memvcs/core/crypto_verify.py

@@ -0,0 +1,291 @@
+"""
+Cryptographic commit verification for agmem.
+
+Merkle tree over commit blobs, optional Ed25519 signing of Merkle root.
+Verification on checkout, pull, and via verify/fsck.
+"""
+
+import hashlib
+import hmac
+import json
+import os
+from pathlib import Path
+from typing import Optional, List, Tuple, Any, Dict
+
+from .objects import ObjectStore, Tree, Commit, Blob
+
+# Ed25519 via cryptography (optional)
+try:
+    from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+        Ed25519PrivateKey,
+        Ed25519PublicKey,
+    )
+    from cryptography.exceptions import InvalidSignature
+    from cryptography.hazmat.primitives import serialization
+
+    ED25519_AVAILABLE = True
+except ImportError:
+    ED25519_AVAILABLE = False
+
+
+def _collect_blob_hashes_from_tree(store: ObjectStore, tree_hash: str) -> List[str]:
+    """Recursively collect all blob hashes from a tree. Returns sorted list for deterministic Merkle."""
+    tree = Tree.load(store, tree_hash)
+    if not tree:
+        return []
+    blobs: List[str] = []
+    for entry in tree.entries:
+        if entry.obj_type == "blob":
+            blobs.append(entry.hash)
+        elif entry.obj_type == "tree":
+            blobs.extend(_collect_blob_hashes_from_tree(store, entry.hash))
+    return sorted(blobs)
+
+
+def _merkle_hash(data: bytes) -> str:
+    """SHA-256 hash for Merkle tree nodes."""
+    return hashlib.sha256(data).hexdigest()
+
+
+def build_merkle_tree(blob_hashes: List[str]) -> str:
+    """
+    Build balanced binary Merkle tree from blob hashes.
+    Leaves are hashes of blob hashes (as hex strings); internal nodes hash(left_hex || right_hex).
+    Returns root hash (hex).
+    """
+    if not blob_hashes:
+        return _merkle_hash(b"empty")
+    # Leaves: hash each blob hash string to fixed-size leaf
+    layer = [_merkle_hash(h.encode()) for h in blob_hashes]
+    while len(layer) > 1:
+        next_layer = []
+        for i in range(0, len(layer), 2):
+            left = layer[i]
+            right = layer[i + 1] if i + 1 < len(layer) else layer[i]
+            combined = (left + right).encode()
+            next_layer.append(_merkle_hash(combined))
+        layer = next_layer
+    return layer[0]
+
+
+def build_merkle_root_for_commit(store: ObjectStore, commit_hash: str) -> Optional[str]:
+    """Build Merkle root for a commit's tree. Returns None if commit/tree missing."""
+    commit = Commit.load(store, commit_hash)
+    if not commit:
+        return None
+    blobs = _collect_blob_hashes_from_tree(store, commit.tree)
+    return build_merkle_tree(blobs)
+
+
+def merkle_proof(blob_hashes: List[str], target_blob_hash: str) -> Optional[List[Tuple[str, str]]]:
+    """
+    Produce Merkle proof for a blob: list of (sibling_hash, "L"|"R") from leaf to root.
+    Returns None if target not in list.
+    """
+    if target_blob_hash not in blob_hashes:
+        return None
+    layer = [_merkle_hash(h.encode()) for h in sorted(blob_hashes)]
+    leaf_index = sorted(blob_hashes).index(target_blob_hash)
+    proof: List[Tuple[str, str]] = []
+    idx = leaf_index
+    while len(layer) > 1:
+        next_layer = []
+        for i in range(0, len(layer), 2):
+            left = layer[i]
+            right = layer[i + 1] if i + 1 < len(layer) else layer[i]
+            combined = (left + right).encode()
+            parent = _merkle_hash(combined)
+            next_layer.append(parent)
+            # If current idx is in this pair, record sibling and advance index
+            pair_idx = i // 2
+            if idx == i:
+                proof.append((right, "R"))
+                idx = pair_idx
+            elif idx == i + 1:
+                proof.append((left, "L"))
+                idx = pair_idx
+        layer = next_layer
+    return proof if proof else []
+
+
+def verify_merkle_proof(blob_hash: str, proof: List[Tuple[str, str]], expected_root: str) -> bool:
+    """Verify a Merkle proof for a blob against expected root."""
+    current = _merkle_hash(blob_hash.encode())
+    for sibling, side in proof:
+        if side == "L":
+            current = _merkle_hash((sibling + current).encode())
+        else:
+            current = _merkle_hash((current + sibling).encode())
+    return current == expected_root
+
+
+# --- Signing (Ed25519) ---
+
+
+def _keys_dir(mem_dir: Path) -> Path:
+    return mem_dir / "keys"
+
+
+def get_signing_key_paths(mem_dir: Path) -> Tuple[Path, Path]:
+    """Return (private_key_path, public_key_path). Private may not exist (env-only)."""
+    kd = _keys_dir(mem_dir)
+    return (kd / "private.pem", kd / "public.pem")
+
+
+def ensure_keys_dir(mem_dir: Path) -> Path:
+    """Ensure .mem/keys exists; return keys dir."""
+    kd = _keys_dir(mem_dir)
+    kd.mkdir(parents=True, exist_ok=True)
+    return kd
+
+
+def generate_keypair(mem_dir: Path) -> Tuple[bytes, bytes]:
+    """Generate Ed25519 keypair. Returns (private_pem, public_pem). Requires cryptography."""
+    if not ED25519_AVAILABLE:
+        raise RuntimeError(
+            "Signing requires 'cryptography'; install with: pip install cryptography"
+        )
+    private_key = Ed25519PrivateKey.generate()
+    public_key = private_key.public_key()
+    private_pem = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.PKCS8,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    public_pem = public_key.public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
+    )
+    return (private_pem, public_pem)
+
+
+def save_public_key(mem_dir: Path, public_pem: bytes) -> Path:
+    """Save public key to .mem/keys/public.pem. Returns path."""
+    ensure_keys_dir(mem_dir)
+    path = _keys_dir(mem_dir) / "public.pem"
+    path.write_bytes(public_pem)
+    return path
+
+
+def load_public_key(mem_dir: Path) -> Optional[bytes]:
+    """Load public key PEM from .mem/keys/public.pem or config. Returns None if not found."""
+    path = _keys_dir(mem_dir) / "public.pem"
+    if path.exists():
+        return path.read_bytes()
+    config_file = mem_dir / "config.json"
+    if config_file.exists():
+        try:
+            config = json.loads(config_file.read_text())
+            return config.get("signing", {}).get("public_key_pem")
+        except Exception:
+            pass
+    return None
+
+
+def load_private_key_from_env() -> Optional[bytes]:
+    """Load private key PEM from env AGMEM_SIGNING_PRIVATE_KEY (or path in AGMEM_SIGNING_PRIVATE_KEY_FILE)."""
+    pem = os.environ.get("AGMEM_SIGNING_PRIVATE_KEY")
+    if pem:
+        return pem.encode() if isinstance(pem, str) else pem
+    path = os.environ.get("AGMEM_SIGNING_PRIVATE_KEY_FILE")
+    if path and os.path.isfile(path):
+        return Path(path).read_bytes()
+    return None
+
+
+def sign_merkle_root(root_hex: str, private_key_pem: bytes) -> str:
+    """Sign Merkle root (hex string). Returns signature as hex."""
+    if not ED25519_AVAILABLE:
+        raise RuntimeError("Signing requires 'cryptography'")
+    key = serialization.load_pem_private_key(private_key_pem, password=None)
+    if not isinstance(key, Ed25519PrivateKey):
+        raise TypeError("Ed25519 private key required")
+    sig = key.sign(root_hex.encode())
+    return sig.hex()
+
+
+def verify_signature(root_hex: str, signature_hex: str, public_key_pem: bytes) -> bool:
+    """Verify signature of Merkle root. Returns True if valid."""
+    if not ED25519_AVAILABLE:
+        return False
+    try:
+        key = serialization.load_pem_public_key(public_key_pem)
+        if not isinstance(key, Ed25519PublicKey):
+            return False
+        key.verify(bytes.fromhex(signature_hex), root_hex.encode())
+        return True
+    except InvalidSignature:
+        return False
+    except Exception:
+        return False
+
+
+def verify_commit(
+    store: ObjectStore,
+    commit_hash: str,
+    public_key_pem: Optional[bytes] = None,
+    *,
+    mem_dir: Optional[Path] = None,
+) -> Tuple[bool, Optional[str]]:
+    """
+    Verify commit: rebuild Merkle tree from blobs, compare root to stored, verify signature.
+    Returns (verified, error_message). verified=True means OK; False + message means tampered or unverified.
+    If public_key_pem is None and mem_dir is set, load from mem_dir.
+    """
+    commit = Commit.load(store, commit_hash)
+    if not commit:
+        return (False, "commit not found")
+    stored_root = (commit.metadata or {}).get("merkle_root")
+    stored_sig = (commit.metadata or {}).get("signature")
+    if not stored_root:
+        return (False, "commit has no merkle_root (unverified)")
+    
+    # Verify that blob objects can be loaded successfully (detects tampering in compressed/encrypted content)
+    blob_hashes = _collect_blob_hashes_from_tree(store, commit.tree)
+    for blob_hash in blob_hashes:
+        try:
+            blob = Blob.load(store, blob_hash)
+            if blob is None:
+                return (False, f"blob {blob_hash[:8]} corrupted or missing")
+        except Exception as e:
+            return (False, f"merkle_root mismatch (commit tampered)")
+    
+    computed_root = build_merkle_root_for_commit(store, commit_hash)
+    if not computed_root:
+        return (False, "could not build Merkle tree (missing tree/blobs)")
+    if not hmac.compare_digest(computed_root, stored_root):
+        return (False, "merkle_root mismatch (commit tampered)")
+    if not stored_sig:
+        return (True, None)  # Root matches; no signature (legacy)
+    pub = public_key_pem
+    if not pub and mem_dir:
+        pub = load_public_key(mem_dir)
+    if not pub:
+        return (False, "signature present but no public key configured")
+    if isinstance(pub, str):
+        pub = pub.encode()
+    if not verify_signature(stored_root, stored_sig, pub):
+        return (False, "signature verification failed")
+    return (True, None)
+
+
+def verify_commit_optional(
+    store: ObjectStore,
+    commit_hash: str,
+    mem_dir: Optional[Path] = None,
+    *,
+    strict: bool = False,
+) -> None:
+    """
+    Verify commit; if strict=True raise on failure. If strict=False, only raise on tamper (root mismatch).
+    Unverified (no merkle_root) is OK when not strict.
+    """
+    ok, err = verify_commit(store, commit_hash, None, mem_dir=mem_dir)
+    if ok:
+        return
+    if not err:
+        return
+    if "tampered" in err or "mismatch" in err or "signature verification failed" in err:
+        raise ValueError(f"Commit verification failed: {err}")
+    if strict:
+        raise ValueError(f"Commit verification failed: {err}")

memvcs/core/distiller.py

@@ -35,6 +35,9 @@ class DistillerConfig:
     llm_provider: Optional[str] = None
     llm_model: Optional[str] = None
     create_safety_branch: bool = True
+    use_dp: bool = False
+    dp_epsilon: Optional[float] = None
+    dp_delta: Optional[float] = None
 
 
 @dataclass
@@ -110,9 +113,32 @@ class Distiller:
                 continue
         combined = "\n---\n".join(contents)
 
-        if self.config.llm_provider == "openai" and self.config.llm_model:
+        if self.config.llm_provider and self.config.llm_model:
             try:
-                return self._extract_with_openai(combined, cluster.topic)
+                from .llm import get_provider
+
+                config = {
+                    "llm_provider": self.config.llm_provider,
+                    "llm_model": self.config.llm_model,
+                }
+                provider = get_provider(config=config)
+                if provider:
+                    text = provider.complete(
+                        [
+                            {
+                                "role": "system",
+                                "content": "Extract factual statements from the text. Output as bullet points (one fact per line). Focus on: user preferences, learned facts, key decisions.",
+                            },
+                            {
+                                "role": "user",
+                                "content": f"Topic: {cluster.topic}\n\n{combined[:4000]}",
+                            },
+                        ],
+                        max_tokens=500,
+                    )
+                    return [
+                        line.strip() for line in text.splitlines() if line.strip().startswith("-")
+                    ][:15]
             except Exception:
                 pass
 
@@ -125,29 +151,6 @@ class Distiller:
                     facts.append(f"- {line[:200]}")
         return facts[:10] if facts else [f"- Learned about {cluster.topic}"]
 
-    def _extract_with_openai(self, content: str, topic: str) -> List[str]:
-        """Extract facts using OpenAI API."""
-        import openai
-
-        response = openai.chat.completions.create(
-            model=self.config.llm_model or "gpt-3.5-turbo",
-            messages=[
-                {
-                    "role": "system",
-                    "content": "Extract factual statements from the text. "
-                    "Output as bullet points (one fact per line). "
-                    "Focus on: user preferences, learned facts, key decisions.",
-                },
-                {
-                    "role": "user",
-                    "content": f"Topic: {topic}\n\n{content[:4000]}",
-                },
-            ],
-            max_tokens=500,
-        )
-        text = response.choices[0].message.content
-        return [line.strip() for line in text.splitlines() if line.strip().startswith("-")][:15]
-
     def write_consolidated(self, cluster: EpisodeCluster, facts: List[str]) -> Path:
         """Write consolidated semantic file."""
         self.target_dir.mkdir(parents=True, exist_ok=True)
@@ -160,13 +163,18 @@ class Distiller:
         except ValueError:
             out_path = self.target_dir / f"consolidated-{ts}.md"
 
+        confidence_score = self.config.extraction_confidence_threshold
+        if self.config.use_dp and self.config.dp_epsilon is not None and self.config.dp_delta is not None:
+            from .privacy_budget import add_noise
+            confidence_score = add_noise(confidence_score, 0.1, self.config.dp_epsilon, self.config.dp_delta)
+            confidence_score = max(0.0, min(1.0, confidence_score))
         frontmatter = {
             "schema_version": "1.0",
             "last_updated": datetime.utcnow().isoformat() + "Z",
             "source_agent_id": "distiller",
             "memory_type": "semantic",
             "tags": cluster.tags + ["auto-generated", "consolidated"],
-            "confidence_score": self.config.extraction_confidence_threshold,
+            "confidence_score": confidence_score,
         }
         body = f"# Consolidated: {cluster.topic}\n\n" + "\n".join(facts)
         if YAML_AVAILABLE:
@@ -266,11 +274,21 @@ class Distiller:
             except Exception:
                 pass
 
+        clusters_processed = len(clusters)
+        facts_extracted = facts_count
+        episodes_archived = archived
+        if self.config.use_dp and self.config.dp_epsilon is not None and self.config.dp_delta is not None:
+            from .privacy_budget import add_noise
+            sensitivity = 1.0
+            clusters_processed = max(0, int(round(add_noise(float(clusters_processed), sensitivity, self.config.dp_epsilon, self.config.dp_delta))))
+            facts_extracted = max(0, int(round(add_noise(float(facts_extracted), sensitivity, self.config.dp_epsilon, self.config.dp_delta))))
+            episodes_archived = max(0, int(round(add_noise(float(episodes_archived), sensitivity, self.config.dp_epsilon, self.config.dp_delta))))
+
         return DistillerResult(
             success=True,
-            clusters_processed=len(clusters),
-            facts_extracted=facts_count,
-            episodes_archived=archived,
+            clusters_processed=clusters_processed,
+            facts_extracted=facts_extracted,
+            episodes_archived=episodes_archived,
             branch_created=branch_name,
             commit_hash=commit_hash,
             message=f"Processed {len(clusters)} clusters, extracted {facts_count} facts",

memvcs/core/encryption.py

@@ -0,0 +1,169 @@
+"""
+Encryption at rest for agmem object store.
+
+AES-256-GCM for object payloads; key derived from passphrase via Argon2id.
+Hash-then-encrypt so content-addressable paths stay based on plaintext hash.
+"""
+
+import json
+import os
+import secrets
+from pathlib import Path
+from typing import Optional, Tuple, Dict, Any, Callable
+
+# AES-GCM and Argon2id via cryptography
+try:
+    from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+    from cryptography.hazmat.primitives.kdf.argon2 import Argon2id
+
+    ENCRYPTION_AVAILABLE = True
+except ImportError:
+    ENCRYPTION_AVAILABLE = False
+
+IV_LEN = 12
+TAG_LEN = 16
+KEY_LEN = 32
+
+
+def _encryption_config_path(mem_dir: Path) -> Path:
+    return mem_dir / "encryption.json"
+
+
+def load_encryption_config(mem_dir: Path) -> Optional[Dict[str, Any]]:
+    """Load encryption config (salt, time_cost, memory_cost) from .mem/encryption.json."""
+    path = _encryption_config_path(mem_dir)
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text())
+    except Exception:
+        return None
+
+
+def save_encryption_config(
+    mem_dir: Path,
+    salt: bytes,
+    time_cost: int = 3,
+    memory_cost: int = 65536,
+    parallelism: int = 4,
+) -> Path:
+    """Save encryption config; salt stored as hex. Returns config path."""
+    mem_dir.mkdir(parents=True, exist_ok=True)
+    path = _encryption_config_path(mem_dir)
+    path.write_text(
+        json.dumps(
+            {
+                "salt_hex": salt.hex(),
+                "time_cost": time_cost,
+                "memory_cost": memory_cost,
+                "parallelism": parallelism,
+            },
+            indent=2,
+        )
+    )
+    return path
+
+
+def derive_key(
+    passphrase: bytes,
+    salt: bytes,
+    time_cost: int = 3,
+    memory_cost: int = 65536,
+    parallelism: int = 4,
+) -> bytes:
+    """Derive 32-byte key from passphrase using Argon2id."""
+    if not ENCRYPTION_AVAILABLE:
+        raise RuntimeError("Encryption requires 'cryptography'")
+    kdf = Argon2id(
+        salt=salt,
+        length=KEY_LEN,
+        time_cost=time_cost,
+        memory_cost=memory_cost,
+        parallelism=parallelism,
+    )
+    return kdf.derive(passphrase)
+
+
+def encrypt(plaintext: bytes, key: bytes) -> Tuple[bytes, bytes]:
+    """Encrypt with AES-256-GCM. Returns (iv, ciphertext_with_tag)."""
+    if not ENCRYPTION_AVAILABLE:
+        raise RuntimeError("Encryption requires 'cryptography'")
+    aes = AESGCM(key)
+    iv = secrets.token_bytes(IV_LEN)
+    ct = aes.encrypt(iv, plaintext, None)  # ct includes 16-byte tag
+    return (iv, ct)
+
+
+def decrypt(iv: bytes, ciphertext_with_tag: bytes, key: bytes) -> bytes:
+    """Decrypt AES-256-GCM. Raises on auth failure."""
+    if not ENCRYPTION_AVAILABLE:
+        raise RuntimeError("Encryption requires 'cryptography'")
+    aes = AESGCM(key)
+    return aes.decrypt(iv, ciphertext_with_tag, None)
+
+
+def init_encryption(mem_dir: Path, time_cost: int = 3, memory_cost: int = 65536) -> bytes:
+    """Create new encryption config with random salt. Returns salt (caller derives key from passphrase)."""
+    salt = secrets.token_bytes(16)
+    save_encryption_config(mem_dir, salt, time_cost=time_cost, memory_cost=memory_cost)
+    return salt
+
+
+class ObjectStoreEncryptor:
+    """
+    Encryptor for object store payloads (compressed bytes).
+    Uses AES-256-GCM; IV and tag stored with ciphertext.
+    """
+
+    def __init__(self, get_key: Callable[[], Optional[bytes]]):
+        self._get_key = get_key
+
+    def encrypt_payload(self, plaintext: bytes) -> bytes:
+        """Encrypt payload. Returns iv (12) + ciphertext_with_tag."""
+        key = self._get_key()
+        if not key:
+            raise ValueError("Encryption key not available (passphrase required)")
+        iv, ct = encrypt(plaintext, key)
+        return iv + ct
+
+    def decrypt_payload(self, raw: bytes) -> bytes:
+        """Decrypt payload. raw = iv (12) + ciphertext_with_tag."""
+        key = self._get_key()
+        if not key:
+            raise ValueError("Encryption key not available (passphrase required)")
+        if len(raw) < IV_LEN + TAG_LEN:
+            raise ValueError("Payload too short for encrypted object")
+        iv = raw[:IV_LEN]
+        ct = raw[IV_LEN:]
+        return decrypt(iv, ct, key)
+
+
+def get_key_from_env_or_cache(
+    mem_dir: Path,
+    env_var: str = "AGMEM_ENCRYPTION_PASSPHRASE",
+    cache_var: str = "_agmem_encryption_key_cache",
+) -> Optional[bytes]:
+    """Get key from env or process cache. Derives key if passphrase in env and config exists."""
+    # Module-level cache for session (same process)
+    import sys
+
+    mod = sys.modules.get("memvcs.core.encryption")
+    if mod and getattr(mod, cache_var, None) is not None:
+        return getattr(mod, cache_var)
+    passphrase = os.environ.get(env_var)
+    if not passphrase:
+        return None
+    cfg = load_encryption_config(mem_dir)
+    if not cfg:
+        return None
+    salt = bytes.fromhex(cfg["salt_hex"])
+    key = derive_key(
+        passphrase.encode() if isinstance(passphrase, str) else passphrase,
+        salt,
+        time_cost=cfg.get("time_cost", 3),
+        memory_cost=cfg.get("memory_cost", 65536),
+        parallelism=cfg.get("parallelism", 4),
+    )
+    if mod is not None:
+        setattr(mod, cache_var, key)
+    return key