agmem 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

memvcs/core/audit.py

@@ -0,0 +1,124 @@
+"""
+Tamper-evident audit trail for agmem.
+
+Append-only, hash-chained log of significant operations.
+"""
+
+import datetime
+import hashlib
+import hmac
+import json
+import os
+from pathlib import Path
+from typing import Optional, List, Dict, Any, Tuple
+
+
+def _audit_dir(mem_dir: Path) -> Path:
+    return mem_dir / "audit"
+
+
+def _log_path(mem_dir: Path) -> Path:
+    return _audit_dir(mem_dir) / "log"
+
+
+def _get_previous_hash(mem_dir: Path) -> str:
+    """Read last line of audit log and return its entry hash, or empty for first entry."""
+    path = _log_path(mem_dir)
+    if not path.exists():
+        return ""
+    lines = path.read_text().strip().split("\n")
+    if not lines:
+        return ""
+    # Format per line: entry_hash\tpayload_json
+    for line in reversed(lines):
+        line = line.strip()
+        if not line:
+            continue
+        if "\t" in line:
+            return line.split("\t", 1)[0]
+        return ""
+    return ""
+
+
+def _hash_entry(prev_hash: str, payload: str) -> str:
+    """Compute this entry's hash: SHA-256(prev_hash + payload)."""
+    return hashlib.sha256((prev_hash + payload).encode()).hexdigest()
+
+
+def append_audit(
+    mem_dir: Path,
+    operation: str,
+    details: Optional[Dict[str, Any]] = None,
+) -> None:
+    """
+    Append a tamper-evident audit entry. Write synchronously.
+    Each entry: entry_hash TAB payload_json (payload has timestamp, operation, details, prev_hash).
+    """
+    mem_dir = Path(mem_dir)
+    _audit_dir(mem_dir).mkdir(parents=True, exist_ok=True)
+    path = _log_path(mem_dir)
+    prev_hash = _get_previous_hash(mem_dir)
+    payload = {
+        "timestamp": datetime.datetime.utcnow().isoformat() + "Z",
+        "operation": operation,
+        "details": details or {},
+        "prev_hash": prev_hash,
+    }
+    payload_str = json.dumps(payload, sort_keys=True)
+    entry_hash = _hash_entry(prev_hash, payload_str)
+    line = f"{entry_hash}\t{payload_str}\n"
+    with open(path, "a", encoding="utf-8") as f:
+        f.write(line)
+        f.flush()
+        try:
+            os.fsync(f.fileno())
+        except (AttributeError, OSError):
+            pass
+
+
+def read_audit(mem_dir: Path, max_entries: int = 1000) -> List[Dict[str, Any]]:
+    """Read audit log entries (newest first). Each entry has entry_hash, prev_hash, timestamp, operation, details."""
+    path = _log_path(mem_dir)
+    if not path.exists():
+        return []
+    entries = []
+    for line in reversed(path.read_text().strip().split("\n")):
+        line = line.strip()
+        if not line:
+            continue
+        if "\t" not in line:
+            continue
+        entry_hash, payload_str = line.split("\t", 1)
+        try:
+            payload = json.loads(payload_str)
+        except json.JSONDecodeError:
+            continue
+        payload["entry_hash"] = entry_hash
+        entries.append(payload)
+        if len(entries) >= max_entries:
+            break
+    return entries
+
+
+def verify_audit(mem_dir: Path) -> Tuple[bool, Optional[int]]:
+    """
+    Verify the audit log chain. Returns (valid, first_bad_index).
+    first_bad_index is 0-based index of first entry that fails chain verification.
+    """
+    path = _log_path(mem_dir)
+    if not path.exists():
+        return (True, None)
+    lines = path.read_text().strip().split("\n")
+    prev_hash = ""
+    for i, line in enumerate(lines):
+        line = line.strip()
+        if not line:
+            continue
+        if "\t" not in line:
+            return (False, i)
+        entry_hash, payload_str = line.split("\t", 1)
+        expected_hash = _hash_entry(prev_hash, payload_str)
+        if not hmac.compare_digest(entry_hash, expected_hash):
+            return (False, i)
+        prev_hash = entry_hash
+    return (True, None)

memvcs/core/consistency.py

@@ -100,23 +100,23 @@ class ConsistencyChecker:
         return triples
 
     def _extract_triples_llm(self, content: str, source: str) -> List[Triple]:
-        """Extract triples using LLM."""
+        """Extract triples using LLM (multi-provider)."""
         try:
-            import openai
+            from .llm import get_provider
 
-            response = openai.chat.completions.create(
-                model="gpt-3.5-turbo",
-                messages=[
+            provider = get_provider(provider_name=self.llm_provider)
+            if not provider:
+                return []
+            text = provider.complete(
+                [
                     {
                         "role": "system",
-                        "content": "Extract factual statements as (subject, predicate, object) triples. "
-                        "One per line, format: SUBJECT | PREDICATE | OBJECT",
+                        "content": "Extract factual statements as (subject, predicate, object) triples. One per line, format: SUBJECT | PREDICATE | OBJECT",
                     },
                     {"role": "user", "content": content[:3000]},
                 ],
                 max_tokens=500,
             )
-            text = response.choices[0].message.content
             triples = []
             for i, line in enumerate(text.splitlines(), 1):
                 if "|" in line:
@@ -138,7 +138,7 @@ class ConsistencyChecker:
 
     def extract_triples(self, content: str, source: str, use_llm: bool = False) -> List[Triple]:
         """Extract triples from content."""
-        if use_llm and self.llm_provider == "openai":
+        if use_llm and self.llm_provider:
             t = self._extract_triples_llm(content, source)
             if t:
                 return t

memvcs/core/crypto_verify.py

@@ -0,0 +1,280 @@
+"""
+Cryptographic commit verification for agmem.
+
+Merkle tree over commit blobs, optional Ed25519 signing of Merkle root.
+Verification on checkout, pull, and via verify/fsck.
+"""
+
+import hashlib
+import hmac
+import json
+import os
+from pathlib import Path
+from typing import Optional, List, Tuple, Any, Dict
+
+from .objects import ObjectStore, Tree, Commit
+
+# Ed25519 via cryptography (optional)
+try:
+    from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+        Ed25519PrivateKey,
+        Ed25519PublicKey,
+    )
+    from cryptography.exceptions import InvalidSignature
+    from cryptography.hazmat.primitives import serialization
+
+    ED25519_AVAILABLE = True
+except ImportError:
+    ED25519_AVAILABLE = False
+
+
+def _collect_blob_hashes_from_tree(store: ObjectStore, tree_hash: str) -> List[str]:
+    """Recursively collect all blob hashes from a tree. Returns sorted list for deterministic Merkle."""
+    tree = Tree.load(store, tree_hash)
+    if not tree:
+        return []
+    blobs: List[str] = []
+    for entry in tree.entries:
+        if entry.obj_type == "blob":
+            blobs.append(entry.hash)
+        elif entry.obj_type == "tree":
+            blobs.extend(_collect_blob_hashes_from_tree(store, entry.hash))
+    return sorted(blobs)
+
+
+def _merkle_hash(data: bytes) -> str:
+    """SHA-256 hash for Merkle tree nodes."""
+    return hashlib.sha256(data).hexdigest()
+
+
+def build_merkle_tree(blob_hashes: List[str]) -> str:
+    """
+    Build balanced binary Merkle tree from blob hashes.
+    Leaves are hashes of blob hashes (as hex strings); internal nodes hash(left_hex || right_hex).
+    Returns root hash (hex).
+    """
+    if not blob_hashes:
+        return _merkle_hash(b"empty")
+    # Leaves: hash each blob hash string to fixed-size leaf
+    layer = [_merkle_hash(h.encode()) for h in blob_hashes]
+    while len(layer) > 1:
+        next_layer = []
+        for i in range(0, len(layer), 2):
+            left = layer[i]
+            right = layer[i + 1] if i + 1 < len(layer) else layer[i]
+            combined = (left + right).encode()
+            next_layer.append(_merkle_hash(combined))
+        layer = next_layer
+    return layer[0]
+
+
+def build_merkle_root_for_commit(store: ObjectStore, commit_hash: str) -> Optional[str]:
+    """Build Merkle root for a commit's tree. Returns None if commit/tree missing."""
+    commit = Commit.load(store, commit_hash)
+    if not commit:
+        return None
+    blobs = _collect_blob_hashes_from_tree(store, commit.tree)
+    return build_merkle_tree(blobs)
+
+
+def merkle_proof(blob_hashes: List[str], target_blob_hash: str) -> Optional[List[Tuple[str, str]]]:
+    """
+    Produce Merkle proof for a blob: list of (sibling_hash, "L"|"R") from leaf to root.
+    Returns None if target not in list.
+    """
+    if target_blob_hash not in blob_hashes:
+        return None
+    layer = [_merkle_hash(h.encode()) for h in sorted(blob_hashes)]
+    leaf_index = sorted(blob_hashes).index(target_blob_hash)
+    proof: List[Tuple[str, str]] = []
+    idx = leaf_index
+    while len(layer) > 1:
+        next_layer = []
+        for i in range(0, len(layer), 2):
+            left = layer[i]
+            right = layer[i + 1] if i + 1 < len(layer) else layer[i]
+            combined = (left + right).encode()
+            parent = _merkle_hash(combined)
+            next_layer.append(parent)
+            # If current idx is in this pair, record sibling and advance index
+            pair_idx = i // 2
+            if idx == i:
+                proof.append((right, "R"))
+                idx = pair_idx
+            elif idx == i + 1:
+                proof.append((left, "L"))
+                idx = pair_idx
+        layer = next_layer
+    return proof if proof else []
+
+
+def verify_merkle_proof(blob_hash: str, proof: List[Tuple[str, str]], expected_root: str) -> bool:
+    """Verify a Merkle proof for a blob against expected root."""
+    current = _merkle_hash(blob_hash.encode())
+    for sibling, side in proof:
+        if side == "L":
+            current = _merkle_hash((sibling + current).encode())
+        else:
+            current = _merkle_hash((current + sibling).encode())
+    return current == expected_root
+
+
+# --- Signing (Ed25519) ---
+
+
+def _keys_dir(mem_dir: Path) -> Path:
+    return mem_dir / "keys"
+
+
+def get_signing_key_paths(mem_dir: Path) -> Tuple[Path, Path]:
+    """Return (private_key_path, public_key_path). Private may not exist (env-only)."""
+    kd = _keys_dir(mem_dir)
+    return (kd / "private.pem", kd / "public.pem")
+
+
+def ensure_keys_dir(mem_dir: Path) -> Path:
+    """Ensure .mem/keys exists; return keys dir."""
+    kd = _keys_dir(mem_dir)
+    kd.mkdir(parents=True, exist_ok=True)
+    return kd
+
+
+def generate_keypair(mem_dir: Path) -> Tuple[bytes, bytes]:
+    """Generate Ed25519 keypair. Returns (private_pem, public_pem). Requires cryptography."""
+    if not ED25519_AVAILABLE:
+        raise RuntimeError(
+            "Signing requires 'cryptography'; install with: pip install cryptography"
+        )
+    private_key = Ed25519PrivateKey.generate()
+    public_key = private_key.public_key()
+    private_pem = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.PKCS8,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    public_pem = public_key.public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
+    )
+    return (private_pem, public_pem)
+
+
+def save_public_key(mem_dir: Path, public_pem: bytes) -> Path:
+    """Save public key to .mem/keys/public.pem. Returns path."""
+    ensure_keys_dir(mem_dir)
+    path = _keys_dir(mem_dir) / "public.pem"
+    path.write_bytes(public_pem)
+    return path
+
+
+def load_public_key(mem_dir: Path) -> Optional[bytes]:
+    """Load public key PEM from .mem/keys/public.pem or config. Returns None if not found."""
+    path = _keys_dir(mem_dir) / "public.pem"
+    if path.exists():
+        return path.read_bytes()
+    config_file = mem_dir / "config.json"
+    if config_file.exists():
+        try:
+            config = json.loads(config_file.read_text())
+            return config.get("signing", {}).get("public_key_pem")
+        except Exception:
+            pass
+    return None
+
+
+def load_private_key_from_env() -> Optional[bytes]:
+    """Load private key PEM from env AGMEM_SIGNING_PRIVATE_KEY (or path in AGMEM_SIGNING_PRIVATE_KEY_FILE)."""
+    pem = os.environ.get("AGMEM_SIGNING_PRIVATE_KEY")
+    if pem:
+        return pem.encode() if isinstance(pem, str) else pem
+    path = os.environ.get("AGMEM_SIGNING_PRIVATE_KEY_FILE")
+    if path and os.path.isfile(path):
+        return Path(path).read_bytes()
+    return None
+
+
+def sign_merkle_root(root_hex: str, private_key_pem: bytes) -> str:
+    """Sign Merkle root (hex string). Returns signature as hex."""
+    if not ED25519_AVAILABLE:
+        raise RuntimeError("Signing requires 'cryptography'")
+    key = serialization.load_pem_private_key(private_key_pem, password=None)
+    if not isinstance(key, Ed25519PrivateKey):
+        raise TypeError("Ed25519 private key required")
+    sig = key.sign(root_hex.encode())
+    return sig.hex()
+
+
+def verify_signature(root_hex: str, signature_hex: str, public_key_pem: bytes) -> bool:
+    """Verify signature of Merkle root. Returns True if valid."""
+    if not ED25519_AVAILABLE:
+        return False
+    try:
+        key = serialization.load_pem_public_key(public_key_pem)
+        if not isinstance(key, Ed25519PublicKey):
+            return False
+        key.verify(bytes.fromhex(signature_hex), root_hex.encode())
+        return True
+    except InvalidSignature:
+        return False
+    except Exception:
+        return False
+
+
+def verify_commit(
+    store: ObjectStore,
+    commit_hash: str,
+    public_key_pem: Optional[bytes] = None,
+    *,
+    mem_dir: Optional[Path] = None,
+) -> Tuple[bool, Optional[str]]:
+    """
+    Verify commit: rebuild Merkle tree from blobs, compare root to stored, verify signature.
+    Returns (verified, error_message). verified=True means OK; False + message means tampered or unverified.
+    If public_key_pem is None and mem_dir is set, load from mem_dir.
+    """
+    commit = Commit.load(store, commit_hash)
+    if not commit:
+        return (False, "commit not found")
+    stored_root = (commit.metadata or {}).get("merkle_root")
+    stored_sig = (commit.metadata or {}).get("signature")
+    if not stored_root:
+        return (False, "commit has no merkle_root (unverified)")
+    computed_root = build_merkle_root_for_commit(store, commit_hash)
+    if not computed_root:
+        return (False, "could not build Merkle tree (missing tree/blobs)")
+    if not hmac.compare_digest(computed_root, stored_root):
+        return (False, "merkle_root mismatch (commit tampered)")
+    if not stored_sig:
+        return (True, None)  # Root matches; no signature (legacy)
+    pub = public_key_pem
+    if not pub and mem_dir:
+        pub = load_public_key(mem_dir)
+    if not pub:
+        return (False, "signature present but no public key configured")
+    if isinstance(pub, str):
+        pub = pub.encode()
+    if not verify_signature(stored_root, stored_sig, pub):
+        return (False, "signature verification failed")
+    return (True, None)
+
+
+def verify_commit_optional(
+    store: ObjectStore,
+    commit_hash: str,
+    mem_dir: Optional[Path] = None,
+    *,
+    strict: bool = False,
+) -> None:
+    """
+    Verify commit; if strict=True raise on failure. If strict=False, only raise on tamper (root mismatch).
+    Unverified (no merkle_root) is OK when not strict.
+    """
+    ok, err = verify_commit(store, commit_hash, None, mem_dir=mem_dir)
+    if ok:
+        return
+    if not err:
+        return
+    if "tampered" in err or "mismatch" in err or "signature verification failed" in err:
+        raise ValueError(f"Commit verification failed: {err}")
+    if strict:
+        raise ValueError(f"Commit verification failed: {err}")

memvcs/core/distiller.py

@@ -110,9 +110,32 @@ class Distiller:
                 continue
         combined = "\n---\n".join(contents)
 
-        if self.config.llm_provider == "openai" and self.config.llm_model:
+        if self.config.llm_provider and self.config.llm_model:
             try:
-                return self._extract_with_openai(combined, cluster.topic)
+                from .llm import get_provider
+
+                config = {
+                    "llm_provider": self.config.llm_provider,
+                    "llm_model": self.config.llm_model,
+                }
+                provider = get_provider(config=config)
+                if provider:
+                    text = provider.complete(
+                        [
+                            {
+                                "role": "system",
+                                "content": "Extract factual statements from the text. Output as bullet points (one fact per line). Focus on: user preferences, learned facts, key decisions.",
+                            },
+                            {
+                                "role": "user",
+                                "content": f"Topic: {cluster.topic}\n\n{combined[:4000]}",
+                            },
+                        ],
+                        max_tokens=500,
+                    )
+                    return [
+                        line.strip() for line in text.splitlines() if line.strip().startswith("-")
+                    ][:15]
             except Exception:
                 pass
 
@@ -125,29 +148,6 @@ class Distiller:
                     facts.append(f"- {line[:200]}")
         return facts[:10] if facts else [f"- Learned about {cluster.topic}"]
 
-    def _extract_with_openai(self, content: str, topic: str) -> List[str]:
-        """Extract facts using OpenAI API."""
-        import openai
-
-        response = openai.chat.completions.create(
-            model=self.config.llm_model or "gpt-3.5-turbo",
-            messages=[
-                {
-                    "role": "system",
-                    "content": "Extract factual statements from the text. "
-                    "Output as bullet points (one fact per line). "
-                    "Focus on: user preferences, learned facts, key decisions.",
-                },
-                {
-                    "role": "user",
-                    "content": f"Topic: {topic}\n\n{content[:4000]}",
-                },
-            ],
-            max_tokens=500,
-        )
-        text = response.choices[0].message.content
-        return [line.strip() for line in text.splitlines() if line.strip().startswith("-")][:15]
-
     def write_consolidated(self, cluster: EpisodeCluster, facts: List[str]) -> Path:
         """Write consolidated semantic file."""
         self.target_dir.mkdir(parents=True, exist_ok=True)

memvcs/core/encryption.py

@@ -0,0 +1,169 @@
+"""
+Encryption at rest for agmem object store.
+
+AES-256-GCM for object payloads; key derived from passphrase via Argon2id.
+Hash-then-encrypt so content-addressable paths stay based on plaintext hash.
+"""
+
+import json
+import os
+import secrets
+from pathlib import Path
+from typing import Optional, Tuple, Dict, Any, Callable
+
+# AES-GCM and Argon2id via cryptography
+try:
+    from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+    from cryptography.hazmat.primitives.kdf.argon2 import Argon2id
+
+    ENCRYPTION_AVAILABLE = True
+except ImportError:
+    ENCRYPTION_AVAILABLE = False
+
+IV_LEN = 12
+TAG_LEN = 16
+KEY_LEN = 32
+
+
+def _encryption_config_path(mem_dir: Path) -> Path:
+    return mem_dir / "encryption.json"
+
+
+def load_encryption_config(mem_dir: Path) -> Optional[Dict[str, Any]]:
+    """Load encryption config (salt, time_cost, memory_cost) from .mem/encryption.json."""
+    path = _encryption_config_path(mem_dir)
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text())
+    except Exception:
+        return None
+
+
+def save_encryption_config(
+    mem_dir: Path,
+    salt: bytes,
+    time_cost: int = 3,
+    memory_cost: int = 65536,
+    parallelism: int = 4,
+) -> Path:
+    """Save encryption config; salt stored as hex. Returns config path."""
+    mem_dir.mkdir(parents=True, exist_ok=True)
+    path = _encryption_config_path(mem_dir)
+    path.write_text(
+        json.dumps(
+            {
+                "salt_hex": salt.hex(),
+                "time_cost": time_cost,
+                "memory_cost": memory_cost,
+                "parallelism": parallelism,
+            },
+            indent=2,
+        )
+    )
+    return path
+
+
+def derive_key(
+    passphrase: bytes,
+    salt: bytes,
+    time_cost: int = 3,
+    memory_cost: int = 65536,
+    parallelism: int = 4,
+) -> bytes:
+    """Derive 32-byte key from passphrase using Argon2id."""
+    if not ENCRYPTION_AVAILABLE:
+        raise RuntimeError("Encryption requires 'cryptography'")
+    kdf = Argon2id(
+        salt=salt,
+        length=KEY_LEN,
+        time_cost=time_cost,
+        memory_cost=memory_cost,
+        parallelism=parallelism,
+    )
+    return kdf.derive(passphrase)
+
+
+def encrypt(plaintext: bytes, key: bytes) -> Tuple[bytes, bytes]:
+    """Encrypt with AES-256-GCM. Returns (iv, ciphertext_with_tag)."""
+    if not ENCRYPTION_AVAILABLE:
+        raise RuntimeError("Encryption requires 'cryptography'")
+    aes = AESGCM(key)
+    iv = secrets.token_bytes(IV_LEN)
+    ct = aes.encrypt(iv, plaintext, None)  # ct includes 16-byte tag
+    return (iv, ct)
+
+
+def decrypt(iv: bytes, ciphertext_with_tag: bytes, key: bytes) -> bytes:
+    """Decrypt AES-256-GCM. Raises on auth failure."""
+    if not ENCRYPTION_AVAILABLE:
+        raise RuntimeError("Encryption requires 'cryptography'")
+    aes = AESGCM(key)
+    return aes.decrypt(iv, ciphertext_with_tag, None)
+
+
+def init_encryption(mem_dir: Path, time_cost: int = 3, memory_cost: int = 65536) -> bytes:
+    """Create new encryption config with random salt. Returns salt (caller derives key from passphrase)."""
+    salt = secrets.token_bytes(16)
+    save_encryption_config(mem_dir, salt, time_cost=time_cost, memory_cost=memory_cost)
+    return salt
+
+
+class ObjectStoreEncryptor:
+    """
+    Encryptor for object store payloads (compressed bytes).
+    Uses AES-256-GCM; IV and tag stored with ciphertext.
+    """
+
+    def __init__(self, get_key: Callable[[], Optional[bytes]]):
+        self._get_key = get_key
+
+    def encrypt_payload(self, plaintext: bytes) -> bytes:
+        """Encrypt payload. Returns iv (12) + ciphertext_with_tag."""
+        key = self._get_key()
+        if not key:
+            raise ValueError("Encryption key not available (passphrase required)")
+        iv, ct = encrypt(plaintext, key)
+        return iv + ct
+
+    def decrypt_payload(self, raw: bytes) -> bytes:
+        """Decrypt payload. raw = iv (12) + ciphertext_with_tag."""
+        key = self._get_key()
+        if not key:
+            raise ValueError("Encryption key not available (passphrase required)")
+        if len(raw) < IV_LEN + TAG_LEN:
+            raise ValueError("Payload too short for encrypted object")
+        iv = raw[:IV_LEN]
+        ct = raw[IV_LEN:]
+        return decrypt(iv, ct, key)
+
+
+def get_key_from_env_or_cache(
+    mem_dir: Path,
+    env_var: str = "AGMEM_ENCRYPTION_PASSPHRASE",
+    cache_var: str = "_agmem_encryption_key_cache",
+) -> Optional[bytes]:
+    """Get key from env or process cache. Derives key if passphrase in env and config exists."""
+    # Module-level cache for session (same process)
+    import sys
+
+    mod = sys.modules.get("memvcs.core.encryption")
+    if mod and getattr(mod, cache_var, None) is not None:
+        return getattr(mod, cache_var)
+    passphrase = os.environ.get(env_var)
+    if not passphrase:
+        return None
+    cfg = load_encryption_config(mem_dir)
+    if not cfg:
+        return None
+    salt = bytes.fromhex(cfg["salt_hex"])
+    key = derive_key(
+        passphrase.encode() if isinstance(passphrase, str) else passphrase,
+        salt,
+        time_cost=cfg.get("time_cost", 3),
+        memory_cost=cfg.get("memory_cost", 65536),
+        parallelism=cfg.get("parallelism", 4),
+    )
+    if mod is not None:
+        setattr(mod, cache_var, key)
+    return key