agmem 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

memvcs/core/crypto_verify.py

@@ -0,0 +1,280 @@
+"""
+Cryptographic commit verification for agmem.
+
+Merkle tree over commit blobs, optional Ed25519 signing of Merkle root.
+Verification on checkout, pull, and via verify/fsck.
+"""
+
+import hashlib
+import hmac
+import json
+import os
+from pathlib import Path
+from typing import Optional, List, Tuple, Any, Dict
+
+from .objects import ObjectStore, Tree, Commit
+
+# Ed25519 via cryptography (optional)
+try:
+    from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+        Ed25519PrivateKey,
+        Ed25519PublicKey,
+    )
+    from cryptography.exceptions import InvalidSignature
+    from cryptography.hazmat.primitives import serialization
+
+    ED25519_AVAILABLE = True
+except ImportError:
+    ED25519_AVAILABLE = False
+
+
+def _collect_blob_hashes_from_tree(store: ObjectStore, tree_hash: str) -> List[str]:
+    """Recursively collect all blob hashes from a tree. Returns sorted list for deterministic Merkle."""
+    tree = Tree.load(store, tree_hash)
+    if not tree:
+        return []
+    blobs: List[str] = []
+    for entry in tree.entries:
+        if entry.obj_type == "blob":
+            blobs.append(entry.hash)
+        elif entry.obj_type == "tree":
+            blobs.extend(_collect_blob_hashes_from_tree(store, entry.hash))
+    return sorted(blobs)
+
+
+def _merkle_hash(data: bytes) -> str:
+    """SHA-256 hash for Merkle tree nodes."""
+    return hashlib.sha256(data).hexdigest()
+
+
+def build_merkle_tree(blob_hashes: List[str]) -> str:
+    """
+    Build balanced binary Merkle tree from blob hashes.
+    Leaves are hashes of blob hashes (as hex strings); internal nodes hash(left_hex || right_hex).
+    Returns root hash (hex).
+    """
+    if not blob_hashes:
+        return _merkle_hash(b"empty")
+    # Leaves: hash each blob hash string to fixed-size leaf
+    layer = [_merkle_hash(h.encode()) for h in blob_hashes]
+    while len(layer) > 1:
+        next_layer = []
+        for i in range(0, len(layer), 2):
+            left = layer[i]
+            right = layer[i + 1] if i + 1 < len(layer) else layer[i]
+            combined = (left + right).encode()
+            next_layer.append(_merkle_hash(combined))
+        layer = next_layer
+    return layer[0]
+
+
+def build_merkle_root_for_commit(store: ObjectStore, commit_hash: str) -> Optional[str]:
+    """Build Merkle root for a commit's tree. Returns None if commit/tree missing."""
+    commit = Commit.load(store, commit_hash)
+    if not commit:
+        return None
+    blobs = _collect_blob_hashes_from_tree(store, commit.tree)
+    return build_merkle_tree(blobs)
+
+
+def merkle_proof(blob_hashes: List[str], target_blob_hash: str) -> Optional[List[Tuple[str, str]]]:
+    """
+    Produce Merkle proof for a blob: list of (sibling_hash, "L"|"R") from leaf to root.
+    Returns None if target not in list.
+    """
+    if target_blob_hash not in blob_hashes:
+        return None
+    layer = [_merkle_hash(h.encode()) for h in sorted(blob_hashes)]
+    leaf_index = sorted(blob_hashes).index(target_blob_hash)
+    proof: List[Tuple[str, str]] = []
+    idx = leaf_index
+    while len(layer) > 1:
+        next_layer = []
+        for i in range(0, len(layer), 2):
+            left = layer[i]
+            right = layer[i + 1] if i + 1 < len(layer) else layer[i]
+            combined = (left + right).encode()
+            parent = _merkle_hash(combined)
+            next_layer.append(parent)
+            # If current idx is in this pair, record sibling and advance index
+            pair_idx = i // 2
+            if idx == i:
+                proof.append((right, "R"))
+                idx = pair_idx
+            elif idx == i + 1:
+                proof.append((left, "L"))
+                idx = pair_idx
+        layer = next_layer
+    return proof if proof else []
+
+
+def verify_merkle_proof(blob_hash: str, proof: List[Tuple[str, str]], expected_root: str) -> bool:
+    """Verify a Merkle proof for a blob against expected root."""
+    current = _merkle_hash(blob_hash.encode())
+    for sibling, side in proof:
+        if side == "L":
+            current = _merkle_hash((sibling + current).encode())
+        else:
+            current = _merkle_hash((current + sibling).encode())
+    return current == expected_root
+
+
+# --- Signing (Ed25519) ---
+
+
+def _keys_dir(mem_dir: Path) -> Path:
+    return mem_dir / "keys"
+
+
+def get_signing_key_paths(mem_dir: Path) -> Tuple[Path, Path]:
+    """Return (private_key_path, public_key_path). Private may not exist (env-only)."""
+    kd = _keys_dir(mem_dir)
+    return (kd / "private.pem", kd / "public.pem")
+
+
+def ensure_keys_dir(mem_dir: Path) -> Path:
+    """Ensure .mem/keys exists; return keys dir."""
+    kd = _keys_dir(mem_dir)
+    kd.mkdir(parents=True, exist_ok=True)
+    return kd
+
+
+def generate_keypair(mem_dir: Path) -> Tuple[bytes, bytes]:
+    """Generate Ed25519 keypair. Returns (private_pem, public_pem). Requires cryptography."""
+    if not ED25519_AVAILABLE:
+        raise RuntimeError(
+            "Signing requires 'cryptography'; install with: pip install cryptography"
+        )
+    private_key = Ed25519PrivateKey.generate()
+    public_key = private_key.public_key()
+    private_pem = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.PKCS8,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    public_pem = public_key.public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
+    )
+    return (private_pem, public_pem)
+
+
+def save_public_key(mem_dir: Path, public_pem: bytes) -> Path:
+    """Save public key to .mem/keys/public.pem. Returns path."""
+    ensure_keys_dir(mem_dir)
+    path = _keys_dir(mem_dir) / "public.pem"
+    path.write_bytes(public_pem)
+    return path
+
+
+def load_public_key(mem_dir: Path) -> Optional[bytes]:
+    """Load public key PEM from .mem/keys/public.pem or config. Returns None if not found."""
+    path = _keys_dir(mem_dir) / "public.pem"
+    if path.exists():
+        return path.read_bytes()
+    config_file = mem_dir / "config.json"
+    if config_file.exists():
+        try:
+            config = json.loads(config_file.read_text())
+            return config.get("signing", {}).get("public_key_pem")
+        except Exception:
+            pass
+    return None
+
+
+def load_private_key_from_env() -> Optional[bytes]:
+    """Load private key PEM from env AGMEM_SIGNING_PRIVATE_KEY (or path in AGMEM_SIGNING_PRIVATE_KEY_FILE)."""
+    pem = os.environ.get("AGMEM_SIGNING_PRIVATE_KEY")
+    if pem:
+        return pem.encode() if isinstance(pem, str) else pem
+    path = os.environ.get("AGMEM_SIGNING_PRIVATE_KEY_FILE")
+    if path and os.path.isfile(path):
+        return Path(path).read_bytes()
+    return None
+
+
+def sign_merkle_root(root_hex: str, private_key_pem: bytes) -> str:
+    """Sign Merkle root (hex string). Returns signature as hex."""
+    if not ED25519_AVAILABLE:
+        raise RuntimeError("Signing requires 'cryptography'")
+    key = serialization.load_pem_private_key(private_key_pem, password=None)
+    if not isinstance(key, Ed25519PrivateKey):
+        raise TypeError("Ed25519 private key required")
+    sig = key.sign(root_hex.encode())
+    return sig.hex()
+
+
+def verify_signature(root_hex: str, signature_hex: str, public_key_pem: bytes) -> bool:
+    """Verify signature of Merkle root. Returns True if valid."""
+    if not ED25519_AVAILABLE:
+        return False
+    try:
+        key = serialization.load_pem_public_key(public_key_pem)
+        if not isinstance(key, Ed25519PublicKey):
+            return False
+        key.verify(bytes.fromhex(signature_hex), root_hex.encode())
+        return True
+    except InvalidSignature:
+        return False
+    except Exception:
+        return False
+
+
+def verify_commit(
+    store: ObjectStore,
+    commit_hash: str,
+    public_key_pem: Optional[bytes] = None,
+    *,
+    mem_dir: Optional[Path] = None,
+) -> Tuple[bool, Optional[str]]:
+    """
+    Verify commit: rebuild Merkle tree from blobs, compare root to stored, verify signature.
+    Returns (verified, error_message). verified=True means OK; False + message means tampered or unverified.
+    If public_key_pem is None and mem_dir is set, load from mem_dir.
+    """
+    commit = Commit.load(store, commit_hash)
+    if not commit:
+        return (False, "commit not found")
+    stored_root = (commit.metadata or {}).get("merkle_root")
+    stored_sig = (commit.metadata or {}).get("signature")
+    if not stored_root:
+        return (False, "commit has no merkle_root (unverified)")
+    computed_root = build_merkle_root_for_commit(store, commit_hash)
+    if not computed_root:
+        return (False, "could not build Merkle tree (missing tree/blobs)")
+    if not hmac.compare_digest(computed_root, stored_root):
+        return (False, "merkle_root mismatch (commit tampered)")
+    if not stored_sig:
+        return (True, None)  # Root matches; no signature (legacy)
+    pub = public_key_pem
+    if not pub and mem_dir:
+        pub = load_public_key(mem_dir)
+    if not pub:
+        return (False, "signature present but no public key configured")
+    if isinstance(pub, str):
+        pub = pub.encode()
+    if not verify_signature(stored_root, stored_sig, pub):
+        return (False, "signature verification failed")
+    return (True, None)
+
+
+def verify_commit_optional(
+    store: ObjectStore,
+    commit_hash: str,
+    mem_dir: Optional[Path] = None,
+    *,
+    strict: bool = False,
+) -> None:
+    """
+    Verify commit; if strict=True raise on failure. If strict=False, only raise on tamper (root mismatch).
+    Unverified (no merkle_root) is OK when not strict.
+    """
+    ok, err = verify_commit(store, commit_hash, None, mem_dir=mem_dir)
+    if ok:
+        return
+    if not err:
+        return
+    if "tampered" in err or "mismatch" in err or "signature verification failed" in err:
+        raise ValueError(f"Commit verification failed: {err}")
+    if strict:
+        raise ValueError(f"Commit verification failed: {err}")

memvcs/core/decay.py

@@ -0,0 +1,185 @@
+"""
+Decay engine - memory decay and forgetting for agmem.
+
+Mimics human forgetting: irrelevant details fade, important ones strengthen.
+Ebbinghaus-inspired time decay + retrieval-induced enhancement.
+"""
+
+import math
+import shutil
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+
+from .constants import MEMORY_TYPES
+from .access_index import AccessIndex
+from .objects import Commit
+from .schema import FrontmatterParser
+
+
+@dataclass
+class DecayConfig:
+    """Configuration for decay engine."""
+
+    episodic_half_life_days: int = 30
+    semantic_min_importance: float = 0.3
+    access_count_threshold: int = 2
+    forgetting_dir: str = "forgetting"
+
+
+@dataclass
+class DecayCandidate:
+    """A memory candidate for decay (archiving)."""
+
+    path: str
+    memory_type: str
+    importance: float
+    last_access_days: Optional[float]
+    access_count: int
+    decay_score: float
+    reason: str
+
+
+class DecayEngine:
+    """Computes decay scores and archives low-importance memories."""
+
+    def __init__(self, repo: Any, config: Optional[DecayConfig] = None):
+        self.repo = repo
+        self.config = config or DecayConfig()
+        self.access_index = AccessIndex(repo.mem_dir)
+        self.forgetting_dir = repo.mem_dir / self.config.forgetting_dir
+        self.current_dir = repo.current_dir
+
+    def _get_importance(self, path: str, content: str) -> float:
+        """Get importance from frontmatter or default."""
+        fm, _ = FrontmatterParser.parse(content)
+        if fm and fm.importance is not None:
+            return float(fm.importance)
+        if fm and fm.confidence_score is not None:
+            return float(fm.confidence_score)
+        return 0.5
+
+    def _get_access_info(self, path: str) -> Tuple[int, Optional[float]]:
+        """Get access count and days since last access."""
+        counts = self.access_index.get_access_counts_by_path()
+        count = counts.get(path, 0)
+        recent = self.access_index.get_recent_accesses(limit=1, path=path)
+        if not recent:
+            return count, None
+        ts_str = recent[0].get("timestamp", "")
+        if not ts_str:
+            return count, None
+        try:
+            if ts_str.endswith("Z"):
+                ts_str = ts_str[:-1] + "+00:00"
+            last = datetime.fromisoformat(ts_str)
+            days = (datetime.utcnow() - last.replace(tzinfo=None)).total_seconds() / 86400
+            return count, days
+        except Exception:
+            return count, None
+
+    def compute_decay_score(
+        self,
+        path: str,
+        content: str,
+        memory_type: str,
+    ) -> DecayCandidate:
+        """
+        Compute decay score for a memory.
+
+        Higher score = more likely to decay (archive).
+        Time decay: importance * 0.5^(days/half_life) when never accessed.
+        Retrieval-induced enhancement: access boosts strength (lower decay).
+        """
+        importance = self._get_importance(path, content)
+        access_count, last_access_days = self._get_access_info(path)
+
+        decay_score = 0.0
+        reason = ""
+
+        if "episodic" in memory_type.lower():
+            half_life = self.config.episodic_half_life_days
+            if last_access_days is not None:
+                decay_score = 1.0 - (importance * math.pow(0.5, last_access_days / half_life))
+                if access_count < self.config.access_count_threshold:
+                    decay_score += 0.2
+                reason = f"episodic: {last_access_days:.0f}d since access, imp={importance:.2f}"
+            else:
+                decay_score = 0.5
+                reason = "episodic: never accessed"
+        else:
+            if importance < self.config.semantic_min_importance:
+                decay_score = 1.0 - importance
+                reason = f"semantic: low importance {importance:.2f}"
+            elif (
+                access_count < self.config.access_count_threshold
+                and last_access_days
+                and last_access_days > 60
+            ):
+                decay_score = 0.4
+                reason = "semantic: rarely accessed"
+
+        return DecayCandidate(
+            path=path,
+            memory_type=memory_type,
+            importance=importance,
+            last_access_days=last_access_days,
+            access_count=access_count,
+            decay_score=decay_score,
+            reason=reason,
+        )
+
+    def get_decay_candidates(self) -> List[DecayCandidate]:
+        """Get list of memories that would be archived (dry-run)."""
+        candidates = []
+        if not self.current_dir.exists():
+            return candidates
+
+        for subdir in MEMORY_TYPES:
+            dir_path = self.current_dir / subdir
+            if not dir_path.exists():
+                continue
+            for f in dir_path.rglob("*"):
+                if not f.is_file() or f.suffix.lower() not in (".md", ".txt"):
+                    continue
+                try:
+                    rel_path = str(f.relative_to(self.current_dir))
+                    content = f.read_text(encoding="utf-8", errors="replace")
+                except Exception:
+                    continue
+                cand = self.compute_decay_score(rel_path, content, subdir)
+                if cand.decay_score > 0.5:
+                    candidates.append(cand)
+
+        candidates.sort(key=lambda x: x.decay_score, reverse=True)
+        return candidates
+
+    def apply_decay(self, candidates: Optional[List[DecayCandidate]] = None) -> int:
+        """
+        Archive low-importance memories to .mem/forgetting/.
+
+        Returns count of files archived.
+        """
+        if candidates is None:
+            candidates = self.get_decay_candidates()
+        self.forgetting_dir.mkdir(parents=True, exist_ok=True)
+        ts = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
+        archive_sub = self.forgetting_dir / ts
+        archive_sub.mkdir(exist_ok=True)
+        count = 0
+        for cand in candidates:
+            if cand.decay_score <= 0.5:
+                continue
+            src = self.current_dir / cand.path
+            if not src.exists():
+                continue
+            try:
+                safe_name = cand.path.replace("/", "_").replace("..", "_")
+                dest = (archive_sub / safe_name).resolve()
+                dest.relative_to(self.forgetting_dir.resolve())
+                shutil.move(str(src), str(dest))
+                count += 1
+            except (ValueError, Exception):
+                continue
+        return count