agmem 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

memvcs/core/distiller.py

@@ -0,0 +1,277 @@
+"""
+Distiller - Episodic-to-semantic distillation pipeline for agmem.
+
+Converts session logs into compact facts (like memory consolidation during sleep).
+Extends Gardener with factual extraction and safety branches.
+"""
+
+import shutil
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+from dataclasses import dataclass, field
+from datetime import datetime
+from collections import defaultdict
+
+try:
+    import yaml
+
+    YAML_AVAILABLE = True
+except ImportError:
+    YAML_AVAILABLE = False
+
+from .gardener import Gardener, GardenerConfig, EpisodeCluster
+
+
+@dataclass
+class DistillerConfig:
+    """Configuration for the Distiller."""
+
+    source_dir: str = "episodic"
+    target_dir: str = "semantic/consolidated"
+    archive_dir: str = "archive"
+    min_cluster_size: int = 3
+    extraction_confidence_threshold: float = 0.7
+    safety_branch_prefix: str = "auto-distill/"
+    llm_provider: Optional[str] = None
+    llm_model: Optional[str] = None
+    create_safety_branch: bool = True
+
+
+@dataclass
+class DistillerResult:
+    """Result of a distillation run."""
+
+    success: bool
+    clusters_processed: int
+    facts_extracted: int
+    episodes_archived: int
+    branch_created: Optional[str] = None
+    commit_hash: Optional[str] = None
+    message: str = ""
+
+
+class Distiller:
+    """
+    Distills episodic memory into semantic facts.
+
+    Pipeline: cluster episodes -> extract facts via LLM -> merge with semantic -> archive.
+    Creates safety branch for human review before merging to main.
+    """
+
+    def __init__(self, repo: Any, config: Optional[DistillerConfig] = None):
+        self.repo = repo
+        self.config = config or DistillerConfig()
+        self.source_dir = repo.root / "current" / self.config.source_dir
+        self.target_dir = repo.root / "current" / self.config.target_dir.rstrip("/")
+        archive_candidate = repo.current_dir / self.config.archive_dir
+        try:
+            archive_candidate.resolve().relative_to(repo.current_dir.resolve())
+            self.archive_dir = archive_candidate
+        except (ValueError, RuntimeError):
+            self.archive_dir = repo.current_dir / "archive"
+        self.gardener = Gardener(
+            repo,
+            GardenerConfig(
+                threshold=1,
+                archive_dir=self.config.archive_dir,
+                min_cluster_size=self.config.min_cluster_size,
+                llm_provider=self.config.llm_provider,
+                llm_model=self.config.llm_model,
+            ),
+        )
+
+    def load_episodes_from(self, source_path: Path) -> List[Tuple[Path, str]]:
+        """Load episodes from source directory."""
+        episodes = []
+        if not source_path.exists():
+            return episodes
+        for f in source_path.glob("**/*.md"):
+            if f.is_file():
+                try:
+                    episodes.append((f, f.read_text(encoding="utf-8", errors="replace")))
+                except Exception:
+                    continue
+        return episodes
+
+    def cluster_episodes(self, episodes: List[Tuple[Path, str]]) -> List[EpisodeCluster]:
+        """Cluster episodes using Gardener's logic."""
+        try:
+            return self.gardener.cluster_episodes_with_embeddings(episodes)
+        except Exception:
+            return self.gardener.cluster_episodes(episodes)
+
+    def extract_facts(self, cluster: EpisodeCluster) -> List[str]:
+        """Extract factual statements from cluster via LLM or heuristics."""
+        contents = []
+        for ep_path in cluster.episodes[:10]:
+            try:
+                contents.append(ep_path.read_text()[:1000])
+            except Exception:
+                continue
+        combined = "\n---\n".join(contents)
+
+        if self.config.llm_provider and self.config.llm_model:
+            try:
+                from .llm import get_provider
+
+                config = {
+                    "llm_provider": self.config.llm_provider,
+                    "llm_model": self.config.llm_model,
+                }
+                provider = get_provider(config=config)
+                if provider:
+                    text = provider.complete(
+                        [
+                            {
+                                "role": "system",
+                                "content": "Extract factual statements from the text. Output as bullet points (one fact per line). Focus on: user preferences, learned facts, key decisions.",
+                            },
+                            {
+                                "role": "user",
+                                "content": f"Topic: {cluster.topic}\n\n{combined[:4000]}",
+                            },
+                        ],
+                        max_tokens=500,
+                    )
+                    return [
+                        line.strip() for line in text.splitlines() if line.strip().startswith("-")
+                    ][:15]
+            except Exception:
+                pass
+
+        # Fallback: simple extraction
+        facts = []
+        for line in combined.splitlines():
+            line = line.strip()
+            if len(line) > 20 and not line.startswith("#") and not line.startswith("-"):
+                if any(w in line.lower() for w in ["prefers", "likes", "uses", "learned", "user"]):
+                    facts.append(f"- {line[:200]}")
+        return facts[:10] if facts else [f"- Learned about {cluster.topic}"]
+
+    def write_consolidated(self, cluster: EpisodeCluster, facts: List[str]) -> Path:
+        """Write consolidated semantic file."""
+        self.target_dir.mkdir(parents=True, exist_ok=True)
+        safe_topic = cluster.topic.replace(" ", "-").lower().replace("/", "_")[:30]
+        ts = datetime.utcnow().strftime("%Y%m%d")
+        filename = f"consolidated-{safe_topic}-{ts}.md"
+        out_path = (self.target_dir / filename).resolve()
+        try:
+            out_path.relative_to(self.repo.current_dir.resolve())
+        except ValueError:
+            out_path = self.target_dir / f"consolidated-{ts}.md"
+
+        frontmatter = {
+            "schema_version": "1.0",
+            "last_updated": datetime.utcnow().isoformat() + "Z",
+            "source_agent_id": "distiller",
+            "memory_type": "semantic",
+            "tags": cluster.tags + ["auto-generated", "consolidated"],
+            "confidence_score": self.config.extraction_confidence_threshold,
+        }
+        body = f"# Consolidated: {cluster.topic}\n\n" + "\n".join(facts)
+        if YAML_AVAILABLE:
+            import yaml
+
+            content = f"---\n{yaml.dump(frontmatter, default_flow_style=False)}---\n\n{body}"
+        else:
+            content = body
+        out_path.write_text(content)
+        return out_path
+
+    def archive_episodes(self, episodes: List[Path]) -> int:
+        """Archive processed episodes to .mem/archive/."""
+        archive_base = self.repo.mem_dir / "archive"
+        archive_base.mkdir(parents=True, exist_ok=True)
+        ts = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
+        archive_sub = archive_base / ts
+        archive_sub.mkdir(exist_ok=True)
+        count = 0
+        for ep in episodes:
+            try:
+                safe_name = ep.name.replace("..", "_").replace("/", "_")
+                dest = (archive_sub / safe_name).resolve()
+                dest.relative_to(archive_base.resolve())
+                shutil.move(str(ep), str(dest))
+                count += 1
+            except (ValueError, Exception):
+                continue
+        return count
+
+    def run(
+        self,
+        source: Optional[str] = None,
+        target: Optional[str] = None,
+        model: Optional[str] = None,
+    ) -> DistillerResult:
+        """Run distillation pipeline."""
+        source_path = Path(source) if source else self.source_dir
+        if not source_path.is_absolute():
+            source_path = self.repo.root / "current" / source_path
+        target_path = Path(target) if target else self.target_dir
+        if not target_path.is_absolute():
+            target_path = self.repo.root / "current" / target_path
+        self.target_dir = target_path
+        if model:
+            self.config.llm_model = model
+
+        episodes = self.load_episodes_from(source_path)
+        if not episodes:
+            return DistillerResult(
+                success=True,
+                clusters_processed=0,
+                facts_extracted=0,
+                episodes_archived=0,
+                message="No episodes to process",
+            )
+
+        clusters = self.cluster_episodes(episodes)
+        if not clusters:
+            return DistillerResult(
+                success=True,
+                clusters_processed=0,
+                facts_extracted=0,
+                episodes_archived=0,
+                message="No clusters formed",
+            )
+
+        # Create safety branch if configured
+        branch_name = None
+        if self.config.create_safety_branch:
+            ts = datetime.utcnow().strftime("%Y-%m-%d")
+            branch_name = f"{self.config.safety_branch_prefix}{ts}"
+            if not self.repo.refs.branch_exists(branch_name):
+                self.repo.refs.create_branch(branch_name)
+                self.repo.checkout(branch_name, force=True)
+
+        facts_count = 0
+        all_archived = []
+        for cluster in clusters:
+            facts = self.extract_facts(cluster)
+            self.write_consolidated(cluster, facts)
+            facts_count += len(facts)
+            all_archived.extend(cluster.episodes)
+
+        archived = self.archive_episodes(all_archived)
+
+        commit_hash = None
+        if facts_count > 0:
+            try:
+                for f in self.target_dir.glob("consolidated-*.md"):
+                    rel = str(f.relative_to(self.repo.root / "current"))
+                    self.repo.stage_file(rel)
+                commit_hash = self.repo.commit(
+                    f"distiller: consolidated {facts_count} facts from {len(episodes)} episodes",
+                    {"distiller": True, "clusters": len(clusters)},
+                )
+            except Exception:
+                pass
+
+        return DistillerResult(
+            success=True,
+            clusters_processed=len(clusters),
+            facts_extracted=facts_count,
+            episodes_archived=archived,
+            branch_created=branch_name,
+            commit_hash=commit_hash,
+            message=f"Processed {len(clusters)} clusters, extracted {facts_count} facts",
+        )

memvcs/core/encryption.py

@@ -0,0 +1,169 @@
+"""
+Encryption at rest for agmem object store.
+
+AES-256-GCM for object payloads; key derived from passphrase via Argon2id.
+Hash-then-encrypt so content-addressable paths stay based on plaintext hash.
+"""
+
+import json
+import os
+import secrets
+from pathlib import Path
+from typing import Optional, Tuple, Dict, Any, Callable
+
+# AES-GCM and Argon2id via cryptography
+try:
+    from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+    from cryptography.hazmat.primitives.kdf.argon2 import Argon2id
+
+    ENCRYPTION_AVAILABLE = True
+except ImportError:
+    ENCRYPTION_AVAILABLE = False
+
+IV_LEN = 12
+TAG_LEN = 16
+KEY_LEN = 32
+
+
+def _encryption_config_path(mem_dir: Path) -> Path:
+    return mem_dir / "encryption.json"
+
+
+def load_encryption_config(mem_dir: Path) -> Optional[Dict[str, Any]]:
+    """Load encryption config (salt, time_cost, memory_cost) from .mem/encryption.json."""
+    path = _encryption_config_path(mem_dir)
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text())
+    except Exception:
+        return None
+
+
+def save_encryption_config(
+    mem_dir: Path,
+    salt: bytes,
+    time_cost: int = 3,
+    memory_cost: int = 65536,
+    parallelism: int = 4,
+) -> Path:
+    """Save encryption config; salt stored as hex. Returns config path."""
+    mem_dir.mkdir(parents=True, exist_ok=True)
+    path = _encryption_config_path(mem_dir)
+    path.write_text(
+        json.dumps(
+            {
+                "salt_hex": salt.hex(),
+                "time_cost": time_cost,
+                "memory_cost": memory_cost,
+                "parallelism": parallelism,
+            },
+            indent=2,
+        )
+    )
+    return path
+
+
+def derive_key(
+    passphrase: bytes,
+    salt: bytes,
+    time_cost: int = 3,
+    memory_cost: int = 65536,
+    parallelism: int = 4,
+) -> bytes:
+    """Derive 32-byte key from passphrase using Argon2id."""
+    if not ENCRYPTION_AVAILABLE:
+        raise RuntimeError("Encryption requires 'cryptography'")
+    kdf = Argon2id(
+        salt=salt,
+        length=KEY_LEN,
+        time_cost=time_cost,
+        memory_cost=memory_cost,
+        parallelism=parallelism,
+    )
+    return kdf.derive(passphrase)
+
+
+def encrypt(plaintext: bytes, key: bytes) -> Tuple[bytes, bytes]:
+    """Encrypt with AES-256-GCM. Returns (iv, ciphertext_with_tag)."""
+    if not ENCRYPTION_AVAILABLE:
+        raise RuntimeError("Encryption requires 'cryptography'")
+    aes = AESGCM(key)
+    iv = secrets.token_bytes(IV_LEN)
+    ct = aes.encrypt(iv, plaintext, None)  # ct includes 16-byte tag
+    return (iv, ct)
+
+
+def decrypt(iv: bytes, ciphertext_with_tag: bytes, key: bytes) -> bytes:
+    """Decrypt AES-256-GCM. Raises on auth failure."""
+    if not ENCRYPTION_AVAILABLE:
+        raise RuntimeError("Encryption requires 'cryptography'")
+    aes = AESGCM(key)
+    return aes.decrypt(iv, ciphertext_with_tag, None)
+
+
+def init_encryption(mem_dir: Path, time_cost: int = 3, memory_cost: int = 65536) -> bytes:
+    """Create new encryption config with random salt. Returns salt (caller derives key from passphrase)."""
+    salt = secrets.token_bytes(16)
+    save_encryption_config(mem_dir, salt, time_cost=time_cost, memory_cost=memory_cost)
+    return salt
+
+
+class ObjectStoreEncryptor:
+    """
+    Encryptor for object store payloads (compressed bytes).
+    Uses AES-256-GCM; IV and tag stored with ciphertext.
+    """
+
+    def __init__(self, get_key: Callable[[], Optional[bytes]]):
+        self._get_key = get_key
+
+    def encrypt_payload(self, plaintext: bytes) -> bytes:
+        """Encrypt payload. Returns iv (12) + ciphertext_with_tag."""
+        key = self._get_key()
+        if not key:
+            raise ValueError("Encryption key not available (passphrase required)")
+        iv, ct = encrypt(plaintext, key)
+        return iv + ct
+
+    def decrypt_payload(self, raw: bytes) -> bytes:
+        """Decrypt payload. raw = iv (12) + ciphertext_with_tag."""
+        key = self._get_key()
+        if not key:
+            raise ValueError("Encryption key not available (passphrase required)")
+        if len(raw) < IV_LEN + TAG_LEN:
+            raise ValueError("Payload too short for encrypted object")
+        iv = raw[:IV_LEN]
+        ct = raw[IV_LEN:]
+        return decrypt(iv, ct, key)
+
+
+def get_key_from_env_or_cache(
+    mem_dir: Path,
+    env_var: str = "AGMEM_ENCRYPTION_PASSPHRASE",
+    cache_var: str = "_agmem_encryption_key_cache",
+) -> Optional[bytes]:
+    """Get key from env or process cache. Derives key if passphrase in env and config exists."""
+    # Module-level cache for session (same process)
+    import sys
+
+    mod = sys.modules.get("memvcs.core.encryption")
+    if mod and getattr(mod, cache_var, None) is not None:
+        return getattr(mod, cache_var)
+    passphrase = os.environ.get(env_var)
+    if not passphrase:
+        return None
+    cfg = load_encryption_config(mem_dir)
+    if not cfg:
+        return None
+    salt = bytes.fromhex(cfg["salt_hex"])
+    key = derive_key(
+        passphrase.encode() if isinstance(passphrase, str) else passphrase,
+        salt,
+        time_cost=cfg.get("time_cost", 3),
+        memory_cost=cfg.get("memory_cost", 65536),
+        parallelism=cfg.get("parallelism", 4),
+    )
+    if mod is not None:
+        setattr(mod, cache_var, key)
+    return key

memvcs/core/federated.py

@@ -0,0 +1,86 @@
+"""
+Federated memory collaboration for agmem.
+
+Agents share model updates or aggregated summaries instead of raw episodic logs.
+Optional coordinator URL; optional differential privacy (Tier 3).
+"""
+
+import json
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+
+from .config_loader import load_agmem_config
+
+
+def get_federated_config(repo_root: Path) -> Optional[Dict[str, Any]]:
+    """Get federated config from repo/user config. Returns None if disabled."""
+    config = load_agmem_config(repo_root)
+    fed = config.get("federated") or {}
+    if not fed.get("enabled"):
+        return None
+    url = fed.get("coordinator_url")
+    if not url:
+        return None
+    return {
+        "coordinator_url": url.rstrip("/"),
+        "memory_types": fed.get("memory_types", ["episodic", "semantic"]),
+    }
+
+
+def produce_local_summary(repo_root: Path, memory_types: List[str]) -> Dict[str, Any]:
+    """
+    Produce a local summary from episodic/semantic data (no raw content).
+    Returns dict suitable for sending to coordinator (e.g. topic counts, fact hashes).
+    """
+    current_dir = repo_root / "current"
+    summary = {"memory_types": memory_types, "topics": {}, "fact_count": 0}
+    for mtype in memory_types:
+        d = current_dir / mtype
+        if not d.exists():
+            continue
+        count = 0
+        for f in d.rglob("*.md"):
+            if f.is_file():
+                count += 1
+        summary["topics"][mtype] = count
+        if mtype == "semantic":
+            summary["fact_count"] = count
+    return summary
+
+
+def push_updates(repo_root: Path, summary: Dict[str, Any]) -> str:
+    """Send local summary to coordinator. Returns status message."""
+    cfg = get_federated_config(repo_root)
+    if not cfg:
+        return "Federated collaboration not configured"
+    url = cfg["coordinator_url"] + "/push"
+    try:
+        import urllib.request
+
+        req = urllib.request.Request(
+            url,
+            data=json.dumps(summary).encode(),
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            if resp.status in (200, 201):
+                return "Pushed updates to coordinator"
+            return f"Coordinator returned {resp.status}"
+    except Exception as e:
+        return f"Push failed: {e}"
+
+
+def pull_merged(repo_root: Path) -> Optional[Dict[str, Any]]:
+    """Pull merged summaries from coordinator. Returns merged data or None."""
+    cfg = get_federated_config(repo_root)
+    if not cfg:
+        return None
+    url = cfg["coordinator_url"] + "/pull"
+    try:
+        import urllib.request
+
+        with urllib.request.urlopen(url, timeout=30) as resp:
+            return json.loads(resp.read().decode())
+    except Exception:
+        return None