agmem 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

memvcs/core/llm/factory.py

@@ -0,0 +1,30 @@
+"""LLM provider factory: select by config or env."""
+
+import os
+from typing import Optional, Dict, Any
+
+from .base import LLMProvider
+from .openai_provider import OpenAIProvider
+from .anthropic_provider import AnthropicProvider
+
+
+def get_provider(
+    provider_name: Optional[str] = None,
+    model: Optional[str] = None,
+    config: Optional[Dict[str, Any]] = None,
+) -> Optional[LLMProvider]:
+    """
+    Return LLM provider by name. Config may have llm_provider, llm_model.
+    Env: AGMEM_LLM_PROVIDER, OPENAI_API_KEY, ANTHROPIC_API_KEY.
+    """
+    name = (
+        provider_name
+        or (config or {}).get("llm_provider")
+        or os.environ.get("AGMEM_LLM_PROVIDER", "openai")
+    )
+    m = model or (config or {}).get("llm_model")
+    if name == "openai":
+        return OpenAIProvider(model=m)
+    if name == "anthropic":
+        return AnthropicProvider(model=m)
+    return OpenAIProvider(model=m)

memvcs/core/llm/openai_provider.py

@@ -0,0 +1,36 @@
+"""OpenAI LLM provider."""
+
+import os
+from typing import Optional, List, Dict, Any
+
+from .base import LLMProvider
+
+
+class OpenAIProvider(LLMProvider):
+    """OpenAI (GPT) provider. API key from OPENAI_API_KEY."""
+
+    def __init__(self, model: Optional[str] = None):
+        self._model = model or os.environ.get("OPENAI_MODEL", "gpt-3.5-turbo")
+
+    @property
+    def name(self) -> str:
+        return "openai"
+
+    def complete(
+        self,
+        messages: List[Dict[str, str]],
+        *,
+        model: Optional[str] = None,
+        max_tokens: int = 1024,
+        **kwargs: Any,
+    ) -> str:
+        import openai
+
+        m = model or self._model
+        response = openai.chat.completions.create(
+            model=m,
+            messages=messages,
+            max_tokens=max_tokens,
+            **kwargs,
+        )
+        return response.choices[0].message.content or ""

memvcs/core/merge.py

@@ -33,6 +33,8 @@ class Conflict:
     ours_content: Optional[str]
     theirs_content: Optional[str]
     message: str
+    memory_type: Optional[str] = None  # episodic, semantic, procedural
+    payload: Optional[Dict[str, Any]] = None  # type-specific (e.g. fact strings, step diffs)
 
 
 @dataclass
@@ -256,31 +258,31 @@ class MergeEngine:
         ours_content: Optional[str],
         theirs_content: Optional[str],
     ) -> Tuple[str, bool]:
-        """LLM arbitration: call LLM to resolve contradiction."""
+        """LLM arbitration: call LLM to resolve contradiction (multi-provider)."""
         try:
-            import openai
-
-            response = openai.chat.completions.create(
-                model="gpt-3.5-turbo",
-                messages=[
-                    {
-                        "role": "system",
-                        "content": "Resolve the contradiction between two memory versions. "
-                        "Output the merged content that best reflects the combined truth.",
-                    },
-                    {
-                        "role": "user",
-                        "content": f"OURS:\n{ours_content}\n\nTHEIRS:\n{theirs_content}",
-                    },
-                ],
-                max_tokens=1000,
-            )
-            merged = response.choices[0].message.content.strip()
-            return merged, False
+            from .llm import get_provider
+
+            provider = get_provider()
+            if provider:
+                merged = provider.complete(
+                    [
+                        {
+                            "role": "system",
+                            "content": "Resolve the contradiction between two memory versions. Output the merged content that best reflects the combined truth.",
+                        },
+                        {
+                            "role": "user",
+                            "content": f"OURS:\n{ours_content}\n\nTHEIRS:\n{theirs_content}",
+                        },
+                    ],
+                    max_tokens=1000,
+                )
+                return (merged or "").strip(), False
         except Exception:
-            # Fallback to conflict markers
-            merged = f"<<<<<<< OURS\n{ours_content}\n=======\n{theirs_content}\n>>>>>>> THEIRS"
-            return merged, True
+            pass
+        # Fallback to conflict markers
+        merged = f"<<<<<<< OURS\n{ours_content}\n=======\n{theirs_content}\n>>>>>>> THEIRS"
+        return merged, True
 
     def merge_procedural(
         self,
@@ -398,6 +400,15 @@ class MergeEngine:
 
             # Record conflict if any
             if had_conflict:
+                payload = {}
+                if ours_content:
+                    payload["ours_preview"] = (
+                        ours_content[:300] if len(ours_content) > 300 else ours_content
+                    )
+                if theirs_content:
+                    payload["theirs_preview"] = (
+                        theirs_content[:300] if len(theirs_content) > 300 else theirs_content
+                    )
                 conflicts.append(
                     Conflict(
                         path=path,
@@ -405,6 +416,8 @@ class MergeEngine:
                         ours_content=ours_content,
                         theirs_content=theirs_content,
                         message=f"{strategy.value} merge conflict in {path}",
+                        memory_type=strategy.value,
+                        payload=payload or None,
                     )
                 )
 

memvcs/core/objects.py

@@ -24,8 +24,9 @@ def _valid_object_hash(hash_id: str) -> bool:
 class ObjectStore:
     """Content-addressable object storage system."""
 
-    def __init__(self, objects_dir: Path):
+    def __init__(self, objects_dir: Path, encryptor: Optional[Any] = None):
         self.objects_dir = Path(objects_dir)
+        self._encryptor = encryptor
         self._ensure_directories()
 
     def _ensure_directories(self):
@@ -68,17 +69,21 @@ class ObjectStore:
         # Create directory if needed
         obj_path.parent.mkdir(parents=True, exist_ok=True)
 
-        # Compress and store
+        # Compress and optionally encrypt
         header = f"{obj_type} {len(content)}\0".encode()
         full_content = header + content
         compressed = zlib.compress(full_content)
-
+        if self._encryptor:
+            try:
+                compressed = self._encryptor.encrypt_payload(compressed)
+            except ValueError:
+                pass  # no key; store plain compressed (legacy behavior)
         obj_path.write_bytes(compressed)
         return hash_id
 
     def retrieve(self, hash_id: str, obj_type: str) -> Optional[bytes]:
         """
-        Retrieve content by hash ID.
+        Retrieve content by hash ID (loose object or pack).
 
         Args:
             hash_id: SHA-256 hash of the object
@@ -89,26 +94,41 @@ class ObjectStore:
         """
         obj_path = self._get_object_path(hash_id, obj_type)
 
-        if not obj_path.exists():
-            return None
-
-        # Decompress and extract content
-        compressed = obj_path.read_bytes()
-        full_content = zlib.decompress(compressed)
-
-        # Parse header
-        null_idx = full_content.index(b"\0")
-        header = full_content[:null_idx].decode()
-        content = full_content[null_idx + 1 :]
-
-        return content
+        if obj_path.exists():
+            raw = obj_path.read_bytes()
+            # Optionally decrypt (iv+tag minimum 12+16 bytes)
+            if self._encryptor and len(raw) >= 12 + 16:
+                try:
+                    raw = self._encryptor.decrypt_payload(raw)
+                except Exception:
+                    pass  # legacy plain compressed
+            full_content = zlib.decompress(raw)
+            null_idx = full_content.index(b"\0")
+            content = full_content[null_idx + 1 :]
+            return content
+
+        # Try pack file when loose object missing
+        try:
+            from .pack import retrieve_from_pack
+            result = retrieve_from_pack(self.objects_dir, hash_id, expected_type=obj_type)
+            if result is not None:
+                return result[1]
+        except Exception:
+            pass
+        return None
 
     def exists(self, hash_id: str, obj_type: str) -> bool:
-        """Check if an object exists. Returns False for invalid hash (no raise)."""
+        """Check if an object exists (loose or pack). Returns False for invalid hash (no raise)."""
         if not _valid_object_hash(hash_id):
             return False
         obj_path = self._get_object_path(hash_id, obj_type)
-        return obj_path.exists()
+        if obj_path.exists():
+            return True
+        try:
+            from .pack import retrieve_from_pack
+            return retrieve_from_pack(self.objects_dir, hash_id, expected_type=obj_type) is not None
+        except Exception:
+            return False
 
     def delete(self, hash_id: str, obj_type: str) -> bool:
         """Delete an object. Returns True if deleted, False if not found."""

memvcs/core/pack.py

@@ -0,0 +1,278 @@
+"""
+Pack files and garbage collection for agmem.
+
+Pack: collect loose objects into single file + index. GC: delete unreachable objects, repack.
+"""
+
+import hashlib
+import struct
+import zlib
+from pathlib import Path
+from typing import Set, Dict, List, Optional, Tuple
+
+from .objects import ObjectStore
+from .refs import RefsManager
+
+PACK_MAGIC = b"PACK"
+PACK_VERSION = 2
+IDX_MAGIC = b"agidx"
+IDX_VERSION = 2
+OBJ_TYPE_BLOB = 1
+OBJ_TYPE_TREE = 2
+OBJ_TYPE_COMMIT = 3
+OBJ_TYPE_TAG = 4
+TYPE_TO_BYTE = {"blob": OBJ_TYPE_BLOB, "tree": OBJ_TYPE_TREE, "commit": OBJ_TYPE_COMMIT, "tag": OBJ_TYPE_TAG}
+BYTE_TO_TYPE = {v: k for k, v in TYPE_TO_BYTE.items()}
+
+
+def _pack_dir(objects_dir: Path) -> Path:
+    return objects_dir / "pack"
+
+
+def _get_loose_object_type(objects_dir: Path, hash_id: str) -> Optional[str]:
+    """Return obj_type for a loose object, or None if not found."""
+    if len(hash_id) < 4:
+        return None
+    prefix, suffix = hash_id[:2], hash_id[2:]
+    for obj_type in ["blob", "tree", "commit", "tag"]:
+        p = objects_dir / obj_type / prefix / suffix
+        if p.exists():
+            return obj_type
+    return None
+
+
+def list_loose_objects(objects_dir: Path) -> Set[str]:
+    """List all loose object hashes (blob, tree, commit, tag)."""
+    hashes = set()
+    for obj_type in ["blob", "tree", "commit", "tag"]:
+        type_dir = objects_dir / obj_type
+        if not type_dir.exists():
+            continue
+        for prefix_dir in type_dir.iterdir():
+            if not prefix_dir.is_dir():
+                continue
+            for f in prefix_dir.iterdir():
+                hash_id = prefix_dir.name + f.name
+                hashes.add(hash_id)
+    return hashes
+
+
+def reachable_from_refs(mem_dir: Path, store: ObjectStore, gc_prune_days: int = 90) -> Set[str]:
+    """Collect all object hashes reachable from branches, tags, and reflog (within prune window)."""
+    refs = RefsManager(mem_dir)
+    reachable = set()
+    # Branch tips
+    for b in refs.list_branches():
+        ch = refs.get_branch_commit(b)
+        if ch:
+            reachable.update(_collect_from_commit(store, ch))
+    # Tags
+    for t in refs.list_tags():
+        ch = refs.get_tag_commit(t)
+        if ch:
+            reachable.update(_collect_from_commit(store, ch))
+    # Reflog (simplified: just HEAD recent)
+    try:
+        log = refs.get_reflog("HEAD", max_count=1000)
+        for e in log:
+            h = e.get("hash")
+            if h:
+                reachable.update(_collect_from_commit(store, h))
+    except Exception:
+        pass
+    return reachable
+
+
+def _collect_from_commit(store: ObjectStore, commit_hash: str) -> Set[str]:
+    """Collect all object hashes reachable from a commit."""
+    from .remote import _collect_objects_from_commit
+
+    return _collect_objects_from_commit(store, commit_hash)
+
+
+def run_gc(
+    mem_dir: Path, store: ObjectStore, gc_prune_days: int = 90, dry_run: bool = False
+) -> Tuple[int, int]:
+    """
+    Garbage collect: delete unreachable loose objects.
+    Returns (deleted_count, bytes_freed). dry_run: only report, do not delete.
+    """
+    loose = list_loose_objects(mem_dir / "objects")
+    reachable = reachable_from_refs(mem_dir, store, gc_prune_days)
+    to_delete = loose - reachable
+    freed = 0
+    for hash_id in to_delete:
+        # Resolve type from path
+        for obj_type in ["blob", "tree", "commit", "tag"]:
+            p = store.objects_dir / obj_type / hash_id[:2] / hash_id[2:]
+            if p.exists():
+                if not dry_run:
+                    size = p.stat().st_size
+                    p.unlink()
+                    freed += size
+                else:
+                    freed += p.stat().st_size
+                break
+    return (len(to_delete), freed)
+
+
+def write_pack(
+    objects_dir: Path, store: ObjectStore, hash_to_type: Dict[str, str]
+) -> Tuple[Path, Path]:
+    """
+    Pack loose objects into a single pack file and index.
+    hash_to_type: map hash_id -> obj_type for objects to include.
+    Returns (pack_path, index_path). Does not delete loose objects.
+    """
+    if not hash_to_type:
+        raise ValueError("Cannot write empty pack")
+    pack_d = _pack_dir(objects_dir)
+    pack_d.mkdir(parents=True, exist_ok=True)
+
+    pack_header_len = len(PACK_MAGIC) + 4 + 4
+    pack_body = bytearray()
+    index_entries: List[Tuple[str, str, int]] = []  # (hash_id, obj_type, offset_in_file)
+    offset_in_file = pack_header_len
+
+    for hash_id in sorted(hash_to_type.keys()):
+        obj_type = hash_to_type[hash_id]
+        content = store.retrieve(hash_id, obj_type)
+        if content is None:
+            continue
+        header = f"{obj_type} {len(content)}\0".encode()
+        full = header + content
+        compressed = zlib.compress(full)
+        type_byte = TYPE_TO_BYTE.get(obj_type, OBJ_TYPE_BLOB)
+        size_bytes = struct.pack(">I", len(compressed))
+        chunk = bytes([type_byte]) + size_bytes + compressed
+        pack_body.extend(chunk)
+        index_entries.append((hash_id, obj_type, offset_in_file))
+        offset_in_file += len(chunk)
+
+    if not index_entries:
+        raise ValueError("No objects to pack")
+
+    pack_content = PACK_MAGIC + struct.pack(">I", PACK_VERSION) + struct.pack(">I", len(index_entries)) + bytes(pack_body)
+    pack_hash = hashlib.sha256(pack_content).digest()
+    pack_content += pack_hash
+
+    pack_name = f"pack-{pack_hash[:16].hex()}.pack"
+    pack_path = pack_d / pack_name
+    pack_path.write_bytes(pack_content)
+
+    index_content = bytearray(IDX_MAGIC + struct.pack(">I", IDX_VERSION) + struct.pack(">I", len(index_entries)))
+    for hash_id, obj_type, off in index_entries:
+        index_content.extend(bytes.fromhex(hash_id))
+        index_content.append(TYPE_TO_BYTE[obj_type])
+        index_content.extend(struct.pack(">I", off))
+    idx_hash = hashlib.sha256(index_content).digest()
+    index_content.extend(idx_hash)
+    idx_path = pack_path.with_suffix(".idx")
+    idx_path.write_bytes(index_content)
+
+    return (pack_path, idx_path)
+
+
+def _find_pack_index(objects_dir: Path) -> Optional[Path]:
+    """Return path to first .idx file in objects/pack, or None."""
+    pack_d = _pack_dir(objects_dir)
+    if not pack_d.exists():
+        return None
+    for p in pack_d.iterdir():
+        if p.suffix == ".idx":
+            return p
+    return None
+
+
+def retrieve_from_pack(objects_dir: Path, hash_id: str, expected_type: Optional[str] = None) -> Optional[Tuple[str, bytes]]:
+    """
+    Retrieve object from pack by hash. Returns (obj_type, content) or None.
+    If expected_type is set, only return if pack type matches.
+    """
+    idx_path = _find_pack_index(objects_dir)
+    if idx_path is None:
+        return None
+    pack_path = idx_path.with_suffix(".pack")
+    if not pack_path.exists():
+        return None
+
+    raw_idx = idx_path.read_bytes()
+    if len(raw_idx) < len(IDX_MAGIC) + 4 + 4 + 32 + 1 + 4 + 32:
+        return None
+    if raw_idx[: len(IDX_MAGIC)] != IDX_MAGIC:
+        return None
+    version = struct.unpack(">I", raw_idx[len(IDX_MAGIC) : len(IDX_MAGIC) + 4])[0]
+    if version != IDX_VERSION:
+        return None
+    count = struct.unpack(">I", raw_idx[len(IDX_MAGIC) + 4 : len(IDX_MAGIC) + 8])[0]
+    entry_size = 32 + 1 + 4
+    entries_start = len(IDX_MAGIC) + 8
+    entries_end = entries_start + count * entry_size
+    if entries_end + 32 > len(raw_idx):
+        return None
+    hash_hex = hash_id
+    if len(hash_hex) != 64:
+        return None
+    hash_bin = bytes.fromhex(hash_hex)
+    for i in range(count):
+        base = entries_start + i * entry_size
+        entry_hash = raw_idx[base : base + 32]
+        if entry_hash != hash_bin:
+            continue
+        type_byte = raw_idx[base + 32]
+        offset = struct.unpack(">I", raw_idx[base + 33 : base + 37])[0]
+        obj_type = BYTE_TO_TYPE.get(type_byte)
+        if obj_type is None:
+            continue
+        if expected_type is not None and obj_type != expected_type:
+            return None
+        pack_raw = pack_path.read_bytes()
+        header_size = len(PACK_MAGIC) + 4 + 4
+        if offset + 1 + 4 > len(pack_raw) - 32:
+            return None
+        size = struct.unpack(">I", pack_raw[offset + 1 : offset + 5])[0]
+        payload_start = offset + 5
+        payload_end = payload_start + size
+        if payload_end > len(pack_raw) - 32:
+            return None
+        compressed = pack_raw[payload_start:payload_end]
+        try:
+            full = zlib.decompress(compressed)
+        except Exception:
+            return None
+        null_idx = full.index(b"\0")
+        content = full[null_idx + 1 :]
+        return (obj_type, content)
+    return None
+
+
+def run_repack(
+    mem_dir: Path, store: ObjectStore, gc_prune_days: int = 90, dry_run: bool = False
+) -> Tuple[int, int]:
+    """
+    After GC: pack all reachable loose objects into a pack file, then delete those loose objects.
+    Returns (objects_packed, bytes_freed_from_loose).
+    """
+    objects_dir = mem_dir / "objects"
+    reachable = reachable_from_refs(mem_dir, store, gc_prune_days)
+    loose = list_loose_objects(objects_dir)
+    to_pack = reachable & loose
+    if not to_pack:
+        return (0, 0)
+    hash_to_type: Dict[str, str] = {}
+    for hash_id in to_pack:
+        obj_type = _get_loose_object_type(objects_dir, hash_id)
+        if obj_type:
+            hash_to_type[hash_id] = obj_type
+    if not hash_to_type:
+        return (0, 0)
+    if dry_run:
+        return (len(hash_to_type), 0)
+    write_pack(objects_dir, store, hash_to_type)
+    freed = 0
+    for hash_id, obj_type in hash_to_type.items():
+        p = store.objects_dir / obj_type / hash_id[:2] / hash_id[2:]
+        if p.exists():
+            freed += p.stat().st_size
+            p.unlink()
+    return (len(hash_to_type), freed)

memvcs/core/privacy_budget.py

@@ -0,0 +1,63 @@
+"""
+Differential privacy budget tracking for agmem.
+
+Per-repo epsilon spent; block when budget exceeded.
+"""
+
+import json
+import math
+from pathlib import Path
+from typing import Optional, Tuple
+
+
+def _budget_path(mem_dir: Path) -> Path:
+    return mem_dir / "privacy_budget.json"
+
+
+def load_budget(mem_dir: Path) -> Tuple[float, float, float]:
+    """Load (epsilon_spent, max_epsilon, delta). Returns (0, max, delta) if no file."""
+    path = _budget_path(mem_dir)
+    if not path.exists():
+        config = mem_dir / "config.json"
+        max_eps = 1.0
+        delta = 1e-5
+        if config.exists():
+            try:
+                c = json.loads(config.read_text())
+                dp = c.get("differential_privacy", {})
+                max_eps = float(dp.get("max_epsilon", 1.0))
+                delta = float(dp.get("delta", 1e-5))
+            except Exception:
+                pass
+        return (0.0, max_eps, delta)
+    try:
+        data = json.loads(path.read_text())
+        return (
+            float(data.get("epsilon_spent", 0)),
+            float(data.get("max_epsilon", 1.0)),
+            float(data.get("delta", 1e-5)),
+        )
+    except Exception:
+        return (0.0, 1.0, 1e-5)
+
+
+def spend_epsilon(mem_dir: Path, epsilon: float, max_epsilon: Optional[float] = None) -> bool:
+    """Record epsilon spent. Returns False if budget would be exceeded."""
+    spent, max_eps, delta = load_budget(mem_dir)
+    if max_epsilon is not None:
+        max_eps = max_epsilon
+    if spent + epsilon > max_eps:
+        return False
+    mem_dir.mkdir(parents=True, exist_ok=True)
+    path = _budget_path(mem_dir)
+    data = {"epsilon_spent": spent + epsilon, "max_epsilon": max_eps, "delta": delta}
+    path.write_text(json.dumps(data, indent=2))
+    return True
+
+
+def add_noise(value: float, sensitivity: float, epsilon: float, delta: float = 1e-5) -> float:
+    """Add Gaussian noise for (epsilon, delta)-DP. sigma = sensitivity * sqrt(2*ln(1.25/delta)) / epsilon."""
+    import random
+
+    sigma = sensitivity * math.sqrt(2 * math.log(1.25 / delta)) / epsilon
+    return value + random.gauss(0, sigma)