agmem 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

memvcs/core/remote.py

@@ -1,19 +1,24 @@
 """
-Remote sync for agmem - file-based push/pull/clone.
+Remote sync for agmem - file-based and cloud (S3/GCS) push/pull/clone.
 
-Supports file:// URLs for local or mounted directories.
+Supports file:// URLs and s3:///gs:// with optional distributed locking.
 """
 
 import json
 import shutil
 from pathlib import Path
-from typing import Optional, Set
+from typing import Optional, Set, Any
 from urllib.parse import urlparse
 
 from .objects import ObjectStore, Commit, Tree, Blob, _valid_object_hash
 from .refs import RefsManager, _ref_path_under_root
 
 
+def _is_cloud_remote(url: str) -> bool:
+    """Return True if URL is S3 or GCS (use storage adapter + optional lock)."""
+    return url.startswith("s3://") or url.startswith("gs://")
+
+
 def parse_remote_url(url: str) -> Path:
     """Parse remote URL to local path. Supports file:// only. Rejects path traversal."""
     parsed = urlparse(url)
@@ -62,6 +67,50 @@ def _collect_objects_from_commit(store: ObjectStore, commit_hash: str) -> Set[st
     return seen
 
 
+def _read_object_from_adapter(adapter: Any, hash_id: str) -> Optional[tuple]:
+    """Read object from storage adapter. Returns (obj_type, content_bytes) or None."""
+    import zlib
+    for obj_type in ["commit", "tree", "blob", "tag"]:
+        rel = f".mem/objects/{obj_type}/{hash_id[:2]}/{hash_id[2:]}"
+        if not adapter.exists(rel):
+            continue
+        try:
+            raw = adapter.read_file(rel)
+            full = zlib.decompress(raw)
+            null_idx = full.index(b"\0")
+            content = full[null_idx + 1:]
+            return (obj_type, content)
+        except Exception:
+            continue
+    return None
+
+
+def _collect_objects_from_commit_remote(adapter: Any, commit_hash: str) -> Set[str]:
+    """Collect object hashes reachable from a commit when reading from storage adapter."""
+    seen = set()
+    todo = [commit_hash]
+    while todo:
+        h = todo.pop()
+        if h in seen:
+            continue
+        seen.add(h)
+        pair = _read_object_from_adapter(adapter, h)
+        if pair is None:
+            continue
+        obj_type, content = pair
+        if obj_type == "commit":
+            data = json.loads(content)
+            todo.extend(data.get("parents", []))
+            if "tree" in data:
+                todo.append(data["tree"])
+        elif obj_type == "tree":
+            data = json.loads(content)
+            for e in data.get("entries", []):
+                if "hash" in e:
+                    todo.append(e["hash"])
+    return seen
+
+
 def _list_local_objects(objects_dir: Path) -> Set[str]:
     """List all object hashes in a .mem/objects directory."""
     hashes = set()
@@ -139,6 +188,113 @@ class Remote:
         self._config["remotes"][self.name]["url"] = url
         self._save_config(self._config)
 
+    def _push_via_storage(self, adapter: Any, branch: Optional[str] = None) -> str:
+        """Push objects and refs via storage adapter. Caller must hold lock if needed."""
+        refs = RefsManager(self.mem_dir)
+        store = ObjectStore(self.objects_dir)
+        to_push = set()
+        for b in refs.list_branches():
+            if branch and b != branch:
+                continue
+            ch = refs.get_branch_commit(b)
+            if ch:
+                to_push.update(_collect_objects_from_commit(store, ch))
+        for t in refs.list_tags():
+            ch = refs.get_tag_commit(t)
+            if ch:
+                to_push.update(_collect_objects_from_commit(store, ch))
+        copied = 0
+        for h in to_push:
+            obj_type = None
+            for otype in ["blob", "tree", "commit", "tag"]:
+                p = self.objects_dir / otype / h[:2] / h[2:]
+                if p.exists():
+                    obj_type = otype
+                    break
+            if not obj_type:
+                continue
+            rel = f".mem/objects/{obj_type}/{h[:2]}/{h[2:]}"
+            if not adapter.exists(rel):
+                try:
+                    data = p.read_bytes()
+                    adapter.makedirs(f".mem/objects/{obj_type}/{h[:2]}")
+                    adapter.write_file(rel, data)
+                    copied += 1
+                except Exception:
+                    pass
+        for b in refs.list_branches():
+            if branch and b != branch:
+                continue
+            ch = refs.get_branch_commit(b)
+            if ch and _ref_path_under_root(b, refs.heads_dir):
+                parent = str(Path(b).parent)
+                if parent != ".":
+                    adapter.makedirs(f".mem/refs/heads/{parent}")
+                adapter.write_file(f".mem/refs/heads/{b}", (ch + "\n").encode())
+        for t in refs.list_tags():
+            ch = refs.get_tag_commit(t)
+            if ch and _ref_path_under_root(t, refs.tags_dir):
+                parent = str(Path(t).parent)
+                if parent != ".":
+                    adapter.makedirs(f".mem/refs/tags/{parent}")
+                adapter.write_file(f".mem/refs/tags/{t}", (ch + "\n").encode())
+        try:
+            from .audit import append_audit
+            append_audit(self.mem_dir, "push", {"remote": self.name, "branch": branch, "copied": copied})
+        except Exception:
+            pass
+        return f"Pushed {copied} object(s) to {self.name}"
+
+    def _fetch_via_storage(self, adapter: Any, branch: Optional[str] = None) -> str:
+        """Fetch objects and refs via storage adapter. Caller must hold lock if needed."""
+        to_fetch = set()
+        try:
+            heads = adapter.list_dir(".mem/refs/heads")
+            for fi in heads:
+                if fi.is_dir:
+                    continue
+                branch_name = fi.path.replace(".mem/refs/heads/", "").replace("\\", "/").strip("/")
+                if branch and branch_name != branch:
+                    continue
+                data = adapter.read_file(fi.path)
+                ch = data.decode().strip()
+                if ch and _valid_object_hash(ch):
+                    to_fetch.update(_collect_objects_from_commit_remote(adapter, ch))
+            tags = adapter.list_dir(".mem/refs/tags")
+            for fi in tags:
+                if fi.is_dir:
+                    continue
+                data = adapter.read_file(fi.path)
+                ch = data.decode().strip()
+                if ch and _valid_object_hash(ch):
+                    to_fetch.update(_collect_objects_from_commit_remote(adapter, ch))
+        except Exception:
+            pass
+        if not to_fetch:
+            return f"Fetched 0 object(s) from {self.name}"
+        local_has = _list_local_objects(self.objects_dir)
+        missing = to_fetch - local_has
+        copied = 0
+        for h in missing:
+            for otype in ["blob", "tree", "commit", "tag"]:
+                rel = f".mem/objects/{otype}/{h[:2]}/{h[2:]}"
+                if adapter.exists(rel):
+                    try:
+                        data = adapter.read_file(rel)
+                        p = self.objects_dir / otype / h[:2] / h[2:]
+                        p.parent.mkdir(parents=True, exist_ok=True)
+                        p.write_bytes(data)
+                        copied += 1
+                    except Exception:
+                        pass
+                    break
+        try:
+            from .audit import append_audit
+            append_audit(self.mem_dir, "fetch", {"remote": self.name, "branch": branch, "copied": copied})
+        except Exception:
+            pass
+        return f"Fetched {copied} object(s) from {self.name}"
+
     def push(self, branch: Optional[str] = None) -> str:
         """
         Push objects and refs to remote.
@@ -148,6 +304,22 @@ class Remote:
         if not url:
             raise ValueError(f"Remote '{self.name}' has no URL configured")
 
+        if _is_cloud_remote(url):
+            try:
+                from .storage import get_adapter
+                from .storage.base import LockError
+                adapter = get_adapter(url, self._config)
+                lock_name = "agmem-push"
+                adapter.acquire_lock(lock_name, 30)
+                try:
+                    return self._push_via_storage(adapter, branch)
+                finally:
+                    adapter.release_lock(lock_name)
+            except LockError as e:
+                raise ValueError(f"Could not acquire remote lock: {e}") from e
+            except Exception as e:
+                raise ValueError(f"Push to cloud failed: {e}") from e
+
         remote_path = parse_remote_url(url)
         remote_mem = remote_path / ".mem"
         remote_objects = remote_mem / "objects"
@@ -164,6 +336,28 @@ class Remote:
         refs = RefsManager(self.mem_dir)
         store = ObjectStore(self.objects_dir)
 
+        # Push conflict detection: remote tip must be ancestor of local tip (non-fast-forward reject)
+        remote_heads = remote_refs / "heads"
+        for b in refs.list_branches():
+            if branch and b != branch:
+                continue
+            local_ch = refs.get_branch_commit(b)
+            if not local_ch:
+                continue
+            remote_branch_file = remote_heads / b
+            if remote_branch_file.exists():
+                remote_ch = remote_branch_file.read_text().strip()
+                if remote_ch and _valid_object_hash(remote_ch):
+                    from .merge import MergeEngine
+                    from .repository import Repository
+
+                    repo = Repository(self.repo_path)
+                    engine = MergeEngine(repo)
+                    if not engine.find_common_ancestor(remote_ch, local_ch) == remote_ch:
+                        raise ValueError(
+                            "Push rejected: remote has diverged. Pull and merge first."
+                        )
+
         # Collect objects to push
         to_push = set()
         for b in refs.list_branches():
@@ -206,6 +400,14 @@ class Remote:
                 (remote_tags_dir / t).parent.mkdir(parents=True, exist_ok=True)
                 (remote_tags_dir / t).write_text(ch + "\n")
 
+        try:
+            from .audit import append_audit
+
+            append_audit(
+                self.mem_dir, "push", {"remote": self.name, "branch": branch, "copied": copied}
+            )
+        except Exception:
+            pass
         return f"Pushed {copied} object(s) to {self.name}"
 
     def fetch(self, branch: Optional[str] = None) -> str:
@@ -217,6 +419,22 @@ class Remote:
         if not url:
             raise ValueError(f"Remote '{self.name}' has no URL configured")
 
+        if _is_cloud_remote(url):
+            try:
+                from .storage import get_adapter
+                from .storage.base import LockError
+                adapter = get_adapter(url, self._config)
+                lock_name = "agmem-fetch"
+                adapter.acquire_lock(lock_name, 30)
+                try:
+                    return self._fetch_via_storage(adapter, branch)
+                finally:
+                    adapter.release_lock(lock_name)
+            except LockError as e:
+                raise ValueError(f"Could not acquire remote lock: {e}") from e
+            except Exception as e:
+                raise ValueError(f"Fetch from cloud failed: {e}") from e
+
         remote_path = parse_remote_url(url)
         remote_objects = remote_path / ".mem" / "objects"
         remote_refs = remote_path / ".mem" / "refs"
@@ -275,4 +493,12 @@ class Remote:
                     if ch:
                         refs.create_tag(tag_name, ch)
 
+        try:
+            from .audit import append_audit
+
+            append_audit(
+                self.mem_dir, "fetch", {"remote": self.name, "branch": branch, "copied": copied}
+            )
+        except Exception:
+            pass
         return f"Fetched {copied} object(s) from {self.name}"

memvcs/core/repository.py

@@ -36,7 +36,23 @@ class Repository:
 
     def _init_components(self):
         """Initialize repository components."""
-        self.object_store = ObjectStore(self.mem_dir / "objects")
+        encryptor = None
+        try:
+            config = self.get_config()
+            if config.get("encryption", {}).get("enabled"):
+                from .encryption import (
+                    load_encryption_config,
+                    ObjectStoreEncryptor,
+                    get_key_from_env_or_cache,
+                )
+
+                if load_encryption_config(self.mem_dir):
+                    encryptor = ObjectStoreEncryptor(
+                        lambda: get_key_from_env_or_cache(self.mem_dir)
+                    )
+        except Exception:
+            pass
+        self.object_store = ObjectStore(self.mem_dir / "objects", encryptor=encryptor)
         self.staging = StagingArea(self.mem_dir)
         self.refs = RefsManager(self.mem_dir)
 
@@ -96,6 +112,14 @@ class Repository:
         # Initialize HEAD
         repo.refs.init_head("main")
 
+        # Tamper-evident audit
+        try:
+            from .audit import append_audit
+
+            append_audit(repo.mem_dir, "init", {"author": author_name, "branch": "main"})
+        except Exception:
+            pass
+
         return repo
 
     def is_valid_repo(self) -> bool:
@@ -115,6 +139,12 @@ class Repository:
     def set_config(self, config: Dict[str, Any]):
         """Set repository configuration."""
         self.config_file.write_text(json.dumps(config, indent=2))
+        try:
+            from .audit import append_audit
+
+            append_audit(self.mem_dir, "config_change", {})
+        except Exception:
+            pass
 
     def get_author(self) -> str:
         """Get the configured author string."""
@@ -298,6 +328,30 @@ class Repository:
         head_commit = self.get_head_commit()
         parents = [head_commit.store(self.object_store)] if head_commit else []
 
+        # Cryptographic verification: Merkle root + optional signing (private key from env)
+        meta = dict(metadata or {})
+        try:
+            from .crypto_verify import (
+                _collect_blob_hashes_from_tree,
+                build_merkle_tree,
+                load_private_key_from_env,
+                sign_merkle_root,
+                ED25519_AVAILABLE,
+            )
+            from .objects import Tree
+
+            tree = Tree.load(self.object_store, tree_hash)
+            if tree:
+                blobs = _collect_blob_hashes_from_tree(self.object_store, tree_hash)
+                merkle_root = build_merkle_tree(blobs)
+                meta["merkle_root"] = merkle_root
+                if ED25519_AVAILABLE:
+                    private_pem = load_private_key_from_env()
+                    if private_pem:
+                        meta["signature"] = sign_merkle_root(merkle_root, private_pem)
+        except Exception:
+            pass
+
         # Create commit
         commit = Commit(
             tree=tree_hash,
@@ -305,7 +359,7 @@ class Repository:
             author=self.get_author(),
             timestamp=datetime.utcnow().isoformat() + "Z",
             message=message,
-            metadata=metadata or {},
+            metadata=meta,
         )
         commit_hash = commit.store(self.object_store)
 
@@ -313,6 +367,14 @@ class Repository:
         old_hash = parents[0] if parents else "0" * 64
         self.refs.append_reflog("HEAD", old_hash, commit_hash, f"commit: {message}")
 
+        # Audit
+        try:
+            from .audit import append_audit
+
+            append_audit(self.mem_dir, "commit", {"commit": commit_hash, "message": message})
+        except Exception:
+            pass
+
         # Update HEAD
         head = self.refs.get_head()
         if head["type"] == "branch":
@@ -354,6 +416,16 @@ class Repository:
         if not tree:
             raise ValueError(f"Reference not found: {ref}")
 
+        # Cryptographic verification: reject if Merkle/signature invalid
+        try:
+            from .crypto_verify import verify_commit_optional
+
+            verify_commit_optional(
+                self.object_store, commit_hash, mem_dir=self.mem_dir, strict=False
+            )
+        except ValueError as e:
+            raise ValueError(str(e))
+
         # Check for uncommitted changes
         if not force:
             staged = self.staging.get_staged_files()
@@ -377,6 +449,14 @@ class Repository:
         # Clear staging
         self.staging.clear()
 
+        # Audit
+        try:
+            from .audit import append_audit
+
+            append_audit(self.mem_dir, "checkout", {"ref": ref, "commit": commit_hash})
+        except Exception:
+            pass
+
         return commit_hash
 
     def get_status(self) -> Dict[str, Any]:

memvcs/core/temporal_index.py

@@ -110,3 +110,12 @@ class TemporalIndex:
         if idx == 0:
             return None  # All commits are after the requested time
         return timeline[idx - 1][1]
+
+    def range_query(self, start_str: str, end_str: str) -> List[Tuple[datetime, str]]:
+        """Return (timestamp, commit_hash) entries in [start, end]. For range queries."""
+        start_dt = _parse_iso_timestamp(start_str)
+        end_dt = _parse_iso_timestamp(end_str)
+        if not start_dt or not end_dt:
+            return []
+        timeline = self._build_commit_timeline()
+        return [(t, h) for t, h in timeline if start_dt <= t <= end_dt]

memvcs/core/trust.py

@@ -0,0 +1,103 @@
+"""
+Multi-agent trust and identity model for agmem.
+
+Trust store: map public keys to levels (full | conditional | untrusted).
+Used on pull/merge to decide auto-merge, prompt, or block.
+"""
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Optional, Dict, List, Any, Union
+
+TRUST_LEVELS = ("full", "conditional", "untrusted")
+
+
+def _trust_dir(mem_dir: Path) -> Path:
+    return mem_dir / "trust"
+
+
+def _trust_file(mem_dir: Path) -> Path:
+    return _trust_dir(mem_dir) / "trust.json"
+
+
+def _key_id(public_key_pem: bytes) -> str:
+    """Stable id for a public key (hash of PEM)."""
+    return hashlib.sha256(public_key_pem).hexdigest()[:16]
+
+
+def _ensure_bytes(pem: Union[bytes, str]) -> bytes:
+    """Normalize PEM to bytes for hashing/serialization."""
+    return pem.encode("utf-8") if isinstance(pem, str) else pem
+
+
+def load_trust_store(mem_dir: Path) -> List[Dict[str, Any]]:
+    """Load trust store: list of { key_id, public_key_pem, level }."""
+    path = _trust_file(mem_dir)
+    if not path.exists():
+        return []
+    try:
+        data = json.loads(path.read_text())
+        return data.get("entries", [])
+    except Exception:
+        return []
+
+
+def get_trust_level(mem_dir: Path, public_key_pem: Union[bytes, str]) -> Optional[str]:
+    """Get trust level for a public key. Returns 'full'|'conditional'|'untrusted' or None if unknown."""
+    pem_b = _ensure_bytes(public_key_pem)
+    kid = _key_id(pem_b)
+    key_pem_str = pem_b.decode("utf-8")
+    for e in load_trust_store(mem_dir):
+        entry_id = e.get("key_id") or _key_id((e.get("public_key_pem") or "").encode())
+        if entry_id == kid:
+            return e.get("level")
+        if e.get("public_key_pem") == key_pem_str:
+            return e.get("level")
+    return None
+
+
+# Reasonable upper bound for PEM to avoid DoS (typical Ed25519 public PEM ~120 bytes)
+_MAX_PEM_BYTES = 8192
+
+
+def set_trust(mem_dir: Path, public_key_pem: Union[bytes, str], level: str) -> None:
+    """Set trust level for a public key. level: full | conditional | untrusted."""
+    if level not in TRUST_LEVELS:
+        raise ValueError(f"level must be one of {TRUST_LEVELS}")
+    pem_b = _ensure_bytes(public_key_pem)
+    if len(pem_b) > _MAX_PEM_BYTES:
+        raise ValueError("Public key PEM exceeds maximum size")
+    kid = _key_id(pem_b)
+    key_pem_str = pem_b.decode("utf-8")
+    _trust_dir(mem_dir).mkdir(parents=True, exist_ok=True)
+    entries = load_trust_store(mem_dir)
+    entries = [
+        e
+        for e in entries
+        if (e.get("key_id") or _key_id((e.get("public_key_pem") or "").encode())) != kid
+    ]
+    entries.append({"key_id": kid, "public_key_pem": key_pem_str, "level": level})
+    _trust_file(mem_dir).write_text(json.dumps({"entries": entries}, indent=2))
+
+
+def find_verifying_key(mem_dir: Path, commit_metadata: Dict[str, Any]) -> Optional[bytes]:
+    """
+    Try each key in trust store to verify the commit's signature.
+    commit_metadata should have merkle_root and signature.
+    Returns public_key_pem of first key that verifies, or None.
+    """
+    from .crypto_verify import verify_signature
+
+    root = commit_metadata.get("merkle_root")
+    sig = commit_metadata.get("signature")
+    if not root or not sig:
+        return None
+    for e in load_trust_store(mem_dir):
+        pem = e.get("public_key_pem")
+        if not pem:
+            continue
+        pem_b = _ensure_bytes(pem)
+        if verify_signature(root, sig, pem_b):
+            return pem_b
+    return None

memvcs/core/vector_store.py

@@ -56,6 +56,19 @@ class VectorStore:
                 "On macOS, try: brew install python (for Homebrew SQLite)"
             ) from e
 
+    def _device(self) -> str:
+        """Return device for embeddings: cuda/mps/cpu. GPU acceleration when available."""
+        try:
+            import torch
+
+            if torch.cuda.is_available():
+                return "cuda"
+            if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+                return "mps"
+        except ImportError:
+            pass
+        return "cpu"
+
     def _get_model(self):
         """Lazy-load the sentence-transformers model."""
         if self._model is not None:
@@ -64,7 +77,8 @@ class VectorStore:
         try:
             from sentence_transformers import SentenceTransformer
 
-            self._model = SentenceTransformer("all-MiniLM-L6-v2")
+            device = self._device()
+            self._model = SentenceTransformer("all-MiniLM-L6-v2", device=device)
             return self._model
         except ImportError as e:
             raise ImportError(