agmem 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

memvcs/core/pack.py

@@ -4,7 +4,8 @@ Pack files and garbage collection for agmem.
 Pack: collect loose objects into single file + index. GC: delete unreachable objects, repack.
 """
 
-import json
+import hashlib
+import struct
 import zlib
 from pathlib import Path
 from typing import Set, Dict, List, Optional, Tuple
@@ -12,11 +13,34 @@ from typing import Set, Dict, List, Optional, Tuple
 from .objects import ObjectStore
 from .refs import RefsManager
 
+PACK_MAGIC = b"PACK"
+PACK_VERSION = 2
+IDX_MAGIC = b"agidx"
+IDX_VERSION = 2
+OBJ_TYPE_BLOB = 1
+OBJ_TYPE_TREE = 2
+OBJ_TYPE_COMMIT = 3
+OBJ_TYPE_TAG = 4
+TYPE_TO_BYTE = {"blob": OBJ_TYPE_BLOB, "tree": OBJ_TYPE_TREE, "commit": OBJ_TYPE_COMMIT, "tag": OBJ_TYPE_TAG}
+BYTE_TO_TYPE = {v: k for k, v in TYPE_TO_BYTE.items()}
+
 
 def _pack_dir(objects_dir: Path) -> Path:
     return objects_dir / "pack"
 
 
+def _get_loose_object_type(objects_dir: Path, hash_id: str) -> Optional[str]:
+    """Return obj_type for a loose object, or None if not found."""
+    if len(hash_id) < 4:
+        return None
+    prefix, suffix = hash_id[:2], hash_id[2:]
+    for obj_type in ["blob", "tree", "commit", "tag"]:
+        p = objects_dir / obj_type / prefix / suffix
+        if p.exists():
+            return obj_type
+    return None
+
+
 def list_loose_objects(objects_dir: Path) -> Set[str]:
     """List all loose object hashes (blob, tree, commit, tag)."""
     hashes = set()
@@ -90,3 +114,165 @@ def run_gc(
                     freed += p.stat().st_size
                 break
     return (len(to_delete), freed)
+
+
+def write_pack(
+    objects_dir: Path, store: ObjectStore, hash_to_type: Dict[str, str]
+) -> Tuple[Path, Path]:
+    """
+    Pack loose objects into a single pack file and index.
+    hash_to_type: map hash_id -> obj_type for objects to include.
+    Returns (pack_path, index_path). Does not delete loose objects.
+    """
+    if not hash_to_type:
+        raise ValueError("Cannot write empty pack")
+    pack_d = _pack_dir(objects_dir)
+    pack_d.mkdir(parents=True, exist_ok=True)
+
+    pack_header_len = len(PACK_MAGIC) + 4 + 4
+    pack_body = bytearray()
+    index_entries: List[Tuple[str, str, int]] = []  # (hash_id, obj_type, offset_in_file)
+    offset_in_file = pack_header_len
+
+    for hash_id in sorted(hash_to_type.keys()):
+        obj_type = hash_to_type[hash_id]
+        content = store.retrieve(hash_id, obj_type)
+        if content is None:
+            continue
+        header = f"{obj_type} {len(content)}\0".encode()
+        full = header + content
+        compressed = zlib.compress(full)
+        type_byte = TYPE_TO_BYTE.get(obj_type, OBJ_TYPE_BLOB)
+        size_bytes = struct.pack(">I", len(compressed))
+        chunk = bytes([type_byte]) + size_bytes + compressed
+        pack_body.extend(chunk)
+        index_entries.append((hash_id, obj_type, offset_in_file))
+        offset_in_file += len(chunk)
+
+    if not index_entries:
+        raise ValueError("No objects to pack")
+
+    pack_content = PACK_MAGIC + struct.pack(">I", PACK_VERSION) + struct.pack(">I", len(index_entries)) + bytes(pack_body)
+    pack_hash = hashlib.sha256(pack_content).digest()
+    pack_content += pack_hash
+
+    pack_name = f"pack-{pack_hash[:16].hex()}.pack"
+    pack_path = pack_d / pack_name
+    pack_path.write_bytes(pack_content)
+
+    index_content = bytearray(IDX_MAGIC + struct.pack(">I", IDX_VERSION) + struct.pack(">I", len(index_entries)))
+    for hash_id, obj_type, off in index_entries:
+        index_content.extend(bytes.fromhex(hash_id))
+        index_content.append(TYPE_TO_BYTE[obj_type])
+        index_content.extend(struct.pack(">I", off))
+    idx_hash = hashlib.sha256(index_content).digest()
+    index_content.extend(idx_hash)
+    idx_path = pack_path.with_suffix(".idx")
+    idx_path.write_bytes(index_content)
+
+    return (pack_path, idx_path)
+
+
+def _find_pack_index(objects_dir: Path) -> Optional[Path]:
+    """Return path to first .idx file in objects/pack, or None."""
+    pack_d = _pack_dir(objects_dir)
+    if not pack_d.exists():
+        return None
+    for p in pack_d.iterdir():
+        if p.suffix == ".idx":
+            return p
+    return None
+
+
+def retrieve_from_pack(objects_dir: Path, hash_id: str, expected_type: Optional[str] = None) -> Optional[Tuple[str, bytes]]:
+    """
+    Retrieve object from pack by hash. Returns (obj_type, content) or None.
+    If expected_type is set, only return if pack type matches.
+    """
+    idx_path = _find_pack_index(objects_dir)
+    if idx_path is None:
+        return None
+    pack_path = idx_path.with_suffix(".pack")
+    if not pack_path.exists():
+        return None
+
+    raw_idx = idx_path.read_bytes()
+    if len(raw_idx) < len(IDX_MAGIC) + 4 + 4 + 32 + 1 + 4 + 32:
+        return None
+    if raw_idx[: len(IDX_MAGIC)] != IDX_MAGIC:
+        return None
+    version = struct.unpack(">I", raw_idx[len(IDX_MAGIC) : len(IDX_MAGIC) + 4])[0]
+    if version != IDX_VERSION:
+        return None
+    count = struct.unpack(">I", raw_idx[len(IDX_MAGIC) + 4 : len(IDX_MAGIC) + 8])[0]
+    entry_size = 32 + 1 + 4
+    entries_start = len(IDX_MAGIC) + 8
+    entries_end = entries_start + count * entry_size
+    if entries_end + 32 > len(raw_idx):
+        return None
+    hash_hex = hash_id
+    if len(hash_hex) != 64:
+        return None
+    hash_bin = bytes.fromhex(hash_hex)
+    for i in range(count):
+        base = entries_start + i * entry_size
+        entry_hash = raw_idx[base : base + 32]
+        if entry_hash != hash_bin:
+            continue
+        type_byte = raw_idx[base + 32]
+        offset = struct.unpack(">I", raw_idx[base + 33 : base + 37])[0]
+        obj_type = BYTE_TO_TYPE.get(type_byte)
+        if obj_type is None:
+            continue
+        if expected_type is not None and obj_type != expected_type:
+            return None
+        pack_raw = pack_path.read_bytes()
+        header_size = len(PACK_MAGIC) + 4 + 4
+        if offset + 1 + 4 > len(pack_raw) - 32:
+            return None
+        size = struct.unpack(">I", pack_raw[offset + 1 : offset + 5])[0]
+        payload_start = offset + 5
+        payload_end = payload_start + size
+        if payload_end > len(pack_raw) - 32:
+            return None
+        compressed = pack_raw[payload_start:payload_end]
+        try:
+            full = zlib.decompress(compressed)
+        except Exception:
+            return None
+        null_idx = full.index(b"\0")
+        content = full[null_idx + 1 :]
+        return (obj_type, content)
+    return None
+
+
+def run_repack(
+    mem_dir: Path, store: ObjectStore, gc_prune_days: int = 90, dry_run: bool = False
+) -> Tuple[int, int]:
+    """
+    After GC: pack all reachable loose objects into a pack file, then delete those loose objects.
+    Returns (objects_packed, bytes_freed_from_loose).
+    """
+    objects_dir = mem_dir / "objects"
+    reachable = reachable_from_refs(mem_dir, store, gc_prune_days)
+    loose = list_loose_objects(objects_dir)
+    to_pack = reachable & loose
+    if not to_pack:
+        return (0, 0)
+    hash_to_type: Dict[str, str] = {}
+    for hash_id in to_pack:
+        obj_type = _get_loose_object_type(objects_dir, hash_id)
+        if obj_type:
+            hash_to_type[hash_id] = obj_type
+    if not hash_to_type:
+        return (0, 0)
+    if dry_run:
+        return (len(hash_to_type), 0)
+    write_pack(objects_dir, store, hash_to_type)
+    freed = 0
+    for hash_id, obj_type in hash_to_type.items():
+        p = store.objects_dir / obj_type / hash_id[:2] / hash_id[2:]
+        if p.exists():
+            freed += p.stat().st_size
+            p.unlink()
+    return (len(hash_to_type), freed)

memvcs/core/remote.py

@@ -1,19 +1,24 @@
 """
-Remote sync for agmem - file-based push/pull/clone.
+Remote sync for agmem - file-based and cloud (S3/GCS) push/pull/clone.
 
-Supports file:// URLs for local or mounted directories.
+Supports file:// URLs and s3:///gs:// with optional distributed locking.
 """
 
 import json
 import shutil
 from pathlib import Path
-from typing import Optional, Set
+from typing import Optional, Set, Any
 from urllib.parse import urlparse
 
 from .objects import ObjectStore, Commit, Tree, Blob, _valid_object_hash
 from .refs import RefsManager, _ref_path_under_root
 
 
+def _is_cloud_remote(url: str) -> bool:
+    """Return True if URL is S3 or GCS (use storage adapter + optional lock)."""
+    return url.startswith("s3://") or url.startswith("gs://")
+
+
 def parse_remote_url(url: str) -> Path:
     """Parse remote URL to local path. Supports file:// only. Rejects path traversal."""
     parsed = urlparse(url)
@@ -62,6 +67,50 @@ def _collect_objects_from_commit(store: ObjectStore, commit_hash: str) -> Set[st
     return seen
 
 
+def _read_object_from_adapter(adapter: Any, hash_id: str) -> Optional[tuple]:
+    """Read object from storage adapter. Returns (obj_type, content_bytes) or None."""
+    import zlib
+    for obj_type in ["commit", "tree", "blob", "tag"]:
+        rel = f".mem/objects/{obj_type}/{hash_id[:2]}/{hash_id[2:]}"
+        if not adapter.exists(rel):
+            continue
+        try:
+            raw = adapter.read_file(rel)
+            full = zlib.decompress(raw)
+            null_idx = full.index(b"\0")
+            content = full[null_idx + 1:]
+            return (obj_type, content)
+        except Exception:
+            continue
+    return None
+
+
+def _collect_objects_from_commit_remote(adapter: Any, commit_hash: str) -> Set[str]:
+    """Collect object hashes reachable from a commit when reading from storage adapter."""
+    seen = set()
+    todo = [commit_hash]
+    while todo:
+        h = todo.pop()
+        if h in seen:
+            continue
+        seen.add(h)
+        pair = _read_object_from_adapter(adapter, h)
+        if pair is None:
+            continue
+        obj_type, content = pair
+        if obj_type == "commit":
+            data = json.loads(content)
+            todo.extend(data.get("parents", []))
+            if "tree" in data:
+                todo.append(data["tree"])
+        elif obj_type == "tree":
+            data = json.loads(content)
+            for e in data.get("entries", []):
+                if "hash" in e:
+                    todo.append(e["hash"])
+    return seen
+
+
 def _list_local_objects(objects_dir: Path) -> Set[str]:
     """List all object hashes in a .mem/objects directory."""
     hashes = set()
@@ -139,6 +188,113 @@ class Remote:
         self._config["remotes"][self.name]["url"] = url
         self._save_config(self._config)
 
+    def _push_via_storage(self, adapter: Any, branch: Optional[str] = None) -> str:
+        """Push objects and refs via storage adapter. Caller must hold lock if needed."""
+        refs = RefsManager(self.mem_dir)
+        store = ObjectStore(self.objects_dir)
+        to_push = set()
+        for b in refs.list_branches():
+            if branch and b != branch:
+                continue
+            ch = refs.get_branch_commit(b)
+            if ch:
+                to_push.update(_collect_objects_from_commit(store, ch))
+        for t in refs.list_tags():
+            ch = refs.get_tag_commit(t)
+            if ch:
+                to_push.update(_collect_objects_from_commit(store, ch))
+        copied = 0
+        for h in to_push:
+            obj_type = None
+            for otype in ["blob", "tree", "commit", "tag"]:
+                p = self.objects_dir / otype / h[:2] / h[2:]
+                if p.exists():
+                    obj_type = otype
+                    break
+            if not obj_type:
+                continue
+            rel = f".mem/objects/{obj_type}/{h[:2]}/{h[2:]}"
+            if not adapter.exists(rel):
+                try:
+                    data = p.read_bytes()
+                    adapter.makedirs(f".mem/objects/{obj_type}/{h[:2]}")
+                    adapter.write_file(rel, data)
+                    copied += 1
+                except Exception:
+                    pass
+        for b in refs.list_branches():
+            if branch and b != branch:
+                continue
+            ch = refs.get_branch_commit(b)
+            if ch and _ref_path_under_root(b, refs.heads_dir):
+                parent = str(Path(b).parent)
+                if parent != ".":
+                    adapter.makedirs(f".mem/refs/heads/{parent}")
+                adapter.write_file(f".mem/refs/heads/{b}", (ch + "\n").encode())
+        for t in refs.list_tags():
+            ch = refs.get_tag_commit(t)
+            if ch and _ref_path_under_root(t, refs.tags_dir):
+                parent = str(Path(t).parent)
+                if parent != ".":
+                    adapter.makedirs(f".mem/refs/tags/{parent}")
+                adapter.write_file(f".mem/refs/tags/{t}", (ch + "\n").encode())
+        try:
+            from .audit import append_audit
+            append_audit(self.mem_dir, "push", {"remote": self.name, "branch": branch, "copied": copied})
+        except Exception:
+            pass
+        return f"Pushed {copied} object(s) to {self.name}"
+
+    def _fetch_via_storage(self, adapter: Any, branch: Optional[str] = None) -> str:
+        """Fetch objects and refs via storage adapter. Caller must hold lock if needed."""
+        to_fetch = set()
+        try:
+            heads = adapter.list_dir(".mem/refs/heads")
+            for fi in heads:
+                if fi.is_dir:
+                    continue
+                branch_name = fi.path.replace(".mem/refs/heads/", "").replace("\\", "/").strip("/")
+                if branch and branch_name != branch:
+                    continue
+                data = adapter.read_file(fi.path)
+                ch = data.decode().strip()
+                if ch and _valid_object_hash(ch):
+                    to_fetch.update(_collect_objects_from_commit_remote(adapter, ch))
+            tags = adapter.list_dir(".mem/refs/tags")
+            for fi in tags:
+                if fi.is_dir:
+                    continue
+                data = adapter.read_file(fi.path)
+                ch = data.decode().strip()
+                if ch and _valid_object_hash(ch):
+                    to_fetch.update(_collect_objects_from_commit_remote(adapter, ch))
+        except Exception:
+            pass
+        if not to_fetch:
+            return f"Fetched 0 object(s) from {self.name}"
+        local_has = _list_local_objects(self.objects_dir)
+        missing = to_fetch - local_has
+        copied = 0
+        for h in missing:
+            for otype in ["blob", "tree", "commit", "tag"]:
+                rel = f".mem/objects/{otype}/{h[:2]}/{h[2:]}"
+                if adapter.exists(rel):
+                    try:
+                        data = adapter.read_file(rel)
+                        p = self.objects_dir / otype / h[:2] / h[2:]
+                        p.parent.mkdir(parents=True, exist_ok=True)
+                        p.write_bytes(data)
+                        copied += 1
+                    except Exception:
+                        pass
+                    break
+        try:
+            from .audit import append_audit
+            append_audit(self.mem_dir, "fetch", {"remote": self.name, "branch": branch, "copied": copied})
+        except Exception:
+            pass
+        return f"Fetched {copied} object(s) from {self.name}"
+
     def push(self, branch: Optional[str] = None) -> str:
         """
         Push objects and refs to remote.
@@ -148,6 +304,22 @@ class Remote:
         if not url:
             raise ValueError(f"Remote '{self.name}' has no URL configured")
 
+        if _is_cloud_remote(url):
+            try:
+                from .storage import get_adapter
+                from .storage.base import LockError
+                adapter = get_adapter(url, self._config)
+                lock_name = "agmem-push"
+                adapter.acquire_lock(lock_name, 30)
+                try:
+                    return self._push_via_storage(adapter, branch)
+                finally:
+                    adapter.release_lock(lock_name)
+            except LockError as e:
+                raise ValueError(f"Could not acquire remote lock: {e}") from e
+            except Exception as e:
+                raise ValueError(f"Push to cloud failed: {e}") from e
+
         remote_path = parse_remote_url(url)
         remote_mem = remote_path / ".mem"
         remote_objects = remote_mem / "objects"
@@ -247,6 +419,22 @@ class Remote:
         if not url:
             raise ValueError(f"Remote '{self.name}' has no URL configured")
 
+        if _is_cloud_remote(url):
+            try:
+                from .storage import get_adapter
+                from .storage.base import LockError
+                adapter = get_adapter(url, self._config)
+                lock_name = "agmem-fetch"
+                adapter.acquire_lock(lock_name, 30)
+                try:
+                    return self._fetch_via_storage(adapter, branch)
+                finally:
+                    adapter.release_lock(lock_name)
+            except LockError as e:
+                raise ValueError(f"Could not acquire remote lock: {e}") from e
+            except Exception as e:
+                raise ValueError(f"Fetch from cloud failed: {e}") from e
+
         remote_path = parse_remote_url(url)
         remote_objects = remote_path / ".mem" / "objects"
         remote_refs = remote_path / ".mem" / "refs"

memvcs/core/zk_proofs.py

@@ -1,26 +1,158 @@
 """
-Zero-knowledge proof system for agmem (stub).
+Zero-knowledge proof system for agmem.
 
-Planned: zk-SNARKs (Groth16) for keyword containment, memory freshness, competence verification.
-Requires optional zk extra (circuit lib, proving system). Trusted setup: public ceremony or small multi-party.
+Hash/signature-based proofs: keyword containment (Merkle set membership),
+memory freshness (signed timestamp). Full zk-SNARK backend can be added later.
 """
 
+import base64
+import hashlib
+import json
+import os
 from pathlib import Path
-from typing import Optional, Tuple
+from typing import Optional, List, Tuple, Any, Dict
+
+from .crypto_verify import (
+    build_merkle_tree,
+    merkle_proof,
+    verify_merkle_proof,
+    load_public_key,
+    load_private_key_from_env,
+    sign_merkle_root,
+    verify_signature,
+    ED25519_AVAILABLE,
+)
+
+
+def _word_hashes(content: str) -> List[str]:
+    """Extract words and return sorted list of SHA-256 hashes (hex)."""
+    words = set()
+    for word in content.split():
+        w = word.strip().lower()
+        if len(w) >= 1:
+            words.add(w)
+    return sorted(hashlib.sha256(w.encode()).hexdigest() for w in words)
 
 
 def prove_keyword_containment(memory_path: Path, keyword: str, output_proof_path: Path) -> bool:
-    """Prove memory file contains keyword without revealing content. Stub: returns False until zk backend added."""
-    return False
+    """
+    Prove memory file contains keyword without revealing content.
+    Proof: Merkle set membership of H(keyword) over word hashes in file.
+    """
+    if not memory_path.exists() or not memory_path.is_file():
+        return False
+    try:
+        content = memory_path.read_text(encoding="utf-8", errors="replace")
+    except Exception:
+        return False
+    word_hashes_list = _word_hashes(content)
+    keyword_hash = hashlib.sha256(keyword.strip().lower().encode()).hexdigest()
+    if keyword_hash not in word_hashes_list:
+        return False
+    root = build_merkle_tree(word_hashes_list)
+    proof_path_list = merkle_proof(word_hashes_list, keyword_hash)
+    if proof_path_list is None:
+        return False
+    proof_data = {
+        "statement_type": "keyword",
+        "keyword_hash": keyword_hash,
+        "root": root,
+        "path": proof_path_list,
+    }
+    output_proof_path.parent.mkdir(parents=True, exist_ok=True)
+    output_proof_path.write_text(json.dumps(proof_data, indent=2))
+    return True
 
 
 def prove_memory_freshness(
-    memory_path: Path, after_timestamp: str, output_proof_path: Path
+    memory_path: Path, after_timestamp: str, output_proof_path: Path, mem_dir: Optional[Path] = None
 ) -> bool:
-    """Prove memory was updated after date without revealing content. Stub: returns False until zk backend added."""
-    return False
+    """
+    Prove memory was updated after date without revealing content.
+    Proof: signed file mtime (or current time) and optional public key.
+    """
+    if not memory_path.exists() or not memory_path.is_file():
+        return False
+    if not ED25519_AVAILABLE:
+        return False
+    try:
+        stat = memory_path.stat()
+        ts = stat.st_mtime
+        from datetime import datetime, timezone
+        iso_ts = datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
+    except Exception:
+        return False
+    private_pem = load_private_key_from_env() if mem_dir is not None else None
+    if private_pem is None:
+        return False
+    try:
+        sig_hex = sign_merkle_root(iso_ts, private_pem)
+    except Exception:
+        return False
+    proof_data = {"statement_type": "freshness", "timestamp": iso_ts, "signature": sig_hex}
+    if mem_dir is not None:
+        pub_pem = load_public_key(mem_dir)
+        if pub_pem is not None:
+            proof_data["public_key_pem_b64"] = base64.b64encode(pub_pem).decode()
+    output_proof_path.parent.mkdir(parents=True, exist_ok=True)
+    output_proof_path.write_text(json.dumps(proof_data, indent=2))
+    return True
 
 
-def verify_proof(proof_path: Path, statement_type: str, **kwargs) -> bool:
-    """Verify a zk proof. Stub: returns False until zk backend added."""
+def verify_proof(proof_path: Path, statement_type: str, **kwargs: Any) -> bool:
+    """
+    Verify a proof. statement_type in ("keyword", "freshness").
+    For keyword: pass keyword=... (the keyword string).
+    For freshness: pass after_timestamp=... (ISO date string). Optional mem_dir=... for public key.
+    """
+    if not proof_path.exists() or not proof_path.is_file():
+        return False
+    try:
+        data = json.loads(proof_path.read_text())
+    except Exception:
+        return False
+    if data.get("statement_type") != statement_type:
+        return False
+    if statement_type == "keyword":
+        keyword = kwargs.get("keyword")
+        if keyword is None:
+            return False
+        keyword_hash = hashlib.sha256(keyword.strip().lower().encode()).hexdigest()
+        if data.get("keyword_hash") != keyword_hash:
+            return False
+        root = data.get("root")
+        path_list = data.get("path")
+        if not root or path_list is None:
+            return False
+        return verify_merkle_proof(keyword_hash, path_list, root)
+    if statement_type == "freshness":
+        after_ts = kwargs.get("after_timestamp")
+        if after_ts is None:
+            return False
+        ts_str = data.get("timestamp")
+        sig_hex = data.get("signature")
+        if not ts_str or not sig_hex:
+            return False
+        pub_pem_b64 = data.get("public_key_pem_b64")
+        if pub_pem_b64:
+            try:
+                pub_pem = base64.b64decode(pub_pem_b64)
+            except Exception:
+                return False
+        else:
+            mem_dir = kwargs.get("mem_dir")
+            if mem_dir is None:
+                return False
+            pub_pem = load_public_key(Path(mem_dir))
+            if pub_pem is None:
+                return False
+        if not verify_signature(ts_str, sig_hex, pub_pem):
+            return False
+        try:
+            from datetime import datetime
+            after_dt = datetime.fromisoformat(after_ts.replace("Z", "+00:00"))
+            ts_dt = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
+            return ts_dt >= after_dt
+        except Exception:
+            return False
     return False