agmem 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

memvcs/core/crypto_verify.py

@@ -12,7 +12,7 @@ import os
 from pathlib import Path
 from typing import Optional, List, Tuple, Any, Dict
 
-from .objects import ObjectStore, Tree, Commit
+from .objects import ObjectStore, Tree, Commit, Blob
 
 # Ed25519 via cryptography (optional)
 try:
@@ -239,6 +239,17 @@ def verify_commit(
     stored_sig = (commit.metadata or {}).get("signature")
     if not stored_root:
         return (False, "commit has no merkle_root (unverified)")
+
+    # Verify that blob objects can be loaded successfully (detects tampering in compressed/encrypted content)
+    blob_hashes = _collect_blob_hashes_from_tree(store, commit.tree)
+    for blob_hash in blob_hashes:
+        try:
+            blob = Blob.load(store, blob_hash)
+            if blob is None:
+                return (False, f"blob {blob_hash[:8]} corrupted or missing")
+        except Exception as e:
+            return (False, f"merkle_root mismatch (commit tampered)")
+
     computed_root = build_merkle_root_for_commit(store, commit_hash)
     if not computed_root:
         return (False, "could not build Merkle tree (missing tree/blobs)")

memvcs/core/distiller.py

@@ -35,6 +35,9 @@ class DistillerConfig:
     llm_provider: Optional[str] = None
     llm_model: Optional[str] = None
     create_safety_branch: bool = True
+    use_dp: bool = False
+    dp_epsilon: Optional[float] = None
+    dp_delta: Optional[float] = None
 
 
 @dataclass
@@ -160,13 +163,25 @@ class Distiller:
         except ValueError:
             out_path = self.target_dir / f"consolidated-{ts}.md"
 
+        confidence_score = self.config.extraction_confidence_threshold
+        if (
+            self.config.use_dp
+            and self.config.dp_epsilon is not None
+            and self.config.dp_delta is not None
+        ):
+            from .privacy_budget import add_noise
+
+            confidence_score = add_noise(
+                confidence_score, 0.1, self.config.dp_epsilon, self.config.dp_delta
+            )
+            confidence_score = max(0.0, min(1.0, confidence_score))
         frontmatter = {
             "schema_version": "1.0",
             "last_updated": datetime.utcnow().isoformat() + "Z",
             "source_agent_id": "distiller",
             "memory_type": "semantic",
             "tags": cluster.tags + ["auto-generated", "consolidated"],
-            "confidence_score": self.config.extraction_confidence_threshold,
+            "confidence_score": confidence_score,
         }
         body = f"# Consolidated: {cluster.topic}\n\n" + "\n".join(facts)
         if YAML_AVAILABLE:
@@ -266,11 +281,62 @@ class Distiller:
             except Exception:
                 pass
 
+        clusters_processed = len(clusters)
+        facts_extracted = facts_count
+        episodes_archived = archived
+        if (
+            self.config.use_dp
+            and self.config.dp_epsilon is not None
+            and self.config.dp_delta is not None
+        ):
+            from .privacy_budget import add_noise
+
+            sensitivity = 1.0
+            clusters_processed = max(
+                0,
+                int(
+                    round(
+                        add_noise(
+                            float(clusters_processed),
+                            sensitivity,
+                            self.config.dp_epsilon,
+                            self.config.dp_delta,
+                        )
+                    )
+                ),
+            )
+            facts_extracted = max(
+                0,
+                int(
+                    round(
+                        add_noise(
+                            float(facts_extracted),
+                            sensitivity,
+                            self.config.dp_epsilon,
+                            self.config.dp_delta,
+                        )
+                    )
+                ),
+            )
+            episodes_archived = max(
+                0,
+                int(
+                    round(
+                        add_noise(
+                            float(episodes_archived),
+                            sensitivity,
+                            self.config.dp_epsilon,
+                            self.config.dp_delta,
+                        )
+                    )
+                ),
+            )
+
         return DistillerResult(
             success=True,
-            clusters_processed=len(clusters),
-            facts_extracted=facts_count,
-            episodes_archived=archived,
+            clusters_processed=clusters_processed,
+            facts_extracted=facts_extracted,
+            episodes_archived=episodes_archived,
             branch_created=branch_name,
             commit_hash=commit_hash,
             message=f"Processed {len(clusters)} clusters, extracted {facts_count} facts",

memvcs/core/federated.py

@@ -5,6 +5,7 @@ Agents share model updates or aggregated summaries instead of raw episodic logs.
 Optional coordinator URL; optional differential privacy (Tier 3).
 """
 
+import hashlib
 import json
 from pathlib import Path
 from typing import Optional, List, Dict, Any
@@ -21,30 +22,100 @@ def get_federated_config(repo_root: Path) -> Optional[Dict[str, Any]]:
     url = fed.get("coordinator_url")
     if not url:
         return None
-    return {
+    out = {
         "coordinator_url": url.rstrip("/"),
         "memory_types": fed.get("memory_types", ["episodic", "semantic"]),
     }
+    dp = fed.get("differential_privacy") or config.get("differential_privacy") or {}
+    if dp.get("enabled"):
+        out["use_dp"] = True
+        out["dp_epsilon"] = float(dp.get("epsilon", 0.1))
+        out["dp_delta"] = float(dp.get("delta", 1e-5))
+    else:
+        out["use_dp"] = False
+    return out
 
 
-def produce_local_summary(repo_root: Path, memory_types: List[str]) -> Dict[str, Any]:
+def _normalize_for_hash(text: str) -> str:
+    """Normalize text for hashing (no raw content sent)."""
+    return " ".join(text.strip().split())
+
+
+def _extract_topic_from_md(path: Path, content: str) -> str:
+    """Extract topic from frontmatter tags or first heading."""
+    if content.startswith("---"):
+        end = content.find("---", 3)
+        if end > 0:
+            try:
+                import yaml
+
+                fm = yaml.safe_load(content[3:end])
+                if isinstance(fm, dict):
+                    tags = fm.get("tags", [])
+                    if tags:
+                        return str(tags[0])[:50]
+            except (ImportError, Exception):
+                pass
+    first_line = content.strip().split("\n")[0] if content.strip() else ""
+    if first_line.startswith("#"):
+        return first_line.lstrip("#").strip()[:50] or "untitled"
+    return "untitled"
+
+
+def produce_local_summary(
+    repo_root: Path,
+    memory_types: List[str],
+    use_dp: bool = False,
+    dp_epsilon: float = 0.1,
+    dp_delta: float = 1e-5,
+) -> Dict[str, Any]:
     """
     Produce a local summary from episodic/semantic data (no raw content).
-    Returns dict suitable for sending to coordinator (e.g. topic counts, fact hashes).
+    Returns dict with topic counts and fact hashes suitable for coordinator.
     """
     current_dir = repo_root / "current"
-    summary = {"memory_types": memory_types, "topics": {}, "fact_count": 0}
+    summary = {"memory_types": memory_types, "topics": {}, "topic_hashes": {}, "fact_count": 0}
+    all_fact_hashes: List[str] = []
+
     for mtype in memory_types:
         d = current_dir / mtype
         if not d.exists():
+            summary["topics"][mtype] = 0
+            summary["topic_hashes"][mtype] = []
             continue
-        count = 0
+        topic_to_count: Dict[str, int] = {}
+        topic_to_hashes: Dict[str, List[str]] = {}
         for f in d.rglob("*.md"):
-            if f.is_file():
-                count += 1
-        summary["topics"][mtype] = count
+            if not f.is_file():
+                continue
+            try:
+                content = f.read_text(encoding="utf-8", errors="replace")
+            except Exception:
+                continue
+            normalized = _normalize_for_hash(content)
+            if normalized:
+                h = hashlib.sha256(normalized.encode()).hexdigest()
+                all_fact_hashes.append(h)
+                topic = _extract_topic_from_md(f, content)
+                topic_to_count[topic] = topic_to_count.get(topic, 0) + 1
+                topic_to_hashes.setdefault(topic, []).append(h)
+        summary["topics"][mtype] = sum(topic_to_count.values())
+        summary["topic_hashes"][mtype] = list(topic_to_hashes.keys())
         if mtype == "semantic":
-            summary["fact_count"] = count
+            summary["fact_count"] = len(all_fact_hashes)
+
+    if use_dp and dp_epsilon and dp_delta:
+        from .privacy_budget import add_noise
+
+        for mtype in summary["topics"]:
+            raw = summary["topics"][mtype]
+            summary["topics"][mtype] = max(
+                0, int(round(add_noise(float(raw), 1.0, dp_epsilon, dp_delta)))
+            )
+        summary["fact_count"] = max(
+            0, int(round(add_noise(float(summary["fact_count"]), 1.0, dp_epsilon, dp_delta)))
+        )
+
     return summary
 
 

memvcs/core/gardener.py

@@ -43,6 +43,9 @@ class GardenerConfig:
     llm_provider: Optional[str] = None  # "openai", "anthropic", etc.
     llm_model: Optional[str] = None
     auto_commit: bool = True
+    use_dp: bool = False
+    dp_epsilon: Optional[float] = None
+    dp_delta: Optional[float] = None
 
 
 @dataclass
@@ -351,14 +354,35 @@ class Gardener:
         except ValueError:
             insight_path = self.semantic_dir / f"insight-{timestamp}.md"
 
-        # Generate frontmatter
+        # Generate frontmatter (optionally noised for differential privacy)
+        source_episodes = len(cluster.episodes)
+        if (
+            self.config.use_dp
+            and self.config.dp_epsilon is not None
+            and self.config.dp_delta is not None
+        ):
+            from .privacy_budget import add_noise
+
+            source_episodes = max(
+                0,
+                int(
+                    round(
+                        add_noise(
+                            float(source_episodes),
+                            1.0,
+                            self.config.dp_epsilon,
+                            self.config.dp_delta,
+                        )
+                    )
+                ),
+            )
         frontmatter = {
             "schema_version": "1.0",
             "last_updated": datetime.utcnow().isoformat() + "Z",
             "source_agent_id": "gardener",
             "memory_type": "semantic",
             "tags": cluster.tags + ["auto-generated", "insight"],
-            "source_episodes": len(cluster.episodes),
+            "source_episodes": source_episodes,
         }
 
         # Write file
@@ -487,11 +511,62 @@ class Gardener:
             except Exception as e:
                 print(f"Warning: Auto-commit failed: {e}")
 
+        clusters_found = len(clusters)
+        insights_generated = insights_written
+        episodes_archived = archived_count
+        if (
+            self.config.use_dp
+            and self.config.dp_epsilon is not None
+            and self.config.dp_delta is not None
+        ):
+            from .privacy_budget import add_noise
+
+            sensitivity = 1.0
+            clusters_found = max(
+                0,
+                int(
+                    round(
+                        add_noise(
+                            float(clusters_found),
+                            sensitivity,
+                            self.config.dp_epsilon,
+                            self.config.dp_delta,
+                        )
+                    )
+                ),
+            )
+            insights_generated = max(
+                0,
+                int(
+                    round(
+                        add_noise(
+                            float(insights_generated),
+                            sensitivity,
+                            self.config.dp_epsilon,
+                            self.config.dp_delta,
+                        )
+                    )
+                ),
+            )
+            episodes_archived = max(
+                0,
+                int(
+                    round(
+                        add_noise(
+                            float(episodes_archived),
+                            sensitivity,
+                            self.config.dp_epsilon,
+                            self.config.dp_delta,
+                        )
+                    )
+                ),
+            )
+
         return GardenerResult(
             success=True,
-            clusters_found=len(clusters),
-            insights_generated=insights_written,
-            episodes_archived=archived_count,
+            clusters_found=clusters_found,
+            insights_generated=insights_generated,
+            episodes_archived=episodes_archived,
             commit_hash=commit_hash,
             message=f"Processed {len(clusters)} clusters, generated {insights_written} insights",
         )

memvcs/core/ipfs_remote.py

@@ -1,25 +1,161 @@
 """
-IPFS remote for agmem (stub).
+IPFS remote for agmem.
 
-Push/pull via CIDs; pinning; gateway fallback when daemon unavailable.
-Requires optional ipfs extra (ipfshttpclient or gateway requests).
+Push/pull via CIDs using HTTP gateway (POST /api/v0/add, GET /ipfs/<cid>).
+Optional ipfshttpclient for local daemon.
 """
 
+import json
+import struct
+import zlib
 from pathlib import Path
-from typing import Optional, Set
+from typing import Optional, Set, Dict, Tuple
 
 from .objects import ObjectStore
 from .remote import _collect_objects_from_commit
 
+# Type byte for bundle (same as pack)
+_TYPE_BLOB = 1
+_TYPE_TREE = 2
+_TYPE_COMMIT = 3
+_TYPE_TAG = 4
+_TYPE_TO_BYTE = {"blob": _TYPE_BLOB, "tree": _TYPE_TREE, "commit": _TYPE_COMMIT, "tag": _TYPE_TAG}
+_BYTE_TO_TYPE = {v: k for k, v in _TYPE_TO_BYTE.items()}
+
+
+def _get_object_type_and_content(store: ObjectStore, hash_id: str) -> Optional[Tuple[str, bytes]]:
+    """Return (obj_type, raw_content) for a hash, or None."""
+    for obj_type in ["commit", "tree", "blob", "tag"]:
+        content = store.retrieve(hash_id, obj_type)
+        if content is not None:
+            return (obj_type, content)
+    return None
+
+
+def _bundle_objects(store: ObjectStore, hash_ids: Set[str]) -> bytes:
+    """Bundle objects into a single byte blob: count + [hash(32) type(1) len(4) zlib_payload]."""
+    entries = []
+    for h in sorted(hash_ids):
+        pair = _get_object_type_and_content(store, h)
+        if pair is None:
+            continue
+        obj_type, content = pair
+        header = f"{obj_type} {len(content)}\0".encode()
+        full = header + content
+        compressed = zlib.compress(full)
+        h_bin = bytes.fromhex(h) if len(h) == 64 else h.encode().ljust(32)[:32]
+        entries.append((h_bin, _TYPE_TO_BYTE.get(obj_type, _TYPE_BLOB), compressed))
+    parts = [struct.pack(">I", len(entries))]
+    for h_bin, type_byte, compressed in entries:
+        parts.append(h_bin)
+        parts.append(bytes([type_byte]))
+        parts.append(struct.pack(">I", len(compressed)))
+        parts.append(compressed)
+    return b"".join(parts)
+
+
+def _unbundle_objects(data: bytes, objects_dir: Path) -> int:
+    """Unbundle and write loose objects. Returns count written."""
+    if len(data) < 4:
+        return 0
+    count = struct.unpack(">I", data[:4])[0]
+    offset = 4
+    written = 0
+    for _ in range(count):
+        if offset + 32 + 1 + 4 > len(data):
+            break
+        h_bin = data[offset : offset + 32]
+        offset += 32
+        type_byte = data[offset]
+        offset += 1
+        comp_len = struct.unpack(">I", data[offset : offset + 4])[0]
+        offset += 4
+        if offset + comp_len > len(data):
+            break
+        compressed = data[offset : offset + comp_len]
+        offset += comp_len
+        obj_type = _BYTE_TO_TYPE.get(type_byte)
+        if obj_type is None:
+            continue
+        try:
+            full = zlib.decompress(compressed)
+        except Exception:
+            continue
+        null_idx = full.index(b"\0")
+        # Validate header
+        prefix = full[:null_idx].decode()
+        if " " not in prefix:
+            continue
+        name, size_str = prefix.split(" ", 1)
+        hash_hex = h_bin.hex() if len(h_bin) == 32 else h_bin.decode().strip()
+        if len(hash_hex) < 4:
+            continue
+        obj_path = objects_dir / obj_type / hash_hex[:2] / hash_hex[2:]
+        obj_path.parent.mkdir(parents=True, exist_ok=True)
+        obj_path.write_bytes(compressed)
+        written += 1
+    return written
+
+
+def _add_to_ipfs_gateway(bundle: bytes, gateway_url: str) -> Optional[str]:
+    """POST bundle to IPFS gateway /api/v0/add (multipart). Returns CID or None."""
+    boundary = "----agmem-boundary-" + str(abs(hash(bundle)))[:12]
+    body = (
+        b"--" + boundary.encode() + b"\r\n"
+        b'Content-Disposition: form-data; name="file"; filename="agmem-bundle.bin"\r\n'
+        b"Content-Type: application/octet-stream\r\n\r\n" + bundle + b"\r\n"
+        b"--" + boundary.encode() + b"--\r\n"
+    )
+    try:
+        import urllib.request
+
+        url = gateway_url.rstrip("/") + "/api/v0/add"
+        req = urllib.request.Request(url, data=body, method="POST")
+        req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary)
+        req.add_header("Content-Length", str(len(body)))
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            if resp.status != 200:
+                return None
+            data = json.loads(resp.read().decode())
+            return data.get("Hash") or data.get("Name")
+    except Exception:
+        try:
+            import requests
+
+            url = gateway_url.rstrip("/") + "/api/v0/add"
+            r = requests.post(
+                url,
+                files={"file": ("agmem-bundle.bin", bundle, "application/octet-stream")},
+                timeout=120,
+            )
+            if r.status_code != 200:
+                return None
+            return r.json().get("Hash") or r.json().get("Name")
+        except Exception:
+            return None
+
 
 def push_to_ipfs(
     objects_dir: Path,
     branch: str,
     commit_hash: str,
     gateway_url: str = "https://ipfs.io",
+    store: Optional[ObjectStore] = None,
 ) -> Optional[str]:
-    """Push branch objects to IPFS and return root CID. Stub: returns None until IPFS client added."""
-    return None
+    """
+    Push branch objects to IPFS and return root CID.
+    Uses gateway POST /api/v0/add (multipart).
+    """
+    if store is None:
+        store = ObjectStore(objects_dir)
+    try:
+        reachable = _collect_objects_from_commit(store, commit_hash)
+    except Exception:
+        return None
+    if not reachable:
+        return None
+    bundle = _bundle_objects(store, reachable)
+    return _add_to_ipfs_gateway(bundle, gateway_url)
 
 
 def pull_from_ipfs(
@@ -27,8 +163,32 @@ def pull_from_ipfs(
     cid: str,
     gateway_url: str = "https://ipfs.io",
 ) -> bool:
-    """Pull objects by CID from IPFS into objects_dir. Stub: returns False until IPFS client added."""
-    return False
+    """
+    Pull objects by CID from IPFS into objects_dir (loose objects).
+    Uses GET gateway_url/ipfs/<cid>.
+    """
+    try:
+        import urllib.request
+
+        url = gateway_url.rstrip("/") + "/ipfs/" + cid
+        req = urllib.request.Request(url, method="GET")
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            if resp.status != 200:
+                return False
+            data = resp.read()
+    except Exception:
+        try:
+            import requests
+
+            url = gateway_url.rstrip("/") + "/ipfs/" + cid
+            r = requests.get(url, timeout=60)
+            if r.status_code != 200:
+                return False
+            data = r.content
+        except Exception:
+            return False
+    written = _unbundle_objects(data, objects_dir)
+    return written > 0
 
 
 def parse_ipfs_url(url: str) -> Optional[str]:

memvcs/core/knowledge_graph.py

@@ -84,7 +84,14 @@ class KnowledgeGraphBuilder:
     1. Wikilinks: [[filename]] references
     2. Semantic similarity: Using embeddings
     3. Shared tags: Files with common tags
-    4. Co-occurrence: Facts in same episodic session (optional)
+    4. Co-occurrence: Files that mention the same entity (e.g. same section/session)
+    5. Causal: Phrases like "caused by", "because of" linking concepts (when derivable)
+    6. Entity: Person/place/thing links (simple keyword or pattern)
+
+    Incremental updates: To update when new files are added without full rebuild,
+    filter the file list to new/changed paths, run build_graph logic for that subset,
+    and merge new nodes/edges into the existing graph (or re-run build_graph; cost is
+    linear in file count).
     """
 
     # Pattern for wikilinks: [[target]] or [[target|display text]]
@@ -262,7 +269,22 @@ class KnowledgeGraphBuilder:
             except Exception:
                 pass  # Skip similarity if vector store fails
 
+        # Add co-occurrence edges (files sharing entities)
+        try:
+            edges.extend(self._build_cooccurrence_edges(file_paths, file_contents))
+        except Exception:
+            pass
+
+        # Add causal edges (phrases like "caused by", "because of" linking to another file)
+        try:
+            edges.extend(self._build_causal_edges(file_contents))
+        except Exception:
+            pass
+
         # Build metadata
+        edge_type_counts = defaultdict(int)
+        for e in edges:
+            edge_type_counts[e.edge_type] += 1
         metadata = {
             "total_nodes": len(nodes),
             "total_edges": len(edges),
@@ -274,15 +296,66 @@ class KnowledgeGraphBuilder:
                     1 for n in nodes if n.memory_type not in ["episodic", "semantic", "procedural"]
                 ),
             },
-            "edge_types": {
-                "reference": sum(1 for e in edges if e.edge_type == "reference"),
-                "similarity": sum(1 for e in edges if e.edge_type == "similarity"),
-                "same_topic": sum(1 for e in edges if e.edge_type == "same_topic"),
-            },
+            "edge_types": dict(edge_type_counts),
         }
 
         return KnowledgeGraphData(nodes=nodes, edges=edges, metadata=metadata)
 
+    def _extract_entities_simple(self, content: str) -> Set[str]:
+        """Extract simple entity tokens (capitalized words, key phrases) for co-occurrence."""
+        entities = set()
+        for word in re.findall(r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b", content):
+            if len(word) > 2:
+                entities.add(word.lower())
+        for phrase in ["user", "project", "agent", "memory", "preference", "workflow"]:
+            if phrase in content.lower():
+                entities.add(phrase)
+        return entities
+
+    def _build_cooccurrence_edges(
+        self, file_paths: List[str], file_contents: Dict[str, str]
+    ) -> List[GraphEdge]:
+        """Build edges between files that share at least one entity (co-occurrence)."""
+        file_entities: Dict[str, Set[str]] = {}
+        for path, content in file_contents.items():
+            file_entities[path] = self._extract_entities_simple(content)
+        edges = []
+        paths_list = list(file_paths)
+        for i, path1 in enumerate(paths_list):
+            for path2 in paths_list[i + 1 :]:
+                common = file_entities.get(path1, set()) & file_entities.get(path2, set())
+                if common:
+                    w = min(1.0, 0.3 + 0.1 * len(common))
+                    edge = GraphEdge(
+                        source=path1, target=path2, edge_type="co_occurrence", weight=w
+                    )
+                    edges.append(edge)
+                    if self._graph is not None:
+                        self._graph.add_edge(path1, path2, type="co_occurrence", weight=w)
+        return edges
+
+    def _build_causal_edges(self, file_contents: Dict[str, str]) -> List[GraphEdge]:
+        """Build edges when content has causal phrases linking to another file (e.g. caused by [[X]])."""
+        causal_phrases = re.compile(
+            r"(?:caused by|because of|led to|due to)\s+(?:\[\[([^\]]+)\]\]|(\w+))",
+            re.IGNORECASE,
+        )
+        edges = []
+        for source_path, content in file_contents.items():
+            for m in causal_phrases.finditer(content):
+                target = m.group(1) or m.group(2)
+                if not target:
+                    continue
+                target_path = self._normalize_link_target(target.strip(), source_path)
+                if target_path and target_path in file_contents and target_path != source_path:
+                    edge = GraphEdge(
+                        source=source_path, target=target_path, edge_type="causal", weight=0.7
+                    )
+                    edges.append(edge)
+                    if self._graph is not None:
+                        self._graph.add_edge(source_path, target_path, type="causal", weight=0.7)
+        return edges
+
     def _build_similarity_edges(
         self, file_paths: List[str], file_contents: Dict[str, str], threshold: float
     ) -> List[GraphEdge]: