agmem 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

memvcs/core/federated.py

@@ -0,0 +1,147 @@
+"""
+Federated memory collaboration for agmem.
+
+Agents share model updates or aggregated summaries instead of raw episodic logs.
+Optional coordinator URL; optional differential privacy (Tier 3).
+"""
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+
+from .config_loader import load_agmem_config
+
+
+def get_federated_config(repo_root: Path) -> Optional[Dict[str, Any]]:
+    """Get federated config from repo/user config. Returns None if disabled."""
+    config = load_agmem_config(repo_root)
+    fed = config.get("federated") or {}
+    if not fed.get("enabled"):
+        return None
+    url = fed.get("coordinator_url")
+    if not url:
+        return None
+    out = {
+        "coordinator_url": url.rstrip("/"),
+        "memory_types": fed.get("memory_types", ["episodic", "semantic"]),
+    }
+    dp = fed.get("differential_privacy") or config.get("differential_privacy") or {}
+    if dp.get("enabled"):
+        out["use_dp"] = True
+        out["dp_epsilon"] = float(dp.get("epsilon", 0.1))
+        out["dp_delta"] = float(dp.get("delta", 1e-5))
+    else:
+        out["use_dp"] = False
+    return out
+
+
+def _normalize_for_hash(text: str) -> str:
+    """Normalize text for hashing (no raw content sent)."""
+    return " ".join(text.strip().split())
+
+
+def _extract_topic_from_md(path: Path, content: str) -> str:
+    """Extract topic from frontmatter tags or first heading."""
+    if content.startswith("---"):
+        end = content.find("---", 3)
+        if end > 0:
+            try:
+                import yaml
+                fm = yaml.safe_load(content[3:end])
+                if isinstance(fm, dict):
+                    tags = fm.get("tags", [])
+                    if tags:
+                        return str(tags[0])[:50]
+            except (ImportError, Exception):
+                pass
+    first_line = content.strip().split("\n")[0] if content.strip() else ""
+    if first_line.startswith("#"):
+        return first_line.lstrip("#").strip()[:50] or "untitled"
+    return "untitled"
+
+
+def produce_local_summary(
+    repo_root: Path, memory_types: List[str], use_dp: bool = False, dp_epsilon: float = 0.1, dp_delta: float = 1e-5
+) -> Dict[str, Any]:
+    """
+    Produce a local summary from episodic/semantic data (no raw content).
+    Returns dict with topic counts and fact hashes suitable for coordinator.
+    """
+    current_dir = repo_root / "current"
+    summary = {"memory_types": memory_types, "topics": {}, "topic_hashes": {}, "fact_count": 0}
+    all_fact_hashes: List[str] = []
+
+    for mtype in memory_types:
+        d = current_dir / mtype
+        if not d.exists():
+            summary["topics"][mtype] = 0
+            summary["topic_hashes"][mtype] = []
+            continue
+        topic_to_count: Dict[str, int] = {}
+        topic_to_hashes: Dict[str, List[str]] = {}
+        for f in d.rglob("*.md"):
+            if not f.is_file():
+                continue
+            try:
+                content = f.read_text(encoding="utf-8", errors="replace")
+            except Exception:
+                continue
+            normalized = _normalize_for_hash(content)
+            if normalized:
+                h = hashlib.sha256(normalized.encode()).hexdigest()
+                all_fact_hashes.append(h)
+                topic = _extract_topic_from_md(f, content)
+                topic_to_count[topic] = topic_to_count.get(topic, 0) + 1
+                topic_to_hashes.setdefault(topic, []).append(h)
+        summary["topics"][mtype] = sum(topic_to_count.values())
+        summary["topic_hashes"][mtype] = list(topic_to_hashes.keys())
+        if mtype == "semantic":
+            summary["fact_count"] = len(all_fact_hashes)
+
+    if use_dp and dp_epsilon and dp_delta:
+        from .privacy_budget import add_noise
+        for mtype in summary["topics"]:
+            raw = summary["topics"][mtype]
+            summary["topics"][mtype] = max(0, int(round(add_noise(float(raw), 1.0, dp_epsilon, dp_delta))))
+        summary["fact_count"] = max(0, int(round(add_noise(float(summary["fact_count"]), 1.0, dp_epsilon, dp_delta))))
+
+    return summary
+
+
+def push_updates(repo_root: Path, summary: Dict[str, Any]) -> str:
+    """Send local summary to coordinator. Returns status message."""
+    cfg = get_federated_config(repo_root)
+    if not cfg:
+        return "Federated collaboration not configured"
+    url = cfg["coordinator_url"] + "/push"
+    try:
+        import urllib.request
+
+        req = urllib.request.Request(
+            url,
+            data=json.dumps(summary).encode(),
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            if resp.status in (200, 201):
+                return "Pushed updates to coordinator"
+            return f"Coordinator returned {resp.status}"
+    except Exception as e:
+        return f"Push failed: {e}"
+
+
+def pull_merged(repo_root: Path) -> Optional[Dict[str, Any]]:
+    """Pull merged summaries from coordinator. Returns merged data or None."""
+    cfg = get_federated_config(repo_root)
+    if not cfg:
+        return None
+    url = cfg["coordinator_url"] + "/pull"
+    try:
+        import urllib.request
+
+        with urllib.request.urlopen(url, timeout=30) as resp:
+            return json.loads(resp.read().decode())
+    except Exception:
+        return None

memvcs/core/gardener.py

@@ -43,6 +43,9 @@ class GardenerConfig:
     llm_provider: Optional[str] = None  # "openai", "anthropic", etc.
     llm_model: Optional[str] = None
     auto_commit: bool = True
+    use_dp: bool = False
+    dp_epsilon: Optional[float] = None
+    dp_delta: Optional[float] = None
 
 
 @dataclass
@@ -284,37 +287,36 @@ class Gardener:
 
         combined = "\n---\n".join(contents)
 
-        # Try LLM summarization
-        if self.config.llm_provider == "openai" and self.config.llm_model:
+        # Try LLM summarization (multi-provider)
+        if self.config.llm_provider and self.config.llm_model:
             try:
-                return self._summarize_with_openai(combined, cluster.topic)
+                from .llm import get_provider
+
+                config = {
+                    "llm_provider": self.config.llm_provider,
+                    "llm_model": self.config.llm_model,
+                }
+                provider = get_provider(config=config)
+                if provider:
+                    return provider.complete(
+                        [
+                            {
+                                "role": "system",
+                                "content": "You are a helpful assistant that summarizes conversation logs into actionable insights.",
+                            },
+                            {
+                                "role": "user",
+                                "content": f"Summarize these conversation logs about '{cluster.topic}' into 2-3 key insights:\n\n{combined[:4000]}",
+                            },
+                        ],
+                        max_tokens=500,
+                    )
             except Exception:
                 pass
 
         # Fall back to simple summary
         return self._simple_summary(cluster, contents)
 
-    def _summarize_with_openai(self, content: str, topic: str) -> str:
-        """Summarize using OpenAI API."""
-        import openai
-
-        response = openai.chat.completions.create(
-            model=self.config.llm_model or "gpt-3.5-turbo",
-            messages=[
-                {
-                    "role": "system",
-                    "content": "You are a helpful assistant that summarizes conversation logs into actionable insights.",
-                },
-                {
-                    "role": "user",
-                    "content": f"Summarize these conversation logs about '{topic}' into 2-3 key insights:\n\n{content[:4000]}",
-                },
-            ],
-            max_tokens=500,
-        )
-
-        return response.choices[0].message.content
-
     def _simple_summary(self, cluster: EpisodeCluster, contents: List[str]) -> str:
         """Generate a simple summary without LLM."""
         return f"""# Insights: {cluster.topic.title()}
@@ -352,14 +354,20 @@ class Gardener:
         except ValueError:
             insight_path = self.semantic_dir / f"insight-{timestamp}.md"
 
-        # Generate frontmatter
+        # Generate frontmatter (optionally noised for differential privacy)
+        source_episodes = len(cluster.episodes)
+        if self.config.use_dp and self.config.dp_epsilon is not None and self.config.dp_delta is not None:
+            from .privacy_budget import add_noise
+            source_episodes = max(0, int(round(add_noise(
+                float(source_episodes), 1.0, self.config.dp_epsilon, self.config.dp_delta
+            ))))
         frontmatter = {
             "schema_version": "1.0",
             "last_updated": datetime.utcnow().isoformat() + "Z",
             "source_agent_id": "gardener",
             "memory_type": "semantic",
             "tags": cluster.tags + ["auto-generated", "insight"],
-            "source_episodes": len(cluster.episodes),
+            "source_episodes": source_episodes,
         }
 
         # Write file
@@ -488,11 +496,21 @@ class Gardener:
             except Exception as e:
                 print(f"Warning: Auto-commit failed: {e}")
 
+        clusters_found = len(clusters)
+        insights_generated = insights_written
+        episodes_archived = archived_count
+        if self.config.use_dp and self.config.dp_epsilon is not None and self.config.dp_delta is not None:
+            from .privacy_budget import add_noise
+            sensitivity = 1.0
+            clusters_found = max(0, int(round(add_noise(float(clusters_found), sensitivity, self.config.dp_epsilon, self.config.dp_delta))))
+            insights_generated = max(0, int(round(add_noise(float(insights_generated), sensitivity, self.config.dp_epsilon, self.config.dp_delta))))
+            episodes_archived = max(0, int(round(add_noise(float(episodes_archived), sensitivity, self.config.dp_epsilon, self.config.dp_delta))))
+
         return GardenerResult(
             success=True,
-            clusters_found=len(clusters),
-            insights_generated=insights_written,
-            episodes_archived=archived_count,
+            clusters_found=clusters_found,
+            insights_generated=insights_generated,
+            episodes_archived=episodes_archived,
             commit_hash=commit_hash,
             message=f"Processed {len(clusters)} clusters, generated {insights_written} insights",
         )

memvcs/core/ipfs_remote.py

@@ -0,0 +1,200 @@
+"""
+IPFS remote for agmem.
+
+Push/pull via CIDs using HTTP gateway (POST /api/v0/add, GET /ipfs/<cid>).
+Optional ipfshttpclient for local daemon.
+"""
+
+import json
+import struct
+import zlib
+from pathlib import Path
+from typing import Optional, Set, Dict, Tuple
+
+from .objects import ObjectStore
+from .remote import _collect_objects_from_commit
+
+# Type byte for bundle (same as pack)
+_TYPE_BLOB = 1
+_TYPE_TREE = 2
+_TYPE_COMMIT = 3
+_TYPE_TAG = 4
+_TYPE_TO_BYTE = {"blob": _TYPE_BLOB, "tree": _TYPE_TREE, "commit": _TYPE_COMMIT, "tag": _TYPE_TAG}
+_BYTE_TO_TYPE = {v: k for k, v in _TYPE_TO_BYTE.items()}
+
+
+def _get_object_type_and_content(store: ObjectStore, hash_id: str) -> Optional[Tuple[str, bytes]]:
+    """Return (obj_type, raw_content) for a hash, or None."""
+    for obj_type in ["commit", "tree", "blob", "tag"]:
+        content = store.retrieve(hash_id, obj_type)
+        if content is not None:
+            return (obj_type, content)
+    return None
+
+
+def _bundle_objects(store: ObjectStore, hash_ids: Set[str]) -> bytes:
+    """Bundle objects into a single byte blob: count + [hash(32) type(1) len(4) zlib_payload]."""
+    entries = []
+    for h in sorted(hash_ids):
+        pair = _get_object_type_and_content(store, h)
+        if pair is None:
+            continue
+        obj_type, content = pair
+        header = f"{obj_type} {len(content)}\0".encode()
+        full = header + content
+        compressed = zlib.compress(full)
+        h_bin = bytes.fromhex(h) if len(h) == 64 else h.encode().ljust(32)[:32]
+        entries.append((h_bin, _TYPE_TO_BYTE.get(obj_type, _TYPE_BLOB), compressed))
+    parts = [struct.pack(">I", len(entries))]
+    for h_bin, type_byte, compressed in entries:
+        parts.append(h_bin)
+        parts.append(bytes([type_byte]))
+        parts.append(struct.pack(">I", len(compressed)))
+        parts.append(compressed)
+    return b"".join(parts)
+
+
+def _unbundle_objects(data: bytes, objects_dir: Path) -> int:
+    """Unbundle and write loose objects. Returns count written."""
+    if len(data) < 4:
+        return 0
+    count = struct.unpack(">I", data[:4])[0]
+    offset = 4
+    written = 0
+    for _ in range(count):
+        if offset + 32 + 1 + 4 > len(data):
+            break
+        h_bin = data[offset : offset + 32]
+        offset += 32
+        type_byte = data[offset]
+        offset += 1
+        comp_len = struct.unpack(">I", data[offset : offset + 4])[0]
+        offset += 4
+        if offset + comp_len > len(data):
+            break
+        compressed = data[offset : offset + comp_len]
+        offset += comp_len
+        obj_type = _BYTE_TO_TYPE.get(type_byte)
+        if obj_type is None:
+            continue
+        try:
+            full = zlib.decompress(compressed)
+        except Exception:
+            continue
+        null_idx = full.index(b"\0")
+        # Validate header
+        prefix = full[:null_idx].decode()
+        if " " not in prefix:
+            continue
+        name, size_str = prefix.split(" ", 1)
+        hash_hex = h_bin.hex() if len(h_bin) == 32 else h_bin.decode().strip()
+        if len(hash_hex) < 4:
+            continue
+        obj_path = objects_dir / obj_type / hash_hex[:2] / hash_hex[2:]
+        obj_path.parent.mkdir(parents=True, exist_ok=True)
+        obj_path.write_bytes(compressed)
+        written += 1
+    return written
+
+
+def _add_to_ipfs_gateway(bundle: bytes, gateway_url: str) -> Optional[str]:
+    """POST bundle to IPFS gateway /api/v0/add (multipart). Returns CID or None."""
+    boundary = "----agmem-boundary-" + str(abs(hash(bundle)))[:12]
+    body = (
+        b"--" + boundary.encode() + b"\r\n"
+        b'Content-Disposition: form-data; name="file"; filename="agmem-bundle.bin"\r\n'
+        b"Content-Type: application/octet-stream\r\n\r\n"
+        + bundle + b"\r\n"
+        b"--" + boundary.encode() + b"--\r\n"
+    )
+    try:
+        import urllib.request
+
+        url = gateway_url.rstrip("/") + "/api/v0/add"
+        req = urllib.request.Request(url, data=body, method="POST")
+        req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary)
+        req.add_header("Content-Length", str(len(body)))
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            if resp.status != 200:
+                return None
+            data = json.loads(resp.read().decode())
+            return data.get("Hash") or data.get("Name")
+    except Exception:
+        try:
+            import requests
+
+            url = gateway_url.rstrip("/") + "/api/v0/add"
+            r = requests.post(
+                url,
+                files={"file": ("agmem-bundle.bin", bundle, "application/octet-stream")},
+                timeout=120,
+            )
+            if r.status_code != 200:
+                return None
+            return r.json().get("Hash") or r.json().get("Name")
+        except Exception:
+            return None
+
+
+def push_to_ipfs(
+    objects_dir: Path,
+    branch: str,
+    commit_hash: str,
+    gateway_url: str = "https://ipfs.io",
+    store: Optional[ObjectStore] = None,
+) -> Optional[str]:
+    """
+    Push branch objects to IPFS and return root CID.
+    Uses gateway POST /api/v0/add (multipart).
+    """
+    if store is None:
+        store = ObjectStore(objects_dir)
+    try:
+        reachable = _collect_objects_from_commit(store, commit_hash)
+    except Exception:
+        return None
+    if not reachable:
+        return None
+    bundle = _bundle_objects(store, reachable)
+    return _add_to_ipfs_gateway(bundle, gateway_url)
+
+
+def pull_from_ipfs(
+    objects_dir: Path,
+    cid: str,
+    gateway_url: str = "https://ipfs.io",
+) -> bool:
+    """
+    Pull objects by CID from IPFS into objects_dir (loose objects).
+    Uses GET gateway_url/ipfs/<cid>.
+    """
+    try:
+        import urllib.request
+
+        url = gateway_url.rstrip("/") + "/ipfs/" + cid
+        req = urllib.request.Request(url, method="GET")
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            if resp.status != 200:
+                return False
+            data = resp.read()
+    except Exception:
+        try:
+            import requests
+
+            url = gateway_url.rstrip("/") + "/ipfs/" + cid
+            r = requests.get(url, timeout=60)
+            if r.status_code != 200:
+                return False
+            data = r.content
+        except Exception:
+            return False
+    written = _unbundle_objects(data, objects_dir)
+    return written > 0
+
+
+def parse_ipfs_url(url: str) -> Optional[str]:
+    """Parse ipfs://<cid> or ipfs://<cid>/path. Returns CID or None."""
+    if not url.startswith("ipfs://"):
+        return None
+    rest = url[7:].lstrip("/")
+    return rest.split("/")[0] or None

memvcs/core/knowledge_graph.py

@@ -84,6 +84,14 @@ class KnowledgeGraphBuilder:
     1. Wikilinks: [[filename]] references
     2. Semantic similarity: Using embeddings
     3. Shared tags: Files with common tags
+    4. Co-occurrence: Files that mention the same entity (e.g. same section/session)
+    5. Causal: Phrases like "caused by", "because of" linking concepts (when derivable)
+    6. Entity: Person/place/thing links (simple keyword or pattern)
+
+    Incremental updates: To update when new files are added without full rebuild,
+    filter the file list to new/changed paths, run build_graph logic for that subset,
+    and merge new nodes/edges into the existing graph (or re-run build_graph; cost is
+    linear in file count).
     """
 
     # Pattern for wikilinks: [[target]] or [[target|display text]]
@@ -261,7 +269,22 @@ class KnowledgeGraphBuilder:
             except Exception:
                 pass  # Skip similarity if vector store fails
 
+        # Add co-occurrence edges (files sharing entities)
+        try:
+            edges.extend(self._build_cooccurrence_edges(file_paths, file_contents))
+        except Exception:
+            pass
+
+        # Add causal edges (phrases like "caused by", "because of" linking to another file)
+        try:
+            edges.extend(self._build_causal_edges(file_contents))
+        except Exception:
+            pass
+
         # Build metadata
+        edge_type_counts = defaultdict(int)
+        for e in edges:
+            edge_type_counts[e.edge_type] += 1
         metadata = {
             "total_nodes": len(nodes),
             "total_edges": len(edges),
@@ -273,15 +296,64 @@ class KnowledgeGraphBuilder:
                     1 for n in nodes if n.memory_type not in ["episodic", "semantic", "procedural"]
                 ),
             },
-            "edge_types": {
-                "reference": sum(1 for e in edges if e.edge_type == "reference"),
-                "similarity": sum(1 for e in edges if e.edge_type == "similarity"),
-                "same_topic": sum(1 for e in edges if e.edge_type == "same_topic"),
-            },
+            "edge_types": dict(edge_type_counts),
         }
 
         return KnowledgeGraphData(nodes=nodes, edges=edges, metadata=metadata)
 
+    def _extract_entities_simple(self, content: str) -> Set[str]:
+        """Extract simple entity tokens (capitalized words, key phrases) for co-occurrence."""
+        entities = set()
+        for word in re.findall(r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b", content):
+            if len(word) > 2:
+                entities.add(word.lower())
+        for phrase in ["user", "project", "agent", "memory", "preference", "workflow"]:
+            if phrase in content.lower():
+                entities.add(phrase)
+        return entities
+
+    def _build_cooccurrence_edges(
+        self, file_paths: List[str], file_contents: Dict[str, str]
+    ) -> List[GraphEdge]:
+        """Build edges between files that share at least one entity (co-occurrence)."""
+        file_entities: Dict[str, Set[str]] = {}
+        for path, content in file_contents.items():
+            file_entities[path] = self._extract_entities_simple(content)
+        edges = []
+        paths_list = list(file_paths)
+        for i, path1 in enumerate(paths_list):
+            for path2 in paths_list[i + 1 :]:
+                common = file_entities.get(path1, set()) & file_entities.get(path2, set())
+                if common:
+                    w = min(1.0, 0.3 + 0.1 * len(common))
+                    edge = GraphEdge(source=path1, target=path2, edge_type="co_occurrence", weight=w)
+                    edges.append(edge)
+                    if self._graph is not None:
+                        self._graph.add_edge(path1, path2, type="co_occurrence", weight=w)
+        return edges
+
+    def _build_causal_edges(self, file_contents: Dict[str, str]) -> List[GraphEdge]:
+        """Build edges when content has causal phrases linking to another file (e.g. caused by [[X]])."""
+        causal_phrases = re.compile(
+            r"(?:caused by|because of|led to|due to)\s+(?:\[\[([^\]]+)\]\]|(\w+))",
+            re.IGNORECASE,
+        )
+        edges = []
+        for source_path, content in file_contents.items():
+            for m in causal_phrases.finditer(content):
+                target = m.group(1) or m.group(2)
+                if not target:
+                    continue
+                target_path = self._normalize_link_target(target.strip(), source_path)
+                if target_path and target_path in file_contents and target_path != source_path:
+                    edge = GraphEdge(
+                        source=source_path, target=target_path, edge_type="causal", weight=0.7
+                    )
+                    edges.append(edge)
+                    if self._graph is not None:
+                        self._graph.add_edge(source_path, target_path, type="causal", weight=0.7)
+        return edges
+
     def _build_similarity_edges(
         self, file_paths: List[str], file_contents: Dict[str, str], threshold: float
     ) -> List[GraphEdge]:

memvcs/core/llm/__init__.py

@@ -0,0 +1,10 @@
+"""
+Multi-provider LLM integration for agmem.
+
+Abstract interface; implementations: OpenAI, Anthropic, Ollama, custom HTTP.
+"""
+
+from .base import LLMProvider
+from .factory import get_provider
+
+__all__ = ["LLMProvider", "get_provider"]

memvcs/core/llm/anthropic_provider.py

@@ -0,0 +1,50 @@
+"""Anthropic (Claude) LLM provider."""
+
+import os
+from typing import Optional, List, Dict, Any
+
+from .base import LLMProvider
+
+
+class AnthropicProvider(LLMProvider):
+    """Anthropic Claude provider. API key from ANTHROPIC_API_KEY."""
+
+    def __init__(self, model: Optional[str] = None):
+        self._model = model or os.environ.get("ANTHROPIC_MODEL", "claude-3-haiku-20240307")
+
+    @property
+    def name(self) -> str:
+        return "anthropic"
+
+    def complete(
+        self,
+        messages: List[Dict[str, str]],
+        *,
+        model: Optional[str] = None,
+        max_tokens: int = 1024,
+        **kwargs: Any,
+    ) -> str:
+        try:
+            import anthropic
+        except ImportError:
+            raise RuntimeError("Anthropic provider requires: pip install anthropic")
+        m = model or self._model
+        client = anthropic.Anthropic()
+        # Convert OpenAI-style messages to Anthropic (system + user/assistant)
+        system = ""
+        anthropic_messages = []
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+            if role == "system":
+                system = content
+            else:
+                anthropic_messages.append({"role": role, "content": content})
+        resp = client.messages.create(
+            model=m,
+            max_tokens=max_tokens,
+            system=system or None,
+            messages=anthropic_messages,
+            **kwargs,
+        )
+        return resp.content[0].text if resp.content else ""

memvcs/core/llm/base.py

@@ -0,0 +1,27 @@
+"""
+LLM provider interface for agmem.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Optional, List, Dict, Any
+
+
+class LLMProvider(ABC):
+    """Abstract LLM provider (complete(messages) -> text)."""
+
+    @abstractmethod
+    def complete(
+        self,
+        messages: List[Dict[str, str]],
+        *,
+        model: Optional[str] = None,
+        max_tokens: int = 1024,
+        **kwargs: Any,
+    ) -> str:
+        """Return completion text for messages. Raises on failure."""
+        pass
+
+    @property
+    def name(self) -> str:
+        """Provider name (e.g. openai, anthropic)."""
+        return "base"