memctrl 1.0.0__py3-none-any.whl

memctrl/extractor.py

@@ -0,0 +1,261 @@
+"""MemCtrl — LLM-powered memory extraction from text.
+
+Extracts structured memories with confidence scoring:
+  - Explicit (1.0): "we decided to use FastAPI"
+  - Inferred (0.7): "import fastapi" ← inferred from code
+  - Mentioned (0.5): "FastAPI was suggested" ← not yet decided
+
+Security: NEVER extracts passwords, API keys, secrets, or PII.
+Uses regex patterns for secret detection + redaction.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from datetime import datetime, timedelta
+from typing import Any, Callable, Coroutine, Dict, List, Optional
+
+# Type alias
+LLMCallable = Callable[[str, bool], Coroutine[Any, Any, str]]
+
+# Secret patterns to redact/detect
+_SECRET_PATTERNS = [
+    (r"\b(sk-[a-zA-Z0-9]{20,})\b", "API_KEY"),
+    (r"\b([A-Za-z0-9/+=]{40,})\b", "TOKEN"),
+    (r"\b(password\s*[=:]\s*\S+)", "PASSWORD"),
+    (r"\b(secret\s*[=:]\s*\S+)", "SECRET"),
+    (r"\b(AKIA[0-9A-Z]{16})\b", "AWS_KEY"),
+    (r"-----BEGIN (RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----", "PRIVATE_KEY"),
+]
+
+_PII_PATTERNS = [
+    (r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", "EMAIL"),
+    (r"\b\d{3}-\d{2}-\d{4}\b", "SSN"),
+    (r"\b\d{3}-\d{3}-\d{4}\b", "PHONE"),
+    (r"\b\d{10,12}\b", "PHONE_INTL"),
+]
+
+
+class MemoryExtractor:
+    """Extract structured memories from text with confidence scoring.
+
+    Distinguishes:
+        - Explicit facts (confidence=1.0): "we use FastAPI"
+        - Inferred facts (confidence=0.7): "import fastapi" ← from code
+        - Mentioned (confidence=0.5): "FastAPI was suggested"
+
+    NEVER extracts passwords, API keys, secrets, or PII.
+    """
+
+    def __init__(
+        self,
+        llm_client: Optional[LLMCallable] = None,
+        rules: Optional[Any] = None,
+    ):
+        self.llm_client = llm_client
+        self.rules = rules
+
+    # --- Public API ---
+
+    async def extract(
+        self,
+        text: str,
+        layer: str,
+        rules,
+    ) -> List[dict]:
+        """Extract structured memories from text.
+
+        text: source text (commit message, chat, file content)
+        layer: target layer (project/session/user)
+        rules: Rules object with confidence levels and forget.never
+
+        Returns list of dicts: {id, layer, content, source, confidence,
+                                  created_at, expires_at, tags}
+        """
+        # 1. Security scan
+        if self._has_secrets(text, rules.forget_never):
+            text = self._sanitize_text(text)
+
+        # 2. LLM extraction
+        if self.llm_client:
+            try:
+                memories = await self._llm_extract(text, layer, rules)
+                if memories:
+                    return memories
+            except Exception:
+                pass  # Fallback to heuristic
+
+        # 3. Fallback heuristic extraction
+        return self._fallback_extract(text, layer, rules)
+
+    # --- Security ---
+
+    def _has_secrets(self, text: str, never_list: List[str]) -> bool:
+        """Check if text contains forbidden patterns."""
+        text_lower = text.lower()
+        for pattern in never_list:
+            if pattern.lower() in text_lower:
+                return True
+        for pattern, _ in _SECRET_PATTERNS:
+            if re.search(pattern, text, re.I):
+                return True
+        return False
+
+    def _sanitize_text(self, text: str) -> str:
+        """Redact secrets and PII from text."""
+        for pattern, label in _SECRET_PATTERNS + _PII_PATTERNS:
+            text = re.sub(pattern, f"[REDACTED_{label}]", text, flags=re.I)
+        return text
+
+    def _detect_pii(self, text: str) -> List[str]:
+        """Detect PII in text. Returns list of found PII types."""
+        found = []
+        for pattern, label in _PII_PATTERNS:
+            if re.search(pattern, text):
+                found.append(label)
+        return found
+
+    # --- LLM extraction ---
+
+    async def _llm_extract(
+        self, text: str, layer: str, rules,
+    ) -> List[dict]:
+        """Use LLM to extract memories with confidence scoring."""
+        prompt = self._build_extraction_prompt(text, layer, rules)
+        response = await self.llm_client(prompt, json_mode=True)
+
+        try:
+            data = {"memories": []}
+            import json as _json
+            data = _json.loads(response)
+        except Exception:
+            return []
+
+        results = []
+        for mem in data.get("memories", []):
+            content = mem.get("content", "").strip()
+            if not content or len(content) < 5:
+                continue
+
+            # Final secret check
+            if any(p.lower() in content.lower() for p in rules.forget_never):
+                continue
+            if self._detect_pii(content):
+                continue
+
+            confidence = mem.get("confidence", 0.5)
+            # Clamp to valid levels
+            valid_levels = list(rules.confidence.values()) if rules.confidence else [0.5, 0.7, 1.0]
+            if valid_levels and confidence not in valid_levels:
+                confidence = min(valid_levels, key=lambda x: abs(x - confidence))
+
+            results.append({
+                "id": str(uuid.uuid4()),
+                "layer": layer,
+                "content": content,
+                "source": "llm_extract",
+                "confidence": confidence,
+                "created_at": datetime.now().isoformat(),
+                "expires_at": None,
+                "tags": mem.get("tags", [layer, "llm-extracted"]),
+            })
+
+        return results
+
+    def _build_extraction_prompt(self, text: str, layer: str, rules) -> str:
+        """Build LLM prompt for memory extraction."""
+        layer_desc = rules.layers.get(layer, layer)
+        explicit_c = rules.confidence.get("explicit", 1.0)
+        inferred_c = rules.confidence.get("inferred", 0.7)
+        mentioned_c = rules.confidence.get("mentioned", 0.5)
+
+        return (
+            f"Extract memories from the following text for the '{layer}' layer.\n\n"
+            f"Layer definition: {layer_desc}\n\n"
+            f"Text:\n{text[:3000]}\n\n"  # Limit to 3K chars
+            f"Confidence levels:\n"
+            f"  {explicit_c} = explicit statement (e.g., 'we decided to use X')\n"
+            f"  {inferred_c} = inferred from context (e.g., 'import X')\n"
+            f"  {mentioned_c} = mentioned but not decided\n\n"
+            f"NEVER extract: passwords, API keys, secrets, PII.\n\n"
+            f"Return ONLY JSON:\n"
+            f'{{"memories": [\n'
+            f'  {{"content": "fact text", "confidence": {explicit_c}, '
+            f'"tags": ["{layer}"]}}\n'
+            f"]}}"
+        )
+
+    # --- Fallback extraction (no LLM) ---
+
+    def _fallback_extract(self, text: str, layer: str, rules) -> List[dict]:
+        """Non-LLM extraction using regex patterns."""
+        results = []
+        lines = text.split("\n")
+
+        patterns = [
+            # Explicit patterns (1.0)
+            (r"(?i)(we\s+(?:use|use[d]|chose|decided|migrated|switched|implemented)\s+.+)",
+             "explicit", "tech_choice"),
+            (r"(?i)(adr[-\s]?\d+\s*[:\-]?\s*.+)",
+             "explicit", "adr"),
+            (r"(?i)(decided\s+to\s+.+)",
+             "explicit", "decision"),
+            # Migration patterns
+            (r"(?i)(migrated?\s+(?:from\s+)?\w+\s+to\s+\w+.+)",
+             "explicit", "migration"),
+            # Inferred patterns (0.7)
+            (r"(?i)^\s*(?:import|from)\s+(\w+).+",
+             "inferred", "dependency"),
+            (r"(?i)(?:built|written|developed)\s+(?:with|on|using)\s+(\w+).+",
+             "inferred", "framework"),
+            # Preference patterns
+            (r"(?i)(?:prefer|like|always|never)\s+.+",
+             "explicit", "preference"),
+        ]
+
+        for line in lines:
+            line = line.strip()
+            if len(line) < 10:
+                continue
+            if len(line) > 500:
+                line = line[:500]
+
+            for pattern, level, tag in patterns:
+                match = re.search(pattern, line)
+                if match:
+                    content = match.group(1) if match.groups() else match.group(0)
+                    content = content.strip(". ;,\t")
+                    if len(content) < 10:
+                        continue
+
+                    # Skip if contains secrets
+                    if any(p.lower() in content.lower()
+                           for p in rules.forget_never):
+                        continue
+                    if self._detect_pii(content):
+                        continue
+
+                    confidence = rules.confidence.get(level, 0.5)
+                    results.append({
+                        "id": str(uuid.uuid4()),
+                        "layer": layer,
+                        "content": content,
+                        "source": "heuristic_extract",
+                        "confidence": confidence,
+                        "created_at": datetime.now().isoformat(),
+                        "expires_at": None,
+                        "tags": [layer, tag, level],
+                    })
+                    break  # One match per line
+
+        # Deduplicate by content similarity
+        seen = set()
+        deduped = []
+        for mem in results:
+            key = mem["content"][:50].lower()
+            if key not in seen:
+                seen.add(key)
+                deduped.append(mem)
+
+        return deduped

memctrl/installer.py

@@ -0,0 +1,122 @@
+"""MemCtrl — SKILL.md installer for AI coding assistants.
+
+Replicates Graphify's install pattern:
+  - `uv tool install graphifyy` → `pip install memctrl`
+  - `graphify install` → `memctrl install`
+  - Writes SKILL.md to ~/.claude/agent/skills/memctrl/SKILL.md etc.
+  - Auto-detects installed tools by checking config dir existence
+
+Research: Graphify writes to ~/.claude/, .claude/, ~/.cursor/, .cursor/,
+~/.codex/, ~/.axga/, ~/.pi/ directories. Uses YAML frontmatter SKILL.md.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import List, Optional
+
+# ---------------------------------------------------------------------------
+# Tool paths
+# ---------------------------------------------------------------------------
+
+TOOL_PATHS = {
+    "claude_code": [
+        "~/.claude/agent/skills/memctrl/SKILL.md",
+        ".claude/agent/skills/memctrl/SKILL.md",
+    ],
+    "cursor": [
+        "~/.cursor/skills/memctrl/SKILL.md",
+        ".cursor/skills/memctrl/SKILL.md",
+    ],
+    "codex": [
+        "~/.codex/skills/memctrl/SKILL.md",
+    ],
+    "axga": [
+        "~/.axga/agent/skills/memctrl/SKILL.md",
+    ],
+    "pi": [
+        "~/.pi/agent/skills/memctrl/SKILL.md",
+    ],
+}
+
+
+# ---------------------------------------------------------------------------
+# Install logic
+# ---------------------------------------------------------------------------
+
+def detect_installed_tools() -> List[str]:
+    """Check which tool config directories exist. Returns tool names."""
+    installed = []
+    for tool_name, paths in TOOL_PATHS.items():
+        for path in paths:
+            expanded = Path(path).expanduser().resolve()
+            if expanded.parent.exists():
+                installed.append(tool_name)
+                break
+    return installed
+
+
+def install_skill(
+    tool: Optional[str] = None,
+    project: bool = False,
+    verbose: bool = True,
+) -> List[str]:
+    """Install SKILL.md for specified tool or all detected tools.
+
+    Args:
+        tool: Specific tool name (claude_code, cursor, codex, etc.)
+        project: If True, install to project-level paths (e.g., .claude/)
+        verbose: Print summary
+
+    Returns:
+        List of paths where SKILL.md was installed.
+    """
+    skill_template = Path(__file__).parent / "templates" / "SKILL.md"
+    if not skill_template.exists():
+        if verbose:
+            print(f"[memctrl] ERROR: SKILL.md template not found at {skill_template}")
+        return []
+
+    targets = [tool] if tool else detect_installed_tools()
+    installed_paths: List[str] = []
+    summary: List[str] = []
+
+    for target in targets:
+        if target not in TOOL_PATHS:
+            if verbose:
+                print(f"[memctrl] Unknown tool: {target}")
+            continue
+
+        paths = TOOL_PATHS[target]
+        if project:
+            project_paths = [p for p in paths if not p.startswith("~/")]
+            for path in project_paths:
+                dest = Path(path)
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(skill_template, dest)
+                installed_paths.append(str(dest))
+                summary.append(f"  {target} (project): {dest}")
+        else:
+            user_paths = [p for p in paths if p.startswith("~/")]
+            for path in user_paths:
+                dest = Path(path).expanduser()
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(skill_template, dest)
+                installed_paths.append(str(dest))
+                summary.append(f"  {target} (user): {dest}")
+
+    if verbose:
+        if installed_paths:
+            print("[memctrl] SKILL.md installed to:")
+            for line in summary:
+                print(line)
+        else:
+            print("[memctrl] No tools detected. Install paths checked:")
+            for tool_name, paths in TOOL_PATHS.items():
+                for p in paths:
+                    print(f"  {tool_name}: {p}")
+            print("\nTo force install for a specific tool, use:")
+            print("  memctrl install --tool claude_code")
+
+    return installed_paths

memctrl/integrations/langgraph.py

@@ -0,0 +1,269 @@
+"""MemCtrl — LangGraph integration.
+
+Provides checkpoint-style persistence and memory nodes for LangGraph agents.
+
+Usage:
+    from memctrl.integrations.langgraph import MemoryNode, MemCtrlMemory
+
+    # As a LangGraph node
+    workflow.add_node("memory", MemoryNode())
+    workflow.add_edge("agent", "memory")
+
+    # As a memory manager inside any node
+    memory = MemCtrlMemory()
+    memory.remember("user prefers dark mode", layer="user")
+    facts = memory.recall("what does the user prefer?")
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Any, Dict, List, Optional
+
+from memctrl.store import MemoryStore
+from memctrl.tree import MemoryTreeBuilder
+from memctrl.retriever import MemoryRetriever
+from memctrl.rules import RuleEngine
+
+# Optional LangGraph import with graceful degradation
+try:
+    from langgraph.checkpoint.base import BaseCheckpointSaver
+    from langgraph.types import StateSnapshot
+    LANGGRAPH_AVAILABLE = True
+except ImportError:
+    BaseCheckpointSaver = object
+    StateSnapshot = Any
+    LANGGRAPH_AVAILABLE = False
+
+
+class MemCtrlMemory:
+    """High-level memory manager for LangGraph agents.
+
+    Wraps MemoryStore with async-friendly methods designed for agent nodes.
+    """
+
+    def __init__(self, db_path: Optional[str] = None):
+        self.store = MemoryStore(db_path)
+        self.builder = MemoryTreeBuilder()
+        self.retriever = MemoryRetriever()
+        self.engine = RuleEngine()
+
+    def remember(
+        self,
+        content: str,
+        layer: str = "session",
+        source: str = "langgraph",
+        confidence: float = 1.0,
+        tags: Optional[List[str]] = None,
+    ) -> str:
+        """Store a memory fact. Returns memory ID."""
+        return self.store.insert_memory(
+            layer=layer,
+            content=content,
+            source=source,
+            confidence=confidence,
+            tags=tags or [],
+        )
+
+    def recall(self, query: str, top_k: int = 5) -> List[str]:
+        """Retrieve relevant memory facts with reasoning trace."""
+        memories = [m.to_dict() for m in self.store.list_memories()]
+        if not memories:
+            return []
+
+        tree = asyncio.run(self.builder.build_tree(memories))
+        tree_dict = tree.to_dict() if tree else {}
+        memory_lookup = {m["id"]: m for m in memories}
+
+        result = asyncio.run(
+            self.retriever.retrieve(query, tree_dict, top_k=top_k, memory_lookup=memory_lookup)
+        )
+        return result.facts
+
+    def recall_with_trace(self, query: str, top_k: int = 5) -> Dict[str, Any]:
+        """Retrieve memories with full trace and metadata."""
+        memories = [m.to_dict() for m in self.store.list_memories()]
+        if not memories:
+            return {"facts": [], "trace": ["empty"], "confidence": 0.0}
+
+        tree = asyncio.run(self.builder.build_tree(memories))
+        tree_dict = tree.to_dict() if tree else {}
+        memory_lookup = {m["id"]: m for m in memories}
+
+        result = asyncio.run(
+            self.retriever.retrieve(query, tree_dict, top_k=top_k, memory_lookup=memory_lookup)
+        )
+        return {
+            "facts": result.facts,
+            "trace": result.trace,
+            "confidence": result.confidence,
+        }
+
+    def consolidate(self, event: str = "on_commit", context: Optional[Dict] = None) -> List[str]:
+        """Fire a trigger rule to consolidate memories."""
+        return self.engine.fire_trigger(event, context or {}, self.store)
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get memory store statistics."""
+        return self.store.stats()
+
+
+class MemoryNode:
+    """LangGraph node that adds persistent memory capabilities.
+
+    Expects state dict with at least:
+        - "messages": list of message dicts (optional, for auto-extraction)
+        - "memory_query": str (optional, for explicit recall)
+        - "memory_facts": list (output, populated by this node)
+
+    Usage:
+        workflow.add_node("memory", MemoryNode())
+        workflow.add_edge("agent", "memory")
+        workflow.add_edge("memory", END)
+    """
+
+    def __init__(self, db_path: Optional[str] = None, auto_extract: bool = True):
+        self.memory = MemCtrlMemory(db_path)
+        self.auto_extract = auto_extract
+
+    def __call__(self, state: Dict[str, Any]) -> Dict[str, Any]:
+        """Process state: extract memories, answer queries, return enriched state."""
+        new_state = dict(state)
+
+        # Auto-extract from latest message if enabled
+        if self.auto_extract and "messages" in state:
+            messages = state["messages"]
+            if messages:
+                latest = messages[-1]
+                content = latest.get("content", "") if isinstance(latest, dict) else str(latest)
+                if len(content) > 20:
+                    self.memory.remember(
+                        content=content[:500],
+                        layer="session",
+                        source="langgraph_conversation",
+                        confidence=0.7,
+                    )
+
+        # Handle explicit memory queries
+        query = state.get("memory_query", "")
+        if query:
+            result = self.memory.recall_with_trace(query)
+            new_state["memory_facts"] = result["facts"]
+            new_state["memory_trace"] = result["trace"]
+            new_state["memory_confidence"] = result["confidence"]
+        else:
+            new_state.setdefault("memory_facts", [])
+            new_state.setdefault("memory_trace", [])
+            new_state.setdefault("memory_confidence", 0.0)
+
+        # Run consolidation if requested
+        if state.get("memory_consolidate"):
+            affected = self.memory.consolidate()
+            new_state["memory_consolidated"] = affected
+
+        return new_state
+
+
+class MemCtrlSaver(BaseCheckpointSaver):
+    """LangGraph checkpoint saver backed by MemCtrl.
+
+    Uses MemoryStore to persist agent state across runs.
+    Provides hierarchical memory + traceability for every checkpoint.
+
+    Usage:
+        from langgraph.graph import StateGraph
+        from memctrl.integrations.langgraph import MemCtrlSaver
+
+        checkpointer = MemCtrlSaver()
+        app = workflow.compile(checkpointer=checkpointer)
+    """
+
+    def __init__(self, db_path: Optional[str] = None):
+        if not LANGGRAPH_AVAILABLE:
+            raise ImportError(
+                "LangGraph is required for MemCtrlSaver. "
+                'Install with: pip install "memctrl[langgraph]"'
+            )
+        super().__init__()
+        self.store = MemoryStore(db_path)
+
+    def get_tuple(self, config: Dict[str, Any]) -> Optional[StateSnapshot]:
+        """Retrieve checkpoint by thread ID."""
+        thread_id = config.get("configurable", {}).get("thread_id", "default")
+        mem = self.store.get_memory(f"checkpoint:{thread_id}")
+        if not mem:
+            return None
+        try:
+            data = json.loads(mem.content)
+            return StateSnapshot(
+                values=data.get("values", {}),
+                next=data.get("next", []),
+                config=config,
+                metadata=data.get("metadata", {}),
+                created_at=mem.created_at,
+                parent_config=data.get("parent_config"),
+                tasks=data.get("tasks", []),
+            )
+        except Exception:
+            return None
+
+    def put(
+        self,
+        config: Dict[str, Any],
+        checkpoint: Dict[str, Any],
+        metadata: Dict[str, Any],
+        new_versions: Any,
+    ) -> Dict[str, Any]:
+        """Store checkpoint."""
+        thread_id = config.get("configurable", {}).get("thread_id", "default")
+        data = {
+            "values": checkpoint.get("values", {}),
+            "next": checkpoint.get("next", []),
+            "metadata": metadata,
+            "parent_config": checkpoint.get("parent_config"),
+            "tasks": checkpoint.get("tasks", []),
+        }
+        # Upsert: delete old then insert
+        self.store.delete_memory(f"checkpoint:{thread_id}")
+        self.store.insert_memory(
+            layer="session",
+            content=json.dumps(data),
+            source=f"checkpoint:{thread_id}",
+            confidence=1.0,
+            tags=["langgraph", "checkpoint", thread_id],
+        )
+        return config
+
+    def list(
+        self,
+        config: Optional[Dict[str, Any]],
+        *,
+        before: Optional[Dict[str, Any]] = None,
+        limit: Optional[int] = None,
+        filter: Optional[Dict[str, Any]] = None,
+    ) -> List[StateSnapshot]:
+        """List checkpoints (returns session-layer checkpoints)."""
+        memories = self.store.list_memories("session")
+        results = []
+        for mem in memories:
+            if not mem.source.startswith("checkpoint:"):
+                continue
+            try:
+                data = json.loads(mem.content)
+                results.append(
+                    StateSnapshot(
+                        values=data.get("values", {}),
+                        next=data.get("next", []),
+                        config=config or {},
+                        metadata=data.get("metadata", {}),
+                        created_at=mem.created_at,
+                        parent_config=data.get("parent_config"),
+                        tasks=data.get("tasks", []),
+                    )
+                )
+            except Exception:
+                continue
+        if limit:
+            results = results[:limit]
+        return results