agmem 0.1.1__py3-none-any.whl

memvcs/core/test_runner.py

@@ -0,0 +1,371 @@
+"""
+Test runner for agmem memory tests.
+
+Implements CI/CD-style testing for agent memory to prevent hallucinated facts
+from corrupting the knowledge base.
+"""
+
+import os
+import json
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from dataclasses import dataclass, field
+from datetime import datetime
+
+try:
+    import yaml
+    YAML_AVAILABLE = True
+except ImportError:
+    YAML_AVAILABLE = False
+
+
+@dataclass
+class TestCase:
+    """A single test case for memory validation."""
+    name: str
+    query: str
+    expected_fact: str
+    confidence_threshold: float = 0.7
+    required: bool = False  # If True, blocks commit on failure
+    tags: List[str] = field(default_factory=list)
+
+
+@dataclass
+class TestFailure:
+    """Represents a failed test."""
+    test_name: str
+    query: str
+    expected: str
+    actual: Optional[str]
+    message: str
+    is_critical: bool = False
+
+
+@dataclass
+class TestResult:
+    """Result of running memory tests."""
+    passed: bool
+    total_count: int
+    passed_count: int
+    failed_count: int
+    failures: List[TestFailure] = field(default_factory=list)
+    duration_ms: int = 0
+
+
+class TestRunner:
+    """
+    Runner for memory regression tests.
+    
+    Tests are defined in YAML files in the tests/ directory of the memory repo.
+    """
+    
+    def __init__(self, repo, vector_store=None):
+        """
+        Initialize test runner.
+        
+        Args:
+            repo: Repository instance
+            vector_store: Optional VectorStore for semantic search tests
+        """
+        self.repo = repo
+        self.vector_store = vector_store
+        self.tests_dir = repo.root / 'tests'
+    
+    def load_tests(self) -> List[TestCase]:
+        """
+        Load all test cases from the tests/ directory.
+        
+        Returns:
+            List of TestCase objects
+        """
+        tests = []
+        
+        if not self.tests_dir.exists():
+            return tests
+        
+        for test_file in self.tests_dir.glob('**/*.yaml'):
+            tests.extend(self._load_test_file(test_file))
+        
+        for test_file in self.tests_dir.glob('**/*.yml'):
+            tests.extend(self._load_test_file(test_file))
+        
+        for test_file in self.tests_dir.glob('**/*.json'):
+            tests.extend(self._load_json_test_file(test_file))
+        
+        return tests
+    
+    def _load_test_file(self, path: Path) -> List[TestCase]:
+        """Load tests from a YAML file."""
+        if not YAML_AVAILABLE:
+            return []
+        
+        try:
+            with open(path) as f:
+                data = yaml.safe_load(f)
+            
+            if not data or 'tests' not in data:
+                return []
+            
+            tests = []
+            file_name = path.stem
+            
+            for i, test_data in enumerate(data['tests']):
+                name = test_data.get('name', f"{file_name}_{i}")
+                tests.append(TestCase(
+                    name=name,
+                    query=test_data['query'],
+                    expected_fact=test_data['expected_fact'],
+                    confidence_threshold=test_data.get('confidence_threshold', 0.7),
+                    required=test_data.get('required', False),
+                    tags=test_data.get('tags', [])
+                ))
+            
+            return tests
+        
+        except Exception as e:
+            print(f"Warning: Failed to load test file {path}: {e}")
+            return []
+    
+    def _load_json_test_file(self, path: Path) -> List[TestCase]:
+        """Load tests from a JSON file."""
+        try:
+            with open(path) as f:
+                data = json.load(f)
+            
+            if not data:
+                return []
+            
+            # Support both array of tests and object with 'tests' key
+            if isinstance(data, list):
+                test_list = data
+            elif 'tests' in data:
+                test_list = data['tests']
+            else:
+                return []
+            
+            tests = []
+            file_name = path.stem
+            
+            for i, test_data in enumerate(test_list):
+                name = test_data.get('name', f"{file_name}_{i}")
+                tests.append(TestCase(
+                    name=name,
+                    query=test_data['query'],
+                    expected_fact=test_data['expected_fact'],
+                    confidence_threshold=test_data.get('confidence_threshold', 0.7),
+                    required=test_data.get('required', False),
+                    tags=test_data.get('tags', [])
+                ))
+            
+            return tests
+        
+        except Exception as e:
+            print(f"Warning: Failed to load test file {path}: {e}")
+            return []
+    
+    def run_test(self, test: TestCase) -> Optional[TestFailure]:
+        """
+        Run a single test case.
+        
+        Returns:
+            TestFailure if test failed, None if passed
+        """
+        # If we have a vector store, use semantic search
+        if self.vector_store:
+            return self._run_semantic_test(test)
+        else:
+            # Fall back to simple text matching
+            return self._run_text_test(test)
+    
+    def _run_semantic_test(self, test: TestCase) -> Optional[TestFailure]:
+        """Run test using semantic search."""
+        try:
+            results = self.vector_store.search(test.query, k=5)
+            
+            if not results:
+                return TestFailure(
+                    test_name=test.name,
+                    query=test.query,
+                    expected=test.expected_fact,
+                    actual=None,
+                    message="No results found for query",
+                    is_critical=test.required
+                )
+            
+            # Check if any result contains the expected fact
+            for result in results:
+                content = result.get('content', '')
+                similarity = result.get('similarity', 0)
+                
+                if similarity >= test.confidence_threshold:
+                    # Use simple string matching as judge
+                    if self._fact_matches(test.expected_fact, content):
+                        return None  # Test passed
+            
+            # No matching result found
+            best_result = results[0] if results else {}
+            return TestFailure(
+                test_name=test.name,
+                query=test.query,
+                expected=test.expected_fact,
+                actual=best_result.get('content', '')[:200],
+                message=f"Expected fact not found in top results (best similarity: {best_result.get('similarity', 0):.2f})",
+                is_critical=test.required
+            )
+        
+        except Exception as e:
+            return TestFailure(
+                test_name=test.name,
+                query=test.query,
+                expected=test.expected_fact,
+                actual=None,
+                message=f"Error running semantic test: {e}",
+                is_critical=test.required
+            )
+    
+    def _run_text_test(self, test: TestCase) -> Optional[TestFailure]:
+        """Run test using simple text search through memory files."""
+        try:
+            current_dir = self.repo.root / 'current'
+            
+            if not current_dir.exists():
+                return TestFailure(
+                    test_name=test.name,
+                    query=test.query,
+                    expected=test.expected_fact,
+                    actual=None,
+                    message="No current/ directory found",
+                    is_critical=test.required
+                )
+            
+            # Search through all memory files
+            for memory_file in current_dir.glob('**/*.md'):
+                try:
+                    content = memory_file.read_text()
+                    if self._fact_matches(test.expected_fact, content):
+                        return None  # Test passed
+                except Exception:
+                    continue
+            
+            return TestFailure(
+                test_name=test.name,
+                query=test.query,
+                expected=test.expected_fact,
+                actual=None,
+                message="Expected fact not found in any memory file",
+                is_critical=test.required
+            )
+        
+        except Exception as e:
+            return TestFailure(
+                test_name=test.name,
+                query=test.query,
+                expected=test.expected_fact,
+                actual=None,
+                message=f"Error running text test: {e}",
+                is_critical=test.required
+            )
+    
+    def _fact_matches(self, expected: str, content: str) -> bool:
+        """
+        Check if expected fact is present in content.
+        
+        Uses case-insensitive substring matching.
+        For more sophisticated matching, this could use an LLM judge.
+        """
+        expected_lower = expected.lower()
+        content_lower = content.lower()
+        
+        # Direct substring match
+        if expected_lower in content_lower:
+            return True
+        
+        # Check if all key words are present
+        key_words = expected_lower.split()
+        if len(key_words) > 2:
+            matches = sum(1 for word in key_words if word in content_lower)
+            if matches >= len(key_words) * 0.8:  # 80% of words match
+                return True
+        
+        return False
+    
+    def run_all(self, tags: Optional[List[str]] = None) -> TestResult:
+        """
+        Run all tests.
+        
+        Args:
+            tags: Optional list of tags to filter tests
+            
+        Returns:
+            TestResult with overall results
+        """
+        start_time = datetime.now()
+        tests = self.load_tests()
+        
+        # Filter by tags if specified
+        if tags:
+            tests = [t for t in tests if any(tag in t.tags for tag in tags)]
+        
+        failures = []
+        passed_count = 0
+        
+        for test in tests:
+            failure = self.run_test(test)
+            if failure:
+                failures.append(failure)
+            else:
+                passed_count += 1
+        
+        duration = (datetime.now() - start_time).total_seconds() * 1000
+        
+        # Check if any critical tests failed
+        critical_failures = [f for f in failures if f.is_critical]
+        passed = len(critical_failures) == 0
+        
+        return TestResult(
+            passed=passed,
+            total_count=len(tests),
+            passed_count=passed_count,
+            failed_count=len(failures),
+            failures=failures,
+            duration_ms=int(duration)
+        )
+    
+    def run_for_branch(self, branch: str) -> TestResult:
+        """
+        Run tests against a specific branch.
+        
+        Creates a temporary vector store with only the branch's data.
+        
+        Args:
+            branch: Branch name to test
+            
+        Returns:
+            TestResult
+        """
+        # For now, just run normal tests
+        # TODO: Implement branch-specific testing with temporary vector store
+        return self.run_all()
+
+
+def create_test_template() -> str:
+    """Create a template test file."""
+    return """# Memory Tests
+# Tests are run with 'agmem test' to validate memory consistency
+
+tests:
+  - name: "example_test"
+    query: "What is the main purpose of this project?"
+    expected_fact: "version control for agent memory"
+    confidence_threshold: 0.7
+    required: false
+    tags:
+      - "core"
+      - "basics"
+
+  # Add more tests below:
+  # - name: "test_name"
+  #   query: "Your query here"
+  #   expected_fact: "Expected answer"
+  #   required: true  # Set to true for critical tests that block commits
+"""

memvcs/core/vector_store.py

@@ -0,0 +1,313 @@
+"""
+Vector store for semantic search over agmem memory.
+
+Uses sqlite-vec for local vector storage and sentence-transformers for embeddings.
+Requires: pip install agmem[vector]
+"""
+
+import logging
+from typing import List, Optional, Tuple
+
+from .constants import MEMORY_TYPES
+import struct
+from pathlib import Path
+
+logger = logging.getLogger("agmem.vector_store")
+
+# Embedding dimension for all-MiniLM-L6-v2
+EMBEDDING_DIM = 384
+
+
+def _serialize_f32(vector: List[float]) -> bytes:
+    """Serialize float list to bytes for sqlite-vec."""
+    return struct.pack(f"{len(vector)}f", *vector)
+
+
+class VectorStore:
+    """Semantic search over memory using vector embeddings."""
+
+    def __init__(self, mem_dir: Path):
+        self.mem_dir = Path(mem_dir)
+        self.db_path = self.mem_dir / "vectors.db"
+        self._model = None
+        self._conn = None
+
+    def _get_connection(self):
+        """Get SQLite connection with sqlite-vec loaded."""
+        if self._conn is not None:
+            return self._conn
+
+        try:
+            import sqlite3
+            import sqlite_vec
+
+            self._conn = sqlite3.connect(str(self.db_path))
+            self._conn.enable_load_extension(True)
+            sqlite_vec.load(self._conn)
+            self._conn.enable_load_extension(False)
+            return self._conn
+        except ImportError as e:
+            raise ImportError(
+                "Vector search requires sqlite-vec. Install with: pip install agmem[vector]"
+            ) from e
+        except AttributeError as e:
+            raise ImportError(
+                "SQLite extension loading not supported. "
+                "On macOS, try: brew install python (for Homebrew SQLite)"
+            ) from e
+
+    def _get_model(self):
+        """Lazy-load the sentence-transformers model."""
+        if self._model is not None:
+            return self._model
+
+        try:
+            from sentence_transformers import SentenceTransformer
+
+            self._model = SentenceTransformer("all-MiniLM-L6-v2")
+            return self._model
+        except ImportError as e:
+            raise ImportError(
+                "Vector search requires sentence-transformers. "
+                "Install with: pip install agmem[vector]"
+            ) from e
+
+    def _ensure_tables(self):
+        """Create vector and metadata tables if they don't exist."""
+        conn = self._get_connection()
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS memory_meta (
+                rowid INTEGER PRIMARY KEY,
+                path TEXT NOT NULL,
+                content TEXT NOT NULL,
+                blob_hash TEXT,
+                commit_hash TEXT,
+                author TEXT,
+                indexed_at TEXT
+            )
+        """)
+        # Try to add new columns to existing tables (for upgrades)
+        for col in ['commit_hash TEXT', 'author TEXT', 'indexed_at TEXT']:
+            try:
+                conn.execute(f"ALTER TABLE memory_meta ADD COLUMN {col}")
+            except Exception:
+                pass  # Column already exists
+        try:
+            conn.execute(f"""
+                CREATE VIRTUAL TABLE IF NOT EXISTS vec_memory
+                USING vec0(embedding float[{EMBEDDING_DIM}])
+            """)
+        except Exception as e:
+            # vec0 might already exist with different schema
+            logger.debug("vec_memory creation: %s", e)
+        conn.commit()
+
+    def _embed(self, text: str) -> List[float]:
+        """Generate embedding for text."""
+        model = self._get_model()
+        emb = model.encode(text, convert_to_numpy=True)
+        return emb.astype("float32").tolist()
+
+    def index_content(
+        self,
+        path: str,
+        content: str,
+        blob_hash: Optional[str] = None,
+        commit_hash: Optional[str] = None,
+        author: Optional[str] = None
+    ) -> None:
+        """
+        Index a memory file for semantic search.
+        
+        Args:
+            path: File path relative to current/
+            content: File content to index
+            blob_hash: Optional blob hash from object store
+            commit_hash: Optional commit hash for provenance tracking
+            author: Optional author string for provenance tracking
+        """
+        from datetime import datetime
+        
+        self._ensure_tables()
+        conn = self._get_connection()
+
+        embedding = self._embed(content)
+        emb_bytes = _serialize_f32(embedding)
+        indexed_at = datetime.utcnow().isoformat() + 'Z'
+
+        with conn:
+            conn.execute(
+                """INSERT INTO memory_meta 
+                   (path, content, blob_hash, commit_hash, author, indexed_at) 
+                   VALUES (?, ?, ?, ?, ?, ?)""",
+                (path, content[:10000], blob_hash, commit_hash, author, indexed_at),
+            )
+            rowid = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
+            conn.execute(
+                "INSERT INTO vec_memory (rowid, embedding) VALUES (?, ?)",
+                (rowid, emb_bytes),
+            )
+        conn.commit()
+
+    def index_directory(self, current_dir: Path) -> int:
+        """Index all memory files in current/ directory. Returns count indexed."""
+        self._ensure_tables()
+        count = 0
+
+        for subdir in MEMORY_TYPES:
+            dir_path = current_dir / subdir
+            if not dir_path.exists():
+                continue
+            for f in dir_path.rglob("*"):
+                if f.is_file():
+                    try:
+                        content = f.read_text(encoding="utf-8", errors="replace")
+                        rel_path = str(f.relative_to(current_dir))
+                        self.index_content(rel_path, content)
+                        count += 1
+                    except Exception as e:
+                        logger.warning("Failed to index %s: %s", f, e)
+
+        return count
+
+    def search(
+        self, query: str, limit: int = 10, min_score: Optional[float] = None
+    ) -> List[Tuple[str, str, float]]:
+        """
+        Semantic search. Returns list of (path, content_snippet, distance).
+        Lower distance = more similar.
+        """
+        self._ensure_tables()
+        conn = self._get_connection()
+
+        query_embedding = self._embed(query)
+        emb_bytes = _serialize_f32(query_embedding)
+
+        rows = conn.execute(
+            """
+            SELECT m.path, m.content, v.distance
+            FROM vec_memory v
+            JOIN memory_meta m ON v.rowid = m.rowid
+            WHERE v.embedding MATCH ?
+            ORDER BY v.distance
+            LIMIT ?
+            """,
+            (emb_bytes, limit),
+        ).fetchall()
+
+        results = []
+        for path, content, distance in rows:
+            if min_score is not None and distance > min_score:
+                continue
+            snippet = content[:500] + ("..." if len(content) > 500 else "")
+            results.append((path, snippet, float(distance)))
+
+        return results
+    
+    def search_with_provenance(
+        self, query: str, limit: int = 10, min_score: Optional[float] = None
+    ) -> List[dict]:
+        """
+        Semantic search with provenance metadata.
+        
+        Returns list of dicts with: path, content, distance, commit_hash, author, indexed_at
+        """
+        self._ensure_tables()
+        conn = self._get_connection()
+
+        query_embedding = self._embed(query)
+        emb_bytes = _serialize_f32(query_embedding)
+
+        rows = conn.execute(
+            """
+            SELECT m.path, m.content, v.distance, m.commit_hash, m.author, m.indexed_at, m.blob_hash
+            FROM vec_memory v
+            JOIN memory_meta m ON v.rowid = m.rowid
+            WHERE v.embedding MATCH ?
+            ORDER BY v.distance
+            LIMIT ?
+            """,
+            (emb_bytes, limit),
+        ).fetchall()
+
+        results = []
+        for path, content, distance, commit_hash, author, indexed_at, blob_hash in rows:
+            if min_score is not None and distance > min_score:
+                continue
+            snippet = content[:500] + ("..." if len(content) > 500 else "")
+            results.append({
+                'path': path,
+                'content': snippet,
+                'distance': float(distance),
+                'similarity': 1.0 - float(distance),  # Convert to similarity score
+                'commit_hash': commit_hash,
+                'author': author,
+                'indexed_at': indexed_at,
+                'blob_hash': blob_hash
+            })
+
+        return results
+    
+    def get_all_entries(self) -> List[dict]:
+        """
+        Get all indexed entries with their metadata.
+        
+        Used for fsck operations to check for dangling vectors.
+        """
+        self._ensure_tables()
+        conn = self._get_connection()
+        
+        rows = conn.execute(
+            """
+            SELECT rowid, path, blob_hash, commit_hash, author, indexed_at
+            FROM memory_meta
+            """
+        ).fetchall()
+        
+        return [
+            {
+                'rowid': rowid,
+                'path': path,
+                'blob_hash': blob_hash,
+                'commit_hash': commit_hash,
+                'author': author,
+                'indexed_at': indexed_at
+            }
+            for rowid, path, blob_hash, commit_hash, author, indexed_at in rows
+        ]
+    
+    def delete_entry(self, rowid: int) -> bool:
+        """
+        Delete an entry by rowid.
+        
+        Used by fsck to remove dangling vectors.
+        """
+        conn = self._get_connection()
+        try:
+            with conn:
+                conn.execute("DELETE FROM memory_meta WHERE rowid = ?", (rowid,))
+                conn.execute("DELETE FROM vec_memory WHERE rowid = ?", (rowid,))
+            conn.commit()
+            return True
+        except Exception as e:
+            logger.warning("Failed to delete entry %s: %s", rowid, e)
+            return False
+
+    def rebuild_index(self, current_dir: Path) -> int:
+        """Clear and rebuild the vector index from current/."""
+        conn = self._get_connection()
+        with conn:
+            try:
+                conn.execute("DROP TABLE IF EXISTS vec_memory")
+            except Exception:
+                pass
+            conn.execute("DELETE FROM memory_meta")
+        conn.commit()
+        self._ensure_tables()
+        return self.index_directory(current_dir)
+
+    def close(self):
+        """Close the database connection."""
+        if self._conn:
+            self._conn.close()
+            self._conn = None

memvcs/integrations/__init__.py

@@ -0,0 +1,5 @@
+"""agmem integrations."""
+
+from .mcp_server import run as run_mcp_server
+
+__all__ = ["run_mcp_server"]