agmem 0.1.1__py3-none-any.whl

memvcs/core/gardener.py

@@ -0,0 +1,466 @@
+"""
+Gardener - The "Hindsight" reflection loop for agmem.
+
+A background process that synthesizes raw episodic logs into semantic insights,
+turning noise into wisdom over time.
+"""
+
+import os
+import json
+import shutil
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+from dataclasses import dataclass, field
+from datetime import datetime
+from collections import defaultdict
+
+try:
+    import yaml
+    YAML_AVAILABLE = True
+except ImportError:
+    YAML_AVAILABLE = False
+
+
+@dataclass
+class EpisodeCluster:
+    """A cluster of related episodes."""
+    topic: str
+    episodes: List[Path]
+    summary: Optional[str] = None
+    tags: List[str] = field(default_factory=list)
+
+
+@dataclass
+class GardenerConfig:
+    """Configuration for the Gardener."""
+    threshold: int = 50  # Number of episodic files before triggering
+    archive_dir: str = "archive"
+    min_cluster_size: int = 3
+    max_clusters: int = 10
+    llm_provider: Optional[str] = None  # "openai", "anthropic", etc.
+    llm_model: Optional[str] = None
+    auto_commit: bool = True
+
+
+@dataclass
+class GardenerResult:
+    """Result of a gardener run."""
+    success: bool
+    clusters_found: int
+    insights_generated: int
+    episodes_archived: int
+    commit_hash: Optional[str] = None
+    message: str = ""
+
+
+class Gardener:
+    """
+    The Gardener agent that refines memory over time.
+    
+    Wakes up when episodic/ files exceed a threshold, clusters them by topic,
+    generates summaries, and archives the raw episodes.
+    """
+    
+    def __init__(self, repo, config: Optional[GardenerConfig] = None):
+        """
+        Initialize the Gardener.
+        
+        Args:
+            repo: Repository instance
+            config: Optional configuration
+        """
+        self.repo = repo
+        self.config = config or GardenerConfig()
+        self.episodic_dir = repo.root / 'current' / 'episodic'
+        self.semantic_dir = repo.root / 'current' / 'semantic'
+        # Ensure archive_dir stays under current/ (path safety)
+        try:
+            archive_candidate = (repo.current_dir / self.config.archive_dir).resolve()
+            archive_candidate.relative_to(repo.current_dir.resolve())
+            self.archive_dir = archive_candidate
+        except (ValueError, RuntimeError):
+            self.archive_dir = repo.current_dir / 'archive'
+    
+    def should_run(self) -> bool:
+        """Check if the Gardener should run based on threshold."""
+        if not self.episodic_dir.exists():
+            return False
+        
+        episode_count = len(list(self.episodic_dir.glob('**/*.md')))
+        return episode_count >= self.config.threshold
+    
+    def get_episode_count(self) -> int:
+        """Get the current number of episodic files."""
+        if not self.episodic_dir.exists():
+            return 0
+        return len(list(self.episodic_dir.glob('**/*.md')))
+    
+    def load_episodes(self) -> List[Tuple[Path, str]]:
+        """
+        Load all episodic files.
+        
+        Returns:
+            List of (path, content) tuples
+        """
+        episodes = []
+        
+        if not self.episodic_dir.exists():
+            return episodes
+        
+        for episode_file in self.episodic_dir.glob('**/*.md'):
+            try:
+                content = episode_file.read_text()
+                episodes.append((episode_file, content))
+            except Exception:
+                continue
+        
+        return episodes
+    
+    def cluster_episodes(self, episodes: List[Tuple[Path, str]]) -> List[EpisodeCluster]:
+        """
+        Cluster episodes by topic using keyword analysis.
+        
+        For more sophisticated clustering, this could use embeddings with k-means.
+        
+        Args:
+            episodes: List of (path, content) tuples
+            
+        Returns:
+            List of EpisodeCluster objects
+        """
+        # Simple keyword-based clustering
+        keyword_to_episodes: Dict[str, List[Path]] = defaultdict(list)
+        
+        # Common programming/tech keywords to look for
+        keywords = [
+            'python', 'javascript', 'typescript', 'rust', 'go',
+            'error', 'bug', 'fix', 'debug', 'issue',
+            'api', 'database', 'server', 'client', 'frontend', 'backend',
+            'test', 'testing', 'deploy', 'deployment',
+            'config', 'setup', 'install', 'environment',
+            'performance', 'optimization', 'memory', 'cache',
+            'security', 'auth', 'authentication', 'permission',
+            'user', 'preference', 'setting', 'option',
+        ]
+        
+        for path, content in episodes:
+            content_lower = content.lower()
+            found_keywords = []
+            
+            for keyword in keywords:
+                if keyword in content_lower:
+                    found_keywords.append(keyword)
+                    keyword_to_episodes[keyword].append(path)
+        
+        # Create clusters from keywords with enough episodes
+        clusters = []
+        used_episodes = set()
+        
+        # Sort by number of episodes (descending)
+        sorted_keywords = sorted(
+            keyword_to_episodes.items(),
+            key=lambda x: len(x[1]),
+            reverse=True
+        )
+        
+        for keyword, episode_paths in sorted_keywords:
+            if len(clusters) >= self.config.max_clusters:
+                break
+            
+            # Filter out already-used episodes
+            unused_paths = [p for p in episode_paths if p not in used_episodes]
+            
+            if len(unused_paths) >= self.config.min_cluster_size:
+                clusters.append(EpisodeCluster(
+                    topic=keyword,
+                    episodes=unused_paths,
+                    tags=[keyword]
+                ))
+                used_episodes.update(unused_paths)
+        
+        return clusters
+    
+    def cluster_episodes_with_embeddings(self, episodes: List[Tuple[Path, str]]) -> List[EpisodeCluster]:
+        """
+        Cluster episodes using embeddings and k-means.
+        
+        Requires scikit-learn and sentence-transformers.
+        """
+        try:
+            from sklearn.cluster import KMeans
+            from sentence_transformers import SentenceTransformer
+        except ImportError:
+            # Fall back to keyword clustering
+            return self.cluster_episodes(episodes)
+        
+        if len(episodes) < self.config.min_cluster_size:
+            return []
+        
+        # Generate embeddings
+        model = SentenceTransformer('all-MiniLM-L6-v2')
+        texts = [content[:2000] for _, content in episodes]  # Truncate long texts
+        embeddings = model.encode(texts)
+        
+        # Determine number of clusters
+        n_clusters = min(self.config.max_clusters, len(episodes) // self.config.min_cluster_size)
+        n_clusters = max(1, n_clusters)
+        
+        # Cluster
+        kmeans = KMeans(n_clusters=n_clusters, random_state=42)
+        labels = kmeans.fit_predict(embeddings)
+        
+        # Group episodes by cluster
+        cluster_episodes: Dict[int, List[Tuple[Path, str]]] = defaultdict(list)
+        for i, (path, content) in enumerate(episodes):
+            cluster_episodes[labels[i]].append((path, content))
+        
+        # Create cluster objects
+        clusters = []
+        for cluster_id, eps in cluster_episodes.items():
+            if len(eps) >= self.config.min_cluster_size:
+                # Extract topic from first few words of first episode
+                first_content = eps[0][1]
+                topic = self._extract_topic(first_content)
+                
+                clusters.append(EpisodeCluster(
+                    topic=topic,
+                    episodes=[p for p, _ in eps]
+                ))
+        
+        return clusters
+    
+    def _extract_topic(self, content: str) -> str:
+        """Extract a topic label from content."""
+        # Take first line or first 50 chars
+        lines = content.strip().split('\n')
+        first_line = lines[0] if lines else content[:50]
+        
+        # Clean up
+        topic = first_line.strip('#').strip()
+        if len(topic) > 50:
+            topic = topic[:47] + '...'
+        
+        return topic or "general"
+    
+    def generate_summary(self, cluster: EpisodeCluster) -> str:
+        """
+        Generate a summary for a cluster of episodes.
+        
+        Uses LLM if configured, otherwise generates a simple summary.
+        """
+        # Collect content from episodes
+        contents = []
+        for episode_path in cluster.episodes[:10]:  # Limit to 10 episodes
+            try:
+                content = episode_path.read_text()
+                contents.append(content[:1000])  # Truncate
+            except Exception:
+                continue
+        
+        combined = '\n---\n'.join(contents)
+        
+        # Try LLM summarization
+        if self.config.llm_provider == 'openai' and self.config.llm_model:
+            try:
+                return self._summarize_with_openai(combined, cluster.topic)
+            except Exception:
+                pass
+        
+        # Fall back to simple summary
+        return self._simple_summary(cluster, contents)
+    
+    def _summarize_with_openai(self, content: str, topic: str) -> str:
+        """Summarize using OpenAI API."""
+        import openai
+        
+        response = openai.chat.completions.create(
+            model=self.config.llm_model or 'gpt-3.5-turbo',
+            messages=[
+                {
+                    'role': 'system',
+                    'content': 'You are a helpful assistant that summarizes conversation logs into actionable insights.'
+                },
+                {
+                    'role': 'user',
+                    'content': f"Summarize these conversation logs about '{topic}' into 2-3 key insights:\n\n{content[:4000]}"
+                }
+            ],
+            max_tokens=500
+        )
+        
+        return response.choices[0].message.content
+    
+    def _simple_summary(self, cluster: EpisodeCluster, contents: List[str]) -> str:
+        """Generate a simple summary without LLM."""
+        return f"""# Insights: {cluster.topic.title()}
+
+**Summary**: The user had {len(cluster.episodes)} conversations related to {cluster.topic}.
+
+**Common themes observed**:
+- Multiple discussions about {cluster.topic}
+- Recurring questions and patterns detected
+
+**Generated**: {datetime.utcnow().isoformat()}Z
+
+---
+*This summary was auto-generated by the Gardener. Review and edit as needed.*
+"""
+    
+    def write_insight(self, cluster: EpisodeCluster) -> Path:
+        """
+        Write cluster summary to semantic memory.
+        
+        Returns:
+            Path to the written insight file
+        """
+        self.semantic_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Generate filename (sanitize topic to avoid path traversal)
+        timestamp = datetime.utcnow().strftime('%Y%m%d')
+        safe_topic = cluster.topic.replace(' ', '-').lower().replace('/', '_').replace('\\', '_')[:30]
+        filename = f"insight-{safe_topic}-{timestamp}.md"
+        insight_path = (self.semantic_dir / filename).resolve()
+        try:
+            insight_path.relative_to(self.repo.current_dir.resolve())
+        except ValueError:
+            insight_path = self.semantic_dir / f"insight-{timestamp}.md"
+        
+        # Generate frontmatter
+        frontmatter = {
+            'schema_version': '1.0',
+            'last_updated': datetime.utcnow().isoformat() + 'Z',
+            'source_agent_id': 'gardener',
+            'memory_type': 'semantic',
+            'tags': cluster.tags + ['auto-generated', 'insight'],
+            'source_episodes': len(cluster.episodes)
+        }
+        
+        # Write file
+        if YAML_AVAILABLE:
+            import yaml
+            content = f"---\n{yaml.dump(frontmatter, default_flow_style=False)}---\n\n{cluster.summary}"
+        else:
+            content = cluster.summary
+        
+        insight_path.write_text(content)
+        return insight_path
+    
+    def archive_episodes(self, episodes: List[Path]) -> int:
+        """
+        Archive processed episodes.
+        
+        Moves files to archive directory with timestamp prefix.
+        
+        Returns:
+            Number of files archived
+        """
+        self.archive_dir.mkdir(parents=True, exist_ok=True)
+        
+        timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
+        archive_subdir = self.archive_dir / timestamp
+        archive_subdir.mkdir(exist_ok=True)
+        
+        count = 0
+        for episode_path in episodes:
+            try:
+                safe_name = episode_path.name.replace('..', '_').replace('/', '_').replace('\\', '_')
+                dest = (archive_subdir / safe_name).resolve()
+                dest.relative_to(self.archive_dir.resolve())
+                shutil.move(str(episode_path), str(dest))
+                count += 1
+            except (ValueError, Exception):
+                continue
+        
+        return count
+    
+    def run(self, force: bool = False) -> GardenerResult:
+        """
+        Run the Gardener process.
+        
+        Args:
+            force: Run even if threshold not met
+            
+        Returns:
+            GardenerResult with operation details
+        """
+        if not force and not self.should_run():
+            return GardenerResult(
+                success=True,
+                clusters_found=0,
+                insights_generated=0,
+                episodes_archived=0,
+                message=f"Threshold not met ({self.get_episode_count()}/{self.config.threshold} episodes)"
+            )
+        
+        # Load episodes
+        episodes = self.load_episodes()
+        if not episodes:
+            return GardenerResult(
+                success=True,
+                clusters_found=0,
+                insights_generated=0,
+                episodes_archived=0,
+                message="No episodes to process"
+            )
+        
+        # Cluster episodes
+        try:
+            clusters = self.cluster_episodes_with_embeddings(episodes)
+        except Exception:
+            clusters = self.cluster_episodes(episodes)
+        
+        if not clusters:
+            return GardenerResult(
+                success=True,
+                clusters_found=0,
+                insights_generated=0,
+                episodes_archived=0,
+                message="No clusters could be formed"
+            )
+        
+        # Generate summaries and write insights
+        insights_written = 0
+        all_archived_episodes = []
+        
+        for cluster in clusters:
+            try:
+                # Generate summary
+                cluster.summary = self.generate_summary(cluster)
+                
+                # Write insight
+                self.write_insight(cluster)
+                insights_written += 1
+                
+                # Track episodes to archive
+                all_archived_episodes.extend(cluster.episodes)
+            except Exception as e:
+                print(f"Warning: Failed to process cluster '{cluster.topic}': {e}")
+        
+        # Archive processed episodes
+        archived_count = self.archive_episodes(all_archived_episodes)
+        
+        # Auto-commit if configured
+        commit_hash = None
+        if self.config.auto_commit and insights_written > 0:
+            try:
+                # Stage new insights
+                for insight_file in self.semantic_dir.glob('insight-*.md'):
+                    rel_path = str(insight_file.relative_to(self.repo.root / 'current'))
+                    self.repo.stage_file(f"current/{rel_path}")
+                
+                # Commit
+                commit_hash = self.repo.commit(
+                    f"gardener: synthesized {insights_written} insights from {archived_count} episodes",
+                    {'gardener': True, 'clusters': len(clusters)}
+                )
+            except Exception as e:
+                print(f"Warning: Auto-commit failed: {e}")
+        
+        return GardenerResult(
+            success=True,
+            clusters_found=len(clusters),
+            insights_generated=insights_written,
+            episodes_archived=archived_count,
+            commit_hash=commit_hash,
+            message=f"Processed {len(clusters)} clusters, generated {insights_written} insights"
+        )

memvcs/core/hooks.py

@@ -0,0 +1,151 @@
+"""
+Pre-commit hooks for agmem.
+
+Provides hook infrastructure for validation before commits.
+PII scanning can be disabled or allowlisted via agmem config (pii.enabled, pii.allowlist).
+"""
+
+import fnmatch
+from dataclasses import dataclass, field
+from typing import List, Dict, Any, Callable, Optional
+from pathlib import Path
+
+PII_SEVERITY_HIGH = "high"
+
+
+@dataclass
+class HookResult:
+    """Result of running hooks."""
+    success: bool
+    errors: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+    
+    def add_error(self, message: str):
+        """Add an error and mark as failed."""
+        self.errors.append(message)
+        self.success = False
+    
+    def add_warning(self, message: str):
+        """Add a warning (doesn't affect success)."""
+        self.warnings.append(message)
+
+
+def _is_path_allowlisted(filepath: str, patterns: List[str]) -> bool:
+    """Return True if filepath matches any allowlist glob pattern."""
+    return any(fnmatch.fnmatch(filepath, pat) for pat in patterns)
+
+
+def _pii_staged_files_to_scan(repo, staged_files: Dict[str, Any]) -> Dict[str, Any]:
+    """Return staged files to scan for PII (excludes allowlisted paths)."""
+    try:
+        from .config_loader import load_agmem_config, pii_enabled, pii_allowlist
+        config = load_agmem_config(getattr(repo, "root", None))
+    except ImportError:
+        return staged_files
+    if not pii_enabled(config):
+        return {}
+    patterns = pii_allowlist(config)
+    if not patterns:
+        return staged_files
+    return {
+        filepath: info
+        for filepath, info in staged_files.items()
+        if not _is_path_allowlisted(filepath, patterns)
+    }
+
+
+def _run_pii_hook(repo, staged_files: Dict[str, Any], result: HookResult) -> None:
+    """Run PII scanner on staged files; high severity → error, else → warning."""
+    try:
+        from .pii_scanner import PIIScanner
+        to_scan = _pii_staged_files_to_scan(repo, staged_files)
+        pii_result = PIIScanner.scan_staged_files(repo, to_scan)
+        if not pii_result.has_issues:
+            return
+        for issue in pii_result.issues:
+            msg = f"PII detected in {issue.filepath}: {issue.description}"
+            if issue.severity == PII_SEVERITY_HIGH:
+                result.add_error(msg)
+            else:
+                result.add_warning(msg)
+    except ImportError:
+        pass
+    except Exception as e:
+        result.add_warning(f"PII scanner failed: {e}")
+
+
+def run_pre_commit_hooks(repo, staged_files: Dict[str, Any]) -> HookResult:
+    """
+    Run all pre-commit hooks on staged files.
+    
+    Args:
+        repo: Repository instance
+        staged_files: Dict of staged files with their info
+        
+    Returns:
+        HookResult with success status and any errors/warnings
+    """
+    result = HookResult(success=True)
+    _run_pii_hook(repo, staged_files, result)
+    file_type_result = validate_file_types(repo, staged_files)
+    if not file_type_result.success:
+        for error in file_type_result.errors:
+            result.add_error(error)
+    for warning in file_type_result.warnings:
+        result.add_warning(warning)
+    
+    return result
+
+
+def validate_file_types(repo, staged_files: Dict[str, Any]) -> HookResult:
+    """
+    Validate that staged files are allowed types.
+    
+    Args:
+        repo: Repository instance
+        staged_files: Dict of staged files
+        
+    Returns:
+        HookResult with validation status
+    """
+    result = HookResult(success=True)
+    
+    # Get config for allowed extensions
+    config = repo.get_config()
+    allowed_extensions = config.get('allowed_extensions', ['.md', '.txt', '.json', '.yaml', '.yml'])
+    
+    for filepath in staged_files.keys():
+        path = Path(filepath)
+        ext = path.suffix.lower()
+        
+        # Skip files without extensions (might be valid)
+        if not ext:
+            continue
+        
+        # Check if extension is allowed
+        if ext not in allowed_extensions:
+            result.add_warning(
+                f"File '{filepath}' has extension '{ext}' which may not be optimal for memory storage. "
+                f"Recommended: {', '.join(allowed_extensions)}"
+            )
+    
+    return result
+
+
+# Hook registry for custom hooks
+_registered_hooks: List[Callable] = []
+
+
+def register_hook(hook_fn: Callable):
+    """
+    Register a custom pre-commit hook.
+    
+    Args:
+        hook_fn: Function that takes (repo, staged_files) and returns HookResult
+    """
+    _registered_hooks.append(hook_fn)
+
+
+def get_registered_hooks() -> List[Callable]:
+    """Get all registered hooks."""
+    return _registered_hooks.copy()