agmem 0.1.1__py3-none-any.whl

memvcs/core/schema.py

@@ -0,0 +1,414 @@
+"""
+Schema validation for agmem memory files.
+
+Implements YAML frontmatter parsing and validation for structured memory metadata.
+"""
+
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Dict, Any, List, Tuple
+from dataclasses import dataclass, field
+from enum import Enum
+
+try:
+    import yaml
+    YAML_AVAILABLE = True
+except ImportError:
+    YAML_AVAILABLE = False
+
+from .constants import MEMORY_TYPES
+
+
+class MemoryType(Enum):
+    """Memory types with their validation requirements."""
+    EPISODIC = "episodic"
+    SEMANTIC = "semantic"
+    PROCEDURAL = "procedural"
+    CHECKPOINTS = "checkpoints"
+    SESSION_SUMMARIES = "session-summaries"
+    UNKNOWN = "unknown"
+
+
+@dataclass
+class FrontmatterData:
+    """Parsed frontmatter data from a memory file."""
+    schema_version: str = "1.0"
+    last_updated: Optional[str] = None
+    source_agent_id: Optional[str] = None
+    confidence_score: Optional[float] = None
+    memory_type: Optional[str] = None
+    tags: List[str] = field(default_factory=list)
+    extra: Dict[str, Any] = field(default_factory=dict)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        result = {
+            "schema_version": self.schema_version,
+        }
+        if self.last_updated:
+            result["last_updated"] = self.last_updated
+        if self.source_agent_id:
+            result["source_agent_id"] = self.source_agent_id
+        if self.confidence_score is not None:
+            result["confidence_score"] = self.confidence_score
+        if self.memory_type:
+            result["memory_type"] = self.memory_type
+        if self.tags:
+            result["tags"] = self.tags
+        result.update(self.extra)
+        return result
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'FrontmatterData':
+        """Create from dictionary."""
+        known_fields = {
+            'schema_version', 'last_updated', 'source_agent_id',
+            'confidence_score', 'memory_type', 'tags'
+        }
+        extra = {k: v for k, v in data.items() if k not in known_fields}
+        
+        return cls(
+            schema_version=data.get('schema_version', '1.0'),
+            last_updated=data.get('last_updated'),
+            source_agent_id=data.get('source_agent_id'),
+            confidence_score=data.get('confidence_score'),
+            memory_type=data.get('memory_type'),
+            tags=data.get('tags', []),
+            extra=extra
+        )
+
+
+@dataclass
+class ValidationError:
+    """A single validation error."""
+    field: str
+    message: str
+    severity: str = "error"  # "error" or "warning"
+
+
+@dataclass
+class ValidationResult:
+    """Result of validating a memory file."""
+    valid: bool
+    errors: List[ValidationError] = field(default_factory=list)
+    warnings: List[ValidationError] = field(default_factory=list)
+    frontmatter: Optional[FrontmatterData] = None
+    
+    def add_error(self, field: str, message: str):
+        """Add a validation error."""
+        self.errors.append(ValidationError(field=field, message=message, severity="error"))
+        self.valid = False
+    
+    def add_warning(self, field: str, message: str):
+        """Add a validation warning."""
+        self.warnings.append(ValidationError(field=field, message=message, severity="warning"))
+
+
+class FrontmatterParser:
+    """Parser for YAML frontmatter in memory files."""
+    
+    # Regex to match YAML frontmatter block
+    FRONTMATTER_PATTERN = re.compile(
+        r'^---\s*\n(.*?)\n---\s*\n',
+        re.DOTALL | re.MULTILINE
+    )
+    
+    @classmethod
+    def parse(cls, content: str) -> Tuple[Optional[FrontmatterData], str]:
+        """
+        Parse frontmatter from content.
+        
+        Args:
+            content: Full file content
+            
+        Returns:
+            Tuple of (frontmatter_data, body_content)
+            frontmatter_data is None if no frontmatter found
+        """
+        if not YAML_AVAILABLE:
+            # Without PyYAML, return None for frontmatter
+            return None, content
+        
+        match = cls.FRONTMATTER_PATTERN.match(content)
+        if not match:
+            return None, content
+        
+        yaml_content = match.group(1)
+        body = content[match.end():]
+        
+        try:
+            data = yaml.safe_load(yaml_content)
+            if not isinstance(data, dict):
+                return None, content
+            
+            frontmatter = FrontmatterData.from_dict(data)
+            return frontmatter, body
+        except yaml.YAMLError:
+            return None, content
+    
+    @classmethod
+    def has_frontmatter(cls, content: str) -> bool:
+        """Check if content has YAML frontmatter."""
+        return bool(cls.FRONTMATTER_PATTERN.match(content))
+    
+    @classmethod
+    def create_frontmatter(cls, data: FrontmatterData) -> str:
+        """
+        Create YAML frontmatter string from data.
+        
+        Args:
+            data: FrontmatterData to serialize
+            
+        Returns:
+            YAML frontmatter string with delimiters
+        """
+        if not YAML_AVAILABLE:
+            # Manual YAML generation without PyYAML
+            lines = ["---"]
+            d = data.to_dict()
+            for key, value in d.items():
+                if isinstance(value, list):
+                    lines.append(f"{key}: [{', '.join(str(v) for v in value)}]")
+                elif value is not None:
+                    lines.append(f"{key}: {value}")
+            lines.append("---")
+            return '\n'.join(lines) + '\n'
+        
+        yaml_str = yaml.dump(data.to_dict(), default_flow_style=False, sort_keys=False)
+        return f"---\n{yaml_str}---\n"
+    
+    @classmethod
+    def add_or_update_frontmatter(cls, content: str, data: FrontmatterData) -> str:
+        """
+        Add or update frontmatter in content.
+        
+        Args:
+            content: Original file content
+            data: FrontmatterData to add/update
+            
+        Returns:
+            Content with updated frontmatter
+        """
+        _, body = cls.parse(content)
+        frontmatter_str = cls.create_frontmatter(data)
+        return frontmatter_str + body
+
+
+class SchemaValidator:
+    """Validates memory files against schema requirements."""
+    
+    # Required fields per memory type
+    REQUIRED_FIELDS: Dict[MemoryType, List[str]] = {
+        MemoryType.SEMANTIC: ['schema_version', 'last_updated'],
+        MemoryType.EPISODIC: ['schema_version'],
+        MemoryType.PROCEDURAL: ['schema_version', 'last_updated'],
+        MemoryType.CHECKPOINTS: ['schema_version', 'last_updated'],
+        MemoryType.SESSION_SUMMARIES: ['schema_version', 'last_updated'],
+        MemoryType.UNKNOWN: ['schema_version'],
+    }
+    
+    # Recommended fields per memory type (generate warnings if missing)
+    RECOMMENDED_FIELDS: Dict[MemoryType, List[str]] = {
+        MemoryType.SEMANTIC: ['source_agent_id', 'confidence_score', 'tags'],
+        MemoryType.EPISODIC: ['source_agent_id'],
+        MemoryType.PROCEDURAL: ['source_agent_id', 'tags'],
+        MemoryType.CHECKPOINTS: ['source_agent_id'],
+        MemoryType.SESSION_SUMMARIES: ['source_agent_id'],
+        MemoryType.UNKNOWN: [],
+    }
+    
+    @classmethod
+    def detect_memory_type(cls, filepath: str) -> MemoryType:
+        """
+        Detect memory type from file path.
+        
+        Args:
+            filepath: Path to the file
+            
+        Returns:
+            MemoryType enum value
+        """
+        path_lower = filepath.lower()
+        
+        if 'episodic' in path_lower:
+            return MemoryType.EPISODIC
+        elif 'semantic' in path_lower:
+            return MemoryType.SEMANTIC
+        elif 'procedural' in path_lower:
+            return MemoryType.PROCEDURAL
+        elif 'checkpoint' in path_lower:
+            return MemoryType.CHECKPOINTS
+        elif 'session-summar' in path_lower or 'session_summar' in path_lower:
+            return MemoryType.SESSION_SUMMARIES
+        
+        return MemoryType.UNKNOWN
+    
+    @classmethod
+    def validate(cls, content: str, filepath: str, strict: bool = False) -> ValidationResult:
+        """
+        Validate a memory file's frontmatter.
+        
+        Args:
+            content: File content
+            filepath: Path to the file (for type detection)
+            strict: If True, treat warnings as errors
+            
+        Returns:
+            ValidationResult with errors and warnings
+        """
+        result = ValidationResult(valid=True)
+        memory_type = cls.detect_memory_type(filepath)
+        
+        # Parse frontmatter
+        frontmatter, body = FrontmatterParser.parse(content)
+        result.frontmatter = frontmatter
+        
+        # Check for missing frontmatter
+        if frontmatter is None:
+            result.add_error('frontmatter', 'Missing YAML frontmatter block')
+            return result
+        
+        # Check required fields
+        required = cls.REQUIRED_FIELDS.get(memory_type, [])
+        frontmatter_dict = frontmatter.to_dict()
+        
+        for field in required:
+            if field not in frontmatter_dict or frontmatter_dict[field] is None:
+                result.add_error(field, f"Required field '{field}' is missing")
+        
+        # Check recommended fields
+        recommended = cls.RECOMMENDED_FIELDS.get(memory_type, [])
+        for field in recommended:
+            if field not in frontmatter_dict or frontmatter_dict[field] is None:
+                if strict:
+                    result.add_error(field, f"Recommended field '{field}' is missing (strict mode)")
+                else:
+                    result.add_warning(field, f"Recommended field '{field}' is missing")
+        
+        # Validate schema_version format
+        if frontmatter.schema_version:
+            if not re.match(r'^\d+\.\d+$', frontmatter.schema_version):
+                result.add_error('schema_version', 
+                    f"Invalid schema_version format: '{frontmatter.schema_version}' (expected X.Y)")
+        
+        # Validate last_updated format (ISO 8601)
+        if frontmatter.last_updated:
+            try:
+                # Try parsing ISO format
+                if frontmatter.last_updated.endswith('Z'):
+                    datetime.fromisoformat(frontmatter.last_updated.replace('Z', '+00:00'))
+                else:
+                    datetime.fromisoformat(frontmatter.last_updated)
+            except ValueError:
+                result.add_error('last_updated',
+                    f"Invalid last_updated format: '{frontmatter.last_updated}' (expected ISO 8601)")
+        
+        # Validate confidence_score range
+        if frontmatter.confidence_score is not None:
+            if not isinstance(frontmatter.confidence_score, (int, float)):
+                result.add_error('confidence_score',
+                    f"confidence_score must be a number, got: {type(frontmatter.confidence_score).__name__}")
+            elif not (0.0 <= frontmatter.confidence_score <= 1.0):
+                result.add_error('confidence_score',
+                    f"confidence_score must be between 0.0 and 1.0, got: {frontmatter.confidence_score}")
+        
+        # Validate memory_type if specified
+        if frontmatter.memory_type:
+            valid_types = [mt.value for mt in MemoryType if mt != MemoryType.UNKNOWN]
+            if frontmatter.memory_type not in valid_types:
+                result.add_warning('memory_type',
+                    f"Unknown memory_type: '{frontmatter.memory_type}' (expected one of: {valid_types})")
+        
+        # Validate tags is a list
+        if frontmatter.tags and not isinstance(frontmatter.tags, list):
+            result.add_error('tags', f"tags must be a list, got: {type(frontmatter.tags).__name__}")
+        
+        return result
+    
+    @classmethod
+    def validate_batch(cls, files: Dict[str, str], strict: bool = False) -> Dict[str, ValidationResult]:
+        """
+        Validate multiple files.
+        
+        Args:
+            files: Dict mapping filepath to content
+            strict: If True, treat warnings as errors
+            
+        Returns:
+            Dict mapping filepath to ValidationResult
+        """
+        results = {}
+        for filepath, content in files.items():
+            results[filepath] = cls.validate(content, filepath, strict)
+        return results
+
+
+def generate_frontmatter(
+    memory_type: str = "semantic",
+    source_agent_id: Optional[str] = None,
+    confidence_score: Optional[float] = None,
+    tags: Optional[List[str]] = None
+) -> FrontmatterData:
+    """
+    Generate frontmatter data with current timestamp.
+    
+    Args:
+        memory_type: Type of memory (episodic, semantic, procedural, etc.)
+        source_agent_id: ID of the agent creating this memory
+        confidence_score: Confidence score (0.0 to 1.0)
+        tags: List of tags for categorization
+        
+    Returns:
+        FrontmatterData with populated fields
+    """
+    return FrontmatterData(
+        schema_version="1.0",
+        last_updated=datetime.utcnow().isoformat() + 'Z',
+        source_agent_id=source_agent_id,
+        confidence_score=confidence_score,
+        memory_type=memory_type,
+        tags=tags or []
+    )
+
+
+def compare_timestamps(timestamp1: Optional[str], timestamp2: Optional[str]) -> int:
+    """
+    Compare two ISO 8601 timestamps.
+    
+    Args:
+        timestamp1: First timestamp
+        timestamp2: Second timestamp
+        
+    Returns:
+        -1 if timestamp1 < timestamp2
+         0 if timestamp1 == timestamp2
+         1 if timestamp1 > timestamp2
+        
+    If either timestamp is None or invalid, the other is considered newer.
+    """
+    def parse_ts(ts: Optional[str]) -> Optional[datetime]:
+        if not ts:
+            return None
+        try:
+            if ts.endswith('Z'):
+                return datetime.fromisoformat(ts.replace('Z', '+00:00'))
+            return datetime.fromisoformat(ts)
+        except ValueError:
+            return None
+    
+    dt1 = parse_ts(timestamp1)
+    dt2 = parse_ts(timestamp2)
+    
+    if dt1 is None and dt2 is None:
+        return 0
+    if dt1 is None:
+        return -1
+    if dt2 is None:
+        return 1
+    
+    if dt1 < dt2:
+        return -1
+    elif dt1 > dt2:
+        return 1
+    return 0

memvcs/core/staging.py

@@ -0,0 +1,227 @@
+"""
+Staging area for agmem.
+
+Manages the index of files staged for commit.
+"""
+
+import json
+import os
+import shutil
+from pathlib import Path
+from typing import Dict, List, Optional, Set, Tuple
+from dataclasses import dataclass, asdict
+
+
+@dataclass
+class StagedFile:
+    """Represents a file in the staging area."""
+    path: str  # Relative path from current/
+    blob_hash: str
+    mode: int = 0o100644  # Regular file
+
+
+def _path_under_root(relative_path: str, root: Path) -> Optional[Path]:
+    """
+    Resolve relative_path under root and ensure it stays inside root.
+    Returns the resolved Path or None if path escapes root (path traversal).
+    """
+    try:
+        resolved = (root / relative_path).resolve()
+        resolved.relative_to(root.resolve())
+        return resolved
+    except ValueError:
+        return None
+
+
+class StagingArea:
+    """Manages the staging area for memory commits."""
+    
+    def __init__(self, mem_dir: Path):
+        self.mem_dir = Path(mem_dir)
+        self.staging_dir = self.mem_dir / 'staging'
+        self.index_file = self.mem_dir / 'index.json'
+        self._index: Dict[str, StagedFile] = {}
+        self._load_index()
+    
+    def _load_index(self):
+        """Load the staging index from disk."""
+        if self.index_file.exists():
+            try:
+                data = json.loads(self.index_file.read_text())
+                for path, info in data.items():
+                    if _path_under_root(path, self.staging_dir) is None:
+                        continue
+                    self._index[path] = StagedFile(
+                        path=path,
+                        blob_hash=info['blob_hash'],
+                        mode=info.get('mode', 0o100644)
+                    )
+            except (json.JSONDecodeError, KeyError):
+                self._index = {}
+    
+    def _save_index(self):
+        """Save the staging index to disk."""
+        data = {
+            path: {
+                'blob_hash': sf.blob_hash,
+                'mode': sf.mode
+            }
+            for path, sf in self._index.items()
+        }
+        self.index_file.write_text(json.dumps(data, indent=2))
+    
+    def add(self, filepath: str, blob_hash: str, content: bytes, mode: int = 0o100644):
+        """
+        Add a file to the staging area.
+        
+        Args:
+            filepath: Relative path from current/
+            blob_hash: Hash of the blob object
+            content: File content bytes
+            mode: File mode (default 0o100644 for regular file)
+            
+        Raises:
+            ValueError: If filepath escapes staging directory (path traversal)
+        """
+        staging_path = _path_under_root(filepath, self.staging_dir)
+        if staging_path is None:
+            raise ValueError(f"Path escapes staging area: {filepath}")
+        
+        self._index[filepath] = StagedFile(
+            path=filepath,
+            blob_hash=blob_hash,
+            mode=mode
+        )
+        
+        staging_path.parent.mkdir(parents=True, exist_ok=True)
+        staging_path.write_bytes(content)
+        
+        self._save_index()
+    
+    def remove(self, filepath: str) -> bool:
+        """
+        Remove a file from the staging area.
+        
+        Returns:
+            True if file was in staging, False otherwise
+        """
+        if filepath in self._index:
+            del self._index[filepath]
+            
+            staging_path = _path_under_root(filepath, self.staging_dir)
+            if staging_path is not None and staging_path.exists():
+                staging_path.unlink()
+                # Clean up empty directories
+                self._cleanup_empty_dirs(staging_path.parent)
+            
+            self._save_index()
+            return True
+        return False
+    
+    def _cleanup_empty_dirs(self, dir_path: Path):
+        """Remove empty directories up to staging root."""
+        try:
+            while dir_path != self.staging_dir:
+                if dir_path.exists() and not any(dir_path.iterdir()):
+                    dir_path.rmdir()
+                    dir_path = dir_path.parent
+                else:
+                    break
+        except OSError:
+            pass
+    
+    def get_staged_files(self) -> Dict[str, StagedFile]:
+        """Get all staged files."""
+        return dict(self._index)
+    
+    def is_staged(self, filepath: str) -> bool:
+        """Check if a file is staged."""
+        return filepath in self._index
+    
+    def get_blob_hash(self, filepath: str) -> Optional[str]:
+        """Get the blob hash for a staged file."""
+        if filepath in self._index:
+            return self._index[filepath].blob_hash
+        return None
+    
+    def clear(self):
+        """Clear the entire staging area."""
+        self._index = {}
+        
+        # Remove staging directory contents
+        if self.staging_dir.exists():
+            shutil.rmtree(self.staging_dir)
+            self.staging_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Remove index file
+        if self.index_file.exists():
+            self.index_file.unlink()
+    
+    def get_status(self) -> Dict[str, List[str]]:
+        """
+        Get staging status.
+        
+        Returns:
+            Dict with 'staged', 'modified', 'deleted', 'untracked' lists
+        """
+        staged = list(self._index.keys())
+        
+        return {
+            'staged': staged,
+            'modified': [],  # TODO: Compare with working directory
+            'deleted': [],   # TODO: Check if files were deleted
+            'untracked': []  # TODO: Find untracked files
+        }
+    
+    def get_tree_entries(self) -> List[Dict]:
+        """
+        Get tree entries for creating a tree object.
+        
+        Returns:
+            List of entry dictionaries for Tree creation
+        """
+        entries = []
+        for path, sf in self._index.items():
+            entries.append({
+                'mode': oct(sf.mode)[2:],  # Convert to string like '100644'
+                'type': 'blob',
+                'hash': sf.blob_hash,
+                'name': Path(path).name,
+                'path': str(Path(path).parent) if str(Path(path).parent) != '.' else ''
+            })
+        return entries
+    
+    def diff_with_head(self, repo) -> Dict[str, Dict]:
+        """
+        Compare staging area with HEAD commit.
+        
+        Returns:
+            Dict mapping file paths to change info
+        """
+        changes = {}
+        
+        # Get HEAD tree
+        head_commit = repo.get_head_commit()
+        if head_commit:
+            head_tree_bytes = repo.object_store.retrieve(head_commit.tree, 'tree')
+            if head_tree_bytes:
+                head_data = json.loads(head_tree_bytes.decode('utf-8'))
+                head_entries = {e['path'] + '/' + e['name'] if e['path'] else e['name']: e 
+                               for e in head_data.get('entries', [])}
+        else:
+            head_entries = {}
+        
+        # Compare with staging
+        for path, sf in self._index.items():
+            if path in head_entries:
+                if head_entries[path]['hash'] != sf.blob_hash:
+                    changes[path] = {'status': 'modified', 'blob_hash': sf.blob_hash}
+            else:
+                changes[path] = {'status': 'added', 'blob_hash': sf.blob_hash}
+        
+        # Check for deleted files
+        for path in head_entries:
+            if path not in self._index:
+                changes[path] = {'status': 'deleted'}
+        
+        return changes