agmem 0.1.1__py3-none-any.whl

memvcs/core/objects.py

@@ -0,0 +1,323 @@
+"""
+Object storage system for agmem.
+
+Implements Git-style content-addressable storage with blob, tree, and commit objects.
+"""
+
+import hashlib
+import json
+import os
+import zlib
+from pathlib import Path
+from typing import Optional, Dict, List, Any, Union
+from dataclasses import dataclass, asdict
+from datetime import datetime
+
+
+def _valid_object_hash(hash_id: str) -> bool:
+    """Return True if hash_id is safe for object paths (hex, 4-64 chars)."""
+    if not hash_id or len(hash_id) < 4 or len(hash_id) > 64:
+        return False
+    return all(c in '0123456789abcdef' for c in hash_id.lower())
+
+
+class ObjectStore:
+    """Content-addressable object storage system."""
+    
+    def __init__(self, objects_dir: Path):
+        self.objects_dir = Path(objects_dir)
+        self._ensure_directories()
+    
+    def _ensure_directories(self):
+        """Create object storage directories."""
+        for obj_type in ['blob', 'tree', 'commit', 'tag']:
+            (self.objects_dir / obj_type).mkdir(parents=True, exist_ok=True)
+    
+    def _get_object_path(self, hash_id: str, obj_type: str) -> Path:
+        """Get storage path for an object. Validates hash_id to prevent path traversal."""
+        if not _valid_object_hash(hash_id):
+            raise ValueError(f"Invalid object hash: {hash_id!r}")
+        prefix = hash_id[:2]
+        suffix = hash_id[2:]
+        return self.objects_dir / obj_type / prefix / suffix
+    
+    def _compute_hash(self, content: bytes, obj_type: str) -> str:
+        """Compute SHA-256 hash of content with type header."""
+        header = f"{obj_type} {len(content)}\0".encode()
+        full_content = header + content
+        return hashlib.sha256(full_content).hexdigest()
+    
+    def store(self, content: bytes, obj_type: str) -> str:
+        """
+        Store content and return its hash ID.
+        
+        Args:
+            content: Raw bytes to store
+            obj_type: Type of object ('blob', 'tree', 'commit', 'tag')
+            
+        Returns:
+            SHA-256 hash ID of stored object
+        """
+        hash_id = self._compute_hash(content, obj_type)
+        obj_path = self._get_object_path(hash_id, obj_type)
+        
+        # Don't store if already exists (deduplication)
+        if obj_path.exists():
+            return hash_id
+        
+        # Create directory if needed
+        obj_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        # Compress and store
+        header = f"{obj_type} {len(content)}\0".encode()
+        full_content = header + content
+        compressed = zlib.compress(full_content)
+        
+        obj_path.write_bytes(compressed)
+        return hash_id
+    
+    def retrieve(self, hash_id: str, obj_type: str) -> Optional[bytes]:
+        """
+        Retrieve content by hash ID.
+        
+        Args:
+            hash_id: SHA-256 hash of the object
+            obj_type: Type of object
+            
+        Returns:
+            Raw bytes content or None if not found
+        """
+        obj_path = self._get_object_path(hash_id, obj_type)
+        
+        if not obj_path.exists():
+            return None
+        
+        # Decompress and extract content
+        compressed = obj_path.read_bytes()
+        full_content = zlib.decompress(compressed)
+        
+        # Parse header
+        null_idx = full_content.index(b'\0')
+        header = full_content[:null_idx].decode()
+        content = full_content[null_idx + 1:]
+        
+        return content
+    
+    def exists(self, hash_id: str, obj_type: str) -> bool:
+        """Check if an object exists. Returns False for invalid hash (no raise)."""
+        if not _valid_object_hash(hash_id):
+            return False
+        obj_path = self._get_object_path(hash_id, obj_type)
+        return obj_path.exists()
+    
+    def delete(self, hash_id: str, obj_type: str) -> bool:
+        """Delete an object. Returns True if deleted, False if not found."""
+        obj_path = self._get_object_path(hash_id, obj_type)
+        if obj_path.exists():
+            obj_path.unlink()
+            # Clean up empty parent directories
+            if not any(obj_path.parent.iterdir()):
+                obj_path.parent.rmdir()
+            return True
+        return False
+    
+    def list_objects(self, obj_type: str) -> List[str]:
+        """List all objects of a given type."""
+        obj_dir = self.objects_dir / obj_type
+        if not obj_dir.exists():
+            return []
+        
+        hashes = []
+        for prefix_dir in obj_dir.iterdir():
+            if prefix_dir.is_dir():
+                for suffix_file in prefix_dir.iterdir():
+                    hash_id = prefix_dir.name + suffix_file.name
+                    hashes.append(hash_id)
+        return hashes
+    
+    def get_size(self, hash_id: str, obj_type: str) -> int:
+        """Get the compressed size of an object."""
+        obj_path = self._get_object_path(hash_id, obj_type)
+        if obj_path.exists():
+            return obj_path.stat().st_size
+        return 0
+
+
+@dataclass
+class Blob:
+    """Blob object for storing raw memory content."""
+    content: bytes
+    
+    def store(self, store: ObjectStore) -> str:
+        """Store this blob and return its hash."""
+        return store.store(self.content, 'blob')
+    
+    @staticmethod
+    def load(store: ObjectStore, hash_id: str) -> Optional['Blob']:
+        """Load a blob from storage."""
+        content = store.retrieve(hash_id, 'blob')
+        if content is not None:
+            return Blob(content=content)
+        return None
+
+
+@dataclass
+class TreeEntry:
+    """Entry in a tree object."""
+    mode: str  # '100644' for file, '040000' for directory
+    obj_type: str  # 'blob' or 'tree'
+    hash: str
+    name: str
+    path: str = ""  # Relative path within tree
+
+
+@dataclass
+class Tree:
+    """Tree object for storing directory structure."""
+    entries: List[TreeEntry]
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            'type': 'tree',
+            'entries': [
+                {
+                    'mode': e.mode,
+                    'type': e.obj_type,
+                    'hash': e.hash,
+                    'name': e.name,
+                    'path': e.path
+                }
+                for e in self.entries
+            ]
+        }
+    
+    def to_bytes(self) -> bytes:
+        """Serialize to bytes."""
+        return json.dumps(self.to_dict(), sort_keys=True).encode()
+    
+    def store(self, store: ObjectStore) -> str:
+        """Store this tree and return its hash."""
+        return store.store(self.to_bytes(), 'tree')
+    
+    @staticmethod
+    def load(store: ObjectStore, hash_id: str) -> Optional['Tree']:
+        """Load a tree from storage."""
+        content = store.retrieve(hash_id, 'tree')
+        if content is None:
+            return None
+        
+        data = json.loads(content)
+        entries = [
+            TreeEntry(
+                mode=e['mode'],
+                obj_type=e['type'],
+                hash=e['hash'],
+                name=e['name'],
+                path=e.get('path', '')
+            )
+            for e in data.get('entries', [])
+        ]
+        return Tree(entries=entries)
+    
+    def get_entry(self, name: str) -> Optional[TreeEntry]:
+        """Get an entry by name."""
+        for entry in self.entries:
+            if entry.name == name:
+                return entry
+        return None
+
+
+@dataclass
+class Commit:
+    """Commit object for storing memory snapshots."""
+    tree: str  # Hash of tree object
+    parents: List[str]  # Hashes of parent commits
+    author: str
+    timestamp: str
+    message: str
+    metadata: Dict[str, Any]  # Additional metadata
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            'type': 'commit',
+            'tree': self.tree,
+            'parents': self.parents,
+            'author': self.author,
+            'timestamp': self.timestamp,
+            'message': self.message,
+            'metadata': self.metadata
+        }
+    
+    def to_bytes(self) -> bytes:
+        """Serialize to bytes."""
+        return json.dumps(self.to_dict(), sort_keys=True).encode()
+    
+    def store(self, store: ObjectStore) -> str:
+        """Store this commit and return its hash."""
+        return store.store(self.to_bytes(), 'commit')
+    
+    @staticmethod
+    def load(store: ObjectStore, hash_id: str) -> Optional['Commit']:
+        """Load a commit from storage."""
+        content = store.retrieve(hash_id, 'commit')
+        if content is None:
+            return None
+        
+        data = json.loads(content)
+        return Commit(
+            tree=data['tree'],
+            parents=data.get('parents', []),
+            author=data['author'],
+            timestamp=data['timestamp'],
+            message=data['message'],
+            metadata=data.get('metadata', {})
+        )
+    
+    def short_hash(self, store: ObjectStore) -> str:
+        """Get short hash for display."""
+        full_hash = self.store(store)
+        return full_hash[:8]
+
+
+@dataclass
+class Tag:
+    """Tag object for marking specific commits."""
+    name: str
+    commit_hash: str
+    message: str
+    timestamp: str
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            'type': 'tag',
+            'name': self.name,
+            'commit_hash': self.commit_hash,
+            'message': self.message,
+            'timestamp': self.timestamp
+        }
+    
+    def to_bytes(self) -> bytes:
+        """Serialize to bytes."""
+        return json.dumps(self.to_dict(), sort_keys=True).encode()
+    
+    def store(self, store: ObjectStore) -> str:
+        """Store this tag and return its hash."""
+        return store.store(self.to_bytes(), 'tag')
+    
+    @staticmethod
+    def load(store: ObjectStore, hash_id: str) -> Optional['Tag']:
+        """Load a tag from storage."""
+        content = store.retrieve(hash_id, 'tag')
+        if content is None:
+            return None
+        
+        data = json.loads(content)
+        return Tag(
+            name=data['name'],
+            commit_hash=data['commit_hash'],
+            message=data['message'],
+            timestamp=data['timestamp']
+        )

memvcs/core/pii_scanner.py

@@ -0,0 +1,343 @@
+"""
+PII (Personally Identifiable Information) scanner for agmem.
+
+Scans staged files for sensitive information before commit.
+"""
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+# IPs to ignore (localhost / internal); not reported as PII
+IP_FALSE_POSITIVES = frozenset(['127.0.0.1', '0.0.0.0', '192.168.0.1', '10.0.0.1'])
+
+
+@dataclass
+class PIIIssue:
+    """A detected PII issue."""
+    filepath: str
+    line_number: int
+    issue_type: str
+    description: str
+    matched_text: str  # Partially redacted
+    severity: str = "high"  # "high", "medium", "low"
+
+
+@dataclass
+class PIIScanResult:
+    """Result of scanning for PII."""
+    has_issues: bool
+    issues: List[PIIIssue] = field(default_factory=list)
+    scanned_files: int = 0
+    
+    def add_issue(self, issue: PIIIssue):
+        self.issues.append(issue)
+        self.has_issues = True
+
+
+class PIIScanner:
+    """
+    Scanner for detecting PII and secrets in memory files.
+    
+    Detects:
+    - API keys and tokens
+    - Credit card numbers
+    - Email addresses
+    - Social Security Numbers
+    - Phone numbers
+    - IP addresses
+    - Private keys
+    - Database connection strings
+    """
+    
+    # Patterns for detecting various types of PII and secrets
+    PATTERNS = {
+        'api_key': {
+            'pattern': re.compile(
+                r'(?i)'
+                r'(?:api[_-]?key|apikey|api[_-]?secret|api[_-]?token|'
+                r'auth[_-]?token|access[_-]?token|bearer[_-]?token|'
+                r'secret[_-]?key|private[_-]?key|password|passwd|pwd)'
+                r'\s*[:=]\s*["\']?([a-zA-Z0-9_\-]{16,})["\']?',
+                re.MULTILINE
+            ),
+            'description': 'API key or secret token detected',
+            'severity': 'high'
+        },
+        'aws_key': {
+            'pattern': re.compile(r'(?:AKIA|ABIA|ACCA|ASIA)[A-Z0-9]{16}'),
+            'description': 'AWS access key detected',
+            'severity': 'high'
+        },
+        'aws_secret': {
+            'pattern': re.compile(
+                r'(?i)aws[_-]?secret[_-]?(?:access[_-]?)?key\s*[:=]\s*["\']?([a-zA-Z0-9+/]{40})["\']?'
+            ),
+            'description': 'AWS secret access key detected',
+            'severity': 'high'
+        },
+        'private_key': {
+            'pattern': re.compile(
+                r'-----BEGIN (?:RSA |DSA |EC |OPENSSH )?PRIVATE KEY-----'
+            ),
+            'description': 'Private key detected',
+            'severity': 'high'
+        },
+        'credit_card': {
+            'pattern': re.compile(
+                r'\b(?:4[0-9]{12}(?:[0-9]{3})?|'  # Visa
+                r'5[1-5][0-9]{14}|'               # Mastercard
+                r'3[47][0-9]{13}|'                # Amex
+                r'6(?:011|5[0-9]{2})[0-9]{12})\b' # Discover
+            ),
+            'description': 'Credit card number detected',
+            'severity': 'high'
+        },
+        'ssn': {
+            'pattern': re.compile(r'\b[0-9]{3}-[0-9]{2}-[0-9]{4}\b'),
+            'description': 'Social Security Number detected',
+            'severity': 'high'
+        },
+        'email': {
+            'pattern': re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'),
+            'description': 'Email address detected',
+            'severity': 'medium'
+        },
+        'phone': {
+            'pattern': re.compile(
+                r'\b(?:\+?1[-.\s]?)?\(?[2-9][0-9]{2}\)?[-.\s]?[2-9][0-9]{2}[-.\s]?[0-9]{4}\b'
+            ),
+            'description': 'Phone number detected',
+            'severity': 'medium'
+        },
+        'ip_address': {
+            'pattern': re.compile(
+                r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}'
+                r'(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b'
+            ),
+            'description': 'IP address detected',
+            'severity': 'low'
+        },
+        'database_url': {
+            'pattern': re.compile(
+                r'(?i)(?:postgres|mysql|mongodb|redis)://[^\s"\'"]+',
+                re.MULTILINE
+            ),
+            'description': 'Database connection string detected',
+            'severity': 'high'
+        },
+        'jwt': {
+            'pattern': re.compile(r'eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+'),
+            'description': 'JWT token detected',
+            'severity': 'high'
+        },
+        'github_token': {
+            'pattern': re.compile(r'(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36,}'),
+            'description': 'GitHub token detected',
+            'severity': 'high'
+        },
+        'slack_token': {
+            'pattern': re.compile(r'xox[baprs]-[0-9]+-[0-9]+-[a-zA-Z0-9]+'),
+            'description': 'Slack token detected',
+            'severity': 'high'
+        },
+        'stripe_key': {
+            'pattern': re.compile(r'(?:sk|pk)_(?:test|live)_[a-zA-Z0-9]{24,}'),
+            'description': 'Stripe API key detected',
+            'severity': 'high'
+        }
+    }
+    
+    # Files/patterns to skip
+    SKIP_PATTERNS = [
+        r'\.git/',
+        r'\.mem/',
+        r'node_modules/',
+        r'__pycache__/',
+        r'\.pyc$',
+        r'\.pyo$',
+    ]
+    
+    @classmethod
+    def _redact(cls, text: str, keep: int = 4) -> str:
+        """Partially redact sensitive text for display."""
+        if len(text) <= keep * 2:
+            return '*' * len(text)
+        return text[:keep] + '*' * (len(text) - keep * 2) + text[-keep:]
+    
+    @classmethod
+    def _should_skip(cls, filepath: str) -> bool:
+        """Check if file should be skipped."""
+        for pattern in cls.SKIP_PATTERNS:
+            if re.search(pattern, filepath):
+                return True
+        return False
+    
+    @classmethod
+    def scan_content(cls, content: str, filepath: str) -> List[PIIIssue]:
+        """
+        Scan content for PII.
+        
+        Args:
+            content: File content to scan
+            filepath: Path to the file (for reporting)
+            
+        Returns:
+            List of PIIIssue objects
+        """
+        issues = []
+        lines = content.split('\n')
+        
+        for line_num, line in enumerate(lines, 1):
+            for pii_type, config in cls.PATTERNS.items():
+                matches = config['pattern'].finditer(line)
+                for match in matches:
+                    matched_text = match.group(0)
+                    
+                    # Skip common false positives
+                    if cls._is_false_positive(pii_type, matched_text, line):
+                        continue
+                    
+                    issues.append(PIIIssue(
+                        filepath=filepath,
+                        line_number=line_num,
+                        issue_type=pii_type,
+                        description=config['description'],
+                        matched_text=cls._redact(matched_text),
+                        severity=config['severity']
+                    ))
+        
+        return issues
+    
+    @classmethod
+    def _is_false_positive(cls, pii_type: str, matched_text: str, line: str) -> bool:
+        """Check for common false positives."""
+        lower_line = line.lower()
+        
+        # Skip example/placeholder values
+        if any(x in lower_line for x in ['example', 'placeholder', 'your_', 'xxx', 'sample']):
+            return True
+        
+        # Skip comments that are likely documentation
+        if line.strip().startswith('#') and 'example' in lower_line:
+            return True
+        
+        if pii_type == 'ip_address':
+            if matched_text in IP_FALSE_POSITIVES:
+                return True
+            # Skip version numbers that look like IPs
+            if 'version' in lower_line or 'v.' in lower_line:
+                return True
+        
+        # Email false positives
+        if pii_type == 'email':
+            # Skip example domains
+            if any(x in matched_text for x in ['example.com', 'test.com', 'localhost']):
+                return True
+        
+        return False
+    
+    @classmethod
+    def scan_file(cls, filepath: Path) -> List[PIIIssue]:
+        """
+        Scan a file for PII.
+        
+        Args:
+            filepath: Path to the file
+            
+        Returns:
+            List of PIIIssue objects
+        """
+        if cls._should_skip(str(filepath)):
+            return []
+        
+        try:
+            content = filepath.read_text(encoding='utf-8', errors='ignore')
+            return cls.scan_content(content, str(filepath))
+        except Exception:
+            return []
+    
+    @classmethod
+    def _get_blob_hash_from_staged(cls, file_info: Any) -> Optional[str]:
+        """Get blob hash from StagedFile or dict (staging returns Dict[str, StagedFile])."""
+        if hasattr(file_info, 'blob_hash'):
+            return file_info.blob_hash
+        if isinstance(file_info, dict):
+            return file_info.get('blob_hash') or file_info.get('hash')
+        return None
+
+    @classmethod
+    def scan_staged_files(cls, repo, staged_files: Dict[str, Any]) -> PIIScanResult:
+        """
+        Scan staged files for PII.
+        
+        Args:
+            repo: Repository instance
+            staged_files: Dict of staged files with their info
+            
+        Returns:
+            PIIScanResult with any issues found
+        """
+        from .objects import Blob
+        
+        result = PIIScanResult(has_issues=False)
+        
+        for filepath, file_info in staged_files.items():
+            if cls._should_skip(filepath):
+                continue
+            
+            result.scanned_files += 1
+            
+            blob_hash = PIIScanner._get_blob_hash_from_staged(file_info)
+            if not blob_hash:
+                continue
+            
+            blob = Blob.load(repo.object_store, blob_hash)
+            if not blob:
+                continue
+            
+            try:
+                content = blob.content.decode('utf-8', errors='ignore')
+            except Exception:
+                continue
+            
+            # Scan content
+            issues = cls.scan_content(content, filepath)
+            for issue in issues:
+                result.add_issue(issue)
+        
+        return result
+    
+    @classmethod
+    def scan_directory(cls, directory: Path, recursive: bool = True) -> PIIScanResult:
+        """
+        Scan a directory for PII.
+        
+        Args:
+            directory: Directory to scan
+            recursive: Whether to scan recursively
+            
+        Returns:
+            PIIScanResult with any issues found
+        """
+        result = PIIScanResult(has_issues=False)
+        
+        if recursive:
+            files = directory.rglob('*')
+        else:
+            files = directory.glob('*')
+        
+        for filepath in files:
+            if not filepath.is_file():
+                continue
+            
+            if cls._should_skip(str(filepath)):
+                continue
+            
+            result.scanned_files += 1
+            issues = cls.scan_file(filepath)
+            for issue in issues:
+                result.add_issue(issue)
+        
+        return result