foodforthought-cli 0.2.7__py3-none-any.whl → 0.3.0__py3-none-any.whl

ate/memory/context.py

@@ -0,0 +1,269 @@
+"""Context management for git-like memory operations.
+
+This module provides the ContextManager that tracks the active memory and train 
+of thought, similar to how git tracks the current repository and branch.
+"""
+
+import json
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Dict, Any
+
+from .store import MemoryStore
+
+
+@dataclass
+class MemoryContext:
+    """Tracks active memory and train of thought."""
+    active_memory: str
+    active_train: str
+    path: str
+
+
+@dataclass
+class MemoryMetadata:
+    """Metadata for a memory."""
+    name: str
+    visibility: str
+    trains: List[str]
+    default_train: str
+    description: str = ""
+    created_at: Optional[str] = None
+    remote: Optional[str] = None
+
+
+class ContextManager:
+    """Manages the active memory context (~/.ate/context.json)."""
+
+    CONFIG_DIR = os.path.expanduser("~/.ate")
+    CONTEXT_FILE = os.path.expanduser("~/.ate/context.json")
+    MEMORIES_DIR = os.path.expanduser("~/.ate/memories")
+
+    @classmethod
+    def get_context(cls) -> MemoryContext:
+        """Get current context. Auto-initializes on first use."""
+        if os.path.exists(cls.CONTEXT_FILE):
+            try:
+                with open(cls.CONTEXT_FILE, 'r') as f:
+                    data = json.load(f)
+                
+                # Provide defaults for missing fields
+                active_memory = data.get("active_memory", "default")
+                active_train = data.get("active_train", "main")
+                path = data.get("path")
+                
+                if not path:
+                    path = cls._train_to_path(active_memory, active_train)
+                
+                return MemoryContext(
+                    active_memory=active_memory,
+                    active_train=active_train,
+                    path=path
+                )
+            except (json.JSONDecodeError, KeyError):
+                pass  # Fall through to auto-init
+        
+        # Auto-initialize on first use
+        return cls._auto_initialize()
+
+    @classmethod
+    def set_context(cls, memory: str, train: str) -> MemoryContext:
+        """Set active context."""
+        # Validate memory name
+        if not cls._is_valid_memory_name(memory):
+            raise ValueError(f"Invalid memory name '{memory}'. Use lowercase alphanumeric and hyphens only.")
+        
+        # Ensure directories exist
+        os.makedirs(cls.CONFIG_DIR, exist_ok=True)
+        os.makedirs(cls.MEMORIES_DIR, exist_ok=True)
+        memory_dir = os.path.join(cls.MEMORIES_DIR, memory)
+        os.makedirs(memory_dir, exist_ok=True)
+        
+        # Create path
+        path = cls._train_to_path(memory, train)
+        
+        # Create .mv2 file if it doesn't exist (auto-create new trains)
+        if not os.path.exists(path):
+            try:
+                store = MemoryStore.create(path)
+                store.close()
+            except Exception:
+                pass
+            # Ensure file exists on disk (memvid create may not touch filesystem)
+            if not os.path.exists(path):
+                Path(path).touch()
+            
+            # Update memory.json trains list
+            safe_train = train.replace('/', '-')
+            memory_json = os.path.join(memory_dir, "memory.json")
+            if os.path.exists(memory_json):
+                try:
+                    with open(memory_json, 'r') as f:
+                        data = json.load(f)
+                    if safe_train not in data.get("trains", []):
+                        data.setdefault("trains", []).append(safe_train)
+                        with open(memory_json, 'w') as f:
+                            json.dump(data, f, indent=2)
+                except (json.JSONDecodeError, KeyError):
+                    pass
+        
+        # Create context
+        context = MemoryContext(
+            active_memory=memory,
+            active_train=train,
+            path=path
+        )
+        
+        # Write context file
+        context_data = {
+            "active_memory": memory,
+            "active_train": train,
+            "path": path
+        }
+        
+        with open(cls.CONTEXT_FILE, 'w') as f:
+            json.dump(context_data, f, indent=2)
+        
+        return context
+
+    @classmethod
+    def resolve_path(cls, memory: Optional[str] = None, train: Optional[str] = None) -> str:
+        """Resolve .mv2 path from context or explicit args."""
+        if memory is not None and train is not None:
+            return cls._train_to_path(memory, train)
+        
+        context = cls.get_context()
+        if memory is not None:
+            return cls._train_to_path(memory, context.active_train)
+        if train is not None:
+            return cls._train_to_path(context.active_memory, train)
+        
+        return context.path
+
+    @classmethod
+    def ensure_memory(cls, name: str) -> str:
+        """Create memory dir + default train if doesn't exist."""
+        if not cls._is_valid_memory_name(name):
+            raise ValueError(f"Invalid memory name '{name}'. Use lowercase alphanumeric and hyphens only.")
+        
+        # Create config and memories directories
+        os.makedirs(cls.CONFIG_DIR, exist_ok=True)
+        os.makedirs(cls.MEMORIES_DIR, exist_ok=True)
+        
+        # Create memory directory
+        memory_dir = os.path.join(cls.MEMORIES_DIR, name)
+        os.makedirs(memory_dir, exist_ok=True)
+        
+        # Create default train (main.mv2) if it doesn't exist
+        main_path = cls._train_to_path(name, "main")
+        if not os.path.exists(main_path):
+            try:
+                store = MemoryStore.create(main_path)
+                store.close()
+            except Exception:
+                pass
+            # Ensure file exists on disk
+            if not os.path.exists(main_path):
+                Path(main_path).touch()
+        
+        # Create memory.json if it doesn't exist
+        memory_json_path = os.path.join(memory_dir, "memory.json")
+        if not os.path.exists(memory_json_path):
+            metadata = {
+                "name": name,
+                "visibility": "private",
+                "trains": ["main"],
+                "default_train": "main",
+                "description": ""
+            }
+            with open(memory_json_path, 'w') as f:
+                json.dump(metadata, f, indent=2)
+        
+        return main_path
+
+    @classmethod
+    def list_memories(cls) -> List[MemoryMetadata]:
+        """List all local memories."""
+        memories = []
+        
+        if not os.path.exists(cls.MEMORIES_DIR):
+            return memories
+        
+        for item in os.listdir(cls.MEMORIES_DIR):
+            memory_dir = os.path.join(cls.MEMORIES_DIR, item)
+            if os.path.isdir(memory_dir):
+                memory_json_path = os.path.join(memory_dir, "memory.json")
+                if os.path.exists(memory_json_path):
+                    try:
+                        with open(memory_json_path, 'r') as f:
+                            data = json.load(f)
+                        
+                        metadata = MemoryMetadata(
+                            name=data["name"],
+                            visibility=data.get("visibility", "private"),
+                            trains=data.get("trains", ["main"]),
+                            default_train=data.get("default_train", "main"),
+                            description=data.get("description", ""),
+                            created_at=data.get("created_at"),
+                            remote=data.get("remote")
+                        )
+                        memories.append(metadata)
+                    except (json.JSONDecodeError, KeyError):
+                        continue  # Skip malformed memory.json files
+        
+        return memories
+
+    @classmethod
+    def list_trains(cls, memory: Optional[str] = None) -> List[str]:
+        """List trains of thought in a memory."""
+        if memory is None:
+            context = cls.get_context()
+            memory = context.active_memory
+        
+        memory_dir = os.path.join(cls.MEMORIES_DIR, memory)
+        if not os.path.exists(memory_dir):
+            raise FileNotFoundError(f"Memory '{memory}' does not exist")
+        
+        trains = set()
+        
+        # Source 1: .mv2 files on disk
+        for item in os.listdir(memory_dir):
+            if item.endswith('.mv2'):
+                train_name = item[:-4]  # Remove .mv2 extension
+                trains.add(train_name)
+        
+        # Source 2: memory.json trains list (in case files haven't been created yet)
+        memory_json = os.path.join(memory_dir, "memory.json")
+        if os.path.exists(memory_json):
+            try:
+                with open(memory_json, 'r') as f:
+                    data = json.load(f)
+                for t in data.get("trains", []):
+                    trains.add(t)
+            except (json.JSONDecodeError, KeyError):
+                pass
+        
+        return sorted(trains)
+
+    @classmethod
+    def _auto_initialize(cls) -> MemoryContext:
+        """Auto-initialize default memory on first use."""
+        # Ensure default memory exists
+        cls.ensure_memory("default")
+        
+        # Set context to default/main
+        return cls.set_context("default", "main")
+
+    @classmethod
+    def _train_to_path(cls, memory: str, train: str) -> str:
+        """Convert memory + train to .mv2 file path."""
+        # Convert train name to filename-safe format (slashes to hyphens)
+        safe_train = train.replace('/', '-')
+        return os.path.join(cls.MEMORIES_DIR, memory, f"{safe_train}.mv2")
+
+    @classmethod
+    def _is_valid_memory_name(cls, name: str) -> bool:
+        """Check if memory name is valid (alphanumeric + hyphens only)."""
+        return bool(re.match(r'^[a-z0-9-]+$', name))

ate/memory/embeddings.py

@@ -0,0 +1,184 @@
+"""Embedding configuration and management for ate memory."""
+
+import os
+import requests
+from dataclasses import dataclass
+from typing import Optional, List, Dict, Any
+
+import memvid_sdk
+
+
+@dataclass
+class EmbeddingConfig:
+    """Embedding provider configuration."""
+    provider: str = "none"
+    model: Optional[str] = None
+    api_key: Optional[str] = None
+
+    def __post_init__(self):
+        """Validate provider after initialization."""
+        valid_providers = ["openai", "cohere", "voyage", "ollama", "none"]
+        if self.provider not in valid_providers:
+            raise ValueError(f"Invalid provider: {self.provider}. Must be one of {valid_providers}")
+
+        # Set default models based on provider
+        if self.model is None:
+            if self.provider == "openai":
+                self.model = "text-embedding-3-small"
+            elif self.provider == "cohere":
+                self.model = "embed-english-v3.0"
+            elif self.provider == "voyage":
+                self.model = "voyage-2"
+            elif self.provider == "ollama":
+                self.model = "nomic-embed-text"
+
+
+class EmbeddingManager:
+    """Detects and manages embedding providers for ate memory."""
+
+    @staticmethod
+    def detect() -> EmbeddingConfig:
+        """Auto-detect best available embedding provider from env.
+        
+        Detection order: OpenAI → Cohere → Voyage → Ollama → BM25-only
+        
+        Returns:
+            EmbeddingConfig with detected provider and settings
+        """
+        # Check OpenAI first (highest priority)
+        openai_key = os.environ.get('OPENAI_API_KEY')
+        if openai_key:
+            return EmbeddingConfig(
+                provider="openai",
+                api_key=openai_key
+            )
+
+        # Check Cohere second
+        cohere_key = os.environ.get('COHERE_API_KEY')
+        if cohere_key:
+            return EmbeddingConfig(
+                provider="cohere",
+                api_key=cohere_key
+            )
+
+        # Check Voyage third
+        voyage_key = os.environ.get('VOYAGE_API_KEY')
+        if voyage_key:
+            return EmbeddingConfig(
+                provider="voyage",
+                api_key=voyage_key
+            )
+
+        # Check Ollama fourth (local service)
+        ollama_host = os.environ.get('OLLAMA_HOST', 'http://localhost:11434')
+        if EmbeddingManager._is_ollama_available(ollama_host):
+            return EmbeddingConfig(
+                provider="ollama",
+                model="nomic-embed-text"
+            )
+
+        # No providers available
+        return EmbeddingConfig(provider="none")
+
+    @staticmethod
+    def _is_ollama_available(host: str) -> bool:
+        """Check if Ollama is reachable at the given host."""
+        try:
+            response = requests.get(f'{host}/api/tags', timeout=2)
+            return response.status_code == 200
+        except:
+            return False
+
+    @staticmethod
+    def get_provider(config: EmbeddingConfig):
+        """Get a memvid_sdk EmbeddingProvider from config.
+        
+        Args:
+            config: EmbeddingConfig instance
+            
+        Returns:
+            EmbeddingProvider instance or None if provider is "none"
+        """
+        if config.provider == "none":
+            return None
+
+        return memvid_sdk.embeddings.get_embedder(
+            provider=config.provider,
+            model=config.model,
+            api_key=config.api_key
+        )
+
+    @staticmethod
+    def available_providers() -> List[Dict[str, Any]]:
+        """List all detected providers with status.
+        
+        Returns:
+            List of provider status dictionaries
+        """
+        providers = []
+
+        # Check OpenAI
+        openai_key = os.environ.get('OPENAI_API_KEY')
+        if openai_key:
+            providers.append({
+                "name": "openai",
+                "available": True,
+                "model": "text-embedding-3-small",
+                "source": "OPENAI_API_KEY"
+            })
+        else:
+            providers.append({
+                "name": "openai",
+                "available": False,
+                "reason": "OPENAI_API_KEY not set"
+            })
+
+        # Check Cohere
+        cohere_key = os.environ.get('COHERE_API_KEY')
+        if cohere_key:
+            providers.append({
+                "name": "cohere",
+                "available": True,
+                "model": "embed-english-v3.0",
+                "source": "COHERE_API_KEY"
+            })
+        else:
+            providers.append({
+                "name": "cohere",
+                "available": False,
+                "reason": "COHERE_API_KEY not set"
+            })
+
+        # Check Voyage
+        voyage_key = os.environ.get('VOYAGE_API_KEY')
+        if voyage_key:
+            providers.append({
+                "name": "voyage",
+                "available": True,
+                "model": "voyage-2",
+                "source": "VOYAGE_API_KEY"
+            })
+        else:
+            providers.append({
+                "name": "voyage",
+                "available": False,
+                "reason": "VOYAGE_API_KEY not set"
+            })
+
+        # Check Ollama
+        ollama_host = os.environ.get('OLLAMA_HOST', 'http://localhost:11434')
+        if EmbeddingManager._is_ollama_available(ollama_host):
+            providers.append({
+                "name": "ollama",
+                "available": True,
+                "model": "nomic-embed-text",
+                "source": ollama_host
+            })
+        else:
+            providers.append({
+                "name": "ollama",
+                "available": False,
+                "reason": "OLLAMA_HOST not set, localhost:11434 not reachable"
+            })
+
+        return providers

ate/memory/export.py

@@ -0,0 +1,26 @@
+"""Export operations and info structures."""
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class MemoryInfo:
+    """Information about a memory store.
+    
+    Attributes:
+        path: Path to the .mv2 file
+        frame_count: Number of memory frames stored
+        size_bytes: Total size in bytes
+        has_lex_index: Whether lexical indexing is enabled
+        has_vec_index: Whether vector indexing is enabled  
+        has_time_index: Whether time indexing is enabled
+        created_at: ISO timestamp when created (optional)
+    """
+    path: str
+    frame_count: int
+    size_bytes: int
+    has_lex_index: bool
+    has_vec_index: bool
+    has_time_index: bool
+    created_at: Optional[str] = None

ate/memory/merge.py

@@ -0,0 +1,146 @@
+"""Memory merging operations."""
+
+from typing import List
+import json
+
+from .store import MemoryStore
+from .export import MemoryInfo
+
+
+def merge_memories(source_paths: List[str], output_path: str, dedup: bool = True) -> MemoryInfo:
+    """Merge multiple .mv2 files into a single output file.
+    
+    Args:
+        source_paths: List of paths to source .mv2 files
+        output_path: Path where merged .mv2 file will be created
+        dedup: Whether to deduplicate identical content (default True)
+        
+    Returns:
+        MemoryInfo about the merged output file
+    """
+    # Create the output memory store
+    output_store = MemoryStore.create(output_path)
+    
+    seen_texts = set() if dedup else None
+    all_items = []
+    
+    try:
+        # Process each source file
+        for source_path in source_paths:
+            source_store = MemoryStore.open(source_path)
+            
+            try:
+                # Try timeline-based iteration first (works with real memvid)
+                items_from_source = []
+                try:
+                    timeline = source_store._mem.timeline()
+                    if timeline:
+                        for entry in timeline:
+                            if isinstance(entry, dict):
+                                uri = entry.get('uri', f"mv2://frames/{entry.get('frame_id', 0)}")
+                                frame_data = source_store._mem.frame(uri)
+                                
+                                title = None
+                                tags = []
+                                metadata = {}
+                                
+                                # Get text from timeline preview (labels are just keywords)
+                                text = entry.get('preview', '').split('\ntitle:')[0].split('\ntags:')[0].strip()
+                                
+                                if isinstance(frame_data, dict):
+                                    title = frame_data.get('title')
+                                    tags = frame_data.get('tags', [])
+                                    for key, value in frame_data.get('extra_metadata', {}).items():
+                                        if key == 'extractous_metadata':
+                                            continue
+                                        try:
+                                            if isinstance(value, str) and (value.startswith('{') or value.startswith('"')):
+                                                metadata[key] = json.loads(value)
+                                            else:
+                                                metadata[key] = value
+                                        except json.JSONDecodeError:
+                                            metadata[key] = value
+                                
+                                if text:
+                                    items_from_source.append({
+                                        'text': text,
+                                        'title': title,
+                                        'tags': tags,
+                                        'metadata': metadata
+                                    })
+                except (AttributeError, TypeError):
+                    pass  # Fall through to search-based approach
+                
+                # Fallback: use search (for mocked tests)
+                if not items_from_source:
+                    search_result = source_store.search("*", top_k=10000)
+                    
+                    if hasattr(search_result, 'hits'):
+                        search_results = search_result.hits
+                    elif isinstance(search_result, list):
+                        search_results = search_result
+                    else:
+                        search_results = []
+                    
+                    for search_item in search_results:
+                        text = getattr(search_item, 'snippet', getattr(search_item, 'text', ''))
+                        title = getattr(search_item, 'title', None)
+                        tags = getattr(search_item, 'tags', [])
+                        frame_id = getattr(search_item, 'frame_id', 0)
+                        
+                        metadata = {}
+                        try:
+                            frame_data = source_store._mem.frame(frame_id)
+                            if hasattr(frame_data, 'metadata'):
+                                metadata = frame_data.metadata or {}
+                        except Exception:
+                            try:
+                                if hasattr(search_item, 'metadata'):
+                                    metadata = search_item.metadata or {}
+                            except Exception:
+                                metadata = {}
+                        
+                        items_from_source.append({
+                            'text': text,
+                            'title': title,
+                            'tags': tags,
+                            'metadata': metadata
+                        })
+                
+                for item in items_from_source:
+                    text = item['text']
+                    
+                    if dedup and text in seen_texts:
+                        continue
+                    
+                    if dedup:
+                        seen_texts.add(text)
+                    
+                    all_items.append(item)
+                    
+            finally:
+                source_store.close()
+        
+        # Add all collected items to the output store
+        output_store.add_batch(all_items)
+        
+        # Get info about the merged result
+        info = output_store.info()
+        
+    finally:
+        output_store.close()
+    
+    # Handle mocked info object vs real MemoryInfo
+    if hasattr(info, 'frame_count') and not isinstance(info, MemoryInfo):
+        # It's a mock, create actual MemoryInfo
+        return MemoryInfo(
+            path=info.path,
+            frame_count=info.frame_count,
+            size_bytes=info.size_bytes,
+            has_lex_index=info.has_lex_index,
+            has_vec_index=info.has_vec_index,
+            has_time_index=info.has_time_index,
+            created_at=getattr(info, 'created_at', None)
+        )
+    
+    return info

ate/memory/migrate/__init__.py

@@ -0,0 +1,34 @@
+"""Migration module for vector database migrations."""
+
+from .base import (
+    VectorRecord,
+    MigrationEstimate, 
+    MigrationResult,
+    MigrationCheckpoint,
+    MigrationSource
+)
+from .pipeline import MigrationPipeline
+from .sources import (
+    PineconeMigrationSource,
+    QdrantMigrationSource,
+    WeaviateMigrationSource,
+    ChromaMigrationSource
+)
+
+__all__ = [
+    # Base classes and data structures
+    'VectorRecord',
+    'MigrationEstimate',
+    'MigrationResult', 
+    'MigrationCheckpoint',
+    'MigrationSource',
+    
+    # Pipeline
+    'MigrationPipeline',
+    
+    # Sources
+    'PineconeMigrationSource',
+    'QdrantMigrationSource',
+    'WeaviateMigrationSource', 
+    'ChromaMigrationSource'
+]