AbstractMemory 0.0.1__py3-none-any.whl → 0.2.1__py3-none-any.whl

abstractmemory/embeddings/__init__.py

@@ -0,0 +1,317 @@
+"""
+Real embedding providers for semantic search capabilities.
+
+Provides a unified interface for generating embeddings from real providers:
+AbstractCore EmbeddingManager, OpenAI, Ollama with semantic capabilities.
+
+NO FALLBACKS - only real semantic embedding providers are supported.
+"""
+
+import logging
+import hashlib
+from typing import List, Optional, Any, Union
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingAdapter:
+    """
+    Unified embedding interface for various providers.
+
+    IMPORTANT: The embedding provider/model must remain consistent within a storage space.
+    You can change LLM providers freely, but changing embedding models requires recreating
+    your vector database as different models produce incompatible vector spaces.
+    """
+
+    def __init__(self, provider: Optional[Any] = None):
+        """
+        Initialize embedding adapter.
+
+        Args:
+            provider: Embedding provider (AbstractCore EmbeddingManager, OpenAI client, etc.)
+                     Note: This is for EMBEDDINGS only, not LLM text generation.
+
+        Raises:
+            ValueError: If no embedding provider is available
+        """
+        if provider is None:
+            raise ValueError(
+                "AbstractMemory semantic search requires a dedicated embedding provider. "
+                "This is separate from your LLM provider for text generation. "
+                "Please provide: AbstractCore EmbeddingManager, OpenAI client, Ollama with embeddings, etc."
+            )
+
+        self.provider = provider
+        self.provider_type = self._detect_provider_type()
+        self.embedding_dimension = self._get_embedding_dimension()
+        self.model_info = self._get_model_info()
+
+    def _detect_provider_type(self) -> str:
+        """Detect the type of provider and its embedding capabilities."""
+        # Check for AbstractCore EmbeddingManager (preferred)
+        try:
+            from abstractllm.embeddings import EmbeddingManager
+            if isinstance(self.provider, EmbeddingManager):
+                return "abstractcore_embeddings"
+        except ImportError:
+            pass
+
+        # Check for AbstractCore provider with embedding support (has specific AbstractCore attributes)
+        if hasattr(self.provider, 'generate_embedding') and hasattr(self.provider, 'provider_name'):
+            return "abstractcore"
+
+        # Check for OpenAI client
+        if hasattr(self.provider, 'embeddings'):
+            return "openai"
+
+        # Check for provider name attribute (Ollama, MLX, etc.)
+        if hasattr(self.provider, 'provider_name'):
+            provider_name = getattr(self.provider, 'provider_name', '').lower()
+            if 'ollama' in provider_name:
+                return "ollama"
+            elif 'mlx' in provider_name:
+                return "mlx"
+
+        # Check if provider has generate_embedding method (generic embedding provider)
+        if hasattr(self.provider, 'generate_embedding') and callable(getattr(self.provider, 'generate_embedding')):
+            return "generic_embedding_provider"
+
+        # If we can't identify an embedding provider, raise an error
+        raise ValueError(
+            f"Unable to identify an embedding provider from: {type(self.provider)}. "
+            "Supported providers: AbstractCore EmbeddingManager, OpenAI client, "
+            "Ollama with embeddings, or any object with 'generate_embedding()' method."
+        )
+
+    def _get_embedding_dimension(self) -> int:
+        """Get the embedding dimension based on provider type."""
+        if self.provider_type == "abstractcore_embeddings":
+            # Get dimension from a test embedding
+            try:
+                test_embedding = self.provider.embed("dimension_test")
+                return len(test_embedding)
+            except Exception as e:
+                logger.error(f"Failed to get embedding dimension from AbstractCore: {e}")
+                raise ValueError("Unable to determine embedding dimension from AbstractCore provider")
+        elif self.provider_type == "openai":
+            return 1536  # text-embedding-3-small default
+        elif self.provider_type == "ollama":
+            # Try to get dimension from test embedding
+            try:
+                test_embedding = self._generate_ollama_embedding("dimension_test")
+                return len(test_embedding)
+            except:
+                return 1024  # Common Ollama embedding dimension
+        elif self.provider_type == "generic_embedding_provider":
+            # For any provider with generate_embedding method
+            try:
+                test_embedding = self.provider.generate_embedding("dimension_test")
+                return len(test_embedding)
+            except Exception as e:
+                logger.error(f"Failed to determine embedding dimension from generic provider: {e}")
+                raise ValueError(f"Unable to determine embedding dimension: {e}")
+        else:
+            # For any other provider, attempt to generate a test embedding
+            try:
+                test_embedding = self.generate_embedding("dimension_test")
+                return len(test_embedding)
+            except Exception as e:
+                logger.error(f"Failed to determine embedding dimension: {e}")
+                raise ValueError(f"Unable to determine embedding dimension for provider type: {self.provider_type}")
+
+    def _get_model_info(self) -> dict:
+        """Get detailed information about the embedding model for consistency tracking."""
+        info = {
+            "provider_type": self.provider_type,
+            "dimension": self.embedding_dimension,
+            "created_at": datetime.now().isoformat()
+        }
+
+        if self.provider_type == "abstractcore_embeddings":
+            # Try to get model name from AbstractCore - only store serializable strings
+            try:
+                if hasattr(self.provider, 'model'):
+                    model_attr = getattr(self.provider, 'model')
+                    if isinstance(model_attr, str):
+                        info["model_name"] = model_attr
+                    else:
+                        # Get string representation of the model
+                        info["model_name"] = str(type(model_attr).__name__)
+                if hasattr(self.provider, 'backend'):
+                    info["backend"] = str(self.provider.backend)
+            except Exception as e:
+                logger.debug(f"Could not extract model info: {e}")
+        elif self.provider_type == "openai":
+            info["model_name"] = "text-embedding-3-small"  # Default assumption
+
+        return info
+
+    def generate_embedding(self, text: str) -> List[float]:
+        """
+        Generate embedding for the given text using the configured provider.
+
+        Args:
+            text: Input text to embed
+
+        Returns:
+            List[float]: Embedding vector
+
+        Raises:
+            EmbeddingError: If embedding generation fails
+        """
+        try:
+            if self.provider_type == "abstractcore_embeddings":
+                return self._generate_abstractcore_embeddings(text)
+            elif self.provider_type == "abstractcore":
+                return self._generate_abstractcore_embedding(text)
+            elif self.provider_type == "openai":
+                return self._generate_openai_embedding(text)
+            elif self.provider_type == "ollama":
+                return self._generate_ollama_embedding(text)
+            elif self.provider_type == "mlx":
+                return self._generate_mlx_embedding(text)
+            elif self.provider_type == "generic_embedding_provider":
+                return self.provider.generate_embedding(text)
+            else:
+                raise EmbeddingError(f"Unknown provider type: {self.provider_type}")
+
+        except Exception as e:
+            logger.error(f"Embedding generation failed with {self.provider_type}: {e}")
+            raise EmbeddingError(f"Failed to generate embedding: {e}") from e
+
+    def _generate_abstractcore_embeddings(self, text: str) -> List[float]:
+        """Generate embedding using AbstractCore EmbeddingManager."""
+        return self.provider.embed(text)
+
+    def _generate_abstractcore_embedding(self, text: str) -> List[float]:
+        """Generate embedding using AbstractCore provider."""
+        return self.provider.generate_embedding(text)
+
+    def _generate_openai_embedding(self, text: str) -> List[float]:
+        """Generate embedding using OpenAI client."""
+        response = self.provider.embeddings.create(
+            model="text-embedding-3-small",
+            input=text
+        )
+        return response.data[0].embedding
+
+    def _generate_ollama_embedding(self, text: str) -> List[float]:
+        """Generate embedding using Ollama provider."""
+        try:
+            import requests
+            response = requests.post(
+                "http://localhost:11434/api/embeddings",
+                json={
+                    "model": "nomic-embed-text",
+                    "prompt": text
+                },
+                timeout=30
+            )
+            if response.status_code == 200:
+                embedding = response.json()["embedding"]
+                if embedding and isinstance(embedding, list):
+                    return embedding
+            raise EmbeddingError(f"Ollama API returned status {response.status_code}")
+        except ImportError:
+            raise EmbeddingError("requests library not available for Ollama embedding API")
+        except Exception as e:
+            raise EmbeddingError(f"Ollama embedding generation failed: {e}") from e
+
+    def _generate_mlx_embedding(self, text: str) -> List[float]:
+        """Generate embedding using MLX provider."""
+        # MLX provider should implement actual MLX embedding model
+        raise EmbeddingError(
+            "MLX embedding implementation not yet available. "
+            "Please use AbstractCore EmbeddingManager or another provider."
+        )
+
+    def is_real_embedding(self) -> bool:
+        """Check if this adapter provides real semantic embeddings."""
+        return self.provider_type in ["abstractcore_embeddings", "abstractcore", "openai", "ollama", "generic_embedding_provider"]
+
+    def get_embedding_info(self) -> dict:
+        """Get comprehensive information about the embedding provider for consistency tracking."""
+        info = self.model_info.copy()
+        info.update({
+            "is_real_embedding": self.is_real_embedding(),
+            "provider_available": self.provider is not None
+        })
+        return info
+
+    def check_consistency_with(self, other_model_info: dict) -> bool:
+        """
+        Check if this adapter is consistent with previously stored model info.
+
+        Args:
+            other_model_info: Previously stored model information
+
+        Returns:
+            bool: True if models are compatible for semantic search
+        """
+        current_info = self.get_embedding_info()
+
+        # Check critical compatibility factors
+        if current_info.get("provider_type") != other_model_info.get("provider_type"):
+            logger.warning(f"Provider type mismatch: {current_info.get('provider_type')} vs {other_model_info.get('provider_type')}")
+            return False
+
+        if current_info.get("dimension") != other_model_info.get("dimension"):
+            logger.warning(f"Dimension mismatch: {current_info.get('dimension')} vs {other_model_info.get('dimension')}")
+            return False
+
+        if current_info.get("model_name") != other_model_info.get("model_name"):
+            logger.warning(f"Model name mismatch: {current_info.get('model_name')} vs {other_model_info.get('model_name')}")
+            return False
+
+        return True
+
+    def warn_about_consistency(self, stored_model_info: dict) -> None:
+        """
+        Issue warnings about embedding model consistency issues.
+
+        Args:
+            stored_model_info: Information about previously stored embeddings
+        """
+        if not self.check_consistency_with(stored_model_info):
+            current_info = self.get_embedding_info()
+            logger.warning(
+                "\n" + "="*80 + "\n"
+                "🚨 CRITICAL: EMBEDDING MODEL INCONSISTENCY DETECTED 🚨\n"
+                "="*80 + "\n"
+                "You are attempting to use a different embedding model than what was\n"
+                "previously used in this storage space. This BREAKS semantic search!\n\n"
+                f"CURRENT embedding model:\n"
+                f"  • Provider: {current_info.get('provider_type', 'Unknown')}\n"
+                f"  • Model: {current_info.get('model_name', 'Unknown')}\n"
+                f"  • Dimensions: {current_info.get('dimension', 'Unknown')}\n\n"
+                f"STORED embedding model:\n"
+                f"  • Provider: {stored_model_info.get('provider_type', 'Unknown')}\n"
+                f"  • Model: {stored_model_info.get('model_name', 'Unknown')}\n"
+                f"  • Dimensions: {stored_model_info.get('dimension', 'Unknown')}\n\n"
+                "IMPORTANT: You can change LLM providers freely (Anthropic ↔ OpenAI ↔ Ollama)\n"
+                "but embedding models must remain consistent within a storage space.\n\n"
+                "TO FIX THIS ISSUE:\n"
+                "1. Use the SAME embedding model as stored, OR\n"
+                "2. Delete your vector database and start fresh with the new model\n"
+                "   (this will re-embed all interactions with the new model)\n"
+                "="*80
+            )
+
+
+class EmbeddingError(Exception):
+    """Base exception for embedding-related errors."""
+    pass
+
+
+def create_embedding_adapter(provider: Optional[Any] = None) -> EmbeddingAdapter:
+    """
+    Create an embedding adapter for the given provider.
+
+    Args:
+        provider: LLM provider instance
+
+    Returns:
+        EmbeddingAdapter: Configured adapter
+    """
+    return EmbeddingAdapter(provider)

abstractmemory/graph/__init__.py

@@ -0,0 +1 @@
+# Temporal knowledge graph implementation

abstractmemory/graph/knowledge_graph.py

@@ -0,0 +1,178 @@
+"""
+Temporal knowledge graph implementation.
+"""
+
+import networkx as nx
+from datetime import datetime
+from typing import Dict, List, Optional, Tuple, Any
+
+from abstractmemory.core.temporal import GroundingAnchor, TemporalSpan, RelationalContext
+
+
+class TemporalKnowledgeGraph:
+    """
+    Knowledge graph with bi-temporal modeling.
+    Based on Zep/Graphiti architecture.
+    """
+
+    def __init__(self):
+        self.graph = nx.MultiDiGraph()
+        self._node_counter = 0
+        self._edge_counter = 0
+        self.ontology = {}  # Auto-built ontology
+
+    def add_entity(self, value: str, entity_type: str = 'entity') -> str:
+        """Add or get entity node"""
+        # Check for existing entity (deduplication)
+        for node_id, data in self.graph.nodes(data=True):
+            if data.get('value') == value:
+                # Update access time
+                self.graph.nodes[node_id]['last_accessed'] = datetime.now()
+                return node_id
+
+        # Create new entity
+        node_id = f"entity_{self._node_counter}"
+        self._node_counter += 1
+
+        self.graph.add_node(
+            node_id,
+            value=value,
+            type=entity_type,
+            created_at=datetime.now(),
+            last_accessed=datetime.now(),
+            importance=1.0
+        )
+
+        # Update ontology
+        if entity_type not in self.ontology:
+            self.ontology[entity_type] = []
+        self.ontology[entity_type].append(node_id)
+
+        return node_id
+
+    def add_fact(self, subject: str, predicate: str, object: str,
+                event_time: datetime, confidence: float = 1.0,
+                source: Optional[str] = None, ingestion_time: Optional[datetime] = None) -> str:
+        """Add temporally anchored fact"""
+
+        # Get or create nodes
+        subj_id = self.add_entity(subject)
+        obj_id = self.add_entity(object)
+
+        # Create grounding anchor
+        anchor = GroundingAnchor(
+            event_time=event_time,
+            ingestion_time=ingestion_time or datetime.now(),
+            validity_span=TemporalSpan(start=event_time),
+            relational=RelationalContext(user_id="default"),  # Will be updated when used in GroundedMemory
+            confidence=confidence,
+            source=source
+        )
+
+        # Check for contradictions
+        self._handle_contradictions(subj_id, predicate, obj_id, anchor)
+
+        # Add edge with temporal data
+        edge_id = f"edge_{self._edge_counter}"
+        self._edge_counter += 1
+
+        self.graph.add_edge(
+            subj_id, obj_id,
+            key=edge_id,
+            predicate=predicate,
+            anchor=anchor,
+            confidence=confidence,
+            valid=True
+        )
+
+        return edge_id
+
+    def _handle_contradictions(self, subj_id: str, predicate: str,
+                              obj_id: str, new_anchor: GroundingAnchor):
+        """Handle temporal contradictions"""
+        # Check existing edges for contradictions
+        for _, _, key, data in self.graph.edges(subj_id, keys=True, data=True):
+            if data.get('predicate') == predicate and data.get('valid'):
+                old_anchor = data.get('anchor')
+                if old_anchor:
+                    # Check for temporal overlap
+                    if self._has_temporal_overlap(old_anchor, new_anchor):
+                        # Invalidate older fact (new info takes precedence)
+                        if old_anchor.ingestion_time < new_anchor.ingestion_time:
+                            data['valid'] = False
+                            old_anchor.validity_span.end = new_anchor.event_time
+                            old_anchor.validity_span.valid = False
+
+    def _has_temporal_overlap(self, anchor1: GroundingAnchor,
+                             anchor2: GroundingAnchor) -> bool:
+        """Check if two anchors have temporal overlap"""
+        span1 = anchor1.validity_span
+        span2 = anchor2.validity_span
+
+        # If either span has no end, check if starts overlap
+        if span1.end is None or span2.end is None:
+            return True  # Conservative: assume overlap
+
+        # Check for actual overlap
+        return not (span1.end < span2.start or span2.end < span1.start)
+
+    def query_at_time(self, query: str, point_in_time: datetime) -> List[Dict[str, Any]]:
+        """Query knowledge state at specific time"""
+        results = []
+
+        for u, v, key, data in self.graph.edges(keys=True, data=True):
+            anchor = data.get('anchor')
+            if not anchor:
+                continue
+
+            # Check if fact was known and valid at this time
+            if (anchor.ingestion_time <= point_in_time and
+                anchor.event_time <= point_in_time and
+                data.get('valid', True)):  # Default to True if not explicitly set
+
+                # Check if still valid at query time
+                if (anchor.validity_span.end is None or
+                    anchor.validity_span.end > point_in_time):
+
+                    # Check if matches query
+                    if query.lower() in data.get('predicate', '').lower():
+                        results.append({
+                            'subject': self.graph.nodes[u]['value'],
+                            'predicate': data['predicate'],
+                            'object': self.graph.nodes[v]['value'],
+                            'confidence': data.get('confidence', 1.0),
+                            'event_time': anchor.event_time,
+                            'source': getattr(anchor, 'source', None)
+                        })
+
+        return results
+
+    def get_entity_evolution(self, entity: str, start: datetime,
+                            end: datetime) -> List[Dict[str, Any]]:
+        """Track how entity's relationships evolved over time"""
+        # Find entity node
+        entity_id = None
+        for node_id, data in self.graph.nodes(data=True):
+            if data.get('value') == entity:
+                entity_id = node_id
+                break
+
+        if not entity_id:
+            return []
+
+        evolution = []
+
+        # Check all edges involving this entity
+        for u, v, key, data in self.graph.edges(keys=True, data=True):
+            if u == entity_id or v == entity_id:
+                anchor = data.get('anchor')
+                if anchor and start <= anchor.event_time <= end:
+                    evolution.append({
+                        'time': anchor.event_time,
+                        'type': 'fact_added' if data.get('valid') else 'fact_invalidated',
+                        'subject': self.graph.nodes[u]['value'],
+                        'predicate': data['predicate'],
+                        'object': self.graph.nodes[v]['value']
+                    })
+
+        return sorted(evolution, key=lambda x: x['time'])

abstractmemory/simple.py

@@ -0,0 +1,151 @@
+"""
+Simple, efficient memory for task-specific agents.
+No over-engineering - just what's needed for the job.
+"""
+
+from typing import List, Optional, Dict, Any
+from collections import deque
+from datetime import datetime
+
+
+class ScratchpadMemory:
+    """
+    Lightweight memory for ReAct agents and single-task tools.
+
+    Use this for:
+    - ReAct agent thought-action-observation cycles
+    - Summarizer working memory
+    - Extractor temporary context
+    - Any agent that doesn't need persistence
+
+    Example:
+        # For a ReAct agent
+        scratchpad = ScratchpadMemory(max_entries=20)
+        scratchpad.add_thought("Need to search for Python tutorials")
+        scratchpad.add_action("search", {"query": "Python basics"})
+        scratchpad.add_observation("Found 10 relevant tutorials")
+
+        # Get full context for next iteration
+        context = scratchpad.get_context()
+    """
+
+    def __init__(self, max_entries: int = 100):
+        """Initialize scratchpad with bounded size"""
+        self.entries: deque = deque(maxlen=max_entries)
+        self.thoughts: List[str] = []
+        self.actions: List[Dict[str, Any]] = []
+        self.observations: List[str] = []
+
+    def add(self, content: str, entry_type: str = "note"):
+        """Add any entry to scratchpad"""
+        entry = {
+            "type": entry_type,
+            "content": content,
+            "timestamp": datetime.now().isoformat()
+        }
+        self.entries.append(entry)
+
+    def add_thought(self, thought: str):
+        """Add a thought (for ReAct pattern)"""
+        self.thoughts.append(thought)
+        self.add(thought, "thought")
+
+    def add_action(self, action: str, params: Optional[Dict] = None):
+        """Add an action (for ReAct pattern)"""
+        action_entry = {"action": action, "params": params or {}}
+        self.actions.append(action_entry)
+        self.add(f"Action: {action} with {params}", "action")
+
+    def add_observation(self, observation: str):
+        """Add an observation (for ReAct pattern)"""
+        self.observations.append(observation)
+        self.add(observation, "observation")
+
+    def get_context(self, last_n: Optional[int] = None) -> str:
+        """Get scratchpad context as string"""
+        entries_to_use = list(self.entries)
+        if last_n:
+            entries_to_use = entries_to_use[-last_n:]
+
+        context_lines = []
+        for entry in entries_to_use:
+            if entry["type"] == "thought":
+                context_lines.append(f"Thought: {entry['content']}")
+            elif entry["type"] == "action":
+                context_lines.append(f"Action: {entry['content']}")
+            elif entry["type"] == "observation":
+                context_lines.append(f"Observation: {entry['content']}")
+            else:
+                context_lines.append(entry['content'])
+
+        return "\n".join(context_lines)
+
+    def get_react_history(self) -> Dict[str, List]:
+        """Get structured ReAct history"""
+        return {
+            "thoughts": self.thoughts,
+            "actions": self.actions,
+            "observations": self.observations
+        }
+
+    def clear(self):
+        """Clear the scratchpad"""
+        self.entries.clear()
+        self.thoughts.clear()
+        self.actions.clear()
+        self.observations.clear()
+
+    def __len__(self) -> int:
+        return len(self.entries)
+
+    def __str__(self) -> str:
+        return f"ScratchpadMemory({len(self.entries)} entries)"
+
+
+class BufferMemory:
+    """
+    Simple conversation buffer (wrapper around BasicSession).
+
+    Use this when BasicSession from AbstractLLM Core is sufficient.
+    This is just a thin adapter for compatibility.
+
+    Example:
+        # For a simple chatbot
+        memory = BufferMemory(max_messages=50)
+        memory.add_message("user", "What's the weather?")
+        memory.add_message("assistant", "I don't have weather data")
+        context = memory.get_context()
+    """
+
+    def __init__(self, max_messages: int = 100):
+        """Initialize buffer with size limit"""
+        self.messages: deque = deque(maxlen=max_messages)
+
+    def add_message(self, role: str, content: str):
+        """Add a message to the buffer"""
+        self.messages.append({
+            "role": role,
+            "content": content,
+            "timestamp": datetime.now().isoformat()
+        })
+
+    def get_messages(self) -> List[Dict[str, str]]:
+        """Get messages for LLM context"""
+        return [{"role": m["role"], "content": m["content"]}
+                for m in self.messages]
+
+    def get_context(self, last_n: Optional[int] = None) -> str:
+        """Get conversation as formatted string"""
+        messages = list(self.messages)
+        if last_n:
+            messages = messages[-last_n:]
+
+        lines = []
+        for msg in messages:
+            lines.append(f"{msg['role']}: {msg['content']}")
+
+        return "\n".join(lines)
+
+    def clear(self):
+        """Clear the buffer"""
+        self.messages.clear()

abstractmemory/storage/__init__.py

@@ -0,0 +1,16 @@
+# Storage backends for memory persistence
+
+from .dual_manager import DualStorageManager
+from .markdown_storage import MarkdownStorage
+
+# LanceDB is optional
+try:
+    from .lancedb_storage import LanceDBStorage
+except ImportError:
+    LanceDBStorage = None
+
+__all__ = [
+    'DualStorageManager',
+    'MarkdownStorage',
+    'LanceDBStorage'
+]