AbstractMemory 0.1.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

abstractmemory/__init__.py

@@ -133,7 +133,7 @@ class GroundedMemory:
         )
 
         # Legacy storage backend for compatibility
-        self.storage = self._init_storage(storage_backend)
+        self.storage = self._init_storage(storage_backend, embedding_provider)
 
     def set_current_user(self, user_id: str, relationship: Optional[str] = None):
         """Set the current user for relational context"""
@@ -545,12 +545,12 @@ class GroundedMemory:
             logging.warning(f"Failed to initialize storage manager: {e}")
             return None
 
-    def _init_storage(self, backend: Optional[str]):
+    def _init_storage(self, backend: Optional[str], embedding_provider: Optional[Any] = None):
         """Initialize storage backend (legacy compatibility)"""
         if backend == 'lancedb':
             try:
                 from .storage.lancedb_storage import LanceDBStorage
-                return LanceDBStorage("./lance.db")
+                return LanceDBStorage("./lance.db", embedding_provider)
             except ImportError:
                 return None
         elif backend == 'file':

abstractmemory/embeddings/__init__.py

@@ -0,0 +1,317 @@
+"""
+Real embedding providers for semantic search capabilities.
+
+Provides a unified interface for generating embeddings from real providers:
+AbstractCore EmbeddingManager, OpenAI, Ollama with semantic capabilities.
+
+NO FALLBACKS - only real semantic embedding providers are supported.
+"""
+
+import logging
+import hashlib
+from typing import List, Optional, Any, Union
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingAdapter:
+    """
+    Unified embedding interface for various providers.
+
+    IMPORTANT: The embedding provider/model must remain consistent within a storage space.
+    You can change LLM providers freely, but changing embedding models requires recreating
+    your vector database as different models produce incompatible vector spaces.
+    """
+
+    def __init__(self, provider: Optional[Any] = None):
+        """
+        Initialize embedding adapter.
+
+        Args:
+            provider: Embedding provider (AbstractCore EmbeddingManager, OpenAI client, etc.)
+                     Note: This is for EMBEDDINGS only, not LLM text generation.
+
+        Raises:
+            ValueError: If no embedding provider is available
+        """
+        if provider is None:
+            raise ValueError(
+                "AbstractMemory semantic search requires a dedicated embedding provider. "
+                "This is separate from your LLM provider for text generation. "
+                "Please provide: AbstractCore EmbeddingManager, OpenAI client, Ollama with embeddings, etc."
+            )
+
+        self.provider = provider
+        self.provider_type = self._detect_provider_type()
+        self.embedding_dimension = self._get_embedding_dimension()
+        self.model_info = self._get_model_info()
+
+    def _detect_provider_type(self) -> str:
+        """Detect the type of provider and its embedding capabilities."""
+        # Check for AbstractCore EmbeddingManager (preferred)
+        try:
+            from abstractllm.embeddings import EmbeddingManager
+            if isinstance(self.provider, EmbeddingManager):
+                return "abstractcore_embeddings"
+        except ImportError:
+            pass
+
+        # Check for AbstractCore provider with embedding support (has specific AbstractCore attributes)
+        if hasattr(self.provider, 'generate_embedding') and hasattr(self.provider, 'provider_name'):
+            return "abstractcore"
+
+        # Check for OpenAI client
+        if hasattr(self.provider, 'embeddings'):
+            return "openai"
+
+        # Check for provider name attribute (Ollama, MLX, etc.)
+        if hasattr(self.provider, 'provider_name'):
+            provider_name = getattr(self.provider, 'provider_name', '').lower()
+            if 'ollama' in provider_name:
+                return "ollama"
+            elif 'mlx' in provider_name:
+                return "mlx"
+
+        # Check if provider has generate_embedding method (generic embedding provider)
+        if hasattr(self.provider, 'generate_embedding') and callable(getattr(self.provider, 'generate_embedding')):
+            return "generic_embedding_provider"
+
+        # If we can't identify an embedding provider, raise an error
+        raise ValueError(
+            f"Unable to identify an embedding provider from: {type(self.provider)}. "
+            "Supported providers: AbstractCore EmbeddingManager, OpenAI client, "
+            "Ollama with embeddings, or any object with 'generate_embedding()' method."
+        )
+
+    def _get_embedding_dimension(self) -> int:
+        """Get the embedding dimension based on provider type."""
+        if self.provider_type == "abstractcore_embeddings":
+            # Get dimension from a test embedding
+            try:
+                test_embedding = self.provider.embed("dimension_test")
+                return len(test_embedding)
+            except Exception as e:
+                logger.error(f"Failed to get embedding dimension from AbstractCore: {e}")
+                raise ValueError("Unable to determine embedding dimension from AbstractCore provider")
+        elif self.provider_type == "openai":
+            return 1536  # text-embedding-3-small default
+        elif self.provider_type == "ollama":
+            # Try to get dimension from test embedding
+            try:
+                test_embedding = self._generate_ollama_embedding("dimension_test")
+                return len(test_embedding)
+            except:
+                return 1024  # Common Ollama embedding dimension
+        elif self.provider_type == "generic_embedding_provider":
+            # For any provider with generate_embedding method
+            try:
+                test_embedding = self.provider.generate_embedding("dimension_test")
+                return len(test_embedding)
+            except Exception as e:
+                logger.error(f"Failed to determine embedding dimension from generic provider: {e}")
+                raise ValueError(f"Unable to determine embedding dimension: {e}")
+        else:
+            # For any other provider, attempt to generate a test embedding
+            try:
+                test_embedding = self.generate_embedding("dimension_test")
+                return len(test_embedding)
+            except Exception as e:
+                logger.error(f"Failed to determine embedding dimension: {e}")
+                raise ValueError(f"Unable to determine embedding dimension for provider type: {self.provider_type}")
+
+    def _get_model_info(self) -> dict:
+        """Get detailed information about the embedding model for consistency tracking."""
+        info = {
+            "provider_type": self.provider_type,
+            "dimension": self.embedding_dimension,
+            "created_at": datetime.now().isoformat()
+        }
+
+        if self.provider_type == "abstractcore_embeddings":
+            # Try to get model name from AbstractCore - only store serializable strings
+            try:
+                if hasattr(self.provider, 'model'):
+                    model_attr = getattr(self.provider, 'model')
+                    if isinstance(model_attr, str):
+                        info["model_name"] = model_attr
+                    else:
+                        # Get string representation of the model
+                        info["model_name"] = str(type(model_attr).__name__)
+                if hasattr(self.provider, 'backend'):
+                    info["backend"] = str(self.provider.backend)
+            except Exception as e:
+                logger.debug(f"Could not extract model info: {e}")
+        elif self.provider_type == "openai":
+            info["model_name"] = "text-embedding-3-small"  # Default assumption
+
+        return info
+
+    def generate_embedding(self, text: str) -> List[float]:
+        """
+        Generate embedding for the given text using the configured provider.
+
+        Args:
+            text: Input text to embed
+
+        Returns:
+            List[float]: Embedding vector
+
+        Raises:
+            EmbeddingError: If embedding generation fails
+        """
+        try:
+            if self.provider_type == "abstractcore_embeddings":
+                return self._generate_abstractcore_embeddings(text)
+            elif self.provider_type == "abstractcore":
+                return self._generate_abstractcore_embedding(text)
+            elif self.provider_type == "openai":
+                return self._generate_openai_embedding(text)
+            elif self.provider_type == "ollama":
+                return self._generate_ollama_embedding(text)
+            elif self.provider_type == "mlx":
+                return self._generate_mlx_embedding(text)
+            elif self.provider_type == "generic_embedding_provider":
+                return self.provider.generate_embedding(text)
+            else:
+                raise EmbeddingError(f"Unknown provider type: {self.provider_type}")
+
+        except Exception as e:
+            logger.error(f"Embedding generation failed with {self.provider_type}: {e}")
+            raise EmbeddingError(f"Failed to generate embedding: {e}") from e
+
+    def _generate_abstractcore_embeddings(self, text: str) -> List[float]:
+        """Generate embedding using AbstractCore EmbeddingManager."""
+        return self.provider.embed(text)
+
+    def _generate_abstractcore_embedding(self, text: str) -> List[float]:
+        """Generate embedding using AbstractCore provider."""
+        return self.provider.generate_embedding(text)
+
+    def _generate_openai_embedding(self, text: str) -> List[float]:
+        """Generate embedding using OpenAI client."""
+        response = self.provider.embeddings.create(
+            model="text-embedding-3-small",
+            input=text
+        )
+        return response.data[0].embedding
+
+    def _generate_ollama_embedding(self, text: str) -> List[float]:
+        """Generate embedding using Ollama provider."""
+        try:
+            import requests
+            response = requests.post(
+                "http://localhost:11434/api/embeddings",
+                json={
+                    "model": "nomic-embed-text",
+                    "prompt": text
+                },
+                timeout=30
+            )
+            if response.status_code == 200:
+                embedding = response.json()["embedding"]
+                if embedding and isinstance(embedding, list):
+                    return embedding
+            raise EmbeddingError(f"Ollama API returned status {response.status_code}")
+        except ImportError:
+            raise EmbeddingError("requests library not available for Ollama embedding API")
+        except Exception as e:
+            raise EmbeddingError(f"Ollama embedding generation failed: {e}") from e
+
+    def _generate_mlx_embedding(self, text: str) -> List[float]:
+        """Generate embedding using MLX provider."""
+        # MLX provider should implement actual MLX embedding model
+        raise EmbeddingError(
+            "MLX embedding implementation not yet available. "
+            "Please use AbstractCore EmbeddingManager or another provider."
+        )
+
+    def is_real_embedding(self) -> bool:
+        """Check if this adapter provides real semantic embeddings."""
+        return self.provider_type in ["abstractcore_embeddings", "abstractcore", "openai", "ollama", "generic_embedding_provider"]
+
+    def get_embedding_info(self) -> dict:
+        """Get comprehensive information about the embedding provider for consistency tracking."""
+        info = self.model_info.copy()
+        info.update({
+            "is_real_embedding": self.is_real_embedding(),
+            "provider_available": self.provider is not None
+        })
+        return info
+
+    def check_consistency_with(self, other_model_info: dict) -> bool:
+        """
+        Check if this adapter is consistent with previously stored model info.
+
+        Args:
+            other_model_info: Previously stored model information
+
+        Returns:
+            bool: True if models are compatible for semantic search
+        """
+        current_info = self.get_embedding_info()
+
+        # Check critical compatibility factors
+        if current_info.get("provider_type") != other_model_info.get("provider_type"):
+            logger.warning(f"Provider type mismatch: {current_info.get('provider_type')} vs {other_model_info.get('provider_type')}")
+            return False
+
+        if current_info.get("dimension") != other_model_info.get("dimension"):
+            logger.warning(f"Dimension mismatch: {current_info.get('dimension')} vs {other_model_info.get('dimension')}")
+            return False
+
+        if current_info.get("model_name") != other_model_info.get("model_name"):
+            logger.warning(f"Model name mismatch: {current_info.get('model_name')} vs {other_model_info.get('model_name')}")
+            return False
+
+        return True
+
+    def warn_about_consistency(self, stored_model_info: dict) -> None:
+        """
+        Issue warnings about embedding model consistency issues.
+
+        Args:
+            stored_model_info: Information about previously stored embeddings
+        """
+        if not self.check_consistency_with(stored_model_info):
+            current_info = self.get_embedding_info()
+            logger.warning(
+                "\n" + "="*80 + "\n"
+                "🚨 CRITICAL: EMBEDDING MODEL INCONSISTENCY DETECTED 🚨\n"
+                "="*80 + "\n"
+                "You are attempting to use a different embedding model than what was\n"
+                "previously used in this storage space. This BREAKS semantic search!\n\n"
+                f"CURRENT embedding model:\n"
+                f"  • Provider: {current_info.get('provider_type', 'Unknown')}\n"
+                f"  • Model: {current_info.get('model_name', 'Unknown')}\n"
+                f"  • Dimensions: {current_info.get('dimension', 'Unknown')}\n\n"
+                f"STORED embedding model:\n"
+                f"  • Provider: {stored_model_info.get('provider_type', 'Unknown')}\n"
+                f"  • Model: {stored_model_info.get('model_name', 'Unknown')}\n"
+                f"  • Dimensions: {stored_model_info.get('dimension', 'Unknown')}\n\n"
+                "IMPORTANT: You can change LLM providers freely (Anthropic ↔ OpenAI ↔ Ollama)\n"
+                "but embedding models must remain consistent within a storage space.\n\n"
+                "TO FIX THIS ISSUE:\n"
+                "1. Use the SAME embedding model as stored, OR\n"
+                "2. Delete your vector database and start fresh with the new model\n"
+                "   (this will re-embed all interactions with the new model)\n"
+                "="*80
+            )
+
+
+class EmbeddingError(Exception):
+    """Base exception for embedding-related errors."""
+    pass
+
+
+def create_embedding_adapter(provider: Optional[Any] = None) -> EmbeddingAdapter:
+    """
+    Create an embedding adapter for the given provider.
+
+    Args:
+        provider: LLM provider instance
+
+    Returns:
+        EmbeddingAdapter: Configured adapter
+    """
+    return EmbeddingAdapter(provider)

abstractmemory/storage/dual_manager.py

@@ -93,12 +93,7 @@ class DualStorageManager:
         # Save to LanceDB storage
         if self.lancedb_storage:
             try:
-                # Generate embedding if provider available
-                embedding = None
-                if self.embedding_provider:
-                    text = f"{user_input} {agent_response}"
-                    embedding = self.embedding_provider.generate_embedding(text)
-
+                # LanceDB storage handles embedding generation internally
                 ldb_id = self.lancedb_storage.save_interaction(
                     user_id, timestamp, user_input, agent_response, topic, metadata
                 )
@@ -138,11 +133,7 @@ class DualStorageManager:
         # Save to LanceDB storage
         if self.lancedb_storage:
             try:
-                # Generate embedding for reflection if provider available
-                embedding = None
-                if self.embedding_provider:
-                    embedding = self.embedding_provider.generate_embedding(reflection)
-
+                # LanceDB storage handles embedding generation internally
                 ldb_note_id = self.lancedb_storage.save_experiential_note(
                     timestamp, reflection, interaction_id, note_type, metadata
                 )

abstractmemory/storage/lancedb_storage.py

@@ -9,6 +9,7 @@ from datetime import datetime
 import logging
 
 from ..core.interfaces import IStorage
+from ..embeddings import create_embedding_adapter
 
 logger = logging.getLogger(__name__)
 
@@ -38,16 +39,27 @@ class LanceDBStorage(IStorage):
         Args:
             uri: LanceDB connection URI (e.g., "./lance.db")
             embedding_provider: AbstractCore instance for generating embeddings
+
+        Raises:
+            ImportError: If LanceDB is not installed
+            ValueError: If no embedding provider is provided
         """
         if not LANCEDB_AVAILABLE:
             raise ImportError("LanceDB is required but not installed. Install with: pip install lancedb")
 
+        if embedding_provider is None:
+            raise ValueError(
+                "LanceDB storage requires a real embedding provider for semantic search. "
+                "Please provide an AbstractCore EmbeddingManager or other embedding provider."
+            )
+
         self.uri = uri
-        self.embedding_provider = embedding_provider
+        self.embedding_adapter = create_embedding_adapter(embedding_provider)
         self.db = lancedb.connect(uri)
 
-        # Initialize tables
+        # Initialize tables and check embedding consistency
         self._init_tables()
+        self._check_embedding_consistency()
 
     def _init_tables(self):
         """Initialize LanceDB tables with schemas"""
@@ -92,43 +104,136 @@ class LanceDBStorage(IStorage):
         ]
 
         # Create tables if they don't exist
+        import pandas as pd
+
         try:
             self.interactions_table = self.db.open_table("interactions")
-        except FileNotFoundError:
-            # Create empty table with schema
-            import pandas as pd
-            empty_df = pd.DataFrame(columns=[col["name"] for col in interactions_schema])
-            self.interactions_table = self.db.create_table("interactions", empty_df)
+        except (FileNotFoundError, ValueError):
+            # Create table with proper schema and sample data
+            import pyarrow as pa
+
+            # Get actual embedding dimension from adapter
+            test_embedding = self.embedding_adapter.generate_embedding("test")
+            embedding_dim = len(test_embedding)
+
+            # Create proper schema with fixed-size list for embeddings
+            schema = pa.schema([
+                pa.field("id", pa.string()),
+                pa.field("user_id", pa.string()),
+                pa.field("timestamp", pa.timestamp('us')),
+                pa.field("user_input", pa.string()),
+                pa.field("agent_response", pa.string()),
+                pa.field("topic", pa.string()),
+                pa.field("metadata", pa.string()),
+                pa.field("embedding", pa.list_(pa.float32(), embedding_dim))
+            ])
+
+            # Create empty table with proper schema
+            self.interactions_table = self.db.create_table("interactions", schema=schema)
 
         try:
             self.notes_table = self.db.open_table("experiential_notes")
-        except FileNotFoundError:
-            import pandas as pd
-            empty_df = pd.DataFrame(columns=[col["name"] for col in notes_schema])
-            self.notes_table = self.db.create_table("experiential_notes", empty_df)
+        except (FileNotFoundError, ValueError):
+            # Create notes table with proper schema
+            notes_schema = pa.schema([
+                pa.field("id", pa.string()),
+                pa.field("timestamp", pa.timestamp('us')),
+                pa.field("reflection", pa.string()),
+                pa.field("interaction_id", pa.string()),
+                pa.field("note_type", pa.string()),
+                pa.field("metadata", pa.string()),
+                pa.field("embedding", pa.list_(pa.float32(), embedding_dim))
+            ])
+            self.notes_table = self.db.create_table("experiential_notes", schema=notes_schema)
 
         try:
             self.links_table = self.db.open_table("links")
-        except FileNotFoundError:
-            import pandas as pd
-            empty_df = pd.DataFrame(columns=[col["name"] for col in links_schema])
-            self.links_table = self.db.create_table("links", empty_df)
+        except (FileNotFoundError, ValueError):
+            sample_data = pd.DataFrame([{
+                "interaction_id": "sample_int",
+                "note_id": "sample_note",
+                "created": datetime.now(),
+                "link_type": "bidirectional"
+            }])
+            self.links_table = self.db.create_table("links", sample_data)
+            self.links_table.delete("interaction_id = 'sample_int'")
 
         try:
             self.components_table = self.db.open_table("memory_components")
-        except FileNotFoundError:
+        except (FileNotFoundError, ValueError):
+            sample_data = pd.DataFrame([{
+                "component_name": "sample",
+                "timestamp": datetime.now(),
+                "data": "{}",
+                "version": 1
+            }])
+            self.components_table = self.db.create_table("memory_components", sample_data)
+            self.components_table.delete("component_name = 'sample'")
+
+        # Embedding metadata table for consistency tracking
+        try:
+            self.embedding_metadata_table = self.db.open_table("embedding_metadata")
+        except (FileNotFoundError, ValueError):
+            sample_data = pd.DataFrame([{
+                "key": "sample",
+                "value": "{}",
+                "created_at": datetime.now()
+            }])
+            self.embedding_metadata_table = self.db.create_table("embedding_metadata", sample_data)
+            self.embedding_metadata_table.delete("key = 'sample'")
+
+    def _check_embedding_consistency(self) -> None:
+        """Check for embedding model consistency with previously stored data."""
+        try:
+            # Get current embedding model info
+            current_info = self.embedding_adapter.get_embedding_info()
+
+            # Try to retrieve previously stored embedding info
+            stored_info_df = self.embedding_metadata_table.search().where("key = 'embedding_model_info'").to_pandas()
+
+            if len(stored_info_df) > 0:
+                # We have previously stored embedding info
+                import json
+                stored_info = json.loads(stored_info_df.iloc[0]['value'])
+
+                # Check consistency and warn if needed
+                self.embedding_adapter.warn_about_consistency(stored_info)
+            else:
+                # First time - store the current embedding info
+                self._store_embedding_info(current_info)
+                logger.info(f"Stored embedding model info for consistency tracking: {current_info}")
+
+        except Exception as e:
+            logger.warning(f"Failed to check embedding consistency: {e}")
+
+    def _store_embedding_info(self, embedding_info: dict) -> None:
+        """Store embedding model information for consistency tracking."""
+        try:
+            import json
             import pandas as pd
-            empty_df = pd.DataFrame(columns=[col["name"] for col in components_schema])
-            self.components_table = self.db.create_table("memory_components", empty_df)
 
-    def _generate_embedding(self, text: str) -> Optional[List[float]]:
-        """Generate embedding using AbstractCore provider"""
-        if self.embedding_provider and hasattr(self.embedding_provider, 'generate_embedding'):
+            # Delete any existing embedding_model_info records
             try:
-                return self.embedding_provider.generate_embedding(text)
-            except Exception as e:
-                logger.error(f"Failed to generate embedding: {e}")
-        return None
+                self.embedding_metadata_table.delete("key = 'embedding_model_info'")
+            except:
+                pass  # Table might be empty
+
+            # Store new info
+            data = pd.DataFrame([{
+                "key": "embedding_model_info",
+                "value": json.dumps(embedding_info),
+                "created_at": datetime.now()
+            }])
+
+            self.embedding_metadata_table.add(data)
+            logger.debug(f"Stored embedding model info: {embedding_info}")
+
+        except Exception as e:
+            logger.error(f"Failed to store embedding info: {e}")
+
+    def _generate_embedding(self, text: str) -> List[float]:
+        """Generate embedding using embedding adapter"""
+        return self.embedding_adapter.generate_embedding(text)
 
     def save_interaction(self, user_id: str, timestamp: datetime,
                         user_input: str, agent_response: str,
@@ -153,7 +258,7 @@ class LanceDBStorage(IStorage):
             "agent_response": agent_response,
             "topic": topic,
             "metadata": json.dumps(metadata or {}),
-            "embedding": embedding or [0.0] * 384  # Default embedding size
+            "embedding": [float(x) for x in embedding]  # Ensure float32 compatibility
         }
 
         # Insert into table
@@ -189,7 +294,7 @@ class LanceDBStorage(IStorage):
             "interaction_id": interaction_id,
             "note_type": note_type,
             "metadata": json.dumps(metadata or {}),
-            "embedding": embedding or [0.0] * 384  # Default embedding size
+            "embedding": [float(x) for x in embedding]  # Ensure float32 compatibility
         }
 
         # Insert into table
@@ -253,13 +358,13 @@ class LanceDBStorage(IStorage):
             # Build WHERE clause
             where_clause = " AND ".join(query_parts) if query_parts else None
 
-            # Try vector search first if embedding provider available
-            if self.embedding_provider:
+            # Try vector search first if embedding adapter available
+            if self.embedding_adapter:
                 try:
                     query_embedding = self._generate_embedding(query)
                     if query_embedding:
                         # Vector similarity search
-                        results = self.interactions_table.search(query_embedding).limit(50)
+                        results = self.interactions_table.search(query_embedding, vector_column_name="embedding").limit(50)
 
                         # Apply additional filters
                         if where_clause:
@@ -408,18 +513,32 @@ class LanceDBStorage(IStorage):
             links_count = len(self.links_table.search().limit(10000).to_pandas())
             components_count = len(self.components_table.search().limit(1000).to_pandas())
 
-            return {
+            stats = {
                 "total_interactions": interactions_count,
                 "total_notes": notes_count,
                 "total_links": links_count,
                 "total_components": components_count,
                 "uri": self.uri,
-                "embedding_provider_available": self.embedding_provider is not None
+                "embedding_provider_available": self.embedding_adapter is not None,
+                "embedding_info": self.embedding_adapter.get_embedding_info() if self.embedding_adapter else None
             }
+
+            # Add stored embedding model info for comparison
+            try:
+                stored_info_df = self.embedding_metadata_table.search().where("key = 'embedding_model_info'").to_pandas()
+                if len(stored_info_df) > 0:
+                    import json
+                    stats["stored_embedding_info"] = json.loads(stored_info_df.iloc[0]['value'])
+                    stats["embedding_consistency"] = self.embedding_adapter.check_consistency_with(stats["stored_embedding_info"]) if self.embedding_adapter else False
+            except Exception as e:
+                logger.debug(f"Could not retrieve stored embedding info: {e}")
+
+            return stats
         except Exception as e:
             logger.error(f"Failed to get stats: {e}")
             return {
                 "error": str(e),
                 "uri": self.uri,
-                "embedding_provider_available": self.embedding_provider is not None
+                "embedding_provider_available": self.embedding_adapter is not None,
+                "embedding_info": self.embedding_adapter.get_embedding_info() if self.embedding_adapter else None
             }

{abstractmemory-0.1.0.dist-info → abstractmemory-0.2.2.dist-info}/METADATA

@@ -1,15 +1,15 @@
 Metadata-Version: 2.4
 Name: AbstractMemory
-Version: 0.1.0
-Summary: Intelligent memory system for LLM agents with two-tier architecture - temporal knowledge graph memory for sophisticated AI agents
+Version: 0.2.2
+Summary: Production-ready memory system for LLM agents - NO MOCKS, real semantic search, clear LLM vs embedding provider separation
 Author-email: AbstractMemory Team <lpalbou@gmail.com>
 Maintainer-email: AbstractMemory Team <palbou@gmail.com>
 License-Expression: MIT
-Project-URL: Homepage, https://github.com/lpalbou/AbstractAgent
-Project-URL: Documentation, https://github.com/lpalbou/AbstractAgent#readme
-Project-URL: Repository, https://github.com/lpalbou/AbstractAgent
-Project-URL: Bug Reports, https://github.com/lpalbou/AbstractAgent/issues
-Keywords: llm,memory,stateful,ai,agents,knowledge-graph,temporal,grounded-memory
+Project-URL: Homepage, https://github.com/lpalbou/AbstractMemory
+Project-URL: Documentation, https://github.com/lpalbou/AbstractMemory#readme
+Project-URL: Repository, https://github.com/lpalbou/AbstractMemory
+Project-URL: Bug Reports, https://github.com/lpalbou/AbstractMemory/issues
+Keywords: llm,memory,semantic-search,embeddings,ai,agents,knowledge-graph,temporal,grounded-memory,vector-search
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -30,9 +30,15 @@ Requires-Dist: pytest; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
 Provides-Extra: llm
-Requires-Dist: abstractllm>=0.5.0; extra == "llm"
+Requires-Dist: abstractcore>=2.1.0; extra == "llm"
+Provides-Extra: embeddings
+Requires-Dist: abstractcore>=2.1.0; extra == "embeddings"
+Requires-Dist: lancedb>=0.6.0; extra == "embeddings"
 Provides-Extra: storage
-Requires-Dist: lancedb; extra == "storage"
+Requires-Dist: lancedb>=0.6.0; extra == "storage"
+Provides-Extra: all
+Requires-Dist: abstractcore>=2.1.0; extra == "all"
+Requires-Dist: lancedb>=0.6.0; extra == "all"
 Dynamic: license-file
 
 # AbstractMemory
@@ -199,20 +205,27 @@ memory = create_memory(
 
 ## 📚 Documentation
 
-- **[Architecture Guide](docs/architecture.md)** - Complete system design
-- **[Memory Types](docs/memory-types.md)** - Detailed component guide
-- **[Storage Systems](docs/storage-systems.md)** - Persistent storage with dual backends
-- **[Usage Patterns](docs/usage-patterns.md)** - Real-world examples
-- **[API Reference](docs/api-reference.md)** - Complete API documentation
-- **[Integration Guide](docs/integration.md)** - AbstractLLM ecosystem integration
-- **[AbstractCore Embedding Specs](docs/abstractcore-embedding-specs.md)** - Embedding integration requirements
+**👉 [START HERE: Complete Documentation Guide](docs/README.md)**
+
+### Core Guides
+- **[🚀 Quick Start](docs/README.md#-start-here)** - Get running in 5 minutes
+- **[🔍 Semantic Search](docs/semantic-search.md)** - Vector embeddings and similarity search
+- **[🧠 Memory Types](docs/memory-types.md)** - ScratchpadMemory, BufferMemory, GroundedMemory
+- **[📊 Performance Guide](docs/semantic-search.md#performance-characteristics)** - Embedding timing and optimization
+
+### Advanced Topics
+- **[🏗️ Architecture](docs/architecture.md)** - System design and two-tier strategy
+- **[💾 Storage Systems](docs/storage-systems.md)** - Markdown + LanceDB dual storage
+- **[🎯 Usage Patterns](docs/usage-patterns.md)** - Real-world examples and best practices
+- **[🔗 Integration Guide](docs/integration.md)** - AbstractLLM ecosystem integration
+- **[📖 API Reference](docs/api-reference.md)** - Complete method documentation
 
 ## 🔬 Key Features
 
 ### ✅ Purpose-Built Memory Types
-- **ScratchpadMemory**: ReAct thought-action-observation cycles
-- **BufferMemory**: Simple conversation history
-- **GroundedMemory**: Multi-dimensional temporal memory
+- **ScratchpadMemory**: ReAct thought-action-observation cycles for task agents
+- **BufferMemory**: Simple conversation history with capacity limits
+- **GroundedMemory**: Four-tier architecture with semantic search and temporal context
 
 ### ✅ State-of-the-Art Research Integration
 - **MemGPT/Letta Pattern**: Self-editing core memory
@@ -238,12 +251,19 @@ Core Memory ──→ Semantic Memory ──→ Working Memory ──→ Episodi
 - **🔗 Bidirectional Links**: Connect interactions to AI insights
 - **📊 Search Capabilities**: Text-based and semantic similarity search
 
+### ✅ Semantic Search with AbstractCore
+- **🎯 Real Embeddings**: Uses AbstractCore's EmbeddingManager with Google's EmbeddingGemma (768D)
+- **⚡ Immediate Indexing**: Embeddings generated instantly during `add_interaction()` (~36ms)
+- **🔍 Vector Similarity**: True semantic search finds contextually relevant content
+- **🗄️ Dual Storage**: Observable markdown files + searchable LanceDB vectors
+- **🎯 Production Ready**: Sub-second search, proven with 200+ real implementation tests
+
 ## 🧪 Testing & Validation
 
-AbstractMemory includes **200+ comprehensive tests** with real implementations:
+AbstractMemory includes **200+ comprehensive tests** using ONLY real implementations:
 
 ```bash
-# Run all tests
+# Run all tests (NO MOCKS - only real implementations)
 python -m pytest tests/ -v
 
 # Run specific test suites
@@ -255,41 +275,150 @@ python -m pytest tests/integration/ -v     # Full system integration
 # Test with real LLM providers (requires AbstractCore)
 python -m pytest tests/integration/test_llm_real_usage.py -v
 
-# Test comprehensive dual storage serialization
+# Test comprehensive dual storage with real embeddings
 python -m pytest tests/storage/test_dual_storage_comprehensive.py -v
 ```
 
+**IMPORTANT**: All tests use real implementations:
+- Real embedding providers (AbstractCore EmbeddingManager)
+- Real LLM providers (Anthropic, OpenAI, Ollama via AbstractCore)
+- Real memory components and storage systems
+- NO MOCKS anywhere in the codebase
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+# Install with semantic search capabilities (recommended)
+pip install abstractmemory[embeddings]
+
+# Or install everything
+pip install abstractmemory[all]
+
+# Basic memory only (no semantic search)
+pip install abstractmemory
+```
+
+### 📋 Upgrading from v0.1.0?
+
+**Version 0.2.0 adds semantic search!** See [Migration Guide](CHANGELOG.md#-migration-guide) for:
+- New AbstractCore dependency (`pip install abstractcore>=2.1.0`)
+- LanceDB schema changes (recreate `.db` files)
+- New `embedding_provider` parameter
+
+### ⚠️  Critical: LLM vs Embedding Provider Separation
+
+**Understanding the difference between LLM and Embedding providers:**
+
+- 🔄 **LLM Providers** (text generation): Change freely between Anthropic, OpenAI, Ollama, etc.
+- 🔒 **Embedding Providers** (semantic search): Must remain consistent within a storage space
+
+**For semantic search consistency:**
+- ✅ **Choose ONE embedding model and stick with it per storage space**
+- ✅ **You can customize which embedding model to use (AbstractCore, OpenAI, Ollama, etc.)**
+- ❌ **Don't change embedding models mid-project - it breaks vector search**
+- 🚨 **AbstractMemory automatically warns when embedding model changes detected**
+
+**Example of correct separation:**
+```python
+# LLM for text generation (can change anytime)
+llm = create_llm("anthropic")  # or "openai", "ollama", etc.
+
+# Dedicated embedding provider (must stay consistent)
+embedder = EmbeddingManager()  # AbstractCore embeddings
+
+memory = create_memory("grounded", embedding_provider=embedder)  # NOT llm!
+```
+
+### Basic Usage
+
+```python
+from abstractllm.embeddings import EmbeddingManager
+from abstractmemory import create_memory
+
+# 1. Create embedding manager for semantic search
+em = EmbeddingManager()  # Uses EmbeddingGemma (768D vectors)
+
+# 2. Create memory with dual storage
+memory = create_memory(
+    "grounded",
+    storage_backend="dual",           # Markdown + LanceDB
+    storage_path="./memory_files",    # Observable files
+    storage_uri="./memory.db",        # Vector search
+    embedding_provider=em             # Real embeddings
+)
+
+# 3. Add interactions (embeddings generated immediately!)
+memory.set_current_user("alice")
+memory.add_interaction(
+    "I'm working on machine learning projects",
+    "Great! ML has amazing applications in many fields."
+)
+# ↳ Takes ~36ms: embedding generated and stored instantly
+
+# 4. Semantic search finds contextually relevant content
+results = memory.search_stored_interactions("artificial intelligence research")
+# ↳ Finds ML interaction via semantic similarity (not keywords!)
+print(f"Found {len(results)} relevant conversations")
+```
+
+### 📋 What Happens When You Add Interactions
+
+```python
+memory.add_interaction("I love Python", "Great choice!")
+# ↓ IMMEDIATE PROCESSING:
+# 1. Text combined: "I love Python Great choice!"
+# 2. EmbeddingManager.embed() called (36ms)
+# 3. 768D vector generated with EmbeddingGemma
+# 4. Saved to markdown file: ./memory_files/verbatim/alice/...
+# 5. Stored in LanceDB: vector + text + metadata
+# 6. Interaction immediately searchable via semantic similarity
+```
+
 ## 🔗 AbstractLLM Ecosystem Integration
 
-AbstractMemory seamlessly integrates with the broader ecosystem:
+AbstractMemory seamlessly integrates with AbstractCore, maintaining clear separation between LLM and embedding providers:
 
-### With AbstractCore
+### Critical Architecture: LLM vs Embedding Separation
 ```python
 from abstractllm import create_llm
+from abstractllm.embeddings import EmbeddingManager
 from abstractmemory import create_memory
 
-# Create LLM provider
-provider = create_llm("anthropic", model="claude-3-5-haiku-latest")
+# SEPARATE PROVIDERS for different purposes:
+
+# 1. LLM Provider - for TEXT GENERATION (can change freely)
+llm_provider = create_llm("anthropic", model="claude-3-5-haiku-latest")
+
+# 2. Embedding Provider - for SEMANTIC SEARCH (must stay consistent)
+embedding_provider = EmbeddingManager()
 
-# Create memory with embedding integration
+# Create memory with DEDICATED embedding provider
 memory = create_memory(
     "grounded",
     enable_kg=True,
     storage_backend="dual",
     storage_path="./memory",
     storage_uri="./memory.db",
-    embedding_provider=provider
+    embedding_provider=embedding_provider  # DEDICATED for embeddings
 )
 
-# Use together in agent reasoning
+# Use in agent reasoning with CLEAR separation
 context = memory.get_full_context(query)
-response = provider.generate(prompt, system_prompt=context)
-memory.add_interaction(query, response.content)
+response = llm_provider.generate(prompt, system_prompt=context)  # LLM for text
+memory.add_interaction(query, response.content)  # Embeddings handled internally
 
-# Search stored memories with semantic similarity
+# Search uses embedding provider for semantic similarity
 similar_memories = memory.search_stored_interactions("related concepts")
 ```
 
+### Key Points:
+- **LLM Provider**: Change freely between Anthropic ↔ OpenAI ↔ Ollama
+- **Embedding Provider**: Must remain consistent within storage space
+- **Never** pass LLM provider as embedding provider
+- **Always** use dedicated embedding provider for semantic search
+
 ### With AbstractAgent (Future)
 ```python
 from abstractagent import create_agent
@@ -306,7 +435,7 @@ response = agent.execute(task, user_id="alice")
 ## 🏛️ Architecture Principles
 
 1. **No Over-Engineering**: Memory complexity matches agent requirements
-2. **Real Implementation Testing**: No mocks - all tests use real implementations
+2. **Real Implementation Testing**: NO MOCKS anywhere - all tests use real implementations
 3. **SOTA Research Foundation**: Built on proven patterns (MemGPT, Zep, Graphiti)
 4. **Clean Abstractions**: Simple interfaces, powerful implementations
 5. **Performance Optimized**: Fast operations for simple agents, scalable for complex ones

{abstractmemory-0.1.0.dist-info → abstractmemory-0.2.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-abstractmemory/__init__.py,sha256=kCU1QIwh4MudeRTiti4KoPP3ZyoBAo2mjHdAAjpZZHA,32391
+abstractmemory/__init__.py,sha256=lqexsZeVlJVPmWzAJZY_ftJJ-jwr3YTjr0HikYa8KwA,32473
 abstractmemory/simple.py,sha256=mZmz7JX0zkzgvWVfC8xrpqCK1Zx43lyr4MBGWD8OQqM,4969
 abstractmemory/cognitive/__init__.py,sha256=6L9dF3Y_OdRct7o8Hn04d4UvzlRyevNh0Etxs6CwLME,40
 abstractmemory/components/__init__.py,sha256=mTMDpRZRaVz6HPQ9C-xvNccFD0xurfVeDP-mCZ5b3SQ,46
@@ -9,14 +9,15 @@ abstractmemory/components/working.py,sha256=4U11V8ZZzKgLs4EdcsCRTKmctQ64k9xk69_0
 abstractmemory/core/__init__.py,sha256=djQvbM76uRfL4ugZgu7gW-ZEKK1wcqm0nVPXgqGq_JA,44
 abstractmemory/core/interfaces.py,sha256=4hvoeHumBCCCyEmvmt46pFSegXHnd0c4UUN0Fp0CFi0,2757
 abstractmemory/core/temporal.py,sha256=4BR_cAJBpX7rlFnYEN4jcdhaOLkUWaCzAYI9aPX6K0M,3526
+abstractmemory/embeddings/__init__.py,sha256=AP_MNSiM9kUiVoenp-m6mHblGOt8ildvoJAtR54YABs,13995
 abstractmemory/graph/__init__.py,sha256=hW927TXsGYJDc3qN4r2zrXiMl7lEk1d81h1siov1GgE,41
 abstractmemory/graph/knowledge_graph.py,sha256=PWWBhsj2Nv4BesVGS_FjBwu88cVi1iNEHjlBRhirMlw,6805
 abstractmemory/storage/__init__.py,sha256=GLvuzeukQ2o84AkN0yH6rB4vhLgMXsd-6up9Mw6icoE,340
-abstractmemory/storage/dual_manager.py,sha256=3RUBqeIm_fa_Tqqv8ymPMBJKEzWj29AeuG8TBD15M0M,11724
-abstractmemory/storage/lancedb_storage.py,sha256=Wp928f-HWVJ4lnBi7XY_loCM5HXD9QVGWwrx2odAgVk,15578
+abstractmemory/storage/dual_manager.py,sha256=HXXYjd2NRg8QTAdrDEaLk34kGcz1WjcpPzfr2qFFKeM,11355
+abstractmemory/storage/lancedb_storage.py,sha256=QGIlJs5_0xQpZ05nfkkBL1zbJpP4QAJ1SgkfMkCXmlQ,20961
 abstractmemory/storage/markdown_storage.py,sha256=h5EVxPrEnYw-X3AhB_8DyfiEc944MywgEDv2e6LcNZA,16054
-abstractmemory-0.1.0.dist-info/licenses/LICENSE,sha256=NKLs197QnnfbXxdwIttTjZb1IHcj5IUTD-qXnr_V8Fw,1082
-abstractmemory-0.1.0.dist-info/METADATA,sha256=4hndUIsA9OvJBlfQRNgYZcHJ3LtM1vqczpDWyDl9e3M,11956
-abstractmemory-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-abstractmemory-0.1.0.dist-info/top_level.txt,sha256=ALpTfrLlzRBy55aam5M8YuOi-4i_shfsq1DcgTR6_NQ,15
-abstractmemory-0.1.0.dist-info/RECORD,,
+abstractmemory-0.2.2.dist-info/licenses/LICENSE,sha256=NKLs197QnnfbXxdwIttTjZb1IHcj5IUTD-qXnr_V8Fw,1082
+abstractmemory-0.2.2.dist-info/METADATA,sha256=ARGL-QSZNiui0lBmUUhx1_0sIUzUhf5HHP2SuZgz-Mg,17391
+abstractmemory-0.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+abstractmemory-0.2.2.dist-info/top_level.txt,sha256=ALpTfrLlzRBy55aam5M8YuOi-4i_shfsq1DcgTR6_NQ,15
+abstractmemory-0.2.2.dist-info/RECORD,,