endee-llamaindex 0.1.2__py3-none-any.whl → 0.1.5__py3-none-any.whl

endee_llamaindex/base.py

@@ -1,16 +1,32 @@
+"""
+EndeeVectorStore: LlamaIndex vector store backed by the Endee API.
+
+Aligned with the local endee package (./endee). API contract:
+
+Endee (endee.endee):
+  - __init__(token, http_library)
+  - create_index(name, dimension, space_type, M, ef_con, precision, version, sparse_dim)
+    Validates: index name (alphanumeric + underscores, max length), dimension <= MAX_DIMENSION_ALLOWED,
+    space_type in SPACE_TYPES_SUPPORTED ('cosine','l2','ip'), precision in PRECISION_TYPES_SUPPORTED,
+    sparse_dim >= 0. Map 'euclidean'->'l2', 'inner_product'->'ip' before calling.
+  - get_index(name) -> Index
+
+Index (endee.index):
+  - upsert(input_array): list of {id, vector, meta?, filter?, sparse_indices?, sparse_values?}; max MAX_VECTORS_PER_BATCH per batch; duplicate IDs in batch raise
+  - query(vector, top_k, filter, ef, include_vectors, sparse_indices, sparse_values)
+  - delete_vector(id), get_vector(id), describe()
+
+No list_ids or batch fetch in endee; filter for query is JSON-serializable (e.g. [{"field":{"$op":value}}] or dict).
+"""
+
 import logging
-from collections import Counter
-from functools import partial
 import json
 from typing import Any, Callable, Dict, List, Optional, cast
-
 from llama_index.core.bridge.pydantic import PrivateAttr
-from llama_index.core.schema import BaseNode, MetadataMode, TextNode
+from llama_index.core.schema import BaseNode, TextNode
 from llama_index.core.vector_stores.types import (
     BasePydanticVectorStore,
-    MetadataFilters,
     VectorStoreQuery,
-    VectorStoreQueryMode,
     VectorStoreQueryResult,
 )
 from llama_index.core.vector_stores.utils import (
@@ -19,83 +35,34 @@ from llama_index.core.vector_stores.utils import (
     metadata_dict_to_node,
     node_to_metadata_dict,
 )
-
-from datetime import datetime
-
-def _import_endee() -> Any:
-    """
-    Try to import endee module. If it's not already installed, instruct user how to install.
-    """
-    try:
-        import endee
-        from endee.endee_client import Endee
-    except ImportError as e:
-        raise ImportError(
-            "Could not import endee python package. "
-            "Please install it with `pip install endee`."
-        ) from e
-    return endee
-
-ID_KEY = "id"
-VECTOR_KEY = "values"
-SPARSE_VECTOR_KEY = "sparse_values"
-METADATA_KEY = "metadata"
-
-DEFAULT_BATCH_SIZE = 100
-
+from .constants import (
+    DEFAULT_BATCH_SIZE,
+    DEFAULT_EF_SEARCH,
+    MAX_DIMENSION_ALLOWED,
+    MAX_EF_SEARCH_ALLOWED,
+    MAX_INDEX_NAME_LENGTH_ALLOWED,
+    MAX_TOP_K_ALLOWED,
+    MAX_VECTORS_PER_BATCH,
+    PRECISION_VALID,
+    REVERSE_OPERATOR_MAP,
+    SPACE_TYPE_MAP,
+    SPACE_TYPES_VALID,
+    SUPPORTED_FILTER_OPERATORS,
+)
+from .utils import get_sparse_encoder
+from endee import Endee
 _logger = logging.getLogger(__name__)
 
-from llama_index.core.vector_stores.types import MetadataFilter, FilterOperator
-
-reverse_operator_map = {
-    FilterOperator.EQ: "$eq",
-    FilterOperator.NE: "$ne",
-    FilterOperator.GT: "$gt",
-    FilterOperator.GTE: "$gte",
-    FilterOperator.LT: "$lt",
-    FilterOperator.LTE: "$lte",
-    FilterOperator.IN: "$in",
-    FilterOperator.NIN: "$nin",
-}
-
-
-
-def build_dict(input_batch: List[List[int]]) -> List[Dict[str, Any]]:
-    """
-    Build a list of sparse dictionaries from a batch of input_ids.
 
-    NOTE: taken from https://www.pinecone.io/learn/hybrid-search-intro/.
 
-    """
-    # store a batch of sparse embeddings
-    sparse_emb = []
-    # iterate through input batch
-    for token_ids in input_batch:
-        indices = []
-        values = []
-        # convert the input_ids list to a dictionary of key to frequency values
-        d = dict(Counter(token_ids))
-        for idx in d:
-            indices.append(idx)
-            values.append(float(d[idx]))
-        sparse_emb.append({"indices": indices, "values": values})
-    # return sparse_emb list
-    return sparse_emb
-
-
-def generate_sparse_vectors(
-    context_batch: List[str], tokenizer: Callable
-) -> List[Dict[str, Any]]:
-    """
-    Generate sparse vectors from a batch of contexts.
+# Supported sparse embedding models
+SUPPORTED_SPARSE_MODELS = {
+    "splade_pp": "prithivida/Splade_PP_en_v1",
+    "splade_cocondenser": "naver/splade-cocondenser-ensembledistil",
+}
 
-    NOTE: taken from https://www.pinecone.io/learn/hybrid-search-intro/.
 
-    """
-    # create batch of input_ids
-    inputs = tokenizer(context_batch)["input_ids"]
-    # create sparse dictionaries
-    return build_dict(inputs)
+# Import sparse encoder utilities from utils module
 
 
 import_err_msg = (
@@ -107,18 +74,20 @@ class EndeeVectorStore(BasePydanticVectorStore):
 
     stores_text: bool = True
     flat_metadata: bool = False
-
     api_token: Optional[str]
     index_name: Optional[str]
     space_type: Optional[str]
     dimension: Optional[int]
-    insert_kwargs: Optional[Dict]
     add_sparse_vector: bool
     text_key: str
     batch_size: int
     remove_text_from_metadata: bool
-
+    hybrid: bool
+    sparse_dim: Optional[int]
+    model_name: Optional[str]
+    precision: Optional[str]
     _endee_index: Any = PrivateAttr()
+    _sparse_encoder: Optional[Callable] = PrivateAttr(default=None)
 
     def __init__(
         self,
@@ -127,31 +96,76 @@ class EndeeVectorStore(BasePydanticVectorStore):
         index_name: Optional[str] = None,
         space_type: Optional[str] = "cosine",
         dimension: Optional[int] = None,
-        insert_kwargs: Optional[Dict] = None,
         add_sparse_vector: bool = False,
         text_key: str = DEFAULT_TEXT_KEY,
         batch_size: int = DEFAULT_BATCH_SIZE,
         remove_text_from_metadata: bool = False,
+        hybrid: bool = False,
+        sparse_dim: Optional[int] = None,
+        model_name: Optional[str] = None,
+        precision: Optional[str] = "float16",
+        M: Optional[int] = None,
+        ef_con: Optional[int] = None,
         **kwargs: Any,
     ) -> None:
-        insert_kwargs = insert_kwargs or {}
-
-        super().__init__(
-            index_name=index_name,
-            api_token=api_token,
-            space_type=space_type,
-            dimension=dimension,
-            insert_kwargs=insert_kwargs,
-            add_sparse_vector=add_sparse_vector,
-            text_key=text_key,
-            batch_size=batch_size,
-            remove_text_from_metadata=remove_text_from_metadata,
-        )
-
-        # Use existing endee_index or initialize a new one
-        self._endee_index = endee_index or self._initialize_endee_index(
-            api_token, index_name, dimension, space_type
-        )
+        try:
+            super().__init__(
+                index_name=index_name,
+                api_token=api_token,
+                space_type=space_type,
+                dimension=dimension,
+                add_sparse_vector=add_sparse_vector,
+                text_key=text_key,
+                batch_size=batch_size,
+                remove_text_from_metadata=remove_text_from_metadata,
+                sparse_dim=sparse_dim,
+                hybrid=hybrid,
+                model_name=model_name,
+                precision=precision,
+            )
+
+            # Initialize index (handles both dense and hybrid)
+            if endee_index is not None:
+                self._endee_index = endee_index
+            else:
+                # sparse_dim=None creates dense index, sparse_dim>0 creates hybrid index
+                self._endee_index = self._initialize_endee_index(
+                    api_token,
+                    index_name,
+                    dimension,
+                    space_type,
+                    precision,
+                    sparse_dim=sparse_dim if hybrid else None,
+                    M=M,
+                    ef_con=ef_con,
+                )
+
+            # Initialize sparse encoder if hybrid mode is enabled
+            if hybrid:
+                # Use default model if none provided
+                if model_name is None:
+                    model_name = "splade_pp"  # Default sparse model
+                    _logger.info(f"Using default sparse model: {model_name}")
+                # If user provided an unsupported model, fall back to one of the supported models
+                elif model_name not in SUPPORTED_SPARSE_MODELS:
+                    _logger.warning(
+                        f"Unsupported sparse model_name {model_name!r} provided. "
+                        "Falling back to default 'splade_pp'. "
+                        f"Supported sparse models: {list(SUPPORTED_SPARSE_MODELS.keys())}"
+                    )
+                    model_name = "splade_pp"
+
+                _logger.info(f"Initializing sparse encoder with model: {model_name}")
+                self._sparse_encoder = get_sparse_encoder(
+                    model_name=model_name,
+                    use_fastembed=True,  # Default to FastEmbed
+                    batch_size=batch_size,
+                )
+            else:
+                self._sparse_encoder = None
+        except Exception as e:
+            _logger.error(f"Error initializing EndeeVectorStore: {e}")
+            raise
 
     @classmethod
     def _initialize_endee_index(
@@ -160,33 +174,139 @@ class EndeeVectorStore(BasePydanticVectorStore):
         index_name: Optional[str],
         dimension: Optional[int] = None,
         space_type: Optional[str] = "cosine",
+        precision: Optional[str] = "float16",
+        sparse_dim: Optional[int] = None,
+        M: Optional[int] = None,
+        ef_con: Optional[int] = None,
     ) -> Any:
-        """Initialize Endee index using the current API."""
-        endee = _import_endee()
-        from endee.endee_client import Endee
-
-        # Initialize Endee client
-        nd = Endee(token=api_token)
+        """
+        Initialize Endee index (dense or hybrid).
 
+        Args:
+            api_token: Endee API token
+            index_name: Name of the index
+            dimension: Dense vector dimension
+            space_type: Distance metric (cosine, l2, ip)
+            precision: Vector precision type
+            sparse_dim: Sparse vector dimension. If None or 0, creates dense-only index.
+                       If > 0, creates hybrid index with both dense and sparse vectors.
+            M: HNSW graph connectivity parameter (optional)
+            ef_con: HNSW construction parameter (optional)
+
+        Returns:
+            Endee Index object
+        """
         try:
-            # Try to get existing index
-            index = nd.get_index(name=index_name)
-            _logger.info(f"Retrieved existing index: {index_name}")
-            return index
+       
+         
+
+            _logger.info("Connecting to Endee service...")
+            nd = Endee(token=api_token)
+            prec = precision if precision is not None else "float16"
+            is_hybrid = sparse_dim is not None and sparse_dim > 0
+            dim_sparse = sparse_dim if is_hybrid else 0
+
+            try:
+                _logger.info(f"Checking if index '{index_name}' exists...")
+                index = nd.get_index(name=index_name)
+                # Check if existing index matches expected type
+                existing_sparse_dim = getattr(index, "sparse_dim", 0)
+                if is_hybrid and existing_sparse_dim > 0:
+                    _logger.info(f"✓ Retrieved existing hybrid index: {index_name}")
+                elif not is_hybrid and existing_sparse_dim == 0:
+                    _logger.info(f"✓ Retrieved existing dense index: {index_name}")
+                elif is_hybrid and existing_sparse_dim == 0:
+                    _logger.warning(
+                        f"Index '{index_name}' exists as dense-only (sparse_dim=0) but hybrid was requested. "
+                        f"Using existing dense index."
+                    )
+                else:
+                    _logger.warning(
+                        f"Index '{index_name}' exists as hybrid (sparse_dim={existing_sparse_dim}) "
+                        f"but dense-only was requested. Using existing hybrid index."
+                    )
+                return index
+
+            except Exception as e:
+                # Index doesn't exist, create new one
+                if dimension is None:
+                    raise ValueError(
+                        f"Must provide dimension when creating a new {'hybrid' if is_hybrid else 'dense'} index"
+                    ) from e
+                if is_hybrid and sparse_dim is None:
+                    raise ValueError(
+                        "Must provide sparse_dim when creating a new hybrid index"
+                    ) from e
+
+                # Validate index name
+                try:
+                    from endee.utils import is_valid_index_name
+
+                    if not is_valid_index_name(index_name):
+                        raise ValueError(
+                            f"Invalid index name. Index name must be alphanumeric and can "
+                            f"contain underscores and should be less than "
+                            f"{MAX_INDEX_NAME_LENGTH_ALLOWED} characters"
+                        )
+                except ImportError:
+                    pass
+
+                # Validate dimension
+                if dimension > MAX_DIMENSION_ALLOWED:
+                    raise ValueError(
+                        f"Dimension cannot be greater than {MAX_DIMENSION_ALLOWED}"
+                    )
+
+                # Validate sparse_dim
+                if dim_sparse < 0:
+                    raise ValueError("sparse_dim cannot be negative")
+
+                # Validate and map space_type
+                space = SPACE_TYPE_MAP.get(
+                    (space_type or "cosine").lower(), (space_type or "cosine").lower()
+                )
+                if space not in SPACE_TYPES_VALID:
+                    raise ValueError(f"Invalid space type: {space}")
+
+                # Validate precision
+                if prec not in PRECISION_VALID:
+                    raise ValueError(
+                        f"Invalid precision: {prec}. Use one of {PRECISION_VALID}"
+                    )
+
+                # Build create_index kwargs
+                create_kwargs = {
+                    "name": index_name,
+                    "dimension": dimension,
+                    "space_type": space,
+                    "precision": prec,
+                    "sparse_dim": dim_sparse,
+                }
+                # Only add M and ef_con if provided
+                if M is not None:
+                    create_kwargs["M"] = M
+                if ef_con is not None:
+                    create_kwargs["ef_con"] = ef_con
+                # Build log message
+                index_type = "hybrid" if is_hybrid else "dense"
+                log_msg = f"Creating new {index_type} index '{index_name}' (dimension={dimension}"
+                if is_hybrid:
+                    log_msg += f", sparse_dim={dim_sparse}"
+                if M is not None:
+                    log_msg += f", M={M}"
+                if ef_con is not None:
+                    log_msg += f", ef_con={ef_con}"
+                log_msg += ")..."
+
+                _logger.info(log_msg)
+             
+                nd.create_index(**create_kwargs)
+                _logger.info("✓ Index created successfully")
+                return nd.get_index(name=index_name)
+
         except Exception as e:
-            if dimension is None:
-                raise ValueError(
-                    "Must provide dimension when creating a new index"
-                ) from e
-            
-            # Create a new index if it doesn't exist
-            _logger.info(f"Creating new index: {index_name}")
-            nd.create_index(
-                name=index_name,
-                dimension=dimension,
-                space_type=space_type,
-            )
-            return nd.get_index(name=index_name)
+            _logger.error(f"Error initializing Endee index: {e}")
+            raise
 
     @classmethod
     def from_params(
@@ -196,24 +316,109 @@ class EndeeVectorStore(BasePydanticVectorStore):
         dimension: Optional[int] = None,
         space_type: str = "cosine",
         batch_size: int = DEFAULT_BATCH_SIZE,
+        hybrid: bool = False,
+        sparse_dim: Optional[int] = None,
+        model_name: Optional[str] = None,
+        precision: Optional[str] = "float16",
+        M: Optional[int] = None,
+        ef_con: Optional[int] = None,
     ) -> "EndeeVectorStore":
-        """Create EndeeVectorStore from parameters."""
-        endee_index = cls._initialize_endee_index(
-            api_token, index_name, dimension, space_type
-        )
-
-        return cls(
-            endee_index=endee_index,
-            api_token=api_token,
-            index_name=index_name,
-            dimension=dimension,
-            space_type=space_type,
-            batch_size=batch_size,
-        )
+        """Create EndeeVectorStore from parameters.
+
+        Args:
+            api_token: API token for Endee service
+            index_name: Name of the index
+            dimension: Vector dimension
+            space_type: Distance metric ("cosine", "l2", or "ip")
+            batch_size: Batch size for operations
+            hybrid: If True, create/use a hybrid index. Auto-set to True if sparse_dim > 0.
+            sparse_dim: Sparse dimension for hybrid index. If > 0, hybrid is automatically enabled.
+            model_name: Model name or alias for sparse embeddings. Defaults to 'splade_pp' if not provided.
+                       Available: 'splade_pp', 'splade_cocondenser', 'bert_base', 'distilbert', etc.
+            precision: Precision for index. Use one of: "binary", "float16", "float32", "int16d", "int8d". Default "float16".
+            M: Optional HNSW M parameter (bi-directional links per node). If not provided, backend uses default.
+            ef_con: Optional HNSW ef_construction parameter. If not provided, backend uses default.
+        """
+        # Auto-enable hybrid if sparse_dim is provided and > 0
+        try:
+            if sparse_dim is not None and sparse_dim > 0:
+                hybrid = True
+                _logger.info(f"Auto-enabling hybrid mode (sparse_dim={sparse_dim} > 0)")
+            if sparse_dim>0:
+                sparse_dim=30522
+
+            # Initialize index (unified method handles both dense and hybrid)
+            endee_index = cls._initialize_endee_index(
+                api_token,
+                index_name,
+                dimension,
+                space_type,
+                precision,
+                sparse_dim=sparse_dim if hybrid else None,
+                M=M,
+                ef_con=ef_con,
+            )
+
+            # Get actual index configuration from the backend
+            try:
+                index_info = endee_index.describe()
+                actual_index_name = index_info.get("name", index_name)
+                actual_dimension = index_info.get("dimension", dimension)
+                actual_space_type = index_info.get("space_type", space_type)
+                actual_precision = index_info.get("precision", precision)
+                actual_sparse_dim = index_info.get("sparse_dim", sparse_dim)
+            except Exception as e:
+                _logger.warning(
+                    f"Could not get index info, using provided parameters: {e}"
+                )
+                # Fallback to provided parameters
+                actual_index_name = index_name
+                actual_dimension = dimension
+                actual_space_type = space_type
+                actual_precision = precision
+                actual_sparse_dim = sparse_dim
+
+            # Determine if index is hybrid based on sparse_dim
+            actual_hybrid = actual_sparse_dim is not None and actual_sparse_dim > 0
+
+            return cls(
+                endee_index=endee_index,
+                api_token=api_token,
+                index_name=actual_index_name,
+                dimension=actual_dimension,
+                space_type=actual_space_type,
+                batch_size=batch_size,
+                sparse_dim=actual_sparse_dim,
+                hybrid=actual_hybrid,
+                model_name=model_name,
+                precision=actual_precision,
+                M=M,
+                ef_con=ef_con,
+            )
+        except Exception as e:
+            _logger.error(f"Error creating EndeeVectorStore from params: {e}")
+            raise
 
     @classmethod
     def class_name(cls) -> str:
-        return "EndeeVectorStore"
+        try:
+            return "EndeeVectorStore"
+        except Exception as e:
+            _logger.error(f"Error getting class name: {e}")
+            raise
+
+    def _compute_sparse_vectors(self, texts: List[str]) -> tuple:
+        """Compute sparse vectors for a list of texts."""
+        try:
+            if self._sparse_encoder is None:
+                raise ValueError(
+                    "Sparse encoder not initialized. "
+                    "Please provide model_name when creating the store with hybrid=True."
+                )
+            return self._sparse_encoder(texts)
+        except Exception as e:
+            _logger.error(f"Error computing sparse vectors: {e}")
+            raise
 
     def add(
         self,
@@ -224,193 +429,336 @@ class EndeeVectorStore(BasePydanticVectorStore):
         Add nodes to index.
 
         Args:
-            nodes: List[BaseNode]: list of nodes with embeddings
+            nodes: List of nodes with embeddings to add to the index.
+                   If index is configured for hybrid search (self.hybrid=True),
+                   sparse vectors will be automatically computed from node text.
         """
-        ids = []
-        entries = []
-        
-        for node in nodes:
-            node_id = node.node_id
-            metadata = node_to_metadata_dict(node)
-            
-            # Filter values must be simple key-value pairs
-            filter_data = {}
-            if "file_name" in metadata:
-                filter_data["file_name"] = metadata["file_name"]
-            if "doc_id" in metadata:
-                filter_data["doc_id"] = metadata["doc_id"]
-            if "category" in metadata:
-                filter_data["category"] = metadata["category"]
-            if "difficulty" in metadata:
-                filter_data["difficulty"] = metadata["difficulty"]
-            if "language" in metadata:
-                filter_data["language"] = metadata["language"]
-            if "field" in metadata:
-                filter_data["field"] = metadata["field"]
-            if "type" in metadata:
-                filter_data["type"] = metadata["type"]
-            if "feature" in metadata:
-                filter_data["feature"] = metadata["feature"]
-
-            
-            entry = {
-                "id": node_id,
-                "vector": node.get_embedding(),
-                "meta": metadata,
-                "filter": filter_data
-            }
-
-            ids.append(node_id)
-            entries.append(entry)
-        
-        # Batch insert to avoid hitting API limits
-        batch_size = self.batch_size
-        for i in range(0, len(entries), batch_size):
-            batch = entries[i : i + batch_size]
-            self._endee_index.upsert(batch)
-        
-        return ids
+        try:
+            # Use instance hybrid setting
+            use_hybrid = self.hybrid
+
+            # Endee Index.upsert rejects duplicate IDs in a batch; dedupe by node_id (keep last)
+            seen: Dict[str, int] = {}
+            for idx, node in enumerate(nodes):
+                seen[node.node_id] = idx
+            deduped_indices = sorted(seen.values())
+            nodes = [nodes[i] for i in deduped_indices]
+
+            ids = []
+            entries = []
+            texts = []
+
+            # Collect texts for sparse encoding if hybrid mode
+            if use_hybrid:
+                for node in nodes:
+                    text = node.get_content()
+                    texts.append(text)
+
+                # Compute sparse vectors in batch
+                if self._sparse_encoder is not None and texts:
+                    sparse_indices, sparse_values = self._compute_sparse_vectors(texts)
+                else:
+                    sparse_indices = [[] for _ in texts]
+                    sparse_values = [[] for _ in texts]
+
+            for i, node in enumerate(nodes):
+                node_id = node.node_id
+                metadata = node_to_metadata_dict(node)
+
+                # Filter values must be simple key-value pairs
+                filter_data = {}
+                ref_id = getattr(node, "ref_doc_id", None) or metadata.get("ref_doc_id")
+                if ref_id is not None:
+                    filter_data["ref_doc_id"] = ref_id
+                if "file_name" in metadata:
+                    filter_data["file_name"] = metadata["file_name"]
+                if "doc_id" in metadata:
+                    filter_data["doc_id"] = metadata["doc_id"]
+                if "category" in metadata:
+                    filter_data["category"] = metadata["category"]
+                if "difficulty" in metadata:
+                    filter_data["difficulty"] = metadata["difficulty"]
+                if "language" in metadata:
+                    filter_data["language"] = metadata["language"]
+                if "field" in metadata:
+                    filter_data["field"] = metadata["field"]
+                if "type" in metadata:
+                    filter_data["type"] = metadata["type"]
+                if "feature" in metadata:
+                    filter_data["feature"] = metadata["feature"]
+
+                # Build entry for endee Index.upsert
+                if use_hybrid:
+                    entry = {
+                        "id": node_id,
+                        "vector": node.get_embedding(),
+                        "sparse_indices": sparse_indices[i],
+                        "sparse_values": sparse_values[i],
+                        "meta": metadata,
+                        "filter": filter_data,
+                    }
+                else:
+                    entry = {
+                        "id": node_id,
+                        "vector": node.get_embedding(),
+                        "meta": metadata,
+                        "filter": filter_data,
+                    }
+
+                ids.append(node_id)
+                entries.append(entry)
+
+            # Batch insert; endee Index.upsert allows max MAX_VECTORS_PER_BATCH per batch
+            batch_size = min(self.batch_size, MAX_VECTORS_PER_BATCH)
+            for i in range(0, len(entries), batch_size):
+                batch = entries[i : i + batch_size]
+                self._endee_index.upsert(batch)
+
+            return ids
+        except Exception as e:
+            _logger.error(f"Error adding nodes to index: {e}")
+            raise
 
     def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None:
         """
-        Delete nodes using with ref_doc_id.
-        
-        Args:
-            ref_doc_id (str): The id of the document to delete.
+        Delete nodes by ref_doc_id using endee Index.delete_with_filter.
+        Only deletes vectors that were stored with ref_doc_id in their filter (see add()).
         """
         try:
-            self._endee_index.delete_with_filter({"doc_id": ref_doc_id})
+            # Filter format consistent with query: list of {field: {$op: value}}
+            filter_dict = [{"ref_doc_id": {"$eq": ref_doc_id}}]
+            self._endee_index.delete_with_filter(filter_dict)
         except Exception as e:
-            _logger.error(f"Error deleting vectors for doc_id {ref_doc_id}: {e}")
+            _logger.error(f"Error deleting by ref_doc_id {ref_doc_id!r}: {e}")
+            raise
 
     @property
     def client(self) -> Any:
         """Return Endee index client."""
-        return self._endee_index
+        try:
+            return self._endee_index
+        except Exception as e:
+            _logger.error(f"Error getting client: {e}")
+            raise
+
+    def describe(self) -> Dict[str, Any]:
+        """Get index metadata (endee Index.describe())."""
+        try:
+            return self._endee_index.describe()
+        except Exception as e:
+            _logger.error(f"Error describing index: {e}")
+            return {}
 
-    def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult:
+    def fetch(self, ids: List[str]) -> List[Dict[str, Any]]:
+        """Fetch vectors by IDs (uses endee Index.get_vector per id)."""
+        out: List[Dict[str, Any]] = []
+        for id_ in ids:
+            try:
+                out.append(self._endee_index.get_vector(id_))
+            except Exception as e:
+                _logger.error(f"Error fetching vector id {id_}: {e}")
+        return out
+
+    def query(
+        self,
+        query: VectorStoreQuery,
+        ef: int = DEFAULT_EF_SEARCH,
+        **kwargs: Any,
+    ) -> VectorStoreQueryResult:
         """
         Query index for top k most similar nodes.
 
         Args:
-            query: VectorStoreQuery object containing query parameters
+            query: VectorStoreQuery object containing query parameters:
+                   - query_embedding: Dense vector for search
+                   - query_str: Text query for sparse search (used if index is hybrid)
+                   - similarity_top_k: Number of results to return
+                   - filters: Optional metadata filters
+                   - alpha: Optional weighting for hybrid search (0=sparse, 1=dense)
+            ef: HNSW ef_search parameter (default 128, max 1024).
+                Controls search quality vs speed tradeoff.
         """
-        if not hasattr(self._endee_index, 'dimension'):
-            # Get dimension from index if available, otherwise try to infer from query
-            try:
-                dimension = self._endee_index.describe()["dimension"]
-            except:
-                if query.query_embedding is not None:
-                    dimension = len(query.query_embedding)
-                else:
-                    raise ValueError("Could not determine vector dimension")
-        else:
-            dimension = self._endee_index.dimension
-        
-        query_embedding = [0.0] * dimension  # Default empty vector
-        filters = {}
-
-        # Apply any metadata filters if provided
-        if query.filters is not None:
-            for filter_item in query.filters.filters:
-                # Case 1: MetadataFilter object
-                if hasattr(filter_item, "key") and hasattr(filter_item, "value") and hasattr(filter_item, "operator"):
-                    op_symbol = reverse_operator_map.get(filter_item.operator)
-                    if not op_symbol:
-                        raise ValueError(f"Unsupported filter operator: {filter_item.operator}")
-                    
-                    if filter_item.key not in filters:
-                        filters[filter_item.key] = {}
-
-                    filters[filter_item.key][op_symbol] = filter_item.value
-
-                # Case 2: Raw dict, e.g. {"category": {"$eq": "programming"}}
-                elif isinstance(filter_item, dict):
-                    for key, op_dict in filter_item.items():
-                        if isinstance(op_dict, dict):
-                            for op, val in op_dict.items():
-                                if key not in filters:
-                                    filters[key] = {}
-                                filters[key][op] = val
-                else:
-                    raise ValueError(f"Unsupported filter format: {filter_item}")
+        # Use index configuration to determine hybrid mode
+        try:
+            use_hybrid = self.hybrid
+
+            # Log the mode being used
+            _logger.info(
+                f"Using {'hybrid' if use_hybrid else 'dense-only'} search (index configured with hybrid={self.hybrid})"
+            )
 
-        _logger.info(f"Final structured filters: {filters}")
+            if not hasattr(self._endee_index, "dimension"):
+                # Get dimension from index if available, otherwise try to infer from query
+                try:
+                    dimension = self._endee_index.describe()["dimension"]
+                except Exception as e:
+                    _logger.warning(f"Could not get dimension from index: {e}")
+                    if query.query_embedding is not None:
+                        dimension = len(query.query_embedding)
+                    else:
+                        raise ValueError("Could not determine vector dimension")
+            else:
+                dimension = self._endee_index.dimension
+
+            query_embedding = [0.0] * dimension  # Default empty vector
+            filters = {}
+            # Apply any metadata filters if provided
+            if query.filters is not None:
+                for filter_item in query.filters.filters:
+                    # Case 1: MetadataFilter object
+                    if (
+                        hasattr(filter_item, "key")
+                        and hasattr(filter_item, "value")
+                        and hasattr(filter_item, "operator")
+                    ):
+                        if filter_item.operator not in SUPPORTED_FILTER_OPERATORS:
+                            raise ValueError(
+                                f"Unsupported filter operator: {filter_item.operator}. "
+                                "Supported filter operations: EQ ($eq), IN ($in)."
+                            )
+                        op_symbol = REVERSE_OPERATOR_MAP[filter_item.operator]
+                        if filter_item.key not in filters:
+                            filters[filter_item.key] = {}
+                        filters[filter_item.key][op_symbol] = filter_item.value
+
+                    # Case 2: Raw dict, e.g. {"category": {"$eq": "programming"}}
+                    elif isinstance(filter_item, dict):
+                        for key, op_dict in filter_item.items():
+                            if isinstance(op_dict, dict):
+                                for op, val in op_dict.items():
+                                    if key not in filters:
+                                        filters[key] = {}
+                                    filters[key][op] = val
+                    else:
+                        raise ValueError(f"Unsupported filter format: {filter_item}")
+
+            _logger.info(f"Final structured filters: {filters}")
+
+            # Endee API expects filter as array: [{"field": {"$op": value}}, ...]
+            filter_for_api: Optional[List[Dict[str, Any]]] = None
+            if filters:
+                filter_for_api = [{field: ops} for field, ops in filters.items()]
+                _logger.info(f"Filter sent to backend API: {filter_for_api}")
+
+            # Use the query embedding if provided
+            if query.query_embedding is not None:
+                query_embedding = cast(List[float], query.query_embedding)
+                if query.alpha is not None and use_hybrid:
+                    # Apply alpha scaling in hybrid mode
+                    query_embedding = [v * query.alpha for v in query_embedding]# Sparse query components for hybrid (endee Index.query uses sparse_indices, sparse_values)
+            sparse_indices_q: Optional[List[int]] = None
+            sparse_values_q: Optional[List[float]] = None
+            if use_hybrid:
+                # Get query text from query.query_str
+                query_text = getattr(query, "query_str", None)
+                if query_text and self._sparse_encoder is not None:
+                    _logger.info(
+                        f"Processing sparse vectors for hybrid search with query_str: '{query_text[:100]}...'"
+                    )
+                    si, sv = self._compute_sparse_vectors([query_text])
+                    sparse_indices_q = si[0]
+                    sparse_values_q = [float(v) for v in sv[0]]
+                    _logger.info(f"Generated {len(sparse_indices_q)} sparse features")
+                elif query_text:
+                    _logger.warning(
+                        "Hybrid mode enabled but no sparse encoder available"
+                    )
+                else:
+                    _logger.warning(
+                        "Hybrid mode enabled but no query_str provided in VectorStoreQuery"
+                    )
+            else:
+                _logger.info("Using dense-only search (not hybrid mode)")
 
-        # Use the query embedding if provided
-        if query.query_embedding is not None:
-            query_embedding = cast(List[float], query.query_embedding)
-            if query.alpha is not None and query.mode == VectorStoreQueryMode.HYBRID:
-                # Apply alpha scaling in hybrid mode
-                query_embedding = [v * query.alpha for v in query_embedding]
+            # Cap to endee limits (MAX_TOP_K_ALLOWED=512, MAX_EF_SEARCH_ALLOWED=1024)
+            requested_top_k = (
+                query.similarity_top_k if query.similarity_top_k is not None else 10
+            )
+            top_k = min(requested_top_k, MAX_TOP_K_ALLOWED)
+            ef_capped = min(ef, MAX_EF_SEARCH_ALLOWED)
+
+            # Build query kwargs - only include optional parameters if they have values
+            query_kwargs = {
+                "vector": query_embedding,
+                "top_k": top_k,
+                "ef": ef_capped,
+                "include_vectors": True,
+            }
 
-        # Execute query
-        try:
-            results = self._endee_index.query(
-                vector=query_embedding,
-                top_k=query.similarity_top_k,
-                filter=filters if filters else None,
-                include_vectors=True
+            # Only add filter if provided
+            if filter_for_api is not None:
+                query_kwargs["filter"] = filter_for_api# Only add sparse vectors if provided (for hybrid search)
+            if sparse_indices_q is not None:
+                query_kwargs["sparse_indices"] = sparse_indices_q
+            if sparse_values_q is not None:
+                query_kwargs["sparse_values"] = sparse_values_q
+            # Use endee Index.query
+            try:
+                results = self._endee_index.query(**query_kwargs)
+            except Exception as e:
+                _logger.error(f"Error querying Endee: {e}")
+                return VectorStoreQueryResult(nodes=[], similarities=[], ids=[])
+
+            # Process results
+            nodes = []
+            similarities = []
+            ids = []
+
+            for result in results:
+                node_id = result["id"]
+                score = result.get("similarity", result.get("score", 0.0))
+                metadata = result.get("meta", {})
+
+                # Create node from metadata
+                if self.flat_metadata:
+                    node = metadata_dict_to_node(
+                        metadata=metadata,
+                        text=metadata.pop(self.text_key, None),
+                        id_=node_id,
+                    )
+                else:
+                    metadata_dict, node_info, relationships = (
+                        legacy_metadata_dict_to_node(
+                            metadata=metadata,
+                            text_key=self.text_key,
+                        )
+                    )
+
+                    # Create TextNode with the extracted metadata
+                    # Step 1: Get the JSON string from "_node_content"
+                    _node_content_str = metadata.get("_node_content", "{}")
+
+                    # Step 2: Convert JSON string to Python dict
+                    try:
+                        node_content = json.loads(_node_content_str)
+                    except json.JSONDecodeError:
+                        node_content = {}
+
+                    # Step 3: Get the text
+                    text = node_content.get(self.text_key, "")
+                    node = TextNode(
+                        text=text,
+                        metadata=metadata_dict,
+                        relationships=relationships,
+                        node_id=node_id,
+                    )
+
+                    # Add any node_info properties to the node
+                    for key, val in node_info.items():
+                        if hasattr(node, key):
+                            setattr(node, key, val)
+
+                # If embedding was returned in the results, add it to the node
+                if "vector" in result:
+                    node.embedding = result["vector"]
+
+                nodes.append(node)
+                similarities.append(score)
+                ids.append(node_id)
+
+            return VectorStoreQueryResult(
+                nodes=nodes, similarities=similarities, ids=ids
             )
         except Exception as e:
-            _logger.error(f"Error querying Endee: {e}")
-            return VectorStoreQueryResult(nodes=[], similarities=[], ids=[])
-
-        # Process results
-        nodes = []
-        similarities = []
-        ids = []
-
-        for result in results:
-            node_id = result["id"]
-            score = result["similarity"]
-            
-            # Get metadata from result
-            metadata = result.get("meta", {})
-            
-            # Create node from metadata
-            if self.flat_metadata:
-                node = metadata_dict_to_node(
-                    metadata=metadata,
-                    text=metadata.pop(self.text_key, None),
-                    id_=node_id,
-                )
-            else:
-                metadata_dict, node_info, relationships = legacy_metadata_dict_to_node(
-                    metadata=metadata,
-                    text_key=self.text_key,
-                )
-                
-                # Create TextNode with the extracted metadata
-                # Step 1: Get the JSON string from "_node_content"
-                _node_content_str = metadata.get("_node_content", "{}")
-
-                # Step 2: Convert JSON string to Python dict
-                try:
-                    node_content = json.loads(_node_content_str)
-                except json.JSONDecodeError:
-                    node_content = {}
-
-                # Step 3: Get the text
-                text = node_content.get(self.text_key, "")
-                node = TextNode(
-                    text=text,
-                    metadata=metadata_dict,
-                    relationships=relationships,
-                    node_id=node_id,
-                )
-                
-                # Add any node_info properties to the node
-                for key, val in node_info.items():
-                    if hasattr(node, key):
-                        setattr(node, key, val)
-            
-            # If embedding was returned in the results, add it to the node
-            if "vector" in result:
-                node.embedding = result["vector"]
-            
-            nodes.append(node)
-            similarities.append(score)
-            ids.append(node_id)
-
-        return VectorStoreQueryResult(nodes=nodes, similarities=similarities, ids=ids)
+            _logger.error(f"Error querying index: {e}")
+            raise