qdrant-hybrid-pipeline 0.1.1__tar.gz

qdrant_hybrid_pipeline-0.1.1/.gitignore

@@ -0,0 +1,174 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc

qdrant_hybrid_pipeline-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Data Parthenon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

qdrant_hybrid_pipeline-0.1.1/PKG-INFO

@@ -0,0 +1,220 @@
+Metadata-Version: 2.4
+Name: qdrant-hybrid-pipeline
+Version: 0.1.1
+Summary: Configurable Hybrid Search Pipeline with Qdrant and FastEmbed
+Author-email: Brian O'Grady <genesysdatallc@gmail.com>
+Maintainer-email: Brian O'Grady <genesysdatallc@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 Data Parthenon
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: colbert,database,embeddings,late-interaction,qdrant,similarity-search,vector
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.11
+Requires-Dist: fastembed>=0.6.1
+Requires-Dist: pydantic>=2.11.3
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: qdrant-client>=1.13.3
+Requires-Dist: sentence-transformers>=4.1.0
+Provides-Extra: dev
+Requires-Dist: build>=1.0.3; extra == 'dev'
+Requires-Dist: mypy>=1.11.0; extra == 'dev'
+Requires-Dist: pre-commit>=3.7.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
+Requires-Dist: pytest-mock>=3.14.0; extra == 'dev'
+Requires-Dist: pytest-xdist>=3.6.0; extra == 'dev'
+Requires-Dist: pytest>=8.2.0; extra == 'dev'
+Requires-Dist: ruff<0.10,>=0.9.7; extra == 'dev'
+Requires-Dist: twine>=4.0.2; extra == 'dev'
+Provides-Extra: test
+Requires-Dist: fastembed-hybrid-pipeline[dev]; extra == 'test'
+Description-Content-Type: text/markdown
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python Version](https://img.shields.io/badge/python-3.11%2B-blue)](https://www.python.org/downloads/)
+[![PyPI version](https://badge.fury.io/py/fastembed-hybrid-pipeline.svg)](https://pypi.org/project/fastembed-hybrid-pipeline/)
+
+# FastEmbed Hybrid Pipeline
+
+A configurable hybrid search pipeline for building semantic search applications with [FastEmbed](https://github.com/qdrant/fastembed) and [Qdrant](https://github.com/qdrant/qdrant).
+
+## Features
+
+- 🚀 **Hybrid Search**: Combines dense embeddings, sparse embeddings, and late interaction embeddings for superior search performance
+- 🔧 **Configurable**: Customize embedding models, vector parameters, and multi-tenancy settings
+- 🔄 **Batch Processing**: Efficiently process and index large document collections
+- 🏢 **Multi-Tenant Support**: Optional partition-based multi-tenancy for SaaS applications
+
+## Installation
+
+```bash
+pip install fastembed-hybrid-pipeline
+```
+
+*Requires Python 3.11+*
+
+## Quick Start
+
+```python
+from qdrant_client import QdrantClient
+from fastembed import TextEmbedding, SparseEmbedding, LateInteractionTextEmbedding
+from qdrant_client.models import Distance, VectorParams, SparseVectorParams, KeywordIndexParams
+from hybrid_search import HybridPipelineConfig, HybridPipeline
+import uuid
+
+# Initialize Qdrant client
+client = QdrantClient(":memory:")  # Use a local instance or Qdrant Cloud
+
+# Configure embedding models
+text_model = TextEmbedding("BAAI/bge-small-en-v1.5")
+sparse_model = SparseEmbedding("sentence-transformers/all-MiniLM-L6-v2")
+late_interaction_model = LateInteractionTextEmbedding("intfloat/e5-small-v2")
+
+# Configure vector parameters
+dense_params = VectorParams(size=text_model.dimensions, distance=Distance.COSINE)
+sparse_params = SparseVectorParams()
+late_interaction_params = VectorParams(size=late_interaction_model.dimensions, distance=Distance.COSINE)
+
+# Optional: Configure multi-tenancy
+partition_field = "tenant_id"
+partition_index = KeywordIndexParams(minWordLength=1, maxWordLength=100)
+partition_config = (partition_field, partition_index)
+
+# Create pipeline configuration
+pipeline_config = HybridPipelineConfig(
+    text_embedding_config=(text_model, dense_params),
+    sparse_embedding_config=(sparse_model, sparse_params),
+    late_interaction_text_embedding_config=(late_interaction_model, late_interaction_params),
+    partition_config=partition_config,  # Optional, for multi-tenant setup
+    multi_tenant=True,                 # Set to False for single-tenant setup
+    replication_factor=1,              # For production, use 2+
+    shard_number=1,                    # For production, use 3+
+)
+
+# Initialize the pipeline
+pipeline = HybridPipeline(
+    qdrant_client=client,
+    collection_name="documents",
+    hybrid_pipeline_config=pipeline_config,
+)
+
+# Index documents
+documents = [
+    "FastEmbed is a lightweight Python library for state-of-the-art text embeddings.",
+    "Qdrant is a vector database for production-ready vector search.",
+    "Hybrid search combines multiple search techniques for better results."
+]
+
+payloads = [
+    {"tenant_id": "acme_corp", "document_type": "library"},
+    {"tenant_id": "acme_corp", "document_type": "database"},
+    {"tenant_id": "acme_corp", "document_type": "technique"}
+]
+
+document_ids = [uuid.uuid4() for _ in range(len(documents))]
+
+# Insert documents
+pipeline.insert_documents(documents, payloads, document_ids)
+
+# Search
+results = pipeline.search(
+    query="Which embedding library should I use?", 
+    top_k=3,
+    partition_filter="acme_corp",  # Only needed for multi-tenant setups
+)
+
+# Process results
+for result in results:
+    print(f"Score: {result.score}")
+    print(f"Document: {result.payload['document']}")
+    print("-" * 30)
+```
+
+## Configuration Options
+
+### Embedding Models
+
+The pipeline requires three types of embedding models from FastEmbed:
+
+1. **Dense Embeddings**: Traditional vector embeddings (TextEmbedding)
+2. **Sparse Embeddings**: Lexical-focused sparse embeddings (SparseEmbedding)  
+3. **Late Interaction**: Special embeddings for late interaction matching (LateInteractionTextEmbedding)
+
+### Vector Parameters
+
+Configure vector parameters for each embedding type:
+
+- **Dense & Late Interaction**: Size, distance metric (cosine, dot, euclidean)
+- **Sparse**: Uses default sparse vector parameters
+
+### Multi-Tenant Configuration
+
+For SaaS applications that need to separate data by tenant:
+
+```python
+# Enable multi-tenancy
+pipeline_config = HybridPipelineConfig(
+    # ... other configs ...
+    partition_config=("tenant_id", KeywordIndexParams(minWordLength=1, maxWordLength=100)),
+    multi_tenant=True,
+)
+
+# When searching, specify the tenant
+results = pipeline.search(query="my search", partition_filter="tenant_123")
+```
+
+### Performance Options
+
+For production deployments:
+
+```python
+pipeline_config = HybridPipelineConfig(
+    # ... other configs ...
+    replication_factor=2,  # Data redundancy for high availability
+    shard_number=3,        # Data distribution for scalability
+)
+```
+
+## Development
+
+```bash
+# Clone the repository
+git clone https://github.com/your-username/fastembed-hybrid-pipeline.git
+cd fastembed-hybrid-pipeline
+
+# Install development dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+```
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

qdrant_hybrid_pipeline-0.1.1/README.md

@@ -0,0 +1,162 @@
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python Version](https://img.shields.io/badge/python-3.11%2B-blue)](https://www.python.org/downloads/)
+[![PyPI version](https://badge.fury.io/py/fastembed-hybrid-pipeline.svg)](https://pypi.org/project/fastembed-hybrid-pipeline/)
+
+# FastEmbed Hybrid Pipeline
+
+A configurable hybrid search pipeline for building semantic search applications with [FastEmbed](https://github.com/qdrant/fastembed) and [Qdrant](https://github.com/qdrant/qdrant).
+
+## Features
+
+- 🚀 **Hybrid Search**: Combines dense embeddings, sparse embeddings, and late interaction embeddings for superior search performance
+- 🔧 **Configurable**: Customize embedding models, vector parameters, and multi-tenancy settings
+- 🔄 **Batch Processing**: Efficiently process and index large document collections
+- 🏢 **Multi-Tenant Support**: Optional partition-based multi-tenancy for SaaS applications
+
+## Installation
+
+```bash
+pip install fastembed-hybrid-pipeline
+```
+
+*Requires Python 3.11+*
+
+## Quick Start
+
+```python
+from qdrant_client import QdrantClient
+from fastembed import TextEmbedding, SparseEmbedding, LateInteractionTextEmbedding
+from qdrant_client.models import Distance, VectorParams, SparseVectorParams, KeywordIndexParams
+from hybrid_search import HybridPipelineConfig, HybridPipeline
+import uuid
+
+# Initialize Qdrant client
+client = QdrantClient(":memory:")  # Use a local instance or Qdrant Cloud
+
+# Configure embedding models
+text_model = TextEmbedding("BAAI/bge-small-en-v1.5")
+sparse_model = SparseEmbedding("sentence-transformers/all-MiniLM-L6-v2")
+late_interaction_model = LateInteractionTextEmbedding("intfloat/e5-small-v2")
+
+# Configure vector parameters
+dense_params = VectorParams(size=text_model.dimensions, distance=Distance.COSINE)
+sparse_params = SparseVectorParams()
+late_interaction_params = VectorParams(size=late_interaction_model.dimensions, distance=Distance.COSINE)
+
+# Optional: Configure multi-tenancy
+partition_field = "tenant_id"
+partition_index = KeywordIndexParams(minWordLength=1, maxWordLength=100)
+partition_config = (partition_field, partition_index)
+
+# Create pipeline configuration
+pipeline_config = HybridPipelineConfig(
+    text_embedding_config=(text_model, dense_params),
+    sparse_embedding_config=(sparse_model, sparse_params),
+    late_interaction_text_embedding_config=(late_interaction_model, late_interaction_params),
+    partition_config=partition_config,  # Optional, for multi-tenant setup
+    multi_tenant=True,                 # Set to False for single-tenant setup
+    replication_factor=1,              # For production, use 2+
+    shard_number=1,                    # For production, use 3+
+)
+
+# Initialize the pipeline
+pipeline = HybridPipeline(
+    qdrant_client=client,
+    collection_name="documents",
+    hybrid_pipeline_config=pipeline_config,
+)
+
+# Index documents
+documents = [
+    "FastEmbed is a lightweight Python library for state-of-the-art text embeddings.",
+    "Qdrant is a vector database for production-ready vector search.",
+    "Hybrid search combines multiple search techniques for better results."
+]
+
+payloads = [
+    {"tenant_id": "acme_corp", "document_type": "library"},
+    {"tenant_id": "acme_corp", "document_type": "database"},
+    {"tenant_id": "acme_corp", "document_type": "technique"}
+]
+
+document_ids = [uuid.uuid4() for _ in range(len(documents))]
+
+# Insert documents
+pipeline.insert_documents(documents, payloads, document_ids)
+
+# Search
+results = pipeline.search(
+    query="Which embedding library should I use?", 
+    top_k=3,
+    partition_filter="acme_corp",  # Only needed for multi-tenant setups
+)
+
+# Process results
+for result in results:
+    print(f"Score: {result.score}")
+    print(f"Document: {result.payload['document']}")
+    print("-" * 30)
+```
+
+## Configuration Options
+
+### Embedding Models
+
+The pipeline requires three types of embedding models from FastEmbed:
+
+1. **Dense Embeddings**: Traditional vector embeddings (TextEmbedding)
+2. **Sparse Embeddings**: Lexical-focused sparse embeddings (SparseEmbedding)  
+3. **Late Interaction**: Special embeddings for late interaction matching (LateInteractionTextEmbedding)
+
+### Vector Parameters
+
+Configure vector parameters for each embedding type:
+
+- **Dense & Late Interaction**: Size, distance metric (cosine, dot, euclidean)
+- **Sparse**: Uses default sparse vector parameters
+
+### Multi-Tenant Configuration
+
+For SaaS applications that need to separate data by tenant:
+
+```python
+# Enable multi-tenancy
+pipeline_config = HybridPipelineConfig(
+    # ... other configs ...
+    partition_config=("tenant_id", KeywordIndexParams(minWordLength=1, maxWordLength=100)),
+    multi_tenant=True,
+)
+
+# When searching, specify the tenant
+results = pipeline.search(query="my search", partition_filter="tenant_123")
+```
+
+### Performance Options
+
+For production deployments:
+
+```python
+pipeline_config = HybridPipelineConfig(
+    # ... other configs ...
+    replication_factor=2,  # Data redundancy for high availability
+    shard_number=3,        # Data distribution for scalability
+)
+```
+
+## Development
+
+```bash
+# Clone the repository
+git clone https://github.com/your-username/fastembed-hybrid-pipeline.git
+cd fastembed-hybrid-pipeline
+
+# Install development dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+```
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

qdrant_hybrid_pipeline-0.1.1/pyproject.toml

@@ -0,0 +1,59 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "qdrant-hybrid-pipeline"
+version = "0.1.1"
+description = "Configurable Hybrid Search Pipeline with Qdrant and FastEmbed"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { file = "LICENSE" }
+authors = [
+    { name = "Brian O'Grady", email = "genesysdatallc@gmail.com" }
+]
+maintainers = [
+    { name = "Brian O'Grady", email = "genesysdatallc@gmail.com" }
+]
+keywords = ["qdrant", "vector", "database", "embeddings", "similarity-search", "colbert", "late-interaction"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Database",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = [
+    "fastembed>=0.6.1",
+    "pydantic>=2.11.3",
+    "python-dotenv>=1.1.0",
+    "qdrant-client>=1.13.3",
+    "sentence-transformers>=4.1.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.2.0",
+    "pytest-cov>=5.0.0",
+    "pytest-asyncio>=0.23.0",
+    "pytest-mock>=3.14.0",
+    "pytest-xdist>=3.6.0",
+    "mypy>=1.11.0",
+    "ruff>=0.9.7,<0.10",
+    "pre-commit>=3.7.0",
+    "build>=1.0.3",
+    "twine>=4.0.2",
+]
+
+test = ["fastembed-hybrid-pipeline[dev]"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/hybrid_search/"]
+
+[tool.hatch.build]
+include = [
+  "src/hybrid_search/**/*.py",
+]

qdrant_hybrid_pipeline-0.1.1/src/hybrid_search/__init__.py

@@ -0,0 +1,15 @@
+"""
+Hybrid Search module for vector search combining dense, sparse, and late interaction embeddings.
+
+This module provides components for creating and managing hybrid search pipelines
+that leverage multiple embedding types for improved search performance.
+"""
+
+from .hybrid_pipeline import HybridPipeline
+from .hybrid_pipeline_config import HybridPipelineConfig, SentenceTransformerEmbedding
+
+__all__ = [
+    "HybridPipeline",
+    "HybridPipelineConfig",
+    "SentenceTransformerEmbedding",
+]

qdrant_hybrid_pipeline-0.1.1/src/hybrid_search/hybrid_pipeline.py

@@ -0,0 +1,372 @@
+"""
+Hybrid Pipeline module for vector search combining dense, sparse, and late interaction embeddings.
+
+This module provides the implementation of the hybrid search pipeline that leverages multiple
+embedding types for improved search performance.
+"""
+
+import uuid
+from typing import Any, Dict, List, Optional, Union
+
+from fastembed import TextEmbedding
+from qdrant_client import QdrantClient
+from qdrant_client.conversions import common_types as types
+from qdrant_client.models import (
+    Filter,
+    FieldCondition,
+    PointStruct,
+    Prefetch,
+    QuantizationSearchParams,
+    SearchParams,
+    MatchValue,
+)
+
+from .hybrid_pipeline_config import HybridPipelineConfig, SentenceTransformerEmbedding
+
+
+class HybridPipeline:
+    """
+    Pipeline for hybrid search using multiple embedding types.
+    
+    This class implements a hybrid search pipeline that combines dense embeddings,
+    sparse embeddings, and late interaction embeddings for improved search performance.
+    It handles the creation and management of a Qdrant collection with the specified
+    configuration, as well as document insertion and search operations.
+    
+    The hybrid approach combines the strengths of different embedding types:
+    - Dense embeddings: Good for semantic similarity
+    - Sparse embeddings: Good for keyword matching
+    - Late interaction embeddings: Good for retrieval with detailed token-level interactions
+    
+    Attributes:
+        collection_name: Name of the Qdrant collection
+        qdrant_client: Client for interacting with the Qdrant vector database
+        config: Configuration for the hybrid pipeline
+        vectors_config_dict: Dictionary of vector configurations
+        sparse_vectors_config_dict: Dictionary of sparse vector configurations
+        multi_tenant: Flag indicating if the pipeline supports multiple tenants
+        replication_factor: Number of replicas for each shard
+        shard_number: Number of shards for the collection
+        partition_field_name: Field name used for partitioning in multi-tenant mode
+        partition_index_params: Index parameters for the partition field
+    """
+
+    def __init__(
+        self,
+        qdrant_client: QdrantClient,
+        collection_name: str,
+        hybrid_pipeline_config: HybridPipelineConfig,
+    ):
+        """
+        Initialize a new HybridPipeline instance.
+        
+        Args:
+            qdrant_client: Client for interacting with the Qdrant vector database
+            collection_name: Name of the Qdrant collection to create
+            hybrid_pipeline_config: Configuration for the hybrid pipeline
+            
+        Raises:
+            ValueError: If the collection already exists
+        """
+        self.collection_name = collection_name
+        self.qdrant_client = qdrant_client
+        self.config = hybrid_pipeline_config
+        self.vectors_config_dict = self.config.get_vectors_config_dict()
+        self.sparse_vectors_config_dict = self.config.get_sparse_vectors_config_dict()
+        self.multi_tenant = self.config.multi_tenant
+        self.replication_factor = self.config.replication_factor
+        self.shard_number = self.config.shard_number
+        self.partition_field_name, self.partition_index_params = self.config.get_partition_config()
+        self._create_collection()
+
+        if self.multi_tenant:
+            self._create_payload_index()
+        
+    def _create_collection(self) -> bool:
+        """
+        Create a new Qdrant collection with the configured parameters.
+        
+        Returns:
+            bool: True if the collection was created successfully
+            
+        Raises:
+            ValueError: If the collection already exists
+        """
+        if self.qdrant_client.collection_exists(self.collection_name):
+            raise ValueError(
+                f"Collection {self.collection_name} already exists"
+            )
+
+        return self.qdrant_client.create_collection(
+            collection_name=self.collection_name,
+            vectors_config=self.vectors_config_dict,
+            sparse_vectors_config=self.sparse_vectors_config_dict,
+            replication_factor=self.replication_factor,
+            shard_number=self.shard_number,
+        )
+
+    def _create_payload_index(self) -> types.UpdateResult:
+        """
+        Create a payload index for the partition field in multi-tenant mode.
+        
+        Returns:
+            types.UpdateResult: Result of the index creation operation
+        """
+        return self.qdrant_client.create_payload_index(
+            collection_name=self.collection_name,
+            field_name=self.partition_field_name,
+            field_schema=self.partition_index_params,
+        )
+    
+    def _embed_documents(self, documents: Union[str, List[str]]) -> Dict[str, List[float]]:
+        """
+        Embed documents using all configured embedding models.
+        
+        Args:
+            documents: A single document string or a list of document strings to embed
+            
+        Returns:
+            Dict[str, List[float]]: Dictionary mapping model names to lists of embeddings
+        """
+        if isinstance(documents, str):
+            documents = [documents]
+        
+        if isinstance(self.config.dense_model, SentenceTransformerEmbedding):
+            dense_embeddings = self.config.dense_model.embed(documents)
+        else:
+            dense_embeddings = [emb.tolist() for emb in list(self.config.dense_model.embed(documents))]
+        
+        sparse_embeddings = [
+            types.SparseVector(
+                indices=emb.indices.tolist(),
+                values=emb.values.tolist()
+            ) for emb in list(self.config.sparse_model.embed(documents))
+        ]
+
+
+        late_interaction_embeddings = list(self.config.late_interaction_model.embed(documents))
+        late_interaction_embeddings = [emb.tolist() for emb in late_interaction_embeddings]
+
+        return {
+            self.config.DENSE_VECTOR_NAME: dense_embeddings,
+            self.config.SPARSE_VECTOR_NAME: sparse_embeddings,
+            self.config.LATE_INTERACTION_VECTOR_NAME: late_interaction_embeddings,
+        }
+
+    def _prepare_documents(
+        self,
+        documents: List[str],
+        payloads: List[Dict[str, Any]],
+        document_ids: List[uuid.UUID],
+    ) -> List[types.PointStruct]:
+        """
+        Prepare documents for insertion into the Qdrant collection.
+        
+        This method embeds the documents using the configured embedding models and
+        creates PointStruct objects that can be inserted into the Qdrant collection.
+        
+        Args:
+            documents: List of document strings to embed and insert
+            payloads: List of payload dictionaries containing metadata for each document
+            document_ids: List of UUIDs to use as IDs for each document
+            
+        Returns:
+            List[types.PointStruct]: List of prepared points ready for insertion
+            
+        Raises:
+            ValueError: If the lengths of documents, payloads, and document_ids don't match,
+                or if multi_tenant is True and a payload is missing the partition field
+        """
+        
+        if not (len(documents) == len(payloads) == len(document_ids)):
+            raise ValueError(
+                "documents, payloads, and document_ids must be the same length"
+            )
+
+        embeddings_dict = self._embed_documents(documents)
+
+        points = []
+        for i in range(len(documents)):
+            if self.multi_tenant and self.partition_field_name not in payloads[i]:
+                raise ValueError(
+                    f"payloads must contain {self.partition_field_name} if multi_tenant is True"
+                )
+            document_id = str(document_ids[i])
+            payloads[i]["document"] = documents[i]
+            payloads[i]["document_id"] = document_id
+            point = PointStruct(
+                id=document_id,
+                vector={
+                    vector_name: embeddings_dict[vector_name][i] for vector_name in embeddings_dict
+                },
+                payload=payloads[i],
+            )
+            points.append(point)
+
+        return points
+
+    def insert_documents(
+        self,
+        documents: List[str],
+        payloads: List[Dict[str, Any]],
+        document_ids: List[uuid.UUID],
+        batch_size: int = 100,
+    ):
+        """
+        Insert documents into the Qdrant collection.
+        
+        This method embeds the documents using the configured embedding models and
+        inserts them into the Qdrant collection in batches.
+        
+        Args:
+            documents: List of document strings to embed and insert
+            payloads: List of payload dictionaries containing metadata for each document
+            document_ids: List of UUIDs to use as IDs for each document
+            batch_size: Number of documents to process in each batch (default: 100)
+            
+        Raises:
+            ValueError: If the lengths of documents, payloads, and document_ids don't match,
+                or if multi_tenant is True and a payload is missing the partition field
+        """
+        if not (len(documents) == len(payloads) == len(document_ids)):
+            raise ValueError(
+                "documents, payloads, and document_ids must be the same length"
+            )
+        
+        for i in range(0, len(documents), batch_size):
+            points = self._prepare_documents(
+                documents=documents[i:i+batch_size],
+                payloads=payloads[i:i+batch_size],
+                document_ids=document_ids[i:i+batch_size]
+            )
+            self.qdrant_client.upsert(
+                collection_name=self.collection_name,
+                points=points,
+            )
+
+    def _embed_query(self, query: str) -> Dict[str, List[float]]:
+        """
+        Embed a query string using all configured embedding models.
+        
+        Args:
+            query: Query string to embed
+            
+        Returns:
+            Dict[str, List[float]]: Dictionary mapping model names to query embeddings
+        """
+        if isinstance(self.config.dense_model, SentenceTransformerEmbedding):
+            dense_embeddings = self.config.dense_model.embed([query])[0]
+        else:
+            dense_embeddings = list(self.config.dense_model.embed([query]))[0].tolist()
+        sparse_embeddings = list(self.config.sparse_model.embed([query]))[0]
+        sparse_embeddings = types.SparseVector(
+            indices=sparse_embeddings.indices.tolist(),
+            values=sparse_embeddings.values.tolist()
+        )
+
+        late_interaction_embeddings = [
+            emb.tolist() for emb in
+            list(self.config.late_interaction_model.embed([query]))[0]
+        ]
+
+        return {
+            self.config.DENSE_VECTOR_NAME: dense_embeddings,
+            self.config.SPARSE_VECTOR_NAME: sparse_embeddings,
+            self.config.LATE_INTERACTION_VECTOR_NAME: late_interaction_embeddings,
+        }
+    
+    def search(
+        self,
+        query: str,
+        top_k: int = 10,
+        partition_filter: Optional[str] = None,
+        overquery_factor: float = 1.0,
+    ) -> List[types.ScoredPoint]:
+        """
+        Search for documents similar to the query using the hybrid approach.
+        
+        This method implements a hybrid search that combines dense embeddings,
+        sparse embeddings, and late interaction embeddings to retrieve the most
+        relevant documents for the query.
+        
+        Args:
+            query: Query string to search for
+            top_k: Number of results to return (default: 10)
+            partition_filter: Value to filter by in the partition field for multi-tenant mode
+                (must be None if multi_tenant is False)
+            overquery_factor: Factor to oversample results during quantization (default: 1.0,
+                must be >= 1.0)
+                
+        Returns:
+            types.QueryResponse: Query response containing the search results
+            
+        Raises:
+            ValueError: If overquery_factor is less than 1.0 or if partition_filter is
+                provided when multi_tenant is False
+        """
+        if overquery_factor < 1.0:
+            raise ValueError("overquery_factor must be greater than or equal to 1.0")
+        
+        filter_condition = None
+        if not self.multi_tenant and partition_filter:
+            raise ValueError("partition_filter must be None if multi_tenant is False")
+
+        filter_condition = Filter(
+            must=[
+                FieldCondition(
+                    key=self.partition_field_name,
+                    match=MatchValue(value=partition_filter)
+                )
+            ]
+        )
+        
+        query_embeddings = self._embed_query(query)
+
+        dense_prefetch = Prefetch(
+            query=query_embeddings[self.config.DENSE_VECTOR_NAME],
+            using=self.config.DENSE_VECTOR_NAME,
+            limit=top_k,
+            filter=filter_condition,
+            params=SearchParams(
+                quantization=QuantizationSearchParams(
+                    ignore=False,
+                    rescore=True,
+                    oversampling=overquery_factor,
+                ),
+            ),
+        )
+
+        sparse_prefetch = Prefetch(
+            query=query_embeddings[self.config.SPARSE_VECTOR_NAME],
+            using=self.config.SPARSE_VECTOR_NAME,
+            limit=top_k,
+            filter=filter_condition,
+        )
+
+        return self.qdrant_client.query_points(
+            collection_name=self.collection_name,
+            prefetch=[
+                dense_prefetch,
+                sparse_prefetch,
+            ],
+            query=query_embeddings[self.config.LATE_INTERACTION_VECTOR_NAME],
+            using=self.config.LATE_INTERACTION_VECTOR_NAME,
+            limit=top_k,
+            with_payload=True,
+        ).points
+    
+    def delete_document(self, document_id: str):
+        """
+        Delete a document from the collection by its ID.
+        
+        Args:
+            document_id: ID of the document to delete
+            
+        Note:
+            This method is currently not implemented.
+        
+        TODO: Implement delete document functionality
+        """
+        #TODO: Implement delete document
+        pass
+    

qdrant_hybrid_pipeline-0.1.1/src/hybrid_search/hybrid_pipeline_config.py

@@ -0,0 +1,261 @@
+"""
+Configuration module for the hybrid search pipeline.
+
+This module provides configuration classes and type definitions for setting up
+hybrid search pipelines that combine dense, sparse, and late interaction embeddings.
+"""
+
+from typing import ClassVar, List, Mapping, Optional, Tuple, TypeVar, Union
+
+from pydantic import BaseModel, model_validator
+from fastembed.late_interaction import LateInteractionTextEmbedding
+from fastembed.sparse import SparseTextEmbedding
+from fastembed.text import TextEmbedding
+from qdrant_client.conversions import common_types as types
+from qdrant_client.models import KeywordIndexParams
+from sentence_transformers import SentenceTransformer
+
+
+class SentenceTransformerEmbedding(SentenceTransformer):
+    """
+    A wrapper around the SentenceTransformer class that adds a model_name attribute.
+    """
+    def __init__(self, model_name_or_path: str, *args, **kwargs):
+        self._model_name_or_path = model_name_or_path
+        super().__init__(model_name_or_path, *args, **kwargs)
+    
+    @property
+    def model_name(self) -> str:
+        return self._model_name_or_path
+    
+    def embed(self, texts: List[str], **kwargs) -> List[List[float]]:
+        return self.encode(texts, **kwargs).tolist()
+
+
+Embedding = TypeVar(
+    "Embedding",
+    TextEmbedding,
+    LateInteractionTextEmbedding,
+    SparseTextEmbedding,
+    SentenceTransformerEmbedding,
+)
+"""Type variable for the different types of embedding models supported."""
+
+
+BaseVectorParams = TypeVar(
+    "BaseVectorParams",
+    types.VectorParams,
+    types.SparseVectorParams
+)
+"""Type variable for the different types of vector parameters supported."""
+
+
+class HybridPipelineConfig(BaseModel):
+    """
+    Configuration for a hybrid search pipeline combining multiple embedding types.
+    
+    This class encapsulates the configuration for a hybrid search pipeline that combines
+    dense embeddings, sparse embeddings, and late interaction embeddings for improved
+    search performance. It also includes configuration for multi-tenancy and sharding.
+    
+    Attributes:
+        text_embedding_config: Configuration for the dense text embedding model.
+            A tuple containing a TextEmbedding model instance and its associated VectorParams.
+        sparse_embedding_config: Configuration for the sparse embedding model.
+            A tuple containing a SparseTextEmbedding model instance and its associated SparseVectorParams.
+        late_interaction_text_embedding_config: Configuration for the late interaction embedding model.
+            A tuple containing a LateInteractionTextEmbedding model instance and its associated VectorParams.
+        partition_config: Configuration for multi-tenant partitioning.
+            A tuple containing the field name to use for partitioning and the KeywordIndexParams 
+            for the partition field. Required if multi_tenant is True.
+        multi_tenant: Flag indicating whether the pipeline should support multiple tenants.
+            If True, the pipeline will create a partitioned collection using the partition_config.
+            Default is False.
+        replication_factor: The number of replicas for each shard in the Qdrant collection.
+            Increases redundancy and read performance. Default is 2.
+        shard_number: The number of shards for the Qdrant collection.
+            Affects write performance and horizontal scalability. Default is 3.
+    """
+    DENSE_VECTOR_NAME: ClassVar[str] = "dense"
+    SPARSE_VECTOR_NAME: ClassVar[str] = "sparse"
+    LATE_INTERACTION_VECTOR_NAME: ClassVar[str] = "multivector"
+
+    text_embedding_config: Tuple[Union[TextEmbedding, SentenceTransformerEmbedding], types.VectorParams]
+    sparse_embedding_config: Tuple[SparseTextEmbedding, types.SparseVectorParams]
+    late_interaction_text_embedding_config: Tuple[LateInteractionTextEmbedding, types.VectorParams]
+    # TODO: Replace PartitionConfig with MultiTenantConfig -> allow user to specify global index or not during collection creation
+    partition_config: Optional[Tuple[str, KeywordIndexParams]] = None
+    multi_tenant: Optional[bool] = False
+    replication_factor: Optional[int] = 2
+    shard_number: Optional[int] = 3
+
+    model_config = {
+        "arbitrary_types_allowed": True,
+    }
+
+    @model_validator(mode='after')
+    def _validate_config(self):
+        """
+        Validate the configuration after model initialization.
+        
+        Ensures that the configuration is valid by checking:
+        - Multi-tenancy and partition configuration compatibility
+        - Replication factor and shard number are valid
+        - Embedding models are of the correct type and have required attributes
+        
+        Returns:
+            self: The validated configuration instance
+            
+        Raises:
+            ValueError: If any validation check fails
+        """
+        if self.multi_tenant and self.partition_config is None:
+            raise ValueError("partition_config must be provided if multi_tenant is True")
+        if not self.multi_tenant and self.partition_config is not None:
+            raise ValueError("partition_config must be None if multi_tenant is False")
+        
+        if not isinstance(self.replication_factor, int) or self.replication_factor < 1:
+            raise ValueError("replication_factor must be an integer greater than 0")
+        
+        if not isinstance(self.shard_number, int) or self.shard_number < 1:
+            raise ValueError("shard_number must be an integer greater than 0") 
+        
+        for config_name, (model, _) in [
+            ("text_embedding_config", self.text_embedding_config),
+            ("sparse_embedding_config", self.sparse_embedding_config),
+            ("late_interaction_text_embedding_config", self.late_interaction_text_embedding_config)
+        ]:
+            if config_name == "text_embedding_config" and not isinstance(model, Union[TextEmbedding, SentenceTransformerEmbedding]):
+                raise ValueError(f"Embedding model in {config_name} must be an instance of TextEmbedding")
+            elif config_name == "sparse_embedding_config" and not isinstance(model, SparseTextEmbedding):
+                raise ValueError(f"Embedding model in {config_name} must be an instance of SparseEmbedding")
+            elif config_name == "late_interaction_text_embedding_config" and not isinstance(model, LateInteractionTextEmbedding):
+                raise ValueError(f"Embedding model in {config_name} must be an instance of LateInteractionTextEmbedding")
+            
+            if not hasattr(model, "model_name"):
+                raise ValueError(f"Embedding model in {config_name} must have a 'model_name' attribute")
+                
+            if not hasattr(model, "embed") or not callable(getattr(model, "embed")):
+                raise ValueError(f"Embedding model in {config_name} must have an 'embed' method")
+        return self
+    
+    @property
+    def dense_model_config(self) -> Tuple[TextEmbedding, types.VectorParams]:
+        """Get the dense embedding model configuration."""
+        return self.text_embedding_config
+    
+    @property
+    def sparse_model_config(self) -> Tuple[SparseTextEmbedding, types.SparseVectorParams]:
+        """Get the sparse embedding model configuration."""
+        return self.sparse_embedding_config
+    
+    @property
+    def late_interaction_model_config(self) -> Tuple[LateInteractionTextEmbedding, types.VectorParams]:
+        """Get the late interaction embedding model configuration."""
+        return self.late_interaction_text_embedding_config
+    
+    @property
+    def dense_model(self) -> TextEmbedding:
+        """Get the dense embedding model."""
+        return self.dense_model_config[0]
+    
+    @property
+    def sparse_model(self) -> SparseTextEmbedding:
+        """Get the sparse embedding model."""
+        return self.sparse_model_config[0]
+    
+    @property
+    def late_interaction_model(self) -> LateInteractionTextEmbedding:
+        """Get the late interaction embedding model."""
+        return self.late_interaction_model_config[0]
+    
+    @property
+    def dense_model_name(self) -> str:
+        """Get the name of the dense embedding model."""
+        return self.dense_model.model_name
+    
+    @property
+    def sparse_model_name(self) -> str:
+        """Get the name of the sparse embedding model."""
+        return self.sparse_model.model_name
+    
+    @property
+    def late_interaction_model_name(self) -> str:
+        """Get the name of the late interaction embedding model."""
+        return self.late_interaction_model.model_name
+
+    def list_embedding_configs(self) -> List[Tuple[Embedding, BaseVectorParams]]:
+        """
+        Get a list of all embedding configurations.
+        
+        Returns:
+            List[Tuple[Embedding, BaseVectorParams]]: A list containing tuples of embedding models 
+            and their associated vector parameters
+        """
+        return [
+            self.text_embedding_config,
+            self.sparse_embedding_config,
+            self.late_interaction_text_embedding_config
+        ]
+    
+    def list_embedding_model_names(self) -> List[str]:
+        """
+        Get a list of all embedding model names.
+        
+        Returns:
+            List[str]: A list of embedding model names
+        """
+        return [
+            config[0].model_name
+            for config in self.list_embedding_configs()
+        ]
+    
+    def list_embedding_models(self) -> List[Embedding]:
+        """
+        Get a list of all embedding models.
+        
+        Returns:
+            List[Embedding]: A list containing all embedding model instances
+        """
+        return [
+            config[0]
+            for config in self.list_embedding_configs()
+        ]
+
+    def get_vectors_config_dict(self) -> Mapping[str, types.VectorParams]:
+        """
+        Get a dictionary mapping dense embedding model names to their vector parameters.
+        
+        Returns:
+            Mapping[str, types.VectorParams]: Dictionary mapping model names to VectorParams
+        """
+        return {
+            self.DENSE_VECTOR_NAME: self.dense_model_config[1],
+            self.LATE_INTERACTION_VECTOR_NAME: self.late_interaction_model_config[1],
+        }
+    
+    def get_sparse_vectors_config_dict(self) -> Mapping[str, types.SparseVectorParams]:
+        """
+        Get a dictionary mapping sparse embedding model names to their vector parameters.
+        
+        Returns:
+            Mapping[str, types.SparseVectorParams]: Dictionary mapping model names to SparseVectorParams
+        """
+        return {
+            self.SPARSE_VECTOR_NAME: self.sparse_model_config[1],
+        }
+    
+    def get_partition_config(self) -> Tuple[str, KeywordIndexParams]:
+        """
+        Get the partition configuration for multi-tenant setup.
+        
+        Returns:
+            Tuple[str, KeywordIndexParams]: A tuple containing the partition field name
+            and the KeywordIndexParams for that field
+            
+        Raises:
+            ValueError: If partition_config is not set but this method is called
+        """
+        if not self.partition_config:
+            raise ValueError("partition_config must be specified during instantiation")
+        return self.partition_config