fastembed 0.2.0__tar.gz → 0.2.2__tar.gz

{fastembed-0.2.0 → fastembed-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastembed
-Version: 0.2.0
+Version: 0.2.2
 Summary: Fast, light, accurate library built for retrieval embedding generation
 Home-page: https://github.com/qdrant/fastembed
 License: Apache License
@@ -29,7 +29,7 @@ Description-Content-Type: text/markdown
 
 # ⚡️ What is FastEmbed?
 
-FastEmbed is a lightweight, fast, Python library built for embedding generation. We [support popular text models](https://qdrant.github.io/fastembed/examples/Supported_Models/). Please [open a Github issue](https://github.com/qdrant/fastembed/issues/new) if you want us to add a new model.
+FastEmbed is a lightweight, fast, Python library built for embedding generation. We [support popular text models](https://qdrant.github.io/fastembed/examples/Supported_Models/). Please [open a GitHub issue](https://github.com/qdrant/fastembed/issues/new) if you want us to add a new model.
 
 The default text embedding (`TextEmbedding`) model is Flag Embedding, the top model in the [MTEB](https://huggingface.co/spaces/mteb/leaderboard) leaderboard. It supports "query" and "passage" prefixes for the input text. Here is an example for [Retrieval Embedding Generation](https://qdrant.github.io/fastembed/examples/Retrieval_with_FastEmbed/) and how to use [FastEmbed with Qdrant](https://qdrant.github.io/fastembed/examples/Usage_With_Qdrant/).
 
@@ -52,7 +52,7 @@ To install the FastEmbed library, pip works:
 pip install fastembed
 ```
 
-## 📖 Usage
+## 📖 Quickstart
 
 ```python
 from fastembed import TextEmbedding
@@ -77,16 +77,14 @@ Installation with Qdrant Client in Python:
 pip install qdrant-client[fastembed]
 ```
 
-Might have to use ```pip install 'qdrant-client[fastembed]'``` on zsh.
+You might have to use ```pip install 'qdrant-client[fastembed]'``` on zsh.
 
 ```python
 from qdrant_client import QdrantClient
 
 # Initialize the client
 client = QdrantClient("localhost", port=6333) # For production
-# OR if you just want to try it out quickly:
-# client = QdrantClient(":memory:")
-# client = QdrantClient(path="path/to/db")
+# client = QdrantClient(":memory:") # For small experiments
 
 # Prepare your documents, metadata, and IDs
 docs = ["Qdrant has Langchain integrations", "Qdrant also has Llama Index integrations"]
@@ -96,7 +94,12 @@ metadata = [
 ]
 ids = [42, 2]
 
-# Use the new add method
+# If you want to change the model:
+# client.set_model("sentence-transformers/all-MiniLM-L6-v2")
+# List of supported models: https://qdrant.github.io/fastembed/examples/Supported_Models
+
+# Use the new add() instead of upsert()
+# This internally calls embed() of the configured embedding model
 client.add(
     collection_name="demo_collection",
     documents=docs,

{fastembed-0.2.0 → fastembed-0.2.2}/README.md

@@ -1,6 +1,6 @@
 # ⚡️ What is FastEmbed?
 
-FastEmbed is a lightweight, fast, Python library built for embedding generation. We [support popular text models](https://qdrant.github.io/fastembed/examples/Supported_Models/). Please [open a Github issue](https://github.com/qdrant/fastembed/issues/new) if you want us to add a new model.
+FastEmbed is a lightweight, fast, Python library built for embedding generation. We [support popular text models](https://qdrant.github.io/fastembed/examples/Supported_Models/). Please [open a GitHub issue](https://github.com/qdrant/fastembed/issues/new) if you want us to add a new model.
 
 The default text embedding (`TextEmbedding`) model is Flag Embedding, the top model in the [MTEB](https://huggingface.co/spaces/mteb/leaderboard) leaderboard. It supports "query" and "passage" prefixes for the input text. Here is an example for [Retrieval Embedding Generation](https://qdrant.github.io/fastembed/examples/Retrieval_with_FastEmbed/) and how to use [FastEmbed with Qdrant](https://qdrant.github.io/fastembed/examples/Usage_With_Qdrant/).
 
@@ -23,7 +23,7 @@ To install the FastEmbed library, pip works:
 pip install fastembed
 ```
 
-## 📖 Usage
+## 📖 Quickstart
 
 ```python
 from fastembed import TextEmbedding
@@ -48,16 +48,14 @@ Installation with Qdrant Client in Python:
 pip install qdrant-client[fastembed]
 ```
 
-Might have to use ```pip install 'qdrant-client[fastembed]'``` on zsh.
+You might have to use ```pip install 'qdrant-client[fastembed]'``` on zsh.
 
 ```python
 from qdrant_client import QdrantClient
 
 # Initialize the client
 client = QdrantClient("localhost", port=6333) # For production
-# OR if you just want to try it out quickly:
-# client = QdrantClient(":memory:")
-# client = QdrantClient(path="path/to/db")
+# client = QdrantClient(":memory:") # For small experiments
 
 # Prepare your documents, metadata, and IDs
 docs = ["Qdrant has Langchain integrations", "Qdrant also has Llama Index integrations"]
@@ -67,7 +65,12 @@ metadata = [
 ]
 ids = [42, 2]
 
-# Use the new add method
+# If you want to change the model:
+# client.set_model("sentence-transformers/all-MiniLM-L6-v2")
+# List of supported models: https://qdrant.github.io/fastembed/examples/Supported_Models
+
+# Use the new add() instead of upsert()
+# This internally calls embed() of the configured embedding model
 client.add(
     collection_name="demo_collection",
     documents=docs,

fastembed-0.2.2/fastembed/__init__.py

@@ -0,0 +1,3 @@
+from fastembed.text.text_embedding import TextEmbedding
+
+__all__ = ["TextEmbedding"]

{fastembed-0.2.0 → fastembed-0.2.2}/fastembed/common/models.py

@@ -33,7 +33,7 @@ def load_tokenizer(model_dir: Path, max_length: int = 512) -> Tokenizer:
 
     tokenizer = Tokenizer.from_file(str(tokenizer_path))
     tokenizer.enable_truncation(max_length=min(tokenizer_config["model_max_length"], max_length))
-    tokenizer.enable_padding(pad_id=config["pad_token_id"], pad_token=tokenizer_config["pad_token"])
+    tokenizer.enable_padding(pad_id=config.get("pad_token_id", 0), pad_token=tokenizer_config["pad_token"])
 
     for token in tokens_map.values():
         if isinstance(token, str):

{fastembed-0.2.0 → fastembed-0.2.2}/fastembed/common/utils.py

@@ -24,10 +24,10 @@ def define_cache_dir(cache_dir: Optional[str] = None) -> Path:
     """
     if cache_dir is None:
         default_cache_dir = os.path.join(tempfile.gettempdir(), "fastembed_cache")
-        cache_dir = Path(os.getenv("FASTEMBED_CACHE_PATH", default_cache_dir))
+        cache_path = Path(os.getenv("FASTEMBED_CACHE_PATH", default_cache_dir))
     else:
-        cache_dir = Path(cache_dir)
+        cache_path = Path(cache_dir)
 
-    cache_dir.mkdir(parents=True, exist_ok=True)
+    cache_path.mkdir(parents=True, exist_ok=True)
 
-    return cache_dir
+    return cache_path

{fastembed-0.2.0 → fastembed-0.2.2}/fastembed/embedding.py

@@ -4,7 +4,7 @@ from loguru import logger
 
 from fastembed.text.text_embedding import TextEmbedding
 
-logger.warning("DefaultEmbedding, FlagEmbedding, JinaEmbedding are deprecated." " Use TextEmbedding instead.")
+logger.warning("DefaultEmbedding, FlagEmbedding, JinaEmbedding are deprecated." "Use from fastembed import TextEmbedding instead.")
 
 DefaultEmbedding = TextEmbedding
 FlagEmbedding = TextEmbedding

{fastembed-0.2.0 → fastembed-0.2.2}/fastembed/text/e5_onnx_embedding.py

@@ -22,8 +22,8 @@ supported_multilingual_e5_models = [
         "size_in_GB": 1.11,
         "sources": {
             "hf": "xenova/paraphrase-multilingual-mpnet-base-v2",
-        }
-    }
+        },
+    },
 ]
 
 
@@ -51,8 +51,8 @@ class E5OnnxEmbedding(OnnxTextEmbedding):
 
 class E5OnnxEmbeddingWorker(OnnxTextEmbeddingWorker):
     def init_embedding(
-            self,
-            model_name: str,
-            cache_dir: str,
+        self,
+        model_name: str,
+        cache_dir: str,
     ) -> E5OnnxEmbedding:
         return E5OnnxEmbedding(model_name=model_name, cache_dir=cache_dir, threads=1)

{fastembed-0.2.0 → fastembed-0.2.2}/fastembed/text/onnx_embedding.py

@@ -1,6 +1,6 @@
 import os
 from multiprocessing import get_all_start_methods
-from typing import List, Dict, Any, Tuple, Union, Iterable, Type
+from typing import List, Dict, Any, Optional, Tuple, Union, Iterable, Type
 
 import numpy as np
 import onnxruntime as ort
@@ -98,6 +98,15 @@ supported_onnx_models = [
             "hf": "qdrant/all-MiniLM-L6-v2-onnx",
         },
     },
+    {
+        "model": "nomic-ai/nomic-embed-text-v1",
+        "dim": 768,
+        "description": "8192 context length english model",
+        "size_in_GB": 0.54,
+        "sources": {
+            "hf": "nomic-ai/nomic-embed-text-v1",
+        },
+    },
     # {
     #     "model": "sentence-transformers/all-MiniLM-L6-v2",
     #     "dim": 384,
@@ -149,8 +158,8 @@ class OnnxTextEmbedding(TextEmbeddingBase):
     def __init__(
         self,
         model_name: str = "BAAI/bge-small-en-v1.5",
-        cache_dir: str = None,
-        threads: int = None,
+        cache_dir: Optional[str] = None,
+        threads: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -193,7 +202,7 @@ class OnnxTextEmbedding(TextEmbeddingBase):
         self,
         documents: Union[str, Iterable[str]],
         batch_size: int = 256,
-        parallel: int = None,
+        parallel: Optional[int] = None,
         **kwargs,
     ) -> Iterable[np.ndarray]:
         """

{fastembed-0.2.0 → fastembed-0.2.2}/fastembed/text/text_embedding.py

@@ -53,24 +53,22 @@ class TextEmbedding(TextEmbeddingBase):
     ):
         super().__init__(model_name, cache_dir, threads, **kwargs)
 
-        self.model = None
         for embedding in self.EMBEDDINGS_REGISTRY:
             supported_models = embedding.list_supported_models()
             if any(model_name == model["model"] for model in supported_models):
                 self.model = embedding(model_name, cache_dir, threads, **kwargs)
-                break
+                return
 
-        if self.model is None:
-            raise ValueError(
-                f"Model {model_name} is not supported in TextEmbedding."
-                "Please check the supported models using `TextEmbedding.list_supported_models()`"
-            )
+        raise ValueError(
+            f"Model {model_name} is not supported in TextEmbedding."
+            "Please check the supported models using `TextEmbedding.list_supported_models()`"
+        )
 
     def embed(
         self,
         documents: Union[str, Iterable[str]],
         batch_size: int = 256,
-        parallel: int = None,
+        parallel: Optional[int] = None,
         **kwargs,
     ) -> Iterable[np.ndarray]:
         """

{fastembed-0.2.0 → fastembed-0.2.2}/fastembed/text/text_embedding_base.py

@@ -1,4 +1,4 @@
-from typing import Optional, Union, Iterable, List, Dict, Any
+from typing import Any, Dict, Iterable, List, Optional, Union
 
 import numpy as np
 
@@ -19,7 +19,7 @@ class TextEmbeddingBase(ModelManagement):
         self,
         documents: Union[str, Iterable[str]],
         batch_size: int = 256,
-        parallel: int = None,
+        parallel: Optional[int] = None,
         **kwargs,
     ) -> Iterable[np.ndarray]:
         raise NotImplementedError()
@@ -39,17 +39,19 @@ class TextEmbeddingBase(ModelManagement):
         # This is model-specific, so that different models can have specialized implementations
         yield from self.embed(texts, **kwargs)
 
-    def query_embed(self, query: str, **kwargs) -> np.ndarray:
+    def query_embed(self, query: Union[str, Iterable[str]], **kwargs) -> Iterable[np.ndarray]:
         """
-        Embeds a query
+        Embeds queries
 
         Args:
-            query (str): The query to search for.
+            query (Union[str, Iterable[str]]): The query to embed, or an iterable e.g. list of queries.
 
         Returns:
-            np.ndarray: The embeddings.
+            Iterable[np.ndarray]: The embeddings.
         """
 
         # This is model-specific, so that different models can have specialized implementations
-        query_embedding = list(self.embed([query], **kwargs))[0]
-        return query_embedding
+        if isinstance(query, str):
+            yield from self.embed([query], **kwargs)
+        if isinstance(query, Iterable):
+            yield from self.embed(query, **kwargs)

{fastembed-0.2.0 → fastembed-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fastembed"
-version = "0.2.0"
+version = "0.2.2"
 description = "Fast, light, accurate library built for retrieval embedding generation"
 authors = ["NirantK <nirant.bits@gmail.com>"]
 license = "Apache License"
@@ -26,14 +26,14 @@ numpy = [
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.2"
-ruff = "^0.1.13"
+ruff = "^0.2.2"
 notebook = ">=7.0.2"
-mkdocs-material = "^9.1.21"
-mkdocstrings = "^0.22.0"
-pillow = "^10.0.0"
+mkdocs-material = "^9.5.10"
+mkdocstrings = "^0.24.0"
+pillow = "^10.2.0"
 cairosvg = "^2.7.1"
 mknotebooks = "^0.8.0"
-pre-commit = { version = "^3.6.0", python = ">=3.9,<3.12" }
+pre-commit = {version = "^3.6.2", python = ">=3.9,<3.12" }
 
 
 [build-system]