fastembed-gpu 0.4.0__tar.gz → 0.4.1__tar.gz

{fastembed_gpu-0.4.0 → fastembed_gpu-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastembed-gpu
-Version: 0.4.0
+Version: 0.4.1
 Summary: Fast, light, accurate library built for retrieval embedding generation
 Home-page: https://github.com/qdrant/fastembed
 License: Apache License
@@ -15,7 +15,6 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: PyStemmer (>=2.2.0,<3.0.0)
 Requires-Dist: huggingface-hub (>=0.20,<1.0)
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
 Requires-Dist: mmh3 (>=4.1.0,<5.0.0)
@@ -23,8 +22,8 @@ Requires-Dist: numpy (>=1.21,<2) ; python_version < "3.12"
 Requires-Dist: numpy (>=1.26,<2) ; python_version >= "3.12"
 Requires-Dist: onnxruntime-gpu (>=1.17.0,<2.0.0)
 Requires-Dist: pillow (>=10.3.0,<11.0.0)
+Requires-Dist: py-rust-stemmers (>=0.1.0,<0.2.0)
 Requires-Dist: requests (>=2.31,<3.0)
-Requires-Dist: snowballstemmer (>=2.2.0,<3.0.0)
 Requires-Dist: tokenizers (>=0.15,<1.0)
 Requires-Dist: tqdm (>=4.66,<5.0)
 Project-URL: Repository, https://github.com/qdrant/fastembed

{fastembed_gpu-0.4.0 → fastembed_gpu-0.4.1}/fastembed/sparse/bm25.py

@@ -6,8 +6,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
 
 import mmh3
 import numpy as np
-from snowballstemmer import stemmer as get_stemmer
-
+from py_rust_stemmers import SnowballStemmer
 from fastembed.common.utils import (
     define_cache_dir,
     iter_batch,
@@ -130,7 +129,7 @@ class Bm25(SparseTextEmbeddingBase):
         self.punctuation = set(get_all_punctuation())
         self.stopwords = set(self._load_stopwords(self._model_dir, self.language))
 
-        self.stemmer = get_stemmer(language)
+        self.stemmer = SnowballStemmer(language)
         self.tokenizer = SimpleTokenizer
 
     @classmethod
@@ -235,7 +234,7 @@ class Bm25(SparseTextEmbeddingBase):
             if len(token) > self.token_max_length:
                 continue
 
-            stemmed_token = self.stemmer.stemWord(token.lower())
+            stemmed_token = self.stemmer.stem_word(token.lower())
 
             if stemmed_token:
                 stemmed_tokens.append(stemmed_token)

{fastembed_gpu-0.4.0 → fastembed_gpu-0.4.1}/fastembed/sparse/bm42.py

@@ -5,7 +5,7 @@ from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Type, U
 
 import mmh3
 import numpy as np
-from snowballstemmer import stemmer as get_stemmer
+from py_rust_stemmers import SnowballStemmer
 
 from fastembed.common import OnnxProvider
 from fastembed.common.onnx_model import OnnxOutputContext
@@ -119,8 +119,8 @@ class Bm42(SparseTextEmbeddingBase, OnnxTextModel[SparseEmbedding]):
         self.special_tokens = set()
         self.special_tokens_ids = set()
         self.punctuation = set(string.punctuation)
-        self.stopwords = set()
-        self.stemmer = get_stemmer(MODEL_TO_LANGUAGE[model_name])
+        self.stopwords = set(self._load_stopwords(self._model_dir))
+        self.stemmer = SnowballStemmer(MODEL_TO_LANGUAGE[model_name])
         self.alpha = alpha
 
         if not self.lazy_load:
@@ -152,7 +152,7 @@ class Bm42(SparseTextEmbeddingBase, OnnxTextModel[SparseEmbedding]):
     def _stem_pair_tokens(self, tokens: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:
         result = []
         for token, value in tokens:
-            processed_token = self.stemmer.stemWord(token)
+            processed_token = self.stemmer.stem_word(token)
             result.append((processed_token, value))
         return result
 

{fastembed_gpu-0.4.0 → fastembed_gpu-0.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fastembed-gpu"
-version = "0.4.0"
+version = "0.4.1"
 description = "Fast, light, accurate library built for retrieval embedding generation"
 authors = ["Qdrant Team <info@qdrant.tech>", "NirantK <nirant.bits@gmail.com>"]
 license = "Apache License"
@@ -23,9 +23,8 @@ numpy = [
     { version = ">=1.26, <2", python = ">=3.12" }
 ]
 pillow = "^10.3.0"
-snowballstemmer = "^2.2.0"
-PyStemmer = "^2.2.0"
 mmh3 = "^4.1.0"
+py-rust-stemmers = "^0.1.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.2"