semantic-compressor 1.7__tar.gz → 1.9__tar.gz

{semantic_compressor-1.7/semantic_compressor.egg-info → semantic_compressor-1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: semantic_compressor
-Version: 1.7
+Version: 1.9
 Author: Carlo Moro
 Author-email: Carlo Moro <cnmoro@gmail.com>
 Classifier: Programming Language :: Python :: 3
@@ -13,8 +13,7 @@ Requires-Dist: numpy<2
 Requires-Dist: nltk
 Requires-Dist: scikit-learn
 Requires-Dist: fasttext
-Requires-Dist: onnxruntime
-Requires-Dist: onnxruntime-extensions
+Requires-Dist: model2vec
 Requires-Dist: pyspellchecker
 
 ```python

{semantic_compressor-1.7 → semantic_compressor-1.9}/compressor/semantic.py

@@ -2,7 +2,6 @@ from sklearn.feature_extraction.text import CountVectorizer, HashingVectorizer
 import numpy as np, pickle, fasttext, os, traceback, importlib
 from sklearn.decomposition import LatentDirichletAllocation
 from sklearn.metrics.pairwise import cosine_similarity
-from onnxruntime_extensions import get_library_path
 from compressor.minbpe.regex import RegexTokenizer
 from concurrent.futures import ProcessPoolExecutor
 from nltk.tokenize import sent_tokenize
@@ -11,7 +10,7 @@ from spellchecker import SpellChecker
 from nltk.stem import PorterStemmer
 from nltk.stem import RSLPStemmer
 from collections import Counter
-import onnxruntime as ort
+from model2vec import StaticModel
 import nltk
 
 tokenizer = RegexTokenizer()
@@ -32,18 +31,7 @@ english_stopwords = pickle.load(open(english_stopwords_path, "rb"))
 portuguese_stopwords = pickle.load(open(portuguese_stopwords_path, "rb"))
 langdetect_model = fasttext.load_model(fasttext_model_path)
 
-embedding_model_cpu_count = os.environ.get('EMBEDDING_MODEL_CPU_COUNT', 1)
-
-_options = ort.SessionOptions()
-_options.inter_op_num_threads, _options.intra_op_num_threads = embedding_model_cpu_count, embedding_model_cpu_count
-_options.register_custom_ops_library(get_library_path())
-_providers = ["CPUExecutionProvider"]
-
-embedding_model = ort.InferenceSession(
-    path_or_bytes = str(importlib.resources.files('compressor').joinpath('resources/embedding_model.onnx')),
-    sess_options=_options,
-    providers=_providers
-)
+embedding_model = StaticModel.from_pretrained("cnmoro/multilingual-e5-small-distilled-16m")
 
 hashing_vectorizer = HashingVectorizer(ngram_range=(1, 6), analyzer='char', n_features=512)
 
@@ -54,7 +42,7 @@ def extract_textual_embeddings(text):
     return fixed_size_matrix.tolist()
 
 def extract_semantic_embeddings(text):
-    return embedding_model.run(output_names=["outputs"], input_feed={"inputs": [text]})[0][0]
+    return embedding_model.encode([text])[0]
 
 def structurize_text(full_text, tokens_per_chunk=300, chunk_overlap=0):
     chunks = []
@@ -339,7 +327,7 @@ def find_needle_in_haystack(
                 haystack_semantic_embeddings = list(executor.map(extract_semantic_embeddings, blocks))
         
         if embedding_mode in {'textual', 'both'}:
-            with ProcessPoolExecutor(max_workers=cpu_count()//1.5) as executor:
+            with ProcessPoolExecutor(max_workers=int(cpu_count()//1.5)) as executor:
                 haystack_textual_embeddings = list(
                     executor.map(preprocess_and_extract_textual_embedding, blocks, [use_stemming]*len(blocks), [lang]*len(blocks))
                 )

{semantic_compressor-1.7 → semantic_compressor-1.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "semantic_compressor"
-version = "1.7"
+version = "1.9"
 authors = [
   { name="Carlo Moro", email="cnmoro@gmail.com" },
 ]
@@ -21,7 +21,6 @@ dependencies = [
     "nltk",
     "scikit-learn",
     "fasttext",
-    "onnxruntime",
-    "onnxruntime-extensions",
+    "model2vec",
     "pyspellchecker"
 ]

{semantic_compressor-1.7 → semantic_compressor-1.9/semantic_compressor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: semantic_compressor
-Version: 1.7
+Version: 1.9
 Author: Carlo Moro
 Author-email: Carlo Moro <cnmoro@gmail.com>
 Classifier: Programming Language :: Python :: 3
@@ -13,8 +13,7 @@ Requires-Dist: numpy<2
 Requires-Dist: nltk
 Requires-Dist: scikit-learn
 Requires-Dist: fasttext
-Requires-Dist: onnxruntime
-Requires-Dist: onnxruntime-extensions
+Requires-Dist: model2vec
 Requires-Dist: pyspellchecker
 
 ```python

{semantic_compressor-1.7 → semantic_compressor-1.9}/semantic_compressor.egg-info/SOURCES.txt

@@ -8,7 +8,6 @@ compressor/minbpe/__init__.py
 compressor/minbpe/base.py
 compressor/minbpe/basic.py
 compressor/minbpe/regex.py
-compressor/resources/embedding_model.onnx
 compressor/resources/en_stopwords.pkl
 compressor/resources/lid.176.ftz
 compressor/resources/pt_stopwords.pkl

{semantic_compressor-1.7 → semantic_compressor-1.9}/semantic_compressor.egg-info/requires.txt

@@ -2,6 +2,5 @@ numpy<2
 nltk
 scikit-learn
 fasttext
-onnxruntime
-onnxruntime-extensions
+model2vec
 pyspellchecker

{semantic_compressor-1.7 → semantic_compressor-1.9}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='semantic_compressor',
-    version='1.7',
+    version='1.9',
     author='Carlo Moro',
     author_email='cnmoro@gmail.com',
     description="Semantic text compression",
@@ -16,8 +16,7 @@ setup(
         "nltk",
         "scikit-learn",
         "fasttext",
-        "onnxruntime",
-        "onnxruntime-extensions",
+        "model2vec",
         "pyspellchecker"
     ],
     classifiers=[