semantic-compressor 2.41__tar.gz → 2.42__tar.gz

semantic_compressor-2.42/.gitignore

@@ -0,0 +1,4 @@
+*__pycache__*
+build/*
+dist/*
+*.egg-info/*

{semantic_compressor-2.41/semantic_compressor.egg-info → semantic_compressor-2.42}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: semantic_compressor
-Version: 2.41
+Version: 2.42
 Author: Carlo Moro
 Author-email: Carlo Moro <cnmoro@gmail.com>
 Classifier: Programming Language :: Python :: 3
@@ -12,10 +12,11 @@ License-File: LICENSE
 Requires-Dist: numpy<2
 Requires-Dist: nltk
 Requires-Dist: scikit-learn
-Requires-Dist: fasttext
+Requires-Dist: lingua-language-detector
 Requires-Dist: model2vec
 Requires-Dist: pyspellchecker
 Dynamic: author
+Dynamic: license-file
 Dynamic: requires-python
 
 ```python

{semantic_compressor-2.41 → semantic_compressor-2.42}/compressor/semantic.py

@@ -6,7 +6,7 @@ from sklearn.decomposition import LatentDirichletAllocation
 from sklearn.metrics.pairwise import cosine_similarity
 from compressor.minbpe.regex import RegexTokenizer
 from concurrent.futures import ProcessPoolExecutor
-import numpy as np, pickle, fasttext, traceback
+import numpy as np, pickle, traceback
 from nltk.tokenize import sent_tokenize
 from multiprocessing import cpu_count
 from spellchecker import SpellChecker
@@ -16,6 +16,10 @@ from collections import Counter
 from model2vec import StaticModel
 import re
 
+from lingua import Language, LanguageDetectorBuilder
+languages = [Language.ENGLISH, Language.PORTUGUESE]
+lang_detector = LanguageDetectorBuilder.from_languages(*languages).build()
+
 tokenizer = RegexTokenizer()
 
 # Inicializando os stemmers
@@ -24,10 +28,8 @@ stemmer_portuguese = RSLPStemmer()
 
 english_stopwords_path = str(importlib.resources.files('compressor').joinpath('resources/en_stopwords.pkl'))
 portuguese_stopwords_path = str(importlib.resources.files('compressor').joinpath('resources/pt_stopwords.pkl'))
-fasttext_model_path = str(importlib.resources.files('compressor').joinpath('resources/lid.176.ftz'))
 english_stopwords = pickle.load(open(english_stopwords_path, "rb"))
 portuguese_stopwords = pickle.load(open(portuguese_stopwords_path, "rb"))
-langdetect_model = fasttext.load_model(fasttext_model_path)
 
 embedding_model = StaticModel.from_pretrained("cnmoro/static-nomic-eng-ptbr-tiny")
 
@@ -91,8 +93,8 @@ def count_tokens(text):
     return len(tokenizer.encode(text))
 
 def detect_language(text):
-    detected_lang = langdetect_model.predict(text.replace('\n', ' '), k=1)[0][0]
-    return 'pt' if (str(detected_lang) == '__label__pt' or str(detected_lang) == 'portuguese') else 'en'
+    detected_lang = lang_detector.detect_language_of(text)
+    return 'pt' if detected_lang == Language.PORTUGUESE else 'en'
 
 def compute_and_remove_repeated_ngrams(text, ngram_size=3, threshold=3):
     words = text.split()

semantic_compressor-2.42/helper_commands.txt

@@ -0,0 +1 @@
+python setup.py sdist bdist_wheel && twine upload dist/*

{semantic_compressor-2.41 → semantic_compressor-2.42}/pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["setuptools>=61.0", "numpy<2", "nltk", "scikit-learn", "fasttext", "onnxruntime", "onnxruntime-extensions", "pyspellchecker"]
+requires = ["setuptools>=61.0", "numpy<2", "nltk", "scikit-learn", "lingua-language-detector", "onnxruntime", "onnxruntime-extensions", "pyspellchecker"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "semantic_compressor"
-version = "2.41"
+version = "2.42"
 authors = [
   { name="Carlo Moro", email="cnmoro@gmail.com" },
 ]
@@ -20,7 +20,7 @@ dependencies = [
     "numpy<2",
     "nltk",
     "scikit-learn",
-    "fasttext",
+    "lingua-language-detector",
     "model2vec",
     "pyspellchecker"
 ]

semantic_compressor-2.42/requirements.txt

@@ -0,0 +1,6 @@
+numpy<2
+nltk
+scikit-learn
+lingua-language-detector
+model2vec
+pyspellchecker

{semantic_compressor-2.41 → semantic_compressor-2.42/semantic_compressor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: semantic_compressor
-Version: 2.41
+Version: 2.42
 Author: Carlo Moro
 Author-email: Carlo Moro <cnmoro@gmail.com>
 Classifier: Programming Language :: Python :: 3
@@ -12,10 +12,11 @@ License-File: LICENSE
 Requires-Dist: numpy<2
 Requires-Dist: nltk
 Requires-Dist: scikit-learn
-Requires-Dist: fasttext
+Requires-Dist: lingua-language-detector
 Requires-Dist: model2vec
 Requires-Dist: pyspellchecker
 Dynamic: author
+Dynamic: license-file
 Dynamic: requires-python
 
 ```python

{semantic_compressor-2.41 → semantic_compressor-2.42}/semantic_compressor.egg-info/SOURCES.txt

@@ -1,7 +1,10 @@
+.gitignore
 LICENSE
 MANIFEST.in
 README.md
+helper_commands.txt
 pyproject.toml
+requirements.txt
 setup.py
 compressor/__init__.py
 compressor/semantic.py
@@ -11,7 +14,6 @@ compressor/minbpe/basic.py
 compressor/minbpe/regex.py
 compressor/resources/__init__.py
 compressor/resources/en_stopwords.pkl
-compressor/resources/lid.176.ftz
 compressor/resources/pt_stopwords.pkl
 compressor/resources/nltk_data/corpora/stopwords.zip
 compressor/resources/nltk_data/corpora/wordnet.zip

{semantic_compressor-2.41 → semantic_compressor-2.42}/semantic_compressor.egg-info/requires.txt

@@ -1,6 +1,6 @@
 numpy<2
 nltk
 scikit-learn
-fasttext
+lingua-language-detector
 model2vec
 pyspellchecker

{semantic_compressor-2.41 → semantic_compressor-2.42}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='semantic_compressor',
-    version='2.41',
+    version='2.42',
     author='Carlo Moro',
     author_email='cnmoro@gmail.com',
     description="Semantic text compression",
@@ -15,7 +15,7 @@ setup(
         "numpy<2",
         "nltk",
         "scikit-learn",
-        "fasttext",
+        "lingua-language-detector",
         "model2vec",
         "pyspellchecker"
     ],