semantic-compressor 2.4__py3-none-any.whl → 2.42__py3-none-any.whl

compressor/semantic.py

@@ -6,7 +6,7 @@ from sklearn.decomposition import LatentDirichletAllocation
 from sklearn.metrics.pairwise import cosine_similarity
 from compressor.minbpe.regex import RegexTokenizer
 from concurrent.futures import ProcessPoolExecutor
-import numpy as np, pickle, fasttext, traceback
+import numpy as np, pickle, traceback
 from nltk.tokenize import sent_tokenize
 from multiprocessing import cpu_count
 from spellchecker import SpellChecker
@@ -16,6 +16,10 @@ from collections import Counter
 from model2vec import StaticModel
 import re
 
+from lingua import Language, LanguageDetectorBuilder
+languages = [Language.ENGLISH, Language.PORTUGUESE]
+lang_detector = LanguageDetectorBuilder.from_languages(*languages).build()
+
 tokenizer = RegexTokenizer()
 
 # Inicializando os stemmers
@@ -24,12 +28,10 @@ stemmer_portuguese = RSLPStemmer()
 
 english_stopwords_path = str(importlib.resources.files('compressor').joinpath('resources/en_stopwords.pkl'))
 portuguese_stopwords_path = str(importlib.resources.files('compressor').joinpath('resources/pt_stopwords.pkl'))
-fasttext_model_path = str(importlib.resources.files('compressor').joinpath('resources/lid.176.ftz'))
 english_stopwords = pickle.load(open(english_stopwords_path, "rb"))
 portuguese_stopwords = pickle.load(open(portuguese_stopwords_path, "rb"))
-langdetect_model = fasttext.load_model(fasttext_model_path)
 
-embedding_model = StaticModel.from_pretrained("cnmoro/Linq-Embed-Mistral-Distilled")
+embedding_model = StaticModel.from_pretrained("cnmoro/static-nomic-eng-ptbr-tiny")
 
 hashing_vectorizer = HashingVectorizer(ngram_range=(1, 6), analyzer='char', n_features=512)
 
@@ -91,8 +93,8 @@ def count_tokens(text):
     return len(tokenizer.encode(text))
 
 def detect_language(text):
-    detected_lang = langdetect_model.predict(text.replace('\n', ' '), k=1)[0][0]
-    return 'pt' if (str(detected_lang) == '__label__pt' or str(detected_lang) == 'portuguese') else 'en'
+    detected_lang = lang_detector.detect_language_of(text)
+    return 'pt' if detected_lang == Language.PORTUGUESE else 'en'
 
 def compute_and_remove_repeated_ngrams(text, ngram_size=3, threshold=3):
     words = text.split()

{semantic_compressor-2.4.dist-info → semantic_compressor-2.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: semantic_compressor
-Version: 2.4
+Version: 2.42
 Author: Carlo Moro
 Author-email: Carlo Moro <cnmoro@gmail.com>
 Classifier: Programming Language :: Python :: 3
@@ -12,10 +12,11 @@ License-File: LICENSE
 Requires-Dist: numpy<2
 Requires-Dist: nltk
 Requires-Dist: scikit-learn
-Requires-Dist: fasttext
+Requires-Dist: lingua-language-detector
 Requires-Dist: model2vec
 Requires-Dist: pyspellchecker
 Dynamic: author
+Dynamic: license-file
 Dynamic: requires-python
 
 ```python

{semantic_compressor-2.4.dist-info → semantic_compressor-2.42.dist-info}/RECORD

@@ -1,12 +1,11 @@
 compressor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-compressor/semantic.py,sha256=sJXbap3_oZpd-XMGkecrqQ7RSon-OE98u8iYcNRIskA,17076
+compressor/semantic.py,sha256=YAjQeipnE3Npo9vr-2uKnjdvo1nZrNEM5iP_n-hO4gY,17023
 compressor/minbpe/__init__.py,sha256=wZ1z2QKkncvGgiZDBc91AP5m7-M-MVenPStKbS6xylE,95
 compressor/minbpe/base.py,sha256=tTKag04RRFnc4ppoieBbDV0V6thzi_ZvZTlhOYIoY7Q,6881
 compressor/minbpe/basic.py,sha256=0kD4tU8l2MZegfPaHMfDo5CnaSzb9i1v9tDBy6GwMbg,2883
 compressor/minbpe/regex.py,sha256=k3bllcxc5c7mi43tUEGg6jX-Zc4Cvfb1CCTGEp7ZcVM,5821
 compressor/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 compressor/resources/en_stopwords.pkl,sha256=Q2PyGQnphPUs_jxN9NMSqp2EQjYv4b4oMJY2aMYvbSY,1310
-compressor/resources/lid.176.ftz,sha256=jzRyz-hzintgmejpmcPL-uDc0VaWqsfXc4qAOdtgPoM,938013
 compressor/resources/pt_stopwords.pkl,sha256=-9bJaxJWjeOFxLHLT9D-rI3XTzGC0iLJfMiwBDnkCYI,1716
 compressor/resources/nltk_data/corpora/stopwords.zip,sha256=Qww7buJgh4OBbo46gkso9BXLKPypB-KlyMmBaBkgC6M,36779
 compressor/resources/nltk_data/corpora/wordnet.zip,sha256=y9pepu7382qXpD1KdfheB_zLtPI2V9J7TMvJPiZGq1k,10775600
@@ -175,8 +174,8 @@ compressor/resources/nltk_data/tokenizers/punkt_tab/turkish/abbrev_types.txt,sha
 compressor/resources/nltk_data/tokenizers/punkt_tab/turkish/collocations.tab,sha256=BhzimBd2qPh12k8kvr1-E4-NodkFe0PQf1gBSOwQajM,273
 compressor/resources/nltk_data/tokenizers/punkt_tab/turkish/ortho_context.tab,sha256=_CFCJ_mdXqPucNII3xaxmE6rN10ZRu03kGHGz1wXGL4,642682
 compressor/resources/nltk_data/tokenizers/punkt_tab/turkish/sent_starters.txt,sha256=kyOftVtdKubZRahKlOEYuoqBYyaxfNwRuoERvqDJeCg,613
-semantic_compressor-2.4.dist-info/LICENSE,sha256=DFRihXonZ3qVRaTrzuXNaDI_-h2jyT2SqWqjtTDHfqI,1067
-semantic_compressor-2.4.dist-info/METADATA,sha256=-QsGOKQoDo4YBW88p-KWPrr0GlHrgvygoRnSN2C09-Y,6178
-semantic_compressor-2.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-semantic_compressor-2.4.dist-info/top_level.txt,sha256=qb2SlKrEmMrQDVrhwxu3Wr7U6JupPXtDGrJpIQr8xSc,11
-semantic_compressor-2.4.dist-info/RECORD,,
+semantic_compressor-2.42.dist-info/licenses/LICENSE,sha256=DFRihXonZ3qVRaTrzuXNaDI_-h2jyT2SqWqjtTDHfqI,1067
+semantic_compressor-2.42.dist-info/METADATA,sha256=1ltMJaaIP6YKZQdREJAo1I-AogbX65WaJbeO_Pbls-E,6217
+semantic_compressor-2.42.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+semantic_compressor-2.42.dist-info/top_level.txt,sha256=qb2SlKrEmMrQDVrhwxu3Wr7U6JupPXtDGrJpIQr8xSc,11
+semantic_compressor-2.42.dist-info/RECORD,,

{semantic_compressor-2.4.dist-info → semantic_compressor-2.42.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any