semantic-compressor 1.4__tar.gz → 1.6__tar.gz

{semantic_compressor-1.4/semantic_compressor.egg-info → semantic_compressor-1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: semantic_compressor
-Version: 1.4
+Version: 1.6
 Author: Carlo Moro
 Author-email: Carlo Moro <cnmoro@gmail.com>
 Classifier: Programming Language :: Python :: 3

{semantic_compressor-1.4 → semantic_compressor-1.6}/compressor/semantic.py

@@ -13,10 +13,6 @@ from collections import Counter
 import onnxruntime as ort
 import nltk
 
-# Inicializando os stemmers
-stemmer_english = PorterStemmer()
-stemmer_portuguese = RSLPStemmer()
-
 tokenizer = RegexTokenizer()
 nltk_data_path = str(importlib.resources.files('compressor').joinpath('resources/nltk_data'))
 
@@ -24,6 +20,10 @@ os.environ['NLTK_DATA'] = nltk_data_path
 
 nltk.download('rslp')
 
+# Inicializando os stemmers
+stemmer_english = PorterStemmer()
+stemmer_portuguese = RSLPStemmer()
+
 english_stopwords_path = str(importlib.resources.files('compressor').joinpath('resources/en_stopwords.pkl'))
 portuguese_stopwords_path = str(importlib.resources.files('compressor').joinpath('resources/pt_stopwords.pkl'))
 fasttext_model_path = str(importlib.resources.files('compressor').joinpath('resources/lid.176.ftz'))
@@ -250,11 +250,18 @@ def stem_text(text, lang='en'):
 
     return stemmed_text
 
-def correct_spelling(frase, detected_lang="pt"):
+def correct_spelling(sentence, detected_lang="pt"):
     spell = SpellChecker(language=detected_lang)
-    words = frase.split()
+    words = sentence.split()
     fixed = [spell.correction(word) for word in words]
-    return " ".join(fixed)
+
+    final_words = []
+
+    # Interpolate original words with fixed words (each word could be "None" in "fixed" when no correction is needed)
+    for original, fixed_word in zip(words, fixed):
+        final_words.append(fixed_word if fixed_word is not None else original)
+
+    return " ".join(final_words)
 
 def find_needle_in_haystack(
         *, haystack: str, needle: str, block_size = 300,

{semantic_compressor-1.4 → semantic_compressor-1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "semantic_compressor"
-version = "1.4"
+version = "1.6"
 authors = [
   { name="Carlo Moro", email="cnmoro@gmail.com" },
 ]

{semantic_compressor-1.4 → semantic_compressor-1.6/semantic_compressor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: semantic_compressor
-Version: 1.4
+Version: 1.6
 Author: Carlo Moro
 Author-email: Carlo Moro <cnmoro@gmail.com>
 Classifier: Programming Language :: Python :: 3

{semantic_compressor-1.4 → semantic_compressor-1.6}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='semantic_compressor',
-    version='1.4',
+    version='1.6',
     author='Carlo Moro',
     author_email='cnmoro@gmail.com',
     description="Semantic text compression",