semantic-compressor 1.4__tar.gz → 1.5__tar.gz

Sign up to get free protection for your applications and to get access to all the features.
Files changed (21) hide show
  1. {semantic_compressor-1.4/semantic_compressor.egg-info → semantic_compressor-1.5}/PKG-INFO +1 -1
  2. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/semantic.py +10 -3
  3. {semantic_compressor-1.4 → semantic_compressor-1.5}/pyproject.toml +1 -1
  4. {semantic_compressor-1.4 → semantic_compressor-1.5/semantic_compressor.egg-info}/PKG-INFO +1 -1
  5. {semantic_compressor-1.4 → semantic_compressor-1.5}/setup.py +1 -1
  6. {semantic_compressor-1.4 → semantic_compressor-1.5}/LICENSE +0 -0
  7. {semantic_compressor-1.4 → semantic_compressor-1.5}/README.md +0 -0
  8. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/__init__.py +0 -0
  9. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/minbpe/__init__.py +0 -0
  10. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/minbpe/base.py +0 -0
  11. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/minbpe/basic.py +0 -0
  12. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/minbpe/regex.py +0 -0
  13. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/resources/embedding_model.onnx +0 -0
  14. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/resources/en_stopwords.pkl +0 -0
  15. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/resources/lid.176.ftz +0 -0
  16. {semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/resources/pt_stopwords.pkl +0 -0
  17. {semantic_compressor-1.4 → semantic_compressor-1.5}/semantic_compressor.egg-info/SOURCES.txt +0 -0
  18. {semantic_compressor-1.4 → semantic_compressor-1.5}/semantic_compressor.egg-info/dependency_links.txt +0 -0
  19. {semantic_compressor-1.4 → semantic_compressor-1.5}/semantic_compressor.egg-info/requires.txt +0 -0
  20. {semantic_compressor-1.4 → semantic_compressor-1.5}/semantic_compressor.egg-info/top_level.txt +0 -0
  21. {semantic_compressor-1.4 → semantic_compressor-1.5}/setup.cfg +0 -0
{semantic_compressor-1.4/semantic_compressor.egg-info → semantic_compressor-1.5}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: semantic_compressor
3
- Version: 1.4
3
+ Version: 1.5
4
4
  Author: Carlo Moro
5
5
  Author-email: Carlo Moro <cnmoro@gmail.com>
6
6
  Classifier: Programming Language :: Python :: 3
{semantic_compressor-1.4 → semantic_compressor-1.5}/compressor/semantic.py RENAMED
@@ -250,11 +250,18 @@ def stem_text(text, lang='en'):
250
250
 
251
251
  return stemmed_text
252
252
 
253
- def correct_spelling(frase, detected_lang="pt"):
253
+ def correct_spelling(sentence, detected_lang="pt"):
254
254
  spell = SpellChecker(language=detected_lang)
255
- words = frase.split()
255
+ words = sentence.split()
256
256
  fixed = [spell.correction(word) for word in words]
257
- return " ".join(fixed)
257
+
258
+ final_words = []
259
+
260
+ # Interpolate original words with fixed words (each word could be "None" in "fixed" when no correction is needed)
261
+ for original, fixed_word in zip(words, fixed):
262
+ final_words.append(fixed_word if fixed_word is not None else original)
263
+
264
+ return " ".join(final_words)
258
265
 
259
266
  def find_needle_in_haystack(
260
267
  *, haystack: str, needle: str, block_size = 300,
{semantic_compressor-1.4 → semantic_compressor-1.5}/pyproject.toml RENAMED
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "semantic_compressor"
7
- version = "1.4"
7
+ version = "1.5"
8
8
  authors = [
9
9
  { name="Carlo Moro", email="cnmoro@gmail.com" },
10
10
  ]
{semantic_compressor-1.4 → semantic_compressor-1.5/semantic_compressor.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: semantic_compressor
3
- Version: 1.4
3
+ Version: 1.5
4
4
  Author: Carlo Moro
5
5
  Author-email: Carlo Moro <cnmoro@gmail.com>
6
6
  Classifier: Programming Language :: Python :: 3
{semantic_compressor-1.4 → semantic_compressor-1.5}/setup.py RENAMED
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
2
2
 
3
3
  setup(
4
4
  name='semantic_compressor',
5
- version='1.4',
5
+ version='1.5',
6
6
  author='Carlo Moro',
7
7
  author_email='cnmoro@gmail.com',
8
8
  description="Semantic text compression",