semantic-compressor 2.2__py3-none-any.whl → 2.4__py3-none-any.whl

compressor/semantic.py

@@ -14,6 +14,7 @@ from nltk.stem import PorterStemmer
 from nltk.stem import RSLPStemmer
 from collections import Counter
 from model2vec import StaticModel
+import re
 
 tokenizer = RegexTokenizer()
 
@@ -28,10 +29,39 @@ english_stopwords = pickle.load(open(english_stopwords_path, "rb"))
 portuguese_stopwords = pickle.load(open(portuguese_stopwords_path, "rb"))
 langdetect_model = fasttext.load_model(fasttext_model_path)
 
-embedding_model = StaticModel.from_pretrained("minishlab/potion-base-2M")
+embedding_model = StaticModel.from_pretrained("cnmoro/Linq-Embed-Mistral-Distilled")
 
 hashing_vectorizer = HashingVectorizer(ngram_range=(1, 6), analyzer='char', n_features=512)
 
+def clean_text(text: str) -> str:
+    # 1) Fix hyphenation at line breaks
+    text = re.sub(r'(\w)-\s*\n\s*(\w)', r'\1\2', text)
+    # 2) Strip stray pipes, bullets, brackets, quotes, unmatched parens
+    text = re.sub(r'[\|\•\[\]\(\)\"“”]', ' ', text)
+    # 3) Remove leading list hyphens
+    text = re.sub(r'(?m)^\s*-\s*', '', text)
+    # 4) Remove hyphens not between letters
+    text = re.sub(r'(?<!\w)-(?!\w)', ' ', text)
+    # 5) Collapse repeated punctuation
+    text = re.sub(r'([!?.,;:]){2,}', r'\1', text)
+    # 6) Normalize whitespace
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = re.sub(r'\n{2,}', '\n', text).strip()
+
+    # 7) Aggressive cleanup if >20% noise, but keep basic punctuation
+    alpha_ratio = sum(c.isalpha() for c in text) / max(len(text), 1)
+    if alpha_ratio < 0.8:
+        text = re.sub(r'[^A-Za-zÀ-ÿ\s\.\,\;\:\?\!]', ' ', text)
+        text = re.sub(r'\s{2,}', ' ', text).strip()
+
+    # 8) Reattach punctuation to preceding word and normalize post-punct spacing
+    #    "word ." → "word."
+    text = re.sub(r'\s+([\.!,\?;:])', r'\1', text)
+    #    "word.Next" → "word. Next"
+    text = re.sub(r'([\.!,\?;:])(?=\S)', r'\1 ', text)
+
+    return text
+
 def extract_textual_embeddings(text):
     X = hashing_vectorizer.fit_transform([text])
     dense_matrix = X.toarray()
@@ -100,7 +130,7 @@ def compute_and_remove_repeated_ngrams(text, ngram_size=3, threshold=3):
 def calculate_similarity(embed1, embed2):
     return cosine_similarity([embed1], [embed2])[0][0]
 
-def semantic_compress_text(full_text, compression_rate=0.7, num_topics=5, reference_text: str = None):
+def semantic_compress_text(full_text, compression_rate=0.7, num_topics=5, reference_text: str = None, perform_cleaning: bool = True):
     def create_lda_model(texts, stopwords):
         vectorizer = CountVectorizer(stop_words=stopwords)
         doc_term_matrix = vectorizer.fit_transform(texts)
@@ -129,6 +159,9 @@ def semantic_compress_text(full_text, compression_rate=0.7, num_topics=5, refere
         return importance
 
     try:
+        if perform_cleaning:
+            full_text = clean_text(full_text)
+        
         # Split the text into sentences
         sentences = sent_tokenize(full_text)
 
@@ -192,7 +225,7 @@ def semantic_compress_text(full_text, compression_rate=0.7, num_topics=5, refere
     
     return full_text
 
-def compress_text(text, *, target_token_count=None, compression_rate=0.7, reference_text_steering=None):
+def compress_text(text, *, target_token_count=None, compression_rate=0.7, reference_text_steering=None, perform_cleaning=True):
     """
     Compress text using either a compression rate or a target token count.
     If both are provided, the compression rate will be used.
@@ -219,7 +252,8 @@ def compress_text(text, *, target_token_count=None, compression_rate=0.7, refere
         return semantic_compress_text(
             full_text = text,
             compression_rate = compression_rate,
-            reference_text = reference_text_steering
+            reference_text = reference_text_steering,
+            perform_cleaning = perform_cleaning
         )
     except Exception:
         traceback.print_exc()

{semantic_compressor-2.2.dist-info → semantic_compressor-2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: semantic_compressor
-Version: 2.2
+Version: 2.4
 Author: Carlo Moro
 Author-email: Carlo Moro <cnmoro@gmail.com>
 Classifier: Programming Language :: Python :: 3

{semantic_compressor-2.2.dist-info → semantic_compressor-2.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 compressor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-compressor/semantic.py,sha256=0TpOSQBhpfBcqyCs-08NbMOBvdMsvMPvKDlZIz-5Q4Q,15669
+compressor/semantic.py,sha256=sJXbap3_oZpd-XMGkecrqQ7RSon-OE98u8iYcNRIskA,17076
 compressor/minbpe/__init__.py,sha256=wZ1z2QKkncvGgiZDBc91AP5m7-M-MVenPStKbS6xylE,95
 compressor/minbpe/base.py,sha256=tTKag04RRFnc4ppoieBbDV0V6thzi_ZvZTlhOYIoY7Q,6881
 compressor/minbpe/basic.py,sha256=0kD4tU8l2MZegfPaHMfDo5CnaSzb9i1v9tDBy6GwMbg,2883
@@ -175,8 +175,8 @@ compressor/resources/nltk_data/tokenizers/punkt_tab/turkish/abbrev_types.txt,sha
 compressor/resources/nltk_data/tokenizers/punkt_tab/turkish/collocations.tab,sha256=BhzimBd2qPh12k8kvr1-E4-NodkFe0PQf1gBSOwQajM,273
 compressor/resources/nltk_data/tokenizers/punkt_tab/turkish/ortho_context.tab,sha256=_CFCJ_mdXqPucNII3xaxmE6rN10ZRu03kGHGz1wXGL4,642682
 compressor/resources/nltk_data/tokenizers/punkt_tab/turkish/sent_starters.txt,sha256=kyOftVtdKubZRahKlOEYuoqBYyaxfNwRuoERvqDJeCg,613
-semantic_compressor-2.2.dist-info/LICENSE,sha256=DFRihXonZ3qVRaTrzuXNaDI_-h2jyT2SqWqjtTDHfqI,1067
-semantic_compressor-2.2.dist-info/METADATA,sha256=XMlw60617-mpHu59sdqmvxMUtcPfKOj7xrlriYb2wvw,6178
-semantic_compressor-2.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-semantic_compressor-2.2.dist-info/top_level.txt,sha256=qb2SlKrEmMrQDVrhwxu3Wr7U6JupPXtDGrJpIQr8xSc,11
-semantic_compressor-2.2.dist-info/RECORD,,
+semantic_compressor-2.4.dist-info/LICENSE,sha256=DFRihXonZ3qVRaTrzuXNaDI_-h2jyT2SqWqjtTDHfqI,1067
+semantic_compressor-2.4.dist-info/METADATA,sha256=-QsGOKQoDo4YBW88p-KWPrr0GlHrgvygoRnSN2C09-Y,6178
+semantic_compressor-2.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+semantic_compressor-2.4.dist-info/top_level.txt,sha256=qb2SlKrEmMrQDVrhwxu3Wr7U6JupPXtDGrJpIQr8xSc,11
+semantic_compressor-2.4.dist-info/RECORD,,