SqueakyCleanText 0.1.0__tar.gz

squeakycleantext-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [2024] [Rehan Fazal]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

squeakycleantext-0.1.0/MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+include LICENSE

squeakycleantext-0.1.0/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: SqueakyCleanText
+Version: 0.1.0
+Summary: A comprehensive text cleaning and preprocessing pipeline.
+Home-page: https://github.com/rhnfzl/SqueakyCleanText
+Author: Rehan Fazal
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: lingua-language-detector
+Requires-Dist: nltk
+Requires-Dist: emoji
+Requires-Dist: ftfy
+Requires-Dist: Unidecode
+Requires-Dist: beautifulsoup4
+Requires-Dist: transformers
+Requires-Dist: torch
+Provides-Extra: dev
+Requires-Dist: hypothesis; extra == "dev"
+Requires-Dist: faker; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+
+# SqueakyCleanText
+Clean your Text for Classical ML and Language Model
+
+# TODO
+- Ability to change the NER MODELS from the config file, which supports AutoModel and AutoTokenizer
+- Add more language support in stopwords

squeakycleantext-0.1.0/README.md

@@ -0,0 +1,6 @@
+# SqueakyCleanText
+Clean your Text for Classical ML and Language Model
+
+# TODO
+- Ability to change the NER MODELS from the config file, which supports AutoModel and AutoTokenizer
+- Add more language support in stopwords

squeakycleantext-0.1.0/SqueakyCleanText.egg-info/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.1
+Name: SqueakyCleanText
+Version: 0.1.0
+Summary: A comprehensive text cleaning and preprocessing pipeline.
+Home-page: https://github.com/rhnfzl/SqueakyCleanText
+Author: Rehan Fazal
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: lingua-language-detector
+Requires-Dist: nltk
+Requires-Dist: emoji
+Requires-Dist: ftfy
+Requires-Dist: Unidecode
+Requires-Dist: beautifulsoup4
+Requires-Dist: transformers
+Requires-Dist: torch
+Provides-Extra: dev
+Requires-Dist: hypothesis; extra == "dev"
+Requires-Dist: faker; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+
+# SqueakyCleanText
+Clean your Text for Classical ML and Language Model
+
+# TODO
+- Ability to change the NER MODELS from the config file, which supports AutoModel and AutoTokenizer
+- Add more language support in stopwords

squeakycleantext-0.1.0/SqueakyCleanText.egg-info/SOURCES.txt

@@ -0,0 +1,26 @@
+LICENSE
+MANIFEST.in
+README.md
+setup.py
+SqueakyCleanText.egg-info/PKG-INFO
+SqueakyCleanText.egg-info/SOURCES.txt
+SqueakyCleanText.egg-info/dependency_links.txt
+SqueakyCleanText.egg-info/entry_points.txt
+SqueakyCleanText.egg-info/requires.txt
+SqueakyCleanText.egg-info/top_level.txt
+sct/__init__.py
+sct/config.py
+sct/sct.py
+sct/scripts/__init__.py
+sct/scripts/download_nltk_stopwords.py
+sct/utils/__init__.py
+sct/utils/constants.py
+sct/utils/contact.py
+sct/utils/datetime.py
+sct/utils/ner.py
+sct/utils/normtext.py
+sct/utils/resources.py
+sct/utils/special.py
+sct/utils/stopwords.py
+tests/__init__.py
+tests/test_sct.py

squeakycleantext-0.1.0/SqueakyCleanText.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

squeakycleantext-0.1.0/SqueakyCleanText.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nltk_downloader = sct.scripts.download_nltk_stopwords:main

squeakycleantext-0.1.0/SqueakyCleanText.egg-info/requires.txt

@@ -0,0 +1,14 @@
+lingua-language-detector
+nltk
+emoji
+ftfy
+Unidecode
+beautifulsoup4
+transformers
+torch
+
+[dev]
+hypothesis
+faker
+flake8
+pytest

squeakycleantext-0.1.0/SqueakyCleanText.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+sct
+tests

squeakycleantext-0.1.0/sct/config.py

@@ -0,0 +1,51 @@
+"""
+    detect_language : to detect the language automatically, but would consume more time if done on a batch
+    fix_bad_unicode : if True, fix "broken" unicode such as mojibake and garbled HTML entities
+    to_ascii_unicode : if True, convert non-to_ascii characters into their closest to_ascii equivalents
+    replace_with_url : special URL token, default "",
+    replace_with_email : special EMAIL token, default "",
+    replace_years : replace year, default "",
+    replace_with_phone_number : special PHONE token, default "",
+    replace_with_number : special NUMBER token, default "",
+    no_currency_symbols : if True, replace all currency symbols with the respective alphabetical ones,
+    ner_process : To execute NER Process to remove the positpositional tags, PER, LOC, ORG, MISC
+    remove_isolated_letters : remove any isolated letters which doesn't add any value to the text
+    remove_isolated_symbols : remove any isolated symbols which shouldn't be present in the text, usually which isn't 
+                            immediatly prefixed and suffixed by letter or number
+    normalize_whitespace : remove any unnecessary whitespace
+    statistical_model_processing : to get the statistical model text, like for fastText, SVM, LR etc
+    casefold : to lower the text
+    remove_stopwords : remove stopwords based on the language, usues NLTK stopwords
+    remove_punctuation : removes all the special symbols
+"""
+
+CHECK_DETECT_LANGUAGE = True
+CHECK_FIX_BAD_UNICODE = True
+CHECK_TO_ASCII_UNICODE = True
+CHECK_REPLACE_HTML = True
+CHECK_REPLACE_URLS = True
+CHECK_REPLACE_EMAILS = True
+CHECK_REPLACE_YEARS = True
+CHECK_REPLACE_PHONE_NUMBERS = True
+CHECK_REPLACE_NUMBERS = True
+CHECK_REPLACE_CURRENCY_SYMBOLS = True
+CHECK_NER_PROCESS = True
+CHECK_REMOVE_CUSTOM_STOP_WORDS = True
+CHECK_REMOVE_ISOLATED_LETTERS = True
+CHECK_REMOVE_ISOLATED_SPECIAL_SYMBOLS = True
+CHECK_NORMALIZE_WHITESPACE = True
+CHECK_STATISTICAL_MODEL_PROCESSING = True
+CHECK_CASEFOLD = True
+CHECK_REMOVE_STOPWORDS = True
+CHECK_REMOVE_PUNCTUATION = True
+CHECK_REMOVE_STEXT_CUSTOM_STOP_WORDS = True
+REPLACE_WITH_URL = ""
+REPLACE_WITH_HTML = ""
+REPLACE_WITH_EMAIL = ""
+REPLACE_WITH_YEARS = ""
+REPLACE_WITH_PHONE_NUMBERS = ""
+REPLACE_WITH_NUMBERS = ""
+REPLACE_WITH_CURRENCY_SYMBOLS = None
+POSITIONAL_TAGS = ['PER', 'LOC']
+NER_CONFIDENCE_THRESHOLD = 0.75
+LANGUAGE = None

squeakycleantext-0.1.0/sct/scripts/download_nltk_stopwords.py

@@ -0,0 +1,7 @@
+import nltk
+
+def main():
+    nltk.download('stopwords')
+
+if __name__ == "__main__":
+    main()

squeakycleantext-0.1.0/sct/sct.py

@@ -0,0 +1,126 @@
+"""
+This code provides a comprehensive text cleaning and preprocessing pipeline. 
+It includes functions to normalize, remove personal information and clean text data, 
+which is crucial for natural language processing tasks.
+"""
+from sct import config
+from sct.utils import contact, datetime, ner, normtext, resources, special, stopwords
+
+class TextCleaner:
+    
+    def __init__(self):
+        self.ProcessContacts = contact.ProcessContacts()
+        self.ProcessDateTime = datetime.ProcessDateTime()
+        self.ProcessSpecialSymbols = special.ProcessSpecialSymbols()
+        self.NormaliseText = normtext.NormaliseText()
+        self.ProcessStopwords = stopwords.ProcessStopwords()
+        self.GeneralNER = ner.GeneralNER()
+        self.pipeline = []
+        self.language = None
+        self.init_pipeline()
+    
+    def init_pipeline(self):
+        # Initialize pipeline steps based on config
+        if config.CHECK_DETECT_LANGUAGE:
+            self.pipeline.append(self.detect_language)
+        
+        if config.CHECK_FIX_BAD_UNICODE:
+            self.pipeline.append(self.fix_bad_unicode)
+        if config.CHECK_TO_ASCII_UNICODE:
+            self.pipeline.append(self.to_ascii_unicode)
+        
+        if config.CHECK_REPLACE_HTML:
+            self.pipeline.append(self.replace_html)
+        if config.CHECK_REPLACE_URLS:
+            self.pipeline.append(self.replace_urls)
+        if config.CHECK_REPLACE_EMAILS:
+            self.pipeline.append(self.replace_emails)
+        if config.CHECK_REPLACE_YEARS:
+            self.pipeline.append(self.replace_years)
+        if config.CHECK_REPLACE_PHONE_NUMBERS:
+            self.pipeline.append(self.replace_phone_numbers)
+        if config.CHECK_REPLACE_NUMBERS:
+            self.pipeline.append(self.replace_numbers)
+        if config.CHECK_REPLACE_CURRENCY_SYMBOLS:
+            self.pipeline.append(self.replace_currency_symbols)
+        
+        if config.CHECK_NER_PROCESS:
+            self.pipeline.append(self.ner_process)
+        
+        if config.CHECK_REMOVE_ISOLATED_LETTERS:
+            self.pipeline.append(self.remove_isolated_letters)
+        if config.CHECK_REMOVE_ISOLATED_SPECIAL_SYMBOLS:
+            self.pipeline.append(self.remove_isolated_special_symbols)
+        if config.CHECK_NORMALIZE_WHITESPACE:
+            self.pipeline.append(self.normalize_whitespace)
+    
+    def process(self, text):
+        text = str(text)
+        
+        for step in self.pipeline:
+            # print(f"Before {step.__name__}: {text}")  # Debug print
+            text = step(text)
+            # print(f"After {step.__name__}: {text}")  # Debug print
+            
+        if config.CHECK_STATISTICAL_MODEL_PROCESSING:
+            stext = self.statistical_model_processing(text)
+            return text, stext, self.language
+        
+        return text, self.language
+
+    def detect_language(self, text):
+        self.language = str(resources.DETECTOR.detect_language_of(text)).split(".")[-1]
+        return text
+
+    def fix_bad_unicode(self, text):
+        return self.NormaliseText.fix_bad_unicode(text)
+
+    def to_ascii_unicode(self, text):
+        return self.NormaliseText.to_ascii_unicode(text)
+
+    def replace_html(self, text):
+        return self.ProcessContacts.replace_html(text, replace_with=config.REPLACE_WITH_HTML)
+
+    def replace_urls(self, text):
+        return self.ProcessContacts.replace_urls(text, replace_with=config.REPLACE_WITH_URL)
+
+    def replace_emails(self, text):
+        return self.ProcessContacts.replace_emails(text, replace_with=config.REPLACE_WITH_EMAIL)
+
+    def replace_years(self, text):
+        return self.ProcessDateTime.replace_years(text, replace_with=config.REPLACE_WITH_YEARS)
+
+    def replace_phone_numbers(self, text):
+        return self.ProcessContacts.replace_phone_numbers(text, replace_with=config.REPLACE_WITH_PHONE_NUMBERS)
+
+    def replace_numbers(self, text):
+        return self.ProcessContacts.replace_numbers(text, replace_with=config.REPLACE_WITH_NUMBERS)
+
+    def replace_currency_symbols(self, text):
+        return self.ProcessSpecialSymbols.replace_currency_symbols(text, replace_with=config.REPLACE_WITH_CURRENCY_SYMBOLS)
+
+    def ner_process(self, text):
+        ner_words = self.GeneralNER.ner_process(text, config.POSITIONAL_TAGS, config.NER_CONFIDENCE_THRESHOLD, self.language)
+        return self.ProcessStopwords.remove_words_from_string(text, ner_words)
+
+    def remove_isolated_letters(self, text):
+        return self.ProcessSpecialSymbols.remove_isolated_letters(text)
+
+    def remove_isolated_special_symbols(self, text):
+        return self.ProcessSpecialSymbols.remove_isolated_special_symbols(text)
+
+    def normalize_whitespace(self, text):
+        return self.NormaliseText.normalize_whitespace(text, no_line_breaks=True)
+
+    def statistical_model_processing(self, text):
+        if config.CHECK_CASEFOLD:
+            stext = text.casefold()  # lowercase
+        if config.CHECK_REMOVE_STOPWORDS:
+            stext = self.ProcessStopwords.remove_stopwords(stext, self.language)
+        if config.CHECK_REMOVE_PUNCTUATION:
+            stext = self.ProcessSpecialSymbols.remove_punctuation(stext)
+        if config.CHECK_REMOVE_ISOLATED_LETTERS:
+            stext = self.ProcessSpecialSymbols.remove_isolated_letters(stext)
+        if config.CHECK_NORMALIZE_WHITESPACE:
+            stext = self.NormaliseText.normalize_whitespace(stext)
+        return stext

squeakycleantext-0.1.0/sct/utils/constants.py

@@ -0,0 +1,131 @@
+"""
+Constant symbols and compiled RegExs use for cleaning.
+"""
+
+import re
+
+CURRENCIES = {
+    "$": "USD",
+    "zł": "PLN",
+    "£": "GBP",
+    "¥": "JPY",
+    "฿": "THB",
+    "₡": "CRC",
+    "₦": "NGN",
+    "₩": "KRW",
+    "₪": "ILS",
+    "₫": "VND",
+    "€": "EUR",
+    "₱": "PHP",
+    "₲": "PYG",
+    "₴": "UAH",
+    "₹": "INR",
+}
+CURRENCY_REGEX = re.compile(
+    "({})+".format("|".join(re.escape(c) for c in CURRENCIES.keys()))
+)
+
+ACRONYM_REGEX = re.compile(
+    r"(?:^|(?<=\W))(?:(?:(?:(?:[A-Z]\.?)+[a-z0-9&/-]?)+(?:[A-Z][s.]?|[0-9]s?))|(?:[0-9](?:\-?[A-Z])+))(?:$|(?=\W))",
+    flags=re.UNICODE,
+)
+
+# taken hostname, domainname, tld from URL regex below
+EMAIL_REGEX = re.compile(
+    r"(?:^|(?<=[^\w@.)]))([\w+-](\.(?!\.))?)*?[\w+-](@|[(<{\[]at[)>}\]])(?:(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)(?:\.(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)*(?:\.(?:[a-z\\u00a1-\\uffff]{2,}))",
+    flags=re.IGNORECASE | re.UNICODE,
+)
+
+# for more information: https://github.com/jfilter/clean-text/issues/10
+# PHONE_REGEX = re.compile(
+#     r"((?:^|(?<=[^\w)]))(((\+?[01])|(\+\d{2}))[ .-]?)?(\(?\d{3,4}\)?/?[ .-]?)?(\d{3}[ .-]?\d{4})(\s?(?:ext\.?|[#x-])\s?\d{2,6})?(?:$|(?=\W)))|\+?\d{4,5}[ .-/]\d{6,9}"
+# )
+# PHONE_REGEX = re.compile(
+#     r"((?:^|(?<=[^\w)]))((\+?[01]|0{1,2}\d{0,1}|\+\d{2})[ .-]?)?(\(?\d{3,4}\)?/?[ .-]?)?(\d{3}[ .-]?\d{4})(\s?(?:ext\.?|[#x-])\s?\d{2,6})?(?:$|(?=\W)))|\+?\d{4,5}[ .-/]\d{6,9}"
+# )
+
+PHONE_REGEX = re.compile(
+    r"((?:^|(?<=[^\w)]))((\+?\d+|0{1,2}\d*?)[ .-]?)?(\(?\d{3,4}\)?/?[ .-]?)?(\d{3}[ .-]?\d{4})(\s?(?:ext\.?|[#x-])\s?\d{2,6})?(?:$|(?=\W)))|\+?\d{4,5}[ .-/]\d{6,9}"
+)
+
+NUMBERS_REGEX = re.compile(
+    r"(?:^|(?<=[^\w,.]))[+–-]?(([1-9]\d{0,2}(,\d{3})+(\.\d*)?)|([1-9]\d{0,2}([ .]\d{3})+(,\d*)?)|(\d*?[.,]\d+)|\d+)(?:$|(?=\b))"
+)
+
+LINEBREAK_REGEX = re.compile(r"((\r\n)|[\n\v])+")
+TWO_LINEBREAK_REGEX = re.compile(r"((\r\n)|[\n\v])+((\r\n)|[\n\v])+")
+MULTI_WHITESPACE_TO_ONE_REGEX = re.compile(r"\s+")
+NONBREAKING_SPACE_REGEX = re.compile(r"(?!\n)\s+")
+
+
+HTML_REGEX = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});', flags=re.UNICODE | re.IGNORECASE,)
+
+# source: https://gist.github.com/dperini/729294
+URL_REGEX = re.compile(
+    r"(?:^|(?<![\w\/\.]))"
+    # protocol identifier
+    # r"(?:(?:https?|ftp)://)" # <-- alt?
+    r"(?:(?:https?:\/\/|ftp:\/\/|www\d{0,3}\.))"
+    # user:pass authentication
+    r"(?:\S+(?::\S*)?@)?" r"(?:"
+    # IP address exclusion
+    # private & local networks
+    r"(?!(?:10|127)(?:\.\d{1,3}){3})"
+    r"(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})"
+    r"(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})"
+    # IP address dotted notation octets
+    # excludes loopback network 0.0.0.0
+    # excludes reserved space >= 224.0.0.0
+    # excludes network & broadcast addresses
+    # (first & last IP address of each class)
+    r"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])"
+    r"(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}"
+    r"(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))"
+    r"|"
+    # host name
+    r"(?:(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)"
+    # domain name
+    r"(?:\.(?:[a-z\\u00a1-\\uffff0-9]-?)*[a-z\\u00a1-\\uffff0-9]+)*"
+    # TLD identifier
+    r"(?:\.(?:[a-z\\u00a1-\\uffff]{2,}))" r"|" r"(?:(localhost))" r")"
+    # port number
+    # r"(?::\d{2,5})?"
+    r"(?::\d{2,5}\b)?"
+    # resource path
+    r"(?:\/[^\)\]\}\s]*)?",
+    # r"(?:$|(?![\w?!+&\/\)]))",
+    flags=re.UNICODE | re.IGNORECASE,
+)
+
+
+strange_double_quotes = [
+    "«",
+    "‹",
+    "»",
+    "›",
+    "„",
+    "“",
+    "‟",
+    "”",
+    "❝",
+    "❞",
+    "❮",
+    "❯",
+    "〝",
+    "〞",
+    "〟",
+    "＂",
+]
+strange_single_quotes = ["‘", "‛", "’", "❛", "❜", "`", "´", "‘", "’"]
+
+DOUBLE_QUOTE_REGEX = re.compile("|".join(strange_double_quotes))
+SINGLE_QUOTE_REGEX = re.compile("|".join(strange_single_quotes))
+
+YEAR_REGEX = re.compile(r"\b(19|20)\d{2}\b") # Matches years from 1900 to 2099
+
+ISOLATED_LETTERS_REGEX = re.compile(r"(?:^|\s)[B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, V, W, X, Y, Z](?=\s|$)", flags=re.UNICODE | re.IGNORECASE)
+
+ISOLATED_SPECIAL_SYMBOLS_REGEX = re.compile(r"(?<![a-zA-Z0-9])[:_.|><;·}@~!?+#)({,/\\\\^]+(?![a-zA-Z0-9])", flags=re.UNICODE | re.IGNORECASE)
+
+
+SENTENCE_BOUNDARY_PATTERN = re.compile('(?<=[.!?])\s+(?=[^\d])')

squeakycleantext-0.1.0/sct/utils/contact.py

@@ -0,0 +1,50 @@
+from sct.utils import constants
+from bs4 import BeautifulSoup
+
+class ProcessContacts:
+    
+    def __init__(self):
+        pass
+    
+    def replace_urls(self, text, replace_with="<URL>"):
+        """
+        Replace all URLs in ``text`` str with ``replace_with`` str.
+        """
+        # matches = constants.URL_REGEX.finditer(text)
+        # result = text
+        # # Iterate through matches in reverse order (to avoid index issues)
+        # for match in reversed(list(matches)):
+        # # Check if the matched substring contains non-ASCII characters
+        #     if not any(ord(char) > 127 for char in match.group()):
+        #         result = text[:match.start()] + replace_with + text[match.end():]
+        return constants.URL_REGEX.sub(replace_with, text)
+
+    def replace_html(self, text, replace_with="<HTML>"):
+        """
+        Replace all html tags in ``text`` str with ``replace_with`` str.
+        """
+        try:
+            soup = BeautifulSoup(text, 'html.parser')
+            text = soup.get_text()
+        except:
+            text = constants.HTML_REGEX.sub(replace_with, text)
+        
+        return text
+    
+    def replace_emails(self, text, replace_with="<EMAIL>"):
+        """
+        Replace all emails in ``text`` str with ``replace_with`` str.
+        """
+        return constants.EMAIL_REGEX.sub(replace_with, text)
+
+    def replace_phone_numbers(self, text, replace_with="<PHONE>"):
+        """
+        Replace all phone numbers in ``text`` str with ``replace_with`` str.
+        """
+        return constants.PHONE_REGEX.sub(replace_with, text)
+
+    def replace_numbers(self, text, replace_with="<NUMBER>"):
+        """
+        Replace all numbers in ``text`` str with ``replace_with`` str.
+        """
+        return constants.NUMBERS_REGEX.sub(replace_with, text)

squeakycleantext-0.1.0/sct/utils/datetime.py

@@ -0,0 +1,15 @@
+from sct.utils import constants
+
+
+class ProcessDateTime:
+    
+    def __init__(self):
+        pass
+    
+    def replace_years(self, text, replace_with ="<YEAR>"):
+        """
+        Replaces years between 1900 to 2099 in the text with a special token.
+        """
+        cleaned_string = constants.YEAR_REGEX.sub(replace_with, text)
+
+        return cleaned_string

squeakycleantext-0.1.0/sct/utils/ner.py

@@ -0,0 +1,147 @@
+import math
+import itertools
+from collections import defaultdict
+
+import transformers
+from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
+
+from sct.utils import constants
+transformers.logging.set_verbosity_error() 
+
+class GeneralNER:
+    
+    """
+    To tag [PER, LOC, ORG, MISC] postional tags using ensemble technique
+    """
+    
+    def __init__(self):
+        
+        #---- NER Models
+        TOKENIZER_EN = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll03-english")
+        MODEL_EN = AutoModelForTokenClassification.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll03-english")
+
+        TOKENIZER_NL = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll02-dutch")
+        MODEL_NL = AutoModelForTokenClassification.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll02-dutch")
+        
+        TOKENIZER_DE = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll03-german")
+        MODEL_DE = AutoModelForTokenClassification.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll03-german")
+        
+        TOKENIZER_ES = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll02-spanish")
+        MODEL_ES = AutoModelForTokenClassification.from_pretrained("FacebookAI/xlm-roberta-large-finetuned-conll02-spanish")
+
+        TOKENIZER_MULTI = AutoTokenizer.from_pretrained("Babelscape/wikineural-multilingual-ner")
+        MODEL_MULTI = AutoModelForTokenClassification.from_pretrained("Babelscape/wikineural-multilingual-ner")
+        
+        # Length with buffer of 10%
+        self.len_single_sentence = [TOKENIZER_EN.max_len_single_sentence, TOKENIZER_MULTI.max_len_single_sentence]
+        self.min_token_length = math.ceil(min(self.len_single_sentence) * 0.9)
+        tokenizer_indicator = self.len_single_sentence.index(min(self.len_single_sentence))
+        
+        if tokenizer_indicator == 0:
+            self.tokenizer = TOKENIZER_EN
+        elif tokenizer_indicator == 1:
+            self.tokenizer = TOKENIZER_MULTI
+            
+        self.nlp_en = pipeline("ner", model=MODEL_EN, tokenizer=TOKENIZER_EN, aggregation_strategy="simple")
+        self.nlp_nl = pipeline("ner", model=MODEL_NL, tokenizer=TOKENIZER_NL, aggregation_strategy="simple")
+        self.nlp_de = pipeline("ner", model=MODEL_DE, tokenizer=TOKENIZER_DE, aggregation_strategy="simple")
+        self.nlp_es = pipeline("ner", model=MODEL_ES, tokenizer=TOKENIZER_ES, aggregation_strategy="simple")
+        self.nlp_multi = pipeline("ner", model=MODEL_MULTI, tokenizer=TOKENIZER_MULTI, aggregation_strategy="simple")
+
+    def ner_data(self, data, pos):
+        """
+        Formats NER (Named Entity Recognition) files.
+        """
+        return [{'entity_group': ix['entity_group'], 'score': ix['score'], 'word': ix['word'], 'key': str(ix['start']) + str(ix['end'])} for ix in data if ix['entity_group'] in pos]
+
+    def ner_ensemble(self, ner_results, t):
+        """
+        Applies an ensemble method for NER.
+        """
+        ner_keys = defaultdict(lambda: [0, 0])
+        ner_words = set()
+
+        for entity in ner_results:
+            ner_keys[entity['key']][0] += 1
+            ner_keys[entity['key']][1] += entity['score']
+
+        ner_keys = [key for key, val in ner_keys.items() if val[1] / val[0] >= t]
+
+        for entity in ner_results:
+            if entity['key'] in ner_keys:
+                ner_words.add(entity['word'])
+
+        ner_words = sorted(list(ner_words), key=len, reverse=True)
+        return ner_words
+    
+    def ner_process(self, text, positional_tags, ner_confidence_threshold, language):
+        """_summary_
+            Executes NER Process to remove the positional tags, PER, LOC, ORG, MISC.
+        Args:
+            text (string): text from which postional tags need to be recognised
+            positional_tags (list): pass tags as ['PER', 'LOC'] (default), also supports 'ORG', 'MISC'
+            ner_confidence_threshold (int): NER Confidence
+            language (string): language model which need to be used, currently supports ENGLISH, DUTCH
+
+        Returns:
+            list: a list of words sorted based on length for the provided positional tags which meets the threshold
+        """
+        ner_results = []
+        
+        # text length
+        text_token_length = len(self.tokenizer.tokenize(text))
+        
+        # parts it need to get split
+        num_parts = math.ceil(text_token_length/self.min_token_length)
+        
+        if num_parts == 0:
+            texts = [text]
+        else:
+            texts = self.split_text(text, self.min_token_length, self.tokenizer)
+            
+        for text in texts:
+            ner_results.append(self.ner_data(self.nlp_multi(text), positional_tags))
+            ner_results.append(self.ner_data(self.nlp_en(text), positional_tags))
+
+            if language == 'DUTCH':
+                ner_results.append(self.ner_data(self.nlp_nl(text), positional_tags))
+            elif language == 'GERMAN':
+                ner_results.append(self.ner_data(self.nlp_de(text), positional_tags))
+            elif language == 'SPANISH':
+                ner_results.append(self.ner_data(self.nlp_es(text), positional_tags))
+                
+        # flat out the list
+        ner_results = list(itertools.chain.from_iterable(ner_results))
+        ner_words = self.ner_ensemble(ner_results, ner_confidence_threshold)
+        return ner_words
+    
+    
+    def split_text(self, text, max_tokens, tokenizer):
+    
+        sentence_boundaries = [(m.start(), m.end()) for m in constants.SENTENCE_BOUNDARY_PATTERN.finditer(text)]
+        
+        chunks = []
+        current_chunk = []
+        current_token_count = 0
+        current_position = 0
+    
+        for boundary_start, boundary_end in sentence_boundaries:
+            sentence = text[current_position:boundary_start+1]
+            current_position = boundary_end
+    
+            token_count = len(tokenizer(sentence)["input_ids"])
+    
+            if current_token_count + token_count <= max_tokens:
+                current_chunk.append(sentence)
+                current_token_count += token_count
+            else:
+                chunks.append(''.join(current_chunk))
+                current_chunk = [sentence]
+                current_token_count = token_count
+    
+        # Append the last sentence
+        last_sentence = text[current_position:]
+        current_chunk.append(last_sentence)
+        chunks.append(''.join(current_chunk))
+    
+        return chunks

squeakycleantext-0.1.0/sct/utils/normtext.py

@@ -0,0 +1,83 @@
+from ftfy import fix_text
+from unidecode import unidecode
+from emoji import demojize, emojize
+#---
+from sct.utils import constants
+
+
+class NormaliseText:
+    
+    def __init__(self):
+        
+        pass
+    
+    def fix_bad_unicode(self, text, normalization="NFC"):
+        """
+        Fix unicode text that's "broken" using `ftfy <http://ftfy.readthedocs.org/>`_;
+        this includes mojibake, HTML entities and other code cruft,
+        and non-standard forms for display purposes.
+        Args:
+            text (str): raw text
+            normalization ({'NFC', 'NFKC', 'NFD', 'NFKD'}): if 'NFC',
+                combines characters and diacritics written using separate code points,
+                e.g. converting "e" plus an acute accent modifier into "é"; unicode
+                can be converted to NFC form without any change in its meaning!
+                if 'NFKC', additional normalizations are applied that can change
+                the meanings of characters, e.g. ellipsis characters will be replaced
+                with three periods
+        """
+        # trying to fix backslash-replaced strings (via https://stackoverflow.com/a/57192592/4028896)
+        try:
+            text = text.encode("latin", "backslashreplace").decode("unicode-escape")
+        except:
+            pass
+
+        return fix_text(text, normalization=normalization)
+
+    def fix_strange_quotes(self, text):
+        """
+        Replace strange quotes, i.e., 〞with a single quote ' or a double quote " if it fits better.
+        """
+        text = constants.SINGLE_QUOTE_REGEX.sub("'", text)
+        text = constants.DOUBLE_QUOTE_REGEX.sub('"', text)
+        return text
+
+    def to_ascii_unicode(self, text, no_emoji=True):
+        """
+        Try to represent unicode data in ascii characters similar to what a human
+        with a US keyboard would choose.
+        Works great for languages of Western origin, worse the farther the language
+        gets from Latin-based alphabets. It's based on hand-tuned character mappings
+        that also contain ascii approximations for symbols and non-Latin alphabets.
+        """
+        # normalize quotes before since this improves transliteration quality
+        text = self.fix_strange_quotes(text)
+
+        if not no_emoji:
+            text = demojize(text, use_aliases=True)
+
+        text = unidecode(text)
+
+        return text
+
+    def normalize_whitespace(self, text, strip_lines=True, no_line_breaks=False, keep_two_line_breaks=False):
+        """
+        Given ``text`` str, replace one or more spacings with a single space, and one
+        or more line breaks with a single newline. Also strip leading/trailing whitespace.
+        """
+        if strip_lines:
+            text = "\n".join([x.strip() for x in text.splitlines()])
+
+        if no_line_breaks:
+            text = constants.MULTI_WHITESPACE_TO_ONE_REGEX.sub(" ", text)
+        else:
+            if keep_two_line_breaks:
+                text = constants.NONBREAKING_SPACE_REGEX.sub(
+                    " ", constants.TWO_LINEBREAK_REGEX.sub(r"\n\n", text)
+                )
+            else:
+                text = constants.NONBREAKING_SPACE_REGEX.sub(
+                    " ", constants.LINEBREAK_REGEX.sub(r"\n", text)
+                )
+
+        return text.strip()

squeakycleantext-0.1.0/sct/utils/resources.py

@@ -0,0 +1,8 @@
+import warnings
+warnings.filterwarnings('ignore')
+
+from lingua import Language, LanguageDetectorBuilder
+
+#---- Detect the Language / Also add languages to support in Future
+LANGUAGES = [Language.DUTCH, Language.ENGLISH]
+DETECTOR = LanguageDetectorBuilder.from_languages(*LANGUAGES).build()

squeakycleantext-0.1.0/sct/utils/special.py

@@ -0,0 +1,50 @@
+import re
+import string
+#---
+from sct.utils import constants
+
+class ProcessSpecialSymbols:
+    
+    def __init__(self):
+        pass
+    
+    def replace_currency_symbols(self, text, replace_with="<CUR>"):
+        """
+        Replace currency symbols in ``text`` str with string specified by ``replace_with`` str.
+        Args:
+            text (str): raw text
+            replace_with (str): if None (default), replace symbols with
+                their standard 3-letter abbreviations (e.g. '$' with 'USD', '£' with 'GBP');
+                otherwise, pass in a string with which to replace all symbols
+                (e.g. "*CURRENCY*")
+        """
+        if replace_with is None:
+            for k, v in constants.CURRENCIES.items():
+                text = text.replace(k, v)
+            return text
+        else:
+            return constants.CURRENCY_REGEX.sub(replace_with, text)
+        
+        
+    def remove_isolated_letters(self, text):
+        """
+        Removes any isolated letters which doesn't add any value to the text.
+        """
+        cleaned_text = constants.ISOLATED_LETTERS_REGEX.sub('', text)
+        
+        return cleaned_text
+
+    def remove_isolated_special_symbols(self, text):
+        """
+        Removes any isolated symbols which shouldn't be present in the text.
+        """
+        cleaned_text = re.sub(r'\[[^\]]+\]', '', text) # to remove [] content, usianlly they are image or file loc text
+        cleaned_text = re.sub(r'\{[^}]+\}', '', cleaned_text) # to remove {}} content, usianlly they are html links
+        cleaned_text = constants.ISOLATED_SPECIAL_SYMBOLS_REGEX.sub('', cleaned_text)
+        cleaned_text = re.sub(r"(?<![a-zA-Z0-9])['\"\-*%](?![a-zA-Z0-9])", '', cleaned_text, flags=re.UNICODE | re.IGNORECASE)
+        
+        return cleaned_text
+    
+    def remove_punctuation(self, text):
+        chars = re.escape(string.punctuation)
+        return re.sub('['+chars+']', '',text)

squeakycleantext-0.1.0/sct/utils/stopwords.py

@@ -0,0 +1,35 @@
+import re
+from nltk.corpus import stopwords as sw
+
+class ProcessStopwords:
+    
+    def __init__(self):
+        ### stop words
+        self.STOP_WORDS_EN = sw.words('english')
+        self.STOP_WORDS_NL = sw.words('dutch')
+        self.STOP_WORDS_DE = sw.words('german')
+        self.STOP_WORDS_ES = sw.words('spanish')
+
+    def remove_stopwords(self, text, lan):
+        """
+        Removes stopwords based on the language.
+        """
+        if lan == 'DUTCH':
+            stop_words = self.STOP_WORDS_NL
+        elif lan == 'ENGLISH':
+            stop_words = self.STOP_WORDS_EN
+        elif lan == 'GERMAN':
+            stop_words = self.STOP_WORDS_DE
+        elif lan == 'SPANISH':
+            stop_words = self.STOP_WORDS_ES
+
+        text = text.split()
+        return " ".join([word for word in text if word not in stop_words])
+
+    def remove_words_from_string(self, text, words_to_remove):
+        # Join the words with the "|" (OR) operator in regex to create a pattern
+        pattern = r'\b(?:' + '|'.join(re.escape(word) for word in words_to_remove) + r')\b'
+        
+        # Use re.sub() to replace all matches with an empty string
+        result_text = re.sub(pattern, '', text)
+        return result_text

squeakycleantext-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

squeakycleantext-0.1.0/setup.py

@@ -0,0 +1,43 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='SqueakyCleanText',
+    version='0.1.0',
+    author='Rehan Fazal',
+    description='A comprehensive text cleaning and preprocessing pipeline.',
+    long_description=open('README.md').read(),
+    long_description_content_type='text/markdown',
+    url='https://github.com/rhnfzl/SqueakyCleanText',
+    license='MIT',
+    packages=find_packages(),
+    install_requires=[
+        'lingua-language-detector',
+        'nltk',
+        'emoji',
+        'ftfy',
+        'Unidecode',
+        'beautifulsoup4',
+        'transformers',
+        'torch',
+    ],
+    extras_require={
+        'dev': [
+            'hypothesis',
+            'faker',
+            'flake8',
+            'pytest',
+        ],
+    },
+    classifiers=[
+        'Programming Language :: Python :: 3',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: OS Independent',
+    ],
+    python_requires='>=3.7',
+    entry_points={
+        'console_scripts': [
+            'nltk_downloader=sct.scripts.download_nltk_stopwords:main'
+        ],
+    },
+    test_suite='tests',
+)

squeakycleantext-0.1.0/tests/test_sct.py

@@ -0,0 +1,84 @@
+import unittest
+import random
+import string
+from hypothesis import given, settings
+from hypothesis.strategies import text, from_regex
+from faker import Faker
+from sct import config
+from sct.utils import contact, datetime, special, normtext, stopwords, constants
+
+class TextCleanerTest(unittest.TestCase):
+    
+    @classmethod
+    def setUpClass(cls):
+        config.CHECK_NER_PROCESS = False
+        cls.ProcessContacts = contact.ProcessContacts()
+        cls.ProcessDateTime = datetime.ProcessDateTime()
+        cls.ProcessSpecialSymbols = special.ProcessSpecialSymbols()
+        cls.NormaliseText = normtext.NormaliseText()
+        cls.ProcessStopwords = stopwords.ProcessStopwords()
+        cls.fake = Faker()
+
+    @settings(deadline=None)
+    @given(from_regex(constants.EMAIL_REGEX, fullmatch=True))
+    def test_email_regex(self, rx):
+        self.assertEqual("", self.ProcessContacts.replace_emails(rx, ""))
+        
+    @settings(deadline=None)
+    @given(from_regex(constants.PHONE_REGEX, fullmatch=True))
+    def test_phone_regex(self, rx):
+        self.assertEqual("", self.ProcessContacts.replace_phone_numbers(rx, ""))
+        
+    @settings(deadline=None)
+    @given(from_regex(constants.NUMBERS_REGEX, fullmatch=True))
+    def test_number_regex(self, rx):
+        self.assertEqual("", self.ProcessContacts.replace_numbers(rx, ""))
+        
+    @settings(deadline=None)
+    @given(from_regex(constants.URL_REGEX, fullmatch=True))
+    def test_url_regex(self, rx):
+        self.assertNotEqual(rx, self.ProcessContacts.replace_urls(rx, ""))
+        
+    @settings(deadline=None)
+    @given(from_regex(constants.YEAR_REGEX, fullmatch=True))
+    def test_year_regex(self, rx):
+        self.assertEqual("", self.ProcessDateTime.replace_years(rx, ""))
+        
+    @settings(deadline=None)
+    @given(from_regex(constants.ISOLATED_LETTERS_REGEX, fullmatch=True))
+    def test_isolated_letters_regex(self, rx):
+        rx = self.ProcessSpecialSymbols.remove_isolated_letters(rx)
+        rx = self.NormaliseText.normalize_whitespace(rx)
+        self.assertEqual("", rx)
+        
+    @settings(deadline=None)
+    @given(from_regex(constants.ISOLATED_SPECIAL_SYMBOLS_REGEX, fullmatch=True))
+    def test_isolated_symbols_regex(self, rx):
+        self.assertEqual("", self.ProcessSpecialSymbols.remove_isolated_special_symbols(rx))
+
+    @settings(deadline=None)
+    @given(text(alphabet=string.ascii_letters + string.digits, min_size=0, max_size=4))
+    def test_faker_email(self, fkw):
+        """Check if generated emails are replaced correctly."""
+        email = self.fake.email()
+        clean_email = self.ProcessContacts.replace_emails(email, replace_with=fkw)
+        self.assertEqual(clean_email, fkw)
+
+    @settings(deadline=None)
+    @given(text(alphabet=string.ascii_letters + string.digits, min_size=0, max_size=4))
+    def test_faker_phone_number(self, fkw):
+        """Check if generated phone numbers are replaced correctly."""
+        phonenum = self.fake.phone_number()
+        clean_phonenum = self.ProcessContacts.replace_phone_numbers(phonenum, replace_with=fkw)
+        self.assertEqual(clean_phonenum, fkw)
+        
+    @settings(deadline=None)
+    @given(text(alphabet=string.ascii_letters + string.digits, min_size=0, max_size=4))
+    def test_faker_url(self, fkw):
+        """Check if generated URLs are replaced correctly."""
+        url = self.fake.url()
+        clean_url = self.ProcessContacts.replace_urls(url, replace_with=fkw)
+        self.assertEqual(clean_url, fkw)
+        
+if __name__ == "__main__":
+    unittest.main(verbosity=2)