nlp-text-preprocessing 0.2.1__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {nlp_text_preprocessing-0.2.1/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.3.0}/PKG-INFO +76 -58
  2. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/Readme.md +37 -29
  3. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/__init__.py +18 -7
  4. nlp_text_preprocessing-0.3.0/nlp_text_preprocessing/all_text_preprocess.py +1091 -0
  5. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info}/PKG-INFO +76 -58
  6. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +5 -2
  7. nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info/requires.txt +27 -0
  8. nlp_text_preprocessing-0.3.0/pyproject.toml +68 -0
  9. nlp_text_preprocessing-0.3.0/requirements.txt +7 -0
  10. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/tests/test_all_text_preprocess.py +1 -1
  11. nlp_text_preprocessing-0.3.0/tests/test_cleaning.py +168 -0
  12. nlp_text_preprocessing-0.3.0/tests/test_infrastructure.py +110 -0
  13. nlp_text_preprocessing-0.3.0/tests/test_linguistic.py +183 -0
  14. nlp_text_preprocessing-0.2.1/nlp_text_preprocessing/all_text_preprocess.py +0 -720
  15. nlp_text_preprocessing-0.2.1/nlp_text_preprocessing.egg-info/requires.txt +0 -14
  16. nlp_text_preprocessing-0.2.1/requirements.txt +0 -14
  17. nlp_text_preprocessing-0.2.1/setup.py +0 -34
  18. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/LICENSE +0 -0
  19. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/MANIFEST.in +0 -0
  20. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/architecture_diagram.png +0 -0
  21. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_banner.png +0 -0
  22. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/data/contractions.json +0 -0
  23. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
  24. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
  25. {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/setup.cfg +0 -0
@@ -1,44 +1,54 @@
1
1
  Metadata-Version: 2.4
2
- Name: nlp_text_preprocessing
3
- Version: 0.2.1
4
- Summary: This is a Text Processing Package For NLP
5
- Home-page: https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
6
- Author: Uditya Narayan Tiwari
7
- Author-email: tiwarimerit@gmail.com
8
- Project-URL: Author Website, https://udityanarayantiwari.netlify.app/
2
+ Name: nlp-text-preprocessing
3
+ Version: 0.3.0
4
+ Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
5
+ Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
9
8
  Project-URL: Source Code, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
10
9
  Project-URL: Bug Tracker, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/issues
11
- Classifier: Programming Language :: Python :: 3
12
- Classifier: License :: OSI Approved :: MIT License
10
+ Project-URL: Author Website, https://udityanarayantiwari.netlify.app/
11
+ Keywords: nlp,text-preprocessing,text-cleaning,spacy,nltk,textblob
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Education
15
+ Classifier: Intended Audience :: Science/Research
13
16
  Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.9
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Classifier: Topic :: Text Processing :: Linguistic
14
25
  Requires-Python: >=3.9
15
26
  Description-Content-Type: text/markdown
16
27
  License-File: LICENSE
17
28
  Requires-Dist: spacy
18
29
  Requires-Dist: textblob
19
- Requires-Dist: beautifulsoup4
20
30
  Requires-Dist: nltk
21
- Requires-Dist: openpyxl
22
- Requires-Dist: SpeechRecognition==3.10.4
23
- Requires-Dist: pyaudio==0.2.14
24
- Requires-Dist: PrettyTable
25
- Requires-Dist: scikit-learn
26
- Requires-Dist: wordcloud
31
+ Requires-Dist: beautifulsoup4
27
32
  Requires-Dist: lxml
28
- Requires-Dist: pandas
29
- Requires-Dist: numpy
30
- Requires-Dist: matplotlib
31
- Dynamic: author
32
- Dynamic: author-email
33
- Dynamic: classifier
34
- Dynamic: description
35
- Dynamic: description-content-type
36
- Dynamic: home-page
33
+ Requires-Dist: emoji>=2.0
34
+ Requires-Dist: langdetect
35
+ Provides-Extra: pandas
36
+ Requires-Dist: pandas; extra == "pandas"
37
+ Provides-Extra: viz
38
+ Requires-Dist: wordcloud; extra == "viz"
39
+ Requires-Dist: matplotlib; extra == "viz"
40
+ Provides-Extra: all
41
+ Requires-Dist: pandas; extra == "all"
42
+ Requires-Dist: wordcloud; extra == "all"
43
+ Requires-Dist: matplotlib; extra == "all"
44
+ Provides-Extra: dev
45
+ Requires-Dist: pytest; extra == "dev"
46
+ Requires-Dist: build; extra == "dev"
47
+ Requires-Dist: twine; extra == "dev"
48
+ Requires-Dist: pandas; extra == "dev"
49
+ Requires-Dist: wordcloud; extra == "dev"
50
+ Requires-Dist: matplotlib; extra == "dev"
37
51
  Dynamic: license-file
38
- Dynamic: project-url
39
- Dynamic: requires-dist
40
- Dynamic: requires-python
41
- Dynamic: summary
42
52
 
43
53
  # NLP Text Preprocessing Utility Package
44
54
 
@@ -92,19 +102,19 @@ Dynamic: summary
92
102
 
93
103
  ## Package Overview
94
104
 
95
- **`nlp_text_preprocessing`** is a production-grade, modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
105
+ **`nlp_text_preprocessing`** is a modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
96
106
 
97
107
  Maintained by **[Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)**.
98
108
 
99
- It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with built-in defensive guards for `None`, `NaN`, and missing values, making it 100% crash-proof when running over Pandas DataFrames or large raw text datasets.
109
+ It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with defensive guards so `None`, `NaN` and missing values are treated as empty text, which makes it convenient to run over Pandas DataFrame columns and raw text datasets.
100
110
 
101
111
  ---
102
112
 
103
113
  ## Key Features
104
114
 
105
- - **Crash-Proof Pipeline**: Automatic `None`/`NaN` input sanitation across all functions.
115
+ - **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
106
116
  - **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
107
- - **Lazy and Fast Model Caching**: Efficient lazy-loading for spaCy (`en_core_web_sm`) and cached NaiveBayes sentiment analyzers.
117
+ - **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
108
118
  - **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
109
119
  - **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
110
120
  - **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
@@ -128,20 +138,27 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
128
138
  ### 1. Installation
129
139
 
130
140
  ```bash
131
- pip install nlp-text-preprocessing
141
+ pip install nlp-text-preprocessing # core
142
+ pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
143
+ pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
144
+ pip install "nlp-text-preprocessing[all]" # everything optional
132
145
  ```
133
146
 
134
147
  ### 2. Downloading Required Corpora and Models
135
148
 
136
- To automatically download all required NLTK corpora (`stopwords`, `movie_reviews`, `brown`) and spaCy models (`en_core_web_sm`), run:
149
+ Nothing is downloaded behind your back. Run the one-time setup, which fetches the NLTK data (`punkt_tab`, `wordnet`, `omw-1.4`) and the spaCy model `en_core_web_sm`:
137
150
 
138
151
  ```python
139
152
  import nlp_text_preprocessing as tp
140
153
 
141
- # One-time automated setup for NLTK and spaCy resources
142
- tp.download_nltk_packages()
154
+ tp.download_nltk_packages() # one-time setup
155
+ tp.check_resources(verbose=True) # see what is installed / missing
143
156
  ```
144
157
 
158
+ `sentiment_analysis(x, method="naive_bayes")` additionally needs the `movie_reviews` corpus:
159
+ `tp.download_nltk_packages(include_sentiment_corpus=True)`.
160
+ Set the environment variable `NLP_TEXT_PREPROCESSING_AUTO_DOWNLOAD=1` if you want the spaCy model to be downloaded automatically on first use.
161
+
145
162
  ---
146
163
 
147
164
  ## Pipeline Automation Engine
@@ -212,29 +229,29 @@ Below is the complete reference guide for all functions available in the library
212
229
  | `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
213
230
  | `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
214
231
  | `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
215
- | `remove_urls(x)` | `x: str` | `str` | Removes HTTP/HTTPS links and `www` URLs. |
232
+ | `remove_urls(x)` | `x: str` | `str` | Removes `http(s)://`, `ftp://` and `www.` URLs (trailing punctuation is kept). |
216
233
  | `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
217
- | `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user`). |
234
+ | `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user:`). |
218
235
  | `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
219
236
  | `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
220
- | `remove_accented_chars(x)` | `x: str` | `str` | Normalizes accented characters (NFKD -> ASCII). |
237
+ | `remove_accented_chars(x)` | `x: str` | `str` | Strips accents (`café` -> `cafe`); non-Latin scripts are kept. |
221
238
  | `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
222
- | `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and special symbols. |
223
- | `remove_repeated_chars(x)` | `x: str` | `str` | Truncates repeated characters beyond 2 consecutive occurrences. |
224
- | `remove_stop_words(x)` | `x: str` | `str` | Case-insensitive removal of stop words while preserving word case. |
225
- | `remove_emojis(x)` | `x: str` | `str` | Strips emoji characters from text. |
239
+ | `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and symbols (`well-known` -> `well known`). |
240
+ | `remove_repeated_chars(x)` | `x: str` | `str` | Shortens elongated words (`soooo` -> `soo`); digits and punctuation are untouched. |
241
+ | `remove_stop_words(x, keep_negations=True)` | `x: str` | `str` | Removes stop words (case-insensitive); negations like `not` are kept by default. |
242
+ | `remove_emojis(x)` | `x: str` | `str` | Strips emoji (flags, skin tones and ZWJ sequences included). |
226
243
  | `extract_emojis(x)` | `x: str` | `list[str]` | Returns list of emojis found in text. |
227
244
  | `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
228
- | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers from text. |
245
+ | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
229
246
  | `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
230
- | `mask_pii(x)` | `x: str` | `str` | Redacts emails, phone numbers, and card numbers with `[REDACTED]`. |
247
+ | `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
231
248
  | `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
232
- | `expand_hashtags(x)` | `x: str` | `str` | Splits camelCase/PascalCase hashtags (`#MachineLearning` -> `Machine Learning`). |
249
+ | `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
233
250
  | `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
234
- | `remove_numbers(x)` | `x: str` | `str` | Strips all numeric digits. |
235
- | `remove_currency_symbols(x)`| `x: str` | `str` | Strips currency symbols (`$`, `€`, `₹`, `£`, `¥`). |
251
+ | `remove_numbers(x)` | `x: str` | `str` | Removes standalone numbers; digits inside words (`3rd`, `H2O`) stay. |
252
+ | `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
236
253
  | `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
237
- | `clean_text(text)` | `text: str` | `str` | Runs end-to-end cleaning pipeline. |
254
+ | `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
238
255
 
239
256
  ---
240
257
 
@@ -247,15 +264,15 @@ Below is the complete reference guide for all functions available in the library
247
264
  | `extract_entities(x)` | `x: str` | `list[tuple]` | Extracts named entities `(text, label)` using spaCy. |
248
265
  | `extract_pos_tags(x)` | `x: str` | `list[tuple]` | Extracts full POS tag breakdown `(token, pos, tag)` using spaCy. |
249
266
  | `get_dependency_tree(x)`| `x: str` | `list[tuple]` | Extracts dependency parse tree `(token, dep, head)` using spaCy. |
250
- | `detect_language(x)` | `x: str` | `str` | Detects language code (`'en'`, `'fr'`, etc.) using TextBlob. |
267
+ | `detect_language(x)` | `x: str` | `str` | Language code (`'en'`, `'fr'`, `'hi'`, ...) via `langdetect`; `'unknown'` if undetectable. |
251
268
  | `keyword_extraction(x, top_n)`| `x: str, top_n: int`| `list[str]`| Extracts top N keywords by term frequency. |
252
269
  | `text_similarity(x1, x2)` | `x1, x2` | `float` | Calculates token Jaccard similarity between two texts (0.0 to 1.0). |
253
- | `get_synonyms(word)` | `word: str` | `list[str]` | Returns list of synonyms from WordNet. |
254
- | `get_antonyms(word)` | `word: str` | `list[str]` | Returns list of antonyms from WordNet. |
255
- | `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | Extracts noun phrases with character start/end spans. |
256
- | `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob model. |
270
+ | `get_synonyms(word)` | `word: str` | `list[str]` | Sorted WordNet synonyms. |
271
+ | `get_antonyms(word)` | `word: str` | `list[str]` | Sorted WordNet antonyms. |
272
+ | `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | spaCy noun chunks with exact character `start`/`end` offsets. |
273
+ | `correct_spelling(x)` | `x: str` | `str` | Corrects misspelled words (TextBlob); punctuation, case, URLs and acronyms are preserved. |
257
274
  | `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
258
- | `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
275
+ | `n_grams(x, n=2)` | `x: str, n: int` | `list[tuple]` | Word-level n-grams as tuples. |
259
276
  | `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
260
277
  | `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
261
278
 
@@ -269,7 +286,7 @@ Below is the complete reference guide for all functions available in the library
269
286
  | `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
270
287
  | `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
271
288
  | `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
272
- | `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using cached NaiveBayes model. |
289
+ | `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
273
290
  | `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
274
291
  | `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
275
292
 
@@ -363,10 +380,11 @@ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
363
380
 
364
381
  ## Troubleshooting and Resource Setup
365
382
 
366
- If any NLTK corpus or spaCy model throws a missing resource error during runtime, simply run:
383
+ If a function reports a missing NLTK corpus or spaCy model, check what is missing and install it:
367
384
 
368
385
  ```python
369
386
  import nlp_text_preprocessing as tp
387
+ tp.check_resources(verbose=True)
370
388
  tp.download_nltk_packages()
371
389
  ```
372
390
 
@@ -50,19 +50,19 @@
50
50
 
51
51
  ## Package Overview
52
52
 
53
- **`nlp_text_preprocessing`** is a production-grade, modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
53
+ **`nlp_text_preprocessing`** is a modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
54
54
 
55
55
  Maintained by **[Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)**.
56
56
 
57
- It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with built-in defensive guards for `None`, `NaN`, and missing values, making it 100% crash-proof when running over Pandas DataFrames or large raw text datasets.
57
+ It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with defensive guards so `None`, `NaN` and missing values are treated as empty text, which makes it convenient to run over Pandas DataFrame columns and raw text datasets.
58
58
 
59
59
  ---
60
60
 
61
61
  ## Key Features
62
62
 
63
- - **Crash-Proof Pipeline**: Automatic `None`/`NaN` input sanitation across all functions.
63
+ - **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
64
64
  - **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
65
- - **Lazy and Fast Model Caching**: Efficient lazy-loading for spaCy (`en_core_web_sm`) and cached NaiveBayes sentiment analyzers.
65
+ - **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
66
66
  - **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
67
67
  - **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
68
68
  - **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
@@ -86,20 +86,27 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
86
86
  ### 1. Installation
87
87
 
88
88
  ```bash
89
- pip install nlp-text-preprocessing
89
+ pip install nlp-text-preprocessing # core
90
+ pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
91
+ pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
92
+ pip install "nlp-text-preprocessing[all]" # everything optional
90
93
  ```
91
94
 
92
95
  ### 2. Downloading Required Corpora and Models
93
96
 
94
- To automatically download all required NLTK corpora (`stopwords`, `movie_reviews`, `brown`) and spaCy models (`en_core_web_sm`), run:
97
+ Nothing is downloaded behind your back. Run the one-time setup, which fetches the NLTK data (`punkt_tab`, `wordnet`, `omw-1.4`) and the spaCy model `en_core_web_sm`:
95
98
 
96
99
  ```python
97
100
  import nlp_text_preprocessing as tp
98
101
 
99
- # One-time automated setup for NLTK and spaCy resources
100
- tp.download_nltk_packages()
102
+ tp.download_nltk_packages() # one-time setup
103
+ tp.check_resources(verbose=True) # see what is installed / missing
101
104
  ```
102
105
 
106
+ `sentiment_analysis(x, method="naive_bayes")` additionally needs the `movie_reviews` corpus:
107
+ `tp.download_nltk_packages(include_sentiment_corpus=True)`.
108
+ Set the environment variable `NLP_TEXT_PREPROCESSING_AUTO_DOWNLOAD=1` if you want the spaCy model to be downloaded automatically on first use.
109
+
103
110
  ---
104
111
 
105
112
  ## Pipeline Automation Engine
@@ -170,29 +177,29 @@ Below is the complete reference guide for all functions available in the library
170
177
  | `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
171
178
  | `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
172
179
  | `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
173
- | `remove_urls(x)` | `x: str` | `str` | Removes HTTP/HTTPS links and `www` URLs. |
180
+ | `remove_urls(x)` | `x: str` | `str` | Removes `http(s)://`, `ftp://` and `www.` URLs (trailing punctuation is kept). |
174
181
  | `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
175
- | `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user`). |
182
+ | `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user:`). |
176
183
  | `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
177
184
  | `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
178
- | `remove_accented_chars(x)` | `x: str` | `str` | Normalizes accented characters (NFKD -> ASCII). |
185
+ | `remove_accented_chars(x)` | `x: str` | `str` | Strips accents (`café` -> `cafe`); non-Latin scripts are kept. |
179
186
  | `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
180
- | `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and special symbols. |
181
- | `remove_repeated_chars(x)` | `x: str` | `str` | Truncates repeated characters beyond 2 consecutive occurrences. |
182
- | `remove_stop_words(x)` | `x: str` | `str` | Case-insensitive removal of stop words while preserving word case. |
183
- | `remove_emojis(x)` | `x: str` | `str` | Strips emoji characters from text. |
187
+ | `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and symbols (`well-known` -> `well known`). |
188
+ | `remove_repeated_chars(x)` | `x: str` | `str` | Shortens elongated words (`soooo` -> `soo`); digits and punctuation are untouched. |
189
+ | `remove_stop_words(x, keep_negations=True)` | `x: str` | `str` | Removes stop words (case-insensitive); negations like `not` are kept by default. |
190
+ | `remove_emojis(x)` | `x: str` | `str` | Strips emoji (flags, skin tones and ZWJ sequences included). |
184
191
  | `extract_emojis(x)` | `x: str` | `list[str]` | Returns list of emojis found in text. |
185
192
  | `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
186
- | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers from text. |
193
+ | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
187
194
  | `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
188
- | `mask_pii(x)` | `x: str` | `str` | Redacts emails, phone numbers, and card numbers with `[REDACTED]`. |
195
+ | `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
189
196
  | `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
190
- | `expand_hashtags(x)` | `x: str` | `str` | Splits camelCase/PascalCase hashtags (`#MachineLearning` -> `Machine Learning`). |
197
+ | `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
191
198
  | `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
192
- | `remove_numbers(x)` | `x: str` | `str` | Strips all numeric digits. |
193
- | `remove_currency_symbols(x)`| `x: str` | `str` | Strips currency symbols (`$`, `€`, `₹`, `£`, `¥`). |
199
+ | `remove_numbers(x)` | `x: str` | `str` | Removes standalone numbers; digits inside words (`3rd`, `H2O`) stay. |
200
+ | `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
194
201
  | `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
195
- | `clean_text(text)` | `text: str` | `str` | Runs end-to-end cleaning pipeline. |
202
+ | `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
196
203
 
197
204
  ---
198
205
 
@@ -205,15 +212,15 @@ Below is the complete reference guide for all functions available in the library
205
212
  | `extract_entities(x)` | `x: str` | `list[tuple]` | Extracts named entities `(text, label)` using spaCy. |
206
213
  | `extract_pos_tags(x)` | `x: str` | `list[tuple]` | Extracts full POS tag breakdown `(token, pos, tag)` using spaCy. |
207
214
  | `get_dependency_tree(x)`| `x: str` | `list[tuple]` | Extracts dependency parse tree `(token, dep, head)` using spaCy. |
208
- | `detect_language(x)` | `x: str` | `str` | Detects language code (`'en'`, `'fr'`, etc.) using TextBlob. |
215
+ | `detect_language(x)` | `x: str` | `str` | Language code (`'en'`, `'fr'`, `'hi'`, ...) via `langdetect`; `'unknown'` if undetectable. |
209
216
  | `keyword_extraction(x, top_n)`| `x: str, top_n: int`| `list[str]`| Extracts top N keywords by term frequency. |
210
217
  | `text_similarity(x1, x2)` | `x1, x2` | `float` | Calculates token Jaccard similarity between two texts (0.0 to 1.0). |
211
- | `get_synonyms(word)` | `word: str` | `list[str]` | Returns list of synonyms from WordNet. |
212
- | `get_antonyms(word)` | `word: str` | `list[str]` | Returns list of antonyms from WordNet. |
213
- | `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | Extracts noun phrases with character start/end spans. |
214
- | `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob model. |
218
+ | `get_synonyms(word)` | `word: str` | `list[str]` | Sorted WordNet synonyms. |
219
+ | `get_antonyms(word)` | `word: str` | `list[str]` | Sorted WordNet antonyms. |
220
+ | `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | spaCy noun chunks with exact character `start`/`end` offsets. |
221
+ | `correct_spelling(x)` | `x: str` | `str` | Corrects misspelled words (TextBlob); punctuation, case, URLs and acronyms are preserved. |
215
222
  | `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
216
- | `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
223
+ | `n_grams(x, n=2)` | `x: str, n: int` | `list[tuple]` | Word-level n-grams as tuples. |
217
224
  | `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
218
225
  | `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
219
226
 
@@ -227,7 +234,7 @@ Below is the complete reference guide for all functions available in the library
227
234
  | `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
228
235
  | `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
229
236
  | `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
230
- | `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using cached NaiveBayes model. |
237
+ | `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
231
238
  | `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
232
239
  | `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
233
240
 
@@ -321,10 +328,11 @@ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
321
328
 
322
329
  ## Troubleshooting and Resource Setup
323
330
 
324
- If any NLTK corpus or spaCy model throws a missing resource error during runtime, simply run:
331
+ If a function reports a missing NLTK corpus or spaCy model, check what is missing and install it:
325
332
 
326
333
  ```python
327
334
  import nlp_text_preprocessing as tp
335
+ tp.check_resources(verbose=True)
328
336
  tp.download_nltk_packages()
329
337
  ```
330
338
 
@@ -1,9 +1,15 @@
1
1
  # entry level code and import text preprocess
2
2
 
3
+ __version__ = "0.3.0"
4
+
3
5
  from .all_text_preprocess import *
4
6
 
5
7
  __all__ = [
8
+ '__version__',
6
9
  'download_nltk_packages',
10
+ 'check_resources',
11
+ 'get_spacy_nlp',
12
+ 'get_stop_words',
7
13
  'word_count',
8
14
  'char_count',
9
15
  'avg_word_len',
@@ -96,16 +102,21 @@ def extract_features(x):
96
102
  }
97
103
 
98
104
 
99
- def clean_text(text):
105
+ def clean_text(text, remove_stopwords=False):
100
106
  """
101
- Applies a sequence of cleaning and normalization steps on text.
107
+ End-to-end cleaning: HTML, URLs and emails are removed, text is lower-cased,
108
+ contractions are expanded, punctuation is dropped and words are lemmatized.
109
+
110
+ Set remove_stopwords=True to also drop stop words (negations are kept).
102
111
  """
112
+ text = remove_html_tag(text)
113
+ text = remove_urls(text)
114
+ text = remove_emails(text)
103
115
  text = to_lower_case(text)
104
116
  text = contraction_to_expansion(text)
105
- text = remove_emails(text)
106
- text = remove_urls(text)
107
- text = remove_html_tag(text)
108
117
  text = remove_special_chars(text)
118
+ text = remove_extra_whitespace(text)
109
119
  text = lemmatize(text)
110
- text = convert_to_base(text)
111
- return text
120
+ if remove_stopwords:
121
+ text = remove_stop_words(text)
122
+ return text