nlp-text-preprocessing 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {nlp_text_preprocessing-0.2.0/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.3.0}/PKG-INFO +81 -55
  2. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/Readme.md +40 -30
  3. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/__init__.py +18 -7
  4. nlp_text_preprocessing-0.3.0/nlp_text_preprocessing/all_text_preprocess.py +1091 -0
  5. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info}/PKG-INFO +81 -55
  6. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +5 -2
  7. nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info/requires.txt +27 -0
  8. nlp_text_preprocessing-0.3.0/pyproject.toml +68 -0
  9. nlp_text_preprocessing-0.3.0/requirements.txt +7 -0
  10. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/tests/test_all_text_preprocess.py +1 -1
  11. nlp_text_preprocessing-0.3.0/tests/test_cleaning.py +168 -0
  12. nlp_text_preprocessing-0.3.0/tests/test_infrastructure.py +110 -0
  13. nlp_text_preprocessing-0.3.0/tests/test_linguistic.py +183 -0
  14. nlp_text_preprocessing-0.2.0/nlp_text_preprocessing/all_text_preprocess.py +0 -720
  15. nlp_text_preprocessing-0.2.0/nlp_text_preprocessing.egg-info/requires.txt +0 -14
  16. nlp_text_preprocessing-0.2.0/requirements.txt +0 -14
  17. nlp_text_preprocessing-0.2.0/setup.py +0 -28
  18. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/LICENSE +0 -0
  19. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/MANIFEST.in +0 -0
  20. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/architecture_diagram.png +0 -0
  21. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_banner.png +0 -0
  22. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/data/contractions.json +0 -0
  23. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
  24. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
  25. {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/setup.cfg +0 -0
@@ -1,38 +1,54 @@
1
1
  Metadata-Version: 2.4
2
- Name: nlp_text_preprocessing
3
- Version: 0.2.0
4
- Summary: This is a Text Processing Package For NLP
5
- Author: Uditya Narayan Tiwari
6
- Author-email: tiwarimerit@gmail.com
7
- Classifier: Programming Language :: Python :: 3
8
- Classifier: License :: OSI Approved :: MIT License
2
+ Name: nlp-text-preprocessing
3
+ Version: 0.3.0
4
+ Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
5
+ Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
8
+ Project-URL: Source Code, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
9
+ Project-URL: Bug Tracker, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/issues
10
+ Project-URL: Author Website, https://udityanarayantiwari.netlify.app/
11
+ Keywords: nlp,text-preprocessing,text-cleaning,spacy,nltk,textblob
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Education
15
+ Classifier: Intended Audience :: Science/Research
9
16
  Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.9
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Classifier: Topic :: Text Processing :: Linguistic
10
25
  Requires-Python: >=3.9
11
26
  Description-Content-Type: text/markdown
12
27
  License-File: LICENSE
13
28
  Requires-Dist: spacy
14
29
  Requires-Dist: textblob
15
- Requires-Dist: beautifulsoup4
16
30
  Requires-Dist: nltk
17
- Requires-Dist: openpyxl
18
- Requires-Dist: SpeechRecognition==3.10.4
19
- Requires-Dist: pyaudio==0.2.14
20
- Requires-Dist: PrettyTable
21
- Requires-Dist: scikit-learn
22
- Requires-Dist: wordcloud
31
+ Requires-Dist: beautifulsoup4
23
32
  Requires-Dist: lxml
24
- Requires-Dist: pandas
25
- Requires-Dist: numpy
26
- Requires-Dist: matplotlib
27
- Dynamic: author
28
- Dynamic: author-email
29
- Dynamic: classifier
30
- Dynamic: description
31
- Dynamic: description-content-type
33
+ Requires-Dist: emoji>=2.0
34
+ Requires-Dist: langdetect
35
+ Provides-Extra: pandas
36
+ Requires-Dist: pandas; extra == "pandas"
37
+ Provides-Extra: viz
38
+ Requires-Dist: wordcloud; extra == "viz"
39
+ Requires-Dist: matplotlib; extra == "viz"
40
+ Provides-Extra: all
41
+ Requires-Dist: pandas; extra == "all"
42
+ Requires-Dist: wordcloud; extra == "all"
43
+ Requires-Dist: matplotlib; extra == "all"
44
+ Provides-Extra: dev
45
+ Requires-Dist: pytest; extra == "dev"
46
+ Requires-Dist: build; extra == "dev"
47
+ Requires-Dist: twine; extra == "dev"
48
+ Requires-Dist: pandas; extra == "dev"
49
+ Requires-Dist: wordcloud; extra == "dev"
50
+ Requires-Dist: matplotlib; extra == "dev"
32
51
  Dynamic: license-file
33
- Dynamic: requires-dist
34
- Dynamic: requires-python
35
- Dynamic: summary
36
52
 
37
53
  # NLP Text Preprocessing Utility Package
38
54
 
@@ -86,17 +102,19 @@ Dynamic: summary
86
102
 
87
103
  ## Package Overview
88
104
 
89
- **`nlp_text_preprocessing`** is a production-grade, modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
105
+ **`nlp_text_preprocessing`** is a modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
90
106
 
91
- It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with built-in defensive guards for `None`, `NaN`, and missing values, making it 100% crash-proof when running over Pandas DataFrames or large raw text datasets.
107
+ Maintained by **[Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)**.
108
+
109
+ It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with defensive guards so `None`, `NaN` and missing values are treated as empty text, which makes it convenient to run over Pandas DataFrame columns and raw text datasets.
92
110
 
93
111
  ---
94
112
 
95
113
  ## Key Features
96
114
 
97
- - **Crash-Proof Pipeline**: Automatic `None`/`NaN` input sanitation across all functions.
115
+ - **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
98
116
  - **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
99
- - **Lazy and Fast Model Caching**: Efficient lazy-loading for spaCy (`en_core_web_sm`) and cached NaiveBayes sentiment analyzers.
117
+ - **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
100
118
  - **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
101
119
  - **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
102
120
  - **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
@@ -120,20 +138,27 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
120
138
  ### 1. Installation
121
139
 
122
140
  ```bash
123
- pip install nlp-text-preprocessing
141
+ pip install nlp-text-preprocessing # core
142
+ pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
143
+ pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
144
+ pip install "nlp-text-preprocessing[all]" # everything optional
124
145
  ```
125
146
 
126
147
  ### 2. Downloading Required Corpora and Models
127
148
 
128
- To automatically download all required NLTK corpora (`stopwords`, `movie_reviews`, `brown`) and spaCy models (`en_core_web_sm`), run:
149
+ Nothing is downloaded behind your back. Run the one-time setup, which fetches the NLTK data (`punkt_tab`, `wordnet`, `omw-1.4`) and the spaCy model `en_core_web_sm`:
129
150
 
130
151
  ```python
131
152
  import nlp_text_preprocessing as tp
132
153
 
133
- # One-time automated setup for NLTK and spaCy resources
134
- tp.download_nltk_packages()
154
+ tp.download_nltk_packages() # one-time setup
155
+ tp.check_resources(verbose=True) # see what is installed / missing
135
156
  ```
136
157
 
158
+ `sentiment_analysis(x, method="naive_bayes")` additionally needs the `movie_reviews` corpus:
159
+ `tp.download_nltk_packages(include_sentiment_corpus=True)`.
160
+ Set the environment variable `NLP_TEXT_PREPROCESSING_AUTO_DOWNLOAD=1` if you want the spaCy model to be downloaded automatically on first use.
161
+
137
162
  ---
138
163
 
139
164
  ## Pipeline Automation Engine
@@ -204,29 +229,29 @@ Below is the complete reference guide for all functions available in the library
204
229
  | `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
205
230
  | `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
206
231
  | `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
207
- | `remove_urls(x)` | `x: str` | `str` | Removes HTTP/HTTPS links and `www` URLs. |
232
+ | `remove_urls(x)` | `x: str` | `str` | Removes `http(s)://`, `ftp://` and `www.` URLs (trailing punctuation is kept). |
208
233
  | `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
209
- | `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user`). |
234
+ | `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user:`). |
210
235
  | `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
211
236
  | `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
212
- | `remove_accented_chars(x)` | `x: str` | `str` | Normalizes accented characters (NFKD -> ASCII). |
237
+ | `remove_accented_chars(x)` | `x: str` | `str` | Strips accents (`café` -> `cafe`); non-Latin scripts are kept. |
213
238
  | `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
214
- | `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and special symbols. |
215
- | `remove_repeated_chars(x)` | `x: str` | `str` | Truncates repeated characters beyond 2 consecutive occurrences. |
216
- | `remove_stop_words(x)` | `x: str` | `str` | Case-insensitive removal of stop words while preserving word case. |
217
- | `remove_emojis(x)` | `x: str` | `str` | Strips emoji characters from text. |
239
+ | `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and symbols (`well-known` -> `well known`). |
240
+ | `remove_repeated_chars(x)` | `x: str` | `str` | Shortens elongated words (`soooo` -> `soo`); digits and punctuation are untouched. |
241
+ | `remove_stop_words(x, keep_negations=True)` | `x: str` | `str` | Removes stop words (case-insensitive); negations like `not` are kept by default. |
242
+ | `remove_emojis(x)` | `x: str` | `str` | Strips emoji (flags, skin tones and ZWJ sequences included). |
218
243
  | `extract_emojis(x)` | `x: str` | `list[str]` | Returns list of emojis found in text. |
219
244
  | `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
220
- | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers from text. |
245
+ | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
221
246
  | `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
222
- | `mask_pii(x)` | `x: str` | `str` | Redacts emails, phone numbers, and card numbers with `[REDACTED]`. |
247
+ | `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
223
248
  | `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
224
- | `expand_hashtags(x)` | `x: str` | `str` | Splits camelCase/PascalCase hashtags (`#MachineLearning` -> `Machine Learning`). |
249
+ | `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
225
250
  | `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
226
- | `remove_numbers(x)` | `x: str` | `str` | Strips all numeric digits. |
227
- | `remove_currency_symbols(x)`| `x: str` | `str` | Strips currency symbols (`$`, `€`, `₹`, `£`, `¥`). |
251
+ | `remove_numbers(x)` | `x: str` | `str` | Removes standalone numbers; digits inside words (`3rd`, `H2O`) stay. |
252
+ | `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
228
253
  | `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
229
- | `clean_text(text)` | `text: str` | `str` | Runs end-to-end cleaning pipeline. |
254
+ | `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
230
255
 
231
256
  ---
232
257
 
@@ -239,15 +264,15 @@ Below is the complete reference guide for all functions available in the library
239
264
  | `extract_entities(x)` | `x: str` | `list[tuple]` | Extracts named entities `(text, label)` using spaCy. |
240
265
  | `extract_pos_tags(x)` | `x: str` | `list[tuple]` | Extracts full POS tag breakdown `(token, pos, tag)` using spaCy. |
241
266
  | `get_dependency_tree(x)`| `x: str` | `list[tuple]` | Extracts dependency parse tree `(token, dep, head)` using spaCy. |
242
- | `detect_language(x)` | `x: str` | `str` | Detects language code (`'en'`, `'fr'`, etc.) using TextBlob. |
267
+ | `detect_language(x)` | `x: str` | `str` | Language code (`'en'`, `'fr'`, `'hi'`, ...) via `langdetect`; `'unknown'` if undetectable. |
243
268
  | `keyword_extraction(x, top_n)`| `x: str, top_n: int`| `list[str]`| Extracts top N keywords by term frequency. |
244
269
  | `text_similarity(x1, x2)` | `x1, x2` | `float` | Calculates token Jaccard similarity between two texts (0.0 to 1.0). |
245
- | `get_synonyms(word)` | `word: str` | `list[str]` | Returns list of synonyms from WordNet. |
246
- | `get_antonyms(word)` | `word: str` | `list[str]` | Returns list of antonyms from WordNet. |
247
- | `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | Extracts noun phrases with character start/end spans. |
248
- | `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob model. |
270
+ | `get_synonyms(word)` | `word: str` | `list[str]` | Sorted WordNet synonyms. |
271
+ | `get_antonyms(word)` | `word: str` | `list[str]` | Sorted WordNet antonyms. |
272
+ | `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | spaCy noun chunks with exact character `start`/`end` offsets. |
273
+ | `correct_spelling(x)` | `x: str` | `str` | Corrects misspelled words (TextBlob); punctuation, case, URLs and acronyms are preserved. |
249
274
  | `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
250
- | `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
275
+ | `n_grams(x, n=2)` | `x: str, n: int` | `list[tuple]` | Word-level n-grams as tuples. |
251
276
  | `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
252
277
  | `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
253
278
 
@@ -261,7 +286,7 @@ Below is the complete reference guide for all functions available in the library
261
286
  | `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
262
287
  | `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
263
288
  | `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
264
- | `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using cached NaiveBayes model. |
289
+ | `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
265
290
  | `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
266
291
  | `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
267
292
 
@@ -355,10 +380,11 @@ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
355
380
 
356
381
  ## Troubleshooting and Resource Setup
357
382
 
358
- If any NLTK corpus or spaCy model throws a missing resource error during runtime, simply run:
383
+ If a function reports a missing NLTK corpus or spaCy model, check what is missing and install it:
359
384
 
360
385
  ```python
361
386
  import nlp_text_preprocessing as tp
387
+ tp.check_resources(verbose=True)
362
388
  tp.download_nltk_packages()
363
389
  ```
364
390
 
@@ -371,5 +397,5 @@ python -m spacy download en_core_web_sm
371
397
 
372
398
  ## License and Author
373
399
 
374
- - **Author**: [Uditya Narayan Tiwari](https://youtube.com/kgptalkie)
400
+ - **Author**: [Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)
375
401
  - **License**: MIT License ([LICENSE](LICENSE))
@@ -50,17 +50,19 @@
50
50
 
51
51
  ## Package Overview
52
52
 
53
- **`nlp_text_preprocessing`** is a production-grade, modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
53
+ **`nlp_text_preprocessing`** is a modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
54
54
 
55
- It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with built-in defensive guards for `None`, `NaN`, and missing values, making it 100% crash-proof when running over Pandas DataFrames or large raw text datasets.
55
+ Maintained by **[Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)**.
56
+
57
+ It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with defensive guards so `None`, `NaN` and missing values are treated as empty text, which makes it convenient to run over Pandas DataFrame columns and raw text datasets.
56
58
 
57
59
  ---
58
60
 
59
61
  ## Key Features
60
62
 
61
- - **Crash-Proof Pipeline**: Automatic `None`/`NaN` input sanitation across all functions.
63
+ - **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
62
64
  - **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
63
- - **Lazy and Fast Model Caching**: Efficient lazy-loading for spaCy (`en_core_web_sm`) and cached NaiveBayes sentiment analyzers.
65
+ - **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
64
66
  - **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
65
67
  - **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
66
68
  - **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
@@ -84,20 +86,27 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
84
86
  ### 1. Installation
85
87
 
86
88
  ```bash
87
- pip install nlp-text-preprocessing
89
+ pip install nlp-text-preprocessing # core
90
+ pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
91
+ pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
92
+ pip install "nlp-text-preprocessing[all]" # everything optional
88
93
  ```
89
94
 
90
95
  ### 2. Downloading Required Corpora and Models
91
96
 
92
- To automatically download all required NLTK corpora (`stopwords`, `movie_reviews`, `brown`) and spaCy models (`en_core_web_sm`), run:
97
+ Nothing is downloaded behind your back. Run the one-time setup, which fetches the NLTK data (`punkt_tab`, `wordnet`, `omw-1.4`) and the spaCy model `en_core_web_sm`:
93
98
 
94
99
  ```python
95
100
  import nlp_text_preprocessing as tp
96
101
 
97
- # One-time automated setup for NLTK and spaCy resources
98
- tp.download_nltk_packages()
102
+ tp.download_nltk_packages() # one-time setup
103
+ tp.check_resources(verbose=True) # see what is installed / missing
99
104
  ```
100
105
 
106
+ `sentiment_analysis(x, method="naive_bayes")` additionally needs the `movie_reviews` corpus:
107
+ `tp.download_nltk_packages(include_sentiment_corpus=True)`.
108
+ Set the environment variable `NLP_TEXT_PREPROCESSING_AUTO_DOWNLOAD=1` if you want the spaCy model to be downloaded automatically on first use.
109
+
101
110
  ---
102
111
 
103
112
  ## Pipeline Automation Engine
@@ -168,29 +177,29 @@ Below is the complete reference guide for all functions available in the library
168
177
  | `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
169
178
  | `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
170
179
  | `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
171
- | `remove_urls(x)` | `x: str` | `str` | Removes HTTP/HTTPS links and `www` URLs. |
180
+ | `remove_urls(x)` | `x: str` | `str` | Removes `http(s)://`, `ftp://` and `www.` URLs (trailing punctuation is kept). |
172
181
  | `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
173
- | `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user`). |
182
+ | `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user:`). |
174
183
  | `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
175
184
  | `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
176
- | `remove_accented_chars(x)` | `x: str` | `str` | Normalizes accented characters (NFKD -> ASCII). |
185
+ | `remove_accented_chars(x)` | `x: str` | `str` | Strips accents (`café` -> `cafe`); non-Latin scripts are kept. |
177
186
  | `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
178
- | `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and special symbols. |
179
- | `remove_repeated_chars(x)` | `x: str` | `str` | Truncates repeated characters beyond 2 consecutive occurrences. |
180
- | `remove_stop_words(x)` | `x: str` | `str` | Case-insensitive removal of stop words while preserving word case. |
181
- | `remove_emojis(x)` | `x: str` | `str` | Strips emoji characters from text. |
187
+ | `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and symbols (`well-known` -> `well known`). |
188
+ | `remove_repeated_chars(x)` | `x: str` | `str` | Shortens elongated words (`soooo` -> `soo`); digits and punctuation are untouched. |
189
+ | `remove_stop_words(x, keep_negations=True)` | `x: str` | `str` | Removes stop words (case-insensitive); negations like `not` are kept by default. |
190
+ | `remove_emojis(x)` | `x: str` | `str` | Strips emoji (flags, skin tones and ZWJ sequences included). |
182
191
  | `extract_emojis(x)` | `x: str` | `list[str]` | Returns list of emojis found in text. |
183
192
  | `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
184
- | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers from text. |
193
+ | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
185
194
  | `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
186
- | `mask_pii(x)` | `x: str` | `str` | Redacts emails, phone numbers, and card numbers with `[REDACTED]`. |
195
+ | `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
187
196
  | `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
188
- | `expand_hashtags(x)` | `x: str` | `str` | Splits camelCase/PascalCase hashtags (`#MachineLearning` -> `Machine Learning`). |
197
+ | `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
189
198
  | `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
190
- | `remove_numbers(x)` | `x: str` | `str` | Strips all numeric digits. |
191
- | `remove_currency_symbols(x)`| `x: str` | `str` | Strips currency symbols (`$`, `€`, `₹`, `£`, `¥`). |
199
+ | `remove_numbers(x)` | `x: str` | `str` | Removes standalone numbers; digits inside words (`3rd`, `H2O`) stay. |
200
+ | `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
192
201
  | `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
193
- | `clean_text(text)` | `text: str` | `str` | Runs end-to-end cleaning pipeline. |
202
+ | `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
194
203
 
195
204
  ---
196
205
 
@@ -203,15 +212,15 @@ Below is the complete reference guide for all functions available in the library
203
212
  | `extract_entities(x)` | `x: str` | `list[tuple]` | Extracts named entities `(text, label)` using spaCy. |
204
213
  | `extract_pos_tags(x)` | `x: str` | `list[tuple]` | Extracts full POS tag breakdown `(token, pos, tag)` using spaCy. |
205
214
  | `get_dependency_tree(x)`| `x: str` | `list[tuple]` | Extracts dependency parse tree `(token, dep, head)` using spaCy. |
206
- | `detect_language(x)` | `x: str` | `str` | Detects language code (`'en'`, `'fr'`, etc.) using TextBlob. |
215
+ | `detect_language(x)` | `x: str` | `str` | Language code (`'en'`, `'fr'`, `'hi'`, ...) via `langdetect`; `'unknown'` if undetectable. |
207
216
  | `keyword_extraction(x, top_n)`| `x: str, top_n: int`| `list[str]`| Extracts top N keywords by term frequency. |
208
217
  | `text_similarity(x1, x2)` | `x1, x2` | `float` | Calculates token Jaccard similarity between two texts (0.0 to 1.0). |
209
- | `get_synonyms(word)` | `word: str` | `list[str]` | Returns list of synonyms from WordNet. |
210
- | `get_antonyms(word)` | `word: str` | `list[str]` | Returns list of antonyms from WordNet. |
211
- | `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | Extracts noun phrases with character start/end spans. |
212
- | `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob model. |
218
+ | `get_synonyms(word)` | `word: str` | `list[str]` | Sorted WordNet synonyms. |
219
+ | `get_antonyms(word)` | `word: str` | `list[str]` | Sorted WordNet antonyms. |
220
+ | `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | spaCy noun chunks with exact character `start`/`end` offsets. |
221
+ | `correct_spelling(x)` | `x: str` | `str` | Corrects misspelled words (TextBlob); punctuation, case, URLs and acronyms are preserved. |
213
222
  | `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
214
- | `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
223
+ | `n_grams(x, n=2)` | `x: str, n: int` | `list[tuple]` | Word-level n-grams as tuples. |
215
224
  | `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
216
225
  | `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
217
226
 
@@ -225,7 +234,7 @@ Below is the complete reference guide for all functions available in the library
225
234
  | `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
226
235
  | `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
227
236
  | `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
228
- | `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using cached NaiveBayes model. |
237
+ | `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
229
238
  | `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
230
239
  | `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
231
240
 
@@ -319,10 +328,11 @@ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
319
328
 
320
329
  ## Troubleshooting and Resource Setup
321
330
 
322
- If any NLTK corpus or spaCy model throws a missing resource error during runtime, simply run:
331
+ If a function reports a missing NLTK corpus or spaCy model, check what is missing and install it:
323
332
 
324
333
  ```python
325
334
  import nlp_text_preprocessing as tp
335
+ tp.check_resources(verbose=True)
326
336
  tp.download_nltk_packages()
327
337
  ```
328
338
 
@@ -335,5 +345,5 @@ python -m spacy download en_core_web_sm
335
345
 
336
346
  ## License and Author
337
347
 
338
- - **Author**: [Uditya Narayan Tiwari](https://youtube.com/kgptalkie)
348
+ - **Author**: [Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)
339
349
  - **License**: MIT License ([LICENSE](LICENSE))
@@ -1,9 +1,15 @@
1
1
  # entry level code and import text preprocess
2
2
 
3
+ __version__ = "0.3.0"
4
+
3
5
  from .all_text_preprocess import *
4
6
 
5
7
  __all__ = [
8
+ '__version__',
6
9
  'download_nltk_packages',
10
+ 'check_resources',
11
+ 'get_spacy_nlp',
12
+ 'get_stop_words',
7
13
  'word_count',
8
14
  'char_count',
9
15
  'avg_word_len',
@@ -96,16 +102,21 @@ def extract_features(x):
96
102
  }
97
103
 
98
104
 
99
- def clean_text(text):
105
+ def clean_text(text, remove_stopwords=False):
100
106
  """
101
- Applies a sequence of cleaning and normalization steps on text.
107
+ End-to-end cleaning: HTML, URLs and emails are removed, text is lower-cased,
108
+ contractions are expanded, punctuation is dropped and words are lemmatized.
109
+
110
+ Set remove_stopwords=True to also drop stop words (negations are kept).
102
111
  """
112
+ text = remove_html_tag(text)
113
+ text = remove_urls(text)
114
+ text = remove_emails(text)
103
115
  text = to_lower_case(text)
104
116
  text = contraction_to_expansion(text)
105
- text = remove_emails(text)
106
- text = remove_urls(text)
107
- text = remove_html_tag(text)
108
117
  text = remove_special_chars(text)
118
+ text = remove_extra_whitespace(text)
109
119
  text = lemmatize(text)
110
- text = convert_to_base(text)
111
- return text
120
+ if remove_stopwords:
121
+ text = remove_stop_words(text)
122
+ return text