nlp-text-preprocessing 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nlp_text_preprocessing-0.2.0/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.3.0}/PKG-INFO +81 -55
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/Readme.md +40 -30
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/__init__.py +18 -7
- nlp_text_preprocessing-0.3.0/nlp_text_preprocessing/all_text_preprocess.py +1091 -0
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info}/PKG-INFO +81 -55
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +5 -2
- nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info/requires.txt +27 -0
- nlp_text_preprocessing-0.3.0/pyproject.toml +68 -0
- nlp_text_preprocessing-0.3.0/requirements.txt +7 -0
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/tests/test_all_text_preprocess.py +1 -1
- nlp_text_preprocessing-0.3.0/tests/test_cleaning.py +168 -0
- nlp_text_preprocessing-0.3.0/tests/test_infrastructure.py +110 -0
- nlp_text_preprocessing-0.3.0/tests/test_linguistic.py +183 -0
- nlp_text_preprocessing-0.2.0/nlp_text_preprocessing/all_text_preprocess.py +0 -720
- nlp_text_preprocessing-0.2.0/nlp_text_preprocessing.egg-info/requires.txt +0 -14
- nlp_text_preprocessing-0.2.0/requirements.txt +0 -14
- nlp_text_preprocessing-0.2.0/setup.py +0 -28
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/LICENSE +0 -0
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/MANIFEST.in +0 -0
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/architecture_diagram.png +0 -0
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_banner.png +0 -0
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/setup.cfg +0 -0
|
@@ -1,38 +1,54 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
|
-
Name:
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
5
|
-
Author: Uditya Narayan Tiwari
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
2
|
+
Name: nlp-text-preprocessing
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
|
|
5
|
+
Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
|
|
8
|
+
Project-URL: Source Code, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/issues
|
|
10
|
+
Project-URL: Author Website, https://udityanarayantiwari.netlify.app/
|
|
11
|
+
Keywords: nlp,text-preprocessing,text-cleaning,spacy,nltk,textblob
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Education
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
9
16
|
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
10
25
|
Requires-Python: >=3.9
|
|
11
26
|
Description-Content-Type: text/markdown
|
|
12
27
|
License-File: LICENSE
|
|
13
28
|
Requires-Dist: spacy
|
|
14
29
|
Requires-Dist: textblob
|
|
15
|
-
Requires-Dist: beautifulsoup4
|
|
16
30
|
Requires-Dist: nltk
|
|
17
|
-
Requires-Dist:
|
|
18
|
-
Requires-Dist: SpeechRecognition==3.10.4
|
|
19
|
-
Requires-Dist: pyaudio==0.2.14
|
|
20
|
-
Requires-Dist: PrettyTable
|
|
21
|
-
Requires-Dist: scikit-learn
|
|
22
|
-
Requires-Dist: wordcloud
|
|
31
|
+
Requires-Dist: beautifulsoup4
|
|
23
32
|
Requires-Dist: lxml
|
|
24
|
-
Requires-Dist:
|
|
25
|
-
Requires-Dist:
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
33
|
+
Requires-Dist: emoji>=2.0
|
|
34
|
+
Requires-Dist: langdetect
|
|
35
|
+
Provides-Extra: pandas
|
|
36
|
+
Requires-Dist: pandas; extra == "pandas"
|
|
37
|
+
Provides-Extra: viz
|
|
38
|
+
Requires-Dist: wordcloud; extra == "viz"
|
|
39
|
+
Requires-Dist: matplotlib; extra == "viz"
|
|
40
|
+
Provides-Extra: all
|
|
41
|
+
Requires-Dist: pandas; extra == "all"
|
|
42
|
+
Requires-Dist: wordcloud; extra == "all"
|
|
43
|
+
Requires-Dist: matplotlib; extra == "all"
|
|
44
|
+
Provides-Extra: dev
|
|
45
|
+
Requires-Dist: pytest; extra == "dev"
|
|
46
|
+
Requires-Dist: build; extra == "dev"
|
|
47
|
+
Requires-Dist: twine; extra == "dev"
|
|
48
|
+
Requires-Dist: pandas; extra == "dev"
|
|
49
|
+
Requires-Dist: wordcloud; extra == "dev"
|
|
50
|
+
Requires-Dist: matplotlib; extra == "dev"
|
|
32
51
|
Dynamic: license-file
|
|
33
|
-
Dynamic: requires-dist
|
|
34
|
-
Dynamic: requires-python
|
|
35
|
-
Dynamic: summary
|
|
36
52
|
|
|
37
53
|
# NLP Text Preprocessing Utility Package
|
|
38
54
|
|
|
@@ -86,17 +102,19 @@ Dynamic: summary
|
|
|
86
102
|
|
|
87
103
|
## Package Overview
|
|
88
104
|
|
|
89
|
-
**`nlp_text_preprocessing`** is a
|
|
105
|
+
**`nlp_text_preprocessing`** is a modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
|
|
90
106
|
|
|
91
|
-
|
|
107
|
+
Maintained by **[Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)**.
|
|
108
|
+
|
|
109
|
+
It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with defensive guards so `None`, `NaN` and missing values are treated as empty text, which makes it convenient to run over Pandas DataFrame columns and raw text datasets.
|
|
92
110
|
|
|
93
111
|
---
|
|
94
112
|
|
|
95
113
|
## Key Features
|
|
96
114
|
|
|
97
|
-
- **
|
|
115
|
+
- **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
|
|
98
116
|
- **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
|
|
99
|
-
- **Lazy
|
|
117
|
+
- **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
|
|
100
118
|
- **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
|
|
101
119
|
- **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
|
|
102
120
|
- **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
|
|
@@ -120,20 +138,27 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
|
|
|
120
138
|
### 1. Installation
|
|
121
139
|
|
|
122
140
|
```bash
|
|
123
|
-
pip install nlp-text-preprocessing
|
|
141
|
+
pip install nlp-text-preprocessing # core
|
|
142
|
+
pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
|
|
143
|
+
pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
|
|
144
|
+
pip install "nlp-text-preprocessing[all]" # everything optional
|
|
124
145
|
```
|
|
125
146
|
|
|
126
147
|
### 2. Downloading Required Corpora and Models
|
|
127
148
|
|
|
128
|
-
|
|
149
|
+
Nothing is downloaded behind your back. Run the one-time setup, which fetches the NLTK data (`punkt_tab`, `wordnet`, `omw-1.4`) and the spaCy model `en_core_web_sm`:
|
|
129
150
|
|
|
130
151
|
```python
|
|
131
152
|
import nlp_text_preprocessing as tp
|
|
132
153
|
|
|
133
|
-
#
|
|
134
|
-
tp.
|
|
154
|
+
tp.download_nltk_packages() # one-time setup
|
|
155
|
+
tp.check_resources(verbose=True) # see what is installed / missing
|
|
135
156
|
```
|
|
136
157
|
|
|
158
|
+
`sentiment_analysis(x, method="naive_bayes")` additionally needs the `movie_reviews` corpus:
|
|
159
|
+
`tp.download_nltk_packages(include_sentiment_corpus=True)`.
|
|
160
|
+
Set the environment variable `NLP_TEXT_PREPROCESSING_AUTO_DOWNLOAD=1` if you want the spaCy model to be downloaded automatically on first use.
|
|
161
|
+
|
|
137
162
|
---
|
|
138
163
|
|
|
139
164
|
## Pipeline Automation Engine
|
|
@@ -204,29 +229,29 @@ Below is the complete reference guide for all functions available in the library
|
|
|
204
229
|
| `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
|
|
205
230
|
| `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
|
|
206
231
|
| `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
|
|
207
|
-
| `remove_urls(x)` | `x: str` | `str` | Removes
|
|
232
|
+
| `remove_urls(x)` | `x: str` | `str` | Removes `http(s)://`, `ftp://` and `www.` URLs (trailing punctuation is kept). |
|
|
208
233
|
| `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
|
|
209
|
-
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user
|
|
234
|
+
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user:`). |
|
|
210
235
|
| `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
|
|
211
236
|
| `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
|
|
212
|
-
| `remove_accented_chars(x)` | `x: str` | `str` |
|
|
237
|
+
| `remove_accented_chars(x)` | `x: str` | `str` | Strips accents (`café` -> `cafe`); non-Latin scripts are kept. |
|
|
213
238
|
| `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
|
|
214
|
-
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and
|
|
215
|
-
| `remove_repeated_chars(x)` | `x: str` | `str` |
|
|
216
|
-
| `remove_stop_words(x)` | `x: str` | `str` |
|
|
217
|
-
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji
|
|
239
|
+
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and symbols (`well-known` -> `well known`). |
|
|
240
|
+
| `remove_repeated_chars(x)` | `x: str` | `str` | Shortens elongated words (`soooo` -> `soo`); digits and punctuation are untouched. |
|
|
241
|
+
| `remove_stop_words(x, keep_negations=True)` | `x: str` | `str` | Removes stop words (case-insensitive); negations like `not` are kept by default. |
|
|
242
|
+
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji (flags, skin tones and ZWJ sequences included). |
|
|
218
243
|
| `extract_emojis(x)` | `x: str` | `list[str]` | Returns list of emojis found in text. |
|
|
219
244
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
220
|
-
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers
|
|
245
|
+
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
221
246
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
222
|
-
| `mask_pii(x)` | `x: str` | `str` | Redacts emails,
|
|
247
|
+
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
223
248
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
224
|
-
| `expand_hashtags(x)` | `x: str` | `str` | Splits
|
|
249
|
+
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
225
250
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
|
226
|
-
| `remove_numbers(x)` | `x: str` | `str` |
|
|
227
|
-
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips
|
|
251
|
+
| `remove_numbers(x)` | `x: str` | `str` | Removes standalone numbers; digits inside words (`3rd`, `H2O`) stay. |
|
|
252
|
+
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
|
|
228
253
|
| `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
|
|
229
|
-
| `clean_text(text)` | `text: str` | `str` |
|
|
254
|
+
| `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
|
|
230
255
|
|
|
231
256
|
---
|
|
232
257
|
|
|
@@ -239,15 +264,15 @@ Below is the complete reference guide for all functions available in the library
|
|
|
239
264
|
| `extract_entities(x)` | `x: str` | `list[tuple]` | Extracts named entities `(text, label)` using spaCy. |
|
|
240
265
|
| `extract_pos_tags(x)` | `x: str` | `list[tuple]` | Extracts full POS tag breakdown `(token, pos, tag)` using spaCy. |
|
|
241
266
|
| `get_dependency_tree(x)`| `x: str` | `list[tuple]` | Extracts dependency parse tree `(token, dep, head)` using spaCy. |
|
|
242
|
-
| `detect_language(x)` | `x: str` | `str` |
|
|
267
|
+
| `detect_language(x)` | `x: str` | `str` | Language code (`'en'`, `'fr'`, `'hi'`, ...) via `langdetect`; `'unknown'` if undetectable. |
|
|
243
268
|
| `keyword_extraction(x, top_n)`| `x: str, top_n: int`| `list[str]`| Extracts top N keywords by term frequency. |
|
|
244
269
|
| `text_similarity(x1, x2)` | `x1, x2` | `float` | Calculates token Jaccard similarity between two texts (0.0 to 1.0). |
|
|
245
|
-
| `get_synonyms(word)` | `word: str` | `list[str]` |
|
|
246
|
-
| `get_antonyms(word)` | `word: str` | `list[str]` |
|
|
247
|
-
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` |
|
|
248
|
-
| `correct_spelling(x)` | `x: str` | `str` | Corrects
|
|
270
|
+
| `get_synonyms(word)` | `word: str` | `list[str]` | Sorted WordNet synonyms. |
|
|
271
|
+
| `get_antonyms(word)` | `word: str` | `list[str]` | Sorted WordNet antonyms. |
|
|
272
|
+
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | spaCy noun chunks with exact character `start`/`end` offsets. |
|
|
273
|
+
| `correct_spelling(x)` | `x: str` | `str` | Corrects misspelled words (TextBlob); punctuation, case, URLs and acronyms are preserved. |
|
|
249
274
|
| `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
|
|
250
|
-
| `n_grams(x, n=2)` | `x: str, n: int` | `list` |
|
|
275
|
+
| `n_grams(x, n=2)` | `x: str, n: int` | `list[tuple]` | Word-level n-grams as tuples. |
|
|
251
276
|
| `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
|
|
252
277
|
| `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
|
|
253
278
|
|
|
@@ -261,7 +286,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
261
286
|
| `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
|
|
262
287
|
| `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
|
|
263
288
|
| `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
|
|
264
|
-
| `sentiment_analysis(x)` | `x: str` | `str` |
|
|
289
|
+
| `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
|
|
265
290
|
| `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
|
|
266
291
|
| `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
|
|
267
292
|
|
|
@@ -355,10 +380,11 @@ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
|
355
380
|
|
|
356
381
|
## Troubleshooting and Resource Setup
|
|
357
382
|
|
|
358
|
-
If
|
|
383
|
+
If a function reports a missing NLTK corpus or spaCy model, check what is missing and install it:
|
|
359
384
|
|
|
360
385
|
```python
|
|
361
386
|
import nlp_text_preprocessing as tp
|
|
387
|
+
tp.check_resources(verbose=True)
|
|
362
388
|
tp.download_nltk_packages()
|
|
363
389
|
```
|
|
364
390
|
|
|
@@ -371,5 +397,5 @@ python -m spacy download en_core_web_sm
|
|
|
371
397
|
|
|
372
398
|
## License and Author
|
|
373
399
|
|
|
374
|
-
- **Author**: [Uditya Narayan Tiwari](https://
|
|
400
|
+
- **Author**: [Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)
|
|
375
401
|
- **License**: MIT License ([LICENSE](LICENSE))
|
|
@@ -50,17 +50,19 @@
|
|
|
50
50
|
|
|
51
51
|
## Package Overview
|
|
52
52
|
|
|
53
|
-
**`nlp_text_preprocessing`** is a
|
|
53
|
+
**`nlp_text_preprocessing`** is a modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
|
|
54
54
|
|
|
55
|
-
|
|
55
|
+
Maintained by **[Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)**.
|
|
56
|
+
|
|
57
|
+
It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with defensive guards so `None`, `NaN` and missing values are treated as empty text, which makes it convenient to run over Pandas DataFrame columns and raw text datasets.
|
|
56
58
|
|
|
57
59
|
---
|
|
58
60
|
|
|
59
61
|
## Key Features
|
|
60
62
|
|
|
61
|
-
- **
|
|
63
|
+
- **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
|
|
62
64
|
- **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
|
|
63
|
-
- **Lazy
|
|
65
|
+
- **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
|
|
64
66
|
- **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
|
|
65
67
|
- **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
|
|
66
68
|
- **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
|
|
@@ -84,20 +86,27 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
|
|
|
84
86
|
### 1. Installation
|
|
85
87
|
|
|
86
88
|
```bash
|
|
87
|
-
pip install nlp-text-preprocessing
|
|
89
|
+
pip install nlp-text-preprocessing # core
|
|
90
|
+
pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
|
|
91
|
+
pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
|
|
92
|
+
pip install "nlp-text-preprocessing[all]" # everything optional
|
|
88
93
|
```
|
|
89
94
|
|
|
90
95
|
### 2. Downloading Required Corpora and Models
|
|
91
96
|
|
|
92
|
-
|
|
97
|
+
Nothing is downloaded behind your back. Run the one-time setup, which fetches the NLTK data (`punkt_tab`, `wordnet`, `omw-1.4`) and the spaCy model `en_core_web_sm`:
|
|
93
98
|
|
|
94
99
|
```python
|
|
95
100
|
import nlp_text_preprocessing as tp
|
|
96
101
|
|
|
97
|
-
#
|
|
98
|
-
tp.
|
|
102
|
+
tp.download_nltk_packages() # one-time setup
|
|
103
|
+
tp.check_resources(verbose=True) # see what is installed / missing
|
|
99
104
|
```
|
|
100
105
|
|
|
106
|
+
`sentiment_analysis(x, method="naive_bayes")` additionally needs the `movie_reviews` corpus:
|
|
107
|
+
`tp.download_nltk_packages(include_sentiment_corpus=True)`.
|
|
108
|
+
Set the environment variable `NLP_TEXT_PREPROCESSING_AUTO_DOWNLOAD=1` if you want the spaCy model to be downloaded automatically on first use.
|
|
109
|
+
|
|
101
110
|
---
|
|
102
111
|
|
|
103
112
|
## Pipeline Automation Engine
|
|
@@ -168,29 +177,29 @@ Below is the complete reference guide for all functions available in the library
|
|
|
168
177
|
| `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
|
|
169
178
|
| `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
|
|
170
179
|
| `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
|
|
171
|
-
| `remove_urls(x)` | `x: str` | `str` | Removes
|
|
180
|
+
| `remove_urls(x)` | `x: str` | `str` | Removes `http(s)://`, `ftp://` and `www.` URLs (trailing punctuation is kept). |
|
|
172
181
|
| `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
|
|
173
|
-
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user
|
|
182
|
+
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user:`). |
|
|
174
183
|
| `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
|
|
175
184
|
| `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
|
|
176
|
-
| `remove_accented_chars(x)` | `x: str` | `str` |
|
|
185
|
+
| `remove_accented_chars(x)` | `x: str` | `str` | Strips accents (`café` -> `cafe`); non-Latin scripts are kept. |
|
|
177
186
|
| `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
|
|
178
|
-
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and
|
|
179
|
-
| `remove_repeated_chars(x)` | `x: str` | `str` |
|
|
180
|
-
| `remove_stop_words(x)` | `x: str` | `str` |
|
|
181
|
-
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji
|
|
187
|
+
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and symbols (`well-known` -> `well known`). |
|
|
188
|
+
| `remove_repeated_chars(x)` | `x: str` | `str` | Shortens elongated words (`soooo` -> `soo`); digits and punctuation are untouched. |
|
|
189
|
+
| `remove_stop_words(x, keep_negations=True)` | `x: str` | `str` | Removes stop words (case-insensitive); negations like `not` are kept by default. |
|
|
190
|
+
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji (flags, skin tones and ZWJ sequences included). |
|
|
182
191
|
| `extract_emojis(x)` | `x: str` | `list[str]` | Returns list of emojis found in text. |
|
|
183
192
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
184
|
-
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers
|
|
193
|
+
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
185
194
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
186
|
-
| `mask_pii(x)` | `x: str` | `str` | Redacts emails,
|
|
195
|
+
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
187
196
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
188
|
-
| `expand_hashtags(x)` | `x: str` | `str` | Splits
|
|
197
|
+
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
189
198
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
|
190
|
-
| `remove_numbers(x)` | `x: str` | `str` |
|
|
191
|
-
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips
|
|
199
|
+
| `remove_numbers(x)` | `x: str` | `str` | Removes standalone numbers; digits inside words (`3rd`, `H2O`) stay. |
|
|
200
|
+
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
|
|
192
201
|
| `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
|
|
193
|
-
| `clean_text(text)` | `text: str` | `str` |
|
|
202
|
+
| `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
|
|
194
203
|
|
|
195
204
|
---
|
|
196
205
|
|
|
@@ -203,15 +212,15 @@ Below is the complete reference guide for all functions available in the library
|
|
|
203
212
|
| `extract_entities(x)` | `x: str` | `list[tuple]` | Extracts named entities `(text, label)` using spaCy. |
|
|
204
213
|
| `extract_pos_tags(x)` | `x: str` | `list[tuple]` | Extracts full POS tag breakdown `(token, pos, tag)` using spaCy. |
|
|
205
214
|
| `get_dependency_tree(x)`| `x: str` | `list[tuple]` | Extracts dependency parse tree `(token, dep, head)` using spaCy. |
|
|
206
|
-
| `detect_language(x)` | `x: str` | `str` |
|
|
215
|
+
| `detect_language(x)` | `x: str` | `str` | Language code (`'en'`, `'fr'`, `'hi'`, ...) via `langdetect`; `'unknown'` if undetectable. |
|
|
207
216
|
| `keyword_extraction(x, top_n)`| `x: str, top_n: int`| `list[str]`| Extracts top N keywords by term frequency. |
|
|
208
217
|
| `text_similarity(x1, x2)` | `x1, x2` | `float` | Calculates token Jaccard similarity between two texts (0.0 to 1.0). |
|
|
209
|
-
| `get_synonyms(word)` | `word: str` | `list[str]` |
|
|
210
|
-
| `get_antonyms(word)` | `word: str` | `list[str]` |
|
|
211
|
-
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` |
|
|
212
|
-
| `correct_spelling(x)` | `x: str` | `str` | Corrects
|
|
218
|
+
| `get_synonyms(word)` | `word: str` | `list[str]` | Sorted WordNet synonyms. |
|
|
219
|
+
| `get_antonyms(word)` | `word: str` | `list[str]` | Sorted WordNet antonyms. |
|
|
220
|
+
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | spaCy noun chunks with exact character `start`/`end` offsets. |
|
|
221
|
+
| `correct_spelling(x)` | `x: str` | `str` | Corrects misspelled words (TextBlob); punctuation, case, URLs and acronyms are preserved. |
|
|
213
222
|
| `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
|
|
214
|
-
| `n_grams(x, n=2)` | `x: str, n: int` | `list` |
|
|
223
|
+
| `n_grams(x, n=2)` | `x: str, n: int` | `list[tuple]` | Word-level n-grams as tuples. |
|
|
215
224
|
| `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
|
|
216
225
|
| `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
|
|
217
226
|
|
|
@@ -225,7 +234,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
225
234
|
| `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
|
|
226
235
|
| `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
|
|
227
236
|
| `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
|
|
228
|
-
| `sentiment_analysis(x)` | `x: str` | `str` |
|
|
237
|
+
| `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
|
|
229
238
|
| `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
|
|
230
239
|
| `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
|
|
231
240
|
|
|
@@ -319,10 +328,11 @@ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
|
319
328
|
|
|
320
329
|
## Troubleshooting and Resource Setup
|
|
321
330
|
|
|
322
|
-
If
|
|
331
|
+
If a function reports a missing NLTK corpus or spaCy model, check what is missing and install it:
|
|
323
332
|
|
|
324
333
|
```python
|
|
325
334
|
import nlp_text_preprocessing as tp
|
|
335
|
+
tp.check_resources(verbose=True)
|
|
326
336
|
tp.download_nltk_packages()
|
|
327
337
|
```
|
|
328
338
|
|
|
@@ -335,5 +345,5 @@ python -m spacy download en_core_web_sm
|
|
|
335
345
|
|
|
336
346
|
## License and Author
|
|
337
347
|
|
|
338
|
-
- **Author**: [Uditya Narayan Tiwari](https://
|
|
348
|
+
- **Author**: [Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)
|
|
339
349
|
- **License**: MIT License ([LICENSE](LICENSE))
|
{nlp_text_preprocessing-0.2.0 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/__init__.py
RENAMED
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
# entry level code and import text preprocess
|
|
2
2
|
|
|
3
|
+
__version__ = "0.3.0"
|
|
4
|
+
|
|
3
5
|
from .all_text_preprocess import *
|
|
4
6
|
|
|
5
7
|
__all__ = [
|
|
8
|
+
'__version__',
|
|
6
9
|
'download_nltk_packages',
|
|
10
|
+
'check_resources',
|
|
11
|
+
'get_spacy_nlp',
|
|
12
|
+
'get_stop_words',
|
|
7
13
|
'word_count',
|
|
8
14
|
'char_count',
|
|
9
15
|
'avg_word_len',
|
|
@@ -96,16 +102,21 @@ def extract_features(x):
|
|
|
96
102
|
}
|
|
97
103
|
|
|
98
104
|
|
|
99
|
-
def clean_text(text):
|
|
105
|
+
def clean_text(text, remove_stopwords=False):
|
|
100
106
|
"""
|
|
101
|
-
|
|
107
|
+
End-to-end cleaning: HTML, URLs and emails are removed, text is lower-cased,
|
|
108
|
+
contractions are expanded, punctuation is dropped and words are lemmatized.
|
|
109
|
+
|
|
110
|
+
Set remove_stopwords=True to also drop stop words (negations are kept).
|
|
102
111
|
"""
|
|
112
|
+
text = remove_html_tag(text)
|
|
113
|
+
text = remove_urls(text)
|
|
114
|
+
text = remove_emails(text)
|
|
103
115
|
text = to_lower_case(text)
|
|
104
116
|
text = contraction_to_expansion(text)
|
|
105
|
-
text = remove_emails(text)
|
|
106
|
-
text = remove_urls(text)
|
|
107
|
-
text = remove_html_tag(text)
|
|
108
117
|
text = remove_special_chars(text)
|
|
118
|
+
text = remove_extra_whitespace(text)
|
|
109
119
|
text = lemmatize(text)
|
|
110
|
-
|
|
111
|
-
|
|
120
|
+
if remove_stopwords:
|
|
121
|
+
text = remove_stop_words(text)
|
|
122
|
+
return text
|