nlp-text-preprocessing 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nlp_text_preprocessing-0.1.2/MANIFEST.in +5 -0
- nlp_text_preprocessing-0.1.2/PKG-INFO +283 -0
- nlp_text_preprocessing-0.1.2/Readme.md +247 -0
- nlp_text_preprocessing-0.1.2/architecture_diagram.png +0 -0
- nlp_text_preprocessing-0.1.2/nlp_banner.png +0 -0
- nlp_text_preprocessing-0.1.2/nlp_text_preprocessing/__init__.py +73 -0
- nlp_text_preprocessing-0.1.2/nlp_text_preprocessing/all_text_preprocess.py +318 -0
- nlp_text_preprocessing-0.1.2/nlp_text_preprocessing.egg-info/PKG-INFO +283 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.2}/nlp_text_preprocessing.egg-info/SOURCES.txt +4 -1
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.2}/setup.py +1 -1
- nlp_text_preprocessing-0.1.2/tests/test_all_text_preprocess.py +139 -0
- nlp_text_preprocessing-0.1.0/MANIFEST.in +0 -3
- nlp_text_preprocessing-0.1.0/PKG-INFO +0 -267
- nlp_text_preprocessing-0.1.0/Readme.md +0 -231
- nlp_text_preprocessing-0.1.0/nlp_text_preprocessing/__init__.py +0 -31
- nlp_text_preprocessing-0.1.0/nlp_text_preprocessing/all_text_preprocess.py +0 -190
- nlp_text_preprocessing-0.1.0/nlp_text_preprocessing.egg-info/PKG-INFO +0 -267
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.2}/LICENSE +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.2}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.2}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.2}/nlp_text_preprocessing.egg-info/requires.txt +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.2}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.2}/requirements.txt +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.2}/setup.cfg +0 -0
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nlp_text_preprocessing
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: This is a Text Processing Package For NLP
|
|
5
|
+
Author: Uditya Narayan Tiwari
|
|
6
|
+
Author-email: tiwarimerit@gmail.com
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: spacy
|
|
14
|
+
Requires-Dist: textblob
|
|
15
|
+
Requires-Dist: beautifulsoup4
|
|
16
|
+
Requires-Dist: nltk
|
|
17
|
+
Requires-Dist: openpyxl
|
|
18
|
+
Requires-Dist: SpeechRecognition==3.10.4
|
|
19
|
+
Requires-Dist: pyaudio==0.2.14
|
|
20
|
+
Requires-Dist: PrettyTable
|
|
21
|
+
Requires-Dist: scikit-learn
|
|
22
|
+
Requires-Dist: wordcloud
|
|
23
|
+
Requires-Dist: lxml
|
|
24
|
+
Requires-Dist: pandas
|
|
25
|
+
Requires-Dist: numpy
|
|
26
|
+
Requires-Dist: matplotlib
|
|
27
|
+
Dynamic: author
|
|
28
|
+
Dynamic: author-email
|
|
29
|
+
Dynamic: classifier
|
|
30
|
+
Dynamic: description
|
|
31
|
+
Dynamic: description-content-type
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
Dynamic: requires-dist
|
|
34
|
+
Dynamic: requires-python
|
|
35
|
+
Dynamic: summary
|
|
36
|
+
|
|
37
|
+
# NLP Text Preprocessing Utility Package
|
|
38
|
+
|
|
39
|
+
<p align="center">
|
|
40
|
+
<img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/nlp_banner.png" alt="NLP Text Preprocessing Banner" width="100%" />
|
|
41
|
+
</p>
|
|
42
|
+
|
|
43
|
+
<p align="center">
|
|
44
|
+
<a href="https://pypi.org/project/nlp-text-preprocessing/">
|
|
45
|
+
<img src="https://img.shields.io/pypi/v/nlp-text-preprocessing" alt="PyPI Version">
|
|
46
|
+
</a>
|
|
47
|
+
<a href="https://pypi.org/project/nlp-text-preprocessing/">
|
|
48
|
+
<img src="https://img.shields.io/pypi/pyversions/nlp-text-preprocessing" alt="Python Version">
|
|
49
|
+
</a>
|
|
50
|
+
<a href="https://pepy.tech/projects/nlp-text-preprocessing">
|
|
51
|
+
<img src="https://static.pepy.tech/badge/nlp-text-preprocessing" alt="Total Downloads">
|
|
52
|
+
</a>
|
|
53
|
+
<a href="https://pepy.tech/projects/nlp-text-preprocessing">
|
|
54
|
+
<img src="https://static.pepy.tech/badge/nlp-text-preprocessing/month" alt="Monthly Downloads">
|
|
55
|
+
</a>
|
|
56
|
+
<a href="https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/blob/main/LICENSE">
|
|
57
|
+
<img src="https://img.shields.io/github/license/udityamerit/Text-Processing-Package-For-Natural-Language-Processing" alt="License">
|
|
58
|
+
</a>
|
|
59
|
+
</p>
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## Table of Contents
|
|
64
|
+
|
|
65
|
+
- [Package Overview](#package-overview)
|
|
66
|
+
- [Key Features](#key-features)
|
|
67
|
+
- [Architecture Diagram](#architecture-diagram)
|
|
68
|
+
- [Quickstart and Setup](#quickstart-and-setup)
|
|
69
|
+
- [Complete API Reference](#complete-api-reference)
|
|
70
|
+
- [1. General Feature Extraction](#1-general-feature-extraction)
|
|
71
|
+
- [2. Text Cleaning and Normalization](#2-text-cleaning-and-normalization)
|
|
72
|
+
- [3. Linguistic and Morphological Processing](#3-linguistic-and-morphological-processing)
|
|
73
|
+
- [4. Visualization Suite](#4-visualization-suite)
|
|
74
|
+
- [Code Examples](#code-examples)
|
|
75
|
+
- [Basic Text Cleaning](#1-basic-text-cleaning)
|
|
76
|
+
- [Pandas DataFrame Integration](#2-pandas-dataframe-integration)
|
|
77
|
+
- [Batch Feature Extraction](#3-batch-feature-extraction)
|
|
78
|
+
- [WordCloud Generation](#4-wordcloud-generation)
|
|
79
|
+
- [Troubleshooting and Resource Setup](#troubleshooting-and-resource-setup)
|
|
80
|
+
- [License and Author](#license-and-author)
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## Package Overview
|
|
85
|
+
|
|
86
|
+
**`nlp_text_preprocessing`** is a production-grade, modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
|
|
87
|
+
|
|
88
|
+
It provides **34 single-purpose functions** with built-in defensive guards for `None`, `NaN`, and missing values, making it 100% crash-proof when running over Pandas DataFrames or large raw text datasets.
|
|
89
|
+
|
|
90
|
+
---
|
|
91
|
+
|
|
92
|
+
## Key Features
|
|
93
|
+
|
|
94
|
+
- **Crash-Proof Pipeline**: Automatic `None`/`NaN` input sanitation across all functions.
|
|
95
|
+
- **Lazy and Fast Model Caching**: Efficient lazy-loading for spaCy (`en_core_web_sm`) and cached NaiveBayes sentiment analyzers.
|
|
96
|
+
- **Comprehensive Cleaning**: Strip emails, URLs, HTML, retweets, special symbols, and expand contractions.
|
|
97
|
+
- **Feature Extraction**: Extract word counts, character counts, uppercase counts, numerics, hashtags, mentions, and full feature matrices in one call.
|
|
98
|
+
- **Linguistic Utilities**: Perform lemmatization, noun phrase extraction, spelling correction, n-grams, singularization, and pluralization.
|
|
99
|
+
- **Flexible Visualizations**: Generate and save custom WordClouds to disk or display interactively.
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Architecture Diagram
|
|
104
|
+
|
|
105
|
+
The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, and **WordCloud** through a unified API surface:
|
|
106
|
+
|
|
107
|
+
<p align="center">
|
|
108
|
+
<img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/architecture_diagram.png" alt="NLP Text Preprocessing Architecture Diagram" width="100%" />
|
|
109
|
+
</p>
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## Quickstart and Setup
|
|
114
|
+
|
|
115
|
+
### 1. Installation
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
pip install nlp-text-preprocessing
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### 2. Downloading Required Corpora and Models
|
|
122
|
+
|
|
123
|
+
To automatically download all required NLTK corpora (`stopwords`, `movie_reviews`, `brown`) and spaCy models (`en_core_web_sm`), run:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
import nlp_text_preprocessing as tp
|
|
127
|
+
|
|
128
|
+
# One-time automated setup for NLTK and spaCy resources
|
|
129
|
+
tp.download_nltk_packages()
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## Complete API Reference
|
|
135
|
+
|
|
136
|
+
Below is the complete reference guide for all **34 functions** available in the library:
|
|
137
|
+
|
|
138
|
+
### 1. General Feature Extraction
|
|
139
|
+
|
|
140
|
+
| Function | Parameters | Return Type | Description |
|
|
141
|
+
| :--- | :--- | :--- | :--- |
|
|
142
|
+
| `word_count(x)` | `x: str` | `int` | Returns total whitespace-separated word count. |
|
|
143
|
+
| `char_count(x)` | `x: str` | `int` | Returns total characters excluding spaces. |
|
|
144
|
+
| `avg_word_len(x)` | `x: str` | `float` | Returns average word length (`0.0` for empty inputs). |
|
|
145
|
+
| `stop_words_count(x)` | `x: str` | `int` | Returns count of stop words (case-insensitive). |
|
|
146
|
+
| `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtag tokens (`#tag`). |
|
|
147
|
+
| `mentions_count(x)` | `x: str` | `int` | Returns count of user handle mentions (`@user`). |
|
|
148
|
+
| `numerics_count(x)` | `x: str` | `int` | Returns count of numeric tokens. |
|
|
149
|
+
| `upper_case_count(x)` | `x: str` | `int` | Returns count of uppercase words. |
|
|
150
|
+
| `extract_features(x)` | `x: str` | `dict` | Returns a dictionary containing all extracted feature counts at once. |
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
### 2. Text Cleaning and Normalization
|
|
155
|
+
|
|
156
|
+
| Function | Parameters | Return Type | Description |
|
|
157
|
+
| :--- | :--- | :--- | :--- |
|
|
158
|
+
| `to_lower_case(x)` | `x: str` | `str` | Converts input text to lowercase. |
|
|
159
|
+
| `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
|
|
160
|
+
| `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
|
|
161
|
+
| `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
|
|
162
|
+
| `remove_urls(x)` | `x: str` | `str` | Removes HTTP/HTTPS links and `www` URLs. |
|
|
163
|
+
| `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
|
|
164
|
+
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user`). |
|
|
165
|
+
| `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
|
|
166
|
+
| `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
|
|
167
|
+
| `remove_accented_chars(x)` | `x: str` | `str` | Normalizes accented characters (NFKD -> ASCII). |
|
|
168
|
+
| `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
|
|
169
|
+
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and special symbols. |
|
|
170
|
+
| `remove_repeated_chars(x)` | `x: str` | `str` | Truncates repeated characters beyond 2 consecutive occurrences. |
|
|
171
|
+
| `remove_stop_words(x)` | `x: str` | `str` | Case-insensitive removal of stop words while preserving word case. |
|
|
172
|
+
| `clean_text(text)` | `text: str` | `str` | Runs end-to-end cleaning pipeline (lowercasing, contractions, emails, URLs, HTML, special chars, lemmatization). |
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
### 3. Linguistic and Morphological Processing
|
|
177
|
+
|
|
178
|
+
| Function | Parameters | Return Type | Description |
|
|
179
|
+
| :--- | :--- | :--- | :--- |
|
|
180
|
+
| `convert_to_base(x)` | `x: str` | `str` | Lemmatizes nouns and verbs via spaCy while keeping other POS tags. |
|
|
181
|
+
| `lemmatize(x)` | `x: str` | `str` | Performs full lemmatization across all tokens via spaCy. |
|
|
182
|
+
| `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob model. |
|
|
183
|
+
| `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
|
|
184
|
+
| `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
|
|
185
|
+
| `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
|
|
186
|
+
| `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
|
|
187
|
+
| `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using cached NaiveBayes model. |
|
|
188
|
+
|
|
189
|
+
---
|
|
190
|
+
|
|
191
|
+
### 4. Visualization Suite
|
|
192
|
+
|
|
193
|
+
| Function | Parameters | Return Type | Description |
|
|
194
|
+
| :--- | :--- | :--- | :--- |
|
|
195
|
+
| `get_wordcloud(x, save_path=None, show=True)` | `x: str, save_path: str, show: bool` | `WordCloud` | Generates a WordCloud object, with optional file saving and GUI toggle options. |
|
|
196
|
+
|
|
197
|
+
---
|
|
198
|
+
|
|
199
|
+
## Code Examples
|
|
200
|
+
|
|
201
|
+
### 1. Basic Text Cleaning
|
|
202
|
+
|
|
203
|
+
```python
|
|
204
|
+
import nlp_text_preprocessing as tp
|
|
205
|
+
|
|
206
|
+
raw_text = "Check out https://example.com! Contact support@company.org or RT @user 'I'm loving #NLP'."
|
|
207
|
+
cleaned = tp.clean_text(raw_text)
|
|
208
|
+
|
|
209
|
+
print(cleaned)
|
|
210
|
+
# Output: check out contact support company org or i am love nlp
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### 2. Pandas DataFrame Integration
|
|
214
|
+
|
|
215
|
+
All functions seamlessly support Pandas `.apply()` and handle missing values (`NaN`/`None`):
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
import pandas as pd
|
|
219
|
+
import nlp_text_preprocessing as tp
|
|
220
|
+
|
|
221
|
+
df = pd.DataFrame({
|
|
222
|
+
'text': [
|
|
223
|
+
"I'm loving this NLP package! #awesome",
|
|
224
|
+
"Contact me at info@test.com or visit https://test.com",
|
|
225
|
+
None
|
|
226
|
+
]
|
|
227
|
+
})
|
|
228
|
+
|
|
229
|
+
# Apply end-to-end text cleaning
|
|
230
|
+
df['clean_text'] = df['text'].apply(tp.clean_text)
|
|
231
|
+
|
|
232
|
+
# Extract word counts safely
|
|
233
|
+
df['word_count'] = df['text'].apply(tp.word_count)
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
### 3. Batch Feature Extraction
|
|
237
|
+
|
|
238
|
+
Extract full feature matrices directly into Pandas columns:
|
|
239
|
+
|
|
240
|
+
```python
|
|
241
|
+
import pandas as pd
|
|
242
|
+
import nlp_text_preprocessing as tp
|
|
243
|
+
|
|
244
|
+
df = pd.DataFrame({'text': ["Hello #world! Visit https://example.com", "Contact user@test.com"]})
|
|
245
|
+
|
|
246
|
+
# Extract all features into a new DataFrame
|
|
247
|
+
features_df = df['text'].apply(tp.extract_features).apply(pd.Series)
|
|
248
|
+
print(features_df)
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
### 4. WordCloud Generation
|
|
252
|
+
|
|
253
|
+
```python
|
|
254
|
+
import nlp_text_preprocessing as tp
|
|
255
|
+
|
|
256
|
+
text = "python natural language processing machine learning text mining data science spaCy nltk textblob"
|
|
257
|
+
|
|
258
|
+
# Save directly to disk without opening interactive GUI window
|
|
259
|
+
tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
---
|
|
263
|
+
|
|
264
|
+
## Troubleshooting and Resource Setup
|
|
265
|
+
|
|
266
|
+
If any NLTK corpus or spaCy model throws a missing resource error during runtime, simply run:
|
|
267
|
+
|
|
268
|
+
```python
|
|
269
|
+
import nlp_text_preprocessing as tp
|
|
270
|
+
tp.download_nltk_packages()
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
Or manually download spaCy model via terminal:
|
|
274
|
+
```bash
|
|
275
|
+
python -m spacy download en_core_web_sm
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
---
|
|
279
|
+
|
|
280
|
+
## License and Author
|
|
281
|
+
|
|
282
|
+
- **Author**: [Uditya Narayan Tiwari](https://youtube.com/kgptalkie)
|
|
283
|
+
- **License**: MIT License ([LICENSE](LICENSE))
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
# NLP Text Preprocessing Utility Package
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/nlp_banner.png" alt="NLP Text Preprocessing Banner" width="100%" />
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
<a href="https://pypi.org/project/nlp-text-preprocessing/">
|
|
9
|
+
<img src="https://img.shields.io/pypi/v/nlp-text-preprocessing" alt="PyPI Version">
|
|
10
|
+
</a>
|
|
11
|
+
<a href="https://pypi.org/project/nlp-text-preprocessing/">
|
|
12
|
+
<img src="https://img.shields.io/pypi/pyversions/nlp-text-preprocessing" alt="Python Version">
|
|
13
|
+
</a>
|
|
14
|
+
<a href="https://pepy.tech/projects/nlp-text-preprocessing">
|
|
15
|
+
<img src="https://static.pepy.tech/badge/nlp-text-preprocessing" alt="Total Downloads">
|
|
16
|
+
</a>
|
|
17
|
+
<a href="https://pepy.tech/projects/nlp-text-preprocessing">
|
|
18
|
+
<img src="https://static.pepy.tech/badge/nlp-text-preprocessing/month" alt="Monthly Downloads">
|
|
19
|
+
</a>
|
|
20
|
+
<a href="https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/blob/main/LICENSE">
|
|
21
|
+
<img src="https://img.shields.io/github/license/udityamerit/Text-Processing-Package-For-Natural-Language-Processing" alt="License">
|
|
22
|
+
</a>
|
|
23
|
+
</p>
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## Table of Contents
|
|
28
|
+
|
|
29
|
+
- [Package Overview](#package-overview)
|
|
30
|
+
- [Key Features](#key-features)
|
|
31
|
+
- [Architecture Diagram](#architecture-diagram)
|
|
32
|
+
- [Quickstart and Setup](#quickstart-and-setup)
|
|
33
|
+
- [Complete API Reference](#complete-api-reference)
|
|
34
|
+
- [1. General Feature Extraction](#1-general-feature-extraction)
|
|
35
|
+
- [2. Text Cleaning and Normalization](#2-text-cleaning-and-normalization)
|
|
36
|
+
- [3. Linguistic and Morphological Processing](#3-linguistic-and-morphological-processing)
|
|
37
|
+
- [4. Visualization Suite](#4-visualization-suite)
|
|
38
|
+
- [Code Examples](#code-examples)
|
|
39
|
+
- [Basic Text Cleaning](#1-basic-text-cleaning)
|
|
40
|
+
- [Pandas DataFrame Integration](#2-pandas-dataframe-integration)
|
|
41
|
+
- [Batch Feature Extraction](#3-batch-feature-extraction)
|
|
42
|
+
- [WordCloud Generation](#4-wordcloud-generation)
|
|
43
|
+
- [Troubleshooting and Resource Setup](#troubleshooting-and-resource-setup)
|
|
44
|
+
- [License and Author](#license-and-author)
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## Package Overview
|
|
49
|
+
|
|
50
|
+
**`nlp_text_preprocessing`** is a production-grade, modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
|
|
51
|
+
|
|
52
|
+
It provides **34 single-purpose functions** with built-in defensive guards for `None`, `NaN`, and missing values, making it 100% crash-proof when running over Pandas DataFrames or large raw text datasets.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## Key Features
|
|
57
|
+
|
|
58
|
+
- **Crash-Proof Pipeline**: Automatic `None`/`NaN` input sanitation across all functions.
|
|
59
|
+
- **Lazy and Fast Model Caching**: Efficient lazy-loading for spaCy (`en_core_web_sm`) and cached NaiveBayes sentiment analyzers.
|
|
60
|
+
- **Comprehensive Cleaning**: Strip emails, URLs, HTML, retweets, special symbols, and expand contractions.
|
|
61
|
+
- **Feature Extraction**: Extract word counts, character counts, uppercase counts, numerics, hashtags, mentions, and full feature matrices in one call.
|
|
62
|
+
- **Linguistic Utilities**: Perform lemmatization, noun phrase extraction, spelling correction, n-grams, singularization, and pluralization.
|
|
63
|
+
- **Flexible Visualizations**: Generate and save custom WordClouds to disk or display interactively.
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## Architecture Diagram
|
|
68
|
+
|
|
69
|
+
The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, and **WordCloud** through a unified API surface:
|
|
70
|
+
|
|
71
|
+
<p align="center">
|
|
72
|
+
<img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/architecture_diagram.png" alt="NLP Text Preprocessing Architecture Diagram" width="100%" />
|
|
73
|
+
</p>
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## Quickstart and Setup
|
|
78
|
+
|
|
79
|
+
### 1. Installation
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install nlp-text-preprocessing
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### 2. Downloading Required Corpora and Models
|
|
86
|
+
|
|
87
|
+
To automatically download all required NLTK corpora (`stopwords`, `movie_reviews`, `brown`) and spaCy models (`en_core_web_sm`), run:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
import nlp_text_preprocessing as tp
|
|
91
|
+
|
|
92
|
+
# One-time automated setup for NLTK and spaCy resources
|
|
93
|
+
tp.download_nltk_packages()
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## Complete API Reference
|
|
99
|
+
|
|
100
|
+
Below is the complete reference guide for all **34 functions** available in the library:
|
|
101
|
+
|
|
102
|
+
### 1. General Feature Extraction
|
|
103
|
+
|
|
104
|
+
| Function | Parameters | Return Type | Description |
|
|
105
|
+
| :--- | :--- | :--- | :--- |
|
|
106
|
+
| `word_count(x)` | `x: str` | `int` | Returns total whitespace-separated word count. |
|
|
107
|
+
| `char_count(x)` | `x: str` | `int` | Returns total characters excluding spaces. |
|
|
108
|
+
| `avg_word_len(x)` | `x: str` | `float` | Returns average word length (`0.0` for empty inputs). |
|
|
109
|
+
| `stop_words_count(x)` | `x: str` | `int` | Returns count of stop words (case-insensitive). |
|
|
110
|
+
| `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtag tokens (`#tag`). |
|
|
111
|
+
| `mentions_count(x)` | `x: str` | `int` | Returns count of user handle mentions (`@user`). |
|
|
112
|
+
| `numerics_count(x)` | `x: str` | `int` | Returns count of numeric tokens. |
|
|
113
|
+
| `upper_case_count(x)` | `x: str` | `int` | Returns count of uppercase words. |
|
|
114
|
+
| `extract_features(x)` | `x: str` | `dict` | Returns a dictionary containing all extracted feature counts at once. |
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
### 2. Text Cleaning and Normalization
|
|
119
|
+
|
|
120
|
+
| Function | Parameters | Return Type | Description |
|
|
121
|
+
| :--- | :--- | :--- | :--- |
|
|
122
|
+
| `to_lower_case(x)` | `x: str` | `str` | Converts input text to lowercase. |
|
|
123
|
+
| `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
|
|
124
|
+
| `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
|
|
125
|
+
| `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
|
|
126
|
+
| `remove_urls(x)` | `x: str` | `str` | Removes HTTP/HTTPS links and `www` URLs. |
|
|
127
|
+
| `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
|
|
128
|
+
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user`). |
|
|
129
|
+
| `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
|
|
130
|
+
| `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
|
|
131
|
+
| `remove_accented_chars(x)` | `x: str` | `str` | Normalizes accented characters (NFKD -> ASCII). |
|
|
132
|
+
| `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
|
|
133
|
+
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and special symbols. |
|
|
134
|
+
| `remove_repeated_chars(x)` | `x: str` | `str` | Truncates repeated characters beyond 2 consecutive occurrences. |
|
|
135
|
+
| `remove_stop_words(x)` | `x: str` | `str` | Case-insensitive removal of stop words while preserving word case. |
|
|
136
|
+
| `clean_text(text)` | `text: str` | `str` | Runs end-to-end cleaning pipeline (lowercasing, contractions, emails, URLs, HTML, special chars, lemmatization). |
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
### 3. Linguistic and Morphological Processing
|
|
141
|
+
|
|
142
|
+
| Function | Parameters | Return Type | Description |
|
|
143
|
+
| :--- | :--- | :--- | :--- |
|
|
144
|
+
| `convert_to_base(x)` | `x: str` | `str` | Lemmatizes nouns and verbs via spaCy while keeping other POS tags. |
|
|
145
|
+
| `lemmatize(x)` | `x: str` | `str` | Performs full lemmatization across all tokens via spaCy. |
|
|
146
|
+
| `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob model. |
|
|
147
|
+
| `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
|
|
148
|
+
| `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
|
|
149
|
+
| `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
|
|
150
|
+
| `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
|
|
151
|
+
| `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using cached NaiveBayes model. |
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
### 4. Visualization Suite
|
|
156
|
+
|
|
157
|
+
| Function | Parameters | Return Type | Description |
|
|
158
|
+
| :--- | :--- | :--- | :--- |
|
|
159
|
+
| `get_wordcloud(x, save_path=None, show=True)` | `x: str, save_path: str, show: bool` | `WordCloud` | Generates a WordCloud object, with optional file saving and GUI toggle options. |
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## Code Examples
|
|
164
|
+
|
|
165
|
+
### 1. Basic Text Cleaning
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
import nlp_text_preprocessing as tp
|
|
169
|
+
|
|
170
|
+
raw_text = "Check out https://example.com! Contact support@company.org or RT @user 'I'm loving #NLP'."
|
|
171
|
+
cleaned = tp.clean_text(raw_text)
|
|
172
|
+
|
|
173
|
+
print(cleaned)
|
|
174
|
+
# Output: check out contact support company org or i am love nlp
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### 2. Pandas DataFrame Integration
|
|
178
|
+
|
|
179
|
+
All functions seamlessly support Pandas `.apply()` and handle missing values (`NaN`/`None`):
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
import pandas as pd
|
|
183
|
+
import nlp_text_preprocessing as tp
|
|
184
|
+
|
|
185
|
+
df = pd.DataFrame({
|
|
186
|
+
'text': [
|
|
187
|
+
"I'm loving this NLP package! #awesome",
|
|
188
|
+
"Contact me at info@test.com or visit https://test.com",
|
|
189
|
+
None
|
|
190
|
+
]
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
# Apply end-to-end text cleaning
|
|
194
|
+
df['clean_text'] = df['text'].apply(tp.clean_text)
|
|
195
|
+
|
|
196
|
+
# Extract word counts safely
|
|
197
|
+
df['word_count'] = df['text'].apply(tp.word_count)
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
### 3. Batch Feature Extraction
|
|
201
|
+
|
|
202
|
+
Extract full feature matrices directly into Pandas columns:
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
import pandas as pd
|
|
206
|
+
import nlp_text_preprocessing as tp
|
|
207
|
+
|
|
208
|
+
df = pd.DataFrame({'text': ["Hello #world! Visit https://example.com", "Contact user@test.com"]})
|
|
209
|
+
|
|
210
|
+
# Extract all features into a new DataFrame
|
|
211
|
+
features_df = df['text'].apply(tp.extract_features).apply(pd.Series)
|
|
212
|
+
print(features_df)
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### 4. WordCloud Generation
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
import nlp_text_preprocessing as tp
|
|
219
|
+
|
|
220
|
+
text = "python natural language processing machine learning text mining data science spaCy nltk textblob"
|
|
221
|
+
|
|
222
|
+
# Save directly to disk without opening interactive GUI window
|
|
223
|
+
tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
---
|
|
227
|
+
|
|
228
|
+
## Troubleshooting and Resource Setup
|
|
229
|
+
|
|
230
|
+
If any NLTK corpus or spaCy model throws a missing resource error during runtime, simply run:
|
|
231
|
+
|
|
232
|
+
```python
|
|
233
|
+
import nlp_text_preprocessing as tp
|
|
234
|
+
tp.download_nltk_packages()
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
Or manually download spaCy model via terminal:
|
|
238
|
+
```bash
|
|
239
|
+
python -m spacy download en_core_web_sm
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
---
|
|
243
|
+
|
|
244
|
+
## License and Author
|
|
245
|
+
|
|
246
|
+
- **Author**: [Uditya Narayan Tiwari](https://youtube.com/kgptalkie)
|
|
247
|
+
- **License**: MIT License ([LICENSE](LICENSE))
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# entry level code and import text preprocess
|
|
2
|
+
|
|
3
|
+
from .all_text_preprocess import *
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
'download_nltk_packages',
|
|
7
|
+
'word_count',
|
|
8
|
+
'char_count',
|
|
9
|
+
'avg_word_len',
|
|
10
|
+
'stop_words_count',
|
|
11
|
+
'hashtags_count',
|
|
12
|
+
'mentions_count',
|
|
13
|
+
'numerics_count',
|
|
14
|
+
'upper_case_count',
|
|
15
|
+
'to_lower_case',
|
|
16
|
+
'contraction_to_expansion',
|
|
17
|
+
'remove_emails',
|
|
18
|
+
'count_emails',
|
|
19
|
+
'count_urls',
|
|
20
|
+
'remove_urls',
|
|
21
|
+
'count_rt',
|
|
22
|
+
'remove_rt',
|
|
23
|
+
'remove_html_tag',
|
|
24
|
+
'remove_accented_chars',
|
|
25
|
+
'remove_mentions',
|
|
26
|
+
'remove_special_chars',
|
|
27
|
+
'remove_repeated_chars',
|
|
28
|
+
'remove_stop_words',
|
|
29
|
+
'convert_to_base',
|
|
30
|
+
'lemmatize',
|
|
31
|
+
'get_wordcloud',
|
|
32
|
+
'correct_spelling',
|
|
33
|
+
'get_noun_phrase',
|
|
34
|
+
'n_grams',
|
|
35
|
+
'singularize_words',
|
|
36
|
+
'pluralize_words',
|
|
37
|
+
'sentiment_analysis',
|
|
38
|
+
'extract_features',
|
|
39
|
+
'clean_text',
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def extract_features(x):
|
|
44
|
+
"""
|
|
45
|
+
Extracts a dictionary of text features from text x.
|
|
46
|
+
"""
|
|
47
|
+
return {
|
|
48
|
+
'word_count': word_count(x),
|
|
49
|
+
'char_count': char_count(x),
|
|
50
|
+
'avg_word_len': avg_word_len(x),
|
|
51
|
+
'stop_words_count': stop_words_count(x),
|
|
52
|
+
'hashtags_count': hashtags_count(x),
|
|
53
|
+
'mentions_count': mentions_count(x),
|
|
54
|
+
'numerics_count': numerics_count(x),
|
|
55
|
+
'upper_case_count': upper_case_count(x),
|
|
56
|
+
'noun_phrases': get_noun_phrase(x),
|
|
57
|
+
'sentiment_analysis': sentiment_analysis(x)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def clean_text(text):
|
|
62
|
+
"""
|
|
63
|
+
Applies a sequence of cleaning and normalization steps on text.
|
|
64
|
+
"""
|
|
65
|
+
text = to_lower_case(text)
|
|
66
|
+
text = contraction_to_expansion(text)
|
|
67
|
+
text = remove_emails(text)
|
|
68
|
+
text = remove_urls(text)
|
|
69
|
+
text = remove_html_tag(text)
|
|
70
|
+
text = remove_special_chars(text)
|
|
71
|
+
text = lemmatize(text)
|
|
72
|
+
text = convert_to_base(text)
|
|
73
|
+
return text
|