nlp-text-preprocessing 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nlp_text_preprocessing-0.1.1/MANIFEST.in +5 -0
- nlp_text_preprocessing-0.1.1/PKG-INFO +270 -0
- nlp_text_preprocessing-0.1.1/Readme.md +234 -0
- nlp_text_preprocessing-0.1.1/architecture_diagram.png +0 -0
- nlp_text_preprocessing-0.1.1/nlp_banner.png +0 -0
- nlp_text_preprocessing-0.1.1/nlp_text_preprocessing/__init__.py +73 -0
- nlp_text_preprocessing-0.1.1/nlp_text_preprocessing/all_text_preprocess.py +318 -0
- nlp_text_preprocessing-0.1.1/nlp_text_preprocessing.egg-info/PKG-INFO +270 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing.egg-info/SOURCES.txt +4 -1
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.1}/setup.py +1 -1
- nlp_text_preprocessing-0.1.1/tests/test_all_text_preprocess.py +139 -0
- nlp_text_preprocessing-0.1.0/MANIFEST.in +0 -3
- nlp_text_preprocessing-0.1.0/PKG-INFO +0 -267
- nlp_text_preprocessing-0.1.0/Readme.md +0 -231
- nlp_text_preprocessing-0.1.0/nlp_text_preprocessing/__init__.py +0 -31
- nlp_text_preprocessing-0.1.0/nlp_text_preprocessing/all_text_preprocess.py +0 -190
- nlp_text_preprocessing-0.1.0/nlp_text_preprocessing.egg-info/PKG-INFO +0 -267
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.1}/LICENSE +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing.egg-info/requires.txt +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.1}/requirements.txt +0 -0
- {nlp_text_preprocessing-0.1.0 → nlp_text_preprocessing-0.1.1}/setup.cfg +0 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nlp_text_preprocessing
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: This is a Text Processing Package For NLP
|
|
5
|
+
Author: Uditya Narayan Tiwari
|
|
6
|
+
Author-email: tiwarimerit@gmail.com
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: spacy
|
|
14
|
+
Requires-Dist: textblob
|
|
15
|
+
Requires-Dist: beautifulsoup4
|
|
16
|
+
Requires-Dist: nltk
|
|
17
|
+
Requires-Dist: openpyxl
|
|
18
|
+
Requires-Dist: SpeechRecognition==3.10.4
|
|
19
|
+
Requires-Dist: pyaudio==0.2.14
|
|
20
|
+
Requires-Dist: PrettyTable
|
|
21
|
+
Requires-Dist: scikit-learn
|
|
22
|
+
Requires-Dist: wordcloud
|
|
23
|
+
Requires-Dist: lxml
|
|
24
|
+
Requires-Dist: pandas
|
|
25
|
+
Requires-Dist: numpy
|
|
26
|
+
Requires-Dist: matplotlib
|
|
27
|
+
Dynamic: author
|
|
28
|
+
Dynamic: author-email
|
|
29
|
+
Dynamic: classifier
|
|
30
|
+
Dynamic: description
|
|
31
|
+
Dynamic: description-content-type
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
Dynamic: requires-dist
|
|
34
|
+
Dynamic: requires-python
|
|
35
|
+
Dynamic: summary
|
|
36
|
+
|
|
37
|
+
# NLP Text Preprocessing Utility Package
|
|
38
|
+
|
|
39
|
+
<p align="center">
|
|
40
|
+
<img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/nlp_banner.png" alt="NLP Text Preprocessing Banner" width="100%" />
|
|
41
|
+
</p>
|
|
42
|
+
|
|
43
|
+
<p align="center">
|
|
44
|
+
<a href="https://pypi.org/project/nlp-text-preprocessing/">
|
|
45
|
+
<img src="https://img.shields.io/pypi/v/nlp-text-preprocessing" alt="PyPI Version">
|
|
46
|
+
</a>
|
|
47
|
+
<a href="https://pypi.org/project/nlp-text-preprocessing/">
|
|
48
|
+
<img src="https://img.shields.io/pypi/pyversions/nlp-text-preprocessing" alt="Python Version">
|
|
49
|
+
</a>
|
|
50
|
+
<a href="https://pepy.tech/projects/nlp-text-preprocessing">
|
|
51
|
+
<img src="https://static.pepy.tech/badge/nlp-text-preprocessing" alt="Total Downloads">
|
|
52
|
+
</a>
|
|
53
|
+
<a href="https://pepy.tech/projects/nlp-text-preprocessing">
|
|
54
|
+
<img src="https://static.pepy.tech/badge/nlp-text-preprocessing/month" alt="Monthly Downloads">
|
|
55
|
+
</a>
|
|
56
|
+
<a href="https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/blob/main/LICENSE">
|
|
57
|
+
<img src="https://img.shields.io/github/license/udityamerit/Text-Processing-Package-For-Natural-Language-Processing" alt="License">
|
|
58
|
+
</a>
|
|
59
|
+
</p>
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## 📌 Overview
|
|
64
|
+
|
|
65
|
+
**`nlp_text_preprocessing`** is a production-grade, fast, and modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
|
|
66
|
+
|
|
67
|
+
Maintained by **[Uditya Narayan Tiwari](https://youtube.com/kgptalkie)**.
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## 🏗 Architecture Diagram
|
|
72
|
+
|
|
73
|
+
The package is built with a decoupled, high-performance architecture integrating **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, and **WordCloud** with safe null/NaN defensive wrappers for seamless Pandas DataFrame processing.
|
|
74
|
+
|
|
75
|
+
<p align="center">
|
|
76
|
+
<img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/architecture_diagram.png" alt="NLP Text Preprocessing Architecture Diagram" width="100%" />
|
|
77
|
+
</p>
|
|
78
|
+
|
|
79
|
+
### Pipeline Architecture Workflow
|
|
80
|
+
|
|
81
|
+
```mermaid
|
|
82
|
+
flowchart TD
|
|
83
|
+
Raw[Raw Input Text / Pandas Series] --> Guard[_clean_input Guard: None/NaN Safety]
|
|
84
|
+
|
|
85
|
+
Guard --> Cleaning[Cleaning & Normalization Module]
|
|
86
|
+
Guard --> Features[Feature Extraction Engine]
|
|
87
|
+
Guard --> Linguistic[Linguistic & NLP Processing]
|
|
88
|
+
Guard --> Viz[Visualization Suite]
|
|
89
|
+
|
|
90
|
+
Cleaning --> Lower[to_lower_case]
|
|
91
|
+
Cleaning --> Expand[contraction_to_expansion]
|
|
92
|
+
Cleaning --> Email[remove_emails / count_emails]
|
|
93
|
+
Cleaning --> URL[remove_urls / count_urls]
|
|
94
|
+
Cleaning --> HTML[remove_html_tag]
|
|
95
|
+
Cleaning --> Regex[remove_special_chars / remove_mentions / remove_rt]
|
|
96
|
+
Cleaning --> Stops[remove_stop_words - Case Insensitive]
|
|
97
|
+
|
|
98
|
+
Features --> Stats[word_count / char_count / avg_word_len]
|
|
99
|
+
Features --> Counts[hashtags_count / mentions_count / numerics_count / upper_case_count]
|
|
100
|
+
Features --> Summary[extract_features - Full Feature Dict]
|
|
101
|
+
|
|
102
|
+
Linguistic --> SpaCy[spaCy Lemmatizer / convert_to_base]
|
|
103
|
+
Linguistic --> NLTKCorpus[Auto-cached NLTK Corpora Check]
|
|
104
|
+
Linguistic --> Sentiment[TextBlob Sentiment - NaiveBayes Cached]
|
|
105
|
+
Linguistic --> Grams[n_grams / get_noun_phrase / singularize / pluralize]
|
|
106
|
+
|
|
107
|
+
Viz --> Cloud[get_wordcloud - File Save & GUI options]
|
|
108
|
+
|
|
109
|
+
Cleaning --> Pipeline[clean_text - End-to-End Pipeline]
|
|
110
|
+
Features --> Output[ML Ready Feature Matrix / Clean Text]
|
|
111
|
+
Linguistic --> Output
|
|
112
|
+
Pipeline --> Output
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## 🚀 Quick Installation
|
|
118
|
+
|
|
119
|
+
### From PyPI
|
|
120
|
+
```bash
|
|
121
|
+
pip install nlp_text_preprocessing
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### From GitHub (Latest Development Version)
|
|
125
|
+
```bash
|
|
126
|
+
pip install git+https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing.git --upgrade --force-reinstall
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### Automatic Resource Setup
|
|
130
|
+
Download required NLTK corpora (`stopwords`, `movie_reviews`, `brown`, etc.) and spaCy language models (`en_core_web_sm`):
|
|
131
|
+
```python
|
|
132
|
+
import nlp_text_preprocessing as tp
|
|
133
|
+
|
|
134
|
+
# Downloads all required NLTK corpora automatically
|
|
135
|
+
tp.download_nltk_packages()
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## 📚 Complete API Reference & Function List
|
|
141
|
+
|
|
142
|
+
Below is the complete list of all **34 functions** available in the `nlp_text_preprocessing` package:
|
|
143
|
+
|
|
144
|
+
### 1. General Feature Extraction Functions
|
|
145
|
+
|
|
146
|
+
| Function Name | Parameters | Return Type | Description |
|
|
147
|
+
| :--- | :--- | :--- | :--- |
|
|
148
|
+
| `word_count(x)` | `x: str` | `int` | Returns total number of whitespace-separated words. |
|
|
149
|
+
| `char_count(x)` | `x: str` | `int` | Returns total characters excluding whitespace. |
|
|
150
|
+
| `avg_word_len(x)` | `x: str` | `float` | Returns average word length (safely returns `0.0` on empty input). |
|
|
151
|
+
| `stop_words_count(x)` | `x: str` | `int` | Returns count of stop words (case-insensitive check). |
|
|
152
|
+
| `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtags (`#tag`). |
|
|
153
|
+
| `mentions_count(x)` | `x: str` | `int` | Returns count of user mentions (`@user`). |
|
|
154
|
+
| `numerics_count(x)` | `x: str` | `int` | Returns count of standalone numeric tokens. |
|
|
155
|
+
| `upper_case_count(x)` | `x: str` | `int` | Returns count of uppercase words. |
|
|
156
|
+
| `extract_features(x)` | `x: str` | `dict` | Returns a dictionary containing all extracted text features at once. |
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
### 2. Text Cleaning & Normalization Functions
|
|
161
|
+
|
|
162
|
+
| Function Name | Parameters | Return Type | Description |
|
|
163
|
+
| :--- | :--- | :--- | :--- |
|
|
164
|
+
| `to_lower_case(x)` | `x: str` | `str` | Converts input text to lowercase. |
|
|
165
|
+
| `contraction_to_expansion(x)` | `x: str` | `str` | Expands abbreviations and contractions using contractions lookup table. |
|
|
166
|
+
| `remove_emails(x)` | `x: str` | `str` | Strips email addresses from text. |
|
|
167
|
+
| `count_emails(x)` | `x: str` | `int` | Returns count of email addresses found in text. |
|
|
168
|
+
| `remove_urls(x)` | `x: str` | `str` | Strips HTTP/HTTPS URLs and `www` links from text. |
|
|
169
|
+
| `count_urls(x)` | `x: str` | `int` | Returns count of URLs found in text. |
|
|
170
|
+
| `remove_rt(x)` | `x: str` | `str` | Removes retweet tags (`RT @user`). |
|
|
171
|
+
| `count_rt(x)` | `x: str` | `int` | Returns count of retweet markers in text. |
|
|
172
|
+
| `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup parser. |
|
|
173
|
+
| `remove_accented_chars(x)` | `x: str` | `str` | Normalizes and converts accented characters (NFKD -> ASCII). |
|
|
174
|
+
| `remove_mentions(x)` | `x: str` | `str` | Strips user handles (`@username`). |
|
|
175
|
+
| `remove_special_chars(x)` | `x: str` | `str` | Removes punctuation and non-alphanumeric special symbols. |
|
|
176
|
+
| `remove_repeated_chars(x)` | `x: str` | `str` | Truncates character repetitions beyond 2 consecutive occurrences. |
|
|
177
|
+
| `remove_stop_words(x)` | `x: str` | `str` | Removes stop words (case-insensitive, preserving capitalized text structure). |
|
|
178
|
+
| `clean_text(text)` | `text: str` | `str` | Executes full end-to-end cleaning pipeline (lowercasing, contractions, emails, URLs, HTML, special chars, lemmatization). |
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
### 3. Linguistic & Morphological Processing Functions
|
|
183
|
+
|
|
184
|
+
| Function Name | Parameters | Return Type | Description |
|
|
185
|
+
| :--- | :--- | :--- | :--- |
|
|
186
|
+
| `convert_to_base(x)` | `x: str` | `str` | Lemmatizes nouns and verbs via spaCy while keeping other POS tags intact. |
|
|
187
|
+
| `lemmatize(x)` | `x: str` | `str` | Performs full lemmatization across all tokens via spaCy (`en_core_web_sm`). |
|
|
188
|
+
| `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob dictionary model. |
|
|
189
|
+
| `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
|
|
190
|
+
| `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
|
|
191
|
+
| `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
|
|
192
|
+
| `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
|
|
193
|
+
| `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using module-cached NaiveBayesAnalyzer. |
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
### 4. Visualization Functions
|
|
198
|
+
|
|
199
|
+
| Function Name | Parameters | Return Type | Description |
|
|
200
|
+
| :--- | :--- | :--- | :--- |
|
|
201
|
+
| `get_wordcloud(x, save_path=None, show=True)` | `x: str, save_path: str, show: bool` | `WordCloud` | Generates a WordCloud image object, with optional file saving and GUI display options. |
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## 💻 Usage Code Examples
|
|
206
|
+
|
|
207
|
+
### 1. Cleaning Raw Text
|
|
208
|
+
```python
|
|
209
|
+
import nlp_text_preprocessing as tp
|
|
210
|
+
|
|
211
|
+
raw_text = "Check out https://example.com! Contact support@company.org or RT @user 'I'm loving #NLP'."
|
|
212
|
+
cleaned = tp.clean_text(raw_text)
|
|
213
|
+
|
|
214
|
+
print(cleaned)
|
|
215
|
+
# Output: check out contact support company org or i am love nlp
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
### 2. Processing Pandas DataFrames
|
|
219
|
+
All functions safely handle `None`, `NaN`, and missing values:
|
|
220
|
+
```python
|
|
221
|
+
import pandas as pd
|
|
222
|
+
import nlp_text_preprocessing as tp
|
|
223
|
+
|
|
224
|
+
df = pd.DataFrame({
|
|
225
|
+
'text': [
|
|
226
|
+
"I'm loving this NLP package! #awesome",
|
|
227
|
+
"Contact me at info@test.com or visit https://test.com",
|
|
228
|
+
None
|
|
229
|
+
]
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
# Apply full cleaning pipeline
|
|
233
|
+
df['clean_text'] = df['text'].apply(tp.clean_text)
|
|
234
|
+
|
|
235
|
+
# Extract word counts
|
|
236
|
+
df['word_count'] = df['text'].apply(tp.word_count)
|
|
237
|
+
|
|
238
|
+
# Extract full feature dictionary as DataFrame columns
|
|
239
|
+
features_df = df['text'].apply(tp.extract_features).apply(pd.Series)
|
|
240
|
+
print(features_df)
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
### 3. WordCloud Generation & Saving
|
|
244
|
+
```python
|
|
245
|
+
import nlp_text_preprocessing as tp
|
|
246
|
+
|
|
247
|
+
text = "python natural language processing machine learning text mining data science spaCy nltk textblob"
|
|
248
|
+
|
|
249
|
+
# Save directly to file without requiring GUI display window
|
|
250
|
+
tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
---
|
|
254
|
+
|
|
255
|
+
## 🛠 Setup & Requirements
|
|
256
|
+
|
|
257
|
+
- `spacy >= 3.0`
|
|
258
|
+
- `textblob`
|
|
259
|
+
- `beautifulsoup4`
|
|
260
|
+
- `nltk`
|
|
261
|
+
- `wordcloud`
|
|
262
|
+
- `pandas`
|
|
263
|
+
- `numpy`
|
|
264
|
+
- `matplotlib`
|
|
265
|
+
|
|
266
|
+
---
|
|
267
|
+
|
|
268
|
+
## 📄 License
|
|
269
|
+
|
|
270
|
+
Distributed under the MIT License. See [`LICENSE`](LICENSE) for more information.
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
# NLP Text Preprocessing Utility Package
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/nlp_banner.png" alt="NLP Text Preprocessing Banner" width="100%" />
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
<a href="https://pypi.org/project/nlp-text-preprocessing/">
|
|
9
|
+
<img src="https://img.shields.io/pypi/v/nlp-text-preprocessing" alt="PyPI Version">
|
|
10
|
+
</a>
|
|
11
|
+
<a href="https://pypi.org/project/nlp-text-preprocessing/">
|
|
12
|
+
<img src="https://img.shields.io/pypi/pyversions/nlp-text-preprocessing" alt="Python Version">
|
|
13
|
+
</a>
|
|
14
|
+
<a href="https://pepy.tech/projects/nlp-text-preprocessing">
|
|
15
|
+
<img src="https://static.pepy.tech/badge/nlp-text-preprocessing" alt="Total Downloads">
|
|
16
|
+
</a>
|
|
17
|
+
<a href="https://pepy.tech/projects/nlp-text-preprocessing">
|
|
18
|
+
<img src="https://static.pepy.tech/badge/nlp-text-preprocessing/month" alt="Monthly Downloads">
|
|
19
|
+
</a>
|
|
20
|
+
<a href="https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/blob/main/LICENSE">
|
|
21
|
+
<img src="https://img.shields.io/github/license/udityamerit/Text-Processing-Package-For-Natural-Language-Processing" alt="License">
|
|
22
|
+
</a>
|
|
23
|
+
</p>
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## 📌 Overview
|
|
28
|
+
|
|
29
|
+
**`nlp_text_preprocessing`** is a production-grade, fast, and modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
|
|
30
|
+
|
|
31
|
+
Maintained by **[Uditya Narayan Tiwari](https://youtube.com/kgptalkie)**.
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## 🏗 Architecture Diagram
|
|
36
|
+
|
|
37
|
+
The package is built with a decoupled, high-performance architecture integrating **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, and **WordCloud** with safe null/NaN defensive wrappers for seamless Pandas DataFrame processing.
|
|
38
|
+
|
|
39
|
+
<p align="center">
|
|
40
|
+
<img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/architecture_diagram.png" alt="NLP Text Preprocessing Architecture Diagram" width="100%" />
|
|
41
|
+
</p>
|
|
42
|
+
|
|
43
|
+
### Pipeline Architecture Workflow
|
|
44
|
+
|
|
45
|
+
```mermaid
|
|
46
|
+
flowchart TD
|
|
47
|
+
Raw[Raw Input Text / Pandas Series] --> Guard[_clean_input Guard: None/NaN Safety]
|
|
48
|
+
|
|
49
|
+
Guard --> Cleaning[Cleaning & Normalization Module]
|
|
50
|
+
Guard --> Features[Feature Extraction Engine]
|
|
51
|
+
Guard --> Linguistic[Linguistic & NLP Processing]
|
|
52
|
+
Guard --> Viz[Visualization Suite]
|
|
53
|
+
|
|
54
|
+
Cleaning --> Lower[to_lower_case]
|
|
55
|
+
Cleaning --> Expand[contraction_to_expansion]
|
|
56
|
+
Cleaning --> Email[remove_emails / count_emails]
|
|
57
|
+
Cleaning --> URL[remove_urls / count_urls]
|
|
58
|
+
Cleaning --> HTML[remove_html_tag]
|
|
59
|
+
Cleaning --> Regex[remove_special_chars / remove_mentions / remove_rt]
|
|
60
|
+
Cleaning --> Stops[remove_stop_words - Case Insensitive]
|
|
61
|
+
|
|
62
|
+
Features --> Stats[word_count / char_count / avg_word_len]
|
|
63
|
+
Features --> Counts[hashtags_count / mentions_count / numerics_count / upper_case_count]
|
|
64
|
+
Features --> Summary[extract_features - Full Feature Dict]
|
|
65
|
+
|
|
66
|
+
Linguistic --> SpaCy[spaCy Lemmatizer / convert_to_base]
|
|
67
|
+
Linguistic --> NLTKCorpus[Auto-cached NLTK Corpora Check]
|
|
68
|
+
Linguistic --> Sentiment[TextBlob Sentiment - NaiveBayes Cached]
|
|
69
|
+
Linguistic --> Grams[n_grams / get_noun_phrase / singularize / pluralize]
|
|
70
|
+
|
|
71
|
+
Viz --> Cloud[get_wordcloud - File Save & GUI options]
|
|
72
|
+
|
|
73
|
+
Cleaning --> Pipeline[clean_text - End-to-End Pipeline]
|
|
74
|
+
Features --> Output[ML Ready Feature Matrix / Clean Text]
|
|
75
|
+
Linguistic --> Output
|
|
76
|
+
Pipeline --> Output
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
## 🚀 Quick Installation
|
|
82
|
+
|
|
83
|
+
### From PyPI
|
|
84
|
+
```bash
|
|
85
|
+
pip install nlp_text_preprocessing
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### From GitHub (Latest Development Version)
|
|
89
|
+
```bash
|
|
90
|
+
pip install git+https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing.git --upgrade --force-reinstall
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### Automatic Resource Setup
|
|
94
|
+
Download required NLTK corpora (`stopwords`, `movie_reviews`, `brown`, etc.) and spaCy language models (`en_core_web_sm`):
|
|
95
|
+
```python
|
|
96
|
+
import nlp_text_preprocessing as tp
|
|
97
|
+
|
|
98
|
+
# Downloads all required NLTK corpora automatically
|
|
99
|
+
tp.download_nltk_packages()
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## 📚 Complete API Reference & Function List
|
|
105
|
+
|
|
106
|
+
Below is the complete list of all **34 functions** available in the `nlp_text_preprocessing` package:
|
|
107
|
+
|
|
108
|
+
### 1. General Feature Extraction Functions
|
|
109
|
+
|
|
110
|
+
| Function Name | Parameters | Return Type | Description |
|
|
111
|
+
| :--- | :--- | :--- | :--- |
|
|
112
|
+
| `word_count(x)` | `x: str` | `int` | Returns total number of whitespace-separated words. |
|
|
113
|
+
| `char_count(x)` | `x: str` | `int` | Returns total characters excluding whitespace. |
|
|
114
|
+
| `avg_word_len(x)` | `x: str` | `float` | Returns average word length (safely returns `0.0` on empty input). |
|
|
115
|
+
| `stop_words_count(x)` | `x: str` | `int` | Returns count of stop words (case-insensitive check). |
|
|
116
|
+
| `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtags (`#tag`). |
|
|
117
|
+
| `mentions_count(x)` | `x: str` | `int` | Returns count of user mentions (`@user`). |
|
|
118
|
+
| `numerics_count(x)` | `x: str` | `int` | Returns count of standalone numeric tokens. |
|
|
119
|
+
| `upper_case_count(x)` | `x: str` | `int` | Returns count of uppercase words. |
|
|
120
|
+
| `extract_features(x)` | `x: str` | `dict` | Returns a dictionary containing all extracted text features at once. |
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
### 2. Text Cleaning & Normalization Functions
|
|
125
|
+
|
|
126
|
+
| Function Name | Parameters | Return Type | Description |
|
|
127
|
+
| :--- | :--- | :--- | :--- |
|
|
128
|
+
| `to_lower_case(x)` | `x: str` | `str` | Converts input text to lowercase. |
|
|
129
|
+
| `contraction_to_expansion(x)` | `x: str` | `str` | Expands abbreviations and contractions using contractions lookup table. |
|
|
130
|
+
| `remove_emails(x)` | `x: str` | `str` | Strips email addresses from text. |
|
|
131
|
+
| `count_emails(x)` | `x: str` | `int` | Returns count of email addresses found in text. |
|
|
132
|
+
| `remove_urls(x)` | `x: str` | `str` | Strips HTTP/HTTPS URLs and `www` links from text. |
|
|
133
|
+
| `count_urls(x)` | `x: str` | `int` | Returns count of URLs found in text. |
|
|
134
|
+
| `remove_rt(x)` | `x: str` | `str` | Removes retweet tags (`RT @user`). |
|
|
135
|
+
| `count_rt(x)` | `x: str` | `int` | Returns count of retweet markers in text. |
|
|
136
|
+
| `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup parser. |
|
|
137
|
+
| `remove_accented_chars(x)` | `x: str` | `str` | Normalizes and converts accented characters (NFKD -> ASCII). |
|
|
138
|
+
| `remove_mentions(x)` | `x: str` | `str` | Strips user handles (`@username`). |
|
|
139
|
+
| `remove_special_chars(x)` | `x: str` | `str` | Removes punctuation and non-alphanumeric special symbols. |
|
|
140
|
+
| `remove_repeated_chars(x)` | `x: str` | `str` | Truncates character repetitions beyond 2 consecutive occurrences. |
|
|
141
|
+
| `remove_stop_words(x)` | `x: str` | `str` | Removes stop words (case-insensitive, preserving capitalized text structure). |
|
|
142
|
+
| `clean_text(text)` | `text: str` | `str` | Executes full end-to-end cleaning pipeline (lowercasing, contractions, emails, URLs, HTML, special chars, lemmatization). |
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
### 3. Linguistic & Morphological Processing Functions
|
|
147
|
+
|
|
148
|
+
| Function Name | Parameters | Return Type | Description |
|
|
149
|
+
| :--- | :--- | :--- | :--- |
|
|
150
|
+
| `convert_to_base(x)` | `x: str` | `str` | Lemmatizes nouns and verbs via spaCy while keeping other POS tags intact. |
|
|
151
|
+
| `lemmatize(x)` | `x: str` | `str` | Performs full lemmatization across all tokens via spaCy (`en_core_web_sm`). |
|
|
152
|
+
| `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob dictionary model. |
|
|
153
|
+
| `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
|
|
154
|
+
| `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
|
|
155
|
+
| `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
|
|
156
|
+
| `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
|
|
157
|
+
| `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using module-cached NaiveBayesAnalyzer. |
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
### 4. Visualization Functions
|
|
162
|
+
|
|
163
|
+
| Function Name | Parameters | Return Type | Description |
|
|
164
|
+
| :--- | :--- | :--- | :--- |
|
|
165
|
+
| `get_wordcloud(x, save_path=None, show=True)` | `x: str, save_path: str, show: bool` | `WordCloud` | Generates a WordCloud image object, with optional file saving and GUI display options. |
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## 💻 Usage Code Examples
|
|
170
|
+
|
|
171
|
+
### 1. Cleaning Raw Text
|
|
172
|
+
```python
|
|
173
|
+
import nlp_text_preprocessing as tp
|
|
174
|
+
|
|
175
|
+
raw_text = "Check out https://example.com! Contact support@company.org or RT @user 'I'm loving #NLP'."
|
|
176
|
+
cleaned = tp.clean_text(raw_text)
|
|
177
|
+
|
|
178
|
+
print(cleaned)
|
|
179
|
+
# Output: check out contact support company org or i am love nlp
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### 2. Processing Pandas DataFrames
|
|
183
|
+
All functions safely handle `None`, `NaN`, and missing values:
|
|
184
|
+
```python
|
|
185
|
+
import pandas as pd
|
|
186
|
+
import nlp_text_preprocessing as tp
|
|
187
|
+
|
|
188
|
+
df = pd.DataFrame({
|
|
189
|
+
'text': [
|
|
190
|
+
"I'm loving this NLP package! #awesome",
|
|
191
|
+
"Contact me at info@test.com or visit https://test.com",
|
|
192
|
+
None
|
|
193
|
+
]
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
# Apply full cleaning pipeline
|
|
197
|
+
df['clean_text'] = df['text'].apply(tp.clean_text)
|
|
198
|
+
|
|
199
|
+
# Extract word counts
|
|
200
|
+
df['word_count'] = df['text'].apply(tp.word_count)
|
|
201
|
+
|
|
202
|
+
# Extract full feature dictionary as DataFrame columns
|
|
203
|
+
features_df = df['text'].apply(tp.extract_features).apply(pd.Series)
|
|
204
|
+
print(features_df)
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### 3. WordCloud Generation & Saving
|
|
208
|
+
```python
|
|
209
|
+
import nlp_text_preprocessing as tp
|
|
210
|
+
|
|
211
|
+
text = "python natural language processing machine learning text mining data science spaCy nltk textblob"
|
|
212
|
+
|
|
213
|
+
# Save directly to file without requiring GUI display window
|
|
214
|
+
tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
---
|
|
218
|
+
|
|
219
|
+
## 🛠 Setup & Requirements
|
|
220
|
+
|
|
221
|
+
- `spacy >= 3.0`
|
|
222
|
+
- `textblob`
|
|
223
|
+
- `beautifulsoup4`
|
|
224
|
+
- `nltk`
|
|
225
|
+
- `wordcloud`
|
|
226
|
+
- `pandas`
|
|
227
|
+
- `numpy`
|
|
228
|
+
- `matplotlib`
|
|
229
|
+
|
|
230
|
+
---
|
|
231
|
+
|
|
232
|
+
## 📄 License
|
|
233
|
+
|
|
234
|
+
Distributed under the MIT License. See [`LICENSE`](LICENSE) for more information.
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# entry level code and import text preprocess
|
|
2
|
+
|
|
3
|
+
from .all_text_preprocess import *
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
'download_nltk_packages',
|
|
7
|
+
'word_count',
|
|
8
|
+
'char_count',
|
|
9
|
+
'avg_word_len',
|
|
10
|
+
'stop_words_count',
|
|
11
|
+
'hashtags_count',
|
|
12
|
+
'mentions_count',
|
|
13
|
+
'numerics_count',
|
|
14
|
+
'upper_case_count',
|
|
15
|
+
'to_lower_case',
|
|
16
|
+
'contraction_to_expansion',
|
|
17
|
+
'remove_emails',
|
|
18
|
+
'count_emails',
|
|
19
|
+
'count_urls',
|
|
20
|
+
'remove_urls',
|
|
21
|
+
'count_rt',
|
|
22
|
+
'remove_rt',
|
|
23
|
+
'remove_html_tag',
|
|
24
|
+
'remove_accented_chars',
|
|
25
|
+
'remove_mentions',
|
|
26
|
+
'remove_special_chars',
|
|
27
|
+
'remove_repeated_chars',
|
|
28
|
+
'remove_stop_words',
|
|
29
|
+
'convert_to_base',
|
|
30
|
+
'lemmatize',
|
|
31
|
+
'get_wordcloud',
|
|
32
|
+
'correct_spelling',
|
|
33
|
+
'get_noun_phrase',
|
|
34
|
+
'n_grams',
|
|
35
|
+
'singularize_words',
|
|
36
|
+
'pluralize_words',
|
|
37
|
+
'sentiment_analysis',
|
|
38
|
+
'extract_features',
|
|
39
|
+
'clean_text',
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def extract_features(x):
|
|
44
|
+
"""
|
|
45
|
+
Extracts a dictionary of text features from text x.
|
|
46
|
+
"""
|
|
47
|
+
return {
|
|
48
|
+
'word_count': word_count(x),
|
|
49
|
+
'char_count': char_count(x),
|
|
50
|
+
'avg_word_len': avg_word_len(x),
|
|
51
|
+
'stop_words_count': stop_words_count(x),
|
|
52
|
+
'hashtags_count': hashtags_count(x),
|
|
53
|
+
'mentions_count': mentions_count(x),
|
|
54
|
+
'numerics_count': numerics_count(x),
|
|
55
|
+
'upper_case_count': upper_case_count(x),
|
|
56
|
+
'noun_phrases': get_noun_phrase(x),
|
|
57
|
+
'sentiment_analysis': sentiment_analysis(x)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def clean_text(text):
|
|
62
|
+
"""
|
|
63
|
+
Applies a sequence of cleaning and normalization steps on text.
|
|
64
|
+
"""
|
|
65
|
+
text = to_lower_case(text)
|
|
66
|
+
text = contraction_to_expansion(text)
|
|
67
|
+
text = remove_emails(text)
|
|
68
|
+
text = remove_urls(text)
|
|
69
|
+
text = remove_html_tag(text)
|
|
70
|
+
text = remove_special_chars(text)
|
|
71
|
+
text = lemmatize(text)
|
|
72
|
+
text = convert_to_base(text)
|
|
73
|
+
return text
|