nlp-text-preprocessing 0.0.9__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. nlp_text_preprocessing-0.1.1/MANIFEST.in +5 -0
  2. nlp_text_preprocessing-0.1.1/PKG-INFO +270 -0
  3. nlp_text_preprocessing-0.1.1/Readme.md +234 -0
  4. nlp_text_preprocessing-0.1.1/architecture_diagram.png +0 -0
  5. nlp_text_preprocessing-0.1.1/nlp_banner.png +0 -0
  6. nlp_text_preprocessing-0.1.1/nlp_text_preprocessing/__init__.py +73 -0
  7. nlp_text_preprocessing-0.1.1/nlp_text_preprocessing/all_text_preprocess.py +318 -0
  8. nlp_text_preprocessing-0.1.1/nlp_text_preprocessing.egg-info/PKG-INFO +270 -0
  9. {nlp_text_preprocessing-0.0.9 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing.egg-info/SOURCES.txt +6 -1
  10. nlp_text_preprocessing-0.1.1/requirements.txt +14 -0
  11. {nlp_text_preprocessing-0.0.9 → nlp_text_preprocessing-0.1.1}/setup.py +2 -2
  12. nlp_text_preprocessing-0.1.1/tests/test_all_text_preprocess.py +139 -0
  13. nlp_text_preprocessing-0.0.9/MANIFEST.in +0 -1
  14. nlp_text_preprocessing-0.0.9/PKG-INFO +0 -252
  15. nlp_text_preprocessing-0.0.9/nlp_text_preprocessing/__init__.py +0 -31
  16. nlp_text_preprocessing-0.0.9/nlp_text_preprocessing/all_text_preprocess.py +0 -190
  17. nlp_text_preprocessing-0.0.9/nlp_text_preprocessing.egg-info/PKG-INFO +0 -252
  18. {nlp_text_preprocessing-0.0.9 → nlp_text_preprocessing-0.1.1}/LICENSE +0 -0
  19. {nlp_text_preprocessing-0.0.9 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing/data/contractions.json +0 -0
  20. {nlp_text_preprocessing-0.0.9 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
  21. {nlp_text_preprocessing-0.0.9 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing.egg-info/requires.txt +0 -0
  22. {nlp_text_preprocessing-0.0.9 → nlp_text_preprocessing-0.1.1}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
  23. {nlp_text_preprocessing-0.0.9 → nlp_text_preprocessing-0.1.1}/setup.cfg +0 -0
@@ -0,0 +1,5 @@
1
+ include Readme.md
2
+ include requirements.txt
3
+ include nlp_banner.png
4
+ include architecture_diagram.png
5
+ include nlp_text_preprocessing/data/contractions.json
@@ -0,0 +1,270 @@
1
+ Metadata-Version: 2.4
2
+ Name: nlp_text_preprocessing
3
+ Version: 0.1.1
4
+ Summary: This is a Text Processing Package For NLP
5
+ Author: Uditya Narayan Tiwari
6
+ Author-email: tiwarimerit@gmail.com
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: License :: OSI Approved :: MIT License
9
+ Classifier: Operating System :: OS Independent
10
+ Requires-Python: >=3.9
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: spacy
14
+ Requires-Dist: textblob
15
+ Requires-Dist: beautifulsoup4
16
+ Requires-Dist: nltk
17
+ Requires-Dist: openpyxl
18
+ Requires-Dist: SpeechRecognition==3.10.4
19
+ Requires-Dist: pyaudio==0.2.14
20
+ Requires-Dist: PrettyTable
21
+ Requires-Dist: scikit-learn
22
+ Requires-Dist: wordcloud
23
+ Requires-Dist: lxml
24
+ Requires-Dist: pandas
25
+ Requires-Dist: numpy
26
+ Requires-Dist: matplotlib
27
+ Dynamic: author
28
+ Dynamic: author-email
29
+ Dynamic: classifier
30
+ Dynamic: description
31
+ Dynamic: description-content-type
32
+ Dynamic: license-file
33
+ Dynamic: requires-dist
34
+ Dynamic: requires-python
35
+ Dynamic: summary
36
+
37
+ # NLP Text Preprocessing Utility Package
38
+
39
+ <p align="center">
40
+ <img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/nlp_banner.png" alt="NLP Text Preprocessing Banner" width="100%" />
41
+ </p>
42
+
43
+ <p align="center">
44
+ <a href="https://pypi.org/project/nlp-text-preprocessing/">
45
+ <img src="https://img.shields.io/pypi/v/nlp-text-preprocessing" alt="PyPI Version">
46
+ </a>
47
+ <a href="https://pypi.org/project/nlp-text-preprocessing/">
48
+ <img src="https://img.shields.io/pypi/pyversions/nlp-text-preprocessing" alt="Python Version">
49
+ </a>
50
+ <a href="https://pepy.tech/projects/nlp-text-preprocessing">
51
+ <img src="https://static.pepy.tech/badge/nlp-text-preprocessing" alt="Total Downloads">
52
+ </a>
53
+ <a href="https://pepy.tech/projects/nlp-text-preprocessing">
54
+ <img src="https://static.pepy.tech/badge/nlp-text-preprocessing/month" alt="Monthly Downloads">
55
+ </a>
56
+ <a href="https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/blob/main/LICENSE">
57
+ <img src="https://img.shields.io/github/license/udityamerit/Text-Processing-Package-For-Natural-Language-Processing" alt="License">
58
+ </a>
59
+ </p>
60
+
61
+ ---
62
+
63
+ ## 📌 Overview
64
+
65
+ **`nlp_text_preprocessing`** is a production-grade, fast, and modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
66
+
67
+ Maintained by **[Uditya Narayan Tiwari](https://youtube.com/kgptalkie)**.
68
+
69
+ ---
70
+
71
+ ## 🏗 Architecture Diagram
72
+
73
+ The package is built with a decoupled, high-performance architecture integrating **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, and **WordCloud** with safe null/NaN defensive wrappers for seamless Pandas DataFrame processing.
74
+
75
+ <p align="center">
76
+ <img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/architecture_diagram.png" alt="NLP Text Preprocessing Architecture Diagram" width="100%" />
77
+ </p>
78
+
79
+ ### Pipeline Architecture Workflow
80
+
81
+ ```mermaid
82
+ flowchart TD
83
+ Raw[Raw Input Text / Pandas Series] --> Guard[_clean_input Guard: None/NaN Safety]
84
+
85
+ Guard --> Cleaning[Cleaning & Normalization Module]
86
+ Guard --> Features[Feature Extraction Engine]
87
+ Guard --> Linguistic[Linguistic & NLP Processing]
88
+ Guard --> Viz[Visualization Suite]
89
+
90
+ Cleaning --> Lower[to_lower_case]
91
+ Cleaning --> Expand[contraction_to_expansion]
92
+ Cleaning --> Email[remove_emails / count_emails]
93
+ Cleaning --> URL[remove_urls / count_urls]
94
+ Cleaning --> HTML[remove_html_tag]
95
+ Cleaning --> Regex[remove_special_chars / remove_mentions / remove_rt]
96
+ Cleaning --> Stops[remove_stop_words - Case Insensitive]
97
+
98
+ Features --> Stats[word_count / char_count / avg_word_len]
99
+ Features --> Counts[hashtags_count / mentions_count / numerics_count / upper_case_count]
100
+ Features --> Summary[extract_features - Full Feature Dict]
101
+
102
+ Linguistic --> SpaCy[spaCy Lemmatizer / convert_to_base]
103
+ Linguistic --> NLTKCorpus[Auto-cached NLTK Corpora Check]
104
+ Linguistic --> Sentiment[TextBlob Sentiment - NaiveBayes Cached]
105
+ Linguistic --> Grams[n_grams / get_noun_phrase / singularize / pluralize]
106
+
107
+ Viz --> Cloud[get_wordcloud - File Save & GUI options]
108
+
109
+ Cleaning --> Pipeline[clean_text - End-to-End Pipeline]
110
+ Features --> Output[ML Ready Feature Matrix / Clean Text]
111
+ Linguistic --> Output
112
+ Pipeline --> Output
113
+ ```
114
+
115
+ ---
116
+
117
+ ## 🚀 Quick Installation
118
+
119
+ ### From PyPI
120
+ ```bash
121
+ pip install nlp_text_preprocessing
122
+ ```
123
+
124
+ ### From GitHub (Latest Development Version)
125
+ ```bash
126
+ pip install git+https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing.git --upgrade --force-reinstall
127
+ ```
128
+
129
+ ### Automatic Resource Setup
130
+ Download required NLTK corpora (`stopwords`, `movie_reviews`, `brown`, etc.) and spaCy language models (`en_core_web_sm`):
131
+ ```python
132
+ import nlp_text_preprocessing as tp
133
+
134
+ # Downloads all required NLTK corpora automatically
135
+ tp.download_nltk_packages()
136
+ ```
137
+
138
+ ---
139
+
140
+ ## 📚 Complete API Reference & Function List
141
+
142
+ Below is the complete list of all **34 functions** available in the `nlp_text_preprocessing` package:
143
+
144
+ ### 1. General Feature Extraction Functions
145
+
146
+ | Function Name | Parameters | Return Type | Description |
147
+ | :--- | :--- | :--- | :--- |
148
+ | `word_count(x)` | `x: str` | `int` | Returns total number of whitespace-separated words. |
149
+ | `char_count(x)` | `x: str` | `int` | Returns total characters excluding whitespace. |
150
+ | `avg_word_len(x)` | `x: str` | `float` | Returns average word length (safely returns `0.0` on empty input). |
151
+ | `stop_words_count(x)` | `x: str` | `int` | Returns count of stop words (case-insensitive check). |
152
+ | `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtags (`#tag`). |
153
+ | `mentions_count(x)` | `x: str` | `int` | Returns count of user mentions (`@user`). |
154
+ | `numerics_count(x)` | `x: str` | `int` | Returns count of standalone numeric tokens. |
155
+ | `upper_case_count(x)` | `x: str` | `int` | Returns count of uppercase words. |
156
+ | `extract_features(x)` | `x: str` | `dict` | Returns a dictionary containing all extracted text features at once. |
157
+
158
+ ---
159
+
160
+ ### 2. Text Cleaning & Normalization Functions
161
+
162
+ | Function Name | Parameters | Return Type | Description |
163
+ | :--- | :--- | :--- | :--- |
164
+ | `to_lower_case(x)` | `x: str` | `str` | Converts input text to lowercase. |
165
+ | `contraction_to_expansion(x)` | `x: str` | `str` | Expands abbreviations and contractions using contractions lookup table. |
166
+ | `remove_emails(x)` | `x: str` | `str` | Strips email addresses from text. |
167
+ | `count_emails(x)` | `x: str` | `int` | Returns count of email addresses found in text. |
168
+ | `remove_urls(x)` | `x: str` | `str` | Strips HTTP/HTTPS URLs and `www` links from text. |
169
+ | `count_urls(x)` | `x: str` | `int` | Returns count of URLs found in text. |
170
+ | `remove_rt(x)` | `x: str` | `str` | Removes retweet tags (`RT @user`). |
171
+ | `count_rt(x)` | `x: str` | `int` | Returns count of retweet markers in text. |
172
+ | `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup parser. |
173
+ | `remove_accented_chars(x)` | `x: str` | `str` | Normalizes and converts accented characters (NFKD -> ASCII). |
174
+ | `remove_mentions(x)` | `x: str` | `str` | Strips user handles (`@username`). |
175
+ | `remove_special_chars(x)` | `x: str` | `str` | Removes punctuation and non-alphanumeric special symbols. |
176
+ | `remove_repeated_chars(x)` | `x: str` | `str` | Truncates character repetitions beyond 2 consecutive occurrences. |
177
+ | `remove_stop_words(x)` | `x: str` | `str` | Removes stop words (case-insensitive, preserving capitalized text structure). |
178
+ | `clean_text(text)` | `text: str` | `str` | Executes full end-to-end cleaning pipeline (lowercasing, contractions, emails, URLs, HTML, special chars, lemmatization). |
179
+
180
+ ---
181
+
182
+ ### 3. Linguistic & Morphological Processing Functions
183
+
184
+ | Function Name | Parameters | Return Type | Description |
185
+ | :--- | :--- | :--- | :--- |
186
+ | `convert_to_base(x)` | `x: str` | `str` | Lemmatizes nouns and verbs via spaCy while keeping other POS tags intact. |
187
+ | `lemmatize(x)` | `x: str` | `str` | Performs full lemmatization across all tokens via spaCy (`en_core_web_sm`). |
188
+ | `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob dictionary model. |
189
+ | `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
190
+ | `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
191
+ | `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
192
+ | `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
193
+ | `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using module-cached NaiveBayesAnalyzer. |
194
+
195
+ ---
196
+
197
+ ### 4. Visualization Functions
198
+
199
+ | Function Name | Parameters | Return Type | Description |
200
+ | :--- | :--- | :--- | :--- |
201
+ | `get_wordcloud(x, save_path=None, show=True)` | `x: str, save_path: str, show: bool` | `WordCloud` | Generates a WordCloud image object, with optional file saving and GUI display options. |
202
+
203
+ ---
204
+
205
+ ## 💻 Usage Code Examples
206
+
207
+ ### 1. Cleaning Raw Text
208
+ ```python
209
+ import nlp_text_preprocessing as tp
210
+
211
+ raw_text = "Check out https://example.com! Contact support@company.org or RT @user 'I'm loving #NLP'."
212
+ cleaned = tp.clean_text(raw_text)
213
+
214
+ print(cleaned)
215
+ # Output: check out contact support company org or i am love nlp
216
+ ```
217
+
218
+ ### 2. Processing Pandas DataFrames
219
+ All functions safely handle `None`, `NaN`, and missing values:
220
+ ```python
221
+ import pandas as pd
222
+ import nlp_text_preprocessing as tp
223
+
224
+ df = pd.DataFrame({
225
+ 'text': [
226
+ "I'm loving this NLP package! #awesome",
227
+ "Contact me at info@test.com or visit https://test.com",
228
+ None
229
+ ]
230
+ })
231
+
232
+ # Apply full cleaning pipeline
233
+ df['clean_text'] = df['text'].apply(tp.clean_text)
234
+
235
+ # Extract word counts
236
+ df['word_count'] = df['text'].apply(tp.word_count)
237
+
238
+ # Extract full feature dictionary as DataFrame columns
239
+ features_df = df['text'].apply(tp.extract_features).apply(pd.Series)
240
+ print(features_df)
241
+ ```
242
+
243
+ ### 3. WordCloud Generation & Saving
244
+ ```python
245
+ import nlp_text_preprocessing as tp
246
+
247
+ text = "python natural language processing machine learning text mining data science spaCy nltk textblob"
248
+
249
+ # Save directly to file without requiring GUI display window
250
+ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
251
+ ```
252
+
253
+ ---
254
+
255
+ ## 🛠 Setup & Requirements
256
+
257
+ - `spacy >= 3.0`
258
+ - `textblob`
259
+ - `beautifulsoup4`
260
+ - `nltk`
261
+ - `wordcloud`
262
+ - `pandas`
263
+ - `numpy`
264
+ - `matplotlib`
265
+
266
+ ---
267
+
268
+ ## 📄 License
269
+
270
+ Distributed under the MIT License. See [`LICENSE`](LICENSE) for more information.
@@ -0,0 +1,234 @@
1
+ # NLP Text Preprocessing Utility Package
2
+
3
+ <p align="center">
4
+ <img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/nlp_banner.png" alt="NLP Text Preprocessing Banner" width="100%" />
5
+ </p>
6
+
7
+ <p align="center">
8
+ <a href="https://pypi.org/project/nlp-text-preprocessing/">
9
+ <img src="https://img.shields.io/pypi/v/nlp-text-preprocessing" alt="PyPI Version">
10
+ </a>
11
+ <a href="https://pypi.org/project/nlp-text-preprocessing/">
12
+ <img src="https://img.shields.io/pypi/pyversions/nlp-text-preprocessing" alt="Python Version">
13
+ </a>
14
+ <a href="https://pepy.tech/projects/nlp-text-preprocessing">
15
+ <img src="https://static.pepy.tech/badge/nlp-text-preprocessing" alt="Total Downloads">
16
+ </a>
17
+ <a href="https://pepy.tech/projects/nlp-text-preprocessing">
18
+ <img src="https://static.pepy.tech/badge/nlp-text-preprocessing/month" alt="Monthly Downloads">
19
+ </a>
20
+ <a href="https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/blob/main/LICENSE">
21
+ <img src="https://img.shields.io/github/license/udityamerit/Text-Processing-Package-For-Natural-Language-Processing" alt="License">
22
+ </a>
23
+ </p>
24
+
25
+ ---
26
+
27
+ ## 📌 Overview
28
+
29
+ **`nlp_text_preprocessing`** is a production-grade, fast, and modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
30
+
31
+ Maintained by **[Uditya Narayan Tiwari](https://youtube.com/kgptalkie)**.
32
+
33
+ ---
34
+
35
+ ## 🏗 Architecture Diagram
36
+
37
+ The package is built with a decoupled, high-performance architecture integrating **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, and **WordCloud** with safe null/NaN defensive wrappers for seamless Pandas DataFrame processing.
38
+
39
+ <p align="center">
40
+ <img src="https://raw.githubusercontent.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/main/architecture_diagram.png" alt="NLP Text Preprocessing Architecture Diagram" width="100%" />
41
+ </p>
42
+
43
+ ### Pipeline Architecture Workflow
44
+
45
+ ```mermaid
46
+ flowchart TD
47
+ Raw[Raw Input Text / Pandas Series] --> Guard[_clean_input Guard: None/NaN Safety]
48
+
49
+ Guard --> Cleaning[Cleaning & Normalization Module]
50
+ Guard --> Features[Feature Extraction Engine]
51
+ Guard --> Linguistic[Linguistic & NLP Processing]
52
+ Guard --> Viz[Visualization Suite]
53
+
54
+ Cleaning --> Lower[to_lower_case]
55
+ Cleaning --> Expand[contraction_to_expansion]
56
+ Cleaning --> Email[remove_emails / count_emails]
57
+ Cleaning --> URL[remove_urls / count_urls]
58
+ Cleaning --> HTML[remove_html_tag]
59
+ Cleaning --> Regex[remove_special_chars / remove_mentions / remove_rt]
60
+ Cleaning --> Stops[remove_stop_words - Case Insensitive]
61
+
62
+ Features --> Stats[word_count / char_count / avg_word_len]
63
+ Features --> Counts[hashtags_count / mentions_count / numerics_count / upper_case_count]
64
+ Features --> Summary[extract_features - Full Feature Dict]
65
+
66
+ Linguistic --> SpaCy[spaCy Lemmatizer / convert_to_base]
67
+ Linguistic --> NLTKCorpus[Auto-cached NLTK Corpora Check]
68
+ Linguistic --> Sentiment[TextBlob Sentiment - NaiveBayes Cached]
69
+ Linguistic --> Grams[n_grams / get_noun_phrase / singularize / pluralize]
70
+
71
+ Viz --> Cloud[get_wordcloud - File Save & GUI options]
72
+
73
+ Cleaning --> Pipeline[clean_text - End-to-End Pipeline]
74
+ Features --> Output[ML Ready Feature Matrix / Clean Text]
75
+ Linguistic --> Output
76
+ Pipeline --> Output
77
+ ```
78
+
79
+ ---
80
+
81
+ ## 🚀 Quick Installation
82
+
83
+ ### From PyPI
84
+ ```bash
85
+ pip install nlp_text_preprocessing
86
+ ```
87
+
88
+ ### From GitHub (Latest Development Version)
89
+ ```bash
90
+ pip install git+https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing.git --upgrade --force-reinstall
91
+ ```
92
+
93
+ ### Automatic Resource Setup
94
+ Download required NLTK corpora (`stopwords`, `movie_reviews`, `brown`, etc.) and spaCy language models (`en_core_web_sm`):
95
+ ```python
96
+ import nlp_text_preprocessing as tp
97
+
98
+ # Downloads all required NLTK corpora automatically
99
+ tp.download_nltk_packages()
100
+ ```
101
+
102
+ ---
103
+
104
+ ## 📚 Complete API Reference & Function List
105
+
106
+ Below is the complete list of all **34 functions** available in the `nlp_text_preprocessing` package:
107
+
108
+ ### 1. General Feature Extraction Functions
109
+
110
+ | Function Name | Parameters | Return Type | Description |
111
+ | :--- | :--- | :--- | :--- |
112
+ | `word_count(x)` | `x: str` | `int` | Returns total number of whitespace-separated words. |
113
+ | `char_count(x)` | `x: str` | `int` | Returns total characters excluding whitespace. |
114
+ | `avg_word_len(x)` | `x: str` | `float` | Returns average word length (safely returns `0.0` on empty input). |
115
+ | `stop_words_count(x)` | `x: str` | `int` | Returns count of stop words (case-insensitive check). |
116
+ | `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtags (`#tag`). |
117
+ | `mentions_count(x)` | `x: str` | `int` | Returns count of user mentions (`@user`). |
118
+ | `numerics_count(x)` | `x: str` | `int` | Returns count of standalone numeric tokens. |
119
+ | `upper_case_count(x)` | `x: str` | `int` | Returns count of uppercase words. |
120
+ | `extract_features(x)` | `x: str` | `dict` | Returns a dictionary containing all extracted text features at once. |
121
+
122
+ ---
123
+
124
+ ### 2. Text Cleaning & Normalization Functions
125
+
126
+ | Function Name | Parameters | Return Type | Description |
127
+ | :--- | :--- | :--- | :--- |
128
+ | `to_lower_case(x)` | `x: str` | `str` | Converts input text to lowercase. |
129
+ | `contraction_to_expansion(x)` | `x: str` | `str` | Expands abbreviations and contractions using contractions lookup table. |
130
+ | `remove_emails(x)` | `x: str` | `str` | Strips email addresses from text. |
131
+ | `count_emails(x)` | `x: str` | `int` | Returns count of email addresses found in text. |
132
+ | `remove_urls(x)` | `x: str` | `str` | Strips HTTP/HTTPS URLs and `www` links from text. |
133
+ | `count_urls(x)` | `x: str` | `int` | Returns count of URLs found in text. |
134
+ | `remove_rt(x)` | `x: str` | `str` | Removes retweet tags (`RT @user`). |
135
+ | `count_rt(x)` | `x: str` | `int` | Returns count of retweet markers in text. |
136
+ | `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup parser. |
137
+ | `remove_accented_chars(x)` | `x: str` | `str` | Normalizes and converts accented characters (NFKD -> ASCII). |
138
+ | `remove_mentions(x)` | `x: str` | `str` | Strips user handles (`@username`). |
139
+ | `remove_special_chars(x)` | `x: str` | `str` | Removes punctuation and non-alphanumeric special symbols. |
140
+ | `remove_repeated_chars(x)` | `x: str` | `str` | Truncates character repetitions beyond 2 consecutive occurrences. |
141
+ | `remove_stop_words(x)` | `x: str` | `str` | Removes stop words (case-insensitive, preserving capitalized text structure). |
142
+ | `clean_text(text)` | `text: str` | `str` | Executes full end-to-end cleaning pipeline (lowercasing, contractions, emails, URLs, HTML, special chars, lemmatization). |
143
+
144
+ ---
145
+
146
+ ### 3. Linguistic & Morphological Processing Functions
147
+
148
+ | Function Name | Parameters | Return Type | Description |
149
+ | :--- | :--- | :--- | :--- |
150
+ | `convert_to_base(x)` | `x: str` | `str` | Lemmatizes nouns and verbs via spaCy while keeping other POS tags intact. |
151
+ | `lemmatize(x)` | `x: str` | `str` | Performs full lemmatization across all tokens via spaCy (`en_core_web_sm`). |
152
+ | `correct_spelling(x)` | `x: str` | `str` | Corrects word spelling using TextBlob dictionary model. |
153
+ | `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
154
+ | `n_grams(x, n=2)` | `x: str, n: int` | `list` | Generates word-level n-grams. |
155
+ | `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
156
+ | `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
157
+ | `sentiment_analysis(x)` | `x: str` | `str` | Returns sentiment classification (`'pos'` / `'neg'`) using module-cached NaiveBayesAnalyzer. |
158
+
159
+ ---
160
+
161
+ ### 4. Visualization Functions
162
+
163
+ | Function Name | Parameters | Return Type | Description |
164
+ | :--- | :--- | :--- | :--- |
165
+ | `get_wordcloud(x, save_path=None, show=True)` | `x: str, save_path: str, show: bool` | `WordCloud` | Generates a WordCloud image object, with optional file saving and GUI display options. |
166
+
167
+ ---
168
+
169
+ ## 💻 Usage Code Examples
170
+
171
+ ### 1. Cleaning Raw Text
172
+ ```python
173
+ import nlp_text_preprocessing as tp
174
+
175
+ raw_text = "Check out https://example.com! Contact support@company.org or RT @user 'I'm loving #NLP'."
176
+ cleaned = tp.clean_text(raw_text)
177
+
178
+ print(cleaned)
179
+ # Output: check out contact support company org or i am love nlp
180
+ ```
181
+
182
+ ### 2. Processing Pandas DataFrames
183
+ All functions safely handle `None`, `NaN`, and missing values:
184
+ ```python
185
+ import pandas as pd
186
+ import nlp_text_preprocessing as tp
187
+
188
+ df = pd.DataFrame({
189
+ 'text': [
190
+ "I'm loving this NLP package! #awesome",
191
+ "Contact me at info@test.com or visit https://test.com",
192
+ None
193
+ ]
194
+ })
195
+
196
+ # Apply full cleaning pipeline
197
+ df['clean_text'] = df['text'].apply(tp.clean_text)
198
+
199
+ # Extract word counts
200
+ df['word_count'] = df['text'].apply(tp.word_count)
201
+
202
+ # Extract full feature dictionary as DataFrame columns
203
+ features_df = df['text'].apply(tp.extract_features).apply(pd.Series)
204
+ print(features_df)
205
+ ```
206
+
207
+ ### 3. WordCloud Generation & Saving
208
+ ```python
209
+ import nlp_text_preprocessing as tp
210
+
211
+ text = "python natural language processing machine learning text mining data science spaCy nltk textblob"
212
+
213
+ # Save directly to file without requiring GUI display window
214
+ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
215
+ ```
216
+
217
+ ---
218
+
219
+ ## 🛠 Setup & Requirements
220
+
221
+ - `spacy >= 3.0`
222
+ - `textblob`
223
+ - `beautifulsoup4`
224
+ - `nltk`
225
+ - `wordcloud`
226
+ - `pandas`
227
+ - `numpy`
228
+ - `matplotlib`
229
+
230
+ ---
231
+
232
+ ## 📄 License
233
+
234
+ Distributed under the MIT License. See [`LICENSE`](LICENSE) for more information.
@@ -0,0 +1,73 @@
1
+ # entry level code and import text preprocess
2
+
3
+ from .all_text_preprocess import *
4
+
5
+ __all__ = [
6
+ 'download_nltk_packages',
7
+ 'word_count',
8
+ 'char_count',
9
+ 'avg_word_len',
10
+ 'stop_words_count',
11
+ 'hashtags_count',
12
+ 'mentions_count',
13
+ 'numerics_count',
14
+ 'upper_case_count',
15
+ 'to_lower_case',
16
+ 'contraction_to_expansion',
17
+ 'remove_emails',
18
+ 'count_emails',
19
+ 'count_urls',
20
+ 'remove_urls',
21
+ 'count_rt',
22
+ 'remove_rt',
23
+ 'remove_html_tag',
24
+ 'remove_accented_chars',
25
+ 'remove_mentions',
26
+ 'remove_special_chars',
27
+ 'remove_repeated_chars',
28
+ 'remove_stop_words',
29
+ 'convert_to_base',
30
+ 'lemmatize',
31
+ 'get_wordcloud',
32
+ 'correct_spelling',
33
+ 'get_noun_phrase',
34
+ 'n_grams',
35
+ 'singularize_words',
36
+ 'pluralize_words',
37
+ 'sentiment_analysis',
38
+ 'extract_features',
39
+ 'clean_text',
40
+ ]
41
+
42
+
43
+ def extract_features(x):
44
+ """
45
+ Extracts a dictionary of text features from text x.
46
+ """
47
+ return {
48
+ 'word_count': word_count(x),
49
+ 'char_count': char_count(x),
50
+ 'avg_word_len': avg_word_len(x),
51
+ 'stop_words_count': stop_words_count(x),
52
+ 'hashtags_count': hashtags_count(x),
53
+ 'mentions_count': mentions_count(x),
54
+ 'numerics_count': numerics_count(x),
55
+ 'upper_case_count': upper_case_count(x),
56
+ 'noun_phrases': get_noun_phrase(x),
57
+ 'sentiment_analysis': sentiment_analysis(x)
58
+ }
59
+
60
+
61
+ def clean_text(text):
62
+ """
63
+ Applies a sequence of cleaning and normalization steps on text.
64
+ """
65
+ text = to_lower_case(text)
66
+ text = contraction_to_expansion(text)
67
+ text = remove_emails(text)
68
+ text = remove_urls(text)
69
+ text = remove_html_tag(text)
70
+ text = remove_special_chars(text)
71
+ text = lemmatize(text)
72
+ text = convert_to_base(text)
73
+ return text