nlp-text-preprocessing 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nlp_text_preprocessing-0.2.1/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.3.0}/PKG-INFO +76 -58
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/Readme.md +37 -29
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/__init__.py +18 -7
- nlp_text_preprocessing-0.3.0/nlp_text_preprocessing/all_text_preprocess.py +1091 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info}/PKG-INFO +76 -58
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +5 -2
- nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info/requires.txt +27 -0
- nlp_text_preprocessing-0.3.0/pyproject.toml +68 -0
- nlp_text_preprocessing-0.3.0/requirements.txt +7 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/tests/test_all_text_preprocess.py +1 -1
- nlp_text_preprocessing-0.3.0/tests/test_cleaning.py +168 -0
- nlp_text_preprocessing-0.3.0/tests/test_infrastructure.py +110 -0
- nlp_text_preprocessing-0.3.0/tests/test_linguistic.py +183 -0
- nlp_text_preprocessing-0.2.1/nlp_text_preprocessing/all_text_preprocess.py +0 -720
- nlp_text_preprocessing-0.2.1/nlp_text_preprocessing.egg-info/requires.txt +0 -14
- nlp_text_preprocessing-0.2.1/requirements.txt +0 -14
- nlp_text_preprocessing-0.2.1/setup.py +0 -34
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/LICENSE +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/MANIFEST.in +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/architecture_diagram.png +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_banner.png +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/setup.cfg +0 -0
|
@@ -1,44 +1,54 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
|
-
Name:
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
Project-URL: Author Website, https://udityanarayantiwari.netlify.app/
|
|
2
|
+
Name: nlp-text-preprocessing
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
|
|
5
|
+
Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
|
|
9
8
|
Project-URL: Source Code, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
|
|
10
9
|
Project-URL: Bug Tracker, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/issues
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
Project-URL: Author Website, https://udityanarayantiwari.netlify.app/
|
|
11
|
+
Keywords: nlp,text-preprocessing,text-cleaning,spacy,nltk,textblob
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Education
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
16
|
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
14
25
|
Requires-Python: >=3.9
|
|
15
26
|
Description-Content-Type: text/markdown
|
|
16
27
|
License-File: LICENSE
|
|
17
28
|
Requires-Dist: spacy
|
|
18
29
|
Requires-Dist: textblob
|
|
19
|
-
Requires-Dist: beautifulsoup4
|
|
20
30
|
Requires-Dist: nltk
|
|
21
|
-
Requires-Dist:
|
|
22
|
-
Requires-Dist: SpeechRecognition==3.10.4
|
|
23
|
-
Requires-Dist: pyaudio==0.2.14
|
|
24
|
-
Requires-Dist: PrettyTable
|
|
25
|
-
Requires-Dist: scikit-learn
|
|
26
|
-
Requires-Dist: wordcloud
|
|
31
|
+
Requires-Dist: beautifulsoup4
|
|
27
32
|
Requires-Dist: lxml
|
|
28
|
-
Requires-Dist:
|
|
29
|
-
Requires-Dist:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
Requires-Dist: emoji>=2.0
|
|
34
|
+
Requires-Dist: langdetect
|
|
35
|
+
Provides-Extra: pandas
|
|
36
|
+
Requires-Dist: pandas; extra == "pandas"
|
|
37
|
+
Provides-Extra: viz
|
|
38
|
+
Requires-Dist: wordcloud; extra == "viz"
|
|
39
|
+
Requires-Dist: matplotlib; extra == "viz"
|
|
40
|
+
Provides-Extra: all
|
|
41
|
+
Requires-Dist: pandas; extra == "all"
|
|
42
|
+
Requires-Dist: wordcloud; extra == "all"
|
|
43
|
+
Requires-Dist: matplotlib; extra == "all"
|
|
44
|
+
Provides-Extra: dev
|
|
45
|
+
Requires-Dist: pytest; extra == "dev"
|
|
46
|
+
Requires-Dist: build; extra == "dev"
|
|
47
|
+
Requires-Dist: twine; extra == "dev"
|
|
48
|
+
Requires-Dist: pandas; extra == "dev"
|
|
49
|
+
Requires-Dist: wordcloud; extra == "dev"
|
|
50
|
+
Requires-Dist: matplotlib; extra == "dev"
|
|
37
51
|
Dynamic: license-file
|
|
38
|
-
Dynamic: project-url
|
|
39
|
-
Dynamic: requires-dist
|
|
40
|
-
Dynamic: requires-python
|
|
41
|
-
Dynamic: summary
|
|
42
52
|
|
|
43
53
|
# NLP Text Preprocessing Utility Package
|
|
44
54
|
|
|
@@ -92,19 +102,19 @@ Dynamic: summary
|
|
|
92
102
|
|
|
93
103
|
## Package Overview
|
|
94
104
|
|
|
95
|
-
**`nlp_text_preprocessing`** is a
|
|
105
|
+
**`nlp_text_preprocessing`** is a modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
|
|
96
106
|
|
|
97
107
|
Maintained by **[Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)**.
|
|
98
108
|
|
|
99
|
-
It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with
|
|
109
|
+
It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with defensive guards so `None`, `NaN` and missing values are treated as empty text, which makes it convenient to run over Pandas DataFrame columns and raw text datasets.
|
|
100
110
|
|
|
101
111
|
---
|
|
102
112
|
|
|
103
113
|
## Key Features
|
|
104
114
|
|
|
105
|
-
- **
|
|
115
|
+
- **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
|
|
106
116
|
- **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
|
|
107
|
-
- **Lazy
|
|
117
|
+
- **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
|
|
108
118
|
- **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
|
|
109
119
|
- **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
|
|
110
120
|
- **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
|
|
@@ -128,20 +138,27 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
|
|
|
128
138
|
### 1. Installation
|
|
129
139
|
|
|
130
140
|
```bash
|
|
131
|
-
pip install nlp-text-preprocessing
|
|
141
|
+
pip install nlp-text-preprocessing # core
|
|
142
|
+
pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
|
|
143
|
+
pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
|
|
144
|
+
pip install "nlp-text-preprocessing[all]" # everything optional
|
|
132
145
|
```
|
|
133
146
|
|
|
134
147
|
### 2. Downloading Required Corpora and Models
|
|
135
148
|
|
|
136
|
-
|
|
149
|
+
Nothing is downloaded behind your back. Run the one-time setup, which fetches the NLTK data (`punkt_tab`, `wordnet`, `omw-1.4`) and the spaCy model `en_core_web_sm`:
|
|
137
150
|
|
|
138
151
|
```python
|
|
139
152
|
import nlp_text_preprocessing as tp
|
|
140
153
|
|
|
141
|
-
#
|
|
142
|
-
tp.
|
|
154
|
+
tp.download_nltk_packages() # one-time setup
|
|
155
|
+
tp.check_resources(verbose=True) # see what is installed / missing
|
|
143
156
|
```
|
|
144
157
|
|
|
158
|
+
`sentiment_analysis(x, method="naive_bayes")` additionally needs the `movie_reviews` corpus:
|
|
159
|
+
`tp.download_nltk_packages(include_sentiment_corpus=True)`.
|
|
160
|
+
Set the environment variable `NLP_TEXT_PREPROCESSING_AUTO_DOWNLOAD=1` if you want the spaCy model to be downloaded automatically on first use.
|
|
161
|
+
|
|
145
162
|
---
|
|
146
163
|
|
|
147
164
|
## Pipeline Automation Engine
|
|
@@ -212,29 +229,29 @@ Below is the complete reference guide for all functions available in the library
|
|
|
212
229
|
| `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
|
|
213
230
|
| `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
|
|
214
231
|
| `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
|
|
215
|
-
| `remove_urls(x)` | `x: str` | `str` | Removes
|
|
232
|
+
| `remove_urls(x)` | `x: str` | `str` | Removes `http(s)://`, `ftp://` and `www.` URLs (trailing punctuation is kept). |
|
|
216
233
|
| `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
|
|
217
|
-
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user
|
|
234
|
+
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user:`). |
|
|
218
235
|
| `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
|
|
219
236
|
| `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
|
|
220
|
-
| `remove_accented_chars(x)` | `x: str` | `str` |
|
|
237
|
+
| `remove_accented_chars(x)` | `x: str` | `str` | Strips accents (`café` -> `cafe`); non-Latin scripts are kept. |
|
|
221
238
|
| `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
|
|
222
|
-
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and
|
|
223
|
-
| `remove_repeated_chars(x)` | `x: str` | `str` |
|
|
224
|
-
| `remove_stop_words(x)` | `x: str` | `str` |
|
|
225
|
-
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji
|
|
239
|
+
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and symbols (`well-known` -> `well known`). |
|
|
240
|
+
| `remove_repeated_chars(x)` | `x: str` | `str` | Shortens elongated words (`soooo` -> `soo`); digits and punctuation are untouched. |
|
|
241
|
+
| `remove_stop_words(x, keep_negations=True)` | `x: str` | `str` | Removes stop words (case-insensitive); negations like `not` are kept by default. |
|
|
242
|
+
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji (flags, skin tones and ZWJ sequences included). |
|
|
226
243
|
| `extract_emojis(x)` | `x: str` | `list[str]` | Returns list of emojis found in text. |
|
|
227
244
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
228
|
-
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers
|
|
245
|
+
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
229
246
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
230
|
-
| `mask_pii(x)` | `x: str` | `str` | Redacts emails,
|
|
247
|
+
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
231
248
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
232
|
-
| `expand_hashtags(x)` | `x: str` | `str` | Splits
|
|
249
|
+
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
233
250
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
|
234
|
-
| `remove_numbers(x)` | `x: str` | `str` |
|
|
235
|
-
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips
|
|
251
|
+
| `remove_numbers(x)` | `x: str` | `str` | Removes standalone numbers; digits inside words (`3rd`, `H2O`) stay. |
|
|
252
|
+
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
|
|
236
253
|
| `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
|
|
237
|
-
| `clean_text(text)` | `text: str` | `str` |
|
|
254
|
+
| `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
|
|
238
255
|
|
|
239
256
|
---
|
|
240
257
|
|
|
@@ -247,15 +264,15 @@ Below is the complete reference guide for all functions available in the library
|
|
|
247
264
|
| `extract_entities(x)` | `x: str` | `list[tuple]` | Extracts named entities `(text, label)` using spaCy. |
|
|
248
265
|
| `extract_pos_tags(x)` | `x: str` | `list[tuple]` | Extracts full POS tag breakdown `(token, pos, tag)` using spaCy. |
|
|
249
266
|
| `get_dependency_tree(x)`| `x: str` | `list[tuple]` | Extracts dependency parse tree `(token, dep, head)` using spaCy. |
|
|
250
|
-
| `detect_language(x)` | `x: str` | `str` |
|
|
267
|
+
| `detect_language(x)` | `x: str` | `str` | Language code (`'en'`, `'fr'`, `'hi'`, ...) via `langdetect`; `'unknown'` if undetectable. |
|
|
251
268
|
| `keyword_extraction(x, top_n)`| `x: str, top_n: int`| `list[str]`| Extracts top N keywords by term frequency. |
|
|
252
269
|
| `text_similarity(x1, x2)` | `x1, x2` | `float` | Calculates token Jaccard similarity between two texts (0.0 to 1.0). |
|
|
253
|
-
| `get_synonyms(word)` | `word: str` | `list[str]` |
|
|
254
|
-
| `get_antonyms(word)` | `word: str` | `list[str]` |
|
|
255
|
-
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` |
|
|
256
|
-
| `correct_spelling(x)` | `x: str` | `str` | Corrects
|
|
270
|
+
| `get_synonyms(word)` | `word: str` | `list[str]` | Sorted WordNet synonyms. |
|
|
271
|
+
| `get_antonyms(word)` | `word: str` | `list[str]` | Sorted WordNet antonyms. |
|
|
272
|
+
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | spaCy noun chunks with exact character `start`/`end` offsets. |
|
|
273
|
+
| `correct_spelling(x)` | `x: str` | `str` | Corrects misspelled words (TextBlob); punctuation, case, URLs and acronyms are preserved. |
|
|
257
274
|
| `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
|
|
258
|
-
| `n_grams(x, n=2)` | `x: str, n: int` | `list` |
|
|
275
|
+
| `n_grams(x, n=2)` | `x: str, n: int` | `list[tuple]` | Word-level n-grams as tuples. |
|
|
259
276
|
| `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
|
|
260
277
|
| `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
|
|
261
278
|
|
|
@@ -269,7 +286,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
269
286
|
| `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
|
|
270
287
|
| `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
|
|
271
288
|
| `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
|
|
272
|
-
| `sentiment_analysis(x)` | `x: str` | `str` |
|
|
289
|
+
| `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
|
|
273
290
|
| `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
|
|
274
291
|
| `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
|
|
275
292
|
|
|
@@ -363,10 +380,11 @@ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
|
363
380
|
|
|
364
381
|
## Troubleshooting and Resource Setup
|
|
365
382
|
|
|
366
|
-
If
|
|
383
|
+
If a function reports a missing NLTK corpus or spaCy model, check what is missing and install it:
|
|
367
384
|
|
|
368
385
|
```python
|
|
369
386
|
import nlp_text_preprocessing as tp
|
|
387
|
+
tp.check_resources(verbose=True)
|
|
370
388
|
tp.download_nltk_packages()
|
|
371
389
|
```
|
|
372
390
|
|
|
@@ -50,19 +50,19 @@
|
|
|
50
50
|
|
|
51
51
|
## Package Overview
|
|
52
52
|
|
|
53
|
-
**`nlp_text_preprocessing`** is a
|
|
53
|
+
**`nlp_text_preprocessing`** is a modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
|
|
54
54
|
|
|
55
55
|
Maintained by **[Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)**.
|
|
56
56
|
|
|
57
|
-
It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with
|
|
57
|
+
It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with defensive guards so `None`, `NaN` and missing values are treated as empty text, which makes it convenient to run over Pandas DataFrame columns and raw text datasets.
|
|
58
58
|
|
|
59
59
|
---
|
|
60
60
|
|
|
61
61
|
## Key Features
|
|
62
62
|
|
|
63
|
-
- **
|
|
63
|
+
- **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
|
|
64
64
|
- **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
|
|
65
|
-
- **Lazy
|
|
65
|
+
- **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
|
|
66
66
|
- **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
|
|
67
67
|
- **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
|
|
68
68
|
- **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
|
|
@@ -86,20 +86,27 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
|
|
|
86
86
|
### 1. Installation
|
|
87
87
|
|
|
88
88
|
```bash
|
|
89
|
-
pip install nlp-text-preprocessing
|
|
89
|
+
pip install nlp-text-preprocessing # core
|
|
90
|
+
pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
|
|
91
|
+
pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
|
|
92
|
+
pip install "nlp-text-preprocessing[all]" # everything optional
|
|
90
93
|
```
|
|
91
94
|
|
|
92
95
|
### 2. Downloading Required Corpora and Models
|
|
93
96
|
|
|
94
|
-
|
|
97
|
+
Nothing is downloaded behind your back. Run the one-time setup, which fetches the NLTK data (`punkt_tab`, `wordnet`, `omw-1.4`) and the spaCy model `en_core_web_sm`:
|
|
95
98
|
|
|
96
99
|
```python
|
|
97
100
|
import nlp_text_preprocessing as tp
|
|
98
101
|
|
|
99
|
-
#
|
|
100
|
-
tp.
|
|
102
|
+
tp.download_nltk_packages() # one-time setup
|
|
103
|
+
tp.check_resources(verbose=True) # see what is installed / missing
|
|
101
104
|
```
|
|
102
105
|
|
|
106
|
+
`sentiment_analysis(x, method="naive_bayes")` additionally needs the `movie_reviews` corpus:
|
|
107
|
+
`tp.download_nltk_packages(include_sentiment_corpus=True)`.
|
|
108
|
+
Set the environment variable `NLP_TEXT_PREPROCESSING_AUTO_DOWNLOAD=1` if you want the spaCy model to be downloaded automatically on first use.
|
|
109
|
+
|
|
103
110
|
---
|
|
104
111
|
|
|
105
112
|
## Pipeline Automation Engine
|
|
@@ -170,29 +177,29 @@ Below is the complete reference guide for all functions available in the library
|
|
|
170
177
|
| `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
|
|
171
178
|
| `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
|
|
172
179
|
| `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
|
|
173
|
-
| `remove_urls(x)` | `x: str` | `str` | Removes
|
|
180
|
+
| `remove_urls(x)` | `x: str` | `str` | Removes `http(s)://`, `ftp://` and `www.` URLs (trailing punctuation is kept). |
|
|
174
181
|
| `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
|
|
175
|
-
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user
|
|
182
|
+
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user:`). |
|
|
176
183
|
| `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
|
|
177
184
|
| `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
|
|
178
|
-
| `remove_accented_chars(x)` | `x: str` | `str` |
|
|
185
|
+
| `remove_accented_chars(x)` | `x: str` | `str` | Strips accents (`café` -> `cafe`); non-Latin scripts are kept. |
|
|
179
186
|
| `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
|
|
180
|
-
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and
|
|
181
|
-
| `remove_repeated_chars(x)` | `x: str` | `str` |
|
|
182
|
-
| `remove_stop_words(x)` | `x: str` | `str` |
|
|
183
|
-
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji
|
|
187
|
+
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and symbols (`well-known` -> `well known`). |
|
|
188
|
+
| `remove_repeated_chars(x)` | `x: str` | `str` | Shortens elongated words (`soooo` -> `soo`); digits and punctuation are untouched. |
|
|
189
|
+
| `remove_stop_words(x, keep_negations=True)` | `x: str` | `str` | Removes stop words (case-insensitive); negations like `not` are kept by default. |
|
|
190
|
+
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji (flags, skin tones and ZWJ sequences included). |
|
|
184
191
|
| `extract_emojis(x)` | `x: str` | `list[str]` | Returns list of emojis found in text. |
|
|
185
192
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
186
|
-
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers
|
|
193
|
+
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
187
194
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
188
|
-
| `mask_pii(x)` | `x: str` | `str` | Redacts emails,
|
|
195
|
+
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
189
196
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
190
|
-
| `expand_hashtags(x)` | `x: str` | `str` | Splits
|
|
197
|
+
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
191
198
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
|
192
|
-
| `remove_numbers(x)` | `x: str` | `str` |
|
|
193
|
-
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips
|
|
199
|
+
| `remove_numbers(x)` | `x: str` | `str` | Removes standalone numbers; digits inside words (`3rd`, `H2O`) stay. |
|
|
200
|
+
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
|
|
194
201
|
| `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
|
|
195
|
-
| `clean_text(text)` | `text: str` | `str` |
|
|
202
|
+
| `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
|
|
196
203
|
|
|
197
204
|
---
|
|
198
205
|
|
|
@@ -205,15 +212,15 @@ Below is the complete reference guide for all functions available in the library
|
|
|
205
212
|
| `extract_entities(x)` | `x: str` | `list[tuple]` | Extracts named entities `(text, label)` using spaCy. |
|
|
206
213
|
| `extract_pos_tags(x)` | `x: str` | `list[tuple]` | Extracts full POS tag breakdown `(token, pos, tag)` using spaCy. |
|
|
207
214
|
| `get_dependency_tree(x)`| `x: str` | `list[tuple]` | Extracts dependency parse tree `(token, dep, head)` using spaCy. |
|
|
208
|
-
| `detect_language(x)` | `x: str` | `str` |
|
|
215
|
+
| `detect_language(x)` | `x: str` | `str` | Language code (`'en'`, `'fr'`, `'hi'`, ...) via `langdetect`; `'unknown'` if undetectable. |
|
|
209
216
|
| `keyword_extraction(x, top_n)`| `x: str, top_n: int`| `list[str]`| Extracts top N keywords by term frequency. |
|
|
210
217
|
| `text_similarity(x1, x2)` | `x1, x2` | `float` | Calculates token Jaccard similarity between two texts (0.0 to 1.0). |
|
|
211
|
-
| `get_synonyms(word)` | `word: str` | `list[str]` |
|
|
212
|
-
| `get_antonyms(word)` | `word: str` | `list[str]` |
|
|
213
|
-
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` |
|
|
214
|
-
| `correct_spelling(x)` | `x: str` | `str` | Corrects
|
|
218
|
+
| `get_synonyms(word)` | `word: str` | `list[str]` | Sorted WordNet synonyms. |
|
|
219
|
+
| `get_antonyms(word)` | `word: str` | `list[str]` | Sorted WordNet antonyms. |
|
|
220
|
+
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | spaCy noun chunks with exact character `start`/`end` offsets. |
|
|
221
|
+
| `correct_spelling(x)` | `x: str` | `str` | Corrects misspelled words (TextBlob); punctuation, case, URLs and acronyms are preserved. |
|
|
215
222
|
| `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
|
|
216
|
-
| `n_grams(x, n=2)` | `x: str, n: int` | `list` |
|
|
223
|
+
| `n_grams(x, n=2)` | `x: str, n: int` | `list[tuple]` | Word-level n-grams as tuples. |
|
|
217
224
|
| `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
|
|
218
225
|
| `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
|
|
219
226
|
|
|
@@ -227,7 +234,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
227
234
|
| `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
|
|
228
235
|
| `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
|
|
229
236
|
| `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
|
|
230
|
-
| `sentiment_analysis(x)` | `x: str` | `str` |
|
|
237
|
+
| `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
|
|
231
238
|
| `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
|
|
232
239
|
| `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
|
|
233
240
|
|
|
@@ -321,10 +328,11 @@ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
|
321
328
|
|
|
322
329
|
## Troubleshooting and Resource Setup
|
|
323
330
|
|
|
324
|
-
If
|
|
331
|
+
If a function reports a missing NLTK corpus or spaCy model, check what is missing and install it:
|
|
325
332
|
|
|
326
333
|
```python
|
|
327
334
|
import nlp_text_preprocessing as tp
|
|
335
|
+
tp.check_resources(verbose=True)
|
|
328
336
|
tp.download_nltk_packages()
|
|
329
337
|
```
|
|
330
338
|
|
{nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.3.0}/nlp_text_preprocessing/__init__.py
RENAMED
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
# entry level code and import text preprocess
|
|
2
2
|
|
|
3
|
+
__version__ = "0.3.0"
|
|
4
|
+
|
|
3
5
|
from .all_text_preprocess import *
|
|
4
6
|
|
|
5
7
|
__all__ = [
|
|
8
|
+
'__version__',
|
|
6
9
|
'download_nltk_packages',
|
|
10
|
+
'check_resources',
|
|
11
|
+
'get_spacy_nlp',
|
|
12
|
+
'get_stop_words',
|
|
7
13
|
'word_count',
|
|
8
14
|
'char_count',
|
|
9
15
|
'avg_word_len',
|
|
@@ -96,16 +102,21 @@ def extract_features(x):
|
|
|
96
102
|
}
|
|
97
103
|
|
|
98
104
|
|
|
99
|
-
def clean_text(text):
|
|
105
|
+
def clean_text(text, remove_stopwords=False):
|
|
100
106
|
"""
|
|
101
|
-
|
|
107
|
+
End-to-end cleaning: HTML, URLs and emails are removed, text is lower-cased,
|
|
108
|
+
contractions are expanded, punctuation is dropped and words are lemmatized.
|
|
109
|
+
|
|
110
|
+
Set remove_stopwords=True to also drop stop words (negations are kept).
|
|
102
111
|
"""
|
|
112
|
+
text = remove_html_tag(text)
|
|
113
|
+
text = remove_urls(text)
|
|
114
|
+
text = remove_emails(text)
|
|
103
115
|
text = to_lower_case(text)
|
|
104
116
|
text = contraction_to_expansion(text)
|
|
105
|
-
text = remove_emails(text)
|
|
106
|
-
text = remove_urls(text)
|
|
107
|
-
text = remove_html_tag(text)
|
|
108
117
|
text = remove_special_chars(text)
|
|
118
|
+
text = remove_extra_whitespace(text)
|
|
109
119
|
text = lemmatize(text)
|
|
110
|
-
|
|
111
|
-
|
|
120
|
+
if remove_stopwords:
|
|
121
|
+
text = remove_stop_words(text)
|
|
122
|
+
return text
|