nlp-text-preprocessing 0.2.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/PKG-INFO +171 -64
- nlp_text_preprocessing-0.2.1/nlp_text_preprocessing.egg-info/PKG-INFO → nlp_text_preprocessing-0.4.0/Readme.md +434 -383
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing/__init__.py +31 -7
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/__main__.py +5 -0
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/all_text_preprocess.py +1475 -0
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/cli.py +150 -0
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/sklearn_compat.py +44 -0
- nlp_text_preprocessing-0.2.1/Readme.md → nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info/PKG-INFO +490 -341
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +12 -2
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info/entry_points.txt +2 -0
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info/requires.txt +32 -0
- nlp_text_preprocessing-0.4.0/pyproject.toml +72 -0
- nlp_text_preprocessing-0.4.0/requirements.txt +7 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/tests/test_all_text_preprocess.py +1 -1
- nlp_text_preprocessing-0.4.0/tests/test_cleaning.py +168 -0
- nlp_text_preprocessing-0.4.0/tests/test_infrastructure.py +110 -0
- nlp_text_preprocessing-0.4.0/tests/test_light_and_cli.py +195 -0
- nlp_text_preprocessing-0.4.0/tests/test_linguistic.py +183 -0
- nlp_text_preprocessing-0.4.0/tests/test_moderation.py +102 -0
- nlp_text_preprocessing-0.4.0/tests/test_pipeline_api.py +100 -0
- nlp_text_preprocessing-0.2.1/nlp_text_preprocessing/all_text_preprocess.py +0 -720
- nlp_text_preprocessing-0.2.1/nlp_text_preprocessing.egg-info/requires.txt +0 -14
- nlp_text_preprocessing-0.2.1/requirements.txt +0 -14
- nlp_text_preprocessing-0.2.1/setup.py +0 -34
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/LICENSE +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/MANIFEST.in +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/architecture_diagram.png +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/nlp_banner.png +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.2.1 → nlp_text_preprocessing-0.4.0}/setup.cfg +0 -0
|
@@ -1,44 +1,58 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
|
-
Name:
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
Project-URL: Author Website, https://udityanarayantiwari.netlify.app/
|
|
2
|
+
Name: nlp-text-preprocessing
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
|
|
5
|
+
Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
|
|
9
8
|
Project-URL: Source Code, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing
|
|
10
9
|
Project-URL: Bug Tracker, https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing/issues
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
Project-URL: Author Website, https://udityanarayantiwari.netlify.app/
|
|
11
|
+
Keywords: nlp,text-preprocessing,text-cleaning,spacy,nltk,textblob
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Education
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
16
|
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
14
25
|
Requires-Python: >=3.9
|
|
15
26
|
Description-Content-Type: text/markdown
|
|
16
27
|
License-File: LICENSE
|
|
17
28
|
Requires-Dist: spacy
|
|
18
29
|
Requires-Dist: textblob
|
|
19
|
-
Requires-Dist: beautifulsoup4
|
|
20
30
|
Requires-Dist: nltk
|
|
21
|
-
Requires-Dist:
|
|
22
|
-
Requires-Dist: SpeechRecognition==3.10.4
|
|
23
|
-
Requires-Dist: pyaudio==0.2.14
|
|
24
|
-
Requires-Dist: PrettyTable
|
|
25
|
-
Requires-Dist: scikit-learn
|
|
26
|
-
Requires-Dist: wordcloud
|
|
31
|
+
Requires-Dist: beautifulsoup4
|
|
27
32
|
Requires-Dist: lxml
|
|
28
|
-
Requires-Dist:
|
|
29
|
-
Requires-Dist:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
Requires-Dist: emoji>=2.0
|
|
34
|
+
Requires-Dist: langdetect
|
|
35
|
+
Provides-Extra: pandas
|
|
36
|
+
Requires-Dist: pandas; extra == "pandas"
|
|
37
|
+
Provides-Extra: viz
|
|
38
|
+
Requires-Dist: wordcloud; extra == "viz"
|
|
39
|
+
Requires-Dist: matplotlib; extra == "viz"
|
|
40
|
+
Provides-Extra: sklearn
|
|
41
|
+
Requires-Dist: scikit-learn; extra == "sklearn"
|
|
42
|
+
Provides-Extra: all
|
|
43
|
+
Requires-Dist: pandas; extra == "all"
|
|
44
|
+
Requires-Dist: wordcloud; extra == "all"
|
|
45
|
+
Requires-Dist: matplotlib; extra == "all"
|
|
46
|
+
Requires-Dist: scikit-learn; extra == "all"
|
|
47
|
+
Provides-Extra: dev
|
|
48
|
+
Requires-Dist: pytest; extra == "dev"
|
|
49
|
+
Requires-Dist: build; extra == "dev"
|
|
50
|
+
Requires-Dist: twine; extra == "dev"
|
|
51
|
+
Requires-Dist: pandas; extra == "dev"
|
|
52
|
+
Requires-Dist: wordcloud; extra == "dev"
|
|
53
|
+
Requires-Dist: matplotlib; extra == "dev"
|
|
54
|
+
Requires-Dist: scikit-learn; extra == "dev"
|
|
37
55
|
Dynamic: license-file
|
|
38
|
-
Dynamic: project-url
|
|
39
|
-
Dynamic: requires-dist
|
|
40
|
-
Dynamic: requires-python
|
|
41
|
-
Dynamic: summary
|
|
42
56
|
|
|
43
57
|
# NLP Text Preprocessing Utility Package
|
|
44
58
|
|
|
@@ -73,6 +87,8 @@ Dynamic: summary
|
|
|
73
87
|
- [Architecture Diagram](#architecture-diagram)
|
|
74
88
|
- [Quickstart and Setup](#quickstart-and-setup)
|
|
75
89
|
- [Pipeline Automation Engine](#pipeline-automation-engine)
|
|
90
|
+
- [Preparing Text for Transformers](#preparing-text-for-transformers-bert-gpt-)
|
|
91
|
+
- [Command Line (`nlp-prep`)](#command-line-nlp-prep)
|
|
76
92
|
- [Complete API Reference](#complete-api-reference)
|
|
77
93
|
- [1. General Feature Extraction](#1-general-feature-extraction)
|
|
78
94
|
- [2. Text Cleaning and Normalization](#2-text-cleaning-and-normalization)
|
|
@@ -92,23 +108,23 @@ Dynamic: summary
|
|
|
92
108
|
|
|
93
109
|
## Package Overview
|
|
94
110
|
|
|
95
|
-
**`nlp_text_preprocessing`** is a
|
|
111
|
+
**`nlp_text_preprocessing`** is a modular Python library designed to streamline text cleaning, feature extraction, linguistic analysis, content moderation, and visualization for Natural Language Processing (NLP) pipelines and Machine Learning workflows.
|
|
96
112
|
|
|
97
113
|
Maintained by **[Uditya Narayan Tiwari](https://udityanarayantiwari.netlify.app/)**.
|
|
98
114
|
|
|
99
|
-
It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with
|
|
115
|
+
It provides over **60 single-purpose functions** and an extensible **Pipeline Engine** with defensive guards so `None`, `NaN` and missing values are treated as empty text, which makes it convenient to run over Pandas DataFrame columns and raw text datasets.
|
|
100
116
|
|
|
101
117
|
---
|
|
102
118
|
|
|
103
119
|
## Key Features
|
|
104
120
|
|
|
105
|
-
- **
|
|
121
|
+
- **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
|
|
106
122
|
- **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
|
|
107
|
-
- **Lazy
|
|
123
|
+
- **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
|
|
108
124
|
- **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
|
|
109
125
|
- **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
|
|
110
126
|
- **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
|
|
111
|
-
- **Safety & Moderation**:
|
|
127
|
+
- **Safety & Moderation (baseline)**: Lexicon-based profanity, toxicity, spam and emotion helpers. They are transparent keyword baselines, not a moderation system: use a trained model for anything high-stakes.
|
|
112
128
|
- **Visualization**: Generate and save custom WordClouds to disk or display interactively.
|
|
113
129
|
|
|
114
130
|
---
|
|
@@ -128,20 +144,28 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
|
|
|
128
144
|
### 1. Installation
|
|
129
145
|
|
|
130
146
|
```bash
|
|
131
|
-
pip install nlp-text-preprocessing
|
|
147
|
+
pip install nlp-text-preprocessing # core
|
|
148
|
+
pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
|
|
149
|
+
pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
|
|
150
|
+
pip install "nlp-text-preprocessing[sklearn]" # + scikit-learn (TextPreprocessor)
|
|
151
|
+
pip install "nlp-text-preprocessing[all]" # everything optional
|
|
132
152
|
```
|
|
133
153
|
|
|
134
154
|
### 2. Downloading Required Corpora and Models
|
|
135
155
|
|
|
136
|
-
|
|
156
|
+
Nothing is downloaded behind your back. Run the one-time setup, which fetches the NLTK data (`punkt_tab`, `wordnet`, `omw-1.4`) and the spaCy model `en_core_web_sm`:
|
|
137
157
|
|
|
138
158
|
```python
|
|
139
159
|
import nlp_text_preprocessing as tp
|
|
140
160
|
|
|
141
|
-
#
|
|
142
|
-
tp.
|
|
161
|
+
tp.download_nltk_packages() # one-time setup
|
|
162
|
+
tp.check_resources(verbose=True) # see what is installed / missing
|
|
143
163
|
```
|
|
144
164
|
|
|
165
|
+
`sentiment_analysis(x, method="naive_bayes")` additionally needs the `movie_reviews` corpus:
|
|
166
|
+
`tp.download_nltk_packages(include_sentiment_corpus=True)`.
|
|
167
|
+
Set the environment variable `NLP_TEXT_PREPROCESSING_AUTO_DOWNLOAD=1` if you want the spaCy model to be downloaded automatically on first use.
|
|
168
|
+
|
|
145
169
|
---
|
|
146
170
|
|
|
147
171
|
## Pipeline Automation Engine
|
|
@@ -175,6 +199,84 @@ clean_series = pipe.run_series(df['text_column'])
|
|
|
175
199
|
report = tp.diff_report(raw_texts, clean_batch)
|
|
176
200
|
```
|
|
177
201
|
|
|
202
|
+
### Steps with arguments, saving and scikit-learn
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
# Pass arguments as (name, {argument: value})
|
|
206
|
+
pipe = tp.Pipeline([
|
|
207
|
+
'to_lower_case',
|
|
208
|
+
('remove_stop_words', {'keep_negations': False}),
|
|
209
|
+
])
|
|
210
|
+
print(pipe) # Pipeline(['to_lower_case', "remove_stop_words{'keep_negations': False}"])
|
|
211
|
+
|
|
212
|
+
# Save and reload (custom functions must be registered with @tp.register_step('name'))
|
|
213
|
+
pipe.save('my_pipeline.json')
|
|
214
|
+
pipe = tp.Pipeline.load('my_pipeline.json')
|
|
215
|
+
|
|
216
|
+
# Use it inside scikit-learn: pip install "nlp-text-preprocessing[sklearn]"
|
|
217
|
+
from sklearn.pipeline import make_pipeline
|
|
218
|
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
219
|
+
|
|
220
|
+
model = make_pipeline(tp.TextPreprocessor(['to_lower_case', 'remove_special_chars']),
|
|
221
|
+
TfidfVectorizer())
|
|
222
|
+
X = model.fit_transform(["Hello, World!", "HELLO there"])
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Every step except the last must return a string. The last step may return anything,
|
|
226
|
+
for example `Pipeline(['to_lower_case', 'text_stats'])`.
|
|
227
|
+
|
|
228
|
+
---
|
|
229
|
+
|
|
230
|
+
## Preparing Text for Transformers (BERT, GPT, ...)
|
|
231
|
+
|
|
232
|
+
Modern models work best with *lightly* cleaned text: keep the case, punctuation and
|
|
233
|
+
word forms, and only remove noise. `light_clean` does exactly that, and `truncate_text`
|
|
234
|
+
/ `chunk_text` help you stay inside a model's token limit.
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
import nlp_text_preprocessing as tp
|
|
238
|
+
|
|
239
|
+
tp.light_clean("<p>Great & fast! Visit https://x.com or mail me@x.com</p>",
|
|
240
|
+
urls='[URL]', emails='[EMAIL]')
|
|
241
|
+
# 'Great & fast! Visit [URL] or mail [EMAIL]'
|
|
242
|
+
|
|
243
|
+
# Use the real tokenizer so the limits are counted in model tokens
|
|
244
|
+
from transformers import AutoTokenizer
|
|
245
|
+
tok = AutoTokenizer.from_pretrained('bert-base-uncased')
|
|
246
|
+
|
|
247
|
+
tp.truncate_text(long_text, 256, tokenizer=tok)
|
|
248
|
+
chunks = tp.chunk_text(long_text, 256, overlap=32, tokenizer=tok)
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Without a `tokenizer`, the limits are counted in whitespace-separated words.
|
|
252
|
+
`tokenizer` may be a HuggingFace tokenizer (anything with `.tokenize()` and
|
|
253
|
+
`.convert_tokens_to_string()`) or a plain function that returns a list of tokens.
|
|
254
|
+
|
|
255
|
+
---
|
|
256
|
+
|
|
257
|
+
## Command Line (`nlp-prep`)
|
|
258
|
+
|
|
259
|
+
Installing the package also installs the `nlp-prep` command (`python -m nlp_text_preprocessing` works too).
|
|
260
|
+
|
|
261
|
+
```bash
|
|
262
|
+
# one document per line, written to a new file
|
|
263
|
+
nlp-prep clean reviews.txt -o reviews_clean.txt
|
|
264
|
+
|
|
265
|
+
# a CSV column (adds a 'text_clean' column; use --output-column text to overwrite)
|
|
266
|
+
nlp-prep clean data.csv --column text -o data_clean.csv
|
|
267
|
+
|
|
268
|
+
# standard input / output
|
|
269
|
+
cat notes.txt | nlp-prep clean - --lowercase --url-token "[URL]"
|
|
270
|
+
|
|
271
|
+
# classic cleaning, or your own steps / a saved Pipeline
|
|
272
|
+
nlp-prep clean reviews.txt --mode standard --stopwords
|
|
273
|
+
nlp-prep clean reviews.txt --steps to_lower_case,remove_urls
|
|
274
|
+
nlp-prep clean reviews.txt --pipeline my_pipeline.json
|
|
275
|
+
|
|
276
|
+
# check or install the NLTK data and spaCy model
|
|
277
|
+
nlp-prep resources --download
|
|
278
|
+
```
|
|
279
|
+
|
|
178
280
|
---
|
|
179
281
|
|
|
180
282
|
## Complete API Reference
|
|
@@ -212,29 +314,32 @@ Below is the complete reference guide for all functions available in the library
|
|
|
212
314
|
| `contraction_to_expansion(x)`| `x: str` | `str` | Expands contractions using contractions dictionary. |
|
|
213
315
|
| `remove_emails(x)` | `x: str` | `str` | Removes email addresses from text. |
|
|
214
316
|
| `count_emails(x)` | `x: str` | `int` | Counts email addresses present in text. |
|
|
215
|
-
| `remove_urls(x)` | `x: str` | `str` | Removes
|
|
317
|
+
| `remove_urls(x)` | `x: str` | `str` | Removes `http(s)://`, `ftp://` and `www.` URLs (trailing punctuation is kept). |
|
|
216
318
|
| `count_urls(x)` | `x: str` | `int` | Counts URLs present in text. |
|
|
217
|
-
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user
|
|
319
|
+
| `remove_rt(x)` | `x: str` | `str` | Removes retweet headers (`RT @user:`). |
|
|
218
320
|
| `count_rt(x)` | `x: str` | `int` | Counts retweet occurrences in text. |
|
|
219
321
|
| `remove_html_tag(x)` | `x: str` | `str` | Strips HTML/XML tags using BeautifulSoup. |
|
|
220
|
-
| `remove_accented_chars(x)` | `x: str` | `str` |
|
|
322
|
+
| `remove_accented_chars(x)` | `x: str` | `str` | Strips accents (`café` -> `cafe`); non-Latin scripts are kept. |
|
|
221
323
|
| `remove_mentions(x)` | `x: str` | `str` | Removes user handle mentions (`@username`). |
|
|
222
|
-
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and
|
|
223
|
-
| `remove_repeated_chars(x)` | `x: str` | `str` |
|
|
224
|
-
| `remove_stop_words(x)` | `x: str` | `str` |
|
|
225
|
-
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji
|
|
324
|
+
| `remove_special_chars(x)` | `x: str` | `str` | Strips punctuation and symbols (`well-known` -> `well known`). |
|
|
325
|
+
| `remove_repeated_chars(x)` | `x: str` | `str` | Shortens elongated words (`soooo` -> `soo`); digits and punctuation are untouched. |
|
|
326
|
+
| `remove_stop_words(x, keep_negations=True)` | `x: str` | `str` | Removes stop words (case-insensitive); negations like `not` are kept by default. |
|
|
327
|
+
| `remove_emojis(x)` | `x: str` | `str` | Strips emoji (flags, skin tones and ZWJ sequences included). |
|
|
226
328
|
| `extract_emojis(x)` | `x: str` | `list[str]` | Returns list of emojis found in text. |
|
|
227
329
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
228
|
-
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers
|
|
330
|
+
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
229
331
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
230
|
-
| `mask_pii(x)` | `x: str` | `str` | Redacts emails,
|
|
332
|
+
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
231
333
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
232
|
-
| `expand_hashtags(x)` | `x: str` | `str` | Splits
|
|
334
|
+
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
233
335
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
|
234
|
-
| `remove_numbers(x)` | `x: str` | `str` |
|
|
235
|
-
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips
|
|
336
|
+
| `remove_numbers(x)` | `x: str` | `str` | Removes standalone numbers; digits inside words (`3rd`, `H2O`) stay. |
|
|
337
|
+
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
|
|
236
338
|
| `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
|
|
237
|
-
| `clean_text(text)` | `text: str` | `str` |
|
|
339
|
+
| `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
|
|
340
|
+
| `light_clean(x, html=True, urls='', emails='', mentions=None, emojis=None, normalize='NFKC', keep_newlines=False, lowercase=False)` | `x: str` | `str` | Minimal cleaning for transformer models: Unicode normalisation, zero-width/control characters, HTML and entities, URLs/emails/mentions (remove, replace or keep), optional emoji handling. Case and punctuation are kept. |
|
|
341
|
+
| `truncate_text(x, max_tokens, tokenizer=None)` | `x: str` | `str` | Keeps the first `max_tokens` tokens (words, or tokens of a HuggingFace-style tokenizer). |
|
|
342
|
+
| `chunk_text(x, max_tokens, overlap=0, tokenizer=None)` | `x: str` | `list[str]` | Splits long text into pieces of at most `max_tokens`, with optional overlap. |
|
|
238
343
|
|
|
239
344
|
---
|
|
240
345
|
|
|
@@ -247,15 +352,15 @@ Below is the complete reference guide for all functions available in the library
|
|
|
247
352
|
| `extract_entities(x)` | `x: str` | `list[tuple]` | Extracts named entities `(text, label)` using spaCy. |
|
|
248
353
|
| `extract_pos_tags(x)` | `x: str` | `list[tuple]` | Extracts full POS tag breakdown `(token, pos, tag)` using spaCy. |
|
|
249
354
|
| `get_dependency_tree(x)`| `x: str` | `list[tuple]` | Extracts dependency parse tree `(token, dep, head)` using spaCy. |
|
|
250
|
-
| `detect_language(x)` | `x: str` | `str` |
|
|
355
|
+
| `detect_language(x)` | `x: str` | `str` | Language code (`'en'`, `'fr'`, `'hi'`, ...) via `langdetect`; `'unknown'` if undetectable. |
|
|
251
356
|
| `keyword_extraction(x, top_n)`| `x: str, top_n: int`| `list[str]`| Extracts top N keywords by term frequency. |
|
|
252
357
|
| `text_similarity(x1, x2)` | `x1, x2` | `float` | Calculates token Jaccard similarity between two texts (0.0 to 1.0). |
|
|
253
|
-
| `get_synonyms(word)` | `word: str` | `list[str]` |
|
|
254
|
-
| `get_antonyms(word)` | `word: str` | `list[str]` |
|
|
255
|
-
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` |
|
|
256
|
-
| `correct_spelling(x)` | `x: str` | `str` | Corrects
|
|
358
|
+
| `get_synonyms(word)` | `word: str` | `list[str]` | Sorted WordNet synonyms. |
|
|
359
|
+
| `get_antonyms(word)` | `word: str` | `list[str]` | Sorted WordNet antonyms. |
|
|
360
|
+
| `chunk_noun_phrases(x)` | `x: str` | `list[dict]` | spaCy noun chunks with exact character `start`/`end` offsets. |
|
|
361
|
+
| `correct_spelling(x)` | `x: str` | `str` | Corrects misspelled words (TextBlob); punctuation, case, URLs and acronyms are preserved. |
|
|
257
362
|
| `get_noun_phrase(x)` | `x: str` | `list[str]` | Extracts noun phrases from text. |
|
|
258
|
-
| `n_grams(x, n=2)` | `x: str, n: int` | `list` |
|
|
363
|
+
| `n_grams(x, n=2)` | `x: str, n: int` | `list[tuple]` | Word-level n-grams as tuples. |
|
|
259
364
|
| `singularize_words(x)` | `x: str` | `str` | Converts plural nouns (`NNS`) into singular forms. |
|
|
260
365
|
| `pluralize_words(x)` | `x: str` | `str` | Converts singular nouns (`NN`) into plural forms. |
|
|
261
366
|
|
|
@@ -265,13 +370,14 @@ Below is the complete reference guide for all functions available in the library
|
|
|
265
370
|
|
|
266
371
|
| Function | Parameters | Return Type | Description |
|
|
267
372
|
| :--- | :--- | :--- | :--- |
|
|
268
|
-
| `detect_profanity(x)` | `x: str` | `bool` |
|
|
269
|
-
| `censor_profanity(x)` | `x: str` | `str` |
|
|
270
|
-
| `detect_toxicity(x)` | `x: str` | `float` |
|
|
271
|
-
| `
|
|
272
|
-
| `
|
|
373
|
+
| `detect_profanity(x, include_mild=True, extra_words=(), allow_words=())` | `x: str` | `bool` | True if the text has profanity. Matches endings (`shitty`) and leetspeak (`sh1t`); ignores `Hell's Kitchen`; `include_mild=False` skips words like `damn`. Masked words such as `f***` are not detected. |
|
|
374
|
+
| `censor_profanity(x, include_mild=True, extra_words=(), allow_words=(), mask_char='*')` | `x: str` | `str` | Masks each profane word, keeping its length and the text layout. |
|
|
375
|
+
| `detect_toxicity(x)` | `x: str` | `float` | Score from 0.0 to 1.0. Counts insults and threats aimed at people; `kill the process` and `I do not hate you` score 0. |
|
|
376
|
+
| `toxicity_report(x)` | `x: str` | `dict` | Explains the score: `score`, `matches` (word, weight, targeted) and `ignored` (negated) words. |
|
|
377
|
+
| `detect_spam_signals(x)` | `x: str` | `dict` | Flags shouting, punctuation runs, links and spam phrases (`spam_phrases`); `is_spam_candidate` needs at least two signals. Short texts like `OK` are not flagged. |
|
|
378
|
+
| `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
|
|
273
379
|
| `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
|
|
274
|
-
| `emotion_detection(x)` | `x: str` | `dict` |
|
|
380
|
+
| `emotion_detection(x)` | `x: str` | `dict` | Emotion counts (joy, anger, sadness, fear). Handles endings (`loved`, `happier`) and ignores negated words (`not happy`). |
|
|
275
381
|
|
|
276
382
|
---
|
|
277
383
|
|
|
@@ -363,10 +469,11 @@ tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
|
363
469
|
|
|
364
470
|
## Troubleshooting and Resource Setup
|
|
365
471
|
|
|
366
|
-
If
|
|
472
|
+
If a function reports a missing NLTK corpus or spaCy model, check what is missing and install it:
|
|
367
473
|
|
|
368
474
|
```python
|
|
369
475
|
import nlp_text_preprocessing as tp
|
|
476
|
+
tp.check_resources(verbose=True)
|
|
370
477
|
tp.download_nltk_packages()
|
|
371
478
|
```
|
|
372
479
|
|