nlp-text-preprocessing 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.4.0}/PKG-INFO +96 -7
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/Readme.md +91 -6
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing/__init__.py +14 -1
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/__main__.py +5 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing/all_text_preprocess.py +431 -47
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/cli.py +150 -0
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/sklearn_compat.py +44 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info}/PKG-INFO +96 -7
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +8 -1
- nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info/entry_points.txt +2 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/requires.txt +5 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/pyproject.toml +6 -2
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/tests/test_infrastructure.py +1 -1
- nlp_text_preprocessing-0.4.0/tests/test_light_and_cli.py +195 -0
- nlp_text_preprocessing-0.4.0/tests/test_moderation.py +102 -0
- nlp_text_preprocessing-0.4.0/tests/test_pipeline_api.py +100 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/LICENSE +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/MANIFEST.in +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/architecture_diagram.png +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_banner.png +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/requirements.txt +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/setup.cfg +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/tests/test_all_text_preprocess.py +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/tests/test_cleaning.py +0 -0
- {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/tests/test_linguistic.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nlp-text-preprocessing
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
|
|
5
5
|
Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -37,10 +37,13 @@ Requires-Dist: pandas; extra == "pandas"
|
|
|
37
37
|
Provides-Extra: viz
|
|
38
38
|
Requires-Dist: wordcloud; extra == "viz"
|
|
39
39
|
Requires-Dist: matplotlib; extra == "viz"
|
|
40
|
+
Provides-Extra: sklearn
|
|
41
|
+
Requires-Dist: scikit-learn; extra == "sklearn"
|
|
40
42
|
Provides-Extra: all
|
|
41
43
|
Requires-Dist: pandas; extra == "all"
|
|
42
44
|
Requires-Dist: wordcloud; extra == "all"
|
|
43
45
|
Requires-Dist: matplotlib; extra == "all"
|
|
46
|
+
Requires-Dist: scikit-learn; extra == "all"
|
|
44
47
|
Provides-Extra: dev
|
|
45
48
|
Requires-Dist: pytest; extra == "dev"
|
|
46
49
|
Requires-Dist: build; extra == "dev"
|
|
@@ -48,6 +51,7 @@ Requires-Dist: twine; extra == "dev"
|
|
|
48
51
|
Requires-Dist: pandas; extra == "dev"
|
|
49
52
|
Requires-Dist: wordcloud; extra == "dev"
|
|
50
53
|
Requires-Dist: matplotlib; extra == "dev"
|
|
54
|
+
Requires-Dist: scikit-learn; extra == "dev"
|
|
51
55
|
Dynamic: license-file
|
|
52
56
|
|
|
53
57
|
# NLP Text Preprocessing Utility Package
|
|
@@ -83,6 +87,8 @@ Dynamic: license-file
|
|
|
83
87
|
- [Architecture Diagram](#architecture-diagram)
|
|
84
88
|
- [Quickstart and Setup](#quickstart-and-setup)
|
|
85
89
|
- [Pipeline Automation Engine](#pipeline-automation-engine)
|
|
90
|
+
- [Preparing Text for Transformers](#preparing-text-for-transformers-bert-gpt-)
|
|
91
|
+
- [Command Line (`nlp-prep`)](#command-line-nlp-prep)
|
|
86
92
|
- [Complete API Reference](#complete-api-reference)
|
|
87
93
|
- [1. General Feature Extraction](#1-general-feature-extraction)
|
|
88
94
|
- [2. Text Cleaning and Normalization](#2-text-cleaning-and-normalization)
|
|
@@ -118,7 +124,7 @@ It provides over **60 single-purpose functions** and an extensible **Pipeline En
|
|
|
118
124
|
- **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
|
|
119
125
|
- **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
|
|
120
126
|
- **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
|
|
121
|
-
- **Safety & Moderation**:
|
|
127
|
+
- **Safety & Moderation (baseline)**: Lexicon-based profanity, toxicity, spam and emotion helpers. They are transparent keyword baselines, not a moderation system: use a trained model for anything high-stakes.
|
|
122
128
|
- **Visualization**: Generate and save custom WordClouds to disk or display interactively.
|
|
123
129
|
|
|
124
130
|
---
|
|
@@ -141,6 +147,7 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
|
|
|
141
147
|
pip install nlp-text-preprocessing # core
|
|
142
148
|
pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
|
|
143
149
|
pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
|
|
150
|
+
pip install "nlp-text-preprocessing[sklearn]" # + scikit-learn (TextPreprocessor)
|
|
144
151
|
pip install "nlp-text-preprocessing[all]" # everything optional
|
|
145
152
|
```
|
|
146
153
|
|
|
@@ -192,6 +199,84 @@ clean_series = pipe.run_series(df['text_column'])
|
|
|
192
199
|
report = tp.diff_report(raw_texts, clean_batch)
|
|
193
200
|
```
|
|
194
201
|
|
|
202
|
+
### Steps with arguments, saving and scikit-learn
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
# Pass arguments as (name, {argument: value})
|
|
206
|
+
pipe = tp.Pipeline([
|
|
207
|
+
'to_lower_case',
|
|
208
|
+
('remove_stop_words', {'keep_negations': False}),
|
|
209
|
+
])
|
|
210
|
+
print(pipe) # Pipeline(['to_lower_case', "remove_stop_words{'keep_negations': False}"])
|
|
211
|
+
|
|
212
|
+
# Save and reload (custom functions must be registered with @tp.register_step('name'))
|
|
213
|
+
pipe.save('my_pipeline.json')
|
|
214
|
+
pipe = tp.Pipeline.load('my_pipeline.json')
|
|
215
|
+
|
|
216
|
+
# Use it inside scikit-learn: pip install "nlp-text-preprocessing[sklearn]"
|
|
217
|
+
from sklearn.pipeline import make_pipeline
|
|
218
|
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
219
|
+
|
|
220
|
+
model = make_pipeline(tp.TextPreprocessor(['to_lower_case', 'remove_special_chars']),
|
|
221
|
+
TfidfVectorizer())
|
|
222
|
+
X = model.fit_transform(["Hello, World!", "HELLO there"])
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Every step except the last must return a string. The last step may return anything,
|
|
226
|
+
for example `Pipeline(['to_lower_case', 'text_stats'])`.
|
|
227
|
+
|
|
228
|
+
---
|
|
229
|
+
|
|
230
|
+
## Preparing Text for Transformers (BERT, GPT, ...)
|
|
231
|
+
|
|
232
|
+
Modern models work best with *lightly* cleaned text: keep the case, punctuation and
|
|
233
|
+
word forms, and only remove noise. `light_clean` does exactly that, and `truncate_text`
|
|
234
|
+
/ `chunk_text` help you stay inside a model's token limit.
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
import nlp_text_preprocessing as tp
|
|
238
|
+
|
|
239
|
+
tp.light_clean("<p>Great & fast! Visit https://x.com or mail me@x.com</p>",
|
|
240
|
+
urls='[URL]', emails='[EMAIL]')
|
|
241
|
+
# 'Great & fast! Visit [URL] or mail [EMAIL]'
|
|
242
|
+
|
|
243
|
+
# Use the real tokenizer so the limits are counted in model tokens
|
|
244
|
+
from transformers import AutoTokenizer
|
|
245
|
+
tok = AutoTokenizer.from_pretrained('bert-base-uncased')
|
|
246
|
+
|
|
247
|
+
tp.truncate_text(long_text, 256, tokenizer=tok)
|
|
248
|
+
chunks = tp.chunk_text(long_text, 256, overlap=32, tokenizer=tok)
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Without a `tokenizer`, the limits are counted in whitespace-separated words.
|
|
252
|
+
`tokenizer` may be a HuggingFace tokenizer (anything with `.tokenize()` and
|
|
253
|
+
`.convert_tokens_to_string()`) or a plain function that returns a list of tokens.
|
|
254
|
+
|
|
255
|
+
---
|
|
256
|
+
|
|
257
|
+
## Command Line (`nlp-prep`)
|
|
258
|
+
|
|
259
|
+
Installing the package also installs the `nlp-prep` command (`python -m nlp_text_preprocessing` works too).
|
|
260
|
+
|
|
261
|
+
```bash
|
|
262
|
+
# one document per line, written to a new file
|
|
263
|
+
nlp-prep clean reviews.txt -o reviews_clean.txt
|
|
264
|
+
|
|
265
|
+
# a CSV column (adds a 'text_clean' column; use --output-column text to overwrite)
|
|
266
|
+
nlp-prep clean data.csv --column text -o data_clean.csv
|
|
267
|
+
|
|
268
|
+
# standard input / output
|
|
269
|
+
cat notes.txt | nlp-prep clean - --lowercase --url-token "[URL]"
|
|
270
|
+
|
|
271
|
+
# classic cleaning, or your own steps / a saved Pipeline
|
|
272
|
+
nlp-prep clean reviews.txt --mode standard --stopwords
|
|
273
|
+
nlp-prep clean reviews.txt --steps to_lower_case,remove_urls
|
|
274
|
+
nlp-prep clean reviews.txt --pipeline my_pipeline.json
|
|
275
|
+
|
|
276
|
+
# check or install the NLTK data and spaCy model
|
|
277
|
+
nlp-prep resources --download
|
|
278
|
+
```
|
|
279
|
+
|
|
195
280
|
---
|
|
196
281
|
|
|
197
282
|
## Complete API Reference
|
|
@@ -252,6 +337,9 @@ Below is the complete reference guide for all functions available in the library
|
|
|
252
337
|
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
|
|
253
338
|
| `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
|
|
254
339
|
| `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
|
|
340
|
+
| `light_clean(x, html=True, urls='', emails='', mentions=None, emojis=None, normalize='NFKC', keep_newlines=False, lowercase=False)` | `x: str` | `str` | Minimal cleaning for transformer models: Unicode normalisation, zero-width/control characters, HTML and entities, URLs/emails/mentions (remove, replace or keep), optional emoji handling. Case and punctuation are kept. |
|
|
341
|
+
| `truncate_text(x, max_tokens, tokenizer=None)` | `x: str` | `str` | Keeps the first `max_tokens` tokens (words, or tokens of a HuggingFace-style tokenizer). |
|
|
342
|
+
| `chunk_text(x, max_tokens, overlap=0, tokenizer=None)` | `x: str` | `list[str]` | Splits long text into pieces of at most `max_tokens`, with optional overlap. |
|
|
255
343
|
|
|
256
344
|
---
|
|
257
345
|
|
|
@@ -282,13 +370,14 @@ Below is the complete reference guide for all functions available in the library
|
|
|
282
370
|
|
|
283
371
|
| Function | Parameters | Return Type | Description |
|
|
284
372
|
| :--- | :--- | :--- | :--- |
|
|
285
|
-
| `detect_profanity(x)` | `x: str` | `bool` |
|
|
286
|
-
| `censor_profanity(x)` | `x: str` | `str` |
|
|
287
|
-
| `detect_toxicity(x)` | `x: str` | `float` |
|
|
288
|
-
| `
|
|
373
|
+
| `detect_profanity(x, include_mild=True, extra_words=(), allow_words=())` | `x: str` | `bool` | True if the text has profanity. Matches endings (`shitty`) and leetspeak (`sh1t`); ignores `Hell's Kitchen`; `include_mild=False` skips words like `damn`. Masked words such as `f***` are not detected. |
|
|
374
|
+
| `censor_profanity(x, include_mild=True, extra_words=(), allow_words=(), mask_char='*')` | `x: str` | `str` | Masks each profane word, keeping its length and the text layout. |
|
|
375
|
+
| `detect_toxicity(x)` | `x: str` | `float` | Score from 0.0 to 1.0. Counts insults and threats aimed at people; `kill the process` and `I do not hate you` score 0. |
|
|
376
|
+
| `toxicity_report(x)` | `x: str` | `dict` | Explains the score: `score`, `matches` (word, weight, targeted) and `ignored` (negated) words. |
|
|
377
|
+
| `detect_spam_signals(x)` | `x: str` | `dict` | Flags shouting, punctuation runs, links and spam phrases (`spam_phrases`); `is_spam_candidate` needs at least two signals. Short texts like `OK` are not flagged. |
|
|
289
378
|
| `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
|
|
290
379
|
| `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
|
|
291
|
-
| `emotion_detection(x)` | `x: str` | `dict` |
|
|
380
|
+
| `emotion_detection(x)` | `x: str` | `dict` | Emotion counts (joy, anger, sadness, fear). Handles endings (`loved`, `happier`) and ignores negated words (`not happy`). |
|
|
292
381
|
|
|
293
382
|
---
|
|
294
383
|
|
|
@@ -31,6 +31,8 @@
|
|
|
31
31
|
- [Architecture Diagram](#architecture-diagram)
|
|
32
32
|
- [Quickstart and Setup](#quickstart-and-setup)
|
|
33
33
|
- [Pipeline Automation Engine](#pipeline-automation-engine)
|
|
34
|
+
- [Preparing Text for Transformers](#preparing-text-for-transformers-bert-gpt-)
|
|
35
|
+
- [Command Line (`nlp-prep`)](#command-line-nlp-prep)
|
|
34
36
|
- [Complete API Reference](#complete-api-reference)
|
|
35
37
|
- [1. General Feature Extraction](#1-general-feature-extraction)
|
|
36
38
|
- [2. Text Cleaning and Normalization](#2-text-cleaning-and-normalization)
|
|
@@ -66,7 +68,7 @@ It provides over **60 single-purpose functions** and an extensible **Pipeline En
|
|
|
66
68
|
- **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
|
|
67
69
|
- **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
|
|
68
70
|
- **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
|
|
69
|
-
- **Safety & Moderation**:
|
|
71
|
+
- **Safety & Moderation (baseline)**: Lexicon-based profanity, toxicity, spam and emotion helpers. They are transparent keyword baselines, not a moderation system: use a trained model for anything high-stakes.
|
|
70
72
|
- **Visualization**: Generate and save custom WordClouds to disk or display interactively.
|
|
71
73
|
|
|
72
74
|
---
|
|
@@ -89,6 +91,7 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
|
|
|
89
91
|
pip install nlp-text-preprocessing # core
|
|
90
92
|
pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
|
|
91
93
|
pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
|
|
94
|
+
pip install "nlp-text-preprocessing[sklearn]" # + scikit-learn (TextPreprocessor)
|
|
92
95
|
pip install "nlp-text-preprocessing[all]" # everything optional
|
|
93
96
|
```
|
|
94
97
|
|
|
@@ -140,6 +143,84 @@ clean_series = pipe.run_series(df['text_column'])
|
|
|
140
143
|
report = tp.diff_report(raw_texts, clean_batch)
|
|
141
144
|
```
|
|
142
145
|
|
|
146
|
+
### Steps with arguments, saving and scikit-learn
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
# Pass arguments as (name, {argument: value})
|
|
150
|
+
pipe = tp.Pipeline([
|
|
151
|
+
'to_lower_case',
|
|
152
|
+
('remove_stop_words', {'keep_negations': False}),
|
|
153
|
+
])
|
|
154
|
+
print(pipe) # Pipeline(['to_lower_case', "remove_stop_words{'keep_negations': False}"])
|
|
155
|
+
|
|
156
|
+
# Save and reload (custom functions must be registered with @tp.register_step('name'))
|
|
157
|
+
pipe.save('my_pipeline.json')
|
|
158
|
+
pipe = tp.Pipeline.load('my_pipeline.json')
|
|
159
|
+
|
|
160
|
+
# Use it inside scikit-learn: pip install "nlp-text-preprocessing[sklearn]"
|
|
161
|
+
from sklearn.pipeline import make_pipeline
|
|
162
|
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
163
|
+
|
|
164
|
+
model = make_pipeline(tp.TextPreprocessor(['to_lower_case', 'remove_special_chars']),
|
|
165
|
+
TfidfVectorizer())
|
|
166
|
+
X = model.fit_transform(["Hello, World!", "HELLO there"])
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Every step except the last must return a string. The last step may return anything,
|
|
170
|
+
for example `Pipeline(['to_lower_case', 'text_stats'])`.
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## Preparing Text for Transformers (BERT, GPT, ...)
|
|
175
|
+
|
|
176
|
+
Modern models work best with *lightly* cleaned text: keep the case, punctuation and
|
|
177
|
+
word forms, and only remove noise. `light_clean` does exactly that, and `truncate_text`
|
|
178
|
+
/ `chunk_text` help you stay inside a model's token limit.
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
import nlp_text_preprocessing as tp
|
|
182
|
+
|
|
183
|
+
tp.light_clean("<p>Great & fast! Visit https://x.com or mail me@x.com</p>",
|
|
184
|
+
urls='[URL]', emails='[EMAIL]')
|
|
185
|
+
# 'Great & fast! Visit [URL] or mail [EMAIL]'
|
|
186
|
+
|
|
187
|
+
# Use the real tokenizer so the limits are counted in model tokens
|
|
188
|
+
from transformers import AutoTokenizer
|
|
189
|
+
tok = AutoTokenizer.from_pretrained('bert-base-uncased')
|
|
190
|
+
|
|
191
|
+
tp.truncate_text(long_text, 256, tokenizer=tok)
|
|
192
|
+
chunks = tp.chunk_text(long_text, 256, overlap=32, tokenizer=tok)
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Without a `tokenizer`, the limits are counted in whitespace-separated words.
|
|
196
|
+
`tokenizer` may be a HuggingFace tokenizer (anything with `.tokenize()` and
|
|
197
|
+
`.convert_tokens_to_string()`) or a plain function that returns a list of tokens.
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
## Command Line (`nlp-prep`)
|
|
202
|
+
|
|
203
|
+
Installing the package also installs the `nlp-prep` command (`python -m nlp_text_preprocessing` works too).
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
# one document per line, written to a new file
|
|
207
|
+
nlp-prep clean reviews.txt -o reviews_clean.txt
|
|
208
|
+
|
|
209
|
+
# a CSV column (adds a 'text_clean' column; use --output-column text to overwrite)
|
|
210
|
+
nlp-prep clean data.csv --column text -o data_clean.csv
|
|
211
|
+
|
|
212
|
+
# standard input / output
|
|
213
|
+
cat notes.txt | nlp-prep clean - --lowercase --url-token "[URL]"
|
|
214
|
+
|
|
215
|
+
# classic cleaning, or your own steps / a saved Pipeline
|
|
216
|
+
nlp-prep clean reviews.txt --mode standard --stopwords
|
|
217
|
+
nlp-prep clean reviews.txt --steps to_lower_case,remove_urls
|
|
218
|
+
nlp-prep clean reviews.txt --pipeline my_pipeline.json
|
|
219
|
+
|
|
220
|
+
# check or install the NLTK data and spaCy model
|
|
221
|
+
nlp-prep resources --download
|
|
222
|
+
```
|
|
223
|
+
|
|
143
224
|
---
|
|
144
225
|
|
|
145
226
|
## Complete API Reference
|
|
@@ -200,6 +281,9 @@ Below is the complete reference guide for all functions available in the library
|
|
|
200
281
|
| `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
|
|
201
282
|
| `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
|
|
202
283
|
| `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
|
|
284
|
+
| `light_clean(x, html=True, urls='', emails='', mentions=None, emojis=None, normalize='NFKC', keep_newlines=False, lowercase=False)` | `x: str` | `str` | Minimal cleaning for transformer models: Unicode normalisation, zero-width/control characters, HTML and entities, URLs/emails/mentions (remove, replace or keep), optional emoji handling. Case and punctuation are kept. |
|
|
285
|
+
| `truncate_text(x, max_tokens, tokenizer=None)` | `x: str` | `str` | Keeps the first `max_tokens` tokens (words, or tokens of a HuggingFace-style tokenizer). |
|
|
286
|
+
| `chunk_text(x, max_tokens, overlap=0, tokenizer=None)` | `x: str` | `list[str]` | Splits long text into pieces of at most `max_tokens`, with optional overlap. |
|
|
203
287
|
|
|
204
288
|
---
|
|
205
289
|
|
|
@@ -230,13 +314,14 @@ Below is the complete reference guide for all functions available in the library
|
|
|
230
314
|
|
|
231
315
|
| Function | Parameters | Return Type | Description |
|
|
232
316
|
| :--- | :--- | :--- | :--- |
|
|
233
|
-
| `detect_profanity(x)` | `x: str` | `bool` |
|
|
234
|
-
| `censor_profanity(x)` | `x: str` | `str` |
|
|
235
|
-
| `detect_toxicity(x)` | `x: str` | `float` |
|
|
236
|
-
| `
|
|
317
|
+
| `detect_profanity(x, include_mild=True, extra_words=(), allow_words=())` | `x: str` | `bool` | True if the text has profanity. Matches endings (`shitty`) and leetspeak (`sh1t`); ignores `Hell's Kitchen`; `include_mild=False` skips words like `damn`. Masked words such as `f***` are not detected. |
|
|
318
|
+
| `censor_profanity(x, include_mild=True, extra_words=(), allow_words=(), mask_char='*')` | `x: str` | `str` | Masks each profane word, keeping its length and the text layout. |
|
|
319
|
+
| `detect_toxicity(x)` | `x: str` | `float` | Score from 0.0 to 1.0. Counts insults and threats aimed at people; `kill the process` and `I do not hate you` score 0. |
|
|
320
|
+
| `toxicity_report(x)` | `x: str` | `dict` | Explains the score: `score`, `matches` (word, weight, targeted) and `ignored` (negated) words. |
|
|
321
|
+
| `detect_spam_signals(x)` | `x: str` | `dict` | Flags shouting, punctuation runs, links and spam phrases (`spam_phrases`); `is_spam_candidate` needs at least two signals. Short texts like `OK` are not flagged. |
|
|
237
322
|
| `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
|
|
238
323
|
| `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
|
|
239
|
-
| `emotion_detection(x)` | `x: str` | `dict` |
|
|
324
|
+
| `emotion_detection(x)` | `x: str` | `dict` | Emotion counts (joy, anger, sadness, fear). Handles endings (`loved`, `happier`) and ignores negated words (`not happy`). |
|
|
240
325
|
|
|
241
326
|
---
|
|
242
327
|
|
{nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing/__init__.py
RENAMED
|
@@ -1,9 +1,18 @@
|
|
|
1
1
|
# entry level code and import text preprocess
|
|
2
2
|
|
|
3
|
-
__version__ = "0.
|
|
3
|
+
__version__ = "0.4.0"
|
|
4
4
|
|
|
5
5
|
from .all_text_preprocess import *
|
|
6
6
|
|
|
7
|
+
|
|
8
|
+
def __getattr__(name):
|
|
9
|
+
# Loaded on first access so scikit-learn stays an optional dependency.
|
|
10
|
+
if name == "TextPreprocessor":
|
|
11
|
+
from .sklearn_compat import TextPreprocessor
|
|
12
|
+
return TextPreprocessor
|
|
13
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
14
|
+
|
|
15
|
+
|
|
7
16
|
__all__ = [
|
|
8
17
|
'__version__',
|
|
9
18
|
'download_nltk_packages',
|
|
@@ -76,6 +85,10 @@ __all__ = [
|
|
|
76
85
|
'sentiment_analysis',
|
|
77
86
|
'polarity_subjectivity',
|
|
78
87
|
'emotion_detection',
|
|
88
|
+
'toxicity_report',
|
|
89
|
+
'light_clean',
|
|
90
|
+
'truncate_text',
|
|
91
|
+
'chunk_text',
|
|
79
92
|
'Pipeline',
|
|
80
93
|
'register_step',
|
|
81
94
|
'diff_report',
|