nlp-text-preprocessing 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.5.0}/PKG-INFO +11 -2
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/Readme.md +5 -1
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/__init__.py +2 -1
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/all_text_preprocess.py +1627 -1475
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/cli.py +5 -2
- nlp_text_preprocessing-0.5.0/nlp_text_preprocessing/py.typed +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/sklearn_compat.py +8 -4
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0/nlp_text_preprocessing.egg-info}/PKG-INFO +11 -2
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +4 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/requires.txt +6 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/pyproject.toml +26 -2
- nlp_text_preprocessing-0.5.0/requirements.txt +15 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_all_text_preprocess.py +1 -23
- nlp_text_preprocessing-0.5.0/tests/test_batching.py +91 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_infrastructure.py +1 -1
- nlp_text_preprocessing-0.5.0/tests/test_known_limits.py +38 -0
- nlp_text_preprocessing-0.5.0/tests/test_new_features.py +55 -0
- nlp_text_preprocessing-0.4.0/requirements.txt +0 -7
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/LICENSE +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/MANIFEST.in +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/architecture_diagram.png +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_banner.png +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/__main__.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/entry_points.txt +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/setup.cfg +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_cleaning.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_light_and_cli.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_linguistic.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_moderation.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_pipeline_api.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nlp-text-preprocessing
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
|
|
5
5
|
Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -39,6 +39,8 @@ Requires-Dist: wordcloud; extra == "viz"
|
|
|
39
39
|
Requires-Dist: matplotlib; extra == "viz"
|
|
40
40
|
Provides-Extra: sklearn
|
|
41
41
|
Requires-Dist: scikit-learn; extra == "sklearn"
|
|
42
|
+
Provides-Extra: transformers
|
|
43
|
+
Requires-Dist: transformers; extra == "transformers"
|
|
42
44
|
Provides-Extra: all
|
|
43
45
|
Requires-Dist: pandas; extra == "all"
|
|
44
46
|
Requires-Dist: wordcloud; extra == "all"
|
|
@@ -46,6 +48,9 @@ Requires-Dist: matplotlib; extra == "all"
|
|
|
46
48
|
Requires-Dist: scikit-learn; extra == "all"
|
|
47
49
|
Provides-Extra: dev
|
|
48
50
|
Requires-Dist: pytest; extra == "dev"
|
|
51
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
52
|
+
Requires-Dist: ruff; extra == "dev"
|
|
53
|
+
Requires-Dist: mypy; extra == "dev"
|
|
49
54
|
Requires-Dist: build; extra == "dev"
|
|
50
55
|
Requires-Dist: twine; extra == "dev"
|
|
51
56
|
Requires-Dist: pandas; extra == "dev"
|
|
@@ -199,6 +204,8 @@ clean_series = pipe.run_series(df['text_column'])
|
|
|
199
204
|
report = tp.diff_report(raw_texts, clean_batch)
|
|
200
205
|
```
|
|
201
206
|
|
|
207
|
+
`run_batch` and `run_series` are faster than looping over `run`: the spaCy steps (`lemmatize`, `convert_to_base`, `singularize_words`, `pluralize_words`) process the whole list with spaCy's `nlp.pipe` (about 2x faster in our test; the result is identical). `run_series` keeps the Series index and name.
|
|
208
|
+
|
|
202
209
|
### Steps with arguments, saving and scikit-learn
|
|
203
210
|
|
|
204
211
|
```python
|
|
@@ -241,6 +248,7 @@ tp.light_clean("<p>Great & fast! Visit https://x.com or mail me@x.com</p>
|
|
|
241
248
|
# 'Great & fast! Visit [URL] or mail [EMAIL]'
|
|
242
249
|
|
|
243
250
|
# Use the real tokenizer so the limits are counted in model tokens
|
|
251
|
+
# (pip install "nlp-text-preprocessing[transformers]")
|
|
244
252
|
from transformers import AutoTokenizer
|
|
245
253
|
tok = AutoTokenizer.from_pretrained('bert-base-uncased')
|
|
246
254
|
|
|
@@ -297,6 +305,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
297
305
|
| `capital_ratio(x)` | `x: str` | `float` | Returns proportion of uppercase words relative to total words. |
|
|
298
306
|
| `unique_word_ratio(x)` | `x: str` | `float` | Returns lexical diversity ratio (unique words / total words). |
|
|
299
307
|
| `readability_score(x)` | `x: str` | `float` | Calculates Flesch Reading Ease score. |
|
|
308
|
+
| `readability_scores(x)` | `x: str` | `dict` | Flesch Reading Ease, Flesch-Kincaid grade, Gunning Fog, Automated Readability Index and Coleman-Liau (English, approximate). |
|
|
300
309
|
| `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtag tokens (`#tag`). |
|
|
301
310
|
| `mentions_count(x)` | `x: str` | `int` | Returns count of user handle mentions (`@user`). |
|
|
302
311
|
| `numerics_count(x)` | `x: str` | `int` | Returns count of numeric tokens. |
|
|
@@ -329,7 +338,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
329
338
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
330
339
|
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
331
340
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
332
|
-
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
341
|
+
| `mask_pii(x, token='[REDACTED]', names=False)` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. `names=True` also redacts people's names with spaCy's entity recogniser (statistical, so review the result). |
|
|
333
342
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
334
343
|
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
335
344
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
|
@@ -143,6 +143,8 @@ clean_series = pipe.run_series(df['text_column'])
|
|
|
143
143
|
report = tp.diff_report(raw_texts, clean_batch)
|
|
144
144
|
```
|
|
145
145
|
|
|
146
|
+
`run_batch` and `run_series` are faster than looping over `run`: the spaCy steps (`lemmatize`, `convert_to_base`, `singularize_words`, `pluralize_words`) process the whole list with spaCy's `nlp.pipe` (about 2x faster in our test; the result is identical). `run_series` keeps the Series index and name.
|
|
147
|
+
|
|
146
148
|
### Steps with arguments, saving and scikit-learn
|
|
147
149
|
|
|
148
150
|
```python
|
|
@@ -185,6 +187,7 @@ tp.light_clean("<p>Great & fast! Visit https://x.com or mail me@x.com</p>
|
|
|
185
187
|
# 'Great & fast! Visit [URL] or mail [EMAIL]'
|
|
186
188
|
|
|
187
189
|
# Use the real tokenizer so the limits are counted in model tokens
|
|
190
|
+
# (pip install "nlp-text-preprocessing[transformers]")
|
|
188
191
|
from transformers import AutoTokenizer
|
|
189
192
|
tok = AutoTokenizer.from_pretrained('bert-base-uncased')
|
|
190
193
|
|
|
@@ -241,6 +244,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
241
244
|
| `capital_ratio(x)` | `x: str` | `float` | Returns proportion of uppercase words relative to total words. |
|
|
242
245
|
| `unique_word_ratio(x)` | `x: str` | `float` | Returns lexical diversity ratio (unique words / total words). |
|
|
243
246
|
| `readability_score(x)` | `x: str` | `float` | Calculates Flesch Reading Ease score. |
|
|
247
|
+
| `readability_scores(x)` | `x: str` | `dict` | Flesch Reading Ease, Flesch-Kincaid grade, Gunning Fog, Automated Readability Index and Coleman-Liau (English, approximate). |
|
|
244
248
|
| `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtag tokens (`#tag`). |
|
|
245
249
|
| `mentions_count(x)` | `x: str` | `int` | Returns count of user handle mentions (`@user`). |
|
|
246
250
|
| `numerics_count(x)` | `x: str` | `int` | Returns count of numeric tokens. |
|
|
@@ -273,7 +277,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
273
277
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
274
278
|
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
275
279
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
276
|
-
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
280
|
+
| `mask_pii(x, token='[REDACTED]', names=False)` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. `names=True` also redacts people's names with spaCy's entity recogniser (statistical, so review the result). |
|
|
277
281
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
278
282
|
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
279
283
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
{nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/__init__.py
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# entry level code and import text preprocess
|
|
2
2
|
|
|
3
|
-
__version__ = "0.
|
|
3
|
+
__version__ = "0.5.0"
|
|
4
4
|
|
|
5
5
|
from .all_text_preprocess import *
|
|
6
6
|
|
|
@@ -33,6 +33,7 @@ __all__ = [
|
|
|
33
33
|
'capital_ratio',
|
|
34
34
|
'unique_word_ratio',
|
|
35
35
|
'readability_score',
|
|
36
|
+
'readability_scores',
|
|
36
37
|
'text_stats',
|
|
37
38
|
'text_stats_batch',
|
|
38
39
|
'to_lower_case',
|