nlp-text-preprocessing 0.4.1__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nlp_text_preprocessing-0.4.1/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.5.0}/PKG-INFO +8 -2
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/Readme.md +4 -1
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/__init__.py +2 -1
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/all_text_preprocess.py +1627 -1503
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/cli.py +5 -2
- nlp_text_preprocessing-0.5.0/nlp_text_preprocessing/py.typed +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/sklearn_compat.py +8 -4
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0/nlp_text_preprocessing.egg-info}/PKG-INFO +8 -2
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +3 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/requires.txt +3 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/pyproject.toml +25 -2
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/tests/test_all_text_preprocess.py +1 -23
- nlp_text_preprocessing-0.5.0/tests/test_batching.py +91 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/tests/test_infrastructure.py +1 -1
- nlp_text_preprocessing-0.5.0/tests/test_new_features.py +55 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/LICENSE +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/MANIFEST.in +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/architecture_diagram.png +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_banner.png +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/__main__.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/entry_points.txt +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/requirements.txt +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/setup.cfg +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/tests/test_cleaning.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/tests/test_known_limits.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/tests/test_light_and_cli.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/tests/test_linguistic.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/tests/test_moderation.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/tests/test_pipeline_api.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nlp-text-preprocessing
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
|
|
5
5
|
Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -48,6 +48,9 @@ Requires-Dist: matplotlib; extra == "all"
|
|
|
48
48
|
Requires-Dist: scikit-learn; extra == "all"
|
|
49
49
|
Provides-Extra: dev
|
|
50
50
|
Requires-Dist: pytest; extra == "dev"
|
|
51
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
52
|
+
Requires-Dist: ruff; extra == "dev"
|
|
53
|
+
Requires-Dist: mypy; extra == "dev"
|
|
51
54
|
Requires-Dist: build; extra == "dev"
|
|
52
55
|
Requires-Dist: twine; extra == "dev"
|
|
53
56
|
Requires-Dist: pandas; extra == "dev"
|
|
@@ -201,6 +204,8 @@ clean_series = pipe.run_series(df['text_column'])
|
|
|
201
204
|
report = tp.diff_report(raw_texts, clean_batch)
|
|
202
205
|
```
|
|
203
206
|
|
|
207
|
+
`run_batch` and `run_series` are faster than looping over `run`: the spaCy steps (`lemmatize`, `convert_to_base`, `singularize_words`, `pluralize_words`) process the whole list with spaCy's `nlp.pipe` (about 2x faster in our test; the result is identical). `run_series` keeps the Series index and name.
|
|
208
|
+
|
|
204
209
|
### Steps with arguments, saving and scikit-learn
|
|
205
210
|
|
|
206
211
|
```python
|
|
@@ -300,6 +305,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
300
305
|
| `capital_ratio(x)` | `x: str` | `float` | Returns proportion of uppercase words relative to total words. |
|
|
301
306
|
| `unique_word_ratio(x)` | `x: str` | `float` | Returns lexical diversity ratio (unique words / total words). |
|
|
302
307
|
| `readability_score(x)` | `x: str` | `float` | Calculates Flesch Reading Ease score. |
|
|
308
|
+
| `readability_scores(x)` | `x: str` | `dict` | Flesch Reading Ease, Flesch-Kincaid grade, Gunning Fog, Automated Readability Index and Coleman-Liau (English, approximate). |
|
|
303
309
|
| `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtag tokens (`#tag`). |
|
|
304
310
|
| `mentions_count(x)` | `x: str` | `int` | Returns count of user handle mentions (`@user`). |
|
|
305
311
|
| `numerics_count(x)` | `x: str` | `int` | Returns count of numeric tokens. |
|
|
@@ -332,7 +338,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
332
338
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
333
339
|
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
334
340
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
335
|
-
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
341
|
+
| `mask_pii(x, token='[REDACTED]', names=False)` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. `names=True` also redacts people's names with spaCy's entity recogniser (statistical, so review the result). |
|
|
336
342
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
337
343
|
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
338
344
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
|
@@ -143,6 +143,8 @@ clean_series = pipe.run_series(df['text_column'])
|
|
|
143
143
|
report = tp.diff_report(raw_texts, clean_batch)
|
|
144
144
|
```
|
|
145
145
|
|
|
146
|
+
`run_batch` and `run_series` are faster than looping over `run`: the spaCy steps (`lemmatize`, `convert_to_base`, `singularize_words`, `pluralize_words`) process the whole list with spaCy's `nlp.pipe` (about 2x faster in our test; the result is identical). `run_series` keeps the Series index and name.
|
|
147
|
+
|
|
146
148
|
### Steps with arguments, saving and scikit-learn
|
|
147
149
|
|
|
148
150
|
```python
|
|
@@ -242,6 +244,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
242
244
|
| `capital_ratio(x)` | `x: str` | `float` | Returns proportion of uppercase words relative to total words. |
|
|
243
245
|
| `unique_word_ratio(x)` | `x: str` | `float` | Returns lexical diversity ratio (unique words / total words). |
|
|
244
246
|
| `readability_score(x)` | `x: str` | `float` | Calculates Flesch Reading Ease score. |
|
|
247
|
+
| `readability_scores(x)` | `x: str` | `dict` | Flesch Reading Ease, Flesch-Kincaid grade, Gunning Fog, Automated Readability Index and Coleman-Liau (English, approximate). |
|
|
245
248
|
| `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtag tokens (`#tag`). |
|
|
246
249
|
| `mentions_count(x)` | `x: str` | `int` | Returns count of user handle mentions (`@user`). |
|
|
247
250
|
| `numerics_count(x)` | `x: str` | `int` | Returns count of numeric tokens. |
|
|
@@ -274,7 +277,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
274
277
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
275
278
|
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
276
279
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
277
|
-
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
280
|
+
| `mask_pii(x, token='[REDACTED]', names=False)` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. `names=True` also redacts people's names with spaCy's entity recogniser (statistical, so review the result). |
|
|
278
281
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
279
282
|
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
280
283
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
{nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/__init__.py
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# entry level code and import text preprocess
|
|
2
2
|
|
|
3
|
-
__version__ = "0.
|
|
3
|
+
__version__ = "0.5.0"
|
|
4
4
|
|
|
5
5
|
from .all_text_preprocess import *
|
|
6
6
|
|
|
@@ -33,6 +33,7 @@ __all__ = [
|
|
|
33
33
|
'capital_ratio',
|
|
34
34
|
'unique_word_ratio',
|
|
35
35
|
'readability_score',
|
|
36
|
+
'readability_scores',
|
|
36
37
|
'text_stats',
|
|
37
38
|
'text_stats_batch',
|
|
38
39
|
'to_lower_case',
|