nlp-text-preprocessing 0.4.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. {nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.5.0}/PKG-INFO +11 -2
  2. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/Readme.md +5 -1
  3. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/__init__.py +2 -1
  4. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/all_text_preprocess.py +1627 -1475
  5. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/cli.py +5 -2
  6. nlp_text_preprocessing-0.5.0/nlp_text_preprocessing/py.typed +0 -0
  7. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/sklearn_compat.py +8 -4
  8. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0/nlp_text_preprocessing.egg-info}/PKG-INFO +11 -2
  9. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +4 -0
  10. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/requires.txt +6 -0
  11. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/pyproject.toml +26 -2
  12. nlp_text_preprocessing-0.5.0/requirements.txt +15 -0
  13. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_all_text_preprocess.py +1 -23
  14. nlp_text_preprocessing-0.5.0/tests/test_batching.py +91 -0
  15. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_infrastructure.py +1 -1
  16. nlp_text_preprocessing-0.5.0/tests/test_known_limits.py +38 -0
  17. nlp_text_preprocessing-0.5.0/tests/test_new_features.py +55 -0
  18. nlp_text_preprocessing-0.4.0/requirements.txt +0 -7
  19. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/LICENSE +0 -0
  20. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/MANIFEST.in +0 -0
  21. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/architecture_diagram.png +0 -0
  22. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_banner.png +0 -0
  23. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/__main__.py +0 -0
  24. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing/data/contractions.json +0 -0
  25. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
  26. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/entry_points.txt +0 -0
  27. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
  28. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/setup.cfg +0 -0
  29. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_cleaning.py +0 -0
  30. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_light_and_cli.py +0 -0
  31. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_linguistic.py +0 -0
  32. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_moderation.py +0 -0
  33. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.5.0}/tests/test_pipeline_api.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nlp-text-preprocessing
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
5
5
  Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
6
6
  License-Expression: MIT
@@ -39,6 +39,8 @@ Requires-Dist: wordcloud; extra == "viz"
39
39
  Requires-Dist: matplotlib; extra == "viz"
40
40
  Provides-Extra: sklearn
41
41
  Requires-Dist: scikit-learn; extra == "sklearn"
42
+ Provides-Extra: transformers
43
+ Requires-Dist: transformers; extra == "transformers"
42
44
  Provides-Extra: all
43
45
  Requires-Dist: pandas; extra == "all"
44
46
  Requires-Dist: wordcloud; extra == "all"
@@ -46,6 +48,9 @@ Requires-Dist: matplotlib; extra == "all"
46
48
  Requires-Dist: scikit-learn; extra == "all"
47
49
  Provides-Extra: dev
48
50
  Requires-Dist: pytest; extra == "dev"
51
+ Requires-Dist: pytest-cov; extra == "dev"
52
+ Requires-Dist: ruff; extra == "dev"
53
+ Requires-Dist: mypy; extra == "dev"
49
54
  Requires-Dist: build; extra == "dev"
50
55
  Requires-Dist: twine; extra == "dev"
51
56
  Requires-Dist: pandas; extra == "dev"
@@ -199,6 +204,8 @@ clean_series = pipe.run_series(df['text_column'])
199
204
  report = tp.diff_report(raw_texts, clean_batch)
200
205
  ```
201
206
 
207
+ `run_batch` and `run_series` are faster than looping over `run`: the spaCy steps (`lemmatize`, `convert_to_base`, `singularize_words`, `pluralize_words`) process the whole list with spaCy's `nlp.pipe` (about 2x faster in our test; the result is identical). `run_series` keeps the Series index and name.
208
+
202
209
  ### Steps with arguments, saving and scikit-learn
203
210
 
204
211
  ```python
@@ -241,6 +248,7 @@ tp.light_clean("<p>Great &amp; fast!​ Visit https://x.com or mail me@x.com</p>
241
248
  # 'Great & fast! Visit [URL] or mail [EMAIL]'
242
249
 
243
250
  # Use the real tokenizer so the limits are counted in model tokens
251
+ # (pip install "nlp-text-preprocessing[transformers]")
244
252
  from transformers import AutoTokenizer
245
253
  tok = AutoTokenizer.from_pretrained('bert-base-uncased')
246
254
 
@@ -297,6 +305,7 @@ Below is the complete reference guide for all functions available in the library
297
305
  | `capital_ratio(x)` | `x: str` | `float` | Returns proportion of uppercase words relative to total words. |
298
306
  | `unique_word_ratio(x)` | `x: str` | `float` | Returns lexical diversity ratio (unique words / total words). |
299
307
  | `readability_score(x)` | `x: str` | `float` | Calculates Flesch Reading Ease score. |
308
+ | `readability_scores(x)` | `x: str` | `dict` | Flesch Reading Ease, Flesch-Kincaid grade, Gunning Fog, Automated Readability Index and Coleman-Liau (English, approximate). |
300
309
  | `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtag tokens (`#tag`). |
301
310
  | `mentions_count(x)` | `x: str` | `int` | Returns count of user handle mentions (`@user`). |
302
311
  | `numerics_count(x)` | `x: str` | `int` | Returns count of numeric tokens. |
@@ -329,7 +338,7 @@ Below is the complete reference guide for all functions available in the library
329
338
  | `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
330
339
  | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
331
340
  | `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
332
- | `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
341
+ | `mask_pii(x, token='[REDACTED]', names=False)` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. `names=True` also redacts people's names with spaCy's entity recogniser (statistical, so review the result). |
333
342
  | `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
334
343
  | `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
335
344
  | `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
@@ -143,6 +143,8 @@ clean_series = pipe.run_series(df['text_column'])
143
143
  report = tp.diff_report(raw_texts, clean_batch)
144
144
  ```
145
145
 
146
+ `run_batch` and `run_series` are faster than looping over `run`: the spaCy steps (`lemmatize`, `convert_to_base`, `singularize_words`, `pluralize_words`) process the whole list with spaCy's `nlp.pipe` (about 2x faster in our test; the result is identical). `run_series` keeps the Series index and name.
147
+
146
148
  ### Steps with arguments, saving and scikit-learn
147
149
 
148
150
  ```python
@@ -185,6 +187,7 @@ tp.light_clean("<p>Great &amp; fast!​ Visit https://x.com or mail me@x.com</p>
185
187
  # 'Great & fast! Visit [URL] or mail [EMAIL]'
186
188
 
187
189
  # Use the real tokenizer so the limits are counted in model tokens
190
+ # (pip install "nlp-text-preprocessing[transformers]")
188
191
  from transformers import AutoTokenizer
189
192
  tok = AutoTokenizer.from_pretrained('bert-base-uncased')
190
193
 
@@ -241,6 +244,7 @@ Below is the complete reference guide for all functions available in the library
241
244
  | `capital_ratio(x)` | `x: str` | `float` | Returns proportion of uppercase words relative to total words. |
242
245
  | `unique_word_ratio(x)` | `x: str` | `float` | Returns lexical diversity ratio (unique words / total words). |
243
246
  | `readability_score(x)` | `x: str` | `float` | Calculates Flesch Reading Ease score. |
247
+ | `readability_scores(x)` | `x: str` | `dict` | Flesch Reading Ease, Flesch-Kincaid grade, Gunning Fog, Automated Readability Index and Coleman-Liau (English, approximate). |
244
248
  | `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtag tokens (`#tag`). |
245
249
  | `mentions_count(x)` | `x: str` | `int` | Returns count of user handle mentions (`@user`). |
246
250
  | `numerics_count(x)` | `x: str` | `int` | Returns count of numeric tokens. |
@@ -273,7 +277,7 @@ Below is the complete reference guide for all functions available in the library
273
277
  | `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
274
278
  | `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
275
279
  | `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
276
- | `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
280
+ | `mask_pii(x, token='[REDACTED]', names=False)` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. `names=True` also redacts people's names with spaCy's entity recogniser (statistical, so review the result). |
277
281
  | `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
278
282
  | `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
279
283
  | `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
@@ -1,6 +1,6 @@
1
1
  # entry level code and import text preprocess
2
2
 
3
- __version__ = "0.4.0"
3
+ __version__ = "0.5.0"
4
4
 
5
5
  from .all_text_preprocess import *
6
6
 
@@ -33,6 +33,7 @@ __all__ = [
33
33
  'capital_ratio',
34
34
  'unique_word_ratio',
35
35
  'readability_score',
36
+ 'readability_scores',
36
37
  'text_stats',
37
38
  'text_stats_batch',
38
39
  'to_lower_case',