nlp-text-preprocessing 0.4.1__tar.gz → 0.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/PKG-INFO +107 -3
- nlp_text_preprocessing-0.4.1/nlp_text_preprocessing.egg-info/PKG-INFO → nlp_text_preprocessing-0.5.1/Readme.md +536 -493
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing/__init__.py +2 -1
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing/all_text_preprocess.py +1629 -1503
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing/cli.py +5 -2
- nlp_text_preprocessing-0.5.1/nlp_text_preprocessing/py.typed +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing/sklearn_compat.py +8 -4
- nlp_text_preprocessing-0.4.1/Readme.md → nlp_text_preprocessing-0.5.1/nlp_text_preprocessing.egg-info/PKG-INFO +597 -435
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing.egg-info/SOURCES.txt +3 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing.egg-info/requires.txt +3 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/pyproject.toml +25 -2
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/tests/test_all_text_preprocess.py +1 -23
- nlp_text_preprocessing-0.5.1/tests/test_batching.py +91 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/tests/test_infrastructure.py +1 -1
- nlp_text_preprocessing-0.5.1/tests/test_new_features.py +64 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/LICENSE +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/MANIFEST.in +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/architecture_diagram.png +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_banner.png +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing/__main__.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing.egg-info/entry_points.txt +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/requirements.txt +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/setup.cfg +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/tests/test_cleaning.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/tests/test_known_limits.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/tests/test_light_and_cli.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/tests/test_linguistic.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/tests/test_moderation.py +0 -0
- {nlp_text_preprocessing-0.4.1 → nlp_text_preprocessing-0.5.1}/tests/test_pipeline_api.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nlp-text-preprocessing
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
|
|
5
5
|
Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -48,6 +48,9 @@ Requires-Dist: matplotlib; extra == "all"
|
|
|
48
48
|
Requires-Dist: scikit-learn; extra == "all"
|
|
49
49
|
Provides-Extra: dev
|
|
50
50
|
Requires-Dist: pytest; extra == "dev"
|
|
51
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
52
|
+
Requires-Dist: ruff; extra == "dev"
|
|
53
|
+
Requires-Dist: mypy; extra == "dev"
|
|
51
54
|
Requires-Dist: build; extra == "dev"
|
|
52
55
|
Requires-Dist: twine; extra == "dev"
|
|
53
56
|
Requires-Dist: pandas; extra == "dev"
|
|
@@ -85,6 +88,7 @@ Dynamic: license-file
|
|
|
85
88
|
## Table of Contents
|
|
86
89
|
|
|
87
90
|
- [Package Overview](#package-overview)
|
|
91
|
+
- [What's New in 0.5.0](#whats-new-in-050)
|
|
88
92
|
- [Key Features](#key-features)
|
|
89
93
|
- [Architecture Diagram](#architecture-diagram)
|
|
90
94
|
- [Quickstart and Setup](#quickstart-and-setup)
|
|
@@ -103,6 +107,11 @@ Dynamic: license-file
|
|
|
103
107
|
- [Pandas DataFrame Integration](#3-pandas-dataframe-integration)
|
|
104
108
|
- [Batch Feature Extraction](#4-batch-feature-extraction)
|
|
105
109
|
- [WordCloud Generation](#5-wordcloud-generation)
|
|
110
|
+
- [Readability Scores](#6-readability-scores)
|
|
111
|
+
- [Masking Personal Data, Including Names](#7-masking-personal-data-including-names)
|
|
112
|
+
- [Fast Batch Processing](#8-fast-batch-processing)
|
|
113
|
+
- [Type Hints](#type-hints)
|
|
114
|
+
- [Development](#development)
|
|
106
115
|
- [Troubleshooting and Resource Setup](#troubleshooting-and-resource-setup)
|
|
107
116
|
- [License and Author](#license-and-author)
|
|
108
117
|
|
|
@@ -118,16 +127,30 @@ It provides over **60 single-purpose functions** and an extensible **Pipeline En
|
|
|
118
127
|
|
|
119
128
|
---
|
|
120
129
|
|
|
130
|
+
## What's New in 0.5.0
|
|
131
|
+
|
|
132
|
+
- **`readability_scores(x)`**: five readability measures in one dictionary (Flesch Reading Ease, Flesch-Kincaid grade, Gunning Fog, ARI, Coleman-Liau). See [Readability Scores](#6-readability-scores).
|
|
133
|
+
- **`mask_pii(x, names=True)`**: also redacts people's names with spaCy's entity recogniser. Off by default. See [Masking Personal Data](#7-masking-personal-data-including-names).
|
|
134
|
+
- **Faster batches**: `Pipeline.run_batch`, `transform` and `run_series` process the spaCy steps with `nlp.pipe` (about 2x faster, identical output).
|
|
135
|
+
- **Type hints** on the public API and a `py.typed` marker, so mypy, Pyright and IDEs use them. See [Type Hints](#type-hints).
|
|
136
|
+
- **Developer tooling**: ruff, mypy, coverage, Python 3.13 in CI and Dependabot. See [Development](#development).
|
|
137
|
+
|
|
138
|
+
Everything from earlier versions still works the same way. The full history is in [CHANGELOG.md](CHANGELOG.md).
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
121
142
|
## Key Features
|
|
122
143
|
|
|
123
144
|
- **Safe Inputs**: `None`, `NaN`, `pd.NA` and bytes are handled by every function (treated as empty text).
|
|
124
145
|
- **Pipeline Automation Engine**: Chain cleaning steps by name or custom functions via `Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])`.
|
|
125
146
|
- **Light Import, Lazy Models**: `import nlp_text_preprocessing` loads no heavy library and downloads nothing. spaCy, TextBlob and NLTK data are loaded on first use and cached.
|
|
126
147
|
- **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
|
|
127
|
-
- **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
|
|
148
|
+
- **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease, plus Flesch-Kincaid, Gunning Fog, ARI and Coleman-Liau via `readability_scores`), punctuation density, capital ratios, and full batch statistics.
|
|
128
149
|
- **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
|
|
129
150
|
- **Safety & Moderation (baseline)**: Lexicon-based profanity, toxicity, spam and emotion helpers. They are transparent keyword baselines, not a moderation system: use a trained model for anything high-stakes.
|
|
130
151
|
- **Visualization**: Generate and save custom WordClouds to disk or display interactively.
|
|
152
|
+
- **Fast Batches**: `Pipeline.run_batch` / `run_series` use spaCy's `nlp.pipe` for the spaCy steps.
|
|
153
|
+
- **Typed**: annotated public API with a `py.typed` marker.
|
|
131
154
|
|
|
132
155
|
---
|
|
133
156
|
|
|
@@ -150,7 +173,9 @@ pip install nlp-text-preprocessing # core
|
|
|
150
173
|
pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
|
|
151
174
|
pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
|
|
152
175
|
pip install "nlp-text-preprocessing[sklearn]" # + scikit-learn (TextPreprocessor)
|
|
176
|
+
pip install "nlp-text-preprocessing[transformers]" # + transformers (tokenizers for truncate_text / chunk_text)
|
|
153
177
|
pip install "nlp-text-preprocessing[all]" # everything optional
|
|
178
|
+
pip install "nlp-text-preprocessing[dev]" # tests, ruff, mypy (for contributors)
|
|
154
179
|
```
|
|
155
180
|
|
|
156
181
|
### 2. Downloading Required Corpora and Models
|
|
@@ -201,6 +226,8 @@ clean_series = pipe.run_series(df['text_column'])
|
|
|
201
226
|
report = tp.diff_report(raw_texts, clean_batch)
|
|
202
227
|
```
|
|
203
228
|
|
|
229
|
+
`run_batch` and `run_series` are faster than looping over `run`: the spaCy steps (`lemmatize`, `convert_to_base`, `singularize_words`, `pluralize_words`) process the whole list with spaCy's `nlp.pipe` (about 2x faster in our test; the result is identical). `run_series` keeps the Series index and name.
|
|
230
|
+
|
|
204
231
|
### Steps with arguments, saving and scikit-learn
|
|
205
232
|
|
|
206
233
|
```python
|
|
@@ -300,6 +327,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
300
327
|
| `capital_ratio(x)` | `x: str` | `float` | Returns proportion of uppercase words relative to total words. |
|
|
301
328
|
| `unique_word_ratio(x)` | `x: str` | `float` | Returns lexical diversity ratio (unique words / total words). |
|
|
302
329
|
| `readability_score(x)` | `x: str` | `float` | Calculates Flesch Reading Ease score. |
|
|
330
|
+
| `readability_scores(x)` | `x: str` | `dict` | Flesch Reading Ease, Flesch-Kincaid grade, Gunning Fog, Automated Readability Index and Coleman-Liau (English, approximate). |
|
|
303
331
|
| `hashtags_count(x)` | `x: str` | `int` | Returns count of hashtag tokens (`#tag`). |
|
|
304
332
|
| `mentions_count(x)` | `x: str` | `int` | Returns count of user handle mentions (`@user`). |
|
|
305
333
|
| `numerics_count(x)` | `x: str` | `int` | Returns count of numeric tokens. |
|
|
@@ -332,7 +360,7 @@ Below is the complete reference guide for all functions available in the library
|
|
|
332
360
|
| `emoji_to_text(x)` | `x: str` | `str` | Converts emojis to descriptive text (e.g. 😂 -> `face_with_tears_of_joy`). |
|
|
333
361
|
| `remove_phone_numbers(x)` | `x: str` | `str` | Strips phone numbers (international formats; dates and decimals are ignored). |
|
|
334
362
|
| `count_phone_numbers(x)` | `x: str` | `int` | Counts phone numbers present in text. |
|
|
335
|
-
| `mask_pii(x, token='[REDACTED]')` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. |
|
|
363
|
+
| `mask_pii(x, token='[REDACTED]', names=False)` | `x: str` | `str` | Redacts emails, card numbers, US SSNs and phone numbers. `names=True` also redacts people's names with spaCy's entity recogniser (statistical, so review the result). |
|
|
336
364
|
| `remove_extra_whitespace(x)`| `x: str` | `str` | Collapses consecutive spaces, tabs, and newlines to a single space. |
|
|
337
365
|
| `expand_hashtags(x)` | `x: str` | `str` | Splits hashtags (`#MachineLearning2024` -> `Machine Learning 2024`). |
|
|
338
366
|
| `normalize_unicode_punctuation(x)`| `x: str` | `str` | Replaces smart quotes, em-dashes, and ellipsis with ASCII equivalents. |
|
|
@@ -468,6 +496,82 @@ text = "python natural language processing machine learning text mining data sci
|
|
|
468
496
|
tp.get_wordcloud(text, save_path="wordcloud.png", show=False)
|
|
469
497
|
```
|
|
470
498
|
|
|
499
|
+
### 6. Readability Scores
|
|
500
|
+
|
|
501
|
+
```python
|
|
502
|
+
import nlp_text_preprocessing as tp
|
|
503
|
+
|
|
504
|
+
scores = tp.readability_scores("The cat sat on the mat. It was a sunny day.")
|
|
505
|
+
print(scores)
|
|
506
|
+
# {'flesch_reading_ease': ..., 'flesch_kincaid_grade': ..., 'gunning_fog': ...,
|
|
507
|
+
# 'automated_readability_index': ..., 'coleman_liau': ...}
|
|
508
|
+
|
|
509
|
+
# Higher Flesch = easier. The other four are US school grade levels (higher = harder).
|
|
510
|
+
# Empty or None text returns 0.0 for every key.
|
|
511
|
+
```
|
|
512
|
+
|
|
513
|
+
`tp.readability_score(x)` still returns only the Flesch Reading Ease number. The scores are approximate and meant for English.
|
|
514
|
+
|
|
515
|
+
### 7. Masking Personal Data, Including Names
|
|
516
|
+
|
|
517
|
+
```python
|
|
518
|
+
import nlp_text_preprocessing as tp
|
|
519
|
+
|
|
520
|
+
text = "Barack Obama emailed me at obama@example.com or 555-123-4567."
|
|
521
|
+
|
|
522
|
+
tp.mask_pii(text)
|
|
523
|
+
# 'Barack Obama emailed me at [REDACTED] or [REDACTED].' (names are left alone by default)
|
|
524
|
+
|
|
525
|
+
tp.mask_pii(text, names=True)
|
|
526
|
+
# '[REDACTED] emailed me at [REDACTED] or [REDACTED].' (needs the spaCy model; run tp.download_nltk_packages() once)
|
|
527
|
+
|
|
528
|
+
tp.mask_pii(text, token='<PII>', names=True) # custom replacement token
|
|
529
|
+
```
|
|
530
|
+
|
|
531
|
+
Names are found by spaCy's statistical entity recogniser, so it can miss or over-match some names. Review the output when privacy matters.
|
|
532
|
+
|
|
533
|
+
### 8. Fast Batch Processing
|
|
534
|
+
|
|
535
|
+
```python
|
|
536
|
+
import nlp_text_preprocessing as tp
|
|
537
|
+
|
|
538
|
+
pipe = tp.Pipeline(['to_lower_case', 'remove_urls', 'lemmatize'])
|
|
539
|
+
|
|
540
|
+
texts = ["Dogs are barking at https://example.com", "Cats were running"] * 500
|
|
541
|
+
clean = pipe.run_batch(texts) # list in, list out
|
|
542
|
+
clean_series = pipe.run_series(df['text']) # keeps the Series index and name
|
|
543
|
+
```
|
|
544
|
+
|
|
545
|
+
---
|
|
546
|
+
|
|
547
|
+
## Type Hints
|
|
548
|
+
|
|
549
|
+
The package ships a `py.typed` marker and annotated function signatures, so type checkers and IDEs show parameter and return types:
|
|
550
|
+
|
|
551
|
+
```python
|
|
552
|
+
import nlp_text_preprocessing as tp
|
|
553
|
+
|
|
554
|
+
n: int = tp.word_count("hello world")
|
|
555
|
+
scores: dict = tp.readability_scores("Hello there.")
|
|
556
|
+
```
|
|
557
|
+
|
|
558
|
+
---
|
|
559
|
+
|
|
560
|
+
## Development
|
|
561
|
+
|
|
562
|
+
```bash
|
|
563
|
+
git clone https://github.com/udityamerit/Text-Processing-Package-For-Natural-Language-Processing.git
|
|
564
|
+
cd Text-Processing-Package-For-Natural-Language-Processing
|
|
565
|
+
pip install -e ".[dev,all]"
|
|
566
|
+
python -m spacy download en_core_web_sm
|
|
567
|
+
|
|
568
|
+
pytest -q --cov # tests with coverage
|
|
569
|
+
ruff check . # lint
|
|
570
|
+
mypy # type check
|
|
571
|
+
```
|
|
572
|
+
|
|
573
|
+
CI runs the same checks on Python 3.10 to 3.13. Release steps, and how to set up PyPI Trusted Publishing, are in [RELEASING.md](RELEASING.md).
|
|
574
|
+
|
|
471
575
|
---
|
|
472
576
|
|
|
473
577
|
## Troubleshooting and Resource Setup
|