nlp-text-preprocessing 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. {nlp_text_preprocessing-0.3.0/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.4.0}/PKG-INFO +96 -7
  2. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/Readme.md +91 -6
  3. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing/__init__.py +14 -1
  4. nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/__main__.py +5 -0
  5. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing/all_text_preprocess.py +431 -47
  6. nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/cli.py +150 -0
  7. nlp_text_preprocessing-0.4.0/nlp_text_preprocessing/sklearn_compat.py +44 -0
  8. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info}/PKG-INFO +96 -7
  9. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/SOURCES.txt +8 -1
  10. nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info/entry_points.txt +2 -0
  11. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/requires.txt +5 -0
  12. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/pyproject.toml +6 -2
  13. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/tests/test_infrastructure.py +1 -1
  14. nlp_text_preprocessing-0.4.0/tests/test_light_and_cli.py +195 -0
  15. nlp_text_preprocessing-0.4.0/tests/test_moderation.py +102 -0
  16. nlp_text_preprocessing-0.4.0/tests/test_pipeline_api.py +100 -0
  17. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/LICENSE +0 -0
  18. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/MANIFEST.in +0 -0
  19. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/architecture_diagram.png +0 -0
  20. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_banner.png +0 -0
  21. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing/data/contractions.json +0 -0
  22. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
  23. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
  24. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/requirements.txt +0 -0
  25. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/setup.cfg +0 -0
  26. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/tests/test_all_text_preprocess.py +0 -0
  27. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/tests/test_cleaning.py +0 -0
  28. {nlp_text_preprocessing-0.3.0 → nlp_text_preprocessing-0.4.0}/tests/test_linguistic.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nlp-text-preprocessing
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
5
5
  Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
6
6
  License-Expression: MIT
@@ -37,10 +37,13 @@ Requires-Dist: pandas; extra == "pandas"
37
37
  Provides-Extra: viz
38
38
  Requires-Dist: wordcloud; extra == "viz"
39
39
  Requires-Dist: matplotlib; extra == "viz"
40
+ Provides-Extra: sklearn
41
+ Requires-Dist: scikit-learn; extra == "sklearn"
40
42
  Provides-Extra: all
41
43
  Requires-Dist: pandas; extra == "all"
42
44
  Requires-Dist: wordcloud; extra == "all"
43
45
  Requires-Dist: matplotlib; extra == "all"
46
+ Requires-Dist: scikit-learn; extra == "all"
44
47
  Provides-Extra: dev
45
48
  Requires-Dist: pytest; extra == "dev"
46
49
  Requires-Dist: build; extra == "dev"
@@ -48,6 +51,7 @@ Requires-Dist: twine; extra == "dev"
48
51
  Requires-Dist: pandas; extra == "dev"
49
52
  Requires-Dist: wordcloud; extra == "dev"
50
53
  Requires-Dist: matplotlib; extra == "dev"
54
+ Requires-Dist: scikit-learn; extra == "dev"
51
55
  Dynamic: license-file
52
56
 
53
57
  # NLP Text Preprocessing Utility Package
@@ -83,6 +87,8 @@ Dynamic: license-file
83
87
  - [Architecture Diagram](#architecture-diagram)
84
88
  - [Quickstart and Setup](#quickstart-and-setup)
85
89
  - [Pipeline Automation Engine](#pipeline-automation-engine)
90
+ - [Preparing Text for Transformers](#preparing-text-for-transformers-bert-gpt-)
91
+ - [Command Line (`nlp-prep`)](#command-line-nlp-prep)
86
92
  - [Complete API Reference](#complete-api-reference)
87
93
  - [1. General Feature Extraction](#1-general-feature-extraction)
88
94
  - [2. Text Cleaning and Normalization](#2-text-cleaning-and-normalization)
@@ -118,7 +124,7 @@ It provides over **60 single-purpose functions** and an extensible **Pipeline En
118
124
  - **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
119
125
  - **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
120
126
  - **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
121
- - **Safety & Moderation**: Profanity detection/censoring, toxicity scores, and spam signal detection.
127
+ - **Safety & Moderation (baseline)**: Lexicon-based profanity, toxicity, spam and emotion helpers. They are transparent keyword baselines, not a moderation system: use a trained model for anything high-stakes.
122
128
  - **Visualization**: Generate and save custom WordClouds to disk or display interactively.
123
129
 
124
130
  ---
@@ -141,6 +147,7 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
141
147
  pip install nlp-text-preprocessing # core
142
148
  pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
143
149
  pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
150
+ pip install "nlp-text-preprocessing[sklearn]" # + scikit-learn (TextPreprocessor)
144
151
  pip install "nlp-text-preprocessing[all]" # everything optional
145
152
  ```
146
153
 
@@ -192,6 +199,84 @@ clean_series = pipe.run_series(df['text_column'])
192
199
  report = tp.diff_report(raw_texts, clean_batch)
193
200
  ```
194
201
 
202
+ ### Steps with arguments, saving and scikit-learn
203
+
204
+ ```python
205
+ # Pass arguments as (name, {argument: value})
206
+ pipe = tp.Pipeline([
207
+ 'to_lower_case',
208
+ ('remove_stop_words', {'keep_negations': False}),
209
+ ])
210
+ print(pipe) # Pipeline(['to_lower_case', "remove_stop_words{'keep_negations': False}"])
211
+
212
+ # Save and reload (custom functions must be registered with @tp.register_step('name'))
213
+ pipe.save('my_pipeline.json')
214
+ pipe = tp.Pipeline.load('my_pipeline.json')
215
+
216
+ # Use it inside scikit-learn: pip install "nlp-text-preprocessing[sklearn]"
217
+ from sklearn.pipeline import make_pipeline
218
+ from sklearn.feature_extraction.text import TfidfVectorizer
219
+
220
+ model = make_pipeline(tp.TextPreprocessor(['to_lower_case', 'remove_special_chars']),
221
+ TfidfVectorizer())
222
+ X = model.fit_transform(["Hello, World!", "HELLO there"])
223
+ ```
224
+
225
+ Every step except the last must return a string. The last step may return anything,
226
+ for example `Pipeline(['to_lower_case', 'text_stats'])`.
227
+
228
+ ---
229
+
230
+ ## Preparing Text for Transformers (BERT, GPT, ...)
231
+
232
+ Modern models work best with *lightly* cleaned text: keep the case, punctuation and
233
+ word forms, and only remove noise. `light_clean` does exactly that, and `truncate_text`
234
+ / `chunk_text` help you stay inside a model's token limit.
235
+
236
+ ```python
237
+ import nlp_text_preprocessing as tp
238
+
239
+ tp.light_clean("<p>Great &amp; fast!​ Visit https://x.com or mail me@x.com</p>",
240
+ urls='[URL]', emails='[EMAIL]')
241
+ # 'Great & fast! Visit [URL] or mail [EMAIL]'
242
+
243
+ # Use the real tokenizer so the limits are counted in model tokens
244
+ from transformers import AutoTokenizer
245
+ tok = AutoTokenizer.from_pretrained('bert-base-uncased')
246
+
247
+ tp.truncate_text(long_text, 256, tokenizer=tok)
248
+ chunks = tp.chunk_text(long_text, 256, overlap=32, tokenizer=tok)
249
+ ```
250
+
251
+ Without a `tokenizer`, the limits are counted in whitespace-separated words.
252
+ `tokenizer` may be a HuggingFace tokenizer (anything with `.tokenize()` and
253
+ `.convert_tokens_to_string()`) or a plain function that returns a list of tokens.
254
+
255
+ ---
256
+
257
+ ## Command Line (`nlp-prep`)
258
+
259
+ Installing the package also installs the `nlp-prep` command (`python -m nlp_text_preprocessing` works too).
260
+
261
+ ```bash
262
+ # one document per line, written to a new file
263
+ nlp-prep clean reviews.txt -o reviews_clean.txt
264
+
265
+ # a CSV column (adds a 'text_clean' column; use --output-column text to overwrite)
266
+ nlp-prep clean data.csv --column text -o data_clean.csv
267
+
268
+ # standard input / output
269
+ cat notes.txt | nlp-prep clean - --lowercase --url-token "[URL]"
270
+
271
+ # classic cleaning, or your own steps / a saved Pipeline
272
+ nlp-prep clean reviews.txt --mode standard --stopwords
273
+ nlp-prep clean reviews.txt --steps to_lower_case,remove_urls
274
+ nlp-prep clean reviews.txt --pipeline my_pipeline.json
275
+
276
+ # check or install the NLTK data and spaCy model
277
+ nlp-prep resources --download
278
+ ```
279
+
195
280
  ---
196
281
 
197
282
  ## Complete API Reference
@@ -252,6 +337,9 @@ Below is the complete reference guide for all functions available in the library
252
337
  | `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
253
338
  | `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
254
339
  | `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
340
+ | `light_clean(x, html=True, urls='', emails='', mentions=None, emojis=None, normalize='NFKC', keep_newlines=False, lowercase=False)` | `x: str` | `str` | Minimal cleaning for transformer models: Unicode normalisation, zero-width/control characters, HTML and entities, URLs/emails/mentions (remove, replace or keep), optional emoji handling. Case and punctuation are kept. |
341
+ | `truncate_text(x, max_tokens, tokenizer=None)` | `x: str` | `str` | Keeps the first `max_tokens` tokens (words, or tokens of a HuggingFace-style tokenizer). |
342
+ | `chunk_text(x, max_tokens, overlap=0, tokenizer=None)` | `x: str` | `list[str]` | Splits long text into pieces of at most `max_tokens`, with optional overlap. |
255
343
 
256
344
  ---
257
345
 
@@ -282,13 +370,14 @@ Below is the complete reference guide for all functions available in the library
282
370
 
283
371
  | Function | Parameters | Return Type | Description |
284
372
  | :--- | :--- | :--- | :--- |
285
- | `detect_profanity(x)` | `x: str` | `bool` | Returns True if text contains profanity words. |
286
- | `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
287
- | `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
288
- | `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
373
+ | `detect_profanity(x, include_mild=True, extra_words=(), allow_words=())` | `x: str` | `bool` | True if the text has profanity. Matches endings (`shitty`) and leetspeak (`sh1t`); ignores `Hell's Kitchen`; `include_mild=False` skips words like `damn`. Masked words such as `f***` are not detected. |
374
+ | `censor_profanity(x, include_mild=True, extra_words=(), allow_words=(), mask_char='*')` | `x: str` | `str` | Masks each profane word, keeping its length and the text layout. |
375
+ | `detect_toxicity(x)` | `x: str` | `float` | Score from 0.0 to 1.0. Counts insults and threats aimed at people; `kill the process` and `I do not hate you` score 0. |
376
+ | `toxicity_report(x)` | `x: str` | `dict` | Explains the score: `score`, `matches` (word, weight, targeted) and `ignored` (negated) words. |
377
+ | `detect_spam_signals(x)` | `x: str` | `dict` | Flags shouting, punctuation runs, links and spam phrases (`spam_phrases`); `is_spam_candidate` needs at least two signals. Short texts like `OK` are not flagged. |
289
378
  | `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
290
379
  | `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
291
- | `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
380
+ | `emotion_detection(x)` | `x: str` | `dict` | Emotion counts (joy, anger, sadness, fear). Handles endings (`loved`, `happier`) and ignores negated words (`not happy`). |
292
381
 
293
382
  ---
294
383
 
@@ -31,6 +31,8 @@
31
31
  - [Architecture Diagram](#architecture-diagram)
32
32
  - [Quickstart and Setup](#quickstart-and-setup)
33
33
  - [Pipeline Automation Engine](#pipeline-automation-engine)
34
+ - [Preparing Text for Transformers](#preparing-text-for-transformers-bert-gpt-)
35
+ - [Command Line (`nlp-prep`)](#command-line-nlp-prep)
34
36
  - [Complete API Reference](#complete-api-reference)
35
37
  - [1. General Feature Extraction](#1-general-feature-extraction)
36
38
  - [2. Text Cleaning and Normalization](#2-text-cleaning-and-normalization)
@@ -66,7 +68,7 @@ It provides over **60 single-purpose functions** and an extensible **Pipeline En
66
68
  - **Advanced Cleaning**: Remove emojis, phone numbers, PII mask, URLs, emails, HTML, retweets, special symbols, and expand contractions/hashtags.
67
69
  - **Feature & Readability Extraction**: Extract word counts, sentence counts, readability scores (Flesch Ease), punctuation density, capital ratios, and full batch statistics.
68
70
  - **Linguistic Utilities**: Named Entity Recognition (NER), POS tagging, dependency parsing, language detection, keyword extraction, synonyms, antonyms, and n-grams.
69
- - **Safety & Moderation**: Profanity detection/censoring, toxicity scores, and spam signal detection.
71
+ - **Safety & Moderation (baseline)**: Lexicon-based profanity, toxicity, spam and emotion helpers. They are transparent keyword baselines, not a moderation system: use a trained model for anything high-stakes.
70
72
  - **Visualization**: Generate and save custom WordClouds to disk or display interactively.
71
73
 
72
74
  ---
@@ -89,6 +91,7 @@ The package integrates **spaCy**, **NLTK**, **TextBlob**, **BeautifulSoup4**, an
89
91
  pip install nlp-text-preprocessing # core
90
92
  pip install "nlp-text-preprocessing[viz]" # + wordcloud, matplotlib (get_wordcloud)
91
93
  pip install "nlp-text-preprocessing[pandas]" # + pandas (text_stats_batch, Pipeline.run_series)
94
+ pip install "nlp-text-preprocessing[sklearn]" # + scikit-learn (TextPreprocessor)
92
95
  pip install "nlp-text-preprocessing[all]" # everything optional
93
96
  ```
94
97
 
@@ -140,6 +143,84 @@ clean_series = pipe.run_series(df['text_column'])
140
143
  report = tp.diff_report(raw_texts, clean_batch)
141
144
  ```
142
145
 
146
+ ### Steps with arguments, saving and scikit-learn
147
+
148
+ ```python
149
+ # Pass arguments as (name, {argument: value})
150
+ pipe = tp.Pipeline([
151
+ 'to_lower_case',
152
+ ('remove_stop_words', {'keep_negations': False}),
153
+ ])
154
+ print(pipe) # Pipeline(['to_lower_case', "remove_stop_words{'keep_negations': False}"])
155
+
156
+ # Save and reload (custom functions must be registered with @tp.register_step('name'))
157
+ pipe.save('my_pipeline.json')
158
+ pipe = tp.Pipeline.load('my_pipeline.json')
159
+
160
+ # Use it inside scikit-learn: pip install "nlp-text-preprocessing[sklearn]"
161
+ from sklearn.pipeline import make_pipeline
162
+ from sklearn.feature_extraction.text import TfidfVectorizer
163
+
164
+ model = make_pipeline(tp.TextPreprocessor(['to_lower_case', 'remove_special_chars']),
165
+ TfidfVectorizer())
166
+ X = model.fit_transform(["Hello, World!", "HELLO there"])
167
+ ```
168
+
169
+ Every step except the last must return a string. The last step may return anything,
170
+ for example `Pipeline(['to_lower_case', 'text_stats'])`.
171
+
172
+ ---
173
+
174
+ ## Preparing Text for Transformers (BERT, GPT, ...)
175
+
176
+ Modern models work best with *lightly* cleaned text: keep the case, punctuation and
177
+ word forms, and only remove noise. `light_clean` does exactly that, and `truncate_text`
178
+ / `chunk_text` help you stay inside a model's token limit.
179
+
180
+ ```python
181
+ import nlp_text_preprocessing as tp
182
+
183
+ tp.light_clean("<p>Great &amp; fast!​ Visit https://x.com or mail me@x.com</p>",
184
+ urls='[URL]', emails='[EMAIL]')
185
+ # 'Great & fast! Visit [URL] or mail [EMAIL]'
186
+
187
+ # Use the real tokenizer so the limits are counted in model tokens
188
+ from transformers import AutoTokenizer
189
+ tok = AutoTokenizer.from_pretrained('bert-base-uncased')
190
+
191
+ tp.truncate_text(long_text, 256, tokenizer=tok)
192
+ chunks = tp.chunk_text(long_text, 256, overlap=32, tokenizer=tok)
193
+ ```
194
+
195
+ Without a `tokenizer`, the limits are counted in whitespace-separated words.
196
+ `tokenizer` may be a HuggingFace tokenizer (anything with `.tokenize()` and
197
+ `.convert_tokens_to_string()`) or a plain function that returns a list of tokens.
198
+
199
+ ---
200
+
201
+ ## Command Line (`nlp-prep`)
202
+
203
+ Installing the package also installs the `nlp-prep` command (`python -m nlp_text_preprocessing` works too).
204
+
205
+ ```bash
206
+ # one document per line, written to a new file
207
+ nlp-prep clean reviews.txt -o reviews_clean.txt
208
+
209
+ # a CSV column (adds a 'text_clean' column; use --output-column text to overwrite)
210
+ nlp-prep clean data.csv --column text -o data_clean.csv
211
+
212
+ # standard input / output
213
+ cat notes.txt | nlp-prep clean - --lowercase --url-token "[URL]"
214
+
215
+ # classic cleaning, or your own steps / a saved Pipeline
216
+ nlp-prep clean reviews.txt --mode standard --stopwords
217
+ nlp-prep clean reviews.txt --steps to_lower_case,remove_urls
218
+ nlp-prep clean reviews.txt --pipeline my_pipeline.json
219
+
220
+ # check or install the NLTK data and spaCy model
221
+ nlp-prep resources --download
222
+ ```
223
+
143
224
  ---
144
225
 
145
226
  ## Complete API Reference
@@ -200,6 +281,9 @@ Below is the complete reference guide for all functions available in the library
200
281
  | `remove_currency_symbols(x)`| `x: str` | `str` | Strips every Unicode currency symbol. |
201
282
  | `normalize_whitespace_and_casing(x)`| `x: str` | `str` | Canonical combination of lowercasing and whitespace collapse. |
202
283
  | `clean_text(text, remove_stopwords=False)` | `text: str` | `str` | End-to-end cleaning (HTML, URLs, emails, lower-case, contractions, punctuation, lemmatization). |
284
+ | `light_clean(x, html=True, urls='', emails='', mentions=None, emojis=None, normalize='NFKC', keep_newlines=False, lowercase=False)` | `x: str` | `str` | Minimal cleaning for transformer models: Unicode normalisation, zero-width/control characters, HTML and entities, URLs/emails/mentions (remove, replace or keep), optional emoji handling. Case and punctuation are kept. |
285
+ | `truncate_text(x, max_tokens, tokenizer=None)` | `x: str` | `str` | Keeps the first `max_tokens` tokens (words, or tokens of a HuggingFace-style tokenizer). |
286
+ | `chunk_text(x, max_tokens, overlap=0, tokenizer=None)` | `x: str` | `list[str]` | Splits long text into pieces of at most `max_tokens`, with optional overlap. |
203
287
 
204
288
  ---
205
289
 
@@ -230,13 +314,14 @@ Below is the complete reference guide for all functions available in the library
230
314
 
231
315
  | Function | Parameters | Return Type | Description |
232
316
  | :--- | :--- | :--- | :--- |
233
- | `detect_profanity(x)` | `x: str` | `bool` | Returns True if text contains profanity words. |
234
- | `censor_profanity(x)` | `x: str` | `str` | Censors profanity words with asterisks (`****`). |
235
- | `detect_toxicity(x)` | `x: str` | `float` | Calculates baseline toxicity score (0.0 to 1.0). |
236
- | `detect_spam_signals(x)` | `x: str` | `dict` | Returns spam flags for excessive caps, punctuation, and URL density. |
317
+ | `detect_profanity(x, include_mild=True, extra_words=(), allow_words=())` | `x: str` | `bool` | True if the text has profanity. Matches endings (`shitty`) and leetspeak (`sh1t`); ignores `Hell's Kitchen`; `include_mild=False` skips words like `damn`. Masked words such as `f***` are not detected. |
318
+ | `censor_profanity(x, include_mild=True, extra_words=(), allow_words=(), mask_char='*')` | `x: str` | `str` | Masks each profane word, keeping its length and the text layout. |
319
+ | `detect_toxicity(x)` | `x: str` | `float` | Score from 0.0 to 1.0. Counts insults and threats aimed at people; `kill the process` and `I do not hate you` score 0. |
320
+ | `toxicity_report(x)` | `x: str` | `dict` | Explains the score: `score`, `matches` (word, weight, targeted) and `ignored` (negated) words. |
321
+ | `detect_spam_signals(x)` | `x: str` | `dict` | Flags shouting, punctuation runs, links and spam phrases (`spam_phrases`); `is_spam_candidate` needs at least two signals. Short texts like `OK` are not flagged. |
237
322
  | `sentiment_analysis(x, method='pattern')` | `x: str` | `str` | `'pos'` / `'neg'` / `'neu'`. Default is instant; `method='naive_bayes'` is the legacy opt-in model. |
238
323
  | `polarity_subjectivity(x)`| `x: str` | `dict` | Returns fast pattern-based sentiment polarity (-1.0 to 1.0) and subjectivity. |
239
- | `emotion_detection(x)` | `x: str` | `dict` | Calculates lexicon-based emotion counts (joy, anger, sadness, fear). |
324
+ | `emotion_detection(x)` | `x: str` | `dict` | Emotion counts (joy, anger, sadness, fear). Handles endings (`loved`, `happier`) and ignores negated words (`not happy`). |
240
325
 
241
326
  ---
242
327
 
@@ -1,9 +1,18 @@
1
1
  # entry level code and import text preprocess
2
2
 
3
- __version__ = "0.3.0"
3
+ __version__ = "0.4.0"
4
4
 
5
5
  from .all_text_preprocess import *
6
6
 
7
+
8
+ def __getattr__(name):
9
+ # Loaded on first access so scikit-learn stays an optional dependency.
10
+ if name == "TextPreprocessor":
11
+ from .sklearn_compat import TextPreprocessor
12
+ return TextPreprocessor
13
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
14
+
15
+
7
16
  __all__ = [
8
17
  '__version__',
9
18
  'download_nltk_packages',
@@ -76,6 +85,10 @@ __all__ = [
76
85
  'sentiment_analysis',
77
86
  'polarity_subjectivity',
78
87
  'emotion_detection',
88
+ 'toxicity_report',
89
+ 'light_clean',
90
+ 'truncate_text',
91
+ 'chunk_text',
79
92
  'Pipeline',
80
93
  'register_step',
81
94
  'diff_report',
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())