nlp-text-preprocessing 0.4.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.4.1}/PKG-INFO +4 -1
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/Readme.md +1 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/__init__.py +1 -1
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/all_text_preprocess.py +32 -4
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1/nlp_text_preprocessing.egg-info}/PKG-INFO +4 -1
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/SOURCES.txt +1 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/requires.txt +3 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/pyproject.toml +1 -0
- nlp_text_preprocessing-0.4.1/requirements.txt +15 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_infrastructure.py +1 -1
- nlp_text_preprocessing-0.4.1/tests/test_known_limits.py +38 -0
- nlp_text_preprocessing-0.4.0/requirements.txt +0 -7
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/LICENSE +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/MANIFEST.in +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/architecture_diagram.png +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_banner.png +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/__main__.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/cli.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/data/contractions.json +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/sklearn_compat.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/entry_points.txt +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/setup.cfg +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_all_text_preprocess.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_cleaning.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_light_and_cli.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_linguistic.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_moderation.py +0 -0
- {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_pipeline_api.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nlp-text-preprocessing
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
|
|
5
5
|
Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -39,6 +39,8 @@ Requires-Dist: wordcloud; extra == "viz"
|
|
|
39
39
|
Requires-Dist: matplotlib; extra == "viz"
|
|
40
40
|
Provides-Extra: sklearn
|
|
41
41
|
Requires-Dist: scikit-learn; extra == "sklearn"
|
|
42
|
+
Provides-Extra: transformers
|
|
43
|
+
Requires-Dist: transformers; extra == "transformers"
|
|
42
44
|
Provides-Extra: all
|
|
43
45
|
Requires-Dist: pandas; extra == "all"
|
|
44
46
|
Requires-Dist: wordcloud; extra == "all"
|
|
@@ -241,6 +243,7 @@ tp.light_clean("<p>Great & fast! Visit https://x.com or mail me@x.com</p>
|
|
|
241
243
|
# 'Great & fast! Visit [URL] or mail [EMAIL]'
|
|
242
244
|
|
|
243
245
|
# Use the real tokenizer so the limits are counted in model tokens
|
|
246
|
+
# (pip install "nlp-text-preprocessing[transformers]")
|
|
244
247
|
from transformers import AutoTokenizer
|
|
245
248
|
tok = AutoTokenizer.from_pretrained('bert-base-uncased')
|
|
246
249
|
|
|
@@ -185,6 +185,7 @@ tp.light_clean("<p>Great & fast! Visit https://x.com or mail me@x.com</p>
|
|
|
185
185
|
# 'Great & fast! Visit [URL] or mail [EMAIL]'
|
|
186
186
|
|
|
187
187
|
# Use the real tokenizer so the limits are counted in model tokens
|
|
188
|
+
# (pip install "nlp-text-preprocessing[transformers]")
|
|
188
189
|
from transformers import AutoTokenizer
|
|
189
190
|
tok = AutoTokenizer.from_pretrained('bert-base-uncased')
|
|
190
191
|
|
|
@@ -442,13 +442,38 @@ def upper_case_count(x):
|
|
|
442
442
|
text = _clean_input(x)
|
|
443
443
|
return len([word for word in text.split() if len(word) > 1 and word.isupper()])
|
|
444
444
|
|
|
445
|
+
_NON_TERMINAL_ABBREVIATIONS = frozenset({
|
|
446
|
+
'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', 'jr', 'st', 'mt', 'vs', 'capt', 'col', 'gen', 'gov',
|
|
447
|
+
'lt', 'sgt', 'rev', 'hon', 'fig', 'no', 'inc', 'ltd', 'co', 'corp', 'e.g', 'i.e', 'cf',
|
|
448
|
+
})
|
|
449
|
+
_SENTENCE_END_RE = re.compile(r'(\S*?)([.!?]+)(?=\s|$)')
|
|
450
|
+
|
|
451
|
+
|
|
445
452
|
def sentence_count(x):
|
|
446
|
-
"""
|
|
453
|
+
"""
|
|
454
|
+
Returns the number of sentences in text x.
|
|
455
|
+
|
|
456
|
+
A '.' inside 3.14 or e.g.com does not split, and neither does one after a common abbreviation
|
|
457
|
+
("Dr.", "Mr.", "vs.", "e.g.") or a single-letter initial ("J. K. Rowling") that is followed by more text.
|
|
458
|
+
"""
|
|
447
459
|
text = _clean_input(x)
|
|
448
460
|
if not text.strip():
|
|
449
461
|
return 0
|
|
450
|
-
|
|
451
|
-
|
|
462
|
+
count = 0
|
|
463
|
+
last_end = 0
|
|
464
|
+
for match in _SENTENCE_END_RE.finditer(text):
|
|
465
|
+
word, punct = match.group(1), match.group(2)
|
|
466
|
+
rest = text[match.end():].strip()
|
|
467
|
+
if punct == '.' and rest:
|
|
468
|
+
bare = word.strip('()[]"\'').lower()
|
|
469
|
+
if bare in _NON_TERMINAL_ABBREVIATIONS or (len(bare) == 1 and bare.isalpha()):
|
|
470
|
+
continue
|
|
471
|
+
if text[last_end:match.end()].strip(' .!?'):
|
|
472
|
+
count += 1
|
|
473
|
+
last_end = match.end()
|
|
474
|
+
if text[last_end:].strip(' .!?'):
|
|
475
|
+
count += 1
|
|
476
|
+
return count
|
|
452
477
|
|
|
453
478
|
def avg_sentence_len(x):
|
|
454
479
|
"""Returns average sentence length in words."""
|
|
@@ -737,7 +762,10 @@ def _join_tokens(doc, attr='lemma_', only_pos=None):
|
|
|
737
762
|
"""Rebuilds text from a spaCy doc keeping the original spacing (no "word ." artefacts)."""
|
|
738
763
|
parts = []
|
|
739
764
|
for token in doc:
|
|
740
|
-
|
|
765
|
+
# Pronouns are kept as written: spaCy lemmatizes "me"/"him"/"us" to "I"/"he"/"we".
|
|
766
|
+
if token.pos_ == 'PRON' and only_pos is None:
|
|
767
|
+
parts.append(token.text_with_ws)
|
|
768
|
+
elif only_pos is None or token.pos_ in only_pos:
|
|
741
769
|
parts.append(getattr(token, attr) + token.whitespace_)
|
|
742
770
|
else:
|
|
743
771
|
parts.append(token.text_with_ws)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nlp-text-preprocessing
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
|
|
5
5
|
Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -39,6 +39,8 @@ Requires-Dist: wordcloud; extra == "viz"
|
|
|
39
39
|
Requires-Dist: matplotlib; extra == "viz"
|
|
40
40
|
Provides-Extra: sklearn
|
|
41
41
|
Requires-Dist: scikit-learn; extra == "sklearn"
|
|
42
|
+
Provides-Extra: transformers
|
|
43
|
+
Requires-Dist: transformers; extra == "transformers"
|
|
42
44
|
Provides-Extra: all
|
|
43
45
|
Requires-Dist: pandas; extra == "all"
|
|
44
46
|
Requires-Dist: wordcloud; extra == "all"
|
|
@@ -241,6 +243,7 @@ tp.light_clean("<p>Great & fast! Visit https://x.com or mail me@x.com</p>
|
|
|
241
243
|
# 'Great & fast! Visit [URL] or mail [EMAIL]'
|
|
242
244
|
|
|
243
245
|
# Use the real tokenizer so the limits are counted in model tokens
|
|
246
|
+
# (pip install "nlp-text-preprocessing[transformers]")
|
|
244
247
|
from transformers import AutoTokenizer
|
|
245
248
|
tok = AutoTokenizer.from_pretrained('bert-base-uncased')
|
|
246
249
|
|
|
@@ -44,6 +44,7 @@ dependencies = [
|
|
|
44
44
|
pandas = ["pandas"]
|
|
45
45
|
viz = ["wordcloud", "matplotlib"]
|
|
46
46
|
sklearn = ["scikit-learn"]
|
|
47
|
+
transformers = ["transformers"]
|
|
47
48
|
all = ["pandas", "wordcloud", "matplotlib", "scikit-learn"]
|
|
48
49
|
dev = ["pytest", "build", "twine", "pandas", "wordcloud", "matplotlib", "scikit-learn"]
|
|
49
50
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Core dependencies (the same list as in pyproject.toml)
|
|
2
|
+
spacy
|
|
3
|
+
textblob
|
|
4
|
+
nltk
|
|
5
|
+
beautifulsoup4
|
|
6
|
+
lxml
|
|
7
|
+
emoji>=2.0
|
|
8
|
+
langdetect
|
|
9
|
+
|
|
10
|
+
# Optional extras - install only what you need:
|
|
11
|
+
# pip install "nlp-text-preprocessing[pandas]" pandas
|
|
12
|
+
# pip install "nlp-text-preprocessing[viz]" wordcloud, matplotlib
|
|
13
|
+
# pip install "nlp-text-preprocessing[sklearn]" scikit-learn
|
|
14
|
+
# pip install "nlp-text-preprocessing[transformers]" transformers (tokenizers for truncate_text / chunk_text)
|
|
15
|
+
# pip install "nlp-text-preprocessing[all]" everything above
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Regression tests for limits that were fixed after 0.4.0."""
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
import nlp_text_preprocessing as tp
|
|
5
|
+
|
|
6
|
+
_report = tp.check_resources()
|
|
7
|
+
needs_spacy = pytest.mark.skipif(not _report["spacy_model"], reason="spaCy model not installed")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.mark.parametrize("text, expected", [
|
|
11
|
+
("Hello world. How are you? Fine!", 3),
|
|
12
|
+
("I saw Dr. Smith today. He was fine.", 2),
|
|
13
|
+
("Mr. and Mrs. Jones left.", 1),
|
|
14
|
+
("J. K. Rowling wrote it. Yes.", 2),
|
|
15
|
+
("Fruits, e.g. apples, are good. Eat.", 2),
|
|
16
|
+
("It costs $5 vs. $6. Fine.", 2),
|
|
17
|
+
("Wait... what?! Really.", 3),
|
|
18
|
+
("Pi is 3.14 and that is ok.", 1),
|
|
19
|
+
("Ends with Dr.", 1),
|
|
20
|
+
("No punctuation here", 1),
|
|
21
|
+
("", 0),
|
|
22
|
+
])
|
|
23
|
+
def test_sentence_count_abbreviations(text, expected):
|
|
24
|
+
assert tp.sentence_count(text) == expected
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_avg_sentence_len_uses_new_count():
|
|
28
|
+
assert tp.avg_sentence_len("I saw Dr. Smith today. He was fine.") == pytest.approx(4.0)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@needs_spacy
|
|
32
|
+
def test_lemmatize_keeps_pronouns():
|
|
33
|
+
out = tp.lemmatize("She gave me the books and told them to us.")
|
|
34
|
+
words = out.rstrip(".").split()
|
|
35
|
+
for pronoun in ("me", "them", "us"):
|
|
36
|
+
assert pronoun in words
|
|
37
|
+
assert "book" in out.split()
|
|
38
|
+
assert " I " not in f" {out} "
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/__main__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_all_text_preprocess.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|