nlp-text-preprocessing 0.4.0__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {nlp_text_preprocessing-0.4.0/nlp_text_preprocessing.egg-info → nlp_text_preprocessing-0.4.1}/PKG-INFO +4 -1
  2. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/Readme.md +1 -0
  3. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/__init__.py +1 -1
  4. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/all_text_preprocess.py +32 -4
  5. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1/nlp_text_preprocessing.egg-info}/PKG-INFO +4 -1
  6. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/SOURCES.txt +1 -0
  7. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/requires.txt +3 -0
  8. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/pyproject.toml +1 -0
  9. nlp_text_preprocessing-0.4.1/requirements.txt +15 -0
  10. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_infrastructure.py +1 -1
  11. nlp_text_preprocessing-0.4.1/tests/test_known_limits.py +38 -0
  12. nlp_text_preprocessing-0.4.0/requirements.txt +0 -7
  13. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/LICENSE +0 -0
  14. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/MANIFEST.in +0 -0
  15. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/architecture_diagram.png +0 -0
  16. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_banner.png +0 -0
  17. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/__main__.py +0 -0
  18. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/cli.py +0 -0
  19. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/data/contractions.json +0 -0
  20. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing/sklearn_compat.py +0 -0
  21. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/dependency_links.txt +0 -0
  22. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/entry_points.txt +0 -0
  23. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/nlp_text_preprocessing.egg-info/top_level.txt +0 -0
  24. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/setup.cfg +0 -0
  25. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_all_text_preprocess.py +0 -0
  26. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_cleaning.py +0 -0
  27. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_light_and_cli.py +0 -0
  28. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_linguistic.py +0 -0
  29. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_moderation.py +0 -0
  30. {nlp_text_preprocessing-0.4.0 → nlp_text_preprocessing-0.4.1}/tests/test_pipeline_api.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nlp-text-preprocessing
3
- Version: 0.4.0
3
+ Version: 0.4.1
4
4
  Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
5
5
  Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
6
6
  License-Expression: MIT
@@ -39,6 +39,8 @@ Requires-Dist: wordcloud; extra == "viz"
39
39
  Requires-Dist: matplotlib; extra == "viz"
40
40
  Provides-Extra: sklearn
41
41
  Requires-Dist: scikit-learn; extra == "sklearn"
42
+ Provides-Extra: transformers
43
+ Requires-Dist: transformers; extra == "transformers"
42
44
  Provides-Extra: all
43
45
  Requires-Dist: pandas; extra == "all"
44
46
  Requires-Dist: wordcloud; extra == "all"
@@ -241,6 +243,7 @@ tp.light_clean("<p>Great &amp; fast!​ Visit https://x.com or mail me@x.com</p>
241
243
  # 'Great & fast! Visit [URL] or mail [EMAIL]'
242
244
 
243
245
  # Use the real tokenizer so the limits are counted in model tokens
246
+ # (pip install "nlp-text-preprocessing[transformers]")
244
247
  from transformers import AutoTokenizer
245
248
  tok = AutoTokenizer.from_pretrained('bert-base-uncased')
246
249
 
@@ -185,6 +185,7 @@ tp.light_clean("<p>Great &amp; fast!​ Visit https://x.com or mail me@x.com</p>
185
185
  # 'Great & fast! Visit [URL] or mail [EMAIL]'
186
186
 
187
187
  # Use the real tokenizer so the limits are counted in model tokens
188
+ # (pip install "nlp-text-preprocessing[transformers]")
188
189
  from transformers import AutoTokenizer
189
190
  tok = AutoTokenizer.from_pretrained('bert-base-uncased')
190
191
 
@@ -1,6 +1,6 @@
1
1
  # entry level code and import text preprocess
2
2
 
3
- __version__ = "0.4.0"
3
+ __version__ = "0.4.1"
4
4
 
5
5
  from .all_text_preprocess import *
6
6
 
@@ -442,13 +442,38 @@ def upper_case_count(x):
442
442
  text = _clean_input(x)
443
443
  return len([word for word in text.split() if len(word) > 1 and word.isupper()])
444
444
 
445
+ _NON_TERMINAL_ABBREVIATIONS = frozenset({
446
+ 'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', 'jr', 'st', 'mt', 'vs', 'capt', 'col', 'gen', 'gov',
447
+ 'lt', 'sgt', 'rev', 'hon', 'fig', 'no', 'inc', 'ltd', 'co', 'corp', 'e.g', 'i.e', 'cf',
448
+ })
449
+ _SENTENCE_END_RE = re.compile(r'(\S*?)([.!?]+)(?=\s|$)')
450
+
451
+
445
452
  def sentence_count(x):
446
- """Returns the number of sentences in text x (a '.' inside 3.14 or e.g.com does not split)."""
453
+ """
454
+ Returns the number of sentences in text x.
455
+
456
+ A '.' inside 3.14 or e.g.com does not split, and neither does one after a common abbreviation
457
+ ("Dr.", "Mr.", "vs.", "e.g.") or a single-letter initial ("J. K. Rowling") that is followed by more text.
458
+ """
447
459
  text = _clean_input(x)
448
460
  if not text.strip():
449
461
  return 0
450
- sentences = re.split(r'[.!?]+(?=\s|$)', text)
451
- return len([s for s in sentences if s.strip()])
462
+ count = 0
463
+ last_end = 0
464
+ for match in _SENTENCE_END_RE.finditer(text):
465
+ word, punct = match.group(1), match.group(2)
466
+ rest = text[match.end():].strip()
467
+ if punct == '.' and rest:
468
+ bare = word.strip('()[]"\'').lower()
469
+ if bare in _NON_TERMINAL_ABBREVIATIONS or (len(bare) == 1 and bare.isalpha()):
470
+ continue
471
+ if text[last_end:match.end()].strip(' .!?'):
472
+ count += 1
473
+ last_end = match.end()
474
+ if text[last_end:].strip(' .!?'):
475
+ count += 1
476
+ return count
452
477
 
453
478
  def avg_sentence_len(x):
454
479
  """Returns average sentence length in words."""
@@ -737,7 +762,10 @@ def _join_tokens(doc, attr='lemma_', only_pos=None):
737
762
  """Rebuilds text from a spaCy doc keeping the original spacing (no "word ." artefacts)."""
738
763
  parts = []
739
764
  for token in doc:
740
- if only_pos is None or token.pos_ in only_pos:
765
+ # Pronouns are kept as written: spaCy lemmatizes "me"/"him"/"us" to "I"/"he"/"we".
766
+ if token.pos_ == 'PRON' and only_pos is None:
767
+ parts.append(token.text_with_ws)
768
+ elif only_pos is None or token.pos_ in only_pos:
741
769
  parts.append(getattr(token, attr) + token.whitespace_)
742
770
  else:
743
771
  parts.append(token.text_with_ws)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nlp-text-preprocessing
3
- Version: 0.4.0
3
+ Version: 0.4.1
4
4
  Summary: Text cleaning, feature extraction and linguistic utilities for NLP and ML pipelines
5
5
  Author-email: Uditya Narayan Tiwari <tiwarimerit@gmail.com>
6
6
  License-Expression: MIT
@@ -39,6 +39,8 @@ Requires-Dist: wordcloud; extra == "viz"
39
39
  Requires-Dist: matplotlib; extra == "viz"
40
40
  Provides-Extra: sklearn
41
41
  Requires-Dist: scikit-learn; extra == "sklearn"
42
+ Provides-Extra: transformers
43
+ Requires-Dist: transformers; extra == "transformers"
42
44
  Provides-Extra: all
43
45
  Requires-Dist: pandas; extra == "all"
44
46
  Requires-Dist: wordcloud; extra == "all"
@@ -241,6 +243,7 @@ tp.light_clean("<p>Great &amp; fast!​ Visit https://x.com or mail me@x.com</p>
241
243
  # 'Great & fast! Visit [URL] or mail [EMAIL]'
242
244
 
243
245
  # Use the real tokenizer so the limits are counted in model tokens
246
+ # (pip install "nlp-text-preprocessing[transformers]")
244
247
  from transformers import AutoTokenizer
245
248
  tok = AutoTokenizer.from_pretrained('bert-base-uncased')
246
249
 
@@ -20,6 +20,7 @@ nlp_text_preprocessing/data/contractions.json
20
20
  tests/test_all_text_preprocess.py
21
21
  tests/test_cleaning.py
22
22
  tests/test_infrastructure.py
23
+ tests/test_known_limits.py
23
24
  tests/test_light_and_cli.py
24
25
  tests/test_linguistic.py
25
26
  tests/test_moderation.py
@@ -27,6 +27,9 @@ pandas
27
27
  [sklearn]
28
28
  scikit-learn
29
29
 
30
+ [transformers]
31
+ transformers
32
+
30
33
  [viz]
31
34
  wordcloud
32
35
  matplotlib
@@ -44,6 +44,7 @@ dependencies = [
44
44
  pandas = ["pandas"]
45
45
  viz = ["wordcloud", "matplotlib"]
46
46
  sklearn = ["scikit-learn"]
47
+ transformers = ["transformers"]
47
48
  all = ["pandas", "wordcloud", "matplotlib", "scikit-learn"]
48
49
  dev = ["pytest", "build", "twine", "pandas", "wordcloud", "matplotlib", "scikit-learn"]
49
50
 
@@ -0,0 +1,15 @@
1
+ # Core dependencies (the same list as in pyproject.toml)
2
+ spacy
3
+ textblob
4
+ nltk
5
+ beautifulsoup4
6
+ lxml
7
+ emoji>=2.0
8
+ langdetect
9
+
10
+ # Optional extras - install only what you need:
11
+ # pip install "nlp-text-preprocessing[pandas]" pandas
12
+ # pip install "nlp-text-preprocessing[viz]" wordcloud, matplotlib
13
+ # pip install "nlp-text-preprocessing[sklearn]" scikit-learn
14
+ # pip install "nlp-text-preprocessing[transformers]" transformers (tokenizers for truncate_text / chunk_text)
15
+ # pip install "nlp-text-preprocessing[all]" everything above
@@ -27,7 +27,7 @@ def test_import_is_light_and_silent():
27
27
 
28
28
 
29
29
  def test_version_exposed():
30
- assert tp.__version__ == "0.4.0"
30
+ assert tp.__version__ == "0.4.1"
31
31
 
32
32
 
33
33
  def test_nltk_zip_only_resource_is_detected(monkeypatch):
@@ -0,0 +1,38 @@
1
+ """Regression tests for limits that were fixed after 0.4.0."""
2
+ import pytest
3
+
4
+ import nlp_text_preprocessing as tp
5
+
6
+ _report = tp.check_resources()
7
+ needs_spacy = pytest.mark.skipif(not _report["spacy_model"], reason="spaCy model not installed")
8
+
9
+
10
+ @pytest.mark.parametrize("text, expected", [
11
+ ("Hello world. How are you? Fine!", 3),
12
+ ("I saw Dr. Smith today. He was fine.", 2),
13
+ ("Mr. and Mrs. Jones left.", 1),
14
+ ("J. K. Rowling wrote it. Yes.", 2),
15
+ ("Fruits, e.g. apples, are good. Eat.", 2),
16
+ ("It costs $5 vs. $6. Fine.", 2),
17
+ ("Wait... what?! Really.", 3),
18
+ ("Pi is 3.14 and that is ok.", 1),
19
+ ("Ends with Dr.", 1),
20
+ ("No punctuation here", 1),
21
+ ("", 0),
22
+ ])
23
+ def test_sentence_count_abbreviations(text, expected):
24
+ assert tp.sentence_count(text) == expected
25
+
26
+
27
+ def test_avg_sentence_len_uses_new_count():
28
+ assert tp.avg_sentence_len("I saw Dr. Smith today. He was fine.") == pytest.approx(4.0)
29
+
30
+
31
+ @needs_spacy
32
+ def test_lemmatize_keeps_pronouns():
33
+ out = tp.lemmatize("She gave me the books and told them to us.")
34
+ words = out.rstrip(".").split()
35
+ for pronoun in ("me", "them", "us"):
36
+ assert pronoun in words
37
+ assert "book" in out.split()
38
+ assert " I " not in f" {out} "
@@ -1,7 +0,0 @@
1
- spacy
2
- textblob
3
- nltk
4
- beautifulsoup4
5
- lxml
6
- emoji>=2.0
7
- langdetect