fast-sentence-segment 1.4.1__tar.gz → 1.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/PKG-INFO +2 -2
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/README.md +1 -1
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/cli.py +53 -8
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/svc/perform_sentence_segmentation.py +37 -1
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/pyproject.toml +1 -1
- fast_sentence_segment-1.4.3/setup.py +39 -0
- fast_sentence_segment-1.4.1/setup.py +0 -39
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/LICENSE +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/__init__.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/bp/__init__.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/bp/segmenter.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/core/__init__.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/core/base_object.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/core/stopwatch.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/__init__.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/abbreviation_merger.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/abbreviation_splitter.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/abbreviations.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/bullet_point_cleaner.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/dehyphenator.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/ellipsis_normalizer.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/group_quoted_sentences.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/newlines_to_periods.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/normalize_quotes.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/numbered_list_normalizer.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/post_process_sentences.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/question_exclamation_splitter.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/spacy_doc_segmenter.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/strip_trailing_period_after_quote.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/title_name_merger.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/svc/__init__.py +0 -0
- {fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/svc/perform_paragraph_segmentation.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: fast-sentence-segment
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.3
|
|
4
4
|
Summary: Fast and Efficient Sentence Segmentation
|
|
5
5
|
Home-page: https://github.com/craigtrim/fast-sentence-segment
|
|
6
6
|
License: MIT
|
|
@@ -33,7 +33,7 @@ Description-Content-Type: text/markdown
|
|
|
33
33
|
|
|
34
34
|
[](https://pypi.org/project/fast-sentence-segment/)
|
|
35
35
|
[](https://pypi.org/project/fast-sentence-segment/)
|
|
36
|
-
[](https://github.com/craigtrim/fast-sentence-segment/tree/master/tests)
|
|
37
37
|
[](https://opensource.org/licenses/MIT)
|
|
38
38
|
[](https://github.com/astral-sh/ruff)
|
|
39
39
|
[](https://pepy.tech/project/fast-sentence-segment)
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://pypi.org/project/fast-sentence-segment/)
|
|
4
4
|
[](https://pypi.org/project/fast-sentence-segment/)
|
|
5
|
-
[](https://github.com/craigtrim/fast-sentence-segment/tree/master/tests)
|
|
6
6
|
[](https://opensource.org/licenses/MIT)
|
|
7
7
|
[](https://github.com/astral-sh/ruff)
|
|
8
8
|
[](https://pepy.tech/project/fast-sentence-segment)
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
"""CLI for fast-sentence-segment."""
|
|
3
3
|
|
|
4
4
|
import argparse
|
|
5
|
+
import itertools
|
|
5
6
|
import logging
|
|
6
7
|
import os
|
|
7
8
|
import sys
|
|
9
|
+
import threading
|
|
8
10
|
import time
|
|
9
11
|
|
|
10
12
|
from fast_sentence_segment import segment_text
|
|
@@ -21,6 +23,34 @@ YELLOW = "\033[33m"
|
|
|
21
23
|
RESET = "\033[0m"
|
|
22
24
|
|
|
23
25
|
|
|
26
|
+
class Spinner:
|
|
27
|
+
"""Animated spinner for long-running operations."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, message: str):
|
|
30
|
+
self.message = message
|
|
31
|
+
self.running = False
|
|
32
|
+
self.thread = None
|
|
33
|
+
self.frames = itertools.cycle(["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"])
|
|
34
|
+
|
|
35
|
+
def _spin(self):
|
|
36
|
+
while self.running:
|
|
37
|
+
frame = next(self.frames)
|
|
38
|
+
print(f"\r {YELLOW}{frame}{RESET} {self.message}", end="", flush=True)
|
|
39
|
+
time.sleep(0.08)
|
|
40
|
+
|
|
41
|
+
def __enter__(self):
|
|
42
|
+
self.running = True
|
|
43
|
+
self.thread = threading.Thread(target=self._spin)
|
|
44
|
+
self.thread.start()
|
|
45
|
+
return self
|
|
46
|
+
|
|
47
|
+
def __exit__(self, *args):
|
|
48
|
+
self.running = False
|
|
49
|
+
if self.thread:
|
|
50
|
+
self.thread.join()
|
|
51
|
+
print(f"\r {' ' * (len(self.message) + 4)}\r", end="", flush=True)
|
|
52
|
+
|
|
53
|
+
|
|
24
54
|
def _header(title: str):
|
|
25
55
|
print(f"\n{BOLD}{CYAN}{title}{RESET}")
|
|
26
56
|
print(f"{DIM}{'─' * 40}{RESET}")
|
|
@@ -113,35 +143,50 @@ def file_main():
|
|
|
113
143
|
)
|
|
114
144
|
args = parser.parse_args()
|
|
115
145
|
|
|
146
|
+
# Echo command immediately
|
|
116
147
|
_header("segment-file")
|
|
148
|
+
print(f" {DIM}Segmenting text file into sentences{RESET}")
|
|
149
|
+
print()
|
|
150
|
+
|
|
151
|
+
# Show configuration
|
|
117
152
|
_param("Input", args.input_file)
|
|
118
153
|
_param("Output", args.output_file)
|
|
119
154
|
_param("Size", _file_size(args.input_file))
|
|
120
|
-
if args.unwrap
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
print(f"\n {YELLOW}Segmenting...{RESET}", end="", flush=True)
|
|
155
|
+
_param("Unwrap", "enabled" if args.unwrap else "disabled")
|
|
156
|
+
_param("Normalize quotes", "disabled" if args.no_normalize_quotes else "enabled")
|
|
157
|
+
print()
|
|
124
158
|
|
|
159
|
+
# Step 1: Read file
|
|
160
|
+
print(f" {YELLOW}→{RESET} Reading input file...")
|
|
125
161
|
with open(args.input_file, "r", encoding="utf-8") as f:
|
|
126
162
|
text = f.read()
|
|
163
|
+
print(f" {GREEN}✓{RESET} Read {len(text):,} characters")
|
|
127
164
|
|
|
165
|
+
# Step 2: Segment text
|
|
166
|
+
print(f" {YELLOW}→{RESET} Segmenting text...", end="", flush=True)
|
|
128
167
|
start = time.perf_counter()
|
|
129
168
|
normalize = not args.no_normalize_quotes
|
|
130
169
|
sentences = segment_text(
|
|
131
170
|
text.strip(), flatten=True, unwrap=args.unwrap, normalize=normalize,
|
|
132
171
|
)
|
|
133
172
|
elapsed = time.perf_counter() - start
|
|
173
|
+
print(f"\r {GREEN}✓{RESET} Segmented into {len(sentences):,} sentences ({elapsed:.2f}s)")
|
|
134
174
|
|
|
175
|
+
# Step 3: Write output
|
|
176
|
+
total = len(sentences)
|
|
135
177
|
with open(args.output_file, "w", encoding="utf-8") as f:
|
|
136
178
|
if args.unwrap:
|
|
137
179
|
f.write(format_grouped_sentences(sentences) + "\n")
|
|
180
|
+
print(f" {GREEN}✓{RESET} Written {total:,} sentences to {args.output_file}")
|
|
138
181
|
else:
|
|
139
|
-
for sentence in sentences:
|
|
182
|
+
for i, sentence in enumerate(sentences, 1):
|
|
140
183
|
f.write(sentence + "\n")
|
|
184
|
+
if i % 500 == 0 or i == total:
|
|
185
|
+
pct = (i / total) * 100
|
|
186
|
+
print(f"\r {YELLOW}→{RESET} Writing... {pct:.0f}% ({i:,}/{total:,})", end="", flush=True)
|
|
187
|
+
print(f"\r {GREEN}✓{RESET} Written {total:,} sentences to {args.output_file} ")
|
|
141
188
|
|
|
142
|
-
print(f"\
|
|
143
|
-
_done(f"{len(sentences):,} sentences in {elapsed:.2f}s")
|
|
144
|
-
_done(f"Written to {args.output_file}")
|
|
189
|
+
print(f"\n {GREEN}Done!{RESET}")
|
|
145
190
|
print()
|
|
146
191
|
|
|
147
192
|
|
|
@@ -3,10 +3,46 @@
|
|
|
3
3
|
""" Sentence Segmentation """
|
|
4
4
|
|
|
5
5
|
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
|
|
6
9
|
import spacy
|
|
7
10
|
|
|
8
11
|
from fast_sentence_segment.core import BaseObject
|
|
9
12
|
|
|
13
|
+
|
|
14
|
+
def _load_spacy_model(model_name: str = "en_core_web_sm"):
|
|
15
|
+
"""Load spaCy model, auto-downloading if not found."""
|
|
16
|
+
# ANSI color codes
|
|
17
|
+
bold = "\033[1m"
|
|
18
|
+
cyan = "\033[36m"
|
|
19
|
+
green = "\033[32m"
|
|
20
|
+
yellow = "\033[33m"
|
|
21
|
+
reset = "\033[0m"
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
return spacy.load(model_name)
|
|
25
|
+
except OSError:
|
|
26
|
+
print(f"\n{bold}{cyan}fast-sentence-segment{reset}", file=sys.stderr)
|
|
27
|
+
print(f"{'─' * 40}", file=sys.stderr)
|
|
28
|
+
print(
|
|
29
|
+
f" {yellow}⚠{reset} spaCy model '{model_name}' not found.",
|
|
30
|
+
file=sys.stderr,
|
|
31
|
+
)
|
|
32
|
+
print(f" {yellow}⏳{reset} Downloading model (one-time setup)...", file=sys.stderr)
|
|
33
|
+
print(file=sys.stderr)
|
|
34
|
+
|
|
35
|
+
subprocess.check_call(
|
|
36
|
+
[sys.executable, "-m", "spacy", "download", model_name],
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
print(file=sys.stderr)
|
|
40
|
+
print(f" {green}✓{reset} Model '{model_name}' installed successfully.", file=sys.stderr)
|
|
41
|
+
print(f" {green}✓{reset} You won't see this message again.", file=sys.stderr)
|
|
42
|
+
print(file=sys.stderr)
|
|
43
|
+
|
|
44
|
+
return spacy.load(model_name)
|
|
45
|
+
|
|
10
46
|
from fast_sentence_segment.dmo import AbbreviationMerger
|
|
11
47
|
from fast_sentence_segment.dmo import AbbreviationSplitter
|
|
12
48
|
from fast_sentence_segment.dmo import TitleNameMerger
|
|
@@ -45,7 +81,7 @@ class PerformSentenceSegmentation(BaseObject):
|
|
|
45
81
|
"""
|
|
46
82
|
BaseObject.__init__(self, __name__)
|
|
47
83
|
if not self.__nlp:
|
|
48
|
-
self.__nlp =
|
|
84
|
+
self.__nlp = _load_spacy_model("en_core_web_sm")
|
|
49
85
|
|
|
50
86
|
self._dehyphenate = Dehyphenator.process
|
|
51
87
|
self._newlines_to_periods = NewlinesToPeriods.process
|
|
@@ -11,7 +11,7 @@ description = "Fast and Efficient Sentence Segmentation"
|
|
|
11
11
|
license = "MIT"
|
|
12
12
|
name = "fast-sentence-segment"
|
|
13
13
|
readme = "README.md"
|
|
14
|
-
version = "1.4.
|
|
14
|
+
version = "1.4.3"
|
|
15
15
|
|
|
16
16
|
keywords = ["nlp", "text", "preprocess", "segment"]
|
|
17
17
|
repository = "https://github.com/craigtrim/fast-sentence-segment"
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
from setuptools import setup
|
|
3
|
+
|
|
4
|
+
packages = \
|
|
5
|
+
['fast_sentence_segment',
|
|
6
|
+
'fast_sentence_segment.bp',
|
|
7
|
+
'fast_sentence_segment.core',
|
|
8
|
+
'fast_sentence_segment.dmo',
|
|
9
|
+
'fast_sentence_segment.svc']
|
|
10
|
+
|
|
11
|
+
package_data = \
|
|
12
|
+
{'': ['*']}
|
|
13
|
+
|
|
14
|
+
install_requires = \
|
|
15
|
+
['spacy>=3.8.0,<4.0.0']
|
|
16
|
+
|
|
17
|
+
entry_points = \
|
|
18
|
+
{'console_scripts': ['segment = fast_sentence_segment.cli:main',
|
|
19
|
+
'segment-file = fast_sentence_segment.cli:file_main']}
|
|
20
|
+
|
|
21
|
+
setup_kwargs = {
|
|
22
|
+
'name': 'fast-sentence-segment',
|
|
23
|
+
'version': '1.4.3',
|
|
24
|
+
'description': 'Fast and Efficient Sentence Segmentation',
|
|
25
|
+
'long_description': '# Fast Sentence Segmentation\n\n[](https://pypi.org/project/fast-sentence-segment/)\n[](https://pypi.org/project/fast-sentence-segment/)\n[](https://github.com/craigtrim/fast-sentence-segment/tree/master/tests)\n[](https://opensource.org/licenses/MIT)\n[](https://github.com/astral-sh/ruff)\n[](https://pepy.tech/project/fast-sentence-segment)\n[](https://pepy.tech/project/fast-sentence-segment)\n\nFast and efficient sentence segmentation using spaCy with surgical post-processing fixes. Handles complex edge cases like abbreviations (Dr., Mr., etc.), ellipses, quoted text, and multi-paragraph documents.\n\n## Why This Library?\n\n1. **Keep it local**: LLM API calls cost money and send your data to third parties. Run sentence segmentation entirely on your machine.\n2. **spaCy perfected**: spaCy is a great local model, but it makes mistakes. This library fixes most of spaCy\'s shortcomings.\n\n## Features\n\n- **Paragraph-aware segmentation**: Returns sentences grouped by paragraph\n- **Abbreviation handling**: Correctly handles "Dr.", "Mr.", "etc.", "p.m.", "a.m." without false splits\n- **Ellipsis preservation**: Keeps `...` intact while detecting sentence boundaries\n- **Question/exclamation splitting**: Properly splits on `?` and `!` followed by capital letters\n- **Cached processing**: LRU cache for repeated text processing\n- **Flexible output**: Nested lists (by paragraph) or flattened list of sentences\n- **Bullet point & numbered list normalization**: Cleans common list formats\n- **CLI tool**: Command-line interface for quick segmentation\n\n## Installation\n\n```bash\npip install fast-sentence-segment\n```\n\nAfter installation, download the spaCy model:\n\n```bash\npython -m spacy download en_core_web_sm\n```\n\n## Quick Start\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = "Do you like Dr. Who? I prefer Dr. Strange! Mr. T is also cool."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n "Do you like Dr. Who?",\n "I prefer Dr. Strange!",\n "Mr. T is also cool."\n]\n```\n\nNotice how "Dr. Who?" stays together as a single sentence—the library correctly recognizes that a title followed by a single-word name ending in `?` or `!` is a name reference, not a sentence boundary.\n\n## Usage\n\n### Basic Segmentation\n\nThe `segment_text` function returns a list of lists, where each inner list represents a paragraph containing its sentences:\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = """Gandalf spoke softly. "All we have to decide is what to do with the time given us."\n\nFrodo nodded. The weight of the Ring pressed against his chest."""\n\nresults = segment_text(text)\n```\n\n```json\n[\n [\n "Gandalf spoke softly.",\n "\\"All we have to decide is what to do with the time given us.\\"."\n ],\n [\n "Frodo nodded.",\n "The weight of the Ring pressed against his chest."\n ]\n]\n```\n\n### Flattened Output\n\nIf you don\'t need paragraph boundaries, use the `flatten` parameter:\n\n```python\ntext = "At 9 a.m. the hobbits set out. By 3 p.m. they reached Rivendell. Mr. Frodo was exhausted."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n "At 9 a.m. the hobbits set out.",\n "By 3 p.m. they reached Rivendell.",\n "Mr. Frodo was exhausted."\n]\n```\n\n### Direct Segmenter Access\n\nFor more control, use the `Segmenter` class directly:\n\n```python\nfrom fast_sentence_segment import Segmenter\n\nsegmenter = Segmenter()\nresults = segmenter.input_text("Your text here.")\n```\n\n### Command Line Interface\n\n```bash\n# Inline text\nsegment "Gandalf paused... You shall not pass! The Balrog roared."\n\n# Pipe from stdin\necho "Have you seen Dr. Who? It\'s brilliant!" | segment\n\n# Numbered output\nsegment -n -f silmarillion.txt\n\n# File-to-file (one sentence per line)\nsegment-file --input-file book.txt --output-file sentences.txt\n\n# Unwrap hard-wrapped e-texts (Project Gutenberg, etc.)\nsegment-file --input-file book.txt --output-file sentences.txt --unwrap\n```\n\n## API Reference\n\n| Function | Parameters | Returns | Description |\n|----------|------------|---------|-------------|\n| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False` | `list` | Main entry point for segmentation |\n| `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |\n\n### CLI Commands\n\n| Command | Description |\n|---------|-------------|\n| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output. |\n| `segment-file --input-file IN --output-file OUT [--unwrap]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts. |\n\n## Why Nested Lists?\n\nThe segmentation process preserves document structure by segmenting into both paragraphs and sentences. Each outer list represents a paragraph, and each inner list contains that paragraph\'s sentences. This is useful for:\n\n- Document structure analysis\n- Paragraph-level processing\n- Maintaining original text organization\n\nUse `flatten=True` when you only need sentences without paragraph context.\n\n## Requirements\n\n- Python 3.9+\n- spaCy 3.8+\n- en_core_web_sm spaCy model\n\n## How It Works\n\nThis library uses spaCy for initial sentence segmentation, then applies surgical post-processing fixes for cases where spaCy\'s default behavior is incorrect:\n\n1. **Pre-processing**: Normalize numbered lists, preserve ellipses with placeholders\n2. **spaCy segmentation**: Use spaCy\'s sentence boundary detection\n3. **Post-processing**: Split on abbreviation boundaries, handle `?`/`!` + capital patterns\n4. **Denormalization**: Restore placeholders to original text\n\n## License\n\nMIT License - see [LICENSE](LICENSE) for details.\n\n## Contributing\n\nContributions are welcome! Please feel free to submit a Pull Request.\n\n1. Fork the repository\n2. Create your feature branch (`git checkout -b feature/amazing-feature`)\n3. Run tests (`make test`)\n4. Commit your changes\n5. Push to the branch\n6. Open a Pull Request\n',
|
|
26
|
+
'author': 'Craig Trim',
|
|
27
|
+
'author_email': 'craigtrim@gmail.com',
|
|
28
|
+
'maintainer': 'Craig Trim',
|
|
29
|
+
'maintainer_email': 'craigtrim@gmail.com',
|
|
30
|
+
'url': 'https://github.com/craigtrim/fast-sentence-segment',
|
|
31
|
+
'packages': packages,
|
|
32
|
+
'package_data': package_data,
|
|
33
|
+
'install_requires': install_requires,
|
|
34
|
+
'entry_points': entry_points,
|
|
35
|
+
'python_requires': '>=3.9,<3.13',
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
setup(**setup_kwargs)
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
from setuptools import setup
|
|
3
|
-
|
|
4
|
-
packages = \
|
|
5
|
-
['fast_sentence_segment',
|
|
6
|
-
'fast_sentence_segment.bp',
|
|
7
|
-
'fast_sentence_segment.core',
|
|
8
|
-
'fast_sentence_segment.dmo',
|
|
9
|
-
'fast_sentence_segment.svc']
|
|
10
|
-
|
|
11
|
-
package_data = \
|
|
12
|
-
{'': ['*']}
|
|
13
|
-
|
|
14
|
-
install_requires = \
|
|
15
|
-
['spacy>=3.8.0,<4.0.0']
|
|
16
|
-
|
|
17
|
-
entry_points = \
|
|
18
|
-
{'console_scripts': ['segment = fast_sentence_segment.cli:main',
|
|
19
|
-
'segment-file = fast_sentence_segment.cli:file_main']}
|
|
20
|
-
|
|
21
|
-
setup_kwargs = {
|
|
22
|
-
'name': 'fast-sentence-segment',
|
|
23
|
-
'version': '1.4.1',
|
|
24
|
-
'description': 'Fast and Efficient Sentence Segmentation',
|
|
25
|
-
'long_description': '# Fast Sentence Segmentation\n\n[](https://pypi.org/project/fast-sentence-segment/)\n[](https://pypi.org/project/fast-sentence-segment/)\n[](https://github.com/craigtrim/fast-sentence-segment/actions/workflows/ci.yml)\n[](https://opensource.org/licenses/MIT)\n[](https://github.com/astral-sh/ruff)\n[](https://pepy.tech/project/fast-sentence-segment)\n[](https://pepy.tech/project/fast-sentence-segment)\n\nFast and efficient sentence segmentation using spaCy with surgical post-processing fixes. Handles complex edge cases like abbreviations (Dr., Mr., etc.), ellipses, quoted text, and multi-paragraph documents.\n\n## Why This Library?\n\n1. **Keep it local**: LLM API calls cost money and send your data to third parties. Run sentence segmentation entirely on your machine.\n2. **spaCy perfected**: spaCy is a great local model, but it makes mistakes. This library fixes most of spaCy\'s shortcomings.\n\n## Features\n\n- **Paragraph-aware segmentation**: Returns sentences grouped by paragraph\n- **Abbreviation handling**: Correctly handles "Dr.", "Mr.", "etc.", "p.m.", "a.m." without false splits\n- **Ellipsis preservation**: Keeps `...` intact while detecting sentence boundaries\n- **Question/exclamation splitting**: Properly splits on `?` and `!` followed by capital letters\n- **Cached processing**: LRU cache for repeated text processing\n- **Flexible output**: Nested lists (by paragraph) or flattened list of sentences\n- **Bullet point & numbered list normalization**: Cleans common list formats\n- **CLI tool**: Command-line interface for quick segmentation\n\n## Installation\n\n```bash\npip install fast-sentence-segment\n```\n\nAfter installation, download the spaCy model:\n\n```bash\npython -m spacy download en_core_web_sm\n```\n\n## Quick Start\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = "Do you like Dr. Who? I prefer Dr. Strange! Mr. T is also cool."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n "Do you like Dr. Who?",\n "I prefer Dr. Strange!",\n "Mr. T is also cool."\n]\n```\n\nNotice how "Dr. Who?" stays together as a single sentence—the library correctly recognizes that a title followed by a single-word name ending in `?` or `!` is a name reference, not a sentence boundary.\n\n## Usage\n\n### Basic Segmentation\n\nThe `segment_text` function returns a list of lists, where each inner list represents a paragraph containing its sentences:\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = """Gandalf spoke softly. "All we have to decide is what to do with the time given us."\n\nFrodo nodded. The weight of the Ring pressed against his chest."""\n\nresults = segment_text(text)\n```\n\n```json\n[\n [\n "Gandalf spoke softly.",\n "\\"All we have to decide is what to do with the time given us.\\"."\n ],\n [\n "Frodo nodded.",\n "The weight of the Ring pressed against his chest."\n ]\n]\n```\n\n### Flattened Output\n\nIf you don\'t need paragraph boundaries, use the `flatten` parameter:\n\n```python\ntext = "At 9 a.m. the hobbits set out. By 3 p.m. they reached Rivendell. Mr. Frodo was exhausted."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n "At 9 a.m. the hobbits set out.",\n "By 3 p.m. they reached Rivendell.",\n "Mr. Frodo was exhausted."\n]\n```\n\n### Direct Segmenter Access\n\nFor more control, use the `Segmenter` class directly:\n\n```python\nfrom fast_sentence_segment import Segmenter\n\nsegmenter = Segmenter()\nresults = segmenter.input_text("Your text here.")\n```\n\n### Command Line Interface\n\n```bash\n# Inline text\nsegment "Gandalf paused... You shall not pass! The Balrog roared."\n\n# Pipe from stdin\necho "Have you seen Dr. Who? It\'s brilliant!" | segment\n\n# Numbered output\nsegment -n -f silmarillion.txt\n\n# File-to-file (one sentence per line)\nsegment-file --input-file book.txt --output-file sentences.txt\n\n# Unwrap hard-wrapped e-texts (Project Gutenberg, etc.)\nsegment-file --input-file book.txt --output-file sentences.txt --unwrap\n```\n\n## API Reference\n\n| Function | Parameters | Returns | Description |\n|----------|------------|---------|-------------|\n| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False` | `list` | Main entry point for segmentation |\n| `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |\n\n### CLI Commands\n\n| Command | Description |\n|---------|-------------|\n| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output. |\n| `segment-file --input-file IN --output-file OUT [--unwrap]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts. |\n\n## Why Nested Lists?\n\nThe segmentation process preserves document structure by segmenting into both paragraphs and sentences. Each outer list represents a paragraph, and each inner list contains that paragraph\'s sentences. This is useful for:\n\n- Document structure analysis\n- Paragraph-level processing\n- Maintaining original text organization\n\nUse `flatten=True` when you only need sentences without paragraph context.\n\n## Requirements\n\n- Python 3.9+\n- spaCy 3.8+\n- en_core_web_sm spaCy model\n\n## How It Works\n\nThis library uses spaCy for initial sentence segmentation, then applies surgical post-processing fixes for cases where spaCy\'s default behavior is incorrect:\n\n1. **Pre-processing**: Normalize numbered lists, preserve ellipses with placeholders\n2. **spaCy segmentation**: Use spaCy\'s sentence boundary detection\n3. **Post-processing**: Split on abbreviation boundaries, handle `?`/`!` + capital patterns\n4. **Denormalization**: Restore placeholders to original text\n\n## License\n\nMIT License - see [LICENSE](LICENSE) for details.\n\n## Contributing\n\nContributions are welcome! Please feel free to submit a Pull Request.\n\n1. Fork the repository\n2. Create your feature branch (`git checkout -b feature/amazing-feature`)\n3. Run tests (`make test`)\n4. Commit your changes\n5. Push to the branch\n6. Open a Pull Request\n',
|
|
26
|
-
'author': 'Craig Trim',
|
|
27
|
-
'author_email': 'craigtrim@gmail.com',
|
|
28
|
-
'maintainer': 'Craig Trim',
|
|
29
|
-
'maintainer_email': 'craigtrim@gmail.com',
|
|
30
|
-
'url': 'https://github.com/craigtrim/fast-sentence-segment',
|
|
31
|
-
'packages': packages,
|
|
32
|
-
'package_data': package_data,
|
|
33
|
-
'install_requires': install_requires,
|
|
34
|
-
'entry_points': entry_points,
|
|
35
|
-
'python_requires': '>=3.9,<3.13',
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
setup(**setup_kwargs)
|
|
File without changes
|
{fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/__init__.py
RENAMED
|
File without changes
|
{fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/bp/__init__.py
RENAMED
|
File without changes
|
{fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/bp/segmenter.py
RENAMED
|
File without changes
|
{fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/core/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/core/stopwatch.py
RENAMED
|
File without changes
|
{fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/dmo/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fast_sentence_segment-1.4.1 → fast_sentence_segment-1.4.3}/fast_sentence_segment/svc/__init__.py
RENAMED
|
File without changes
|