fast-sentence-segment 0.1.9__tar.gz → 1.2.0__tar.gz

fast_sentence_segment-1.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Craig Trim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fast_sentence_segment-1.2.0/PKG-INFO

@@ -0,0 +1,189 @@
+Metadata-Version: 2.4
+Name: fast-sentence-segment
+Version: 1.2.0
+Summary: Fast and Efficient Sentence Segmentation
+License: MIT
+License-File: LICENSE
+Keywords: nlp,text,preprocess,segment
+Author: Craig Trim
+Author-email: craigtrim@gmail.com
+Maintainer: Craig Trim
+Maintainer-email: craigtrim@gmail.com
+Requires-Python: >=3.9,<4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: spacy (>=3.8.0,<4.0.0)
+Project-URL: Bug Tracker, https://github.com/craigtrim/fast-sentence-segment/issues
+Project-URL: Repository, https://github.com/craigtrim/fast-sentence-segment
+Description-Content-Type: text/markdown
+
+# Fast Sentence Segmentation
+
+[![PyPI version](https://img.shields.io/pypi/v/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
+[![Python versions](https://img.shields.io/pypi/pyversions/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![spaCy](https://img.shields.io/badge/spaCy-3.8-blue.svg)](https://spacy.io/)
+
+Fast and efficient sentence segmentation using spaCy with surgical post-processing fixes. Handles complex edge cases like abbreviations (Dr., Mr., etc.), ellipses, quoted text, and multi-paragraph documents.
+
+## Why This Library?
+
+1. **Keep it local**: LLM API calls cost money and send your data to third parties. Run sentence segmentation entirely on your machine.
+2. **spaCy perfected**: spaCy is a great local model, but it makes mistakes. This library fixes most of spaCy's shortcomings.
+
+## Features
+
+- **Paragraph-aware segmentation**: Returns sentences grouped by paragraph
+- **Abbreviation handling**: Correctly handles "Dr.", "Mr.", "etc.", "p.m.", "a.m." without false splits
+- **Ellipsis preservation**: Keeps `...` intact while detecting sentence boundaries
+- **Question/exclamation splitting**: Properly splits on `?` and `!` followed by capital letters
+- **Cached processing**: LRU cache for repeated text processing
+- **Flexible output**: Nested lists (by paragraph) or flattened list of sentences
+- **Bullet point & numbered list normalization**: Cleans common list formats
+- **CLI tool**: Command-line interface for quick segmentation
+
+## Installation
+
+```bash
+pip install fast-sentence-segment
+```
+
+After installation, download the spaCy model:
+
+```bash
+python -m spacy download en_core_web_sm
+```
+
+## Quick Start
+
+```python
+from fast_sentence_segment import segment_text
+
+text = "Here is a Dr. who says something. And then again, what else? I don't know. Do you?"
+
+results = segment_text(text)
+# Returns: [['Here is a Dr. who says something.', 'And then again, what else?', "I don't know.", 'Do you?']]
+```
+
+## Usage
+
+### Basic Segmentation
+
+The `segment_text` function returns a list of lists, where each inner list represents a paragraph containing its sentences:
+
+```python
+from fast_sentence_segment import segment_text
+
+text = """First paragraph here. It has two sentences.
+
+Second paragraph starts here. This one also has multiple sentences. And a third."""
+
+results = segment_text(text)
+# Returns:
+# [
+#     ['First paragraph here.', 'It has two sentences.'],
+#     ['Second paragraph starts here.', 'This one also has multiple sentences.', 'And a third.']
+# ]
+```
+
+### Flattened Output
+
+If you don't need paragraph boundaries, use the `flatten` parameter:
+
+```python
+results = segment_text(text, flatten=True)
+# Returns: ['First paragraph here.', 'It has two sentences.', 'Second paragraph starts here.', ...]
+```
+
+### Direct Segmenter Access
+
+For more control, use the `Segmenter` class directly:
+
+```python
+from fast_sentence_segment import Segmenter
+
+segmenter = Segmenter()
+results = segmenter.input_text("Your text here.")
+```
+
+### Command Line Interface
+
+Segment text directly from the terminal:
+
+```bash
+# Direct text input
+segment "Hello world. How are you? I am fine."
+
+# Numbered output
+segment -n "First sentence. Second sentence."
+
+# From stdin
+echo "Some text here. Another sentence." | segment
+
+# From file
+segment -f document.txt
+```
+
+## API Reference
+
+| Function | Parameters | Returns | Description |
+|----------|------------|---------|-------------|
+| `segment_text()` | `input_text: str`, `flatten: bool = False` | `list` | Main entry point for segmentation |
+| `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |
+
+### CLI Options
+
+| Option | Description |
+|--------|-------------|
+| `text` | Text to segment (positional argument) |
+| `-f, --file` | Read text from file |
+| `-n, --numbered` | Number output lines |
+
+## Why Nested Lists?
+
+The segmentation process preserves document structure by segmenting into both paragraphs and sentences. Each outer list represents a paragraph, and each inner list contains that paragraph's sentences. This is useful for:
+
+- Document structure analysis
+- Paragraph-level processing
+- Maintaining original text organization
+
+Use `flatten=True` when you only need sentences without paragraph context.
+
+## Requirements
+
+- Python 3.9+
+- spaCy 3.8+
+- en_core_web_sm spaCy model
+
+## How It Works
+
+This library uses spaCy for initial sentence segmentation, then applies surgical post-processing fixes for cases where spaCy's default behavior is incorrect:
+
+1. **Pre-processing**: Normalize numbered lists, preserve ellipses with placeholders
+2. **spaCy segmentation**: Use spaCy's sentence boundary detection
+3. **Post-processing**: Split on abbreviation boundaries, handle `?`/`!` + capital patterns
+4. **Denormalization**: Restore placeholders to original text
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+1. Fork the repository
+2. Create your feature branch (`git checkout -b feature/amazing-feature`)
+3. Run tests (`make test`)
+4. Commit your changes
+5. Push to the branch
+6. Open a Pull Request
+

fast_sentence_segment-1.2.0/README.md

@@ -0,0 +1,161 @@
+# Fast Sentence Segmentation
+
+[![PyPI version](https://img.shields.io/pypi/v/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
+[![Python versions](https://img.shields.io/pypi/pyversions/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![spaCy](https://img.shields.io/badge/spaCy-3.8-blue.svg)](https://spacy.io/)
+
+Fast and efficient sentence segmentation using spaCy with surgical post-processing fixes. Handles complex edge cases like abbreviations (Dr., Mr., etc.), ellipses, quoted text, and multi-paragraph documents.
+
+## Why This Library?
+
+1. **Keep it local**: LLM API calls cost money and send your data to third parties. Run sentence segmentation entirely on your machine.
+2. **spaCy perfected**: spaCy is a great local model, but it makes mistakes. This library fixes most of spaCy's shortcomings.
+
+## Features
+
+- **Paragraph-aware segmentation**: Returns sentences grouped by paragraph
+- **Abbreviation handling**: Correctly handles "Dr.", "Mr.", "etc.", "p.m.", "a.m." without false splits
+- **Ellipsis preservation**: Keeps `...` intact while detecting sentence boundaries
+- **Question/exclamation splitting**: Properly splits on `?` and `!` followed by capital letters
+- **Cached processing**: LRU cache for repeated text processing
+- **Flexible output**: Nested lists (by paragraph) or flattened list of sentences
+- **Bullet point & numbered list normalization**: Cleans common list formats
+- **CLI tool**: Command-line interface for quick segmentation
+
+## Installation
+
+```bash
+pip install fast-sentence-segment
+```
+
+After installation, download the spaCy model:
+
+```bash
+python -m spacy download en_core_web_sm
+```
+
+## Quick Start
+
+```python
+from fast_sentence_segment import segment_text
+
+text = "Here is a Dr. who says something. And then again, what else? I don't know. Do you?"
+
+results = segment_text(text)
+# Returns: [['Here is a Dr. who says something.', 'And then again, what else?', "I don't know.", 'Do you?']]
+```
+
+## Usage
+
+### Basic Segmentation
+
+The `segment_text` function returns a list of lists, where each inner list represents a paragraph containing its sentences:
+
+```python
+from fast_sentence_segment import segment_text
+
+text = """First paragraph here. It has two sentences.
+
+Second paragraph starts here. This one also has multiple sentences. And a third."""
+
+results = segment_text(text)
+# Returns:
+# [
+#     ['First paragraph here.', 'It has two sentences.'],
+#     ['Second paragraph starts here.', 'This one also has multiple sentences.', 'And a third.']
+# ]
+```
+
+### Flattened Output
+
+If you don't need paragraph boundaries, use the `flatten` parameter:
+
+```python
+results = segment_text(text, flatten=True)
+# Returns: ['First paragraph here.', 'It has two sentences.', 'Second paragraph starts here.', ...]
+```
+
+### Direct Segmenter Access
+
+For more control, use the `Segmenter` class directly:
+
+```python
+from fast_sentence_segment import Segmenter
+
+segmenter = Segmenter()
+results = segmenter.input_text("Your text here.")
+```
+
+### Command Line Interface
+
+Segment text directly from the terminal:
+
+```bash
+# Direct text input
+segment "Hello world. How are you? I am fine."
+
+# Numbered output
+segment -n "First sentence. Second sentence."
+
+# From stdin
+echo "Some text here. Another sentence." | segment
+
+# From file
+segment -f document.txt
+```
+
+## API Reference
+
+| Function | Parameters | Returns | Description |
+|----------|------------|---------|-------------|
+| `segment_text()` | `input_text: str`, `flatten: bool = False` | `list` | Main entry point for segmentation |
+| `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |
+
+### CLI Options
+
+| Option | Description |
+|--------|-------------|
+| `text` | Text to segment (positional argument) |
+| `-f, --file` | Read text from file |
+| `-n, --numbered` | Number output lines |
+
+## Why Nested Lists?
+
+The segmentation process preserves document structure by segmenting into both paragraphs and sentences. Each outer list represents a paragraph, and each inner list contains that paragraph's sentences. This is useful for:
+
+- Document structure analysis
+- Paragraph-level processing
+- Maintaining original text organization
+
+Use `flatten=True` when you only need sentences without paragraph context.
+
+## Requirements
+
+- Python 3.9+
+- spaCy 3.8+
+- en_core_web_sm spaCy model
+
+## How It Works
+
+This library uses spaCy for initial sentence segmentation, then applies surgical post-processing fixes for cases where spaCy's default behavior is incorrect:
+
+1. **Pre-processing**: Normalize numbered lists, preserve ellipses with placeholders
+2. **spaCy segmentation**: Use spaCy's sentence boundary detection
+3. **Post-processing**: Split on abbreviation boundaries, handle `?`/`!` + capital patterns
+4. **Denormalization**: Restore placeholders to original text
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+1. Fork the repository
+2. Create your feature branch (`git checkout -b feature/amazing-feature`)
+3. Run tests (`make test`)
+4. Commit your changes
+5. Push to the branch
+6. Open a Pull Request

{fast-sentence-segment-0.1.9 → fast_sentence_segment-1.2.0}/fast_sentence_segment/__init__.py

@@ -1,18 +1,18 @@
-from .bp import *
-from .svc import *
-from .dmo import *
-
-from .bp.segmenter import Segmenter
-
-segment = Segmenter().input_text
-
-
-def segment_text(input_text: str, flatten: bool = False) -> list:
-    results = segment(input_text)
-
-    if flatten:
-        flat = []
-        [[flat.append(y) for y in x] for x in results]
-        return flat
-
-    return results
+from .bp import *
+from .svc import *
+from .dmo import *
+
+from .bp.segmenter import Segmenter
+
+segment = Segmenter().input_text
+
+
+def segment_text(input_text: str, flatten: bool = False) -> list:
+    results = segment(input_text)
+
+    if flatten:
+        flat = []
+        [[flat.append(y) for y in x] for x in results]
+        return flat
+
+    return results

{fast-sentence-segment-0.1.9 → fast_sentence_segment-1.2.0}/fast_sentence_segment/bp/__init__.py

@@ -1 +1 @@
-from .segmenter import Segmenter
+from .segmenter import Segmenter

{fast-sentence-segment-0.1.9 → fast_sentence_segment-1.2.0}/fast_sentence_segment/bp/segmenter.py

@@ -1,68 +1,65 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Orchestrate Sentence Segmentation """
-
-
-from functools import lru_cache
-
-from baseblock import Enforcer
-from baseblock import Stopwatch
-from baseblock import BaseObject
-
-from fast_sentence_segment.svc import PerformParagraphSegmentation
-from fast_sentence_segment.svc import PerformSentenceSegmentation
-
-
-class Segmenter(BaseObject):
-    """ Orchestrate Sentence Segmentation """
-
-    def __init__(self):
-        """ Change Log
-
-        Created:
-            30-Sept-2021
-        """
-        BaseObject.__init__(self, __name__)
-        self._segment_paragraphs = PerformParagraphSegmentation().process
-        self._segment_sentences = PerformSentenceSegmentation().process
-
-    def _input_text(self,
-                    input_text: str) -> list:
-        paragraphs = []
-
-        for paragraph in self._segment_paragraphs(input_text):
-            paragraphs.append(self._segment_sentences(paragraph))
-
-        return paragraphs
-
-    @lru_cache(maxsize=1024, typed=True)
-    def input_text(self,
-                   input_text: str) -> list:
-        """Segment Input Text into Paragraphs and Sentences
-
-        Args:
-            input_text (str): An input string of any length or type
-
-        Raises:
-            ValueError: input must be a string
-
-        Returns:
-            list:   returns a list of lists.
-                    Each outer list is a paragraph.
-                    Each inner list contains 1..* sentences
-        """
-
-        if self.isEnabledForDebug:
-            Enforcer.is_str(input_text)
-
-        sw = Stopwatch()
-
-        paragraphs = self._input_text(input_text)
-
-        if self.isEnabledForInfo:
-            self.logger.info('\n'.join([
-                "Segmentation of Input Text Complete",
-                f"\tTotal Paragraphs: {len(paragraphs)}",
-                f"\tTotal Time: {str(sw)}"]))
-
-        return paragraphs
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Orchestrate Sentence Segmentation """
+
+
+from functools import lru_cache
+
+from fast_sentence_segment.core import BaseObject, Stopwatch
+from fast_sentence_segment.svc import PerformParagraphSegmentation
+from fast_sentence_segment.svc import PerformSentenceSegmentation
+
+
+class Segmenter(BaseObject):
+    """ Orchestrate Sentence Segmentation """
+
+    def __init__(self):
+        """ Change Log
+
+        Created:
+            30-Sept-2021
+        """
+        BaseObject.__init__(self, __name__)
+        self._segment_paragraphs = PerformParagraphSegmentation().process
+        self._segment_sentences = PerformSentenceSegmentation().process
+
+    def _input_text(self,
+                    input_text: str) -> list:
+        paragraphs = []
+
+        for paragraph in self._segment_paragraphs(input_text):
+            paragraphs.append(self._segment_sentences(paragraph))
+
+        return paragraphs
+
+    @lru_cache(maxsize=1024, typed=True)
+    def input_text(self,
+                   input_text: str) -> list:
+        """Segment Input Text into Paragraphs and Sentences
+
+        Args:
+            input_text (str): An input string of any length or type
+
+        Raises:
+            ValueError: input must be a string
+
+        Returns:
+            list:   returns a list of lists.
+                    Each outer list is a paragraph.
+                    Each inner list contains 1..* sentences
+        """
+
+        if self.isEnabledForDebug and not isinstance(input_text, str):
+            raise ValueError(f"Expected str, got {type(input_text)}")
+
+        sw = Stopwatch()
+
+        paragraphs = self._input_text(input_text)
+
+        if self.isEnabledForInfo:
+            self.logger.info('\n'.join([
+                "Segmentation of Input Text Complete",
+                f"\tTotal Paragraphs: {len(paragraphs)}",
+                f"\tTotal Time: {str(sw)}"]))
+
+        return paragraphs

fast_sentence_segment-1.2.0/fast_sentence_segment/cli.py

@@ -0,0 +1,56 @@
+# -*- coding: UTF-8 -*-
+"""CLI for fast-sentence-segment."""
+
+import argparse
+import logging
+import sys
+
+from fast_sentence_segment import segment_text
+
+logging.disable(logging.CRITICAL)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="segment",
+        description="Segment text into sentences",
+    )
+    parser.add_argument(
+        "text",
+        nargs="?",
+        help="Text to segment (or use stdin)",
+    )
+    parser.add_argument(
+        "-f", "--file",
+        help="Read text from file",
+    )
+    parser.add_argument(
+        "-n", "--numbered",
+        action="store_true",
+        help="Number output lines",
+    )
+    args = parser.parse_args()
+
+    # Get input text
+    if args.file:
+        with open(args.file, "r", encoding="utf-8") as f:
+            text = f.read()
+    elif args.text:
+        text = args.text
+    elif not sys.stdin.isatty():
+        text = sys.stdin.read()
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    # Segment and output
+    sentences = segment_text(text.strip(), flatten=True)
+    for i, sentence in enumerate(sentences, 1):
+        if args.numbered:
+            print(f"{i}. {sentence}")
+        else:
+            print(sentence)
+
+
+if __name__ == "__main__":
+    main()

fast_sentence_segment-1.2.0/fast_sentence_segment/core/__init__.py

@@ -0,0 +1,4 @@
+from .base_object import BaseObject
+from .stopwatch import Stopwatch
+
+__all__ = ["BaseObject", "Stopwatch"]

fast_sentence_segment-1.2.0/fast_sentence_segment/core/base_object.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""Base object providing logging functionality."""
+
+import logging
+
+logging.basicConfig(
+    format='%(asctime)s : %(levelname)s : %(filename)s : %(funcName)s() : %(lineno)d : %(message)s',
+    level=logging.DEBUG)
+
+
+class BaseObject:
+    """Base class providing logging capabilities."""
+
+    def __init__(self, component_name: str):
+        self.logger = logging.getLogger(component_name)
+        self.isEnabledForDebug = self.logger.isEnabledFor(logging.DEBUG)
+        self.isEnabledForInfo = self.logger.isEnabledFor(logging.INFO)
+        self.isEnabledForWarning = self.logger.isEnabledFor(logging.WARNING)

fast_sentence_segment-1.2.0/fast_sentence_segment/core/stopwatch.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""Simple stopwatch for timing operations."""
+
+import time
+
+
+class Stopwatch:
+    """A simple stopwatch for measuring elapsed time."""
+
+    def __init__(self):
+        self._start = time.perf_counter()
+        self._end = None
+
+    @property
+    def duration(self):
+        return self._end - self._start if self._end else time.perf_counter() - self._start
+
+    @property
+    def running(self):
+        return not self._end
+
+    def restart(self):
+        self._start = time.perf_counter()
+        self._end = None
+        return self
+
+    def stop(self):
+        if self.running:
+            self._end = time.perf_counter()
+        return self
+
+    def __str__(self):
+        ms = self.duration * 1000
+        if ms >= 1000:
+            return f'{ms / 1000:.2f}s'
+        if ms >= 1:
+            return f'{ms:.2f}ms'
+        return f'{ms * 1000:.2f}μs'

fast_sentence_segment-1.2.0/fast_sentence_segment/dmo/__init__.py

@@ -0,0 +1,10 @@
+from .abbreviation_merger import AbbreviationMerger
+from .abbreviation_splitter import AbbreviationSplitter
+from .title_name_merger import TitleNameMerger
+from .bullet_point_cleaner import BulletPointCleaner
+from .ellipsis_normalizer import EllipsisNormalizer
+from .newlines_to_periods import NewlinesToPeriods
+from .post_process_sentences import PostProcessStructure
+from .question_exclamation_splitter import QuestionExclamationSplitter
+from .spacy_doc_segmenter import SpacyDocSegmenter
+from .numbered_list_normalizer import NumberedListNormalizer