fast-sentence-segment 0.1.9__py3-none-any.whl → 1.2.0__py3-none-any.whl

fast_sentence_segment/__init__.py

@@ -1,18 +1,18 @@
-from .bp import *
-from .svc import *
-from .dmo import *
-
-from .bp.segmenter import Segmenter
-
-segment = Segmenter().input_text
-
-
-def segment_text(input_text: str, flatten: bool = False) -> list:
-    results = segment(input_text)
-
-    if flatten:
-        flat = []
-        [[flat.append(y) for y in x] for x in results]
-        return flat
-
-    return results
+from .bp import *
+from .svc import *
+from .dmo import *
+
+from .bp.segmenter import Segmenter
+
+segment = Segmenter().input_text
+
+
+def segment_text(input_text: str, flatten: bool = False) -> list:
+    results = segment(input_text)
+
+    if flatten:
+        flat = []
+        [[flat.append(y) for y in x] for x in results]
+        return flat
+
+    return results

fast_sentence_segment/bp/__init__.py

@@ -1 +1 @@
-from .segmenter import Segmenter
+from .segmenter import Segmenter

fast_sentence_segment/bp/segmenter.py

@@ -1,68 +1,65 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Orchestrate Sentence Segmentation """
-
-
-from functools import lru_cache
-
-from baseblock import Enforcer
-from baseblock import Stopwatch
-from baseblock import BaseObject
-
-from fast_sentence_segment.svc import PerformParagraphSegmentation
-from fast_sentence_segment.svc import PerformSentenceSegmentation
-
-
-class Segmenter(BaseObject):
-    """ Orchestrate Sentence Segmentation """
-
-    def __init__(self):
-        """ Change Log
-
-        Created:
-            30-Sept-2021
-        """
-        BaseObject.__init__(self, __name__)
-        self._segment_paragraphs = PerformParagraphSegmentation().process
-        self._segment_sentences = PerformSentenceSegmentation().process
-
-    def _input_text(self,
-                    input_text: str) -> list:
-        paragraphs = []
-
-        for paragraph in self._segment_paragraphs(input_text):
-            paragraphs.append(self._segment_sentences(paragraph))
-
-        return paragraphs
-
-    @lru_cache(maxsize=1024, typed=True)
-    def input_text(self,
-                   input_text: str) -> list:
-        """Segment Input Text into Paragraphs and Sentences
-
-        Args:
-            input_text (str): An input string of any length or type
-
-        Raises:
-            ValueError: input must be a string
-
-        Returns:
-            list:   returns a list of lists.
-                    Each outer list is a paragraph.
-                    Each inner list contains 1..* sentences
-        """
-
-        if self.isEnabledForDebug:
-            Enforcer.is_str(input_text)
-
-        sw = Stopwatch()
-
-        paragraphs = self._input_text(input_text)
-
-        if self.isEnabledForInfo:
-            self.logger.info('\n'.join([
-                "Segmentation of Input Text Complete",
-                f"\tTotal Paragraphs: {len(paragraphs)}",
-                f"\tTotal Time: {str(sw)}"]))
-
-        return paragraphs
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Orchestrate Sentence Segmentation """
+
+
+from functools import lru_cache
+
+from fast_sentence_segment.core import BaseObject, Stopwatch
+from fast_sentence_segment.svc import PerformParagraphSegmentation
+from fast_sentence_segment.svc import PerformSentenceSegmentation
+
+
+class Segmenter(BaseObject):
+    """ Orchestrate Sentence Segmentation """
+
+    def __init__(self):
+        """ Change Log
+
+        Created:
+            30-Sept-2021
+        """
+        BaseObject.__init__(self, __name__)
+        self._segment_paragraphs = PerformParagraphSegmentation().process
+        self._segment_sentences = PerformSentenceSegmentation().process
+
+    def _input_text(self,
+                    input_text: str) -> list:
+        paragraphs = []
+
+        for paragraph in self._segment_paragraphs(input_text):
+            paragraphs.append(self._segment_sentences(paragraph))
+
+        return paragraphs
+
+    @lru_cache(maxsize=1024, typed=True)
+    def input_text(self,
+                   input_text: str) -> list:
+        """Segment Input Text into Paragraphs and Sentences
+
+        Args:
+            input_text (str): An input string of any length or type
+
+        Raises:
+            ValueError: input must be a string
+
+        Returns:
+            list:   returns a list of lists.
+                    Each outer list is a paragraph.
+                    Each inner list contains 1..* sentences
+        """
+
+        if self.isEnabledForDebug and not isinstance(input_text, str):
+            raise ValueError(f"Expected str, got {type(input_text)}")
+
+        sw = Stopwatch()
+
+        paragraphs = self._input_text(input_text)
+
+        if self.isEnabledForInfo:
+            self.logger.info('\n'.join([
+                "Segmentation of Input Text Complete",
+                f"\tTotal Paragraphs: {len(paragraphs)}",
+                f"\tTotal Time: {str(sw)}"]))
+
+        return paragraphs

fast_sentence_segment/cli.py

@@ -0,0 +1,56 @@
+# -*- coding: UTF-8 -*-
+"""CLI for fast-sentence-segment."""
+
+import argparse
+import logging
+import sys
+
+from fast_sentence_segment import segment_text
+
+logging.disable(logging.CRITICAL)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="segment",
+        description="Segment text into sentences",
+    )
+    parser.add_argument(
+        "text",
+        nargs="?",
+        help="Text to segment (or use stdin)",
+    )
+    parser.add_argument(
+        "-f", "--file",
+        help="Read text from file",
+    )
+    parser.add_argument(
+        "-n", "--numbered",
+        action="store_true",
+        help="Number output lines",
+    )
+    args = parser.parse_args()
+
+    # Get input text
+    if args.file:
+        with open(args.file, "r", encoding="utf-8") as f:
+            text = f.read()
+    elif args.text:
+        text = args.text
+    elif not sys.stdin.isatty():
+        text = sys.stdin.read()
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    # Segment and output
+    sentences = segment_text(text.strip(), flatten=True)
+    for i, sentence in enumerate(sentences, 1):
+        if args.numbered:
+            print(f"{i}. {sentence}")
+        else:
+            print(sentence)
+
+
+if __name__ == "__main__":
+    main()

fast_sentence_segment/core/__init__.py

@@ -0,0 +1,4 @@
+from .base_object import BaseObject
+from .stopwatch import Stopwatch
+
+__all__ = ["BaseObject", "Stopwatch"]

fast_sentence_segment/core/base_object.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""Base object providing logging functionality."""
+
+import logging
+
+logging.basicConfig(
+    format='%(asctime)s : %(levelname)s : %(filename)s : %(funcName)s() : %(lineno)d : %(message)s',
+    level=logging.DEBUG)
+
+
+class BaseObject:
+    """Base class providing logging capabilities."""
+
+    def __init__(self, component_name: str):
+        self.logger = logging.getLogger(component_name)
+        self.isEnabledForDebug = self.logger.isEnabledFor(logging.DEBUG)
+        self.isEnabledForInfo = self.logger.isEnabledFor(logging.INFO)
+        self.isEnabledForWarning = self.logger.isEnabledFor(logging.WARNING)

fast_sentence_segment/core/stopwatch.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""Simple stopwatch for timing operations."""
+
+import time
+
+
+class Stopwatch:
+    """A simple stopwatch for measuring elapsed time."""
+
+    def __init__(self):
+        self._start = time.perf_counter()
+        self._end = None
+
+    @property
+    def duration(self):
+        return self._end - self._start if self._end else time.perf_counter() - self._start
+
+    @property
+    def running(self):
+        return not self._end
+
+    def restart(self):
+        self._start = time.perf_counter()
+        self._end = None
+        return self
+
+    def stop(self):
+        if self.running:
+            self._end = time.perf_counter()
+        return self
+
+    def __str__(self):
+        ms = self.duration * 1000
+        if ms >= 1000:
+            return f'{ms / 1000:.2f}s'
+        if ms >= 1:
+            return f'{ms:.2f}ms'
+        return f'{ms * 1000:.2f}μs'

fast_sentence_segment/dmo/__init__.py

@@ -1,6 +1,10 @@
-from .bullet_point_cleaner import BulletPointCleaner
-from .delimiters_to_periods import DelimitersToPeriods
-from .newlines_to_periods import NewlinesToPeriods
-from .post_process_sentences import PostProcessStructure
-from .spacy_doc_segmenter import SpacyDocSegmenter
-from .numbered_list_normalizer import NumberedListNormalizer
+from .abbreviation_merger import AbbreviationMerger
+from .abbreviation_splitter import AbbreviationSplitter
+from .title_name_merger import TitleNameMerger
+from .bullet_point_cleaner import BulletPointCleaner
+from .ellipsis_normalizer import EllipsisNormalizer
+from .newlines_to_periods import NewlinesToPeriods
+from .post_process_sentences import PostProcessStructure
+from .question_exclamation_splitter import QuestionExclamationSplitter
+from .spacy_doc_segmenter import SpacyDocSegmenter
+from .numbered_list_normalizer import NumberedListNormalizer

fast_sentence_segment/dmo/abbreviation_merger.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+Merge sentences that spaCy incorrectly split at abbreviation boundaries.
+
+When spaCy incorrectly splits after an abbreviation (e.g., "ext. 5" becomes
+["ext.", "5. Ask for help."]), this component merges them back together
+using specific known patterns.
+
+Reference: https://github.com/craigtrim/fast-sentence-segment/issues/3
+"""
+
+import re
+from typing import List, Optional, Tuple
+
+from fast_sentence_segment.core import BaseObject
+
+
+# Patterns where spaCy incorrectly splits after an abbreviation.
+# Format: (ending_pattern, extract_pattern)
+#   - ending_pattern: regex to match end of current sentence
+#   - extract_pattern: regex to extract the portion to merge from next sentence
+#
+# The extract_pattern MUST have a capture group for the portion to merge.
+# Whatever is NOT captured remains as a separate sentence.
+
+MERGE_PATTERNS: List[Tuple[str, str]] = [
+
+    # ext. 5, Ext. 123, EXT. 42
+    (r"(?i)\bext\.$", r"^(\d+\.?)\s*"),
+
+    # no. 5, No. 42, NO. 100
+    (r"(?i)\bno\.$", r"^(\d+\.?)\s*"),
+
+    # vol. 3, Vol. 42, VOL. 1
+    (r"(?i)\bvol\.$", r"^(\d+\.?)\s*"),
+
+    # pt. 2, Pt. 1, PT. 3
+    (r"(?i)\bpt\.$", r"^(\d+\.?)\s*"),
+
+    # ch. 5, Ch. 10, CH. 3
+    (r"(?i)\bch\.$", r"^(\d+\.?)\s*"),
+
+    # sec. 3, Sec. 14, SEC. 2
+    (r"(?i)\bsec\.$", r"^(\d+(?:\.\d+)?\.?)\s*"),
+
+    # fig. 1, Fig. 3.2, FIG. 10
+    (r"(?i)\bfig\.$", r"^(\d+(?:\.\d+)?\.?)\s*"),
+
+    # p. 42, P. 100
+    (r"(?i)\bp\.$", r"^(\d+\.?)\s*"),
+
+    # pp. 42-50, PP. 100-110
+    (r"(?i)\bpp\.$", r"^(\d+(?:-\d+)?\.?)\s*"),
+
+    # art. 5, Art. 12, ART. 1
+    (r"(?i)\bart\.$", r"^(\d+\.?)\s*"),
+
+]
+
+
+class AbbreviationMerger(BaseObject):
+    """Merge sentences incorrectly split at abbreviation boundaries."""
+
+    def __init__(self):
+        """
+        Created:
+            27-Dec-2024
+            craigtrim@gmail.com
+        Reference:
+            https://github.com/craigtrim/fast-sentence-segment/issues/3
+        """
+        BaseObject.__init__(self, __name__)
+        # Compile patterns for efficiency
+        self._patterns = [
+            (re.compile(ending), re.compile(extract))
+            for ending, extract in MERGE_PATTERNS
+        ]
+
+    def _try_merge(self, current: str, next_sent: str) -> Optional[Tuple[str, str]]:
+        """Try to merge two sentences based on known patterns.
+
+        Args:
+            current: Current sentence
+            next_sent: Next sentence
+
+        Returns:
+            Tuple of (merged_sentence, remainder) if merge needed, else None
+        """
+        current = current.strip()
+        next_sent = next_sent.strip()
+
+        for ending_pattern, extract_pattern in self._patterns:
+            if ending_pattern.search(current):
+                match = extract_pattern.match(next_sent)
+                if match:
+                    # Extract the portion to merge
+                    extracted = match.group(1)
+                    # Get the remainder (everything after the match)
+                    remainder = next_sent[match.end():].strip()
+                    # Build merged sentence
+                    merged = current + " " + extracted
+                    return (merged, remainder)
+
+        return None
+
+    def process(self, sentences: List[str]) -> List[str]:
+        """Process a list of sentences, merging incorrectly split ones.
+
+        Args:
+            sentences: List of sentences from spaCy
+
+        Returns:
+            List of sentences with incorrect splits merged
+        """
+        if not sentences:
+            return sentences
+
+        result = []
+        i = 0
+
+        while i < len(sentences):
+            current = sentences[i]
+
+            # Check if we should merge with next sentence
+            if i + 1 < len(sentences):
+                next_sent = sentences[i + 1]
+                merge_result = self._try_merge(current, next_sent)
+
+                if merge_result:
+                    merged, remainder = merge_result
+                    result.append(merged)
+
+                    # If there's a remainder, it becomes a new sentence to process
+                    if remainder:
+                        # Insert remainder back for processing
+                        sentences = sentences[:i+2] + [remainder] + sentences[i+2:]
+                        sentences[i+1] = remainder
+
+                    i += 2
+                    continue
+
+            result.append(current)
+            i += 1
+
+        return result

fast_sentence_segment/dmo/abbreviation_splitter.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+Split sentences at abbreviation boundaries.
+
+When spaCy fails to detect a sentence boundary after an abbreviation
+(e.g., "I woke at 6 a.m. It was dark."), this component splits the
+sentence by detecting the pattern: abbreviation + space + Capital letter.
+
+Reference: https://github.com/craigtrim/fast-sentence-segment/issues/3
+"""
+
+import re
+from typing import List
+
+from fast_sentence_segment.core import BaseObject
+from fast_sentence_segment.dmo.abbreviations import SENTENCE_ENDING_ABBREVIATIONS
+
+
+class AbbreviationSplitter(BaseObject):
+    """Split sentences at abbreviation boundaries."""
+
+    def __init__(self):
+        """
+        Created:
+            27-Dec-2024
+            craigtrim@gmail.com
+        Reference:
+            https://github.com/craigtrim/fast-sentence-segment/issues/3
+        """
+        BaseObject.__init__(self, __name__)
+        self._pattern = self._build_pattern()
+
+    def _build_pattern(self) -> re.Pattern:
+        """Build regex pattern to match abbreviation + capital letter.
+
+        Pattern matches:
+        - A known sentence-ending abbreviation (escaped for regex)
+        - Followed by one or more spaces
+        - Followed by a capital letter (start of new sentence)
+
+        Note: Title abbreviations (Dr., Mr., etc.) are excluded because
+        they are typically followed by names, not new sentences.
+
+        Returns:
+            Compiled regex pattern
+        """
+        escaped_abbrevs = [re.escape(abbr) for abbr in SENTENCE_ENDING_ABBREVIATIONS]
+        abbrev_pattern = "|".join(escaped_abbrevs)
+        pattern = rf"({abbrev_pattern})\s+([A-Z])"
+        return re.compile(pattern)
+
+    def _split_sentence(self, sentence: str) -> List[str]:
+        """Split a single sentence at abbreviation boundaries.
+
+        Args:
+            sentence: A sentence that may contain abbreviation boundaries
+
+        Returns:
+            List of one or more sentences
+        """
+        results = []
+        remaining = sentence
+
+        while True:
+            match = self._pattern.search(remaining)
+            if not match:
+                if remaining.strip():
+                    results.append(remaining.strip())
+                break
+
+            split_pos = match.end(1)
+
+            before = remaining[:split_pos].strip()
+            if before:
+                results.append(before)
+
+            remaining = remaining[split_pos:].strip()
+
+        return results if results else [sentence]
+
+    def process(self, sentences: List[str]) -> List[str]:
+        """Process a list of sentences, splitting at abbreviation boundaries.
+
+        Args:
+            sentences: List of sentences from spaCy
+
+        Returns:
+            List of sentences with abbreviation boundaries properly split
+        """
+        result = []
+        for sentence in sentences:
+            split_sentences = self._split_sentence(sentence)
+            result.extend(split_sentences)
+        return result

fast_sentence_segment/dmo/abbreviations.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+Known abbreviations that end with periods.
+
+Reference: https://github.com/craigtrim/fast-sentence-segment/issues/3
+"""
+
+# Abbreviations that can END a sentence and be followed by a new sentence.
+# When these are followed by a capital letter, it likely indicates a sentence break.
+from typing import List
+
+SENTENCE_ENDING_ABBREVIATIONS: List[str] = [
+    # Time
+    "a.m.",
+    "p.m.",
+    "A.M.",
+    "P.M.",
+
+    # Common sentence-enders
+    "etc.",
+    "ext.",
+
+    # Academic degrees (when at end of sentence)
+    "Ph.D.",
+    "M.D.",
+    "B.A.",
+    "B.S.",
+    "M.A.",
+    "M.S.",
+    "Ed.D.",
+    "J.D.",
+    "D.D.S.",
+    "R.N.",
+
+    # Business (when at end of sentence)
+    "Inc.",
+    "Corp.",
+    "Ltd.",
+    "Co.",
+    "Bros.",
+
+    # Countries/Regions (when at end of sentence)
+    "U.S.",
+    "U.S.A.",
+    "U.K.",
+    "U.N.",
+    "E.U.",
+    "D.C.",
+]
+
+# Abbreviations that are NEVER sentence-enders because they're
+# typically followed by a name or noun (e.g., "Dr. Smith", "Mt. Everest").
+# Do NOT split after these even when followed by a capital letter.
+TITLE_ABBREVIATIONS: List[str] = [
+    # Personal titles
+    "Dr.",
+    "Mr.",
+    "Mrs.",
+    "Ms.",
+    "Prof.",
+    "Sr.",
+    "Jr.",
+    "Rev.",
+    "Gen.",
+    "Col.",
+    "Capt.",
+    "Lt.",
+    "Sgt.",
+    "Rep.",
+    "Sen.",
+    "Gov.",
+    "Pres.",
+    "Hon.",
+
+    # Geographic prefixes
+    "St.",
+    "Mt.",
+    "Ft.",
+
+    # Other prefixes
+    "Fig.",
+    "fig.",
+    "Sec.",
+    "sec.",
+    "Ch.",
+    "ch.",
+    "Art.",
+    "art.",
+    "Vol.",
+    "vol.",
+    "No.",
+    "no.",
+    "Pt.",
+    "pt.",
+]