fast-sentence-segment 1.1.8__py3-none-any.whl → 1.2.0__py3-none-any.whl

fast_sentence_segment/cli.py

@@ -0,0 +1,56 @@
+# -*- coding: UTF-8 -*-
+"""CLI for fast-sentence-segment."""
+
+import argparse
+import logging
+import sys
+
+from fast_sentence_segment import segment_text
+
+logging.disable(logging.CRITICAL)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="segment",
+        description="Segment text into sentences",
+    )
+    parser.add_argument(
+        "text",
+        nargs="?",
+        help="Text to segment (or use stdin)",
+    )
+    parser.add_argument(
+        "-f", "--file",
+        help="Read text from file",
+    )
+    parser.add_argument(
+        "-n", "--numbered",
+        action="store_true",
+        help="Number output lines",
+    )
+    args = parser.parse_args()
+
+    # Get input text
+    if args.file:
+        with open(args.file, "r", encoding="utf-8") as f:
+            text = f.read()
+    elif args.text:
+        text = args.text
+    elif not sys.stdin.isatty():
+        text = sys.stdin.read()
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    # Segment and output
+    sentences = segment_text(text.strip(), flatten=True)
+    for i, sentence in enumerate(sentences, 1):
+        if args.numbered:
+            print(f"{i}. {sentence}")
+        else:
+            print(sentence)
+
+
+if __name__ == "__main__":
+    main()

fast_sentence_segment/dmo/__init__.py

@@ -1,6 +1,10 @@
+from .abbreviation_merger import AbbreviationMerger
+from .abbreviation_splitter import AbbreviationSplitter
+from .title_name_merger import TitleNameMerger
 from .bullet_point_cleaner import BulletPointCleaner
-from .delimiters_to_periods import DelimitersToPeriods
+from .ellipsis_normalizer import EllipsisNormalizer
 from .newlines_to_periods import NewlinesToPeriods
 from .post_process_sentences import PostProcessStructure
+from .question_exclamation_splitter import QuestionExclamationSplitter
 from .spacy_doc_segmenter import SpacyDocSegmenter
 from .numbered_list_normalizer import NumberedListNormalizer

fast_sentence_segment/dmo/abbreviation_merger.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+Merge sentences that spaCy incorrectly split at abbreviation boundaries.
+
+When spaCy incorrectly splits after an abbreviation (e.g., "ext. 5" becomes
+["ext.", "5. Ask for help."]), this component merges them back together
+using specific known patterns.
+
+Reference: https://github.com/craigtrim/fast-sentence-segment/issues/3
+"""
+
+import re
+from typing import List, Optional, Tuple
+
+from fast_sentence_segment.core import BaseObject
+
+
+# Patterns where spaCy incorrectly splits after an abbreviation.
+# Format: (ending_pattern, extract_pattern)
+#   - ending_pattern: regex to match end of current sentence
+#   - extract_pattern: regex to extract the portion to merge from next sentence
+#
+# The extract_pattern MUST have a capture group for the portion to merge.
+# Whatever is NOT captured remains as a separate sentence.
+
+MERGE_PATTERNS: List[Tuple[str, str]] = [
+
+    # ext. 5, Ext. 123, EXT. 42
+    (r"(?i)\bext\.$", r"^(\d+\.?)\s*"),
+
+    # no. 5, No. 42, NO. 100
+    (r"(?i)\bno\.$", r"^(\d+\.?)\s*"),
+
+    # vol. 3, Vol. 42, VOL. 1
+    (r"(?i)\bvol\.$", r"^(\d+\.?)\s*"),
+
+    # pt. 2, Pt. 1, PT. 3
+    (r"(?i)\bpt\.$", r"^(\d+\.?)\s*"),
+
+    # ch. 5, Ch. 10, CH. 3
+    (r"(?i)\bch\.$", r"^(\d+\.?)\s*"),
+
+    # sec. 3, Sec. 14, SEC. 2
+    (r"(?i)\bsec\.$", r"^(\d+(?:\.\d+)?\.?)\s*"),
+
+    # fig. 1, Fig. 3.2, FIG. 10
+    (r"(?i)\bfig\.$", r"^(\d+(?:\.\d+)?\.?)\s*"),
+
+    # p. 42, P. 100
+    (r"(?i)\bp\.$", r"^(\d+\.?)\s*"),
+
+    # pp. 42-50, PP. 100-110
+    (r"(?i)\bpp\.$", r"^(\d+(?:-\d+)?\.?)\s*"),
+
+    # art. 5, Art. 12, ART. 1
+    (r"(?i)\bart\.$", r"^(\d+\.?)\s*"),
+
+]
+
+
+class AbbreviationMerger(BaseObject):
+    """Merge sentences incorrectly split at abbreviation boundaries."""
+
+    def __init__(self):
+        """
+        Created:
+            27-Dec-2024
+            craigtrim@gmail.com
+        Reference:
+            https://github.com/craigtrim/fast-sentence-segment/issues/3
+        """
+        BaseObject.__init__(self, __name__)
+        # Compile patterns for efficiency
+        self._patterns = [
+            (re.compile(ending), re.compile(extract))
+            for ending, extract in MERGE_PATTERNS
+        ]
+
+    def _try_merge(self, current: str, next_sent: str) -> Optional[Tuple[str, str]]:
+        """Try to merge two sentences based on known patterns.
+
+        Args:
+            current: Current sentence
+            next_sent: Next sentence
+
+        Returns:
+            Tuple of (merged_sentence, remainder) if merge needed, else None
+        """
+        current = current.strip()
+        next_sent = next_sent.strip()
+
+        for ending_pattern, extract_pattern in self._patterns:
+            if ending_pattern.search(current):
+                match = extract_pattern.match(next_sent)
+                if match:
+                    # Extract the portion to merge
+                    extracted = match.group(1)
+                    # Get the remainder (everything after the match)
+                    remainder = next_sent[match.end():].strip()
+                    # Build merged sentence
+                    merged = current + " " + extracted
+                    return (merged, remainder)
+
+        return None
+
+    def process(self, sentences: List[str]) -> List[str]:
+        """Process a list of sentences, merging incorrectly split ones.
+
+        Args:
+            sentences: List of sentences from spaCy
+
+        Returns:
+            List of sentences with incorrect splits merged
+        """
+        if not sentences:
+            return sentences
+
+        result = []
+        i = 0
+
+        while i < len(sentences):
+            current = sentences[i]
+
+            # Check if we should merge with next sentence
+            if i + 1 < len(sentences):
+                next_sent = sentences[i + 1]
+                merge_result = self._try_merge(current, next_sent)
+
+                if merge_result:
+                    merged, remainder = merge_result
+                    result.append(merged)
+
+                    # If there's a remainder, it becomes a new sentence to process
+                    if remainder:
+                        # Insert remainder back for processing
+                        sentences = sentences[:i+2] + [remainder] + sentences[i+2:]
+                        sentences[i+1] = remainder
+
+                    i += 2
+                    continue
+
+            result.append(current)
+            i += 1
+
+        return result

fast_sentence_segment/dmo/abbreviation_splitter.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+Split sentences at abbreviation boundaries.
+
+When spaCy fails to detect a sentence boundary after an abbreviation
+(e.g., "I woke at 6 a.m. It was dark."), this component splits the
+sentence by detecting the pattern: abbreviation + space + Capital letter.
+
+Reference: https://github.com/craigtrim/fast-sentence-segment/issues/3
+"""
+
+import re
+from typing import List
+
+from fast_sentence_segment.core import BaseObject
+from fast_sentence_segment.dmo.abbreviations import SENTENCE_ENDING_ABBREVIATIONS
+
+
+class AbbreviationSplitter(BaseObject):
+    """Split sentences at abbreviation boundaries."""
+
+    def __init__(self):
+        """
+        Created:
+            27-Dec-2024
+            craigtrim@gmail.com
+        Reference:
+            https://github.com/craigtrim/fast-sentence-segment/issues/3
+        """
+        BaseObject.__init__(self, __name__)
+        self._pattern = self._build_pattern()
+
+    def _build_pattern(self) -> re.Pattern:
+        """Build regex pattern to match abbreviation + capital letter.
+
+        Pattern matches:
+        - A known sentence-ending abbreviation (escaped for regex)
+        - Followed by one or more spaces
+        - Followed by a capital letter (start of new sentence)
+
+        Note: Title abbreviations (Dr., Mr., etc.) are excluded because
+        they are typically followed by names, not new sentences.
+
+        Returns:
+            Compiled regex pattern
+        """
+        escaped_abbrevs = [re.escape(abbr) for abbr in SENTENCE_ENDING_ABBREVIATIONS]
+        abbrev_pattern = "|".join(escaped_abbrevs)
+        pattern = rf"({abbrev_pattern})\s+([A-Z])"
+        return re.compile(pattern)
+
+    def _split_sentence(self, sentence: str) -> List[str]:
+        """Split a single sentence at abbreviation boundaries.
+
+        Args:
+            sentence: A sentence that may contain abbreviation boundaries
+
+        Returns:
+            List of one or more sentences
+        """
+        results = []
+        remaining = sentence
+
+        while True:
+            match = self._pattern.search(remaining)
+            if not match:
+                if remaining.strip():
+                    results.append(remaining.strip())
+                break
+
+            split_pos = match.end(1)
+
+            before = remaining[:split_pos].strip()
+            if before:
+                results.append(before)
+
+            remaining = remaining[split_pos:].strip()
+
+        return results if results else [sentence]
+
+    def process(self, sentences: List[str]) -> List[str]:
+        """Process a list of sentences, splitting at abbreviation boundaries.
+
+        Args:
+            sentences: List of sentences from spaCy
+
+        Returns:
+            List of sentences with abbreviation boundaries properly split
+        """
+        result = []
+        for sentence in sentences:
+            split_sentences = self._split_sentence(sentence)
+            result.extend(split_sentences)
+        return result

fast_sentence_segment/dmo/abbreviations.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+Known abbreviations that end with periods.
+
+Reference: https://github.com/craigtrim/fast-sentence-segment/issues/3
+"""
+
+# Abbreviations that can END a sentence and be followed by a new sentence.
+# When these are followed by a capital letter, it likely indicates a sentence break.
+from typing import List
+
+SENTENCE_ENDING_ABBREVIATIONS: List[str] = [
+    # Time
+    "a.m.",
+    "p.m.",
+    "A.M.",
+    "P.M.",
+
+    # Common sentence-enders
+    "etc.",
+    "ext.",
+
+    # Academic degrees (when at end of sentence)
+    "Ph.D.",
+    "M.D.",
+    "B.A.",
+    "B.S.",
+    "M.A.",
+    "M.S.",
+    "Ed.D.",
+    "J.D.",
+    "D.D.S.",
+    "R.N.",
+
+    # Business (when at end of sentence)
+    "Inc.",
+    "Corp.",
+    "Ltd.",
+    "Co.",
+    "Bros.",
+
+    # Countries/Regions (when at end of sentence)
+    "U.S.",
+    "U.S.A.",
+    "U.K.",
+    "U.N.",
+    "E.U.",
+    "D.C.",
+]
+
+# Abbreviations that are NEVER sentence-enders because they're
+# typically followed by a name or noun (e.g., "Dr. Smith", "Mt. Everest").
+# Do NOT split after these even when followed by a capital letter.
+TITLE_ABBREVIATIONS: List[str] = [
+    # Personal titles
+    "Dr.",
+    "Mr.",
+    "Mrs.",
+    "Ms.",
+    "Prof.",
+    "Sr.",
+    "Jr.",
+    "Rev.",
+    "Gen.",
+    "Col.",
+    "Capt.",
+    "Lt.",
+    "Sgt.",
+    "Rep.",
+    "Sen.",
+    "Gov.",
+    "Pres.",
+    "Hon.",
+
+    # Geographic prefixes
+    "St.",
+    "Mt.",
+    "Ft.",
+
+    # Other prefixes
+    "Fig.",
+    "fig.",
+    "Sec.",
+    "sec.",
+    "Ch.",
+    "ch.",
+    "Art.",
+    "art.",
+    "Vol.",
+    "vol.",
+    "No.",
+    "no.",
+    "Pt.",
+    "pt.",
+]

fast_sentence_segment/dmo/ellipsis_normalizer.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Normalize Ellipses to prevent them being stripped by cleanup routines """
+
+
+import re
+
+from fast_sentence_segment.core import BaseObject
+
+
+PLACEHOLDER = "〈ELLIPSIS〉"
+
+# Pattern: ... followed by space and capital letter
+BOUNDARY_PATTERN = re.compile(r'\.\.\.(\s+)([A-Z])')
+
+
+class EllipsisNormalizer(BaseObject):
+    """ Normalize Ellipses to prevent them being stripped by cleanup routines """
+
+    def __init__(self):
+        """
+        Created:
+            27-Dec-2024
+            craigtrim@gmail.com
+            *   preserve ellipses through the pipeline
+                https://github.com/craigtrim/fast-sentence-segment/issues/3
+        """
+        BaseObject.__init__(self, __name__)
+
+    def process(self,
+                input_text: str,
+                denormalize: bool = False) -> str:
+
+        if not denormalize:
+            # "... [Capital]" → "〈ELLIPSIS〉. [Capital]" (adds period for spaCy to split)
+            input_text = BOUNDARY_PATTERN.sub(PLACEHOLDER + r'.\1\2', input_text)
+            # Remaining ellipses (mid-sentence): "..." → "〈ELLIPSIS〉"
+            input_text = input_text.replace("...", PLACEHOLDER)
+        else:
+            # "〈ELLIPSIS〉." → "..." (remove extra period added for boundary)
+            input_text = input_text.replace(PLACEHOLDER + ".", "...")
+            # Remaining placeholders: "〈ELLIPSIS〉" → "..."
+            input_text = input_text.replace(PLACEHOLDER, "...")
+
+        return input_text

fast_sentence_segment/dmo/numbered_list_normalizer.py

@@ -3,24 +3,21 @@
 """ Normalize Numbered Lists to prevent False Positive Segmentation """
 
 
+import re
+
 from fast_sentence_segment.core import BaseObject
 
 
 class NumberedListNormalizer(BaseObject):
     """ Normalize Numbered Lists to prevent False Positive Segmentation """
 
-    __d_candidate_list_elements = {
-        "1. ": "1_ ",
-        "2. ": "2_ ",
-        "3. ": "3_ ",
-        "4. ": "4_ ",
-        "5. ": "5_ ",
-        "6. ": "6_ ",
-        "7. ": "7_ ",
-        "8. ": "8_ ",
-        "9. ": "9_ ",
-        "10. ": "10_ ",
-    }
+    # Pattern 1: start of string OR newline, followed by number, period, space
+    __normalize_line_start = re.compile(r'(^|\n\s*)(\d{1,2})\. ')
+    __denormalize_line_start = re.compile(r'(^|\n\s*)(\d{1,2})_ ')
+
+    # Pattern 2: inline numbered list ". N. " (period + space + number + period + space)
+    __normalize_inline = re.compile(r'(\. )(\d{1,2})\. ')
+    __denormalize_inline = re.compile(r'(\. )(\d{1,2})_ ')
 
     def __init__(self):
         """
@@ -28,6 +25,11 @@ class NumberedListNormalizer(BaseObject):
             19-Oct-2022
             craigtrim@gmail.com
             *   https://github.com/craigtrim/fast-sentence-segment/issues/1
+        Updated:
+            27-Dec-2024
+            craigtrim@gmail.com
+            *   fix to only match at line starts, not mid-sentence
+                https://github.com/craigtrim/fast-sentence-segment/issues/3
         """
         BaseObject.__init__(self, __name__)
 
@@ -36,18 +38,10 @@ class NumberedListNormalizer(BaseObject):
                 denormalize: bool = False) -> str:
 
         if not denormalize:
-            for candidate in self.__d_candidate_list_elements:
-                if candidate in input_text:
-                    input_text = input_text.replace(
-                        candidate, self.__d_candidate_list_elements[candidate])
-
-        else:  # reverse the process
-            d_rev = {self.__d_candidate_list_elements[k]: k
-                     for k in self.__d_candidate_list_elements}
-
-            for candidate in d_rev:
-                if candidate in input_text:
-                    input_text = input_text.replace(
-                        candidate, d_rev[candidate])
+            input_text = self.__normalize_line_start.sub(r'\1\2_ ', input_text)
+            input_text = self.__normalize_inline.sub(r'\1\2_ ', input_text)
+        else:
+            input_text = self.__denormalize_line_start.sub(r'\1\2. ', input_text)
+            input_text = self.__denormalize_inline.sub(r'\1\2. ', input_text)
 
         return input_text

fast_sentence_segment/dmo/question_exclamation_splitter.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Split sentences at ? and ! followed by capital letter """
+
+
+import re
+from typing import List
+
+from fast_sentence_segment.core import BaseObject
+
+
+# Pattern: ? or ! followed by space and capital letter
+BOUNDARY_PATTERN = re.compile(r'([?!])(\s+)([A-Z])')
+
+
+class QuestionExclamationSplitter(BaseObject):
+    """ Split sentences at ? and ! followed by capital letter """
+
+    def __init__(self):
+        """
+        Created:
+            27-Dec-2024
+            craigtrim@gmail.com
+            *   spaCy doesn't always split on ? and ! boundaries
+                https://github.com/craigtrim/fast-sentence-segment/issues/3
+        """
+        BaseObject.__init__(self, __name__)
+
+    def process(self, sentences: List[str]) -> List[str]:
+        """Split sentences that contain ? or ! followed by capital letter.
+
+        Args:
+            sentences: List of sentences from earlier processing
+
+        Returns:
+            List of sentences with ? and ! boundaries split
+        """
+        result = []
+        for sent in sentences:
+            # Split on pattern, keeping the punctuation with the first part
+            parts = BOUNDARY_PATTERN.split(sent)
+            if len(parts) == 1:
+                result.append(sent)
+            else:
+                # Reassemble: parts = [before, punct, space, capital, after, ...]
+                i = 0
+                while i < len(parts):
+                    if i + 3 < len(parts):
+                        # before + punct
+                        result.append(parts[i] + parts[i + 1])
+                        # capital + rest will be handled in next iteration
+                        parts[i + 4] = parts[i + 3] + parts[i + 4] if i + 4 < len(parts) else parts[i + 3]
+                        i += 4
+                    else:
+                        if parts[i].strip():
+                            result.append(parts[i])
+                        i += 1
+
+        return [s.strip() for s in result if s.strip()]

fast_sentence_segment/dmo/title_name_merger.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+Merge sentences incorrectly split when a title abbreviation is followed by a
+single-word name ending in punctuation.
+
+When the question/exclamation splitter splits "Dr. Who?" into ["Dr.", "Who?"],
+this component merges them back together because a title + single capitalized
+word is almost certainly a name, not two sentences.
+
+Examples that should be merged:
+    ["Dr.", "Who?"] -> ["Dr. Who?"]
+    ["Mr.", "T!"] -> ["Mr. T!"]
+    ["Do you like Dr.", "Who?"] -> ["Do you like Dr. Who?"]
+
+Examples that should NOT be merged:
+    ["Dr.", "Where did he go?"] -> stays split (multi-word sentence)
+    ["Dr.", "who can help."] -> stays split (lowercase = not a name)
+
+Reference: https://github.com/craigtrim/fast-sentence-segment/issues/3
+"""
+
+import re
+from typing import List, Optional, Tuple
+
+from fast_sentence_segment.core import BaseObject
+from fast_sentence_segment.dmo.abbreviations import TITLE_ABBREVIATIONS
+
+
+# Subset of titles that commonly precede names (not geographic like Mt., St.)
+PERSONAL_TITLES: List[str] = [
+    "Dr.",
+    "Mr.",
+    "Mrs.",
+    "Ms.",
+    "Prof.",
+    "Sr.",
+    "Jr.",
+    "Rev.",
+    "Gen.",
+    "Col.",
+    "Capt.",
+    "Lt.",
+    "Sgt.",
+    "Rep.",
+    "Sen.",
+    "Gov.",
+    "Pres.",
+    "Hon.",
+]
+
+
+class TitleNameMerger(BaseObject):
+    """Merge sentences incorrectly split at title + single-word name boundaries."""
+
+    def __init__(self):
+        """
+        Created:
+            28-Dec-2024
+            craigtrim@gmail.com
+        Reference:
+            https://github.com/craigtrim/fast-sentence-segment/issues/3
+        """
+        BaseObject.__init__(self, __name__)
+
+        # Build pattern to match sentences ending with a title abbreviation
+        # Escape dots in abbreviations for regex
+        escaped_titles = [re.escape(t) for t in PERSONAL_TITLES]
+        titles_pattern = "|".join(escaped_titles)
+        self._ending_with_title = re.compile(rf"({titles_pattern})$", re.IGNORECASE)
+
+        # Pattern to match a single capitalized word followed by sentence-ending punctuation
+        # at the START of a sentence (may have more content after)
+        # Matches: "Who?", "Who? More text", "T!", "T! More", "Who?." (with trailing period), etc.
+        # Captures the word+punctuation part for extraction
+        # Note: The spaCy segmenter may add a trailing period to sentences ending in ?/!
+        self._single_word_with_punct = re.compile(r"^([A-Z][a-zA-Z\-]*[?!]+\.?)\s*(.*)$")
+
+    def _try_merge(self, current: str, next_sent: str) -> Optional[Tuple[str, str]]:
+        """Try to merge two sentences if they match the title + single-word name pattern.
+
+        Args:
+            current: Current sentence (may end with title abbreviation)
+            next_sent: Next sentence (may start with single-word name with punctuation)
+
+        Returns:
+            Tuple of (merged_sentence, remainder) if merge needed, else None
+        """
+        current = current.strip()
+        next_sent = next_sent.strip()
+
+        # Current sentence must end with a title abbreviation
+        if not self._ending_with_title.search(current):
+            return None
+
+        # Next sentence must start with a single capitalized word with ?/! punctuation
+        match = self._single_word_with_punct.match(next_sent)
+        if not match:
+            return None
+
+        # Extract the name part and any remainder
+        name_part = match.group(1)
+        remainder = match.group(2).strip() if match.group(2) else ""
+
+        # Clean up trailing period from name if present (added by spaCy)
+        if name_part.endswith('?.') or name_part.endswith('!.'):
+            name_part = name_part[:-1]
+
+        merged = current + " " + name_part
+        return (merged, remainder)
+
+    def process(self, sentences: List[str]) -> List[str]:
+        """Process a list of sentences, merging title + single-word name splits.
+
+        Args:
+            sentences: List of sentences
+
+        Returns:
+            List of sentences with title+name splits merged
+        """
+        if not sentences or len(sentences) < 2:
+            return sentences
+
+        # Work with a mutable copy
+        sentences = list(sentences)
+        result = []
+        i = 0
+
+        while i < len(sentences):
+            current = sentences[i]
+
+            # Check if we should merge with next sentence
+            if i + 1 < len(sentences):
+                next_sent = sentences[i + 1]
+                merge_result = self._try_merge(current, next_sent)
+
+                if merge_result:
+                    merged, remainder = merge_result
+                    result.append(merged)
+
+                    # If there's a remainder, replace next_sent with it for further processing
+                    if remainder:
+                        sentences[i + 1] = remainder
+                        i += 1  # Move to process the remainder (now at i+1, will be i after increment)
+                    else:
+                        i += 2  # Skip both merged sentences
+                    continue
+
+            result.append(current)
+            i += 1
+
+        return result

fast_sentence_segment/svc/perform_sentence_segmentation.py

@@ -7,10 +7,14 @@ import spacy
 
 from fast_sentence_segment.core import BaseObject
 
+from fast_sentence_segment.dmo import AbbreviationMerger
+from fast_sentence_segment.dmo import AbbreviationSplitter
+from fast_sentence_segment.dmo import TitleNameMerger
+from fast_sentence_segment.dmo import EllipsisNormalizer
 from fast_sentence_segment.dmo import NewlinesToPeriods
-from fast_sentence_segment.dmo import DelimitersToPeriods
 from fast_sentence_segment.dmo import BulletPointCleaner
 from fast_sentence_segment.dmo import NumberedListNormalizer
+from fast_sentence_segment.dmo import QuestionExclamationSplitter
 from fast_sentence_segment.dmo import SpacyDocSegmenter
 from fast_sentence_segment.dmo import PostProcessStructure
 
@@ -31,18 +35,38 @@ class PerformSentenceSegmentation(BaseObject):
             craigtrim@gmail.com
             *   add numbered-list normalization
                 https://github.com/craigtrim/fast-sentence-segment/issues/1
+        Updated:
+            27-Dec-2024
+            craigtrim@gmail.com
+            *   add abbreviation-aware sentence splitting
+                https://github.com/craigtrim/fast-sentence-segment/issues/3
         """
         BaseObject.__init__(self, __name__)
         if not self.__nlp:
             self.__nlp = spacy.load("en_core_web_sm")
 
-        self._delimiters_to_periods = DelimitersToPeriods.process
         self._newlines_to_periods = NewlinesToPeriods.process
         self._normalize_numbered_lists = NumberedListNormalizer().process
+        self._normalize_ellipses = EllipsisNormalizer().process
         self._clean_bullet_points = BulletPointCleaner.process
         self._spacy_segmenter = SpacyDocSegmenter(self.__nlp).process
+        self._abbreviation_merger = AbbreviationMerger().process
+        self._abbreviation_splitter = AbbreviationSplitter().process
+        self._question_exclamation_splitter = QuestionExclamationSplitter().process
+        self._title_name_merger = TitleNameMerger().process
         self._post_process = PostProcessStructure().process
 
+    def _denormalize(self, text: str) -> str:
+        """ Restore normalized placeholders to original form """
+        text = self._normalize_numbered_lists(text, denormalize=True)
+        text = self._normalize_ellipses(text, denormalize=True)
+        return text
+
+    @staticmethod
+    def _has_sentence_punct(text: str) -> bool:
+        """ Check if text has sentence-ending punctuation """
+        return "." in text or "?" in text or "!" in text
+
     @staticmethod
     def _clean_punctuation(input_text: str) -> str:
         """ Purpose:
@@ -67,33 +91,41 @@ class PerformSentenceSegmentation(BaseObject):
     def _process(self,
                  input_text: str) -> list:
 
-        input_text = self._delimiters_to_periods(
-            delimiter=',',
-            input_text=input_text)
-
-        input_text = self._delimiters_to_periods(
-            delimiter=';',
-            input_text=input_text)
+        # Normalize tabs to spaces
+        input_text = input_text.replace('\t', ' ')
 
         input_text = self._normalize_numbered_lists(input_text)
+        input_text = self._normalize_ellipses(input_text)
 
         input_text = self._newlines_to_periods(input_text)
 
         input_text = self._clean_spacing(input_text)
-        if "." not in input_text:
-            return [input_text]
+        if not self._has_sentence_punct(input_text):
+            return [self._denormalize(input_text)]
 
         input_text = self._clean_bullet_points(input_text)
-        if "." not in input_text:
-            return [input_text]
+        if not self._has_sentence_punct(input_text):
+            return [self._denormalize(input_text)]
 
         input_text = self._clean_punctuation(input_text)
-        if "." not in input_text:
-            return [input_text]
+        if not self._has_sentence_punct(input_text):
+            return [self._denormalize(input_text)]
 
         sentences = self._spacy_segmenter(input_text)
-        if "." not in input_text:
-            return [input_text]
+        if not self._has_sentence_punct(input_text):
+            return [self._denormalize(input_text)]
+
+        # Merge sentences incorrectly split at abbreviations (issue #3)
+        sentences = self._abbreviation_merger(sentences)
+
+        # Merge title + single-word name splits (e.g., "Dr." + "Who?" -> "Dr. Who?")
+        sentences = self._title_name_merger(sentences)
+
+        # Split sentences at abbreviation boundaries (issue #3)
+        sentences = self._abbreviation_splitter(sentences)
+
+        # Split sentences at ? and ! boundaries (issue #3)
+        sentences = self._question_exclamation_splitter(sentences)
 
         sentences = self._post_process(sentences)
 
@@ -101,6 +133,10 @@ class PerformSentenceSegmentation(BaseObject):
             self._normalize_numbered_lists(x, denormalize=True)
             for x in sentences
         ]
+        sentences = [
+            self._normalize_ellipses(x, denormalize=True)
+            for x in sentences
+        ]
 
         return sentences
 

{fast_sentence_segment-1.1.8.dist-info → fast_sentence_segment-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fast-sentence-segment
-Version: 1.1.8
+Version: 1.2.0
 Summary: Fast and Efficient Sentence Segmentation
 License: MIT
 License-File: LICENSE
@@ -30,17 +30,25 @@ Description-Content-Type: text/markdown
 [![PyPI version](https://img.shields.io/pypi/v/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
 [![Python versions](https://img.shields.io/pypi/pyversions/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![spaCy](https://img.shields.io/badge/spaCy-3.5-blue.svg)](https://spacy.io/)
+[![spaCy](https://img.shields.io/badge/spaCy-3.8-blue.svg)](https://spacy.io/)
 
-Fast and efficient sentence segmentation using spaCy. Handles complex edge cases like abbreviations (Dr., Mr., etc.), quoted text, and multi-paragraph documents.
+Fast and efficient sentence segmentation using spaCy with surgical post-processing fixes. Handles complex edge cases like abbreviations (Dr., Mr., etc.), ellipses, quoted text, and multi-paragraph documents.
+
+## Why This Library?
+
+1. **Keep it local**: LLM API calls cost money and send your data to third parties. Run sentence segmentation entirely on your machine.
+2. **spaCy perfected**: spaCy is a great local model, but it makes mistakes. This library fixes most of spaCy's shortcomings.
 
 ## Features
 
 - **Paragraph-aware segmentation**: Returns sentences grouped by paragraph
-- **Abbreviation handling**: Correctly handles "Dr.", "Mr.", "etc." without false splits
+- **Abbreviation handling**: Correctly handles "Dr.", "Mr.", "etc.", "p.m.", "a.m." without false splits
+- **Ellipsis preservation**: Keeps `...` intact while detecting sentence boundaries
+- **Question/exclamation splitting**: Properly splits on `?` and `!` followed by capital letters
 - **Cached processing**: LRU cache for repeated text processing
 - **Flexible output**: Nested lists (by paragraph) or flattened list of sentences
 - **Bullet point & numbered list normalization**: Cleans common list formats
+- **CLI tool**: Command-line interface for quick segmentation
 
 ## Installation
 
@@ -106,6 +114,24 @@ segmenter = Segmenter()
 results = segmenter.input_text("Your text here.")
 ```
 
+### Command Line Interface
+
+Segment text directly from the terminal:
+
+```bash
+# Direct text input
+segment "Hello world. How are you? I am fine."
+
+# Numbered output
+segment -n "First sentence. Second sentence."
+
+# From stdin
+echo "Some text here. Another sentence." | segment
+
+# From file
+segment -f document.txt
+```
+
 ## API Reference
 
 | Function | Parameters | Returns | Description |
@@ -113,6 +139,14 @@ results = segmenter.input_text("Your text here.")
 | `segment_text()` | `input_text: str`, `flatten: bool = False` | `list` | Main entry point for segmentation |
 | `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |
 
+### CLI Options
+
+| Option | Description |
+|--------|-------------|
+| `text` | Text to segment (positional argument) |
+| `-f, --file` | Read text from file |
+| `-n, --numbered` | Number output lines |
+
 ## Why Nested Lists?
 
 The segmentation process preserves document structure by segmenting into both paragraphs and sentences. Each outer list represents a paragraph, and each inner list contains that paragraph's sentences. This is useful for:
@@ -125,10 +159,19 @@ Use `flatten=True` when you only need sentences without paragraph context.
 
 ## Requirements
 
-- Python 3.8.5+
-- spaCy 3.5.3
+- Python 3.9+
+- spaCy 3.8+
 - en_core_web_sm spaCy model
 
+## How It Works
+
+This library uses spaCy for initial sentence segmentation, then applies surgical post-processing fixes for cases where spaCy's default behavior is incorrect:
+
+1. **Pre-processing**: Normalize numbered lists, preserve ellipses with placeholders
+2. **spaCy segmentation**: Use spaCy's sentence boundary detection
+3. **Post-processing**: Split on abbreviation boundaries, handle `?`/`!` + capital patterns
+4. **Denormalization**: Restore placeholders to original text
+
 ## License
 
 MIT License - see [LICENSE](LICENSE) for details.

fast_sentence_segment-1.2.0.dist-info/RECORD

@@ -0,0 +1,27 @@
+fast_sentence_segment/__init__.py,sha256=HTONyC0JLVWTAHyvJO6rMINmxymbfMpARtGeRw5iIsQ,359
+fast_sentence_segment/bp/__init__.py,sha256=j2-WfQ9WwVuXeGSjvV6XLVwEdvau8sdAQe4Pa4DrYi8,33
+fast_sentence_segment/bp/segmenter.py,sha256=UW6DguPgA56h-pPYRsfJhjIzBe40j6NdjkwYxamASyA,1928
+fast_sentence_segment/cli.py,sha256=X2dMLkfc2dtheig62wKC75AohfW0Y9oTU0ORhGUFkbQ,1250
+fast_sentence_segment/core/__init__.py,sha256=uoBersYyVStJ5a8zJpQz1GDGaloEdAv2jGHw1292hRM,108
+fast_sentence_segment/core/base_object.py,sha256=AYr7yzusIwawjbKdvcv4yTEnhmx6M583kDZzhzPOmq4,635
+fast_sentence_segment/core/stopwatch.py,sha256=hE6hMz2q6rduaKi58KZmiAL-lRtyh_wWCANhl4KLkRQ,879
+fast_sentence_segment/dmo/__init__.py,sha256=E70tkpdHu86KP2dwBX5Dy5D7eNiU6fzucrfDJOY1ui4,551
+fast_sentence_segment/dmo/abbreviation_merger.py,sha256=tCXM6yCfMryJvMIVWIxP_EocoibZi8vohFzJ5tvMYr0,4432
+fast_sentence_segment/dmo/abbreviation_splitter.py,sha256=03mSyJcLooNyIjXx6mPlrnjmKgZW-uhUIqG4U-MbIGw,2981
+fast_sentence_segment/dmo/abbreviations.py,sha256=7mpEoOnw5MH8URYmmpxaYs3Wc2eqy4pC0hAnYfYSdck,1639
+fast_sentence_segment/dmo/bullet_point_cleaner.py,sha256=WOZQRWXiiyRi8rOuEIw36EmkaXmATHL9_Dxb2rderw4,1606
+fast_sentence_segment/dmo/ellipsis_normalizer.py,sha256=lHs9dLFfKJe-2vFNe17Hik90g3_kXX347OzGP_IOT08,1521
+fast_sentence_segment/dmo/newlines_to_periods.py,sha256=PUrXreqZWiITINfoJL5xRRlXJH6noH0cdXtW1EqAh8I,1517
+fast_sentence_segment/dmo/numbered_list_normalizer.py,sha256=q0sOCW8Jkn2vTXlUcVhmDvYES3yvJx1oUVl_8y7eL4E,1672
+fast_sentence_segment/dmo/post_process_sentences.py,sha256=5jxG3TmFjxIExMPLhnCB5JT1lXQvFU9r4qQGoATGrWk,916
+fast_sentence_segment/dmo/question_exclamation_splitter.py,sha256=cRsWRu8zb6wOWG-BjMahHfz4YGutKiV9lW7dE-q3tgc,2006
+fast_sentence_segment/dmo/spacy_doc_segmenter.py,sha256=0icAkSQwAUQo3VYqQ2PUjW6-MOU5RNCGPX3-fB5YfCc,2554
+fast_sentence_segment/dmo/title_name_merger.py,sha256=zbG04_VjwM8TtT8LhavvmZqIZL_2xgT2OTxWkK_Zt1s,5133
+fast_sentence_segment/svc/__init__.py,sha256=9B12mXxBnlalH4OAm1AMLwUMa-RLi2ilv7qhqv26q7g,144
+fast_sentence_segment/svc/perform_paragraph_segmentation.py,sha256=zLKw9rSzb0NNfx4MyEeoGrHwhxTtH5oDrYcAL2LMVHY,1378
+fast_sentence_segment/svc/perform_sentence_segmentation.py,sha256=dqGxFsJoP6ox_MJwtB85R9avEbBAR4x9YKaRaQ5fAXo,5723
+fast_sentence_segment-1.2.0.dist-info/METADATA,sha256=05V3aFKHCD9JaYN8va_vIuMtaoAbGmKgFAOUDJWfM80,6405
+fast_sentence_segment-1.2.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+fast_sentence_segment-1.2.0.dist-info/entry_points.txt,sha256=mDiRuKOZlOeqmtH1eZwqGEGM6KUh0RTzwyETGMpxSDI,58
+fast_sentence_segment-1.2.0.dist-info/licenses/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
+fast_sentence_segment-1.2.0.dist-info/RECORD,,

fast_sentence_segment-1.2.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+segment=fast_sentence_segment.cli:main
+

fast_sentence_segment/dmo/delimiters_to_periods.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Convert Delimiters into Periods """
-
-
-from fast_sentence_segment.core import BaseObject
-
-
-class DelimitersToPeriods(BaseObject):
-    """ Convert Delimiters into Periods """
-
-    def __init__(self):
-        """
-        Created:
-            30-Sept-2021
-        """
-        BaseObject.__init__(self, __name__)
-
-    @staticmethod
-    def process(input_text: str,
-                delimiter: str):
-        """
-        Purpose:
-            Take a CSV list and transform to sentences
-        :param input_text:
-        :return:
-        """
-        total_len = len(input_text)
-        total_delims = input_text.count(delimiter)
-
-        if total_delims == 0:
-            return input_text
-
-        if total_delims / total_len > 0.04:
-            return input_text.replace(delimiter, '.')
-
-        return input_text

fast_sentence_segment-1.1.8.dist-info/RECORD

@@ -1,20 +0,0 @@
-fast_sentence_segment/__init__.py,sha256=HTONyC0JLVWTAHyvJO6rMINmxymbfMpARtGeRw5iIsQ,359
-fast_sentence_segment/bp/__init__.py,sha256=j2-WfQ9WwVuXeGSjvV6XLVwEdvau8sdAQe4Pa4DrYi8,33
-fast_sentence_segment/bp/segmenter.py,sha256=UW6DguPgA56h-pPYRsfJhjIzBe40j6NdjkwYxamASyA,1928
-fast_sentence_segment/core/__init__.py,sha256=uoBersYyVStJ5a8zJpQz1GDGaloEdAv2jGHw1292hRM,108
-fast_sentence_segment/core/base_object.py,sha256=AYr7yzusIwawjbKdvcv4yTEnhmx6M583kDZzhzPOmq4,635
-fast_sentence_segment/core/stopwatch.py,sha256=hE6hMz2q6rduaKi58KZmiAL-lRtyh_wWCANhl4KLkRQ,879
-fast_sentence_segment/dmo/__init__.py,sha256=Tz1ICwhQLimQ3KoZM4f5HQFFgYamnMAn4opcyIB7Chk,328
-fast_sentence_segment/dmo/bullet_point_cleaner.py,sha256=WOZQRWXiiyRi8rOuEIw36EmkaXmATHL9_Dxb2rderw4,1606
-fast_sentence_segment/dmo/delimiters_to_periods.py,sha256=cshN3TU2YzCHWVGGDnd_FOQJluyRExpYPrpZ_BQQeko,857
-fast_sentence_segment/dmo/newlines_to_periods.py,sha256=PUrXreqZWiITINfoJL5xRRlXJH6noH0cdXtW1EqAh8I,1517
-fast_sentence_segment/dmo/numbered_list_normalizer.py,sha256=q0cLiTUt6ITX_cCwCRrIR7A3o92WOKVWpFWs8P5Gu5M,1549
-fast_sentence_segment/dmo/post_process_sentences.py,sha256=5jxG3TmFjxIExMPLhnCB5JT1lXQvFU9r4qQGoATGrWk,916
-fast_sentence_segment/dmo/spacy_doc_segmenter.py,sha256=0icAkSQwAUQo3VYqQ2PUjW6-MOU5RNCGPX3-fB5YfCc,2554
-fast_sentence_segment/svc/__init__.py,sha256=9B12mXxBnlalH4OAm1AMLwUMa-RLi2ilv7qhqv26q7g,144
-fast_sentence_segment/svc/perform_paragraph_segmentation.py,sha256=zLKw9rSzb0NNfx4MyEeoGrHwhxTtH5oDrYcAL2LMVHY,1378
-fast_sentence_segment/svc/perform_sentence_segmentation.py,sha256=ZT231A5CJ5v9tGpYbny_eq8SzgEdEwVTCVu9OV_UO9g,3839
-fast_sentence_segment-1.1.8.dist-info/METADATA,sha256=mCHUoRPcbOcSlffv6-da66W2KZU6P5OUBQFnGFgJhQU,4817
-fast_sentence_segment-1.1.8.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-fast_sentence_segment-1.1.8.dist-info/licenses/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
-fast_sentence_segment-1.1.8.dist-info/RECORD,,