fast-sentence-segment 0.1.9__py3-none-any.whl → 1.2.0__py3-none-any.whl

fast_sentence_segment/dmo/spacy_doc_segmenter.py

@@ -1,101 +1,101 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Run Sentence Segmentation using spaCy """
-
-
-from spacy.lang.en import English
-
-
-from baseblock import BaseObject
-
-
-class SpacyDocSegmenter(BaseObject):
-    """ Run Sentence Segmentation using spaCy """
-
-    def __init__(self,
-                 nlp: English):
-        """
-        Created:
-            30-Sept-2021
-        """
-        BaseObject.__init__(self, __name__)
-        self._nlp = nlp
-
-    @staticmethod
-    def _append_period(a_sentence: str) -> str:
-        """
-        Purpose:
-            if the sentence is not terminated with a period, then add one
-        :return:
-            a sentence terminated by a period
-        """
-        __blacklist = [':', '?', '!']
-        if not a_sentence.strip().endswith('.'):
-            for ch in __blacklist:
-                if not a_sentence.endswith(ch):
-                    return f"{a_sentence}."
-        return a_sentence
-
-    @staticmethod
-    def _is_valid_sentence(a_sentence: str) -> bool:
-        """
-        Purpose:
-            enable filtering of invalid sentences
-        :return:
-            True        if the sentence is a valid one
-        """
-        if not a_sentence:
-            return False
-        if not len(a_sentence):
-            return False
-        if a_sentence.strip() == '.':
-            return False
-        return True
-
-    @staticmethod
-    def _cleanse(sentences: list) -> str:
-        sentences = [sent for sent in sentences
-                     if sent != '..']
-
-        normalized = []
-
-        for s in sentences:
-            s = s.replace('\n', ' ')
-
-            if s.startswith('.. '):
-                s = s[3:]
-
-            if s.endswith('.  ..'):
-                s = s[:len(s) - 3].strip()
-
-            normalized.append(s)
-
-        return normalized
-
-    def process(self,
-                input_text: str) -> list:
-        """
-        Purpose:
-            Perform Sentence Segmentation
-        :param input_text:
-            any input text
-        :return:
-            a list of 0-or-More sentences
-        """
-
-        doc = self._nlp(input_text)
-
-        sentences = [str(sent) for sent in doc.sents]
-
-        sentences = [sent for sent in sentences if
-                     sent and len(sent) and sent != 'None']
-
-        sentences = [self._append_period(sent)
-                     for sent in sentences]
-
-        sentences = [sent.strip() for sent in sentences
-                     if self._is_valid_sentence(sent)]
-
-        sentences = self._cleanse(sentences)
-
-        return sentences
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Run Sentence Segmentation using spaCy """
+
+
+from spacy.lang.en import English
+
+
+from fast_sentence_segment.core import BaseObject
+
+
+class SpacyDocSegmenter(BaseObject):
+    """ Run Sentence Segmentation using spaCy """
+
+    def __init__(self,
+                 nlp: English):
+        """
+        Created:
+            30-Sept-2021
+        """
+        BaseObject.__init__(self, __name__)
+        self._nlp = nlp
+
+    @staticmethod
+    def _append_period(a_sentence: str) -> str:
+        """
+        Purpose:
+            if the sentence is not terminated with a period, then add one
+        :return:
+            a sentence terminated by a period
+        """
+        __blacklist = [':', '?', '!']
+        if not a_sentence.strip().endswith('.'):
+            for ch in __blacklist:
+                if not a_sentence.endswith(ch):
+                    return f"{a_sentence}."
+        return a_sentence
+
+    @staticmethod
+    def _is_valid_sentence(a_sentence: str) -> bool:
+        """
+        Purpose:
+            enable filtering of invalid sentences
+        :return:
+            True        if the sentence is a valid one
+        """
+        if not a_sentence:
+            return False
+        if not len(a_sentence):
+            return False
+        if a_sentence.strip() == '.':
+            return False
+        return True
+
+    @staticmethod
+    def _cleanse(sentences: list) -> str:
+        sentences = [sent for sent in sentences
+                     if sent != '..']
+
+        normalized = []
+
+        for s in sentences:
+            s = s.replace('\n', ' ')
+
+            if s.startswith('.. '):
+                s = s[3:]
+
+            if s.endswith('.  ..'):
+                s = s[:len(s) - 3].strip()
+
+            normalized.append(s)
+
+        return normalized
+
+    def process(self,
+                input_text: str) -> list:
+        """
+        Purpose:
+            Perform Sentence Segmentation
+        :param input_text:
+            any input text
+        :return:
+            a list of 0-or-More sentences
+        """
+
+        doc = self._nlp(input_text)
+
+        sentences = [str(sent) for sent in doc.sents]
+
+        sentences = [sent for sent in sentences if
+                     sent and len(sent) and sent != 'None']
+
+        sentences = [self._append_period(sent)
+                     for sent in sentences]
+
+        sentences = [sent.strip() for sent in sentences
+                     if self._is_valid_sentence(sent)]
+
+        sentences = self._cleanse(sentences)
+
+        return sentences

fast_sentence_segment/dmo/title_name_merger.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+Merge sentences incorrectly split when a title abbreviation is followed by a
+single-word name ending in punctuation.
+
+When the question/exclamation splitter splits "Dr. Who?" into ["Dr.", "Who?"],
+this component merges them back together because a title + single capitalized
+word is almost certainly a name, not two sentences.
+
+Examples that should be merged:
+    ["Dr.", "Who?"] -> ["Dr. Who?"]
+    ["Mr.", "T!"] -> ["Mr. T!"]
+    ["Do you like Dr.", "Who?"] -> ["Do you like Dr. Who?"]
+
+Examples that should NOT be merged:
+    ["Dr.", "Where did he go?"] -> stays split (multi-word sentence)
+    ["Dr.", "who can help."] -> stays split (lowercase = not a name)
+
+Reference: https://github.com/craigtrim/fast-sentence-segment/issues/3
+"""
+
+import re
+from typing import List, Optional, Tuple
+
+from fast_sentence_segment.core import BaseObject
+from fast_sentence_segment.dmo.abbreviations import TITLE_ABBREVIATIONS
+
+
+# Subset of titles that commonly precede names (not geographic like Mt., St.)
+PERSONAL_TITLES: List[str] = [
+    "Dr.",
+    "Mr.",
+    "Mrs.",
+    "Ms.",
+    "Prof.",
+    "Sr.",
+    "Jr.",
+    "Rev.",
+    "Gen.",
+    "Col.",
+    "Capt.",
+    "Lt.",
+    "Sgt.",
+    "Rep.",
+    "Sen.",
+    "Gov.",
+    "Pres.",
+    "Hon.",
+]
+
+
+class TitleNameMerger(BaseObject):
+    """Merge sentences incorrectly split at title + single-word name boundaries."""
+
+    def __init__(self):
+        """
+        Created:
+            28-Dec-2024
+            craigtrim@gmail.com
+        Reference:
+            https://github.com/craigtrim/fast-sentence-segment/issues/3
+        """
+        BaseObject.__init__(self, __name__)
+
+        # Build pattern to match sentences ending with a title abbreviation
+        # Escape dots in abbreviations for regex
+        escaped_titles = [re.escape(t) for t in PERSONAL_TITLES]
+        titles_pattern = "|".join(escaped_titles)
+        self._ending_with_title = re.compile(rf"({titles_pattern})$", re.IGNORECASE)
+
+        # Pattern to match a single capitalized word followed by sentence-ending punctuation
+        # at the START of a sentence (may have more content after)
+        # Matches: "Who?", "Who? More text", "T!", "T! More", "Who?." (with trailing period), etc.
+        # Captures the word+punctuation part for extraction
+        # Note: The spaCy segmenter may add a trailing period to sentences ending in ?/!
+        self._single_word_with_punct = re.compile(r"^([A-Z][a-zA-Z\-]*[?!]+\.?)\s*(.*)$")
+
+    def _try_merge(self, current: str, next_sent: str) -> Optional[Tuple[str, str]]:
+        """Try to merge two sentences if they match the title + single-word name pattern.
+
+        Args:
+            current: Current sentence (may end with title abbreviation)
+            next_sent: Next sentence (may start with single-word name with punctuation)
+
+        Returns:
+            Tuple of (merged_sentence, remainder) if merge needed, else None
+        """
+        current = current.strip()
+        next_sent = next_sent.strip()
+
+        # Current sentence must end with a title abbreviation
+        if not self._ending_with_title.search(current):
+            return None
+
+        # Next sentence must start with a single capitalized word with ?/! punctuation
+        match = self._single_word_with_punct.match(next_sent)
+        if not match:
+            return None
+
+        # Extract the name part and any remainder
+        name_part = match.group(1)
+        remainder = match.group(2).strip() if match.group(2) else ""
+
+        # Clean up trailing period from name if present (added by spaCy)
+        if name_part.endswith('?.') or name_part.endswith('!.'):
+            name_part = name_part[:-1]
+
+        merged = current + " " + name_part
+        return (merged, remainder)
+
+    def process(self, sentences: List[str]) -> List[str]:
+        """Process a list of sentences, merging title + single-word name splits.
+
+        Args:
+            sentences: List of sentences
+
+        Returns:
+            List of sentences with title+name splits merged
+        """
+        if not sentences or len(sentences) < 2:
+            return sentences
+
+        # Work with a mutable copy
+        sentences = list(sentences)
+        result = []
+        i = 0
+
+        while i < len(sentences):
+            current = sentences[i]
+
+            # Check if we should merge with next sentence
+            if i + 1 < len(sentences):
+                next_sent = sentences[i + 1]
+                merge_result = self._try_merge(current, next_sent)
+
+                if merge_result:
+                    merged, remainder = merge_result
+                    result.append(merged)
+
+                    # If there's a remainder, replace next_sent with it for further processing
+                    if remainder:
+                        sentences[i + 1] = remainder
+                        i += 1  # Move to process the remainder (now at i+1, will be i after increment)
+                    else:
+                        i += 2  # Skip both merged sentences
+                    continue
+
+            result.append(current)
+            i += 1
+
+        return result

fast_sentence_segment/svc/__init__.py

@@ -1,2 +1,2 @@
-from .perform_paragraph_segmentation import PerformParagraphSegmentation
-from .perform_sentence_segmentation import PerformSentenceSegmentation
+from .perform_paragraph_segmentation import PerformParagraphSegmentation
+from .perform_sentence_segmentation import PerformSentenceSegmentation

fast_sentence_segment/svc/perform_paragraph_segmentation.py

@@ -1,50 +1,50 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Paragraph Segmentation """
-
-
-from baseblock import BaseObject
-
-
-class PerformParagraphSegmentation(BaseObject):
-    """ Paragraph Segmentation """
-
-    def __init__(self):
-        """
-        Created:
-            1-Oct-2021
-        """
-        BaseObject.__init__(self, __name__)
-
-    def _process(self,
-                 input_text: str) -> list:
-        paragraphs = input_text.split('\n\n')
-
-        paragraphs = [x.strip() for x in paragraphs if x]
-        paragraphs = [x for x in paragraphs if len(x)]
-
-        return paragraphs
-
-    def process(self,
-                input_text: str) -> list:
-        """Perform Paragraph Segmentation
-
-        Args:
-            input_text (str): An input string of any length or type
-
-        Raises:
-            ValueError: input must be a string
-
-        Returns:
-            list:   a list of 1..* paragraphs
-                    each list item is an input string of any length, but is a paragraph
-                    A paragraph is a structural concept rather than a semantic one
-        """
-        if input_text is None or not len(input_text):
-            raise ValueError("Empty Input")
-
-        if type(input_text) != str:
-            self.logger.warning(f"Invalid Input Text: {input_text}")
-            return []
-
-        return self._process(input_text)
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Paragraph Segmentation """
+
+
+from fast_sentence_segment.core import BaseObject
+
+
+class PerformParagraphSegmentation(BaseObject):
+    """ Paragraph Segmentation """
+
+    def __init__(self):
+        """
+        Created:
+            1-Oct-2021
+        """
+        BaseObject.__init__(self, __name__)
+
+    def _process(self,
+                 input_text: str) -> list:
+        paragraphs = input_text.split('\n\n')
+
+        paragraphs = [x.strip() for x in paragraphs if x]
+        paragraphs = [x for x in paragraphs if len(x)]
+
+        return paragraphs
+
+    def process(self,
+                input_text: str) -> list:
+        """Perform Paragraph Segmentation
+
+        Args:
+            input_text (str): An input string of any length or type
+
+        Raises:
+            ValueError: input must be a string
+
+        Returns:
+            list:   a list of 1..* paragraphs
+                    each list item is an input string of any length, but is a paragraph
+                    A paragraph is a structural concept rather than a semantic one
+        """
+        if input_text is None or not len(input_text):
+            raise ValueError("Empty Input")
+
+        if not isinstance(input_text, str):
+            self.logger.warning(f"Invalid Input Text: {input_text}")
+            return []
+
+        return self._process(input_text)