fast-sentence-segment 0.1.9__py3-none-any.whl → 1.1.8__py3-none-any.whl

fast_sentence_segment/__init__.py

@@ -1,18 +1,18 @@
-from .bp import *
-from .svc import *
-from .dmo import *
-
-from .bp.segmenter import Segmenter
-
-segment = Segmenter().input_text
-
-
-def segment_text(input_text: str, flatten: bool = False) -> list:
-    results = segment(input_text)
-
-    if flatten:
-        flat = []
-        [[flat.append(y) for y in x] for x in results]
-        return flat
-
-    return results
+from .bp import *
+from .svc import *
+from .dmo import *
+
+from .bp.segmenter import Segmenter
+
+segment = Segmenter().input_text
+
+
+def segment_text(input_text: str, flatten: bool = False) -> list:
+    results = segment(input_text)
+
+    if flatten:
+        flat = []
+        [[flat.append(y) for y in x] for x in results]
+        return flat
+
+    return results

fast_sentence_segment/bp/__init__.py

@@ -1 +1 @@
-from .segmenter import Segmenter
+from .segmenter import Segmenter

fast_sentence_segment/bp/segmenter.py

@@ -1,68 +1,65 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Orchestrate Sentence Segmentation """
-
-
-from functools import lru_cache
-
-from baseblock import Enforcer
-from baseblock import Stopwatch
-from baseblock import BaseObject
-
-from fast_sentence_segment.svc import PerformParagraphSegmentation
-from fast_sentence_segment.svc import PerformSentenceSegmentation
-
-
-class Segmenter(BaseObject):
-    """ Orchestrate Sentence Segmentation """
-
-    def __init__(self):
-        """ Change Log
-
-        Created:
-            30-Sept-2021
-        """
-        BaseObject.__init__(self, __name__)
-        self._segment_paragraphs = PerformParagraphSegmentation().process
-        self._segment_sentences = PerformSentenceSegmentation().process
-
-    def _input_text(self,
-                    input_text: str) -> list:
-        paragraphs = []
-
-        for paragraph in self._segment_paragraphs(input_text):
-            paragraphs.append(self._segment_sentences(paragraph))
-
-        return paragraphs
-
-    @lru_cache(maxsize=1024, typed=True)
-    def input_text(self,
-                   input_text: str) -> list:
-        """Segment Input Text into Paragraphs and Sentences
-
-        Args:
-            input_text (str): An input string of any length or type
-
-        Raises:
-            ValueError: input must be a string
-
-        Returns:
-            list:   returns a list of lists.
-                    Each outer list is a paragraph.
-                    Each inner list contains 1..* sentences
-        """
-
-        if self.isEnabledForDebug:
-            Enforcer.is_str(input_text)
-
-        sw = Stopwatch()
-
-        paragraphs = self._input_text(input_text)
-
-        if self.isEnabledForInfo:
-            self.logger.info('\n'.join([
-                "Segmentation of Input Text Complete",
-                f"\tTotal Paragraphs: {len(paragraphs)}",
-                f"\tTotal Time: {str(sw)}"]))
-
-        return paragraphs
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Orchestrate Sentence Segmentation """
+
+
+from functools import lru_cache
+
+from fast_sentence_segment.core import BaseObject, Stopwatch
+from fast_sentence_segment.svc import PerformParagraphSegmentation
+from fast_sentence_segment.svc import PerformSentenceSegmentation
+
+
+class Segmenter(BaseObject):
+    """ Orchestrate Sentence Segmentation """
+
+    def __init__(self):
+        """ Change Log
+
+        Created:
+            30-Sept-2021
+        """
+        BaseObject.__init__(self, __name__)
+        self._segment_paragraphs = PerformParagraphSegmentation().process
+        self._segment_sentences = PerformSentenceSegmentation().process
+
+    def _input_text(self,
+                    input_text: str) -> list:
+        paragraphs = []
+
+        for paragraph in self._segment_paragraphs(input_text):
+            paragraphs.append(self._segment_sentences(paragraph))
+
+        return paragraphs
+
+    @lru_cache(maxsize=1024, typed=True)
+    def input_text(self,
+                   input_text: str) -> list:
+        """Segment Input Text into Paragraphs and Sentences
+
+        Args:
+            input_text (str): An input string of any length or type
+
+        Raises:
+            ValueError: input must be a string
+
+        Returns:
+            list:   returns a list of lists.
+                    Each outer list is a paragraph.
+                    Each inner list contains 1..* sentences
+        """
+
+        if self.isEnabledForDebug and not isinstance(input_text, str):
+            raise ValueError(f"Expected str, got {type(input_text)}")
+
+        sw = Stopwatch()
+
+        paragraphs = self._input_text(input_text)
+
+        if self.isEnabledForInfo:
+            self.logger.info('\n'.join([
+                "Segmentation of Input Text Complete",
+                f"\tTotal Paragraphs: {len(paragraphs)}",
+                f"\tTotal Time: {str(sw)}"]))
+
+        return paragraphs

fast_sentence_segment/core/__init__.py

@@ -0,0 +1,4 @@
+from .base_object import BaseObject
+from .stopwatch import Stopwatch
+
+__all__ = ["BaseObject", "Stopwatch"]

fast_sentence_segment/core/base_object.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""Base object providing logging functionality."""
+
+import logging
+
+logging.basicConfig(
+    format='%(asctime)s : %(levelname)s : %(filename)s : %(funcName)s() : %(lineno)d : %(message)s',
+    level=logging.DEBUG)
+
+
+class BaseObject:
+    """Base class providing logging capabilities."""
+
+    def __init__(self, component_name: str):
+        self.logger = logging.getLogger(component_name)
+        self.isEnabledForDebug = self.logger.isEnabledFor(logging.DEBUG)
+        self.isEnabledForInfo = self.logger.isEnabledFor(logging.INFO)
+        self.isEnabledForWarning = self.logger.isEnabledFor(logging.WARNING)

fast_sentence_segment/core/stopwatch.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""Simple stopwatch for timing operations."""
+
+import time
+
+
+class Stopwatch:
+    """A simple stopwatch for measuring elapsed time."""
+
+    def __init__(self):
+        self._start = time.perf_counter()
+        self._end = None
+
+    @property
+    def duration(self):
+        return self._end - self._start if self._end else time.perf_counter() - self._start
+
+    @property
+    def running(self):
+        return not self._end
+
+    def restart(self):
+        self._start = time.perf_counter()
+        self._end = None
+        return self
+
+    def stop(self):
+        if self.running:
+            self._end = time.perf_counter()
+        return self
+
+    def __str__(self):
+        ms = self.duration * 1000
+        if ms >= 1000:
+            return f'{ms / 1000:.2f}s'
+        if ms >= 1:
+            return f'{ms:.2f}ms'
+        return f'{ms * 1000:.2f}μs'

fast_sentence_segment/dmo/__init__.py

@@ -1,6 +1,6 @@
-from .bullet_point_cleaner import BulletPointCleaner
-from .delimiters_to_periods import DelimitersToPeriods
-from .newlines_to_periods import NewlinesToPeriods
-from .post_process_sentences import PostProcessStructure
-from .spacy_doc_segmenter import SpacyDocSegmenter
-from .numbered_list_normalizer import NumberedListNormalizer
+from .bullet_point_cleaner import BulletPointCleaner
+from .delimiters_to_periods import DelimitersToPeriods
+from .newlines_to_periods import NewlinesToPeriods
+from .post_process_sentences import PostProcessStructure
+from .spacy_doc_segmenter import SpacyDocSegmenter
+from .numbered_list_normalizer import NumberedListNormalizer

fast_sentence_segment/dmo/bullet_point_cleaner.py

@@ -1,55 +1,55 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Prevent Bullet Points from Triggering False Positive Segmentation """
-
-
-from baseblock import BaseObject
-
-
-class BulletPointCleaner(BaseObject):
-    """ Prevent Bullet Points from Triggering False Positive Segmentation """
-
-    def __init__(self):
-        """ Change Log
-
-        Created:
-            30-Sept-2021
-            craigtrim@gmail.com
-        Updated:
-            19-Oct-2022
-            craigtrim@gmail.com
-            *   clean up for segment_text_3_test.py
-        """
-        BaseObject.__init__(self, __name__)
-
-    @staticmethod
-    def process(input_text: str) -> str:
-        """
-        Purpose:
-            prevent numbered bullet points from triggering sentence detection
-        :param input_text:
-            any input text
-        :return:
-            preprocessed input text
-        """
-        if input_text.startswith("-"):
-            input_text = input_text[1:]  # segment_text_3_test.py
-
-        if "  " in input_text:
-            input_text = input_text.replace("  ", " ")
-
-        # the replacement routine above leaves double '..' in the text
-        # this replacement will solve that
-        while ".." in input_text:
-            input_text = input_text.replace("..", ".")
-
-        while ". -" in input_text:  # segment_text_3_test.py
-            input_text = input_text.replace(". -", ". ")
-
-        while ". . " in input_text:
-            input_text = input_text.replace(". . ", ".")
-
-        while '  ' in input_text:
-            input_text = input_text.replace('  ', ' ')
-
-        return input_text
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Prevent Bullet Points from Triggering False Positive Segmentation """
+
+
+from fast_sentence_segment.core import BaseObject
+
+
+class BulletPointCleaner(BaseObject):
+    """ Prevent Bullet Points from Triggering False Positive Segmentation """
+
+    def __init__(self):
+        """ Change Log
+
+        Created:
+            30-Sept-2021
+            craigtrim@gmail.com
+        Updated:
+            19-Oct-2022
+            craigtrim@gmail.com
+            *   clean up for segment_text_3_test.py
+        """
+        BaseObject.__init__(self, __name__)
+
+    @staticmethod
+    def process(input_text: str) -> str:
+        """
+        Purpose:
+            prevent numbered bullet points from triggering sentence detection
+        :param input_text:
+            any input text
+        :return:
+            preprocessed input text
+        """
+        if input_text.startswith("-"):
+            input_text = input_text[1:]  # segment_text_3_test.py
+
+        if "  " in input_text:
+            input_text = input_text.replace("  ", " ")
+
+        # the replacement routine above leaves double '..' in the text
+        # this replacement will solve that
+        while ".." in input_text:
+            input_text = input_text.replace("..", ".")
+
+        while ". -" in input_text:  # segment_text_3_test.py
+            input_text = input_text.replace(". -", ". ")
+
+        while ". . " in input_text:
+            input_text = input_text.replace(". . ", ".")
+
+        while '  ' in input_text:
+            input_text = input_text.replace('  ', ' ')
+
+        return input_text

fast_sentence_segment/dmo/delimiters_to_periods.py

@@ -1,37 +1,37 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Convert Delimiters into Periods """
-
-
-from baseblock import BaseObject
-
-
-class DelimitersToPeriods(BaseObject):
-    """ Convert Delimiters into Periods """
-
-    def __init__(self):
-        """
-        Created:
-            30-Sept-2021
-        """
-        BaseObject.__init__(self, __name__)
-
-    @staticmethod
-    def process(input_text: str,
-                delimiter: str):
-        """
-        Purpose:
-            Take a CSV list and transform to sentences
-        :param input_text:
-        :return:
-        """
-        total_len = len(input_text)
-        total_delims = input_text.count(delimiter)
-
-        if total_delims == 0:
-            return input_text
-
-        if total_delims / total_len > 0.04:
-            return input_text.replace(delimiter, '.')
-
-        return input_text
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Convert Delimiters into Periods """
+
+
+from fast_sentence_segment.core import BaseObject
+
+
+class DelimitersToPeriods(BaseObject):
+    """ Convert Delimiters into Periods """
+
+    def __init__(self):
+        """
+        Created:
+            30-Sept-2021
+        """
+        BaseObject.__init__(self, __name__)
+
+    @staticmethod
+    def process(input_text: str,
+                delimiter: str):
+        """
+        Purpose:
+            Take a CSV list and transform to sentences
+        :param input_text:
+        :return:
+        """
+        total_len = len(input_text)
+        total_delims = input_text.count(delimiter)
+
+        if total_delims == 0:
+            return input_text
+
+        if total_delims / total_len > 0.04:
+            return input_text.replace(delimiter, '.')
+
+        return input_text

fast_sentence_segment/dmo/newlines_to_periods.py

@@ -1,57 +1,57 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Convert New Lines into Periods """
-
-
-from baseblock import BaseObject
-
-
-class NewlinesToPeriods(BaseObject):
-    """ Convert New Lines into Periods """
-
-    def __init__(self):
-        """
-        Created:
-            30-Sept-2021
-        """
-        BaseObject.__init__(self, __name__)
-
-    @staticmethod
-    def process(input_text: str):
-        """
-        Purpose:
-            Take a CSV list and transform to sentences
-        :param input_text:
-        :return:
-        """
-
-        # def replace(input_text: str,
-        #             variant: str,
-        #             canon: str) -> str:
-
-        #     v1 = f" {variant} "
-        #     if v1 in input_text:
-        #         return input_text.replace(
-        #             v1, f" {canon} ")
-
-        #     v2 = f"{variant} "
-        #     if v2 in input_text:
-        #         return input_text.replace(
-        #             v2, f"{canon} ")
-
-        #     v3 = f" {variant}"
-        #     if v3 in input_text:
-        #         return input_text.replace(
-        #             v3, f" {canon}")
-
-        #     return input_text
-
-        # result = replace(input_text=input_text,
-        #                  variant='\n',
-        #                  canon=' . ')
-
-        # 20230309; don't replace a newline with a period
-        #           that too often causes confusion and puts a period where one should not exist
-        result = input_text.replace('\n', ' ')
-
-        return result
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Convert New Lines into Periods """
+
+
+from fast_sentence_segment.core import BaseObject
+
+
+class NewlinesToPeriods(BaseObject):
+    """ Convert New Lines into Periods """
+
+    def __init__(self):
+        """
+        Created:
+            30-Sept-2021
+        """
+        BaseObject.__init__(self, __name__)
+
+    @staticmethod
+    def process(input_text: str):
+        """
+        Purpose:
+            Take a CSV list and transform to sentences
+        :param input_text:
+        :return:
+        """
+
+        # def replace(input_text: str,
+        #             variant: str,
+        #             canon: str) -> str:
+
+        #     v1 = f" {variant} "
+        #     if v1 in input_text:
+        #         return input_text.replace(
+        #             v1, f" {canon} ")
+
+        #     v2 = f"{variant} "
+        #     if v2 in input_text:
+        #         return input_text.replace(
+        #             v2, f"{canon} ")
+
+        #     v3 = f" {variant}"
+        #     if v3 in input_text:
+        #         return input_text.replace(
+        #             v3, f" {canon}")
+
+        #     return input_text
+
+        # result = replace(input_text=input_text,
+        #                  variant='\n',
+        #                  canon=' . ')
+
+        # 20230309; don't replace a newline with a period
+        #           that too often causes confusion and puts a period where one should not exist
+        result = input_text.replace('\n', ' ')
+
+        return result

fast_sentence_segment/dmo/numbered_list_normalizer.py

@@ -1,53 +1,53 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Normalize Numbered Lists to prevent False Positive Segmentation """
-
-
-from baseblock import BaseObject
-
-
-class NumberedListNormalizer(BaseObject):
-    """ Normalize Numbered Lists to prevent False Positive Segmentation """
-
-    __d_candidate_list_elements = {
-        "1. ": "1_ ",
-        "2. ": "2_ ",
-        "3. ": "3_ ",
-        "4. ": "4_ ",
-        "5. ": "5_ ",
-        "6. ": "6_ ",
-        "7. ": "7_ ",
-        "8. ": "8_ ",
-        "9. ": "9_ ",
-        "10. ": "10_ ",
-    }
-
-    def __init__(self):
-        """
-        Created:
-            19-Oct-2022
-            craigtrim@gmail.com
-            *   https://github.com/craigtrim/fast-sentence-segment/issues/1
-        """
-        BaseObject.__init__(self, __name__)
-
-    def process(self,
-                input_text: str,
-                denormalize: bool = False) -> str:
-
-        if not denormalize:
-            for candidate in self.__d_candidate_list_elements:
-                if candidate in input_text:
-                    input_text = input_text.replace(
-                        candidate, self.__d_candidate_list_elements[candidate])
-
-        else:  # reverse the process
-            d_rev = {self.__d_candidate_list_elements[k]: k
-                     for k in self.__d_candidate_list_elements}
-
-            for candidate in d_rev:
-                if candidate in input_text:
-                    input_text = input_text.replace(
-                        candidate, d_rev[candidate])
-
-        return input_text
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Normalize Numbered Lists to prevent False Positive Segmentation """
+
+
+from fast_sentence_segment.core import BaseObject
+
+
+class NumberedListNormalizer(BaseObject):
+    """ Normalize Numbered Lists to prevent False Positive Segmentation """
+
+    __d_candidate_list_elements = {
+        "1. ": "1_ ",
+        "2. ": "2_ ",
+        "3. ": "3_ ",
+        "4. ": "4_ ",
+        "5. ": "5_ ",
+        "6. ": "6_ ",
+        "7. ": "7_ ",
+        "8. ": "8_ ",
+        "9. ": "9_ ",
+        "10. ": "10_ ",
+    }
+
+    def __init__(self):
+        """
+        Created:
+            19-Oct-2022
+            craigtrim@gmail.com
+            *   https://github.com/craigtrim/fast-sentence-segment/issues/1
+        """
+        BaseObject.__init__(self, __name__)
+
+    def process(self,
+                input_text: str,
+                denormalize: bool = False) -> str:
+
+        if not denormalize:
+            for candidate in self.__d_candidate_list_elements:
+                if candidate in input_text:
+                    input_text = input_text.replace(
+                        candidate, self.__d_candidate_list_elements[candidate])
+
+        else:  # reverse the process
+            d_rev = {self.__d_candidate_list_elements[k]: k
+                     for k in self.__d_candidate_list_elements}
+
+            for candidate in d_rev:
+                if candidate in input_text:
+                    input_text = input_text.replace(
+                        candidate, d_rev[candidate])
+
+        return input_text