fast-sentence-segment 0.1.9__py3-none-any.whl → 1.1.8__py3-none-any.whl

fast_sentence_segment/dmo/post_process_sentences.py

@@ -1,48 +1,48 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Post Process Sentences """
-
-
-from baseblock import BaseObject
-
-
-class PostProcessStructure(BaseObject):
-    """ Post Process Sentences """
-
-    __replace = {
-        '..': '. ',
-        '. .': '. ',
-
-        ',.': ', ',
-        ', .': ', ',
-
-        '!.': '! ',
-        '! .': '! ',
-
-        '?.': '? ',
-        '? .': '? ',
-
-        ':.': ': ',
-        ': .': ': ',
-    }
-
-    def __init__(self):
-        """
-        Created:
-            1-Oct-2021
-        """
-        BaseObject.__init__(self, __name__)
-
-    def process(self,
-                sentences: list) -> list:
-        normalized = []
-
-        for sentence in sentences:
-
-            for k in self.__replace:
-                if k in sentence:
-                    sentence = sentence.replace(k, self.__replace[k]).strip()
-
-            normalized.append(sentence)
-
-        return normalized
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Post Process Sentences """
+
+
+from fast_sentence_segment.core import BaseObject
+
+
+class PostProcessStructure(BaseObject):
+    """ Post Process Sentences """
+
+    __replace = {
+        '..': '. ',
+        '. .': '. ',
+
+        ',.': ', ',
+        ', .': ', ',
+
+        '!.': '! ',
+        '! .': '! ',
+
+        '?.': '? ',
+        '? .': '? ',
+
+        ':.': ': ',
+        ': .': ': ',
+    }
+
+    def __init__(self):
+        """
+        Created:
+            1-Oct-2021
+        """
+        BaseObject.__init__(self, __name__)
+
+    def process(self,
+                sentences: list) -> list:
+        normalized = []
+
+        for sentence in sentences:
+
+            for k in self.__replace:
+                if k in sentence:
+                    sentence = sentence.replace(k, self.__replace[k]).strip()
+
+            normalized.append(sentence)
+
+        return normalized

fast_sentence_segment/dmo/spacy_doc_segmenter.py

@@ -1,101 +1,101 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Run Sentence Segmentation using spaCy """
-
-
-from spacy.lang.en import English
-
-
-from baseblock import BaseObject
-
-
-class SpacyDocSegmenter(BaseObject):
-    """ Run Sentence Segmentation using spaCy """
-
-    def __init__(self,
-                 nlp: English):
-        """
-        Created:
-            30-Sept-2021
-        """
-        BaseObject.__init__(self, __name__)
-        self._nlp = nlp
-
-    @staticmethod
-    def _append_period(a_sentence: str) -> str:
-        """
-        Purpose:
-            if the sentence is not terminated with a period, then add one
-        :return:
-            a sentence terminated by a period
-        """
-        __blacklist = [':', '?', '!']
-        if not a_sentence.strip().endswith('.'):
-            for ch in __blacklist:
-                if not a_sentence.endswith(ch):
-                    return f"{a_sentence}."
-        return a_sentence
-
-    @staticmethod
-    def _is_valid_sentence(a_sentence: str) -> bool:
-        """
-        Purpose:
-            enable filtering of invalid sentences
-        :return:
-            True        if the sentence is a valid one
-        """
-        if not a_sentence:
-            return False
-        if not len(a_sentence):
-            return False
-        if a_sentence.strip() == '.':
-            return False
-        return True
-
-    @staticmethod
-    def _cleanse(sentences: list) -> str:
-        sentences = [sent for sent in sentences
-                     if sent != '..']
-
-        normalized = []
-
-        for s in sentences:
-            s = s.replace('\n', ' ')
-
-            if s.startswith('.. '):
-                s = s[3:]
-
-            if s.endswith('.  ..'):
-                s = s[:len(s) - 3].strip()
-
-            normalized.append(s)
-
-        return normalized
-
-    def process(self,
-                input_text: str) -> list:
-        """
-        Purpose:
-            Perform Sentence Segmentation
-        :param input_text:
-            any input text
-        :return:
-            a list of 0-or-More sentences
-        """
-
-        doc = self._nlp(input_text)
-
-        sentences = [str(sent) for sent in doc.sents]
-
-        sentences = [sent for sent in sentences if
-                     sent and len(sent) and sent != 'None']
-
-        sentences = [self._append_period(sent)
-                     for sent in sentences]
-
-        sentences = [sent.strip() for sent in sentences
-                     if self._is_valid_sentence(sent)]
-
-        sentences = self._cleanse(sentences)
-
-        return sentences
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Run Sentence Segmentation using spaCy """
+
+
+from spacy.lang.en import English
+
+
+from fast_sentence_segment.core import BaseObject
+
+
+class SpacyDocSegmenter(BaseObject):
+    """ Run Sentence Segmentation using spaCy """
+
+    def __init__(self,
+                 nlp: English):
+        """
+        Created:
+            30-Sept-2021
+        """
+        BaseObject.__init__(self, __name__)
+        self._nlp = nlp
+
+    @staticmethod
+    def _append_period(a_sentence: str) -> str:
+        """
+        Purpose:
+            if the sentence is not terminated with a period, then add one
+        :return:
+            a sentence terminated by a period
+        """
+        __blacklist = [':', '?', '!']
+        if not a_sentence.strip().endswith('.'):
+            for ch in __blacklist:
+                if not a_sentence.endswith(ch):
+                    return f"{a_sentence}."
+        return a_sentence
+
+    @staticmethod
+    def _is_valid_sentence(a_sentence: str) -> bool:
+        """
+        Purpose:
+            enable filtering of invalid sentences
+        :return:
+            True        if the sentence is a valid one
+        """
+        if not a_sentence:
+            return False
+        if not len(a_sentence):
+            return False
+        if a_sentence.strip() == '.':
+            return False
+        return True
+
+    @staticmethod
+    def _cleanse(sentences: list) -> str:
+        sentences = [sent for sent in sentences
+                     if sent != '..']
+
+        normalized = []
+
+        for s in sentences:
+            s = s.replace('\n', ' ')
+
+            if s.startswith('.. '):
+                s = s[3:]
+
+            if s.endswith('.  ..'):
+                s = s[:len(s) - 3].strip()
+
+            normalized.append(s)
+
+        return normalized
+
+    def process(self,
+                input_text: str) -> list:
+        """
+        Purpose:
+            Perform Sentence Segmentation
+        :param input_text:
+            any input text
+        :return:
+            a list of 0-or-More sentences
+        """
+
+        doc = self._nlp(input_text)
+
+        sentences = [str(sent) for sent in doc.sents]
+
+        sentences = [sent for sent in sentences if
+                     sent and len(sent) and sent != 'None']
+
+        sentences = [self._append_period(sent)
+                     for sent in sentences]
+
+        sentences = [sent.strip() for sent in sentences
+                     if self._is_valid_sentence(sent)]
+
+        sentences = self._cleanse(sentences)
+
+        return sentences

fast_sentence_segment/svc/__init__.py

@@ -1,2 +1,2 @@
-from .perform_paragraph_segmentation import PerformParagraphSegmentation
-from .perform_sentence_segmentation import PerformSentenceSegmentation
+from .perform_paragraph_segmentation import PerformParagraphSegmentation
+from .perform_sentence_segmentation import PerformSentenceSegmentation

fast_sentence_segment/svc/perform_paragraph_segmentation.py

@@ -1,50 +1,50 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Paragraph Segmentation """
-
-
-from baseblock import BaseObject
-
-
-class PerformParagraphSegmentation(BaseObject):
-    """ Paragraph Segmentation """
-
-    def __init__(self):
-        """
-        Created:
-            1-Oct-2021
-        """
-        BaseObject.__init__(self, __name__)
-
-    def _process(self,
-                 input_text: str) -> list:
-        paragraphs = input_text.split('\n\n')
-
-        paragraphs = [x.strip() for x in paragraphs if x]
-        paragraphs = [x for x in paragraphs if len(x)]
-
-        return paragraphs
-
-    def process(self,
-                input_text: str) -> list:
-        """Perform Paragraph Segmentation
-
-        Args:
-            input_text (str): An input string of any length or type
-
-        Raises:
-            ValueError: input must be a string
-
-        Returns:
-            list:   a list of 1..* paragraphs
-                    each list item is an input string of any length, but is a paragraph
-                    A paragraph is a structural concept rather than a semantic one
-        """
-        if input_text is None or not len(input_text):
-            raise ValueError("Empty Input")
-
-        if type(input_text) != str:
-            self.logger.warning(f"Invalid Input Text: {input_text}")
-            return []
-
-        return self._process(input_text)
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Paragraph Segmentation """
+
+
+from fast_sentence_segment.core import BaseObject
+
+
+class PerformParagraphSegmentation(BaseObject):
+    """ Paragraph Segmentation """
+
+    def __init__(self):
+        """
+        Created:
+            1-Oct-2021
+        """
+        BaseObject.__init__(self, __name__)
+
+    def _process(self,
+                 input_text: str) -> list:
+        paragraphs = input_text.split('\n\n')
+
+        paragraphs = [x.strip() for x in paragraphs if x]
+        paragraphs = [x for x in paragraphs if len(x)]
+
+        return paragraphs
+
+    def process(self,
+                input_text: str) -> list:
+        """Perform Paragraph Segmentation
+
+        Args:
+            input_text (str): An input string of any length or type
+
+        Raises:
+            ValueError: input must be a string
+
+        Returns:
+            list:   a list of 1..* paragraphs
+                    each list item is an input string of any length, but is a paragraph
+                    A paragraph is a structural concept rather than a semantic one
+        """
+        if input_text is None or not len(input_text):
+            raise ValueError("Empty Input")
+
+        if not isinstance(input_text, str):
+            self.logger.warning(f"Invalid Input Text: {input_text}")
+            return []
+
+        return self._process(input_text)

fast_sentence_segment/svc/perform_sentence_segmentation.py

@@ -1,129 +1,129 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-""" Sentence Segmentation """
-
-
-import spacy
-
-from baseblock import BaseObject
-
-from fast_sentence_segment.dmo import NewlinesToPeriods
-from fast_sentence_segment.dmo import DelimitersToPeriods
-from fast_sentence_segment.dmo import BulletPointCleaner
-from fast_sentence_segment.dmo import NumberedListNormalizer
-from fast_sentence_segment.dmo import SpacyDocSegmenter
-from fast_sentence_segment.dmo import PostProcessStructure
-
-
-class PerformSentenceSegmentation(BaseObject):
-    """ Sentence Segmentation """
-
-    __nlp = None
-
-    def __init__(self):
-        """ Change Log
-
-        Created:
-            30-Sept-2021
-            craigtrim@gmail.com
-        Updated:
-            19-Oct-2022
-            craigtrim@gmail.com
-            *   add numbered-list normalization
-                https://github.com/craigtrim/fast-sentence-segment/issues/1
-        """
-        BaseObject.__init__(self, __name__)
-        if not self.__nlp:
-            self.__nlp = spacy.load("en_core_web_sm")
-
-        self._delimiters_to_periods = DelimitersToPeriods.process
-        self._newlines_to_periods = NewlinesToPeriods.process
-        self._normalize_numbered_lists = NumberedListNormalizer().process
-        self._clean_bullet_points = BulletPointCleaner.process
-        self._spacy_segmenter = SpacyDocSegmenter(self.__nlp).process
-        self._post_process = PostProcessStructure().process
-
-    @staticmethod
-    def _clean_punctuation(input_text: str) -> str:
-        """ Purpose:
-            Clean punctuation oddities; this is likely highly overfitted (for now)
-        """
-        if ", Inc" in input_text:
-            input_text = input_text.replace(", Inc", " Inc")
-
-        return input_text
-
-    @staticmethod
-    def _clean_spacing(a_sentence: str) -> str:
-
-        # eliminate triple-space
-        a_sentence = a_sentence.replace('   ', '  ')
-
-        # treat double-space as delimiter
-        a_sentence = a_sentence.replace('  ', '. ')
-
-        return a_sentence
-
-    def _process(self,
-                 input_text: str) -> list:
-
-        input_text = self._delimiters_to_periods(
-            delimiter=',',
-            input_text=input_text)
-
-        input_text = self._delimiters_to_periods(
-            delimiter=';',
-            input_text=input_text)
-
-        input_text = self._normalize_numbered_lists(input_text)
-
-        input_text = self._newlines_to_periods(input_text)
-
-        input_text = self._clean_spacing(input_text)
-        if "." not in input_text:
-            return [input_text]
-
-        input_text = self._clean_bullet_points(input_text)
-        if "." not in input_text:
-            return [input_text]
-
-        input_text = self._clean_punctuation(input_text)
-        if "." not in input_text:
-            return [input_text]
-
-        sentences = self._spacy_segmenter(input_text)
-        if "." not in input_text:
-            return [input_text]
-
-        sentences = self._post_process(sentences)
-
-        sentences = [
-            self._normalize_numbered_lists(x, denormalize=True)
-            for x in sentences
-        ]
-
-        return sentences
-
-    def process(self,
-                input_text: str) -> list:
-        """Perform Sentence Segmentation
-
-        Args:
-            input_text (str): An input string of any length or type
-
-        Raises:
-            ValueError: input must be a string
-
-        Returns:
-            list:   a list of sentences
-                    each list item is an input string of any length, but is a semantic sentence
-        """
-
-        if input_text is None or not len(input_text):
-            raise ValueError("Empty Input")
-
-        if type(input_text) != str:
-            self.logger.warning(f"Invalid Input Text: {input_text}")
-            return []
-
-        return self._process(input_text)
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Sentence Segmentation """
+
+
+import spacy
+
+from fast_sentence_segment.core import BaseObject
+
+from fast_sentence_segment.dmo import NewlinesToPeriods
+from fast_sentence_segment.dmo import DelimitersToPeriods
+from fast_sentence_segment.dmo import BulletPointCleaner
+from fast_sentence_segment.dmo import NumberedListNormalizer
+from fast_sentence_segment.dmo import SpacyDocSegmenter
+from fast_sentence_segment.dmo import PostProcessStructure
+
+
+class PerformSentenceSegmentation(BaseObject):
+    """ Sentence Segmentation """
+
+    __nlp = None
+
+    def __init__(self):
+        """ Change Log
+
+        Created:
+            30-Sept-2021
+            craigtrim@gmail.com
+        Updated:
+            19-Oct-2022
+            craigtrim@gmail.com
+            *   add numbered-list normalization
+                https://github.com/craigtrim/fast-sentence-segment/issues/1
+        """
+        BaseObject.__init__(self, __name__)
+        if not self.__nlp:
+            self.__nlp = spacy.load("en_core_web_sm")
+
+        self._delimiters_to_periods = DelimitersToPeriods.process
+        self._newlines_to_periods = NewlinesToPeriods.process
+        self._normalize_numbered_lists = NumberedListNormalizer().process
+        self._clean_bullet_points = BulletPointCleaner.process
+        self._spacy_segmenter = SpacyDocSegmenter(self.__nlp).process
+        self._post_process = PostProcessStructure().process
+
+    @staticmethod
+    def _clean_punctuation(input_text: str) -> str:
+        """ Purpose:
+            Clean punctuation oddities; this is likely highly overfitted (for now)
+        """
+        if ", Inc" in input_text:
+            input_text = input_text.replace(", Inc", " Inc")
+
+        return input_text
+
+    @staticmethod
+    def _clean_spacing(a_sentence: str) -> str:
+
+        # eliminate triple-space
+        a_sentence = a_sentence.replace('   ', '  ')
+
+        # treat double-space as delimiter
+        a_sentence = a_sentence.replace('  ', '. ')
+
+        return a_sentence
+
+    def _process(self,
+                 input_text: str) -> list:
+
+        input_text = self._delimiters_to_periods(
+            delimiter=',',
+            input_text=input_text)
+
+        input_text = self._delimiters_to_periods(
+            delimiter=';',
+            input_text=input_text)
+
+        input_text = self._normalize_numbered_lists(input_text)
+
+        input_text = self._newlines_to_periods(input_text)
+
+        input_text = self._clean_spacing(input_text)
+        if "." not in input_text:
+            return [input_text]
+
+        input_text = self._clean_bullet_points(input_text)
+        if "." not in input_text:
+            return [input_text]
+
+        input_text = self._clean_punctuation(input_text)
+        if "." not in input_text:
+            return [input_text]
+
+        sentences = self._spacy_segmenter(input_text)
+        if "." not in input_text:
+            return [input_text]
+
+        sentences = self._post_process(sentences)
+
+        sentences = [
+            self._normalize_numbered_lists(x, denormalize=True)
+            for x in sentences
+        ]
+
+        return sentences
+
+    def process(self,
+                input_text: str) -> list:
+        """Perform Sentence Segmentation
+
+        Args:
+            input_text (str): An input string of any length or type
+
+        Raises:
+            ValueError: input must be a string
+
+        Returns:
+            list:   a list of sentences
+                    each list item is an input string of any length, but is a semantic sentence
+        """
+
+        if input_text is None or not len(input_text):
+            raise ValueError("Empty Input")
+
+        if not isinstance(input_text, str):
+            self.logger.warning(f"Invalid Input Text: {input_text}")
+            return []
+
+        return self._process(input_text)