fast-sentence-segment 1.4.3__py3-none-any.whl → 1.4.4__py3-none-any.whl

fast_sentence_segment/dmo/spacy_doc_segmenter.py

@@ -65,6 +65,54 @@ class SpacyDocSegmenter(BaseObject):
             return False
         return True
 
+    @staticmethod
+    def _merge_orphaned_quotes(sentences: list) -> list:
+        """Merge orphaned opening quotes with the following sentence.
+
+        spaCy sometimes splits on opening quotes, producing sentences like:
+            ["'", "Oh, the funeral..."]
+        This merges them into:
+            ["'Oh, the funeral..."]
+
+        Also handles trailing orphaned quotes that should belong to next sentence:
+            ["He said. '", "Hello!'"]
+        Becomes:
+            ["He said.", "'Hello!'"]
+        """
+        if not sentences:
+            return sentences
+
+        result = []
+        i = 0
+        while i < len(sentences):
+            sent = sentences[i]
+            # Check if this sentence is just an opening quote
+            if sent.strip() in ("'", '"', "'.", '".'):
+                # Merge with the next sentence if available
+                if i + 1 < len(sentences):
+                    quote_char = sent.strip().rstrip('.')
+                    result.append(quote_char + sentences[i + 1])
+                    i += 2
+                    continue
+            result.append(sent)
+            i += 1
+
+        # Second pass: handle trailing orphaned quotes
+        # Pattern: sentence ends with `. '` or `. "` - move quote to next sentence
+        fixed = []
+        for i, sent in enumerate(result):
+            # Check for trailing orphaned quote (`. '` or `? '` or `! '`)
+            if len(sent) >= 3 and sent[-2:] in (" '", ' "') and sent[-3] in '.?!':
+                # Strip the trailing quote
+                trailing_quote = sent[-1]
+                sent = sent[:-2]
+                # Prepend to next sentence if available
+                if i + 1 < len(result) and not result[i + 1].startswith(('"', "'")):
+                    result[i + 1] = trailing_quote + result[i + 1]
+            fixed.append(sent)
+
+        return fixed
+
     @staticmethod
     def _cleanse(sentences: list) -> str:
         sentences = [sent for sent in sentences
@@ -103,6 +151,9 @@ class SpacyDocSegmenter(BaseObject):
         sentences = [sent for sent in sentences if
                      sent and len(sent) and sent != 'None']
 
+        # Merge orphaned opening quotes with following sentence
+        sentences = self._merge_orphaned_quotes(sentences)
+
         sentences = [self._append_period(sent)
                      for sent in sentences]
 

{fast_sentence_segment-1.4.3.dist-info → fast_sentence_segment-1.4.4.dist-info}/METADATA

@@ -1,9 +1,9 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: fast-sentence-segment
-Version: 1.4.3
+Version: 1.4.4
 Summary: Fast and Efficient Sentence Segmentation
-Home-page: https://github.com/craigtrim/fast-sentence-segment
 License: MIT
+License-File: LICENSE
 Keywords: nlp,text,preprocess,segment
 Author: Craig Trim
 Author-email: craigtrim@gmail.com
@@ -33,6 +33,7 @@ Description-Content-Type: text/markdown
 
 [![PyPI version](https://img.shields.io/pypi/v/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
 [![Python versions](https://img.shields.io/pypi/pyversions/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
+[![CI](https://img.shields.io/github/actions/workflow/status/craigtrim/fast-sentence-segment/ci.yml?branch=master&label=CI)](https://github.com/craigtrim/fast-sentence-segment/actions/workflows/ci.yml)
 [![Tests](https://img.shields.io/badge/tests-664-brightgreen)](https://github.com/craigtrim/fast-sentence-segment/tree/master/tests)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)

{fast_sentence_segment-1.4.3.dist-info → fast_sentence_segment-1.4.4.dist-info}/RECORD

@@ -18,15 +18,15 @@ fast_sentence_segment/dmo/normalize_quotes.py,sha256=mr53qo_tj_I9XzElOKjUQvCtDQh
 fast_sentence_segment/dmo/numbered_list_normalizer.py,sha256=q0sOCW8Jkn2vTXlUcVhmDvYES3yvJx1oUVl_8y7eL4E,1672
 fast_sentence_segment/dmo/post_process_sentences.py,sha256=5jxG3TmFjxIExMPLhnCB5JT1lXQvFU9r4qQGoATGrWk,916
 fast_sentence_segment/dmo/question_exclamation_splitter.py,sha256=cRsWRu8zb6wOWG-BjMahHfz4YGutKiV9lW7dE-q3tgc,2006
-fast_sentence_segment/dmo/spacy_doc_segmenter.py,sha256=_oTsrIL2rjysjt_8bPJVNTn230pUtL-geCC8g174iC4,3163
+fast_sentence_segment/dmo/spacy_doc_segmenter.py,sha256=Kb65TYMhrbpTYEey5vb7TyhCjUHVxmugHYIeKkntCwk,5147
 fast_sentence_segment/dmo/strip_trailing_period_after_quote.py,sha256=wYkoLy5XJKZIblJXBvDAB8-a81UTQOhOf2u91wjJWUw,2259
 fast_sentence_segment/dmo/title_name_merger.py,sha256=zbG04_VjwM8TtT8LhavvmZqIZL_2xgT2OTxWkK_Zt1s,5133
 fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py,sha256=V1T5RsJBaII_iGJMyWvv6rb2mny8pnVd428oVZL0n5I,2457
 fast_sentence_segment/svc/__init__.py,sha256=9B12mXxBnlalH4OAm1AMLwUMa-RLi2ilv7qhqv26q7g,144
 fast_sentence_segment/svc/perform_paragraph_segmentation.py,sha256=zLKw9rSzb0NNfx4MyEeoGrHwhxTtH5oDrYcAL2LMVHY,1378
 fast_sentence_segment/svc/perform_sentence_segmentation.py,sha256=mAJEPWqNQFbnlj7Rb7yiXIRHCAdlgsN0jAbg7e2qpMU,7421
-fast_sentence_segment-1.4.3.dist-info/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
-fast_sentence_segment-1.4.3.dist-info/METADATA,sha256=5LGK9z9ip2AtOr2FgaIgkrR2mLvIQaeeuh8gVi3GBaA,7785
-fast_sentence_segment-1.4.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-fast_sentence_segment-1.4.3.dist-info/entry_points.txt,sha256=Zc8OwFKj3ofnjy5ZIFqHzDkIEWweV1AP1xap1ZFGD8M,107
-fast_sentence_segment-1.4.3.dist-info/RECORD,,
+fast_sentence_segment-1.4.4.dist-info/METADATA,sha256=8EZrIvdE8CWTW64_HqkMH8eF6CzXs6UDdaFjcce7LTA,7947
+fast_sentence_segment-1.4.4.dist-info/WHEEL,sha256=kJCRJT_g0adfAJzTx2GUMmS80rTJIVHRCfG0DQgLq3o,88
+fast_sentence_segment-1.4.4.dist-info/entry_points.txt,sha256=Zc8OwFKj3ofnjy5ZIFqHzDkIEWweV1AP1xap1ZFGD8M,107
+fast_sentence_segment-1.4.4.dist-info/licenses/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
+fast_sentence_segment-1.4.4.dist-info/RECORD,,

{fast_sentence_segment-1.4.3.dist-info → fast_sentence_segment-1.4.4.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 2.3.1
 Root-Is-Purelib: true
 Tag: py3-none-any