fast-sentence-segment 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

fast_sentence_segment/cli.py

@@ -62,6 +62,11 @@ def main():
         action="store_true",
         help="Number output lines",
     )
+    parser.add_argument(
+        "--unwrap",
+        action="store_true",
+        help="Unwrap hard-wrapped lines and dehyphenate split words",
+    )
     args = parser.parse_args()
 
     # Get input text
@@ -77,7 +82,7 @@ def main():
         sys.exit(1)
 
     # Segment and output
-    sentences = segment_text(text.strip(), flatten=True)
+    sentences = segment_text(text.strip(), flatten=True, unwrap=args.unwrap)
     for i, sentence in enumerate(sentences, 1):
         if args.numbered:
             print(f"{i}. {sentence}")

fast_sentence_segment/dmo/__init__.py

@@ -2,6 +2,7 @@ from .abbreviation_merger import AbbreviationMerger
 from .abbreviation_splitter import AbbreviationSplitter
 from .title_name_merger import TitleNameMerger
 from .bullet_point_cleaner import BulletPointCleaner
+from .dehyphenator import Dehyphenator
 from .ellipsis_normalizer import EllipsisNormalizer
 from .newlines_to_periods import NewlinesToPeriods
 from .post_process_sentences import PostProcessStructure

fast_sentence_segment/dmo/dehyphenator.py

@@ -0,0 +1,55 @@
+# -*- coding: UTF-8 -*-
+"""Dehyphenate words split across lines.
+
+Related GitHub Issue:
+    #8 - Add dehyphenation support for words split across lines
+    https://github.com/craigtrim/fast-sentence-segment/issues/8
+
+When processing ebooks and scanned documents, words are often hyphenated
+at line breaks for typesetting purposes. This module rejoins those words.
+"""
+
+import re
+
+from fast_sentence_segment.core import BaseObject
+
+# Pattern to match hyphenated word breaks at end of line:
+# - A single hyphen (not -- em-dash)
+# - Followed by newline and optional whitespace
+# - Followed by a lowercase letter (continuation of word)
+_HYPHEN_LINE_BREAK_PATTERN = re.compile(r'(?<!-)-\n\s*([a-z])')
+
+
+class Dehyphenator(BaseObject):
+    """Rejoin words that were hyphenated across line breaks."""
+
+    def __init__(self):
+        """Change Log
+
+        Created:
+            3-Feb-2026
+            craigtrim@gmail.com
+            *   add dehyphenation support for words split across lines
+                https://github.com/craigtrim/fast-sentence-segment/issues/8
+        """
+        BaseObject.__init__(self, __name__)
+
+    @staticmethod
+    def process(input_text: str) -> str:
+        """Rejoin words that were hyphenated across line breaks.
+
+        Detects the pattern of a word fragment ending with a hyphen
+        at the end of a line, followed by the word continuation
+        starting with a lowercase letter on the next line.
+
+        Examples:
+            "bot-\\ntle" -> "bottle"
+            "cham-\\n    bermaid" -> "chambermaid"
+
+        Args:
+            input_text: Text that may contain hyphenated line breaks.
+
+        Returns:
+            Text with hyphenated word breaks rejoined.
+        """
+        return _HYPHEN_LINE_BREAK_PATTERN.sub(r'\1', input_text)

fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py

@@ -2,30 +2,71 @@
 """Unwrap hard-wrapped text (e.g., Project Gutenberg e-texts).
 
 Joins lines within paragraphs into continuous strings while
-preserving paragraph boundaries (blank lines).
+preserving paragraph boundaries (blank lines). Also dehyphenates
+words that were split across lines for typesetting.
+
+Related GitHub Issue:
+    #8 - Add dehyphenation support for words split across lines
+    https://github.com/craigtrim/fast-sentence-segment/issues/8
 """
 
 import re
 
+# Pattern to match hyphenated word breaks at end of line:
+# - A single hyphen (not -- em-dash)
+# - Followed by newline and optional whitespace
+# - Followed by a lowercase letter (continuation of word)
+_HYPHEN_LINE_BREAK_PATTERN = re.compile(r'(?<!-)-\n\s*([a-z])')
+
+
+def _dehyphenate_block(block: str) -> str:
+    """Remove hyphens from words split across lines.
+
+    Detects the pattern of a word fragment ending with a hyphen
+    at the end of a line, followed by the word continuation
+    starting with a lowercase letter on the next line.
+
+    Examples:
+        "bot-\\ntle" -> "bottle"
+        "cham-\\n    bermaid" -> "chambermaid"
+
+    Args:
+        block: A paragraph block that may contain hyphenated line breaks.
+
+    Returns:
+        The block with hyphenated word breaks rejoined.
+    """
+    return _HYPHEN_LINE_BREAK_PATTERN.sub(r'\1', block)
+
 
 def unwrap_hard_wrapped_text(text: str) -> str:
     """Unwrap hard-wrapped paragraphs into continuous lines.
 
     Splits on blank lines to identify paragraphs, then joins
     lines within each paragraph into a single string with
-    single spaces.
+    single spaces. Also dehyphenates words that were split
+    across lines for typesetting purposes.
+
+    Examples:
+        >>> unwrap_hard_wrapped_text("a bot-\\ntle of wine")
+        'a bottle of wine'
+        >>> unwrap_hard_wrapped_text("line one\\nline two")
+        'line one line two'
 
     Args:
         text: Raw text with hard-wrapped lines.
 
     Returns:
         Text with paragraphs unwrapped into continuous strings,
-        separated by double newlines.
+        separated by double newlines, with hyphenated words rejoined.
     """
     blocks = re.split(r'\n\s*\n', text)
     unwrapped = []
 
     for block in blocks:
+        # First, dehyphenate words split across lines
+        block = _dehyphenate_block(block)
+        # Then join remaining lines with spaces
         lines = block.splitlines()
         joined = ' '.join(line.strip() for line in lines if line.strip())
         if joined:

fast_sentence_segment/svc/perform_sentence_segmentation.py

@@ -18,6 +18,7 @@ from fast_sentence_segment.dmo import QuestionExclamationSplitter
 from fast_sentence_segment.dmo import SpacyDocSegmenter
 from fast_sentence_segment.dmo import PostProcessStructure
 from fast_sentence_segment.dmo import StripTrailingPeriodAfterQuote
+from fast_sentence_segment.dmo import Dehyphenator
 
 
 class PerformSentenceSegmentation(BaseObject):
@@ -46,6 +47,7 @@ class PerformSentenceSegmentation(BaseObject):
         if not self.__nlp:
             self.__nlp = spacy.load("en_core_web_sm")
 
+        self._dehyphenate = Dehyphenator.process
         self._newlines_to_periods = NewlinesToPeriods.process
         self._normalize_numbered_lists = NumberedListNormalizer().process
         self._normalize_ellipses = EllipsisNormalizer().process
@@ -96,6 +98,10 @@ class PerformSentenceSegmentation(BaseObject):
         # Normalize tabs to spaces
         input_text = input_text.replace('\t', ' ')
 
+        # Dehyphenate words split across lines (issue #8)
+        # Must happen before newlines are converted to periods
+        input_text = self._dehyphenate(input_text)
+
         input_text = self._normalize_numbered_lists(input_text)
         input_text = self._normalize_ellipses(input_text)
 

{fast_sentence_segment-1.3.0.dist-info → fast_sentence_segment-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fast-sentence-segment
-Version: 1.3.0
+Version: 1.4.0
 Summary: Fast and Efficient Sentence Segmentation
 Home-page: https://github.com/craigtrim/fast-sentence-segment
 License: MIT
@@ -33,8 +33,9 @@ Description-Content-Type: text/markdown
 
 [![PyPI version](https://img.shields.io/pypi/v/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
 [![Python versions](https://img.shields.io/pypi/pyversions/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
+[![CI](https://img.shields.io/github/actions/workflow/status/craigtrim/fast-sentence-segment/ci.yml?branch=master&label=CI)](https://github.com/craigtrim/fast-sentence-segment/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![spaCy](https://img.shields.io/badge/spaCy-3.8-blue.svg)](https://spacy.io/)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![Downloads](https://static.pepy.tech/badge/fast-sentence-segment)](https://pepy.tech/project/fast-sentence-segment)
 [![Downloads/Month](https://static.pepy.tech/badge/fast-sentence-segment/month)](https://pepy.tech/project/fast-sentence-segment)
 

{fast_sentence_segment-1.3.0.dist-info → fast_sentence_segment-1.4.0.dist-info}/RECORD

@@ -1,15 +1,16 @@
 fast_sentence_segment/__init__.py,sha256=jeb4yCy89ivyqbo-4ldJLquPAG_XR_33Q7nrDjqPxvE,1465
 fast_sentence_segment/bp/__init__.py,sha256=j2-WfQ9WwVuXeGSjvV6XLVwEdvau8sdAQe4Pa4DrYi8,33
 fast_sentence_segment/bp/segmenter.py,sha256=UW6DguPgA56h-pPYRsfJhjIzBe40j6NdjkwYxamASyA,1928
-fast_sentence_segment/cli.py,sha256=y6PIj3k6QKSYheRof8UXGhZXZZevgvlWXISyZFZ1J30,3699
+fast_sentence_segment/cli.py,sha256=Y89BH-xbJ0vykg301D2543MtGP4kYLnA6i3UQ7Hg5YA,3869
 fast_sentence_segment/core/__init__.py,sha256=uoBersYyVStJ5a8zJpQz1GDGaloEdAv2jGHw1292hRM,108
 fast_sentence_segment/core/base_object.py,sha256=AYr7yzusIwawjbKdvcv4yTEnhmx6M583kDZzhzPOmq4,635
 fast_sentence_segment/core/stopwatch.py,sha256=hE6hMz2q6rduaKi58KZmiAL-lRtyh_wWCANhl4KLkRQ,879
-fast_sentence_segment/dmo/__init__.py,sha256=pt8i82Gj2O3UwK8_v0hyn343EI4mzdbf8imkfDe48C0,823
+fast_sentence_segment/dmo/__init__.py,sha256=N0lLHVn6zKeg6h1LIfoc4XeXPUY-uSbMT45dP2_vn8M,862
 fast_sentence_segment/dmo/abbreviation_merger.py,sha256=tCXM6yCfMryJvMIVWIxP_EocoibZi8vohFzJ5tvMYr0,4432
 fast_sentence_segment/dmo/abbreviation_splitter.py,sha256=03mSyJcLooNyIjXx6mPlrnjmKgZW-uhUIqG4U-MbIGw,2981
 fast_sentence_segment/dmo/abbreviations.py,sha256=CGJrJDo6pmYd3pTNEQbdOo8N6tnkCnwyL2X7Si663Os,2530
 fast_sentence_segment/dmo/bullet_point_cleaner.py,sha256=WOZQRWXiiyRi8rOuEIw36EmkaXmATHL9_Dxb2rderw4,1606
+fast_sentence_segment/dmo/dehyphenator.py,sha256=6BJTie7tClRAifeiW8V2CdAAbcbknhtqmKylAdRZ7ko,1776
 fast_sentence_segment/dmo/ellipsis_normalizer.py,sha256=lHs9dLFfKJe-2vFNe17Hik90g3_kXX347OzGP_IOT08,1521
 fast_sentence_segment/dmo/group_quoted_sentences.py,sha256=Ifh_kUwi7sMbzbZvrTgEKkzWe50AafUDhVKVPR9h7wQ,5092
 fast_sentence_segment/dmo/newlines_to_periods.py,sha256=PUrXreqZWiITINfoJL5xRRlXJH6noH0cdXtW1EqAh8I,1517
@@ -20,12 +21,12 @@ fast_sentence_segment/dmo/question_exclamation_splitter.py,sha256=cRsWRu8zb6wOWG
 fast_sentence_segment/dmo/spacy_doc_segmenter.py,sha256=_oTsrIL2rjysjt_8bPJVNTn230pUtL-geCC8g174iC4,3163
 fast_sentence_segment/dmo/strip_trailing_period_after_quote.py,sha256=wYkoLy5XJKZIblJXBvDAB8-a81UTQOhOf2u91wjJWUw,2259
 fast_sentence_segment/dmo/title_name_merger.py,sha256=zbG04_VjwM8TtT8LhavvmZqIZL_2xgT2OTxWkK_Zt1s,5133
-fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py,sha256=mrqfvUU1wN0aDhFmCQ9XwA5aNyPXNiBKvXI11AqCgJQ,928
+fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py,sha256=V1T5RsJBaII_iGJMyWvv6rb2mny8pnVd428oVZL0n5I,2457
 fast_sentence_segment/svc/__init__.py,sha256=9B12mXxBnlalH4OAm1AMLwUMa-RLi2ilv7qhqv26q7g,144
 fast_sentence_segment/svc/perform_paragraph_segmentation.py,sha256=zLKw9rSzb0NNfx4MyEeoGrHwhxTtH5oDrYcAL2LMVHY,1378
-fast_sentence_segment/svc/perform_sentence_segmentation.py,sha256=_MsUBHebx0R2MREUM9VM6VyBQDIJiC0x1aYeCZnWThQ,6003
-fast_sentence_segment-1.3.0.dist-info/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
-fast_sentence_segment-1.3.0.dist-info/METADATA,sha256=frEQAshukVK3-b6RzDWxDYKh0hIKUZZ91j8P_LVTeyo,7573
-fast_sentence_segment-1.3.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-fast_sentence_segment-1.3.0.dist-info/entry_points.txt,sha256=Zc8OwFKj3ofnjy5ZIFqHzDkIEWweV1AP1xap1ZFGD8M,107
-fast_sentence_segment-1.3.0.dist-info/RECORD,,
+fast_sentence_segment/svc/perform_sentence_segmentation.py,sha256=Qaj7oxVHfUd6pIlwCD1O8P14LaGUdolFGuykmrF6gw8,6276
+fast_sentence_segment-1.4.0.dist-info/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
+fast_sentence_segment-1.4.0.dist-info/METADATA,sha256=XernLjKJbfSBxiqhQm-SZC8JFtggSoPe1hUD2X6D9N8,7853
+fast_sentence_segment-1.4.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fast_sentence_segment-1.4.0.dist-info/entry_points.txt,sha256=Zc8OwFKj3ofnjy5ZIFqHzDkIEWweV1AP1xap1ZFGD8M,107
+fast_sentence_segment-1.4.0.dist-info/RECORD,,