fast-sentence-segment 1.2.1__py3-none-any.whl → 1.4.0__py3-none-any.whl

fast_sentence_segment/__init__.py

@@ -3,11 +3,44 @@ from .svc import *
 from .dmo import *
 
 from .bp.segmenter import Segmenter
+from .dmo.unwrap_hard_wrapped_text import unwrap_hard_wrapped_text
+from .dmo.normalize_quotes import normalize_quotes
 
 segment = Segmenter().input_text
 
 
-def segment_text(input_text: str, flatten: bool = False) -> list:
+def segment_text(
+    input_text: str,
+    flatten: bool = False,
+    unwrap: bool = False,
+    normalize: bool = True,
+) -> list:
+    """Segment text into sentences.
+
+    Args:
+        input_text: The text to segment.
+        flatten: If True, return a flat list of sentences instead of
+            nested paragraphs.
+        unwrap: If True, unwrap hard-wrapped lines (e.g., Project
+            Gutenberg e-texts) before segmenting.
+        normalize: If True (default), normalize unicode quote variants
+            to ASCII equivalents before segmenting. Ensures consistent
+            quote characters for downstream processing.
+
+    Returns:
+        List of sentences (if flatten=True) or list of paragraph
+        groups, each containing a list of sentences.
+
+    Related GitHub Issue:
+        #6 - Review findings from Issue #5
+        https://github.com/craigtrim/fast-sentence-segment/issues/6
+    """
+    if unwrap:
+        input_text = unwrap_hard_wrapped_text(input_text)
+
+    if normalize:
+        input_text = normalize_quotes(input_text)
+
     results = segment(input_text)
 
     if flatten:

fast_sentence_segment/cli.py

@@ -3,12 +3,45 @@
 
 import argparse
 import logging
+import os
 import sys
+import time
 
 from fast_sentence_segment import segment_text
+from fast_sentence_segment.dmo.group_quoted_sentences import format_grouped_sentences
 
 logging.disable(logging.CRITICAL)
 
+# ANSI color codes
+BOLD = "\033[1m"
+DIM = "\033[2m"
+CYAN = "\033[36m"
+GREEN = "\033[32m"
+YELLOW = "\033[33m"
+RESET = "\033[0m"
+
+
+def _header(title: str):
+    print(f"\n{BOLD}{CYAN}{title}{RESET}")
+    print(f"{DIM}{'─' * 40}{RESET}")
+
+
+def _param(label: str, value: str):
+    print(f"  {DIM}{label}:{RESET} {value}")
+
+
+def _done(msg: str):
+    print(f"\n  {GREEN}✓{RESET} {msg}")
+
+
+def _file_size(path: str) -> str:
+    size = os.path.getsize(path)
+    if size < 1024:
+        return f"{size} B"
+    elif size < 1024 * 1024:
+        return f"{size / 1024:.1f} KB"
+    return f"{size / (1024 * 1024):.1f} MB"
+
 
 def main():
     parser = argparse.ArgumentParser(
@@ -29,6 +62,11 @@ def main():
         action="store_true",
         help="Number output lines",
     )
+    parser.add_argument(
+        "--unwrap",
+        action="store_true",
+        help="Unwrap hard-wrapped lines and dehyphenate split words",
+    )
     args = parser.parse_args()
 
     # Get input text
@@ -44,7 +82,7 @@ def main():
         sys.exit(1)
 
     # Segment and output
-    sentences = segment_text(text.strip(), flatten=True)
+    sentences = segment_text(text.strip(), flatten=True, unwrap=args.unwrap)
     for i, sentence in enumerate(sentences, 1):
         if args.numbered:
             print(f"{i}. {sentence}")
@@ -52,5 +90,60 @@ def main():
             print(sentence)
 
 
+def file_main():
+    parser = argparse.ArgumentParser(
+        prog="segment-file",
+        description="Segment a text file into sentences and write to an output file",
+    )
+    parser.add_argument(
+        "--input-file", required=True,
+        help="Path to input text file",
+    )
+    parser.add_argument(
+        "--output-file", required=True,
+        help="Path to output file",
+    )
+    parser.add_argument(
+        "--unwrap", action="store_true",
+        help="Unwrap hard-wrapped lines (e.g., Project Gutenberg e-texts)",
+    )
+    parser.add_argument(
+        "--no-normalize-quotes", action="store_true",
+        help="Disable unicode quote normalization to ASCII equivalents",
+    )
+    args = parser.parse_args()
+
+    _header("segment-file")
+    _param("Input", args.input_file)
+    _param("Output", args.output_file)
+    _param("Size", _file_size(args.input_file))
+    if args.unwrap:
+        _param("Unwrap", "enabled")
+
+    print(f"\n  {YELLOW}Segmenting...{RESET}", end="", flush=True)
+
+    with open(args.input_file, "r", encoding="utf-8") as f:
+        text = f.read()
+
+    start = time.perf_counter()
+    normalize = not args.no_normalize_quotes
+    sentences = segment_text(
+        text.strip(), flatten=True, unwrap=args.unwrap, normalize=normalize,
+    )
+    elapsed = time.perf_counter() - start
+
+    with open(args.output_file, "w", encoding="utf-8") as f:
+        if args.unwrap:
+            f.write(format_grouped_sentences(sentences) + "\n")
+        else:
+            for sentence in sentences:
+                f.write(sentence + "\n")
+
+    print(f"\r  {' ' * 20}\r", end="")
+    _done(f"{len(sentences):,} sentences in {elapsed:.2f}s")
+    _done(f"Written to {args.output_file}")
+    print()
+
+
 if __name__ == "__main__":
     main()

fast_sentence_segment/dmo/__init__.py

@@ -2,9 +2,14 @@ from .abbreviation_merger import AbbreviationMerger
 from .abbreviation_splitter import AbbreviationSplitter
 from .title_name_merger import TitleNameMerger
 from .bullet_point_cleaner import BulletPointCleaner
+from .dehyphenator import Dehyphenator
 from .ellipsis_normalizer import EllipsisNormalizer
 from .newlines_to_periods import NewlinesToPeriods
 from .post_process_sentences import PostProcessStructure
 from .question_exclamation_splitter import QuestionExclamationSplitter
 from .spacy_doc_segmenter import SpacyDocSegmenter
 from .numbered_list_normalizer import NumberedListNormalizer
+from .unwrap_hard_wrapped_text import unwrap_hard_wrapped_text
+from .normalize_quotes import normalize_quotes
+from .group_quoted_sentences import group_quoted_sentences, format_grouped_sentences
+from .strip_trailing_period_after_quote import StripTrailingPeriodAfterQuote

fast_sentence_segment/dmo/abbreviations.py

@@ -20,6 +20,8 @@ SENTENCE_ENDING_ABBREVIATIONS: List[str] = [
     # Common sentence-enders
     "etc.",
     "ext.",
+    "approx.",
+    "dept.",
 
     # Academic degrees (when at end of sentence)
     "Ph.D.",
@@ -32,6 +34,9 @@ SENTENCE_ENDING_ABBREVIATIONS: List[str] = [
     "J.D.",
     "D.D.S.",
     "R.N.",
+    "M.B.A.",
+    "LL.B.",
+    "LL.M.",
 
     # Business (when at end of sentence)
     "Inc.",
@@ -39,6 +44,14 @@ SENTENCE_ENDING_ABBREVIATIONS: List[str] = [
     "Ltd.",
     "Co.",
     "Bros.",
+    "LLC.",
+    "LLP.",
+
+    # Academic/legal citations (can end sentences)
+    "ibid.",
+    "Ibid.",
+    "cf.",
+    "Cf.",
 
     # Countries/Regions (when at end of sentence)
     "U.S.",
@@ -62,23 +75,59 @@ TITLE_ABBREVIATIONS: List[str] = [
     "Sr.",
     "Jr.",
     "Rev.",
+    "Hon.",
+    "Esq.",
+
+    # French/formal titles (common in translated literature)
+    "Mme.",
+    "Mlle.",
+    "Messrs.",
+
+    # Military ranks
     "Gen.",
     "Col.",
     "Capt.",
     "Lt.",
     "Sgt.",
+    "Maj.",
+    "Cpl.",
+    "Pvt.",
+    "Adm.",
+    "Cmdr.",
+
+    # Political titles
     "Rep.",
     "Sen.",
     "Gov.",
     "Pres.",
-    "Hon.",
+
+    # Ecclesiastical titles
+    "Fr.",
+    "Msgr.",
 
     # Geographic prefixes
     "St.",
     "Mt.",
     "Ft.",
+    "Ave.",
+    "Blvd.",
+    "Rd.",
 
-    # Other prefixes
+    # Latin terms (never end sentences -- always introduce clauses)
+    # Include common inconsistent forms: with/without internal periods,
+    # and with trailing comma (the most common real-world form)
+    "i.e.",
+    "i.e.,",
+    "ie.",
+    "ie.,",
+    "e.g.",
+    "e.g.,",
+    "eg.",
+    "eg.,",
+    "viz.",
+    "viz.,",
+
+    # Reference/numbering prefixes
     "Fig.",
     "fig.",
     "Sec.",
@@ -93,4 +142,8 @@ TITLE_ABBREVIATIONS: List[str] = [
     "no.",
     "Pt.",
     "pt.",
+
+    # Legal / adversarial
+    "vs.",
+    "Vs.",
 ]

fast_sentence_segment/dmo/dehyphenator.py

@@ -0,0 +1,55 @@
+# -*- coding: UTF-8 -*-
+"""Dehyphenate words split across lines.
+
+Related GitHub Issue:
+    #8 - Add dehyphenation support for words split across lines
+    https://github.com/craigtrim/fast-sentence-segment/issues/8
+
+When processing ebooks and scanned documents, words are often hyphenated
+at line breaks for typesetting purposes. This module rejoins those words.
+"""
+
+import re
+
+from fast_sentence_segment.core import BaseObject
+
+# Pattern to match hyphenated word breaks at end of line:
+# - A single hyphen (not -- em-dash)
+# - Followed by newline and optional whitespace
+# - Followed by a lowercase letter (continuation of word)
+_HYPHEN_LINE_BREAK_PATTERN = re.compile(r'(?<!-)-\n\s*([a-z])')
+
+
+class Dehyphenator(BaseObject):
+    """Rejoin words that were hyphenated across line breaks."""
+
+    def __init__(self):
+        """Change Log
+
+        Created:
+            3-Feb-2026
+            craigtrim@gmail.com
+            *   add dehyphenation support for words split across lines
+                https://github.com/craigtrim/fast-sentence-segment/issues/8
+        """
+        BaseObject.__init__(self, __name__)
+
+    @staticmethod
+    def process(input_text: str) -> str:
+        """Rejoin words that were hyphenated across line breaks.
+
+        Detects the pattern of a word fragment ending with a hyphen
+        at the end of a line, followed by the word continuation
+        starting with a lowercase letter on the next line.
+
+        Examples:
+            "bot-\\ntle" -> "bottle"
+            "cham-\\n    bermaid" -> "chambermaid"
+
+        Args:
+            input_text: Text that may contain hyphenated line breaks.
+
+        Returns:
+            Text with hyphenated word breaks rejoined.
+        """
+        return _HYPHEN_LINE_BREAK_PATTERN.sub(r'\1', input_text)

fast_sentence_segment/dmo/group_quoted_sentences.py

@@ -0,0 +1,141 @@
+# -*- coding: UTF-8 -*-
+"""Group sentences that belong to the same open-quote span.
+
+When outputting segmented text with blank-line separators, sentences
+that open with a double quote but do not close it should be grouped
+with subsequent sentences (no blank line between them) until the
+closing quote is found.
+
+Related GitHub Issues:
+    #5 - Normalize quotes and group open-quote sentences in unwrap mode
+    https://github.com/craigtrim/fast-sentence-segment/issues/5
+
+    #6 - Review findings from Issue #5
+    https://github.com/craigtrim/fast-sentence-segment/issues/6
+"""
+
+from typing import List
+
+# Maximum number of sentences that can be grouped under a single
+# open-quote span before the quote state is forcibly reset. This
+# bounds the damage from a stray quote character (e.g., OCR artifact)
+# which would otherwise corrupt grouping for all subsequent sentences.
+#
+# A typical quoted passage in literature rarely exceeds 20 sentences.
+# This limit is deliberately generous to avoid false resets on
+# legitimately long quoted passages while still preventing runaway
+# grouping on malformed input.
+MAX_QUOTE_GROUP_SIZE = 20
+
+
+def group_quoted_sentences(sentences: List[str]) -> List[List[str]]:
+    """Group sentences into blocks based on open/close quote tracking.
+
+    Sentences within an unclosed double-quote span are grouped together
+    into the same block. Sentences outside of a quote span each form
+    their own block.
+
+    When rendered, each block is joined by newlines, and blocks are
+    separated by blank lines (double newlines).
+
+    The algorithm tracks the quote state by counting ASCII double quote
+    characters in each sentence. An odd count toggles the open/close
+    state. When a quote is open, subsequent sentences are appended to
+    the current group rather than starting a new one.
+
+    A safety limit (MAX_QUOTE_GROUP_SIZE) prevents a stray or malformed
+    quote from swallowing all remaining sentences into one group. When
+    the limit is reached, the current group is flushed and the quote
+    state is reset. This bounds corruption from OCR artifacts or
+    encoding errors to a bounded window rather than the entire document.
+
+    Args:
+        sentences: Flat list of segmented sentences.
+
+    Returns:
+        List of sentence groups. Each group is a list of sentences
+        that should be rendered together without blank-line separators.
+
+    Example:
+        >>> groups = group_quoted_sentences([
+        ...     '"The probability lies in that direction.',
+        ...     'And if we take this as a working hypothesis."',
+        ...     'He paused.',
+        ... ])
+        >>> groups
+        [['"The probability lies in that direction.',
+          'And if we take this as a working hypothesis."'],
+         ['He paused.']]
+
+    Related GitHub Issues:
+        #5 - Normalize quotes and group open-quote sentences in unwrap mode
+        https://github.com/craigtrim/fast-sentence-segment/issues/5
+
+        #6 - Review findings from Issue #5
+        https://github.com/craigtrim/fast-sentence-segment/issues/6
+    """
+    if not sentences:
+        return []
+
+    groups: List[List[str]] = []
+    current_group: List[str] = []
+    quote_open = False
+
+    for sentence in sentences:
+        quote_count = sentence.count('"')
+
+        if not quote_open:
+            # Starting a new group
+            if current_group:
+                groups.append(current_group)
+            current_group = [sentence]
+        else:
+            # Inside an open quote span -- append to current group
+            current_group.append(sentence)
+
+        # Toggle quote state on odd quote count
+        if quote_count % 2 == 1:
+            quote_open = not quote_open
+
+        # Safety: if a group grows beyond the limit, the quote is
+        # likely corrupted (stray quote character). Flush the group
+        # and reset state to prevent runaway grouping.
+        if quote_open and len(current_group) >= MAX_QUOTE_GROUP_SIZE:
+            groups.append(current_group)
+            current_group = []
+            quote_open = False
+
+    # Flush the final group
+    if current_group:
+        groups.append(current_group)
+
+    return groups
+
+
+def format_grouped_sentences(sentences: List[str]) -> str:
+    """Format sentences with quote-aware blank-line separation.
+
+    Sentences within the same quoted span are separated by single
+    newlines. Sentence groups are separated by blank lines (double
+    newlines).
+
+    Args:
+        sentences: Flat list of segmented sentences.
+
+    Returns:
+        Formatted string with appropriate line separation.
+
+    Example:
+        >>> text = format_grouped_sentences([
+        ...     '"The probability lies in that direction.',
+        ...     'And if we take this as a working hypothesis."',
+        ...     'He paused.',
+        ... ])
+        >>> print(text)
+        "The probability lies in that direction.
+        And if we take this as a working hypothesis."
+        <BLANKLINE>
+        He paused.
+    """
+    groups = group_quoted_sentences(sentences)
+    return '\n\n'.join('\n'.join(group) for group in groups)

fast_sentence_segment/dmo/normalize_quotes.py

@@ -0,0 +1,80 @@
+# -*- coding: UTF-8 -*-
+"""Normalize unicode quote variants to ASCII equivalents.
+
+E-texts use a variety of quote characters (curly/smart quotes, unicode
+variants, primes, guillemets). This module normalizes all quote variants
+to their standard ASCII equivalents: double quote (") and single
+quote/apostrophe (').
+
+Related GitHub Issues:
+    #5 - Normalize quotes and group open-quote sentences in unwrap mode
+    https://github.com/craigtrim/fast-sentence-segment/issues/5
+
+    #6 - Review findings from Issue #5
+    https://github.com/craigtrim/fast-sentence-segment/issues/6
+"""
+
+import re
+
+# Unicode double quote variants to normalize to ASCII " (U+0022).
+#
+# U+201C  " LEFT DOUBLE QUOTATION MARK
+# U+201D  " RIGHT DOUBLE QUOTATION MARK
+# U+201E  „ DOUBLE LOW-9 QUOTATION MARK
+# U+201F  ‟ DOUBLE HIGH-REVERSED-9 QUOTATION MARK
+# U+00AB  « LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+# U+00BB  » RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+# U+2033  ″ DOUBLE PRIME
+# U+301D  〝 REVERSED DOUBLE PRIME QUOTATION MARK
+# U+301E  〞 DOUBLE PRIME QUOTATION MARK
+# U+301F  〟 LOW DOUBLE PRIME QUOTATION MARK
+# U+FF02  ＂ FULLWIDTH QUOTATION MARK
+DOUBLE_QUOTE_PATTERN = re.compile(
+    '[\u201c\u201d\u201e\u201f\u00ab\u00bb\u2033\u301d\u301e\u301f\uff02]'
+)
+
+# Unicode single quote variants to normalize to ASCII ' (U+0027).
+#
+# U+2018  ' LEFT SINGLE QUOTATION MARK
+# U+2019  ' RIGHT SINGLE QUOTATION MARK
+# U+201A  ‚ SINGLE LOW-9 QUOTATION MARK
+# U+201B  ‛ SINGLE HIGH-REVERSED-9 QUOTATION MARK
+# U+2039  ‹ SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+# U+203A  › SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
+# U+2032  ′ PRIME
+# U+FF07  ＇ FULLWIDTH APOSTROPHE
+# U+0060  ` GRAVE ACCENT (used as opening quote in some e-texts)
+# U+00B4  ´ ACUTE ACCENT (used as closing quote in some e-texts)
+SINGLE_QUOTE_PATTERN = re.compile(
+    '[\u2018\u2019\u201a\u201b\u2039\u203a\u2032\uff07\u0060\u00b4]'
+)
+
+
+def normalize_quotes(text: str) -> str:
+    """Replace all unicode quote variants with their ASCII equivalents.
+
+    Double quote variants are normalized to ASCII " (U+0022).
+    Single quote variants are normalized to ASCII ' (U+0027).
+
+    Args:
+        text: Input text potentially containing unicode quotes.
+
+    Returns:
+        Text with all quote variants replaced by ASCII equivalents.
+
+    Example:
+        >>> normalize_quotes('\u201cHello,\u201d she said.')
+        '"Hello," she said.'
+        >>> normalize_quotes('It\u2019s fine.')
+        "It's fine."
+
+    Related GitHub Issues:
+        #5 - Normalize quotes and group open-quote sentences in unwrap mode
+        https://github.com/craigtrim/fast-sentence-segment/issues/5
+
+        #6 - Review findings from Issue #5
+        https://github.com/craigtrim/fast-sentence-segment/issues/6
+    """
+    text = DOUBLE_QUOTE_PATTERN.sub('"', text)
+    text = SINGLE_QUOTE_PATTERN.sub("'", text)
+    return text

fast_sentence_segment/dmo/spacy_doc_segmenter.py

@@ -23,18 +23,31 @@ class SpacyDocSegmenter(BaseObject):
 
     @staticmethod
     def _append_period(a_sentence: str) -> str:
+        """Append a period if the sentence lacks terminal punctuation.
+
+        Checks for terminal punctuation (. ? ! :) after stripping any
+        trailing quote characters (" '). This prevents a spurious period
+        from being appended to sentences like:
+            'He said "Hello."'  ->  unchanged (not 'He said "Hello.".')
+
+        Related GitHub Issue:
+            #7 - Spurious trailing period appended after sentence-final
+                 closing quote
+            https://github.com/craigtrim/fast-sentence-segment/issues/7
+
+        Args:
+            a_sentence: A sentence that may or may not have terminal
+                punctuation.
+
+        Returns:
+            The sentence with a period appended if it lacked terminal
+            punctuation, otherwise unchanged.
         """
-        Purpose:
-            if the sentence is not terminated with a period, then add one
-        :return:
-            a sentence terminated by a period
-        """
-        __blacklist = [':', '?', '!']
-        if not a_sentence.strip().endswith('.'):
-            for ch in __blacklist:
-                if not a_sentence.endswith(ch):
-                    return f"{a_sentence}."
-        return a_sentence
+        # Strip trailing quotes to inspect the actual punctuation
+        stripped = a_sentence.strip().rstrip('"\'')
+        if stripped and stripped[-1] in '.?!:':
+            return a_sentence
+        return f"{a_sentence}."
 
     @staticmethod
     def _is_valid_sentence(a_sentence: str) -> bool:

fast_sentence_segment/dmo/strip_trailing_period_after_quote.py

@@ -0,0 +1,70 @@
+# -*- coding: UTF-8 -*-
+"""Strip spurious trailing periods appended after sentence-final closing quotes.
+
+The spaCy segmenter's _append_period method can produce sentences like:
+    'He said "Hello.".'   (spurious trailing period)
+    'She asked "Why?".'   (spurious trailing period)
+    'He yelled "Stop!".'  (spurious trailing period)
+
+This post-processor removes the trailing period when the sentence ends
+with a closing double quote preceded by terminal punctuation.
+
+Related GitHub Issue:
+    #7 - Spurious trailing period appended after sentence-final
+         closing quote
+    https://github.com/craigtrim/fast-sentence-segment/issues/7
+"""
+
+import re
+
+from fast_sentence_segment.core import BaseObject
+
+# Matches a sentence that ends with terminal punctuation (. ? !)
+# followed by a closing double quote, followed by a spurious period.
+# The fix strips the final period.
+_SPURIOUS_PERIOD_PATTERN = re.compile(r'([.?!]")\.$')
+
+
+class StripTrailingPeriodAfterQuote(BaseObject):
+    """Strip spurious trailing periods after sentence-final closing quotes.
+
+    Detects sentences ending with patterns like:
+        ."."  ->  ."
+        ?"."  ->  ?"
+        !"."  ->  !"
+
+    Applied as a post-processing step in the sentence segmentation
+    pipeline, after spaCy segmentation and after the existing
+    PostProcessStructure step.
+
+    Related GitHub Issue:
+        #7 - Spurious trailing period appended after sentence-final
+             closing quote
+        https://github.com/craigtrim/fast-sentence-segment/issues/7
+    """
+
+    def __init__(self):
+        """
+        Created:
+            29-Jan-2026
+        """
+        BaseObject.__init__(self, __name__)
+
+    def process(self, sentences: list) -> list:
+        """Remove spurious trailing periods after closing quotes.
+
+        Args:
+            sentences: List of segmented sentences.
+
+        Returns:
+            List of sentences with spurious trailing periods removed.
+
+        Example:
+            >>> proc = StripTrailingPeriodAfterQuote()
+            >>> proc.process(['He said "Hello.".', 'She waved.'])
+            ['He said "Hello."', 'She waved.']
+        """
+        return [
+            _SPURIOUS_PERIOD_PATTERN.sub(r'\1', sentence)
+            for sentence in sentences
+        ]

fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py

@@ -0,0 +1,75 @@
+# -*- coding: UTF-8 -*-
+"""Unwrap hard-wrapped text (e.g., Project Gutenberg e-texts).
+
+Joins lines within paragraphs into continuous strings while
+preserving paragraph boundaries (blank lines). Also dehyphenates
+words that were split across lines for typesetting.
+
+Related GitHub Issue:
+    #8 - Add dehyphenation support for words split across lines
+    https://github.com/craigtrim/fast-sentence-segment/issues/8
+"""
+
+import re
+
+# Pattern to match hyphenated word breaks at end of line:
+# - A single hyphen (not -- em-dash)
+# - Followed by newline and optional whitespace
+# - Followed by a lowercase letter (continuation of word)
+_HYPHEN_LINE_BREAK_PATTERN = re.compile(r'(?<!-)-\n\s*([a-z])')
+
+
+def _dehyphenate_block(block: str) -> str:
+    """Remove hyphens from words split across lines.
+
+    Detects the pattern of a word fragment ending with a hyphen
+    at the end of a line, followed by the word continuation
+    starting with a lowercase letter on the next line.
+
+    Examples:
+        "bot-\\ntle" -> "bottle"
+        "cham-\\n    bermaid" -> "chambermaid"
+
+    Args:
+        block: A paragraph block that may contain hyphenated line breaks.
+
+    Returns:
+        The block with hyphenated word breaks rejoined.
+    """
+    return _HYPHEN_LINE_BREAK_PATTERN.sub(r'\1', block)
+
+
+def unwrap_hard_wrapped_text(text: str) -> str:
+    """Unwrap hard-wrapped paragraphs into continuous lines.
+
+    Splits on blank lines to identify paragraphs, then joins
+    lines within each paragraph into a single string with
+    single spaces. Also dehyphenates words that were split
+    across lines for typesetting purposes.
+
+    Examples:
+        >>> unwrap_hard_wrapped_text("a bot-\\ntle of wine")
+        'a bottle of wine'
+        >>> unwrap_hard_wrapped_text("line one\\nline two")
+        'line one line two'
+
+    Args:
+        text: Raw text with hard-wrapped lines.
+
+    Returns:
+        Text with paragraphs unwrapped into continuous strings,
+        separated by double newlines, with hyphenated words rejoined.
+    """
+    blocks = re.split(r'\n\s*\n', text)
+    unwrapped = []
+
+    for block in blocks:
+        # First, dehyphenate words split across lines
+        block = _dehyphenate_block(block)
+        # Then join remaining lines with spaces
+        lines = block.splitlines()
+        joined = ' '.join(line.strip() for line in lines if line.strip())
+        if joined:
+            unwrapped.append(joined)
+
+    return '\n\n'.join(unwrapped)

fast_sentence_segment/svc/perform_sentence_segmentation.py

@@ -17,6 +17,8 @@ from fast_sentence_segment.dmo import NumberedListNormalizer
 from fast_sentence_segment.dmo import QuestionExclamationSplitter
 from fast_sentence_segment.dmo import SpacyDocSegmenter
 from fast_sentence_segment.dmo import PostProcessStructure
+from fast_sentence_segment.dmo import StripTrailingPeriodAfterQuote
+from fast_sentence_segment.dmo import Dehyphenator
 
 
 class PerformSentenceSegmentation(BaseObject):
@@ -45,6 +47,7 @@ class PerformSentenceSegmentation(BaseObject):
         if not self.__nlp:
             self.__nlp = spacy.load("en_core_web_sm")
 
+        self._dehyphenate = Dehyphenator.process
         self._newlines_to_periods = NewlinesToPeriods.process
         self._normalize_numbered_lists = NumberedListNormalizer().process
         self._normalize_ellipses = EllipsisNormalizer().process
@@ -55,6 +58,7 @@ class PerformSentenceSegmentation(BaseObject):
         self._question_exclamation_splitter = QuestionExclamationSplitter().process
         self._title_name_merger = TitleNameMerger().process
         self._post_process = PostProcessStructure().process
+        self._strip_trailing_period = StripTrailingPeriodAfterQuote().process
 
     def _denormalize(self, text: str) -> str:
         """ Restore normalized placeholders to original form """
@@ -94,6 +98,10 @@ class PerformSentenceSegmentation(BaseObject):
         # Normalize tabs to spaces
         input_text = input_text.replace('\t', ' ')
 
+        # Dehyphenate words split across lines (issue #8)
+        # Must happen before newlines are converted to periods
+        input_text = self._dehyphenate(input_text)
+
         input_text = self._normalize_numbered_lists(input_text)
         input_text = self._normalize_ellipses(input_text)
 
@@ -129,6 +137,9 @@ class PerformSentenceSegmentation(BaseObject):
 
         sentences = self._post_process(sentences)
 
+        # Strip spurious trailing periods after closing quotes (issue #7)
+        sentences = self._strip_trailing_period(sentences)
+
         sentences = [
             self._normalize_numbered_lists(x, denormalize=True)
             for x in sentences

{fast_sentence_segment-1.2.1.dist-info → fast_sentence_segment-1.4.0.dist-info}/METADATA

@@ -1,25 +1,29 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: fast-sentence-segment
-Version: 1.2.1
+Version: 1.4.0
 Summary: Fast and Efficient Sentence Segmentation
+Home-page: https://github.com/craigtrim/fast-sentence-segment
 License: MIT
-License-File: LICENSE
 Keywords: nlp,text,preprocess,segment
 Author: Craig Trim
 Author-email: craigtrim@gmail.com
 Maintainer: Craig Trim
 Maintainer-email: craigtrim@gmail.com
 Requires-Python: >=3.9,<4.0
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
 Requires-Dist: spacy (>=3.8.0,<4.0.0)
 Project-URL: Bug Tracker, https://github.com/craigtrim/fast-sentence-segment/issues
 Project-URL: Repository, https://github.com/craigtrim/fast-sentence-segment
@@ -29,8 +33,11 @@ Description-Content-Type: text/markdown
 
 [![PyPI version](https://img.shields.io/pypi/v/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
 [![Python versions](https://img.shields.io/pypi/pyversions/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)
+[![CI](https://img.shields.io/github/actions/workflow/status/craigtrim/fast-sentence-segment/ci.yml?branch=master&label=CI)](https://github.com/craigtrim/fast-sentence-segment/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![spaCy](https://img.shields.io/badge/spaCy-3.8-blue.svg)](https://spacy.io/)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Downloads](https://static.pepy.tech/badge/fast-sentence-segment)](https://pepy.tech/project/fast-sentence-segment)
+[![Downloads/Month](https://static.pepy.tech/badge/fast-sentence-segment/month)](https://pepy.tech/project/fast-sentence-segment)
 
 Fast and efficient sentence segmentation using spaCy with surgical post-processing fixes. Handles complex edge cases like abbreviations (Dr., Mr., etc.), ellipses, quoted text, and multi-paragraph documents.
 
@@ -142,48 +149,36 @@ results = segmenter.input_text("Your text here.")
 
 ### Command Line Interface
 
-Segment text directly from the terminal:
-
 ```bash
-# Direct text input
-echo "Have you seen Dr. Who? It's brilliant!" | segment
-```
+# Inline text
+segment "Gandalf paused... You shall not pass! The Balrog roared."
 
-```
-Have you seen Dr. Who?
-It's brilliant!
-```
+# Pipe from stdin
+echo "Have you seen Dr. Who? It's brilliant!" | segment
 
-```bash
 # Numbered output
-segment -n "Gandalf paused... You shall not pass! The Balrog roared."
-```
+segment -n -f silmarillion.txt
 
-```
-1. Gandalf paused...
-2. You shall not pass!
-3. The Balrog roared.
-```
+# File-to-file (one sentence per line)
+segment-file --input-file book.txt --output-file sentences.txt
 
-```bash
-# From file
-segment -f silmarillion.txt
+# Unwrap hard-wrapped e-texts (Project Gutenberg, etc.)
+segment-file --input-file book.txt --output-file sentences.txt --unwrap
 ```
 
 ## API Reference
 
 | Function | Parameters | Returns | Description |
 |----------|------------|---------|-------------|
-| `segment_text()` | `input_text: str`, `flatten: bool = False` | `list` | Main entry point for segmentation |
+| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False` | `list` | Main entry point for segmentation |
 | `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |
 
-### CLI Options
+### CLI Commands
 
-| Option | Description |
-|--------|-------------|
-| `text` | Text to segment (positional argument) |
-| `-f, --file` | Read text from file |
-| `-n, --numbered` | Number output lines |
+| Command | Description |
+|---------|-------------|
+| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output. |
+| `segment-file --input-file IN --output-file OUT [--unwrap]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts. |
 
 ## Why Nested Lists?
 

{fast_sentence_segment-1.2.1.dist-info → fast_sentence_segment-1.4.0.dist-info}/RECORD

@@ -1,27 +1,32 @@
-fast_sentence_segment/__init__.py,sha256=HTONyC0JLVWTAHyvJO6rMINmxymbfMpARtGeRw5iIsQ,359
+fast_sentence_segment/__init__.py,sha256=jeb4yCy89ivyqbo-4ldJLquPAG_XR_33Q7nrDjqPxvE,1465
 fast_sentence_segment/bp/__init__.py,sha256=j2-WfQ9WwVuXeGSjvV6XLVwEdvau8sdAQe4Pa4DrYi8,33
 fast_sentence_segment/bp/segmenter.py,sha256=UW6DguPgA56h-pPYRsfJhjIzBe40j6NdjkwYxamASyA,1928
-fast_sentence_segment/cli.py,sha256=X2dMLkfc2dtheig62wKC75AohfW0Y9oTU0ORhGUFkbQ,1250
+fast_sentence_segment/cli.py,sha256=Y89BH-xbJ0vykg301D2543MtGP4kYLnA6i3UQ7Hg5YA,3869
 fast_sentence_segment/core/__init__.py,sha256=uoBersYyVStJ5a8zJpQz1GDGaloEdAv2jGHw1292hRM,108
 fast_sentence_segment/core/base_object.py,sha256=AYr7yzusIwawjbKdvcv4yTEnhmx6M583kDZzhzPOmq4,635
 fast_sentence_segment/core/stopwatch.py,sha256=hE6hMz2q6rduaKi58KZmiAL-lRtyh_wWCANhl4KLkRQ,879
-fast_sentence_segment/dmo/__init__.py,sha256=E70tkpdHu86KP2dwBX5Dy5D7eNiU6fzucrfDJOY1ui4,551
+fast_sentence_segment/dmo/__init__.py,sha256=N0lLHVn6zKeg6h1LIfoc4XeXPUY-uSbMT45dP2_vn8M,862
 fast_sentence_segment/dmo/abbreviation_merger.py,sha256=tCXM6yCfMryJvMIVWIxP_EocoibZi8vohFzJ5tvMYr0,4432
 fast_sentence_segment/dmo/abbreviation_splitter.py,sha256=03mSyJcLooNyIjXx6mPlrnjmKgZW-uhUIqG4U-MbIGw,2981
-fast_sentence_segment/dmo/abbreviations.py,sha256=7mpEoOnw5MH8URYmmpxaYs3Wc2eqy4pC0hAnYfYSdck,1639
+fast_sentence_segment/dmo/abbreviations.py,sha256=CGJrJDo6pmYd3pTNEQbdOo8N6tnkCnwyL2X7Si663Os,2530
 fast_sentence_segment/dmo/bullet_point_cleaner.py,sha256=WOZQRWXiiyRi8rOuEIw36EmkaXmATHL9_Dxb2rderw4,1606
+fast_sentence_segment/dmo/dehyphenator.py,sha256=6BJTie7tClRAifeiW8V2CdAAbcbknhtqmKylAdRZ7ko,1776
 fast_sentence_segment/dmo/ellipsis_normalizer.py,sha256=lHs9dLFfKJe-2vFNe17Hik90g3_kXX347OzGP_IOT08,1521
+fast_sentence_segment/dmo/group_quoted_sentences.py,sha256=Ifh_kUwi7sMbzbZvrTgEKkzWe50AafUDhVKVPR9h7wQ,5092
 fast_sentence_segment/dmo/newlines_to_periods.py,sha256=PUrXreqZWiITINfoJL5xRRlXJH6noH0cdXtW1EqAh8I,1517
+fast_sentence_segment/dmo/normalize_quotes.py,sha256=mr53qo_tj_I9XzElOKjUQvCtDQh7mBCGy-iqsHZDX14,2881
 fast_sentence_segment/dmo/numbered_list_normalizer.py,sha256=q0sOCW8Jkn2vTXlUcVhmDvYES3yvJx1oUVl_8y7eL4E,1672
 fast_sentence_segment/dmo/post_process_sentences.py,sha256=5jxG3TmFjxIExMPLhnCB5JT1lXQvFU9r4qQGoATGrWk,916
 fast_sentence_segment/dmo/question_exclamation_splitter.py,sha256=cRsWRu8zb6wOWG-BjMahHfz4YGutKiV9lW7dE-q3tgc,2006
-fast_sentence_segment/dmo/spacy_doc_segmenter.py,sha256=0icAkSQwAUQo3VYqQ2PUjW6-MOU5RNCGPX3-fB5YfCc,2554
+fast_sentence_segment/dmo/spacy_doc_segmenter.py,sha256=_oTsrIL2rjysjt_8bPJVNTn230pUtL-geCC8g174iC4,3163
+fast_sentence_segment/dmo/strip_trailing_period_after_quote.py,sha256=wYkoLy5XJKZIblJXBvDAB8-a81UTQOhOf2u91wjJWUw,2259
 fast_sentence_segment/dmo/title_name_merger.py,sha256=zbG04_VjwM8TtT8LhavvmZqIZL_2xgT2OTxWkK_Zt1s,5133
+fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py,sha256=V1T5RsJBaII_iGJMyWvv6rb2mny8pnVd428oVZL0n5I,2457
 fast_sentence_segment/svc/__init__.py,sha256=9B12mXxBnlalH4OAm1AMLwUMa-RLi2ilv7qhqv26q7g,144
 fast_sentence_segment/svc/perform_paragraph_segmentation.py,sha256=zLKw9rSzb0NNfx4MyEeoGrHwhxTtH5oDrYcAL2LMVHY,1378
-fast_sentence_segment/svc/perform_sentence_segmentation.py,sha256=dqGxFsJoP6ox_MJwtB85R9avEbBAR4x9YKaRaQ5fAXo,5723
-fast_sentence_segment-1.2.1.dist-info/METADATA,sha256=OsUlH-UhmI6fw-ChvsF83G_WwTXBlhZPINo243CaziQ,6889
-fast_sentence_segment-1.2.1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-fast_sentence_segment-1.2.1.dist-info/entry_points.txt,sha256=mDiRuKOZlOeqmtH1eZwqGEGM6KUh0RTzwyETGMpxSDI,58
-fast_sentence_segment-1.2.1.dist-info/licenses/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
-fast_sentence_segment-1.2.1.dist-info/RECORD,,
+fast_sentence_segment/svc/perform_sentence_segmentation.py,sha256=Qaj7oxVHfUd6pIlwCD1O8P14LaGUdolFGuykmrF6gw8,6276
+fast_sentence_segment-1.4.0.dist-info/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
+fast_sentence_segment-1.4.0.dist-info/METADATA,sha256=XernLjKJbfSBxiqhQm-SZC8JFtggSoPe1hUD2X6D9N8,7853
+fast_sentence_segment-1.4.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fast_sentence_segment-1.4.0.dist-info/entry_points.txt,sha256=Zc8OwFKj3ofnjy5ZIFqHzDkIEWweV1AP1xap1ZFGD8M,107
+fast_sentence_segment-1.4.0.dist-info/RECORD,,

{fast_sentence_segment-1.2.1.dist-info → fast_sentence_segment-1.4.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.2.1
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any

{fast_sentence_segment-1.2.1.dist-info → fast_sentence_segment-1.4.0.dist-info}/entry_points.txt

@@ -1,3 +1,4 @@
 [console_scripts]
 segment=fast_sentence_segment.cli:main
+segment-file=fast_sentence_segment.cli:file_main