fast-sentence-segment 1.4.5__tar.gz → 1.5.3__tar.gz

{fast_sentence_segment-1.4.5 → fast_sentence_segment-1.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fast-sentence-segment
-Version: 1.4.5
+Version: 1.5.3
 Summary: Fast and Efficient Sentence Segmentation
 Home-page: https://github.com/craigtrim/fast-sentence-segment
 License: MIT
@@ -164,21 +164,24 @@ segment-file --input-file book.txt --output-file sentences.txt
 
 # Unwrap hard-wrapped e-texts (Project Gutenberg, etc.)
 segment-file --input-file book.txt --output-file sentences.txt --unwrap
+
+# Dialog-aware formatting (implies --unwrap)
+segment -f book.txt --format
 ```
 
 ## API Reference
 
 | Function | Parameters | Returns | Description |
 |----------|------------|---------|-------------|
-| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False` | `list` | Main entry point for segmentation |
+| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False`, `format: str = None` | `list` or `str` | Main entry point for segmentation. Use `format="dialog"` for dialog-aware output. |
 | `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |
 
 ### CLI Commands
 
 | Command | Description |
 |---------|-------------|
-| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output. |
-| `segment-file --input-file IN --output-file OUT [--unwrap]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts. |
+| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output, `--format` for dialog-aware paragraph grouping. |
+| `segment-file --input-file IN --output-file OUT [--unwrap] [--format]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts, `--format` for dialog-aware formatting. |
 
 ## Why Nested Lists?
 

{fast_sentence_segment-1.4.5 → fast_sentence_segment-1.5.3}/README.md

@@ -133,21 +133,24 @@ segment-file --input-file book.txt --output-file sentences.txt
 
 # Unwrap hard-wrapped e-texts (Project Gutenberg, etc.)
 segment-file --input-file book.txt --output-file sentences.txt --unwrap
+
+# Dialog-aware formatting (implies --unwrap)
+segment -f book.txt --format
 ```
 
 ## API Reference
 
 | Function | Parameters | Returns | Description |
 |----------|------------|---------|-------------|
-| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False` | `list` | Main entry point for segmentation |
+| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False`, `format: str = None` | `list` or `str` | Main entry point for segmentation. Use `format="dialog"` for dialog-aware output. |
 | `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |
 
 ### CLI Commands
 
 | Command | Description |
 |---------|-------------|
-| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output. |
-| `segment-file --input-file IN --output-file OUT [--unwrap]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts. |
+| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output, `--format` for dialog-aware paragraph grouping. |
+| `segment-file --input-file IN --output-file OUT [--unwrap] [--format]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts, `--format` for dialog-aware formatting. |
 
 ## Why Nested Lists?
 

{fast_sentence_segment-1.4.5 → fast_sentence_segment-1.5.3}/fast_sentence_segment/cli.py

@@ -100,10 +100,13 @@ def main():
     parser.add_argument(
         "--format",
         action="store_true",
-        help="Format output with dialog-aware paragraph grouping",
+        help="Format output with dialog-aware paragraph grouping (implies --unwrap)",
     )
     args = parser.parse_args()
 
+    # --format implies --unwrap
+    unwrap = args.unwrap or args.format
+
     # Get input text
     if args.file:
         with open(args.file, "r", encoding="utf-8") as f:
@@ -118,7 +121,7 @@ def main():
 
     # Segment and output
     result = segment_text(
-        text.strip(), flatten=True, unwrap=args.unwrap,
+        text.strip(), flatten=True, unwrap=unwrap,
         format="dialog" if args.format else None
     )
 
@@ -170,7 +173,6 @@ def _process_single_file(
     # Step 3: Write output
     if format:
         # Format mode returns a string
-        sentence_count = result.count("\n") + 1 if result else 0
         print(f"\r  {GREEN}✓{RESET} Segmented text ({elapsed:.2f}s)")
         with open(output_file, "w", encoding="utf-8") as f:
             f.write(result + "\n")
@@ -221,10 +223,13 @@ def file_main():
     parser.add_argument(
         "--format",
         action="store_true",
-        help="Format output with dialog-aware paragraph grouping",
+        help="Format output with dialog-aware paragraph grouping (implies --unwrap)",
     )
     args = parser.parse_args()
 
+    # --format implies --unwrap
+    unwrap = args.unwrap or args.format
+
     # Validate arguments
     if not args.input_file and not args.input_dir:
         print(f"  {YELLOW}Error:{RESET} Either --input-file or --input-dir is required")
@@ -240,36 +245,37 @@ def file_main():
 
     # Process directory
     if args.input_dir:
-        if not os.path.isdir(args.input_dir):
-            print(f"  {YELLOW}Error:{RESET} Directory not found: {args.input_dir}")
+        input_dir = os.path.expanduser(args.input_dir)
+        if not os.path.isdir(input_dir):
+            print(f"  {YELLOW}Error:{RESET} Directory not found: {input_dir}")
             sys.exit(1)
 
         # Find all .txt files
         txt_files = sorted([
-            f for f in os.listdir(args.input_dir)
+            f for f in os.listdir(input_dir)
             if f.endswith(".txt") and not f.endswith("-clean.txt")
         ])
 
         if not txt_files:
-            print(f"  {YELLOW}Error:{RESET} No .txt files found in {args.input_dir}")
+            print(f"  {YELLOW}Error:{RESET} No .txt files found in {input_dir}")
             sys.exit(1)
 
         _header("segment-file (batch)")
         print(f"  {DIM}Processing {len(txt_files)} files in directory{RESET}")
         print()
-        _param("Directory", args.input_dir)
+        _param("Directory", input_dir)
         _param("Files", str(len(txt_files)))
-        _param("Unwrap", "enabled" if args.unwrap else "disabled")
+        _param("Unwrap", "enabled" if unwrap else "disabled")
         _param("Normalize quotes", "disabled" if not normalize else "enabled")
         _param("Format", "dialog" if args.format else "default (one sentence per line)")
         print()
 
         format_value = "dialog" if args.format else None
         for i, filename in enumerate(txt_files, 1):
-            input_path = os.path.join(args.input_dir, filename)
+            input_path = os.path.join(input_dir, filename)
             output_path = _generate_output_path(input_path)
             print(f"  {BOLD}[{i}/{len(txt_files)}]{RESET} {filename}")
-            _process_single_file(input_path, output_path, args.unwrap, normalize, format_value)
+            _process_single_file(input_path, output_path, unwrap, normalize, format_value)
             print()
 
         print(f"  {GREEN}Done! Processed {len(txt_files)} files.{RESET}")
@@ -277,18 +283,20 @@ def file_main():
         return
 
     # Process single file
-    if not os.path.isfile(args.input_file):
-        print(f"  {YELLOW}Error:{RESET} File not found: {args.input_file}")
+    input_file = os.path.expanduser(args.input_file)
+    if not os.path.isfile(input_file):
+        print(f"  {YELLOW}Error:{RESET} File not found: {input_file}")
         sys.exit(1)
 
-    output_file = args.output_file or _generate_output_path(args.input_file)
+    output_file = args.output_file or _generate_output_path(input_file)
+    output_file = os.path.expanduser(output_file)
 
     _header("segment-file")
     print(f"  {DIM}Segmenting text file into sentences{RESET}")
     print()
 
     format_value = "dialog" if args.format else None
-    _process_single_file(args.input_file, output_file, args.unwrap, normalize, format_value)
+    _process_single_file(input_file, output_file, unwrap, normalize, format_value)
 
     print(f"\n  {GREEN}Done!{RESET}")
     print()

{fast_sentence_segment-1.4.5 → fast_sentence_segment-1.5.3}/fast_sentence_segment/dmo/dialog_formatter.py

@@ -37,16 +37,129 @@ DOUBLE_QUOTES = '""\""'
 SINGLE_QUOTES = "'''"
 ALL_QUOTES = DOUBLE_QUOTES + SINGLE_QUOTES
 
+# Known elision words (case-insensitive)
+# These are specific words where an apostrophe replaces omitted letters at the start
+KNOWN_ELISION_WORDS = {
+    # 'it' elisions
+    "tis", "twas", "twere", "twill", "twould", "taint", "tother",
+    # Archaic oaths (very specific words)
+    "sblood", "sdeath", "swounds", "sbodikins", "slid", "strewth", "zounds",
+    # Common elisions with a-/be- prefix dropped
+    "bout", "bove", "cross", "fore", "fraid", "gainst", "live", "loft", "lone",
+    "long", "mid", "midst", "mong", "mongst", "neath", "round", "sleep", "tween",
+    "twixt", "wake", "ware", "way", "cause", "cuz", "coz", "hind", "low", "side",
+    "yond", "cept", "scaped", "specially", "splain", "spect",
+    # Cockney/dialect h-dropping (common words and their forms)
+    "e", "em", "er", "ere", "im", "is", "ave", "avin", "ead", "ear", "eard",
+    "eart", "eaven", "eavens", "eavy", "eck", "edge", "eel", "eight", "ell",
+    "elp", "en", "ero", "igh", "ill", "imself", "int", "it", "itch", "obby",
+    "old", "ole", "oliday", "oller", "ollow", "oly", "ome", "onest", "oney",
+    "onor", "onour", "ood", "ook", "oop", "ope", "orizon", "orn", "orrible",
+    "orse", "ospital", "ot", "otel", "our", "ouse", "ow", "owever", "uge",
+    "undred", "ungry", "unt", "urry", "urt", "usband", "alf", "all", "am",
+    "and", "andsome", "appen", "appy", "ard", "arm", "at", "ate",
+    # Cockney th-dropping
+    "at", "ese", "ey", "ose", "ough", "rough",
+    # Other prefix elisions
+    "count", "fter", "gain", "gin", "less", "nother", "nough", "nuff", "pears",
+    "pon", "prentice", "scuse", "spite", "spose", "stead", "tarnal", "tend",
+    "thout", "til", "till", "un",
+    # Modern colloquial
+    "kay", "sup", "dya", "ja", "yer", "copter",
+    # Musical
+    "cello",
+    # Year abbreviations handled separately (digits)
+    # 'member (remember)
+    "member",
+}
+
+
+def _is_elision(text: str, pos: int) -> bool:
+    """Check if apostrophe at position is a word-initial elision.
+
+    Elisions like 'tis, 'twas, 'cello, 'em replace omitted letters at word start.
+    Dialog quotes like 'Hello!' surround quoted speech.
+
+    Args:
+        text: The full text.
+        pos: Position of the apostrophe character.
+
+    Returns:
+        True if this appears to be an elision, not a dialog quote.
+    """
+    if pos >= len(text) - 1:
+        return False
+
+    next_char = text[pos + 1]
+
+    # If followed by a digit, it's a year abbreviation ('99, '20s)
+    if next_char.isdigit():
+        return True
+
+    # Extract the word after the apostrophe (letters only, up to non-letter)
+    word_start = pos + 1
+    word_end = word_start
+    while word_end < len(text) and text[word_end].isalpha():
+        word_end += 1
+
+    if word_end == word_start:
+        return False  # No letters after apostrophe
+
+    word = text[word_start:word_end].lower()
+
+    # Check if it's a known elision word
+    return word in KNOWN_ELISION_WORDS
+
 
 def _count_quotes(text: str) -> int:
-    """Count quote characters in text (both single and double)."""
-    return sum(1 for c in text if c in ALL_QUOTES)
+    """Count actual quote characters in text, excluding apostrophes.
+
+    Apostrophes in contractions (don't, can't), possessives (Jack's, Joselito's),
+    and word-initial elisions ('tis, 'twas, 'cello, 'em) are NOT counted as quotes
+    because they don't indicate dialog boundaries.
+
+    A quote character is considered an apostrophe (not a quote) if:
+    - It's preceded by a letter AND followed by a letter (mid-word: don't, Joselito's)
+    - It's a word-initial elision ('tis, 'Twas, 'cello, '99)
+    """
+    count = 0
+    for i, c in enumerate(text):
+        if c not in ALL_QUOTES:
+            continue
+
+        # Check if this is a mid-word apostrophe (contraction/possessive)
+        prev_is_letter = i > 0 and text[i - 1].isalpha()
+        next_is_letter = i < len(text) - 1 and text[i + 1].isalpha()
+
+        if prev_is_letter and next_is_letter:
+            # Mid-word apostrophe (contraction/possessive) - don't count
+            continue
+
+        # Check if this is a word-initial elision
+        prev_is_word_boundary = i == 0 or not text[i - 1].isalnum()
+        if prev_is_word_boundary and c in SINGLE_QUOTES and _is_elision(text, i):
+            # Word-initial elision - don't count
+            continue
+
+        count += 1
+    return count
 
 
 def _starts_with_quote(text: str) -> bool:
-    """Check if text starts with a quote character."""
+    """Check if text starts with a dialog quote (not an elision).
+
+    Returns True only for actual dialog openings, not for elisions
+    like 'tis, 'twas, 'cello, etc.
+    """
     text = text.lstrip()
-    return text and text[0] in ALL_QUOTES
+    if not text or text[0] not in ALL_QUOTES:
+        return False
+
+    # Check if this is an elision rather than a dialog quote
+    if text[0] in SINGLE_QUOTES and _is_elision(text, 0):
+        return False
+
+    return True
 
 
 def _ends_with_closing_quote(text: str) -> bool:
@@ -211,7 +324,6 @@ def format_dialog(sentences: List[str]) -> str:
         quote_count = _count_quotes(sentence)
         starts_quote = _starts_with_quote(sentence)
         is_narrative = _is_narrative(sentence)
-        is_complete = _is_complete_quote(sentence)
 
         # Get info about previous sentence
         prev_sentence = current_para[-1] if current_para else ""
@@ -230,6 +342,10 @@ def format_dialog(sentences: List[str]) -> str:
         elif starts_quote:
             # New quote starting - always new paragraph
             should_start_new_para = True
+        elif is_narrative and prev_was_narrative:
+            # Consecutive narrative sentences - each gets its own paragraph
+            # This gives clean ebook formatting with paragraph breaks
+            should_start_new_para = True
         elif is_narrative and prev_was_complete and not prev_was_narrative:
             # Narrative after complete dialog - new paragraph
             should_start_new_para = True

fast_sentence_segment-1.5.3/fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py

@@ -0,0 +1,319 @@
+# -*- coding: UTF-8 -*-
+"""Unwrap hard-wrapped text (e.g., Project Gutenberg e-texts).
+
+Joins lines within paragraphs into continuous strings while
+preserving paragraph boundaries (blank lines). Also dehyphenates
+words that were split across lines for typesetting.
+
+Related GitHub Issues:
+    #8 - Add dehyphenation support for words split across lines
+    https://github.com/craigtrim/fast-sentence-segment/issues/8
+
+    #12 - Unwrap fails when hard-wrap splits across blank line
+    https://github.com/craigtrim/fast-sentence-segment/issues/12
+
+    #14 - Make unwrap quote-aware to join text across blank lines inside open quotes
+    https://github.com/craigtrim/fast-sentence-segment/issues/14
+"""
+
+import re
+
+# Quote characters to track for dialog detection
+# Includes straight and curly/smart quotes
+DOUBLE_QUOTES = '"""\u201c\u201d'  # " " " " "
+SINGLE_QUOTES = "'''\u2018\u2019"  # ' ' ' ' '
+ALL_QUOTES = DOUBLE_QUOTES + SINGLE_QUOTES
+
+# Known elision words where an apostrophe replaces omitted letters at word start
+# These should NOT be counted as dialog quotes
+KNOWN_ELISION_WORDS = {
+    # 'it' elisions
+    "tis", "twas", "twere", "twill", "twould", "taint", "tother",
+    # Archaic oaths
+    "sblood", "sdeath", "swounds", "sbodikins", "slid", "strewth", "zounds",
+    # Common elisions with a-/be- prefix dropped
+    "bout", "bove", "cross", "fore", "fraid", "gainst", "live", "loft", "lone",
+    "long", "mid", "midst", "mong", "mongst", "neath", "round", "sleep", "tween",
+    "twixt", "wake", "ware", "way", "cause", "cuz", "coz", "hind", "low", "side",
+    "yond", "cept", "scaped", "specially", "splain", "spect",
+    # Cockney/dialect h-dropping
+    "e", "em", "er", "ere", "im", "is", "ave", "avin", "ead", "ear", "eard",
+    "eart", "eaven", "eavens", "eavy", "eck", "edge", "eel", "eight", "ell",
+    "elp", "en", "ero", "igh", "ill", "imself", "int", "it", "itch", "obby",
+    "old", "ole", "oliday", "oller", "ollow", "oly", "ome", "onest", "oney",
+    "onor", "onour", "ood", "ook", "oop", "ope", "orizon", "orn", "orrible",
+    "orse", "ospital", "ot", "otel", "our", "ouse", "ow", "owever", "uge",
+    "undred", "ungry", "unt", "urry", "urt", "usband", "alf", "all", "am",
+    "and", "andsome", "appen", "appy", "ard", "arm", "at", "ate",
+    # Cockney th-dropping
+    "ese", "ey", "ose", "ough", "rough",
+    # Other prefix elisions
+    "count", "fter", "gain", "gin", "less", "nother", "nough", "nuff", "pears",
+    "pon", "prentice", "scuse", "spite", "spose", "stead", "tarnal", "tend",
+    "thout", "til", "till", "un",
+    # Modern colloquial
+    "kay", "sup", "dya", "ja", "yer", "copter",
+    # Musical
+    "cello",
+    # 'member (remember)
+    "member",
+}
+
+# Pattern to match hyphenated word breaks at end of line:
+# - A single hyphen (not -- em-dash)
+# - Followed by newline and optional whitespace
+# - Followed by a lowercase letter (continuation of word)
+_HYPHEN_LINE_BREAK_PATTERN = re.compile(r'(?<!-)-\n\s*([a-z])')
+
+
+def _is_elision(text: str, pos: int) -> bool:
+    """Check if apostrophe at position is a word-initial elision.
+
+    Elisions like 'tis, 'twas, 'cello, 'em replace omitted letters at word start.
+    Dialog quotes like 'Hello!' surround quoted speech.
+
+    Args:
+        text: The full text.
+        pos: Position of the apostrophe character.
+
+    Returns:
+        True if this appears to be an elision, not a dialog quote.
+    """
+    if pos >= len(text) - 1:
+        return False
+
+    next_char = text[pos + 1]
+
+    # If followed by a digit, it's a year abbreviation ('99, '20s)
+    if next_char.isdigit():
+        return True
+
+    # Extract the word after the apostrophe (letters only, up to non-letter)
+    word_start = pos + 1
+    word_end = word_start
+    while word_end < len(text) and text[word_end].isalpha():
+        word_end += 1
+
+    if word_end == word_start:
+        return False  # No letters after apostrophe
+
+    word = text[word_start:word_end].lower()
+
+    # Check if it's a known elision word
+    return word in KNOWN_ELISION_WORDS
+
+
+def _count_quotes(text: str) -> int:
+    """Count actual quote characters in text, excluding apostrophes.
+
+    Apostrophes in contractions (don't, can't), possessives (Jack's, Joselito's),
+    and word-initial elisions ('tis, 'twas, 'cello, 'em) are NOT counted as quotes
+    because they don't indicate dialog boundaries.
+
+    A quote character is considered an apostrophe (not a quote) if:
+    - It's preceded by a letter AND followed by a letter (mid-word: don't, Joselito's)
+    - It's a word-initial elision ('tis, 'Twas, 'cello, '99)
+
+    Args:
+        text: The text to count quotes in.
+
+    Returns:
+        Number of actual quote characters (not apostrophes).
+    """
+    count = 0
+    for i, c in enumerate(text):
+        if c not in ALL_QUOTES:
+            continue
+
+        # Check if this is a mid-word apostrophe (contraction/possessive)
+        prev_is_letter = i > 0 and text[i - 1].isalpha()
+        next_is_letter = i < len(text) - 1 and text[i + 1].isalpha()
+
+        if prev_is_letter and next_is_letter:
+            # Mid-word apostrophe (contraction/possessive) - don't count
+            continue
+
+        # Check if this is a word-initial elision
+        prev_is_word_boundary = i == 0 or not text[i - 1].isalnum()
+        if prev_is_word_boundary and c in SINGLE_QUOTES and _is_elision(text, i):
+            # Word-initial elision - don't count
+            continue
+
+        count += 1
+    return count
+
+
+def _is_inside_open_quote(text: str) -> bool:
+    """Check if text ends with an unclosed quote.
+
+    Counts quote characters (excluding apostrophes) and returns True
+    if the count is odd, meaning there's an open quote.
+
+    Args:
+        text: The text to check.
+
+    Returns:
+        True if there's an unclosed quote at the end of text.
+    """
+    return _count_quotes(text) % 2 == 1
+
+# Characters that indicate end of sentence
+_SENTENCE_END_PUNCT = {'.', '?', '!'}
+
+
+def _ends_with_sentence_punct(text: str) -> bool:
+    """Check if text ends with sentence-ending punctuation.
+
+    Handles trailing quotes/parens: 'He said "Hello."' -> True
+    Handles ellipsis: 'He wondered...' -> True
+
+    Args:
+        text: The text to check.
+
+    Returns:
+        True if text ends with . ? ! or ... (possibly followed by quotes/parens).
+    """
+    if not text:
+        return False
+
+    # Strip trailing whitespace and quotes/parens (including curly quotes)
+    stripped = text.rstrip()
+    trailing_chars = {'"', "'", ')', ']', '\u201d', '\u2019'}  # " ' ) ] " '
+    while stripped and stripped[-1] in trailing_chars:
+        stripped = stripped[:-1]
+
+    if not stripped:
+        return False
+
+    return stripped[-1] in _SENTENCE_END_PUNCT
+
+
+def _dehyphenate_block(block: str) -> str:
+    """Remove hyphens from words split across lines.
+
+    Detects the pattern of a word fragment ending with a hyphen
+    at the end of a line, followed by the word continuation
+    starting with a lowercase letter on the next line.
+
+    Examples:
+        "bot-\\ntle" -> "bottle"
+        "cham-\\n    bermaid" -> "chambermaid"
+
+    Args:
+        block: A paragraph block that may contain hyphenated line breaks.
+
+    Returns:
+        The block with hyphenated word breaks rejoined.
+    """
+    return _HYPHEN_LINE_BREAK_PATTERN.sub(r'\1', block)
+
+
+def unwrap_hard_wrapped_text(text: str) -> str:
+    """Unwrap hard-wrapped paragraphs into continuous lines.
+
+    Splits on blank lines to identify paragraphs, then joins
+    lines within each paragraph into a single string with
+    single spaces. Also dehyphenates words that were split
+    across lines for typesetting purposes.
+
+    Special handling for spurious blank lines (issue #12):
+    When a single blank line appears mid-sentence (previous line
+    doesn't end with .?! and next line starts lowercase), the
+    text is joined rather than treated as a paragraph break.
+
+    Quote-aware joining (issue #14):
+    When we're inside an open quote (odd number of quote characters),
+    join across blank lines even if the previous line ends with
+    sentence punctuation and the next starts uppercase. This keeps
+    multi-sentence dialog together.
+
+    Examples:
+        >>> unwrap_hard_wrapped_text("a bot-\\ntle of wine")
+        'a bottle of wine'
+        >>> unwrap_hard_wrapped_text("line one\\nline two")
+        'line one line two'
+        >>> unwrap_hard_wrapped_text("His colour\\n\\nmounted;")
+        'His colour mounted;'
+        >>> unwrap_hard_wrapped_text("'First.\\n\\nSecond.'")
+        "'First. Second.'"
+
+    Args:
+        text: Raw text with hard-wrapped lines.
+
+    Returns:
+        Text with paragraphs unwrapped into continuous strings,
+        separated by double newlines, with hyphenated words rejoined.
+    """
+    lines = text.splitlines()
+    paragraphs: list[list[str]] = []
+    current_para_lines: list[str] = []
+    blank_line_count = 0
+
+    for line in lines:
+        stripped = line.strip()
+
+        if not stripped:
+            # Blank line (or whitespace-only)
+            blank_line_count += 1
+        else:
+            # Non-blank line
+            if current_para_lines and blank_line_count > 0:
+                # We have previous content and saw blank line(s)
+                # Build the current paragraph text to check ending
+                prev_para = ' '.join(ln.strip() for ln in current_para_lines if ln.strip())
+
+                # Issue #12: Join across single blank line if:
+                # 1. Exactly one blank line
+                # 2. Previous paragraph doesn't end with sentence punctuation
+                # 3. Current line starts with lowercase
+                issue_12_join = (
+                    blank_line_count == 1
+                    and prev_para
+                    and not _ends_with_sentence_punct(prev_para)
+                    and stripped[0].islower()
+                )
+
+                # Issue #14: Join across blank line if inside open quote
+                # Even if previous ends with sentence punct and next starts uppercase,
+                # we should join if we're inside an unclosed quote
+                issue_14_join = (
+                    blank_line_count == 1
+                    and prev_para
+                    and _is_inside_open_quote(prev_para)
+                )
+
+                should_join = issue_12_join or issue_14_join
+
+                if should_join:
+                    # Treat as continuation of current paragraph
+                    current_para_lines.append(line)
+                else:
+                    # Finish current paragraph, start new one
+                    paragraphs.append(current_para_lines)
+                    current_para_lines = [line]
+            else:
+                # No blank lines seen, add to current paragraph
+                current_para_lines.append(line)
+
+            blank_line_count = 0
+
+    # Don't forget the last paragraph
+    if current_para_lines:
+        paragraphs.append(current_para_lines)
+
+    # Process each paragraph: dehyphenate and join lines
+    unwrapped = []
+    for para_lines in paragraphs:
+        block = '\n'.join(para_lines)
+        # First, dehyphenate words split across lines
+        block = _dehyphenate_block(block)
+        # Then join remaining lines with spaces
+        joined_lines = block.splitlines()
+        joined = ' '.join(ln.strip() for ln in joined_lines if ln.strip())
+        # Normalize multiple spaces to single space (OCR artifacts, formatting)
+        while '  ' in joined:
+            joined = joined.replace('  ', ' ')
+        if joined:
+            unwrapped.append(joined)
+
+    return '\n\n'.join(unwrapped)

{fast_sentence_segment-1.4.5 → fast_sentence_segment-1.5.3}/fast_sentence_segment/svc/perform_sentence_segmentation.py

@@ -43,19 +43,20 @@ def _load_spacy_model(model_name: str = "en_core_web_sm"):
 
         return spacy.load(model_name)
 
-from fast_sentence_segment.dmo import AbbreviationMerger
-from fast_sentence_segment.dmo import AbbreviationSplitter
-from fast_sentence_segment.dmo import TitleNameMerger
-from fast_sentence_segment.dmo import EllipsisNormalizer
-from fast_sentence_segment.dmo import NewlinesToPeriods
-from fast_sentence_segment.dmo import BulletPointCleaner
-from fast_sentence_segment.dmo import NumberedListNormalizer
-from fast_sentence_segment.dmo import QuestionExclamationSplitter
-from fast_sentence_segment.dmo import SpacyDocSegmenter
-from fast_sentence_segment.dmo import PostProcessStructure
-from fast_sentence_segment.dmo import StripTrailingPeriodAfterQuote
-from fast_sentence_segment.dmo import Dehyphenator
-from fast_sentence_segment.dmo import OcrArtifactFixer
+# Imports after lazy spacy loading function (intentional)
+from fast_sentence_segment.dmo import AbbreviationMerger  # noqa: E402
+from fast_sentence_segment.dmo import AbbreviationSplitter  # noqa: E402
+from fast_sentence_segment.dmo import TitleNameMerger  # noqa: E402
+from fast_sentence_segment.dmo import EllipsisNormalizer  # noqa: E402
+from fast_sentence_segment.dmo import NewlinesToPeriods  # noqa: E402
+from fast_sentence_segment.dmo import BulletPointCleaner  # noqa: E402
+from fast_sentence_segment.dmo import NumberedListNormalizer  # noqa: E402
+from fast_sentence_segment.dmo import QuestionExclamationSplitter  # noqa: E402
+from fast_sentence_segment.dmo import SpacyDocSegmenter  # noqa: E402
+from fast_sentence_segment.dmo import PostProcessStructure  # noqa: E402
+from fast_sentence_segment.dmo import StripTrailingPeriodAfterQuote  # noqa: E402
+from fast_sentence_segment.dmo import Dehyphenator  # noqa: E402
+from fast_sentence_segment.dmo import OcrArtifactFixer  # noqa: E402
 
 
 class PerformSentenceSegmentation(BaseObject):

{fast_sentence_segment-1.4.5 → fast_sentence_segment-1.5.3}/pyproject.toml

@@ -11,7 +11,7 @@ description = "Fast and Efficient Sentence Segmentation"
 license = "MIT"
 name = "fast-sentence-segment"
 readme = "README.md"
-version = "1.4.5"
+version = "1.5.3"
 
 keywords = ["nlp", "text", "preprocess", "segment"]
 repository = "https://github.com/craigtrim/fast-sentence-segment"

{fast_sentence_segment-1.4.5 → fast_sentence_segment-1.5.3}/setup.py

@@ -20,9 +20,9 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'fast-sentence-segment',
-    'version': '1.4.5',
+    'version': '1.5.3',
     'description': 'Fast and Efficient Sentence Segmentation',
-    'long_description': '# Fast Sentence Segmentation\n\n[![PyPI version](https://img.shields.io/pypi/v/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)\n[![Python versions](https://img.shields.io/pypi/pyversions/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)\n[![Tests](https://img.shields.io/badge/tests-664-brightgreen)](https://github.com/craigtrim/fast-sentence-segment/tree/master/tests)\n[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)\n[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)\n[![Downloads](https://static.pepy.tech/badge/fast-sentence-segment)](https://pepy.tech/project/fast-sentence-segment)\n[![Downloads/Month](https://static.pepy.tech/badge/fast-sentence-segment/month)](https://pepy.tech/project/fast-sentence-segment)\n\nFast and efficient sentence segmentation using spaCy with surgical post-processing fixes. Handles complex edge cases like abbreviations (Dr., Mr., etc.), ellipses, quoted text, and multi-paragraph documents.\n\n## Why This Library?\n\n1. **Keep it local**: LLM API calls cost money and send your data to third parties. Run sentence segmentation entirely on your machine.\n2. **spaCy perfected**: spaCy is a great local model, but it makes mistakes. This library fixes most of spaCy\'s shortcomings.\n\n## Features\n\n- **Paragraph-aware segmentation**: Returns sentences grouped by paragraph\n- **Abbreviation handling**: Correctly handles "Dr.", "Mr.", "etc.", "p.m.", "a.m." without false splits\n- **Ellipsis preservation**: Keeps `...` intact while detecting sentence boundaries\n- **Question/exclamation splitting**: Properly splits on `?` and `!` followed by capital letters\n- **Cached processing**: LRU cache for repeated text processing\n- **Flexible output**: Nested lists (by paragraph) or flattened list of sentences\n- **Bullet point & numbered list normalization**: Cleans common list formats\n- **CLI tool**: Command-line interface for quick segmentation\n\n## Installation\n\n```bash\npip install fast-sentence-segment\n```\n\nAfter installation, download the spaCy model:\n\n```bash\npython -m spacy download en_core_web_sm\n```\n\n## Quick Start\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = "Do you like Dr. Who? I prefer Dr. Strange! Mr. T is also cool."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n  "Do you like Dr. Who?",\n  "I prefer Dr. Strange!",\n  "Mr. T is also cool."\n]\n```\n\nNotice how "Dr. Who?" stays together as a single sentence—the library correctly recognizes that a title followed by a single-word name ending in `?` or `!` is a name reference, not a sentence boundary.\n\n## Usage\n\n### Basic Segmentation\n\nThe `segment_text` function returns a list of lists, where each inner list represents a paragraph containing its sentences:\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = """Gandalf spoke softly. "All we have to decide is what to do with the time given us."\n\nFrodo nodded. The weight of the Ring pressed against his chest."""\n\nresults = segment_text(text)\n```\n\n```json\n[\n  [\n    "Gandalf spoke softly.",\n    "\\"All we have to decide is what to do with the time given us.\\"."\n  ],\n  [\n    "Frodo nodded.",\n    "The weight of the Ring pressed against his chest."\n  ]\n]\n```\n\n### Flattened Output\n\nIf you don\'t need paragraph boundaries, use the `flatten` parameter:\n\n```python\ntext = "At 9 a.m. the hobbits set out. By 3 p.m. they reached Rivendell. Mr. Frodo was exhausted."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n  "At 9 a.m. the hobbits set out.",\n  "By 3 p.m. they reached Rivendell.",\n  "Mr. Frodo was exhausted."\n]\n```\n\n### Direct Segmenter Access\n\nFor more control, use the `Segmenter` class directly:\n\n```python\nfrom fast_sentence_segment import Segmenter\n\nsegmenter = Segmenter()\nresults = segmenter.input_text("Your text here.")\n```\n\n### Command Line Interface\n\n```bash\n# Inline text\nsegment "Gandalf paused... You shall not pass! The Balrog roared."\n\n# Pipe from stdin\necho "Have you seen Dr. Who? It\'s brilliant!" | segment\n\n# Numbered output\nsegment -n -f silmarillion.txt\n\n# File-to-file (one sentence per line)\nsegment-file --input-file book.txt --output-file sentences.txt\n\n# Unwrap hard-wrapped e-texts (Project Gutenberg, etc.)\nsegment-file --input-file book.txt --output-file sentences.txt --unwrap\n```\n\n## API Reference\n\n| Function | Parameters | Returns | Description |\n|----------|------------|---------|-------------|\n| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False` | `list` | Main entry point for segmentation |\n| `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |\n\n### CLI Commands\n\n| Command | Description |\n|---------|-------------|\n| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output. |\n| `segment-file --input-file IN --output-file OUT [--unwrap]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts. |\n\n## Why Nested Lists?\n\nThe segmentation process preserves document structure by segmenting into both paragraphs and sentences. Each outer list represents a paragraph, and each inner list contains that paragraph\'s sentences. This is useful for:\n\n- Document structure analysis\n- Paragraph-level processing\n- Maintaining original text organization\n\nUse `flatten=True` when you only need sentences without paragraph context.\n\n## Requirements\n\n- Python 3.9+\n- spaCy 3.8+\n- en_core_web_sm spaCy model\n\n## How It Works\n\nThis library uses spaCy for initial sentence segmentation, then applies surgical post-processing fixes for cases where spaCy\'s default behavior is incorrect:\n\n1. **Pre-processing**: Normalize numbered lists, preserve ellipses with placeholders\n2. **spaCy segmentation**: Use spaCy\'s sentence boundary detection\n3. **Post-processing**: Split on abbreviation boundaries, handle `?`/`!` + capital patterns\n4. **Denormalization**: Restore placeholders to original text\n\n## License\n\nMIT License - see [LICENSE](LICENSE) for details.\n\n## Contributing\n\nContributions are welcome! Please feel free to submit a Pull Request.\n\n1. Fork the repository\n2. Create your feature branch (`git checkout -b feature/amazing-feature`)\n3. Run tests (`make test`)\n4. Commit your changes\n5. Push to the branch\n6. Open a Pull Request\n',
+    'long_description': '# Fast Sentence Segmentation\n\n[![PyPI version](https://img.shields.io/pypi/v/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)\n[![Python versions](https://img.shields.io/pypi/pyversions/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)\n[![Tests](https://img.shields.io/badge/tests-664-brightgreen)](https://github.com/craigtrim/fast-sentence-segment/tree/master/tests)\n[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)\n[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)\n[![Downloads](https://static.pepy.tech/badge/fast-sentence-segment)](https://pepy.tech/project/fast-sentence-segment)\n[![Downloads/Month](https://static.pepy.tech/badge/fast-sentence-segment/month)](https://pepy.tech/project/fast-sentence-segment)\n\nFast and efficient sentence segmentation using spaCy with surgical post-processing fixes. Handles complex edge cases like abbreviations (Dr., Mr., etc.), ellipses, quoted text, and multi-paragraph documents.\n\n## Why This Library?\n\n1. **Keep it local**: LLM API calls cost money and send your data to third parties. Run sentence segmentation entirely on your machine.\n2. **spaCy perfected**: spaCy is a great local model, but it makes mistakes. This library fixes most of spaCy\'s shortcomings.\n\n## Features\n\n- **Paragraph-aware segmentation**: Returns sentences grouped by paragraph\n- **Abbreviation handling**: Correctly handles "Dr.", "Mr.", "etc.", "p.m.", "a.m." without false splits\n- **Ellipsis preservation**: Keeps `...` intact while detecting sentence boundaries\n- **Question/exclamation splitting**: Properly splits on `?` and `!` followed by capital letters\n- **Cached processing**: LRU cache for repeated text processing\n- **Flexible output**: Nested lists (by paragraph) or flattened list of sentences\n- **Bullet point & numbered list normalization**: Cleans common list formats\n- **CLI tool**: Command-line interface for quick segmentation\n\n## Installation\n\n```bash\npip install fast-sentence-segment\n```\n\nAfter installation, download the spaCy model:\n\n```bash\npython -m spacy download en_core_web_sm\n```\n\n## Quick Start\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = "Do you like Dr. Who? I prefer Dr. Strange! Mr. T is also cool."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n  "Do you like Dr. Who?",\n  "I prefer Dr. Strange!",\n  "Mr. T is also cool."\n]\n```\n\nNotice how "Dr. Who?" stays together as a single sentence—the library correctly recognizes that a title followed by a single-word name ending in `?` or `!` is a name reference, not a sentence boundary.\n\n## Usage\n\n### Basic Segmentation\n\nThe `segment_text` function returns a list of lists, where each inner list represents a paragraph containing its sentences:\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = """Gandalf spoke softly. "All we have to decide is what to do with the time given us."\n\nFrodo nodded. The weight of the Ring pressed against his chest."""\n\nresults = segment_text(text)\n```\n\n```json\n[\n  [\n    "Gandalf spoke softly.",\n    "\\"All we have to decide is what to do with the time given us.\\"."\n  ],\n  [\n    "Frodo nodded.",\n    "The weight of the Ring pressed against his chest."\n  ]\n]\n```\n\n### Flattened Output\n\nIf you don\'t need paragraph boundaries, use the `flatten` parameter:\n\n```python\ntext = "At 9 a.m. the hobbits set out. By 3 p.m. they reached Rivendell. Mr. Frodo was exhausted."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n  "At 9 a.m. the hobbits set out.",\n  "By 3 p.m. they reached Rivendell.",\n  "Mr. Frodo was exhausted."\n]\n```\n\n### Direct Segmenter Access\n\nFor more control, use the `Segmenter` class directly:\n\n```python\nfrom fast_sentence_segment import Segmenter\n\nsegmenter = Segmenter()\nresults = segmenter.input_text("Your text here.")\n```\n\n### Command Line Interface\n\n```bash\n# Inline text\nsegment "Gandalf paused... You shall not pass! The Balrog roared."\n\n# Pipe from stdin\necho "Have you seen Dr. Who? It\'s brilliant!" | segment\n\n# Numbered output\nsegment -n -f silmarillion.txt\n\n# File-to-file (one sentence per line)\nsegment-file --input-file book.txt --output-file sentences.txt\n\n# Unwrap hard-wrapped e-texts (Project Gutenberg, etc.)\nsegment-file --input-file book.txt --output-file sentences.txt --unwrap\n\n# Dialog-aware formatting (implies --unwrap)\nsegment -f book.txt --format\n```\n\n## API Reference\n\n| Function | Parameters | Returns | Description |\n|----------|------------|---------|-------------|\n| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False`, `format: str = None` | `list` or `str` | Main entry point for segmentation. Use `format="dialog"` for dialog-aware output. |\n| `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |\n\n### CLI Commands\n\n| Command | Description |\n|---------|-------------|\n| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output, `--format` for dialog-aware paragraph grouping. |\n| `segment-file --input-file IN --output-file OUT [--unwrap] [--format]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts, `--format` for dialog-aware formatting. |\n\n## Why Nested Lists?\n\nThe segmentation process preserves document structure by segmenting into both paragraphs and sentences. Each outer list represents a paragraph, and each inner list contains that paragraph\'s sentences. This is useful for:\n\n- Document structure analysis\n- Paragraph-level processing\n- Maintaining original text organization\n\nUse `flatten=True` when you only need sentences without paragraph context.\n\n## Requirements\n\n- Python 3.9+\n- spaCy 3.8+\n- en_core_web_sm spaCy model\n\n## How It Works\n\nThis library uses spaCy for initial sentence segmentation, then applies surgical post-processing fixes for cases where spaCy\'s default behavior is incorrect:\n\n1. **Pre-processing**: Normalize numbered lists, preserve ellipses with placeholders\n2. **spaCy segmentation**: Use spaCy\'s sentence boundary detection\n3. **Post-processing**: Split on abbreviation boundaries, handle `?`/`!` + capital patterns\n4. **Denormalization**: Restore placeholders to original text\n\n## License\n\nMIT License - see [LICENSE](LICENSE) for details.\n\n## Contributing\n\nContributions are welcome! Please feel free to submit a Pull Request.\n\n1. Fork the repository\n2. Create your feature branch (`git checkout -b feature/amazing-feature`)\n3. Run tests (`make test`)\n4. Commit your changes\n5. Push to the branch\n6. Open a Pull Request\n',
     'author': 'Craig Trim',
     'author_email': 'craigtrim@gmail.com',
     'maintainer': 'Craig Trim',

fast_sentence_segment-1.4.5/fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py

@@ -1,75 +0,0 @@
-# -*- coding: UTF-8 -*-
-"""Unwrap hard-wrapped text (e.g., Project Gutenberg e-texts).
-
-Joins lines within paragraphs into continuous strings while
-preserving paragraph boundaries (blank lines). Also dehyphenates
-words that were split across lines for typesetting.
-
-Related GitHub Issue:
-    #8 - Add dehyphenation support for words split across lines
-    https://github.com/craigtrim/fast-sentence-segment/issues/8
-"""
-
-import re
-
-# Pattern to match hyphenated word breaks at end of line:
-# - A single hyphen (not -- em-dash)
-# - Followed by newline and optional whitespace
-# - Followed by a lowercase letter (continuation of word)
-_HYPHEN_LINE_BREAK_PATTERN = re.compile(r'(?<!-)-\n\s*([a-z])')
-
-
-def _dehyphenate_block(block: str) -> str:
-    """Remove hyphens from words split across lines.
-
-    Detects the pattern of a word fragment ending with a hyphen
-    at the end of a line, followed by the word continuation
-    starting with a lowercase letter on the next line.
-
-    Examples:
-        "bot-\\ntle" -> "bottle"
-        "cham-\\n    bermaid" -> "chambermaid"
-
-    Args:
-        block: A paragraph block that may contain hyphenated line breaks.
-
-    Returns:
-        The block with hyphenated word breaks rejoined.
-    """
-    return _HYPHEN_LINE_BREAK_PATTERN.sub(r'\1', block)
-
-
-def unwrap_hard_wrapped_text(text: str) -> str:
-    """Unwrap hard-wrapped paragraphs into continuous lines.
-
-    Splits on blank lines to identify paragraphs, then joins
-    lines within each paragraph into a single string with
-    single spaces. Also dehyphenates words that were split
-    across lines for typesetting purposes.
-
-    Examples:
-        >>> unwrap_hard_wrapped_text("a bot-\\ntle of wine")
-        'a bottle of wine'
-        >>> unwrap_hard_wrapped_text("line one\\nline two")
-        'line one line two'
-
-    Args:
-        text: Raw text with hard-wrapped lines.
-
-    Returns:
-        Text with paragraphs unwrapped into continuous strings,
-        separated by double newlines, with hyphenated words rejoined.
-    """
-    blocks = re.split(r'\n\s*\n', text)
-    unwrapped = []
-
-    for block in blocks:
-        # First, dehyphenate words split across lines
-        block = _dehyphenate_block(block)
-        # Then join remaining lines with spaces
-        lines = block.splitlines()
-        joined = ' '.join(line.strip() for line in lines if line.strip())
-        if joined:
-            unwrapped.append(joined)
-
-    return '\n\n'.join(unwrapped)