fast-sentence-segment 1.4.3__tar.gz → 1.4.5__tar.gz

{fast_sentence_segment-1.4.3 → fast_sentence_segment-1.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fast-sentence-segment
-Version: 1.4.3
+Version: 1.4.5
 Summary: Fast and Efficient Sentence Segmentation
 Home-page: https://github.com/craigtrim/fast-sentence-segment
 License: MIT

{fast_sentence_segment-1.4.3 → fast_sentence_segment-1.4.5}/fast_sentence_segment/__init__.py

@@ -1,3 +1,5 @@
+from typing import List, Optional, Union
+
 from .bp import *
 from .svc import *
 from .dmo import *
@@ -5,6 +7,7 @@ from .dmo import *
 from .bp.segmenter import Segmenter
 from .dmo.unwrap_hard_wrapped_text import unwrap_hard_wrapped_text
 from .dmo.normalize_quotes import normalize_quotes
+from .dmo.dialog_formatter import format_dialog
 
 segment = Segmenter().input_text
 
@@ -14,7 +17,8 @@ def segment_text(
     flatten: bool = False,
     unwrap: bool = False,
     normalize: bool = True,
-) -> list:
+    format: Optional[str] = None,
+) -> Union[List, str]:
     """Segment text into sentences.
 
     Args:
@@ -26,14 +30,23 @@ def segment_text(
         normalize: If True (default), normalize unicode quote variants
             to ASCII equivalents before segmenting. Ensures consistent
             quote characters for downstream processing.
+        format: Optional output format. Supported values:
+            - None (default): Return list of sentences/paragraphs
+            - "dialog": Return formatted string with dialog-aware
+              paragraph grouping (keeps multi-sentence quotes together,
+              adds paragraph breaks between speakers)
 
     Returns:
-        List of sentences (if flatten=True) or list of paragraph
-        groups, each containing a list of sentences.
+        If format is None: List of sentences (if flatten=True) or list
+        of paragraph groups, each containing a list of sentences.
+        If format="dialog": Formatted string with paragraph breaks.
 
-    Related GitHub Issue:
+    Related GitHub Issues:
         #6 - Review findings from Issue #5
         https://github.com/craigtrim/fast-sentence-segment/issues/6
+
+        #10 - feat: Add --format flag for dialog-aware paragraph formatting
+        https://github.com/craigtrim/fast-sentence-segment/issues/10
     """
     if unwrap:
         input_text = unwrap_hard_wrapped_text(input_text)
@@ -43,9 +56,15 @@ def segment_text(
 
     results = segment(input_text)
 
+    # Flatten to list of sentences
+    flat = []
+    [[flat.append(y) for y in x] for x in results]
+
+    # Apply formatting if requested
+    if format == "dialog":
+        return format_dialog(flat)
+
     if flatten:
-        flat = []
-        [[flat.append(y) for y in x] for x in results]
         return flat
 
     return results

fast_sentence_segment-1.4.5/fast_sentence_segment/cli.py

@@ -0,0 +1,298 @@
+# -*- coding: UTF-8 -*-
+"""CLI for fast-sentence-segment."""
+
+import argparse
+import itertools
+import logging
+import os
+import sys
+import threading
+import time
+
+from fast_sentence_segment import segment_text
+from fast_sentence_segment.dmo.group_quoted_sentences import format_grouped_sentences
+
+logging.disable(logging.CRITICAL)
+
+# ANSI color codes
+BOLD = "\033[1m"
+DIM = "\033[2m"
+CYAN = "\033[36m"
+GREEN = "\033[32m"
+YELLOW = "\033[33m"
+RESET = "\033[0m"
+
+
+class Spinner:
+    """Animated spinner for long-running operations."""
+
+    def __init__(self, message: str):
+        self.message = message
+        self.running = False
+        self.thread = None
+        self.frames = itertools.cycle(["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"])
+
+    def _spin(self):
+        while self.running:
+            frame = next(self.frames)
+            print(f"\r  {YELLOW}{frame}{RESET} {self.message}", end="", flush=True)
+            time.sleep(0.08)
+
+    def __enter__(self):
+        self.running = True
+        self.thread = threading.Thread(target=self._spin)
+        self.thread.start()
+        return self
+
+    def __exit__(self, *args):
+        self.running = False
+        if self.thread:
+            self.thread.join()
+        print(f"\r  {' ' * (len(self.message) + 4)}\r", end="", flush=True)
+
+
+def _header(title: str):
+    print(f"\n{BOLD}{CYAN}{title}{RESET}")
+    print(f"{DIM}{'─' * 40}{RESET}")
+
+
+def _param(label: str, value: str):
+    print(f"  {DIM}{label}:{RESET} {value}")
+
+
+def _done(msg: str):
+    print(f"\n  {GREEN}✓{RESET} {msg}")
+
+
+def _file_size(path: str) -> str:
+    size = os.path.getsize(path)
+    if size < 1024:
+        return f"{size} B"
+    elif size < 1024 * 1024:
+        return f"{size / 1024:.1f} KB"
+    return f"{size / (1024 * 1024):.1f} MB"
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="segment",
+        description="Segment text into sentences",
+    )
+    parser.add_argument(
+        "text",
+        nargs="?",
+        help="Text to segment (or use stdin)",
+    )
+    parser.add_argument(
+        "-f", "--file",
+        help="Read text from file",
+    )
+    parser.add_argument(
+        "-n", "--numbered",
+        action="store_true",
+        help="Number output lines",
+    )
+    parser.add_argument(
+        "--unwrap",
+        action="store_true",
+        help="Unwrap hard-wrapped lines and dehyphenate split words",
+    )
+    parser.add_argument(
+        "--format",
+        action="store_true",
+        help="Format output with dialog-aware paragraph grouping",
+    )
+    args = parser.parse_args()
+
+    # Get input text
+    if args.file:
+        with open(args.file, "r", encoding="utf-8") as f:
+            text = f.read()
+    elif args.text:
+        text = args.text
+    elif not sys.stdin.isatty():
+        text = sys.stdin.read()
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    # Segment and output
+    result = segment_text(
+        text.strip(), flatten=True, unwrap=args.unwrap,
+        format="dialog" if args.format else None
+    )
+
+    # If format is used, result is a string
+    if args.format:
+        print(result)
+    else:
+        # Result is a list of sentences
+        for i, sentence in enumerate(result, 1):
+            if args.numbered:
+                print(f"{i}. {sentence}")
+            else:
+                print(sentence)
+
+
+def _generate_output_path(input_path: str) -> str:
+    """Generate output path by inserting -clean before extension."""
+    base, ext = os.path.splitext(input_path)
+    return f"{base}-clean{ext}"
+
+
+def _process_single_file(
+    input_file: str, output_file: str, unwrap: bool, normalize: bool, format: str = None
+):
+    """Process a single file and write output."""
+    # Show configuration
+    _param("Input", input_file)
+    _param("Output", output_file)
+    _param("Size", _file_size(input_file))
+    _param("Unwrap", "enabled" if unwrap else "disabled")
+    _param("Normalize quotes", "disabled" if not normalize else "enabled")
+    _param("Format", format if format else "default (one sentence per line)")
+    print()
+
+    # Step 1: Read file
+    print(f"  {YELLOW}→{RESET} Reading input file...")
+    with open(input_file, "r", encoding="utf-8") as f:
+        text = f.read()
+    print(f"  {GREEN}✓{RESET} Read {len(text):,} characters")
+
+    # Step 2: Segment text
+    print(f"  {YELLOW}→{RESET} Segmenting text...", end="", flush=True)
+    start = time.perf_counter()
+    result = segment_text(
+        text.strip(), flatten=True, unwrap=unwrap, normalize=normalize, format=format,
+    )
+    elapsed = time.perf_counter() - start
+
+    # Step 3: Write output
+    if format:
+        # Format mode returns a string
+        sentence_count = result.count("\n") + 1 if result else 0
+        print(f"\r  {GREEN}✓{RESET} Segmented text ({elapsed:.2f}s)")
+        with open(output_file, "w", encoding="utf-8") as f:
+            f.write(result + "\n")
+        print(f"  {GREEN}✓{RESET} Written formatted output to {output_file}")
+    else:
+        # Default mode returns a list
+        sentences = result
+        print(f"\r  {GREEN}✓{RESET} Segmented into {len(sentences):,} sentences ({elapsed:.2f}s)")
+        total = len(sentences)
+        with open(output_file, "w", encoding="utf-8") as f:
+            if unwrap:
+                f.write(format_grouped_sentences(sentences) + "\n")
+                print(f"  {GREEN}✓{RESET} Written {total:,} sentences to {output_file}")
+            else:
+                for i, sentence in enumerate(sentences, 1):
+                    f.write(sentence + "\n")
+                    if i % 500 == 0 or i == total:
+                        pct = (i / total) * 100
+                        print(f"\r  {YELLOW}→{RESET} Writing... {pct:.0f}% ({i:,}/{total:,})", end="", flush=True)
+                print(f"\r  {GREEN}✓{RESET} Written {total:,} sentences to {output_file}       ")
+
+
+def file_main():
+    parser = argparse.ArgumentParser(
+        prog="segment-file",
+        description="Segment a text file into sentences and write to an output file",
+    )
+    parser.add_argument(
+        "--input-file",
+        help="Path to input text file",
+    )
+    parser.add_argument(
+        "--input-dir",
+        help="Path to directory containing text files to process",
+    )
+    parser.add_argument(
+        "--output-file",
+        help="Path to output file (optional, defaults to input-file with -clean suffix)",
+    )
+    parser.add_argument(
+        "--unwrap", action="store_true",
+        help="Unwrap hard-wrapped lines (e.g., Project Gutenberg e-texts)",
+    )
+    parser.add_argument(
+        "--no-normalize-quotes", action="store_true",
+        help="Disable unicode quote normalization to ASCII equivalents",
+    )
+    parser.add_argument(
+        "--format",
+        action="store_true",
+        help="Format output with dialog-aware paragraph grouping",
+    )
+    args = parser.parse_args()
+
+    # Validate arguments
+    if not args.input_file and not args.input_dir:
+        print(f"  {YELLOW}Error:{RESET} Either --input-file or --input-dir is required")
+        sys.exit(1)
+    if args.input_file and args.input_dir:
+        print(f"  {YELLOW}Error:{RESET} Cannot specify both --input-file and --input-dir")
+        sys.exit(1)
+    if args.input_dir and args.output_file:
+        print(f"  {YELLOW}Error:{RESET} --output-file cannot be used with --input-dir")
+        sys.exit(1)
+
+    normalize = not args.no_normalize_quotes
+
+    # Process directory
+    if args.input_dir:
+        if not os.path.isdir(args.input_dir):
+            print(f"  {YELLOW}Error:{RESET} Directory not found: {args.input_dir}")
+            sys.exit(1)
+
+        # Find all .txt files
+        txt_files = sorted([
+            f for f in os.listdir(args.input_dir)
+            if f.endswith(".txt") and not f.endswith("-clean.txt")
+        ])
+
+        if not txt_files:
+            print(f"  {YELLOW}Error:{RESET} No .txt files found in {args.input_dir}")
+            sys.exit(1)
+
+        _header("segment-file (batch)")
+        print(f"  {DIM}Processing {len(txt_files)} files in directory{RESET}")
+        print()
+        _param("Directory", args.input_dir)
+        _param("Files", str(len(txt_files)))
+        _param("Unwrap", "enabled" if args.unwrap else "disabled")
+        _param("Normalize quotes", "disabled" if not normalize else "enabled")
+        _param("Format", "dialog" if args.format else "default (one sentence per line)")
+        print()
+
+        format_value = "dialog" if args.format else None
+        for i, filename in enumerate(txt_files, 1):
+            input_path = os.path.join(args.input_dir, filename)
+            output_path = _generate_output_path(input_path)
+            print(f"  {BOLD}[{i}/{len(txt_files)}]{RESET} {filename}")
+            _process_single_file(input_path, output_path, args.unwrap, normalize, format_value)
+            print()
+
+        print(f"  {GREEN}Done! Processed {len(txt_files)} files.{RESET}")
+        print()
+        return
+
+    # Process single file
+    if not os.path.isfile(args.input_file):
+        print(f"  {YELLOW}Error:{RESET} File not found: {args.input_file}")
+        sys.exit(1)
+
+    output_file = args.output_file or _generate_output_path(args.input_file)
+
+    _header("segment-file")
+    print(f"  {DIM}Segmenting text file into sentences{RESET}")
+    print()
+
+    format_value = "dialog" if args.format else None
+    _process_single_file(args.input_file, output_file, args.unwrap, normalize, format_value)
+
+    print(f"\n  {GREEN}Done!{RESET}")
+    print()
+
+
+if __name__ == "__main__":
+    main()

{fast_sentence_segment-1.4.3 → fast_sentence_segment-1.4.5}/fast_sentence_segment/dmo/__init__.py

@@ -13,3 +13,5 @@ from .unwrap_hard_wrapped_text import unwrap_hard_wrapped_text
 from .normalize_quotes import normalize_quotes
 from .group_quoted_sentences import group_quoted_sentences, format_grouped_sentences
 from .strip_trailing_period_after_quote import StripTrailingPeriodAfterQuote
+from .ocr_artifact_fixer import OcrArtifactFixer
+from .dialog_formatter import DialogFormatter, format_dialog

fast_sentence_segment-1.4.5/fast_sentence_segment/dmo/dialog_formatter.py

@@ -0,0 +1,255 @@
+# -*- coding: UTF-8 -*-
+"""Dialog-aware paragraph formatter for segmented text.
+
+Formats segmented sentences into readable paragraphs with intelligent
+grouping of dialog and narrative text. Keeps multi-sentence quoted speech
+together and adds paragraph breaks between different speakers.
+
+Related GitHub Issue:
+    #10 - feat: Add --format flag for dialog-aware paragraph formatting
+    https://github.com/craigtrim/fast-sentence-segment/issues/10
+
+Example:
+    >>> from fast_sentence_segment.dmo.dialog_formatter import format_dialog
+    >>> sentences = [
+    ...     '"Hello," said Jack.',
+    ...     '"How are you today?',
+    ...     'I hope you are well."',
+    ...     '"I am fine," replied Mary.',
+    ... ]
+    >>> print(format_dialog(sentences))
+    "Hello," said Jack.
+
+    "How are you today?
+    I hope you are well."
+
+    "I am fine," replied Mary.
+"""
+
+import re
+from typing import List
+
+from fast_sentence_segment.core import BaseObject
+
+
+# Quote characters to track for dialog detection
+DOUBLE_QUOTES = '""\""'
+SINGLE_QUOTES = "'''"
+ALL_QUOTES = DOUBLE_QUOTES + SINGLE_QUOTES
+
+
+def _count_quotes(text: str) -> int:
+    """Count quote characters in text (both single and double)."""
+    return sum(1 for c in text if c in ALL_QUOTES)
+
+
+def _starts_with_quote(text: str) -> bool:
+    """Check if text starts with a quote character."""
+    text = text.lstrip()
+    return text and text[0] in ALL_QUOTES
+
+
+def _ends_with_closing_quote(text: str) -> bool:
+    """Check if text ends with a closing quote (possibly followed by punctuation)."""
+    text = text.rstrip()
+    if not text:
+        return False
+    # Check last few characters for closing quote pattern
+    # e.g., '" or "' or .' or ."
+    for i in range(min(3, len(text)), 0, -1):
+        if text[-i] in ALL_QUOTES:
+            return True
+    return False
+
+
+def _is_complete_quote(text: str) -> bool:
+    """Check if text contains a complete (balanced) quote.
+
+    A complete quote has an even number of quote characters,
+    meaning all opened quotes are closed.
+    """
+    quote_count = _count_quotes(text)
+    return quote_count > 0 and quote_count % 2 == 0
+
+
+def _sentence_is_dialog_continuation(sentence: str, in_quote: bool) -> bool:
+    """Determine if sentence continues an open quote.
+
+    Args:
+        sentence: The sentence to check.
+        in_quote: Whether we're currently inside an unclosed quote.
+
+    Returns:
+        True if this sentence is a continuation of open dialog.
+    """
+    if in_quote:
+        return True
+    return False
+
+
+def _get_quote_delta(sentence: str) -> int:
+    """Get the net change in quote depth for a sentence.
+
+    Returns:
+        Positive if more quotes opened than closed,
+        negative if more closed than opened,
+        zero if balanced.
+    """
+    return _count_quotes(sentence) % 2
+
+
+class DialogFormatter(BaseObject):
+    """Formats segmented sentences with dialog-aware paragraph grouping.
+
+    This formatter analyzes sentence structure to intelligently group
+    text into paragraphs:
+
+    - Multi-sentence quoted speech stays together (same speaker)
+    - Paragraph breaks added between different speakers
+    - Narrative text grouped appropriately
+    - Handles both single and double quote styles
+
+    Example:
+        >>> formatter = DialogFormatter()
+        >>> sentences = ['"Hello," he said.', 'The door opened.']
+        >>> print(formatter.process(sentences))
+        "Hello," he said.
+
+        The door opened.
+    """
+
+    def __init__(self):
+        """Initialize the DialogFormatter."""
+        BaseObject.__init__(self, __name__)
+
+    def process(self, sentences: List[str]) -> str:
+        """Format sentences into dialog-aware paragraphs.
+
+        Args:
+            sentences: List of segmented sentences.
+
+        Returns:
+            Formatted string with appropriate paragraph breaks.
+        """
+        return format_dialog(sentences)
+
+
+def _is_narrative(sentence: str) -> bool:
+    """Check if a sentence is narrative (no quotes at start)."""
+    return not _starts_with_quote(sentence)
+
+
+def _ends_dialog_turn(sentence: str) -> bool:
+    """Check if a sentence ends a dialog turn.
+
+    A dialog turn ends when the sentence ends with a closing quote
+    followed by optional punctuation or dialog tag ending.
+    """
+    sentence = sentence.rstrip()
+    if not sentence:
+        return False
+
+    # Pattern: ends with quote + optional punctuation
+    # e.g., ." or .' or "' or '" or ," he said. etc.
+    # Check if there's a closing quote near the end
+    last_chars = sentence[-10:] if len(sentence) >= 10 else sentence
+
+    # Count quotes in last part - if odd from end, likely closes
+    for i, c in enumerate(reversed(last_chars)):
+        if c in ALL_QUOTES:
+            # Found a quote - check if it's likely a closer
+            # A closer is typically followed by punctuation or end
+            remaining = last_chars[len(last_chars) - i:]
+            if not remaining or all(ch in '.,!?;: ' for ch in remaining):
+                return True
+            # Also handle dialog tags: ,' he said.
+            if remaining and remaining[0] in '.,!?' and 'said' not in sentence.lower()[-20:]:
+                return True
+            break
+
+    return False
+
+
+def format_dialog(sentences: List[str]) -> str:
+    """Format sentences into dialog-aware paragraphs.
+
+    Groups sentences intelligently based on dialog structure:
+    - Sentences within an unclosed quote stay grouped
+    - Complete quoted sentences become their own paragraphs
+    - Narrative text is grouped together
+    - Paragraph breaks separate different speakers/turns
+
+    Args:
+        sentences: List of segmented sentences.
+
+    Returns:
+        Formatted string with paragraph breaks (double newlines)
+        between logical groups and single newlines within groups.
+
+    Example:
+        >>> sentences = [
+        ...     '"My dear sir," cried the man.',
+        ...     '"You had every reason to be carried away."',
+        ... ]
+        >>> print(format_dialog(sentences))
+        "My dear sir," cried the man.
+
+        "You had every reason to be carried away."
+    """
+    if not sentences:
+        return ""
+
+    paragraphs: List[List[str]] = []
+    current_para: List[str] = []
+    in_quote = False  # Track if we're inside an unclosed quote
+
+    for i, sentence in enumerate(sentences):
+        sentence = sentence.strip()
+        if not sentence:
+            continue
+
+        quote_count = _count_quotes(sentence)
+        starts_quote = _starts_with_quote(sentence)
+        is_narrative = _is_narrative(sentence)
+        is_complete = _is_complete_quote(sentence)
+
+        # Get info about previous sentence
+        prev_sentence = current_para[-1] if current_para else ""
+        prev_was_narrative = _is_narrative(prev_sentence) if prev_sentence else False
+        prev_was_complete = _is_complete_quote(prev_sentence) if prev_sentence else False
+
+        # Determine if this sentence starts a new paragraph
+        should_start_new_para = False
+
+        if not current_para:
+            # First sentence always starts a new paragraph
+            should_start_new_para = True
+        elif in_quote:
+            # Inside an open quote - continue current paragraph
+            should_start_new_para = False
+        elif starts_quote:
+            # New quote starting - always new paragraph
+            should_start_new_para = True
+        elif is_narrative and prev_was_complete and not prev_was_narrative:
+            # Narrative after complete dialog - new paragraph
+            should_start_new_para = True
+        elif is_narrative and not prev_was_narrative and _ends_dialog_turn(prev_sentence):
+            # Narrative after dialog that ends a turn - new paragraph
+            should_start_new_para = True
+
+        if should_start_new_para and current_para:
+            paragraphs.append(current_para)
+            current_para = []
+
+        current_para.append(sentence)
+
+        # Update quote tracking
+        if quote_count % 2 == 1:
+            in_quote = not in_quote
+
+    # Don't forget the last paragraph
+    if current_para:
+        paragraphs.append(current_para)
+
+    # Format: join sentences in paragraph with newline, paragraphs with double newline
+    return "\n\n".join("\n".join(para) for para in paragraphs)

fast_sentence_segment-1.4.5/fast_sentence_segment/dmo/ocr_artifact_fixer.py

@@ -0,0 +1,70 @@
+# -*- coding: UTF-8 -*-
+"""Fix common OCR/text extraction artifacts.
+
+Ebook text files often contain artifacts where common word pairs
+are incorrectly joined. This module fixes known patterns.
+
+Related GitHub Issue:
+    #9 - Fix common OCR/cleaning artifacts (Iam, witha)
+    https://github.com/craigtrim/fast-sentence-segment/issues/9
+"""
+
+from fast_sentence_segment.core import BaseObject
+
+# Known OCR artifact patterns: (pattern, replacement)
+# All patterns include surrounding spaces to ensure exact word boundaries
+_OCR_ARTIFACTS = [
+    (" Iam ", " I am "),
+    (" Ihave ", " I have "),
+    (" ihave ", " I have "),
+    (" Ithink ", " I think "),
+    (" ithink ", " I think "),
+    (" anda ", " and a "),
+    (" witha ", " with a "),
+    (" sucha ", " such a "),
+    (" aliquid ", " a liquid "),
+]
+
+
+class OcrArtifactFixer(BaseObject):
+    """Fix common OCR/text extraction artifacts.
+
+    Detects and corrects known patterns where words are incorrectly
+    joined during OCR or text extraction processes.
+
+    Related GitHub Issue:
+        #9 - Fix common OCR/cleaning artifacts (Iam, witha)
+        https://github.com/craigtrim/fast-sentence-segment/issues/9
+    """
+
+    def __init__(self):
+        """Change Log
+
+        Created:
+            3-Feb-2026
+            craigtrim@gmail.com
+            *   fix common OCR/cleaning artifacts
+                https://github.com/craigtrim/fast-sentence-segment/issues/9
+        """
+        BaseObject.__init__(self, __name__)
+
+    @staticmethod
+    def process(input_text: str) -> str:
+        """Fix known OCR artifact patterns.
+
+        Args:
+            input_text: Text that may contain OCR artifacts.
+
+        Returns:
+            Text with known OCR artifacts corrected.
+
+        Examples:
+            >>> OcrArtifactFixer.process("Jack, Iam so happy")
+            'Jack, I am so happy'
+            >>> OcrArtifactFixer.process("horizon witha hint")
+            'horizon with a hint'
+        """
+        for pattern, replacement in _OCR_ARTIFACTS:
+            if pattern in input_text:
+                input_text = input_text.replace(pattern, replacement)
+        return input_text

{fast_sentence_segment-1.4.3 → fast_sentence_segment-1.4.5}/fast_sentence_segment/dmo/spacy_doc_segmenter.py

@@ -65,6 +65,54 @@ class SpacyDocSegmenter(BaseObject):
             return False
         return True
 
+    @staticmethod
+    def _merge_orphaned_quotes(sentences: list) -> list:
+        """Merge orphaned opening quotes with the following sentence.
+
+        spaCy sometimes splits on opening quotes, producing sentences like:
+            ["'", "Oh, the funeral..."]
+        This merges them into:
+            ["'Oh, the funeral..."]
+
+        Also handles trailing orphaned quotes that should belong to next sentence:
+            ["He said. '", "Hello!'"]
+        Becomes:
+            ["He said.", "'Hello!'"]
+        """
+        if not sentences:
+            return sentences
+
+        result = []
+        i = 0
+        while i < len(sentences):
+            sent = sentences[i]
+            # Check if this sentence is just an opening quote
+            if sent.strip() in ("'", '"', "'.", '".'):
+                # Merge with the next sentence if available
+                if i + 1 < len(sentences):
+                    quote_char = sent.strip().rstrip('.')
+                    result.append(quote_char + sentences[i + 1])
+                    i += 2
+                    continue
+            result.append(sent)
+            i += 1
+
+        # Second pass: handle trailing orphaned quotes
+        # Pattern: sentence ends with `. '` or `. "` - move quote to next sentence
+        fixed = []
+        for i, sent in enumerate(result):
+            # Check for trailing orphaned quote (`. '` or `? '` or `! '`)
+            if len(sent) >= 3 and sent[-2:] in (" '", ' "') and sent[-3] in '.?!':
+                # Strip the trailing quote
+                trailing_quote = sent[-1]
+                sent = sent[:-2]
+                # Prepend to next sentence if available
+                if i + 1 < len(result) and not result[i + 1].startswith(('"', "'")):
+                    result[i + 1] = trailing_quote + result[i + 1]
+            fixed.append(sent)
+
+        return fixed
+
     @staticmethod
     def _cleanse(sentences: list) -> str:
         sentences = [sent for sent in sentences
@@ -103,6 +151,9 @@ class SpacyDocSegmenter(BaseObject):
         sentences = [sent for sent in sentences if
                      sent and len(sent) and sent != 'None']
 
+        # Merge orphaned opening quotes with following sentence
+        sentences = self._merge_orphaned_quotes(sentences)
+
         sentences = [self._append_period(sent)
                      for sent in sentences]
 

{fast_sentence_segment-1.4.3 → fast_sentence_segment-1.4.5}/fast_sentence_segment/svc/perform_sentence_segmentation.py

@@ -55,6 +55,7 @@ from fast_sentence_segment.dmo import SpacyDocSegmenter
 from fast_sentence_segment.dmo import PostProcessStructure
 from fast_sentence_segment.dmo import StripTrailingPeriodAfterQuote
 from fast_sentence_segment.dmo import Dehyphenator
+from fast_sentence_segment.dmo import OcrArtifactFixer
 
 
 class PerformSentenceSegmentation(BaseObject):
@@ -84,6 +85,7 @@ class PerformSentenceSegmentation(BaseObject):
             self.__nlp = _load_spacy_model("en_core_web_sm")
 
         self._dehyphenate = Dehyphenator.process
+        self._fix_ocr_artifacts = OcrArtifactFixer.process
         self._newlines_to_periods = NewlinesToPeriods.process
         self._normalize_numbered_lists = NumberedListNormalizer().process
         self._normalize_ellipses = EllipsisNormalizer().process
@@ -138,6 +140,9 @@ class PerformSentenceSegmentation(BaseObject):
         # Must happen before newlines are converted to periods
         input_text = self._dehyphenate(input_text)
 
+        # Fix common OCR artifacts (issue #9)
+        input_text = self._fix_ocr_artifacts(input_text)
+
         input_text = self._normalize_numbered_lists(input_text)
         input_text = self._normalize_ellipses(input_text)
 

{fast_sentence_segment-1.4.3 → fast_sentence_segment-1.4.5}/pyproject.toml

@@ -11,7 +11,7 @@ description = "Fast and Efficient Sentence Segmentation"
 license = "MIT"
 name = "fast-sentence-segment"
 readme = "README.md"
-version = "1.4.3"
+version = "1.4.5"
 
 keywords = ["nlp", "text", "preprocess", "segment"]
 repository = "https://github.com/craigtrim/fast-sentence-segment"

{fast_sentence_segment-1.4.3 → fast_sentence_segment-1.4.5}/setup.py

@@ -20,7 +20,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'fast-sentence-segment',
-    'version': '1.4.3',
+    'version': '1.4.5',
     'description': 'Fast and Efficient Sentence Segmentation',
     'long_description': '# Fast Sentence Segmentation\n\n[![PyPI version](https://img.shields.io/pypi/v/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)\n[![Python versions](https://img.shields.io/pypi/pyversions/fast-sentence-segment.svg)](https://pypi.org/project/fast-sentence-segment/)\n[![Tests](https://img.shields.io/badge/tests-664-brightgreen)](https://github.com/craigtrim/fast-sentence-segment/tree/master/tests)\n[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)\n[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)\n[![Downloads](https://static.pepy.tech/badge/fast-sentence-segment)](https://pepy.tech/project/fast-sentence-segment)\n[![Downloads/Month](https://static.pepy.tech/badge/fast-sentence-segment/month)](https://pepy.tech/project/fast-sentence-segment)\n\nFast and efficient sentence segmentation using spaCy with surgical post-processing fixes. Handles complex edge cases like abbreviations (Dr., Mr., etc.), ellipses, quoted text, and multi-paragraph documents.\n\n## Why This Library?\n\n1. **Keep it local**: LLM API calls cost money and send your data to third parties. Run sentence segmentation entirely on your machine.\n2. **spaCy perfected**: spaCy is a great local model, but it makes mistakes. This library fixes most of spaCy\'s shortcomings.\n\n## Features\n\n- **Paragraph-aware segmentation**: Returns sentences grouped by paragraph\n- **Abbreviation handling**: Correctly handles "Dr.", "Mr.", "etc.", "p.m.", "a.m." without false splits\n- **Ellipsis preservation**: Keeps `...` intact while detecting sentence boundaries\n- **Question/exclamation splitting**: Properly splits on `?` and `!` followed by capital letters\n- **Cached processing**: LRU cache for repeated text processing\n- **Flexible output**: Nested lists (by paragraph) or flattened list of sentences\n- **Bullet point & numbered list normalization**: Cleans common list formats\n- **CLI tool**: Command-line interface for quick segmentation\n\n## Installation\n\n```bash\npip install fast-sentence-segment\n```\n\nAfter installation, download the spaCy model:\n\n```bash\npython -m spacy download en_core_web_sm\n```\n\n## Quick Start\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = "Do you like Dr. Who? I prefer Dr. Strange! Mr. T is also cool."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n  "Do you like Dr. Who?",\n  "I prefer Dr. Strange!",\n  "Mr. T is also cool."\n]\n```\n\nNotice how "Dr. Who?" stays together as a single sentence—the library correctly recognizes that a title followed by a single-word name ending in `?` or `!` is a name reference, not a sentence boundary.\n\n## Usage\n\n### Basic Segmentation\n\nThe `segment_text` function returns a list of lists, where each inner list represents a paragraph containing its sentences:\n\n```python\nfrom fast_sentence_segment import segment_text\n\ntext = """Gandalf spoke softly. "All we have to decide is what to do with the time given us."\n\nFrodo nodded. The weight of the Ring pressed against his chest."""\n\nresults = segment_text(text)\n```\n\n```json\n[\n  [\n    "Gandalf spoke softly.",\n    "\\"All we have to decide is what to do with the time given us.\\"."\n  ],\n  [\n    "Frodo nodded.",\n    "The weight of the Ring pressed against his chest."\n  ]\n]\n```\n\n### Flattened Output\n\nIf you don\'t need paragraph boundaries, use the `flatten` parameter:\n\n```python\ntext = "At 9 a.m. the hobbits set out. By 3 p.m. they reached Rivendell. Mr. Frodo was exhausted."\n\nresults = segment_text(text, flatten=True)\n```\n\n```json\n[\n  "At 9 a.m. the hobbits set out.",\n  "By 3 p.m. they reached Rivendell.",\n  "Mr. Frodo was exhausted."\n]\n```\n\n### Direct Segmenter Access\n\nFor more control, use the `Segmenter` class directly:\n\n```python\nfrom fast_sentence_segment import Segmenter\n\nsegmenter = Segmenter()\nresults = segmenter.input_text("Your text here.")\n```\n\n### Command Line Interface\n\n```bash\n# Inline text\nsegment "Gandalf paused... You shall not pass! The Balrog roared."\n\n# Pipe from stdin\necho "Have you seen Dr. Who? It\'s brilliant!" | segment\n\n# Numbered output\nsegment -n -f silmarillion.txt\n\n# File-to-file (one sentence per line)\nsegment-file --input-file book.txt --output-file sentences.txt\n\n# Unwrap hard-wrapped e-texts (Project Gutenberg, etc.)\nsegment-file --input-file book.txt --output-file sentences.txt --unwrap\n```\n\n## API Reference\n\n| Function | Parameters | Returns | Description |\n|----------|------------|---------|-------------|\n| `segment_text()` | `input_text: str`, `flatten: bool = False`, `unwrap: bool = False` | `list` | Main entry point for segmentation |\n| `Segmenter.input_text()` | `input_text: str` | `list[list[str]]` | Cached paragraph-aware segmentation |\n\n### CLI Commands\n\n| Command | Description |\n|---------|-------------|\n| `segment [text]` | Segment text from argument, `-f FILE`, or stdin. Use `-n` for numbered output. |\n| `segment-file --input-file IN --output-file OUT [--unwrap]` | Segment a file and write one sentence per line. Use `--unwrap` for hard-wrapped e-texts. |\n\n## Why Nested Lists?\n\nThe segmentation process preserves document structure by segmenting into both paragraphs and sentences. Each outer list represents a paragraph, and each inner list contains that paragraph\'s sentences. This is useful for:\n\n- Document structure analysis\n- Paragraph-level processing\n- Maintaining original text organization\n\nUse `flatten=True` when you only need sentences without paragraph context.\n\n## Requirements\n\n- Python 3.9+\n- spaCy 3.8+\n- en_core_web_sm spaCy model\n\n## How It Works\n\nThis library uses spaCy for initial sentence segmentation, then applies surgical post-processing fixes for cases where spaCy\'s default behavior is incorrect:\n\n1. **Pre-processing**: Normalize numbered lists, preserve ellipses with placeholders\n2. **spaCy segmentation**: Use spaCy\'s sentence boundary detection\n3. **Post-processing**: Split on abbreviation boundaries, handle `?`/`!` + capital patterns\n4. **Denormalization**: Restore placeholders to original text\n\n## License\n\nMIT License - see [LICENSE](LICENSE) for details.\n\n## Contributing\n\nContributions are welcome! Please feel free to submit a Pull Request.\n\n1. Fork the repository\n2. Create your feature branch (`git checkout -b feature/amazing-feature`)\n3. Run tests (`make test`)\n4. Commit your changes\n5. Push to the branch\n6. Open a Pull Request\n',
     'author': 'Craig Trim',

fast_sentence_segment-1.4.3/fast_sentence_segment/cli.py

@@ -1,194 +0,0 @@
-# -*- coding: UTF-8 -*-
-"""CLI for fast-sentence-segment."""
-
-import argparse
-import itertools
-import logging
-import os
-import sys
-import threading
-import time
-
-from fast_sentence_segment import segment_text
-from fast_sentence_segment.dmo.group_quoted_sentences import format_grouped_sentences
-
-logging.disable(logging.CRITICAL)
-
-# ANSI color codes
-BOLD = "\033[1m"
-DIM = "\033[2m"
-CYAN = "\033[36m"
-GREEN = "\033[32m"
-YELLOW = "\033[33m"
-RESET = "\033[0m"
-
-
-class Spinner:
-    """Animated spinner for long-running operations."""
-
-    def __init__(self, message: str):
-        self.message = message
-        self.running = False
-        self.thread = None
-        self.frames = itertools.cycle(["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"])
-
-    def _spin(self):
-        while self.running:
-            frame = next(self.frames)
-            print(f"\r  {YELLOW}{frame}{RESET} {self.message}", end="", flush=True)
-            time.sleep(0.08)
-
-    def __enter__(self):
-        self.running = True
-        self.thread = threading.Thread(target=self._spin)
-        self.thread.start()
-        return self
-
-    def __exit__(self, *args):
-        self.running = False
-        if self.thread:
-            self.thread.join()
-        print(f"\r  {' ' * (len(self.message) + 4)}\r", end="", flush=True)
-
-
-def _header(title: str):
-    print(f"\n{BOLD}{CYAN}{title}{RESET}")
-    print(f"{DIM}{'─' * 40}{RESET}")
-
-
-def _param(label: str, value: str):
-    print(f"  {DIM}{label}:{RESET} {value}")
-
-
-def _done(msg: str):
-    print(f"\n  {GREEN}✓{RESET} {msg}")
-
-
-def _file_size(path: str) -> str:
-    size = os.path.getsize(path)
-    if size < 1024:
-        return f"{size} B"
-    elif size < 1024 * 1024:
-        return f"{size / 1024:.1f} KB"
-    return f"{size / (1024 * 1024):.1f} MB"
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        prog="segment",
-        description="Segment text into sentences",
-    )
-    parser.add_argument(
-        "text",
-        nargs="?",
-        help="Text to segment (or use stdin)",
-    )
-    parser.add_argument(
-        "-f", "--file",
-        help="Read text from file",
-    )
-    parser.add_argument(
-        "-n", "--numbered",
-        action="store_true",
-        help="Number output lines",
-    )
-    parser.add_argument(
-        "--unwrap",
-        action="store_true",
-        help="Unwrap hard-wrapped lines and dehyphenate split words",
-    )
-    args = parser.parse_args()
-
-    # Get input text
-    if args.file:
-        with open(args.file, "r", encoding="utf-8") as f:
-            text = f.read()
-    elif args.text:
-        text = args.text
-    elif not sys.stdin.isatty():
-        text = sys.stdin.read()
-    else:
-        parser.print_help()
-        sys.exit(1)
-
-    # Segment and output
-    sentences = segment_text(text.strip(), flatten=True, unwrap=args.unwrap)
-    for i, sentence in enumerate(sentences, 1):
-        if args.numbered:
-            print(f"{i}. {sentence}")
-        else:
-            print(sentence)
-
-
-def file_main():
-    parser = argparse.ArgumentParser(
-        prog="segment-file",
-        description="Segment a text file into sentences and write to an output file",
-    )
-    parser.add_argument(
-        "--input-file", required=True,
-        help="Path to input text file",
-    )
-    parser.add_argument(
-        "--output-file", required=True,
-        help="Path to output file",
-    )
-    parser.add_argument(
-        "--unwrap", action="store_true",
-        help="Unwrap hard-wrapped lines (e.g., Project Gutenberg e-texts)",
-    )
-    parser.add_argument(
-        "--no-normalize-quotes", action="store_true",
-        help="Disable unicode quote normalization to ASCII equivalents",
-    )
-    args = parser.parse_args()
-
-    # Echo command immediately
-    _header("segment-file")
-    print(f"  {DIM}Segmenting text file into sentences{RESET}")
-    print()
-
-    # Show configuration
-    _param("Input", args.input_file)
-    _param("Output", args.output_file)
-    _param("Size", _file_size(args.input_file))
-    _param("Unwrap", "enabled" if args.unwrap else "disabled")
-    _param("Normalize quotes", "disabled" if args.no_normalize_quotes else "enabled")
-    print()
-
-    # Step 1: Read file
-    print(f"  {YELLOW}→{RESET} Reading input file...")
-    with open(args.input_file, "r", encoding="utf-8") as f:
-        text = f.read()
-    print(f"  {GREEN}✓{RESET} Read {len(text):,} characters")
-
-    # Step 2: Segment text
-    print(f"  {YELLOW}→{RESET} Segmenting text...", end="", flush=True)
-    start = time.perf_counter()
-    normalize = not args.no_normalize_quotes
-    sentences = segment_text(
-        text.strip(), flatten=True, unwrap=args.unwrap, normalize=normalize,
-    )
-    elapsed = time.perf_counter() - start
-    print(f"\r  {GREEN}✓{RESET} Segmented into {len(sentences):,} sentences ({elapsed:.2f}s)")
-
-    # Step 3: Write output
-    total = len(sentences)
-    with open(args.output_file, "w", encoding="utf-8") as f:
-        if args.unwrap:
-            f.write(format_grouped_sentences(sentences) + "\n")
-            print(f"  {GREEN}✓{RESET} Written {total:,} sentences to {args.output_file}")
-        else:
-            for i, sentence in enumerate(sentences, 1):
-                f.write(sentence + "\n")
-                if i % 500 == 0 or i == total:
-                    pct = (i / total) * 100
-                    print(f"\r  {YELLOW}→{RESET} Writing... {pct:.0f}% ({i:,}/{total:,})", end="", flush=True)
-            print(f"\r  {GREEN}✓{RESET} Written {total:,} sentences to {args.output_file}       ")
-
-    print(f"\n  {GREEN}Done!{RESET}")
-    print()
-
-
-if __name__ == "__main__":
-    main()