fast-sentence-segment 1.4.3__py3-none-any.whl → 1.4.5__py3-none-any.whl

fast_sentence_segment/__init__.py

@@ -1,3 +1,5 @@
+from typing import List, Optional, Union
+
 from .bp import *
 from .svc import *
 from .dmo import *
@@ -5,6 +7,7 @@ from .dmo import *
 from .bp.segmenter import Segmenter
 from .dmo.unwrap_hard_wrapped_text import unwrap_hard_wrapped_text
 from .dmo.normalize_quotes import normalize_quotes
+from .dmo.dialog_formatter import format_dialog
 
 segment = Segmenter().input_text
 
@@ -14,7 +17,8 @@ def segment_text(
     flatten: bool = False,
     unwrap: bool = False,
     normalize: bool = True,
-) -> list:
+    format: Optional[str] = None,
+) -> Union[List, str]:
     """Segment text into sentences.
 
     Args:
@@ -26,14 +30,23 @@ def segment_text(
         normalize: If True (default), normalize unicode quote variants
             to ASCII equivalents before segmenting. Ensures consistent
             quote characters for downstream processing.
+        format: Optional output format. Supported values:
+            - None (default): Return list of sentences/paragraphs
+            - "dialog": Return formatted string with dialog-aware
+              paragraph grouping (keeps multi-sentence quotes together,
+              adds paragraph breaks between speakers)
 
     Returns:
-        List of sentences (if flatten=True) or list of paragraph
-        groups, each containing a list of sentences.
+        If format is None: List of sentences (if flatten=True) or list
+        of paragraph groups, each containing a list of sentences.
+        If format="dialog": Formatted string with paragraph breaks.
 
-    Related GitHub Issue:
+    Related GitHub Issues:
         #6 - Review findings from Issue #5
         https://github.com/craigtrim/fast-sentence-segment/issues/6
+
+        #10 - feat: Add --format flag for dialog-aware paragraph formatting
+        https://github.com/craigtrim/fast-sentence-segment/issues/10
     """
     if unwrap:
         input_text = unwrap_hard_wrapped_text(input_text)
@@ -43,9 +56,15 @@ def segment_text(
 
     results = segment(input_text)
 
+    # Flatten to list of sentences
+    flat = []
+    [[flat.append(y) for y in x] for x in results]
+
+    # Apply formatting if requested
+    if format == "dialog":
+        return format_dialog(flat)
+
     if flatten:
-        flat = []
-        [[flat.append(y) for y in x] for x in results]
         return flat
 
     return results

fast_sentence_segment/cli.py

@@ -97,6 +97,11 @@ def main():
         action="store_true",
         help="Unwrap hard-wrapped lines and dehyphenate split words",
     )
+    parser.add_argument(
+        "--format",
+        action="store_true",
+        help="Format output with dialog-aware paragraph grouping",
+    )
     args = parser.parse_args()
 
     # Get input text
@@ -112,12 +117,80 @@ def main():
         sys.exit(1)
 
     # Segment and output
-    sentences = segment_text(text.strip(), flatten=True, unwrap=args.unwrap)
-    for i, sentence in enumerate(sentences, 1):
-        if args.numbered:
-            print(f"{i}. {sentence}")
-        else:
-            print(sentence)
+    result = segment_text(
+        text.strip(), flatten=True, unwrap=args.unwrap,
+        format="dialog" if args.format else None
+    )
+
+    # If format is used, result is a string
+    if args.format:
+        print(result)
+    else:
+        # Result is a list of sentences
+        for i, sentence in enumerate(result, 1):
+            if args.numbered:
+                print(f"{i}. {sentence}")
+            else:
+                print(sentence)
+
+
+def _generate_output_path(input_path: str) -> str:
+    """Generate output path by inserting -clean before extension."""
+    base, ext = os.path.splitext(input_path)
+    return f"{base}-clean{ext}"
+
+
+def _process_single_file(
+    input_file: str, output_file: str, unwrap: bool, normalize: bool, format: str = None
+):
+    """Process a single file and write output."""
+    # Show configuration
+    _param("Input", input_file)
+    _param("Output", output_file)
+    _param("Size", _file_size(input_file))
+    _param("Unwrap", "enabled" if unwrap else "disabled")
+    _param("Normalize quotes", "disabled" if not normalize else "enabled")
+    _param("Format", format if format else "default (one sentence per line)")
+    print()
+
+    # Step 1: Read file
+    print(f"  {YELLOW}→{RESET} Reading input file...")
+    with open(input_file, "r", encoding="utf-8") as f:
+        text = f.read()
+    print(f"  {GREEN}✓{RESET} Read {len(text):,} characters")
+
+    # Step 2: Segment text
+    print(f"  {YELLOW}→{RESET} Segmenting text...", end="", flush=True)
+    start = time.perf_counter()
+    result = segment_text(
+        text.strip(), flatten=True, unwrap=unwrap, normalize=normalize, format=format,
+    )
+    elapsed = time.perf_counter() - start
+
+    # Step 3: Write output
+    if format:
+        # Format mode returns a string
+        sentence_count = result.count("\n") + 1 if result else 0
+        print(f"\r  {GREEN}✓{RESET} Segmented text ({elapsed:.2f}s)")
+        with open(output_file, "w", encoding="utf-8") as f:
+            f.write(result + "\n")
+        print(f"  {GREEN}✓{RESET} Written formatted output to {output_file}")
+    else:
+        # Default mode returns a list
+        sentences = result
+        print(f"\r  {GREEN}✓{RESET} Segmented into {len(sentences):,} sentences ({elapsed:.2f}s)")
+        total = len(sentences)
+        with open(output_file, "w", encoding="utf-8") as f:
+            if unwrap:
+                f.write(format_grouped_sentences(sentences) + "\n")
+                print(f"  {GREEN}✓{RESET} Written {total:,} sentences to {output_file}")
+            else:
+                for i, sentence in enumerate(sentences, 1):
+                    f.write(sentence + "\n")
+                    if i % 500 == 0 or i == total:
+                        pct = (i / total) * 100
+                        print(f"\r  {YELLOW}→{RESET} Writing... {pct:.0f}% ({i:,}/{total:,})", end="", flush=True)
+                print(f"\r  {GREEN}✓{RESET} Written {total:,} sentences to {output_file}       ")
 
 
 def file_main():
@@ -126,12 +199,16 @@ def file_main():
         description="Segment a text file into sentences and write to an output file",
     )
     parser.add_argument(
-        "--input-file", required=True,
+        "--input-file",
         help="Path to input text file",
     )
     parser.add_argument(
-        "--output-file", required=True,
-        help="Path to output file",
+        "--input-dir",
+        help="Path to directory containing text files to process",
+    )
+    parser.add_argument(
+        "--output-file",
+        help="Path to output file (optional, defaults to input-file with -clean suffix)",
     )
     parser.add_argument(
         "--unwrap", action="store_true",
@@ -141,50 +218,77 @@ def file_main():
         "--no-normalize-quotes", action="store_true",
         help="Disable unicode quote normalization to ASCII equivalents",
     )
+    parser.add_argument(
+        "--format",
+        action="store_true",
+        help="Format output with dialog-aware paragraph grouping",
+    )
     args = parser.parse_args()
 
-    # Echo command immediately
-    _header("segment-file")
-    print(f"  {DIM}Segmenting text file into sentences{RESET}")
-    print()
+    # Validate arguments
+    if not args.input_file and not args.input_dir:
+        print(f"  {YELLOW}Error:{RESET} Either --input-file or --input-dir is required")
+        sys.exit(1)
+    if args.input_file and args.input_dir:
+        print(f"  {YELLOW}Error:{RESET} Cannot specify both --input-file and --input-dir")
+        sys.exit(1)
+    if args.input_dir and args.output_file:
+        print(f"  {YELLOW}Error:{RESET} --output-file cannot be used with --input-dir")
+        sys.exit(1)
 
-    # Show configuration
-    _param("Input", args.input_file)
-    _param("Output", args.output_file)
-    _param("Size", _file_size(args.input_file))
-    _param("Unwrap", "enabled" if args.unwrap else "disabled")
-    _param("Normalize quotes", "disabled" if args.no_normalize_quotes else "enabled")
-    print()
+    normalize = not args.no_normalize_quotes
 
-    # Step 1: Read file
-    print(f"  {YELLOW}→{RESET} Reading input file...")
-    with open(args.input_file, "r", encoding="utf-8") as f:
-        text = f.read()
-    print(f"  {GREEN}✓{RESET} Read {len(text):,} characters")
+    # Process directory
+    if args.input_dir:
+        if not os.path.isdir(args.input_dir):
+            print(f"  {YELLOW}Error:{RESET} Directory not found: {args.input_dir}")
+            sys.exit(1)
+
+        # Find all .txt files
+        txt_files = sorted([
+            f for f in os.listdir(args.input_dir)
+            if f.endswith(".txt") and not f.endswith("-clean.txt")
+        ])
+
+        if not txt_files:
+            print(f"  {YELLOW}Error:{RESET} No .txt files found in {args.input_dir}")
+            sys.exit(1)
+
+        _header("segment-file (batch)")
+        print(f"  {DIM}Processing {len(txt_files)} files in directory{RESET}")
+        print()
+        _param("Directory", args.input_dir)
+        _param("Files", str(len(txt_files)))
+        _param("Unwrap", "enabled" if args.unwrap else "disabled")
+        _param("Normalize quotes", "disabled" if not normalize else "enabled")
+        _param("Format", "dialog" if args.format else "default (one sentence per line)")
+        print()
+
+        format_value = "dialog" if args.format else None
+        for i, filename in enumerate(txt_files, 1):
+            input_path = os.path.join(args.input_dir, filename)
+            output_path = _generate_output_path(input_path)
+            print(f"  {BOLD}[{i}/{len(txt_files)}]{RESET} {filename}")
+            _process_single_file(input_path, output_path, args.unwrap, normalize, format_value)
+            print()
+
+        print(f"  {GREEN}Done! Processed {len(txt_files)} files.{RESET}")
+        print()
+        return
+
+    # Process single file
+    if not os.path.isfile(args.input_file):
+        print(f"  {YELLOW}Error:{RESET} File not found: {args.input_file}")
+        sys.exit(1)
 
-    # Step 2: Segment text
-    print(f"  {YELLOW}→{RESET} Segmenting text...", end="", flush=True)
-    start = time.perf_counter()
-    normalize = not args.no_normalize_quotes
-    sentences = segment_text(
-        text.strip(), flatten=True, unwrap=args.unwrap, normalize=normalize,
-    )
-    elapsed = time.perf_counter() - start
-    print(f"\r  {GREEN}✓{RESET} Segmented into {len(sentences):,} sentences ({elapsed:.2f}s)")
+    output_file = args.output_file or _generate_output_path(args.input_file)
 
-    # Step 3: Write output
-    total = len(sentences)
-    with open(args.output_file, "w", encoding="utf-8") as f:
-        if args.unwrap:
-            f.write(format_grouped_sentences(sentences) + "\n")
-            print(f"  {GREEN}✓{RESET} Written {total:,} sentences to {args.output_file}")
-        else:
-            for i, sentence in enumerate(sentences, 1):
-                f.write(sentence + "\n")
-                if i % 500 == 0 or i == total:
-                    pct = (i / total) * 100
-                    print(f"\r  {YELLOW}→{RESET} Writing... {pct:.0f}% ({i:,}/{total:,})", end="", flush=True)
-            print(f"\r  {GREEN}✓{RESET} Written {total:,} sentences to {args.output_file}       ")
+    _header("segment-file")
+    print(f"  {DIM}Segmenting text file into sentences{RESET}")
+    print()
+
+    format_value = "dialog" if args.format else None
+    _process_single_file(args.input_file, output_file, args.unwrap, normalize, format_value)
 
     print(f"\n  {GREEN}Done!{RESET}")
     print()

fast_sentence_segment/dmo/__init__.py

@@ -13,3 +13,5 @@ from .unwrap_hard_wrapped_text import unwrap_hard_wrapped_text
 from .normalize_quotes import normalize_quotes
 from .group_quoted_sentences import group_quoted_sentences, format_grouped_sentences
 from .strip_trailing_period_after_quote import StripTrailingPeriodAfterQuote
+from .ocr_artifact_fixer import OcrArtifactFixer
+from .dialog_formatter import DialogFormatter, format_dialog

fast_sentence_segment/dmo/dialog_formatter.py

@@ -0,0 +1,255 @@
+# -*- coding: UTF-8 -*-
+"""Dialog-aware paragraph formatter for segmented text.
+
+Formats segmented sentences into readable paragraphs with intelligent
+grouping of dialog and narrative text. Keeps multi-sentence quoted speech
+together and adds paragraph breaks between different speakers.
+
+Related GitHub Issue:
+    #10 - feat: Add --format flag for dialog-aware paragraph formatting
+    https://github.com/craigtrim/fast-sentence-segment/issues/10
+
+Example:
+    >>> from fast_sentence_segment.dmo.dialog_formatter import format_dialog
+    >>> sentences = [
+    ...     '"Hello," said Jack.',
+    ...     '"How are you today?',
+    ...     'I hope you are well."',
+    ...     '"I am fine," replied Mary.',
+    ... ]
+    >>> print(format_dialog(sentences))
+    "Hello," said Jack.
+
+    "How are you today?
+    I hope you are well."
+
+    "I am fine," replied Mary.
+"""
+
+import re
+from typing import List
+
+from fast_sentence_segment.core import BaseObject
+
+
+# Quote characters to track for dialog detection
+DOUBLE_QUOTES = '""\""'
+SINGLE_QUOTES = "'''"
+ALL_QUOTES = DOUBLE_QUOTES + SINGLE_QUOTES
+
+
+def _count_quotes(text: str) -> int:
+    """Count quote characters in text (both single and double)."""
+    return sum(1 for c in text if c in ALL_QUOTES)
+
+
+def _starts_with_quote(text: str) -> bool:
+    """Check if text starts with a quote character."""
+    text = text.lstrip()
+    return text and text[0] in ALL_QUOTES
+
+
+def _ends_with_closing_quote(text: str) -> bool:
+    """Check if text ends with a closing quote (possibly followed by punctuation)."""
+    text = text.rstrip()
+    if not text:
+        return False
+    # Check last few characters for closing quote pattern
+    # e.g., '" or "' or .' or ."
+    for i in range(min(3, len(text)), 0, -1):
+        if text[-i] in ALL_QUOTES:
+            return True
+    return False
+
+
+def _is_complete_quote(text: str) -> bool:
+    """Check if text contains a complete (balanced) quote.
+
+    A complete quote has an even number of quote characters,
+    meaning all opened quotes are closed.
+    """
+    quote_count = _count_quotes(text)
+    return quote_count > 0 and quote_count % 2 == 0
+
+
+def _sentence_is_dialog_continuation(sentence: str, in_quote: bool) -> bool:
+    """Determine if sentence continues an open quote.
+
+    Args:
+        sentence: The sentence to check.
+        in_quote: Whether we're currently inside an unclosed quote.
+
+    Returns:
+        True if this sentence is a continuation of open dialog.
+    """
+    if in_quote:
+        return True
+    return False
+
+
+def _get_quote_delta(sentence: str) -> int:
+    """Get the net change in quote depth for a sentence.
+
+    Returns:
+        Positive if more quotes opened than closed,
+        negative if more closed than opened,
+        zero if balanced.
+    """
+    return _count_quotes(sentence) % 2
+
+
+class DialogFormatter(BaseObject):
+    """Formats segmented sentences with dialog-aware paragraph grouping.
+
+    This formatter analyzes sentence structure to intelligently group
+    text into paragraphs:
+
+    - Multi-sentence quoted speech stays together (same speaker)
+    - Paragraph breaks added between different speakers
+    - Narrative text grouped appropriately
+    - Handles both single and double quote styles
+
+    Example:
+        >>> formatter = DialogFormatter()
+        >>> sentences = ['"Hello," he said.', 'The door opened.']
+        >>> print(formatter.process(sentences))
+        "Hello," he said.
+
+        The door opened.
+    """
+
+    def __init__(self):
+        """Initialize the DialogFormatter."""
+        BaseObject.__init__(self, __name__)
+
+    def process(self, sentences: List[str]) -> str:
+        """Format sentences into dialog-aware paragraphs.
+
+        Args:
+            sentences: List of segmented sentences.
+
+        Returns:
+            Formatted string with appropriate paragraph breaks.
+        """
+        return format_dialog(sentences)
+
+
+def _is_narrative(sentence: str) -> bool:
+    """Check if a sentence is narrative (no quotes at start)."""
+    return not _starts_with_quote(sentence)
+
+
+def _ends_dialog_turn(sentence: str) -> bool:
+    """Check if a sentence ends a dialog turn.
+
+    A dialog turn ends when the sentence ends with a closing quote
+    followed by optional punctuation or dialog tag ending.
+    """
+    sentence = sentence.rstrip()
+    if not sentence:
+        return False
+
+    # Pattern: ends with quote + optional punctuation
+    # e.g., ." or .' or "' or '" or ," he said. etc.
+    # Check if there's a closing quote near the end
+    last_chars = sentence[-10:] if len(sentence) >= 10 else sentence
+
+    # Count quotes in last part - if odd from end, likely closes
+    for i, c in enumerate(reversed(last_chars)):
+        if c in ALL_QUOTES:
+            # Found a quote - check if it's likely a closer
+            # A closer is typically followed by punctuation or end
+            remaining = last_chars[len(last_chars) - i:]
+            if not remaining or all(ch in '.,!?;: ' for ch in remaining):
+                return True
+            # Also handle dialog tags: ,' he said.
+            if remaining and remaining[0] in '.,!?' and 'said' not in sentence.lower()[-20:]:
+                return True
+            break
+
+    return False
+
+
+def format_dialog(sentences: List[str]) -> str:
+    """Format sentences into dialog-aware paragraphs.
+
+    Groups sentences intelligently based on dialog structure:
+    - Sentences within an unclosed quote stay grouped
+    - Complete quoted sentences become their own paragraphs
+    - Narrative text is grouped together
+    - Paragraph breaks separate different speakers/turns
+
+    Args:
+        sentences: List of segmented sentences.
+
+    Returns:
+        Formatted string with paragraph breaks (double newlines)
+        between logical groups and single newlines within groups.
+
+    Example:
+        >>> sentences = [
+        ...     '"My dear sir," cried the man.',
+        ...     '"You had every reason to be carried away."',
+        ... ]
+        >>> print(format_dialog(sentences))
+        "My dear sir," cried the man.
+
+        "You had every reason to be carried away."
+    """
+    if not sentences:
+        return ""
+
+    paragraphs: List[List[str]] = []
+    current_para: List[str] = []
+    in_quote = False  # Track if we're inside an unclosed quote
+
+    for i, sentence in enumerate(sentences):
+        sentence = sentence.strip()
+        if not sentence:
+            continue
+
+        quote_count = _count_quotes(sentence)
+        starts_quote = _starts_with_quote(sentence)
+        is_narrative = _is_narrative(sentence)
+        is_complete = _is_complete_quote(sentence)
+
+        # Get info about previous sentence
+        prev_sentence = current_para[-1] if current_para else ""
+        prev_was_narrative = _is_narrative(prev_sentence) if prev_sentence else False
+        prev_was_complete = _is_complete_quote(prev_sentence) if prev_sentence else False
+
+        # Determine if this sentence starts a new paragraph
+        should_start_new_para = False
+
+        if not current_para:
+            # First sentence always starts a new paragraph
+            should_start_new_para = True
+        elif in_quote:
+            # Inside an open quote - continue current paragraph
+            should_start_new_para = False
+        elif starts_quote:
+            # New quote starting - always new paragraph
+            should_start_new_para = True
+        elif is_narrative and prev_was_complete and not prev_was_narrative:
+            # Narrative after complete dialog - new paragraph
+            should_start_new_para = True
+        elif is_narrative and not prev_was_narrative and _ends_dialog_turn(prev_sentence):
+            # Narrative after dialog that ends a turn - new paragraph
+            should_start_new_para = True
+
+        if should_start_new_para and current_para:
+            paragraphs.append(current_para)
+            current_para = []
+
+        current_para.append(sentence)
+
+        # Update quote tracking
+        if quote_count % 2 == 1:
+            in_quote = not in_quote
+
+    # Don't forget the last paragraph
+    if current_para:
+        paragraphs.append(current_para)
+
+    # Format: join sentences in paragraph with newline, paragraphs with double newline
+    return "\n\n".join("\n".join(para) for para in paragraphs)

fast_sentence_segment/dmo/ocr_artifact_fixer.py

@@ -0,0 +1,70 @@
+# -*- coding: UTF-8 -*-
+"""Fix common OCR/text extraction artifacts.
+
+Ebook text files often contain artifacts where common word pairs
+are incorrectly joined. This module fixes known patterns.
+
+Related GitHub Issue:
+    #9 - Fix common OCR/cleaning artifacts (Iam, witha)
+    https://github.com/craigtrim/fast-sentence-segment/issues/9
+"""
+
+from fast_sentence_segment.core import BaseObject
+
+# Known OCR artifact patterns: (pattern, replacement)
+# All patterns include surrounding spaces to ensure exact word boundaries
+_OCR_ARTIFACTS = [
+    (" Iam ", " I am "),
+    (" Ihave ", " I have "),
+    (" ihave ", " I have "),
+    (" Ithink ", " I think "),
+    (" ithink ", " I think "),
+    (" anda ", " and a "),
+    (" witha ", " with a "),
+    (" sucha ", " such a "),
+    (" aliquid ", " a liquid "),
+]
+
+
+class OcrArtifactFixer(BaseObject):
+    """Fix common OCR/text extraction artifacts.
+
+    Detects and corrects known patterns where words are incorrectly
+    joined during OCR or text extraction processes.
+
+    Related GitHub Issue:
+        #9 - Fix common OCR/cleaning artifacts (Iam, witha)
+        https://github.com/craigtrim/fast-sentence-segment/issues/9
+    """
+
+    def __init__(self):
+        """Change Log
+
+        Created:
+            3-Feb-2026
+            craigtrim@gmail.com
+            *   fix common OCR/cleaning artifacts
+                https://github.com/craigtrim/fast-sentence-segment/issues/9
+        """
+        BaseObject.__init__(self, __name__)
+
+    @staticmethod
+    def process(input_text: str) -> str:
+        """Fix known OCR artifact patterns.
+
+        Args:
+            input_text: Text that may contain OCR artifacts.
+
+        Returns:
+            Text with known OCR artifacts corrected.
+
+        Examples:
+            >>> OcrArtifactFixer.process("Jack, Iam so happy")
+            'Jack, I am so happy'
+            >>> OcrArtifactFixer.process("horizon witha hint")
+            'horizon with a hint'
+        """
+        for pattern, replacement in _OCR_ARTIFACTS:
+            if pattern in input_text:
+                input_text = input_text.replace(pattern, replacement)
+        return input_text

fast_sentence_segment/dmo/spacy_doc_segmenter.py

@@ -65,6 +65,54 @@ class SpacyDocSegmenter(BaseObject):
             return False
         return True
 
+    @staticmethod
+    def _merge_orphaned_quotes(sentences: list) -> list:
+        """Merge orphaned opening quotes with the following sentence.
+
+        spaCy sometimes splits on opening quotes, producing sentences like:
+            ["'", "Oh, the funeral..."]
+        This merges them into:
+            ["'Oh, the funeral..."]
+
+        Also handles trailing orphaned quotes that should belong to next sentence:
+            ["He said. '", "Hello!'"]
+        Becomes:
+            ["He said.", "'Hello!'"]
+        """
+        if not sentences:
+            return sentences
+
+        result = []
+        i = 0
+        while i < len(sentences):
+            sent = sentences[i]
+            # Check if this sentence is just an opening quote
+            if sent.strip() in ("'", '"', "'.", '".'):
+                # Merge with the next sentence if available
+                if i + 1 < len(sentences):
+                    quote_char = sent.strip().rstrip('.')
+                    result.append(quote_char + sentences[i + 1])
+                    i += 2
+                    continue
+            result.append(sent)
+            i += 1
+
+        # Second pass: handle trailing orphaned quotes
+        # Pattern: sentence ends with `. '` or `. "` - move quote to next sentence
+        fixed = []
+        for i, sent in enumerate(result):
+            # Check for trailing orphaned quote (`. '` or `? '` or `! '`)
+            if len(sent) >= 3 and sent[-2:] in (" '", ' "') and sent[-3] in '.?!':
+                # Strip the trailing quote
+                trailing_quote = sent[-1]
+                sent = sent[:-2]
+                # Prepend to next sentence if available
+                if i + 1 < len(result) and not result[i + 1].startswith(('"', "'")):
+                    result[i + 1] = trailing_quote + result[i + 1]
+            fixed.append(sent)
+
+        return fixed
+
     @staticmethod
     def _cleanse(sentences: list) -> str:
         sentences = [sent for sent in sentences
@@ -103,6 +151,9 @@ class SpacyDocSegmenter(BaseObject):
         sentences = [sent for sent in sentences if
                      sent and len(sent) and sent != 'None']
 
+        # Merge orphaned opening quotes with following sentence
+        sentences = self._merge_orphaned_quotes(sentences)
+
         sentences = [self._append_period(sent)
                      for sent in sentences]
 

fast_sentence_segment/svc/perform_sentence_segmentation.py

@@ -55,6 +55,7 @@ from fast_sentence_segment.dmo import SpacyDocSegmenter
 from fast_sentence_segment.dmo import PostProcessStructure
 from fast_sentence_segment.dmo import StripTrailingPeriodAfterQuote
 from fast_sentence_segment.dmo import Dehyphenator
+from fast_sentence_segment.dmo import OcrArtifactFixer
 
 
 class PerformSentenceSegmentation(BaseObject):
@@ -84,6 +85,7 @@ class PerformSentenceSegmentation(BaseObject):
             self.__nlp = _load_spacy_model("en_core_web_sm")
 
         self._dehyphenate = Dehyphenator.process
+        self._fix_ocr_artifacts = OcrArtifactFixer.process
         self._newlines_to_periods = NewlinesToPeriods.process
         self._normalize_numbered_lists = NumberedListNormalizer().process
         self._normalize_ellipses = EllipsisNormalizer().process
@@ -138,6 +140,9 @@ class PerformSentenceSegmentation(BaseObject):
         # Must happen before newlines are converted to periods
         input_text = self._dehyphenate(input_text)
 
+        # Fix common OCR artifacts (issue #9)
+        input_text = self._fix_ocr_artifacts(input_text)
+
         input_text = self._normalize_numbered_lists(input_text)
         input_text = self._normalize_ellipses(input_text)
 

{fast_sentence_segment-1.4.3.dist-info → fast_sentence_segment-1.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fast-sentence-segment
-Version: 1.4.3
+Version: 1.4.5
 Summary: Fast and Efficient Sentence Segmentation
 Home-page: https://github.com/craigtrim/fast-sentence-segment
 License: MIT

{fast_sentence_segment-1.4.3.dist-info → fast_sentence_segment-1.4.5.dist-info}/RECORD

@@ -1,32 +1,34 @@
-fast_sentence_segment/__init__.py,sha256=jeb4yCy89ivyqbo-4ldJLquPAG_XR_33Q7nrDjqPxvE,1465
+fast_sentence_segment/__init__.py,sha256=DI7cyxtqnWd_5lrtGXqkIm8Aje0h55nGAHGgP6zXRyE,2278
 fast_sentence_segment/bp/__init__.py,sha256=j2-WfQ9WwVuXeGSjvV6XLVwEdvau8sdAQe4Pa4DrYi8,33
 fast_sentence_segment/bp/segmenter.py,sha256=UW6DguPgA56h-pPYRsfJhjIzBe40j6NdjkwYxamASyA,1928
-fast_sentence_segment/cli.py,sha256=vr1Gh-pq4bIPcnhUF6c7ckGdEfoyrI_r0XcrJrIfjEA,5640
+fast_sentence_segment/cli.py,sha256=8AA_V3hcOVL8ENXboFDuJTimFbHKK5KbjzIeS9Cs-xA,9568
 fast_sentence_segment/core/__init__.py,sha256=uoBersYyVStJ5a8zJpQz1GDGaloEdAv2jGHw1292hRM,108
 fast_sentence_segment/core/base_object.py,sha256=AYr7yzusIwawjbKdvcv4yTEnhmx6M583kDZzhzPOmq4,635
 fast_sentence_segment/core/stopwatch.py,sha256=hE6hMz2q6rduaKi58KZmiAL-lRtyh_wWCANhl4KLkRQ,879
-fast_sentence_segment/dmo/__init__.py,sha256=N0lLHVn6zKeg6h1LIfoc4XeXPUY-uSbMT45dP2_vn8M,862
+fast_sentence_segment/dmo/__init__.py,sha256=emn-F46GMpR5dTQXbhMWGexj7sOfWWuTpRWiENjIadQ,972
 fast_sentence_segment/dmo/abbreviation_merger.py,sha256=tCXM6yCfMryJvMIVWIxP_EocoibZi8vohFzJ5tvMYr0,4432
 fast_sentence_segment/dmo/abbreviation_splitter.py,sha256=03mSyJcLooNyIjXx6mPlrnjmKgZW-uhUIqG4U-MbIGw,2981
 fast_sentence_segment/dmo/abbreviations.py,sha256=CGJrJDo6pmYd3pTNEQbdOo8N6tnkCnwyL2X7Si663Os,2530
 fast_sentence_segment/dmo/bullet_point_cleaner.py,sha256=WOZQRWXiiyRi8rOuEIw36EmkaXmATHL9_Dxb2rderw4,1606
 fast_sentence_segment/dmo/dehyphenator.py,sha256=6BJTie7tClRAifeiW8V2CdAAbcbknhtqmKylAdRZ7ko,1776
+fast_sentence_segment/dmo/dialog_formatter.py,sha256=C5r3t3vY6ZbU26g0JGIk8BRlXhMn1LGPljw_WPTuCSg,8276
 fast_sentence_segment/dmo/ellipsis_normalizer.py,sha256=lHs9dLFfKJe-2vFNe17Hik90g3_kXX347OzGP_IOT08,1521
 fast_sentence_segment/dmo/group_quoted_sentences.py,sha256=Ifh_kUwi7sMbzbZvrTgEKkzWe50AafUDhVKVPR9h7wQ,5092
 fast_sentence_segment/dmo/newlines_to_periods.py,sha256=PUrXreqZWiITINfoJL5xRRlXJH6noH0cdXtW1EqAh8I,1517
 fast_sentence_segment/dmo/normalize_quotes.py,sha256=mr53qo_tj_I9XzElOKjUQvCtDQh7mBCGy-iqsHZDX14,2881
 fast_sentence_segment/dmo/numbered_list_normalizer.py,sha256=q0sOCW8Jkn2vTXlUcVhmDvYES3yvJx1oUVl_8y7eL4E,1672
+fast_sentence_segment/dmo/ocr_artifact_fixer.py,sha256=lhU6Nfp4_g5yChm1zxgwM5R5ixk3pzhrk1qEgNJa8Hc,2139
 fast_sentence_segment/dmo/post_process_sentences.py,sha256=5jxG3TmFjxIExMPLhnCB5JT1lXQvFU9r4qQGoATGrWk,916
 fast_sentence_segment/dmo/question_exclamation_splitter.py,sha256=cRsWRu8zb6wOWG-BjMahHfz4YGutKiV9lW7dE-q3tgc,2006
-fast_sentence_segment/dmo/spacy_doc_segmenter.py,sha256=_oTsrIL2rjysjt_8bPJVNTn230pUtL-geCC8g174iC4,3163
+fast_sentence_segment/dmo/spacy_doc_segmenter.py,sha256=Kb65TYMhrbpTYEey5vb7TyhCjUHVxmugHYIeKkntCwk,5147
 fast_sentence_segment/dmo/strip_trailing_period_after_quote.py,sha256=wYkoLy5XJKZIblJXBvDAB8-a81UTQOhOf2u91wjJWUw,2259
 fast_sentence_segment/dmo/title_name_merger.py,sha256=zbG04_VjwM8TtT8LhavvmZqIZL_2xgT2OTxWkK_Zt1s,5133
 fast_sentence_segment/dmo/unwrap_hard_wrapped_text.py,sha256=V1T5RsJBaII_iGJMyWvv6rb2mny8pnVd428oVZL0n5I,2457
 fast_sentence_segment/svc/__init__.py,sha256=9B12mXxBnlalH4OAm1AMLwUMa-RLi2ilv7qhqv26q7g,144
 fast_sentence_segment/svc/perform_paragraph_segmentation.py,sha256=zLKw9rSzb0NNfx4MyEeoGrHwhxTtH5oDrYcAL2LMVHY,1378
-fast_sentence_segment/svc/perform_sentence_segmentation.py,sha256=mAJEPWqNQFbnlj7Rb7yiXIRHCAdlgsN0jAbg7e2qpMU,7421
-fast_sentence_segment-1.4.3.dist-info/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
-fast_sentence_segment-1.4.3.dist-info/METADATA,sha256=5LGK9z9ip2AtOr2FgaIgkrR2mLvIQaeeuh8gVi3GBaA,7785
-fast_sentence_segment-1.4.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-fast_sentence_segment-1.4.3.dist-info/entry_points.txt,sha256=Zc8OwFKj3ofnjy5ZIFqHzDkIEWweV1AP1xap1ZFGD8M,107
-fast_sentence_segment-1.4.3.dist-info/RECORD,,
+fast_sentence_segment/svc/perform_sentence_segmentation.py,sha256=vpZ7lyOKxIAgHRk-rfgNI4BtAC38vU5UbYokkw8JhK8,7639
+fast_sentence_segment-1.4.5.dist-info/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
+fast_sentence_segment-1.4.5.dist-info/METADATA,sha256=1xZQ63f20VnbsLC_leQ40nIXLRbSkfUwnbxZmtBGYo0,7785
+fast_sentence_segment-1.4.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fast_sentence_segment-1.4.5.dist-info/entry_points.txt,sha256=Zc8OwFKj3ofnjy5ZIFqHzDkIEWweV1AP1xap1ZFGD8M,107
+fast_sentence_segment-1.4.5.dist-info/RECORD,,