markdowncleaner 0.1.1__py3-none-any.whl

markdowncleaner/__init__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+__version__ = version("markdowncleaner")
+from .markdowncleaner import CleanerOptions, MarkdownCleaner

markdowncleaner/cli.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""
+Command-line interface for the markdowncleaner package.
+"""
+
+import argparse
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from markdowncleaner.markdowncleaner import MarkdownCleaner, CleanerOptions
+from markdowncleaner.config.loader import get_default_patterns, CleaningPatterns
+
+
+def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Clean and format markdown files by removing unwanted sections, lines, and patterns."
+    )
+
+    parser.add_argument(
+        "input_file",
+        type=Path,
+        help="Path to the markdown file to clean"
+    )
+
+    parser.add_argument(
+        "-o", "--output",
+        type=Path,
+        help="Path to save the cleaned markdown file (default: input_file with '_cleaned' suffix)"
+    )
+
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        help="Directory to save the cleaned file (default: same as input file)"
+    )
+
+    parser.add_argument(
+        "--config",
+        type=Path,
+        help="Path to custom YAML configuration file with cleaning patterns"
+    )
+
+    # Options for customizing the cleaning process
+    parser.add_argument(
+        "--keep-short-lines",
+        action="store_true",
+        help="Don't remove lines shorter than minimum length"
+    )
+
+    parser.add_argument(
+        "--min-line-length",
+        type=int,
+        default=70,
+        help="Minimum line length to keep (default: 70)"
+    )
+
+    parser.add_argument(
+        "--keep-bad-lines",
+        action="store_true",
+        help="Don't remove lines matching bad line patterns"
+    )
+
+    parser.add_argument(
+        "--keep-sections",
+        action="store_true",
+        help="Don't remove sections like References, Acknowledgements, etc."
+    )
+
+    parser.add_argument(
+        "--keep-footnotes",
+        action="store_true",
+        help="Don't remove footnote references"
+    )
+
+    parser.add_argument(
+        "--no-replacements",
+        action="store_true",
+        help="Don't perform text replacements"
+    )
+
+    parser.add_argument(
+        "--keep-inline-patterns",
+        action="store_true",
+        help="Don't remove inline patterns like citations"
+    )
+
+    parser.add_argument(
+        "--keep-empty-lines",
+        action="store_true",
+        help="Don't contract consecutive empty lines"
+    )
+
+    parser.add_argument(
+        "--no-crimp-linebreaks",
+        action="store_true",
+        help="Don't crimp linebreaks"
+    )
+
+    return parser.parse_args(args)
+
+
+def main(args: Optional[List[str]] = None) -> int:
+    """Main entry point for the CLI."""
+    parsed_args = parse_args(args)
+
+    # Configure cleaner options based on arguments
+    options = CleanerOptions()
+    options.remove_short_lines = not parsed_args.keep_short_lines
+    options.min_line_length = parsed_args.min_line_length
+    options.remove_whole_lines = not parsed_args.keep_bad_lines
+    options.remove_sections = not parsed_args.keep_sections
+    options.remove_footnotes_in_text = not parsed_args.keep_footnotes
+    options.replace_within_lines = not parsed_args.no_replacements
+    options.remove_within_lines = not parsed_args.keep_inline_patterns
+    options.contract_empty_lines = not parsed_args.keep_empty_lines
+    options.crimp_linebreaks = not parsed_args.no_crimp_linebreaks
+
+    # Load patterns from custom config or use defaults
+    if parsed_args.config:
+        try:
+            patterns = CleaningPatterns.from_yaml(parsed_args.config)
+        except Exception as e:
+            print(f"Error loading custom configuration: {e}", file=sys.stderr)
+            return 1
+    else:
+        patterns = get_default_patterns()
+
+    # Initialize the cleaner with patterns and configured options
+    cleaner = MarkdownCleaner(patterns=patterns, options=options)
+
+    try:
+        # Clean the markdown file
+        output_file = cleaner.clean_markdown_file(
+            input_file=parsed_args.input_file,
+            output_path=parsed_args.output_dir,
+            output_file=parsed_args.output
+        )
+        print(f"Cleaned markdown saved to: {output_file}")
+        return 0
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

markdowncleaner/config/default_cleaning_patterns.yaml

@@ -0,0 +1,296 @@
+sections_to_remove:
+  - |-
+    Acknowledge?ments?\s?$
+  - Declaration of Competing Interest
+  - Declaration of Conflicting Interests
+  - Funding
+  - Endnotes
+  - Credit authorship contribution statement
+  - |-
+    ACM\s?Reference\?Format
+  - Contributors
+  - CCS CONCEPTS
+  - pdf version of the entry
+  - |-
+    Correspondence\s?$
+  - Funding information
+  - ORCID
+  - RESEARCH ARTICLE
+  - Compliance with Ethical Standards
+  - License
+  - |-
+    Index\s?$
+  - |-
+    References\s?\.?$
+  - |-
+    \s?\d{1,2}\.\s*References\s?$
+  - |-
+    Bibliography\s?
+  - |-
+    ^\s?Notes\s?\:?\.?$
+  - Works Cited
+  - Author Biographies
+  - Author Biography
+  - About the author
+  - Oxford Scholarship Online
+  - University Press Scholarship Online
+  - Corresponding author
+  - |-
+    ^Appendix\*
+  - Contents
+  - Table of Contents
+  - List of Figures
+  - List of Tables
+  - List of Illustrations
+  - |-
+    ^\s?Keywords\s?\.?$
+  - Authors' Note
+  - |-
+    ^[a-zA-Z0-9]+-[a-zA-Z0-9]+-?[a-zA-Z0-9]+[^\s]*$
+  - |-
+    Dedication\:?$
+  - Citing this paper
+  - General rights
+  - Disclosure statement
+  - PLEASE SCROLL DOWN FOR ARTICLE
+  - About the Contributors
+  - List of Key Terms
+  - |-
+    Citations\:
+  - |-
+    Provided by\:
+  - |-
+    Published by\:
+  - Preface
+
+# Patterns for inline text that should be removed
+bad_inline_patterns:
+  - |-
+    \[\d{1,3}(?:\s*,\s*\d{1,3})*\]
+  - |-
+    \(see.*?\)
+  - |-
+    \(Figure\s*.*?\)
+  - |-
+    \(Table\s*.*?\)
+  - |-
+    \(Section\s*.*?\)
+  - |-
+    \(Chapter\s*.*?\)
+  - |-
+    \* 1 ,
+  - |-
+    \(p\.?\d{1,3}\)
+
+# Patterns for entire lines that should be removed
+bad_lines_patterns:
+  # Copyright and legal
+  - |-
+    (?m)^.*?\b[Aa]ll [Rr]ights [Rr]eserved\..*?$|.*?\b[Aa]ll [Rr]ights [Rr]eserved\.$
+  - |-
+    Copyright \d{4} by
+  - |-
+    ^#{1,3} ORIGINAL RESEARCH$
+  - |-
+    ^#{1,3} ORIGINAL ARTICLE$
+  - |-
+    ^[A-Z\s]+$
+  - |-
+    <!--.*?-->
+  - |-
+    ^- (\d{1,3})?\.
+  - |-
+    ^Received:
+  - |-
+    ^\(Received
+  - |-
+    Received.*?(\d{4}).*?(accepted|Accepted).*
+  - |-
+    doi:
+  - |-
+    10\.\d{4,9}/[-._;()/:\w]+
+  - Digital Object Identifier
+  - |-
+    ^[Kk]eywords
+  - |-
+     ^Conflict of Interest
+  - ^Library of Congress
+  - ^Open Access
+  - ^OpenAccess
+  - ^Department of
+  - |-
+   ^[A-Z][a-zA-Z]+ University Press, Inc
+  - |-
+    ^Author\(s\):
+  - ^Acknowledgements
+  - ^Acknowledgments
+  - ^Funding Statement
+  - ^Table of Contents
+  - |-
+    ^Source:
+  - |-
+    ^Published by
+  - |-
+   ^CCS Concepts
+  - ISSN
+  - ISBN
+  - |-
+    arxiv:
+  - JSTOR
+  - Contributed Paper
+  - |-
+    ^Fig\.
+  - |-
+    ^\s?Figure\s?\d{1,2}\s?[.:]
+  - |-
+    ^\s?Table\s?\d{1,2}\s?[.:]
+  - ^First published
+  - |-
+    National Science Foundation\s*\(NSF\).*?[Gg]rant.*?\d{4,}
+  - |-
+    [Gg]rant\s*[Nn]umber\s*\d{4,}
+  - |-
+    [A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}
+  - |-
+    \{\s*[\w,\s]+\s*\}\s*@\s*[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}
+  - |-
+    https?://?[^\s]+
+  - ©
+  - ®
+  - ™
+  - |-
+    c \d{4}
+  - |-
+    ^\d{1,3}\s?.\s?[Ss]ee
+  - |-
+    ^\-\s?\d{1,3}\s?.\s?[Ss]ee
+  - |-
+    ^\d{1,2}[A-Za-z]+
+  - |-
+    ^#{1,3}[\d\s]+$
+  - |-
+    ^[^/]*(/[^/]*){5,}
+  - †
+  - ‡
+  - |-
+    ^\d{1,3}.*supra note
+  - |-
+    ^\d{1,3}.*\(\d{4}\)
+  - |-
+    ^\d{1,3}.*cf
+  - |-
+    ^\d{1,3}.\s?Id\.
+  - |-
+    (?:.*?GLYPH\&lt){3,}.*
+  - on previous versions of this
+  - |-
+    ^We are grateful for
+  - |-
+    ^I am grateful
+  - |-
+    ^Subject:
+  - open access article under the terms of the Creative Commons
+  - |-
+    Key words:
+  - |-
+    Published online:
+  - |-
+    Correspondence:
+  - |-
+    Print Publication Date:
+  - |-
+    Accepted:
+  - |-
+    This article was downloaded by:
+  - This content downloaded from
+  - |-
+    Registered office:
+  - |-
+    DATE DOWNLOADED:
+  - NBER working papers are circulated
+  - FirstView Article
+  - Published in
+  - Published online
+  - Typeset by
+  - All rights reserved
+  - No portion of this publication may be reproduced
+  - authors have asserted their rights to be identified as the authors
+  - material in this work has been asserted in accordance
+  - imprint of Macmillan Publishers
+  - A catalogue record for this book is available
+  - Your use of this HeinOnline
+  - obtain permission to use this article
+  - This article is published with open access
+  - Links to third party websites
+  - Registered in England and Wales
+  - |-
+    \*I thank
+  - |-
+    \*We thank
+  - |-
+    \*For comments
+  - |-
+    \*I would like to thank
+  - |-
+    \*We would like to thank
+  - |-
+    \*Thanks to
+  - |-
+    \*\s?This article is
+  - I am grateful for
+  - Oxford University Press is a department of
+  - a registered trademark of
+  - Published in the United States
+  - Printed in the United States
+  - The paper used in this publication
+  - Trademark notice
+  - 1 Department of
+  - catalogue recordfor this publication
+  - An earlier version of this paper was
+  - Springer International Publishing AG
+  - part of Springer Nature
+  - This work is subject to copyright.
+  - permission to reuse the copyright
+  - distributed under the terms of a Creative Commons
+  - |-
+    , [A-Z]{2} \d{5}(?:-\d{4})?,
+
+# Footnotes
+# Pattern matches:
+# 1. A period
+# 2. Optional space
+# 3. 1-3 digits
+# 4. Optionally followed by comma and space and more 1-3 digit numbers
+# 5. This pattern repeats for any number of comma-separated numbers
+# 6. Followed by either end of string, whitespace, or newline
+footnote_patterns:
+  - |-
+    \.\s?(\d{1,3}(?:,\s*\d{1,3})*?)(?=\s|$)
+
+replacements:
+  'GLYPH&lt;28&gt;': 'fi'
+  'GLYPH&lt;29&gt;': 'fl'
+  'GLYPH&lt;3&gt;': '*'
+  'GLYPH&lt;21&gt;': '-'
+  'GLYPH&lt;22&gt;': '—'
+  'GLYPH&lt;15&gt;': '-'
+  'GLYPH&lt;129&gt;': '-'
+  'GLYPH&lt;133&gt;': 'fi'
+  'GLYPH<28>': 'fi'
+  'GLYPH<29>': 'fl'
+  'GLYPH<3>': '*'
+  'GLYPH<21>': '-'
+  'GLYPH<22>': '—'
+  'GLYPH<15>': '•'
+  '/uniFB01': 'fi'
+  '/uniFB03': 'ffi'
+  '/uniFB00': 'ff'
+  '/uniFB02': 'fl'
+  'GLYPH<146>': 'Š'
+  'GLYPH<228>': 'ä'
+  'GLYPH<252>': 'ü'
+  'GLYPH<237>': 'í'
+  'GLYPH<2>': 'Š'
+  '/ornm58.ornm': ''
+  '/ornm69.ornm': ''
+  '&amp;': '&'

markdowncleaner/config/loader.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+import yaml
+import re
+from typing import List, Pattern, Optional, Dict
+from dataclasses import dataclass
+import logging
+
+_logger = logging.getLogger(__name__)
+
+@dataclass
+class CleaningPatterns:
+    """Container for document cleaning patterns."""
+    sections_to_remove: List[str]
+    bad_inline_patterns: List[Pattern]
+    bad_lines_patterns: List[Pattern]
+    footnote_patterns: List[Pattern]
+    replacements: Dict[Pattern | str, str]
+
+    @classmethod
+    def from_yaml(cls, yaml_path: Path) -> 'CleaningPatterns':
+        """
+        Create a CleaningPatterns instance from a YAML file.
+        
+        Args:
+            yaml_path: Path to the YAML configuration file
+            
+        Returns:
+            CleaningPatterns instance with compiled regex patterns
+        """
+        try:
+            with open(yaml_path, 'r', encoding='utf-8') as f:
+                config = yaml.safe_load(f)
+
+            def compile_pattern(pattern: str) -> Pattern:
+                """Safely compile a regex pattern."""
+                if not isinstance(pattern, str):
+                    raise TypeError(f"Pattern must be a string, got {type(pattern)}: {pattern}")
+                try:
+                    return re.compile(pattern, re.IGNORECASE)
+                except re.error as e:
+                    _logger.error(f"Invalid regex pattern '{pattern}': {e}")
+                    raise
+
+            # Ensure patterns are strings before compiling
+            sections = [str(p).strip() for p in config.get('sections_to_remove', [])]
+            inline_patterns = [str(p).strip() for p in config.get('bad_inline_patterns', [])]
+            line_patterns = [str(p).strip() for p in config.get('bad_lines_patterns', [])]
+            footnotes = [str(p).strip() for p in config.get('footnote_patterns', [])]
+            replace = {str(k).strip(): str(v).strip() for k, v in config.get('replacements', {}).items()}
+
+            return cls(
+                sections_to_remove=sections,
+                bad_inline_patterns=[compile_pattern(p) for p in inline_patterns],
+                bad_lines_patterns=[compile_pattern(p) for p in line_patterns],
+                footnote_patterns=[compile_pattern(p) for p in footnotes],
+                replacements=replace # just taking this a str dict, no patterns to compile
+            )
+        except Exception as e:
+            _logger.error(f"Error loading cleaning patterns from {yaml_path}: {e}")
+            raise
+
+
+_cached_patterns: Optional[CleaningPatterns] = None
+
+def get_default_patterns() -> CleaningPatterns:
+    """Load the default cleaning patterns from the package's configuration file."""
+    global _cached_patterns
+    if _cached_patterns is None:
+        config_dir = Path(__file__).parent
+        yaml_path = config_dir / 'default_cleaning_patterns.yaml'
+        _cached_patterns = CleaningPatterns.from_yaml(yaml_path)
+    return _cached_patterns

markdowncleaner/markdowncleaner.py

@@ -0,0 +1,372 @@
+from pathlib import Path
+from dataclasses import dataclass
+from typing import Pattern, Optional
+import logging
+import re
+import ftfy
+from markdowncleaner.config.loader import get_default_patterns, CleaningPatterns
+
+_logger = logging.getLogger(__name__)
+
+@dataclass
+class CleanerOptions:
+    """Countainer for Cleaner options"""
+    remove_short_lines: bool = True
+    min_line_length: int = 70
+    remove_whole_lines: bool = True
+    remove_sections: bool = True
+    remove_footnotes_in_text: bool = True
+    replace_within_lines: bool = True
+    remove_within_lines: bool = True
+    contract_empty_lines: bool = True
+    crimp_linebreaks: bool = True
+
+
+class MarkdownCleaner:
+    """Class to handle markdown document cleaning operations."""
+
+    def __init__(self, patterns: CleaningPatterns = None, options: Optional[CleanerOptions] = None):
+        """
+        Initialize the cleaner with patterns.
+
+        Args:
+            patterns: CleaningPatterns instance, or None to use defaults
+        """
+        self.patterns = patterns or get_default_patterns()
+        self.options = options or CleanerOptions()
+
+    def clean_markdown_file(self,
+                            input_file: Path,
+                            output_path: Optional[Path] = None,
+                            output_file: Optional[Path | str] = None) -> Path:
+        """
+        Clean a markdown file using the configured patterns.
+
+        Args:
+            input_file: Path to the input markdown file
+            output_path: Optional directory to save the cleaned file
+            output_file: Optional path to save the cleaned file directly (overrides output_path)
+
+        Returns:
+            Path: The path to the cleaned output file
+        """
+        # Read the content of the input file
+        with open(input_file, 'r', encoding='utf-8') as file:
+            cleaned_content = file.read()
+
+        # Apply cleaning operations
+        cleaned_content = self.clean_markdown_string(cleaned_content)
+
+        if output_file is not None:
+            cleaned_filepath = Path(output_file)
+            # Ensure parent directory exists
+            cleaned_filepath.parent.mkdir(parents=True, exist_ok=True)
+        elif output_path is not None:
+            # Determine the output filepath
+            output_path.mkdir(parents=True, exist_ok=True)
+            cleaned_filepath = output_path / input_file.name
+        else:  # both output_path and output_file are None
+            # Generate a new filename with "_cleaned" suffix
+            cleaned_filename = f"{input_file.stem}_cleaned{input_file.suffix}"
+            cleaned_filepath = input_file.parent / cleaned_filename
+        # Ensure parent directory exists (might be unnecessary if same as input, but added for safety)
+        cleaned_filepath.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write the cleaned content
+        with open(cleaned_filepath, 'w', encoding='utf-8') as file:
+            file.write(cleaned_content)
+
+        _logger.info(f"Cleaned file saved to: {cleaned_filepath}")
+
+        return cleaned_filepath
+
+    def clean_markdown_string(self, content: str) -> str:
+        """Apply all cleaning operations to the content."""
+
+        # Apply all default ftfy fixes if mojibake is detected
+        if ftfy.is_bad(content):
+            content = ftfy.fix_text(content)
+
+        # Reduce two or more subsequent spaces to a single space
+        content = re.sub(r' {2,}', ' ', content)
+
+        # Normalize quotes
+        content = self._normalize_quotation_symbols(content)
+
+        # Remove lines shorter than min_line_length (default: 70 characters)
+        if self.options.remove_short_lines:
+            content = self._remove_short_lines(content, self.options.min_line_length)
+        # Clean out "bad" lines
+        if self.options.remove_whole_lines:
+            content = self._remove_whole_lines(content, self.patterns.bad_lines_patterns)
+
+        # Remove unwanted sections
+        if self.options.remove_sections:
+            for title in self.patterns.sections_to_remove:
+                content = self._remove_sections(content, title)
+
+        # Replace strings for string
+        if self.options.replace_within_lines:
+            for k, v in self.patterns.replacements.items():
+                content = self._replace_within_lines(content, k, v)
+
+        # Replace footnote pattern (numbers at end of sentence) with '.'
+        if self.options.remove_footnotes_in_text:
+            content = self._replace_within_lines(content, self.patterns.footnote_patterns, '.')
+
+        # Remove remaining unwanted inline patterns (some may have been replaced by replacements)
+        if self.options.remove_within_lines:
+            content = self._replace_within_lines(content, self.patterns.bad_inline_patterns, '')
+
+        # Clean up formatting
+        if self.options.crimp_linebreaks:
+            content = self._crimp_linebreaks(content)
+        if self.options.contract_empty_lines:
+            content = self._contract_empty_lines(content)
+
+        return content
+    
+    def _normalize_quotation_symbols(self, text: str) -> str:
+        """
+        Normalizes quotation symbols in the input text.
+
+        Args:
+            text (str): Input text to clean
+        Returns:
+            str: Text with all single and double quotation symbols replaced with standard ones.
+        """
+        double_quotes = [
+            "«",
+            "‹",
+            "»",
+            "›",
+            "„",
+            "“",
+            "‟",
+            "”",
+            "❝",
+            "❞",
+            "❮",
+            "❯",
+            "〝",
+            "〞",
+            "〟",
+            "＂",
+        ]
+        single_quotes = ["‘", "‛", "’", "❛", "❜", "`", "´", "‘", "’"]
+
+        double_quotes_regex = re.compile("|".join(double_quotes))
+        single_quotes_regex = re.compile("|".join(single_quotes))
+
+        text = single_quotes_regex.sub("'", text)
+        text = double_quotes_regex.sub('"', text)
+
+        return text 
+
+    def _replace_within_lines(self, text: str, patterns: str | Pattern | list[str | Pattern], replacement: str = '') -> str:
+        """
+        Removes multiple patterns from text, applying them sequentially.
+
+        Args:
+            text (str): Input text to clean
+            patterns (str | Pattern | list): Single regex pattern or list of regex patterns to remove
+            replacement (str, optional): String to replace the matched patterns with. Defaults to ''.
+
+        Returns:
+            str: Text with all patterns removed or replaced
+        """
+        # Ensure patterns is a list, converting single pattern to a list if needed
+        if not isinstance(patterns, list):
+            patterns = [patterns]
+
+        cleaned_text = text
+        for pattern in patterns:
+            # Make sure each pattern is a compiled regex
+            if isinstance(pattern, str):
+                pattern = re.compile(pattern)
+
+            cleaned_text = pattern.sub(replacement, cleaned_text)
+        return cleaned_text
+
+    def _remove_short_lines(self, multiline_string: str, length: int = 70) -> str:
+        """
+        Remove lines from a multiline string that are shorter than a specified length.
+
+        Args:
+            multiline_string: The input string to clean
+            length: The minimum length of lines to keep
+
+        Returns:
+            Cleaned string with matching lines removed
+        """
+
+        # Split the content into lines
+        lines = multiline_string.splitlines()
+
+        # Filter out lines that are shorter than length but that are neither empty nor start with '#'
+        filtered_lines = []
+        for line in lines:
+            if not line.strip() == '' and not line.startswith('#') and len(line) < length:
+                continue
+            filtered_lines.append(line)
+
+        # Join the remaining lines back into a single string
+        return '\n'.join(filtered_lines)
+
+    def _remove_whole_lines(self, multiline_string: str, patterns: str | Pattern | list[str | Pattern]) -> str:
+        """
+        Remove lines from a multiline string that match specified regex pattern(s).
+
+        Args:
+            multiline_string: The input string to clean
+            patterns: A single regex pattern (str or compiled Pattern) or a list of patterns
+
+        Returns:
+            Cleaned string with matching lines removed
+        """
+
+        # Split the content into lines
+        lines = multiline_string.splitlines()
+
+        # Make sure patterns is a list
+        if not isinstance(patterns, list):
+            patterns = [patterns]
+        # Filter out lines that match any of the patterns, except empty lines
+        filtered_lines = []
+        for line in lines:
+            if line.strip() == '':  # Keep empty lines
+                filtered_lines.append(line)
+                continue
+            if any(pattern.search(line) for pattern in patterns):
+                continue
+            filtered_lines.append(line)
+
+        # Join the remaining lines back into a single string
+        return '\n'.join(filtered_lines)
+
+    def _contract_empty_lines(self, multiline_string: str) -> str:
+        """Contract two or more consecutive empty lines from a multiline string."""
+        lines = multiline_string.splitlines()
+        result = []
+        prev_empty = False
+
+        for line in lines:
+            is_empty = not line.strip()
+            if not (is_empty and prev_empty):
+                result.append(line)
+            prev_empty = is_empty
+
+        return '\n'.join(result)
+
+    def _remove_sections(self, markdown_text: str, section_pattern: str) -> str:
+        """
+        Removes a first or second level section from a markdown document if its title
+        matches the given regular expression pattern.
+
+        Args:
+            markdown_text (str): The input markdown text
+            section_pattern (str): Regular expression pattern to match the section title
+
+        Returns:
+            str: The markdown text with the matching section removed
+        """
+        # Compile the pattern with IGNORECASE flag
+        pattern = re.compile(section_pattern, re.IGNORECASE)
+
+        # Split the markdown into lines
+        lines = markdown_text.splitlines()
+        result_lines = []
+
+        i = 0
+        while i < len(lines):
+            line = lines[i].strip()
+
+            # Check if line is a first or second level heading
+            if line.startswith(('#', '##')):
+                # Extract the heading text
+                heading_text = line.lstrip('#').strip() #.lower()
+
+                # Check if heading matches the pattern
+                if pattern.search(heading_text):
+                    # Skip this heading and find the end of its section
+                    i += 1
+                    section_level = line.count('#')
+
+                    # Continue until we find a heading of same or higher level, or end of document
+                    while i < len(lines):
+                        next_line = lines[i].strip()
+                        if next_line.startswith('#'):
+                            next_level = next_line.count('#')
+                            if next_level <= section_level:
+                                break
+                        i += 1
+                    continue
+
+            result_lines.append(lines[i])
+            i += 1
+
+        # Return the modified markdown, preserving original line endings
+        return '\n'.join(result_lines)
+
+    def _crimp_linebreaks(self, markdown_text: str) -> str:
+        """
+        Fix line break errors in markdown text converted from PDF.
+
+        Args:
+            markdown_text (str): Input markdown text with potential line break errors
+
+        The function handles two cases:
+        1. Hyphenated words split across lines (even with one empty line in between)
+        2. Paragraphs incorrectly split by empty lines when a line ends with a letter
+
+        Returns:
+            str: Text with all patterns removed
+        """
+
+        lines = markdown_text.splitlines()
+        result_lines = []
+        i = 0
+
+        while i < len(lines):
+            current_line = lines[i].strip()
+            
+            # Try to join as many consecutive lines as possible
+            while True:
+                joined = False
+                
+                # Case 1: Handle hyphenated words
+                if current_line.endswith('-'):
+                    j = i + 1
+                    while j < len(lines) and not lines[j].strip():
+                        j += 1
+                    
+                    if j < len(lines) and lines[j].strip() and lines[j].strip()[0].islower():
+                        current_line = current_line[:-1] + lines[j].strip()
+                        i = j  # Update i to the last joined line
+                        joined = True
+                        continue  # Skip to next join check
+                
+                # Case 2: Handle paragraph merging
+                if not current_line.startswith('#') and current_line and (current_line[-1].isalpha() or current_line[-1] in ',;\'\"'):
+                    _logger.debug(f'Crimping line:... {current_line[-50:]}')
+                    j = i + 1
+                    while j < len(lines) and not lines[j].strip():
+                        j += 1
+                    
+                    if j < len(lines) and lines[j].strip() and \
+                       not lines[j].strip().startswith('#') and \
+                       not lines[j].strip().startswith('*') and \
+                       not lines[j].strip().startswith('-'):
+                        current_line = current_line + ' ' + lines[j].strip()
+                        i = j  # Update i to the last joined line
+                        joined = True # noqa: F841
+                        continue  # Skip to next join check
+                
+                # If no joins were made, break the loop
+                break
+            
+            # Add the fully processed line to results
+            result_lines.append(current_line)
+            i += 1  # Move to the next line
+        
+        return '\n'.join(result_lines)

markdowncleaner-0.1.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Johannes Himmelreich
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

markdowncleaner-0.1.1.dist-info/METADATA

@@ -0,0 +1,132 @@
+Metadata-Version: 2.2
+Name: markdowncleaner
+Version: 0.1.1
+Summary: A tool for cleaning and formatting markdown documents
+Author-email: Johannes Himmelreich <jrhimmel@syr.edu>
+License: MIT
+Project-URL: Repository, https://github.com/josk0/markdowncleaner
+Project-URL: Issues, https://github.com/josk0/markdowncleaner/issues
+Keywords: markdown,cleaning,formatting,text processing
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Utilities
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: ftfy>=6.0.3
+
+# markdowncleaner
+
+A simple Python tool for cleaning and formatting markdown documents. Default configuration with regex patterns for PDFs of academic papers that have been converted to markdown.
+
+## Description
+
+`markdowncleaner` helps you clean up markdown files by removing unwanted content such as:
+- References, bibliographies, and citations
+- Footnotes and endnotes
+- Copyright notices and legal disclaimers
+- Acknowledgements and funding information
+- Author information and contact details
+- Specific patterns like DOIs, URLs, and email addresses
+- Short lines and excessive whitespace
+
+This tool is particularly useful for processing academic papers, books, or any markdown document that needs formatting cleanup.
+
+## Installation
+
+```bash
+pip install markdowncleaner
+```
+
+## Usage
+
+### Basic Usage
+
+```python
+from markdowncleaner import MarkdownCleaner
+from pathlib import Path
+
+# Create a cleaner with default patterns
+cleaner = MarkdownCleaner()
+
+# Clean a markdown file
+result_path = cleaner.clean_markdown_file(Path("input.md"))
+
+# Clean a markdown string
+text = "# Title\nSome content here. [1]\n\nReferences\n1. Citation"
+cleaned_text = cleaner.clean_markdown_string(text)
+print(cleaned_text)
+```
+
+### Customizing Cleaning Options
+
+```python
+from markdowncleaner import MarkdownCleaner, CleanerOptions
+
+# Create custom options
+options = CleanerOptions()
+options.remove_short_lines = True
+options.min_line_length = 50  # custom minimum line length
+options.remove_footnotes_in_text = True
+options.contract_empty_lines = True
+
+# Initialize cleaner with custom options
+cleaner = MarkdownCleaner(options=options)
+
+# Use the cleaner as before
+```
+
+### Custom Cleaning Patterns
+
+You can also provide custom cleaning patterns:
+
+```python
+from markdowncleaner import MarkdownCleaner, CleaningPatterns
+from pathlib import Path
+
+# Load custom patterns from a YAML file
+custom_patterns = CleaningPatterns.from_yaml(Path("my_patterns.yaml"))
+
+# Initialize cleaner with custom patterns
+cleaner = MarkdownCleaner(patterns=custom_patterns)
+```
+
+## Configuration
+
+The default cleaning patterns are defined in `default_cleaning_patterns.yaml` and include:
+
+- **Sections to Remove**: Acknowledgements, References, Bibliography, etc.
+- **Bad Inline Patterns**: Citations, figure references, etc.
+- **Bad Lines Patterns**: Copyright notices, DOIs, URLs, etc.
+- **Footnote Patterns**: Footnote references in text that fit the pattern '.1'
+- **Replacements**: Various character replacements for PDF parsing errors
+
+## Options
+
+- `remove_short_lines`: Remove lines shorter than `min_line_length` (default: 70 characters)
+- `remove_whole_lines`: Remove lines matching specific patterns
+- `remove_sections`: Remove entire sections based on section headings
+- `remove_footnotes_in_text`: Remove footnote references
+- `replace_within_lines`: Replace specific patterns within lines
+- `remove_within_lines`: Remove specific patterns within lines
+- `contract_empty_lines`: Normalize whitespace
+- `crimp_linebreaks`: Improve line break formatting
+
+## License
+
+MIT License
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.

markdowncleaner-0.1.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+markdowncleaner/__init__.py,sha256=5P_nDL6T2WXoX2jVEtf-3fpGb-rEG1A_Bo-H0-upQFs,140
+markdowncleaner/cli.py,sha256=qpqEl3AL0ekQe9JoMljuEZQQlMRnEH9kFz7np7wF_64,4356
+markdowncleaner/markdowncleaner.py,sha256=wLWOBw_Oy1buw1Av7KvDDaN1V9w5gqvM27CMS9SzF3c,14150
+markdowncleaner/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+markdowncleaner/config/default_cleaning_patterns.yaml,sha256=VI2tDtgHelJbEMUVR9dUUlXqagfsIpS86_Zss-I6Qsg,6005
+markdowncleaner/config/loader.py,sha256=SX9ZgsGFV-25sFCeTy9XW5UQARz3EeaUhrVHu2Y-XeU,2927
+markdowncleaner-0.1.1.dist-info/LICENSE,sha256=yH2dxfGn0hRK5hPAnNFarbaFq1jx8wrPdqy-nsMi2II,1076
+markdowncleaner-0.1.1.dist-info/METADATA,sha256=8SXdYIt669JCe07MsNBG3zx8vr2-_zqt5yRthZO-duE,4296
+markdowncleaner-0.1.1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+markdowncleaner-0.1.1.dist-info/entry_points.txt,sha256=C_FJkZY4OeKfdTacBhqh9hzjZEk8gQJGzGSI8qLr-lU,61
+markdowncleaner-0.1.1.dist-info/top_level.txt,sha256=dMBEsk9DuO55cKIbeM4Ih2e0Y5aP6AfcKEYo2SoyFs0,16
+markdowncleaner-0.1.1.dist-info/RECORD,,

markdowncleaner-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

markdowncleaner-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+markdowncleaner = markdowncleaner.cli:main

markdowncleaner-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+markdowncleaner