markdowncleaner 0.1.1__tar.gz

markdowncleaner-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Johannes Himmelreich
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

markdowncleaner-0.1.1/PKG-INFO

@@ -0,0 +1,132 @@
+Metadata-Version: 2.2
+Name: markdowncleaner
+Version: 0.1.1
+Summary: A tool for cleaning and formatting markdown documents
+Author-email: Johannes Himmelreich <jrhimmel@syr.edu>
+License: MIT
+Project-URL: Repository, https://github.com/josk0/markdowncleaner
+Project-URL: Issues, https://github.com/josk0/markdowncleaner/issues
+Keywords: markdown,cleaning,formatting,text processing
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Utilities
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: ftfy>=6.0.3
+
+# markdowncleaner
+
+A simple Python tool for cleaning and formatting markdown documents. Default configuration with regex patterns for PDFs of academic papers that have been converted to markdown.
+
+## Description
+
+`markdowncleaner` helps you clean up markdown files by removing unwanted content such as:
+- References, bibliographies, and citations
+- Footnotes and endnotes
+- Copyright notices and legal disclaimers
+- Acknowledgements and funding information
+- Author information and contact details
+- Specific patterns like DOIs, URLs, and email addresses
+- Short lines and excessive whitespace
+
+This tool is particularly useful for processing academic papers, books, or any markdown document that needs formatting cleanup.
+
+## Installation
+
+```bash
+pip install markdowncleaner
+```
+
+## Usage
+
+### Basic Usage
+
+```python
+from markdowncleaner import MarkdownCleaner
+from pathlib import Path
+
+# Create a cleaner with default patterns
+cleaner = MarkdownCleaner()
+
+# Clean a markdown file
+result_path = cleaner.clean_markdown_file(Path("input.md"))
+
+# Clean a markdown string
+text = "# Title\nSome content here. [1]\n\nReferences\n1. Citation"
+cleaned_text = cleaner.clean_markdown_string(text)
+print(cleaned_text)
+```
+
+### Customizing Cleaning Options
+
+```python
+from markdowncleaner import MarkdownCleaner, CleanerOptions
+
+# Create custom options
+options = CleanerOptions()
+options.remove_short_lines = True
+options.min_line_length = 50  # custom minimum line length
+options.remove_footnotes_in_text = True
+options.contract_empty_lines = True
+
+# Initialize cleaner with custom options
+cleaner = MarkdownCleaner(options=options)
+
+# Use the cleaner as before
+```
+
+### Custom Cleaning Patterns
+
+You can also provide custom cleaning patterns:
+
+```python
+from markdowncleaner import MarkdownCleaner, CleaningPatterns
+from pathlib import Path
+
+# Load custom patterns from a YAML file
+custom_patterns = CleaningPatterns.from_yaml(Path("my_patterns.yaml"))
+
+# Initialize cleaner with custom patterns
+cleaner = MarkdownCleaner(patterns=custom_patterns)
+```
+
+## Configuration
+
+The default cleaning patterns are defined in `default_cleaning_patterns.yaml` and include:
+
+- **Sections to Remove**: Acknowledgements, References, Bibliography, etc.
+- **Bad Inline Patterns**: Citations, figure references, etc.
+- **Bad Lines Patterns**: Copyright notices, DOIs, URLs, etc.
+- **Footnote Patterns**: Footnote references in text that fit the pattern '.1'
+- **Replacements**: Various character replacements for PDF parsing errors
+
+## Options
+
+- `remove_short_lines`: Remove lines shorter than `min_line_length` (default: 70 characters)
+- `remove_whole_lines`: Remove lines matching specific patterns
+- `remove_sections`: Remove entire sections based on section headings
+- `remove_footnotes_in_text`: Remove footnote references
+- `replace_within_lines`: Replace specific patterns within lines
+- `remove_within_lines`: Remove specific patterns within lines
+- `contract_empty_lines`: Normalize whitespace
+- `crimp_linebreaks`: Improve line break formatting
+
+## License
+
+MIT License
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.

markdowncleaner-0.1.1/README.md

@@ -0,0 +1,104 @@
+# markdowncleaner
+
+A simple Python tool for cleaning and formatting markdown documents. Default configuration with regex patterns for PDFs of academic papers that have been converted to markdown.
+
+## Description
+
+`markdowncleaner` helps you clean up markdown files by removing unwanted content such as:
+- References, bibliographies, and citations
+- Footnotes and endnotes
+- Copyright notices and legal disclaimers
+- Acknowledgements and funding information
+- Author information and contact details
+- Specific patterns like DOIs, URLs, and email addresses
+- Short lines and excessive whitespace
+
+This tool is particularly useful for processing academic papers, books, or any markdown document that needs formatting cleanup.
+
+## Installation
+
+```bash
+pip install markdowncleaner
+```
+
+## Usage
+
+### Basic Usage
+
+```python
+from markdowncleaner import MarkdownCleaner
+from pathlib import Path
+
+# Create a cleaner with default patterns
+cleaner = MarkdownCleaner()
+
+# Clean a markdown file
+result_path = cleaner.clean_markdown_file(Path("input.md"))
+
+# Clean a markdown string
+text = "# Title\nSome content here. [1]\n\nReferences\n1. Citation"
+cleaned_text = cleaner.clean_markdown_string(text)
+print(cleaned_text)
+```
+
+### Customizing Cleaning Options
+
+```python
+from markdowncleaner import MarkdownCleaner, CleanerOptions
+
+# Create custom options
+options = CleanerOptions()
+options.remove_short_lines = True
+options.min_line_length = 50  # custom minimum line length
+options.remove_footnotes_in_text = True
+options.contract_empty_lines = True
+
+# Initialize cleaner with custom options
+cleaner = MarkdownCleaner(options=options)
+
+# Use the cleaner as before
+```
+
+### Custom Cleaning Patterns
+
+You can also provide custom cleaning patterns:
+
+```python
+from markdowncleaner import MarkdownCleaner, CleaningPatterns
+from pathlib import Path
+
+# Load custom patterns from a YAML file
+custom_patterns = CleaningPatterns.from_yaml(Path("my_patterns.yaml"))
+
+# Initialize cleaner with custom patterns
+cleaner = MarkdownCleaner(patterns=custom_patterns)
+```
+
+## Configuration
+
+The default cleaning patterns are defined in `default_cleaning_patterns.yaml` and include:
+
+- **Sections to Remove**: Acknowledgements, References, Bibliography, etc.
+- **Bad Inline Patterns**: Citations, figure references, etc.
+- **Bad Lines Patterns**: Copyright notices, DOIs, URLs, etc.
+- **Footnote Patterns**: Footnote references in text that fit the pattern '.1'
+- **Replacements**: Various character replacements for PDF parsing errors
+
+## Options
+
+- `remove_short_lines`: Remove lines shorter than `min_line_length` (default: 70 characters)
+- `remove_whole_lines`: Remove lines matching specific patterns
+- `remove_sections`: Remove entire sections based on section headings
+- `remove_footnotes_in_text`: Remove footnote references
+- `replace_within_lines`: Replace specific patterns within lines
+- `remove_within_lines`: Remove specific patterns within lines
+- `contract_empty_lines`: Normalize whitespace
+- `crimp_linebreaks`: Improve line break formatting
+
+## License
+
+MIT License
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.

markdowncleaner-0.1.1/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "markdowncleaner"
+version = "0.1.1"
+description = "A tool for cleaning and formatting markdown documents"
+readme = "README.md"
+authors = [
+    {name = "Johannes Himmelreich", email = "jrhimmel@syr.edu"}
+]
+license = {text = "MIT"}
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Topic :: Text Processing :: Markup :: Markdown",
+    "Topic :: Utilities",
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+]
+keywords = ["markdown", "cleaning", "formatting", "text processing"]
+dependencies = [
+    "pyyaml>=6.0",
+    "ftfy>=6.0.3",
+]
+requires-python = ">=3.8"
+
+[project.urls]
+"Repository" = "https://github.com/josk0/markdowncleaner"
+"Issues" = "https://github.com/josk0/markdowncleaner/issues"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+markdowncleaner = ["config/*.yaml"]
+
+[project.scripts]
+markdowncleaner = "markdowncleaner.cli:main"

markdowncleaner-0.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

markdowncleaner-0.1.1/src/markdowncleaner/__init__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+__version__ = version("markdowncleaner")
+from .markdowncleaner import CleanerOptions, MarkdownCleaner

markdowncleaner-0.1.1/src/markdowncleaner/cli.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""
+Command-line interface for the markdowncleaner package.
+"""
+
+import argparse
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from markdowncleaner.markdowncleaner import MarkdownCleaner, CleanerOptions
+from markdowncleaner.config.loader import get_default_patterns, CleaningPatterns
+
+
+def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Clean and format markdown files by removing unwanted sections, lines, and patterns."
+    )
+
+    parser.add_argument(
+        "input_file",
+        type=Path,
+        help="Path to the markdown file to clean"
+    )
+
+    parser.add_argument(
+        "-o", "--output",
+        type=Path,
+        help="Path to save the cleaned markdown file (default: input_file with '_cleaned' suffix)"
+    )
+
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        help="Directory to save the cleaned file (default: same as input file)"
+    )
+
+    parser.add_argument(
+        "--config",
+        type=Path,
+        help="Path to custom YAML configuration file with cleaning patterns"
+    )
+
+    # Options for customizing the cleaning process
+    parser.add_argument(
+        "--keep-short-lines",
+        action="store_true",
+        help="Don't remove lines shorter than minimum length"
+    )
+
+    parser.add_argument(
+        "--min-line-length",
+        type=int,
+        default=70,
+        help="Minimum line length to keep (default: 70)"
+    )
+
+    parser.add_argument(
+        "--keep-bad-lines",
+        action="store_true",
+        help="Don't remove lines matching bad line patterns"
+    )
+
+    parser.add_argument(
+        "--keep-sections",
+        action="store_true",
+        help="Don't remove sections like References, Acknowledgements, etc."
+    )
+
+    parser.add_argument(
+        "--keep-footnotes",
+        action="store_true",
+        help="Don't remove footnote references"
+    )
+
+    parser.add_argument(
+        "--no-replacements",
+        action="store_true",
+        help="Don't perform text replacements"
+    )
+
+    parser.add_argument(
+        "--keep-inline-patterns",
+        action="store_true",
+        help="Don't remove inline patterns like citations"
+    )
+
+    parser.add_argument(
+        "--keep-empty-lines",
+        action="store_true",
+        help="Don't contract consecutive empty lines"
+    )
+
+    parser.add_argument(
+        "--no-crimp-linebreaks",
+        action="store_true",
+        help="Don't crimp linebreaks"
+    )
+
+    return parser.parse_args(args)
+
+
+def main(args: Optional[List[str]] = None) -> int:
+    """Main entry point for the CLI."""
+    parsed_args = parse_args(args)
+
+    # Configure cleaner options based on arguments
+    options = CleanerOptions()
+    options.remove_short_lines = not parsed_args.keep_short_lines
+    options.min_line_length = parsed_args.min_line_length
+    options.remove_whole_lines = not parsed_args.keep_bad_lines
+    options.remove_sections = not parsed_args.keep_sections
+    options.remove_footnotes_in_text = not parsed_args.keep_footnotes
+    options.replace_within_lines = not parsed_args.no_replacements
+    options.remove_within_lines = not parsed_args.keep_inline_patterns
+    options.contract_empty_lines = not parsed_args.keep_empty_lines
+    options.crimp_linebreaks = not parsed_args.no_crimp_linebreaks
+
+    # Load patterns from custom config or use defaults
+    if parsed_args.config:
+        try:
+            patterns = CleaningPatterns.from_yaml(parsed_args.config)
+        except Exception as e:
+            print(f"Error loading custom configuration: {e}", file=sys.stderr)
+            return 1
+    else:
+        patterns = get_default_patterns()
+
+    # Initialize the cleaner with patterns and configured options
+    cleaner = MarkdownCleaner(patterns=patterns, options=options)
+
+    try:
+        # Clean the markdown file
+        output_file = cleaner.clean_markdown_file(
+            input_file=parsed_args.input_file,
+            output_path=parsed_args.output_dir,
+            output_file=parsed_args.output
+        )
+        print(f"Cleaned markdown saved to: {output_file}")
+        return 0
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())