pdf-to-markdown-cli 0.2.0__tar.gz

pdf_to_markdown_cli-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 SokolskyNikita
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pdf_to_markdown_cli-0.2.0/MANIFEST.in

@@ -0,0 +1,24 @@
+# Include the README and LICENSE
+include README.md
+include LICENSE
+
+# Include the package source directory
+recursive-include docs_to_md *
+
+# Exclude development/cache/temporary files and directories
+global-exclude *.py[cod]
+global-exclude __pycache__
+global-exclude .DS_Store
+prune .git
+prune .vscode
+prune .marker_cache
+prune pdfs
+prune converted
+prune examples
+prune chunks
+prune __pycache__
+
+# Exclude files not needed for the installed package
+exclude marker_api_docs.md
+exclude requirements.txt
+# exclude setup.py  # This MUST be included in the sdist for building 

pdf_to_markdown_cli-0.2.0/PKG-INFO

@@ -0,0 +1,179 @@
+Metadata-Version: 2.4
+Name: pdf-to-markdown-cli
+Version: 0.2.0
+Summary: CLI tool to convert PDF files (and other documents) to markdown using the Marker API.
+Home-page: https://github.com/SokolskyNikita/pdf-to-markdown-cli
+Author: Nikita Sokolsky
+Project-URL: Bug Tracker, https://github.com/SokolskyNikita/pdf-to-markdown-cli/issues
+Keywords: pdf,markdown,converter,cli,document,marker,md
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: backoff>=2.0
+Requires-Dist: diskcache>=5.0
+Requires-Dist: filetype>=1.0
+Requires-Dist: pikepdf>=8.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: ratelimit>=2.0
+Requires-Dist: requests>=2.0
+Requires-Dist: tqdm>=4.0
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# PDF to Markdown CLI (using Marker API)
+
+Convert PDF files (and other documents) to markdown using the [Marker API](https://www.marker.io/) via a command-line tool.
+
+## Overview
+
+This package provides a convenient command-line interface (`pdf-to-md`) for converting PDF files (and other document formats like Word, EPUB, images, etc.) to markdown. It leverages the Marker API for high-quality PDF conversion and handles chunking, parallel processing, and result combination.
+
+## Features
+
+- Convert PDFs, Word documents, PowerPoint files, spreadsheets, epub, HTML, and images to markdown
+- Optionally use the Marker API for enhanced PDF conversion
+- Handle large documents by splitting them into chunks
+- Progress tracking for long-running operations
+- Customizable OCR options
+- Local caching of results
+- Output in markdown, JSON, or HTML format
+
+## Installation
+
+### From PyPI
+
+```bash
+pip install pdf-to-markdown-cli
+```
+
+### From source
+
+```bash
+git clone https://github.com/SokolskyNikita/pdf-to-markdown-cli.git 
+cd pdf-to-markdown-cli
+pip install -e .
+```
+
+## Usage
+
+### Command-line interface
+
+```bash
+# Set your Marker API key
+export MARKER_PDF_KEY=your_api_key_here
+
+# Basic usage
+pdf-to-md /path/to/file.pdf
+
+# Process all files in a directory
+pdf-to-md /path/to/directory
+
+# Output in JSON format
+pdf-to-md /path/to/file.pdf --json
+
+# Use additional languages for OCR
+pdf-to-md /path/to/file.pdf --langs "English,French,German"
+
+# Use all Marker OCR enhancements
+pdf-to-md /path/to/file.pdf --max
+```
+
+### API
+
+```python
+# Internal module paths remain the same
+from docs_to_md.config.settings import Config
+from docs_to_md.core.processor import MarkerProcessor
+
+# Create configuration
+config = Config(
+    api_key="your_api_key_here",
+    input_path="/path/to/file.pdf",
+    output_format="markdown",
+    use_llm=True
+)
+
+# Create processor and run
+processor = MarkerProcessor(config)
+processor.process()
+```
+
+## Command-line Options
+
+- `input`: Input file or directory path
+- `--json`: Output in JSON format (default is markdown)
+- `--langs`: Comma-separated OCR languages (default: "English")
+- `--llm`: Use LLM for enhanced processing
+- `--strip`: Redo OCR processing
+- `--noimg`: Disable image extraction
+- `--force`: Force OCR on all pages
+- `--pages`: Add page delimiters
+- `--max`: Enable all OCR enhancements (equivalent to --llm --strip --force)
+- `--max-pages`: Maximum number of pages to process from the start of the file
+- `--no-chunk`: Disable PDF chunking
+- `--chunk-size`: Set PDF chunk size in pages (default: 25)
+- `--output-dir`: Output directory (default: "converted")
+- `--cache-dir`: Cache directory (default: ".marker_cache")
+
+## Project Structure
+
+The package is organized as follows:
+
+```
+pdf-to-markdown-cli/ (Project Root)
+├── docs_to_md/      (Python Package Source)
+│   ├── api/
+│   ├── config/
+│   ├── core/
+│   ├── pdf/
+│   ├── storage/
+│   └── utils/
+├── setup.py
+├── README.md
+└── ... (other config files)
+```
+
+## Requirements
+
+- Python 3.10 or higher
+- Runtime dependencies are listed in `setup.py` and automatically installed via pip.
+- The `requirements.txt` file lists exact versions used for development and testing, and can be used to set up a development environment: `pip install -r requirements.txt`
+
+## Development
+
+To run the code directly from the source tree without installation:
+
+```bash
+# Using the installed entry point (requires editable install):
+# pip install -e .
+# pdf-to-md /path/to/file.pdf
+
+# Or running the module directly:
+python -m docs_to_md /path/to/file.pdf 
+```
+
+For regular use after installation, use the `pdf-to-md` command.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

pdf_to_markdown_cli-0.2.0/README.md

@@ -0,0 +1,137 @@
+# PDF to Markdown CLI (using Marker API)
+
+Convert PDF files (and other documents) to markdown using the [Marker API](https://www.marker.io/) via a command-line tool.
+
+## Overview
+
+This package provides a convenient command-line interface (`pdf-to-md`) for converting PDF files (and other document formats like Word, EPUB, images, etc.) to markdown. It leverages the Marker API for high-quality PDF conversion and handles chunking, parallel processing, and result combination.
+
+## Features
+
+- Convert PDFs, Word documents, PowerPoint files, spreadsheets, epub, HTML, and images to markdown
+- Optionally use the Marker API for enhanced PDF conversion
+- Handle large documents by splitting them into chunks
+- Progress tracking for long-running operations
+- Customizable OCR options
+- Local caching of results
+- Output in markdown, JSON, or HTML format
+
+## Installation
+
+### From PyPI
+
+```bash
+pip install pdf-to-markdown-cli
+```
+
+### From source
+
+```bash
+git clone https://github.com/SokolskyNikita/pdf-to-markdown-cli.git 
+cd pdf-to-markdown-cli
+pip install -e .
+```
+
+## Usage
+
+### Command-line interface
+
+```bash
+# Set your Marker API key
+export MARKER_PDF_KEY=your_api_key_here
+
+# Basic usage
+pdf-to-md /path/to/file.pdf
+
+# Process all files in a directory
+pdf-to-md /path/to/directory
+
+# Output in JSON format
+pdf-to-md /path/to/file.pdf --json
+
+# Use additional languages for OCR
+pdf-to-md /path/to/file.pdf --langs "English,French,German"
+
+# Use all Marker OCR enhancements
+pdf-to-md /path/to/file.pdf --max
+```
+
+### API
+
+```python
+# Internal module paths remain the same
+from docs_to_md.config.settings import Config
+from docs_to_md.core.processor import MarkerProcessor
+
+# Create configuration
+config = Config(
+    api_key="your_api_key_here",
+    input_path="/path/to/file.pdf",
+    output_format="markdown",
+    use_llm=True
+)
+
+# Create processor and run
+processor = MarkerProcessor(config)
+processor.process()
+```
+
+## Command-line Options
+
+- `input`: Input file or directory path
+- `--json`: Output in JSON format (default is markdown)
+- `--langs`: Comma-separated OCR languages (default: "English")
+- `--llm`: Use LLM for enhanced processing
+- `--strip`: Redo OCR processing
+- `--noimg`: Disable image extraction
+- `--force`: Force OCR on all pages
+- `--pages`: Add page delimiters
+- `--max`: Enable all OCR enhancements (equivalent to --llm --strip --force)
+- `--max-pages`: Maximum number of pages to process from the start of the file
+- `--no-chunk`: Disable PDF chunking
+- `--chunk-size`: Set PDF chunk size in pages (default: 25)
+- `--output-dir`: Output directory (default: "converted")
+- `--cache-dir`: Cache directory (default: ".marker_cache")
+
+## Project Structure
+
+The package is organized as follows:
+
+```
+pdf-to-markdown-cli/ (Project Root)
+├── docs_to_md/      (Python Package Source)
+│   ├── api/
+│   ├── config/
+│   ├── core/
+│   ├── pdf/
+│   ├── storage/
+│   └── utils/
+├── setup.py
+├── README.md
+└── ... (other config files)
+```
+
+## Requirements
+
+- Python 3.10 or higher
+- Runtime dependencies are listed in `setup.py` and automatically installed via pip.
+- The `requirements.txt` file lists exact versions used for development and testing, and can be used to set up a development environment: `pip install -r requirements.txt`
+
+## Development
+
+To run the code directly from the source tree without installation:
+
+```bash
+# Using the installed entry point (requires editable install):
+# pip install -e .
+# pdf-to-md /path/to/file.pdf
+
+# Or running the module directly:
+python -m docs_to_md /path/to/file.pdf 
+```
+
+For regular use after installation, use the `pdf-to-md` command.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

pdf_to_markdown_cli-0.2.0/docs_to_md/__main__.py

@@ -0,0 +1,5 @@
+import sys
+from .main import main
+
+if __name__ == "__main__":
+    sys.exit(main()) 

pdf_to_markdown_cli-0.2.0/docs_to_md/api/client.py

@@ -0,0 +1,218 @@
+import json
+import logging
+from pathlib import Path
+from typing import Dict, Optional, Any
+
+import backoff
+import filetype
+import requests
+from ratelimit import limits, sleep_and_retry
+
+from docs_to_md.api.models import MarkerStatus, StatusEnum, SubmitResponse, SUPPORTED_MIME_TYPES
+from docs_to_md.utils.exceptions import APIError
+from docs_to_md.utils.file_utils import FileIO
+
+# Constants
+MAX_REQUESTS_PER_MINUTE = 150
+REQUEST_TIMEOUT = 30  # seconds
+MAX_RETRIES = 3
+
+logger = logging.getLogger(__name__)
+
+
+class MarkerClient:
+    """Client for interacting with the Marker API."""
+
+    BASE_URL = "https://www.datalab.to/api/v1/marker"
+
+    def __init__(self, api_key: str):
+        """
+        Initialize the API client.
+        
+        Args:
+            api_key: API key for authentication
+            
+        Raises:
+            APIError: If API key is not provided or invalid
+        """
+        # Check for empty API key
+        if not api_key:
+            raise APIError("API key is required")
+            
+        # Basic validation - API keys should typically be alphanumeric
+        # and have a reasonable length. The exact format depends on Marker's specs.
+        api_key = api_key.strip()  # Remove accidental whitespace
+        if len(api_key) < 8:  # Assuming a minimum sensible length
+            raise APIError("API key appears to be too short")
+            
+        self.headers = {"X-Api-Key": api_key}
+
+    @sleep_and_retry
+    @limits(calls=MAX_REQUESTS_PER_MINUTE, period=60)
+    @backoff.on_exception(
+        backoff.expo,
+        (requests.exceptions.RequestException, json.JSONDecodeError),
+        max_tries=MAX_RETRIES
+    )
+    def submit_file(
+        self,
+        file_path: Path,
+        output_format: str = "markdown",
+        langs: str = "English",
+        use_llm: bool = False,
+        strip_existing_ocr: bool = False,
+        disable_image_extraction: bool = False,
+        force_ocr: bool = False,
+        paginate: bool = False,
+        max_pages: Optional[int] = None,
+    ) -> Optional[str]:
+        """
+        Submit a file for conversion.
+        
+        Args:
+            file_path: Path to the file to submit
+            output_format: Desired output format (markdown, json, html)
+            langs: Comma-separated OCR languages
+            use_llm: Whether to use LLM for enhanced processing
+            strip_existing_ocr: Whether to redo OCR processing
+            disable_image_extraction: Whether to disable image extraction
+            force_ocr: Whether to force OCR on all pages
+            paginate: Whether to add page delimiters
+            max_pages: Maximum pages to process
+            
+        Returns:
+            Request ID if successful, None otherwise
+            
+        Raises:
+            APIError: If file is invalid or API request fails
+        """
+        try:
+            # Validate file exists
+            if not file_path.exists():
+                raise APIError(f"File not found: {file_path}")
+
+            # Read file and check type
+            file_data = FileIO.read_file(file_path)
+            kind = filetype.guess(file_data)
+            if not kind or kind.mime not in SUPPORTED_MIME_TYPES:
+                raise APIError(f"Unsupported file type: {kind.mime if kind else 'unknown'}")
+
+            # Build form data
+            files = {
+                'file': (file_path.name, file_data, kind.mime),
+                'langs': (None, langs),
+                'force_ocr': (None, force_ocr),
+                'paginate': (None, paginate),
+                'strip_existing_ocr': (None, strip_existing_ocr),
+                'disable_image_extraction': (None, disable_image_extraction),
+                'use_llm': (None, use_llm),
+                'output_format': (None, output_format),
+            }
+            # Add max_pages only if it's provided
+            if max_pages is not None:
+                files['max_pages'] = (None, max_pages)
+
+            # Send request
+            response = requests.post(
+                self.BASE_URL,
+                files=files,
+                headers=self.headers,
+                timeout=REQUEST_TIMEOUT
+            )
+            response.raise_for_status()
+
+            # Parse response
+            data = response.json()
+            submit_response = SubmitResponse.model_validate(data)
+
+            if not submit_response.success:
+                logger.error(f"API request failed: {submit_response.error or 'Unknown error'}")
+                return None
+
+            logger.info(f"Successfully submitted file. Request ID: {submit_response.request_id}")
+            return submit_response.request_id
+
+        except Exception as e:
+            logger.error(f"Error submitting file {file_path}: {e}")
+            return None
+
+    @sleep_and_retry
+    @limits(calls=MAX_REQUESTS_PER_MINUTE, period=60)
+    @backoff.on_exception(
+        backoff.expo,
+        (requests.exceptions.RequestException, json.JSONDecodeError),
+        max_tries=MAX_RETRIES
+    )
+    def check_status(self, request_id: str) -> Optional[MarkerStatus]:
+        """
+        Check the status of a conversion request.
+        
+        Args:
+            request_id: Request ID to check
+            
+        Returns:
+            MarkerStatus object with current status, or None if request fails
+        """
+        if not request_id:
+            logger.error("Cannot check status: empty request_id provided")
+            return None
+            
+        try:
+            # Make API request with timeout
+            response = requests.get(
+                f"{self.BASE_URL}/{request_id}",
+                headers=self.headers,
+                timeout=REQUEST_TIMEOUT
+            )
+            
+            # Handle non-200 responses properly
+            if response.status_code != 200:
+                logger.error(f"API returned status code {response.status_code} for request {request_id}")
+                if response.status_code == 404:
+                    return MarkerStatus(status=StatusEnum.PROCESSING, error="Request not found")
+                elif response.status_code == 401:
+                    return MarkerStatus(status=StatusEnum.FAILED, error="Authentication failed")
+                elif response.status_code == 429:
+                    return MarkerStatus(status=StatusEnum.PROCESSING, error="Rate limit exceeded")
+                return None
+                
+            # Parse response JSON
+            try:
+                data = response.json()
+            except json.JSONDecodeError as e:
+                logger.error(f"Invalid JSON response for request {request_id}: {e}")
+                return None
+
+            # Handle empty response
+            if not data:
+                logger.error(f"Empty response for request {request_id}")
+                return None
+
+            # Validate and create status object
+            try:
+                status = MarkerStatus.model_validate(data)
+                return status
+            except Exception as e:
+                logger.error(f"Failed to parse status response for {request_id}: {e}")
+                return None
+
+        except requests.exceptions.Timeout:
+            logger.error(f"Timeout checking status for {request_id}")
+            return None
+        except requests.exceptions.ConnectionError:
+            logger.error(f"Connection error checking status for {request_id}")
+            return None
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Request error checking status for {request_id}: {e}")
+            return None
+        except Exception as e:
+            logger.error(f"Unexpected error checking status for {request_id}: {e}")
+            return None
+            
+    def __enter__(self):
+        """Support for context manager."""
+        return self
+        
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Clean up resources if needed."""
+        pass  # No cleanup needed for API client 

pdf_to_markdown_cli-0.2.0/docs_to_md/api/models.py

@@ -0,0 +1,66 @@
+from enum import Enum
+from typing import Any, Dict, Optional, Set
+from dataclasses import dataclass
+
+from pydantic import BaseModel
+
+
+class StatusEnum(str, Enum):
+    """Enum for API status values."""
+    COMPLETE = "complete"
+    PROCESSING = "processing"
+    FAILED = "failed"
+
+
+class MarkerStatus(BaseModel):
+    """Model for API status response."""
+    status: StatusEnum  # Indicates the status of the request (`complete`, or `processing`).
+    output_format: Optional[str] = None  # The requested output format, `json`, `html`, or `markdown`.
+    success: Optional[bool] = None  # Indicates if the request completed successfully. `True` or `False`.
+    error: Optional[str] = None  # If there was an error, this contains the error message.
+    markdown: Optional[str] = None  # The output from the file if `output_format` is `markdown`.
+    json_data: Optional[Dict[str, Any]] = None  # The output from the file if `output_format` is `json`.
+    images: Optional[Dict[str, str]] = None  # Dictionary of image filenames (keys) and base64 encoded images (values).
+    meta: Optional[Dict[str, Any]] = None  # Metadata about the markdown conversion.
+    page_count: Optional[int] = None  # Number of pages that were converted.
+
+
+class SubmitResponse(BaseModel):
+    """Model for API submit response."""
+    success: bool
+    error: Optional[str] = None
+    request_id: str
+    request_check_url: Optional[str] = None
+
+
+@dataclass
+class ApiParams:
+    """Parameters for the Marker API submit call."""
+    output_format: str = "markdown"
+    langs: str = "English"
+    use_llm: bool = False
+    strip_existing_ocr: bool = False
+    disable_image_extraction: bool = False
+    force_ocr: bool = False
+    paginate: bool = False
+    max_pages: Optional[int] = None
+
+
+# Supported mime types according to API docs
+SUPPORTED_MIME_TYPES: Set[str] = {
+    # PDF
+    'application/pdf',
+    # Word documents
+    'application/msword',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    # Powerpoint
+    'application/vnd.ms-powerpoint',
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+    # Images
+    'image/png',
+    'image/jpeg',
+    'image/webp',
+    'image/gif',
+    'image/tiff',
+    'image/jpg'
+}