pdf-to-markdown-cli 0.2.0__py3-none-any.whl

docs_to_md/__main__.py

@@ -0,0 +1,5 @@
+import sys
+from .main import main
+
+if __name__ == "__main__":
+    sys.exit(main()) 

docs_to_md/api/client.py

@@ -0,0 +1,218 @@
+import json
+import logging
+from pathlib import Path
+from typing import Dict, Optional, Any
+
+import backoff
+import filetype
+import requests
+from ratelimit import limits, sleep_and_retry
+
+from docs_to_md.api.models import MarkerStatus, StatusEnum, SubmitResponse, SUPPORTED_MIME_TYPES
+from docs_to_md.utils.exceptions import APIError
+from docs_to_md.utils.file_utils import FileIO
+
+# Constants
+MAX_REQUESTS_PER_MINUTE = 150
+REQUEST_TIMEOUT = 30  # seconds
+MAX_RETRIES = 3
+
+logger = logging.getLogger(__name__)
+
+
+class MarkerClient:
+    """Client for interacting with the Marker API."""
+
+    BASE_URL = "https://www.datalab.to/api/v1/marker"
+
+    def __init__(self, api_key: str):
+        """
+        Initialize the API client.
+        
+        Args:
+            api_key: API key for authentication
+            
+        Raises:
+            APIError: If API key is not provided or invalid
+        """
+        # Check for empty API key
+        if not api_key:
+            raise APIError("API key is required")
+            
+        # Basic validation - API keys should typically be alphanumeric
+        # and have a reasonable length. The exact format depends on Marker's specs.
+        api_key = api_key.strip()  # Remove accidental whitespace
+        if len(api_key) < 8:  # Assuming a minimum sensible length
+            raise APIError("API key appears to be too short")
+            
+        self.headers = {"X-Api-Key": api_key}
+
+    @sleep_and_retry
+    @limits(calls=MAX_REQUESTS_PER_MINUTE, period=60)
+    @backoff.on_exception(
+        backoff.expo,
+        (requests.exceptions.RequestException, json.JSONDecodeError),
+        max_tries=MAX_RETRIES
+    )
+    def submit_file(
+        self,
+        file_path: Path,
+        output_format: str = "markdown",
+        langs: str = "English",
+        use_llm: bool = False,
+        strip_existing_ocr: bool = False,
+        disable_image_extraction: bool = False,
+        force_ocr: bool = False,
+        paginate: bool = False,
+        max_pages: Optional[int] = None,
+    ) -> Optional[str]:
+        """
+        Submit a file for conversion.
+        
+        Args:
+            file_path: Path to the file to submit
+            output_format: Desired output format (markdown, json, html)
+            langs: Comma-separated OCR languages
+            use_llm: Whether to use LLM for enhanced processing
+            strip_existing_ocr: Whether to redo OCR processing
+            disable_image_extraction: Whether to disable image extraction
+            force_ocr: Whether to force OCR on all pages
+            paginate: Whether to add page delimiters
+            max_pages: Maximum pages to process
+            
+        Returns:
+            Request ID if successful, None otherwise
+            
+        Raises:
+            APIError: If file is invalid or API request fails
+        """
+        try:
+            # Validate file exists
+            if not file_path.exists():
+                raise APIError(f"File not found: {file_path}")
+
+            # Read file and check type
+            file_data = FileIO.read_file(file_path)
+            kind = filetype.guess(file_data)
+            if not kind or kind.mime not in SUPPORTED_MIME_TYPES:
+                raise APIError(f"Unsupported file type: {kind.mime if kind else 'unknown'}")
+
+            # Build form data
+            files = {
+                'file': (file_path.name, file_data, kind.mime),
+                'langs': (None, langs),
+                'force_ocr': (None, force_ocr),
+                'paginate': (None, paginate),
+                'strip_existing_ocr': (None, strip_existing_ocr),
+                'disable_image_extraction': (None, disable_image_extraction),
+                'use_llm': (None, use_llm),
+                'output_format': (None, output_format),
+            }
+            # Add max_pages only if it's provided
+            if max_pages is not None:
+                files['max_pages'] = (None, max_pages)
+
+            # Send request
+            response = requests.post(
+                self.BASE_URL,
+                files=files,
+                headers=self.headers,
+                timeout=REQUEST_TIMEOUT
+            )
+            response.raise_for_status()
+
+            # Parse response
+            data = response.json()
+            submit_response = SubmitResponse.model_validate(data)
+
+            if not submit_response.success:
+                logger.error(f"API request failed: {submit_response.error or 'Unknown error'}")
+                return None
+
+            logger.info(f"Successfully submitted file. Request ID: {submit_response.request_id}")
+            return submit_response.request_id
+
+        except Exception as e:
+            logger.error(f"Error submitting file {file_path}: {e}")
+            return None
+
+    @sleep_and_retry
+    @limits(calls=MAX_REQUESTS_PER_MINUTE, period=60)
+    @backoff.on_exception(
+        backoff.expo,
+        (requests.exceptions.RequestException, json.JSONDecodeError),
+        max_tries=MAX_RETRIES
+    )
+    def check_status(self, request_id: str) -> Optional[MarkerStatus]:
+        """
+        Check the status of a conversion request.
+        
+        Args:
+            request_id: Request ID to check
+            
+        Returns:
+            MarkerStatus object with current status, or None if request fails
+        """
+        if not request_id:
+            logger.error("Cannot check status: empty request_id provided")
+            return None
+            
+        try:
+            # Make API request with timeout
+            response = requests.get(
+                f"{self.BASE_URL}/{request_id}",
+                headers=self.headers,
+                timeout=REQUEST_TIMEOUT
+            )
+            
+            # Handle non-200 responses properly
+            if response.status_code != 200:
+                logger.error(f"API returned status code {response.status_code} for request {request_id}")
+                if response.status_code == 404:
+                    return MarkerStatus(status=StatusEnum.PROCESSING, error="Request not found")
+                elif response.status_code == 401:
+                    return MarkerStatus(status=StatusEnum.FAILED, error="Authentication failed")
+                elif response.status_code == 429:
+                    return MarkerStatus(status=StatusEnum.PROCESSING, error="Rate limit exceeded")
+                return None
+                
+            # Parse response JSON
+            try:
+                data = response.json()
+            except json.JSONDecodeError as e:
+                logger.error(f"Invalid JSON response for request {request_id}: {e}")
+                return None
+
+            # Handle empty response
+            if not data:
+                logger.error(f"Empty response for request {request_id}")
+                return None
+
+            # Validate and create status object
+            try:
+                status = MarkerStatus.model_validate(data)
+                return status
+            except Exception as e:
+                logger.error(f"Failed to parse status response for {request_id}: {e}")
+                return None
+
+        except requests.exceptions.Timeout:
+            logger.error(f"Timeout checking status for {request_id}")
+            return None
+        except requests.exceptions.ConnectionError:
+            logger.error(f"Connection error checking status for {request_id}")
+            return None
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Request error checking status for {request_id}: {e}")
+            return None
+        except Exception as e:
+            logger.error(f"Unexpected error checking status for {request_id}: {e}")
+            return None
+            
+    def __enter__(self):
+        """Support for context manager."""
+        return self
+        
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Clean up resources if needed."""
+        pass  # No cleanup needed for API client 

docs_to_md/api/models.py

@@ -0,0 +1,66 @@
+from enum import Enum
+from typing import Any, Dict, Optional, Set
+from dataclasses import dataclass
+
+from pydantic import BaseModel
+
+
+class StatusEnum(str, Enum):
+    """Enum for API status values."""
+    COMPLETE = "complete"
+    PROCESSING = "processing"
+    FAILED = "failed"
+
+
+class MarkerStatus(BaseModel):
+    """Model for API status response."""
+    status: StatusEnum  # Indicates the status of the request (`complete`, or `processing`).
+    output_format: Optional[str] = None  # The requested output format, `json`, `html`, or `markdown`.
+    success: Optional[bool] = None  # Indicates if the request completed successfully. `True` or `False`.
+    error: Optional[str] = None  # If there was an error, this contains the error message.
+    markdown: Optional[str] = None  # The output from the file if `output_format` is `markdown`.
+    json_data: Optional[Dict[str, Any]] = None  # The output from the file if `output_format` is `json`.
+    images: Optional[Dict[str, str]] = None  # Dictionary of image filenames (keys) and base64 encoded images (values).
+    meta: Optional[Dict[str, Any]] = None  # Metadata about the markdown conversion.
+    page_count: Optional[int] = None  # Number of pages that were converted.
+
+
+class SubmitResponse(BaseModel):
+    """Model for API submit response."""
+    success: bool
+    error: Optional[str] = None
+    request_id: str
+    request_check_url: Optional[str] = None
+
+
+@dataclass
+class ApiParams:
+    """Parameters for the Marker API submit call."""
+    output_format: str = "markdown"
+    langs: str = "English"
+    use_llm: bool = False
+    strip_existing_ocr: bool = False
+    disable_image_extraction: bool = False
+    force_ocr: bool = False
+    paginate: bool = False
+    max_pages: Optional[int] = None
+
+
+# Supported mime types according to API docs
+SUPPORTED_MIME_TYPES: Set[str] = {
+    # PDF
+    'application/pdf',
+    # Word documents
+    'application/msword',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    # Powerpoint
+    'application/vnd.ms-powerpoint',
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+    # Images
+    'image/png',
+    'image/jpeg',
+    'image/webp',
+    'image/gif',
+    'image/tiff',
+    'image/jpg'
+} 

docs_to_md/config/cli.py

@@ -0,0 +1,86 @@
+import argparse
+from pathlib import Path
+
+from docs_to_md.config.settings import Config
+from docs_to_md.utils.exceptions import ConfigurationError
+from docs_to_md.utils.file_utils import get_env_var
+
+
+def parse_args() -> argparse.Namespace:
+    """
+    Parse command line arguments.
+    
+    Returns:
+        Parsed arguments namespace
+    """
+    parser = argparse.ArgumentParser(
+        description="Process PDF files using Marker API.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    
+    # Required arguments
+    parser.add_argument("input", help="Input file or directory path")
+    
+    # Output format
+    parser.add_argument("--json", action="store_true", help="Output in JSON format")
+    
+    # OCR settings
+    parser.add_argument("--langs", default="English", help="Comma-separated OCR languages")
+    parser.add_argument("--llm", action="store_true", help="Use LLM for enhanced processing")
+    parser.add_argument("--strip", action="store_true", help="Redo OCR processing")
+    parser.add_argument("--noimg", action="store_true", help="Disable image extraction")
+    parser.add_argument("--force", action="store_true", help="Force OCR on all pages")
+    parser.add_argument("--pages", action="store_true", help="Add page delimiters")
+    parser.add_argument("--max-pages", type=int, help="Maximum number of pages to process from the start of the file")
+    
+    # Advanced settings
+    parser.add_argument("--max", action="store_true", help="Enable all OCR enhancements (LLM, strip OCR, force OCR)")
+    parser.add_argument("--no-chunk", action="store_true", help="Disable PDF chunking (sets chunk size to 1 million)")
+    parser.add_argument("-cs", "--chunk-size", type=int, help="Set PDF chunk size in pages", default=25)
+    parser.add_argument("--output-dir", help="Output directory", default="converted")
+    parser.add_argument("--cache-dir", help="Cache directory", default=".docs_to_md_cache")
+    
+    return parser.parse_args()
+
+
+def create_config_from_args() -> Config:
+    """
+    Create configuration from command line arguments.
+    
+    Returns:
+        Config object with settings from command line
+        
+    Raises:
+        ConfigurationError: If required arguments are missing
+    """
+    args = parse_args()
+    
+    # Get API key from environment
+    try:
+        api_key = get_env_var("MARKER_PDF_KEY")
+    except Exception as e:
+        raise ConfigurationError(f"API key not found: {e}. Set the MARKER_PDF_KEY environment variable.")
+        
+    # If --no-chunk is specified, override chunk size to effectively disable chunking
+    chunk_size = 1_000_000 if args.no_chunk else args.chunk_size
+    
+    config = Config(
+        api_key=api_key,
+        input_path=args.input,
+        output_dir=Path(args.output_dir),
+        cache_dir=Path(args.cache_dir),
+        output_format="json" if args.json else "markdown",
+        langs=args.langs,
+        use_llm=args.llm or args.max,
+        strip_existing_ocr=args.strip or args.max,
+        disable_image_extraction=args.noimg,
+        force_ocr=args.force or args.max,
+        paginate=args.pages,
+        chunk_size=chunk_size,
+        max_pages=args.max_pages
+    )
+    
+    # Validate the configuration
+    config.validate()
+    
+    return config 

docs_to_md/config/settings.py

@@ -0,0 +1,108 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional, List
+import logging
+import os
+
+from docs_to_md.utils.exceptions import ConfigurationError
+from docs_to_md.utils.file_utils import ensure_directory, get_env_var
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class Config:
+    """Global configuration for marker PDF conversion."""
+    # API settings
+    api_key: str
+    
+    # Input/output settings
+    input_path: str
+    output_dir: Path = Path("converted")
+    cache_dir: Path = Path(".docs_to_md_cache")
+    tmp_dir: Path = Path("chunks")
+    
+    # Conversion settings
+    output_format: str = "markdown"
+    langs: str = "English"
+    chunk_size: int = 25
+    
+    # Feature flags
+    use_llm: bool = False
+    strip_existing_ocr: bool = False
+    disable_image_extraction: bool = False
+    force_ocr: bool = False
+    paginate: bool = False
+    max_pages: Optional[int] = None
+    
+    @classmethod
+    def from_env(cls, api_key_var: str = "MARKER_PDF_KEY") -> "Config":
+        """
+        Create a configuration from environment variables.
+        
+        Args:
+            api_key_var: Environment variable name for API key
+            
+        Returns:
+            Config instance with settings from environment
+            
+        Raises:
+            ConfigurationError: If required environment variables are missing
+        """
+        try:
+            api_key = get_env_var(api_key_var)
+            
+            # This will raise an error if MARKER_INPUT_PATH is not set
+            input_path = get_env_var("MARKER_INPUT_PATH", required=False)
+            
+            if not input_path:
+                raise ConfigurationError("No input path specified. Use MARKER_INPUT_PATH environment variable.")
+                
+            return cls(
+                api_key=api_key,
+                input_path=input_path,
+                output_dir=Path(get_env_var("MARKER_OUTPUT_DIR", required=False) or "converted"),
+                cache_dir=Path(get_env_var("DOCS_TO_MD_CACHE_DIR", required=False) or ".docs_to_md_cache"),
+                output_format=get_env_var("MARKER_OUTPUT_FORMAT", required=False) or "markdown",
+                langs=get_env_var("MARKER_LANGS", required=False) or "English",
+                use_llm=get_env_var("MARKER_USE_LLM", required=False) == "1",
+                chunk_size=int(get_env_var("MARKER_CHUNK_SIZE", required=False) or "25")
+            )
+        except ValueError as e:
+            raise ConfigurationError(f"Invalid environment configuration: {e}")
+            
+    def ensure_directories(self) -> None:
+        """Ensure all required directories exist."""
+        ensure_directory(self.output_dir)
+        ensure_directory(self.cache_dir)
+        ensure_directory(self.tmp_dir)
+        
+    def validate(self) -> None:
+        """
+        Validate configuration values.
+        
+        Raises:
+            ConfigurationError: If configuration is invalid
+        """
+        if not self.api_key:
+            raise ConfigurationError("API key is required")
+            
+        if not self.input_path:
+            raise ConfigurationError("Input path is required")
+            
+        if self.chunk_size < 1:
+            raise ConfigurationError("Chunk size must be at least 1")
+            
+        if self.max_pages is not None and self.max_pages < 1:
+            raise ConfigurationError("Max pages must be at least 1")
+            
+        if self.output_format not in ["markdown", "json", "html", "txt"]:
+            raise ConfigurationError(f"Unsupported output format: {self.output_format}")
+            
+        # Additional validations can be added here 
+
+        if not self.output_dir.exists():
+            try:
+                ensure_directory(self.output_dir)
+                logger.info(f"Created output directory: {self.output_dir}")
+            except Exception as e:
+                raise ConfigurationError(f"Failed to create output directory: {e}")