mdify-cli 2.7.0__py3-none-any.whl

mdify/container.py

@@ -0,0 +1,167 @@
+"""Container lifecycle management for docling-serve."""
+
+import subprocess
+import time
+import uuid
+from typing import Optional
+
+from mdify.docling_client import check_health
+
+
+class DoclingContainer:
+    """Manages docling-serve container lifecycle.
+
+    Provides context manager support for automatic startup and cleanup.
+
+    Usage:
+        with DoclingContainer("docker", "ghcr.io/docling-project/docling-serve-cpu:main") as container:
+            # Container is running and healthy
+            response = requests.post(f"{container.base_url}/v1/convert/file", ...)
+        # Container automatically stopped and removed
+    """
+
+    def __init__(self, runtime: str, image: str, port: int = 5001, timeout: int = 1200):
+        """Initialize container manager.
+
+        Args:
+            runtime: Container runtime ("docker" or "podman")
+            image: Container image to use
+            port: Host port to bind (default: 5001)
+            timeout: Conversion timeout in seconds (default: 1200)
+        """
+        self.runtime = runtime
+        self.image = image
+        self.port = port
+        self.timeout = timeout
+        self.container_name = f"mdify-serve-{uuid.uuid4().hex[:8]}"
+        self.container_id: Optional[str] = None
+
+    @property
+    def base_url(self) -> str:
+        """Return base URL for API requests."""
+        return f"http://localhost:{self.port}"
+
+    def _cleanup_stale_containers(self) -> None:
+        """Stop any existing mdify-serve containers.
+
+        This handles the case where a previous run left a container running
+        (e.g., due to crash, interrupt, or timeout).
+        """
+        # Find running containers matching mdify-serve-* pattern
+        result = subprocess.run(
+            [
+                self.runtime,
+                "ps",
+                "--filter",
+                "name=mdify-serve-",
+                "--format",
+                "{{.Names}}",
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+        if result.returncode != 0 or not result.stdout.strip():
+            return
+
+        # Stop each stale container
+        for container_name in result.stdout.strip().split("\n"):
+            if container_name:
+                subprocess.run(
+                    [self.runtime, "stop", container_name],
+                    capture_output=True,
+                    check=False,
+                )
+
+    def start(self, timeout: int = 120) -> None:
+        """Start container and wait for health check.
+
+        Args:
+            timeout: Maximum seconds to wait for health (default: 120)
+
+        Raises:
+            subprocess.CalledProcessError: If container fails to start
+            TimeoutError: If health check doesn't pass within timeout
+        """
+        self._cleanup_stale_containers()
+
+        # Start container in detached mode
+        cmd = [
+            self.runtime,
+            "run",
+            "-d",  # Detached mode
+            "--rm",  # Auto-remove on stop
+            "--name",
+            self.container_name,
+            "-p",
+            f"{self.port}:5001",
+            "-e",
+            f"DOCLING_SERVE_MAX_SYNC_WAIT={self.timeout}",
+            self.image,
+        ]
+
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+            self.container_id = result.stdout.strip()
+        except subprocess.CalledProcessError as e:
+            error_msg = e.stderr.strip() or e.stdout.strip() or "Unknown error"
+            raise subprocess.CalledProcessError(
+                e.returncode,
+                e.cmd,
+                output=e.stdout,
+                stderr=f"Failed to start container: {error_msg}",
+            )
+
+        # Wait for health check
+        self._wait_for_health(timeout)
+
+    def stop(self) -> None:
+        """Stop and remove container. Safe to call multiple times."""
+        if self.container_name:
+            subprocess.run(
+                [self.runtime, "stop", self.container_name],
+                capture_output=True,
+                check=False,
+            )
+
+    def is_ready(self) -> bool:
+        """Check if container is healthy.
+
+        Returns:
+            True if container is healthy, False otherwise
+        """
+        try:
+            return check_health(self.base_url)
+        except Exception:
+            return False
+
+    def _wait_for_health(self, timeout: int) -> None:
+        """Poll health endpoint until ready.
+
+        Args:
+            timeout: Maximum seconds to wait
+
+        Raises:
+            TimeoutError: If health check doesn't pass within timeout
+        """
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            try:
+                if check_health(self.base_url):
+                    return
+            except Exception:
+                pass
+            time.sleep(2)  # Poll every 2 seconds
+
+        raise TimeoutError(f"Container failed to become healthy within {timeout}s")
+
+    def __enter__(self):
+        """Context manager entry."""
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit - ensures cleanup."""
+        self.stop()
+        return False

mdify/docling_client.py

@@ -0,0 +1,232 @@
+"""HTTP client for docling-serve REST API."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+import mimetypes
+
+import requests
+
+
+@dataclass
+class ConvertResult:
+    """Result from document conversion."""
+
+    content: str
+    format: str
+    success: bool
+    error: Optional[str] = None
+
+
+@dataclass
+class StatusResult:
+    """Status of async conversion task."""
+
+    status: str  # "pending", "completed", "failed"
+    task_id: str
+    error: Optional[str] = None
+
+
+class DoclingClientError(Exception):
+    """Base exception for docling client errors."""
+
+    pass
+
+
+class DoclingHTTPError(DoclingClientError):
+    """HTTP error from docling-serve API."""
+
+    def __init__(self, status_code: int, message: str):
+        self.status_code = status_code
+        super().__init__(f"HTTP {status_code}: {message}")
+
+
+def _get_mime_type(file_path: Path) -> str:
+    """Get MIME type for file, with fallback for unknown types."""
+    mime_type, _ = mimetypes.guess_type(str(file_path))
+    return mime_type or "application/octet-stream"
+
+
+def check_health(base_url: str) -> bool:
+    """Check if docling-serve is healthy.
+
+    Args:
+        base_url: Base URL of docling-serve (e.g., "http://localhost:8000")
+
+    Returns:
+        True if healthy, False otherwise
+    """
+    try:
+        response = requests.get(f"{base_url}/health")
+        return response.status_code == 200
+    except requests.RequestException:
+        return False
+
+
+def convert_file(
+    base_url: str, file_path: Path, to_format: str = "md", do_ocr: bool = True
+) -> ConvertResult:
+    """Convert a file synchronously.
+
+    Args:
+        base_url: Base URL of docling-serve
+        file_path: Path to file to convert
+        to_format: Output format (default: "md")
+        do_ocr: Whether to perform OCR (default: True)
+
+    Returns:
+        ConvertResult with conversion output
+
+    Raises:
+        DoclingHTTPError: If HTTP request fails
+    """
+    try:
+        with open(file_path, "rb") as f:
+            response = requests.post(
+                f"{base_url}/v1/convert/file",
+                files={"files": (file_path.name, f, _get_mime_type(file_path))},
+                data={"to_formats": to_format, "do_ocr": str(do_ocr).lower()},
+            )
+
+        if response.status_code != 200:
+            raise DoclingHTTPError(
+                response.status_code, response.text or "Conversion failed"
+            )
+
+        result_data = response.json()
+
+        # docling-serve returns results in a list format
+        if isinstance(result_data, list) and len(result_data) > 0:
+            first_result = result_data[0]
+            return ConvertResult(
+                content=first_result.get("content", ""), format=to_format, success=True
+            )
+        elif isinstance(result_data, dict):
+            return ConvertResult(
+                content=result_data.get("content", ""), format=to_format, success=True
+            )
+        else:
+            raise DoclingHTTPError(200, f"Unexpected response format: {result_data}")
+
+    except requests.RequestException as e:
+        return ConvertResult(content="", format=to_format, success=False, error=str(e))
+
+
+def convert_file_async(
+    base_url: str, file_path: Path, to_format: str = "md", do_ocr: bool = True
+) -> str:
+    """Start async file conversion.
+
+    Args:
+        base_url: Base URL of docling-serve
+        file_path: Path to file to convert
+        to_format: Output format (default: "md")
+        do_ocr: Whether to perform OCR (default: True)
+
+    Returns:
+        Task ID for polling
+
+    Raises:
+        DoclingHTTPError: If HTTP request fails
+    """
+    try:
+        with open(file_path, "rb") as f:
+            response = requests.post(
+                f"{base_url}/v1/convert/file/async",
+                files={"files": (file_path.name, f, _get_mime_type(file_path))},
+                data={"to_formats": to_format, "do_ocr": str(do_ocr).lower()},
+            )
+
+        if response.status_code != 200:
+            raise DoclingHTTPError(
+                response.status_code, response.text or "Async conversion failed"
+            )
+
+        result_data = response.json()
+        task_id = result_data.get("task_id")
+
+        if not task_id:
+            raise DoclingHTTPError(200, f"No task_id in response: {result_data}")
+
+        return task_id
+
+    except requests.RequestException as e:
+        raise DoclingHTTPError(500, str(e))
+
+
+def poll_status(base_url: str, task_id: str) -> StatusResult:
+    """Poll status of async conversion task.
+
+    Args:
+        base_url: Base URL of docling-serve
+        task_id: Task ID from convert_file_async
+
+    Returns:
+        StatusResult with current status
+
+    Raises:
+        DoclingHTTPError: If HTTP request fails
+    """
+    try:
+        response = requests.get(f"{base_url}/v1/status/poll/{task_id}")
+
+        if response.status_code != 200:
+            raise DoclingHTTPError(
+                response.status_code, response.text or "Status poll failed"
+            )
+
+        result_data = response.json()
+
+        return StatusResult(
+            status=result_data.get("status", "unknown"),
+            task_id=task_id,
+            error=result_data.get("error"),
+        )
+
+    except requests.RequestException as e:
+        raise DoclingHTTPError(500, str(e))
+
+
+def get_result(base_url: str, task_id: str) -> ConvertResult:
+    """Get result of completed async conversion.
+
+    Args:
+        base_url: Base URL of docling-serve
+        task_id: Task ID from convert_file_async
+
+    Returns:
+        ConvertResult with conversion output
+
+    Raises:
+        DoclingHTTPError: If HTTP request fails or task not completed
+    """
+    try:
+        response = requests.get(f"{base_url}/v1/result/{task_id}")
+
+        if response.status_code != 200:
+            raise DoclingHTTPError(
+                response.status_code, response.text or "Result retrieval failed"
+            )
+
+        result_data = response.json()
+
+        # Similar to sync conversion, handle list or dict format
+        if isinstance(result_data, list) and len(result_data) > 0:
+            first_result = result_data[0]
+            return ConvertResult(
+                content=first_result.get("content", ""),
+                format=first_result.get("format", "md"),
+                success=True,
+            )
+        elif isinstance(result_data, dict):
+            return ConvertResult(
+                content=result_data.get("content", ""),
+                format=result_data.get("format", "md"),
+                success=True,
+            )
+        else:
+            raise DoclingHTTPError(200, f"Unexpected response format: {result_data}")
+
+    except requests.RequestException as e:
+        return ConvertResult(content="", format="md", success=False, error=str(e))

mdify_cli-2.7.0.dist-info/METADATA

@@ -0,0 +1,274 @@
+Metadata-Version: 2.4
+Name: mdify-cli
+Version: 2.7.0
+Summary: Convert PDFs and document images into structured Markdown for LLM workflows
+Author: tiroq
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/tiroq/mdify
+Project-URL: Repository, https://github.com/tiroq/mdify
+Project-URL: Issues, https://github.com/tiroq/mdify/issues
+Keywords: markdown,conversion,pdf,docling,cli,document,docker
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Utilities
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Dynamic: license-file
+
+# mdify
+
+![mdify banner](https://raw.githubusercontent.com/tiroq/mdify/main/assets/mdify.png)
+
+[![PyPI](https://img.shields.io/pypi/v/mdify-cli?logo=python&style=flat-square)](https://pypi.org/project/mdify-cli/)
+[![Container](https://img.shields.io/badge/container-ghcr.io-blue?logo=docker&style=flat-square)](https://github.com/tiroq/mdify/pkgs/container/mdify-runtime)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](https://opensource.org/licenses/MIT)
+
+A lightweight CLI for converting documents to Markdown. The CLI is fast to install via pipx, while the heavy ML conversion runs inside a container.
+
+## Requirements
+
+- **Python 3.8+**
+- **Docker** or **Podman** (for document conversion)
+
+## Installation
+
+### macOS (recommended)
+
+```bash
+brew install pipx
+pipx ensurepath
+pipx install mdify-cli
+```
+
+Restart your terminal after installation.
+
+### Linux
+
+```bash
+python3 -m pip install --user pipx
+pipx ensurepath
+pipx install mdify-cli
+```
+
+### Install via pip
+
+```bash
+pip install mdify-cli
+```
+
+### Development install
+
+```bash
+git clone https://github.com/tiroq/mdify.git
+cd mdify
+pip install -e .
+```
+
+## Usage
+
+### Basic conversion
+
+Convert a single file:
+```bash
+mdify document.pdf
+```
+
+The first run will automatically pull the container image (~2GB) if not present.
+
+### Convert multiple files
+
+Convert all PDFs in a directory:
+```bash
+mdify /path/to/documents -g "*.pdf"
+```
+
+Recursively convert files:
+```bash
+mdify /path/to/documents -r -g "*.pdf"
+```
+
+### GPU Acceleration
+
+For faster processing with NVIDIA GPU:
+```bash
+mdify --gpu documents/*.pdf
+```
+
+Requires NVIDIA GPU with CUDA support and nvidia-container-toolkit.
+
+### ⚠️ PII Masking (Deprecated)
+
+The `--mask` flag is deprecated and will be ignored in this version. PII masking functionality was available in older versions using a custom runtime but is not supported with the current docling-serve backend.
+
+If PII masking is critical for your use case, please use mdify v1.5.x or earlier versions.
+
+## Performance
+
+mdify now uses docling-serve for significantly faster batch processing:
+
+- **Single model load**: Models are loaded once per session, not per file
+- **~10-20x speedup** for multiple file conversions compared to previous versions
+- **GPU acceleration**: Use `--gpu` for additional 2-6x speedup (requires NVIDIA GPU)
+
+### First Run Behavior
+
+The first conversion takes longer (~30-60s) as the container loads ML models into memory. Subsequent files in the same batch process quickly, typically in 1-3 seconds per file.
+
+## Options
+
+| Option | Description |
+|--------|-------------|
+| `input` | Input file or directory to convert (required) |
+| `-o, --out-dir DIR` | Output directory for converted files (default: output) |
+| `-g, --glob PATTERN` | Glob pattern for filtering files (default: *) |
+| `-r, --recursive` | Recursively scan directories |
+| `--flat` | Disable directory structure preservation |
+| `--overwrite` | Overwrite existing output files |
+| `-q, --quiet` | Suppress progress messages |
+| `-m, --mask` | ⚠️ **Deprecated**: PII masking not supported in current version |
+| `--gpu` | Use GPU-accelerated container (requires NVIDIA GPU and nvidia-container-toolkit) |
+| `--port PORT` | Container port (default: 5001) |
+| `--runtime RUNTIME` | Container runtime: docker or podman (auto-detected) |
+| `--image IMAGE` | Custom container image (default: ghcr.io/docling-project/docling-serve-cpu:main) |
+| `--pull POLICY` | Image pull policy: always, missing, never (default: missing) |
+| `--check-update` | Check for available updates and exit |
+| `--version` | Show version and exit |
+
+### Flat Mode
+
+With `--flat`, all output files are placed directly in the output directory. Directory paths are incorporated into filenames to prevent collisions:
+
+- `docs/subdir1/file.pdf` → `output/subdir1_file.md`
+- `docs/subdir2/file.pdf` → `output/subdir2_file.md`
+
+## Examples
+
+Convert all PDFs recursively, preserving structure:
+```bash
+mdify documents/ -r -g "*.pdf" -o markdown_output
+```
+
+Convert with Podman instead of Docker:
+```bash
+mdify document.pdf --runtime podman
+```
+
+Use a custom/local container image:
+```bash
+mdify document.pdf --image my-custom-image:latest
+```
+
+Force pull latest container image:
+```bash
+mdify document.pdf --pull
+```
+
+## Architecture
+
+```
+┌──────────────────┐     ┌─────────────────────────────────┐
+│   mdify CLI      │     │  Container (Docker/Podman)      │
+│   (lightweight)  │────▶│  ┌───────────────────────────┐  │
+│                  │     │  │  Docling + ML Models      │  │
+│  - File handling │◀────│  │  - PDF parsing            │  │
+│  - Container     │     │  │  - OCR (Tesseract)        │  │
+│    orchestration │     │  │  - Document conversion    │  │
+└──────────────────┘     │  └───────────────────────────┘  │
+                         └─────────────────────────────────┘
+```
+
+The CLI:
+- Installs in seconds via pipx (no ML dependencies)
+- Automatically detects Docker or Podman
+- Pulls the runtime container on first use
+- Mounts files and runs conversions in the container
+
+## Container Images
+
+mdify uses official docling-serve containers:
+
+**CPU Version** (default):
+```
+ghcr.io/docling-project/docling-serve-cpu:main
+```
+
+**GPU Version** (use with `--gpu` flag):
+```
+ghcr.io/docling-project/docling-serve-cu126:main
+```
+
+These are official images from the [docling-serve project](https://github.com/DS4SD/docling-serve).
+
+## Updates
+
+mdify checks for updates daily. When a new version is available:
+
+```
+==================================================
+A new version of mdify is available!
+  Current version: 0.3.0
+  Latest version:  0.4.0
+==================================================
+
+Run upgrade now? [y/N]
+```
+
+### Disable update checks
+
+```bash
+export MDIFY_NO_UPDATE_CHECK=1
+```
+
+## Uninstall
+
+```bash
+pipx uninstall mdify-cli
+```
+
+Or if installed via pip:
+
+```bash
+pip uninstall mdify-cli
+```
+
+## Development
+
+### Task automation
+
+This project uses [Task](https://taskfile.dev) for automation:
+
+```bash
+# Show available tasks
+task
+
+# Build package
+task build
+
+# Build container locally
+task container-build
+
+# Release workflow
+task release-patch
+```
+
+### Building for PyPI
+
+See [PUBLISHING.md](PUBLISHING.md) for complete publishing instructions.
+
+## License
+
+MIT

mdify_cli-2.7.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+assets/mdify.png,sha256=qUj7WXWqNwpI2KNXOW79XJwqFqa-UI0JEkmt1mmy4Rg,1820418
+mdify/__init__.py,sha256=ymBvtqVt-BtORLCI0ZO674etO8tlMJxzghl39z6gCUg,90
+mdify/__main__.py,sha256=bhpJ00co6MfaVOdH4XLoW04NtLYDa_oJK7ODzfLrn9M,143
+mdify/cli.py,sha256=LqIibolYSKGCNYqxuIyFnvPkjJyNlXvfWeKaSaoOrqo,28542
+mdify/container.py,sha256=tkk0nv7EquL-rKUY4nkS_yGITb7mqw8B7eEfuqaeVrg,5239
+mdify/docling_client.py,sha256=9QWPmd0W5APzf6LeUrdDBAru6E4d89w2q8WqGVlJoHg,6807
+mdify_cli-2.7.0.dist-info/licenses/LICENSE,sha256=NWM66Uv-XuSMKaU-gaPmvfyk4WgE6zcIPr78wyg6GAo,1065
+mdify_cli-2.7.0.dist-info/METADATA,sha256=4v5CMHOhZ2LKgRgH7xm7hOUUYwahYCRJSCMcGtNja5g,7923
+mdify_cli-2.7.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+mdify_cli-2.7.0.dist-info/entry_points.txt,sha256=0Xki8f5lADQUtwdt6Eq_FEaieI6Byhk8UE7BuDhChMg,41
+mdify_cli-2.7.0.dist-info/top_level.txt,sha256=qltzf7h8owHq7dxCdfCkSHY8gT21hn1_E8P-VWS_OKM,6
+mdify_cli-2.7.0.dist-info/RECORD,,

mdify_cli-2.7.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

mdify_cli-2.7.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+mdify = mdify.cli:main