opendataloader-pdf 0.0.0__py3-none-any.whl → 1.8.2__py3-none-any.whl

opendataloader_pdf/NOTICE.md

@@ -1,5 +1,5 @@
 
-Copyright © 2025 Hancom, Inc.
+Copyright © 2025-2026 Hancom, Inc.
 
 This Source Code Form is subject to the terms of the Mozilla Public License 2.0 (MPL-2.0).
 If a copy of the MPL was not distributed with this file, you can obtain one at https://mozilla.org/MPL/2.0/.

opendataloader_pdf/__init__.py

@@ -1,3 +1,3 @@
-from .wrapper import run
+from .wrapper import run, convert, run_jar
 
-__all__ = ["run"]
+__all__ = ["run", "convert", "run_jar"]

opendataloader_pdf/__main__.py

@@ -0,0 +1,5 @@
+from .wrapper import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

opendataloader_pdf/cli_options_generated.py

@@ -0,0 +1,236 @@
+# AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
+# Run `npm run generate-options` to regenerate
+
+"""
+CLI option definitions for opendataloader-pdf.
+"""
+from typing import Any, Dict, List
+
+
+# Option metadata list
+CLI_OPTIONS: List[Dict[str, Any]] = [
+    {
+        "name": "output-dir",
+        "python_name": "output_dir",
+        "short_name": "o",
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Directory where output files are written. Default: input file directory",
+    },
+    {
+        "name": "password",
+        "python_name": "password",
+        "short_name": "p",
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Password for encrypted PDF files",
+    },
+    {
+        "name": "format",
+        "python_name": "format",
+        "short_name": "f",
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Output formats (comma-separated). Values: json, text, html, pdf, markdown, markdown-with-html, markdown-with-images. Default: json",
+    },
+    {
+        "name": "quiet",
+        "python_name": "quiet",
+        "short_name": "q",
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Suppress console logging output",
+    },
+    {
+        "name": "content-safety-off",
+        "python_name": "content_safety_off",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg",
+    },
+    {
+        "name": "keep-line-breaks",
+        "python_name": "keep_line_breaks",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Preserve original line breaks in extracted text",
+    },
+    {
+        "name": "replace-invalid-chars",
+        "python_name": "replace_invalid_chars",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": " ",
+        "description": "Replacement character for invalid/unrecognized characters. Default: space",
+    },
+    {
+        "name": "use-struct-tree",
+        "python_name": "use_struct_tree",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": False,
+        "description": "Use PDF structure tree (tagged PDF) for reading order and semantic structure",
+    },
+    {
+        "name": "table-method",
+        "python_name": "table_method",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "default",
+        "description": "Table detection method. Values: default (border-based), cluster (border + cluster). Default: default",
+    },
+    {
+        "name": "reading-order",
+        "python_name": "reading_order",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "xycut",
+        "description": "Reading order algorithm. Values: off, xycut. Default: xycut",
+    },
+    {
+        "name": "markdown-page-separator",
+        "python_name": "markdown_page_separator",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Separator between pages in Markdown output. Use %%page-number%% for page numbers. Default: none",
+    },
+    {
+        "name": "text-page-separator",
+        "python_name": "text_page_separator",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Separator between pages in text output. Use %%page-number%% for page numbers. Default: none",
+    },
+    {
+        "name": "html-page-separator",
+        "python_name": "html_page_separator",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Separator between pages in HTML output. Use %%page-number%% for page numbers. Default: none",
+    },
+    {
+        "name": "image-output",
+        "python_name": "image_output",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "external",
+        "description": "Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external",
+    },
+    {
+        "name": "image-format",
+        "python_name": "image_format",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "png",
+        "description": "Output format for extracted images. Values: png, jpeg. Default: png",
+    },
+    {
+        "name": "image-dir",
+        "python_name": "image_dir",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Directory for extracted images",
+    },
+    {
+        "name": "pages",
+        "python_name": "pages",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Pages to extract (e.g., \"1,3,5-7\"). Default: all pages",
+    },
+    {
+        "name": "hybrid",
+        "python_name": "hybrid",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "off",
+        "description": "Hybrid backend for AI processing. Values: off (default), docling (docling-fast is deprecated alias)",
+    },
+    {
+        "name": "hybrid-mode",
+        "python_name": "hybrid_mode",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "auto",
+        "description": "Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend)",
+    },
+    {
+        "name": "hybrid-ocr",
+        "python_name": "hybrid_ocr",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "auto",
+        "description": "Hybrid OCR mode for Docling backend. Values: auto (default, OCR only where needed), force (force full-page OCR)",
+    },
+    {
+        "name": "hybrid-url",
+        "python_name": "hybrid_url",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": None,
+        "description": "Hybrid backend server URL (overrides default)",
+    },
+    {
+        "name": "hybrid-timeout",
+        "python_name": "hybrid_timeout",
+        "short_name": None,
+        "type": "string",
+        "required": False,
+        "default": "30000",
+        "description": "Hybrid backend request timeout in milliseconds. Default: 30000",
+    },
+    {
+        "name": "hybrid-fallback",
+        "python_name": "hybrid_fallback",
+        "short_name": None,
+        "type": "boolean",
+        "required": False,
+        "default": True,
+        "description": "Fallback to Java processing on hybrid backend error. Default: true",
+    },
+]
+
+
+def add_options_to_parser(parser) -> None:
+    """Add all CLI options to an argparse.ArgumentParser."""
+    for opt in CLI_OPTIONS:
+        flags = []
+        if opt["short_name"]:
+            flags.append(f'-{opt["short_name"]}')
+        flags.append(f'--{opt["name"]}')
+
+        kwargs = {"help": opt["description"]}
+        if opt["type"] == "boolean":
+            kwargs["action"] = "store_true"
+        else:
+            kwargs["default"] = None
+
+        parser.add_argument(*flags, **kwargs)

opendataloader_pdf/convert_generated.py

@@ -0,0 +1,130 @@
+# AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
+# Run `npm run generate-options` to regenerate
+
+"""
+Auto-generated convert function for opendataloader-pdf.
+"""
+from typing import List, Optional, Union
+
+from .runner import run_jar
+
+
+def convert(
+    input_path: Union[str, List[str]],
+    output_dir: Optional[str] = None,
+    password: Optional[str] = None,
+    format: Optional[Union[str, List[str]]] = None,
+    quiet: bool = False,
+    content_safety_off: Optional[Union[str, List[str]]] = None,
+    keep_line_breaks: bool = False,
+    replace_invalid_chars: Optional[str] = None,
+    use_struct_tree: bool = False,
+    table_method: Optional[str] = None,
+    reading_order: Optional[str] = None,
+    markdown_page_separator: Optional[str] = None,
+    text_page_separator: Optional[str] = None,
+    html_page_separator: Optional[str] = None,
+    image_output: Optional[str] = None,
+    image_format: Optional[str] = None,
+    image_dir: Optional[str] = None,
+    pages: Optional[str] = None,
+    hybrid: Optional[str] = None,
+    hybrid_mode: Optional[str] = None,
+    hybrid_ocr: Optional[str] = None,
+    hybrid_url: Optional[str] = None,
+    hybrid_timeout: Optional[str] = None,
+    hybrid_fallback: bool = True,
+) -> None:
+    """
+    Convert PDF(s) into the requested output format(s).
+
+    Args:
+        input_path: One or more input PDF file paths or directories
+        output_dir: Directory where output files are written. Default: input file directory
+        password: Password for encrypted PDF files
+        format: Output formats (comma-separated). Values: json, text, html, pdf, markdown, markdown-with-html, markdown-with-images. Default: json
+        quiet: Suppress console logging output
+        content_safety_off: Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg
+        keep_line_breaks: Preserve original line breaks in extracted text
+        replace_invalid_chars: Replacement character for invalid/unrecognized characters. Default: space
+        use_struct_tree: Use PDF structure tree (tagged PDF) for reading order and semantic structure
+        table_method: Table detection method. Values: default (border-based), cluster (border + cluster). Default: default
+        reading_order: Reading order algorithm. Values: off, xycut. Default: xycut
+        markdown_page_separator: Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none
+        text_page_separator: Separator between pages in text output. Use %page-number% for page numbers. Default: none
+        html_page_separator: Separator between pages in HTML output. Use %page-number% for page numbers. Default: none
+        image_output: Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external
+        image_format: Output format for extracted images. Values: png, jpeg. Default: png
+        image_dir: Directory for extracted images
+        pages: Pages to extract (e.g., "1,3,5-7"). Default: all pages
+        hybrid: Hybrid backend for AI processing. Values: off (default), docling (docling-fast is deprecated alias)
+        hybrid_mode: Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend)
+        hybrid_ocr: Hybrid OCR mode for Docling backend. Values: auto (default, OCR only where needed), force (force full-page OCR)
+        hybrid_url: Hybrid backend server URL (overrides default)
+        hybrid_timeout: Hybrid backend request timeout in milliseconds. Default: 30000
+        hybrid_fallback: Fallback to Java processing on hybrid backend error. Default: true
+    """
+    args: List[str] = []
+
+    # Build input paths
+    if isinstance(input_path, list):
+        args.extend(input_path)
+    else:
+        args.append(input_path)
+
+    if output_dir:
+        args.extend(["--output-dir", output_dir])
+    if password:
+        args.extend(["--password", password])
+    if format:
+        if isinstance(format, list):
+            if format:
+                args.extend(["--format", ",".join(format)])
+        else:
+            args.extend(["--format", format])
+    if quiet:
+        args.append("--quiet")
+    if content_safety_off:
+        if isinstance(content_safety_off, list):
+            if content_safety_off:
+                args.extend(["--content-safety-off", ",".join(content_safety_off)])
+        else:
+            args.extend(["--content-safety-off", content_safety_off])
+    if keep_line_breaks:
+        args.append("--keep-line-breaks")
+    if replace_invalid_chars:
+        args.extend(["--replace-invalid-chars", replace_invalid_chars])
+    if use_struct_tree:
+        args.append("--use-struct-tree")
+    if table_method:
+        args.extend(["--table-method", table_method])
+    if reading_order:
+        args.extend(["--reading-order", reading_order])
+    if markdown_page_separator:
+        args.extend(["--markdown-page-separator", markdown_page_separator])
+    if text_page_separator:
+        args.extend(["--text-page-separator", text_page_separator])
+    if html_page_separator:
+        args.extend(["--html-page-separator", html_page_separator])
+    if image_output:
+        args.extend(["--image-output", image_output])
+    if image_format:
+        args.extend(["--image-format", image_format])
+    if image_dir:
+        args.extend(["--image-dir", image_dir])
+    if pages:
+        args.extend(["--pages", pages])
+    if hybrid:
+        args.extend(["--hybrid", hybrid])
+    if hybrid_mode:
+        args.extend(["--hybrid-mode", hybrid_mode])
+    if hybrid_ocr:
+        args.extend(["--hybrid-ocr", hybrid_ocr])
+    if hybrid_url:
+        args.extend(["--hybrid-url", hybrid_url])
+    if hybrid_timeout:
+        args.extend(["--hybrid-timeout", hybrid_timeout])
+    if hybrid_fallback:
+        args.append("--hybrid-fallback")
+
+    run_jar(args, quiet)

opendataloader_pdf/hybrid_server.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+"""Fast docling server using DocumentConverter singleton.
+
+A lightweight FastAPI server optimized for hybrid PDF processing:
+1. Using DocumentConverter singletons (no per-request initialization)
+2. Returns only JSON (DoclingDocument format) - markdown/HTML generated by Java
+
+Usage:
+    opendataloader-pdf-hybrid [--port PORT] [--host HOST]
+
+    # Default: http://localhost:5002
+    opendataloader-pdf-hybrid
+
+    # Custom port
+    opendataloader-pdf-hybrid --port 5003
+
+API Endpoints:
+    GET  /health              - Health check
+    POST /v1/convert/file     - Convert PDF to JSON
+
+The /v1/convert/file endpoint parameters:
+    - files: PDF file (multipart/form-data)
+    - page_ranges: Page range to process (optional)
+    - force_ocr: Force full-page OCR mode (optional, default: false)
+
+Requirements:
+    Install with hybrid extra: pip install opendataloader-pdf[hybrid]
+"""
+
+import argparse
+import logging
+import os
+import tempfile
+import time
+import traceback
+from contextlib import asynccontextmanager
+from typing import Optional
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+# Configuration
+DEFAULT_HOST = "0.0.0.0"
+DEFAULT_PORT = 5002
+MAX_FILE_SIZE = 100 * 1024 * 1024  # 100MB max file size
+
+# Global converter instances (initialized on startup)
+converter_auto = None   # force_full_page_ocr=False
+converter_force = None  # force_full_page_ocr=True
+
+
+def _check_dependencies():
+    """Check if hybrid dependencies are installed."""
+    missing = []
+    try:
+        import uvicorn  # noqa: F401
+    except ImportError:
+        missing.append("uvicorn")
+    try:
+        import fastapi  # noqa: F401
+    except ImportError:
+        missing.append("fastapi")
+    try:
+        import docling  # noqa: F401
+    except ImportError:
+        missing.append("docling")
+
+    if missing:
+        raise ImportError(
+            f"Missing dependencies: {', '.join(missing)}. "
+            "Install with: pip install opendataloader-pdf[hybrid]"
+        )
+
+
+def create_converter(force_full_page_ocr: bool = False):
+    """Create a DocumentConverter with the specified OCR options.
+
+    Args:
+        force_full_page_ocr: If True, force OCR on all pages regardless of text content.
+                            If False (default), OCR only where needed.
+    """
+    from docling.datamodel.base_models import InputFormat
+    from docling.datamodel.pipeline_options import (
+        EasyOcrOptions,
+        PdfPipelineOptions,
+        TableFormerMode,
+        TableStructureOptions,
+    )
+    from docling.document_converter import DocumentConverter, PdfFormatOption
+
+    pipeline_options = PdfPipelineOptions(
+        do_ocr=True,
+        do_table_structure=True,
+        ocr_options=EasyOcrOptions(force_full_page_ocr=force_full_page_ocr),
+        table_structure_options=TableStructureOptions(
+            mode=TableFormerMode.ACCURATE
+        ),
+    )
+
+    return DocumentConverter(
+        format_options={
+            InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
+        }
+    )
+
+
+def create_app():
+    """Create and configure the FastAPI application."""
+    from fastapi import FastAPI, File, Form, UploadFile
+    from fastapi.responses import JSONResponse
+
+    @asynccontextmanager
+    async def lifespan(_app: FastAPI):
+        """Lifespan context manager for startup and shutdown events."""
+        global converter_auto, converter_force
+        logger.info("Initializing DocumentConverters...")
+        start = time.perf_counter()
+
+        # Initialize both converters at startup
+        converter_auto = create_converter(force_full_page_ocr=False)
+        converter_force = create_converter(force_full_page_ocr=True)
+
+        elapsed = time.perf_counter() - start
+        logger.info(f"DocumentConverters initialized in {elapsed:.2f}s")
+        yield
+        # Cleanup on shutdown (if needed)
+
+    app = FastAPI(
+        title="Docling Fast Server",
+        description="Fast PDF conversion using docling SDK with singleton pattern",
+        version="1.0.0",
+        lifespan=lifespan,
+    )
+
+    @app.get("/health")
+    def health():
+        """Health check endpoint."""
+        return {"status": "ok"}
+
+    @app.post("/v1/convert/file")
+    async def convert_file(
+        files: UploadFile = File(...),
+        page_ranges: Optional[str] = Form(default=None),
+        force_ocr: Optional[str] = Form(default=None),
+    ):
+        """Convert PDF file to JSON (DoclingDocument format).
+
+        Only JSON output is provided - markdown and HTML are generated by
+        Java processors for consistent reading order application.
+
+        Args:
+            files: The PDF file to convert
+            page_ranges: Page range string "start-end" (e.g., "1-5") (optional)
+            force_ocr: If "true", use force full-page OCR mode (optional)
+
+        Returns:
+            JSON response with document content.
+        """
+        global converter_auto, converter_force
+
+        # Select converter based on force_ocr parameter
+        use_force_ocr = force_ocr and force_ocr.lower() == "true"
+        converter = converter_force if use_force_ocr else converter_auto
+
+        if converter is None:
+            return JSONResponse(
+                {"status": "failure", "errors": ["Server not initialized"]},
+                status_code=503,
+            )
+
+        # Parse page_ranges string to tuple
+        page_range_tuple = None
+        if page_ranges:
+            try:
+                parts = page_ranges.split("-")
+                if len(parts) == 2:
+                    page_range_tuple = (int(parts[0]), int(parts[1]))
+            except ValueError:
+                pass
+
+        # Read and validate file size
+        content = await files.read()
+        if len(content) > MAX_FILE_SIZE:
+            return JSONResponse(
+                {
+                    "status": "failure",
+                    "errors": [f"File size exceeds maximum allowed ({MAX_FILE_SIZE // (1024*1024)}MB)"],
+                },
+                status_code=413,
+            )
+
+        # Save uploaded file to temp location
+        tmp_path = None
+        with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
+            tmp.write(content)
+            tmp_path = tmp.name
+
+        try:
+            start = time.perf_counter()
+            if page_range_tuple:
+                result = converter.convert(tmp_path, page_range=page_range_tuple)
+            else:
+                result = converter.convert(tmp_path)
+            processing_time = time.perf_counter() - start
+
+            # Export to JSON (DoclingDocument format)
+            json_content = result.document.export_to_dict()
+
+            # Build response compatible with docling-serve format
+            response = {
+                "status": "success",
+                "document": {
+                    "json_content": json_content,
+                },
+                "processing_time": processing_time,
+                "force_ocr": use_force_ocr,
+            }
+
+            return JSONResponse(response)
+
+        except Exception as e:
+            logger.error(f"PDF conversion failed: {e}\n{traceback.format_exc()}")
+            return JSONResponse(
+                {"status": "failure", "errors": ["PDF conversion failed"]},
+                status_code=500,
+            )
+        finally:
+            if tmp_path and os.path.exists(tmp_path):
+                os.unlink(tmp_path)
+
+    return app
+
+
+def main():
+    """Run the server."""
+    _check_dependencies()
+    import uvicorn
+
+    parser = argparse.ArgumentParser(description="Docling Fast Server for opendataloader-pdf")
+    parser.add_argument(
+        "--host",
+        default=DEFAULT_HOST,
+        help=f"Host to bind to (default: {DEFAULT_HOST})",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=DEFAULT_PORT,
+        help=f"Port to bind to (default: {DEFAULT_PORT})",
+    )
+    parser.add_argument(
+        "--log-level",
+        default="info",
+        choices=["debug", "info", "warning", "error"],
+        help="Log level (default: info)",
+    )
+    args = parser.parse_args()
+
+    logger.info(f"Starting Docling Fast Server on http://{args.host}:{args.port}")
+    app = create_app()
+    uvicorn.run(
+        app,
+        host=args.host,
+        port=args.port,
+        log_level=args.log_level,
+    )
+
+
+if __name__ == "__main__":
+    main()