extract-python 0.10.4__py3-none-any.whl → 0.11.1__py3-none-any.whl

extract_python/__init__.py

@@ -5,20 +5,20 @@ logger = logging.getLogger(__name__)
 try:
     from .docling_ import DOCLING_DEFAULT_ARTIFACTS_PATH, DoclingPipeline
 except ModuleNotFoundError:
-    logger.exception("docling is not available, might be optional...")
+    logger.warning("docling is not available, might be optional...")
     DOCKING_DEFAULT_ARTIFACTS_PATH, DoclingPipeline = None, None
 
 try:
     from .marker_ import MarkerPipeline
 except ModuleNotFoundError:
-    logger.exception("marker is not available, might be optional...")
+    logger.warning("marker is not available, might be optional...")
     MarkerPipeline = None
 
 
 try:
     from .miner_u import MinerUPipeline
 except ModuleNotFoundError:
-    logger.exception("mineru is not available, might be optional...")
+    logger.warning("mineru is not available, might be optional...")
     MinerUPipeline = None
 
 

extract_python/__main__.py

@@ -0,0 +1,9 @@
+from .cli import cli_app
+
+
+def main() -> None:
+    cli_app()
+
+
+if __name__ == "__main__":
+    main()

extract_python/cli/__init__.py

@@ -0,0 +1,50 @@
+import importlib
+import logging
+import os
+from typing import Annotated
+
+import typer
+from icij_common.logging_utils import setup_loggers
+
+import extract_python
+
+from .doc import configs_app
+from .utils import AsyncTyper
+
+cli_app = AsyncTyper(
+    context_settings={"help_option_names": ["-h", "--help"]},
+    pretty_exceptions_enable=False,
+)
+cli_app.add_typer(configs_app)
+
+
+def version_callback(value: bool) -> None:  # noqa: FBT001
+    if value:
+        package_version = importlib.metadata.version(extract_python.__name__)
+        print(package_version)
+        raise typer.Exit()
+
+
+def pretty_exc_callback(value: bool) -> None:  # noqa: FBT001
+    if not value:
+        os.environ["TYPER_STANDARD_TRACEBACK"] = "1"
+
+
+@cli_app.callback()
+def main(
+    version: Annotated[  # noqa: ARG001
+        bool | None,
+        typer.Option("--version", callback=version_callback, is_eager=True),
+    ] = None,
+    *,
+    pretty_exceptions: Annotated[  # noqa: ARG001
+        bool,
+        typer.Option(
+            "--pretty-exceptions", callback=pretty_exc_callback, is_eager=True
+        ),
+    ] = False,
+) -> None:
+    setup_loggers(
+        ["__main__", extract_python.__name__, "docling", "mineru", "marker"],
+        level=logging.DEBUG,
+    )

extract_python/cli/doc.py

@@ -0,0 +1,78 @@
+import logging
+import os
+from pathlib import Path
+from typing import Annotated, Never
+
+import typer
+from extract_core import (
+    Device,
+    InputDoc,
+    OutputFormat,
+    Pipeline,
+    PipelineConfig,
+    PipelineSize,
+    PipelineType,
+)
+from extract_core.default_config import default_config
+from pydantic import TypeAdapter
+
+from .utils import AsyncTyper
+
+logger = logging.getLogger(__name__)
+
+_DOC = "doc"
+
+_DOC_CONVERT_HELP = "Converts documents to the provided output format"
+_DOC_CONVERT_DOCS_HELP = "input documents (can be file or directories)"
+_DOC_CONVERT_OUTPUT_PATH_HELP = "directory where the converted documents will be saved"
+_DOC_CONVERT_DEVICE_HELP = "device used to run inference"
+_DOC_CONVERT_OUTPUT_FMT_HELP = "output format"
+_DOC_CONVERT_PIPELINE_CONFIG = "path to a JSON pipeline configuration"
+
+
+configs_app = AsyncTyper(name=_DOC)
+
+
+@configs_app.async_command(help=_DOC_CONVERT_HELP)
+async def convert(
+    docs: Annotated[list[Path], typer.Argument(help=_DOC_CONVERT_DOCS_HELP)],
+    output_path: Annotated[Path, typer.Argument(help=_DOC_CONVERT_OUTPUT_PATH_HELP)],
+    output_format: Annotated[
+        OutputFormat, typer.Option(help=_DOC_CONVERT_OUTPUT_FMT_HELP)
+    ] = OutputFormat.MARKDOWN,
+    config: Annotated[
+        Path | None,
+        typer.Option("-c", "--config", help=_DOC_CONVERT_PIPELINE_CONFIG),
+    ] = None,
+) -> None:
+    if config is not None:
+        config = TypeAdapter(PipelineConfig).validate_json(config.read_text())
+    else:
+        config = default_config(
+            pipeline_type=PipelineType.DOCLING,
+            device=Device.CPU,
+            size=PipelineSize.SMALL,
+        )
+    config = config.to_device()
+    docs = [InputDoc.from_path(d, n_pages=1) for p in docs for d in _walkdir(p)]
+    pipeline = Pipeline.from_config(config)
+    output_paths = []
+    async for res in pipeline.extract_content(docs, output_format, output_path):
+        output_path = res.output.path
+        logger.info("converted %s to %s", res.input.path, output_path / output_path)
+        output_paths.append(output_path / output_path)
+    print("\n".join(output_paths))
+
+
+def _raise(exc: OSError) -> Never:
+    raise exc
+
+
+def _walkdir(path: Path) -> list[Path]:
+    if path.is_file():
+        return [path]
+    paths = []
+    for root, _, files in os.walk(path, onerror=_raise):
+        root = Path(root)  # noqa: PLW2901
+        paths.extend(root / f for f in files)
+    return paths

extract_python/cli/utils.py

@@ -0,0 +1,25 @@
+import asyncio
+import sys
+from collections.abc import Callable
+from functools import wraps
+from typing import Any
+
+import typer
+
+
+class AsyncTyper(typer.Typer):
+    def async_command(self, *args, **kwargs) -> Callable[[Callable], Callable]:
+        def decorator(async_func: Callable) -> Callable:
+            @wraps(async_func)
+            def sync_func(*_args, **_kwargs) -> Any:
+                res = asyncio.run(async_func(*_args, **_kwargs))
+                return res
+
+            self.command(*args, **kwargs)(sync_func)
+            return async_func
+
+        return decorator
+
+
+def eprint(*args, **kwargs) -> None:
+    print(*args, file=sys.stderr, **kwargs)

extract_python/docling_.py

@@ -59,8 +59,7 @@ class DoclingPipeline(Pipeline):
     def __init__(self, config: DoclingPipelineConfig):
         super().__init__(config)
         format_options = {
-            k: v.to_docling(self._device)
-            for k, v in self._config.format_options.items()
+            k: v.to_docling() for k, v in self._config.format_options.items()
         }
         logger.info(
             "resolved format options to: %s",

{extract_python-0.10.4.dist-info → extract_python-0.11.1.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.5
 Name: extract-python
-Version: 0.10.4
+Version: 0.11.1
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python
 Project-URL: Issues, https://github.com/ICIJ/extract-python/issues
 Author-email: Clément Doumouro <cdoumouro@icij.org>
 Requires-Python: <3.15,>=3.13
-Requires-Dist: extract-core~=0.10.4
+Requires-Dist: extract-core~=0.11.0
 Requires-Dist: icij-common~=0.8.2
 Requires-Dist: pympler~=1.1
 Provides-Extra: benches
@@ -15,6 +15,8 @@ Requires-Dist: html2image~=2.0.7; extra == 'benches'
 Requires-Dist: markdown2>=2.5.4; extra == 'benches'
 Requires-Dist: notebook>=7.4.5; extra == 'benches'
 Requires-Dist: pypdfium2>=4.30.0; extra == 'benches'
+Provides-Extra: cli
+Requires-Dist: typer~=0.21.2; extra == 'cli'
 Provides-Extra: docling
 Requires-Dist: docling-slim[feat-ocr-easyocr,feat-ocr-mac,feat-ocr-rapidocr,feat-ocr-tesserocr,standard]~=2.127; extra == 'docling'
 Provides-Extra: docling-gpu

extract_python-0.11.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+extract_python/__init__.py,sha256=mUWwAYj8dRhM79J_Ns_yNM3WG2GDqWRNXZd3XVkkUaY,753
+extract_python/__main__.py,sha256=g-fvS46zl9umKmGrSpl-OG-8PSuZgjqvTCqjpsZtSps,101
+extract_python/constants.py,sha256=659V40LcTWJhX3IbuJLSSvI5AsGJh9ciMrGCfzJn2zA,98
+extract_python/docling_.py,sha256=iZha9zh6uyFo67RVB0YqHMJt9auzJJV0kJ1OCPTOan8,10030
+extract_python/marker_.py,sha256=bb0DJXrSWOznIuPnw4-adR5_R9QiwtFGGBh5KlXFxjM,3323
+extract_python/miner_u.py,sha256=sEU1PhPpfPVo4jP7flNlbGO9PfsLFLz973-ufZEd0ws,3966
+extract_python/utils.py,sha256=LrR1tlOgpWEri6TLDlYYPRBmlka-AjOepWnbySf2Haw,9974
+extract_python/cli/__init__.py,sha256=Xt3MB74fO8cimm0Xas8dIMmXu62O0dV-4o3-OUzZggo,1245
+extract_python/cli/doc.py,sha256=SMwRs-7-u_lw3wpHO6m3vFYnbjYCn_qIVC-q2TIxcbg,2435
+extract_python/cli/utils.py,sha256=Png7oGKKxDeStn27k-o4Hxlzl-G-u-QDSH6IfJqUk1I,667
+extract_python-0.11.1.dist-info/METADATA,sha256=B4sG_uDFLruPQE3ytKMq5o7ODQOKGRlYGgvyuFiBmkE,1429
+extract_python-0.11.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+extract_python-0.11.1.dist-info/entry_points.txt,sha256=noSkCWA0LRm0V7WbmYDPBnJrKeiYdpvXynAW6y72Z4M,64
+extract_python-0.11.1.dist-info/RECORD,,

extract_python-0.11.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+extract_python = extract_python.__main__:main

extract_python-0.10.4.dist-info/RECORD

@@ -1,9 +0,0 @@
-extract_python/__init__.py,sha256=DA2LUro6vMjfS8fb2MsqO95FbJEZHyZ7kFyn42q02Wk,759
-extract_python/constants.py,sha256=659V40LcTWJhX3IbuJLSSvI5AsGJh9ciMrGCfzJn2zA,98
-extract_python/docling_.py,sha256=zD5rIXV1pAQBNGZc0RliP9buHjttz4CdSOj21Z4FbFo,10054
-extract_python/marker_.py,sha256=bb0DJXrSWOznIuPnw4-adR5_R9QiwtFGGBh5KlXFxjM,3323
-extract_python/miner_u.py,sha256=sEU1PhPpfPVo4jP7flNlbGO9PfsLFLz973-ufZEd0ws,3966
-extract_python/utils.py,sha256=LrR1tlOgpWEri6TLDlYYPRBmlka-AjOepWnbySf2Haw,9974
-extract_python-0.10.4.dist-info/METADATA,sha256=5UcF7UU0SgfLmsb-oSXbCfZWzUBOKSYSvZFUILttM4o,1364
-extract_python-0.10.4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-extract_python-0.10.4.dist-info/RECORD,,