extract-core 0.5.4__tar.gz

extract_core-0.5.4/.gitignore

@@ -0,0 +1,19 @@
+.idea
+
+# Python
+*.log
+venv
+*.egg-info
+.eggs
+__pycache__
+*.pytest_cache
+*.pyc
+build
+dist
+test/.env
+.cache
+tmp
+./*.csv
+
+# VS code
+.vscode

extract_core-0.5.4/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: extract-core
+Version: 0.5.4
+Summary: Core of extract-python
+Project-URL: Homepage, https://github.com/ICIJ/extract-python
+Project-URL: Repository, https://github.com/ICIJ/extract-python
+Project-URL: Issues, https://github.com/ICIJ/extract-python/issues
+Author-email: Clément Doumouro <cdoumouro@icij.org>
+Requires-Python: <3.14,>=3.11
+Requires-Dist: docling-slim~=2.96
+Requires-Dist: icij-common~=0.8.2
+Requires-Dist: marker-pdf~=1.10
+Requires-Dist: mineru~=3.2
+Requires-Dist: pydantic-extra-types[pycountry]>=2.11.1
+Requires-Dist: pydantic~=2.13

extract_core-0.5.4/extract_core/__init__.py

@@ -0,0 +1,63 @@
+from typing import Annotated
+
+from icij_common.pydantic_utils import make_enum_discriminator, tagged_union
+from pydantic import Discriminator
+
+from .configs import BasePipelineConfig, PipelineType
+from .objects import (
+    BaseModel,
+    ConversionOutput,
+    Error,
+    InputDoc,
+    MarkdownDoc,
+    OutputFormat,
+    PageIndexes,
+    Result,
+    Status,
+)
+from .pipeline import Pipeline
+
+try:
+    from .docling_ import DoclingFormatOption, DoclingPipelineConfig
+except ImportError:
+    DoclingPipelineConfig, DoclingFormatOption = None, None
+try:
+    from .marker_ import MarkerPipelineConfig
+except ImportError:
+    MarkerPipelineConfig = None
+
+
+try:
+    from .miner_u import MinerUBackend, MinerUConfig, MinerUPipelineConfig
+except ImportError:
+    MinerUBackend, MinerUPipelineConfig, MinerUConfig = None, None, None
+
+
+pipeline_type_discriminator = make_enum_discriminator("pipeline", PipelineType)
+PipelineConfig = Annotated[
+    tagged_union(
+        BasePipelineConfig.__subclasses__(), lambda t: t.pipeline.default.value
+    ),
+    Discriminator(pipeline_type_discriminator),
+]
+
+
+__all__ = [
+    "BaseModel",
+    "BasePipelineConfig",
+    "ConversionOutput",
+    "DoclingPipelineConfig",
+    "Error",
+    "InputDoc",
+    "MarkdownDoc",
+    "MarkerPipelineConfig",
+    "MinerUBackend",
+    "MinerUConfig",
+    "MinerUPipelineConfig",
+    "OutputFormat",
+    "PageIndexes",
+    "Pipeline",
+    "PipelineType",
+    "Result",
+    "Status",
+]

extract_core-0.5.4/extract_core/configs.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+from enum import StrEnum
+from typing import ClassVar
+
+from icij_common.pydantic_utils import icij_config, merge_configs, no_enum_values_config
+from icij_common.registrable import RegistrableConfig
+from pydantic import Field
+
+from .objects import SupportedExt
+
+
+class PipelineType(StrEnum):
+    DOCLING = "docling"
+    MARKER = "marker"
+    MINER_U = "miner_u"
+
+
+class BasePipelineConfig(RegistrableConfig, ABC):
+    # TODO: move this icij_config() to RegistrableConfig
+    model_config = merge_configs(icij_config(), no_enum_values_config())
+
+    registry_key: ClassVar[str] = Field(frozen=True, default="pipeline")
+    pipeline: ClassVar[PipelineType]
+
+    @classmethod
+    @abstractmethod
+    def supported_exts(cls) -> set[SupportedExt]: ...

extract_core-0.5.4/extract_core/docling_.py

@@ -0,0 +1,268 @@
+import importlib
+from functools import cache
+from typing import Annotated, Any, ClassVar, TypeVar, get_type_hints
+
+from docling.datamodel.backend_options import BackendOptions, BaseBackendOptions
+from docling.datamodel.base_models import (
+    BaseFormatOption,
+    FormatToExtensions,
+    InputFormat,
+)
+from docling.datamodel.pipeline_options import (
+    BaseLayoutOptions,
+    BaseTableStructureOptions,
+    EasyOcrOptions,
+    LayoutOptions,
+    OcrOptions,
+    PictureDescriptionBaseOptions,
+    PictureDescriptionVlmEngineOptions,
+    PipelineOptions,
+    TableStructureOptions,
+    ThreadedPdfPipelineOptions,
+)
+from icij_common.pydantic_utils import (
+    merge_configs,
+    tagged_union,
+    to_lower_snake_case,
+)
+from pydantic import (
+    ConfigDict,
+    Discriminator,
+    Field,
+    TypeAdapter,
+    WrapSerializer,
+)
+from pydantic_core.core_schema import SerializerFunctionWrapHandler
+
+from .configs import BasePipelineConfig, PipelineType
+from .objects import BaseModel, SupportedExt
+from .utils import all_subclasses
+
+
+@cache
+def _ext_to_docling_input_format() -> dict:
+
+    mapping = dict()
+    supported = DoclingPipelineConfig.supported_exts()
+    for input_f, exts in FormatToExtensions.items():
+        for ext in exts:
+            try:
+                ext = SupportedExt(f".{ext.lower()}")  # noqa: PLW2901
+            except ValueError:
+                continue
+            if ext in supported:
+                mapping[ext] = input_f
+    return mapping
+
+
+def _validate_pipeline_opts(v: PipelineOptions) -> PipelineOptions:
+    generate_picture_images = getattr(v, "generate_picture_images", None)
+    if generate_picture_images is False:
+        msg = "generate_picture_images should be set to True"
+        raise ValueError(msg)
+    return v
+
+
+T = TypeVar("T")
+
+
+def _find_subcls(cls: type[T], name: str) -> type[T]:
+    # Check if the class available
+    for c in all_subclasses(cls):
+        if c.__name__ == name:
+            return c
+    # Then apply ad-hoc search
+    if "pipeline" in cls.__name__.lower():
+        module_name = f"docling.pipeline.{to_lower_snake_case(name)}"
+        try:
+            module = importlib.import_module(module_name)
+            return getattr(module, name)
+        except (ModuleNotFoundError, AttributeError):
+            pass
+    raise ValueError(f"unknown {cls.__name__} subclass {name}")
+
+
+def _find_init_arg_type(cls: type[Any], arg: str) -> type[BaseModel]:
+    hints = get_type_hints(cls.__init__)
+    return hints[arg]
+
+
+def _resolve_pipeline_cls(v: str) -> Any:
+    if isinstance(v, str):
+        from docling.pipeline.base_pipeline import BasePipeline  # noqa: PLC0415
+
+        return _find_subcls(BasePipeline, v)
+    return v
+
+
+def _ser_as_str(v: type) -> str:
+    return v.__name__
+
+
+def _ser_with_backend_option_kind(
+    v: Any, handler: SerializerFunctionWrapHandler
+) -> Any:
+    serialized = handler(v)
+    if isinstance(v, BaseBackendOptions):
+        kind = getattr(v, "kind", None)
+        if kind is not None:
+            serialized["kind"] = kind
+    return serialized
+
+
+def _resolve_backend(v: Any) -> Any:
+    from docling.backend.abstract_backend import (  # noqa: PLC0415
+        AbstractDocumentBackend,
+    )
+
+    if isinstance(v, str):
+        return _find_subcls(AbstractDocumentBackend, v)
+    return v
+
+
+@cache
+def _picture_descr_opts_type_adapter() -> TypeAdapter:
+    _PictureDescriptionModel = Annotated[  # noqa: N806
+        tagged_union(
+            PictureDescriptionBaseOptions.__subclasses__(), tag_getter=lambda x: x.kind
+        ),
+        Discriminator(lambda x: x["kind"]),
+    ]
+    return TypeAdapter(_PictureDescriptionModel)
+
+
+@cache
+def _ocr_opts_type_adapter() -> TypeAdapter:
+    _OcrOptions = Annotated[  # noqa: N806
+        tagged_union(OcrOptions.__subclasses__(), tag_getter=lambda x: x.kind),
+        Discriminator(lambda x: x.pop("kind")),
+    ]
+    return TypeAdapter(_OcrOptions)
+
+
+@cache
+def _layout_opts_type_adapter() -> TypeAdapter:
+    _LayoutOptions = Annotated[  # noqa: N806
+        tagged_union(BaseLayoutOptions.__subclasses__(), tag_getter=lambda x: x.kind),
+        Discriminator(lambda x: x["kind"]),
+    ]
+    return TypeAdapter(_LayoutOptions)
+
+
+@cache
+def _table_structure_opts_type_adapter() -> TypeAdapter:
+    _TableStructureOptions = Annotated[  # noqa: N806
+        tagged_union(
+            BaseTableStructureOptions.__subclasses__(), tag_getter=lambda x: x.kind
+        ),
+        Discriminator(lambda x: x["kind"]),
+    ]
+    return TypeAdapter(_TableStructureOptions)
+
+
+def _resolve_pipeline_options(
+    pipeline_options: dict[str, Any] | None | PipelineOptions, pipeline_cls: type
+) -> PipelineOptions:
+    option_cls = _find_init_arg_type(pipeline_cls, "pipeline_options")
+    picture_descr_opts = pipeline_options.get("picture_description_options")
+    if picture_descr_opts is not None:
+        if "kind" not in picture_descr_opts:
+            msg = f"missing picture description options kind: {picture_descr_opts}"
+            raise ValueError(msg)
+
+        picture_descr_opts = _picture_descr_opts_type_adapter().validate_python(
+            picture_descr_opts
+        )
+        pipeline_options["picture_description_options"] = picture_descr_opts
+    ocr_opts = pipeline_options.get("ocr_options")
+    if ocr_opts is not None:
+        if "kind" not in ocr_opts:
+            msg = f"missing ocr options kind: {ocr_opts}"
+            raise ValueError(msg)
+        ocr_opts = _ocr_opts_type_adapter().validate_python(ocr_opts)
+        pipeline_options["ocr_options"] = ocr_opts
+    layout_opts = pipeline_options.get("layout_options")
+    if layout_opts is not None:
+        if "kind" not in layout_opts:
+            msg = f"missing layout options kind: {layout_opts}"
+            raise ValueError(msg)
+        layout_opts = _layout_opts_type_adapter().validate_python(layout_opts)
+        pipeline_options["layout_options"] = layout_opts
+    table_structure_opts = pipeline_options.get("table_structure_options")
+    if table_structure_opts is not None:
+        if "kind" not in table_structure_opts:
+            msg = f"missing table structure options kind: {table_structure_opts}"
+            raise ValueError(msg)
+        table_structure_opts = _table_structure_opts_type_adapter().validate_python(
+            table_structure_opts
+        )
+        pipeline_options["table_structure_options"] = table_structure_opts
+    pipeline_options = option_cls.model_validate(pipeline_options)
+    return pipeline_options
+
+
+# Mimics the docling FormatOption but only with lightweight types,
+# the heavy convertion is done at runtime
+class DoclingFormatOption(BaseFormatOption):
+    model_config = merge_configs(
+        BaseModel.model_config, ConfigDict(polymorphic_serialization=True)
+    )
+    backend: str
+    backend_options: Annotated[
+        BackendOptions | None, WrapSerializer(_ser_with_backend_option_kind)
+    ] = None
+    pipeline_cls: str
+    pipeline_options: dict[str, Any] | None = None
+
+    def to_docling(self) -> BaseFormatOption:  # noqa: ANN201
+        from docling.document_converter import FormatOption  # noqa: PLC0415
+
+        pipeline_cls = _resolve_pipeline_cls(self.pipeline_cls)
+        pipeline_opts = _resolve_pipeline_options(self.pipeline_options, pipeline_cls)
+        pipeline_opts = _validate_pipeline_opts(pipeline_opts)
+        return FormatOption(
+            pipeline_cls=pipeline_cls,
+            pipeline_options=pipeline_opts,
+            backend=_resolve_backend(self.backend),
+            backend_options=self.backend_options,
+        )
+
+
+@cache
+def _default_format_opts() -> dict[InputFormat, DoclingFormatOption]:
+    pipeline_opts = ThreadedPdfPipelineOptions(
+        ocr_options=EasyOcrOptions(), generate_picture_images=True
+    ).model_dump(polymorphic_serialization=True)
+    pipeline_opts["picture_description_options"]["kind"] = (
+        PictureDescriptionVlmEngineOptions.kind
+    )
+    pipeline_opts["ocr_options"]["kind"] = EasyOcrOptions.kind
+    pipeline_opts["layout_options"]["kind"] = LayoutOptions.kind
+    pipeline_opts["table_structure_options"]["kind"] = TableStructureOptions.kind
+    return {
+        InputFormat.PDF: DoclingFormatOption(
+            pipeline_cls="StandardPdfPipeline",
+            backend="DoclingParseDocumentBackend",
+            pipeline_options=pipeline_opts,
+        ),
+    }
+
+
+class DoclingPipelineConfig(BasePipelineConfig):
+    pipeline: ClassVar[PipelineType] = Field(frozen=True, default=PipelineType.DOCLING)
+
+    format_options: dict[InputFormat, DoclingFormatOption] = Field(
+        default_factory=_default_format_opts
+    )
+
+    @classmethod
+    @cache
+    def supported_exts(cls) -> set[SupportedExt]:
+        unsupported = {InputFormat.AUDIO, InputFormat.METS_GBS, InputFormat.VTT}
+        supported = set()
+        for f in InputFormat:
+            if f in unsupported:
+                continue
+            for ext in FormatToExtensions[f]:
+                supported.add(SupportedExt(f".{ext.lower()}"))
+        return supported

extract_core-0.5.4/extract_core/marker_.py

@@ -0,0 +1,40 @@
+from functools import cache
+from typing import Any, ClassVar
+
+from pydantic import Field
+
+from .configs import BasePipelineConfig, PipelineType
+from .objects import SupportedExt
+
+
+class MarkerPipelineConfig(BasePipelineConfig):
+    pipeline: ClassVar[PipelineType] = Field(frozen=True, default=PipelineType.MARKER)
+
+    config: dict[str, Any] = dict()
+
+    @classmethod
+    @cache
+    def supported_exts(cls) -> set[SupportedExt]:
+        # Subset of https://documentation.datalab.to/docs/common/supportedfiletypes
+        return {
+            SupportedExt.PDF,
+            SupportedExt.XLS,
+            SupportedExt.XLSX,
+            SupportedExt.XLSM,
+            SupportedExt.CSV,
+            SupportedExt.ODS,
+            SupportedExt.DOC,
+            SupportedExt.DOCX,
+            SupportedExt.ODT,
+            SupportedExt.PPT,
+            SupportedExt.PPTX,
+            SupportedExt.ODP,
+            SupportedExt.HTLM,
+            SupportedExt.EPUB,
+            SupportedExt.PNG,
+            SupportedExt.JPG,
+            SupportedExt.JPEG,
+            SupportedExt.WEBP,
+            SupportedExt.GIF,
+            SupportedExt.TIFF,
+        }

extract_core-0.5.4/extract_core/miner_u.py

@@ -0,0 +1,76 @@
+from collections.abc import Callable
+from copy import copy
+from enum import StrEnum
+from functools import cache
+from typing import Any, ClassVar
+
+from pydantic import Field
+from pydantic_extra_types.language_code import LanguageAlpha2
+
+from .configs import BasePipelineConfig, PipelineType
+from .objects import BaseModel, SupportedExt
+
+_MINER_U_CONVERSION_ERRORS = tuple()
+MDMakeFunction = Callable[[list, str, str], str | None]
+
+
+class MinerUBackend(StrEnum):
+    PIPELINE = "pipeline"
+    VLM = "vlm"
+
+
+class MinerUConfig(BaseModel):
+    backend: MinerUBackend = MinerUBackend.PIPELINE
+    enable_formula_extraction: bool = True
+    enable_table_extraction: bool = True
+    # TODO: use enum or literal here
+    parse_method: str = "auto"
+
+    def as_parse_kwargs(self) -> dict[str, Any]:
+        kwargs = copy(self._get_default_kwargs())
+        kwargs["backend"] = self.backend
+        kwargs["parse_method"] = self.parse_method
+        kwargs["formula_enable"] = self.enable_formula_extraction
+        kwargs["table_enable"] = self.enable_table_extraction
+        return kwargs
+
+    @classmethod
+    @cache
+    def _get_default_kwargs(cls) -> dict[str, Any]:
+        from mineru.utils.enum_class import MakeMode  # noqa: PLC0415
+
+        return {
+            "server_url": None,
+            # We don't dump md directly we process, we dump the middle json in order
+            # to be able to get page indexes
+            "parse_method": "auto",
+            "dump_md": False,
+            "dump_middle_json": True,
+            "f_draw_layout_bbox": False,
+            "f_draw_span_bbox": False,
+            "f_dump_model_output": False,  # might be useful for debug though
+            "f_dump_orig_pdf": False,
+            "f_dump_content_list": False,  # might be useful for debug though
+            "start_page_id": 0,
+            "f_make_md_mode": MakeMode.MM_MD,
+            "image_analysis": True,
+            "end_page_id": None,
+            "client_side_output_generation": False,
+        }
+
+
+class MinerUPipelineConfig(BasePipelineConfig):  # noqa: F821
+    pipeline: ClassVar[PipelineType] = Field(frozen=True, default=PipelineType.MINER_U)
+
+    config: MinerUConfig = Field(frozen=True, default=MinerUConfig())
+    language: LanguageAlpha2 = Field(frozen=True, default="en")
+
+    @classmethod
+    @cache
+    def supported_exts(cls) -> set[SupportedExt]:
+        return {
+            SupportedExt.PDF,
+            SupportedExt.DOCX,
+            SupportedExt.PPTX,
+            SupportedExt.XLSX,
+        }