extract-core 0.5.4__py3-none-any.whl

extract_core/__init__.py

@@ -0,0 +1,63 @@
+from typing import Annotated
+
+from icij_common.pydantic_utils import make_enum_discriminator, tagged_union
+from pydantic import Discriminator
+
+from .configs import BasePipelineConfig, PipelineType
+from .objects import (
+    BaseModel,
+    ConversionOutput,
+    Error,
+    InputDoc,
+    MarkdownDoc,
+    OutputFormat,
+    PageIndexes,
+    Result,
+    Status,
+)
+from .pipeline import Pipeline
+
+try:
+    from .docling_ import DoclingFormatOption, DoclingPipelineConfig
+except ImportError:
+    DoclingPipelineConfig, DoclingFormatOption = None, None
+try:
+    from .marker_ import MarkerPipelineConfig
+except ImportError:
+    MarkerPipelineConfig = None
+
+
+try:
+    from .miner_u import MinerUBackend, MinerUConfig, MinerUPipelineConfig
+except ImportError:
+    MinerUBackend, MinerUPipelineConfig, MinerUConfig = None, None, None
+
+
+pipeline_type_discriminator = make_enum_discriminator("pipeline", PipelineType)
+PipelineConfig = Annotated[
+    tagged_union(
+        BasePipelineConfig.__subclasses__(), lambda t: t.pipeline.default.value
+    ),
+    Discriminator(pipeline_type_discriminator),
+]
+
+
+__all__ = [
+    "BaseModel",
+    "BasePipelineConfig",
+    "ConversionOutput",
+    "DoclingPipelineConfig",
+    "Error",
+    "InputDoc",
+    "MarkdownDoc",
+    "MarkerPipelineConfig",
+    "MinerUBackend",
+    "MinerUConfig",
+    "MinerUPipelineConfig",
+    "OutputFormat",
+    "PageIndexes",
+    "Pipeline",
+    "PipelineType",
+    "Result",
+    "Status",
+]

extract_core/configs.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+from enum import StrEnum
+from typing import ClassVar
+
+from icij_common.pydantic_utils import icij_config, merge_configs, no_enum_values_config
+from icij_common.registrable import RegistrableConfig
+from pydantic import Field
+
+from .objects import SupportedExt
+
+
+class PipelineType(StrEnum):
+    DOCLING = "docling"
+    MARKER = "marker"
+    MINER_U = "miner_u"
+
+
+class BasePipelineConfig(RegistrableConfig, ABC):
+    # TODO: move this icij_config() to RegistrableConfig
+    model_config = merge_configs(icij_config(), no_enum_values_config())
+
+    registry_key: ClassVar[str] = Field(frozen=True, default="pipeline")
+    pipeline: ClassVar[PipelineType]
+
+    @classmethod
+    @abstractmethod
+    def supported_exts(cls) -> set[SupportedExt]: ...

extract_core/docling_.py

@@ -0,0 +1,268 @@
+import importlib
+from functools import cache
+from typing import Annotated, Any, ClassVar, TypeVar, get_type_hints
+
+from docling.datamodel.backend_options import BackendOptions, BaseBackendOptions
+from docling.datamodel.base_models import (
+    BaseFormatOption,
+    FormatToExtensions,
+    InputFormat,
+)
+from docling.datamodel.pipeline_options import (
+    BaseLayoutOptions,
+    BaseTableStructureOptions,
+    EasyOcrOptions,
+    LayoutOptions,
+    OcrOptions,
+    PictureDescriptionBaseOptions,
+    PictureDescriptionVlmEngineOptions,
+    PipelineOptions,
+    TableStructureOptions,
+    ThreadedPdfPipelineOptions,
+)
+from icij_common.pydantic_utils import (
+    merge_configs,
+    tagged_union,
+    to_lower_snake_case,
+)
+from pydantic import (
+    ConfigDict,
+    Discriminator,
+    Field,
+    TypeAdapter,
+    WrapSerializer,
+)
+from pydantic_core.core_schema import SerializerFunctionWrapHandler
+
+from .configs import BasePipelineConfig, PipelineType
+from .objects import BaseModel, SupportedExt
+from .utils import all_subclasses
+
+
+@cache
+def _ext_to_docling_input_format() -> dict:
+
+    mapping = dict()
+    supported = DoclingPipelineConfig.supported_exts()
+    for input_f, exts in FormatToExtensions.items():
+        for ext in exts:
+            try:
+                ext = SupportedExt(f".{ext.lower()}")  # noqa: PLW2901
+            except ValueError:
+                continue
+            if ext in supported:
+                mapping[ext] = input_f
+    return mapping
+
+
+def _validate_pipeline_opts(v: PipelineOptions) -> PipelineOptions:
+    generate_picture_images = getattr(v, "generate_picture_images", None)
+    if generate_picture_images is False:
+        msg = "generate_picture_images should be set to True"
+        raise ValueError(msg)
+    return v
+
+
+T = TypeVar("T")
+
+
+def _find_subcls(cls: type[T], name: str) -> type[T]:
+    # Check if the class available
+    for c in all_subclasses(cls):
+        if c.__name__ == name:
+            return c
+    # Then apply ad-hoc search
+    if "pipeline" in cls.__name__.lower():
+        module_name = f"docling.pipeline.{to_lower_snake_case(name)}"
+        try:
+            module = importlib.import_module(module_name)
+            return getattr(module, name)
+        except (ModuleNotFoundError, AttributeError):
+            pass
+    raise ValueError(f"unknown {cls.__name__} subclass {name}")
+
+
+def _find_init_arg_type(cls: type[Any], arg: str) -> type[BaseModel]:
+    hints = get_type_hints(cls.__init__)
+    return hints[arg]
+
+
+def _resolve_pipeline_cls(v: str) -> Any:
+    if isinstance(v, str):
+        from docling.pipeline.base_pipeline import BasePipeline  # noqa: PLC0415
+
+        return _find_subcls(BasePipeline, v)
+    return v
+
+
+def _ser_as_str(v: type) -> str:
+    return v.__name__
+
+
+def _ser_with_backend_option_kind(
+    v: Any, handler: SerializerFunctionWrapHandler
+) -> Any:
+    serialized = handler(v)
+    if isinstance(v, BaseBackendOptions):
+        kind = getattr(v, "kind", None)
+        if kind is not None:
+            serialized["kind"] = kind
+    return serialized
+
+
+def _resolve_backend(v: Any) -> Any:
+    from docling.backend.abstract_backend import (  # noqa: PLC0415
+        AbstractDocumentBackend,
+    )
+
+    if isinstance(v, str):
+        return _find_subcls(AbstractDocumentBackend, v)
+    return v
+
+
+@cache
+def _picture_descr_opts_type_adapter() -> TypeAdapter:
+    _PictureDescriptionModel = Annotated[  # noqa: N806
+        tagged_union(
+            PictureDescriptionBaseOptions.__subclasses__(), tag_getter=lambda x: x.kind
+        ),
+        Discriminator(lambda x: x["kind"]),
+    ]
+    return TypeAdapter(_PictureDescriptionModel)
+
+
+@cache
+def _ocr_opts_type_adapter() -> TypeAdapter:
+    _OcrOptions = Annotated[  # noqa: N806
+        tagged_union(OcrOptions.__subclasses__(), tag_getter=lambda x: x.kind),
+        Discriminator(lambda x: x.pop("kind")),
+    ]
+    return TypeAdapter(_OcrOptions)
+
+
+@cache
+def _layout_opts_type_adapter() -> TypeAdapter:
+    _LayoutOptions = Annotated[  # noqa: N806
+        tagged_union(BaseLayoutOptions.__subclasses__(), tag_getter=lambda x: x.kind),
+        Discriminator(lambda x: x["kind"]),
+    ]
+    return TypeAdapter(_LayoutOptions)
+
+
+@cache
+def _table_structure_opts_type_adapter() -> TypeAdapter:
+    _TableStructureOptions = Annotated[  # noqa: N806
+        tagged_union(
+            BaseTableStructureOptions.__subclasses__(), tag_getter=lambda x: x.kind
+        ),
+        Discriminator(lambda x: x["kind"]),
+    ]
+    return TypeAdapter(_TableStructureOptions)
+
+
+def _resolve_pipeline_options(
+    pipeline_options: dict[str, Any] | None | PipelineOptions, pipeline_cls: type
+) -> PipelineOptions:
+    option_cls = _find_init_arg_type(pipeline_cls, "pipeline_options")
+    picture_descr_opts = pipeline_options.get("picture_description_options")
+    if picture_descr_opts is not None:
+        if "kind" not in picture_descr_opts:
+            msg = f"missing picture description options kind: {picture_descr_opts}"
+            raise ValueError(msg)
+
+        picture_descr_opts = _picture_descr_opts_type_adapter().validate_python(
+            picture_descr_opts
+        )
+        pipeline_options["picture_description_options"] = picture_descr_opts
+    ocr_opts = pipeline_options.get("ocr_options")
+    if ocr_opts is not None:
+        if "kind" not in ocr_opts:
+            msg = f"missing ocr options kind: {ocr_opts}"
+            raise ValueError(msg)
+        ocr_opts = _ocr_opts_type_adapter().validate_python(ocr_opts)
+        pipeline_options["ocr_options"] = ocr_opts
+    layout_opts = pipeline_options.get("layout_options")
+    if layout_opts is not None:
+        if "kind" not in layout_opts:
+            msg = f"missing layout options kind: {layout_opts}"
+            raise ValueError(msg)
+        layout_opts = _layout_opts_type_adapter().validate_python(layout_opts)
+        pipeline_options["layout_options"] = layout_opts
+    table_structure_opts = pipeline_options.get("table_structure_options")
+    if table_structure_opts is not None:
+        if "kind" not in table_structure_opts:
+            msg = f"missing table structure options kind: {table_structure_opts}"
+            raise ValueError(msg)
+        table_structure_opts = _table_structure_opts_type_adapter().validate_python(
+            table_structure_opts
+        )
+        pipeline_options["table_structure_options"] = table_structure_opts
+    pipeline_options = option_cls.model_validate(pipeline_options)
+    return pipeline_options
+
+
+# Mimics the docling FormatOption but only with lightweight types,
+# the heavy convertion is done at runtime
+class DoclingFormatOption(BaseFormatOption):
+    model_config = merge_configs(
+        BaseModel.model_config, ConfigDict(polymorphic_serialization=True)
+    )
+    backend: str
+    backend_options: Annotated[
+        BackendOptions | None, WrapSerializer(_ser_with_backend_option_kind)
+    ] = None
+    pipeline_cls: str
+    pipeline_options: dict[str, Any] | None = None
+
+    def to_docling(self) -> BaseFormatOption:  # noqa: ANN201
+        from docling.document_converter import FormatOption  # noqa: PLC0415
+
+        pipeline_cls = _resolve_pipeline_cls(self.pipeline_cls)
+        pipeline_opts = _resolve_pipeline_options(self.pipeline_options, pipeline_cls)
+        pipeline_opts = _validate_pipeline_opts(pipeline_opts)
+        return FormatOption(
+            pipeline_cls=pipeline_cls,
+            pipeline_options=pipeline_opts,
+            backend=_resolve_backend(self.backend),
+            backend_options=self.backend_options,
+        )
+
+
+@cache
+def _default_format_opts() -> dict[InputFormat, DoclingFormatOption]:
+    pipeline_opts = ThreadedPdfPipelineOptions(
+        ocr_options=EasyOcrOptions(), generate_picture_images=True
+    ).model_dump(polymorphic_serialization=True)
+    pipeline_opts["picture_description_options"]["kind"] = (
+        PictureDescriptionVlmEngineOptions.kind
+    )
+    pipeline_opts["ocr_options"]["kind"] = EasyOcrOptions.kind
+    pipeline_opts["layout_options"]["kind"] = LayoutOptions.kind
+    pipeline_opts["table_structure_options"]["kind"] = TableStructureOptions.kind
+    return {
+        InputFormat.PDF: DoclingFormatOption(
+            pipeline_cls="StandardPdfPipeline",
+            backend="DoclingParseDocumentBackend",
+            pipeline_options=pipeline_opts,
+        ),
+    }
+
+
+class DoclingPipelineConfig(BasePipelineConfig):
+    pipeline: ClassVar[PipelineType] = Field(frozen=True, default=PipelineType.DOCLING)
+
+    format_options: dict[InputFormat, DoclingFormatOption] = Field(
+        default_factory=_default_format_opts
+    )
+
+    @classmethod
+    @cache
+    def supported_exts(cls) -> set[SupportedExt]:
+        unsupported = {InputFormat.AUDIO, InputFormat.METS_GBS, InputFormat.VTT}
+        supported = set()
+        for f in InputFormat:
+            if f in unsupported:
+                continue
+            for ext in FormatToExtensions[f]:
+                supported.add(SupportedExt(f".{ext.lower()}"))
+        return supported

extract_core/marker_.py

@@ -0,0 +1,40 @@
+from functools import cache
+from typing import Any, ClassVar
+
+from pydantic import Field
+
+from .configs import BasePipelineConfig, PipelineType
+from .objects import SupportedExt
+
+
+class MarkerPipelineConfig(BasePipelineConfig):
+    pipeline: ClassVar[PipelineType] = Field(frozen=True, default=PipelineType.MARKER)
+
+    config: dict[str, Any] = dict()
+
+    @classmethod
+    @cache
+    def supported_exts(cls) -> set[SupportedExt]:
+        # Subset of https://documentation.datalab.to/docs/common/supportedfiletypes
+        return {
+            SupportedExt.PDF,
+            SupportedExt.XLS,
+            SupportedExt.XLSX,
+            SupportedExt.XLSM,
+            SupportedExt.CSV,
+            SupportedExt.ODS,
+            SupportedExt.DOC,
+            SupportedExt.DOCX,
+            SupportedExt.ODT,
+            SupportedExt.PPT,
+            SupportedExt.PPTX,
+            SupportedExt.ODP,
+            SupportedExt.HTLM,
+            SupportedExt.EPUB,
+            SupportedExt.PNG,
+            SupportedExt.JPG,
+            SupportedExt.JPEG,
+            SupportedExt.WEBP,
+            SupportedExt.GIF,
+            SupportedExt.TIFF,
+        }

extract_core/miner_u.py

@@ -0,0 +1,76 @@
+from collections.abc import Callable
+from copy import copy
+from enum import StrEnum
+from functools import cache
+from typing import Any, ClassVar
+
+from pydantic import Field
+from pydantic_extra_types.language_code import LanguageAlpha2
+
+from .configs import BasePipelineConfig, PipelineType
+from .objects import BaseModel, SupportedExt
+
+_MINER_U_CONVERSION_ERRORS = tuple()
+MDMakeFunction = Callable[[list, str, str], str | None]
+
+
+class MinerUBackend(StrEnum):
+    PIPELINE = "pipeline"
+    VLM = "vlm"
+
+
+class MinerUConfig(BaseModel):
+    backend: MinerUBackend = MinerUBackend.PIPELINE
+    enable_formula_extraction: bool = True
+    enable_table_extraction: bool = True
+    # TODO: use enum or literal here
+    parse_method: str = "auto"
+
+    def as_parse_kwargs(self) -> dict[str, Any]:
+        kwargs = copy(self._get_default_kwargs())
+        kwargs["backend"] = self.backend
+        kwargs["parse_method"] = self.parse_method
+        kwargs["formula_enable"] = self.enable_formula_extraction
+        kwargs["table_enable"] = self.enable_table_extraction
+        return kwargs
+
+    @classmethod
+    @cache
+    def _get_default_kwargs(cls) -> dict[str, Any]:
+        from mineru.utils.enum_class import MakeMode  # noqa: PLC0415
+
+        return {
+            "server_url": None,
+            # We don't dump md directly we process, we dump the middle json in order
+            # to be able to get page indexes
+            "parse_method": "auto",
+            "dump_md": False,
+            "dump_middle_json": True,
+            "f_draw_layout_bbox": False,
+            "f_draw_span_bbox": False,
+            "f_dump_model_output": False,  # might be useful for debug though
+            "f_dump_orig_pdf": False,
+            "f_dump_content_list": False,  # might be useful for debug though
+            "start_page_id": 0,
+            "f_make_md_mode": MakeMode.MM_MD,
+            "image_analysis": True,
+            "end_page_id": None,
+            "client_side_output_generation": False,
+        }
+
+
+class MinerUPipelineConfig(BasePipelineConfig):  # noqa: F821
+    pipeline: ClassVar[PipelineType] = Field(frozen=True, default=PipelineType.MINER_U)
+
+    config: MinerUConfig = Field(frozen=True, default=MinerUConfig())
+    language: LanguageAlpha2 = Field(frozen=True, default="en")
+
+    @classmethod
+    @cache
+    def supported_exts(cls) -> set[SupportedExt]:
+        return {
+            SupportedExt.PDF,
+            SupportedExt.DOCX,
+            SupportedExt.PPTX,
+            SupportedExt.XLSX,
+        }

extract_core/objects.py

@@ -0,0 +1,295 @@
+import logging
+import os
+import traceback
+import uuid
+from abc import ABC
+from enum import StrEnum
+from functools import cache
+from io import BytesIO
+from pathlib import Path
+from typing import Annotated, Any, NoReturn, Self
+
+from icij_common.pydantic_utils import (
+    icij_config,
+    merge_configs,
+    no_enum_values_config,
+    safe_copy,
+)
+from pydantic import AfterValidator, RootModel, TypeAdapter
+from pydantic import BaseModel as _BaseModel
+
+logger = logging.getLogger(__name__)
+base_config = merge_configs(icij_config(), no_enum_values_config())
+
+
+class BaseModel(_BaseModel):
+    model_config = base_config
+
+
+class SupportedExt(StrEnum):
+    ADOC = ".adoc"
+    ASC = ".asc"
+    ASCIIDOC = ".asciidoc"
+    BMP = ".bmp"
+    CSV = ".csv"
+    DOC = ".doc"
+    DOCX = ".docx"
+    DOTX = ".dotx"
+    DOTM = ".dotm"
+    DOCM = ".docm"
+    EPUB = ".epub"
+    EML = ".eml"
+    GIF = ".gif"
+    HTLM = ".html"
+    HTM = ".htm"
+    JPEG = ".jpeg"
+    JPG = ".jpg"
+    JSON = ".json"
+    LATEX = ".latex"
+    MD = ".md"
+    NXML = ".nxml"
+    ODP = ".odp"
+    ODS = ".ods"
+    ODT = ".odt"
+    PDF = ".pdf"
+    PNG = ".png"
+    PPSX = ".ppsx"
+    PPT = ".ppt"
+    PPTM = ".pptm"
+    PPSM = ".ppsm"
+    POTX = ".potx"
+    POTM = ".potm"
+    PPTX = ".pptx"
+    QMD = ".qmd"
+    RMD = ".rmd"
+    TEX = ".tex"
+    TIF = ".tif"
+    TIFF = ".tiff"
+    TXT = ".txt"
+    TEXT = ".text"
+    WEBP = ".webp"
+    XBRL = ".xbrl"
+    XHTML = ".xhtml"
+    XLS = ".xls"
+    XLSM = ".xlsm"
+    XLSX = ".xlsx"
+    XLTX = ".xltx"
+    XML = ".xml"
+
+    def to_docling(self):  # noqa: ANN201
+        from .docling_ import _ext_to_docling_input_format  # noqa: PLC0415
+
+        return _ext_to_docling_input_format()[self]
+
+
+class OutputFormat(StrEnum):
+    MARKDOWN = ".md"
+
+    @property
+    def suffix(self) -> str:
+        return self.value[1:]
+
+    def to_marker(self) -> str:
+        match self:
+            case OutputFormat.MARKDOWN:
+                return "markdown"
+            case _:
+                raise ValueError(f"{self} is unsupported by marker")
+
+
+class Status(StrEnum):
+    FAILURE = "failure"
+    SUCCESS = "success"
+    PARTIAL_SUCCESS = "partial_success"
+
+    @classmethod
+    def from_docling(cls, v: Any) -> Self:
+        from docling.datamodel.base_models import ConversionStatus  # noqa: PLC0415
+
+        if v is ConversionStatus.SUCCESS:
+            return cls.SUCCESS
+        if v is ConversionStatus.PARTIAL_SUCCESS:
+            return cls.PARTIAL_SUCCESS
+        if isinstance(v, ConversionStatus):
+            return cls.FAILURE
+        raise TypeError(f"can't convert {v!r} to {cls.__name__!r}")
+
+    @property
+    def allows_conversion(self) -> bool:
+        return self is Status.SUCCESS or self is Status.PARTIAL_SUCCESS
+
+
+class Error(BaseModel):
+    id: str
+    title: str
+    detail: str
+
+    @classmethod
+    def from_exception(cls, exception: BaseException) -> Self:
+        title = exception.__class__.__name__
+        trace_lines = traceback.format_exception(
+            None, value=exception, tb=exception.__traceback__
+        )
+        detail = f"{exception}\n{''.join(trace_lines)}"
+        error_id = f"{_id_title(title)}-{uuid.uuid4().hex}"
+        error = cls(id=error_id, title=title, detail=detail)
+        return error
+
+    @classmethod
+    def from_docling(cls, docling_error) -> Self:  # noqa: ANN001
+        title = "DoclingConversionError"
+        error_id = f"{_id_title(title)}-{uuid.uuid4().hex}"
+        detail = (
+            f"error in module {docling_error.module_name} of"
+            f" {docling_error.component_type}:\n{docling_error.error_message}"
+        )
+        return cls(id=error_id, title=title, detail=detail)
+
+
+def _id_title(title: str) -> str:
+    id_title = []
+    for i, letter in enumerate(title):
+        if i and letter.isupper():
+            id_title.append("-")
+        id_title.append(letter.lower())
+    return "".join(id_title)
+
+
+class InputDoc(BaseModel):
+    ext: SupportedExt
+    path: Path
+    content: bytes | None = None
+
+    @classmethod
+    def from_path(cls, path: str | Path) -> Self:
+        if isinstance(path, str):
+            path = Path(path)
+        ext = SupportedExt(path.suffix)
+        return cls(path=path, ext=ext)
+
+    def to_docling(self):  # noqa: ANN201
+        from docling_core.types.io import DocumentStream  # noqa: PLC0415
+
+        if self.content is not None:
+            return DocumentStream(name=str(self.path), stream=BytesIO(self.content))
+
+        if not self.path.suffix:
+            return DocumentStream(
+                name=str(self.path), stream=BytesIO(self.path.read_bytes())
+            )
+        return self.path
+
+    def without_content(self) -> Self:
+        return safe_copy(self, update={"content": None})
+
+
+class PageIndexes(RootModel[list[tuple[int, int]]]):
+    # Stores page end index
+    @classmethod
+    def from_page_end_indices(cls, lengths: list[int]) -> Self:
+        return [
+            ((lengths[p - 1] if p > 0 else 0), lengths[p]) for p in range(len(lengths))
+        ]
+
+
+class ConversionOutput(BaseModel):
+    path: Path
+    pages: PageIndexes = []
+
+
+class MarkdownDoc(ConversionOutput):
+    @classmethod
+    @property
+    @cache
+    def _valid_conversion_statuses(cls) -> set:
+        from docling.datamodel.base_models import ConversionStatus  # noqa: PLC0415
+
+        return {ConversionStatus.SUCCESS, ConversionStatus.PARTIAL_SUCCESS}
+
+
+def _input_should_not_have_content(value: InputDoc) -> InputDoc:
+    if value.content is not None:
+        raise ValueError(f"response input can't have content, but got {value}")
+    return value
+
+
+class _BaseResult(BaseModel, ABC):
+    input: InputDoc
+    status: Status
+    errors: list[Error] = []
+
+
+class ResponseResult(_BaseResult):
+    input: Annotated[InputDoc, AfterValidator(func=_input_should_not_have_content)]
+    output_path: Path
+
+
+class Result(_BaseResult):
+    # TODO: we could also use generics here when we add more output formats
+    output: ConversionOutput | None
+
+    def to_response(self) -> ResponseResult:
+        return ResponseResult(
+            input=self.input.without_content(),
+            status=self.status,
+            errors=self.errors,
+            output_path=self.output.path,
+        )
+
+
+class ExtractionResponse(BaseModel):
+    results: list[ResponseResult]
+
+
+_INPUT_DOCS_ADAPTER = TypeAdapter(list[InputDoc | Path])
+
+
+def parse_extraction_request(
+    docs: str | list[dict | str], *, data_dir: Path
+) -> list[InputDoc]:
+    if isinstance(docs, str):
+        logger.debug("exploring files in %s", data_dir.absolute())
+        docs_dir = Path(data_dir) / docs
+        docs = _as_input_docs(docs_dir)
+        msg = "found %s"
+        if len(docs) > 10:
+            msg = msg + ", and more..."
+        logger.debug("found %s", docs[:10])
+        return docs
+    docs = _INPUT_DOCS_ADAPTER.validate_python(docs)
+    if not docs:
+        return []
+    if isinstance(docs[0], Path):
+        doc_meta = []
+        unknown_exts = []
+        for doc in docs:
+            _, ext = os.path.splitext(str(doc))
+            if not ext:
+                unknown_exts.append(doc)
+            else:
+                doc_meta.append(InputDoc.from_path(path=doc.relative_to(data_dir)))
+        if unknown_exts:
+            raise ValueError(f"found files with unknown extensions {unknown_exts}")
+        return doc_meta
+    return docs
+
+
+def _raise(err: OSError) -> NoReturn:
+    raise err
+
+
+def _as_input_docs(
+    docs_dir: Path, *, supported_ext: set[str] | None = None
+) -> list[InputDoc]:
+    if supported_ext is None:
+        supported_ext = {v.value for v in SupportedExt}
+    docs = []
+    for root, _, files in os.walk(docs_dir, onerror=_raise):
+        root = Path(root)  # noqa: PLW2901
+        for f in files:
+            ext = Path(f).suffix
+            if not ext or ext not in supported_ext:
+                continue
+            docs.append(InputDoc.from_path(path=root / f))
+    docs = sorted(docs, key=lambda x: x.path)
+    return docs

extract_core/pipeline.py

@@ -0,0 +1,14 @@
+from abc import ABC, abstractmethod
+from collections.abc import AsyncGenerator, Iterable
+from pathlib import Path
+
+from icij_common.registrable import RegistrableFromConfig
+
+from .objects import InputDoc, OutputFormat, Result
+
+
+class Pipeline(RegistrableFromConfig, ABC):
+    @abstractmethod
+    async def extract_content(
+        self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
+    ) -> AsyncGenerator[Result, None]: ...

extract_core/utils.py

@@ -0,0 +1,9 @@
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def all_subclasses(cls: type[T]) -> set[type[T]]:
+    return set(cls.__subclasses__()).union(
+        [s for c in cls.__subclasses__() for s in all_subclasses(c)]
+    )

extract_core-0.5.4.dist-info/METADATA

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: extract-core
+Version: 0.5.4
+Summary: Core of extract-python
+Project-URL: Homepage, https://github.com/ICIJ/extract-python
+Project-URL: Repository, https://github.com/ICIJ/extract-python
+Project-URL: Issues, https://github.com/ICIJ/extract-python/issues
+Author-email: Clément Doumouro <cdoumouro@icij.org>
+Requires-Python: <3.14,>=3.11
+Requires-Dist: docling-slim~=2.96
+Requires-Dist: icij-common~=0.8.2
+Requires-Dist: marker-pdf~=1.10
+Requires-Dist: mineru~=3.2
+Requires-Dist: pydantic-extra-types[pycountry]>=2.11.1
+Requires-Dist: pydantic~=2.13

extract_core-0.5.4.dist-info/RECORD

@@ -0,0 +1,11 @@
+extract_core/__init__.py,sha256=ClGxkohwwJDt75FJoIfje23FuZNrXiEmA4FOu3kNcgY,1449
+extract_core/configs.py,sha256=jTg9eqT4ifqZopjVvQiNqYa88EbW6nmiFXV7O24rTOg,781
+extract_core/docling_.py,sha256=qxJHrQkoSUGunZTbedmeioFuMApavgM73QHc8NW9SKU,8987
+extract_core/marker_.py,sha256=IPHOlxJ6yLw4dkm-BnFEJWrma6_ItqW0m7L1w8_KpLI,1151
+extract_core/miner_u.py,sha256=q7gLPqVRSW1QdVUgPiBoBibLncm9GbmdfQI8yMRFzpk,2516
+extract_core/objects.py,sha256=UvjbPN--kbHOz5KXcCWvboBpIwilvVajndvdp-Og5Tc,7950
+extract_core/pipeline.py,sha256=Q5ZEBM3J50B2pQT2E-uhSIpd1i1tlsb47WfqBAdsFZ8,453
+extract_core/utils.py,sha256=MaxiOac9YS_fBU_-DhFFSLoEC_VrRuSR96-9M99-NTg,216
+extract_core-0.5.4.dist-info/METADATA,sha256=uN4ejrGJeJ7bFtQQ0F3dAUywy113c2EI4JE4Qgb7x4k,576
+extract_core-0.5.4.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+extract_core-0.5.4.dist-info/RECORD,,

extract_core-0.5.4.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any