extract-python 0.2.1__py3-none-any.whl → 0.3.1__py3-none-any.whl

extract_python/docling_.py

@@ -3,28 +3,13 @@ import tempfile
 from collections.abc import AsyncGenerator, Iterable, Iterator
 from functools import cache
 from pathlib import Path
-from typing import Any, ClassVar, Literal, TypeVar
-
-from docling.backend.abstract_backend import AbstractDocumentBackend
-from docling.datamodel.base_models import InputFormat
-from docling.datamodel.document import ConversionResult
-from docling.datamodel.pipeline_options import (
-    EasyOcrOptions,
-    PdfPipelineOptions,
-    PipelineOptions,
-    VlmPipelineOptions,
-)
-from docling.document_converter import DocumentConverter, FormatOption
-from docling.models.factories import get_ocr_factory
-from docling.pipeline.base_pipeline import BasePipeline
-from docling_core.types.doc import ImageRefMode
-from docling_core.types.io import DocumentStream
+from typing import TYPE_CHECKING, Annotated, ClassVar, TypeVar
+
 from icij_common.registrable import FromConfig
-from pydantic import Field, model_validator
+from pydantic import AfterValidator, Field
 
 from .constants import ARTIFACTS, CPU_GROUP, DEFAULT_MD_PAGE_SEP
 from .objects import (
-    BaseModel,
     Error,
     InputDoc,
     MarkdownDoc,
@@ -39,72 +24,43 @@ from .utils import all_subclasses, chdir, map_and_preserve, path_to_artifacts_di
 
 DOCLING_DEFAULT_ARTIFACTS_PATH = Path.home().joinpath(".cache", "docling", "models")
 
+if TYPE_CHECKING:
+    from docling.datamodel.base_models import InputFormat
+    from docling.datamodel.pipeline_options import PipelineOptions
+    from docling.document_converter import ConversionResult, FormatOption
+    from docling_core.types.io import DocumentStream
 
-class _PdfPipelineOptions(PdfPipelineOptions):
-    generate_picture_images: bool = Field(default=True, frozen=True)
 
-    @model_validator(mode="before")
-    @classmethod
-    def validate_ocr_options(cls, data: Any) -> Any:
-        if isinstance(data, dict):
-            ocr_options = data.get("ocr_options")
-            if not isinstance(ocr_options, dict):
-                return data
-            allow_external_plugins = ocr_options.get("allow_external_plugins", False)
-            ocr_factory = get_ocr_factory(allow_external_plugins=allow_external_plugins)
-            kind = ocr_options.pop("kind")
-            data["ocr_options"] = ocr_factory.create_options(kind=kind, **ocr_options)
-        return data
-
-
-OptionsByPipeline = list[
-    tuple[Literal["pdf"], _PdfPipelineOptions]
-    | tuple[Literal["vlm"], VlmPipelineOptions]
-]
-
-
-def _default_pipeline_options() -> OptionsByPipeline:
-    pipeline_options = _PdfPipelineOptions(ocr_options=EasyOcrOptions())
-    return [("pdf", pipeline_options), ("vlm", VlmPipelineOptions())]
-
-
-class DoclingFormatOption(BaseModel):
-    pipeline_cls: str
-    backend_cls: str
-
-    def to_docling(
-        self, pipeline_options: dict[Literal["pdf", "vlm"], PipelineOptions]
-    ) -> FormatOption:
-        pipeline_cls = _find_subcls(BasePipeline, self.pipeline_cls)
-        backend_cls = _find_subcls(AbstractDocumentBackend, self.backend_cls)
-        if "vlm" in self.pipeline_cls.lower():
-            pipeline_options = pipeline_options.get("vlm")
-            if pipeline_options is not None:
-                pipeline_options = VlmPipelineOptions.model_validate(pipeline_options)
-        elif "pdf" in self.pipeline_cls.lower():
-            pipeline_options = pipeline_options.get("pdf")
-            if pipeline_options is not None:
-                pipeline_options = _PdfPipelineOptions.model_validate(pipeline_options)
-        else:
-            raise ValueError(
-                f"invalid pipeline_cls: {pipeline_cls}, expected a VLM or PDF pipeline"
-            )
-        return FormatOption(
-            pipeline_cls=pipeline_cls,
-            pipeline_options=pipeline_options,
-            backend=backend_cls,
-        )
+def _validate_pipeline_opts(opts: "PipelineOptions") -> None:
+    from docling.datamodel.pipeline_options import PdfPipelineOptions
+
+    if isinstance(opts, PdfPipelineOptions) and not opts.generate_picture_images:
+        msg = "generate_picture_images should be set to true"
+        raise ValueError(msg)
+
+
+def _validate_options(
+    data: dict["InputFormat", "FormatOption"],
+) -> dict["InputFormat", "FormatOption"]:
+    for opts in data.values():
+        _validate_pipeline_opts(opts.pipeline_options)
+    return data
 
 
 @cache
-def _default_format_options() -> dict[InputFormat, DoclingFormatOption]:
-    supported_fmt = {InputFormat.PDF}
+def _default_format_opts() -> dict["InputFormat", "FormatOption"]:
+    from docling.datamodel.pipeline_options import (
+        EasyOcrOptions,
+        PdfPipelineOptions,
+    )
+    from docling.document_converter import PdfFormatOption
+
     return {
-        fmt: DoclingFormatOption(
-            pipeline_cls=opt.pipeline_cls.__name__, backend_cls=opt.backend.__name__
-        )
-        for fmt, opt in DocumentConverter().format_to_options.items()
-        if fmt in supported_fmt
+        InputFormat.PDF: PdfFormatOption(
+            pipeline_options=PdfPipelineOptions(
+                ocr_options=EasyOcrOptions(), generate_picture_images=True
+            )
+        ),
     }
 
 
@@ -123,54 +79,41 @@ class DoclingPipelineConfig(PipelineConfig):
     pipeline: PipelineType = Field(frozen=True, default=PipelineType.DOCLING)
     task_group: ClassVar[str] = Field(frozen=True, default=CPU_GROUP)
 
-    pipeline_options: OptionsByPipeline = Field(
-        default_factory=_default_pipeline_options
-    )
-    format_options: dict[InputFormat, DoclingFormatOption] = Field(
-        default_factory=_default_format_options
-    )
+    format_options: Annotated[
+        dict[InputFormat, FormatOption] | None, AfterValidator(_validate_options)
+    ] = Field(default_factory=_default_format_opts)
 
-    def to_format_options(self) -> dict[InputFormat, FormatOption]:
-        pipeline_options = dict(self.pipeline_options)
-        return {
-            InputFormat(f): opt.to_docling(pipeline_options)
-            for f, opt in self.format_options.items()
-        }
+    _unsupported_input_formats: ClassVar[set[InputFormat]] = {
+        InputFormat.AUDIO,
+        InputFormat.METS_GBS,
+        InputFormat.VTT,
+    }
 
     @classmethod
     @cache
-    def supported_formats(cls) -> set[SupportedExt]:
-        # Subset of https://docling-project.github.io/docling/usage/supported_formats/
-        return {
-            SupportedExt.ADOC,
-            SupportedExt.ASCIIDOC,
-            SupportedExt.BMP,
-            SupportedExt.CSV,
-            SupportedExt.DOCX,
-            SupportedExt.HTLM,
-            SupportedExt.JPG,
-            SupportedExt.MD,
-            SupportedExt.PDF,
-            SupportedExt.PNG,
-            SupportedExt.PPTX,
-            SupportedExt.TEX,
-            SupportedExt.TIFF,
-            SupportedExt.TXT,
-            SupportedExt.WEBP,
-            SupportedExt.XHTML,
-            SupportedExt.XLSX,
-        }
-
-
-DEFAULT_FORMAT_OPTIONS = DoclingPipelineConfig().to_format_options()
+    def supported_exts(cls) -> set[SupportedExt]:
+        from docling.datamodel.base_models import FormatToExtensions, InputFormat
+
+        supported = set()
+        for f in InputFormat:
+            if f in cls._unsupported_input_formats:
+                continue
+            for ext in FormatToExtensions[f]:
+                supported.add(SupportedExt(f".{ext.lower()}"))
+        return supported
 
 
 @Pipeline.register(PipelineType.DOCLING)
 class DoclingPipeline(Pipeline):
     def __init__(self, format_options: dict[InputFormat, FormatOption] | None = None):
-        if format_options is None:
-            format_options = DEFAULT_FORMAT_OPTIONS
-        self._converter = DocumentConverter(format_options=format_options)
+        from docling.document_converter import DocumentConverter
+
+        allowed_format = [
+            f.to_docling() for f in DoclingPipelineConfig.supported_exts()
+        ]
+        self._converter = DocumentConverter(
+            allowed_formats=allowed_format, format_options=format_options
+        )
 
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
@@ -182,16 +125,16 @@ class DoclingPipeline(Pipeline):
 
     @classmethod
     def _from_config(cls, config: DoclingPipelineConfig) -> FromConfig:
-        return cls(config.to_format_options())
+        return cls(config.format_options)
 
 
-def _to_docling(docs: Iterable[InputDoc]) -> Iterator[Path | DocumentStream]:
+def _to_docling(docs: Iterable[InputDoc]) -> Iterator[Path | "DocumentStream"]:
     for d in docs:
         yield d.to_docling()
 
 
 def _to_result(
-    res: ConversionResult,
+    res: "ConversionResult",
     input_document: InputDoc,
     output_format: OutputFormat,
     output_path: Path,
@@ -212,11 +155,13 @@ def _to_result(
 
 
 def _to_markdown_doc(
-    res: ConversionResult,
+    res: "ConversionResult",
     output_path: Path,
     page_sep: str = DEFAULT_MD_PAGE_SEP,
     **kwargs,
 ) -> MarkdownDoc:
+    from docling_core.types.doc import ImageRefMode
+
     # TODO: Should we add a hash to avoid collision between files with same names
     #  nested in the tree structured
     md_dir_name = path_to_artifacts_dirname(res.input.file)

extract_python/marker_.py

@@ -3,14 +3,8 @@ from collections.abc import AsyncGenerator, Iterable
 from copy import deepcopy
 from functools import cache
 from pathlib import Path
-from typing import Any, ClassVar, Self
-
-from marker.config.parser import ConfigParser
-from marker.converters.pdf import PdfConverter
-from marker.models import create_model_dict
-from marker.output import text_from_rendered
-from marker.renderers.markdown import MarkdownRenderer
-from PIL.Image import Image
+from typing import TYPE_CHECKING, Any, ClassVar, Self
+
 from pydantic import Field
 
 from .constants import ARTIFACTS, CPU_GROUP
@@ -26,6 +20,10 @@ from .objects import (
 from .pipeline import Pipeline, PipelineConfig, PipelineType
 from .utils import path_to_artifacts_dirname, report_recoverable_errors
 
+if TYPE_CHECKING:
+    from marker.converters.pdf import PdfConverter
+    from PIL import Image
+
 
 @PipelineConfig.register()
 class MarkerPipelineConfig(PipelineConfig):
@@ -36,7 +34,7 @@ class MarkerPipelineConfig(PipelineConfig):
 
     @classmethod
     @cache
-    def supported_formats(cls) -> set[SupportedExt]:
+    def supported_exts(cls) -> set[SupportedExt]:
         # Subset of https://documentation.datalab.to/docs/common/supportedfiletypes
         return {
             SupportedExt.PDF,
@@ -75,6 +73,10 @@ class MarkerPipeline(Pipeline):
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
     ) -> AsyncGenerator[Result, None]:
+        from marker.config.parser import ConfigParser
+        from marker.converters.pdf import PdfConverter
+        from marker.models import create_model_dict
+
         config = deepcopy(self._marker_config)
         config["output_format"] = output_format.to_marker()
         config_parser = ConfigParser(config)
@@ -96,10 +98,12 @@ class MarkerPipeline(Pipeline):
 @report_recoverable_errors(_MARKER_CONVERSION_ERRORS)
 def _process_doc(
     doc: InputDoc,
-    converter: PdfConverter,
+    converter: "PdfConverter",
     output_format: OutputFormat,
     output_path: Path,
 ) -> Result:
+    from marker.output import text_from_rendered
+
     rendered = converter(str(doc.path))
     content, _, images = text_from_rendered(rendered)
     match output_format:
@@ -112,8 +116,10 @@ def _process_doc(
 
 
 def _to_markdown_doc(
-    input_doc: InputDoc, content: str, images: dict[str, Image], output_path: Path
+    input_doc: InputDoc, content: str, images: dict[str, "Image"], output_path: Path
 ) -> MarkdownDoc:
+    from marker.renderers.markdown import MarkdownRenderer
+
     # TODO: Should we add a hash to avoid collision between files with same names
     #  nested in the tree structured
     md_dir_name = path_to_artifacts_dirname(input_doc.path)

extract_python/miner_u.py

@@ -8,12 +8,6 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, ClassVar, Self
 
-from mineru.backend.pipeline.pipeline_middle_json_mkcontent import (
-    union_make as pipeline_union_make,
-)
-from mineru.backend.vlm.vlm_middle_json_mkcontent import union_make as vlm_union_make
-from mineru.cli.common import aio_do_parse
-from mineru.utils.enum_class import MakeMode
 from pydantic import Field
 from pydantic_extra_types.language_code import LanguageAlpha2
 
@@ -47,33 +41,39 @@ class MinerUConfig(BaseModel):
     # TODO: use enum or literal here
     parse_method: str = "auto"
 
-    default_kwargs: ClassVar[dict] = {
-        "server_url": None,
-        # We don't dump md directly we process, we dump the middle json in order to be
-        # able to get page indexes
-        "parse_method": "auto",
-        "dump_md": False,
-        "dump_middle_json": True,
-        "f_draw_layout_bbox": False,
-        "f_draw_span_bbox": False,
-        "f_dump_model_output": False,  # might be useful for debug though
-        "f_dump_orig_pdf": False,
-        "f_dump_content_list": False,  # might be useful for debug though
-        "start_page_id": 0,
-        "f_make_md_mode": MakeMode.MM_MD,
-        "image_analysis": True,
-        "end_page_id": None,
-        "client_side_output_generation": False,
-    }
-
     def as_parse_kwargs(self) -> dict[str, Any]:
-        kwargs = copy(self.default_kwargs)
+        kwargs = copy(self._get_default_kwargs())
         kwargs["backend"] = self.backend
         kwargs["parse_method"] = self.parse_method
         kwargs["formula_enable"] = self.enable_formula_extraction
         kwargs["table_enable"] = self.enable_table_extraction
         return kwargs
 
+    @classmethod
+    @cache
+    def _get_default_kwargs(cls) -> dict[str, Any]:
+
+        from mineru.utils.enum_class import MakeMode
+
+        return {
+            "server_url": None,
+            # We don't dump md directly we process, we dump the middle json in order to be
+            # able to get page indexes
+            "parse_method": "auto",
+            "dump_md": False,
+            "dump_middle_json": True,
+            "f_draw_layout_bbox": False,
+            "f_draw_span_bbox": False,
+            "f_dump_model_output": False,  # might be useful for debug though
+            "f_dump_orig_pdf": False,
+            "f_dump_content_list": False,  # might be useful for debug though
+            "start_page_id": 0,
+            "f_make_md_mode": MakeMode.MM_MD,
+            "image_analysis": True,
+            "end_page_id": None,
+            "client_side_output_generation": False,
+        }
+
 
 @PipelineConfig.register()  # noqa: F821
 class MinerUPipelineConfig(PipelineConfig):  # noqa: F821
@@ -85,7 +85,7 @@ class MinerUPipelineConfig(PipelineConfig):  # noqa: F821
 
     @classmethod
     @cache
-    def supported_formats(cls) -> set[SupportedExt]:
+    def supported_exts(cls) -> set[SupportedExt]:
         return {
             SupportedExt.PDF,
             SupportedExt.DOCX,
@@ -104,6 +104,8 @@ class MinerUPipeline(Pipeline):
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
     ) -> AsyncGenerator[Result, None]:
+        from mineru.cli.common import aio_do_parse
+
         docs = list(docs)
         # TODO: exclude files which are not pdf and return an error
         pdfs_bytes = [d.path.read_bytes() for d in docs]
@@ -149,11 +151,18 @@ def _revert_mineru_output(output_dir: Path, *, pdf_filename: str) -> Path:
 
 
 def _parse_md_make_fn(backend: MinerUBackend) -> MDMakeFunction:
+
     match backend:
         case MinerUBackend.PIPELINE:
-            return pipeline_union_make
+            from mineru.backend.pipeline.pipeline_middle_json_mkcontent import (
+                union_make,
+            )
+
+            return union_make
         case MinerUBackend.VLM:
-            return vlm_union_make
+            from mineru.backend.vlm.vlm_middle_json_mkcontent import union_make
+
+            return union_make
         case _:
             raise ValueError(f"Unsupported backend: {backend}")
 
@@ -201,8 +210,12 @@ def _dump_md_content(
     output_path: Path,
     md_path: Path,
     im_dir: Path,
-    md_make_mode: str = MakeMode.MM_MD,
+    md_make_mode: str | None = None,
 ) -> ConversionOutput:
+    from mineru.utils.enum_class import MakeMode
+
+    if md_make_mode is None:
+        md_make_mode = MakeMode.MM_MD
     total_length = 0
     end_indices = []
     with md_path.open("w") as f:

extract_python/objects.py

@@ -21,7 +21,12 @@ from pydantic import AfterValidator, RootModel, TypeAdapter
 from pydantic import BaseModel as _BaseModel
 
 try:
-    from docling.datamodel.base_models import ConversionStatus, ErrorItem, InputFormat
+    from docling.datamodel.base_models import (
+        ConversionStatus,
+        ErrorItem,
+        FormatToExtensions,
+        InputFormat,
+    )
     from docling.datamodel.document import InputDocument
     from docling_core.types.io import DocumentStream
 except ImportError:
@@ -33,42 +38,78 @@ logger = logging.getLogger(__name__)
 base_config = merge_configs(icij_config(), no_enum_values_config())
 
 
+@cache
+def _ext_to_docling_input_format() -> dict:
+    from .docling_ import DoclingPipelineConfig  # noqa: PLC0415
+
+    mapping = dict()
+    supported = DoclingPipelineConfig.supported_exts()
+    for input_f, exts in FormatToExtensions.items():
+        for ext in exts:
+            try:
+                ext = SupportedExt(f".{ext.lower()}")  # noqa: PLW2901
+            except ValueError:
+                continue
+            if ext in supported:
+                mapping[ext] = input_f
+    return mapping
+
+
 class BaseModel(_BaseModel):
     model_config = base_config
 
 
 class SupportedExt(StrEnum):
     ADOC = ".adoc"
+    ASC = ".asc"
     ASCIIDOC = ".asciidoc"
     BMP = ".bmp"
     CSV = ".csv"
     DOC = ".doc"
     DOCX = ".docx"
+    DOTX = ".dotx"
+    DOTM = ".dotm"
+    DOCM = ".docm"
     EPUB = ".epub"
     GIF = ".gif"
     HTLM = ".html"
+    HTM = ".htm"
     JPEG = ".jpeg"
     JPG = ".jpg"
+    JSON = ".json"
+    LATEX = ".latex"
     MD = ".md"
+    NXML = ".nxml"
     ODP = ".odp"
     ODS = ".ods"
     ODT = ".odt"
     PDF = ".pdf"
     PNG = ".png"
+    PPSX = ".ppsx"
     PPT = ".ppt"
+    PPTM = ".pptm"
+    PPSM = ".ppsm"
+    POTX = ".potx"
+    POTM = ".potm"
     PPTX = ".pptx"
+    QMD = ".qmd"
+    RMD = ".rmd"
     TEX = ".tex"
+    TIF = ".tif"
     TIFF = ".tiff"
     TXT = ".txt"
+    TEXT = ".text"
     WEBP = ".webp"
+    XBRL = ".xbrl"
     XHTML = ".xhtml"
     XLS = ".xls"
     XLSM = ".xlsm"
     XLSX = ".xlsx"
     XLTX = ".xltx"
+    XML = ".xml"
 
     def to_docling(self) -> InputFormat:
-        return InputFormat(self.value[1:])
+        return _ext_to_docling_input_format()[self]
 
 
 class OutputFormat(StrEnum):

extract_python/pipeline.py

@@ -30,7 +30,7 @@ class PipelineConfig(RegistrableConfig, ABC):
 
     @classmethod
     @abstractmethod
-    def supported_formats(cls) -> set[SupportedExt]: ...
+    def supported_exts(cls) -> set[SupportedExt]: ...
 
 
 class Pipeline(RegistrableFromConfig, ABC):

{extract_python-0.2.1.dist-info → extract_python-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: extract-python
-Version: 0.2.1
+Version: 0.3.1
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python

extract_python-0.3.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+extract_python/__init__.py,sha256=Y-lcFbJd5sX3wt1UnyYAbm7H81Sgg7p2fG07CuuJUco,945
+extract_python/constants.py,sha256=JmAkjXyQMYwpTod9DCLc11zOc3caK2j_2ji_r3hGZws,236
+extract_python/docling_.py,sha256=lWWQ2PT5qOFUcJkeKw8ibF4JxzxQBgf93_CfvNcykDg,7041
+extract_python/marker_.py,sha256=ocRFxWX__A-M31z7Qr67OMcWRvgGO_C3tyZpiKc-bXw,5027
+extract_python/miner_u.py,sha256=hwRFTvtWGN_mRuv0p6H7nKS89dTErQxI1yOrvh6238M,8010
+extract_python/objects.py,sha256=kCxg6m7j01Z1sNAGi8vniokMnw6Ry0gU2lXXH2uan8A,8744
+extract_python/pipeline.py,sha256=VhDvfCxMEKvhFbMA-yxWO7FEeErDoLQCHiTRNnrbI8Y,1204
+extract_python/utils.py,sha256=kXe0CyT6zGYDTOvxs3BJYxq2-cgJZ0d_IT3hdMbuXa8,1943
+extract_python-0.3.1.dist-info/METADATA,sha256=qtfZpwEIKgWzkfbxGYMVP-pNFMFAbLrZo1-hmDXcgvE,1132
+extract_python-0.3.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+extract_python-0.3.1.dist-info/RECORD,,

extract_python-0.2.1.dist-info/RECORD

@@ -1,11 +0,0 @@
-extract_python/__init__.py,sha256=Y-lcFbJd5sX3wt1UnyYAbm7H81Sgg7p2fG07CuuJUco,945
-extract_python/constants.py,sha256=JmAkjXyQMYwpTod9DCLc11zOc3caK2j_2ji_r3hGZws,236
-extract_python/docling_.py,sha256=L7RPOYTN4kuodsgfi7NVsXILRerbAZUDaYxqVofxOog,9267
-extract_python/marker_.py,sha256=EUGpBRAe9mE0QbSdMFdvE16_m-c-DeAtwZ8F79w2Mcg,4908
-extract_python/miner_u.py,sha256=Bse66I5Yj-PiOgejr3JjXXDkjCh46M9KuwTEB8QK5g4,7750
-extract_python/objects.py,sha256=hqa9ONk9KwylvQa6DvKIEQnnCgfy-T-d5SU2LpfmTcQ,7815
-extract_python/pipeline.py,sha256=0qkuqEcxEbc3_sy8gNbLPwq8IIlC8cfGaqk_5fNpOCM,1207
-extract_python/utils.py,sha256=kXe0CyT6zGYDTOvxs3BJYxq2-cgJZ0d_IT3hdMbuXa8,1943
-extract_python-0.2.1.dist-info/METADATA,sha256=iaIXzaha4s-kqJzkedwJn5VsvQGZhy3KJQJpDns8pR4,1132
-extract_python-0.2.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-extract_python-0.2.1.dist-info/RECORD,,