extract-python 0.5.14__tar.gz → 0.6.0__tar.gz

{extract_python-0.5.14 → extract_python-0.6.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: extract-python
-Version: 0.5.14
+Version: 0.6.0
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python
 Project-URL: Issues, https://github.com/ICIJ/extract-python/issues
 Author-email: Clément Doumouro <cdoumouro@icij.org>
 Requires-Python: <3.14,>=3.11
-Requires-Dist: extract-core~=0.5.5
+Requires-Dist: extract-core~=0.6.0
 Requires-Dist: icij-common~=0.8.2
 Provides-Extra: benches
 Requires-Dist: html2image~=2.0.7; extra == 'benches'

{extract_python-0.5.14 → extract_python-0.6.0}/extract_python/docling_.py

@@ -1,11 +1,13 @@
 import asyncio
+import json
+import logging
 import shutil
 import tempfile
 from collections.abc import AsyncGenerator, Iterable, Iterator
+from functools import partial
 from pathlib import Path
 from typing import Any, Self
 
-from docling.datamodel.base_models import InputFormat
 from docling.datamodel.document import ConversionResult
 from docling.datamodel.pipeline_options import PipelineOptions
 from docling.document_converter import DocumentConverter, FormatOption
@@ -15,7 +17,6 @@ from docling_core.types.doc import ImageRefMode
 from docling_core.types.io import DocumentStream
 from extract_core import (
     BaseModel,
-    Device,
     DoclingFormatOption,
     DoclingPipelineConfig,
     Error,
@@ -29,28 +30,29 @@ from extract_core import (
     Status,
 )
 from icij_common.pydantic_utils import merge_configs
-from icij_common.registrable import FromConfig
 from pydantic import ConfigDict, field_serializer
 from pydantic_core.core_schema import SerializerFunctionWrapHandler
 
 from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP
 from .utils import chdir, map_and_preserve, path_to_artifacts_dirname
 
+logger = logging.getLogger(__name__)
+
 DOCLING_DEFAULT_ARTIFACTS_PATH = Path.home().joinpath(".cache", "docling", "models")
 
 
 @Pipeline.register(PipelineType.DOCLING)
 class DoclingPipeline(Pipeline):
-    def __init__(
-        self,
-        format_options: dict["InputFormat", DoclingFormatOption] | None = None,
-        *,
-        device: Device = Device.CPU,
-    ):
-        super().__init__(device)
+    def __init__(self, config: DoclingPipelineConfig):
+        super().__init__(config)
         format_options = {
-            k: v.to_docling(self._device) for k, v in format_options.items()
+            k: v.to_docling(self._device)
+            for k, v in self._config.format_options.items()
         }
+        logger.info(
+            "resolved format options to: %s",
+            lambda: partial(json.dumps, format_options, indent=2),
+        )
         allowed_format = [
             f.to_docling() for f in DoclingPipelineConfig.supported_exts()
         ]
@@ -72,15 +74,6 @@ class DoclingPipeline(Pipeline):
             doc = next(docs)
             yield _to_result(res, doc, output_format, output_path=output_path)
 
-    @classmethod
-    def _from_config(
-        cls,
-        config: DoclingPipelineConfig,
-        *,
-        device: Device = Device.CPU,
-    ) -> FromConfig:
-        return cls(config.format_options, device=device)
-
 
 def _to_docling(docs: Iterable[InputDoc]) -> Iterator["Path | DocumentStream"]:
     for d in docs:

{extract_python-0.5.14 → extract_python-0.6.0}/extract_python/marker_.py

@@ -2,13 +2,10 @@ import asyncio
 import gc
 from collections.abc import AsyncGenerator, Iterable
 from copy import deepcopy
-from functools import cache
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar, Self
+from typing import TYPE_CHECKING
 
 from extract_core import (
-    BasePipelineConfig,
-    Device,
     InputDoc,
     MarkdownDoc,
     OutputFormat,
@@ -17,9 +14,7 @@ from extract_core import (
     PipelineType,
     Result,
     Status,
-    SupportedExt,
 )
-from pydantic import Field
 
 from .constants import ARTIFACTS
 from .utils import path_to_artifacts_dirname, report_recoverable_errors
@@ -29,55 +24,11 @@ if TYPE_CHECKING:
     from PIL import Image
 
 
-class MarkerPipelineConfig(BasePipelineConfig):
-    pipeline: ClassVar[PipelineType] = Field(frozen=True, default=PipelineType.MARKER)
-
-    config: dict[str, Any] = Field(default_factory=dict)
-
-    @classmethod
-    @cache
-    def supported_exts(cls) -> set[SupportedExt]:
-        # Subset of https://documentation.datalab.to/docs/common/supportedfiletypes
-        return {
-            SupportedExt.PDF,
-            SupportedExt.XLS,
-            SupportedExt.XLSX,
-            SupportedExt.XLSM,
-            SupportedExt.CSV,
-            SupportedExt.ODS,
-            SupportedExt.DOC,
-            SupportedExt.DOCX,
-            SupportedExt.ODT,
-            SupportedExt.PPT,
-            SupportedExt.PPTX,
-            SupportedExt.ODP,
-            SupportedExt.HTLM,
-            SupportedExt.EPUB,
-            SupportedExt.PNG,
-            SupportedExt.JPG,
-            SupportedExt.JPEG,
-            SupportedExt.WEBP,
-            SupportedExt.GIF,
-            SupportedExt.TIFF,
-        }
-
-
 _MARKER_CONVERSION_ERRORS = tuple()
 
 
 @Pipeline.register(PipelineType.MARKER)
 class MarkerPipeline(Pipeline):
-    def __init__(
-        self,
-        marker_config: dict[str, Any] | None = None,
-        *,
-        device: Device = Device.CPU,
-    ):
-        super().__init__(device)
-        if marker_config is None:
-            marker_config = dict()
-        self._marker_config = marker_config
-
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
     ) -> AsyncGenerator[Result, None]:
@@ -85,7 +36,7 @@ class MarkerPipeline(Pipeline):
         from marker.converters.pdf import PdfConverter  # noqa: PLC0415
         from marker.models import create_model_dict  # noqa: PLC0415
 
-        config = deepcopy(self._marker_config)
+        config = deepcopy(self._config.config)
         config["output_format"] = output_format.to_marker()
         config_parser = ConfigParser(config)
         renderer = config_parser.get_renderer()
@@ -98,15 +49,6 @@ class MarkerPipeline(Pipeline):
         for doc in docs:
             yield await _process_doc(doc, converter, output_format, output_path)
 
-    @classmethod
-    def _from_config(
-        cls,
-        config: MarkerPipelineConfig,
-        *,
-        device: Device = Device.CPU,
-    ) -> Self:
-        return cls(config.config, device=device)
-
 
 @report_recoverable_errors(_MARKER_CONVERSION_ERRORS)
 async def _process_doc(

{extract_python-0.5.14 → extract_python-0.6.0}/extract_python/miner_u.py

@@ -5,14 +5,11 @@ from collections.abc import AsyncGenerator, Callable, Iterable
 from functools import partial
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Self
 
 from extract_core import (
     ConversionOutput,
-    Device,
     InputDoc,
     MinerUBackend,
-    MinerUConfig,
     MinerUPipelineConfig,
     OutputFormat,
     PageIndexes,
@@ -31,13 +28,10 @@ MDMakeFunction = Callable[[list, str, str], str | None]
 
 @Pipeline.register(PipelineType.MINER_U)
 class MinerUPipeline(Pipeline):
-    def __init__(
-        self, config: MinerUConfig, language: str, *, device: Device = Device.CPU
-    ):
-        super().__init__(device)
-        self._config = config
-        self._language = language
-        self._md_make_fn = _parse_md_make_fn(config.backend)
+    def __init__(self, config: MinerUPipelineConfig):
+        super().__init__(config)
+        self._language = self._config.language
+        self._md_make_fn = _parse_md_make_fn(self._config.config.backend)
 
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
@@ -59,7 +53,7 @@ class MinerUPipeline(Pipeline):
                     pdf_file_names=pdfs_names,
                     pdf_bytes_list=pdfs_bytes,
                     p_lang_list=p_lang_list,
-                    **self._config.as_parse_kwargs(),
+                    **self._config.config.as_parse_kwargs(),
                 )
                 res_paths = [
                     _revert_mineru_output(workdir, pdf_filename=p) for p in pdfs_names
@@ -73,15 +67,6 @@ class MinerUPipeline(Pipeline):
                         output_path=output_path,
                     )
 
-    @classmethod
-    def _from_config(
-        cls,
-        config: MinerUPipelineConfig,
-        *,
-        device: Device = Device.CPU,
-    ) -> Self:
-        return cls(config.config, language=config.language, device=device)
-
 
 def _revert_mineru_output(output_dir: Path, *, pdf_filename: str) -> Path:
     output_path = output_dir / pdf_filename

{extract_python-0.5.14 → extract_python-0.6.0}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 requires-python = ">=3.11,<3.14"
 dependencies = [
   "icij-common~=0.8.2",
-  "extract-core~=0.5.5",
+  "extract-core~=0.6.0",
 ]
 
 [project.optional-dependencies]
@@ -51,14 +51,28 @@ override-dependencies = [
   "pillow==11.3.0",
 ]
 
+[[tool.uv.index]]
+name = "pytorch-cpu"
+url = "https://download.pytorch.org/whl/cpu"
+explicit = true
+
+
 [tool.uv.sources]
 extract-core = { path = "../extract-core", editable = true }
+torch = [
+  { index = "pytorch-cpu" },
+]
+torchvision = [
+  { index = "pytorch-cpu" },
+]
 
 [dependency-groups]
 dev = [
   "pytest~=8.3.5",
   "pytest-asyncio~=0.25.3",
   "ruff==0.15.2",
+  "torch==2.12.0",
+  "torchvision==0.27.0",
 ]
 
 [project.urls]