extract-python 0.4.1__tar.gz → 0.5.4__tar.gz

{extract_python-0.4.1 → extract_python-0.5.4}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: extract-python
-Version: 0.4.1
+Version: 0.5.4
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python
 Project-URL: Issues, https://github.com/ICIJ/extract-python/issues
 Author-email: Clément Doumouro <cdoumouro@icij.org>
 Requires-Python: <3.14,>=3.11
+Requires-Dist: extract-core~=0.1
 Requires-Dist: icij-common~=0.8.2
 Provides-Extra: benches
 Requires-Dist: html2image~=2.0.7; extra == 'benches'
@@ -21,4 +22,5 @@ Provides-Extra: mineru
 Requires-Dist: mineru[mlx]~=3.2; (sys_platform == 'darwin') and extra == 'mineru'
 Requires-Dist: mineru[pipeline,vlm]~=3.2; extra == 'mineru'
 Requires-Dist: pydantic-extra-types[pycountry]~=2.11; extra == 'mineru'
+Requires-Dist: python-pptx~=1.0; extra == 'mineru'
 Requires-Dist: six~=1.17; extra == 'mineru'

{extract_python-0.4.1 → extract_python-0.5.4}/benches/compare.ipynb

@@ -13,8 +13,8 @@
     "from extract_python.benches.compare import (\n",
     "    compare,\n",
     ")\n",
-    "from extract_python.pipelines import DoclingPipeline, MarkerPipeline\n",
-    "from extract_python.objects import InputDoc, OutputFormat"
+    "from extract_python.objects import InputDoc, OutputFormat\n",
+    "from extract_python.pipelines import DoclingPipeline, MarkerPipeline"
    ]
   },
   {

{extract_python-0.4.1 → extract_python-0.5.4}/benches/compare.py

@@ -3,12 +3,11 @@ from tempfile import TemporaryDirectory
 
 import markdown2
 import pypdfium2
-from extract_python.pipelines.utils import chdir
+from extract_core import BaseModel, OutputFormat, PageIndexes
+from extract_python.utils import chdir
 from html2image import Html2Image
 from PIL import Image, ImageDraw
 
-from extract_python.objects import BaseModel, OutputFormat, PageIndexes
-
 _WHITE_BACKGROUND_CSS = "body {background: white;}"
 
 
@@ -146,7 +145,10 @@ def _scan_pages(
         ).root
         for compared in comparison.compared
     ]
-    all_pages = zip(*all_pages)
+    all_pages = zip(*all_pages, strict=True)
     compared_names = (p.parent.name for p in comparison.compared)
-    pages = [dict(zip(compared_names, page_comp_ixs)) for page_comp_ixs in all_pages]
+    pages = [
+        dict(zip(compared_names, page_comp_ixs, strict=True))
+        for page_comp_ixs in all_pages
+    ]
     return pages

extract_python-0.5.4/extract_python/__init__.py

@@ -0,0 +1,23 @@
+try:
+    from .docling_ import DOCLING_DEFAULT_ARTIFACTS_PATH, DoclingPipeline
+except ImportError:
+    DOCKING_DEFAULT_ARTIFACTS_PATH, DoclingPipeline = None, None
+
+try:
+    from .marker_ import MarkerPipeline
+except ImportError:
+    MarkerPipeline = None
+
+
+try:
+    from .miner_u import MinerUPipeline
+except ImportError:
+    MinerUPipeline = None
+
+
+__all__ = [
+    "DoclingPipeline",
+    "DOCLING_DEFAULT_ARTIFACTS_PATH",
+    "MarkerPipeline",
+    "MinerUPipeline",
+]

extract_python-0.5.4/extract_python/constants.py

@@ -0,0 +1,2 @@
+ARTIFACTS = "artifacts"
+DEFAULT_MD_PAGE_SEP = '<div style="page-break-after: always;"></div>'

extract_python-0.5.4/extract_python/docling_.py

@@ -0,0 +1,130 @@
+import shutil
+import tempfile
+from collections.abc import AsyncGenerator, Iterable, Iterator
+from pathlib import Path
+
+from docling.datamodel.base_models import InputFormat
+from docling.datamodel.document import ConversionResult
+from docling.document_converter import DocumentConverter
+
+# TODO: this is long to load improve it
+from docling_core.types.doc import ImageRefMode
+from docling_core.types.io import DocumentStream
+from extract_core import (
+    DoclingFormatOption,
+    DoclingPipelineConfig,
+    Error,
+    InputDoc,
+    MarkdownDoc,
+    OutputFormat,
+    PageIndexes,
+    Pipeline,
+    PipelineType,
+    Result,
+    Status,
+)
+from icij_common.registrable import FromConfig
+
+from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP
+from .utils import chdir, map_and_preserve, path_to_artifacts_dirname
+
+DOCLING_DEFAULT_ARTIFACTS_PATH = Path.home().joinpath(".cache", "docling", "models")
+
+
+@Pipeline.register(PipelineType.DOCLING)
+class DoclingPipeline(Pipeline):
+    def __init__(
+        self, format_options: dict["InputFormat", DoclingFormatOption] | None = None
+    ):
+        format_options = {k: v.to_docling() for k, v in format_options.items()}
+        allowed_format = [
+            f.to_docling() for f in DoclingPipelineConfig.supported_exts()
+        ]
+        self._converter = DocumentConverter(
+            allowed_formats=allowed_format, format_options=format_options
+        )
+
+    async def extract_content(
+        self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
+    ) -> AsyncGenerator[Result, None]:
+        docs, path_or_streams = map_and_preserve(_to_docling, docs)
+        outputs = self._converter.convert_all(path_or_streams, raises_on_error=False)
+        for doc, res in zip(docs, outputs, strict=True):
+            yield _to_result(res, doc, output_format, output_path=output_path)
+
+    @classmethod
+    def _from_config(cls, config: DoclingPipelineConfig) -> FromConfig:
+        return cls(config.format_options)
+
+
+def _to_docling(docs: Iterable[InputDoc]) -> Iterator["Path | DocumentStream"]:
+    for d in docs:
+        yield d.to_docling()
+
+
+def _to_result(
+    res: ConversionResult,
+    input_document: InputDoc,
+    output_format: OutputFormat,
+    output_path: Path,
+    **kwargs,
+) -> Result:
+    output_path.mkdir(parents=True, exist_ok=True)
+    output = None
+    status = Status.from_docling(res.status)
+    if status.allows_conversion:
+        match output_format:
+            case OutputFormat.MARKDOWN:
+                output = _to_markdown_doc(res, output_path, **kwargs)
+            case _:
+                raise NotImplementedError(f"unsupported output format {output_format}")
+    errors = [Error.from_docling(e) for e in res.errors]
+    input_doc = input_document.without_content()
+    return Result(input=input_doc, status=status, errors=errors, output=output)
+
+
+def _to_markdown_doc(
+    res: ConversionResult,
+    output_path: Path,
+    page_sep: str = DEFAULT_MD_PAGE_SEP,
+    **kwargs,
+) -> MarkdownDoc:
+    # TODO: Should we add a hash to avoid collision between files with same names
+    #  nested in the tree structured
+    md_dir_name = path_to_artifacts_dirname(res.input.file)
+    md_dir = output_path / md_dir_name
+    if md_dir.exists():
+        raise FileExistsError(f"directory {md_dir} already exists")
+    # Let's avoid issue of duplicated input file names flattened top level
+    md_filename = md_dir_name + OutputFormat.MARKDOWN
+    total_length = 0
+    n_pages = len(res.pages)
+
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as td:
+        tmp_dir = Path(td)
+        page_path = Path("page.md")
+        # We do a chdir to bypass a Docling bug which only allows to maintain relative
+        # image ref when saving the markdown to a relative path
+        with (tmp_dir / md_filename).open("w") as f, chdir(tmp_dir):
+            end_indices = []
+            for page_i in range(n_pages):
+                res.document.save_as_markdown(
+                    page_path,
+                    page_no=page_i + 1,
+                    image_mode=ImageRefMode.REFERENCED,
+                    artifacts_dir=Path(ARTIFACTS),
+                    **kwargs,
+                )
+                content = page_path.read_text()
+                if page_i > 0:
+                    content += "\n"
+                if page_i < n_pages - 1:
+                    content += page_sep
+                total_length += len(content)
+                end_indices.append(total_length)
+                f.write(content)
+                f.flush()
+                page_path.unlink()
+        shutil.move(tmp_dir, md_dir)
+    pages = PageIndexes.from_page_end_indices(end_indices)
+    return MarkdownDoc(path=Path(md_dir_name), pages=pages)

{extract_python-0.4.1 → extract_python-0.5.4}/extract_python/marker_.py

@@ -5,10 +5,8 @@ from functools import cache
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Self
 
-from pydantic import Field
-
-from .constants import ARTIFACTS
-from .objects import (
+from extract_core import BasePipelineConfig, Pipeline, PipelineType
+from extract_core.objects import (
     InputDoc,
     MarkdownDoc,
     OutputFormat,
@@ -17,7 +15,9 @@ from .objects import (
     Status,
     SupportedExt,
 )
-from .pipeline import Pipeline, PipelineConfig, PipelineType
+from pydantic import Field
+
+from .constants import ARTIFACTS
 from .utils import path_to_artifacts_dirname, report_recoverable_errors
 
 if TYPE_CHECKING:
@@ -25,10 +25,10 @@ if TYPE_CHECKING:
     from PIL import Image
 
 
-class MarkerPipelineConfig(PipelineConfig):
+class MarkerPipelineConfig(BasePipelineConfig):
     pipeline: ClassVar[PipelineType] = Field(frozen=True, default=PipelineType.MARKER)
 
-    config: dict[str, Any] = dict()
+    config: dict[str, Any] = Field(default_factory=dict)
 
     @classmethod
     @cache

{extract_python-0.4.1 → extract_python-0.5.4}/extract_python/miner_u.py

@@ -1,96 +1,32 @@
 import json
 import shutil
 from collections.abc import AsyncGenerator, Callable, Iterable
-from copy import copy
-from enum import StrEnum
-from functools import cache, partial
+from functools import partial
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Any, ClassVar, Self
+from typing import Self
 
-from pydantic import Field
-from pydantic_extra_types.language_code import LanguageAlpha2
-
-from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP
-from .objects import (
-    BaseModel,
+from extract_core import (
     ConversionOutput,
     InputDoc,
+    MinerUBackend,
+    MinerUConfig,
+    MinerUPipelineConfig,
     OutputFormat,
     PageIndexes,
+    Pipeline,
+    PipelineType,
     Result,
     Status,
-    SupportedExt,
 )
-from .pipeline import Pipeline, PipelineConfig, PipelineType
+
+from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP
 from .utils import path_to_artifacts_dirname
 
 _MINER_U_CONVERSION_ERRORS = tuple()
 MDMakeFunction = Callable[[list, str, str], str | None]
 
 
-class MinerUBackend(StrEnum):
-    PIPELINE = "pipeline"
-    VLM = "vlm"
-
-
-class MinerUConfig(BaseModel):
-    backend: MinerUBackend = MinerUBackend.PIPELINE
-    enable_formula_extraction: bool = True
-    enable_table_extraction: bool = True
-    # TODO: use enum or literal here
-    parse_method: str = "auto"
-
-    def as_parse_kwargs(self) -> dict[str, Any]:
-        kwargs = copy(self._get_default_kwargs())
-        kwargs["backend"] = self.backend
-        kwargs["parse_method"] = self.parse_method
-        kwargs["formula_enable"] = self.enable_formula_extraction
-        kwargs["table_enable"] = self.enable_table_extraction
-        return kwargs
-
-    @classmethod
-    @cache
-    def _get_default_kwargs(cls) -> dict[str, Any]:
-        from mineru.utils.enum_class import MakeMode  # noqa: PLC0415
-
-        return {
-            "server_url": None,
-            # We don't dump md directly we process, we dump the middle json in order
-            # to be able to get page indexes
-            "parse_method": "auto",
-            "dump_md": False,
-            "dump_middle_json": True,
-            "f_draw_layout_bbox": False,
-            "f_draw_span_bbox": False,
-            "f_dump_model_output": False,  # might be useful for debug though
-            "f_dump_orig_pdf": False,
-            "f_dump_content_list": False,  # might be useful for debug though
-            "start_page_id": 0,
-            "f_make_md_mode": MakeMode.MM_MD,
-            "image_analysis": True,
-            "end_page_id": None,
-            "client_side_output_generation": False,
-        }
-
-
-class MinerUPipelineConfig(PipelineConfig):  # noqa: F821
-    pipeline: ClassVar[PipelineType] = Field(frozen=True, default=PipelineType.MINER_U)
-
-    config: MinerUConfig = Field(frozen=True, default=MinerUConfig())
-    language: LanguageAlpha2 = Field(frozen=True, default="en")
-
-    @classmethod
-    @cache
-    def supported_exts(cls) -> set[SupportedExt]:
-        return {
-            SupportedExt.PDF,
-            SupportedExt.DOCX,
-            SupportedExt.PPTX,
-            SupportedExt.XLSX,
-        }
-
-
 @Pipeline.register(PipelineType.MINER_U)
 class MinerUPipeline(Pipeline):
     def __init__(self, config: MinerUConfig, language: str):

{extract_python-0.4.1 → extract_python-0.5.4}/extract_python/utils.py

@@ -6,26 +6,20 @@ from itertools import tee
 from pathlib import Path, PurePath
 from typing import Protocol, TypeVar
 
-from .objects import Error, InputDoc, Result, Status
+from extract_core import Error, InputDoc, Result, Status
 
 R = TypeVar("R")
-T = TypeVar("T")
+In = TypeVar("In")
 
 
 def map_and_preserve(
-    fn: Callable[[Iterable[T]], Iterator[R]], inputs: Iterable[T]
-) -> tuple[Iterable[T], Iterator[R]]:
+    fn: Callable[[Iterable[In]], Iterator[R]], inputs: Iterable[In]
+) -> tuple[Iterable[In], Iterator[R]]:
     save_inputs, function_inputs = tee(inputs)
     outputs = iter(fn(function_inputs))
     return save_inputs, outputs
 
 
-def all_subclasses(cls: type[T]) -> set[type[T]]:
-    return set(cls.__subclasses__()).union(
-        [s for c in cls.__subclasses__() for s in all_subclasses(c)]
-    )
-
-
 def path_to_artifacts_dirname(path: PurePath, sep: str = "_") -> str:
     dirname = f"{path.name[: -len(path.suffix)]}"
     ext = path.suffix

{extract_python-0.4.1 → extract_python-0.5.4}/pyproject.toml

@@ -9,6 +9,7 @@ readme = "README.md"
 requires-python = ">=3.11,<3.14"
 dependencies = [
   "icij-common~=0.8.2",
+  "extract-core~=0.1",
 ]
 
 [project.optional-dependencies]
@@ -31,6 +32,7 @@ mineru = [
   "mineru[pipeline,vlm]~=3.2",
   "mineru[mlx]~=3.2; sys_platform == 'darwin'",
   "pydantic-extra-types[pycountry]~=2.11",
+  "python-pptx~=1.0",
   "six~=1.17",
 ]
 
@@ -45,7 +47,12 @@ required-environments = [
   "sys_platform == 'darwin' and platform_machine == 'arm64'",
   "sys_platform == 'linux'",
 ]
+override-dependencies = [
+  "pillow==11.3.0",
+]
 
+[tool.uv.sources]
+extract-core = { path = "../extract-core", editable = true }
 
 [dependency-groups]
 dev = [
@@ -87,4 +94,5 @@ exclude = [
 ]
 [tool.uv-dynamic-versioning]
 fallback-version = "0.0.0"
-pattern = "default-unprefixed"
+pattern-prefix = "extract-python-"
+pattern = "default-unprefixed"