extract-python 0.2.1__tar.gz → 0.3.0__tar.gz

{extract_python-0.2.1 → extract_python-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: extract-python
-Version: 0.2.1
+Version: 0.3.0
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python

{extract_python-0.2.1 → extract_python-0.3.0}/extract_python/docling_.py

@@ -3,28 +3,23 @@ import tempfile
 from collections.abc import AsyncGenerator, Iterable, Iterator
 from functools import cache
 from pathlib import Path
-from typing import Any, ClassVar, Literal, TypeVar
+from typing import Annotated, ClassVar, TypeVar
 
-from docling.backend.abstract_backend import AbstractDocumentBackend
-from docling.datamodel.base_models import InputFormat
+from docling.datamodel.base_models import FormatToExtensions, InputFormat
 from docling.datamodel.document import ConversionResult
 from docling.datamodel.pipeline_options import (
     EasyOcrOptions,
     PdfPipelineOptions,
     PipelineOptions,
-    VlmPipelineOptions,
 )
-from docling.document_converter import DocumentConverter, FormatOption
-from docling.models.factories import get_ocr_factory
-from docling.pipeline.base_pipeline import BasePipeline
+from docling.document_converter import DocumentConverter, FormatOption, PdfFormatOption
 from docling_core.types.doc import ImageRefMode
 from docling_core.types.io import DocumentStream
 from icij_common.registrable import FromConfig
-from pydantic import Field, model_validator
+from pydantic import AfterValidator, Field
 
 from .constants import ARTIFACTS, CPU_GROUP, DEFAULT_MD_PAGE_SEP
 from .objects import (
-    BaseModel,
     Error,
     InputDoc,
     MarkdownDoc,
@@ -40,73 +35,27 @@ from .utils import all_subclasses, chdir, map_and_preserve, path_to_artifacts_di
 DOCLING_DEFAULT_ARTIFACTS_PATH = Path.home().joinpath(".cache", "docling", "models")
 
 
-class _PdfPipelineOptions(PdfPipelineOptions):
-    generate_picture_images: bool = Field(default=True, frozen=True)
+def _validate_pipeline_opts(opts: PipelineOptions) -> None:
+    if isinstance(opts, PdfPipelineOptions) and not opts.generate_picture_images:
+        msg = "generate_picture_images should be set to true"
+        raise ValueError(msg)
 
-    @model_validator(mode="before")
-    @classmethod
-    def validate_ocr_options(cls, data: Any) -> Any:
-        if isinstance(data, dict):
-            ocr_options = data.get("ocr_options")
-            if not isinstance(ocr_options, dict):
-                return data
-            allow_external_plugins = ocr_options.get("allow_external_plugins", False)
-            ocr_factory = get_ocr_factory(allow_external_plugins=allow_external_plugins)
-            kind = ocr_options.pop("kind")
-            data["ocr_options"] = ocr_factory.create_options(kind=kind, **ocr_options)
-        return data
-
-
-OptionsByPipeline = list[
-    tuple[Literal["pdf"], _PdfPipelineOptions]
-    | tuple[Literal["vlm"], VlmPipelineOptions]
-]
-
-
-def _default_pipeline_options() -> OptionsByPipeline:
-    pipeline_options = _PdfPipelineOptions(ocr_options=EasyOcrOptions())
-    return [("pdf", pipeline_options), ("vlm", VlmPipelineOptions())]
-
-
-class DoclingFormatOption(BaseModel):
-    pipeline_cls: str
-    backend_cls: str
 
-    def to_docling(
-        self, pipeline_options: dict[Literal["pdf", "vlm"], PipelineOptions]
-    ) -> FormatOption:
-        pipeline_cls = _find_subcls(BasePipeline, self.pipeline_cls)
-        backend_cls = _find_subcls(AbstractDocumentBackend, self.backend_cls)
-        if "vlm" in self.pipeline_cls.lower():
-            pipeline_options = pipeline_options.get("vlm")
-            if pipeline_options is not None:
-                pipeline_options = VlmPipelineOptions.model_validate(pipeline_options)
-        elif "pdf" in self.pipeline_cls.lower():
-            pipeline_options = pipeline_options.get("pdf")
-            if pipeline_options is not None:
-                pipeline_options = _PdfPipelineOptions.model_validate(pipeline_options)
-        else:
-            raise ValueError(
-                f"invalid pipeline_cls: {pipeline_cls}, expected a VLM or PDF pipeline"
-            )
-        return FormatOption(
-            pipeline_cls=pipeline_cls,
-            pipeline_options=pipeline_options,
-            backend=backend_cls,
-        )
+def _validate_options(
+    data: dict[InputFormat, FormatOption],
+) -> dict[InputFormat, FormatOption]:
+    for opts in data.values():
+        _validate_pipeline_opts(opts.pipeline_options)
+    return data
 
 
-@cache
-def _default_format_options() -> dict[InputFormat, DoclingFormatOption]:
-    supported_fmt = {InputFormat.PDF}
-    return {
-        fmt: DoclingFormatOption(
-            pipeline_cls=opt.pipeline_cls.__name__, backend_cls=opt.backend.__name__
+_DEFAULT_FORMAT_OPTS = {
+    InputFormat.PDF: PdfFormatOption(
+        pipeline_options=PdfPipelineOptions(
+            ocr_options=EasyOcrOptions(), generate_picture_images=True
         )
-        for fmt, opt in DocumentConverter().format_to_options.items()
-        if fmt in supported_fmt
-    }
-
+    ),
+}
 
 T = TypeVar("T")
 
@@ -123,54 +72,37 @@ class DoclingPipelineConfig(PipelineConfig):
     pipeline: PipelineType = Field(frozen=True, default=PipelineType.DOCLING)
     task_group: ClassVar[str] = Field(frozen=True, default=CPU_GROUP)
 
-    pipeline_options: OptionsByPipeline = Field(
-        default_factory=_default_pipeline_options
-    )
-    format_options: dict[InputFormat, DoclingFormatOption] = Field(
-        default_factory=_default_format_options
-    )
+    format_options: Annotated[
+        dict[InputFormat, FormatOption] | None, AfterValidator(_validate_options)
+    ] = _DEFAULT_FORMAT_OPTS
 
-    def to_format_options(self) -> dict[InputFormat, FormatOption]:
-        pipeline_options = dict(self.pipeline_options)
-        return {
-            InputFormat(f): opt.to_docling(pipeline_options)
-            for f, opt in self.format_options.items()
-        }
+    _unsupported_input_formats: ClassVar[set[InputFormat]] = {
+        InputFormat.AUDIO,
+        InputFormat.METS_GBS,
+        InputFormat.VTT,
+    }
 
     @classmethod
     @cache
-    def supported_formats(cls) -> set[SupportedExt]:
-        # Subset of https://docling-project.github.io/docling/usage/supported_formats/
-        return {
-            SupportedExt.ADOC,
-            SupportedExt.ASCIIDOC,
-            SupportedExt.BMP,
-            SupportedExt.CSV,
-            SupportedExt.DOCX,
-            SupportedExt.HTLM,
-            SupportedExt.JPG,
-            SupportedExt.MD,
-            SupportedExt.PDF,
-            SupportedExt.PNG,
-            SupportedExt.PPTX,
-            SupportedExt.TEX,
-            SupportedExt.TIFF,
-            SupportedExt.TXT,
-            SupportedExt.WEBP,
-            SupportedExt.XHTML,
-            SupportedExt.XLSX,
-        }
-
-
-DEFAULT_FORMAT_OPTIONS = DoclingPipelineConfig().to_format_options()
+    def supported_exts(cls) -> set[SupportedExt]:
+        supported = set()
+        for f in InputFormat:
+            if f in cls._unsupported_input_formats:
+                continue
+            for ext in FormatToExtensions[f]:
+                supported.add(SupportedExt(f".{ext.lower()}"))
+        return supported
 
 
 @Pipeline.register(PipelineType.DOCLING)
 class DoclingPipeline(Pipeline):
     def __init__(self, format_options: dict[InputFormat, FormatOption] | None = None):
-        if format_options is None:
-            format_options = DEFAULT_FORMAT_OPTIONS
-        self._converter = DocumentConverter(format_options=format_options)
+        allowed_format = [
+            f.to_docling() for f in DoclingPipelineConfig.supported_exts()
+        ]
+        self._converter = DocumentConverter(
+            allowed_formats=allowed_format, format_options=format_options
+        )
 
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
@@ -182,7 +114,7 @@ class DoclingPipeline(Pipeline):
 
     @classmethod
     def _from_config(cls, config: DoclingPipelineConfig) -> FromConfig:
-        return cls(config.to_format_options())
+        return cls(config.format_options)
 
 
 def _to_docling(docs: Iterable[InputDoc]) -> Iterator[Path | DocumentStream]:

{extract_python-0.2.1 → extract_python-0.3.0}/extract_python/marker_.py

@@ -36,7 +36,7 @@ class MarkerPipelineConfig(PipelineConfig):
 
     @classmethod
     @cache
-    def supported_formats(cls) -> set[SupportedExt]:
+    def supported_exts(cls) -> set[SupportedExt]:
         # Subset of https://documentation.datalab.to/docs/common/supportedfiletypes
         return {
             SupportedExt.PDF,

{extract_python-0.2.1 → extract_python-0.3.0}/extract_python/miner_u.py

@@ -85,7 +85,7 @@ class MinerUPipelineConfig(PipelineConfig):  # noqa: F821
 
     @classmethod
     @cache
-    def supported_formats(cls) -> set[SupportedExt]:
+    def supported_exts(cls) -> set[SupportedExt]:
         return {
             SupportedExt.PDF,
             SupportedExt.DOCX,

{extract_python-0.2.1 → extract_python-0.3.0}/extract_python/objects.py

@@ -21,7 +21,12 @@ from pydantic import AfterValidator, RootModel, TypeAdapter
 from pydantic import BaseModel as _BaseModel
 
 try:
-    from docling.datamodel.base_models import ConversionStatus, ErrorItem, InputFormat
+    from docling.datamodel.base_models import (
+        ConversionStatus,
+        ErrorItem,
+        FormatToExtensions,
+        InputFormat,
+    )
     from docling.datamodel.document import InputDocument
     from docling_core.types.io import DocumentStream
 except ImportError:
@@ -33,42 +38,78 @@ logger = logging.getLogger(__name__)
 base_config = merge_configs(icij_config(), no_enum_values_config())
 
 
+@cache
+def _ext_to_docling_input_format() -> dict:
+    from .docling_ import DoclingPipelineConfig  # noqa: PLC0415
+
+    mapping = dict()
+    supported = DoclingPipelineConfig.supported_exts()
+    for input_f, exts in FormatToExtensions.items():
+        for ext in exts:
+            try:
+                ext = SupportedExt(f".{ext.lower()}")  # noqa: PLW2901
+            except ValueError:
+                continue
+            if ext in supported:
+                mapping[ext] = input_f
+    return mapping
+
+
 class BaseModel(_BaseModel):
     model_config = base_config
 
 
 class SupportedExt(StrEnum):
     ADOC = ".adoc"
+    ASC = ".asc"
     ASCIIDOC = ".asciidoc"
     BMP = ".bmp"
     CSV = ".csv"
     DOC = ".doc"
     DOCX = ".docx"
+    DOTX = ".dotx"
+    DOTM = ".dotm"
+    DOCM = ".docm"
     EPUB = ".epub"
     GIF = ".gif"
     HTLM = ".html"
+    HTM = ".htm"
     JPEG = ".jpeg"
     JPG = ".jpg"
+    JSON = ".json"
+    LATEX = ".latex"
     MD = ".md"
+    NXML = ".nxml"
     ODP = ".odp"
     ODS = ".ods"
     ODT = ".odt"
     PDF = ".pdf"
     PNG = ".png"
+    PPSX = ".ppsx"
     PPT = ".ppt"
+    PPTM = ".pptm"
+    PPSM = ".ppsm"
+    POTX = ".potx"
+    POTM = ".potm"
     PPTX = ".pptx"
+    QMD = ".qmd"
+    RMD = ".rmd"
     TEX = ".tex"
+    TIF = ".tif"
     TIFF = ".tiff"
     TXT = ".txt"
+    TEXT = ".text"
     WEBP = ".webp"
+    XBRL = ".xbrl"
     XHTML = ".xhtml"
     XLS = ".xls"
     XLSM = ".xlsm"
     XLSX = ".xlsx"
     XLTX = ".xltx"
+    XML = ".xml"
 
     def to_docling(self) -> InputFormat:
-        return InputFormat(self.value[1:])
+        return _ext_to_docling_input_format()[self]
 
 
 class OutputFormat(StrEnum):

{extract_python-0.2.1 → extract_python-0.3.0}/extract_python/pipeline.py

@@ -30,7 +30,7 @@ class PipelineConfig(RegistrableConfig, ABC):
 
     @classmethod
     @abstractmethod
-    def supported_formats(cls) -> set[SupportedExt]: ...
+    def supported_exts(cls) -> set[SupportedExt]: ...
 
 
 class Pipeline(RegistrableFromConfig, ABC):

{extract_python-0.2.1 → extract_python-0.3.0}/pyproject.toml

@@ -51,6 +51,7 @@ required-environments = [
 dev = [
   "pytest~=8.3.5",
   "pytest-asyncio~=0.25.3",
+  "ruff==0.15.2",
 ]
 
 [project.urls]

{extract_python-0.2.1 → extract_python-0.3.0}/uv.lock

@@ -868,7 +868,6 @@ wheels = [
 
 [[package]]
 name = "extract-python"
-version = "0.1.0"
 source = { editable = "." }
 dependencies = [
     { name = "icij-common" },
@@ -898,11 +897,12 @@ mineru = [
 dev = [
     { name = "pytest" },
     { name = "pytest-asyncio" },
+    { name = "ruff" },
 ]
 
 [package.metadata]
 requires-dist = [
-    { name = "docling-slim", extras = ["standard", "feat-ocr-easyocr", "feat-ocr-tesserocr", "feat-ocr-mac"], marker = "extra == 'docling'", specifier = "~=2.96" },
+    { name = "docling-slim", extras = ["feat-ocr-easyocr", "feat-ocr-mac", "feat-ocr-tesserocr", "standard"], marker = "extra == 'docling'", specifier = "~=2.96" },
     { name = "html2image", marker = "extra == 'benches'", specifier = "~=2.0.7" },
     { name = "icij-common", specifier = "~=0.8.2" },
     { name = "markdown2", marker = "extra == 'benches'", specifier = ">=2.5.4" },
@@ -920,6 +920,7 @@ provides-extras = ["benches", "docling", "marker", "mineru"]
 dev = [
     { name = "pytest", specifier = "~=8.3.5" },
     { name = "pytest-asyncio", specifier = "~=0.25.3" },
+    { name = "ruff", specifier = "==0.15.2" },
 ]
 
 [[package]]
@@ -4334,6 +4335,31 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/3f/50/0a9e7e7afe7339bd5e36911f0ceb15fed51945836ed803ae5afd661057fd/rtree-1.4.1-py3-none-win_arm64.whl", hash = "sha256:3d46f55729b28138e897ffef32f7ce93ac335cb67f9120125ad3742a220800f0", size = 355253, upload-time = "2025-08-13T19:32:00.296Z" },
 ]
 
+[[package]]
+name = "ruff"
+version = "0.15.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/06/04/eab13a954e763b0606f460443fcbf6bb5a0faf06890ea3754ff16523dce5/ruff-0.15.2.tar.gz", hash = "sha256:14b965afee0969e68bb871eba625343b8673375f457af4abe98553e8bbb98342", size = 4558148, upload-time = "2026-02-19T22:32:20.271Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2f/70/3a4dc6d09b13cb3e695f28307e5d889b2e1a66b7af9c5e257e796695b0e6/ruff-0.15.2-py3-none-linux_armv6l.whl", hash = "sha256:120691a6fdae2f16d65435648160f5b81a9625288f75544dc40637436b5d3c0d", size = 10430565, upload-time = "2026-02-19T22:32:41.824Z" },
+    { url = "https://files.pythonhosted.org/packages/71/0b/bb8457b56185ece1305c666dc895832946d24055be90692381c31d57466d/ruff-0.15.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a89056d831256099658b6bba4037ac6dd06f49d194199215befe2bb10457ea5e", size = 10820354, upload-time = "2026-02-19T22:32:07.366Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/c1/e0532d7f9c9e0b14c46f61b14afd563298b8b83f337b6789ddd987e46121/ruff-0.15.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e36dee3a64be0ebd23c86ffa3aa3fd3ac9a712ff295e192243f814a830b6bd87", size = 10170767, upload-time = "2026-02-19T22:32:13.188Z" },
+    { url = "https://files.pythonhosted.org/packages/47/e8/da1aa341d3af017a21c7a62fb5ec31d4e7ad0a93ab80e3a508316efbcb23/ruff-0.15.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9fb47b6d9764677f8c0a193c0943ce9a05d6763523f132325af8a858eadc2b9", size = 10529591, upload-time = "2026-02-19T22:32:02.547Z" },
+    { url = "https://files.pythonhosted.org/packages/93/74/184fbf38e9f3510231fbc5e437e808f0b48c42d1df9434b208821efcd8d6/ruff-0.15.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f376990f9d0d6442ea9014b19621d8f2aaf2b8e39fdbfc79220b7f0c596c9b80", size = 10260771, upload-time = "2026-02-19T22:32:36.938Z" },
+    { url = "https://files.pythonhosted.org/packages/05/ac/605c20b8e059a0bc4b42360414baa4892ff278cec1c91fff4be0dceedefd/ruff-0.15.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dcc987551952d73cbf5c88d9fdee815618d497e4df86cd4c4824cc59d5dd75f", size = 11045791, upload-time = "2026-02-19T22:32:31.642Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/52/db6e419908f45a894924d410ac77d64bdd98ff86901d833364251bd08e22/ruff-0.15.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:42a47fd785cbe8c01b9ff45031af875d101b040ad8f4de7bbb716487c74c9a77", size = 11879271, upload-time = "2026-02-19T22:32:29.305Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/d8/7992b18f2008bdc9231d0f10b16df7dda964dbf639e2b8b4c1b4e91b83af/ruff-0.15.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbe9f49354866e575b4c6943856989f966421870e85cd2ac94dccb0a9dcb2fea", size = 11303707, upload-time = "2026-02-19T22:32:22.492Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/02/849b46184bcfdd4b64cde61752cc9a146c54759ed036edd11857e9b8443b/ruff-0.15.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7a672c82b5f9887576087d97be5ce439f04bbaf548ee987b92d3a7dede41d3a", size = 11149151, upload-time = "2026-02-19T22:32:44.234Z" },
+    { url = "https://files.pythonhosted.org/packages/70/04/f5284e388bab60d1d3b99614a5a9aeb03e0f333847e2429bebd2aaa1feec/ruff-0.15.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ecc64f46f7019e2bcc3cdc05d4a7da958b629a5ab7033195e11a438403d956", size = 11091132, upload-time = "2026-02-19T22:32:24.691Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/ae/88d844a21110e14d92cf73d57363fab59b727ebeabe78009b9ccb23500af/ruff-0.15.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:8dcf243b15b561c655c1ef2f2b0050e5d50db37fe90115507f6ff37d865dc8b4", size = 10504717, upload-time = "2026-02-19T22:32:26.75Z" },
+    { url = "https://files.pythonhosted.org/packages/64/27/867076a6ada7f2b9c8292884ab44d08fd2ba71bd2b5364d4136f3cd537e1/ruff-0.15.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dab6941c862c05739774677c6273166d2510d254dac0695c0e3f5efa1b5585de", size = 10263122, upload-time = "2026-02-19T22:32:10.036Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/ef/faf9321d550f8ebf0c6373696e70d1758e20ccdc3951ad7af00c0956be7c/ruff-0.15.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1b9164f57fc36058e9a6806eb92af185b0697c9fe4c7c52caa431c6554521e5c", size = 10735295, upload-time = "2026-02-19T22:32:39.227Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/55/e8089fec62e050ba84d71b70e7834b97709ca9b7aba10c1a0b196e493f97/ruff-0.15.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:80d24fcae24d42659db7e335b9e1531697a7102c19185b8dc4a028b952865fd8", size = 11241641, upload-time = "2026-02-19T22:32:34.617Z" },
+    { url = "https://files.pythonhosted.org/packages/23/01/1c30526460f4d23222d0fabd5888868262fd0e2b71a00570ca26483cd993/ruff-0.15.2-py3-none-win32.whl", hash = "sha256:fd5ff9e5f519a7e1bd99cbe8daa324010a74f5e2ebc97c6242c08f26f3714f6f", size = 10507885, upload-time = "2026-02-19T22:32:15.635Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/10/3d18e3bbdf8fc50bbb4ac3cc45970aa5a9753c5cb51bf9ed9a3cd8b79fa3/ruff-0.15.2-py3-none-win_amd64.whl", hash = "sha256:d20014e3dfa400f3ff84830dfb5755ece2de45ab62ecea4af6b7262d0fb4f7c5", size = 11623725, upload-time = "2026-02-19T22:32:04.947Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/78/097c0798b1dab9f8affe73da9642bb4500e098cb27fd8dc9724816ac747b/ruff-0.15.2-py3-none-win_arm64.whl", hash = "sha256:cabddc5822acdc8f7b5527b36ceac55cc51eec7b1946e60181de8fe83ca8876e", size = 10941649, upload-time = "2026-02-19T22:32:18.108Z" },
+]
+
 [[package]]
 name = "safetensors"
 version = "0.6.2"