extract-python 0.2.1__tar.gz → 0.3.1__tar.gz

{extract_python-0.2.1 → extract_python-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: extract-python
-Version: 0.2.1
+Version: 0.3.1
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python

extract_python-0.3.1/extract_python/docling_.py

@@ -0,0 +1,203 @@
+import shutil
+import tempfile
+from collections.abc import AsyncGenerator, Iterable, Iterator
+from functools import cache
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated, ClassVar, TypeVar
+
+from icij_common.registrable import FromConfig
+from pydantic import AfterValidator, Field
+
+from .constants import ARTIFACTS, CPU_GROUP, DEFAULT_MD_PAGE_SEP
+from .objects import (
+    Error,
+    InputDoc,
+    MarkdownDoc,
+    OutputFormat,
+    PageIndexes,
+    Result,
+    Status,
+    SupportedExt,
+)
+from .pipeline import Pipeline, PipelineConfig, PipelineType
+from .utils import all_subclasses, chdir, map_and_preserve, path_to_artifacts_dirname
+
+DOCLING_DEFAULT_ARTIFACTS_PATH = Path.home().joinpath(".cache", "docling", "models")
+
+if TYPE_CHECKING:
+    from docling.datamodel.base_models import InputFormat
+    from docling.datamodel.pipeline_options import PipelineOptions
+    from docling.document_converter import ConversionResult, FormatOption
+    from docling_core.types.io import DocumentStream
+
+
+def _validate_pipeline_opts(opts: "PipelineOptions") -> None:
+    from docling.datamodel.pipeline_options import PdfPipelineOptions
+
+    if isinstance(opts, PdfPipelineOptions) and not opts.generate_picture_images:
+        msg = "generate_picture_images should be set to true"
+        raise ValueError(msg)
+
+
+def _validate_options(
+    data: dict["InputFormat", "FormatOption"],
+) -> dict["InputFormat", "FormatOption"]:
+    for opts in data.values():
+        _validate_pipeline_opts(opts.pipeline_options)
+    return data
+
+
+@cache
+def _default_format_opts() -> dict["InputFormat", "FormatOption"]:
+    from docling.datamodel.pipeline_options import (
+        EasyOcrOptions,
+        PdfPipelineOptions,
+    )
+    from docling.document_converter import PdfFormatOption
+
+    return {
+        InputFormat.PDF: PdfFormatOption(
+            pipeline_options=PdfPipelineOptions(
+                ocr_options=EasyOcrOptions(), generate_picture_images=True
+            )
+        ),
+    }
+
+
+T = TypeVar("T")
+
+
+def _find_subcls(cls: type[T], name: str) -> type[T]:
+    for c in all_subclasses(cls):
+        if c.__name__ == name:
+            return c
+    raise ValueError(f"unknown {cls.__name__} subclass {name}")
+
+
+@PipelineConfig.register()
+class DoclingPipelineConfig(PipelineConfig):
+    pipeline: PipelineType = Field(frozen=True, default=PipelineType.DOCLING)
+    task_group: ClassVar[str] = Field(frozen=True, default=CPU_GROUP)
+
+    format_options: Annotated[
+        dict[InputFormat, FormatOption] | None, AfterValidator(_validate_options)
+    ] = Field(default_factory=_default_format_opts)
+
+    _unsupported_input_formats: ClassVar[set[InputFormat]] = {
+        InputFormat.AUDIO,
+        InputFormat.METS_GBS,
+        InputFormat.VTT,
+    }
+
+    @classmethod
+    @cache
+    def supported_exts(cls) -> set[SupportedExt]:
+        from docling.datamodel.base_models import FormatToExtensions, InputFormat
+
+        supported = set()
+        for f in InputFormat:
+            if f in cls._unsupported_input_formats:
+                continue
+            for ext in FormatToExtensions[f]:
+                supported.add(SupportedExt(f".{ext.lower()}"))
+        return supported
+
+
+@Pipeline.register(PipelineType.DOCLING)
+class DoclingPipeline(Pipeline):
+    def __init__(self, format_options: dict[InputFormat, FormatOption] | None = None):
+        from docling.document_converter import DocumentConverter
+
+        allowed_format = [
+            f.to_docling() for f in DoclingPipelineConfig.supported_exts()
+        ]
+        self._converter = DocumentConverter(
+            allowed_formats=allowed_format, format_options=format_options
+        )
+
+    async def extract_content(
+        self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
+    ) -> AsyncGenerator[Result, None]:
+        docs, path_or_streams = map_and_preserve(_to_docling, docs)
+        outputs = self._converter.convert_all(path_or_streams, raises_on_error=False)
+        for doc, res in zip(docs, outputs, strict=True):
+            yield _to_result(res, doc, output_format, output_path=output_path)
+
+    @classmethod
+    def _from_config(cls, config: DoclingPipelineConfig) -> FromConfig:
+        return cls(config.format_options)
+
+
+def _to_docling(docs: Iterable[InputDoc]) -> Iterator[Path | "DocumentStream"]:
+    for d in docs:
+        yield d.to_docling()
+
+
+def _to_result(
+    res: "ConversionResult",
+    input_document: InputDoc,
+    output_format: OutputFormat,
+    output_path: Path,
+    **kwargs,
+) -> Result:
+    output_path.mkdir(parents=True, exist_ok=True)
+    output = None
+    status = Status.from_docling(res.status)
+    if status.allows_conversion:
+        match output_format:
+            case OutputFormat.MARKDOWN:
+                output = _to_markdown_doc(res, output_path, **kwargs)
+            case _:
+                raise NotImplementedError(f"unsupported output format {output_format}")
+    errors = [Error.from_docling(e) for e in res.errors]
+    input_doc = input_document.without_content()
+    return Result(input=input_doc, status=status, errors=errors, output=output)
+
+
+def _to_markdown_doc(
+    res: "ConversionResult",
+    output_path: Path,
+    page_sep: str = DEFAULT_MD_PAGE_SEP,
+    **kwargs,
+) -> MarkdownDoc:
+    from docling_core.types.doc import ImageRefMode
+
+    # TODO: Should we add a hash to avoid collision between files with same names
+    #  nested in the tree structured
+    md_dir_name = path_to_artifacts_dirname(res.input.file)
+    md_dir = output_path / md_dir_name
+    if md_dir.exists():
+        raise FileExistsError(f"directory {md_dir} already exists")
+    # Let's avoid issue of duplicated input file names flattened top level
+    md_filename = md_dir_name + OutputFormat.MARKDOWN
+    total_length = 0
+    n_pages = len(res.pages)
+
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as td:
+        tmp_dir = Path(td)
+        page_path = Path("page.md")
+        # We do a chdir to bypass a Docling bug which only allows to maintain relative
+        # image ref when saving the markdown to a relative path
+        with (tmp_dir / md_filename).open("w") as f, chdir(tmp_dir):
+            end_indices = []
+            for page_i in range(n_pages):
+                res.document.save_as_markdown(
+                    page_path,
+                    page_no=page_i + 1,
+                    image_mode=ImageRefMode.REFERENCED,
+                    artifacts_dir=Path(ARTIFACTS),
+                    **kwargs,
+                )
+                content = page_path.read_text()
+                if page_i > 0:
+                    content += "\n"
+                if page_i < n_pages - 1:
+                    content += page_sep
+                total_length += len(content)
+                end_indices.append(total_length)
+                f.write(content)
+                f.flush()
+                page_path.unlink()
+        shutil.move(tmp_dir, md_dir)
+    pages = PageIndexes.from_page_end_indices(end_indices)
+    return MarkdownDoc(path=Path(md_dir_name), pages=pages)

{extract_python-0.2.1 → extract_python-0.3.1}/extract_python/marker_.py

@@ -3,14 +3,8 @@ from collections.abc import AsyncGenerator, Iterable
 from copy import deepcopy
 from functools import cache
 from pathlib import Path
-from typing import Any, ClassVar, Self
-
-from marker.config.parser import ConfigParser
-from marker.converters.pdf import PdfConverter
-from marker.models import create_model_dict
-from marker.output import text_from_rendered
-from marker.renderers.markdown import MarkdownRenderer
-from PIL.Image import Image
+from typing import TYPE_CHECKING, Any, ClassVar, Self
+
 from pydantic import Field
 
 from .constants import ARTIFACTS, CPU_GROUP
@@ -26,6 +20,10 @@ from .objects import (
 from .pipeline import Pipeline, PipelineConfig, PipelineType
 from .utils import path_to_artifacts_dirname, report_recoverable_errors
 
+if TYPE_CHECKING:
+    from marker.converters.pdf import PdfConverter
+    from PIL import Image
+
 
 @PipelineConfig.register()
 class MarkerPipelineConfig(PipelineConfig):
@@ -36,7 +34,7 @@ class MarkerPipelineConfig(PipelineConfig):
 
     @classmethod
     @cache
-    def supported_formats(cls) -> set[SupportedExt]:
+    def supported_exts(cls) -> set[SupportedExt]:
         # Subset of https://documentation.datalab.to/docs/common/supportedfiletypes
         return {
             SupportedExt.PDF,
@@ -75,6 +73,10 @@ class MarkerPipeline(Pipeline):
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
     ) -> AsyncGenerator[Result, None]:
+        from marker.config.parser import ConfigParser
+        from marker.converters.pdf import PdfConverter
+        from marker.models import create_model_dict
+
         config = deepcopy(self._marker_config)
         config["output_format"] = output_format.to_marker()
         config_parser = ConfigParser(config)
@@ -96,10 +98,12 @@ class MarkerPipeline(Pipeline):
 @report_recoverable_errors(_MARKER_CONVERSION_ERRORS)
 def _process_doc(
     doc: InputDoc,
-    converter: PdfConverter,
+    converter: "PdfConverter",
     output_format: OutputFormat,
     output_path: Path,
 ) -> Result:
+    from marker.output import text_from_rendered
+
     rendered = converter(str(doc.path))
     content, _, images = text_from_rendered(rendered)
     match output_format:
@@ -112,8 +116,10 @@ def _process_doc(
 
 
 def _to_markdown_doc(
-    input_doc: InputDoc, content: str, images: dict[str, Image], output_path: Path
+    input_doc: InputDoc, content: str, images: dict[str, "Image"], output_path: Path
 ) -> MarkdownDoc:
+    from marker.renderers.markdown import MarkdownRenderer
+
     # TODO: Should we add a hash to avoid collision between files with same names
     #  nested in the tree structured
     md_dir_name = path_to_artifacts_dirname(input_doc.path)

{extract_python-0.2.1 → extract_python-0.3.1}/extract_python/miner_u.py

@@ -8,12 +8,6 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, ClassVar, Self
 
-from mineru.backend.pipeline.pipeline_middle_json_mkcontent import (
-    union_make as pipeline_union_make,
-)
-from mineru.backend.vlm.vlm_middle_json_mkcontent import union_make as vlm_union_make
-from mineru.cli.common import aio_do_parse
-from mineru.utils.enum_class import MakeMode
 from pydantic import Field
 from pydantic_extra_types.language_code import LanguageAlpha2
 
@@ -47,33 +41,39 @@ class MinerUConfig(BaseModel):
     # TODO: use enum or literal here
     parse_method: str = "auto"
 
-    default_kwargs: ClassVar[dict] = {
-        "server_url": None,
-        # We don't dump md directly we process, we dump the middle json in order to be
-        # able to get page indexes
-        "parse_method": "auto",
-        "dump_md": False,
-        "dump_middle_json": True,
-        "f_draw_layout_bbox": False,
-        "f_draw_span_bbox": False,
-        "f_dump_model_output": False,  # might be useful for debug though
-        "f_dump_orig_pdf": False,
-        "f_dump_content_list": False,  # might be useful for debug though
-        "start_page_id": 0,
-        "f_make_md_mode": MakeMode.MM_MD,
-        "image_analysis": True,
-        "end_page_id": None,
-        "client_side_output_generation": False,
-    }
-
     def as_parse_kwargs(self) -> dict[str, Any]:
-        kwargs = copy(self.default_kwargs)
+        kwargs = copy(self._get_default_kwargs())
         kwargs["backend"] = self.backend
         kwargs["parse_method"] = self.parse_method
         kwargs["formula_enable"] = self.enable_formula_extraction
         kwargs["table_enable"] = self.enable_table_extraction
         return kwargs
 
+    @classmethod
+    @cache
+    def _get_default_kwargs(cls) -> dict[str, Any]:
+
+        from mineru.utils.enum_class import MakeMode
+
+        return {
+            "server_url": None,
+            # We don't dump md directly we process, we dump the middle json in order to be
+            # able to get page indexes
+            "parse_method": "auto",
+            "dump_md": False,
+            "dump_middle_json": True,
+            "f_draw_layout_bbox": False,
+            "f_draw_span_bbox": False,
+            "f_dump_model_output": False,  # might be useful for debug though
+            "f_dump_orig_pdf": False,
+            "f_dump_content_list": False,  # might be useful for debug though
+            "start_page_id": 0,
+            "f_make_md_mode": MakeMode.MM_MD,
+            "image_analysis": True,
+            "end_page_id": None,
+            "client_side_output_generation": False,
+        }
+
 
 @PipelineConfig.register()  # noqa: F821
 class MinerUPipelineConfig(PipelineConfig):  # noqa: F821
@@ -85,7 +85,7 @@ class MinerUPipelineConfig(PipelineConfig):  # noqa: F821
 
     @classmethod
     @cache
-    def supported_formats(cls) -> set[SupportedExt]:
+    def supported_exts(cls) -> set[SupportedExt]:
         return {
             SupportedExt.PDF,
             SupportedExt.DOCX,
@@ -104,6 +104,8 @@ class MinerUPipeline(Pipeline):
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
     ) -> AsyncGenerator[Result, None]:
+        from mineru.cli.common import aio_do_parse
+
         docs = list(docs)
         # TODO: exclude files which are not pdf and return an error
         pdfs_bytes = [d.path.read_bytes() for d in docs]
@@ -149,11 +151,18 @@ def _revert_mineru_output(output_dir: Path, *, pdf_filename: str) -> Path:
 
 
 def _parse_md_make_fn(backend: MinerUBackend) -> MDMakeFunction:
+
     match backend:
         case MinerUBackend.PIPELINE:
-            return pipeline_union_make
+            from mineru.backend.pipeline.pipeline_middle_json_mkcontent import (
+                union_make,
+            )
+
+            return union_make
         case MinerUBackend.VLM:
-            return vlm_union_make
+            from mineru.backend.vlm.vlm_middle_json_mkcontent import union_make
+
+            return union_make
         case _:
             raise ValueError(f"Unsupported backend: {backend}")
 
@@ -201,8 +210,12 @@ def _dump_md_content(
     output_path: Path,
     md_path: Path,
     im_dir: Path,
-    md_make_mode: str = MakeMode.MM_MD,
+    md_make_mode: str | None = None,
 ) -> ConversionOutput:
+    from mineru.utils.enum_class import MakeMode
+
+    if md_make_mode is None:
+        md_make_mode = MakeMode.MM_MD
     total_length = 0
     end_indices = []
     with md_path.open("w") as f:

{extract_python-0.2.1 → extract_python-0.3.1}/extract_python/objects.py

@@ -21,7 +21,12 @@ from pydantic import AfterValidator, RootModel, TypeAdapter
 from pydantic import BaseModel as _BaseModel
 
 try:
-    from docling.datamodel.base_models import ConversionStatus, ErrorItem, InputFormat
+    from docling.datamodel.base_models import (
+        ConversionStatus,
+        ErrorItem,
+        FormatToExtensions,
+        InputFormat,
+    )
     from docling.datamodel.document import InputDocument
     from docling_core.types.io import DocumentStream
 except ImportError:
@@ -33,42 +38,78 @@ logger = logging.getLogger(__name__)
 base_config = merge_configs(icij_config(), no_enum_values_config())
 
 
+@cache
+def _ext_to_docling_input_format() -> dict:
+    from .docling_ import DoclingPipelineConfig  # noqa: PLC0415
+
+    mapping = dict()
+    supported = DoclingPipelineConfig.supported_exts()
+    for input_f, exts in FormatToExtensions.items():
+        for ext in exts:
+            try:
+                ext = SupportedExt(f".{ext.lower()}")  # noqa: PLW2901
+            except ValueError:
+                continue
+            if ext in supported:
+                mapping[ext] = input_f
+    return mapping
+
+
 class BaseModel(_BaseModel):
     model_config = base_config
 
 
 class SupportedExt(StrEnum):
     ADOC = ".adoc"
+    ASC = ".asc"
     ASCIIDOC = ".asciidoc"
     BMP = ".bmp"
     CSV = ".csv"
     DOC = ".doc"
     DOCX = ".docx"
+    DOTX = ".dotx"
+    DOTM = ".dotm"
+    DOCM = ".docm"
     EPUB = ".epub"
     GIF = ".gif"
     HTLM = ".html"
+    HTM = ".htm"
     JPEG = ".jpeg"
     JPG = ".jpg"
+    JSON = ".json"
+    LATEX = ".latex"
     MD = ".md"
+    NXML = ".nxml"
     ODP = ".odp"
     ODS = ".ods"
     ODT = ".odt"
     PDF = ".pdf"
     PNG = ".png"
+    PPSX = ".ppsx"
     PPT = ".ppt"
+    PPTM = ".pptm"
+    PPSM = ".ppsm"
+    POTX = ".potx"
+    POTM = ".potm"
     PPTX = ".pptx"
+    QMD = ".qmd"
+    RMD = ".rmd"
     TEX = ".tex"
+    TIF = ".tif"
     TIFF = ".tiff"
     TXT = ".txt"
+    TEXT = ".text"
     WEBP = ".webp"
+    XBRL = ".xbrl"
     XHTML = ".xhtml"
     XLS = ".xls"
     XLSM = ".xlsm"
     XLSX = ".xlsx"
     XLTX = ".xltx"
+    XML = ".xml"
 
     def to_docling(self) -> InputFormat:
-        return InputFormat(self.value[1:])
+        return _ext_to_docling_input_format()[self]
 
 
 class OutputFormat(StrEnum):

{extract_python-0.2.1 → extract_python-0.3.1}/extract_python/pipeline.py

@@ -30,7 +30,7 @@ class PipelineConfig(RegistrableConfig, ABC):
 
     @classmethod
     @abstractmethod
-    def supported_formats(cls) -> set[SupportedExt]: ...
+    def supported_exts(cls) -> set[SupportedExt]: ...
 
 
 class Pipeline(RegistrableFromConfig, ABC):

{extract_python-0.2.1 → extract_python-0.3.1}/pyproject.toml

@@ -51,6 +51,7 @@ required-environments = [
 dev = [
   "pytest~=8.3.5",
   "pytest-asyncio~=0.25.3",
+  "ruff==0.15.2",
 ]
 
 [project.urls]

{extract_python-0.2.1 → extract_python-0.3.1}/uv.lock

@@ -868,7 +868,6 @@ wheels = [
 
 [[package]]
 name = "extract-python"
-version = "0.1.0"
 source = { editable = "." }
 dependencies = [
     { name = "icij-common" },
@@ -898,11 +897,12 @@ mineru = [
 dev = [
     { name = "pytest" },
     { name = "pytest-asyncio" },
+    { name = "ruff" },
 ]
 
 [package.metadata]
 requires-dist = [
-    { name = "docling-slim", extras = ["standard", "feat-ocr-easyocr", "feat-ocr-tesserocr", "feat-ocr-mac"], marker = "extra == 'docling'", specifier = "~=2.96" },
+    { name = "docling-slim", extras = ["feat-ocr-easyocr", "feat-ocr-mac", "feat-ocr-tesserocr", "standard"], marker = "extra == 'docling'", specifier = "~=2.96" },
     { name = "html2image", marker = "extra == 'benches'", specifier = "~=2.0.7" },
     { name = "icij-common", specifier = "~=0.8.2" },
     { name = "markdown2", marker = "extra == 'benches'", specifier = ">=2.5.4" },
@@ -920,6 +920,7 @@ provides-extras = ["benches", "docling", "marker", "mineru"]
 dev = [
     { name = "pytest", specifier = "~=8.3.5" },
     { name = "pytest-asyncio", specifier = "~=0.25.3" },
+    { name = "ruff", specifier = "==0.15.2" },
 ]
 
 [[package]]
@@ -4334,6 +4335,31 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/3f/50/0a9e7e7afe7339bd5e36911f0ceb15fed51945836ed803ae5afd661057fd/rtree-1.4.1-py3-none-win_arm64.whl", hash = "sha256:3d46f55729b28138e897ffef32f7ce93ac335cb67f9120125ad3742a220800f0", size = 355253, upload-time = "2025-08-13T19:32:00.296Z" },
 ]
 
+[[package]]
+name = "ruff"
+version = "0.15.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/06/04/eab13a954e763b0606f460443fcbf6bb5a0faf06890ea3754ff16523dce5/ruff-0.15.2.tar.gz", hash = "sha256:14b965afee0969e68bb871eba625343b8673375f457af4abe98553e8bbb98342", size = 4558148, upload-time = "2026-02-19T22:32:20.271Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2f/70/3a4dc6d09b13cb3e695f28307e5d889b2e1a66b7af9c5e257e796695b0e6/ruff-0.15.2-py3-none-linux_armv6l.whl", hash = "sha256:120691a6fdae2f16d65435648160f5b81a9625288f75544dc40637436b5d3c0d", size = 10430565, upload-time = "2026-02-19T22:32:41.824Z" },
+    { url = "https://files.pythonhosted.org/packages/71/0b/bb8457b56185ece1305c666dc895832946d24055be90692381c31d57466d/ruff-0.15.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a89056d831256099658b6bba4037ac6dd06f49d194199215befe2bb10457ea5e", size = 10820354, upload-time = "2026-02-19T22:32:07.366Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/c1/e0532d7f9c9e0b14c46f61b14afd563298b8b83f337b6789ddd987e46121/ruff-0.15.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e36dee3a64be0ebd23c86ffa3aa3fd3ac9a712ff295e192243f814a830b6bd87", size = 10170767, upload-time = "2026-02-19T22:32:13.188Z" },
+    { url = "https://files.pythonhosted.org/packages/47/e8/da1aa341d3af017a21c7a62fb5ec31d4e7ad0a93ab80e3a508316efbcb23/ruff-0.15.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9fb47b6d9764677f8c0a193c0943ce9a05d6763523f132325af8a858eadc2b9", size = 10529591, upload-time = "2026-02-19T22:32:02.547Z" },
+    { url = "https://files.pythonhosted.org/packages/93/74/184fbf38e9f3510231fbc5e437e808f0b48c42d1df9434b208821efcd8d6/ruff-0.15.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f376990f9d0d6442ea9014b19621d8f2aaf2b8e39fdbfc79220b7f0c596c9b80", size = 10260771, upload-time = "2026-02-19T22:32:36.938Z" },
+    { url = "https://files.pythonhosted.org/packages/05/ac/605c20b8e059a0bc4b42360414baa4892ff278cec1c91fff4be0dceedefd/ruff-0.15.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dcc987551952d73cbf5c88d9fdee815618d497e4df86cd4c4824cc59d5dd75f", size = 11045791, upload-time = "2026-02-19T22:32:31.642Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/52/db6e419908f45a894924d410ac77d64bdd98ff86901d833364251bd08e22/ruff-0.15.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:42a47fd785cbe8c01b9ff45031af875d101b040ad8f4de7bbb716487c74c9a77", size = 11879271, upload-time = "2026-02-19T22:32:29.305Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/d8/7992b18f2008bdc9231d0f10b16df7dda964dbf639e2b8b4c1b4e91b83af/ruff-0.15.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbe9f49354866e575b4c6943856989f966421870e85cd2ac94dccb0a9dcb2fea", size = 11303707, upload-time = "2026-02-19T22:32:22.492Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/02/849b46184bcfdd4b64cde61752cc9a146c54759ed036edd11857e9b8443b/ruff-0.15.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7a672c82b5f9887576087d97be5ce439f04bbaf548ee987b92d3a7dede41d3a", size = 11149151, upload-time = "2026-02-19T22:32:44.234Z" },
+    { url = "https://files.pythonhosted.org/packages/70/04/f5284e388bab60d1d3b99614a5a9aeb03e0f333847e2429bebd2aaa1feec/ruff-0.15.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ecc64f46f7019e2bcc3cdc05d4a7da958b629a5ab7033195e11a438403d956", size = 11091132, upload-time = "2026-02-19T22:32:24.691Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/ae/88d844a21110e14d92cf73d57363fab59b727ebeabe78009b9ccb23500af/ruff-0.15.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:8dcf243b15b561c655c1ef2f2b0050e5d50db37fe90115507f6ff37d865dc8b4", size = 10504717, upload-time = "2026-02-19T22:32:26.75Z" },
+    { url = "https://files.pythonhosted.org/packages/64/27/867076a6ada7f2b9c8292884ab44d08fd2ba71bd2b5364d4136f3cd537e1/ruff-0.15.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dab6941c862c05739774677c6273166d2510d254dac0695c0e3f5efa1b5585de", size = 10263122, upload-time = "2026-02-19T22:32:10.036Z" },
+    { url = "https://files.pythonhosted.org/packages/e7/ef/faf9321d550f8ebf0c6373696e70d1758e20ccdc3951ad7af00c0956be7c/ruff-0.15.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1b9164f57fc36058e9a6806eb92af185b0697c9fe4c7c52caa431c6554521e5c", size = 10735295, upload-time = "2026-02-19T22:32:39.227Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/55/e8089fec62e050ba84d71b70e7834b97709ca9b7aba10c1a0b196e493f97/ruff-0.15.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:80d24fcae24d42659db7e335b9e1531697a7102c19185b8dc4a028b952865fd8", size = 11241641, upload-time = "2026-02-19T22:32:34.617Z" },
+    { url = "https://files.pythonhosted.org/packages/23/01/1c30526460f4d23222d0fabd5888868262fd0e2b71a00570ca26483cd993/ruff-0.15.2-py3-none-win32.whl", hash = "sha256:fd5ff9e5f519a7e1bd99cbe8daa324010a74f5e2ebc97c6242c08f26f3714f6f", size = 10507885, upload-time = "2026-02-19T22:32:15.635Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/10/3d18e3bbdf8fc50bbb4ac3cc45970aa5a9753c5cb51bf9ed9a3cd8b79fa3/ruff-0.15.2-py3-none-win_amd64.whl", hash = "sha256:d20014e3dfa400f3ff84830dfb5755ece2de45ab62ecea4af6b7262d0fb4f7c5", size = 11623725, upload-time = "2026-02-19T22:32:04.947Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/78/097c0798b1dab9f8affe73da9642bb4500e098cb27fd8dc9724816ac747b/ruff-0.15.2-py3-none-win_arm64.whl", hash = "sha256:cabddc5822acdc8f7b5527b36ceac55cc51eec7b1946e60181de8fe83ca8876e", size = 10941649, upload-time = "2026-02-19T22:32:18.108Z" },
+]
+
 [[package]]
 name = "safetensors"
 version = "0.6.2"

extract_python-0.2.1/extract_python/docling_.py

@@ -1,258 +0,0 @@
-import shutil
-import tempfile
-from collections.abc import AsyncGenerator, Iterable, Iterator
-from functools import cache
-from pathlib import Path
-from typing import Any, ClassVar, Literal, TypeVar
-
-from docling.backend.abstract_backend import AbstractDocumentBackend
-from docling.datamodel.base_models import InputFormat
-from docling.datamodel.document import ConversionResult
-from docling.datamodel.pipeline_options import (
-    EasyOcrOptions,
-    PdfPipelineOptions,
-    PipelineOptions,
-    VlmPipelineOptions,
-)
-from docling.document_converter import DocumentConverter, FormatOption
-from docling.models.factories import get_ocr_factory
-from docling.pipeline.base_pipeline import BasePipeline
-from docling_core.types.doc import ImageRefMode
-from docling_core.types.io import DocumentStream
-from icij_common.registrable import FromConfig
-from pydantic import Field, model_validator
-
-from .constants import ARTIFACTS, CPU_GROUP, DEFAULT_MD_PAGE_SEP
-from .objects import (
-    BaseModel,
-    Error,
-    InputDoc,
-    MarkdownDoc,
-    OutputFormat,
-    PageIndexes,
-    Result,
-    Status,
-    SupportedExt,
-)
-from .pipeline import Pipeline, PipelineConfig, PipelineType
-from .utils import all_subclasses, chdir, map_and_preserve, path_to_artifacts_dirname
-
-DOCLING_DEFAULT_ARTIFACTS_PATH = Path.home().joinpath(".cache", "docling", "models")
-
-
-class _PdfPipelineOptions(PdfPipelineOptions):
-    generate_picture_images: bool = Field(default=True, frozen=True)
-
-    @model_validator(mode="before")
-    @classmethod
-    def validate_ocr_options(cls, data: Any) -> Any:
-        if isinstance(data, dict):
-            ocr_options = data.get("ocr_options")
-            if not isinstance(ocr_options, dict):
-                return data
-            allow_external_plugins = ocr_options.get("allow_external_plugins", False)
-            ocr_factory = get_ocr_factory(allow_external_plugins=allow_external_plugins)
-            kind = ocr_options.pop("kind")
-            data["ocr_options"] = ocr_factory.create_options(kind=kind, **ocr_options)
-        return data
-
-
-OptionsByPipeline = list[
-    tuple[Literal["pdf"], _PdfPipelineOptions]
-    | tuple[Literal["vlm"], VlmPipelineOptions]
-]
-
-
-def _default_pipeline_options() -> OptionsByPipeline:
-    pipeline_options = _PdfPipelineOptions(ocr_options=EasyOcrOptions())
-    return [("pdf", pipeline_options), ("vlm", VlmPipelineOptions())]
-
-
-class DoclingFormatOption(BaseModel):
-    pipeline_cls: str
-    backend_cls: str
-
-    def to_docling(
-        self, pipeline_options: dict[Literal["pdf", "vlm"], PipelineOptions]
-    ) -> FormatOption:
-        pipeline_cls = _find_subcls(BasePipeline, self.pipeline_cls)
-        backend_cls = _find_subcls(AbstractDocumentBackend, self.backend_cls)
-        if "vlm" in self.pipeline_cls.lower():
-            pipeline_options = pipeline_options.get("vlm")
-            if pipeline_options is not None:
-                pipeline_options = VlmPipelineOptions.model_validate(pipeline_options)
-        elif "pdf" in self.pipeline_cls.lower():
-            pipeline_options = pipeline_options.get("pdf")
-            if pipeline_options is not None:
-                pipeline_options = _PdfPipelineOptions.model_validate(pipeline_options)
-        else:
-            raise ValueError(
-                f"invalid pipeline_cls: {pipeline_cls}, expected a VLM or PDF pipeline"
-            )
-        return FormatOption(
-            pipeline_cls=pipeline_cls,
-            pipeline_options=pipeline_options,
-            backend=backend_cls,
-        )
-
-
-@cache
-def _default_format_options() -> dict[InputFormat, DoclingFormatOption]:
-    supported_fmt = {InputFormat.PDF}
-    return {
-        fmt: DoclingFormatOption(
-            pipeline_cls=opt.pipeline_cls.__name__, backend_cls=opt.backend.__name__
-        )
-        for fmt, opt in DocumentConverter().format_to_options.items()
-        if fmt in supported_fmt
-    }
-
-
-T = TypeVar("T")
-
-
-def _find_subcls(cls: type[T], name: str) -> type[T]:
-    for c in all_subclasses(cls):
-        if c.__name__ == name:
-            return c
-    raise ValueError(f"unknown {cls.__name__} subclass {name}")
-
-
-@PipelineConfig.register()
-class DoclingPipelineConfig(PipelineConfig):
-    pipeline: PipelineType = Field(frozen=True, default=PipelineType.DOCLING)
-    task_group: ClassVar[str] = Field(frozen=True, default=CPU_GROUP)
-
-    pipeline_options: OptionsByPipeline = Field(
-        default_factory=_default_pipeline_options
-    )
-    format_options: dict[InputFormat, DoclingFormatOption] = Field(
-        default_factory=_default_format_options
-    )
-
-    def to_format_options(self) -> dict[InputFormat, FormatOption]:
-        pipeline_options = dict(self.pipeline_options)
-        return {
-            InputFormat(f): opt.to_docling(pipeline_options)
-            for f, opt in self.format_options.items()
-        }
-
-    @classmethod
-    @cache
-    def supported_formats(cls) -> set[SupportedExt]:
-        # Subset of https://docling-project.github.io/docling/usage/supported_formats/
-        return {
-            SupportedExt.ADOC,
-            SupportedExt.ASCIIDOC,
-            SupportedExt.BMP,
-            SupportedExt.CSV,
-            SupportedExt.DOCX,
-            SupportedExt.HTLM,
-            SupportedExt.JPG,
-            SupportedExt.MD,
-            SupportedExt.PDF,
-            SupportedExt.PNG,
-            SupportedExt.PPTX,
-            SupportedExt.TEX,
-            SupportedExt.TIFF,
-            SupportedExt.TXT,
-            SupportedExt.WEBP,
-            SupportedExt.XHTML,
-            SupportedExt.XLSX,
-        }
-
-
-DEFAULT_FORMAT_OPTIONS = DoclingPipelineConfig().to_format_options()
-
-
-@Pipeline.register(PipelineType.DOCLING)
-class DoclingPipeline(Pipeline):
-    def __init__(self, format_options: dict[InputFormat, FormatOption] | None = None):
-        if format_options is None:
-            format_options = DEFAULT_FORMAT_OPTIONS
-        self._converter = DocumentConverter(format_options=format_options)
-
-    async def extract_content(
-        self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
-    ) -> AsyncGenerator[Result, None]:
-        docs, path_or_streams = map_and_preserve(_to_docling, docs)
-        outputs = self._converter.convert_all(path_or_streams, raises_on_error=False)
-        for doc, res in zip(docs, outputs, strict=True):
-            yield _to_result(res, doc, output_format, output_path=output_path)
-
-    @classmethod
-    def _from_config(cls, config: DoclingPipelineConfig) -> FromConfig:
-        return cls(config.to_format_options())
-
-
-def _to_docling(docs: Iterable[InputDoc]) -> Iterator[Path | DocumentStream]:
-    for d in docs:
-        yield d.to_docling()
-
-
-def _to_result(
-    res: ConversionResult,
-    input_document: InputDoc,
-    output_format: OutputFormat,
-    output_path: Path,
-    **kwargs,
-) -> Result:
-    output_path.mkdir(parents=True, exist_ok=True)
-    output = None
-    status = Status.from_docling(res.status)
-    if status.allows_conversion:
-        match output_format:
-            case OutputFormat.MARKDOWN:
-                output = _to_markdown_doc(res, output_path, **kwargs)
-            case _:
-                raise NotImplementedError(f"unsupported output format {output_format}")
-    errors = [Error.from_docling(e) for e in res.errors]
-    input_doc = input_document.without_content()
-    return Result(input=input_doc, status=status, errors=errors, output=output)
-
-
-def _to_markdown_doc(
-    res: ConversionResult,
-    output_path: Path,
-    page_sep: str = DEFAULT_MD_PAGE_SEP,
-    **kwargs,
-) -> MarkdownDoc:
-    # TODO: Should we add a hash to avoid collision between files with same names
-    #  nested in the tree structured
-    md_dir_name = path_to_artifacts_dirname(res.input.file)
-    md_dir = output_path / md_dir_name
-    if md_dir.exists():
-        raise FileExistsError(f"directory {md_dir} already exists")
-    # Let's avoid issue of duplicated input file names flattened top level
-    md_filename = md_dir_name + OutputFormat.MARKDOWN
-    total_length = 0
-    n_pages = len(res.pages)
-
-    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as td:
-        tmp_dir = Path(td)
-        page_path = Path("page.md")
-        # We do a chdir to bypass a Docling bug which only allows to maintain relative
-        # image ref when saving the markdown to a relative path
-        with (tmp_dir / md_filename).open("w") as f, chdir(tmp_dir):
-            end_indices = []
-            for page_i in range(n_pages):
-                res.document.save_as_markdown(
-                    page_path,
-                    page_no=page_i + 1,
-                    image_mode=ImageRefMode.REFERENCED,
-                    artifacts_dir=Path(ARTIFACTS),
-                    **kwargs,
-                )
-                content = page_path.read_text()
-                if page_i > 0:
-                    content += "\n"
-                if page_i < n_pages - 1:
-                    content += page_sep
-                total_length += len(content)
-                end_indices.append(total_length)
-                f.write(content)
-                f.flush()
-                page_path.unlink()
-        shutil.move(tmp_dir, md_dir)
-    pages = PageIndexes.from_page_end_indices(end_indices)
-    return MarkdownDoc(path=Path(md_dir_name), pages=pages)