extract-python 0.7.3__tar.gz → 0.8.1__tar.gz

{extract_python-0.7.3 → extract_python-0.8.1}/PKG-INFO

@@ -1,14 +1,15 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: extract-python
-Version: 0.7.3
+Version: 0.8.1
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python
 Project-URL: Issues, https://github.com/ICIJ/extract-python/issues
 Author-email: Clément Doumouro <cdoumouro@icij.org>
 Requires-Python: <3.15,>=3.13
-Requires-Dist: extract-core~=0.7.0
+Requires-Dist: extract-core~=0.8.0
 Requires-Dist: icij-common~=0.8.2
+Requires-Dist: pympler~=1.1
 Provides-Extra: benches
 Requires-Dist: html2image~=2.0.7; extra == 'benches'
 Requires-Dist: markdown2>=2.5.4; extra == 'benches'
@@ -24,3 +25,6 @@ Requires-Dist: mineru[pipeline,vlm]~=3.2; extra == 'mineru'
 Requires-Dist: pydantic-extra-types[pycountry]~=2.11; extra == 'mineru'
 Requires-Dist: python-pptx~=1.0; extra == 'mineru'
 Requires-Dist: six~=1.17; extra == 'mineru'
+Description-Content-Type: text/markdown
+
+ô

extract_python-0.8.1/README.md

@@ -0,0 +1 @@
+ô

{extract_python-0.7.3 → extract_python-0.8.1}/extract_python/docling_.py

@@ -1,20 +1,27 @@
 import asyncio
 import json
 import logging
+import operator
 import shutil
 import tempfile
-from collections.abc import AsyncGenerator, Iterable, Iterator
-from functools import partial
+from collections.abc import AsyncIterable, Iterable
+from contextlib import AbstractContextManager
+from functools import partial, reduce
 from pathlib import Path
 from typing import Any, Self
 
-from docling.datamodel.document import ConversionResult
+from docling.datamodel.document import ConversionAssets
 from docling.datamodel.pipeline_options import PipelineOptions
+from docling.datamodel.settings import (
+    DEFAULT_PAGE_RANGE,
+    AppSettings,
+    scoped,
+)
 from docling.document_converter import DocumentConverter, FormatOption
+from docling_core.types import DoclingDocument
 
 # TODO: this is long to load improve it
 from docling_core.types.doc import ImageRefMode
-from docling_core.types.io import DocumentStream
 from extract_core import (
     BaseModel,
     DoclingFormatOption,
@@ -33,7 +40,14 @@ from pydantic import ConfigDict, field_serializer
 from pydantic_core.core_schema import SerializerFunctionWrapHandler
 
 from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP
-from .utils import chdir, map_and_preserve, path_to_artifacts_dirname, write_pages
+from .utils import (
+    Range,
+    ResultBuffer,
+    batch_per_pages,
+    chdir,
+    path_to_artifacts_dirname,
+    write_pages,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -61,54 +75,112 @@ class DoclingPipeline(Pipeline):
 
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
-    ) -> AsyncGenerator[Result, None]:
-        docs, path_or_streams = map_and_preserve(_to_docling, docs)
-        outputs = self._converter.convert_all(path_or_streams, raises_on_error=False)
+    ) -> AsyncIterable[Result]:
+        settings = self._config.settings
+        logger.info("starting extraction with settings: %s", settings)
+        with self._scoped_settings, self._result_buffer as buffer:
+            max_page_batches = settings.perf.max_page_batches
+            page_batch_size = settings.perf.page_batch_size
+            batches = batch_per_pages(
+                docs, page_batch_size, max_page_batches=max_page_batches
+            )
+            for batch in batches:
+                page_range = list({pages.page_range for pages in batch})
+                if len(page_range) > 1:
+                    msg = "convert_all only accept 1 page range for all docs"
+                    raise ValueError(msg)
+                page_range = page_range[0]
+                page_range = _docling_range(page_range)
+                docling_docs = (pages.doc.to_docling() for pages in batch)
+                outputs = self._converter.convert_all(
+                    docling_docs, raises_on_error=False, page_range=page_range
+                )
+                processed = iter(batch)
+                sentinel = object()
+                while True:
+                    res = await asyncio.to_thread(next, outputs, sentinel)
+                    if res is sentinel:
+                        break
+                    pages = next(processed)
+                    buffer.add(pages, res)
+                    if buffer.is_complete(pages.doc_idx):
+                        doc_pages = buffer.pop_complete(pages.doc_idx)
+                        yield _to_result(
+                            doc_pages, pages.doc, output_format, output_path=output_path
+                        )
+
+    @property
+    def _scoped_settings(self) -> AbstractContextManager[AppSettings]:
+        settings = self._config.settings
+        docling_settings = scoped(
+            perf=settings.perf, debug=settings.debug, inference=settings.inference
+        )
+        return docling_settings
+
+    @property
+    def _result_buffer(self) -> ResultBuffer:
+        buffer = ResultBuffer(
+            max_size_bytes=self._config.result_buffer.max_size,
+            root=self._config.result_buffer.root,
+            save_fn=_save_conversion_result,
+            load_fn=_load_conversion_result,
+        )
+        return buffer
+
 
-        sentinel = object()
-        while True:
-            res = await asyncio.to_thread(next, outputs, sentinel)
-            if res is sentinel:
-                return
-            doc = next(docs)
-            yield _to_result(res, doc, output_format, output_path=output_path)
+def _save_conversion_result(res: ConversionAssets, path: Path) -> None:
+    return res.save(filename=path)
 
 
-def _to_docling(docs: Iterable[InputDoc]) -> Iterator["Path | DocumentStream"]:
-    for d in docs:
-        yield d.to_docling()
+def _load_conversion_result(path: Path) -> ConversionAssets:
+    return ConversionAssets.load(path)
 
 
 def _to_result(
-    res: ConversionResult,
-    input_document: InputDoc,
+    buffer: list[ConversionAssets],
+    input_doc: InputDoc,
     output_format: OutputFormat,
     output_path: Path,
     **kwargs,
 ) -> Result:
+    import numpy as np  # noqa: PLC0415
+
+    if not buffer:
+        raise ValueError("empty buffer")
+    merged = DoclingDocument.concatenate([res.document for res in buffer])
     output_path.mkdir(parents=True, exist_ok=True)
+    status = reduce(operator.iadd, (Status.from_docling(d.status) for d in buffer))
+    # TODO: implement confidence weight
+    confidence = np.mean([res.confidence.mean_score for res in buffer])
     output = None
-    status = Status.from_docling(res.status)
     if status.allows_conversion:
         match output_format:
             case OutputFormat.MARKDOWN:
-                output = _to_markdown_doc(res, output_path, **kwargs)
+                output = _to_markdown_doc(
+                    merged,
+                    input_path=input_doc.path,
+                    output_path=output_path,
+                    confidence=confidence,
+                    **kwargs,
+                )
             case _:
                 raise NotImplementedError(f"unsupported output format {output_format}")
-    errors = [Error.from_docling(e) for e in res.errors]
-    input_doc = input_document.without_content()
+    errors = [Error.from_docling(e) for res in buffer for e in res.errors]
     return Result(input=input_doc, status=status, errors=errors, output=output)
 
 
 def _to_markdown_doc(
-    res: ConversionResult,
+    doc: DoclingDocument,
+    input_path: Path,
+    *,
     output_path: Path,
     page_sep: str = DEFAULT_MD_PAGE_SEP,
+    confidence: float,
     **kwargs,
 ) -> MarkdownDoc:
     # TODO: Should we add a hash to avoid collision between files with same names
     #  nested in the tree structured
-    md_dir_name = path_to_artifacts_dirname(res.input.file)
+    md_dir_name = path_to_artifacts_dirname(input_path)
     md_dir = output_path / md_dir_name
     if md_dir.exists():
         raise FileExistsError(f"directory {md_dir} already exists")
@@ -121,21 +193,21 @@ def _to_markdown_doc(
         with chdir(tmp_dir):
             # We do a chdir to bypass a Docling bug which only allows to maintain
             # relative image ref when saving the markdown to a relative path
-            pages = _docling_pages_it(res, current_page_path, **kwargs)
+            pages = _docling_pages_it(doc, current_page_path, **kwargs)
             with md_path.open("wb") as f:
                 pages = write_pages(pages, page_sep, f)
         # Clean up the tmp page file before move everything to the end destination
         current_page_path.unlink(missing_ok=True)
         shutil.move(tmp_dir, md_dir)
-    return MarkdownDoc(path=Path(md_dir_name), pages=pages)
+    return MarkdownDoc(path=Path(md_dir_name), pages=pages, confidence=confidence)
 
 
 def _docling_pages_it(
-    res: ConversionResult, output_path: Path, **kwargs
+    doc: DoclingDocument, output_path: Path, **kwargs
 ) -> Iterable[str]:
-    n_pages = len(res.pages)
+    n_pages = len(doc.pages)
     for page_i in range(n_pages):
-        res.document.save_as_markdown(
+        doc.save_as_markdown(
             output_path,
             page_no=page_i + 1,
             image_mode=ImageRefMode.REFERENCED,
@@ -193,3 +265,9 @@ class SerializableFormatOptions(DoclingFormatOption):
                 serialized["table_structure_options"] = dict()
             serialized["table_structure_options"]["kind"] = table_structure_opts.kind
         return serialized
+
+
+def _docling_range(rng: Range | None) -> tuple[int, int]:
+    if rng is None:
+        return DEFAULT_PAGE_RANGE
+    return (rng[0] + 1, rng[1] + 1)

{extract_python-0.7.3 → extract_python-0.8.1}/extract_python/marker_.py

@@ -1,6 +1,6 @@
 import asyncio
 import gc
-from collections.abc import AsyncGenerator, Iterable
+from collections.abc import AsyncIterable, Iterable
 from copy import deepcopy
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -30,7 +30,7 @@ _MARKER_CONVERSION_ERRORS = tuple()
 class MarkerPipeline(Pipeline):
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
-    ) -> AsyncGenerator[Result, None]:
+    ) -> AsyncIterable[Result]:
         from marker.config.parser import ConfigParser  # noqa: PLC0415
         from marker.converters.pdf import PdfConverter  # noqa: PLC0415
         from marker.models import create_model_dict  # noqa: PLC0415
@@ -67,8 +67,7 @@ async def _process_doc(
             )
         case _:
             raise NotImplementedError(f"unsupported output format {output_format}")
-    input_doc = doc.without_content()
-    return Result(input=input_doc, status=Status.SUCCESS, output=output)
+    return Result(input=doc, status=Status.SUCCESS, output=output)
 
 
 def _to_markdown_doc(
@@ -96,4 +95,4 @@ def _to_markdown_doc(
     md_path = md_path.with_suffix(OutputFormat.MARKDOWN.value)
     with md_path.open("wb") as f:
         pages = write_pages(pages, page_sep, f)
-    return MarkdownDoc(path=Path(md_dir_name), pages=pages)
+    return MarkdownDoc(path=Path(md_dir_name), pages=pages, confidence=None)

{extract_python-0.7.3 → extract_python-0.8.1}/extract_python/miner_u.py

@@ -1,7 +1,7 @@
 import json
 import os
 import shutil
-from collections.abc import AsyncGenerator, Callable, Iterable
+from collections.abc import AsyncIterable, Callable, Iterable
 from functools import partial
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -34,7 +34,7 @@ class MinerUPipeline(Pipeline):
 
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
-    ) -> AsyncGenerator[Result, None]:
+    ) -> AsyncIterable[Result]:
         from mineru.cli.common import aio_do_parse  # noqa: PLC0415
 
         with reset_env():
@@ -126,15 +126,13 @@ def _process_doc(
             raise NotImplementedError(f"unsupported output format {output_format}")
     middle_json_path = res_path / f"{doc.path.name}_middle.json"
     middle_json = json.loads(middle_json_path.read_text())
-    pdf_info = middle_json["pdf_info"]
     shutil.move(res_path / "images", artifacts_dir)
-    output = dump_content_fn(pdf_info)
-    input_doc = doc.without_content()
-    return Result(input=input_doc, status=Status.SUCCESS, output=output)
+    output = dump_content_fn(middle_json)
+    return Result(input=doc, status=Status.SUCCESS, output=output)
 
 
 def _dump_md_content(
-    pdf_info: list[dict],
+    middle_json: dict,
     *,
     md_make_fn: MDMakeFunction,
     page_sep: str = DEFAULT_MD_PAGE_SEP,
@@ -145,11 +143,72 @@ def _dump_md_content(
 ) -> ConversionOutput:
     from mineru.utils.enum_class import MakeMode  # noqa: PLC0415
 
+    pdf_info = middle_json["pdf_info"]
     if md_make_mode is None:
         md_make_mode = MakeMode.MM_MD
     pages = (md_make_fn([p], md_make_mode, str(im_dir)) for p in pdf_info)
     with md_path.open("wb") as f:
         pages = write_pages(pages, page_sep, f)
     output_path = md_path.parent.relative_to(output_path)
-    output = ConversionOutput(path=output_path, pages=pages)
+    confidence = _mineru_confidence(pdf_info)
+    output = ConversionOutput(path=output_path, pages=pages, confidence=confidence)
     return output
+
+
+def _mineru_confidence(pdf_info: list[dict]) -> float:
+    if not pdf_info:
+        return 1.0
+    block_conf = _mineru_block_confidence(pdf_info)
+    line_config = _mineru_line_confidence(pdf_info)
+    return (block_conf + line_config) / 2.0
+
+
+def _mineru_block_confidence(pdf_info: list[dict]) -> float:
+    import numpy as np  # noqa: PLC0415
+
+    scores = []
+    for info in pdf_info:
+        for block in info["para_blocks"]:
+            score = block.get("score")
+            if score is not None:
+                scores.append(score)
+    if scores:
+        return np.average(scores)
+    return 1.0
+
+
+def _mineru_line_confidence(pdf_info: list[dict]) -> float:
+    import numpy as np  # noqa: PLC0415
+
+    scores = []
+    lengths = []
+    for info in pdf_info:
+        for block in info["para_blocks"]:
+            for line in block.get("lines", []):
+                for span in line["spans"]:
+                    score = span.get("score")
+                    if score is not None:
+                        scores.append(score)
+                        lengths.append(len(span["content"]))
+    if scores:
+        return np.average(scores, weights=lengths)
+    return 1.0
+
+
+def _parse_block(block: dict) -> tuple[list[float], list[float]]:
+    if "lines" in block:
+        scores = []
+        lengths = []
+        for line in block.get("lines", []):
+            for span in line["spans"]:
+                score = span.get("score")
+                if score is not None:
+                    scores.append(score)
+                    lengths.append(len(span["content"]))
+        return scores, lengths
+    if "blocs" in block:
+        scores, lengths = (_parse_block(b) for b in block["blocs"])
+        scores = sum(*scores, start=[])
+        lengths = sum(*lengths, start=[])
+        return scores, lengths
+    raise NotImplementedError(f"unsupported block: {block}")

extract_python-0.8.1/extract_python/utils.py

@@ -0,0 +1,299 @@
+import gc
+import itertools
+import logging
+import os
+import shutil
+import uuid
+from collections import defaultdict, deque
+from collections.abc import Callable, Generator, Iterable, Iterator
+from contextlib import contextmanager
+from copy import copy
+from dataclasses import dataclass
+from functools import wraps
+from itertools import tee
+from pathlib import Path, PurePath
+from tempfile import TemporaryDirectory
+from types import TracebackType
+from typing import BinaryIO, Protocol, Self
+
+from extract_core import Error, InputDoc, Pages, Result, Status
+from pympler import asizeof
+
+logger = logging.getLogger(__name__)
+
+
+def map_and_preserve[I, R](
+    fn: Callable[[Iterable[I]], Iterator[R]], inputs: Iterable[I]
+) -> tuple[Iterable[I], Iterator[R]]:
+    save_inputs, function_inputs = tee(inputs)
+    outputs = iter(fn(function_inputs))
+    return save_inputs, outputs
+
+
+def path_to_artifacts_dirname(path: PurePath, sep: str = "_") -> str:
+    dirname = f"{path.name[: -len(path.suffix)]}"
+    ext = path.suffix
+    if ext:
+        dirname += sep + ext[1:]
+    return dirname
+
+
+class DocProcessingFn(Protocol):
+    def __call__(self, doc: InputDoc, *arg, **kwargs) -> Result: ...
+
+
+def report_recoverable_errors(
+    recoverable_errors: tuple[type[Exception], ...] = tuple(),
+) -> Callable[[DocProcessingFn], DocProcessingFn]:
+    def make_decorator(f: DocProcessingFn) -> DocProcessingFn:
+        @wraps(f)
+        def wrapped(doc: InputDoc, *args, **kwargs) -> Result:
+            try:
+                return f(doc, *args, **kwargs)
+            except recoverable_errors as e:
+                error = Error.from_exception(e)
+                return Result(
+                    input=doc, status=Status.FAILURE, errors=[error], output=None
+                )
+
+        return wrapped
+
+    return make_decorator
+
+
+@contextmanager
+def chdir(path: Path) -> Generator[None]:
+    cwd = Path.cwd()
+    try:
+        os.chdir(path)
+        yield
+    finally:
+        os.chdir(cwd)
+
+
+@contextmanager
+def reset_env() -> Generator[None]:
+    old_env = copy(dict(os.environ))
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(old_env)
+
+
+def write_pages(pages: Iterable[str], page_sep: str, out: BinaryIO) -> Pages:
+    pages_byte_sizes = []
+    pages = iter(pages)
+    content = None
+    for p in pages:
+        if content:
+            pages_byte_sizes.append(out.write((content + page_sep).encode()))
+        content = p
+    if content:
+        pages_byte_sizes.append(out.write(content.encode()))
+    return Pages.from_pages_bytes_sizes(pages_byte_sizes)
+
+
+Range = tuple[int, int]
+
+
+@dataclass(frozen=True)
+class ProcessedPages:
+    doc: InputDoc
+    doc_idx: int
+    page_range: Range | None = None
+
+    @property
+    def page_length(self) -> int:
+        if self.page_range is None:
+            return self.doc.n_pages
+        return self.page_range[1] - self.page_range[0]
+
+
+class ResultBuffer[R]:
+    def __init__(
+        self,
+        max_size_bytes: int,
+        save_fn: Callable[[R, Path], None],
+        *,
+        load_fn: Callable[[Path], R],
+        root: Path | None = None,
+    ):
+        self._max_bytes = max_size_bytes
+        self._save_fn = save_fn
+        self._load_fn = load_fn
+        self._tmp_dir = None
+        if root is None:
+            self._tmp_dir = TemporaryDirectory()
+            root = Path(self._tmp_dir.name)
+        self._root = root
+        self.__fs_buffer_path = None
+        self._mem_buffer: dict[int, list[R | Path]] = defaultdict(list)
+        self._missing_pages: dict[int, int] = dict()
+        self._current_size: int = 0
+
+    def __enter__(self) -> Self:
+        if self._tmp_dir is not None:
+            self._tmp_dir.__enter__()
+        self.__fs_buffer_path = self._root / uuid.uuid4().hex
+        self.__fs_buffer_path.mkdir()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        if self._mem_buffer or self._missing_pages:
+            logger.warning("closing an non empty buffer")
+        if self._tmp_dir is not None:
+            self._tmp_dir.__exit__(exc_type, exc_val, exc_tb)
+        if self._fs_buffer_path.exists():
+            shutil.rmtree(self._fs_buffer_path)
+        self._mem_buffer = dict()
+        self._missing_pages = dict()
+
+    @property
+    def _fs_buffer_path(self) -> Path:
+        if not self.__fs_buffer_path:
+            msg = (
+                f"inconsistent state, {ResultBuffer.__class__.__name__} is a context"
+                f" manager, call __enter__ before using it"
+            )
+            raise ValueError(msg)
+        return self.__fs_buffer_path
+
+    def add(self, processed: ProcessedPages, result: R) -> None:
+        size = asizeof.asizeof(result)
+        if self._current_size + size > self._max_bytes:
+            path = self._page_path(processed.doc_idx)
+            self._save_fn(result, path)
+            result = path
+        else:
+            self._current_size += size
+        self._mem_buffer[processed.doc_idx].append(result)
+        if processed.doc_idx not in self._missing_pages:
+            self._missing_pages[processed.doc_idx] = processed.doc.n_pages
+        self._missing_pages[processed.doc_idx] -= processed.page_length
+
+    def is_complete(self, doc: int) -> bool:
+        return self._missing_pages[doc] == 0
+
+    def pop_complete(self, doc: int) -> list[R]:
+        if not self.is_complete(doc):
+            raise ValueError(f"{doc} is incomplete")
+        pages = self._mem_buffer.pop(doc)
+        self._missing_pages.pop(doc)
+        for i, page in enumerate(pages):
+            if isinstance(page, Path):
+                page = self._load_fn(page)  # noqa: PLW2901
+            else:
+                self._current_size -= asizeof.asizeof(page)
+            pages[i] = page
+        return pages
+
+    def _page_path(self, doc_id: int) -> Path:
+        pages = self._mem_buffer[doc_id]
+        return self._fs_buffer_path / f"doc-{doc_id}-pages-{len(pages)}"
+
+    def __len__(self) -> int:
+        return len(self._mem_buffer)
+
+
+# The converter process page_batch_size in parallel (GPU sees page_batch_size batches).
+#
+# The batching tradeoff is: avoid calling convert_all to many times vs. releasing the
+# GIL often enough.
+#
+# Calling convert_all to many times on the same doc results in overhead. Each time we
+# call the function, we create a doc processing backend + reload the doc.
+#
+# On the other hand we have to use a reasonable max_page_batches otherwise we process
+# all the stream in a single call and take the risk to lock the GIL for too long. Some
+# docling ops are sadly not async (numpy or torch inference are, but document loading
+# and conversion aren't, so the asyncio.to_thread is not helping)
+def batch_per_pages(
+    docs: Iterable[InputDoc],
+    page_batch_size: int,
+    *,
+    max_page_batches: int,
+    chunk_size: int = 1000,
+) -> Iterable[tuple[ProcessedPages]]:
+    # convert_all only accept to process docs on the exact same page_range
+    #
+    # We collect by chunk to avoid collecting too many inputs, input docs are
+    # lightweight anyway so memory impact should stay limited
+    #
+    # Additionally, results can be output unordered and partial results are buffered
+    # it's OK to process doc pages unordered.
+    # TODO: if it's not OK to sort because inputs is l
+    max_pages = page_batch_size * max_page_batches
+    docs = itertools.batched(docs, chunk_size, strict=False)
+    for chunk in docs:
+        short_docs = [d for d in chunk if d.n_pages <= max_pages]
+        long_docs = [d for d in chunk if d.n_pages > max_pages]
+        del chunk
+        gc.collect()
+        # Bin fill for docs smaller than max_pages
+        offset = yield from _bin_fill(short_docs, max_pages=max_pages)
+        # otherwise we just yield chunks of max_pages except the last chunk which is
+        # grouped by page_range
+        yield from _by_page_ranges(long_docs, max_pages=max_pages, offset=offset)
+
+
+def _bin_fill(
+    docs: Iterable[InputDoc], max_pages: int, offset: int = 0
+) -> Generator[tuple[ProcessedPages], None, int]:
+    bins = defaultdict(deque)
+    doc_idx = offset
+    for doc in docs:
+        if doc.n_pages > max_pages:
+            msg = f"expected docs to have <= {max_pages} pages"
+            raise ValueError(msg)
+        pages = ProcessedPages(doc=doc, doc_idx=doc_idx)
+        doc_idx += 1
+        available_space = (s for s in sorted(bins.keys()) if doc.n_pages <= s)
+        available_space = next(available_space, max_pages)
+        selected = bins[available_space]
+        selected = selected.pop() if selected else []
+        selected.append(pages)
+        available_space -= doc.n_pages
+        if available_space == 0:
+            yield tuple(selected)
+            continue
+        bins[available_space].append(selected)
+    for range_bins in bins.values():
+        for b in range_bins:
+            yield tuple(b)
+    return doc_idx
+
+
+def _by_page_ranges(
+    docs: Iterable[InputDoc], max_pages: int, offset: int = 0
+) -> Generator[tuple[ProcessedPages], None, int]:
+    by_range = defaultdict(list)
+    doc_idx = offset
+    for doc in docs:
+        if doc.n_pages < max_pages:
+            msg = f"expected docs to have >= {max_pages} pages"
+            raise ValueError(msg)
+
+        for i in range(0, doc.n_pages, max_pages):
+            start = i
+            end = min(start + max_pages, doc.n_pages)
+            rng = (start, end)
+            rng_size = end - start
+            alone_in_batch = rng_size == max_pages
+            pages = ProcessedPages(doc=doc, doc_idx=doc_idx, page_range=rng)
+            if alone_in_batch:
+                yield (pages,)
+                continue
+            by_range[rng].append(pages)
+            is_complete = len(by_range[rng]) == (max_pages // rng_size)
+            if is_complete:
+                yield tuple(by_range.pop(rng))
+        doc_idx += 1
+    for v in by_range.values():
+        yield tuple(v)
+    return doc_idx

{extract_python-0.7.3 → extract_python-0.8.1}/pyproject.toml

@@ -8,8 +8,9 @@ authors = [
 readme = "README.md"
 requires-python = ">=3.13,<3.15"
 dependencies = [
+  "extract-core~=0.8.0",
   "icij-common~=0.8.2",
-  "extract-core~=0.7.0",
+  "pympler~=1.1",
 ]
 
 [project.optional-dependencies]

{extract_python-0.7.3 → extract_python-0.8.1}/uv.lock

@@ -919,6 +919,7 @@ source = { editable = "." }
 dependencies = [
     { name = "extract-core" },
     { name = "icij-common" },
+    { name = "pympler" },
 ]
 
 [package.optional-dependencies]
@@ -965,6 +966,7 @@ requires-dist = [
     { name = "mineru", extras = ["pipeline", "vlm"], marker = "extra == 'mineru'", specifier = "~=3.2" },
     { name = "notebook", marker = "extra == 'benches'", specifier = ">=7.4.5" },
     { name = "pydantic-extra-types", extras = ["pycountry"], marker = "extra == 'mineru'", specifier = "~=2.11" },
+    { name = "pympler", specifier = "~=1.1" },
     { name = "pypdfium2", marker = "extra == 'benches'", specifier = ">=4.30.0" },
     { name = "python-pptx", marker = "extra == 'mineru'", specifier = "~=1.0" },
     { name = "six", marker = "extra == 'mineru'", specifier = "~=1.17" },
@@ -2600,9 +2602,9 @@ name = "ocrmac"
 version = "1.0.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "click", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
-    { name = "pillow", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
-    { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
+    { name = "click", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
+    { name = "pillow", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
+    { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/5e/07/3e15ab404f75875c5e48c47163300eb90b7409044d8711fc3aaf52503f2e/ocrmac-1.0.1.tar.gz", hash = "sha256:507fe5e4cbd67b2d03f6729a52bbc11f9d0b58241134eb958a5daafd4b9d93d9", size = 1454317, upload-time = "2026-01-08T16:44:26.412Z" }
 wheels = [
@@ -3291,6 +3293,18 @@ version = "2.10"
 source = { registry = "https://pypi.org/simple" }
 sdist = { url = "https://files.pythonhosted.org/packages/5d/ab/34ec41718af73c00119d0351b7a2531d2ebddb51833a36448fc7b862be60/pylatexenc-2.10.tar.gz", hash = "sha256:3dd8fd84eb46dc30bee1e23eaab8d8fb5a7f507347b23e5f38ad9675c84f40d3", size = 162597, upload-time = "2021-04-06T07:56:07.854Z" }
 
+[[package]]
+name = "pympler"
+version = "1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/dd/37/c384631908029676d8e7213dd956bb686af303a80db7afbc9be36bc49495/pympler-1.1.tar.gz", hash = "sha256:1eaa867cb8992c218430f1708fdaccda53df064144d1c5656b1e6f1ee6000424", size = 179954, upload-time = "2024-06-28T19:56:06.563Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/79/4f/a6a2e2b202d7fd97eadfe90979845b8706676b41cbd3b42ba75adf329d1f/Pympler-1.1-py3-none-any.whl", hash = "sha256:5b223d6027d0619584116a0cbc28e8d2e378f7a79c1e5e024f9ff3b673c58506", size = 165766, upload-time = "2024-06-28T19:56:05.087Z" },
+]
+
 [[package]]
 name = "pyobjc-core"
 version = "12.2.1"
@@ -3308,7 +3322,7 @@ name = "pyobjc-framework-cocoa"
 version = "12.2.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "pyobjc-core", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
+    { name = "pyobjc-core", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/51/34/fbe38a204643aa4e1b91391cdce07a34da565a69171ebcad08de7438a556/pyobjc_framework_cocoa-12.2.1.tar.gz", hash = "sha256:b94b37fe5730e5ae1fb0052912cd174e6ec329b0bfba4a012ae5db1014b5864b", size = 3125751, upload-time = "2026-06-19T16:20:05.159Z" }
 wheels = [
@@ -3323,8 +3337,8 @@ name = "pyobjc-framework-coreml"
 version = "12.2.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "pyobjc-core", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
-    { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
+    { name = "pyobjc-core", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
+    { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/98/1e/7d2db3e4468eb04cc92264be83113d86eea4f96302742437de695a445d6d/pyobjc_framework_coreml-12.2.1.tar.gz", hash = "sha256:ef3c2b6a160891b44173235603d10174929656b9c206d6f2f443fe2aa903c2cb", size = 49272, upload-time = "2026-06-19T16:20:18.459Z" }
 wheels = [
@@ -3339,8 +3353,8 @@ name = "pyobjc-framework-quartz"
 version = "12.2.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "pyobjc-core", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
-    { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
+    { name = "pyobjc-core", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
+    { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/3b/f6/2a8b84dbf1fe7c04dd96ea73d991678d4e09a909f51971ecc51629bb2ab4/pyobjc_framework_quartz-12.2.1.tar.gz", hash = "sha256:b3b8b6f71e66147f8ff9e6213864cc8527e3a0b1ee90835b93ce221f4802d9b0", size = 3215521, upload-time = "2026-06-19T16:21:30.199Z" }
 wheels = [
@@ -3355,10 +3369,10 @@ name = "pyobjc-framework-vision"
 version = "12.2.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "pyobjc-core", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
-    { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
-    { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
-    { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" },
+    { name = "pyobjc-core", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
+    { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
+    { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
+    { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/0e/7a/1fdffff1b6bf124b260a2169869f4b71a08b9f6603698f7dec990d5ae5f3/pyobjc_framework_vision-12.2.1.tar.gz", hash = "sha256:debfd59dd7d962a6053bf733370148c11a9ec44091b517a0966f48d81c305879", size = 72683, upload-time = "2026-06-19T16:22:01.102Z" }
 wheels = [

extract_python-0.7.3/extract_python/utils.py

@@ -1,88 +0,0 @@
-import os
-from collections.abc import Callable, Generator, Iterable, Iterator
-from contextlib import contextmanager
-from copy import copy
-from functools import wraps
-from itertools import tee
-from pathlib import Path, PurePath
-from typing import BinaryIO, Protocol, TypeVar
-
-from extract_core import Error, InputDoc, Pages, Result, Status
-
-R = TypeVar("R")
-In = TypeVar("In")
-
-
-def map_and_preserve(
-    fn: Callable[[Iterable[In]], Iterator[R]], inputs: Iterable[In]
-) -> tuple[Iterable[In], Iterator[R]]:
-    save_inputs, function_inputs = tee(inputs)
-    outputs = iter(fn(function_inputs))
-    return save_inputs, outputs
-
-
-def path_to_artifacts_dirname(path: PurePath, sep: str = "_") -> str:
-    dirname = f"{path.name[: -len(path.suffix)]}"
-    ext = path.suffix
-    if ext:
-        dirname += sep + ext[1:]
-    return dirname
-
-
-class DocProcessingFn(Protocol):
-    def __call__(self, doc: InputDoc, *arg, **kwargs) -> Result: ...
-
-
-def report_recoverable_errors(
-    recoverable_errors: tuple[type[Exception], ...] = tuple(),
-) -> Callable[[DocProcessingFn], DocProcessingFn]:
-    def make_decorator(f: DocProcessingFn) -> DocProcessingFn:
-        @wraps(f)
-        def wrapped(doc: InputDoc, *args, **kwargs) -> Result:
-            try:
-                return f(doc, *args, **kwargs)
-            except recoverable_errors as e:
-                error = Error.from_exception(e)
-                return Result(
-                    input=doc.without_content(),
-                    status=Status.FAILURE,
-                    errors=[error],
-                    output=None,
-                )
-
-        return wrapped
-
-    return make_decorator
-
-
-@contextmanager
-def chdir(path: Path) -> Generator[None, None, None]:
-    cwd = Path.cwd()
-    try:
-        os.chdir(path)
-        yield
-    finally:
-        os.chdir(cwd)
-
-
-@contextmanager
-def reset_env() -> Generator[None, None, None]:
-    old_env = copy(dict(os.environ))
-    try:
-        yield
-    finally:
-        os.environ.clear()
-        os.environ.update(old_env)
-
-
-def write_pages(pages: Iterable[str], page_sep: str, out: BinaryIO) -> Pages:
-    pages_byte_sizes = []
-    pages = iter(pages)
-    content = None
-    for p in pages:
-        if content:
-            pages_byte_sizes.append(out.write((content + page_sep).encode()))
-        content = p
-    if content:
-        pages_byte_sizes.append(out.write(content.encode()))
-    return Pages.from_pages_bytes_sizes(pages_byte_sizes)