extract-python 0.7.2__tar.gz → 0.8.0__tar.gz

extract_python-0.8.0/.python-version

@@ -0,0 +1 @@
+3.14

{extract_python-0.7.2 → extract_python-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: extract-python
-Version: 0.7.2
+Version: 0.8.0
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python
@@ -9,6 +9,7 @@ Author-email: Clément Doumouro <cdoumouro@icij.org>
 Requires-Python: <3.15,>=3.13
 Requires-Dist: extract-core~=0.7.0
 Requires-Dist: icij-common~=0.8.2
+Requires-Dist: pympler~=1.1
 Provides-Extra: benches
 Requires-Dist: html2image~=2.0.7; extra == 'benches'
 Requires-Dist: markdown2>=2.5.4; extra == 'benches'
@@ -24,3 +25,6 @@ Requires-Dist: mineru[pipeline,vlm]~=3.2; extra == 'mineru'
 Requires-Dist: pydantic-extra-types[pycountry]~=2.11; extra == 'mineru'
 Requires-Dist: python-pptx~=1.0; extra == 'mineru'
 Requires-Dist: six~=1.17; extra == 'mineru'
+Description-Content-Type: text/markdown
+
+ô

extract_python-0.8.0/README.md

@@ -0,0 +1 @@
+ô

{extract_python-0.7.2 → extract_python-0.8.0}/extract_python/docling_.py

@@ -1,20 +1,27 @@
 import asyncio
 import json
 import logging
+import operator
 import shutil
 import tempfile
-from collections.abc import AsyncGenerator, Iterable, Iterator
-from functools import partial
+from collections.abc import AsyncIterable, Iterable
+from contextlib import AbstractContextManager
+from functools import partial, reduce
 from pathlib import Path
 from typing import Any, Self
 
-from docling.datamodel.document import ConversionResult
+from docling.datamodel.document import ConversionAssets
 from docling.datamodel.pipeline_options import PipelineOptions
+from docling.datamodel.settings import (
+    DEFAULT_PAGE_RANGE,
+    AppSettings,
+    scoped,
+)
 from docling.document_converter import DocumentConverter, FormatOption
+from docling_core.types import DoclingDocument
 
 # TODO: this is long to load improve it
 from docling_core.types.doc import ImageRefMode
-from docling_core.types.io import DocumentStream
 from extract_core import (
     BaseModel,
     DoclingFormatOption,
@@ -33,7 +40,14 @@ from pydantic import ConfigDict, field_serializer
 from pydantic_core.core_schema import SerializerFunctionWrapHandler
 
 from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP
-from .utils import chdir, map_and_preserve, path_to_artifacts_dirname, write_pages
+from .utils import (
+    Range,
+    ResultBuffer,
+    batch_per_pages,
+    chdir,
+    path_to_artifacts_dirname,
+    write_pages,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -61,54 +75,112 @@ class DoclingPipeline(Pipeline):
 
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
-    ) -> AsyncGenerator[Result, None]:
-        docs, path_or_streams = map_and_preserve(_to_docling, docs)
-        outputs = self._converter.convert_all(path_or_streams, raises_on_error=False)
+    ) -> AsyncIterable[Result]:
+        settings = self._config.settings
+        logger.info("starting extraction with settings: %s", settings)
+        with self._scoped_settings, self._result_buffer as buffer:
+            max_page_batches = settings.perf.max_page_batches
+            page_batch_size = settings.perf.page_batch_size
+            batches = batch_per_pages(
+                docs, page_batch_size, max_page_batches=max_page_batches
+            )
+            for batch in batches:
+                page_range = list({pages.page_range for pages in batch})
+                if len(page_range) > 1:
+                    msg = "convert_all only accept 1 page range for all docs"
+                    raise ValueError(msg)
+                page_range = page_range[0]
+                page_range = _docling_range(page_range)
+                docling_docs = (pages.doc.to_docling() for pages in batch)
+                outputs = self._converter.convert_all(
+                    docling_docs, raises_on_error=False, page_range=page_range
+                )
+                processed = iter(batch)
+                sentinel = object()
+                while True:
+                    res = await asyncio.to_thread(next, outputs, sentinel)
+                    if res is sentinel:
+                        break
+                    pages = next(processed)
+                    buffer.add(pages, res)
+                    if buffer.is_complete(pages.doc_idx):
+                        doc_pages = buffer.pop_complete(pages.doc_idx)
+                        yield _to_result(
+                            doc_pages, pages.doc, output_format, output_path=output_path
+                        )
+
+    @property
+    def _scoped_settings(self) -> AbstractContextManager[AppSettings]:
+        settings = self._config.settings
+        docling_settings = scoped(
+            perf=settings.perf, debug=settings.debug, inference=settings.inference
+        )
+        return docling_settings
+
+    @property
+    def _result_buffer(self) -> ResultBuffer:
+        buffer = ResultBuffer(
+            max_size_bytes=self._config.result_buffer.max_size,
+            root=self._config.result_buffer.root,
+            save_fn=_save_conversion_result,
+            load_fn=_load_conversion_result,
+        )
+        return buffer
+
 
-        sentinel = object()
-        while True:
-            res = await asyncio.to_thread(next, outputs, sentinel)
-            if res is sentinel:
-                return
-            doc = next(docs)
-            yield _to_result(res, doc, output_format, output_path=output_path)
+def _save_conversion_result(res: ConversionAssets, path: Path) -> None:
+    return res.save(filename=path)
 
 
-def _to_docling(docs: Iterable[InputDoc]) -> Iterator["Path | DocumentStream"]:
-    for d in docs:
-        yield d.to_docling()
+def _load_conversion_result(path: Path) -> ConversionAssets:
+    return ConversionAssets.load(path)
 
 
 def _to_result(
-    res: ConversionResult,
-    input_document: InputDoc,
+    buffer: list[ConversionAssets],
+    input_doc: InputDoc,
     output_format: OutputFormat,
     output_path: Path,
     **kwargs,
 ) -> Result:
+    import numpy as np  # noqa: PLC0415
+
+    if not buffer:
+        raise ValueError("empty buffer")
+    merged = DoclingDocument.concatenate([res.document for res in buffer])
     output_path.mkdir(parents=True, exist_ok=True)
+    status = reduce(operator.iadd, (Status.from_docling(d.status) for d in buffer))
+    # TODO: implement confidence weight
+    confidence = np.mean([res.confidence.mean_score for res in buffer])
     output = None
-    status = Status.from_docling(res.status)
     if status.allows_conversion:
         match output_format:
             case OutputFormat.MARKDOWN:
-                output = _to_markdown_doc(res, output_path, **kwargs)
+                output = _to_markdown_doc(
+                    merged,
+                    input_path=input_doc.path,
+                    output_path=output_path,
+                    confidence=confidence,
+                    **kwargs,
+                )
             case _:
                 raise NotImplementedError(f"unsupported output format {output_format}")
-    errors = [Error.from_docling(e) for e in res.errors]
-    input_doc = input_document.without_content()
+    errors = [Error.from_docling(e) for res in buffer for e in res.errors]
     return Result(input=input_doc, status=status, errors=errors, output=output)
 
 
 def _to_markdown_doc(
-    res: ConversionResult,
+    doc: DoclingDocument,
+    input_path: Path,
+    *,
     output_path: Path,
     page_sep: str = DEFAULT_MD_PAGE_SEP,
+    confidence: float,
     **kwargs,
 ) -> MarkdownDoc:
     # TODO: Should we add a hash to avoid collision between files with same names
     #  nested in the tree structured
-    md_dir_name = path_to_artifacts_dirname(res.input.file)
+    md_dir_name = path_to_artifacts_dirname(input_path)
     md_dir = output_path / md_dir_name
     if md_dir.exists():
         raise FileExistsError(f"directory {md_dir} already exists")
@@ -121,21 +193,21 @@ def _to_markdown_doc(
         with chdir(tmp_dir):
             # We do a chdir to bypass a Docling bug which only allows to maintain
             # relative image ref when saving the markdown to a relative path
-            pages = _docling_pages_it(res, current_page_path, **kwargs)
+            pages = _docling_pages_it(doc, current_page_path, **kwargs)
             with md_path.open("wb") as f:
                 pages = write_pages(pages, page_sep, f)
         # Clean up the tmp page file before move everything to the end destination
         current_page_path.unlink(missing_ok=True)
         shutil.move(tmp_dir, md_dir)
-    return MarkdownDoc(path=Path(md_dir_name), pages=pages)
+    return MarkdownDoc(path=Path(md_dir_name), pages=pages, confidence=confidence)
 
 
 def _docling_pages_it(
-    res: ConversionResult, output_path: Path, **kwargs
+    doc: DoclingDocument, output_path: Path, **kwargs
 ) -> Iterable[str]:
-    n_pages = len(res.pages)
+    n_pages = len(doc.pages)
     for page_i in range(n_pages):
-        res.document.save_as_markdown(
+        doc.save_as_markdown(
             output_path,
             page_no=page_i + 1,
             image_mode=ImageRefMode.REFERENCED,
@@ -193,3 +265,9 @@ class SerializableFormatOptions(DoclingFormatOption):
                 serialized["table_structure_options"] = dict()
             serialized["table_structure_options"]["kind"] = table_structure_opts.kind
         return serialized
+
+
+def _docling_range(rng: Range | None) -> tuple[int, int]:
+    if rng is None:
+        return DEFAULT_PAGE_RANGE
+    return (rng[0] + 1, rng[1] + 1)

{extract_python-0.7.2 → extract_python-0.8.0}/extract_python/marker_.py

@@ -1,6 +1,6 @@
 import asyncio
 import gc
-from collections.abc import AsyncGenerator, Iterable
+from collections.abc import AsyncIterable, Iterable
 from copy import deepcopy
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -30,7 +30,7 @@ _MARKER_CONVERSION_ERRORS = tuple()
 class MarkerPipeline(Pipeline):
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
-    ) -> AsyncGenerator[Result, None]:
+    ) -> AsyncIterable[Result]:
         from marker.config.parser import ConfigParser  # noqa: PLC0415
         from marker.converters.pdf import PdfConverter  # noqa: PLC0415
         from marker.models import create_model_dict  # noqa: PLC0415
@@ -67,8 +67,7 @@ async def _process_doc(
             )
         case _:
             raise NotImplementedError(f"unsupported output format {output_format}")
-    input_doc = doc.without_content()
-    return Result(input=input_doc, status=Status.SUCCESS, output=output)
+    return Result(input=doc, status=Status.SUCCESS, output=output)
 
 
 def _to_markdown_doc(
@@ -96,4 +95,4 @@ def _to_markdown_doc(
     md_path = md_path.with_suffix(OutputFormat.MARKDOWN.value)
     with md_path.open("wb") as f:
         pages = write_pages(pages, page_sep, f)
-    return MarkdownDoc(path=Path(md_dir_name), pages=pages)
+    return MarkdownDoc(path=Path(md_dir_name), pages=pages, confidence=None)

{extract_python-0.7.2 → extract_python-0.8.0}/extract_python/miner_u.py

@@ -1,7 +1,7 @@
 import json
 import os
 import shutil
-from collections.abc import AsyncGenerator, Callable, Iterable
+from collections.abc import AsyncIterable, Callable, Iterable
 from functools import partial
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -34,7 +34,7 @@ class MinerUPipeline(Pipeline):
 
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
-    ) -> AsyncGenerator[Result, None]:
+    ) -> AsyncIterable[Result]:
         from mineru.cli.common import aio_do_parse  # noqa: PLC0415
 
         with reset_env():
@@ -126,15 +126,13 @@ def _process_doc(
             raise NotImplementedError(f"unsupported output format {output_format}")
     middle_json_path = res_path / f"{doc.path.name}_middle.json"
     middle_json = json.loads(middle_json_path.read_text())
-    pdf_info = middle_json["pdf_info"]
     shutil.move(res_path / "images", artifacts_dir)
-    output = dump_content_fn(pdf_info)
-    input_doc = doc.without_content()
-    return Result(input=input_doc, status=Status.SUCCESS, output=output)
+    output = dump_content_fn(middle_json)
+    return Result(input=doc, status=Status.SUCCESS, output=output)
 
 
 def _dump_md_content(
-    pdf_info: list[dict],
+    middle_json: dict,
     *,
     md_make_fn: MDMakeFunction,
     page_sep: str = DEFAULT_MD_PAGE_SEP,
@@ -145,11 +143,72 @@ def _dump_md_content(
 ) -> ConversionOutput:
     from mineru.utils.enum_class import MakeMode  # noqa: PLC0415
 
+    pdf_info = middle_json["pdf_info"]
     if md_make_mode is None:
         md_make_mode = MakeMode.MM_MD
     pages = (md_make_fn([p], md_make_mode, str(im_dir)) for p in pdf_info)
     with md_path.open("wb") as f:
         pages = write_pages(pages, page_sep, f)
     output_path = md_path.parent.relative_to(output_path)
-    output = ConversionOutput(path=output_path, pages=pages)
+    confidence = _mineru_confidence(pdf_info)
+    output = ConversionOutput(path=output_path, pages=pages, confidence=confidence)
     return output
+
+
+def _mineru_confidence(pdf_info: list[dict]) -> float:
+    if not pdf_info:
+        return 1.0
+    block_conf = _mineru_block_confidence(pdf_info)
+    line_config = _mineru_line_confidence(pdf_info)
+    return (block_conf + line_config) / 2.0
+
+
+def _mineru_block_confidence(pdf_info: list[dict]) -> float:
+    import numpy as np  # noqa: PLC0415
+
+    scores = []
+    for info in pdf_info:
+        for block in info["para_blocks"]:
+            score = block.get("score")
+            if score is not None:
+                scores.append(score)
+    if scores:
+        return np.average(scores)
+    return 1.0
+
+
+def _mineru_line_confidence(pdf_info: list[dict]) -> float:
+    import numpy as np  # noqa: PLC0415
+
+    scores = []
+    lengths = []
+    for info in pdf_info:
+        for block in info["para_blocks"]:
+            for line in block.get("lines", []):
+                for span in line["spans"]:
+                    score = span.get("score")
+                    if score is not None:
+                        scores.append(score)
+                        lengths.append(len(span["content"]))
+    if scores:
+        return np.average(scores, weights=lengths)
+    return 1.0
+
+
+def _parse_block(block: dict) -> tuple[list[float], list[float]]:
+    if "lines" in block:
+        scores = []
+        lengths = []
+        for line in block.get("lines", []):
+            for span in line["spans"]:
+                score = span.get("score")
+                if score is not None:
+                    scores.append(score)
+                    lengths.append(len(span["content"]))
+        return scores, lengths
+    if "blocs" in block:
+        scores, lengths = (_parse_block(b) for b in block["blocs"])
+        scores = sum(*scores, start=[])
+        lengths = sum(*lengths, start=[])
+        return scores, lengths
+    raise NotImplementedError(f"unsupported block: {block}")

extract_python-0.8.0/extract_python/utils.py

@@ -0,0 +1,299 @@
+import gc
+import itertools
+import logging
+import os
+import shutil
+import uuid
+from collections import defaultdict, deque
+from collections.abc import Callable, Generator, Iterable, Iterator
+from contextlib import contextmanager
+from copy import copy
+from dataclasses import dataclass
+from functools import wraps
+from itertools import tee
+from pathlib import Path, PurePath
+from tempfile import TemporaryDirectory
+from types import TracebackType
+from typing import BinaryIO, Protocol, Self
+
+from extract_core import Error, InputDoc, Pages, Result, Status
+from pympler import asizeof
+
+logger = logging.getLogger(__name__)
+
+
+def map_and_preserve[I, R](
+    fn: Callable[[Iterable[I]], Iterator[R]], inputs: Iterable[I]
+) -> tuple[Iterable[I], Iterator[R]]:
+    save_inputs, function_inputs = tee(inputs)
+    outputs = iter(fn(function_inputs))
+    return save_inputs, outputs
+
+
+def path_to_artifacts_dirname(path: PurePath, sep: str = "_") -> str:
+    dirname = f"{path.name[: -len(path.suffix)]}"
+    ext = path.suffix
+    if ext:
+        dirname += sep + ext[1:]
+    return dirname
+
+
+class DocProcessingFn(Protocol):
+    def __call__(self, doc: InputDoc, *arg, **kwargs) -> Result: ...
+
+
+def report_recoverable_errors(
+    recoverable_errors: tuple[type[Exception], ...] = tuple(),
+) -> Callable[[DocProcessingFn], DocProcessingFn]:
+    def make_decorator(f: DocProcessingFn) -> DocProcessingFn:
+        @wraps(f)
+        def wrapped(doc: InputDoc, *args, **kwargs) -> Result:
+            try:
+                return f(doc, *args, **kwargs)
+            except recoverable_errors as e:
+                error = Error.from_exception(e)
+                return Result(
+                    input=doc, status=Status.FAILURE, errors=[error], output=None
+                )
+
+        return wrapped
+
+    return make_decorator
+
+
+@contextmanager
+def chdir(path: Path) -> Generator[None]:
+    cwd = Path.cwd()
+    try:
+        os.chdir(path)
+        yield
+    finally:
+        os.chdir(cwd)
+
+
+@contextmanager
+def reset_env() -> Generator[None]:
+    old_env = copy(dict(os.environ))
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(old_env)
+
+
+def write_pages(pages: Iterable[str], page_sep: str, out: BinaryIO) -> Pages:
+    pages_byte_sizes = []
+    pages = iter(pages)
+    content = None
+    for p in pages:
+        if content:
+            pages_byte_sizes.append(out.write((content + page_sep).encode()))
+        content = p
+    if content:
+        pages_byte_sizes.append(out.write(content.encode()))
+    return Pages.from_pages_bytes_sizes(pages_byte_sizes)
+
+
+Range = tuple[int, int]
+
+
+@dataclass(frozen=True)
+class ProcessedPages:
+    doc: InputDoc
+    doc_idx: int
+    page_range: Range | None = None
+
+    @property
+    def page_length(self) -> int:
+        if self.page_range is None:
+            return self.doc.n_pages
+        return self.page_range[1] - self.page_range[0]
+
+
+class ResultBuffer[R]:
+    def __init__(
+        self,
+        max_size_bytes: int,
+        save_fn: Callable[[R, Path], None],
+        *,
+        load_fn: Callable[[Path], R],
+        root: Path | None = None,
+    ):
+        self._max_bytes = max_size_bytes
+        self._save_fn = save_fn
+        self._load_fn = load_fn
+        self._tmp_dir = None
+        if root is None:
+            self._tmp_dir = TemporaryDirectory()
+            root = Path(self._tmp_dir.name)
+        self._root = root
+        self.__fs_buffer_path = None
+        self._mem_buffer: dict[int, list[R | Path]] = defaultdict(list)
+        self._missing_pages: dict[int, int] = dict()
+        self._current_size: int = 0
+
+    def __enter__(self) -> Self:
+        if self._tmp_dir is not None:
+            self._tmp_dir.__enter__()
+        self.__fs_buffer_path = self._root / uuid.uuid4().hex
+        self.__fs_buffer_path.mkdir()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        if self._mem_buffer or self._missing_pages:
+            logger.warning("closing an non empty buffer")
+        if self._tmp_dir is not None:
+            self._tmp_dir.__exit__(exc_type, exc_val, exc_tb)
+        if self._fs_buffer_path.exists():
+            shutil.rmtree(self._fs_buffer_path)
+        self._mem_buffer = dict()
+        self._missing_pages = dict()
+
+    @property
+    def _fs_buffer_path(self) -> Path:
+        if not self.__fs_buffer_path:
+            msg = (
+                f"inconsistent state, {ResultBuffer.__class__.__name__} is a context"
+                f" manager, call __enter__ before using it"
+            )
+            raise ValueError(msg)
+        return self.__fs_buffer_path
+
+    def add(self, processed: ProcessedPages, result: R) -> None:
+        size = asizeof.asizeof(result)
+        if self._current_size + size > self._max_bytes:
+            path = self._page_path(processed.doc_idx)
+            self._save_fn(result, path)
+            result = path
+        else:
+            self._current_size += size
+        self._mem_buffer[processed.doc_idx].append(result)
+        if processed.doc_idx not in self._missing_pages:
+            self._missing_pages[processed.doc_idx] = processed.doc.n_pages
+        self._missing_pages[processed.doc_idx] -= processed.page_length
+
+    def is_complete(self, doc: int) -> bool:
+        return self._missing_pages[doc] == 0
+
+    def pop_complete(self, doc: int) -> list[R]:
+        if not self.is_complete(doc):
+            raise ValueError(f"{doc} is incomplete")
+        pages = self._mem_buffer.pop(doc)
+        self._missing_pages.pop(doc)
+        for i, page in enumerate(pages):
+            if isinstance(page, Path):
+                page = self._load_fn(page)  # noqa: PLW2901
+            else:
+                self._current_size -= asizeof.asizeof(page)
+            pages[i] = page
+        return pages
+
+    def _page_path(self, doc_id: int) -> Path:
+        pages = self._mem_buffer[doc_id]
+        return self._fs_buffer_path / f"doc-{doc_id}-pages-{len(pages)}"
+
+    def __len__(self) -> int:
+        return len(self._mem_buffer)
+
+
+# The converter process page_batch_size in parallel (GPU sees page_batch_size batches).
+#
+# The batching tradeoff is: avoid calling convert_all to many times vs. releasing the
+# GIL often enough.
+#
+# Calling convert_all to many times on the same doc results in overhead. Each time we
+# call the function, we create a doc processing backend + reload the doc.
+#
+# On the other hand we have to use a reasonable max_page_batches otherwise we process
+# all the stream in a single call and take the risk to lock the GIL for too long. Some
+# docling ops are sadly not async (numpy or torch inference are, but document loading
+# and conversion aren't, so the asyncio.to_thread is not helping)
+def batch_per_pages(
+    docs: Iterable[InputDoc],
+    page_batch_size: int,
+    *,
+    max_page_batches: int,
+    chunk_size: int = 1000,
+) -> Iterable[tuple[ProcessedPages]]:
+    # convert_all only accept to process docs on the exact same page_range
+    #
+    # We collect by chunk to avoid collecting too many inputs, input docs are
+    # lightweight anyway so memory impact should stay limited
+    #
+    # Additionally, results can be output unordered and partial results are buffered
+    # it's OK to process doc pages unordered.
+    # TODO: if it's not OK to sort because inputs is l
+    max_pages = page_batch_size * max_page_batches
+    docs = itertools.batched(docs, chunk_size, strict=False)
+    for chunk in docs:
+        short_docs = [d for d in chunk if d.n_pages <= max_pages]
+        long_docs = [d for d in chunk if d.n_pages > max_pages]
+        del chunk
+        gc.collect()
+        # Bin fill for docs smaller than max_pages
+        offset = yield from _bin_fill(short_docs, max_pages=max_pages)
+        # otherwise we just yield chunks of max_pages except the last chunk which is
+        # grouped by page_range
+        yield from _by_page_ranges(long_docs, max_pages=max_pages, offset=offset)
+
+
+def _bin_fill(
+    docs: Iterable[InputDoc], max_pages: int, offset: int = 0
+) -> Generator[tuple[ProcessedPages], None, int]:
+    bins = defaultdict(deque)
+    doc_idx = offset
+    for doc in docs:
+        if doc.n_pages > max_pages:
+            msg = f"expected docs to have <= {max_pages} pages"
+            raise ValueError(msg)
+        pages = ProcessedPages(doc=doc, doc_idx=doc_idx)
+        doc_idx += 1
+        available_space = (s for s in sorted(bins.keys()) if doc.n_pages <= s)
+        available_space = next(available_space, max_pages)
+        selected = bins[available_space]
+        selected = selected.pop() if selected else []
+        selected.append(pages)
+        available_space -= doc.n_pages
+        if available_space == 0:
+            yield tuple(selected)
+            continue
+        bins[available_space].append(selected)
+    for range_bins in bins.values():
+        for b in range_bins:
+            yield tuple(b)
+    return doc_idx
+
+
+def _by_page_ranges(
+    docs: Iterable[InputDoc], max_pages: int, offset: int = 0
+) -> Generator[tuple[ProcessedPages], None, int]:
+    by_range = defaultdict(list)
+    doc_idx = offset
+    for doc in docs:
+        if doc.n_pages < max_pages:
+            msg = f"expected docs to have >= {max_pages} pages"
+            raise ValueError(msg)
+
+        for i in range(0, doc.n_pages, max_pages):
+            start = i
+            end = min(start + max_pages, doc.n_pages)
+            rng = (start, end)
+            rng_size = end - start
+            alone_in_batch = rng_size == max_pages
+            pages = ProcessedPages(doc=doc, doc_idx=doc_idx, page_range=rng)
+            if alone_in_batch:
+                yield (pages,)
+                continue
+            by_range[rng].append(pages)
+            is_complete = len(by_range[rng]) == (max_pages // rng_size)
+            if is_complete:
+                yield tuple(by_range.pop(rng))
+        doc_idx += 1
+    for v in by_range.values():
+        yield tuple(v)
+    return doc_idx