extract-python 0.8.4__py3-none-any.whl → 0.9.0__py3-none-any.whl

extract_python/miner_u.py

@@ -1,15 +1,14 @@
-import json
 import os
-import shutil
 from collections.abc import AsyncIterable, Callable, Iterable
-from functools import partial
+from functools import cache, partial
 from pathlib import Path
-from tempfile import TemporaryDirectory
 
+from docvortex.export import materialize_middle, validate_materialized_assets
+from docvortex.render import RenderMode, render_markdown
+from docvortex.schema import DocumentMetadata, PageInfo, Producer
 from extract_core import (
     ConversionOutput,
     InputDoc,
-    MinerUBackend,
     MinerUPipelineConfig,
     OutputFormat,
     Pipeline,
@@ -17,6 +16,7 @@ from extract_core import (
     Result,
     Status,
 )
+from mineru import MiddleJson, ParseResult
 
 from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP
 from .utils import path_to_artifacts_dirname, reset_env, write_pages
@@ -30,185 +30,85 @@ class MinerUPipeline(Pipeline):
     def __init__(self, config: MinerUPipelineConfig):
         super().__init__(config)
         self._language = self._config.language
-        self._md_make_fn = _parse_md_make_fn(self._config.config.backend)
 
     async def extract_content(
         self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
     ) -> AsyncIterable[Result]:
-        from mineru.cli.common import aio_do_parse  # noqa: PLC0415
+        from mineru import MinerUParser  # noqa: PLC0415
 
         with reset_env():
             os.environ["MINERU_DEVICE_MODE"] = self._device
-            docs = list(docs)
             # TODO: exclude files which are not pdf and return an error
-            pdfs_bytes = [d.path.read_bytes() for d in docs]
-            pdfs_names = [d.path.name for d in docs]
-            p_lang_list = [self._language for _ in pdfs_names]
             # TODO: we should only process valid PDFs
-            with TemporaryDirectory(prefix="mineru-") as workdir:
-                workdir = Path(workdir)  # noqa: PLW2901
-                await aio_do_parse(
-                    output_dir=workdir,
-                    pdf_file_names=pdfs_names,
-                    pdf_bytes_list=pdfs_bytes,
-                    p_lang_list=p_lang_list,
-                    **self._config.config.as_parse_kwargs(),
-                )
-                res_paths = [
-                    _revert_mineru_output(workdir, pdf_filename=p) for p in pdfs_names
-                ]
-                for doc, res_path in zip(docs, res_paths, strict=True):
-                    yield _process_doc(
-                        doc,
-                        md_make_fn=self._md_make_fn,
-                        res_path=res_path,
-                        output_format=output_format,
-                        output_path=output_path,
-                    )
-
-
-def _revert_mineru_output(output_dir: Path, *, pdf_filename: str) -> Path:
-    output_path = output_dir / pdf_filename
-    if not output_path.exists():
-        msg = f"couldn't find result for {pdf_filename}"
-        raise FileNotFoundError(msg)
-    dirs = [p for p in output_path.iterdir() if p.is_dir()]
-    if len(dirs) != 1:
-        msg = f"expected exactly one result directory, found: {dirs}"
-        raise ValueError(msg)
-    return output_dir / dirs[0]
-
-
-def _parse_md_make_fn(backend: MinerUBackend) -> MDMakeFunction:
-
-    match backend:
-        case MinerUBackend.PIPELINE:
-            from mineru.backend.pipeline.pipeline_middle_json_mkcontent import (  # noqa: PLC0415
-                union_make,
+            config = self._config.config
+            parser = MinerUParser(
+                tier=config.tier,
+                parse_mode=config.parse_mode,
+                image_analysis=config.image_analysis,
+                vlm_config=config.vlm_config,
             )
-
-            return union_make
-        case MinerUBackend.VLM:
-            from mineru.backend.vlm.vlm_middle_json_mkcontent import (  # noqa: PLC0415
-                union_make,
-            )
-
-            return union_make
-        case _:
-            raise ValueError(f"Unsupported backend: {backend}")
+            for doc in docs:
+                res = await parser.parse_async(doc.path)
+                yield _process_doc(
+                    doc, res, output_format=output_format, output_path=output_path
+                )
 
 
 def _process_doc(
     doc: InputDoc,
+    result: ParseResult,
     *,
-    md_make_fn: MDMakeFunction,
-    res_path: Path,
     output_format: OutputFormat,
     output_path: Path,
 ) -> Result:
-    md_dir_name = path_to_artifacts_dirname(doc.path)
-    md_dir = Path(output_path) / md_dir_name
-    md_dir.mkdir(parents=True, exist_ok=False)
-    artifacts_dir = md_dir / ARTIFACTS
-    md_path = (md_dir / md_dir_name).with_suffix(OutputFormat.MARKDOWN.value)
+    output_path_dir_name = path_to_artifacts_dirname(doc.path)
+    output_dir = Path(output_path) / output_path_dir_name
+    output_dir.mkdir(parents=True)
     # Fail early
     match output_format:
         case OutputFormat.MARKDOWN:
-            im_rel_dir = artifacts_dir.relative_to(md_dir)
             dump_content_fn = partial(
-                _dump_md_content,
-                md_make_fn=md_make_fn,
-                output_path=output_path,
-                md_path=md_path,
-                im_dir=im_rel_dir,
+                _dump_md_content, output_path=output_path, md_path=output_dir
             )
         case _:
             raise NotImplementedError(f"unsupported output format {output_format}")
-    middle_json_path = res_path / f"{doc.path.name}_middle.json"
-    middle_json = json.loads(middle_json_path.read_text())
-    shutil.move(res_path / "images", artifacts_dir)
-    output = dump_content_fn(middle_json)
+    output = dump_content_fn(result.middle_json)
     return Result(input=doc, status=Status.SUCCESS, output=output)
 
 
+@cache
+def _mineru_md_page_sep() -> str:
+    dummy_json = MiddleJson(
+        pages=[PageInfo(page_idx=0), PageInfo(page_idx=1)],
+        is_full_document=True,
+        metadata=DocumentMetadata(
+            file_suffix="pdf", producer=Producer(name="dummy", version="1.0")
+        ),
+    )
+    md = render_markdown(dummy_json, mode=RenderMode.FULL)
+    return md.replace(" ", "")
+
+
 def _dump_md_content(
-    middle_json: dict,
+    middle_json: MiddleJson,
     *,
-    md_make_fn: MDMakeFunction,
     page_sep: str = DEFAULT_MD_PAGE_SEP,
     output_path: Path,
     md_path: Path,
-    im_dir: Path,
-    md_make_mode: str | None = None,
 ) -> ConversionOutput:
-    from mineru.utils.enum_class import MakeMode  # noqa: PLC0415
-
-    pdf_info = middle_json["pdf_info"]
-    if md_make_mode is None:
-        md_make_mode = MakeMode.MM_MD
-    pages = (md_make_fn([p], md_make_mode, str(im_dir)) for p in pdf_info)
-    with md_path.open("wb") as f:
+    middle_json, assets = materialize_middle(middle_json)
+    exported = ParseResult(middle_json=middle_json)
+    validate_materialized_assets(middle_json, assets)
+    md = exported.markdown(mode=RenderMode.FULL)
+    for path, im_bytes in assets.items():
+        im_path = md_path / ARTIFACTS / path
+        im_path.parent.mkdir(parents=True, exist_ok=True)
+        im_path.write_bytes(im_bytes)
+        md = md.replace(path, str(im_path.relative_to(md_path)))
+    pages = md.split(_mineru_md_page_sep())
+    md_content_path = (md_path / md_path.name).with_suffix(OutputFormat.MARKDOWN)
+    with open(md_content_path, "wb") as f:
         pages = write_pages(pages, page_sep, f)
-    output_path = md_path.parent.relative_to(output_path)
-    confidence = _mineru_confidence(pdf_info)
-    output = ConversionOutput(path=output_path, pages=pages, confidence=confidence)
+    path = md_path.relative_to(output_path)
+    output = ConversionOutput(path=path, pages=pages, confidence=None)
     return output
-
-
-def _mineru_confidence(pdf_info: list[dict]) -> float:
-    if not pdf_info:
-        return 1.0
-    block_conf = _mineru_block_confidence(pdf_info)
-    line_config = _mineru_line_confidence(pdf_info)
-    return (block_conf + line_config) / 2.0
-
-
-def _mineru_block_confidence(pdf_info: list[dict]) -> float:
-    import numpy as np  # noqa: PLC0415
-
-    scores = []
-    for info in pdf_info:
-        for block in info["para_blocks"]:
-            score = block.get("score")
-            if score is not None:
-                scores.append(score)
-    if scores:
-        return np.average(scores)
-    return 1.0
-
-
-def _mineru_line_confidence(pdf_info: list[dict]) -> float:
-    import numpy as np  # noqa: PLC0415
-
-    scores = []
-    lengths = []
-    for info in pdf_info:
-        for block in info["para_blocks"]:
-            for line in block.get("lines", []):
-                for span in line["spans"]:
-                    score = span.get("score")
-                    if score is not None:
-                        scores.append(score)
-                        lengths.append(len(span["content"]))
-    if scores:
-        return np.average(scores, weights=lengths)
-    return 1.0
-
-
-def _parse_block(block: dict) -> tuple[list[float], list[float]]:
-    if "lines" in block:
-        scores = []
-        lengths = []
-        for line in block.get("lines", []):
-            for span in line["spans"]:
-                score = span.get("score")
-                if score is not None:
-                    scores.append(score)
-                    lengths.append(len(span["content"]))
-        return scores, lengths
-    if "blocs" in block:
-        scores, lengths = (_parse_block(b) for b in block["blocs"])
-        scores = sum(*scores, start=[])
-        lengths = sum(*lengths, start=[])
-        return scores, lengths
-    raise NotImplementedError(f"unsupported block: {block}")

{extract_python-0.8.4.dist-info → extract_python-0.9.0.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.5
 Name: extract-python
-Version: 0.8.4
+Version: 0.9.0
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python
 Project-URL: Issues, https://github.com/ICIJ/extract-python/issues
 Author-email: Clément Doumouro <cdoumouro@icij.org>
 Requires-Python: <3.15,>=3.13
-Requires-Dist: extract-core~=0.8.0
+Requires-Dist: extract-core~=0.9.0
 Requires-Dist: icij-common~=0.8.2
 Requires-Dist: pympler~=1.1
 Provides-Extra: benches
@@ -20,8 +20,7 @@ Requires-Dist: docling-slim[feat-ocr-easyocr,feat-ocr-mac,feat-ocr-tesserocr,sta
 Provides-Extra: marker
 Requires-Dist: marker-pdf~=1.10; extra == 'marker'
 Provides-Extra: mineru
-Requires-Dist: mineru[mlx]~=3.2; (sys_platform == 'darwin') and extra == 'mineru'
-Requires-Dist: mineru[pipeline,vlm]~=3.2; extra == 'mineru'
+Requires-Dist: mineru[torch,vlm]~=4.0; extra == 'mineru'
 Requires-Dist: pydantic-extra-types[pycountry]~=2.11; extra == 'mineru'
 Requires-Dist: python-pptx~=1.0; extra == 'mineru'
 Requires-Dist: six~=1.17; extra == 'mineru'

{extract_python-0.8.4.dist-info → extract_python-0.9.0.dist-info}/RECORD

@@ -2,8 +2,8 @@ extract_python/__init__.py,sha256=DA2LUro6vMjfS8fb2MsqO95FbJEZHyZ7kFyn42q02Wk,75
 extract_python/constants.py,sha256=659V40LcTWJhX3IbuJLSSvI5AsGJh9ciMrGCfzJn2zA,98
 extract_python/docling_.py,sha256=H_A-6pMObQ0Q7XA62GCBv0nbGtW5rKjfOVW3giU4NCk,10146
 extract_python/marker_.py,sha256=bb0DJXrSWOznIuPnw4-adR5_R9QiwtFGGBh5KlXFxjM,3323
-extract_python/miner_u.py,sha256=OCIleNu-VvaJlx_zG0VuagJvQvElXV7e-aPXqHbp31A,7302
+extract_python/miner_u.py,sha256=sEU1PhPpfPVo4jP7flNlbGO9PfsLFLz973-ufZEd0ws,3966
 extract_python/utils.py,sha256=LrR1tlOgpWEri6TLDlYYPRBmlka-AjOepWnbySf2Haw,9974
-extract_python-0.8.4.dist-info/METADATA,sha256=iMMjpvJFarhTkbrco3oF1lNZSa2jhDLfFvAionIRJqU,1290
-extract_python-0.8.4.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-extract_python-0.8.4.dist-info/RECORD,,
+extract_python-0.9.0.dist-info/METADATA,sha256=ChieXoOAjl4y-ycPKQCKPvlVqx7ukeogABYdChcCuSs,1205
+extract_python-0.9.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+extract_python-0.9.0.dist-info/RECORD,,

{extract_python-0.8.4.dist-info → extract_python-0.9.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.32.0
+Generator: hatchling 1.32.4
 Root-Is-Purelib: true
 Tag: py3-none-any