dingo-python 2.3.0__py3-none-any.whl → 2.4.0__py3-none-any.whl

dingo/config/input_args.py

@@ -53,6 +53,10 @@ class DatasetFieldArgs(BaseModel):
     image: str = ''
 
 
+class DatasetMinerUArgs(BaseModel):
+    include_types: Optional[List[str]] = None  # 只保留指定的 block 类型，None 表示全部保留
+
+
 class DatasetArgs(BaseModel):
     source: str = 'hugging_face'
     format: str = 'json'
@@ -64,6 +68,7 @@ class DatasetArgs(BaseModel):
     excel_config: DatasetExcelArgs = DatasetExcelArgs()
     csv_config: DatasetCsvArgs = DatasetCsvArgs()
     parquet_config: DatasetParquetArgs = DatasetParquetArgs()
+    mineru_config: DatasetMinerUArgs = DatasetMinerUArgs()
 
 
 class ExecutorResultSaveArgs(BaseModel):

dingo/data/converter/__init__.py

@@ -1,3 +1,4 @@
+import dingo.data.converter.mineru  # noqa: F401  — registers mineru / mineru_v2
 from dingo.data.converter.base import BaseConverter
 
 converters = BaseConverter.converters

dingo/data/converter/mineru.py

@@ -0,0 +1,245 @@
+"""MinerU output format converters.
+
+Supports two MinerU structured output files:
+  - ``content_list.json`` (format: ``"mineru"``) — flat array of blocks
+  - ``content_list_v2.json`` (format: ``"mineru_v2"``) — pages × blocks
+"""
+
+import json
+from typing import Callable, Dict, List, Union
+
+from dingo.config import InputArgs
+from dingo.data.converter.base import BaseConverter
+from dingo.io import Data
+
+
+def _flatten_spans(spans: List[Dict]) -> str:
+    """Concatenate a V2 span list into plain text.
+
+    Spans look like ``[{"type": "text", "content": "..."}, ...]``.
+    Hyperlink spans may carry ``children`` with nested text spans;
+    when present we use the top-level ``content`` which is already the
+    concatenated text.
+    """
+    parts = []
+    for span in spans:
+        parts.append(span.get("content", ""))
+    return "".join(parts)
+
+
+def _wrap_image(img_path) -> List[str]:
+    """Ensure img_path is a list of strings (Dingo convention)."""
+    if not img_path:
+        return []
+    if isinstance(img_path, list):
+        return img_path
+    return [img_path]
+
+
+# ---------------------------------------------------------------------------
+# V1: content_list.json
+# ---------------------------------------------------------------------------
+
+_V1_TEXT_TYPES = frozenset({
+    "text", "equation", "header", "footer",
+    "page_number", "aside_text", "page_footnote",
+})
+
+
+def _map_block_v1(block: dict, block_idx: int) -> dict:
+    """Map a single content_list.json block to a Data-compatible dict."""
+    btype = block.get("type", "")
+    page_idx = block.get("page_idx", 0)
+
+    data_dict = dict(block)
+    data_dict["data_id"] = f"p{page_idx}-b{block_idx}"
+
+    if btype in _V1_TEXT_TYPES:
+        data_dict["content"] = block.get("text", "")
+
+    elif btype == "image":
+        data_dict["content"] = ""
+        data_dict["image"] = _wrap_image(block.get("img_path"))
+
+    elif btype == "table":
+        data_dict["content"] = block.get("table_body", "")
+        data_dict["image"] = _wrap_image(block.get("img_path"))
+
+    elif btype == "chart":
+        data_dict["content"] = block.get("content", "")
+        data_dict["image"] = _wrap_image(block.get("img_path"))
+
+    elif btype == "code":
+        data_dict["content"] = block.get("code_body", "")
+
+    elif btype == "list":
+        items = block.get("list_items", [])
+        data_dict["content"] = "\n".join(items) if items else ""
+
+    else:
+        data_dict.setdefault("content", block.get("text", ""))
+
+    if "img_path" in data_dict and "image" not in data_dict:
+        img = _wrap_image(data_dict["img_path"])
+        if img:
+            data_dict["image"] = img
+
+    return data_dict
+
+
+@BaseConverter.register("mineru")
+class MinerUConverter(BaseConverter):
+    """Converter for MinerU ``content_list.json`` (flat block array)."""
+
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    def convertor(cls, input_args: InputArgs) -> Callable:
+        include = None
+        if hasattr(input_args.dataset, "mineru_config"):
+            cfg = input_args.dataset.mineru_config
+            if cfg.include_types:
+                include = frozenset(cfg.include_types)
+
+        def _convert(raw: Union[str, list]):
+            blocks = raw
+            if isinstance(raw, str):
+                blocks = json.loads(raw)
+            for block_idx, block in enumerate(blocks):
+                if include and block.get("type", "") not in include:
+                    continue
+                data_dict = _map_block_v1(block, block_idx)
+                yield Data(**data_dict)
+
+        return _convert
+
+
+# ---------------------------------------------------------------------------
+# V2: content_list_v2.json
+# ---------------------------------------------------------------------------
+
+def _map_block_v2(block: dict, page_idx: int, block_idx: int) -> dict:
+    """Map a single content_list_v2.json block to a Data-compatible dict."""
+    btype = block.get("type", "")
+    inner = block.get("content")
+
+    if not isinstance(inner, dict):
+        data_dict = {
+            "data_id": f"p{page_idx}-b{block_idx}",
+            "type": btype,
+            "page_idx": page_idx,
+            "content": inner if isinstance(inner, str) else "",
+            "raw_content": inner,
+        }
+        if "bbox" in block:
+            data_dict["bbox"] = block["bbox"]
+        return data_dict
+
+    data_dict = {
+        "data_id": f"p{page_idx}-b{block_idx}",
+        "type": btype,
+        "page_idx": page_idx,
+    }
+
+    if "bbox" in block:
+        data_dict["bbox"] = block["bbox"]
+    if "anchor" in block:
+        data_dict["anchor"] = block["anchor"]
+    if "sub_type" in block:
+        data_dict["sub_type"] = block["sub_type"]
+
+    if btype == "title":
+        spans = inner.get("title_content", [])
+        data_dict["content"] = _flatten_spans(spans)
+        data_dict["text_level"] = inner.get("level", 0)
+
+    elif btype == "paragraph":
+        spans = inner.get("paragraph_content", [])
+        data_dict["content"] = _flatten_spans(spans)
+
+    elif btype == "equation_interline":
+        data_dict["content"] = inner.get("math_content", "")
+        if "math_type" in inner:
+            data_dict["math_type"] = inner["math_type"]
+
+    elif btype == "image":
+        data_dict["content"] = ""
+        data_dict["image"] = _wrap_image(inner.get("img_path"))
+        for key in ("image_caption", "image_footnote"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype == "table":
+        data_dict["content"] = inner.get("table_body", "")
+        data_dict["image"] = _wrap_image(inner.get("img_path"))
+        for key in ("table_caption", "table_footnote"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype == "chart":
+        data_dict["content"] = inner.get("content", "")
+        data_dict["image"] = _wrap_image(inner.get("img_path"))
+        for key in ("chart_caption", "chart_footnote"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype == "code":
+        data_dict["content"] = inner.get("code_content", "")
+        for key in ("code_caption", "code_footnote", "code_language"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype == "algorithm":
+        data_dict["content"] = inner.get("algorithm_content", "")
+        for key in ("algorithm_caption", "algorithm_footnote"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype in ("list", "index"):
+        items = inner.get("list_items", [])
+        data_dict["content"] = "\n".join(items) if items else ""
+        data_dict["list_items"] = items
+
+    else:
+        content_key = f"{btype}_content"
+        spans = inner.get(content_key, [])
+        if isinstance(spans, list) and spans and isinstance(spans[0], dict):
+            data_dict["content"] = _flatten_spans(spans)
+        elif isinstance(inner, str):
+            data_dict["content"] = inner
+        else:
+            data_dict["content"] = ""
+
+    data_dict["raw_content"] = inner
+
+    return data_dict
+
+
+@BaseConverter.register("mineru_v2")
+class MinerUV2Converter(BaseConverter):
+    """Converter for MinerU ``content_list_v2.json`` (pages x blocks)."""
+
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    def convertor(cls, input_args: InputArgs) -> Callable:
+        include = None
+        if hasattr(input_args.dataset, "mineru_config"):
+            cfg = input_args.dataset.mineru_config
+            if cfg.include_types:
+                include = frozenset(cfg.include_types)
+
+        def _convert(raw: Union[str, list]):
+            pages = raw
+            if isinstance(raw, str):
+                pages = json.loads(raw)
+            for page_idx, page_blocks in enumerate(pages):
+                for block_idx, block in enumerate(page_blocks):
+                    if include and block.get("type", "") not in include:
+                        continue
+                    data_dict = _map_block_v2(block, page_idx, block_idx)
+                    yield Data(**data_dict)
+
+        return _convert

dingo/data/datasource/local.py

@@ -395,7 +395,7 @@ class LocalDataSource(DataSource):
         elif os.path.exists(self.path) and os.path.isdir(self.path):
             self._find_all_files(self.path, f_list)
 
-        by_line = self.input_args.dataset.format not in ["json", "listjson"]
+        by_line = self.input_args.dataset.format not in ["json", "listjson", "mineru", "mineru_v2"]
 
         for f in f_list:
             # Check if file is CSV

{dingo_python-2.3.0.dist-info → dingo_python-2.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dingo-python
-Version: 2.3.0
+Version: 2.4.0
 Summary: A Comprehensive AI Data Quality Evaluation Tool for Large Models
 Home-page: https://github.com/MigoXLab/dingo
 Author: Dingo

{dingo_python-2.3.0.dist-info → dingo_python-2.4.0.dist-info}/RECORD

@@ -1,10 +1,11 @@
 dingo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dingo/config/__init__.py,sha256=SaeOmGEUG0Hp5lqHxnHUTE_9ysN5KzA_Icilb9xY2mQ,349
-dingo/config/input_args.py,sha256=ngOJcAyq_LdLqNszMTljfOUp_utXN6FX-nBsD_ADyeg,4388
+dingo/config/input_args.py,sha256=lu4ZcG9me_9PgjXScnB9C17mBosVj7ArTU-JNuE4qgc,4592
 dingo/data/__init__.py,sha256=reCw4XQoInUTtvRW6c1wY_LH1EWJ7XpZDQcBCW61Lf8,214
-dingo/data/converter/__init__.py,sha256=1MiG4H8Sg2sYHQmYdg0F9_1okP_YoMNHyQorPEAf6zw,91
+dingo/data/converter/__init__.py,sha256=nZOTVKQ5B4_VmMhxNM2289KisdWXeom-IcciuAv9iqI,174
 dingo/data/converter/base.py,sha256=_WXa_plKj83iFgQyHABchGbX-dv3d17QuODua-bd83w,12820
 dingo/data/converter/img_utils.py,sha256=ulySpvbmdn-LoeufJw7pVS1_k5XRJMbptzP2qLOIBjU,3527
+dingo/data/converter/mineru.py,sha256=bVB8dKGaiAEIXgms4fM_ykiPVPb5tKMfJf8_hPmh2j8,8136
 dingo/data/dataset/__init__.py,sha256=IWXEBNDBcffxOnteKLJDOuWTyx2rgk0MN1kHv5eirFY,574
 dingo/data/dataset/base.py,sha256=orRyPrgYqRgxO6WiFtCE2hX0yZXloMKfydkosFJzdGU,5437
 dingo/data/dataset/huggingface.py,sha256=DNor2VudY8ckvNZxNHk3XY0PqKvyyvKg6sYZzUE1GZ0,7155
@@ -15,7 +16,7 @@ dingo/data/dataset/sql.py,sha256=hERVYF3Ij7UzQWJmucurBj0OUiKjU5cGtfGF-J1_8AM,259
 dingo/data/datasource/__init__.py,sha256=_cZDMi4xAHGfals0UpXEcNxRDuYOSjJHCV29tQB6dso,386
 dingo/data/datasource/base.py,sha256=pK5h8oxSGnEJtP_vbA4rkzRhaM5GTssQ6mQuF-HVlEY,2936
 dingo/data/datasource/huggingface.py,sha256=wWruojFogbToghatcXnqGCg1inQ1NYyN74yfyDFNZBw,3768
-dingo/data/datasource/local.py,sha256=8IQ9FvW_T_vFSr4k6Pu0l3Am8UVLt94RrBRLsJsuGUs,18787
+dingo/data/datasource/local.py,sha256=O4R6qpS5RGa-M57EdY2nm1jK56Pe-aFogjcaI4LdSyw,18810
 dingo/data/datasource/s3.py,sha256=FKDfrtZfiwby2x1WLUngbBO2ILfb4r-Dz-ho6-ThRSA,2963
 dingo/data/datasource/sql.py,sha256=LQBaPyP7Pigs-Rdm6ODfrbo45TILYY_CFUIPWp8qUk0,3910
 dingo/data/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -154,9 +155,9 @@ dingo/utils/exception.py,sha256=fh58dSLSmYSnwW4MQXg-Jfai2QcZfDruTaYGbaWk7Wc,446
 dingo/utils/image_loader.py,sha256=RqnziJhqDB6zWf1s7S4zKgFqLA9e9CHkf5aU1R4fqnM,4479
 dingo/utils/log_util/__init__.py,sha256=VfzAAHUV8RuN-QaySahfAPfhM__-myigUlKx7ywVerA,717
 dingo/utils/log_util/logger.py,sha256=spGK0w22UgXsCcArd1rpt2teLPy7QPlIuvBaKYioHdY,1414
-dingo_python-2.3.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dingo_python-2.3.0.dist-info/METADATA,sha256=F6glwcTym-X-iH-k5LDOT2bu5lmZdzdDWgwH4_n2Eg4,27555
-dingo_python-2.3.0.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-dingo_python-2.3.0.dist-info/entry_points.txt,sha256=Vo_p8qSVnOENdy1uubqxJRppZIpiQ753JG3WPAUeYps,45
-dingo_python-2.3.0.dist-info/top_level.txt,sha256=gSXQSLowu_WOQRi75wK3qyjbHxeN5PqsaA4ChGmJdek,6
-dingo_python-2.3.0.dist-info/RECORD,,
+dingo_python-2.4.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dingo_python-2.4.0.dist-info/METADATA,sha256=0RovhGosWHE5sqmznPjief9qCa2BN13cKQ1yfNwsmRw,27555
+dingo_python-2.4.0.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+dingo_python-2.4.0.dist-info/entry_points.txt,sha256=Vo_p8qSVnOENdy1uubqxJRppZIpiQ753JG3WPAUeYps,45
+dingo_python-2.4.0.dist-info/top_level.txt,sha256=gSXQSLowu_WOQRi75wK3qyjbHxeN5PqsaA4ChGmJdek,6
+dingo_python-2.4.0.dist-info/RECORD,,