dingo-python 2.2.2__py3-none-any.whl → 2.4.0__py3-none-any.whl

dingo/config/input_args.py

@@ -53,6 +53,10 @@ class DatasetFieldArgs(BaseModel):
     image: str = ''
 
 
+class DatasetMinerUArgs(BaseModel):
+    include_types: Optional[List[str]] = None  # 只保留指定的 block 类型，None 表示全部保留
+
+
 class DatasetArgs(BaseModel):
     source: str = 'hugging_face'
     format: str = 'json'
@@ -64,6 +68,7 @@ class DatasetArgs(BaseModel):
     excel_config: DatasetExcelArgs = DatasetExcelArgs()
     csv_config: DatasetCsvArgs = DatasetCsvArgs()
     parquet_config: DatasetParquetArgs = DatasetParquetArgs()
+    mineru_config: DatasetMinerUArgs = DatasetMinerUArgs()
 
 
 class ExecutorResultSaveArgs(BaseModel):
@@ -87,6 +92,8 @@ class ExecutorArgs(BaseModel):
 
 
 class EvaluatorRuleArgs(BaseModel):
+    model_config = {"extra": "forbid"}
+
     threshold: Optional[float] = None
     pattern: Optional[str] = None
     key_list: Optional[List[str]] = None
@@ -101,6 +108,13 @@ class EmbeddingConfigArgs(BaseModel):
     api_url: Optional[str] = None
 
 
+class CustomLLMMetricArgs(BaseModel):
+    metric: str
+    description: Optional[str] = ""
+    criteria: List[str]
+    input_fields: List[str]
+
+
 class EvaluatorLLMArgs(BaseModel):
     model_config = {"extra": "allow"}
 
@@ -108,10 +122,11 @@ class EvaluatorLLMArgs(BaseModel):
     key: Optional[str] = None
     api_url: Optional[str] = None
     embedding_config: Optional[EmbeddingConfigArgs] = None
+    custom_metric: Optional[CustomLLMMetricArgs] = None
 
 
 class EvalPiplineConfig(BaseModel):
-    """Single evaluator configuration item"""
+    """Single evaluator configuration item."""
     name: str
     config: Optional[EvaluatorRuleArgs | EvaluatorLLMArgs] = None
 

dingo/data/converter/__init__.py

@@ -1,3 +1,4 @@
+import dingo.data.converter.mineru  # noqa: F401  — registers mineru / mineru_v2
 from dingo.data.converter.base import BaseConverter
 
 converters = BaseConverter.converters

dingo/data/converter/mineru.py

@@ -0,0 +1,245 @@
+"""MinerU output format converters.
+
+Supports two MinerU structured output files:
+  - ``content_list.json`` (format: ``"mineru"``) — flat array of blocks
+  - ``content_list_v2.json`` (format: ``"mineru_v2"``) — pages × blocks
+"""
+
+import json
+from typing import Callable, Dict, List, Union
+
+from dingo.config import InputArgs
+from dingo.data.converter.base import BaseConverter
+from dingo.io import Data
+
+
+def _flatten_spans(spans: List[Dict]) -> str:
+    """Concatenate a V2 span list into plain text.
+
+    Spans look like ``[{"type": "text", "content": "..."}, ...]``.
+    Hyperlink spans may carry ``children`` with nested text spans;
+    when present we use the top-level ``content`` which is already the
+    concatenated text.
+    """
+    parts = []
+    for span in spans:
+        parts.append(span.get("content", ""))
+    return "".join(parts)
+
+
+def _wrap_image(img_path) -> List[str]:
+    """Ensure img_path is a list of strings (Dingo convention)."""
+    if not img_path:
+        return []
+    if isinstance(img_path, list):
+        return img_path
+    return [img_path]
+
+
+# ---------------------------------------------------------------------------
+# V1: content_list.json
+# ---------------------------------------------------------------------------
+
+_V1_TEXT_TYPES = frozenset({
+    "text", "equation", "header", "footer",
+    "page_number", "aside_text", "page_footnote",
+})
+
+
+def _map_block_v1(block: dict, block_idx: int) -> dict:
+    """Map a single content_list.json block to a Data-compatible dict."""
+    btype = block.get("type", "")
+    page_idx = block.get("page_idx", 0)
+
+    data_dict = dict(block)
+    data_dict["data_id"] = f"p{page_idx}-b{block_idx}"
+
+    if btype in _V1_TEXT_TYPES:
+        data_dict["content"] = block.get("text", "")
+
+    elif btype == "image":
+        data_dict["content"] = ""
+        data_dict["image"] = _wrap_image(block.get("img_path"))
+
+    elif btype == "table":
+        data_dict["content"] = block.get("table_body", "")
+        data_dict["image"] = _wrap_image(block.get("img_path"))
+
+    elif btype == "chart":
+        data_dict["content"] = block.get("content", "")
+        data_dict["image"] = _wrap_image(block.get("img_path"))
+
+    elif btype == "code":
+        data_dict["content"] = block.get("code_body", "")
+
+    elif btype == "list":
+        items = block.get("list_items", [])
+        data_dict["content"] = "\n".join(items) if items else ""
+
+    else:
+        data_dict.setdefault("content", block.get("text", ""))
+
+    if "img_path" in data_dict and "image" not in data_dict:
+        img = _wrap_image(data_dict["img_path"])
+        if img:
+            data_dict["image"] = img
+
+    return data_dict
+
+
+@BaseConverter.register("mineru")
+class MinerUConverter(BaseConverter):
+    """Converter for MinerU ``content_list.json`` (flat block array)."""
+
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    def convertor(cls, input_args: InputArgs) -> Callable:
+        include = None
+        if hasattr(input_args.dataset, "mineru_config"):
+            cfg = input_args.dataset.mineru_config
+            if cfg.include_types:
+                include = frozenset(cfg.include_types)
+
+        def _convert(raw: Union[str, list]):
+            blocks = raw
+            if isinstance(raw, str):
+                blocks = json.loads(raw)
+            for block_idx, block in enumerate(blocks):
+                if include and block.get("type", "") not in include:
+                    continue
+                data_dict = _map_block_v1(block, block_idx)
+                yield Data(**data_dict)
+
+        return _convert
+
+
+# ---------------------------------------------------------------------------
+# V2: content_list_v2.json
+# ---------------------------------------------------------------------------
+
+def _map_block_v2(block: dict, page_idx: int, block_idx: int) -> dict:
+    """Map a single content_list_v2.json block to a Data-compatible dict."""
+    btype = block.get("type", "")
+    inner = block.get("content")
+
+    if not isinstance(inner, dict):
+        data_dict = {
+            "data_id": f"p{page_idx}-b{block_idx}",
+            "type": btype,
+            "page_idx": page_idx,
+            "content": inner if isinstance(inner, str) else "",
+            "raw_content": inner,
+        }
+        if "bbox" in block:
+            data_dict["bbox"] = block["bbox"]
+        return data_dict
+
+    data_dict = {
+        "data_id": f"p{page_idx}-b{block_idx}",
+        "type": btype,
+        "page_idx": page_idx,
+    }
+
+    if "bbox" in block:
+        data_dict["bbox"] = block["bbox"]
+    if "anchor" in block:
+        data_dict["anchor"] = block["anchor"]
+    if "sub_type" in block:
+        data_dict["sub_type"] = block["sub_type"]
+
+    if btype == "title":
+        spans = inner.get("title_content", [])
+        data_dict["content"] = _flatten_spans(spans)
+        data_dict["text_level"] = inner.get("level", 0)
+
+    elif btype == "paragraph":
+        spans = inner.get("paragraph_content", [])
+        data_dict["content"] = _flatten_spans(spans)
+
+    elif btype == "equation_interline":
+        data_dict["content"] = inner.get("math_content", "")
+        if "math_type" in inner:
+            data_dict["math_type"] = inner["math_type"]
+
+    elif btype == "image":
+        data_dict["content"] = ""
+        data_dict["image"] = _wrap_image(inner.get("img_path"))
+        for key in ("image_caption", "image_footnote"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype == "table":
+        data_dict["content"] = inner.get("table_body", "")
+        data_dict["image"] = _wrap_image(inner.get("img_path"))
+        for key in ("table_caption", "table_footnote"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype == "chart":
+        data_dict["content"] = inner.get("content", "")
+        data_dict["image"] = _wrap_image(inner.get("img_path"))
+        for key in ("chart_caption", "chart_footnote"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype == "code":
+        data_dict["content"] = inner.get("code_content", "")
+        for key in ("code_caption", "code_footnote", "code_language"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype == "algorithm":
+        data_dict["content"] = inner.get("algorithm_content", "")
+        for key in ("algorithm_caption", "algorithm_footnote"):
+            if key in inner:
+                data_dict[key] = inner[key]
+
+    elif btype in ("list", "index"):
+        items = inner.get("list_items", [])
+        data_dict["content"] = "\n".join(items) if items else ""
+        data_dict["list_items"] = items
+
+    else:
+        content_key = f"{btype}_content"
+        spans = inner.get(content_key, [])
+        if isinstance(spans, list) and spans and isinstance(spans[0], dict):
+            data_dict["content"] = _flatten_spans(spans)
+        elif isinstance(inner, str):
+            data_dict["content"] = inner
+        else:
+            data_dict["content"] = ""
+
+    data_dict["raw_content"] = inner
+
+    return data_dict
+
+
+@BaseConverter.register("mineru_v2")
+class MinerUV2Converter(BaseConverter):
+    """Converter for MinerU ``content_list_v2.json`` (pages x blocks)."""
+
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    def convertor(cls, input_args: InputArgs) -> Callable:
+        include = None
+        if hasattr(input_args.dataset, "mineru_config"):
+            cfg = input_args.dataset.mineru_config
+            if cfg.include_types:
+                include = frozenset(cfg.include_types)
+
+        def _convert(raw: Union[str, list]):
+            pages = raw
+            if isinstance(raw, str):
+                pages = json.loads(raw)
+            for page_idx, page_blocks in enumerate(pages):
+                for block_idx, block in enumerate(page_blocks):
+                    if include and block.get("type", "") not in include:
+                        continue
+                    data_dict = _map_block_v2(block, page_idx, block_idx)
+                    yield Data(**data_dict)
+
+        return _convert

dingo/data/datasource/local.py

@@ -395,7 +395,7 @@ class LocalDataSource(DataSource):
         elif os.path.exists(self.path) and os.path.isdir(self.path):
             self._find_all_files(self.path, f_list)
 
-        by_line = self.input_args.dataset.format not in ["json", "listjson"]
+        by_line = self.input_args.dataset.format not in ["json", "listjson", "mineru", "mineru_v2"]
 
         for f in f_list:
             # Check if file is CSV

dingo/exec/local.py

@@ -178,8 +178,9 @@ class LocalExecutor(ExecProto):
                 Model.set_config_rule(model, e_c_i.config)
             elif eval_type == 'llm':
                 model_cls = Model.llm_name_map.get(e_c_i.name)
-                model = model_cls()  # 实例化类为对象，避免多线程配置覆盖
+                model = model_cls()
                 Model.set_config_llm(model, e_c_i.config)
+                Model.set_config_llm(model_cls, e_c_i.config)
             else:
                 raise ValueError(f"Error eval_type: {eval_type}")
 

dingo/io/output/__init__.py

@@ -1,2 +1,3 @@
+# from dingo.io.output.benchmark_report import BenchmarkReport  # noqa E402.
 from dingo.io.output.result_info import ResultInfo  # noqa E402.
 from dingo.io.output.summary_model import SummaryModel  # noqa E402.

dingo/io/output/result_info.py

@@ -33,6 +33,19 @@ class ResultInfo(BaseModel):
         Returns:
             包含原始数据和dingo_result的字典
         """
+        def move_conflict_field(field_name: str):
+            if field_name not in self.raw_data:
+                return
+
+            index = 1
+            while True:
+                backup_field = f'{field_name}_old_v{index}'
+                if backup_field not in self.raw_data:
+                    self.raw_data[backup_field] = self.raw_data[field_name]
+                    del self.raw_data[field_name]
+                    return
+                index += 1
+
         dingo_result = {
             'eval_status': self.eval_status,
             'eval_details': {
@@ -40,5 +53,8 @@ class ResultInfo(BaseModel):
                 for k, v in self.eval_details.items()
             },
         }
+        move_conflict_field('dingo_id')
+        move_conflict_field('dingo_result')
+        self.raw_data['dingo_id'] = self.dingo_id
         self.raw_data['dingo_result'] = dingo_result
         return self.raw_data

dingo/model/llm/compare/llm_html_extract_compare.py

@@ -95,13 +95,28 @@ class LLMHtmlExtractCompare(BaseOpenAI):
 
     @classmethod
     def build_messages(cls, input_data: Data) -> List:
+        raw_data = getattr(input_data, "raw_data", None) or {}
+        # Backward-compatible input handling:
+        # - Preferred: raw_data["magic_md"] and raw_data["content"] (legacy dataset schema)
+        # - Fallback: input_data.prompt (tool A) and input_data.reference (tool B)
+        # - Last resort: input_data.prompt (tool A) and input_data.extra fields if provided
+        tool_a_md = raw_data.get("magic_md", None) or getattr(input_data, "prompt", None)
+        tool_b_md = raw_data.get("content", None) or getattr(input_data, "reference", None)
+
+        if tool_a_md is None or tool_b_md is None:
+            raise ValueError(
+                "LLMHtmlExtractCompare requires Tool A and Tool B markdown. "
+                "Provide raw_data['magic_md'] and raw_data['content'], or provide Data.prompt (tool A) "
+                "and Data.reference (tool B)."
+            )
+
         messages = [
             {
                 "role": "user",
                 "content": cls.prompt.format(
                     input_data.content,
-                    input_data.raw_data["magic_md"],
-                    input_data.raw_data["content"],
+                    tool_a_md,
+                    tool_b_md,
                 ),
             }
         ]

dingo/model/llm/compare/llm_html_extract_compare_v2.py

@@ -36,7 +36,7 @@ class LLMHtmlExtractCompareV2(BaseOpenAI):
         'paper_url': '',
     }
 
-    _required_fields = [RequiredField.CONTENT, RequiredField.PROMPT]
+    _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT]
     prompt = {
         "content_en": r"""Please compare the following two texts, each extracted from the same webpage using different HTML parsing methods. Your task is to determine whether there is a difference in the core informational content between them.
 

dingo/model/llm/compare/llm_html_extract_compare_v3.py

@@ -0,0 +1,221 @@
+import json
+import re
+from typing import List
+
+from dingo.io.input import Data, RequiredField
+from dingo.io.output.eval_detail import EvalDetail
+from dingo.model import Model
+from dingo.model.llm.base_openai import BaseOpenAI
+from dingo.model.response.response_class import ResponseScoreTypeNameReason
+from dingo.utils import log
+from dingo.utils.exception import ConvertJsonError
+
+
+@Model.llm_register("LLMHtmlExtractCompareV3")
+class LLMHtmlExtractCompareV3(BaseOpenAI):
+    """
+    HTML提取工具对比评估 V3 版本
+
+    基于 LLMTextQualityV5 的质量维度（Completeness / Effectiveness / Similarity / Security）
+    对两个 HTML 提取工具的完整输出做对比评估，判断哪个工具的提取质量更高。
+
+    与 V2 的区别：V2 侧重"谁保留了更多信息内容"，V3 侧重"谁引入了更少质量缺陷"。
+    V3 直接发送全文（不做 diff 预处理），保留完整上下文，确保质量缺陷（尤其是
+    Error_Formula 等需要上下文才能正确归因的问题）能被准确识别。
+
+    输入数据要求：
+    - input_data.prompt: 工具A提取的文本（对应 Data.prompt 字段）
+    - input_data.content: 工具B提取的文本（对应 Data.content 字段）
+    - language: 可选，来自 input_data.language 或 raw_data["language"]，缺省为 "en"
+
+    EvalDetail.label 前缀与 Data 字段对齐（避免 TOOL_ONE/TOOL_TWO 歧义）：
+    - PROMPT_BETTER：score=1，Data.prompt 侧提取质量更好
+    - CONTENT_BETTER：score=2，Data.content 侧更好
+    - EXTRACTION_EQUAL：score=0，两者相当
+    """
+
+    _metric_info = {
+        "category": "Pretrain Text Quality Assessment Metrics",
+        "metric_name": "LLMHtmlExtractCompareV3",
+        "description": "Compares two HTML extraction tools using LLM pretraining quality dimensions (completeness, effectiveness, similarity, security) with full-text evaluation for accurate defect attribution",
+    }
+
+    _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT]
+
+    prompt = {
+        "content_en": r"""You are an expert in assessing pretraining data quality for large language models. You will compare two texts extracted from the same HTML page by different tools, and determine which extraction is of higher quality for LLM pretraining.
+
+# Quality Dimensions
+
+Evaluate BOTH texts against these dimensions and compare:
+
+## 1. Completeness
+- **Error_Content_Coverage**: One extraction tool failed to capture the full main-body content of the page — at least one complete paragraph or named section present in the other extraction is entirely absent (e.g., an "Applications" or "Common Algorithms" section is missing). This is about **extraction-level omission** (the tool did not locate or include that block), NOT about individual missing words, broken formatting, or formula stripping (use the specific error types below for those).
+- **Error_Formula**: Mathematical content with broken LaTeX syntax (unmatched delimiters, unclosed environments) OR systematically stripped symbols/formulas (orphan hyphens from stripped Greek letters like "-solutions" instead of "κ-solutions", empty positions after connective words like "thus ;" where a formula was removed)
+- **Error_Table**: Malformed or unreadable table structures (misaligned columns, missing headers, garbled HTML tags)
+- **Error_Code**: Code blocks with formatting corruption (missing code fences, lost indentation, broken identifiers like "sys .argv", line numbers mixed with code)
+
+## 2. Effectiveness
+- **Error_Garbled_Characters**: Encoding issues or anti-crawler artifacts ("’", "□□□", ""); threshold: >1% of characters garbled
+- **Error_Words_Stuck**: Missing spaces breaking tokenization ("Thequickbrownfox"); threshold: >1% of text affected
+- **Error_Lack_Punctuation**: Unclear sentence boundaries ("I like apples they are red also I like oranges")
+
+## 3. Similarity
+- **Error_Duplicate**: Excessive repetition dominating the text; threshold: same phrase repeats >5 times OR duplicate ratio >30%
+
+## 4. Security
+- **Error_Politics**: Content promoting extremism, terrorism, ethnic hatred
+- **Error_Prohibition**: Violence, pornography, gambling, drugs
+
+# Input
+
+**Text A** (Data.prompt — first extraction tool):
+{text_tool_a}
+
+**Text B** (Data.content — second extraction tool):
+{text_tool_b}
+
+# Evaluation Rules
+
+1. Evaluate each text independently against the quality dimensions above, then compare.
+2. Identify the dimension with the **largest quality difference** between the two texts.
+3. Minor formatting or whitespace differences that do not affect training quality should be ignored.
+
+⚠️ The order of Text A and Text B reflects the fixed field mapping: A = `Data.prompt`, B = `Data.content`. Do NOT favor either text based on its position.
+
+# Output Format
+
+Return JSON only:
+{{
+  "score": [0|1|2],
+  "name": "[error_type from the dimension with greatest difference]",
+  "reason": "[objective description of quality differences]"
+}}
+
+Where:
+- `score`: 1 if Text A (`Data.prompt`) is better, 2 if Text B (`Data.content`) is better, 0 if equal
+- `name`: The specific error type with the biggest quality difference (e.g., "Error_Content_Coverage", "Error_Formula", "Error_Table", "Error_Code", "Error_Garbled_Characters", "Error_Words_Stuck", "Error_Lack_Punctuation", "Error_Duplicate", "Error_Politics", "Error_Prohibition"). Use "None" if both are equal.
+- `reason`: Brief objective description (1-3 sentences)
+""",
+        "content_cn": r"""你是一位大语言模型预训练数据质量评估专家。你将对比两个不同 HTML 提取工具从同一网页中提取的文本，判断哪个提取结果的质量更高，更适合用于 LLM 预训练。
+
+# 质量维度
+
+请基于以下维度分别评估两段文本并进行对比：
+
+## 1. 完整性 (Completeness)
+- **Error_Content_Coverage**：一个提取工具未能覆盖网页的完整主体内容——另一方存在的至少一个完整段落或命名小节在这方完全缺失（例如"应用场景"或"常用算法"整节不见）。这针对的是**提取层面的遗漏**（工具未识别或未包含该区块），而非个别词语缺失、格式损坏或公式剥离（这些请用下方对应的专用错误类型）。
+- **Error_Formula**：数学内容存在 LaTeX 语法错误（未匹配的定界符、未关闭的环境）或符号/公式被系统性剥离（如 "κ-solutions" 被剥离为 "-solutions"，连接词后公式缺失如 "thus ;" ）
+- **Error_Table**：表格结构畸形或不可读（列未对齐、缺少表头、HTML标签残留）
+- **Error_Code**：代码块格式损坏（缺少代码围栏、缩进丢失、标识符断裂如 "sys .argv"、行号混入代码）
+
+## 2. 有效性 (Effectiveness)
+- **Error_Garbled_Characters**：编码问题或反爬虫伪影（"’"、"□□□"、""）；阈值：>1% 的字符为乱码
+- **Error_Words_Stuck**：缺失空格导致分词错误（"Thequickbrownfox"）；阈值：>1% 的文本受影响
+- **Error_Lack_Punctuation**：句子边界不清（"I like apples they are red also I like oranges"）
+
+## 3. 相似性 (Similarity)
+- **Error_Duplicate**：过度重复内容；阈值：同一短语重复>5次 或 重复率>30%
+
+## 4. 安全性 (Security)
+- **Error_Politics**：宣扬极端主义、恐怖主义、民族仇恨的内容
+- **Error_Prohibition**：暴力、色情、赌博、毒品相关内容
+
+# 输入
+
+**文本A**（Data.prompt — 第一个提取工具的结果）：
+{text_tool_a}
+
+**文本B**（Data.content — 第二个提取工具的结果）：
+{text_tool_b}
+
+# 评估规则
+
+1. 独立按上述质量维度评估每段文本，再进行对比。
+2. 找出两段文本之间**质量差异最大**的维度。
+3. 不影响训练质量的细微格式差异或空白差异应忽略。
+
+⚠️ 文本A和文本B的顺序反映固定字段映射：A = `Data.prompt`，B = `Data.content`。请勿因位置先后偏好任何一方。
+
+# 输出格式
+
+仅返回 JSON：
+{{
+  "score": [0|1|2],
+  "name": "[差异最大维度中的具体错误类型]",
+  "reason": "[客观描述两段文本的质量差异]"
+}}
+
+其中：
+- `score`：文本A（`Data.prompt`）更好为 1，文本B（`Data.content`）更好为 2，质量相当为 0
+- `name`：差异最大的具体错误类型（如 "Error_Content_Coverage"、"Error_Formula"、"Error_Table"、"Error_Code"、"Error_Garbled_Characters"、"Error_Words_Stuck"、"Error_Lack_Punctuation"、"Error_Duplicate"、"Error_Politics"、"Error_Prohibition"）。如果两者相当则为 "None"。
+- `reason`：简要客观描述（1-3句话）
+""",
+    }
+
+    @classmethod
+    def build_messages(cls, input_data: Data) -> List:
+        text_tool_a = input_data.prompt
+        text_tool_b = input_data.content
+
+        raw_data = getattr(input_data, "raw_data", {}) or {}
+        language = raw_data.get("language", getattr(input_data, "language", "en"))
+
+        if language == "zh":
+            prompt_template = cls.prompt["content_cn"]
+        else:
+            prompt_template = cls.prompt["content_en"]
+
+        prompt_content = prompt_template.format(
+            text_tool_a=text_tool_a,
+            text_tool_b=text_tool_b,
+        )
+
+        return [{"role": "user", "content": prompt_content}]
+
+    @classmethod
+    def process_response(cls, response: str) -> EvalDetail:
+        log.info(response)
+
+        response_think = ""
+        if response.startswith("<think>"):
+            think_content = re.search(
+                r"<think>(.*?)</think>", response, flags=re.DOTALL
+            )
+            if think_content:
+                response_think = think_content.group(1).strip()
+            response = re.sub(r"<think>.*?</think>", "", response, flags=re.DOTALL)
+            response = response.strip()
+
+        if response.startswith("```json"):
+            response = response[7:]
+        if response.startswith("```"):
+            response = response[3:]
+        if response.endswith("```"):
+            response = response[:-3]
+        response = response.strip()
+
+        try:
+            response_json = json.loads(response)
+            if response_think:
+                response_json["reason"] = response_json.get("reason", "") + "\n" + response_think
+        except json.JSONDecodeError:
+            raise ConvertJsonError(f"Convert to JSON format failed: {response}")
+
+        response_model = ResponseScoreTypeNameReason(**response_json)
+
+        result = EvalDetail(metric=cls.__name__)
+
+        # Label prefixes match Data fields: prompt=first extraction, content=second.
+        if response_model.score == 1:
+            tmp_type = "PROMPT_BETTER"
+        elif response_model.score == 2:
+            tmp_type = "CONTENT_BETTER"
+        else:
+            tmp_type = "EXTRACTION_EQUAL"
+
+        result.status = response_model.score != 1
+        result.label = [f"{tmp_type}"]
+        result.reason = [json.dumps(response_json, ensure_ascii=False)]
+
+        return result

dingo/model/llm/hhh/llm_text_3h.py

@@ -10,7 +10,7 @@ from dingo.utils.exception import ConvertJsonError
 
 # @Model.llm_register("LLMText3H")
 class LLMText3H(BaseOpenAI):
-    _required_fields = [RequiredField.CONTENT, RequiredField.PROMPT]
+    _required_fields = [RequiredField.PROMPT, RequiredField.CONTENT]
 
     @classmethod
     def build_messages(cls, input_data):

dingo/model/llm/llm_classify_qr.py

@@ -8,6 +8,7 @@ from dingo.model.llm.base_openai import BaseOpenAI
 from dingo.model.response.response_class import ResponseNameReason
 from dingo.utils import log
 from dingo.utils.exception import ConvertJsonError
+from dingo.utils.image_loader import ImageLoader
 
 
 @Model.llm_register("LLMClassifyQR")
@@ -20,7 +21,7 @@ class LLMClassifyQR(BaseOpenAI):
         "evaluation_results": ""
     }
 
-    _required_fields = [RequiredField.CONTENT]
+    _required_fields = [RequiredField.IMAGE]
     prompt = """
     'Classify the image into one of the following categories: "CAPTCHA", "QR code", or "Normal image". '
     'Return the type as the image category (CAPTCHA or QR code or Normal image) and the reason as the specific type of CAPTCHA or QR code. '
@@ -33,12 +34,13 @@ class LLMClassifyQR(BaseOpenAI):
 
     @classmethod
     def build_messages(cls, input_data: Data) -> List:
+        image_url = ImageLoader.encode_for_api(input_data.image)
         messages = [
             {
                 "role": "user",
                 "content": [
                     {"type": "text", "text": cls.prompt},
-                    {"type": "image_url", "image_url": {"url": input_data.content}},
+                    {"type": "image_url", "image_url": {"url": image_url}},
                 ],
             }
         ]