PorosData-Designer 0.1.0__py3-none-any.whl

porosdata_designer/__init__.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+"""
+PorosData-Designer 包初始化模块
+
+核心能力:
+    - 段落类型识别: ParagraphClassifier 智能分类文档结构
+    - 文本结构化: TextAggregator 全文本聚合引擎
+    - 多模态关联: MultimodalInterleaver 图文交织引擎
+    - 标记与格式化: TokenMarker 添加特殊标记
+    - Schema 校验: SchemaValidator / LaTeXValidator
+    - 资产穿透: AssetAnchoringEngine (Fig/Table → UUID)
+    - 数据挖掘: DataMiningMapper 双视图映射
+"""
+
+import sys
+import os
+from typing import Dict, List
+
+if sys.platform == "win32":
+    os.environ["PYTHONIOENCODING"] = "utf-8"
+    try:
+        import subprocess
+        subprocess.run(["chcp", "65001"], shell=True, capture_output=True)
+    except:
+        pass
+    if hasattr(sys.stdout, "reconfigure"):
+        sys.stdout.reconfigure(encoding="utf-8")
+    if hasattr(sys.stderr, "reconfigure"):
+        sys.stderr.reconfigure(encoding="utf-8")
+
+from .reorganizers.content_filter import ContentFilter, ContentType
+from .reorganizers.paragraph_classifier import ParagraphClassifier, ParagraphType
+from .reorganizers.token_marker import TokenMarker
+from .reorganizers.text_aggregator import TextAggregator
+from .reorganizers.multimodal_interleaver import MultimodalInterleaver
+from .adapters.content_list_adapter import ContentListAdapter
+from .runtime.plugin_system import PluginRegistry
+from .validators.schema_validator import SchemaValidator, SchemaValidationResult, SchemaIssue
+from .validators.latex_validator import LaTeXValidator, LaTeXValidationResult, FormulaQuality
+from .mappers.asset_anchoring import AssetAnchoringEngine
+from .mappers.data_mining_mapper import DataMiningMapper, DataMiningView
+
+from . import adapters
+from . import mappers
+from . import reorganizers
+from . import runtime
+from . import validators
+
+
+def aggregate_text(content_list: List[Dict]) -> str:
+    """便捷的文本聚合函数
+
+    Args:
+        content_list: MinerU解析后的内容列表
+
+    Returns:
+        结构化的XML文本
+
+    Example:
+        >>> from porosdata_designer import aggregate_text
+        >>> xml_text = aggregate_text(content_list)
+    """
+    aggregator = TextAggregator()
+    return aggregator.aggregate(content_list)
+
+
+__version__ = "0.1.0"
+__all__ = [
+    "aggregate_text",
+
+    "ContentFilter",
+    "ContentType",
+    "ParagraphClassifier",
+    "ParagraphType",
+    "TokenMarker",
+    "TextAggregator",
+    "MultimodalInterleaver",
+    "ContentListAdapter",
+    "PluginRegistry",
+    "SchemaValidator",
+    "SchemaValidationResult",
+    "SchemaIssue",
+    "LaTeXValidator",
+    "LaTeXValidationResult",
+    "FormulaQuality",
+    "AssetAnchoringEngine",
+    "DataMiningMapper",
+    "DataMiningView",
+]

porosdata_designer/__main__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+"""Module entry point forwarding to the unified CLI."""
+
+from .cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

porosdata_designer/adapters/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""适配器模块 - 支持多种输入格式"""
+
+from .content_list_adapter import ContentListAdapter
+
+from . import content_list_adapter
+
+__all__ = [
+    "ContentListAdapter",
+]

porosdata_designer/adapters/content_list_adapter.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+"""
+内容列表适配器 (ContentListAdapter) - 数据重组流水线的输入适配组件
+
+详细描述：
+    专门处理MinerU输出的content_list.json格式数据，提供标准化的
+    数据访问接口。负责文件读取、格式验证和数据预处理。
+
+核心功能:
+    - 文件读取: 从JSON文件加载MinerU content_list数据
+    - 格式验证: 检查数据结构的完整性和正确性
+    - 数据标准化: 统一不同版本MinerU输出的数据格式（含 processed 的 image 优化格式）
+    - 错误处理: 完善的异常捕获和错误报告机制
+
+处理格式:
+    - 输入: MinerU content_list.json文件路径
+    - 输出: 标准化的List[Dict]数据结构
+    - 支持类型: text、image、table等MinerU元素类型
+    - image: image_caption / image_footnote 支持旧版 [str] 与新版 [{"text": "...", "original_text": "..."}]
+"""
+
+import json
+from pathlib import Path
+from typing import List, Dict
+from ..runtime.plugin_system import PluginRegistry
+
+
+def _normalize_caption_footnote_list(raw_list: List) -> List[str]:
+    """将 image_caption 或 image_footnote 的原始列表规范为清理后的文本字符串列表。
+
+    兼容两种格式：
+    - 旧版/原始 MinerU：列表元素为 str
+    - 新版 processed：列表元素为 dict，含 "text"（清理后）与 "original_text"
+    统一返回清理后的文本列表，供下游解析与展示使用。
+    """
+    if not raw_list:
+        return []
+    result = []
+    for elem in raw_list:
+        if isinstance(elem, dict):
+            result.append(elem.get("text", "").strip() or "")
+        elif isinstance(elem, str):
+            result.append(elem.strip())
+        else:
+            result.append(str(elem).strip())
+    return [s for s in result if s]
+
+
+class ContentListAdapter:
+    """content_list.json 适配器"""
+    
+    @staticmethod
+    def load(file_path: str) -> List[Dict]:
+        """加载 content_list.json 文件
+        
+        Args:
+            file_path: JSON 文件路径
+        
+        Returns:
+            内容块列表
+        """
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content_list = json.load(f)
+        
+        # 验证格式（应该是数组）
+        if not isinstance(content_list, list):
+            raise ValueError(
+                f"content_list.json should be a list, got {type(content_list)}"
+            )
+        
+        return content_list
+
+    @staticmethod
+    def get_image_caption_texts(image_item: Dict) -> List[str]:
+        """从图片项中获取清理后的图注文本列表。
+
+        兼容 processed 优化格式：image_caption 为 [{"text": "...", "original_text": "..."}] 时，
+        仅返回 "text" 字段（清理后文本）；旧版 [str] 则原样返回并 strip。
+
+        Args:
+            image_item: content_list 中 type=="image" 的项
+
+        Returns:
+            清理后的图注字符串列表
+        """
+        raw = image_item.get("image_caption", [])
+        return _normalize_caption_footnote_list(raw)
+
+    @staticmethod
+    def get_image_footnote_texts(image_item: Dict) -> List[str]:
+        """从图片项中获取清理后的图脚注文本列表。
+
+        格式约定同 get_image_caption_texts。
+
+        Args:
+            image_item: content_list 中 type=="image" 的项
+
+        Returns:
+            清理后的图脚注字符串列表
+        """
+        raw = image_item.get("image_footnote", [])
+        return _normalize_caption_footnote_list(raw)
+    
+    @staticmethod
+    def save(data: List[Dict], file_path: str) -> None:
+        """保存为 content_list.json 格式
+        
+        Args:
+            data: 内容块列表
+            file_path: 输出文件路径
+        """
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+@PluginRegistry.register("content_list_adapter")
+def load_content_list(file_path: str) -> List[Dict]:
+    """content_list.json 适配器"""
+    return ContentListAdapter.load(file_path)
+

porosdata_designer/cli.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Optional, Sequence
+
+from .runtime.commands import (
+    run_final_acceptance_validation,
+    run_structured_audit,
+    validate_delivery_outputs,
+    validate_multimodal_outputs,
+    validate_structured_outputs,
+)
+from .runtime.pipelines import ensure_console_encoding, run_full_pipeline, run_multimodal_pipeline, run_text_pipeline
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="porosdata-designer", description="PorosData Designer unified command line interface")
+    subparsers = parser.add_subparsers(dest="command")
+
+    run_parser = subparsers.add_parser("run", help="run pipeline stages")
+    run_subparsers = run_parser.add_subparsers(dest="run_command")
+    for name in ("all", "text", "multimodal"):
+        cmd = run_subparsers.add_parser(name, help=f"run {name} pipeline")
+        cmd.add_argument("--input_dir", type=Path, required=True, help="Input data directory, for example data/processed")
+        cmd.add_argument("--output_dir", type=Path, default=None, help="Structured output base directory")
+        cmd.add_argument("--log_dir", type=Path, default=None, help="Log directory")
+
+    audit_parser = subparsers.add_parser("audit", help="run audits")
+    audit_subparsers = audit_parser.add_subparsers(dest="audit_command")
+    audit_structured = audit_subparsers.add_parser("structured", help="audit data/structured outputs")
+    audit_structured.add_argument("--root_dir", type=Path, default=None, help="Structured root directory")
+
+    validate_parser = subparsers.add_parser("validate", help="run validations")
+    validate_subparsers = validate_parser.add_subparsers(dest="validate_command")
+
+    validate_structured = validate_subparsers.add_parser("structured", help="validate full_text structured outputs")
+    validate_structured.add_argument("--output_dir", type=Path, default=None, help="full_text output directory")
+    validate_structured.add_argument("--log_dir", type=Path, default=None, help="Log directory")
+
+    validate_multimodal = validate_subparsers.add_parser("multimodal", help="validate multimodal outputs")
+    validate_multimodal.add_argument("--output_dir", type=Path, default=None, help="multimodal output directory")
+    validate_multimodal.add_argument("--log_dir", type=Path, default=None, help="Log directory")
+
+    validate_acceptance = validate_subparsers.add_parser("acceptance", help="run final acceptance validation")
+    validate_acceptance.add_argument("--output_dir", type=Path, default=None, help="multimodal output directory")
+    validate_acceptance.add_argument("--log_dir", type=Path, default=None, help="Log directory")
+
+    validate_delivery = validate_subparsers.add_parser("delivery", help="validate outputs against delivery standard")
+    validate_delivery.add_argument("--root_dir", type=Path, default=None, help="structured root directory")
+    validate_delivery.add_argument("--log_dir", type=Path, default=None, help="Log directory")
+    return parser
+
+
+def dispatch(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
+    if args.command == "run":
+        if args.run_command == "all":
+            return run_full_pipeline(args.input_dir, output_dir=args.output_dir, log_dir=args.log_dir)
+        if args.run_command == "text":
+            return run_text_pipeline(args.input_dir, output_dir=args.output_dir, log_dir=args.log_dir)
+        if args.run_command == "multimodal":
+            return run_multimodal_pipeline(args.input_dir, output_dir=args.output_dir, log_dir=args.log_dir)
+        run_parser = next(action for action in parser._actions if isinstance(action, argparse._SubParsersAction)).choices["run"]
+        run_parser.print_help()
+        return 0
+
+    if args.command == "audit":
+        if args.audit_command == "structured":
+            return run_structured_audit(structured_root=args.root_dir)
+        audit_parser = next(action for action in parser._actions if isinstance(action, argparse._SubParsersAction)).choices["audit"]
+        audit_parser.print_help()
+        return 0
+
+    if args.command == "validate":
+        if args.validate_command == "structured":
+            return validate_structured_outputs(output_dir=args.output_dir, log_dir=args.log_dir)
+        if args.validate_command == "multimodal":
+            return validate_multimodal_outputs(output_dir=args.output_dir, log_dir=args.log_dir)
+        if args.validate_command == "acceptance":
+            return run_final_acceptance_validation(multimodal_dir=args.output_dir, log_dir=args.log_dir)
+        if args.validate_command == "delivery":
+            return validate_delivery_outputs(structured_root=args.root_dir, log_dir=args.log_dir)
+        validate_parser = next(action for action in parser._actions if isinstance(action, argparse._SubParsersAction)).choices["validate"]
+        validate_parser.print_help()
+        return 0
+
+    parser.print_help()
+    return 0
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    ensure_console_encoding()
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return dispatch(args, parser)

porosdata_designer/config.py

@@ -0,0 +1 @@
+from .runtime.config import *  # noqa: F401,F403

porosdata_designer/mappers/__init__.py

@@ -0,0 +1,3 @@
+"""Output mapping and asset anchoring helpers."""
+from .asset_anchoring import *  # noqa: F401,F403
+from .data_mining_mapper import *  # noqa: F401,F403

porosdata_designer/mappers/asset_anchoring.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+"""
+资产穿透引擎 (Asset Anchoring Engine) - Fig/Table 与物理资产硬链接
+
+核心职责：
+- 自动扫描文本中的 Fig. n 或 Table n 关键字
+- 将其替换为包含 UUID 的资产标签 <poros_asset uuid="xxx">
+- 直接关联 fig_n.md 中的图像元数据
+"""
+
+import re
+import uuid
+from typing import Dict, List, Optional, Tuple
+from pathlib import Path
+
+from ..runtime.config import TAG_PREFIX
+from ..adapters.content_list_adapter import ContentListAdapter
+
+# 使用 uuid5 确保同一 doc+fig 始终得到相同 UUID，便于跨任务关联
+def _make_asset_uuid(doc_id: str, asset_key: str) -> str:
+    namespace = uuid.NAMESPACE_DNS
+    name = f"{doc_id}:{asset_key}"
+    return str(uuid.uuid5(namespace, name))
+
+
+class AssetAnchoringEngine:
+    """资产穿透引擎
+
+    将正文中的 Fig. n / Table n 引用替换为带 UUID 的资产标签，
+    建立与物理文件（fig_n.md、图像）的硬链接。
+    """
+
+    def __init__(self):
+        # Fig 引用模式：Fig. 1, Fig 1, Figure 1, Fig 1a, Fig 2.1
+        self._fig_pattern = re.compile(
+            r'\b(?:Fig\.?|Figure|FIG\.?)\s+(\d+(?:\.\d+|[a-zA-Z])?)\b',
+            re.IGNORECASE
+        )
+        # Table 引用模式
+        self._table_pattern = re.compile(
+            r'\b(?:Table|TABLE)\s+(\d+(?:\.\d+|[a-zA-Z])?)\b',
+            re.IGNORECASE
+        )
+
+    def build_asset_registry(
+        self,
+        content_list: List[Dict],
+        doc_id: str,
+        output_dir: Optional[Path] = None
+    ) -> Dict[str, str]:
+        """建立 Fig/Table ID -> UUID 的注册表
+
+        使用 uuid5 确保同一 doc+fig 始终得到相同 UUID，便于 full_text 与 multimodal 跨任务关联。
+
+        Args:
+            content_list: MinerU 内容列表
+            doc_id: 文档 ID
+            output_dir: 多模态输出目录（可选，用于后续生成 fig_n.md）
+
+        Returns:
+            {"fig_1": "uuid-xxx", "table_1": "uuid-yyy", ...}
+        """
+        registry: Dict[str, str] = {}
+
+        for item in content_list:
+            if item.get("type") == "image":
+                fig_id = self._parse_fig_id_from_image(item)
+                if fig_id:
+                    key = f"fig_{fig_id}"
+                    if key not in registry:
+                        registry[key] = _make_asset_uuid(doc_id, key)
+
+            if item.get("type") == "table" or "table_body" in item:
+                table_id = self._parse_table_id(item)
+                if table_id:
+                    key = f"table_{table_id}"
+                    if key not in registry:
+                        registry[key] = _make_asset_uuid(doc_id, key)
+
+        return registry
+
+    def _parse_fig_id_from_image(self, image_item: Dict) -> Optional[str]:
+        """从图片项解析 Fig ID（使用清理后的图注文本，兼容 processed image 格式）"""
+        captions = ContentListAdapter.get_image_caption_texts(image_item)
+        for cap in captions:
+            m = re.search(r'\b(?:Fig\.?|Figure|FIG\.?)\s+(\d+(?:\.\d+|[a-zA-Z])?)', cap, re.IGNORECASE)
+            if m:
+                return m.group(1)
+        return None
+
+    def _parse_table_id(self, item: Dict) -> Optional[str]:
+        """从表格项解析 Table ID"""
+        text = item.get("text", "") or ""
+        m = re.search(r'\b(?:Table|TABLE)\s+(\d+(?:\.\d+|[a-zA-Z])?)', text, re.IGNORECASE)
+        if m:
+            return m.group(1)
+        return None
+
+    def anchor_text(
+        self,
+        text: str,
+        registry: Dict[str, str],
+        replace_fig: bool = True,
+        replace_table: bool = True
+    ) -> Tuple[str, List[Dict]]:
+        """将文本中的 Fig/Table 引用替换为资产标签
+
+        Args:
+            text: 原始文本
+            registry: {"fig_1": "uuid-xxx", "table_1": "uuid-yyy"}
+            replace_fig: 是否替换 Fig 引用
+            replace_table: 是否替换 Table 引用
+
+        Returns:
+            (替换后的文本, 替换记录列表)
+        """
+        result = text
+        replacements: List[Dict] = []
+
+        def replace_fig_match(m):
+            fig_id = m.group(1)
+            key = f"fig_{fig_id}"
+            if key in registry:
+                uid = registry[key]
+                replacements.append({"type": "fig", "id": fig_id, "uuid": uid})
+                return f'<{TAG_PREFIX}asset uuid="{uid}" type="fig" ref="{fig_id}">Fig. {fig_id}</{TAG_PREFIX}asset>'
+            return m.group(0)
+
+        def replace_table_match(m):
+            table_id = m.group(1)
+            key = f"table_{table_id}"
+            if key in registry:
+                uid = registry[key]
+                replacements.append({"type": "table", "id": table_id, "uuid": uid})
+                return f'<{TAG_PREFIX}asset uuid="{uid}" type="table" ref="{table_id}">Table {table_id}</{TAG_PREFIX}asset>'
+            return m.group(0)
+
+        if replace_fig:
+            result = self._fig_pattern.sub(replace_fig_match, result)
+        if replace_table:
+            result = self._table_pattern.sub(replace_table_match, result)
+
+        return result, replacements