PorosData-Designer 0.1.0__tar.gz

porosdata_designer-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Kivent
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

porosdata_designer-0.1.0/MANIFEST.in

@@ -0,0 +1,60 @@
+# Minimal release metadata
+include README.md
+include README_CN.md
+include LICENSE
+include pyproject.toml
+
+# Package typing marker
+include src/porosdata_designer/py.typed
+
+# Keep the source distribution clean
+prune tests
+prune data
+prune logs
+prune scripts
+prune examples
+prune docs
+prune docs/audit
+prune .cursor
+prune .github
+prune .idea
+prune .pytest_cache
+prune .vscode
+
+exclude .gitignore
+exclude CHANGELOG.md
+exclude requirements-dev.txt
+exclude Pipfile
+exclude Pipfile.lock
+exclude .env
+exclude .env.*
+
+global-exclude *.py[cod]
+global-exclude __pycache__
+global-exclude .pytest_cache
+global-exclude *.log
+global-exclude *.ipynb
+global-exclude .ipynb_checkpoints
+global-exclude *.pem
+global-exclude *.key
+global-exclude *.png
+global-exclude *.jpg
+global-exclude *.jpeg
+global-exclude *.gif
+global-exclude *.svg
+global-exclude *.webp
+global-exclude *.pdf
+global-exclude *.csv
+global-exclude *.tsv
+global-exclude *.xls
+global-exclude *.xlsx
+global-exclude *.db
+global-exclude *.sqlite
+global-exclude *.sqlite3
+global-exclude *.parquet
+global-exclude *.feather
+global-exclude *.pkl
+global-exclude *.joblib
+global-exclude *.npy
+global-exclude *.npz
+global-exclude .DS_Store

porosdata_designer-0.1.0/PKG-INFO

@@ -0,0 +1,177 @@
+Metadata-Version: 2.4
+Name: PorosData-Designer
+Version: 0.1.0
+Summary: Structured delivery toolkit for converting MinerU parses into full_text, datamining, and multimodal outputs.
+Author-email: Kivent <72405514@cityu-dg.edu.cn>
+License: MIT
+Keywords: mineru,document-processing,scientific-data,multimodal,structured-output
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typing-extensions>=4.0; python_version < "3.10"
+Requires-Dist: dataclasses>=0.6; python_version < "3.7"
+Requires-Dist: pydantic>=2.0
+Requires-Dist: loguru>=0.7.0
+Requires-Dist: pathlib2>=2.3.0; python_version < "3.4"
+Requires-Dist: dataclasses-json>=0.5.0
+Requires-Dist: tqdm>=4.64.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Provides-Extra: optional
+Requires-Dist: cleanlit>=0.2.0; extra == "optional"
+Dynamic: license-file
+
+# PorosData-Designer
+
+`PorosData-Designer` converts MinerU-generated document parses into three stable deliverables: `full_text`, `datamining`, and `multimodal`.
+
+It is designed for scientific data processing, structure-aware training preparation, and atomic document design centered on paragraphs, formulas, chemical expressions, and figure assets.
+
+## What It Does
+
+- Builds a structure-aware full-text view from `*_content_list.json`.
+- Maps document sections, formulas, chemical expressions, and asset references into a datamining view.
+- Extracts image-caption-mention relationships into a multimodal view with copied assets and Markdown outputs.
+
+## Install
+
+```bash
+pip install porosdata-designer
+```
+
+## Quick Start
+
+`--input_dir` should point to a directory tree that contains MinerU outputs. In practice, Designer expects:
+
+- one or more `*_content_list.json` files
+- image assets that remain resolvable relative to those input files
+
+For package usage, it is recommended to pass explicit output and log directories.
+
+Run the full pipeline:
+
+```bash
+porosdata-designer run all --input_dir "path/to/input_dir" --output_dir "path/to/output_dir" --log_dir "path/to/log_dir"
+```
+
+Equivalent module mode:
+
+```bash
+python -m porosdata_designer run all --input_dir "path/to/input_dir" --output_dir "path/to/output_dir" --log_dir "path/to/log_dir"
+```
+
+## Stage Commands
+
+Run text structuring only:
+
+```bash
+porosdata-designer run text --input_dir "path/to/input_dir" --output_dir "path/to/output_dir"
+```
+
+Run multimodal extraction only:
+
+```bash
+porosdata-designer run multimodal --input_dir "path/to/input_dir" --output_dir "path/to/output_dir"
+```
+
+## Outputs
+
+Designer produces three delivery views by default:
+
+- `full_text`: for structure-aware training and human review.
+- `datamining`: for retrieval, extraction, indexing, and downstream storage.
+- `multimodal`: for image-text linking, multimodal indexing, and asset delivery.
+
+Typical output layout:
+
+```text
+path/to/output_dir/
+├── full_text/{doc_id}/
+│   ├── {doc_id}_structured.json
+│   └── {doc_id}_structured.txt
+├── datamining/{doc_id}/
+│   └── {doc_id}_datamining.json
+└── multimodal/{doc_id}/
+    ├── {doc_id}_index.json
+    ├── fig_n.md
+    └── assets/
+```
+
+## Validation
+
+Audit structured outputs:
+
+```bash
+porosdata-designer audit structured --root_dir "path/to/output_dir"
+```
+
+Validate text outputs:
+
+```bash
+porosdata-designer validate structured --output_dir "path/to/output_dir/full_text"
+```
+
+Validate multimodal outputs:
+
+```bash
+porosdata-designer validate multimodal --output_dir "path/to/output_dir/multimodal"
+```
+
+Run final acceptance validation:
+
+```bash
+porosdata-designer validate acceptance --output_dir "path/to/output_dir/multimodal"
+```
+
+## Python Usage
+
+You can also use the package directly in Python:
+
+```python
+from porosdata_designer import DataMiningMapper, MultimodalInterleaver, TextAggregator
+
+aggregator = TextAggregator()
+mapper = DataMiningMapper()
+interleaver = MultimodalInterleaver()
+```
+
+A more complete text-side example:
+
+```python
+from porosdata_designer import DataMiningMapper, TextAggregator
+
+content_list = [
+    {"type": "text", "text_level": 1, "text": "Abstract", "page_idx": 0},
+    {"type": "text", "text": "This work studies a Cu-Zr metallic glass system.", "page_idx": 0},
+    {"type": "text", "text_level": 1, "text": "Results and Discussion", "page_idx": 1},
+    {"type": "text", "text": "Figure 1 shows the microstructure evolution at 700 K.", "page_idx": 1},
+]
+
+aggregator = TextAggregator()
+structured_text = aggregator.aggregate(content_list)
+
+mapper = DataMiningMapper()
+datamining_view = mapper.map(structured_text, {"doc_id": "demo-0001"})
+
+print(structured_text)
+print(datamining_view.pure_text_stream)
+print(datamining_view.structured_json["sections"])
+```
+
+Expected outcome:
+
+- `structured_text` contains Poros tags such as `<poros_doc>`, `<poros_section_*>`, and `<poros_paragraph>`.
+- `pure_text_stream` removes the structure tags while keeping readable text.
+- `structured_json` exposes mined fields such as `sections`, `formulas`, `chemical_formulas`, and `asset_refs`.
+

porosdata_designer-0.1.0/README.md

@@ -0,0 +1,143 @@
+# PorosData-Designer
+
+`PorosData-Designer` converts MinerU-generated document parses into three stable deliverables: `full_text`, `datamining`, and `multimodal`.
+
+It is designed for scientific data processing, structure-aware training preparation, and atomic document design centered on paragraphs, formulas, chemical expressions, and figure assets.
+
+## What It Does
+
+- Builds a structure-aware full-text view from `*_content_list.json`.
+- Maps document sections, formulas, chemical expressions, and asset references into a datamining view.
+- Extracts image-caption-mention relationships into a multimodal view with copied assets and Markdown outputs.
+
+## Install
+
+```bash
+pip install porosdata-designer
+```
+
+## Quick Start
+
+`--input_dir` should point to a directory tree that contains MinerU outputs. In practice, Designer expects:
+
+- one or more `*_content_list.json` files
+- image assets that remain resolvable relative to those input files
+
+For package usage, it is recommended to pass explicit output and log directories.
+
+Run the full pipeline:
+
+```bash
+porosdata-designer run all --input_dir "path/to/input_dir" --output_dir "path/to/output_dir" --log_dir "path/to/log_dir"
+```
+
+Equivalent module mode:
+
+```bash
+python -m porosdata_designer run all --input_dir "path/to/input_dir" --output_dir "path/to/output_dir" --log_dir "path/to/log_dir"
+```
+
+## Stage Commands
+
+Run text structuring only:
+
+```bash
+porosdata-designer run text --input_dir "path/to/input_dir" --output_dir "path/to/output_dir"
+```
+
+Run multimodal extraction only:
+
+```bash
+porosdata-designer run multimodal --input_dir "path/to/input_dir" --output_dir "path/to/output_dir"
+```
+
+## Outputs
+
+Designer produces three delivery views by default:
+
+- `full_text`: for structure-aware training and human review.
+- `datamining`: for retrieval, extraction, indexing, and downstream storage.
+- `multimodal`: for image-text linking, multimodal indexing, and asset delivery.
+
+Typical output layout:
+
+```text
+path/to/output_dir/
+├── full_text/{doc_id}/
+│   ├── {doc_id}_structured.json
+│   └── {doc_id}_structured.txt
+├── datamining/{doc_id}/
+│   └── {doc_id}_datamining.json
+└── multimodal/{doc_id}/
+    ├── {doc_id}_index.json
+    ├── fig_n.md
+    └── assets/
+```
+
+## Validation
+
+Audit structured outputs:
+
+```bash
+porosdata-designer audit structured --root_dir "path/to/output_dir"
+```
+
+Validate text outputs:
+
+```bash
+porosdata-designer validate structured --output_dir "path/to/output_dir/full_text"
+```
+
+Validate multimodal outputs:
+
+```bash
+porosdata-designer validate multimodal --output_dir "path/to/output_dir/multimodal"
+```
+
+Run final acceptance validation:
+
+```bash
+porosdata-designer validate acceptance --output_dir "path/to/output_dir/multimodal"
+```
+
+## Python Usage
+
+You can also use the package directly in Python:
+
+```python
+from porosdata_designer import DataMiningMapper, MultimodalInterleaver, TextAggregator
+
+aggregator = TextAggregator()
+mapper = DataMiningMapper()
+interleaver = MultimodalInterleaver()
+```
+
+A more complete text-side example:
+
+```python
+from porosdata_designer import DataMiningMapper, TextAggregator
+
+content_list = [
+    {"type": "text", "text_level": 1, "text": "Abstract", "page_idx": 0},
+    {"type": "text", "text": "This work studies a Cu-Zr metallic glass system.", "page_idx": 0},
+    {"type": "text", "text_level": 1, "text": "Results and Discussion", "page_idx": 1},
+    {"type": "text", "text": "Figure 1 shows the microstructure evolution at 700 K.", "page_idx": 1},
+]
+
+aggregator = TextAggregator()
+structured_text = aggregator.aggregate(content_list)
+
+mapper = DataMiningMapper()
+datamining_view = mapper.map(structured_text, {"doc_id": "demo-0001"})
+
+print(structured_text)
+print(datamining_view.pure_text_stream)
+print(datamining_view.structured_json["sections"])
+```
+
+Expected outcome:
+
+- `structured_text` contains Poros tags such as `<poros_doc>`, `<poros_section_*>`, and `<poros_paragraph>`.
+- `pure_text_stream` removes the structure tags while keeping readable text.
+- `structured_json` exposes mined fields such as `sections`, `formulas`, `chemical_formulas`, and `asset_refs`.
+

porosdata_designer-0.1.0/README_CN.md

@@ -0,0 +1,143 @@
+# PorosData-Designer
+
+`PorosData-Designer` 用来把 MinerU 生成的文档解析结果转换成三类稳定交付物：`full_text`、`datamining` 和 `multimodal`。
+
+它面向科学数据处理、结构感知训练准备，以及围绕段落、公式、化学式和图表资产展开的原子化文档设计。
+
+## 它解决什么问题
+
+- 把 `*_content_list.json` 构造成适合训练和审阅的结构化全文视图。
+- 把章节、公式、化学式和资产引用整理成稳定的数据挖掘视图。
+- 把图像、图注、正文提及和复制后的图片资产组织成多模态输出。
+
+## 安装
+
+```bash
+pip install porosdata-designer
+```
+
+## 快速开始
+
+`--input_dir` 应指向包含 MinerU 输出结果的目录树。通常要求：
+
+- 存在一个或多个 `*_content_list.json`
+- 对应图片资源能通过与输入文件的相对路径被正确定位
+
+作为 PyPI 安装用户，建议显式传入输出目录和日志目录。
+
+安装后可直接运行：
+
+```bash
+porosdata-designer run all --input_dir "path/to/input_dir" --output_dir "path/to/output_dir" --log_dir "path/to/log_dir"
+```
+
+等价的模块调用方式：
+
+```bash
+python -m porosdata_designer run all --input_dir "path/to/input_dir" --output_dir "path/to/output_dir" --log_dir "path/to/log_dir"
+```
+
+## 分阶段运行
+
+只运行文本结构化：
+
+```bash
+porosdata-designer run text --input_dir "path/to/input_dir" --output_dir "path/to/output_dir"
+```
+
+只运行多模态抽取：
+
+```bash
+porosdata-designer run multimodal --input_dir "path/to/input_dir" --output_dir "path/to/output_dir"
+```
+
+## 输出结果
+
+Designer 默认会生成三类交付结果：
+
+- `full_text`：适合结构感知训练和人工审阅。
+- `datamining`：适合检索、抽取、入库和后续数据分析。
+- `multimodal`：适合图文关联、多模态索引和图片资产交付。
+
+典型输出目录如下：
+
+```text
+path/to/output_dir/
+├── full_text/{doc_id}/
+│   ├── {doc_id}_structured.json
+│   └── {doc_id}_structured.txt
+├── datamining/{doc_id}/
+│   └── {doc_id}_datamining.json
+└── multimodal/{doc_id}/
+    ├── {doc_id}_index.json
+    ├── fig_n.md
+    └── assets/
+```
+
+## 校验与验收
+
+审计结构化输出：
+
+```bash
+porosdata-designer audit structured --root_dir "path/to/output_dir"
+```
+
+校验文本输出：
+
+```bash
+porosdata-designer validate structured --output_dir "path/to/output_dir/full_text"
+```
+
+校验多模态输出：
+
+```bash
+porosdata-designer validate multimodal --output_dir "path/to/output_dir/multimodal"
+```
+
+执行最终验收校验：
+
+```bash
+porosdata-designer validate acceptance --output_dir "path/to/output_dir/multimodal"
+```
+
+## Python 用法
+
+你也可以直接在 Python 中使用这些核心组件：
+
+```python
+from porosdata_designer import DataMiningMapper, MultimodalInterleaver, TextAggregator
+
+aggregator = TextAggregator()
+mapper = DataMiningMapper()
+interleaver = MultimodalInterleaver()
+```
+
+下面是一个更完整的文本侧示例：
+
+```python
+from porosdata_designer import DataMiningMapper, TextAggregator
+
+content_list = [
+    {"type": "text", "text_level": 1, "text": "Abstract", "page_idx": 0},
+    {"type": "text", "text": "This work studies a Cu-Zr metallic glass system.", "page_idx": 0},
+    {"type": "text", "text_level": 1, "text": "Results and Discussion", "page_idx": 1},
+    {"type": "text", "text": "Figure 1 shows the microstructure evolution at 700 K.", "page_idx": 1},
+]
+
+aggregator = TextAggregator()
+structured_text = aggregator.aggregate(content_list)
+
+mapper = DataMiningMapper()
+datamining_view = mapper.map(structured_text, {"doc_id": "demo-0001"})
+
+print(structured_text)
+print(datamining_view.pure_text_stream)
+print(datamining_view.structured_json["sections"])
+```
+
+这个例子执行后，你可以看到：
+
+- `structured_text` 会生成带 Poros 标签的结构化文本，例如 `<poros_doc>`、`<poros_section_*>`、`<poros_paragraph>`。
+- `pure_text_stream` 会去掉结构标签，保留适合继续训练或检索的纯文本内容。
+- `structured_json` 会暴露 `sections`、`formulas`、`chemical_formulas`、`asset_refs` 等数据挖掘字段。
+

porosdata_designer-0.1.0/pyproject.toml

@@ -0,0 +1,64 @@
+[project]
+name = "PorosData-Designer"
+version = "0.1.0"
+description = "Structured delivery toolkit for converting MinerU parses into full_text, datamining, and multimodal outputs."
+readme = "README.md"
+requires-python = ">=3.8"
+license = {text = "MIT"}
+authors = [
+    {name = "Kivent", email = "72405514@cityu-dg.edu.cn"},
+]
+keywords = ["mineru", "document-processing", "scientific-data", "multimodal", "structured-output"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+
+dependencies = [
+    # 原有依赖
+    "typing-extensions>=4.0; python_version<'3.10'",
+    "dataclasses>=0.6; python_version<'3.7'",
+    "pydantic>=2.0",
+    "loguru>=0.7.0",
+
+    # 新增必需依赖
+    "pathlib2>=2.3.0; python_version<'3.4'",  # Path处理兼容性
+    "dataclasses-json>=0.5.0",  # JSON序列化支持
+    "tqdm>=4.64.0",  # 进度条显示
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov",
+    "black",
+    "mypy",
+]
+optional = [
+    "cleanlit>=0.2.0",
+]
+
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["porosdata_designer*"]
+
+[tool.setuptools.package-data]
+porosdata_designer = ["py.typed"]
+
+[project.scripts]
+porosdata-designer = "porosdata_designer.cli:main"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+

porosdata_designer-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+