dtflow 0.4.3__tar.gz → 0.5.2__tar.gz

{dtflow-0.4.3 → dtflow-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.4.3
+Version: 0.5.2
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme
@@ -32,16 +32,26 @@ Requires-Dist: orjson>=3.9.0
 Requires-Dist: polars>=0.20.0
 Requires-Dist: pyyaml>=5.4.0
 Requires-Dist: rich>=10.0.0
+Requires-Dist: tiktoken>=0.5.0
 Requires-Dist: typer>=0.9.0
 Provides-Extra: converters
 Requires-Dist: datasets>=2.0.0; extra == 'converters'
 Provides-Extra: dev
 Requires-Dist: black>=21.0; extra == 'dev'
+Requires-Dist: datasets>=2.0.0; extra == 'dev'
+Requires-Dist: datasketch>=1.5.0; extra == 'dev'
 Requires-Dist: flake8>=3.9.0; extra == 'dev'
+Requires-Dist: huggingface-hub>=0.20.0; extra == 'dev'
 Requires-Dist: isort>=5.9.0; extra == 'dev'
 Requires-Dist: mypy>=0.910; extra == 'dev'
+Requires-Dist: pyarrow; extra == 'dev'
 Requires-Dist: pytest-cov>=2.12.0; extra == 'dev'
 Requires-Dist: pytest>=6.0.0; extra == 'dev'
+Requires-Dist: rich>=10.0.0; extra == 'dev'
+Requires-Dist: scikit-learn>=0.24.0; extra == 'dev'
+Requires-Dist: tiktoken>=0.5.0; extra == 'dev'
+Requires-Dist: tokenizers>=0.15.0; extra == 'dev'
+Requires-Dist: toolong>=1.5.0; extra == 'dev'
 Provides-Extra: display
 Provides-Extra: docs
 Requires-Dist: myst-parser>=0.15.0; extra == 'docs'
@@ -129,7 +139,7 @@ dt.filter(lambda x: x.language == "zh")
 ### 数据验证
 
 ```python
-# 验证数据，返回不通过的记录列表
+# 简单验证，返回不通过的记录列表
 errors = dt.validate(lambda x: len(x.messages) >= 2)
 
 if errors:
@@ -137,6 +147,53 @@ if errors:
         print(f"第 {e.index} 行: {e.error}")
 ```
 
+### Schema 验证
+
+使用 Schema 进行结构化数据验证：
+
+```python
+from dtflow import Schema, Field, openai_chat_schema
+
+# 使用预设 Schema
+result = dt.validate_schema(openai_chat_schema)
+print(result)  # ValidationResult(valid=950, invalid=50, errors=[...])
+
+# 自定义 Schema
+schema = Schema({
+    "messages": Field(type="list", required=True, min_length=1),
+    "messages[*].role": Field(type="str", choices=["user", "assistant", "system"]),
+    "messages[*].content": Field(type="str", min_length=1),
+    "score": Field(type="float", min=0, max=1),
+})
+
+result = dt.validate_schema(schema)
+
+# 过滤出有效数据
+valid_dt = dt.validate_schema(schema, filter_invalid=True)
+valid_dt.save("valid.jsonl")
+```
+
+**预设 Schema**：
+
+| Schema 名称 | 用途 |
+|------------|------|
+| `openai_chat_schema` | OpenAI messages 格式验证 |
+| `alpaca_schema` | Alpaca instruction/output 格式 |
+| `sharegpt_schema` | ShareGPT conversations 格式 |
+| `dpo_schema` | DPO prompt/chosen/rejected 格式 |
+
+**Field 参数**：
+
+| 参数 | 说明 | 示例 |
+|------|------|------|
+| `type` | 类型验证 | `"str"`, `"int"`, `"float"`, `"bool"`, `"list"`, `"dict"` |
+| `required` | 是否必填 | `True` / `False` |
+| `min` / `max` | 数值范围 | `min=0, max=1` |
+| `min_length` / `max_length` | 长度范围 | `min_length=1` |
+| `choices` | 枚举值 | `choices=["user", "assistant"]` |
+| `pattern` | 正则匹配 | `pattern=r"^\d{4}-\d{2}-\d{2}$"` |
+| `custom` | 自定义验证 | `custom=lambda x: x > 0` |
+
 ### 数据转换
 
 ```python
@@ -286,6 +343,58 @@ dt.transform(to_swift_vlm(images_field="images")).save("swift_vlm.jsonl")
 # 输出: {"messages": [...], "images": ["/path/to/img.jpg"]}
 ```
 
+### 训练框架一键导出
+
+将数据导出为目标训练框架可直接使用的格式，自动生成配置文件：
+
+```python
+from dtflow import DataTransformer
+
+dt = DataTransformer.load("data.jsonl")
+
+# 1. 检查框架兼容性
+result = dt.check_compatibility("llama-factory")
+print(result)
+# ✅ 兼容 - LLaMA-Factory (openai_chat)
+# 或
+# ❌ 不兼容 - 错误: xxx
+
+# 2. 一键导出到 LLaMA-Factory
+files = dt.export_for("llama-factory", "./llama_ready/")
+# 生成文件:
+# - ./llama_ready/custom_dataset.json      # 数据文件
+# - ./llama_ready/dataset_info.json        # 数据集配置
+# - ./llama_ready/train_args.yaml          # 训练参数模板
+
+# 3. 导出到 ms-swift
+files = dt.export_for("swift", "./swift_ready/")
+# 生成: data.jsonl + train_swift.sh
+
+# 4. 导出到 Axolotl
+files = dt.export_for("axolotl", "./axolotl_ready/")
+# 生成: data.jsonl + config.yaml
+
+# 指定数据集名称
+dt.export_for("llama-factory", "./output/", dataset_name="my_sft_data")
+```
+
+**支持的框架**：
+
+| 框架 | 导出内容 | 使用方式 |
+|------|---------|---------|
+| `llama-factory` | data.json + dataset_info.json + train_args.yaml | `llamafactory-cli train train_args.yaml` |
+| `swift` | data.jsonl + train_swift.sh | `bash train_swift.sh` |
+| `axolotl` | data.jsonl + config.yaml | `accelerate launch -m axolotl.cli.train config.yaml` |
+
+**自动格式检测**：
+
+| 检测到的格式 | 数据结构 |
+|------------|---------|
+| `openai_chat` | `{"messages": [{"role": "user", ...}]}` |
+| `alpaca` | `{"instruction": ..., "output": ...}` |
+| `sharegpt` | `{"conversations": [{"from": "human", ...}]}` |
+| `dpo` | `{"prompt": ..., "chosen": ..., "rejected": ...}` |
+
 ### 其他操作
 
 ```python
@@ -361,6 +470,12 @@ dt concat a.jsonl b.jsonl -o merged.jsonl
 
 # 数据统计
 dt stats data.jsonl
+
+# 数据验证
+dt validate data.jsonl --preset=openai_chat           # 使用预设 schema 验证
+dt validate data.jsonl --preset=alpaca --verbose      # 详细输出
+dt validate data.jsonl --preset=sharegpt --filter-invalid -o valid.jsonl  # 过滤出有效数据
+dt validate data.jsonl --preset=dpo --max-errors=100  # 限制错误输出数量
 ```
 
 ### 字段路径语法

{dtflow-0.4.3 → dtflow-0.5.2}/README.md

@@ -53,7 +53,7 @@ dt.filter(lambda x: x.language == "zh")
 ### 数据验证
 
 ```python
-# 验证数据，返回不通过的记录列表
+# 简单验证，返回不通过的记录列表
 errors = dt.validate(lambda x: len(x.messages) >= 2)
 
 if errors:
@@ -61,6 +61,53 @@ if errors:
         print(f"第 {e.index} 行: {e.error}")
 ```
 
+### Schema 验证
+
+使用 Schema 进行结构化数据验证：
+
+```python
+from dtflow import Schema, Field, openai_chat_schema
+
+# 使用预设 Schema
+result = dt.validate_schema(openai_chat_schema)
+print(result)  # ValidationResult(valid=950, invalid=50, errors=[...])
+
+# 自定义 Schema
+schema = Schema({
+    "messages": Field(type="list", required=True, min_length=1),
+    "messages[*].role": Field(type="str", choices=["user", "assistant", "system"]),
+    "messages[*].content": Field(type="str", min_length=1),
+    "score": Field(type="float", min=0, max=1),
+})
+
+result = dt.validate_schema(schema)
+
+# 过滤出有效数据
+valid_dt = dt.validate_schema(schema, filter_invalid=True)
+valid_dt.save("valid.jsonl")
+```
+
+**预设 Schema**：
+
+| Schema 名称 | 用途 |
+|------------|------|
+| `openai_chat_schema` | OpenAI messages 格式验证 |
+| `alpaca_schema` | Alpaca instruction/output 格式 |
+| `sharegpt_schema` | ShareGPT conversations 格式 |
+| `dpo_schema` | DPO prompt/chosen/rejected 格式 |
+
+**Field 参数**：
+
+| 参数 | 说明 | 示例 |
+|------|------|------|
+| `type` | 类型验证 | `"str"`, `"int"`, `"float"`, `"bool"`, `"list"`, `"dict"` |
+| `required` | 是否必填 | `True` / `False` |
+| `min` / `max` | 数值范围 | `min=0, max=1` |
+| `min_length` / `max_length` | 长度范围 | `min_length=1` |
+| `choices` | 枚举值 | `choices=["user", "assistant"]` |
+| `pattern` | 正则匹配 | `pattern=r"^\d{4}-\d{2}-\d{2}$"` |
+| `custom` | 自定义验证 | `custom=lambda x: x > 0` |
+
 ### 数据转换
 
 ```python
@@ -210,6 +257,58 @@ dt.transform(to_swift_vlm(images_field="images")).save("swift_vlm.jsonl")
 # 输出: {"messages": [...], "images": ["/path/to/img.jpg"]}
 ```
 
+### 训练框架一键导出
+
+将数据导出为目标训练框架可直接使用的格式，自动生成配置文件：
+
+```python
+from dtflow import DataTransformer
+
+dt = DataTransformer.load("data.jsonl")
+
+# 1. 检查框架兼容性
+result = dt.check_compatibility("llama-factory")
+print(result)
+# ✅ 兼容 - LLaMA-Factory (openai_chat)
+# 或
+# ❌ 不兼容 - 错误: xxx
+
+# 2. 一键导出到 LLaMA-Factory
+files = dt.export_for("llama-factory", "./llama_ready/")
+# 生成文件:
+# - ./llama_ready/custom_dataset.json      # 数据文件
+# - ./llama_ready/dataset_info.json        # 数据集配置
+# - ./llama_ready/train_args.yaml          # 训练参数模板
+
+# 3. 导出到 ms-swift
+files = dt.export_for("swift", "./swift_ready/")
+# 生成: data.jsonl + train_swift.sh
+
+# 4. 导出到 Axolotl
+files = dt.export_for("axolotl", "./axolotl_ready/")
+# 生成: data.jsonl + config.yaml
+
+# 指定数据集名称
+dt.export_for("llama-factory", "./output/", dataset_name="my_sft_data")
+```
+
+**支持的框架**：
+
+| 框架 | 导出内容 | 使用方式 |
+|------|---------|---------|
+| `llama-factory` | data.json + dataset_info.json + train_args.yaml | `llamafactory-cli train train_args.yaml` |
+| `swift` | data.jsonl + train_swift.sh | `bash train_swift.sh` |
+| `axolotl` | data.jsonl + config.yaml | `accelerate launch -m axolotl.cli.train config.yaml` |
+
+**自动格式检测**：
+
+| 检测到的格式 | 数据结构 |
+|------------|---------|
+| `openai_chat` | `{"messages": [{"role": "user", ...}]}` |
+| `alpaca` | `{"instruction": ..., "output": ...}` |
+| `sharegpt` | `{"conversations": [{"from": "human", ...}]}` |
+| `dpo` | `{"prompt": ..., "chosen": ..., "rejected": ...}` |
+
 ### 其他操作
 
 ```python
@@ -285,6 +384,12 @@ dt concat a.jsonl b.jsonl -o merged.jsonl
 
 # 数据统计
 dt stats data.jsonl
+
+# 数据验证
+dt validate data.jsonl --preset=openai_chat           # 使用预设 schema 验证
+dt validate data.jsonl --preset=alpaca --verbose      # 详细输出
+dt validate data.jsonl --preset=sharegpt --filter-invalid -o valid.jsonl  # 过滤出有效数据
+dt validate data.jsonl --preset=dpo --max-errors=100  # 限制错误输出数量
 ```
 
 ### 字段路径语法

{dtflow-0.4.3 → dtflow-0.5.2}/dtflow/__init__.py

@@ -4,6 +4,7 @@ DataTransformer: 简洁的数据格式转换工具
 核心功能:
 - DataTransformer: 数据加载、转换、保存
 - presets: 预设转换模板 (openai_chat, alpaca, sharegpt, dpo_pair, simple_qa)
+- schema: 数据结构验证 (Schema, Field)
 - tokenizers: Token 统计和过滤
 - converters: HuggingFace/OpenAI 等格式转换
 """
@@ -26,6 +27,23 @@ from .converters import (  # LLaMA-Factory 扩展; ms-swift
 )
 from .core import DataTransformer, DictWrapper, TransformError, TransformErrors
 from .presets import get_preset, list_presets
+from .schema import (
+    Field,
+    Schema,
+    ValidationError,
+    ValidationResult,
+    alpaca_schema,
+    dpo_schema,
+    openai_chat_schema,
+    sharegpt_schema,
+    validate_data,
+)
+from .framework import (
+    CompatibilityResult,
+    check_compatibility,
+    detect_format,
+    export_for,
+)
 from .storage import load_data, sample_file, save_data
 from .streaming import StreamingTransformer, load_sharded, load_stream, process_shards
 from .tokenizers import (
@@ -42,7 +60,7 @@ from .tokenizers import (
     token_stats,
 )
 
-__version__ = "0.4.3"
+__version__ = "0.5.2"
 
 __all__ = [
     # core
@@ -53,6 +71,21 @@ __all__ = [
     # presets
     "get_preset",
     "list_presets",
+    # schema
+    "Schema",
+    "Field",
+    "ValidationResult",
+    "ValidationError",
+    "validate_data",
+    "openai_chat_schema",
+    "alpaca_schema",
+    "dpo_schema",
+    "sharegpt_schema",
+    # framework
+    "CompatibilityResult",
+    "check_compatibility",
+    "detect_format",
+    "export_for",
     # storage
     "save_data",
     "load_data",

{dtflow-0.4.3 → dtflow-0.5.2}/dtflow/__main__.py

@@ -18,6 +18,7 @@ Commands:
     clean        数据清洗
     run          执行 Pipeline 配置文件
     history      显示数据血缘历史
+    validate     使用 Schema 验证数据格式
     mcp          MCP 服务管理（install/uninstall/status）
     logs         日志查看工具使用说明
 """
@@ -40,6 +41,7 @@ from .cli.commands import stats as _stats
 from .cli.commands import tail as _tail
 from .cli.commands import token_stats as _token_stats
 from .cli.commands import transform as _transform
+from .cli.commands import validate as _validate
 
 # 创建主应用
 app = typer.Typer(
@@ -211,6 +213,26 @@ def history(
     _history(filename, json)
 
 
+# ============ 验证命令 ============
+
+
+@app.command()
+def validate(
+    filename: str = typer.Argument(..., help="输入文件路径"),
+    preset: Optional[str] = typer.Option(
+        None, "--preset", "-p", help="预设 Schema: openai_chat, alpaca, dpo, sharegpt"
+    ),
+    output: Optional[str] = typer.Option(None, "--output", "-o", help="输出有效数据的文件路径"),
+    filter: bool = typer.Option(
+        False, "--filter", "-f", help="过滤无效数据并保存"
+    ),
+    max_errors: int = typer.Option(20, "--max-errors", help="最多显示的错误数量"),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="显示详细信息"),
+):
+    """使用预设 Schema 验证数据格式"""
+    _validate(filename, preset, output, filter, max_errors, verbose)
+
+
 # ============ 工具命令 ============
 
 

{dtflow-0.4.3 → dtflow-0.5.2}/dtflow/cli/commands.py

@@ -33,6 +33,9 @@ from .pipeline import run
 # 血缘追踪命令
 from .lineage import history
 
+# 验证命令
+from .validate import validate
+
 __all__ = [
     # 采样
     "sample",
@@ -53,4 +56,6 @@ __all__ = [
     "run",
     # 血缘
     "history",
+    # 验证
+    "validate",
 ]

{dtflow-0.4.3 → dtflow-0.5.2}/dtflow/cli/common.py

@@ -57,7 +57,7 @@ def _get_file_row_count(filepath: Path) -> Optional[int]:
     return None
 
 
-def _format_value(value: Any, max_len: int = 80) -> str:
+def _format_value(value: Any, max_len: int = 120) -> str:
     """格式化单个值，长文本截断。"""
     if value is None:
         return "[dim]null[/dim]"
@@ -66,18 +66,22 @@ def _format_value(value: Any, max_len: int = 80) -> str:
     if isinstance(value, (int, float)):
         return f"[cyan]{value}[/cyan]"
     if isinstance(value, str):
+        half_len = max_len // 2
         # 处理多行文本
         if "\n" in value:
             lines = value.split("\n")
-            if len(lines) > 3:
-                preview = lines[0][:max_len] + f"... [dim]({len(lines)} 行)[/dim]"
-            else:
-                preview = value.replace("\n", "\\n")
-                if len(preview) > max_len:
-                    preview = preview[:max_len] + "..."
+            preview = value.replace("\n", "\\n")
+            if len(preview) > max_len:
+                # 前半 + 省略标记 + 后半
+                head = preview[:half_len]
+                tail = preview[-half_len:]
+                return f'"{head} [yellow]<<<{len(lines)}行>>>[/yellow] {tail}"'
             return f'"{preview}"'
         if len(value) > max_len:
-            return f'"{value[:max_len]}..." [dim]({len(value)} 字符)[/dim]'
+            # 前半 + 省略标记 + 后半
+            head = value[:half_len]
+            tail = value[-half_len:]
+            return f'"{head} [yellow]<<<{len(value)}字符>>>[/yellow] {tail}"'
         return f'"{value}"'
     return str(value)
 
@@ -86,7 +90,7 @@ def _format_nested(
     value: Any,
     indent: str = "",
     is_last: bool = True,
-    max_len: int = 80,
+    max_len: int = 120,
 ) -> List[str]:
     """
     递归格式化嵌套结构，返回行列表。