dtflow 0.4.3__py3-none-any.whl → 0.5.2__py3-none-any.whl

dtflow/tokenizers.py

@@ -210,7 +210,10 @@ def token_counter(
     创建 token 计数转换函数。
 
     Args:
-        fields: 要统计的字段（单个或多个）
+        fields: 要统计的字段（单个或多个），支持嵌套路径语法
+            - 简单字段: "text"
+            - 嵌套字段: "meta.content", "data.text"
+            - 索引: "messages[0].content", "messages[-1].content"
         model: 模型名称或别名，如 "qwen2.5", "gpt-4", "llama3" 等
         backend: 后端选择，None 则自动检测
         output_field: 输出字段名
@@ -221,6 +224,7 @@ def token_counter(
     Examples:
         >>> dt.transform(token_counter("text"))
         >>> dt.transform(token_counter(["question", "answer"], model="qwen3"))
+        >>> dt.transform(token_counter("messages[-1].content"))  # 最后一条消息
     """
     if isinstance(fields, str):
         fields = [fields]
@@ -229,7 +233,7 @@ def token_counter(
         result = item.to_dict() if hasattr(item, "to_dict") else dict(item)
         total = 0
         for field in fields:
-            value = item.get(field, "") if hasattr(item, "get") else item[field]
+            value = get_field_with_spec(item, field, default="")
             if value:
                 total += count_tokens(str(value), model=model, backend=backend)
         result[output_field] = total
@@ -249,7 +253,10 @@ def token_filter(
     创建基于 token 长度的过滤函数。
 
     Args:
-        fields: 要统计的字段（单个或多个）
+        fields: 要统计的字段（单个或多个），支持嵌套路径语法
+            - 简单字段: "text"
+            - 嵌套字段: "meta.content", "data.text"
+            - 索引: "messages[0].content", "messages[-1].content"
         min_tokens: 最小 token 数（包含）
         max_tokens: 最大 token 数（包含）
         model: 模型名称
@@ -261,6 +268,7 @@ def token_filter(
     Examples:
         >>> dt.filter(token_filter("text", min_tokens=10, max_tokens=512))
         >>> dt.filter(token_filter(["q", "a"], max_tokens=2048))
+        >>> dt.filter(token_filter("messages[-1].content", max_tokens=1024))
     """
     if isinstance(fields, str):
         fields = [fields]
@@ -268,7 +276,7 @@ def token_filter(
     def filter_func(item) -> bool:
         total = 0
         for field in fields:
-            value = item.get(field, "") if hasattr(item, "get") else item[field]
+            value = get_field_with_spec(item, field, default="")
             if value:
                 total += count_tokens(str(value), model=model, backend=backend)
 
@@ -281,11 +289,32 @@ def token_filter(
     return filter_func
 
 
+def _percentile(sorted_data: List[int], p: float) -> int:
+    """计算百分位数"""
+    n = len(sorted_data)
+    if n == 0:
+        return 0
+    idx = (n - 1) * p / 100
+    lower = int(idx)
+    upper = min(lower + 1, n - 1)
+    weight = idx - lower
+    return int(sorted_data[lower] * (1 - weight) + sorted_data[upper] * weight)
+
+
+def _std(counts: List[int], avg: float) -> float:
+    """计算标准差"""
+    if len(counts) < 2:
+        return 0.0
+    variance = sum((x - avg) ** 2 for x in counts) / len(counts)
+    return variance**0.5
+
+
 def token_stats(
     data: List[Dict[str, Any]],
     fields: Union[str, List[str]],
     model: str = DEFAULT_MODEL,
     backend: Optional[str] = None,
+    progress_callback: Optional[Callable[[int, int], None]] = None,
 ) -> Dict[str, Any]:
     """
     统计数据集的 token 信息。
@@ -295,9 +324,17 @@ def token_stats(
         fields: 要统计的字段，支持嵌套路径语法（如 meta.text, messages[-1].content）
         model: 模型名称或别名，如 "qwen2.5", "gpt-4" 等
         backend: 后端选择，None 则自动检测
+        progress_callback: 进度回调函数，接收 (current, total) 两个参数
 
     Returns:
-        统计信息字典
+        统计信息字典，包含:
+        - total_tokens: 总 token 数
+        - count: 样本数
+        - avg_tokens: 平均 token 数
+        - std_tokens: 标准差
+        - min_tokens, max_tokens: 最小/最大值
+        - median_tokens: 中位数 (p50)
+        - p25, p75, p90, p95, p99: 百分位数
     """
     if isinstance(fields, str):
         fields = [fields]
@@ -306,21 +343,33 @@ def token_stats(
         return {"total_tokens": 0, "count": 0}
 
     counts = []
-    for item in data:
+    total_items = len(data)
+    for i, item in enumerate(data):
         total = 0
         for field in fields:
             value = get_field_with_spec(item, field, default="")
             if value:
                 total += count_tokens(str(value), model=model, backend=backend)
         counts.append(total)
+        if progress_callback:
+            progress_callback(i + 1, total_items)
+
+    sorted_counts = sorted(counts)
+    avg = sum(counts) / len(counts)
 
     return {
         "total_tokens": sum(counts),
         "count": len(counts),
-        "avg_tokens": sum(counts) / len(counts),
+        "avg_tokens": avg,
+        "std_tokens": _std(counts, avg),
         "min_tokens": min(counts),
         "max_tokens": max(counts),
-        "median_tokens": sorted(counts)[len(counts) // 2],
+        "median_tokens": _percentile(sorted_counts, 50),
+        "p25": _percentile(sorted_counts, 25),
+        "p75": _percentile(sorted_counts, 75),
+        "p90": _percentile(sorted_counts, 90),
+        "p95": _percentile(sorted_counts, 95),
+        "p99": _percentile(sorted_counts, 99),
     }
 
 
@@ -504,6 +553,7 @@ def messages_token_stats(
     messages_field: str = "messages",
     model: str = DEFAULT_MODEL,
     backend: Optional[str] = None,
+    progress_callback: Optional[Callable[[int, int], None]] = None,
 ) -> Dict[str, Any]:
     """
     统计数据集中 messages 的 token 信息。
@@ -513,25 +563,18 @@ def messages_token_stats(
         messages_field: messages 字段名，支持嵌套路径语法（如 conversation.messages）
         model: 模型名称或别名
         backend: 后端，None 则自动检测
+        progress_callback: 进度回调函数，接收 (current, total) 两个参数
 
     Returns:
-        统计信息字典
-
-    Examples:
-        >>> stats = messages_token_stats(dt.data)  # 使用默认 qwen2.5
-        >>> stats = messages_token_stats(dt.data, model="qwen3")
-        >>> print(stats)
-        {
-            "count": 1000,
-            "total_tokens": 500000,
-            "user_tokens": 200000,
-            "assistant_tokens": 290000,
-            "system_tokens": 10000,
-            "avg_tokens": 500,
-            "max_tokens": 2048,
-            "min_tokens": 50,
-            "avg_turns": 4,
-        }
+        统计信息字典，包含:
+        - count: 样本数
+        - total_tokens: 总 token 数
+        - user_tokens, assistant_tokens, system_tokens: 各角色 token 数
+        - avg_tokens, std_tokens: 平均值和标准差
+        - min_tokens, max_tokens: 最小/最大值
+        - median_tokens: 中位数
+        - p25, p75, p90, p95, p99: 百分位数
+        - avg_turns: 平均对话轮数
     """
     _backend = backend or _auto_backend(model)
 
@@ -539,24 +582,36 @@ def messages_token_stats(
         return {"count": 0, "total_tokens": 0}
 
     all_stats = []
-    for item in data:
+    total_items = len(data)
+    for i, item in enumerate(data):
         messages = get_field_with_spec(item, messages_field, default=[])
         if messages:
             all_stats.append(_count_messages_tokens(messages, model=model, backend=_backend))
+        if progress_callback:
+            progress_callback(i + 1, total_items)
 
     if not all_stats:
         return {"count": 0, "total_tokens": 0}
 
     totals = [s["total"] for s in all_stats]
+    sorted_totals = sorted(totals)
+    avg = sum(totals) / len(totals)
+
     return {
         "count": len(all_stats),
         "total_tokens": sum(totals),
         "user_tokens": sum(s["user"] for s in all_stats),
         "assistant_tokens": sum(s["assistant"] for s in all_stats),
         "system_tokens": sum(s["system"] for s in all_stats),
-        "avg_tokens": sum(totals) // len(totals),
-        "max_tokens": max(totals),
+        "avg_tokens": int(avg),
+        "std_tokens": _std(totals, avg),
         "min_tokens": min(totals),
-        "median_tokens": sorted(totals)[len(totals) // 2],
+        "max_tokens": max(totals),
+        "median_tokens": _percentile(sorted_totals, 50),
+        "p25": _percentile(sorted_totals, 25),
+        "p75": _percentile(sorted_totals, 75),
+        "p90": _percentile(sorted_totals, 90),
+        "p95": _percentile(sorted_totals, 95),
+        "p99": _percentile(sorted_totals, 99),
         "avg_turns": sum(s["turns"] for s in all_stats) // len(all_stats),
     }

dtflow/utils/field_path.py

@@ -96,7 +96,9 @@ def _parse_path(path: str) -> List[Union[str, int, Literal["*", "#"]]]:
             continue
 
         # 解析 field[index] 格式
-        match = re.match(r"([a-zA-Z_\u4e00-\u9fff][a-zA-Z0-9_\u4e00-\u9fff]*)?(?:\[(-?\d+|\*)\])?", part)
+        match = re.match(
+            r"([a-zA-Z_\u4e00-\u9fff][a-zA-Z0-9_\u4e00-\u9fff]*)?(?:\[(-?\d+|\*)\])?", part
+        )
         if match:
             field_name, index = match.groups()
 
@@ -175,10 +177,12 @@ def _get_value_by_segments(
 
             return values
 
-        # 字典字段访问
+        # 字典字段访问（支持 dict 和类 dict 对象如 DictWrapper）
         if isinstance(seg, str):
             if isinstance(current, dict):
                 current = current.get(seg)
+            elif hasattr(current, "get"):
+                current = current.get(seg)
             else:
                 return None
 

{dtflow-0.4.3.dist-info → dtflow-0.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.4.3
+Version: 0.5.2
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme
@@ -32,16 +32,26 @@ Requires-Dist: orjson>=3.9.0
 Requires-Dist: polars>=0.20.0
 Requires-Dist: pyyaml>=5.4.0
 Requires-Dist: rich>=10.0.0
+Requires-Dist: tiktoken>=0.5.0
 Requires-Dist: typer>=0.9.0
 Provides-Extra: converters
 Requires-Dist: datasets>=2.0.0; extra == 'converters'
 Provides-Extra: dev
 Requires-Dist: black>=21.0; extra == 'dev'
+Requires-Dist: datasets>=2.0.0; extra == 'dev'
+Requires-Dist: datasketch>=1.5.0; extra == 'dev'
 Requires-Dist: flake8>=3.9.0; extra == 'dev'
+Requires-Dist: huggingface-hub>=0.20.0; extra == 'dev'
 Requires-Dist: isort>=5.9.0; extra == 'dev'
 Requires-Dist: mypy>=0.910; extra == 'dev'
+Requires-Dist: pyarrow; extra == 'dev'
 Requires-Dist: pytest-cov>=2.12.0; extra == 'dev'
 Requires-Dist: pytest>=6.0.0; extra == 'dev'
+Requires-Dist: rich>=10.0.0; extra == 'dev'
+Requires-Dist: scikit-learn>=0.24.0; extra == 'dev'
+Requires-Dist: tiktoken>=0.5.0; extra == 'dev'
+Requires-Dist: tokenizers>=0.15.0; extra == 'dev'
+Requires-Dist: toolong>=1.5.0; extra == 'dev'
 Provides-Extra: display
 Provides-Extra: docs
 Requires-Dist: myst-parser>=0.15.0; extra == 'docs'
@@ -129,7 +139,7 @@ dt.filter(lambda x: x.language == "zh")
 ### 数据验证
 
 ```python
-# 验证数据，返回不通过的记录列表
+# 简单验证，返回不通过的记录列表
 errors = dt.validate(lambda x: len(x.messages) >= 2)
 
 if errors:
@@ -137,6 +147,53 @@ if errors:
         print(f"第 {e.index} 行: {e.error}")
 ```
 
+### Schema 验证
+
+使用 Schema 进行结构化数据验证：
+
+```python
+from dtflow import Schema, Field, openai_chat_schema
+
+# 使用预设 Schema
+result = dt.validate_schema(openai_chat_schema)
+print(result)  # ValidationResult(valid=950, invalid=50, errors=[...])
+
+# 自定义 Schema
+schema = Schema({
+    "messages": Field(type="list", required=True, min_length=1),
+    "messages[*].role": Field(type="str", choices=["user", "assistant", "system"]),
+    "messages[*].content": Field(type="str", min_length=1),
+    "score": Field(type="float", min=0, max=1),
+})
+
+result = dt.validate_schema(schema)
+
+# 过滤出有效数据
+valid_dt = dt.validate_schema(schema, filter_invalid=True)
+valid_dt.save("valid.jsonl")
+```
+
+**预设 Schema**：
+
+| Schema 名称 | 用途 |
+|------------|------|
+| `openai_chat_schema` | OpenAI messages 格式验证 |
+| `alpaca_schema` | Alpaca instruction/output 格式 |
+| `sharegpt_schema` | ShareGPT conversations 格式 |
+| `dpo_schema` | DPO prompt/chosen/rejected 格式 |
+
+**Field 参数**：
+
+| 参数 | 说明 | 示例 |
+|------|------|------|
+| `type` | 类型验证 | `"str"`, `"int"`, `"float"`, `"bool"`, `"list"`, `"dict"` |
+| `required` | 是否必填 | `True` / `False` |
+| `min` / `max` | 数值范围 | `min=0, max=1` |
+| `min_length` / `max_length` | 长度范围 | `min_length=1` |
+| `choices` | 枚举值 | `choices=["user", "assistant"]` |
+| `pattern` | 正则匹配 | `pattern=r"^\d{4}-\d{2}-\d{2}$"` |
+| `custom` | 自定义验证 | `custom=lambda x: x > 0` |
+
 ### 数据转换
 
 ```python
@@ -286,6 +343,58 @@ dt.transform(to_swift_vlm(images_field="images")).save("swift_vlm.jsonl")
 # 输出: {"messages": [...], "images": ["/path/to/img.jpg"]}
 ```
 
+### 训练框架一键导出
+
+将数据导出为目标训练框架可直接使用的格式，自动生成配置文件：
+
+```python
+from dtflow import DataTransformer
+
+dt = DataTransformer.load("data.jsonl")
+
+# 1. 检查框架兼容性
+result = dt.check_compatibility("llama-factory")
+print(result)
+# ✅ 兼容 - LLaMA-Factory (openai_chat)
+# 或
+# ❌ 不兼容 - 错误: xxx
+
+# 2. 一键导出到 LLaMA-Factory
+files = dt.export_for("llama-factory", "./llama_ready/")
+# 生成文件:
+# - ./llama_ready/custom_dataset.json      # 数据文件
+# - ./llama_ready/dataset_info.json        # 数据集配置
+# - ./llama_ready/train_args.yaml          # 训练参数模板
+
+# 3. 导出到 ms-swift
+files = dt.export_for("swift", "./swift_ready/")
+# 生成: data.jsonl + train_swift.sh
+
+# 4. 导出到 Axolotl
+files = dt.export_for("axolotl", "./axolotl_ready/")
+# 生成: data.jsonl + config.yaml
+
+# 指定数据集名称
+dt.export_for("llama-factory", "./output/", dataset_name="my_sft_data")
+```
+
+**支持的框架**：
+
+| 框架 | 导出内容 | 使用方式 |
+|------|---------|---------|
+| `llama-factory` | data.json + dataset_info.json + train_args.yaml | `llamafactory-cli train train_args.yaml` |
+| `swift` | data.jsonl + train_swift.sh | `bash train_swift.sh` |
+| `axolotl` | data.jsonl + config.yaml | `accelerate launch -m axolotl.cli.train config.yaml` |
+
+**自动格式检测**：
+
+| 检测到的格式 | 数据结构 |
+|------------|---------|
+| `openai_chat` | `{"messages": [{"role": "user", ...}]}` |
+| `alpaca` | `{"instruction": ..., "output": ...}` |
+| `sharegpt` | `{"conversations": [{"from": "human", ...}]}` |
+| `dpo` | `{"prompt": ..., "chosen": ..., "rejected": ...}` |
+
 ### 其他操作
 
 ```python
@@ -361,6 +470,12 @@ dt concat a.jsonl b.jsonl -o merged.jsonl
 
 # 数据统计
 dt stats data.jsonl
+
+# 数据验证
+dt validate data.jsonl --preset=openai_chat           # 使用预设 schema 验证
+dt validate data.jsonl --preset=alpaca --verbose      # 详细输出
+dt validate data.jsonl --preset=sharegpt --filter-invalid -o valid.jsonl  # 过滤出有效数据
+dt validate data.jsonl --preset=dpo --max-errors=100  # 限制错误输出数量
 ```
 
 ### 字段路径语法

{dtflow-0.4.3.dist-info → dtflow-0.5.2.dist-info}/RECORD

@@ -1,22 +1,25 @@
-dtflow/__init__.py,sha256=F5fBna3PxmrOK34SnZxvolzmyi_pjIxNeDkNzGg8wvA,2347
-dtflow/__main__.py,sha256=vuX2--_gXI25vy-Xb1uqhURFzu05VeMUYxIJ2q-XE7M,11656
+dtflow/__init__.py,sha256=PTqh_6-F6eEwg1RxQ0ueP6CYnZauMuqYhlZe2BJphr0,3031
+dtflow/__main__.py,sha256=ySpqvEn7k-vsrYFPx-8O6p-yx_24KccgnOSPd2XybhM,12572
 dtflow/converters.py,sha256=gyy-K15zjzGBawFnZa8D9JX37JZ47rey2GhjKa2pxFo,22081
-dtflow/core.py,sha256=5XivbEdcKMrj1wSfju6MDkqdCqkO_mRS-ALWJ3DOcKo,29937
-dtflow/lineage.py,sha256=vQ06lxBHftu-Ma5HlISp3F2eiIvwagQSnUGaLeABDZY,12190
+dtflow/core.py,sha256=qMo6B3LK--TWRK7ZBKObGcs3pKFnd0NPoaM0T8JC7Jw,38135
+dtflow/framework.py,sha256=jyICi_RWHjX7WfsXdSbWmP1SL7y1OWSPyd5G5Y-lvg4,17578
+dtflow/lineage.py,sha256=jie3OL1qK90-_cOOqqLbhSJ1oGUktDM1x5HRpQ5Qiyc,12800
 dtflow/pipeline.py,sha256=zZaC4fg5vsp_30Fhbg75vu0yggsdvf28bWBiVDWzZ6Y,13901
 dtflow/presets.py,sha256=OP1nnM5NFk5Kli9FsXK0xAot48E5OQ6-VOIJT9ffXPg,5023
-dtflow/streaming.py,sha256=jtWQjkhhZqfyzIaFskXNvooGAYDQBn1b6X8FHgaCZYk,22704
-dtflow/tokenizers.py,sha256=zxE6XZGjZ_DOGCjRSClI9xaAbFVf8FS6jwwssGoi_9U,18111
+dtflow/schema.py,sha256=IFcij22_UFKcgKT1YWwRg2QJO0vcAvCb1arZmsGByts,16824
+dtflow/streaming.py,sha256=dxpNd1-Wz_PTLTdvM5qn06_2TJr5NRlIIuw0LOSS2Iw,24755
+dtflow/tokenizers.py,sha256=7ZAelSmcDxLWH5kICgH9Q1ULH3_BfDZb9suHMjJJRZU,20589
 dtflow/cli/__init__.py,sha256=QhZ-thgx9IBTFII7T_hdoWFUl0CCsdGQHN5ZEZw2XB0,423
 dtflow/cli/clean.py,sha256=y9VCRibgK1j8WIY3h0XZX0m93EdELQC7TdnseMWwS-0,17799
-dtflow/cli/commands.py,sha256=ExcD8Z_uXQhcewvgcPtIlPzsQG4QF93K8Bg6C3uUJHk,1094
-dtflow/cli/common.py,sha256=FsDFVNcLj_874qSg2dGef4V7mqPU9THLchT8PxJpBt8,12955
+dtflow/cli/commands.py,sha256=ST65Ox_MKu-CKAtPVaxECAPXYOJiF7BhL32A4nsZZl0,1175
+dtflow/cli/common.py,sha256=nIPc9GBK61r6kmaI9OS3IyhcfPqShpDEHx1ddjFPnlM,13131
 dtflow/cli/io_ops.py,sha256=BMDisP6dxzzmSjYwmeFwaHmpHHPqirmXAWeNTD-9MQM,13254
 dtflow/cli/lineage.py,sha256=_lNh35nF9AA0Zy6FyZ4g8IzrXH2ZQnp3inF-o2Hs1pw,1383
 dtflow/cli/pipeline.py,sha256=QNEo-BJlaC1CVnVeRZr7TwfuZYloJ4TebIzJ5ALzry0,1426
 dtflow/cli/sample.py,sha256=vPTQlF0OXEry4QjO8uaD9vOae4AQbX9zDwVYOxg59ZI,10339
-dtflow/cli/stats.py,sha256=HByF0sFMqY1kM75dnjTcJbMKDdQNdOt4iDba4au_-pI,20495
+dtflow/cli/stats.py,sha256=u4ehCfgw1X8WuOyAjrApMRgcIO3BVmINbsTjxEscQro,24086
 dtflow/cli/transform.py,sha256=w6xqMOxPxQvL2u_BPCfpDHuPSC9gmcqMPVN8s-B6bbY,15052
+dtflow/cli/validate.py,sha256=65aGVlMS_Rq0Ch0YQ-TclVJ03RQP4CnG137wthzb8Ao,4384
 dtflow/mcp/__init__.py,sha256=huEJ3rXDbxDRjsLPEvjNT2u3tWs6Poiv6fokPIrByjw,897
 dtflow/mcp/__main__.py,sha256=PoT2ZZmJq9xDZxDACJfqDW9Ld_ukHrGNK-0XUd7WGnY,448
 dtflow/mcp/cli.py,sha256=ck0oOS_642cNktxULaMRE7BJfMxsBCwotmCj3PSPwVk,13110
@@ -26,8 +29,8 @@ dtflow/storage/__init__.py,sha256=C0jpWNQU808Ezz7lWneddABal3wILy8ijFUNiSKbHV4,36
 dtflow/storage/io.py,sha256=ZH2aSE-S89gpy3z4oTqhcqWf4u10OdkDoyul7o_YBDI,23374
 dtflow/utils/__init__.py,sha256=f8v9HJZMWRI5AL64Vjr76Pf2Na_whOF9nJBKgPbXXYg,429
 dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
-dtflow/utils/field_path.py,sha256=WcNA-LZh3H61a77FEzB_R7YAyyZl3M8ofdq05ytQGmI,7459
-dtflow-0.4.3.dist-info/METADATA,sha256=LdjtTbPEuHlqxthUuMX2lqrj2l2kBIfVf-u6jDZmCeI,18524
-dtflow-0.4.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dtflow-0.4.3.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
-dtflow-0.4.3.dist-info/RECORD,,
+dtflow/utils/field_path.py,sha256=K8nU196RxTSJ1OoieTWGcYOWl9KjGq2iSxCAkfjECuM,7621
+dtflow-0.5.2.dist-info/METADATA,sha256=RlpGaySrAIgTviom_Wyn6o2LWzQQVihff12Jpazy10o,22544
+dtflow-0.5.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dtflow-0.5.2.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
+dtflow-0.5.2.dist-info/RECORD,,