dtflow 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

dtflow/utils/text_parser.py

@@ -0,0 +1,124 @@
+"""
+文本清洗工具
+
+提供 LLM 输出的常见清洗函数：
+- strip_think_tags: 去除 <think>...</think> 思考链内容
+- extract_code_snippets: 提取 ``` 代码块
+- parse_generic_tags: 解析 <tag>content</tag> 格式标签
+"""
+
+import re
+from typing import Dict, List
+
+
+def strip_think_tags(text: str) -> str:
+    """去除 <think>...</think> 包裹的内容
+
+    Args:
+        text: 输入文本
+
+    Returns:
+        去除思考链后的文本
+
+    Examples:
+        >>> strip_think_tags("<think>让我想想...</think>答案是42")
+        '答案是42'
+    """
+    if not text:
+        return text
+    return re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
+
+
+def extract_code_snippets(text: str, strict: bool = True) -> List[Dict[str, str]]:
+    """提取 ``` 代码块
+
+    Args:
+        text: 输入文本
+        strict: True 仅匹配 ```lang...``` 格式，False 额外匹配 {...} 格式
+
+    Returns:
+        代码片段列表，每项为 {"language": ..., "code": ...}
+
+    Examples:
+        >>> extract_code_snippets("```json\\n{\"a\": 1}\\n```")
+        [{'language': 'json', 'code': '{"a": 1}'}]
+    """
+    pattern = r"```(\w+)?\s*([\s\S]*?)```"
+    matches = re.findall(pattern, text)
+
+    code_snippets = []
+    for lang, code in matches:
+        code_snippets.append(
+            {
+                "language": lang.strip() if lang else "unknown",
+                "code": code.strip(),
+            }
+        )
+
+    if not strict:
+        # 移除已匹配的 ``` 块，在剩余文本中匹配 { ... }
+        text = re.sub(pattern, "", text)
+        brace_matches = re.findall(r"\{[\s\S]*?\}", text)
+        for code in brace_matches:
+            code_snippets.append(
+                {
+                    "language": "unknown",
+                    "code": code.strip(),
+                }
+            )
+
+    return code_snippets
+
+
+def parse_generic_tags(text: str, strict: bool = False) -> Dict[str, str]:
+    """解析 XML 风格标签
+
+    支持两种模式：
+    - strict=True: 仅匹配闭合标签 <label>content</label>
+    - strict=False: 同时匹配开放式标签 <label>content，闭合标签优先
+
+    Args:
+        text: 输入文本
+        strict: 是否严格模式
+
+    Returns:
+        {标签名: 内容} 字典
+
+    Examples:
+        >>> parse_generic_tags("<标签>内容</标签>")
+        {'标签': '内容'}
+        >>> parse_generic_tags("<a>hello<b>world", strict=False)
+        {'a': 'hello', 'b': 'world'}
+    """
+    if not text:
+        return {}
+
+    result = {}
+
+    if strict:
+        pattern_closed = r"<([^>]+)>\s*(.*?)\s*</\1>"
+        matches = re.findall(pattern_closed, text, re.DOTALL)
+        for label, content in matches:
+            result[label.strip()] = content.strip()
+    else:
+        remaining_text = str(text)
+
+        # 1. 优先处理闭合标签
+        def process_closed_tag(match_obj):
+            label = match_obj.group(1).strip()
+            content = match_obj.group(2).strip()
+            result[label] = content
+            return ""
+
+        pattern_closed = r"<([^>]+)>\s*(.*?)\s*</\1>"
+        remaining_text = re.sub(pattern_closed, process_closed_tag, remaining_text, flags=re.DOTALL)
+
+        # 2. 在剩余文本中处理开放式标签
+        pattern_open = r"<([^>]+)>\s*(.*?)(?=<[^>]+>|$)"
+        matches_open = re.findall(pattern_open, remaining_text, re.DOTALL)
+        for label, content in matches_open:
+            label_stripped = label.strip()
+            if label_stripped not in result:
+                result[label_stripped] = content.strip()
+
+    return result

{dtflow-0.5.7.dist-info → dtflow-0.5.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.5.7
+Version: 0.5.9
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme
@@ -44,6 +44,7 @@ Requires-Dist: flake8>=3.9.0; extra == 'dev'
 Requires-Dist: huggingface-hub>=0.20.0; extra == 'dev'
 Requires-Dist: isort>=5.9.0; extra == 'dev'
 Requires-Dist: mypy>=0.910; extra == 'dev'
+Requires-Dist: pandas>=1.3.0; extra == 'dev'
 Requires-Dist: pyarrow; extra == 'dev'
 Requires-Dist: pytest-cov>=2.12.0; extra == 'dev'
 Requires-Dist: pytest>=6.0.0; extra == 'dev'
@@ -57,10 +58,14 @@ Provides-Extra: docs
 Requires-Dist: myst-parser>=0.15.0; extra == 'docs'
 Requires-Dist: sphinx-rtd-theme>=0.5.0; extra == 'docs'
 Requires-Dist: sphinx>=4.0.0; extra == 'docs'
+Provides-Extra: eval
+Requires-Dist: pandas>=1.3.0; extra == 'eval'
+Requires-Dist: scikit-learn>=0.24.0; extra == 'eval'
 Provides-Extra: full
 Requires-Dist: datasets>=2.0.0; extra == 'full'
 Requires-Dist: datasketch>=1.5.0; extra == 'full'
 Requires-Dist: huggingface-hub>=0.20.0; extra == 'full'
+Requires-Dist: pandas>=1.3.0; extra == 'full'
 Requires-Dist: pyarrow; extra == 'full'
 Requires-Dist: rich>=10.0.0; extra == 'full'
 Requires-Dist: scikit-learn>=0.24.0; extra == 'full'
@@ -435,6 +440,13 @@ dt sample data.jsonl 1000 --by=messages.#         # 按消息数量分层采样
 dt sample data.jsonl --where="category=tech"      # 筛选后采样
 dt sample data.jsonl --where="messages.#>=2"      # 多条件筛选
 
+# 按行范围查看（Python 切片语法）
+dt slice data.jsonl 10:20                          # 第 10-19 行（0-based，左闭右开）
+dt slice data.jsonl :100                           # 前 100 行
+dt slice data.jsonl 100:                           # 第 100 行到末尾
+dt slice data.jsonl 10:20 -o sliced.jsonl          # 保存到文件
+dt slice data.jsonl 10:20 -f question,answer       # 只显示指定字段
+
 # 数据转换 - 预设模式
 dt transform data.jsonl --preset=openai_chat
 dt transform data.jsonl --preset=alpaca
@@ -468,6 +480,9 @@ dt clean data.jsonl --max-len=messages[-1].content:500  # 最后一条消息最
 dt clean data.jsonl --keep=question,answer          # 只保留这些字段
 dt clean data.jsonl --drop=metadata                 # 删除指定字段
 dt clean data.jsonl --strip                         # 去除字符串首尾空白
+dt clean data.jsonl --min-tokens=content:10          # 最少 10 tokens
+dt clean data.jsonl --max-tokens=content:1000        # 最多 1000 tokens
+dt clean data.jsonl --min-tokens=text:50 -m gpt-4    # 指定分词器
 
 # 数据去重
 dt dedupe data.jsonl                            # 全量精确去重
@@ -476,11 +491,26 @@ dt dedupe data.jsonl --key=meta.id              # 按嵌套字段去重
 dt dedupe data.jsonl --key=messages[0].content  # 按第一条消息内容去重
 dt dedupe data.jsonl --key=text --similar=0.8   # 相似度去重
 
+# 数据集切分
+dt split data.jsonl --ratio=0.8 --seed=42           # 二分: train/test
+dt split data.jsonl --ratio=0.7,0.15,0.15           # 三分: train/val/test
+dt split data.jsonl --ratio=0.8 -o /tmp/output      # 指定输出目录
+
+# 训练框架导出
+dt export data.jsonl --framework=llama-factory       # 导出到 LLaMA-Factory
+dt export data.jsonl -f swift -o ./swift_out         # 导出到 ms-swift
+dt export data.jsonl -f axolotl                      # 导出到 Axolotl
+dt export data.jsonl -f llama-factory --check        # 仅检查兼容性
+
 # 文件拼接
 dt concat a.jsonl b.jsonl -o merged.jsonl
 
 # 数据统计
-dt stats data.jsonl
+dt stats data.jsonl                                       # 快速模式
+dt stats data.jsonl --full                                # 完整模式（含值分布）
+dt stats data.jsonl --full --field=category               # 指定字段统计
+dt stats data.jsonl --full --expand=tags                  # 展开 list 字段统计元素分布
+dt stats data.jsonl --full --expand='messages[*].role'    # 展开嵌套 list 字段
 
 # Claude Code Skill 安装
 dt install-skill                              # 安装到 ~/.claude/skills/
@@ -516,6 +546,8 @@ CLI 命令中的字段参数支持嵌套路径语法，可访问深层嵌套的
 | `clean` | `--drop-empty=` | `--drop-empty=meta.source` |
 | `clean` | `--min-len=` | `--min-len=messages.#:2` |
 | `clean` | `--max-len=` | `--max-len=messages[-1].content:500` |
+| `clean` | `--min-tokens=` | `--min-tokens=content:10` |
+| `clean` | `--max-tokens=` | `--max-tokens=content:1000` |
 | `token-stats` | `--field=` | `--field=messages[-1].content` |
 | `diff` | `--key=` | `--key=meta.uuid` |
 

{dtflow-0.5.7.dist-info → dtflow-0.5.9.dist-info}/RECORD

@@ -1,8 +1,9 @@
-dtflow/SKILL.md,sha256=sHf6i6DKUCca5zvSJ67VHu05tFlST4mYgnoURXVe1g0,7836
-dtflow/__init__.py,sha256=Ee7CDDxzki69MEGeXB5bczuMts5OwZZ-jVsKjH_rD_0,3031
-dtflow/__main__.py,sha256=3LXTku09Fw1dsgTUtX1UJCmE20qKeZpNga3UqmI3UiY,12145
+dtflow/SKILL.md,sha256=hPxJhroGmNbBv8MLZUkOA2yW1TDdUKEUYYlz9tW2mao,10393
+dtflow/__init__.py,sha256=9ZqhqD8qQM9w2dfHKyUWIaqSX-X4elWtbaQN4CNBhgg,3031
+dtflow/__main__.py,sha256=gg3v7u-Ot7AicgKrP1fuyKtMJXVduNuLmhy7L1LUPDg,17710
 dtflow/converters.py,sha256=X3qeFD7FCOMnfiP3MicL5MXimOm4XUYBs5pczIkudU0,22331
 dtflow/core.py,sha256=qMo6B3LK--TWRK7ZBKObGcs3pKFnd0NPoaM0T8JC7Jw,38135
+dtflow/eval.py,sha256=_c-XP2zsOBznYltSyKEScOqvmPVX2orqepg5cNhXXB0,9836
 dtflow/framework.py,sha256=jyICi_RWHjX7WfsXdSbWmP1SL7y1OWSPyd5G5Y-lvg4,17578
 dtflow/lineage.py,sha256=jie3OL1qK90-_cOOqqLbhSJ1oGUktDM1x5HRpQ5Qiyc,12800
 dtflow/pipeline.py,sha256=zZaC4fg5vsp_30Fhbg75vu0yggsdvf28bWBiVDWzZ6Y,13901
@@ -11,15 +12,18 @@ dtflow/schema.py,sha256=IFcij22_UFKcgKT1YWwRg2QJO0vcAvCb1arZmsGByts,16824
 dtflow/streaming.py,sha256=dxpNd1-Wz_PTLTdvM5qn06_2TJr5NRlIIuw0LOSS2Iw,24755
 dtflow/tokenizers.py,sha256=7ZAelSmcDxLWH5kICgH9Q1ULH3_BfDZb9suHMjJJRZU,20589
 dtflow/cli/__init__.py,sha256=QhZ-thgx9IBTFII7T_hdoWFUl0CCsdGQHN5ZEZw2XB0,423
-dtflow/cli/clean.py,sha256=y9VCRibgK1j8WIY3h0XZX0m93EdELQC7TdnseMWwS-0,17799
-dtflow/cli/commands.py,sha256=zKUG-B9Az-spqyqM00cR8Sgc2UgeOPQDThJFHWDNO_w,1336
+dtflow/cli/clean.py,sha256=BEQQlH2q6luCbx51M3oxxOwcnwlOA8vo9WX3Fp7I6AY,29498
+dtflow/cli/commands.py,sha256=LvyDQ_nWUM7UlPDEFQadRdw5O2ZKDLgF41_xAJRhYxI,1583
 dtflow/cli/common.py,sha256=gCwnF5Sw2ploqfZJO_z3Ms9mR1HNT7Lj6ydHn0uVaIw,13817
+dtflow/cli/eval.py,sha256=c53kCRH86k2Q_6vESKFlcepcNnTpO9O68agWK4_oJj8,9582
+dtflow/cli/export.py,sha256=loRfVPwEVsDw3ZMKEYGp0Hy38kYZG2QT8JCMbz1dRzU,2156
 dtflow/cli/io_ops.py,sha256=BMDisP6dxzzmSjYwmeFwaHmpHHPqirmXAWeNTD-9MQM,13254
 dtflow/cli/lineage.py,sha256=_lNh35nF9AA0Zy6FyZ4g8IzrXH2ZQnp3inF-o2Hs1pw,1383
 dtflow/cli/pipeline.py,sha256=QNEo-BJlaC1CVnVeRZr7TwfuZYloJ4TebIzJ5ALzry0,1426
-dtflow/cli/sample.py,sha256=pubpx4AIzsarBEalD150MC2apYQSt4bal70IZkTfFO0,15475
+dtflow/cli/sample.py,sha256=etbro5I0pyNgn0Qfhp1M6Bh-95JN-AntDa5AwVe_oKY,18269
 dtflow/cli/skill.py,sha256=opiTEBejA7JHKrEMftMOPDQlOgZ4n59rwaHXGU1Nukk,2022
-dtflow/cli/stats.py,sha256=u4ehCfgw1X8WuOyAjrApMRgcIO3BVmINbsTjxEscQro,24086
+dtflow/cli/split.py,sha256=96bhWnxHnjIqifoliLgciApkLbwQU8bWHovK8bcMk9g,3667
+dtflow/cli/stats.py,sha256=Jx3d4X0ftgpzU5q5RAWZEVJWwXviQTF4EAwBmz1IliA,31366
 dtflow/cli/transform.py,sha256=w6xqMOxPxQvL2u_BPCfpDHuPSC9gmcqMPVN8s-B6bbY,15052
 dtflow/cli/validate.py,sha256=65aGVlMS_Rq0Ch0YQ-TclVJ03RQP4CnG137wthzb8Ao,4384
 dtflow/storage/__init__.py,sha256=C0jpWNQU808Ezz7lWneddABal3wILy8ijFUNiSKbHV4,362
@@ -28,7 +32,8 @@ dtflow/utils/__init__.py,sha256=Pn-ltwV04fBQmeZG7FxInDQmzH29LYOi90LgeLMEuQk,506
 dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
 dtflow/utils/field_path.py,sha256=K8nU196RxTSJ1OoieTWGcYOWl9KjGq2iSxCAkfjECuM,7621
 dtflow/utils/helpers.py,sha256=JXN176_B2pm53GLVyZ1wj3wrmBJG52Tkw6AMQSdj7M8,791
-dtflow-0.5.7.dist-info/METADATA,sha256=mlWaRHSM1ZucQrAa8PGcHzjHj2RQPBynnmdA_JoNSNI,23899
-dtflow-0.5.7.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dtflow-0.5.7.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
-dtflow-0.5.7.dist-info/RECORD,,
+dtflow/utils/text_parser.py,sha256=0t2TMOSha4dTiDu9H4ygdb67cI20zhtBH1XavDspL_g,3727
+dtflow-0.5.9.dist-info/METADATA,sha256=Pu92Dz2vj7U_dki4A0e5xgka36BTT9K2PnN1LIeEhN0,25839
+dtflow-0.5.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dtflow-0.5.9.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
+dtflow-0.5.9.dist-info/RECORD,,