dtflow 0.5.6__py3-none-any.whl → 0.5.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtflow/SKILL.md +225 -0
- dtflow/__init__.py +1 -1
- dtflow/__main__.py +31 -51
- dtflow/cli/commands.py +16 -10
- dtflow/cli/skill.py +72 -0
- {dtflow-0.5.6.dist-info → dtflow-0.5.7.dist-info}/METADATA +40 -3
- {dtflow-0.5.6.dist-info → dtflow-0.5.7.dist-info}/RECORD +9 -12
- dtflow/mcp/__init__.py +0 -29
- dtflow/mcp/__main__.py +0 -18
- dtflow/mcp/cli.py +0 -388
- dtflow/mcp/docs.py +0 -416
- dtflow/mcp/server.py +0 -153
- {dtflow-0.5.6.dist-info → dtflow-0.5.7.dist-info}/WHEEL +0 -0
- {dtflow-0.5.6.dist-info → dtflow-0.5.7.dist-info}/entry_points.txt +0 -0
dtflow/SKILL.md
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: dtflow
|
|
3
|
+
description: 数据文件处理(JSONL/CSV/Parquet)- 去重/采样/统计/过滤/转换/Schema验证/训练框架导出
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# dtflow - 机器学习训练数据格式转换工具
|
|
7
|
+
|
|
8
|
+
## 设计理念
|
|
9
|
+
|
|
10
|
+
- **函数式优于类继承**:直接用 lambda/函数做转换,不需要 OOP 抽象
|
|
11
|
+
- **KISS 原则**:一个 `DataTransformer` 类搞定所有操作
|
|
12
|
+
- **链式 API**:`dt.filter(...).to(...).save(...)`
|
|
13
|
+
|
|
14
|
+
## Python API
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
from dtflow import DataTransformer
|
|
18
|
+
|
|
19
|
+
# 加载数据(支持 JSONL/JSON/CSV/Parquet/Arrow,使用 Polars 引擎)
|
|
20
|
+
dt = DataTransformer.load("data.jsonl")
|
|
21
|
+
|
|
22
|
+
# 链式操作
|
|
23
|
+
(dt.filter(lambda x: x.score > 0.8)
|
|
24
|
+
.to(lambda x: {"q": x.question, "a": x.answer})
|
|
25
|
+
.dedupe("text")
|
|
26
|
+
.save("output.jsonl"))
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### 数据过滤
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
dt.filter(lambda x: x.score > 0.8)
|
|
33
|
+
dt.filter(lambda x: x.language == "zh")
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### 数据验证
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
# 简单验证
|
|
40
|
+
errors = dt.validate(lambda x: len(x.messages) >= 2)
|
|
41
|
+
|
|
42
|
+
# Schema 验证
|
|
43
|
+
from dtflow import Schema, Field, openai_chat_schema
|
|
44
|
+
|
|
45
|
+
result = dt.validate_schema(openai_chat_schema) # 预设 Schema
|
|
46
|
+
valid_dt = dt.validate_schema(schema, filter_invalid=True) # 过滤无效数据
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
**预设 Schema**:`openai_chat_schema`、`alpaca_schema`、`sharegpt_schema`、`dpo_schema`
|
|
50
|
+
|
|
51
|
+
### 数据转换
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
# 自定义转换
|
|
55
|
+
dt.to(lambda x: {"question": x.q, "answer": x.a})
|
|
56
|
+
|
|
57
|
+
# 使用预设模板
|
|
58
|
+
dt.to(preset="openai_chat", user_field="q", assistant_field="a")
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
**预设模板**:`openai_chat`、`alpaca`、`sharegpt`、`dpo_pair`、`simple_qa`
|
|
62
|
+
|
|
63
|
+
### Token 统计
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from dtflow import count_tokens, token_counter, token_filter, token_stats
|
|
67
|
+
|
|
68
|
+
count = count_tokens("Hello world", model="gpt-4")
|
|
69
|
+
dt.transform(token_counter("text")).save("with_tokens.jsonl")
|
|
70
|
+
dt.filter(token_filter("text", max_tokens=2048))
|
|
71
|
+
|
|
72
|
+
# Messages Token 统计(多轮对话)
|
|
73
|
+
from dtflow import messages_token_counter, messages_token_filter
|
|
74
|
+
dt.transform(messages_token_counter(model="gpt-4", detailed=True))
|
|
75
|
+
dt.filter(messages_token_filter(min_turns=2, max_turns=10))
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
### 格式转换器
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from dtflow import (
|
|
82
|
+
to_hf_dataset, from_hf_dataset, # HuggingFace Dataset
|
|
83
|
+
to_openai_batch, from_openai_batch, # OpenAI Batch API
|
|
84
|
+
to_llama_factory, to_llama_factory_sharegpt, # LLaMA-Factory
|
|
85
|
+
to_swift_messages, to_swift_query_response, # ms-swift
|
|
86
|
+
messages_to_text, # messages 转纯文本
|
|
87
|
+
)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### 训练框架导出
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
# 检查兼容性
|
|
94
|
+
result = dt.check_compatibility("llama-factory")
|
|
95
|
+
|
|
96
|
+
# 一键导出
|
|
97
|
+
files = dt.export_for("llama-factory", "./output/") # 生成 data.json + dataset_info.json + train_args.yaml
|
|
98
|
+
files = dt.export_for("swift", "./output/") # 生成 data.jsonl + train_swift.sh
|
|
99
|
+
files = dt.export_for("axolotl", "./output/") # 生成 data.jsonl + config.yaml
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### 大文件流式处理
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from dtflow import load_stream, load_sharded
|
|
106
|
+
|
|
107
|
+
# O(1) 内存,100GB 文件也能处理
|
|
108
|
+
(load_stream("huge.jsonl")
|
|
109
|
+
.filter(lambda x: x["score"] > 0.5)
|
|
110
|
+
.save("output.jsonl"))
|
|
111
|
+
|
|
112
|
+
# 分片文件加载
|
|
113
|
+
(load_sharded("data/train_*.parquet")
|
|
114
|
+
.filter(lambda x: len(x["text"]) > 10)
|
|
115
|
+
.save("merged.jsonl"))
|
|
116
|
+
|
|
117
|
+
# 分片保存
|
|
118
|
+
load_stream("huge.jsonl").save_sharded("output/", shard_size=100000)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### 其他操作
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
dt.sample(100) # 随机采样
|
|
125
|
+
dt.head(10) / dt.tail(10) # 取前/后 N 条
|
|
126
|
+
train, test = dt.split(ratio=0.8) # 分割
|
|
127
|
+
dt.shuffle(seed=42) # 打乱
|
|
128
|
+
dt.stats() # 统计
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## CLI 命令
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
# 统计(推荐首先使用)
|
|
135
|
+
dt stats data.jsonl # 基本统计(文件大小、条数、字段)
|
|
136
|
+
dt stats data.jsonl --full # 完整模式:值分布、唯一值、非空率
|
|
137
|
+
dt stats data.jsonl --full -n 20 # 显示 Top 20 值分布
|
|
138
|
+
|
|
139
|
+
# Token 统计
|
|
140
|
+
dt token-stats data.jsonl # 默认统计 messages 字段
|
|
141
|
+
dt token-stats data.jsonl -f text # 指定统计字段
|
|
142
|
+
dt token-stats data.jsonl -m qwen2.5 # 指定分词器 (cl100k_base/qwen2.5/llama3)
|
|
143
|
+
dt token-stats data.jsonl --detailed # 显示详细统计
|
|
144
|
+
|
|
145
|
+
# 采样(支持字段路径语法)
|
|
146
|
+
dt sample data.jsonl 100 # 随机采样 100 条
|
|
147
|
+
dt sample data.jsonl 100 -t head # 取前 100 条 (head/tail/random)
|
|
148
|
+
dt sample data.jsonl 1000 --by=category # 分层采样
|
|
149
|
+
dt sample data.jsonl 1000 --by=category --uniform # 均匀分层采样
|
|
150
|
+
dt sample data.jsonl --where="messages.#>=2" # 条件筛选
|
|
151
|
+
dt sample data.jsonl 10 -f input,output # 只显示指定字段
|
|
152
|
+
dt sample data.jsonl 10 --raw # 输出原始 JSON(不截断)
|
|
153
|
+
dt sample data.jsonl 100 --seed=42 -o out.jsonl # 固定随机种子并保存
|
|
154
|
+
|
|
155
|
+
# 去重
|
|
156
|
+
dt dedupe data.jsonl --key=text # 精确去重
|
|
157
|
+
dt dedupe data.jsonl --key=meta.id # 按嵌套字段去重
|
|
158
|
+
dt dedupe data.jsonl --key=text --similar=0.8 # 相似度去重
|
|
159
|
+
dt dedupe data.jsonl --key=text -o deduped.jsonl # 指定输出文件
|
|
160
|
+
|
|
161
|
+
# 清洗
|
|
162
|
+
dt clean data.jsonl --drop-empty=text,answer # 删除空值记录
|
|
163
|
+
dt clean data.jsonl --min-len=text:10 # 最小长度过滤
|
|
164
|
+
dt clean data.jsonl --max-len=text:2000 # 最大长度过滤
|
|
165
|
+
dt clean data.jsonl --min-len=messages.#:2 # 最少 2 条消息
|
|
166
|
+
dt clean data.jsonl --keep=question,answer # 只保留指定字段
|
|
167
|
+
dt clean data.jsonl --drop=metadata # 删除指定字段
|
|
168
|
+
dt clean data.jsonl --strip # 去除字符串首尾空白
|
|
169
|
+
dt clean data.jsonl --strip --drop-empty=input -o cleaned.jsonl # 组合使用
|
|
170
|
+
|
|
171
|
+
# 验证
|
|
172
|
+
dt validate data.jsonl --preset=openai_chat # 预设: openai_chat/alpaca/dpo/sharegpt
|
|
173
|
+
dt validate data.jsonl -p alpaca -f -o valid.jsonl # 过滤无效数据并保存
|
|
174
|
+
dt validate data.jsonl -p openai_chat -v # 显示详细信息
|
|
175
|
+
dt validate data.jsonl -p openai_chat --max-errors=50 # 最多显示 50 条错误
|
|
176
|
+
|
|
177
|
+
# 转换
|
|
178
|
+
dt transform data.jsonl --preset=openai_chat
|
|
179
|
+
dt transform data.jsonl # 交互式生成配置文件
|
|
180
|
+
|
|
181
|
+
# 合并与对比
|
|
182
|
+
dt concat a.jsonl b.jsonl -o merged.jsonl # 合并文件
|
|
183
|
+
dt concat a.jsonl b.jsonl -o merged.jsonl --strict # 严格模式(字段必须一致)
|
|
184
|
+
dt diff a.jsonl b.jsonl --key=id # 对比差异
|
|
185
|
+
dt diff a.jsonl b.jsonl --key=id -o report.md # 输出对比报告
|
|
186
|
+
|
|
187
|
+
# 查看数据
|
|
188
|
+
dt head data.jsonl 10 # 前 10 条
|
|
189
|
+
dt head data.jsonl 10 -f input,output # 只显示指定字段
|
|
190
|
+
dt head data.jsonl 10 --raw # 输出完整 JSON(不截断)
|
|
191
|
+
dt tail data.jsonl 10 # 后 10 条
|
|
192
|
+
|
|
193
|
+
# 其他
|
|
194
|
+
dt run pipeline.yaml # Pipeline 执行
|
|
195
|
+
dt history processed.jsonl # 数据血缘
|
|
196
|
+
dt install-skill # 安装 Claude Code skill
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
## 字段路径语法
|
|
200
|
+
|
|
201
|
+
| 语法 | 含义 | 示例 |
|
|
202
|
+
|------|------|------|
|
|
203
|
+
| `a.b.c` | 嵌套字段 | `meta.source` |
|
|
204
|
+
| `a[0].b` | 数组索引 | `messages[0].role` |
|
|
205
|
+
| `a[-1].b` | 负索引 | `messages[-1].content` |
|
|
206
|
+
| `a.#` | 数组长度 | `messages.#` |
|
|
207
|
+
| `a[*].b` | 展开所有元素 | `messages[*].role` |
|
|
208
|
+
|
|
209
|
+
## Pipeline 配置
|
|
210
|
+
|
|
211
|
+
```yaml
|
|
212
|
+
# pipeline.yaml
|
|
213
|
+
version: "1.0"
|
|
214
|
+
seed: 42
|
|
215
|
+
input: raw_data.jsonl
|
|
216
|
+
output: processed.jsonl
|
|
217
|
+
|
|
218
|
+
steps:
|
|
219
|
+
- type: filter
|
|
220
|
+
condition: "score > 0.5"
|
|
221
|
+
- type: transform
|
|
222
|
+
preset: openai_chat
|
|
223
|
+
- type: dedupe
|
|
224
|
+
key: text
|
|
225
|
+
```
|
dtflow/__init__.py
CHANGED
dtflow/__main__.py
CHANGED
|
@@ -6,21 +6,21 @@ Usage:
|
|
|
6
6
|
dt --install-completion # 安装 shell 自动补全
|
|
7
7
|
|
|
8
8
|
Commands:
|
|
9
|
-
sample
|
|
10
|
-
head
|
|
11
|
-
tail
|
|
12
|
-
transform
|
|
13
|
-
stats
|
|
14
|
-
token-stats
|
|
15
|
-
diff
|
|
16
|
-
dedupe
|
|
17
|
-
concat
|
|
18
|
-
clean
|
|
19
|
-
run
|
|
20
|
-
history
|
|
21
|
-
validate
|
|
22
|
-
|
|
23
|
-
|
|
9
|
+
sample 从数据文件中采样
|
|
10
|
+
head 显示文件的前 N 条数据
|
|
11
|
+
tail 显示文件的后 N 条数据
|
|
12
|
+
transform 转换数据格式(核心命令)
|
|
13
|
+
stats 显示数据文件的统计信息
|
|
14
|
+
token-stats Token 统计
|
|
15
|
+
diff 数据集对比
|
|
16
|
+
dedupe 数据去重
|
|
17
|
+
concat 拼接多个数据文件
|
|
18
|
+
clean 数据清洗
|
|
19
|
+
run 执行 Pipeline 配置文件
|
|
20
|
+
history 显示数据血缘历史
|
|
21
|
+
validate 使用 Schema 验证数据格式
|
|
22
|
+
logs 日志查看工具使用说明
|
|
23
|
+
install-skill 安装 dtflow skill 到 Claude Code
|
|
24
24
|
"""
|
|
25
25
|
|
|
26
26
|
import os
|
|
@@ -35,12 +35,15 @@ from .cli.commands import dedupe as _dedupe
|
|
|
35
35
|
from .cli.commands import diff as _diff
|
|
36
36
|
from .cli.commands import head as _head
|
|
37
37
|
from .cli.commands import history as _history
|
|
38
|
+
from .cli.commands import install_skill as _install_skill
|
|
38
39
|
from .cli.commands import run as _run
|
|
39
40
|
from .cli.commands import sample as _sample
|
|
41
|
+
from .cli.commands import skill_status as _skill_status
|
|
40
42
|
from .cli.commands import stats as _stats
|
|
41
43
|
from .cli.commands import tail as _tail
|
|
42
44
|
from .cli.commands import token_stats as _token_stats
|
|
43
45
|
from .cli.commands import transform as _transform
|
|
46
|
+
from .cli.commands import uninstall_skill as _uninstall_skill
|
|
44
47
|
from .cli.commands import validate as _validate
|
|
45
48
|
|
|
46
49
|
# 创建主应用
|
|
@@ -263,48 +266,25 @@ dtflow 内置了 toolong 日志查看器,安装后可直接使用 tl 命令:
|
|
|
263
266
|
print(help_text)
|
|
264
267
|
|
|
265
268
|
|
|
266
|
-
# ============
|
|
269
|
+
# ============ Skill 命令 ============
|
|
267
270
|
|
|
268
|
-
mcp_app = typer.Typer(help="MCP 服务管理")
|
|
269
|
-
app.add_typer(mcp_app, name="mcp")
|
|
270
271
|
|
|
272
|
+
@app.command("install-skill")
|
|
273
|
+
def install_skill():
|
|
274
|
+
"""安装 dtflow skill 到 Claude Code"""
|
|
275
|
+
_install_skill()
|
|
271
276
|
|
|
272
|
-
@mcp_app.command()
|
|
273
|
-
def install(
|
|
274
|
-
name: str = typer.Option("datatron", "--name", "-n", help="MCP 服务名称"),
|
|
275
|
-
target: str = typer.Option("code", "--target", "-t", help="安装目标: desktop/code/all"),
|
|
276
|
-
):
|
|
277
|
-
"""安装 Datatron MCP 服务"""
|
|
278
|
-
from .mcp.cli import MCPCommands
|
|
279
|
-
|
|
280
|
-
MCPCommands().install(name, target)
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
@mcp_app.command()
|
|
284
|
-
def uninstall(
|
|
285
|
-
name: str = typer.Option("datatron", "--name", "-n", help="MCP 服务名称"),
|
|
286
|
-
target: str = typer.Option("all", "--target", "-t", help="移除目标: desktop/code/all"),
|
|
287
|
-
):
|
|
288
|
-
"""移除 Datatron MCP 服务"""
|
|
289
|
-
from .mcp.cli import MCPCommands
|
|
290
|
-
|
|
291
|
-
MCPCommands().uninstall(name, target)
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
@mcp_app.command()
|
|
295
|
-
def status():
|
|
296
|
-
"""查看 MCP 服务安装状态"""
|
|
297
|
-
from .mcp.cli import MCPCommands
|
|
298
|
-
|
|
299
|
-
MCPCommands().status()
|
|
300
277
|
|
|
278
|
+
@app.command("uninstall-skill")
|
|
279
|
+
def uninstall_skill():
|
|
280
|
+
"""卸载 dtflow skill"""
|
|
281
|
+
_uninstall_skill()
|
|
301
282
|
|
|
302
|
-
@mcp_app.command()
|
|
303
|
-
def test():
|
|
304
|
-
"""测试 MCP 服务是否正常"""
|
|
305
|
-
from .mcp.cli import MCPCommands
|
|
306
283
|
|
|
307
|
-
|
|
284
|
+
@app.command("skill-status")
|
|
285
|
+
def skill_status():
|
|
286
|
+
"""查看 skill 安装状态"""
|
|
287
|
+
_skill_status()
|
|
308
288
|
|
|
309
289
|
|
|
310
290
|
def _show_completion_hint():
|
dtflow/cli/commands.py
CHANGED
|
@@ -13,25 +13,27 @@ CLI 命令统一导出入口
|
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
15
|
# 采样命令
|
|
16
|
-
from .sample import head, sample, tail
|
|
17
|
-
|
|
18
|
-
# 转换命令
|
|
19
|
-
from .transform import transform
|
|
20
|
-
|
|
21
|
-
# 统计命令
|
|
22
|
-
from .stats import stats, token_stats
|
|
23
|
-
|
|
24
16
|
# 清洗命令
|
|
25
17
|
from .clean import clean, dedupe
|
|
26
18
|
|
|
27
19
|
# IO 操作命令
|
|
28
20
|
from .io_ops import concat, diff
|
|
29
21
|
|
|
22
|
+
# 血缘追踪命令
|
|
23
|
+
from .lineage import history
|
|
24
|
+
|
|
30
25
|
# Pipeline 命令
|
|
31
26
|
from .pipeline import run
|
|
27
|
+
from .sample import head, sample, tail
|
|
32
28
|
|
|
33
|
-
#
|
|
34
|
-
from .
|
|
29
|
+
# Skill 命令
|
|
30
|
+
from .skill import install_skill, skill_status, uninstall_skill
|
|
31
|
+
|
|
32
|
+
# 统计命令
|
|
33
|
+
from .stats import stats, token_stats
|
|
34
|
+
|
|
35
|
+
# 转换命令
|
|
36
|
+
from .transform import transform
|
|
35
37
|
|
|
36
38
|
# 验证命令
|
|
37
39
|
from .validate import validate
|
|
@@ -58,4 +60,8 @@ __all__ = [
|
|
|
58
60
|
"history",
|
|
59
61
|
# 验证
|
|
60
62
|
"validate",
|
|
63
|
+
# Skill
|
|
64
|
+
"install_skill",
|
|
65
|
+
"uninstall_skill",
|
|
66
|
+
"skill_status",
|
|
61
67
|
]
|
dtflow/cli/skill.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Claude Code Skill 安装命令
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import shutil
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
|
|
10
|
+
console = Console()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def get_skill_source_path() -> Path:
|
|
14
|
+
"""获取 SKILL.md 源文件路径"""
|
|
15
|
+
return Path(__file__).parent.parent / "SKILL.md"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_skill_target_dir() -> Path:
|
|
19
|
+
"""获取 skill 安装目标目录"""
|
|
20
|
+
return Path.home() / ".claude" / "skills" / "dtflow"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def install_skill() -> None:
|
|
24
|
+
"""安装 dtflow skill 到 Claude Code"""
|
|
25
|
+
source = get_skill_source_path()
|
|
26
|
+
target_dir = get_skill_target_dir()
|
|
27
|
+
target = target_dir / "SKILL.md"
|
|
28
|
+
|
|
29
|
+
if not source.exists():
|
|
30
|
+
console.print("[red]错误: SKILL.md 源文件不存在[/red]")
|
|
31
|
+
raise SystemExit(1)
|
|
32
|
+
|
|
33
|
+
# 创建目标目录
|
|
34
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
|
|
36
|
+
# 复制文件
|
|
37
|
+
shutil.copy2(source, target)
|
|
38
|
+
|
|
39
|
+
console.print("[green]✓[/green] 已安装 dtflow skill 到 Claude Code")
|
|
40
|
+
console.print(f" [dim]{target}[/dim]")
|
|
41
|
+
console.print()
|
|
42
|
+
console.print("[dim]在 Claude Code 中使用 /dtflow 调用此 skill[/dim]")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def uninstall_skill() -> None:
|
|
46
|
+
"""卸载 dtflow skill"""
|
|
47
|
+
target_dir = get_skill_target_dir()
|
|
48
|
+
target = target_dir / "SKILL.md"
|
|
49
|
+
|
|
50
|
+
if not target.exists():
|
|
51
|
+
console.print("[yellow]dtflow skill 未安装[/yellow]")
|
|
52
|
+
return
|
|
53
|
+
|
|
54
|
+
target.unlink()
|
|
55
|
+
|
|
56
|
+
# 如果目录为空,也删除目录
|
|
57
|
+
if target_dir.exists() and not any(target_dir.iterdir()):
|
|
58
|
+
target_dir.rmdir()
|
|
59
|
+
|
|
60
|
+
console.print("[green]✓[/green] 已卸载 dtflow skill")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def skill_status() -> None:
|
|
64
|
+
"""显示 skill 安装状态"""
|
|
65
|
+
target = get_skill_target_dir() / "SKILL.md"
|
|
66
|
+
|
|
67
|
+
if target.exists():
|
|
68
|
+
console.print("[green]✓[/green] dtflow skill 已安装")
|
|
69
|
+
console.print(f" [dim]{target}[/dim]")
|
|
70
|
+
else:
|
|
71
|
+
console.print("[yellow]✗[/yellow] dtflow skill 未安装")
|
|
72
|
+
console.print(" [dim]运行 dt install-skill 安装[/dim]")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dtflow
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.7
|
|
4
4
|
Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
|
|
5
5
|
Project-URL: Homepage, https://github.com/yourusername/DataTransformer
|
|
6
6
|
Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme
|
|
@@ -69,8 +69,6 @@ Requires-Dist: tokenizers>=0.15.0; extra == 'full'
|
|
|
69
69
|
Requires-Dist: toolong>=1.5.0; extra == 'full'
|
|
70
70
|
Provides-Extra: logs
|
|
71
71
|
Requires-Dist: toolong>=1.5.0; extra == 'logs'
|
|
72
|
-
Provides-Extra: mcp
|
|
73
|
-
Requires-Dist: mcp>=1.0.0; extra == 'mcp'
|
|
74
72
|
Provides-Extra: similarity
|
|
75
73
|
Requires-Dist: datasketch>=1.5.0; extra == 'similarity'
|
|
76
74
|
Requires-Dist: scikit-learn>=0.24.0; extra == 'similarity'
|
|
@@ -99,6 +97,17 @@ pip install transformers # Token 统计(HuggingFace 模型)
|
|
|
99
97
|
pip install datasets # HuggingFace Dataset 转换
|
|
100
98
|
```
|
|
101
99
|
|
|
100
|
+
## 🤖 Claude Code 集成
|
|
101
|
+
|
|
102
|
+
dtflow 内置了 [Claude Code](https://docs.anthropic.com/en/docs/claude-code) skill:
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
dt install-skill # 安装 skill
|
|
106
|
+
dt skill-status # 查看状态
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
安装后在 Claude Code 中输入 `/dtflow`,Claude 将掌握 dtflow 的完整用法,可直接协助你完成数据处理任务。
|
|
110
|
+
|
|
102
111
|
## 快速开始
|
|
103
112
|
|
|
104
113
|
```python
|
|
@@ -473,6 +482,10 @@ dt concat a.jsonl b.jsonl -o merged.jsonl
|
|
|
473
482
|
# 数据统计
|
|
474
483
|
dt stats data.jsonl
|
|
475
484
|
|
|
485
|
+
# Claude Code Skill 安装
|
|
486
|
+
dt install-skill # 安装到 ~/.claude/skills/
|
|
487
|
+
dt skill-status # 查看安装状态
|
|
488
|
+
|
|
476
489
|
# 数据验证
|
|
477
490
|
dt validate data.jsonl --preset=openai_chat # 使用预设 schema 验证
|
|
478
491
|
dt validate data.jsonl --preset=alpaca --verbose # 详细输出
|
|
@@ -506,6 +519,18 @@ CLI 命令中的字段参数支持嵌套路径语法,可访问深层嵌套的
|
|
|
506
519
|
| `token-stats` | `--field=` | `--field=messages[-1].content` |
|
|
507
520
|
| `diff` | `--key=` | `--key=meta.uuid` |
|
|
508
521
|
|
|
522
|
+
`--where` 支持的操作符:
|
|
523
|
+
|
|
524
|
+
| 操作符 | 含义 | 示例 |
|
|
525
|
+
|--------|------|------|
|
|
526
|
+
| `=` | 等于 | `--where="category=tech"` |
|
|
527
|
+
| `!=` | 不等于 | `--where="source!=wiki"` |
|
|
528
|
+
| `~=` | 包含 | `--where="content~=机器学习"` |
|
|
529
|
+
| `>` | 大于 | `--where="score>0.8"` |
|
|
530
|
+
| `>=` | 大于等于 | `--where="messages.#>=2"` |
|
|
531
|
+
| `<` | 小于 | `--where="length<1000"` |
|
|
532
|
+
| `<=` | 小于等于 | `--where="turns<=10"` |
|
|
533
|
+
|
|
509
534
|
示例数据:
|
|
510
535
|
```json
|
|
511
536
|
{"meta": {"source": "wiki"}, "messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}
|
|
@@ -603,6 +628,18 @@ dt history processed.jsonl
|
|
|
603
628
|
dt history processed.jsonl --json # JSON 格式输出
|
|
604
629
|
```
|
|
605
630
|
|
|
631
|
+
### 日志查看
|
|
632
|
+
|
|
633
|
+
dtflow 内置了 [toolong](https://github.com/Textualize/toolong) 日志查看器:
|
|
634
|
+
|
|
635
|
+
```bash
|
|
636
|
+
pip install dtflow[logs] # 安装日志工具
|
|
637
|
+
|
|
638
|
+
tl app.log # 交互式 TUI 查看
|
|
639
|
+
tl --tail app.log # 实时跟踪(类似 tail -f)
|
|
640
|
+
dt logs # 查看使用说明
|
|
641
|
+
```
|
|
642
|
+
|
|
606
643
|
### 大文件流式处理
|
|
607
644
|
|
|
608
645
|
专为超大文件设计的流式处理接口,内存占用 O(1),支持 JSONL、CSV、Parquet、Arrow 格式:
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
dtflow/
|
|
2
|
-
dtflow/
|
|
1
|
+
dtflow/SKILL.md,sha256=sHf6i6DKUCca5zvSJ67VHu05tFlST4mYgnoURXVe1g0,7836
|
|
2
|
+
dtflow/__init__.py,sha256=Ee7CDDxzki69MEGeXB5bczuMts5OwZZ-jVsKjH_rD_0,3031
|
|
3
|
+
dtflow/__main__.py,sha256=3LXTku09Fw1dsgTUtX1UJCmE20qKeZpNga3UqmI3UiY,12145
|
|
3
4
|
dtflow/converters.py,sha256=X3qeFD7FCOMnfiP3MicL5MXimOm4XUYBs5pczIkudU0,22331
|
|
4
5
|
dtflow/core.py,sha256=qMo6B3LK--TWRK7ZBKObGcs3pKFnd0NPoaM0T8JC7Jw,38135
|
|
5
6
|
dtflow/framework.py,sha256=jyICi_RWHjX7WfsXdSbWmP1SL7y1OWSPyd5G5Y-lvg4,17578
|
|
@@ -11,27 +12,23 @@ dtflow/streaming.py,sha256=dxpNd1-Wz_PTLTdvM5qn06_2TJr5NRlIIuw0LOSS2Iw,24755
|
|
|
11
12
|
dtflow/tokenizers.py,sha256=7ZAelSmcDxLWH5kICgH9Q1ULH3_BfDZb9suHMjJJRZU,20589
|
|
12
13
|
dtflow/cli/__init__.py,sha256=QhZ-thgx9IBTFII7T_hdoWFUl0CCsdGQHN5ZEZw2XB0,423
|
|
13
14
|
dtflow/cli/clean.py,sha256=y9VCRibgK1j8WIY3h0XZX0m93EdELQC7TdnseMWwS-0,17799
|
|
14
|
-
dtflow/cli/commands.py,sha256=
|
|
15
|
+
dtflow/cli/commands.py,sha256=zKUG-B9Az-spqyqM00cR8Sgc2UgeOPQDThJFHWDNO_w,1336
|
|
15
16
|
dtflow/cli/common.py,sha256=gCwnF5Sw2ploqfZJO_z3Ms9mR1HNT7Lj6ydHn0uVaIw,13817
|
|
16
17
|
dtflow/cli/io_ops.py,sha256=BMDisP6dxzzmSjYwmeFwaHmpHHPqirmXAWeNTD-9MQM,13254
|
|
17
18
|
dtflow/cli/lineage.py,sha256=_lNh35nF9AA0Zy6FyZ4g8IzrXH2ZQnp3inF-o2Hs1pw,1383
|
|
18
19
|
dtflow/cli/pipeline.py,sha256=QNEo-BJlaC1CVnVeRZr7TwfuZYloJ4TebIzJ5ALzry0,1426
|
|
19
20
|
dtflow/cli/sample.py,sha256=pubpx4AIzsarBEalD150MC2apYQSt4bal70IZkTfFO0,15475
|
|
21
|
+
dtflow/cli/skill.py,sha256=opiTEBejA7JHKrEMftMOPDQlOgZ4n59rwaHXGU1Nukk,2022
|
|
20
22
|
dtflow/cli/stats.py,sha256=u4ehCfgw1X8WuOyAjrApMRgcIO3BVmINbsTjxEscQro,24086
|
|
21
23
|
dtflow/cli/transform.py,sha256=w6xqMOxPxQvL2u_BPCfpDHuPSC9gmcqMPVN8s-B6bbY,15052
|
|
22
24
|
dtflow/cli/validate.py,sha256=65aGVlMS_Rq0Ch0YQ-TclVJ03RQP4CnG137wthzb8Ao,4384
|
|
23
|
-
dtflow/mcp/__init__.py,sha256=huEJ3rXDbxDRjsLPEvjNT2u3tWs6Poiv6fokPIrByjw,897
|
|
24
|
-
dtflow/mcp/__main__.py,sha256=PoT2ZZmJq9xDZxDACJfqDW9Ld_ukHrGNK-0XUd7WGnY,448
|
|
25
|
-
dtflow/mcp/cli.py,sha256=ck0oOS_642cNktxULaMRE7BJfMxsBCwotmCj3PSPwVk,13110
|
|
26
|
-
dtflow/mcp/docs.py,sha256=DI2Vf-eFo4chRP_bDLsv4Uc3kJt8_1emz8N-NBSVirM,8834
|
|
27
|
-
dtflow/mcp/server.py,sha256=Nf0UlqDGhV55ndGuEglfr7VRjDWAC_9rRsNhdr0-ssM,4275
|
|
28
25
|
dtflow/storage/__init__.py,sha256=C0jpWNQU808Ezz7lWneddABal3wILy8ijFUNiSKbHV4,362
|
|
29
26
|
dtflow/storage/io.py,sha256=ZH2aSE-S89gpy3z4oTqhcqWf4u10OdkDoyul7o_YBDI,23374
|
|
30
27
|
dtflow/utils/__init__.py,sha256=Pn-ltwV04fBQmeZG7FxInDQmzH29LYOi90LgeLMEuQk,506
|
|
31
28
|
dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
|
|
32
29
|
dtflow/utils/field_path.py,sha256=K8nU196RxTSJ1OoieTWGcYOWl9KjGq2iSxCAkfjECuM,7621
|
|
33
30
|
dtflow/utils/helpers.py,sha256=JXN176_B2pm53GLVyZ1wj3wrmBJG52Tkw6AMQSdj7M8,791
|
|
34
|
-
dtflow-0.5.
|
|
35
|
-
dtflow-0.5.
|
|
36
|
-
dtflow-0.5.
|
|
37
|
-
dtflow-0.5.
|
|
31
|
+
dtflow-0.5.7.dist-info/METADATA,sha256=mlWaRHSM1ZucQrAa8PGcHzjHj2RQPBynnmdA_JoNSNI,23899
|
|
32
|
+
dtflow-0.5.7.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
33
|
+
dtflow-0.5.7.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
|
|
34
|
+
dtflow-0.5.7.dist-info/RECORD,,
|
dtflow/mcp/__init__.py
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
"""DataTransformer MCP (Model Context Protocol) 服务
|
|
2
|
-
|
|
3
|
-
提供 DataTransformer 的用法查询功能,供 AI 模型调用。
|
|
4
|
-
|
|
5
|
-
使用方式:
|
|
6
|
-
# 安装 MCP 服务到 Claude Code
|
|
7
|
-
dt mcp install
|
|
8
|
-
|
|
9
|
-
# 运行 MCP 服务(通常由 Claude 自动调用)
|
|
10
|
-
dt-mcp
|
|
11
|
-
|
|
12
|
-
注意: MCP 功能需要安装 mcp 依赖: pip install dtflow[mcp]
|
|
13
|
-
"""
|
|
14
|
-
|
|
15
|
-
__all__ = ["main", "mcp"]
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def __getattr__(name):
|
|
19
|
-
"""延迟导入 server 模块,避免在未安装 mcp 依赖时报错"""
|
|
20
|
-
if name in ("main", "mcp"):
|
|
21
|
-
try:
|
|
22
|
-
from .server import main, mcp
|
|
23
|
-
|
|
24
|
-
return main if name == "main" else mcp
|
|
25
|
-
except ImportError as e:
|
|
26
|
-
raise ImportError(
|
|
27
|
-
f"MCP 功能需要安装 mcp 依赖: pip install dtflow[mcp]\n原始错误: {e}"
|
|
28
|
-
) from e
|
|
29
|
-
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
dtflow/mcp/__main__.py
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
"""Datatron MCP 服务入口
|
|
2
|
-
|
|
3
|
-
使用方式:
|
|
4
|
-
python -m dtflow.mcp
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
if __name__ == "__main__":
|
|
8
|
-
try:
|
|
9
|
-
from .server import main
|
|
10
|
-
|
|
11
|
-
main()
|
|
12
|
-
except ImportError as e:
|
|
13
|
-
import sys
|
|
14
|
-
|
|
15
|
-
print(f"错误: MCP 功能需要安装 mcp 依赖", file=sys.stderr)
|
|
16
|
-
print(f"请运行: pip install dtflow[mcp]", file=sys.stderr)
|
|
17
|
-
print(f"\n原始错误: {e}", file=sys.stderr)
|
|
18
|
-
sys.exit(1)
|