dtflow 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

dtflow/cli/clean.py

@@ -132,6 +132,11 @@ def clean(
     max_len: Optional[str] = None,
     keep: Optional[str] = None,
     drop: Optional[str] = None,
+    rename: Optional[str] = None,
+    promote: Optional[str] = None,
+    add_field: Optional[str] = None,
+    fill: Optional[str] = None,
+    reorder: Optional[str] = None,
     strip: bool = False,
     output: Optional[str] = None,
 ) -> None:
@@ -147,18 +152,26 @@ def clean(
         max_len: 最大长度过滤，格式 "字段:长度"，字段支持嵌套路径
         keep: 只保留指定字段（逗号分隔，仅支持顶层字段）
         drop: 删除指定字段（逗号分隔，仅支持顶层字段）
+        rename: 重命名字段，格式 "old:new" 或 "old1:new1,old2:new2"
+        promote: 提升嵌套字段到顶层，格式 "path" 或 "path:name"（逗号分隔多个）
+        add_field: 添加常量字段，格式 "key:value"（逗号分隔多个）
+        fill: 填充空值，格式 "field:default_value"（逗号分隔多个）
+        reorder: 控制字段顺序（逗号分隔），未列出的字段追加在后面
         strip: 去除所有字符串字段的首尾空白
         output: 输出文件路径，不指定则覆盖原文件
 
     Examples:
         dt clean data.jsonl --drop-empty                    # 删除任意空值记录
         dt clean data.jsonl --drop-empty=text,answer        # 删除指定字段为空的记录
-        dt clean data.jsonl --drop-empty=meta.source        # 删除嵌套字段为空的记录
         dt clean data.jsonl --min-len=text:10               # text 字段最少 10 字符
-        dt clean data.jsonl --min-len=messages.#:2          # 至少 2 条消息
-        dt clean data.jsonl --max-len=messages[-1].content:500  # 最后一条消息最多 500 字符
         dt clean data.jsonl --keep=question,answer          # 只保留这些字段
         dt clean data.jsonl --drop=metadata,timestamp       # 删除这些字段
+        dt clean data.jsonl --rename=question:instruction   # 重命名字段
+        dt clean data.jsonl --promote=meta.label            # 提升嵌套字段到顶层
+        dt clean data.jsonl --promote=meta.label:tag        # 提升并自定义名称
+        dt clean data.jsonl --add-field=source:web          # 添加常量字段
+        dt clean data.jsonl --fill=label:unknown            # 填充空值
+        dt clean data.jsonl --reorder=id,text,label         # 控制字段顺序
         dt clean data.jsonl --strip                         # 去除字符串首尾空白
     """
     filepath = Path(filename)
@@ -175,6 +188,11 @@ def clean(
     max_len_field, max_len_value = _parse_len_param(max_len) if max_len else (None, None)
     keep_fields = _parse_field_list(keep) if keep else None
     drop_fields_set = set(_parse_field_list(drop)) if drop else None
+    rename_map = _parse_rename_param(rename) if rename else None
+    promote_list = _parse_promote_param(promote) if promote else None
+    add_field_map = _parse_kv_param(add_field, "add-field") if add_field else None
+    fill_map = _parse_kv_param(fill, "fill") if fill else None
+    reorder_fields = _parse_field_list(reorder) if reorder else None
     keep_set = set(keep_fields) if keep_fields else None
 
     # 构建清洗配置
@@ -197,6 +215,20 @@ def clean(
         print(f"🔄 只保留字段: {', '.join(keep_fields)}")
     if drop_fields_set:
         print(f"🔄 删除字段: {', '.join(drop_fields_set)}")
+    if rename_map:
+        rename_desc = ", ".join(f"{k} → {v}" for k, v in rename_map.items())
+        print(f"🔄 重命名字段: {rename_desc}")
+    if promote_list:
+        promote_desc = ", ".join(f"{src} → {dst}" for src, dst in promote_list)
+        print(f"🔄 提升字段: {promote_desc}")
+    if add_field_map:
+        add_desc = ", ".join(f"{k}={v}" for k, v in add_field_map.items())
+        print(f"🔄 添加字段: {add_desc}")
+    if fill_map:
+        fill_desc = ", ".join(f"{k}={v}" for k, v in fill_map.items())
+        print(f"🔄 填充空值: {fill_desc}")
+    if reorder_fields:
+        print(f"🔄 字段排序: {', '.join(reorder_fields)}")
 
     output_path = output or str(filepath)
 
@@ -234,6 +266,11 @@ def clean(
                 max_len_value=max_len_value,
                 keep_set=keep_set,
                 drop_fields_set=drop_fields_set,
+                rename_map=rename_map,
+                promote_list=promote_list,
+                add_field_map=add_field_map,
+                fill_map=fill_map,
+                reorder_fields=reorder_fields,
             )
 
             # 如果使用了临时文件，移动到目标位置
@@ -274,6 +311,11 @@ def clean(
         max_len_value=max_len_value,
         keep_fields=keep_fields,
         drop_fields=drop_fields_set,
+        rename_map=rename_map,
+        promote_list=promote_list,
+        add_field_map=add_field_map,
+        fill_map=fill_map,
+        reorder_fields=reorder_fields,
     )
 
     # 保存结果
@@ -288,12 +330,106 @@ def clean(
 
     # 打印统计
     removed_count = original_count - final_count
-    print(f"\n✅ 完成!")
+    print("\n✅ 完成!")
     print(f"   原始: {original_count} 条 -> 清洗后: {final_count} 条 (删除 {removed_count} 条)")
     if step_stats:
         print(f"   步骤: {' | '.join(step_stats)}")
 
 
+def _parse_rename_param(param: str) -> Dict[str, str]:
+    """解析重命名参数，格式 'old:new' 或 'old1:new1,old2:new2'"""
+    rename_map = {}
+    for pair in param.split(","):
+        pair = pair.strip()
+        if ":" not in pair:
+            raise ValueError(f"重命名参数格式错误: {pair}，应为 'old:new'")
+        old, new = pair.split(":", 1)
+        old, new = old.strip(), new.strip()
+        if not old or not new:
+            raise ValueError(f"重命名参数格式错误: {pair}，字段名不能为空")
+        rename_map[old] = new
+    return rename_map
+
+
+def _parse_promote_param(param: str) -> List[tuple]:
+    """
+    解析提升参数，格式 'path' 或 'path:name'（逗号分隔多个）。
+
+    Returns:
+        [(source_path, target_name), ...]
+    """
+    result = []
+    for item in param.split(","):
+        item = item.strip()
+        if ":" in item:
+            src, dst = item.split(":", 1)
+            src, dst = src.strip(), dst.strip()
+        else:
+            src = item
+            # 默认用路径最后一段作为目标名
+            dst = src.rsplit(".", 1)[-1] if "." in src else src
+        if not src or not dst:
+            raise ValueError(f"promote 参数格式错误: {item}")
+        result.append((src, dst))
+    return result
+
+
+def _parse_kv_param(param: str, param_name: str) -> Dict[str, str]:
+    """解析 key:value 格式参数（通用），用于 --add-field 和 --fill"""
+    kv_map = {}
+    for pair in param.split(","):
+        pair = pair.strip()
+        if ":" not in pair:
+            raise ValueError(f"{param_name} 参数格式错误: {pair}，应为 'key:value'")
+        key, value = pair.split(":", 1)
+        key, value = key.strip(), value.strip()
+        if not key:
+            raise ValueError(f"{param_name} 参数格式错误: {pair}，key 不能为空")
+        kv_map[key] = value
+    return kv_map
+
+
+def _rename_item(item: Dict, rename_map: Dict[str, str]) -> Dict:
+    """重命名字段，保持字段顺序"""
+    return {rename_map.get(k, k): v for k, v in item.items()}
+
+
+def _promote_fields(item: Dict, promote_list: List[tuple]) -> Dict:
+    """提升嵌套字段到顶层（始终添加字段，即使值为 None）"""
+    item = dict(item)
+    for src_path, dst_name in promote_list:
+        item[dst_name] = get_field_with_spec(item, src_path)
+    return item
+
+
+def _add_fields(item: Dict, add_field_map: Dict[str, str]) -> Dict:
+    """添加常量字段"""
+    item = dict(item)
+    item.update(add_field_map)
+    return item
+
+
+def _fill_empty(item: Dict, fill_map: Dict[str, str]) -> Dict:
+    """填充空值（字段不存在时也会添加）"""
+    item = dict(item)
+    for field, default in fill_map.items():
+        if field not in item or _is_empty_value(item[field]):
+            item[field] = default
+    return item
+
+
+def _reorder_item(item: Dict, reorder_fields: List[str]) -> Dict:
+    """按指定顺序重排字段，未列出的字段追加在后面"""
+    ordered = {}
+    for f in reorder_fields:
+        if f in item:
+            ordered[f] = item[f]
+    for k, v in item.items():
+        if k not in ordered:
+            ordered[k] = v
+    return ordered
+
+
 def _parse_len_param(param: str) -> tuple:
     """解析长度参数，格式 'field:length'"""
     if ":" not in param:
@@ -302,8 +438,8 @@ def _parse_len_param(param: str) -> tuple:
     field = parts[0].strip()
     try:
         length = int(parts[1].strip())
-    except ValueError:
-        raise ValueError(f"长度必须是整数: {parts[1]}")
+    except ValueError as e:
+        raise ValueError(f"长度必须是整数: {parts[1]}") from e
     return field, length
 
 
@@ -317,6 +453,11 @@ def _clean_data_single_pass(
     max_len_value: Optional[int] = None,
     keep_fields: Optional[List[str]] = None,
     drop_fields: Optional[set] = None,
+    rename_map: Optional[Dict[str, str]] = None,
+    promote_list: Optional[List[tuple]] = None,
+    add_field_map: Optional[Dict[str, str]] = None,
+    fill_map: Optional[Dict[str, str]] = None,
+    reorder_fields: Optional[List[str]] = None,
 ) -> tuple:
     """
     单次遍历执行所有清洗操作。
@@ -375,12 +516,32 @@ def _clean_data_single_pass(
                 stats["max_len"] += 1
                 continue
 
-        # 5. 字段管理（keep/drop）
+        # 5. 提升嵌套字段（在 drop 之前，否则父字段被删后无法提取）
+        if promote_list is not None:
+            item = _promote_fields(item, promote_list)
+
+        # 6. 字段管理（keep/drop）
         if keep_set is not None:
             item = {k: v for k, v in item.items() if k in keep_set}
         elif drop_fields is not None:
             item = {k: v for k, v in item.items() if k not in drop_fields}
 
+        # 7. 字段重命名
+        if rename_map is not None:
+            item = _rename_item(item, rename_map)
+
+        # 8. 添加常量字段
+        if add_field_map is not None:
+            item = _add_fields(item, add_field_map)
+
+        # 9. 填充空值
+        if fill_map is not None:
+            item = _fill_empty(item, fill_map)
+
+        # 10. 字段排序（最后执行）
+        if reorder_fields is not None:
+            item = _reorder_item(item, reorder_fields)
+
         result.append(item)
 
     # 构建统计信息字符串列表
@@ -397,6 +558,16 @@ def _clean_data_single_pass(
         step_stats.append(f"keep: {len(keep_fields)} 字段")
     if drop_fields:
         step_stats.append(f"drop: {len(drop_fields)} 字段")
+    if rename_map:
+        step_stats.append(f"rename: {len(rename_map)} 字段")
+    if promote_list:
+        step_stats.append(f"promote: {len(promote_list)} 字段")
+    if add_field_map:
+        step_stats.append(f"add-field: {len(add_field_map)} 字段")
+    if fill_map:
+        step_stats.append(f"fill: {len(fill_map)} 字段")
+    if reorder_fields:
+        step_stats.append("reorder")
 
     return result, step_stats
 
@@ -412,6 +583,11 @@ def _clean_streaming(
     max_len_value: Optional[int] = None,
     keep_set: Optional[set] = None,
     drop_fields_set: Optional[set] = None,
+    rename_map: Optional[Dict[str, str]] = None,
+    promote_list: Optional[List[tuple]] = None,
+    add_field_map: Optional[Dict[str, str]] = None,
+    fill_map: Optional[Dict[str, str]] = None,
+    reorder_fields: Optional[List[str]] = None,
 ) -> int:
     """
     流式清洗数据。
@@ -471,7 +647,11 @@ def _clean_streaming(
     if empty_fields is not None or min_len_field is not None or max_len_field is not None:
         st = st.filter(clean_filter)
 
-    # 执行字段管理（如果没有 strip，也需要在这里处理）
+    # 提升嵌套字段（在 drop 之前，否则父字段被删后无法提取）
+    if promote_list is not None:
+        st = st.transform(lambda item: _promote_fields(item, promote_list))
+
+    # 执行字段管理（keep/drop）
     if keep_set is not None or drop_fields_set is not None:
 
         def field_transform(item):
@@ -483,4 +663,20 @@ def _clean_streaming(
 
         st = st.transform(field_transform)
 
+    # 执行字段重命名
+    if rename_map is not None:
+        st = st.transform(lambda item: _rename_item(item, rename_map))
+
+    # 添加常量字段
+    if add_field_map is not None:
+        st = st.transform(lambda item: _add_fields(item, add_field_map))
+
+    # 填充空值
+    if fill_map is not None:
+        st = st.transform(lambda item: _fill_empty(item, fill_map))
+
+    # 字段排序（最后执行）
+    if reorder_fields is not None:
+        st = st.transform(lambda item: _reorder_item(item, reorder_fields))
+
     return st.save(output_path)

dtflow/cli/commands.py

@@ -13,25 +13,27 @@ CLI 命令统一导出入口
 """
 
 # 采样命令
-from .sample import head, sample, tail
-
-# 转换命令
-from .transform import transform
-
-# 统计命令
-from .stats import stats, token_stats
-
 # 清洗命令
 from .clean import clean, dedupe
 
 # IO 操作命令
 from .io_ops import concat, diff
 
+# 血缘追踪命令
+from .lineage import history
+
 # Pipeline 命令
 from .pipeline import run
+from .sample import head, sample, tail
 
-# 血缘追踪命令
-from .lineage import history
+# Skill 命令
+from .skill import install_skill, skill_status, uninstall_skill
+
+# 统计命令
+from .stats import stats, token_stats
+
+# 转换命令
+from .transform import transform
 
 # 验证命令
 from .validate import validate
@@ -58,4 +60,8 @@ __all__ = [
     "history",
     # 验证
     "validate",
+    # Skill
+    "install_skill",
+    "uninstall_skill",
+    "skill_status",
 ]

dtflow/cli/skill.py

@@ -0,0 +1,72 @@
+"""
+Claude Code Skill 安装命令
+"""
+
+import shutil
+from pathlib import Path
+
+from rich.console import Console
+
+console = Console()
+
+
+def get_skill_source_path() -> Path:
+    """获取 SKILL.md 源文件路径"""
+    return Path(__file__).parent.parent / "SKILL.md"
+
+
+def get_skill_target_dir() -> Path:
+    """获取 skill 安装目标目录"""
+    return Path.home() / ".claude" / "skills" / "dtflow"
+
+
+def install_skill() -> None:
+    """安装 dtflow skill 到 Claude Code"""
+    source = get_skill_source_path()
+    target_dir = get_skill_target_dir()
+    target = target_dir / "SKILL.md"
+
+    if not source.exists():
+        console.print("[red]错误: SKILL.md 源文件不存在[/red]")
+        raise SystemExit(1)
+
+    # 创建目标目录
+    target_dir.mkdir(parents=True, exist_ok=True)
+
+    # 复制文件
+    shutil.copy2(source, target)
+
+    console.print("[green]✓[/green] 已安装 dtflow skill 到 Claude Code")
+    console.print(f"  [dim]{target}[/dim]")
+    console.print()
+    console.print("[dim]在 Claude Code 中使用 /dtflow 调用此 skill[/dim]")
+
+
+def uninstall_skill() -> None:
+    """卸载 dtflow skill"""
+    target_dir = get_skill_target_dir()
+    target = target_dir / "SKILL.md"
+
+    if not target.exists():
+        console.print("[yellow]dtflow skill 未安装[/yellow]")
+        return
+
+    target.unlink()
+
+    # 如果目录为空，也删除目录
+    if target_dir.exists() and not any(target_dir.iterdir()):
+        target_dir.rmdir()
+
+    console.print("[green]✓[/green] 已卸载 dtflow skill")
+
+
+def skill_status() -> None:
+    """显示 skill 安装状态"""
+    target = get_skill_target_dir() / "SKILL.md"
+
+    if target.exists():
+        console.print("[green]✓[/green] dtflow skill 已安装")
+        console.print(f"  [dim]{target}[/dim]")
+    else:
+        console.print("[yellow]✗[/yellow] dtflow skill 未安装")
+        console.print("  [dim]运行 dt install-skill 安装[/dim]")