dtflow 0.5.2__tar.gz → 0.5.4__tar.gz

dtflow-0.5.4/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+## [0.5.2] - 2026-01-18
+
+### Miscellaneous
+
+- Bump version to 0.5.2
+- 添加 pre-commit 配置和发版脚本
+
+## [0.5.1] - 2026-01-18
+
+### Features
+
+- 优化 sample 命令文本预览显示
+
+### Testing
+
+- 添加测试运行说明
+- 补充 tail/token-stats/validate 性能测试

{dtflow-0.5.2 → dtflow-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.5.2
+Version: 0.5.4
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme

{dtflow-0.5.2 → dtflow-0.5.4}/dtflow/__init__.py

@@ -26,6 +26,12 @@ from .converters import (  # LLaMA-Factory 扩展; ms-swift
     to_swift_vlm,
 )
 from .core import DataTransformer, DictWrapper, TransformError, TransformErrors
+from .framework import (
+    CompatibilityResult,
+    check_compatibility,
+    detect_format,
+    export_for,
+)
 from .presets import get_preset, list_presets
 from .schema import (
     Field,
@@ -38,12 +44,6 @@ from .schema import (
     sharegpt_schema,
     validate_data,
 )
-from .framework import (
-    CompatibilityResult,
-    check_compatibility,
-    detect_format,
-    export_for,
-)
 from .storage import load_data, sample_file, save_data
 from .streaming import StreamingTransformer, load_sharded, load_stream, process_shards
 from .tokenizers import (
@@ -60,7 +60,7 @@ from .tokenizers import (
     token_stats,
 )
 
-__version__ = "0.5.2"
+__version__ = "0.5.4"
 
 __all__ = [
     # core

{dtflow-0.5.2 → dtflow-0.5.4}/dtflow/cli/common.py

@@ -100,8 +100,6 @@ def _format_nested(
     └─ 最后一项
     """
     lines = []
-    branch = "└─ " if is_last else "├─ "
-    cont = "   " if is_last else "│  "
 
     if isinstance(value, dict):
         items = list(value.items())
@@ -183,6 +181,7 @@ def _print_samples(
     filename: Optional[str] = None,
     total_count: Optional[int] = None,
     fields: Optional[List[str]] = None,
+    file_size: Optional[int] = None,
 ) -> None:
     """
     打印采样结果。
@@ -190,8 +189,9 @@ def _print_samples(
     Args:
         samples: 采样数据列表
         filename: 文件名（用于显示概览）
-        total_count: 文件总行数（用于显示概览）
+        total_count: 文件总行数（用于显示概览），大文件时可能为 None
         fields: 只显示指定字段
+        file_size: 文件大小（字节），当 total_count 为 None 时显示
     """
     if not samples:
         print("没有数据")
@@ -219,6 +219,8 @@ def _print_samples(
 
             if total_count is not None:
                 info = f"总行数: {total_count:,} | 采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
+            elif file_size is not None:
+                info = f"文件大小: {_format_file_size(file_size)} | 采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
             else:
                 info = f"采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
 
@@ -266,6 +268,10 @@ def _print_samples(
                 print(
                     f"   总行数: {total_count:,} | 采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
                 )
+            elif file_size is not None:
+                print(
+                    f"   文件大小: {_format_file_size(file_size)} | 采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
+                )
             else:
                 print(f"   采样: {len(samples)} 条 | 字段: {len(all_fields)} 个")
             print(f"   字段: {', '.join(sorted(all_fields))}")
@@ -287,6 +293,15 @@ def _parse_field_list(value: Any) -> List[str]:
         return [str(value)]
 
 
+def _format_file_size(size: int) -> str:
+    """格式化文件大小"""
+    for unit in ["B", "KB", "MB", "GB"]:
+        if size < 1024:
+            return f"{size:.1f} {unit}"
+        size /= 1024
+    return f"{size:.1f} TB"
+
+
 def _is_empty_value(v: Any) -> bool:
     """判断值是否为空"""
     if v is None:

{dtflow-0.5.2 → dtflow-0.5.4}/dtflow/cli/sample.py

@@ -99,11 +99,15 @@ def sample(
         for item in sampled:
             print(orjson.dumps(item, option=orjson.OPT_INDENT_2).decode("utf-8"))
     else:
-        # 获取文件总行数用于显示
-        total_count = _get_file_row_count(filepath)
+        # 大文件跳过行数统计（50MB 阈值）
+        file_size = filepath.stat().st_size
+        if file_size < 50 * 1024 * 1024:
+            total_count = _get_file_row_count(filepath)
+        else:
+            total_count = None
         # 解析 fields 参数
         field_list = _parse_field_list(fields) if fields else None
-        _print_samples(sampled, filepath.name, total_count, field_list)
+        _print_samples(sampled, filepath.name, total_count, field_list, file_size)
 
 
 def _stratified_sample(
@@ -196,7 +200,7 @@ def _stratified_sample(
 
     # 执行各组采样
     result = []
-    print(f"🔄 执行采样...")
+    print("🔄 执行采样...")
     for key in group_keys:
         group_data = groups[key]
         target = min(sample_counts[key], len(group_data))
@@ -215,7 +219,7 @@ def _stratified_sample(
         result.extend(sampled)
 
     # 打印采样结果
-    print(f"\n📋 采样结果:")
+    print("\n📋 采样结果:")
     result_groups: Dict[Any, int] = defaultdict(int)
     for item in result:
         key = item.get(stratify_field, "__null__")

{dtflow-0.5.2 → dtflow-0.5.4}/dtflow/converters.py

@@ -4,7 +4,7 @@
 提供与 HuggingFace datasets 等常用格式的互转功能。
 """
 
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional
 
 
 def to_hf_dataset(data: List[Dict[str, Any]]):
@@ -23,8 +23,8 @@ def to_hf_dataset(data: List[Dict[str, Any]]):
     """
     try:
         from datasets import Dataset
-    except ImportError:
-        raise ImportError("需要安装 datasets: pip install datasets")
+    except ImportError as e:
+        raise ImportError("需要安装 datasets: pip install datasets") from e
 
     return Dataset.from_list(data)
 
@@ -45,9 +45,9 @@ def from_hf_dataset(dataset, split: Optional[str] = None) -> List[Dict[str, Any]
         >>> data = from_hf_dataset(my_dataset, split="train")
     """
     try:
-        from datasets import Dataset, DatasetDict, load_dataset
-    except ImportError:
-        raise ImportError("需要安装 datasets: pip install datasets")
+        from datasets import load_dataset
+    except ImportError as e:
+        raise ImportError("需要安装 datasets: pip install datasets") from e
 
     # 如果是字符串，加载数据集
     if isinstance(dataset, str):
@@ -143,14 +143,16 @@ def to_openai_batch(
         >>> batch_input = dt.to(to_openai_batch(model="gpt-4o"))
     """
 
-    def transform(item, idx=[0]) -> dict:
+    counter = {"idx": 0}
+
+    def transform(item) -> dict:
         messages = item.get(messages_field, []) if hasattr(item, "get") else item[messages_field]
 
         if custom_id_field:
             custom_id = item.get(custom_id_field) if hasattr(item, "get") else item[custom_id_field]
         else:
-            custom_id = f"request-{idx[0]}"
-            idx[0] += 1
+            custom_id = f"request-{counter['idx']}"
+            counter["idx"] += 1
 
         return {
             "custom_id": str(custom_id),
@@ -196,7 +198,8 @@ def to_llama_factory(
     """
 
     def transform(item) -> dict:
-        get = lambda f: (item.get(f, "") if hasattr(item, "get") else item.get(f, ""))
+        def get(f):
+            return item.get(f, "") if hasattr(item, "get") else getattr(item, f, "")
 
         result = {
             "instruction": get(instruction_field),
@@ -248,7 +251,7 @@ def to_axolotl(
         conversations = (
             item.get(conversations_field, [])
             if hasattr(item, "get")
-            else item.get(conversations_field, [])
+            else getattr(item, conversations_field, [])
         )
 
         # 如果已经是正确格式，直接返回
@@ -257,7 +260,9 @@ def to_axolotl(
                 return {"conversations": conversations}
 
         # 尝试从 messages 格式转换
-        messages = item.get("messages", []) if hasattr(item, "get") else item.get("messages", [])
+        messages = (
+            item.get("messages", []) if hasattr(item, "get") else getattr(item, "messages", [])
+        )
         if messages:
             role_map = {"user": "human", "assistant": "gpt", "system": "system"}
             conversations = [
@@ -312,7 +317,9 @@ def to_llama_factory_sharegpt(
     }
 
     def transform(item) -> dict:
-        get = lambda f: (item.get(f, "") if hasattr(item, "get") else item.get(f, ""))
+        def get(f):
+            return item.get(f, "") if hasattr(item, "get") else getattr(item, f, "")
+
         messages = get(messages_field) or []
 
         conversations = []
@@ -385,7 +392,9 @@ def to_llama_factory_vlm(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+
         messages = get(messages_field) or []
 
         instruction = ""
@@ -467,7 +476,9 @@ def to_llama_factory_vlm_sharegpt(
     role_map = {"user": "human", "assistant": "gpt", "system": "system"}
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+
         messages = get(messages_field) or []
 
         conversations = []
@@ -541,7 +552,9 @@ def to_swift_messages(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+
         messages = get(messages_field) or []
 
         # 复制 messages，避免修改原数据
@@ -600,7 +613,8 @@ def to_swift_query_response(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
 
         query = get(query_field)
         response = get(response_field)
@@ -613,7 +627,7 @@ def to_swift_query_response(
             current_query = ""
             current_response = ""
 
-            for i, msg in enumerate(messages):
+            for _i, msg in enumerate(messages):
                 role = msg.get("role", "")
                 content = msg.get("content", "")
 
@@ -693,7 +707,9 @@ def to_swift_vlm(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else item.get(f)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+
         messages = get(messages_field) or []
 
         result_messages = []

{dtflow-0.5.2 → dtflow-0.5.4}/dtflow/presets.py

@@ -6,6 +6,8 @@
 
 from typing import Any, Callable
 
+from dtflow.utils.helpers import get_field_value
+
 
 def openai_chat(
     user_field: str = "q", assistant_field: str = "a", system_prompt: str = None
@@ -33,8 +35,8 @@ def openai_chat(
         if system_prompt:
             messages.append({"role": "system", "content": system_prompt})
 
-        user_content = getattr(item, user_field, None) or item.get(user_field, "")
-        assistant_content = getattr(item, assistant_field, None) or item.get(assistant_field, "")
+        user_content = get_field_value(item, user_field)
+        assistant_content = get_field_value(item, assistant_field)
 
         messages.append({"role": "user", "content": user_content})
         messages.append({"role": "assistant", "content": assistant_content})
@@ -60,10 +62,9 @@ def alpaca(
 
     def transform(item: Any) -> dict:
         return {
-            "instruction": getattr(item, instruction_field, None)
-            or item.get(instruction_field, ""),
-            "input": getattr(item, input_field, None) or item.get(input_field, ""),
-            "output": getattr(item, output_field, None) or item.get(output_field, ""),
+            "instruction": get_field_value(item, instruction_field),
+            "input": get_field_value(item, input_field),
+            "output": get_field_value(item, output_field),
         }
 
     return transform
@@ -84,9 +85,7 @@ def sharegpt(conversations_field: str = "conversations", role_mapping: dict = No
     role_mapping = role_mapping or {"user": "human", "assistant": "gpt"}
 
     def transform(item: Any) -> dict:
-        conversations = getattr(item, conversations_field, None) or item.get(
-            conversations_field, []
-        )
+        conversations = get_field_value(item, conversations_field, [])
 
         # 如果已经是对话格式，直接返回
         if conversations:
@@ -102,7 +101,7 @@ def sharegpt(conversations_field: str = "conversations", role_mapping: dict = No
             ("answer", "gpt"),
             ("output", "gpt"),
         ]:
-            value = getattr(item, field, None) or item.get(field, None)
+            value = get_field_value(item, field, None)
             if value:
                 result.append({"from": role, "value": value})
 
@@ -127,9 +126,9 @@ def dpo_pair(
 
     def transform(item: Any) -> dict:
         return {
-            "prompt": getattr(item, prompt_field, None) or item.get(prompt_field, ""),
-            "chosen": getattr(item, chosen_field, None) or item.get(chosen_field, ""),
-            "rejected": getattr(item, rejected_field, None) or item.get(rejected_field, ""),
+            "prompt": get_field_value(item, prompt_field),
+            "chosen": get_field_value(item, chosen_field),
+            "rejected": get_field_value(item, rejected_field),
         }
 
     return transform
@@ -148,8 +147,8 @@ def simple_qa(question_field: str = "q", answer_field: str = "a") -> Callable:
 
     def transform(item: Any) -> dict:
         return {
-            "question": getattr(item, question_field, None) or item.get(question_field, ""),
-            "answer": getattr(item, answer_field, None) or item.get(answer_field, ""),
+            "question": get_field_value(item, question_field),
+            "answer": get_field_value(item, answer_field),
         }
 
     return transform

{dtflow-0.5.2 → dtflow-0.5.4}/dtflow/utils/__init__.py

@@ -9,6 +9,7 @@ from .field_path import (
     get_field_with_spec,
     parse_field_spec,
 )
+from .helpers import get_field_value
 
 __all__ = [
     "display_data",
@@ -20,4 +21,6 @@ __all__ = [
     "extract",
     "extract_with_spec",
     "ExpandMode",
+    # helpers
+    "get_field_value",
 ]

dtflow-0.5.4/dtflow/utils/helpers.py

@@ -0,0 +1,30 @@
+"""公共辅助函数"""
+
+from typing import Any
+
+
+def get_field_value(item: Any, field: str, default: Any = "") -> Any:
+    """
+    获取字段值，支持 DictWrapper 和普通 dict。
+
+    优先尝试 dict.get()，如果没有 get 方法则使用 getattr()。
+
+    Args:
+        item: 数据对象（dict 或 DictWrapper）
+        field: 字段名
+        default: 默认值
+
+    Returns:
+        字段值或默认值
+
+    Examples:
+        >>> get_field_value({"name": "test"}, "name")
+        'test'
+        >>> get_field_value({"name": ""}, "name", "default")
+        'default'
+    """
+    if hasattr(item, "get"):
+        value = item.get(field, default)
+    else:
+        value = getattr(item, field, default)
+    return value if value else default

{dtflow-0.5.2 → dtflow-0.5.4}/pyproject.toml

@@ -300,8 +300,10 @@ ignore_missing_imports = true
 
 # Ruff configuration (optional alternative to flake8)
 [tool.ruff]
-target-version = "py37"
+target-version = "py38"
 line-length = 100
+
+[tool.ruff.lint]
 select = [
     "E",  # pycodestyle errors
     "W",  # pycodestyle warnings
@@ -311,13 +313,13 @@ select = [
     "B",  # flake8-bugbear
 ]
 ignore = [
-    "E501",  # line too long, handled by black
+    "E501",  # line too long, handled by ruff-format
     "B008",  # do not perform function calls in argument defaults
     "C901",  # too complex
 ]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["F401"]
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 known-first-party = ["dtflow"]