dtflow 0.5.3__py3-none-any.whl → 0.5.4__py3-none-any.whl

dtflow/__init__.py

@@ -60,7 +60,7 @@ from .tokenizers import (
     token_stats,
 )
 
-__version__ = "0.5.3"
+__version__ = "0.5.4"
 
 __all__ = [
     # core

dtflow/cli/common.py

@@ -100,8 +100,6 @@ def _format_nested(
     └─ 最后一项
     """
     lines = []
-    branch = "└─ " if is_last else "├─ "
-    cont = "   " if is_last else "│  "
 
     if isinstance(value, dict):
         items = list(value.items())
@@ -183,6 +181,7 @@ def _print_samples(
     filename: Optional[str] = None,
     total_count: Optional[int] = None,
     fields: Optional[List[str]] = None,
+    file_size: Optional[int] = None,
 ) -> None:
     """
     打印采样结果。
@@ -190,8 +189,9 @@ def _print_samples(
     Args:
         samples: 采样数据列表
         filename: 文件名（用于显示概览）
-        total_count: 文件总行数（用于显示概览）
+        total_count: 文件总行数（用于显示概览），大文件时可能为 None
         fields: 只显示指定字段
+        file_size: 文件大小（字节），当 total_count 为 None 时显示
     """
     if not samples:
         print("没有数据")
@@ -219,6 +219,8 @@ def _print_samples(
 
             if total_count is not None:
                 info = f"总行数: {total_count:,} | 采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
+            elif file_size is not None:
+                info = f"文件大小: {_format_file_size(file_size)} | 采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
             else:
                 info = f"采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
 
@@ -266,6 +268,10 @@ def _print_samples(
                 print(
                     f"   总行数: {total_count:,} | 采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
                 )
+            elif file_size is not None:
+                print(
+                    f"   文件大小: {_format_file_size(file_size)} | 采样: {len(samples)} 条 | 字段: {len(all_fields)} 个"
+                )
             else:
                 print(f"   采样: {len(samples)} 条 | 字段: {len(all_fields)} 个")
             print(f"   字段: {', '.join(sorted(all_fields))}")
@@ -287,6 +293,15 @@ def _parse_field_list(value: Any) -> List[str]:
         return [str(value)]
 
 
+def _format_file_size(size: int) -> str:
+    """格式化文件大小"""
+    for unit in ["B", "KB", "MB", "GB"]:
+        if size < 1024:
+            return f"{size:.1f} {unit}"
+        size /= 1024
+    return f"{size:.1f} TB"
+
+
 def _is_empty_value(v: Any) -> bool:
     """判断值是否为空"""
     if v is None:

dtflow/cli/sample.py

@@ -99,11 +99,15 @@ def sample(
         for item in sampled:
             print(orjson.dumps(item, option=orjson.OPT_INDENT_2).decode("utf-8"))
     else:
-        # 获取文件总行数用于显示
-        total_count = _get_file_row_count(filepath)
+        # 大文件跳过行数统计（50MB 阈值）
+        file_size = filepath.stat().st_size
+        if file_size < 50 * 1024 * 1024:
+            total_count = _get_file_row_count(filepath)
+        else:
+            total_count = None
         # 解析 fields 参数
         field_list = _parse_field_list(fields) if fields else None
-        _print_samples(sampled, filepath.name, total_count, field_list)
+        _print_samples(sampled, filepath.name, total_count, field_list, file_size)
 
 
 def _stratified_sample(
@@ -196,7 +200,7 @@ def _stratified_sample(
 
     # 执行各组采样
     result = []
-    print(f"🔄 执行采样...")
+    print("🔄 执行采样...")
     for key in group_keys:
         group_data = groups[key]
         target = min(sample_counts[key], len(group_data))
@@ -215,7 +219,7 @@ def _stratified_sample(
         result.extend(sampled)
 
     # 打印采样结果
-    print(f"\n📋 采样结果:")
+    print("\n📋 采样结果:")
     result_groups: Dict[Any, int] = defaultdict(int)
     for item in result:
         key = item.get(stratify_field, "__null__")

dtflow/converters.py

@@ -23,8 +23,8 @@ def to_hf_dataset(data: List[Dict[str, Any]]):
     """
     try:
         from datasets import Dataset
-    except ImportError:
-        raise ImportError("需要安装 datasets: pip install datasets")
+    except ImportError as e:
+        raise ImportError("需要安装 datasets: pip install datasets") from e
 
     return Dataset.from_list(data)
 
@@ -45,9 +45,9 @@ def from_hf_dataset(dataset, split: Optional[str] = None) -> List[Dict[str, Any]
         >>> data = from_hf_dataset(my_dataset, split="train")
     """
     try:
-        from datasets import Dataset, DatasetDict, load_dataset
-    except ImportError:
-        raise ImportError("需要安装 datasets: pip install datasets")
+        from datasets import load_dataset
+    except ImportError as e:
+        raise ImportError("需要安装 datasets: pip install datasets") from e
 
     # 如果是字符串，加载数据集
     if isinstance(dataset, str):
@@ -198,7 +198,8 @@ def to_llama_factory(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f, "") if hasattr(item, "get") else getattr(item, f, "")
+        def get(f):
+            return item.get(f, "") if hasattr(item, "get") else getattr(item, f, "")
 
         result = {
             "instruction": get(instruction_field),
@@ -316,7 +317,9 @@ def to_llama_factory_sharegpt(
     }
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f, "") if hasattr(item, "get") else getattr(item, f, "")
+        def get(f):
+            return item.get(f, "") if hasattr(item, "get") else getattr(item, f, "")
+
         messages = get(messages_field) or []
 
         conversations = []
@@ -389,7 +392,9 @@ def to_llama_factory_vlm(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+
         messages = get(messages_field) or []
 
         instruction = ""
@@ -471,7 +476,9 @@ def to_llama_factory_vlm_sharegpt(
     role_map = {"user": "human", "assistant": "gpt", "system": "system"}
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+
         messages = get(messages_field) or []
 
         conversations = []
@@ -545,7 +552,9 @@ def to_swift_messages(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+
         messages = get(messages_field) or []
 
         # 复制 messages，避免修改原数据
@@ -604,7 +613,8 @@ def to_swift_query_response(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
 
         query = get(query_field)
         response = get(response_field)
@@ -617,7 +627,7 @@ def to_swift_query_response(
             current_query = ""
             current_response = ""
 
-            for i, msg in enumerate(messages):
+            for _i, msg in enumerate(messages):
                 role = msg.get("role", "")
                 content = msg.get("content", "")
 
@@ -697,7 +707,9 @@ def to_swift_vlm(
     """
 
     def transform(item) -> dict:
-        get = lambda f: item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+        def get(f):
+            return item.get(f) if hasattr(item, "get") else getattr(item, f, None)
+
         messages = get(messages_field) or []
 
         result_messages = []

{dtflow-0.5.3.dist-info → dtflow-0.5.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.5.3
+Version: 0.5.4
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme

{dtflow-0.5.3.dist-info → dtflow-0.5.4.dist-info}/RECORD

@@ -1,6 +1,6 @@
-dtflow/__init__.py,sha256=RJql_KmINJNbq2FEqU7jD9Z0c5ETkxQJPvUUPKiFt74,3031
+dtflow/__init__.py,sha256=yUwvKuVAmhDnp-1tYhZGlZcTdiEnZ3Jh-IJymgMIUhA,3031
 dtflow/__main__.py,sha256=ySpqvEn7k-vsrYFPx-8O6p-yx_24KccgnOSPd2XybhM,12572
-dtflow/converters.py,sha256=yXafSDeRC7DB2MMj8fD1NWjAG8HoAGh5Ay2A5Z7s6xA,22206
+dtflow/converters.py,sha256=X3qeFD7FCOMnfiP3MicL5MXimOm4XUYBs5pczIkudU0,22331
 dtflow/core.py,sha256=qMo6B3LK--TWRK7ZBKObGcs3pKFnd0NPoaM0T8JC7Jw,38135
 dtflow/framework.py,sha256=jyICi_RWHjX7WfsXdSbWmP1SL7y1OWSPyd5G5Y-lvg4,17578
 dtflow/lineage.py,sha256=jie3OL1qK90-_cOOqqLbhSJ1oGUktDM1x5HRpQ5Qiyc,12800
@@ -12,11 +12,11 @@ dtflow/tokenizers.py,sha256=7ZAelSmcDxLWH5kICgH9Q1ULH3_BfDZb9suHMjJJRZU,20589
 dtflow/cli/__init__.py,sha256=QhZ-thgx9IBTFII7T_hdoWFUl0CCsdGQHN5ZEZw2XB0,423
 dtflow/cli/clean.py,sha256=y9VCRibgK1j8WIY3h0XZX0m93EdELQC7TdnseMWwS-0,17799
 dtflow/cli/commands.py,sha256=ST65Ox_MKu-CKAtPVaxECAPXYOJiF7BhL32A4nsZZl0,1175
-dtflow/cli/common.py,sha256=nIPc9GBK61r6kmaI9OS3IyhcfPqShpDEHx1ddjFPnlM,13131
+dtflow/cli/common.py,sha256=gCwnF5Sw2ploqfZJO_z3Ms9mR1HNT7Lj6ydHn0uVaIw,13817
 dtflow/cli/io_ops.py,sha256=BMDisP6dxzzmSjYwmeFwaHmpHHPqirmXAWeNTD-9MQM,13254
 dtflow/cli/lineage.py,sha256=_lNh35nF9AA0Zy6FyZ4g8IzrXH2ZQnp3inF-o2Hs1pw,1383
 dtflow/cli/pipeline.py,sha256=QNEo-BJlaC1CVnVeRZr7TwfuZYloJ4TebIzJ5ALzry0,1426
-dtflow/cli/sample.py,sha256=vPTQlF0OXEry4QjO8uaD9vOae4AQbX9zDwVYOxg59ZI,10339
+dtflow/cli/sample.py,sha256=LRCkpFi9t0CI2QjRKADmvwWMdGfLriqdNkoFG6_wQkY,10497
 dtflow/cli/stats.py,sha256=u4ehCfgw1X8WuOyAjrApMRgcIO3BVmINbsTjxEscQro,24086
 dtflow/cli/transform.py,sha256=w6xqMOxPxQvL2u_BPCfpDHuPSC9gmcqMPVN8s-B6bbY,15052
 dtflow/cli/validate.py,sha256=65aGVlMS_Rq0Ch0YQ-TclVJ03RQP4CnG137wthzb8Ao,4384
@@ -31,7 +31,7 @@ dtflow/utils/__init__.py,sha256=Pn-ltwV04fBQmeZG7FxInDQmzH29LYOi90LgeLMEuQk,506
 dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
 dtflow/utils/field_path.py,sha256=K8nU196RxTSJ1OoieTWGcYOWl9KjGq2iSxCAkfjECuM,7621
 dtflow/utils/helpers.py,sha256=JXN176_B2pm53GLVyZ1wj3wrmBJG52Tkw6AMQSdj7M8,791
-dtflow-0.5.3.dist-info/METADATA,sha256=5joXihL8gkmnNEaUTqRpe0_U-y8osaIfdX0v91WVtK8,22544
-dtflow-0.5.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dtflow-0.5.3.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
-dtflow-0.5.3.dist-info/RECORD,,
+dtflow-0.5.4.dist-info/METADATA,sha256=mQIIV3B-6VBOuNSRiPQjqOwdLTs6Nir6to1_FIER3d0,22544
+dtflow-0.5.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dtflow-0.5.4.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
+dtflow-0.5.4.dist-info/RECORD,,