dtflow 0.4.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

dtflow/__init__.py

@@ -42,7 +42,7 @@ from .tokenizers import (
     token_stats,
 )
 
-__version__ = "0.4.0"
+__version__ = "0.4.1"
 
 __all__ = [
     # core

dtflow/cli/commands.py

@@ -796,6 +796,17 @@ def _generate_default_transform(field_names: List[str]) -> str:
     return "\n".join(lines) if lines else "        # 在这里定义输出字段"
 
 
+def _unwrap(obj: Any) -> Any:
+    """递归将 DictWrapper 转换为普通 dict"""
+    if hasattr(obj, "to_dict"):
+        return _unwrap(obj.to_dict())
+    if isinstance(obj, dict):
+        return {k: _unwrap(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_unwrap(v) for v in obj]
+    return obj
+
+
 def _execute_transform(
     input_path: Path,
     config_path: Path,
@@ -829,7 +840,8 @@ def _execute_transform(
         try:
             # 包装转换函数以支持属性访问（配置文件中定义的 Item 类）
             def wrapped_transform(item):
-                return transform_func(DictWrapper(item))
+                result = transform_func(DictWrapper(item))
+                return _unwrap(result)
 
             st = load_stream(str(input_path))
             if num:
@@ -926,7 +938,8 @@ def _execute_preset_transform(
         try:
             # 包装转换函数以支持属性访问
             def wrapped_transform(item):
-                return transform_func(DictWrapper(item))
+                result = transform_func(DictWrapper(item))
+                return _unwrap(result)
 
             st = load_stream(str(input_path))
             if num:

dtflow/streaming.py

@@ -84,6 +84,8 @@ class StreamingTransformer:
         self._source_path = source_path
         self._total = total
         self._operations: List[Dict[str, Any]] = []
+        self._error_count = 0
+        self._first_error: Optional[str] = None
 
     @classmethod
     def load_stream(cls, filepath: str, batch_size: int = 10000) -> "StreamingTransformer":
@@ -194,17 +196,20 @@ class StreamingTransformer:
         Returns:
             新的 StreamingTransformer（惰性，不立即执行）
         """
+        # transform 是 1:1 转换，保留 total
+        new_st = StreamingTransformer(iter([]), self._source_path, total=self._total)
+        new_st._operations = self._operations + [{"type": "transform", "func": func}]
 
         def transformed_iterator():
             for item in self._iterator:
                 try:
                     yield func(item)
-                except Exception:
-                    pass  # 跳过错误
+                except Exception as e:
+                    new_st._error_count += 1
+                    if new_st._first_error is None:
+                        new_st._first_error = f"{type(e).__name__}: {e}"
 
-        # transform 是 1:1 转换，保留 total
-        new_st = StreamingTransformer(transformed_iterator(), self._source_path, total=self._total)
-        new_st._operations = self._operations + [{"type": "transform", "func": func}]
+        new_st._iterator = transformed_iterator()
         return new_st
 
     def head(self, n: int) -> "StreamingTransformer":
@@ -299,16 +304,21 @@ class StreamingTransformer:
         ext = path.suffix.lower()
 
         if ext == ".jsonl":
-            return self._save_jsonl(filepath, show_progress)
+            count = self._save_jsonl(filepath, show_progress)
         elif ext == ".csv":
-            return self._save_batched(filepath, "csv", batch_size, show_progress)
+            count = self._save_batched(filepath, "csv", batch_size, show_progress)
         elif ext == ".parquet":
-            return self._save_batched(filepath, "parquet", batch_size, show_progress)
+            count = self._save_batched(filepath, "parquet", batch_size, show_progress)
         elif ext in (".arrow", ".feather"):
-            return self._save_batched(filepath, "arrow", batch_size, show_progress)
+            count = self._save_batched(filepath, "arrow", batch_size, show_progress)
         else:
-            # 默认 JSONL
-            return self._save_jsonl(filepath, show_progress)
+            count = self._save_jsonl(filepath, show_progress)
+
+        # 打印错误摘要
+        if self._error_count > 0:
+            print(f"⚠️  跳过 {self._error_count} 条错误记录: {self._first_error}")
+
+        return count
 
     def _save_jsonl(self, filepath: str, show_progress: bool) -> int:
         """JSONL 逐行流式保存（使用 orjson）"""

{dtflow-0.4.0.dist-info → dtflow-0.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.4.0
+Version: 0.4.1
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme

{dtflow-0.4.0.dist-info → dtflow-0.4.1.dist-info}/RECORD

@@ -1,14 +1,14 @@
-dtflow/__init__.py,sha256=OF6TdEQPvEpcAsuBBsHeycXo6OfDY_Ar_YWaMPhiBFI,2347
+dtflow/__init__.py,sha256=Gd9Us_BDXaxmMIGlz51E6OZDohqzweOrvB-2j8k3KVs,2347
 dtflow/__main__.py,sha256=7lKluJTruDPN4CKSK2mWLUxSUlVLtkrqXyRMjlGk7SY,10595
 dtflow/converters.py,sha256=gyy-K15zjzGBawFnZa8D9JX37JZ47rey2GhjKa2pxFo,22081
 dtflow/core.py,sha256=szm9qmRVe1Q97O18UTGz7xTsdV-V8L4D6Bl1bxBJCWk,28778
 dtflow/lineage.py,sha256=vQ06lxBHftu-Ma5HlISp3F2eiIvwagQSnUGaLeABDZY,12190
 dtflow/pipeline.py,sha256=zZaC4fg5vsp_30Fhbg75vu0yggsdvf28bWBiVDWzZ6Y,13901
 dtflow/presets.py,sha256=OP1nnM5NFk5Kli9FsXK0xAot48E5OQ6-VOIJT9ffXPg,5023
-dtflow/streaming.py,sha256=O8waTXDOEk_6ES_H3-TKTc3zyc-EC8DjOfgepAKV96A,21556
+dtflow/streaming.py,sha256=lYf9gi5U-3oqr7oEe5mENx1r-LtRb2YfGNq1fP3_sw4,21972
 dtflow/tokenizers.py,sha256=zxE6XZGjZ_DOGCjRSClI9xaAbFVf8FS6jwwssGoi_9U,18111
 dtflow/cli/__init__.py,sha256=QhZ-thgx9IBTFII7T_hdoWFUl0CCsdGQHN5ZEZw2XB0,423
-dtflow/cli/commands.py,sha256=1NEHcwNq68le-YEy70j5bacn4RLWSJj_HWcZkOUl2bI,84537
+dtflow/cli/commands.py,sha256=8t_HgFuFqGt1HXPpEDV47qB2fwMD5C6d9Bjj-VNb37I,84958
 dtflow/mcp/__init__.py,sha256=huEJ3rXDbxDRjsLPEvjNT2u3tWs6Poiv6fokPIrByjw,897
 dtflow/mcp/__main__.py,sha256=PoT2ZZmJq9xDZxDACJfqDW9Ld_ukHrGNK-0XUd7WGnY,448
 dtflow/mcp/cli.py,sha256=ck0oOS_642cNktxULaMRE7BJfMxsBCwotmCj3PSPwVk,13110
@@ -19,7 +19,7 @@ dtflow/storage/io.py,sha256=XNWLL10a7jgOjM1IfTN9kIuW23dwzFE1nnaw4E3LaiU,21885
 dtflow/utils/__init__.py,sha256=f8v9HJZMWRI5AL64Vjr76Pf2Na_whOF9nJBKgPbXXYg,429
 dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
 dtflow/utils/field_path.py,sha256=WcNA-LZh3H61a77FEzB_R7YAyyZl3M8ofdq05ytQGmI,7459
-dtflow-0.4.0.dist-info/METADATA,sha256=HJhvSYxPG6wmYZPx0qLVQLSbmyK0CIp8qiu3ppe65mA,18306
-dtflow-0.4.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dtflow-0.4.0.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
-dtflow-0.4.0.dist-info/RECORD,,
+dtflow-0.4.1.dist-info/METADATA,sha256=-rdgDNFMy3pPO5mpMcKlB_quxSlD9mUIoe_tIUXoPP4,18306
+dtflow-0.4.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dtflow-0.4.1.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
+dtflow-0.4.1.dist-info/RECORD,,