dtflow 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

dtflow/__init__.py

@@ -42,7 +42,7 @@ from .tokenizers import (
     token_stats,
 )
 
-__version__ = "0.4.0"
+__version__ = "0.4.2"
 
 __all__ = [
     # core

dtflow/__main__.py

@@ -56,7 +56,8 @@ app = typer.Typer(
 @app.command()
 def sample(
     filename: str = typer.Argument(..., help="输入文件路径"),
-    num: int = typer.Argument(10, help="采样数量"),
+    num_arg: Optional[int] = typer.Argument(None, help="采样数量", metavar="NUM"),
+    num: int = typer.Option(10, "--num", "-n", help="采样数量", show_default=True),
     type: str = typer.Option("head", "--type", "-t", help="采样方式: random/head/tail"),
     output: Optional[str] = typer.Option(None, "--output", "-o", help="输出文件路径"),
     seed: Optional[int] = typer.Option(None, "--seed", help="随机种子"),
@@ -65,29 +66,36 @@ def sample(
     fields: Optional[str] = typer.Option(None, "--fields", "-f", help="只显示指定字段（逗号分隔）"),
 ):
     """从数据文件中采样指定数量的数据"""
-    _sample(filename, num, type, output, seed, by, uniform, fields)
+    actual_num = num_arg if num_arg is not None else num
+    _sample(filename, actual_num, type, output, seed, by, uniform, fields)
 
 
 @app.command()
 def head(
     filename: str = typer.Argument(..., help="输入文件路径"),
-    num: int = typer.Argument(10, help="显示数量"),
+    num_arg: Optional[int] = typer.Argument(None, help="显示数量", metavar="NUM"),
+    num: int = typer.Option(10, "--num", "-n", help="显示数量", show_default=True),
     output: Optional[str] = typer.Option(None, "--output", "-o", help="输出文件路径"),
     fields: Optional[str] = typer.Option(None, "--fields", "-f", help="只显示指定字段"),
 ):
     """显示文件的前 N 条数据"""
-    _head(filename, num, output, fields)
+    # 位置参数优先于选项参数
+    actual_num = num_arg if num_arg is not None else num
+    _head(filename, actual_num, output, fields)
 
 
 @app.command()
 def tail(
     filename: str = typer.Argument(..., help="输入文件路径"),
-    num: int = typer.Argument(10, help="显示数量"),
+    num_arg: Optional[int] = typer.Argument(None, help="显示数量", metavar="NUM"),
+    num: int = typer.Option(10, "--num", "-n", help="显示数量", show_default=True),
     output: Optional[str] = typer.Option(None, "--output", "-o", help="输出文件路径"),
     fields: Optional[str] = typer.Option(None, "--fields", "-f", help="只显示指定字段"),
 ):
     """显示文件的后 N 条数据"""
-    _tail(filename, num, output, fields)
+    # 位置参数优先于选项参数
+    actual_num = num_arg if num_arg is not None else num
+    _tail(filename, actual_num, output, fields)
 
 
 # ============ 数据转换命令 ============
@@ -161,9 +169,10 @@ def clean(
 def stats(
     filename: str = typer.Argument(..., help="输入文件路径"),
     top: int = typer.Option(10, "--top", "-n", help="显示 Top N 值"),
+    full: bool = typer.Option(False, "--full", "-f", help="完整模式：统计值分布、唯一值等详细信息"),
 ):
     """显示数据文件的统计信息"""
-    _stats(filename, top)
+    _stats(filename, top, full)
 
 
 @app.command("token-stats")

dtflow/cli/commands.py

@@ -796,6 +796,17 @@ def _generate_default_transform(field_names: List[str]) -> str:
     return "\n".join(lines) if lines else "        # 在这里定义输出字段"
 
 
+def _unwrap(obj: Any) -> Any:
+    """递归将 DictWrapper 转换为普通 dict"""
+    if hasattr(obj, "to_dict"):
+        return _unwrap(obj.to_dict())
+    if isinstance(obj, dict):
+        return {k: _unwrap(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_unwrap(v) for v in obj]
+    return obj
+
+
 def _execute_transform(
     input_path: Path,
     config_path: Path,
@@ -829,7 +840,8 @@ def _execute_transform(
         try:
             # 包装转换函数以支持属性访问（配置文件中定义的 Item 类）
             def wrapped_transform(item):
-                return transform_func(DictWrapper(item))
+                result = transform_func(DictWrapper(item))
+                return _unwrap(result)
 
             st = load_stream(str(input_path))
             if num:
@@ -926,7 +938,8 @@ def _execute_preset_transform(
         try:
             # 包装转换函数以支持属性访问
             def wrapped_transform(item):
-                return transform_func(DictWrapper(item))
+                result = transform_func(DictWrapper(item))
+                return _unwrap(result)
 
             st = load_stream(str(input_path))
             if num:
@@ -1276,17 +1289,23 @@ def _concat_streaming(file_paths: List[Path], output: str) -> int:
 def stats(
     filename: str,
     top: int = 10,
+    full: bool = False,
 ) -> None:
     """
-    显示数据文件的统计信息（类似 pandas df.info() + df.describe()）。
+    显示数据文件的统计信息。
+
+    默认快速模式：只统计行数和字段结构。
+    完整模式（--full）：统计值分布、唯一值、长度等详细信息。
 
     Args:
         filename: 输入文件路径，支持 csv/excel/jsonl/json/parquet/arrow/feather 格式
-        top: 显示频率最高的前 N 个值，默认 10
+        top: 显示频率最高的前 N 个值，默认 10（仅完整模式）
+        full: 完整模式，统计值分布、唯一值等详细信息
 
     Examples:
-        dt stats data.jsonl
-        dt stats data.csv --top=5
+        dt stats data.jsonl            # 快速模式（默认）
+        dt stats data.jsonl --full     # 完整模式
+        dt stats data.csv -f --top=5   # 完整模式，显示 Top 5
     """
     filepath = Path(filename)
 
@@ -1297,6 +1316,10 @@ def stats(
     if not _check_file_format(filepath):
         return
 
+    if not full:
+        _quick_stats(filepath)
+        return
+
     # 加载数据
     try:
         data = load_data(str(filepath))
@@ -1316,6 +1339,142 @@ def stats(
     _print_stats(filepath.name, total, field_stats)
 
 
+def _quick_stats(filepath: Path) -> None:
+    """
+    快速统计模式：只统计行数和字段结构，不遍历全部数据。
+
+    特点:
+    - 使用流式计数，不加载全部数据到内存
+    - 只读取前几条数据来推断字段结构
+    - 不计算值分布、唯一值等耗时统计
+    """
+    import orjson
+
+    from ..streaming import _count_rows_fast
+
+    ext = filepath.suffix.lower()
+    file_size = filepath.stat().st_size
+
+    # 格式化文件大小
+    def format_size(size: int) -> str:
+        for unit in ["B", "KB", "MB", "GB"]:
+            if size < 1024:
+                return f"{size:.1f} {unit}"
+            size /= 1024
+        return f"{size:.1f} TB"
+
+    # 快速统计行数
+    total = _count_rows_fast(str(filepath))
+    if total is None:
+        # 回退：手动计数
+        total = 0
+        try:
+            with open(filepath, "rb") as f:
+                for line in f:
+                    if line.strip():
+                        total += 1
+        except Exception:
+            total = -1
+
+    # 读取前几条数据推断字段结构
+    sample_data = []
+    sample_size = 5
+    try:
+        if ext == ".jsonl":
+            with open(filepath, "rb") as f:
+                for i, line in enumerate(f):
+                    if i >= sample_size:
+                        break
+                    line = line.strip()
+                    if line:
+                        sample_data.append(orjson.loads(line))
+        elif ext == ".csv":
+            import polars as pl
+
+            df = pl.scan_csv(str(filepath)).head(sample_size).collect()
+            sample_data = df.to_dicts()
+        elif ext == ".parquet":
+            import polars as pl
+
+            df = pl.scan_parquet(str(filepath)).head(sample_size).collect()
+            sample_data = df.to_dicts()
+        elif ext in (".arrow", ".feather"):
+            import polars as pl
+
+            df = pl.scan_ipc(str(filepath)).head(sample_size).collect()
+            sample_data = df.to_dicts()
+        elif ext == ".json":
+            with open(filepath, "rb") as f:
+                data = orjson.loads(f.read())
+                if isinstance(data, list):
+                    sample_data = data[:sample_size]
+    except Exception:
+        pass
+
+    # 分析字段结构
+    fields = []
+    if sample_data:
+        all_keys = set()
+        for item in sample_data:
+            all_keys.update(item.keys())
+
+        for key in sorted(all_keys):
+            # 从采样数据中推断类型
+            sample_values = [item.get(key) for item in sample_data if key in item]
+            non_null = [v for v in sample_values if v is not None]
+            if non_null:
+                field_type = _infer_type(non_null)
+            else:
+                field_type = "unknown"
+            fields.append({"field": key, "type": field_type})
+
+    # 输出
+    try:
+        from rich.console import Console
+        from rich.panel import Panel
+        from rich.table import Table
+
+        console = Console()
+
+        # 概览
+        console.print(
+            Panel(
+                f"[bold]文件:[/bold] {filepath.name}\n"
+                f"[bold]大小:[/bold] {format_size(file_size)}\n"
+                f"[bold]总数:[/bold] {total:,} 条\n"
+                f"[bold]字段:[/bold] {len(fields)} 个",
+                title="📊 快速统计",
+                expand=False,
+            )
+        )
+
+        if fields:
+            table = Table(title="📋 字段结构", show_header=True, header_style="bold cyan")
+            table.add_column("#", style="dim", justify="right")
+            table.add_column("字段", style="green")
+            table.add_column("类型", style="yellow")
+
+            for i, f in enumerate(fields, 1):
+                table.add_row(str(i), f["field"], f["type"])
+
+            console.print(table)
+
+    except ImportError:
+        # 没有 rich，使用普通打印
+        print(f"\n{'=' * 40}")
+        print("📊 快速统计")
+        print(f"{'=' * 40}")
+        print(f"文件: {filepath.name}")
+        print(f"大小: {format_size(file_size)}")
+        print(f"总数: {total:,} 条")
+        print(f"字段: {len(fields)} 个")
+
+        if fields:
+            print(f"\n📋 字段结构:")
+            for i, f in enumerate(fields, 1):
+                print(f"  {i}. {f['field']} ({f['type']})")
+
+
 def _compute_field_stats(data: List[Dict], top: int) -> List[Dict[str, Any]]:
     """
     单次遍历计算每个字段的统计信息。
@@ -1397,22 +1556,34 @@ def _count_unique(values: List[Any], field_type: str) -> int:
     """
     计算唯一值数量。
 
-    对于简单类型直接比较，对于 list/dict 使用 hash 节省内存。
+    对于简单类型直接比较，对于 list/dict 或混合类型使用 hash。
     """
     if field_type in ("list", "dict"):
-        # 复杂类型：使用 orjson 序列化后计算 hash
-        import hashlib
+        return _count_unique_by_hash(values)
+    else:
+        # 简单类型：尝试直接比较，失败则回退到 hash 方式
+        try:
+            return len(set(values))
+        except TypeError:
+            # 混合类型（如字段中既有 str 又有 dict），回退到 hash
+            return _count_unique_by_hash(values)
+
 
-        import orjson
+def _count_unique_by_hash(values: List[Any]) -> int:
+    """使用 orjson 序列化后计算 hash 来统计唯一值"""
+    import hashlib
 
-        seen = set()
-        for v in values:
+    import orjson
+
+    seen = set()
+    for v in values:
+        try:
             h = hashlib.md5(orjson.dumps(v, option=orjson.OPT_SORT_KEYS)).digest()
             seen.add(h)
-        return len(seen)
-    else:
-        # 简单类型：直接比较
-        return len(set(values))
+        except TypeError:
+            # 无法序列化的值，用 repr 兜底
+            seen.add(repr(v))
+    return len(seen)
 
 
 def _infer_type(values: List[Any]) -> str:

dtflow/streaming.py

@@ -84,6 +84,8 @@ class StreamingTransformer:
         self._source_path = source_path
         self._total = total
         self._operations: List[Dict[str, Any]] = []
+        self._error_count = 0
+        self._first_error: Optional[str] = None
 
     @classmethod
     def load_stream(cls, filepath: str, batch_size: int = 10000) -> "StreamingTransformer":
@@ -194,17 +196,20 @@ class StreamingTransformer:
         Returns:
             新的 StreamingTransformer（惰性，不立即执行）
         """
+        # transform 是 1:1 转换，保留 total
+        new_st = StreamingTransformer(iter([]), self._source_path, total=self._total)
+        new_st._operations = self._operations + [{"type": "transform", "func": func}]
 
         def transformed_iterator():
             for item in self._iterator:
                 try:
                     yield func(item)
-                except Exception:
-                    pass  # 跳过错误
+                except Exception as e:
+                    new_st._error_count += 1
+                    if new_st._first_error is None:
+                        new_st._first_error = f"{type(e).__name__}: {e}"
 
-        # transform 是 1:1 转换，保留 total
-        new_st = StreamingTransformer(transformed_iterator(), self._source_path, total=self._total)
-        new_st._operations = self._operations + [{"type": "transform", "func": func}]
+        new_st._iterator = transformed_iterator()
         return new_st
 
     def head(self, n: int) -> "StreamingTransformer":
@@ -299,16 +304,21 @@ class StreamingTransformer:
         ext = path.suffix.lower()
 
         if ext == ".jsonl":
-            return self._save_jsonl(filepath, show_progress)
+            count = self._save_jsonl(filepath, show_progress)
         elif ext == ".csv":
-            return self._save_batched(filepath, "csv", batch_size, show_progress)
+            count = self._save_batched(filepath, "csv", batch_size, show_progress)
         elif ext == ".parquet":
-            return self._save_batched(filepath, "parquet", batch_size, show_progress)
+            count = self._save_batched(filepath, "parquet", batch_size, show_progress)
         elif ext in (".arrow", ".feather"):
-            return self._save_batched(filepath, "arrow", batch_size, show_progress)
+            count = self._save_batched(filepath, "arrow", batch_size, show_progress)
         else:
-            # 默认 JSONL
-            return self._save_jsonl(filepath, show_progress)
+            count = self._save_jsonl(filepath, show_progress)
+
+        # 打印错误摘要
+        if self._error_count > 0:
+            print(f"⚠️  跳过 {self._error_count} 条错误记录: {self._first_error}")
+
+        return count
 
     def _save_jsonl(self, filepath: str, show_progress: bool) -> int:
         """JSONL 逐行流式保存（使用 orjson）"""

{dtflow-0.4.0.dist-info → dtflow-0.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.4.0
+Version: 0.4.2
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme

{dtflow-0.4.0.dist-info → dtflow-0.4.2.dist-info}/RECORD

@@ -1,14 +1,14 @@
-dtflow/__init__.py,sha256=OF6TdEQPvEpcAsuBBsHeycXo6OfDY_Ar_YWaMPhiBFI,2347
-dtflow/__main__.py,sha256=7lKluJTruDPN4CKSK2mWLUxSUlVLtkrqXyRMjlGk7SY,10595
+dtflow/__init__.py,sha256=outMIxwn2hlX6wMUfLFrBkDVknGEzJOkNUHOUWfEGuw,2347
+dtflow/__main__.py,sha256=eVzGbJQxhoOvvmk1076AzT7VscOXSJu18VGGdZ9r9h8,11359
 dtflow/converters.py,sha256=gyy-K15zjzGBawFnZa8D9JX37JZ47rey2GhjKa2pxFo,22081
 dtflow/core.py,sha256=szm9qmRVe1Q97O18UTGz7xTsdV-V8L4D6Bl1bxBJCWk,28778
 dtflow/lineage.py,sha256=vQ06lxBHftu-Ma5HlISp3F2eiIvwagQSnUGaLeABDZY,12190
 dtflow/pipeline.py,sha256=zZaC4fg5vsp_30Fhbg75vu0yggsdvf28bWBiVDWzZ6Y,13901
 dtflow/presets.py,sha256=OP1nnM5NFk5Kli9FsXK0xAot48E5OQ6-VOIJT9ffXPg,5023
-dtflow/streaming.py,sha256=O8waTXDOEk_6ES_H3-TKTc3zyc-EC8DjOfgepAKV96A,21556
+dtflow/streaming.py,sha256=lYf9gi5U-3oqr7oEe5mENx1r-LtRb2YfGNq1fP3_sw4,21972
 dtflow/tokenizers.py,sha256=zxE6XZGjZ_DOGCjRSClI9xaAbFVf8FS6jwwssGoi_9U,18111
 dtflow/cli/__init__.py,sha256=QhZ-thgx9IBTFII7T_hdoWFUl0CCsdGQHN5ZEZw2XB0,423
-dtflow/cli/commands.py,sha256=1NEHcwNq68le-YEy70j5bacn4RLWSJj_HWcZkOUl2bI,84537
+dtflow/cli/commands.py,sha256=IZ2rHnJ7RHmXOW19JLjtHPfzbfNj5vq_FT2YDSao2SI,90303
 dtflow/mcp/__init__.py,sha256=huEJ3rXDbxDRjsLPEvjNT2u3tWs6Poiv6fokPIrByjw,897
 dtflow/mcp/__main__.py,sha256=PoT2ZZmJq9xDZxDACJfqDW9Ld_ukHrGNK-0XUd7WGnY,448
 dtflow/mcp/cli.py,sha256=ck0oOS_642cNktxULaMRE7BJfMxsBCwotmCj3PSPwVk,13110
@@ -19,7 +19,7 @@ dtflow/storage/io.py,sha256=XNWLL10a7jgOjM1IfTN9kIuW23dwzFE1nnaw4E3LaiU,21885
 dtflow/utils/__init__.py,sha256=f8v9HJZMWRI5AL64Vjr76Pf2Na_whOF9nJBKgPbXXYg,429
 dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
 dtflow/utils/field_path.py,sha256=WcNA-LZh3H61a77FEzB_R7YAyyZl3M8ofdq05ytQGmI,7459
-dtflow-0.4.0.dist-info/METADATA,sha256=HJhvSYxPG6wmYZPx0qLVQLSbmyK0CIp8qiu3ppe65mA,18306
-dtflow-0.4.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dtflow-0.4.0.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
-dtflow-0.4.0.dist-info/RECORD,,
+dtflow-0.4.2.dist-info/METADATA,sha256=Rck3BDh1Vvpr24fUvCAcnmPXQOrZxTu_OYMAvJHADnU,18306
+dtflow-0.4.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dtflow-0.4.2.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
+dtflow-0.4.2.dist-info/RECORD,,