dtflow 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

dtflow/__init__.py

@@ -60,7 +60,7 @@ from .tokenizers import (
     token_stats,
 )
 
-__version__ = "0.5.0"
+__version__ = "0.5.2"
 
 __all__ = [
     # core

dtflow/cli/common.py

@@ -57,7 +57,7 @@ def _get_file_row_count(filepath: Path) -> Optional[int]:
     return None
 
 
-def _format_value(value: Any, max_len: int = 80) -> str:
+def _format_value(value: Any, max_len: int = 120) -> str:
     """格式化单个值，长文本截断。"""
     if value is None:
         return "[dim]null[/dim]"
@@ -66,18 +66,22 @@ def _format_value(value: Any, max_len: int = 80) -> str:
     if isinstance(value, (int, float)):
         return f"[cyan]{value}[/cyan]"
     if isinstance(value, str):
+        half_len = max_len // 2
         # 处理多行文本
         if "\n" in value:
             lines = value.split("\n")
-            if len(lines) > 3:
-                preview = lines[0][:max_len] + f"... [dim]({len(lines)} 行)[/dim]"
-            else:
-                preview = value.replace("\n", "\\n")
-                if len(preview) > max_len:
-                    preview = preview[:max_len] + "..."
+            preview = value.replace("\n", "\\n")
+            if len(preview) > max_len:
+                # 前半 + 省略标记 + 后半
+                head = preview[:half_len]
+                tail = preview[-half_len:]
+                return f'"{head} [yellow]<<<{len(lines)}行>>>[/yellow] {tail}"'
             return f'"{preview}"'
         if len(value) > max_len:
-            return f'"{value[:max_len]}..." [dim]({len(value)} 字符)[/dim]'
+            # 前半 + 省略标记 + 后半
+            head = value[:half_len]
+            tail = value[-half_len:]
+            return f'"{head} [yellow]<<<{len(value)}字符>>>[/yellow] {tail}"'
         return f'"{value}"'
     return str(value)
 
@@ -86,7 +90,7 @@ def _format_nested(
     value: Any,
     indent: str = "",
     is_last: bool = True,
-    max_len: int = 80,
+    max_len: int = 120,
 ) -> List[str]:
     """
     递归格式化嵌套结构，返回行列表。

dtflow/cli/stats.py

@@ -465,34 +465,65 @@ def token_stats(
         return
 
     total = len(data)
-    print(f"   共 {total} 条数据")
-    print(f"🔢 统计 Token (模型: {model}, 字段: {field})...")
+    print(f"   共 {total:,} 条数据")
 
     # 检查字段类型并选择合适的统计方法（支持嵌套路径）
     sample = data[0]
     field_value = get_field_with_spec(sample, field)
 
+    # 尝试使用 rich 进度条
     try:
-        if isinstance(field_value, list) and field_value and isinstance(field_value[0], dict):
-            # messages 格式
-            from ..tokenizers import messages_token_stats
-
-            stats_result = messages_token_stats(data, messages_field=field, model=model)
-            _print_messages_token_stats(stats_result, detailed)
-        else:
-            # 普通文本字段
-            from ..tokenizers import token_stats as compute_token_stats
-
-            stats_result = compute_token_stats(data, fields=field, model=model)
-            _print_text_token_stats(stats_result, detailed)
-    except ImportError as e:
-        print(f"错误: {e}")
-        return
-    except Exception as e:
-        print(f"错误: 统计失败 - {e}")
-        import traceback
+        from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
+
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[bold blue]统计 Token"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TextColumn(f"(模型: {model})"),
+        ) as progress:
+            task = progress.add_task("", total=total)
+
+            def update_progress(current: int, total_count: int):
+                progress.update(task, completed=current)
+
+            if isinstance(field_value, list) and field_value and isinstance(field_value[0], dict):
+                from ..tokenizers import messages_token_stats
+
+                stats_result = messages_token_stats(
+                    data, messages_field=field, model=model, progress_callback=update_progress
+                )
+                _print_messages_token_stats(stats_result, detailed)
+            else:
+                from ..tokenizers import token_stats as compute_token_stats
+
+                stats_result = compute_token_stats(
+                    data, fields=field, model=model, progress_callback=update_progress
+                )
+                _print_text_token_stats(stats_result, detailed)
+
+    except ImportError:
+        # 没有 rich，显示简单进度
+        print(f"🔢 统计 Token (模型: {model}, 字段: {field})...")
+        try:
+            if isinstance(field_value, list) and field_value and isinstance(field_value[0], dict):
+                from ..tokenizers import messages_token_stats
+
+                stats_result = messages_token_stats(data, messages_field=field, model=model)
+                _print_messages_token_stats(stats_result, detailed)
+            else:
+                from ..tokenizers import token_stats as compute_token_stats
 
-        traceback.print_exc()
+                stats_result = compute_token_stats(data, fields=field, model=model)
+                _print_text_token_stats(stats_result, detailed)
+        except ImportError as e:
+            print(f"错误: {e}")
+            return
+        except Exception as e:
+            print(f"错误: 统计失败 - {e}")
+            import traceback
+
+            traceback.print_exc()
 
 
 def _print_messages_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
@@ -505,21 +536,39 @@ def _print_messages_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
         console = Console()
 
         # 概览
+        std = stats.get("std_tokens", 0)
         overview = (
             f"[bold]总样本数:[/bold] {stats['count']:,}\n"
             f"[bold]总 Token:[/bold] {stats['total_tokens']:,}\n"
-            f"[bold]平均 Token:[/bold] {stats['avg_tokens']:,}\n"
-            f"[bold]中位数:[/bold] {stats['median_tokens']:,}\n"
+            f"[bold]平均 Token:[/bold] {stats['avg_tokens']:,} (std: {std:.1f})\n"
             f"[bold]范围:[/bold] {stats['min_tokens']:,} - {stats['max_tokens']:,}"
         )
         console.print(Panel(overview, title="📊 Token 统计概览", expand=False))
 
+        # 百分位数表格
+        table = Table(title="📈 分布统计")
+        table.add_column("百分位", style="cyan", justify="center")
+        table.add_column("Token 数", justify="right")
+        percentiles = [
+            ("Min", stats["min_tokens"]),
+            ("P25", stats.get("p25", "-")),
+            ("P50 (中位数)", stats.get("median_tokens", "-")),
+            ("P75", stats.get("p75", "-")),
+            ("P90", stats.get("p90", "-")),
+            ("P95", stats.get("p95", "-")),
+            ("P99", stats.get("p99", "-")),
+            ("Max", stats["max_tokens"]),
+        ]
+        for name, val in percentiles:
+            table.add_row(name, f"{val:,}" if isinstance(val, int) else str(val))
+        console.print(table)
+
         if detailed:
-            # 详细统计
-            table = Table(title="📋 分角色统计")
-            table.add_column("角色", style="cyan")
-            table.add_column("Token 数", justify="right")
-            table.add_column("占比", justify="right")
+            # 分角色统计
+            role_table = Table(title="📋 分角色统计")
+            role_table.add_column("角色", style="cyan")
+            role_table.add_column("Token 数", justify="right")
+            role_table.add_column("占比", justify="right")
 
             total = stats["total_tokens"]
             for role, key in [
@@ -529,22 +578,27 @@ def _print_messages_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
             ]:
                 tokens = stats.get(key, 0)
                 pct = tokens / total * 100 if total > 0 else 0
-                table.add_row(role, f"{tokens:,}", f"{pct:.1f}%")
+                role_table.add_row(role, f"{tokens:,}", f"{pct:.1f}%")
 
-            console.print(table)
+            console.print(role_table)
             console.print(f"\n平均对话轮数: {stats.get('avg_turns', 0)}")
 
     except ImportError:
         # 没有 rich，使用普通打印
+        std = stats.get("std_tokens", 0)
         print(f"\n{'=' * 40}")
         print("📊 Token 统计概览")
         print(f"{'=' * 40}")
         print(f"总样本数: {stats['count']:,}")
         print(f"总 Token: {stats['total_tokens']:,}")
-        print(f"平均 Token: {stats['avg_tokens']:,}")
-        print(f"中位数: {stats['median_tokens']:,}")
+        print(f"平均 Token: {stats['avg_tokens']:,} (std: {std:.1f})")
         print(f"范围: {stats['min_tokens']:,} - {stats['max_tokens']:,}")
 
+        print(f"\n📈 百分位分布:")
+        print(f"  P25: {stats.get('p25', '-'):,}  P50: {stats.get('median_tokens', '-'):,}")
+        print(f"  P75: {stats.get('p75', '-'):,}  P90: {stats.get('p90', '-'):,}")
+        print(f"  P95: {stats.get('p95', '-'):,}  P99: {stats.get('p99', '-'):,}")
+
         if detailed:
             print(f"\n{'=' * 40}")
             print("📋 分角色统计")
@@ -566,24 +620,48 @@ def _print_text_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
     try:
         from rich.console import Console
         from rich.panel import Panel
+        from rich.table import Table
 
         console = Console()
 
+        std = stats.get("std_tokens", 0)
         overview = (
             f"[bold]总样本数:[/bold] {stats['count']:,}\n"
             f"[bold]总 Token:[/bold] {stats['total_tokens']:,}\n"
-            f"[bold]平均 Token:[/bold] {stats['avg_tokens']:.1f}\n"
-            f"[bold]中位数:[/bold] {stats['median_tokens']:,}\n"
+            f"[bold]平均 Token:[/bold] {stats['avg_tokens']:.1f} (std: {std:.1f})\n"
             f"[bold]范围:[/bold] {stats['min_tokens']:,} - {stats['max_tokens']:,}"
         )
         console.print(Panel(overview, title="📊 Token 统计", expand=False))
 
+        # 百分位数表格
+        table = Table(title="📈 分布统计")
+        table.add_column("百分位", style="cyan", justify="center")
+        table.add_column("Token 数", justify="right")
+        percentiles = [
+            ("Min", stats["min_tokens"]),
+            ("P25", stats.get("p25", "-")),
+            ("P50 (中位数)", stats.get("median_tokens", "-")),
+            ("P75", stats.get("p75", "-")),
+            ("P90", stats.get("p90", "-")),
+            ("P95", stats.get("p95", "-")),
+            ("P99", stats.get("p99", "-")),
+            ("Max", stats["max_tokens"]),
+        ]
+        for name, val in percentiles:
+            table.add_row(name, f"{val:,}" if isinstance(val, int) else str(val))
+        console.print(table)
+
     except ImportError:
+        std = stats.get("std_tokens", 0)
         print(f"\n{'=' * 40}")
         print("📊 Token 统计")
         print(f"{'=' * 40}")
         print(f"总样本数: {stats['count']:,}")
         print(f"总 Token: {stats['total_tokens']:,}")
-        print(f"平均 Token: {stats['avg_tokens']:.1f}")
-        print(f"中位数: {stats['median_tokens']:,}")
+        print(f"平均 Token: {stats['avg_tokens']:.1f} (std: {std:.1f})")
         print(f"范围: {stats['min_tokens']:,} - {stats['max_tokens']:,}")
+
+        print(f"\n📈 百分位分布:")
+        print(f"  P25: {stats.get('p25', '-'):,}  P50: {stats.get('median_tokens', '-'):,}")
+        print(f"  P75: {stats.get('p75', '-'):,}  P90: {stats.get('p90', '-'):,}")
+        print(f"  P95: {stats.get('p95', '-'):,}  P99: {stats.get('p99', '-'):,}")

dtflow/core.py

@@ -793,19 +793,29 @@ class DataTransformer:
             seed: 随机种子
 
         Returns:
-            (train, test) 两个 DataTransformer
+            (train, test) 两个 DataTransformer，各自拥有独立的血缘追踪器
         """
         data = self.shuffle(seed).data
         split_idx = int(len(data) * ratio)
 
-        # 分割后血缘追踪器各自独立
+        # 分割后血缘追踪器各自独立（使用深拷贝避免相互影响）
         tracker = self._lineage_tracker
+        train_tracker = None
+        test_tracker = None
+
         if tracker:
             tracker.record("split", {"ratio": ratio, "seed": seed}, len(self._data), len(data))
+            # 为每个子数据集创建独立的追踪器副本
+            train_tracker = tracker.copy()
+            train_tracker.record("split_part", {"part": "train", "ratio": ratio}, len(data), split_idx)
+            test_tracker = tracker.copy()
+            test_tracker.record(
+                "split_part", {"part": "test", "ratio": 1 - ratio}, len(data), len(data) - split_idx
+            )
 
         return (
-            DataTransformer(data[:split_idx], _lineage_tracker=tracker),
-            DataTransformer(data[split_idx:], _lineage_tracker=tracker),
+            DataTransformer(data[:split_idx], _lineage_tracker=train_tracker),
+            DataTransformer(data[split_idx:], _lineage_tracker=test_tracker),
         )
 
     # ============ 并行处理 ============
@@ -815,6 +825,7 @@ class DataTransformer:
         func: Callable[[Dict], Any],
         workers: Optional[int] = None,
         chunksize: int = 1000,
+        timeout: Optional[float] = None,
     ) -> List[Any]:
         """
         并行执行转换函数（使用多进程）。
@@ -825,24 +836,46 @@ class DataTransformer:
             func: 转换函数，接收原始 dict，返回转换结果
             workers: 进程数，默认为 CPU 核心数
             chunksize: 每个进程处理的数据块大小
+            timeout: 超时时间（秒），None 表示无超时
 
         Returns:
             转换后的结果列表
 
+        Raises:
+            TypeError: 如果 func 无法被 pickle（如 lambda 函数）
+            RuntimeError: 如果子进程执行出错或超时
+
         Examples:
             >>> def transform(item):
             ...     return {"id": item["id"], "text": item["text"].upper()}
             >>> results = dt.map_parallel(transform)
         """
-        from multiprocessing import Pool, cpu_count
+        from multiprocessing import Pool, TimeoutError, cpu_count
+        import pickle
 
         if not self._data:
             return []
 
+        # 检查函数是否可 pickle
+        try:
+            pickle.dumps(func)
+        except (pickle.PicklingError, AttributeError, TypeError) as e:
+            func_name = getattr(func, "__name__", str(func))
+            raise TypeError(
+                f"函数 '{func_name}' 无法被 pickle，不能用于并行处理。"
+                f"请使用模块级函数而非 lambda 或闭包。错误: {e}"
+            ) from e
+
         workers = workers or cpu_count()
 
-        with Pool(workers) as pool:
-            results = pool.map(func, self._data, chunksize=chunksize)
+        try:
+            with Pool(workers) as pool:
+                async_result = pool.map_async(func, self._data, chunksize=chunksize)
+                results = async_result.get(timeout=timeout)
+        except TimeoutError:
+            raise RuntimeError(f"并行处理超时（{timeout}秒）")
+        except Exception as e:
+            raise RuntimeError(f"并行处理失败: {type(e).__name__}: {e}") from e
 
         return results
 
@@ -851,6 +884,7 @@ class DataTransformer:
         func: Callable[[Dict], bool],
         workers: Optional[int] = None,
         chunksize: int = 1000,
+        timeout: Optional[float] = None,
     ) -> "DataTransformer":
         """
         并行执行过滤函数（使用多进程）。
@@ -861,24 +895,46 @@ class DataTransformer:
             func: 过滤函数，接收原始 dict，返回 True 保留
             workers: 进程数，默认为 CPU 核心数
             chunksize: 每个进程处理的数据块大小
+            timeout: 超时时间（秒），None 表示无超时
 
         Returns:
             过滤后的新 DataTransformer
 
+        Raises:
+            TypeError: 如果 func 无法被 pickle（如 lambda 函数）
+            RuntimeError: 如果子进程执行出错或超时
+
         Examples:
             >>> def is_valid(item):
             ...     return len(item["text"]) > 10
             >>> filtered = dt.filter_parallel(is_valid)
         """
-        from multiprocessing import Pool, cpu_count
+        from multiprocessing import Pool, TimeoutError, cpu_count
+        import pickle
 
         if not self._data:
             return DataTransformer([])
 
+        # 检查函数是否可 pickle
+        try:
+            pickle.dumps(func)
+        except (pickle.PicklingError, AttributeError, TypeError) as e:
+            func_name = getattr(func, "__name__", str(func))
+            raise TypeError(
+                f"函数 '{func_name}' 无法被 pickle，不能用于并行处理。"
+                f"请使用模块级函数而非 lambda 或闭包。错误: {e}"
+            ) from e
+
         workers = workers or cpu_count()
 
-        with Pool(workers) as pool:
-            mask = pool.map(func, self._data, chunksize=chunksize)
+        try:
+            with Pool(workers) as pool:
+                async_result = pool.map_async(func, self._data, chunksize=chunksize)
+                mask = async_result.get(timeout=timeout)
+        except TimeoutError:
+            raise RuntimeError(f"并行处理超时（{timeout}秒）")
+        except Exception as e:
+            raise RuntimeError(f"并行处理失败: {type(e).__name__}: {e}") from e
 
         filtered = [item for item, keep in zip(self._data, mask) if keep]
         return DataTransformer(filtered)

dtflow/lineage.py

@@ -237,6 +237,23 @@ class LineageTracker:
 
         return lineage_path
 
+    def copy(self) -> "LineageTracker":
+        """
+        创建追踪器的深拷贝。
+
+        用于 split() 等场景，确保子数据集有独立的血缘追踪。
+
+        Returns:
+            新的 LineageTracker 实例
+        """
+        import copy as copy_module
+
+        new_tracker = LineageTracker.__new__(LineageTracker)
+        new_tracker.source_path = self.source_path
+        new_tracker.source_lineage = self.source_lineage  # LineageRecord 是不可变的，可共享
+        new_tracker.operations = copy_module.deepcopy(self.operations)
+        return new_tracker
+
 
 def _sanitize_params(params: Dict[str, Any]) -> Dict[str, Any]:
     """

dtflow/streaming.py

@@ -365,50 +365,108 @@ class StreamingTransformer:
         """
         批量流式保存（CSV/Parquet/Arrow）。
 
-        读取和处理是流式的，写入时收集后一次性写入。
+        真正的流式写入：分批处理，每批写入后释放内存。
+        内存占用 O(batch_size) 而非 O(n)。
         """
         path = Path(filepath)
-        all_items = []
+        count = 0
+        batch = []
+        first_batch = True
 
-        if show_progress:
-            # 根据是否有总数选择进度条样式
-            if self._total is not None:
-                columns = [
-                    SpinnerColumn(),
-                    TextColumn("[progress.description]{task.description}"),
-                    BarColumn(),
-                    TaskProgressColumn(),
-                    MofNCompleteColumn(),
-                    TimeElapsedColumn(),
-                    TimeRemainingColumn(),
-                ]
-            else:
-                columns = [
-                    SpinnerColumn(),
-                    TextColumn("[progress.description]{task.description}"),
-                    MofNCompleteColumn(),
-                    TimeElapsedColumn(),
-                ]
+        # 进度条配置
+        progress_columns = self._get_progress_columns()
 
-            with Progress(*columns) as progress:
-                task = progress.add_task("处理中", total=self._total)
-                for item in self._iterator:
-                    all_items.append(item)
-                    progress.update(task, advance=1)
-        else:
-            for item in self._iterator:
-                all_items.append(item)
+        def write_batch(items: List[Dict], is_first: bool, writer_state: Dict):
+            """写入一批数据"""
+            if not items:
+                return
+
+            df = pl.DataFrame(items)
 
-        if all_items:
-            df = pl.DataFrame(all_items)
             if fmt == "csv":
-                df.write_csv(path)
+                if is_first:
+                    df.write_csv(path)
+                else:
+                    # CSV 追加模式：不写表头
+                    with open(path, "ab") as f:
+                        f.write(df.write_csv(include_header=False).encode("utf-8"))
+
             elif fmt == "parquet":
-                df.write_parquet(path)
+                import pyarrow as pa
+                import pyarrow.parquet as pq
+
+                table = df.to_arrow()
+                if is_first:
+                    writer_state["writer"] = pq.ParquetWriter(str(path), table.schema)
+                writer_state["writer"].write_table(table)
+
             elif fmt == "arrow":
-                df.write_ipc(path)
+                import pyarrow as pa
+
+                table = df.to_arrow()
+                if is_first:
+                    writer_state["writer"] = pa.ipc.new_file(str(path), table.schema)
+                for record_batch in table.to_batches():
+                    writer_state["writer"].write_batch(record_batch)
+
+        writer_state: Dict[str, Any] = {}
+
+        try:
+            if show_progress:
+                with Progress(*progress_columns) as progress:
+                    task = progress.add_task("处理中", total=self._total)
+                    for item in self._iterator:
+                        batch.append(item)
+                        count += 1
+                        progress.update(task, advance=1)
+
+                        if len(batch) >= batch_size:
+                            write_batch(batch, first_batch, writer_state)
+                            first_batch = False
+                            batch = []  # 释放内存
+
+                    # 写入最后一批
+                    if batch:
+                        write_batch(batch, first_batch, writer_state)
+            else:
+                for item in self._iterator:
+                    batch.append(item)
+                    count += 1
+
+                    if len(batch) >= batch_size:
+                        write_batch(batch, first_batch, writer_state)
+                        first_batch = False
+                        batch = []
 
-        return len(all_items)
+                if batch:
+                    write_batch(batch, first_batch, writer_state)
+
+        finally:
+            # 关闭 writer
+            if "writer" in writer_state:
+                writer_state["writer"].close()
+
+        return count
+
+    def _get_progress_columns(self):
+        """获取进度条列配置"""
+        if self._total is not None:
+            return [
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                TaskProgressColumn(),
+                MofNCompleteColumn(),
+                TimeElapsedColumn(),
+                TimeRemainingColumn(),
+            ]
+        else:
+            return [
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                MofNCompleteColumn(),
+                TimeElapsedColumn(),
+            ]
 
     def save_sharded(
         self,

dtflow/tokenizers.py

@@ -210,7 +210,10 @@ def token_counter(
     创建 token 计数转换函数。
 
     Args:
-        fields: 要统计的字段（单个或多个）
+        fields: 要统计的字段（单个或多个），支持嵌套路径语法
+            - 简单字段: "text"
+            - 嵌套字段: "meta.content", "data.text"
+            - 索引: "messages[0].content", "messages[-1].content"
         model: 模型名称或别名，如 "qwen2.5", "gpt-4", "llama3" 等
         backend: 后端选择，None 则自动检测
         output_field: 输出字段名
@@ -221,6 +224,7 @@ def token_counter(
     Examples:
         >>> dt.transform(token_counter("text"))
         >>> dt.transform(token_counter(["question", "answer"], model="qwen3"))
+        >>> dt.transform(token_counter("messages[-1].content"))  # 最后一条消息
     """
     if isinstance(fields, str):
         fields = [fields]
@@ -229,7 +233,7 @@ def token_counter(
         result = item.to_dict() if hasattr(item, "to_dict") else dict(item)
         total = 0
         for field in fields:
-            value = item.get(field, "") if hasattr(item, "get") else item[field]
+            value = get_field_with_spec(item, field, default="")
             if value:
                 total += count_tokens(str(value), model=model, backend=backend)
         result[output_field] = total
@@ -249,7 +253,10 @@ def token_filter(
     创建基于 token 长度的过滤函数。
 
     Args:
-        fields: 要统计的字段（单个或多个）
+        fields: 要统计的字段（单个或多个），支持嵌套路径语法
+            - 简单字段: "text"
+            - 嵌套字段: "meta.content", "data.text"
+            - 索引: "messages[0].content", "messages[-1].content"
         min_tokens: 最小 token 数（包含）
         max_tokens: 最大 token 数（包含）
         model: 模型名称
@@ -261,6 +268,7 @@ def token_filter(
     Examples:
         >>> dt.filter(token_filter("text", min_tokens=10, max_tokens=512))
         >>> dt.filter(token_filter(["q", "a"], max_tokens=2048))
+        >>> dt.filter(token_filter("messages[-1].content", max_tokens=1024))
     """
     if isinstance(fields, str):
         fields = [fields]
@@ -268,7 +276,7 @@ def token_filter(
     def filter_func(item) -> bool:
         total = 0
         for field in fields:
-            value = item.get(field, "") if hasattr(item, "get") else item[field]
+            value = get_field_with_spec(item, field, default="")
             if value:
                 total += count_tokens(str(value), model=model, backend=backend)
 
@@ -281,11 +289,32 @@ def token_filter(
     return filter_func
 
 
+def _percentile(sorted_data: List[int], p: float) -> int:
+    """计算百分位数"""
+    n = len(sorted_data)
+    if n == 0:
+        return 0
+    idx = (n - 1) * p / 100
+    lower = int(idx)
+    upper = min(lower + 1, n - 1)
+    weight = idx - lower
+    return int(sorted_data[lower] * (1 - weight) + sorted_data[upper] * weight)
+
+
+def _std(counts: List[int], avg: float) -> float:
+    """计算标准差"""
+    if len(counts) < 2:
+        return 0.0
+    variance = sum((x - avg) ** 2 for x in counts) / len(counts)
+    return variance**0.5
+
+
 def token_stats(
     data: List[Dict[str, Any]],
     fields: Union[str, List[str]],
     model: str = DEFAULT_MODEL,
     backend: Optional[str] = None,
+    progress_callback: Optional[Callable[[int, int], None]] = None,
 ) -> Dict[str, Any]:
     """
     统计数据集的 token 信息。
@@ -295,9 +324,17 @@ def token_stats(
         fields: 要统计的字段，支持嵌套路径语法（如 meta.text, messages[-1].content）
         model: 模型名称或别名，如 "qwen2.5", "gpt-4" 等
         backend: 后端选择，None 则自动检测
+        progress_callback: 进度回调函数，接收 (current, total) 两个参数
 
     Returns:
-        统计信息字典
+        统计信息字典，包含:
+        - total_tokens: 总 token 数
+        - count: 样本数
+        - avg_tokens: 平均 token 数
+        - std_tokens: 标准差
+        - min_tokens, max_tokens: 最小/最大值
+        - median_tokens: 中位数 (p50)
+        - p25, p75, p90, p95, p99: 百分位数
     """
     if isinstance(fields, str):
         fields = [fields]
@@ -306,21 +343,33 @@ def token_stats(
         return {"total_tokens": 0, "count": 0}
 
     counts = []
-    for item in data:
+    total_items = len(data)
+    for i, item in enumerate(data):
         total = 0
         for field in fields:
             value = get_field_with_spec(item, field, default="")
             if value:
                 total += count_tokens(str(value), model=model, backend=backend)
         counts.append(total)
+        if progress_callback:
+            progress_callback(i + 1, total_items)
+
+    sorted_counts = sorted(counts)
+    avg = sum(counts) / len(counts)
 
     return {
         "total_tokens": sum(counts),
         "count": len(counts),
-        "avg_tokens": sum(counts) / len(counts),
+        "avg_tokens": avg,
+        "std_tokens": _std(counts, avg),
         "min_tokens": min(counts),
         "max_tokens": max(counts),
-        "median_tokens": sorted(counts)[len(counts) // 2],
+        "median_tokens": _percentile(sorted_counts, 50),
+        "p25": _percentile(sorted_counts, 25),
+        "p75": _percentile(sorted_counts, 75),
+        "p90": _percentile(sorted_counts, 90),
+        "p95": _percentile(sorted_counts, 95),
+        "p99": _percentile(sorted_counts, 99),
     }
 
 
@@ -504,6 +553,7 @@ def messages_token_stats(
     messages_field: str = "messages",
     model: str = DEFAULT_MODEL,
     backend: Optional[str] = None,
+    progress_callback: Optional[Callable[[int, int], None]] = None,
 ) -> Dict[str, Any]:
     """
     统计数据集中 messages 的 token 信息。
@@ -513,25 +563,18 @@ def messages_token_stats(
         messages_field: messages 字段名，支持嵌套路径语法（如 conversation.messages）
         model: 模型名称或别名
         backend: 后端，None 则自动检测
+        progress_callback: 进度回调函数，接收 (current, total) 两个参数
 
     Returns:
-        统计信息字典
-
-    Examples:
-        >>> stats = messages_token_stats(dt.data)  # 使用默认 qwen2.5
-        >>> stats = messages_token_stats(dt.data, model="qwen3")
-        >>> print(stats)
-        {
-            "count": 1000,
-            "total_tokens": 500000,
-            "user_tokens": 200000,
-            "assistant_tokens": 290000,
-            "system_tokens": 10000,
-            "avg_tokens": 500,
-            "max_tokens": 2048,
-            "min_tokens": 50,
-            "avg_turns": 4,
-        }
+        统计信息字典，包含:
+        - count: 样本数
+        - total_tokens: 总 token 数
+        - user_tokens, assistant_tokens, system_tokens: 各角色 token 数
+        - avg_tokens, std_tokens: 平均值和标准差
+        - min_tokens, max_tokens: 最小/最大值
+        - median_tokens: 中位数
+        - p25, p75, p90, p95, p99: 百分位数
+        - avg_turns: 平均对话轮数
     """
     _backend = backend or _auto_backend(model)
 
@@ -539,24 +582,36 @@ def messages_token_stats(
         return {"count": 0, "total_tokens": 0}
 
     all_stats = []
-    for item in data:
+    total_items = len(data)
+    for i, item in enumerate(data):
         messages = get_field_with_spec(item, messages_field, default=[])
         if messages:
             all_stats.append(_count_messages_tokens(messages, model=model, backend=_backend))
+        if progress_callback:
+            progress_callback(i + 1, total_items)
 
     if not all_stats:
         return {"count": 0, "total_tokens": 0}
 
     totals = [s["total"] for s in all_stats]
+    sorted_totals = sorted(totals)
+    avg = sum(totals) / len(totals)
+
     return {
         "count": len(all_stats),
         "total_tokens": sum(totals),
         "user_tokens": sum(s["user"] for s in all_stats),
         "assistant_tokens": sum(s["assistant"] for s in all_stats),
         "system_tokens": sum(s["system"] for s in all_stats),
-        "avg_tokens": sum(totals) // len(totals),
-        "max_tokens": max(totals),
+        "avg_tokens": int(avg),
+        "std_tokens": _std(totals, avg),
         "min_tokens": min(totals),
-        "median_tokens": sorted(totals)[len(totals) // 2],
+        "max_tokens": max(totals),
+        "median_tokens": _percentile(sorted_totals, 50),
+        "p25": _percentile(sorted_totals, 25),
+        "p75": _percentile(sorted_totals, 75),
+        "p90": _percentile(sorted_totals, 90),
+        "p95": _percentile(sorted_totals, 95),
+        "p99": _percentile(sorted_totals, 99),
         "avg_turns": sum(s["turns"] for s in all_stats) // len(all_stats),
     }

dtflow/utils/field_path.py

@@ -96,7 +96,9 @@ def _parse_path(path: str) -> List[Union[str, int, Literal["*", "#"]]]:
             continue
 
         # 解析 field[index] 格式
-        match = re.match(r"([a-zA-Z_\u4e00-\u9fff][a-zA-Z0-9_\u4e00-\u9fff]*)?(?:\[(-?\d+|\*)\])?", part)
+        match = re.match(
+            r"([a-zA-Z_\u4e00-\u9fff][a-zA-Z0-9_\u4e00-\u9fff]*)?(?:\[(-?\d+|\*)\])?", part
+        )
         if match:
             field_name, index = match.groups()
 
@@ -175,10 +177,12 @@ def _get_value_by_segments(
 
             return values
 
-        # 字典字段访问
+        # 字典字段访问（支持 dict 和类 dict 对象如 DictWrapper）
         if isinstance(seg, str):
             if isinstance(current, dict):
                 current = current.get(seg)
+            elif hasattr(current, "get"):
+                current = current.get(seg)
             else:
                 return None
 

{dtflow-0.5.0.dist-info → dtflow-0.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.5.0
+Version: 0.5.2
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme
@@ -32,16 +32,26 @@ Requires-Dist: orjson>=3.9.0
 Requires-Dist: polars>=0.20.0
 Requires-Dist: pyyaml>=5.4.0
 Requires-Dist: rich>=10.0.0
+Requires-Dist: tiktoken>=0.5.0
 Requires-Dist: typer>=0.9.0
 Provides-Extra: converters
 Requires-Dist: datasets>=2.0.0; extra == 'converters'
 Provides-Extra: dev
 Requires-Dist: black>=21.0; extra == 'dev'
+Requires-Dist: datasets>=2.0.0; extra == 'dev'
+Requires-Dist: datasketch>=1.5.0; extra == 'dev'
 Requires-Dist: flake8>=3.9.0; extra == 'dev'
+Requires-Dist: huggingface-hub>=0.20.0; extra == 'dev'
 Requires-Dist: isort>=5.9.0; extra == 'dev'
 Requires-Dist: mypy>=0.910; extra == 'dev'
+Requires-Dist: pyarrow; extra == 'dev'
 Requires-Dist: pytest-cov>=2.12.0; extra == 'dev'
 Requires-Dist: pytest>=6.0.0; extra == 'dev'
+Requires-Dist: rich>=10.0.0; extra == 'dev'
+Requires-Dist: scikit-learn>=0.24.0; extra == 'dev'
+Requires-Dist: tiktoken>=0.5.0; extra == 'dev'
+Requires-Dist: tokenizers>=0.15.0; extra == 'dev'
+Requires-Dist: toolong>=1.5.0; extra == 'dev'
 Provides-Extra: display
 Provides-Extra: docs
 Requires-Dist: myst-parser>=0.15.0; extra == 'docs'

{dtflow-0.5.0.dist-info → dtflow-0.5.2.dist-info}/RECORD

@@ -1,23 +1,23 @@
-dtflow/__init__.py,sha256=fOkG8g8VXS1HFk2ztmaJpjHBXmArHGBW8WE8tHPHXts,3031
+dtflow/__init__.py,sha256=PTqh_6-F6eEwg1RxQ0ueP6CYnZauMuqYhlZe2BJphr0,3031
 dtflow/__main__.py,sha256=ySpqvEn7k-vsrYFPx-8O6p-yx_24KccgnOSPd2XybhM,12572
 dtflow/converters.py,sha256=gyy-K15zjzGBawFnZa8D9JX37JZ47rey2GhjKa2pxFo,22081
-dtflow/core.py,sha256=HJAlxOaCtwvLOWF9JSC-2li3fsyRE2Q-H9unj9GQJ6M,35445
+dtflow/core.py,sha256=qMo6B3LK--TWRK7ZBKObGcs3pKFnd0NPoaM0T8JC7Jw,38135
 dtflow/framework.py,sha256=jyICi_RWHjX7WfsXdSbWmP1SL7y1OWSPyd5G5Y-lvg4,17578
-dtflow/lineage.py,sha256=vQ06lxBHftu-Ma5HlISp3F2eiIvwagQSnUGaLeABDZY,12190
+dtflow/lineage.py,sha256=jie3OL1qK90-_cOOqqLbhSJ1oGUktDM1x5HRpQ5Qiyc,12800
 dtflow/pipeline.py,sha256=zZaC4fg5vsp_30Fhbg75vu0yggsdvf28bWBiVDWzZ6Y,13901
 dtflow/presets.py,sha256=OP1nnM5NFk5Kli9FsXK0xAot48E5OQ6-VOIJT9ffXPg,5023
 dtflow/schema.py,sha256=IFcij22_UFKcgKT1YWwRg2QJO0vcAvCb1arZmsGByts,16824
-dtflow/streaming.py,sha256=jtWQjkhhZqfyzIaFskXNvooGAYDQBn1b6X8FHgaCZYk,22704
-dtflow/tokenizers.py,sha256=zxE6XZGjZ_DOGCjRSClI9xaAbFVf8FS6jwwssGoi_9U,18111
+dtflow/streaming.py,sha256=dxpNd1-Wz_PTLTdvM5qn06_2TJr5NRlIIuw0LOSS2Iw,24755
+dtflow/tokenizers.py,sha256=7ZAelSmcDxLWH5kICgH9Q1ULH3_BfDZb9suHMjJJRZU,20589
 dtflow/cli/__init__.py,sha256=QhZ-thgx9IBTFII7T_hdoWFUl0CCsdGQHN5ZEZw2XB0,423
 dtflow/cli/clean.py,sha256=y9VCRibgK1j8WIY3h0XZX0m93EdELQC7TdnseMWwS-0,17799
 dtflow/cli/commands.py,sha256=ST65Ox_MKu-CKAtPVaxECAPXYOJiF7BhL32A4nsZZl0,1175
-dtflow/cli/common.py,sha256=FsDFVNcLj_874qSg2dGef4V7mqPU9THLchT8PxJpBt8,12955
+dtflow/cli/common.py,sha256=nIPc9GBK61r6kmaI9OS3IyhcfPqShpDEHx1ddjFPnlM,13131
 dtflow/cli/io_ops.py,sha256=BMDisP6dxzzmSjYwmeFwaHmpHHPqirmXAWeNTD-9MQM,13254
 dtflow/cli/lineage.py,sha256=_lNh35nF9AA0Zy6FyZ4g8IzrXH2ZQnp3inF-o2Hs1pw,1383
 dtflow/cli/pipeline.py,sha256=QNEo-BJlaC1CVnVeRZr7TwfuZYloJ4TebIzJ5ALzry0,1426
 dtflow/cli/sample.py,sha256=vPTQlF0OXEry4QjO8uaD9vOae4AQbX9zDwVYOxg59ZI,10339
-dtflow/cli/stats.py,sha256=HByF0sFMqY1kM75dnjTcJbMKDdQNdOt4iDba4au_-pI,20495
+dtflow/cli/stats.py,sha256=u4ehCfgw1X8WuOyAjrApMRgcIO3BVmINbsTjxEscQro,24086
 dtflow/cli/transform.py,sha256=w6xqMOxPxQvL2u_BPCfpDHuPSC9gmcqMPVN8s-B6bbY,15052
 dtflow/cli/validate.py,sha256=65aGVlMS_Rq0Ch0YQ-TclVJ03RQP4CnG137wthzb8Ao,4384
 dtflow/mcp/__init__.py,sha256=huEJ3rXDbxDRjsLPEvjNT2u3tWs6Poiv6fokPIrByjw,897
@@ -29,8 +29,8 @@ dtflow/storage/__init__.py,sha256=C0jpWNQU808Ezz7lWneddABal3wILy8ijFUNiSKbHV4,36
 dtflow/storage/io.py,sha256=ZH2aSE-S89gpy3z4oTqhcqWf4u10OdkDoyul7o_YBDI,23374
 dtflow/utils/__init__.py,sha256=f8v9HJZMWRI5AL64Vjr76Pf2Na_whOF9nJBKgPbXXYg,429
 dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
-dtflow/utils/field_path.py,sha256=WcNA-LZh3H61a77FEzB_R7YAyyZl3M8ofdq05ytQGmI,7459
-dtflow-0.5.0.dist-info/METADATA,sha256=chELFIevPb1h7ZydbWtH9rM7RiA2n3Ep-XWL1qbaHk0,22084
-dtflow-0.5.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dtflow-0.5.0.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
-dtflow-0.5.0.dist-info/RECORD,,
+dtflow/utils/field_path.py,sha256=K8nU196RxTSJ1OoieTWGcYOWl9KjGq2iSxCAkfjECuM,7621
+dtflow-0.5.2.dist-info/METADATA,sha256=RlpGaySrAIgTviom_Wyn6o2LWzQQVihff12Jpazy10o,22544
+dtflow-0.5.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dtflow-0.5.2.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
+dtflow-0.5.2.dist-info/RECORD,,