dtflow 0.5.9__py3-none-any.whl → 0.5.10__py3-none-any.whl

dtflow/SKILL.md

@@ -154,6 +154,7 @@ dt token-stats data.jsonl                         # 默认统计 messages 字段
 dt token-stats data.jsonl -f text                 # 指定统计字段
 dt token-stats data.jsonl -m qwen2.5              # 指定分词器 (cl100k_base/qwen2.5/llama3)
 dt token-stats data.jsonl --detailed              # 显示详细统计
+dt token-stats data.jsonl -w 4                    # 多进程加速（数据量>=1000时自动启用）
 
 # 采样（支持字段路径语法）
 dt sample data.jsonl 100                          # 随机采样 100 条
@@ -204,6 +205,7 @@ dt validate data.jsonl --preset=openai_chat       # 预设: openai_chat/alpaca/d
 dt validate data.jsonl -p alpaca -f -o valid.jsonl  # 过滤无效数据并保存
 dt validate data.jsonl -p openai_chat -v          # 显示详细信息
 dt validate data.jsonl -p openai_chat --max-errors=50  # 最多显示 50 条错误
+dt validate data.jsonl -p openai_chat -w 4        # 多进程加速
 
 # 转换
 dt transform data.jsonl --preset=openai_chat

dtflow/__init__.py

@@ -60,7 +60,7 @@ from .tokenizers import (
     token_stats,
 )
 
-__version__ = "0.5.9"
+__version__ = "0.5.10"
 
 __all__ = [
     # core

dtflow/__main__.py

@@ -256,9 +256,12 @@ def token_stats(
         "cl100k_base", "--model", "-m", help="分词器: cl100k_base (默认), qwen2.5, llama3, gpt-4 等"
     ),
     detailed: bool = typer.Option(False, "--detailed", "-d", help="显示详细统计"),
+    workers: Optional[int] = typer.Option(
+        None, "--workers", "-w", help="并行进程数 (默认自动, 1 禁用并行)"
+    ),
 ):
     """统计数据集的 Token 信息"""
-    _token_stats(filename, field, model, detailed)
+    _token_stats(filename, field, model, detailed, workers)
 
 
 @app.command()
@@ -359,9 +362,12 @@ def validate(
     filter: bool = typer.Option(False, "--filter", "-f", help="过滤无效数据并保存"),
     max_errors: int = typer.Option(20, "--max-errors", help="最多显示的错误数量"),
     verbose: bool = typer.Option(False, "--verbose", "-v", help="显示详细信息"),
+    workers: Optional[int] = typer.Option(
+        None, "--workers", "-w", help="并行进程数 (默认自动, 1 禁用并行)"
+    ),
 ):
     """使用预设 Schema 验证数据格式"""
-    _validate(filename, preset, output, filter, max_errors, verbose)
+    _validate(filename, preset, output, filter, max_errors, verbose, workers)
 
 
 # ============ 工具命令 ============

dtflow/cli/stats.py

@@ -209,7 +209,7 @@ def _quick_stats(filepath: Path) -> None:
         print(f"字段: {len(fields)} 个")
 
         if fields:
-            print(f"\n📋 字段结构:")
+            print("\n📋 字段结构:")
             for i, f in enumerate(fields, 1):
                 print(f"  {i}. {f['field']} ({f['type']})")
 
@@ -597,14 +597,14 @@ def _print_stats(filename: str, total: int, field_stats: List[Dict[str, Any]]) -
     except ImportError:
         # 没有 rich，使用普通打印
         print(f"\n{'=' * 50}")
-        print(f"📊 数据概览")
+        print("📊 数据概览")
         print(f"{'=' * 50}")
         print(f"文件: {filename}")
         print(f"总数: {total:,} 条")
         print(f"字段: {len(field_stats)} 个")
 
         print(f"\n{'=' * 50}")
-        print(f"📋 字段统计")
+        print("📋 字段统计")
         print(f"{'=' * 50}")
         print(f"{'字段':<20} {'类型':<8} {'非空率':<8} {'唯一值':<8}")
         print("-" * 50)
@@ -620,6 +620,7 @@ def token_stats(
     field: str = "messages",
     model: str = "cl100k_base",
     detailed: bool = False,
+    workers: Optional[int] = None,
 ) -> None:
     """
     统计数据集的 Token 信息。
@@ -629,6 +630,7 @@ def token_stats(
         field: 要统计的字段（默认 messages），支持嵌套路径语法
         model: 分词器: cl100k_base (默认), qwen2.5, llama3, gpt-4 等
         detailed: 是否显示详细统计
+        workers: 并行进程数，None 自动检测，1 禁用并行
 
     Examples:
         dt token-stats data.jsonl
@@ -636,6 +638,7 @@ def token_stats(
         dt token-stats data.jsonl --field=conversation.messages
         dt token-stats data.jsonl --field=messages[-1].content   # 统计最后一条消息
         dt token-stats data.jsonl --detailed
+        dt token-stats data.jsonl --workers=4   # 使用 4 进程
     """
     filepath = Path(filename)
 
@@ -667,7 +670,7 @@ def token_stats(
 
     # 尝试使用 rich 进度条
     try:
-        from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
+        from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
 
         with Progress(
             SpinnerColumn(),
@@ -685,14 +688,22 @@ def token_stats(
                 from ..tokenizers import messages_token_stats
 
                 stats_result = messages_token_stats(
-                    data, messages_field=field, model=model, progress_callback=update_progress
+                    data,
+                    messages_field=field,
+                    model=model,
+                    progress_callback=update_progress,
+                    workers=workers,
                 )
                 _print_messages_token_stats(stats_result, detailed)
             else:
                 from ..tokenizers import token_stats as compute_token_stats
 
                 stats_result = compute_token_stats(
-                    data, fields=field, model=model, progress_callback=update_progress
+                    data,
+                    fields=field,
+                    model=model,
+                    progress_callback=update_progress,
+                    workers=workers,
                 )
                 _print_text_token_stats(stats_result, detailed)
 
@@ -703,12 +714,14 @@ def token_stats(
             if isinstance(field_value, list) and field_value and isinstance(field_value[0], dict):
                 from ..tokenizers import messages_token_stats
 
-                stats_result = messages_token_stats(data, messages_field=field, model=model)
+                stats_result = messages_token_stats(
+                    data, messages_field=field, model=model, workers=workers
+                )
                 _print_messages_token_stats(stats_result, detailed)
             else:
                 from ..tokenizers import token_stats as compute_token_stats
 
-                stats_result = compute_token_stats(data, fields=field, model=model)
+                stats_result = compute_token_stats(data, fields=field, model=model, workers=workers)
                 _print_text_token_stats(stats_result, detailed)
         except ImportError as e:
             print(f"错误: {e}")
@@ -788,7 +801,7 @@ def _print_messages_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
         print(f"平均 Token: {stats['avg_tokens']:,} (std: {std:.1f})")
         print(f"范围: {stats['min_tokens']:,} - {stats['max_tokens']:,}")
 
-        print(f"\n📈 百分位分布:")
+        print("\n📈 百分位分布:")
         print(f"  P25: {stats.get('p25', '-'):,}  P50: {stats.get('median_tokens', '-'):,}")
         print(f"  P75: {stats.get('p75', '-'):,}  P90: {stats.get('p90', '-'):,}")
         print(f"  P95: {stats.get('p95', '-'):,}  P99: {stats.get('p99', '-'):,}")
@@ -855,7 +868,7 @@ def _print_text_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
         print(f"平均 Token: {stats['avg_tokens']:.1f} (std: {std:.1f})")
         print(f"范围: {stats['min_tokens']:,} - {stats['max_tokens']:,}")
 
-        print(f"\n📈 百分位分布:")
+        print("\n📈 百分位分布:")
         print(f"  P25: {stats.get('p25', '-'):,}  P50: {stats.get('median_tokens', '-'):,}")
         print(f"  P75: {stats.get('p75', '-'):,}  P90: {stats.get('p90', '-'):,}")
         print(f"  P95: {stats.get('p95', '-'):,}  P99: {stats.get('p99', '-'):,}")

dtflow/cli/validate.py

@@ -6,8 +6,6 @@ from pathlib import Path
 from typing import Optional
 
 from ..schema import (
-    Schema,
-    Field,
     alpaca_schema,
     dpo_schema,
     openai_chat_schema,
@@ -16,7 +14,6 @@ from ..schema import (
 from ..storage.io import load_data, save_data
 from .common import _check_file_format
 
-
 # 预设 Schema 映射
 PRESET_SCHEMAS = {
     "openai_chat": openai_chat_schema,
@@ -36,6 +33,7 @@ def validate(
     filter_invalid: bool = False,
     max_errors: int = 20,
     verbose: bool = False,
+    workers: Optional[int] = None,
 ) -> None:
     """
     使用 Schema 验证数据文件。
@@ -47,11 +45,13 @@ def validate(
         filter_invalid: 过滤无效数据并保存
         max_errors: 最多显示的错误数量
         verbose: 显示详细信息
+        workers: 并行进程数，None 自动检测，1 禁用并行
 
     Examples:
         dt validate data.jsonl --preset=openai_chat
         dt validate data.jsonl --preset=alpaca -o valid.jsonl
         dt validate data.jsonl --preset=chat --filter
+        dt validate data.jsonl --preset=chat --workers=4
     """
     filepath = Path(filename)
 
@@ -99,19 +99,54 @@ def validate(
     print(f"总记录数: {total}")
     print()
 
-    # 验证
-    valid_data = []
-    invalid_count = 0
-    error_samples = []
-
-    for i, item in enumerate(data):
-        result = schema.validate(item)
-        if result.valid:
-            valid_data.append(item)
-        else:
-            invalid_count += 1
-            if len(error_samples) < max_errors:
-                error_samples.append((i, result))
+    # 验证（使用并行或串行）
+    use_parallel = workers != 1 and total >= 1000
+
+    if use_parallel:
+        # 使用进度条（如果有 rich）
+        try:
+            from rich.progress import (
+                BarColumn,
+                Progress,
+                SpinnerColumn,
+                TaskProgressColumn,
+                TextColumn,
+            )
+
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[bold blue]验证数据"),
+                BarColumn(),
+                TaskProgressColumn(),
+            ) as progress:
+                task = progress.add_task("", total=total)
+
+                def update_progress(current: int, total_count: int):
+                    progress.update(task, completed=current)
+
+                valid_data, invalid_results = schema.validate_parallel(
+                    data, workers=workers, progress_callback=update_progress
+                )
+        except ImportError:
+            print("🔍 验证数据...")
+            valid_data, invalid_results = schema.validate_parallel(data, workers=workers)
+
+        invalid_count = len(invalid_results)
+        error_samples = invalid_results[:max_errors]
+    else:
+        # 串行验证
+        valid_data = []
+        invalid_count = 0
+        error_samples = []
+
+        for i, item in enumerate(data):
+            result = schema.validate(item)
+            if result.valid:
+                valid_data.append(item)
+            else:
+                invalid_count += 1
+                if len(error_samples) < max_errors:
+                    error_samples.append((i, result))
 
     valid_count = len(valid_data)
     valid_ratio = valid_count / total * 100 if total > 0 else 0
@@ -138,9 +173,7 @@ def validate(
 
     # 保存有效数据
     if output or filter_invalid:
-        output_path = output or str(filepath).replace(
-            filepath.suffix, f"_valid{filepath.suffix}"
-        )
+        output_path = output or str(filepath).replace(filepath.suffix, f"_valid{filepath.suffix}")
         save_data(valid_data, output_path)
         print(f"✅ 有效数据已保存: {output_path} ({valid_count} 条)")
 

dtflow/parallel.py

@@ -0,0 +1,115 @@
+"""
+并行处理模块
+
+提供多进程并行处理工具，用于加速大数据集的 token 统计和 schema 验证。
+"""
+
+from multiprocessing import Pool, cpu_count
+from typing import Callable, List, Optional, TypeVar
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+def parallel_map(
+    func: Callable[[T], R],
+    data: List[T],
+    workers: Optional[int] = None,
+    threshold: int = 1000,
+    chunksize: Optional[int] = None,
+) -> List[R]:
+    """
+    并行 map 操作。
+
+    Args:
+        func: 处理函数（必须可 pickle，不能是 lambda 或闭包）
+        data: 数据列表
+        workers: 进程数，None 则使用 CPU 核数
+        threshold: 数据量阈值，低于此值使用串行
+        chunksize: 每个进程的任务块大小，None 则自动计算
+
+    Returns:
+        处理结果列表（保持顺序）
+    """
+    n = len(data)
+
+    # 数据量小或指定单进程，使用串行
+    if n < threshold or workers == 1:
+        return [func(item) for item in data]
+
+    workers = workers or cpu_count()
+    workers = min(workers, n)  # 进程数不超过数据量
+
+    # 自动计算 chunksize
+    if chunksize is None:
+        chunksize = max(1, n // (workers * 4))
+
+    with Pool(processes=workers) as pool:
+        return pool.map(func, data, chunksize=chunksize)
+
+
+def parallel_imap(
+    func: Callable[[T], R],
+    data: List[T],
+    workers: Optional[int] = None,
+    threshold: int = 1000,
+    chunksize: Optional[int] = None,
+):
+    """
+    并行 imap 操作（惰性迭代器版本，支持进度回调）。
+
+    Args:
+        func: 处理函数（必须可 pickle）
+        data: 数据列表
+        workers: 进程数，None 则使用 CPU 核数
+        threshold: 数据量阈值，低于此值使用串行
+        chunksize: 每个进程的任务块大小
+
+    Yields:
+        处理结果（按顺序）
+    """
+    n = len(data)
+
+    # 数据量小或指定单进程，使用串行
+    if n < threshold or workers == 1:
+        for item in data:
+            yield func(item)
+        return
+
+    workers = workers or cpu_count()
+    workers = min(workers, n)
+
+    if chunksize is None:
+        chunksize = max(1, n // (workers * 4))
+
+    with Pool(processes=workers) as pool:
+        for result in pool.imap(func, data, chunksize=chunksize):
+            yield result
+
+
+def get_optimal_workers(data_size: int, default: Optional[int] = None) -> int:
+    """
+    根据数据量计算最优进程数。
+
+    Args:
+        data_size: 数据量
+        default: 用户指定的进程数，None 则自动计算
+
+    Returns:
+        最优进程数
+    """
+    if default is not None:
+        return default
+
+    cpu_cores = cpu_count()
+
+    # 数据量小于阈值，单进程
+    if data_size < 1000:
+        return 1
+
+    # 数据量适中，使用一半 CPU
+    if data_size < 10000:
+        return max(1, cpu_cores // 2)
+
+    # 大数据量，使用全部 CPU
+    return cpu_cores

dtflow/schema.py

@@ -26,10 +26,35 @@ Schema 验证模块
     results = dt.validate_schema(schema)
 """
 
-from dataclasses import dataclass, field as dataclass_field
-from typing import Any, Callable, Dict, List, Literal, Optional, Set, Union
+from dataclasses import dataclass
+from dataclasses import field as dataclass_field
+from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
+
+from .utils.field_path import _parse_path, get_field
+
+
+def _validate_item_wrapper(args: tuple) -> Tuple[int, bool, list]:
+    """
+    验证单条数据（用于多进程）。
+
+    Args:
+        args: (index, item, schema_fields) 元组
+
+    Returns:
+        (index, is_valid, errors_as_dicts) - 返回字典列表而非对象（pickle 兼容）
+    """
+    idx, item, fields = args
+    # 在子进程中重建 Schema
+    schema = Schema(fields)
+    result = schema.validate(item)
+
+    if result.valid:
+        return (idx, True, [])
+    else:
+        # 将错误转换为字典（pickle 兼容）
+        errors = [{"path": e.path, "message": e.message, "value": e.value} for e in result.errors]
+        return (idx, False, errors)
 
-from .utils.field_path import get_field, _parse_path, _get_value_by_segments
 
 # 支持的类型
 FieldType = Literal["str", "int", "float", "bool", "list", "dict", "any"]
@@ -162,9 +187,7 @@ class Field:
 
         # 选项检查
         if self.choices is not None and value not in self.choices:
-            errors.append(
-                ValidationError(path, f"值必须是 {self.choices} 之一", value)
-            )
+            errors.append(ValidationError(path, f"值必须是 {self.choices} 之一", value))
 
         # 正则表达式检查
         if self.pattern is not None and isinstance(value, str):
@@ -324,9 +347,7 @@ class Schema:
 
         return errors
 
-    def validate_batch(
-        self, data: List[dict], max_errors: int = 100
-    ) -> List[tuple]:
+    def validate_batch(self, data: List[dict], max_errors: int = 100) -> List[tuple]:
         """
         批量验证数据
 
@@ -350,9 +371,76 @@ class Schema:
 
         return failed
 
+    def validate_parallel(
+        self,
+        data: List[dict],
+        workers: Optional[int] = None,
+        progress_callback: Optional[Callable[[int, int], None]] = None,
+    ) -> tuple:
+        """
+        并行验证数据列表。
+
+        Args:
+            data: 数据列表
+            workers: 进程数，None 自动检测，1 禁用并行
+            progress_callback: 进度回调函数
+
+        Returns:
+            (valid_data, invalid_indices_results) 元组
+            - valid_data: 有效数据列表
+            - invalid_indices_results: [(index, ValidationResult), ...] 无效数据
+        """
+        if not data:
+            return [], []
+
+        total = len(data)
+        use_parallel = workers != 1 and total >= 1000
+
+        valid_data = []
+        invalid_results = []
+
+        if use_parallel:
+            from .parallel import get_optimal_workers, parallel_imap
+
+            actual_workers = get_optimal_workers(total, workers)
+            # 准备参数：(index, item, schema_fields)
+            args_list = [(i, item, self._fields) for i, item in enumerate(data)]
+
+            for i, (idx, is_valid, result_data) in enumerate(
+                parallel_imap(
+                    _validate_item_wrapper,
+                    args_list,
+                    workers=actual_workers,
+                    threshold=1000,
+                )
+            ):
+                if is_valid:
+                    valid_data.append(data[idx])
+                else:
+                    # 重建 ValidationResult（因为不能直接 pickle）
+                    errors = [
+                        ValidationError(path=e["path"], message=e["message"], value=e.get("value"))
+                        for e in result_data
+                    ]
+                    invalid_results.append((idx, ValidationResult(valid=False, errors=errors)))
+                if progress_callback:
+                    progress_callback(i + 1, total)
+        else:
+            # 串行处理
+            for i, item in enumerate(data):
+                result = self.validate(item)
+                if result.valid:
+                    valid_data.append(item)
+                else:
+                    invalid_results.append((i, result))
+                if progress_callback:
+                    progress_callback(i + 1, total)
+
+        return valid_data, invalid_results
+
     def __repr__(self) -> str:
         field_strs = [f"  {path}: {field_def}" for path, field_def in self._fields.items()]
-        return f"Schema({{\n" + ",\n".join(field_strs) + "\n}})"
+        return "Schema({\n" + ",\n".join(field_strs) + "\n}})"
 
 
 # ============================================================================
@@ -461,9 +549,7 @@ def sharegpt_schema(
     """
     return Schema(
         {
-            "conversations": Field(
-                type="list", required=True, min_length=min_conversations
-            ),
+            "conversations": Field(type="list", required=True, min_length=min_conversations),
             "conversations[*].from": Field(
                 type="str", required=True, choices=[human_role, gpt_role]
             ),

dtflow/tokenizers.py

@@ -122,8 +122,8 @@ def _get_tiktoken_encoder(model: str):
                 _tokenizer_cache[model] = tiktoken.get_encoding(model)
             else:
                 _tokenizer_cache[model] = tiktoken.encoding_for_model(model)
-        except ImportError:
-            raise ImportError("需要安装 tiktoken: pip install tiktoken")
+        except ImportError as e:
+            raise ImportError("需要安装 tiktoken: pip install tiktoken") from e
     return _tokenizer_cache[model]
 
 
@@ -149,12 +149,12 @@ def _get_hf_tokenizer(model: str):
 
                 tokenizer = AutoTokenizer.from_pretrained(resolved, trust_remote_code=True)
                 _tokenizer_cache[resolved] = ("transformers", tokenizer)
-            except ImportError:
+            except ImportError as e:
                 raise ImportError(
                     "需要安装 tokenizers 或 transformers:\n"
                     "  pip install tokenizers huggingface_hub  (推荐，更轻量)\n"
                     "  pip install transformers"
-                )
+                ) from e
     return _tokenizer_cache[resolved]
 
 
@@ -309,12 +309,29 @@ def _std(counts: List[int], avg: float) -> float:
     return variance**0.5
 
 
+def _count_item_tokens(args: tuple) -> int:
+    """
+    计算单条数据的 token 数（用于多进程）。
+
+    Args:
+        args: (item, fields, model, backend) 元组
+    """
+    item, fields, model, backend = args
+    total = 0
+    for field in fields:
+        value = get_field_with_spec(item, field, default="")
+        if value:
+            total += count_tokens(str(value), model=model, backend=backend)
+    return total
+
+
 def token_stats(
     data: List[Dict[str, Any]],
     fields: Union[str, List[str]],
     model: str = DEFAULT_MODEL,
     backend: Optional[str] = None,
     progress_callback: Optional[Callable[[int, int], None]] = None,
+    workers: Optional[int] = None,
 ) -> Dict[str, Any]:
     """
     统计数据集的 token 信息。
@@ -325,6 +342,7 @@ def token_stats(
         model: 模型名称或别名，如 "qwen2.5", "gpt-4" 等
         backend: 后端选择，None 则自动检测
         progress_callback: 进度回调函数，接收 (current, total) 两个参数
+        workers: 进程数，None 自动检测，1 表示禁用并行
 
     Returns:
         统计信息字典，包含:
@@ -342,17 +360,42 @@ def token_stats(
     if not data:
         return {"total_tokens": 0, "count": 0}
 
-    counts = []
     total_items = len(data)
-    for i, item in enumerate(data):
-        total = 0
-        for field in fields:
-            value = get_field_with_spec(item, field, default="")
-            if value:
-                total += count_tokens(str(value), model=model, backend=backend)
-        counts.append(total)
-        if progress_callback:
-            progress_callback(i + 1, total_items)
+    _backend = backend or _auto_backend(model)
+
+    # 判断是否使用多进程
+    use_parallel = workers != 1 and total_items >= 1000
+
+    if use_parallel:
+        from .parallel import get_optimal_workers, parallel_imap
+
+        actual_workers = get_optimal_workers(total_items, workers)
+        # 准备参数
+        args_list = [(item, fields, model, _backend) for item in data]
+        counts = []
+        for i, result in enumerate(
+            parallel_imap(
+                _count_item_tokens,
+                args_list,
+                workers=actual_workers,
+                threshold=1000,
+            )
+        ):
+            counts.append(result)
+            if progress_callback:
+                progress_callback(i + 1, total_items)
+    else:
+        # 串行处理
+        counts = []
+        for i, item in enumerate(data):
+            total = 0
+            for field in fields:
+                value = get_field_with_spec(item, field, default="")
+                if value:
+                    total += count_tokens(str(value), model=model, backend=_backend)
+            counts.append(total)
+            if progress_callback:
+                progress_callback(i + 1, total_items)
 
     sorted_counts = sorted(counts)
     avg = sum(counts) / len(counts)
@@ -548,12 +591,27 @@ def messages_token_filter(
     return filter_func
 
 
+def _count_messages_tokens_wrapper(args: tuple) -> Optional[Dict[str, int]]:
+    """
+    计算单条 messages 的 token 数（用于多进程）。
+
+    Args:
+        args: (item, messages_field, model, backend) 元组
+    """
+    item, messages_field, model, backend = args
+    messages = get_field_with_spec(item, messages_field, default=[])
+    if messages:
+        return _count_messages_tokens(messages, model=model, backend=backend)
+    return None
+
+
 def messages_token_stats(
     data: List[Dict[str, Any]],
     messages_field: str = "messages",
     model: str = DEFAULT_MODEL,
     backend: Optional[str] = None,
     progress_callback: Optional[Callable[[int, int], None]] = None,
+    workers: Optional[int] = None,
 ) -> Dict[str, Any]:
     """
     统计数据集中 messages 的 token 信息。
@@ -564,6 +622,7 @@ def messages_token_stats(
         model: 模型名称或别名
         backend: 后端，None 则自动检测
         progress_callback: 进度回调函数，接收 (current, total) 两个参数
+        workers: 进程数，None 自动检测，1 表示禁用并行
 
     Returns:
         统计信息字典，包含:
@@ -581,14 +640,38 @@ def messages_token_stats(
     if not data:
         return {"count": 0, "total_tokens": 0}
 
-    all_stats = []
     total_items = len(data)
-    for i, item in enumerate(data):
-        messages = get_field_with_spec(item, messages_field, default=[])
-        if messages:
-            all_stats.append(_count_messages_tokens(messages, model=model, backend=_backend))
-        if progress_callback:
-            progress_callback(i + 1, total_items)
+
+    # 判断是否使用多进程
+    use_parallel = workers != 1 and total_items >= 1000
+
+    all_stats = []
+    if use_parallel:
+        from .parallel import get_optimal_workers, parallel_imap
+
+        actual_workers = get_optimal_workers(total_items, workers)
+        args_list = [(item, messages_field, model, _backend) for item in data]
+
+        for i, result in enumerate(
+            parallel_imap(
+                _count_messages_tokens_wrapper,
+                args_list,
+                workers=actual_workers,
+                threshold=1000,
+            )
+        ):
+            if result is not None:
+                all_stats.append(result)
+            if progress_callback:
+                progress_callback(i + 1, total_items)
+    else:
+        # 串行处理
+        for i, item in enumerate(data):
+            messages = get_field_with_spec(item, messages_field, default=[])
+            if messages:
+                all_stats.append(_count_messages_tokens(messages, model=model, backend=_backend))
+            if progress_callback:
+                progress_callback(i + 1, total_items)
 
     if not all_stats:
         return {"count": 0, "total_tokens": 0}

{dtflow-0.5.9.dist-info → dtflow-0.5.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtflow
-Version: 0.5.9
+Version: 0.5.10
 Summary: A flexible data transformation tool for ML training formats (SFT, RLHF, Pretrain)
 Project-URL: Homepage, https://github.com/yourusername/DataTransformer
 Project-URL: Documentation, https://github.com/yourusername/DataTransformer#readme
@@ -464,6 +464,7 @@ dt run pipeline.yaml --input=new_data.jsonl --output=result.jsonl
 dt token-stats data.jsonl --field=messages --model=gpt-4
 dt token-stats data.jsonl --field=messages[-1].content   # 统计最后一条消息
 dt token-stats data.jsonl --field=text --detailed
+dt token-stats data.jsonl --workers=4                    # 多进程加速（数据量大时自动启用）
 
 # 数据对比
 dt diff v1/train.jsonl v2/train.jsonl
@@ -521,6 +522,7 @@ dt validate data.jsonl --preset=openai_chat           # 使用预设 schema 验
 dt validate data.jsonl --preset=alpaca --verbose      # 详细输出
 dt validate data.jsonl --preset=sharegpt --filter-invalid -o valid.jsonl  # 过滤出有效数据
 dt validate data.jsonl --preset=dpo --max-errors=100  # 限制错误输出数量
+dt validate data.jsonl --preset=openai_chat --workers=4  # 多进程加速
 ```
 
 ### 字段路径语法

{dtflow-0.5.9.dist-info → dtflow-0.5.10.dist-info}/RECORD

@@ -1,16 +1,17 @@
-dtflow/SKILL.md,sha256=hPxJhroGmNbBv8MLZUkOA2yW1TDdUKEUYYlz9tW2mao,10393
-dtflow/__init__.py,sha256=9ZqhqD8qQM9w2dfHKyUWIaqSX-X4elWtbaQN4CNBhgg,3031
-dtflow/__main__.py,sha256=gg3v7u-Ot7AicgKrP1fuyKtMJXVduNuLmhy7L1LUPDg,17710
+dtflow/SKILL.md,sha256=Oq8Kb5JghZMJ1WoP8OWhX3qAWaUY9Sip_iWAv8S2eMg,10567
+dtflow/__init__.py,sha256=2A-P6k9VBIWZXRgXwYPFOwHMCmgkfKZVYuGuBziqqhc,3032
+dtflow/__main__.py,sha256=_wrpYfOog6G83I17yuBe-hryBsaCrIwbXSEnzT-r28g,18008
 dtflow/converters.py,sha256=X3qeFD7FCOMnfiP3MicL5MXimOm4XUYBs5pczIkudU0,22331
 dtflow/core.py,sha256=qMo6B3LK--TWRK7ZBKObGcs3pKFnd0NPoaM0T8JC7Jw,38135
 dtflow/eval.py,sha256=_c-XP2zsOBznYltSyKEScOqvmPVX2orqepg5cNhXXB0,9836
 dtflow/framework.py,sha256=jyICi_RWHjX7WfsXdSbWmP1SL7y1OWSPyd5G5Y-lvg4,17578
 dtflow/lineage.py,sha256=jie3OL1qK90-_cOOqqLbhSJ1oGUktDM1x5HRpQ5Qiyc,12800
+dtflow/parallel.py,sha256=EnIdGEGMrZUNT2-CBIV93UFfpqr_jU_heqqvdGXcP-Y,3046
 dtflow/pipeline.py,sha256=zZaC4fg5vsp_30Fhbg75vu0yggsdvf28bWBiVDWzZ6Y,13901
 dtflow/presets.py,sha256=qa8WQJhbNMuGxqqgA9BFadEBwDB9s0zWNxxhzF3q1K8,4701
-dtflow/schema.py,sha256=IFcij22_UFKcgKT1YWwRg2QJO0vcAvCb1arZmsGByts,16824
+dtflow/schema.py,sha256=zCZNEAqTMT1BS_p2t0CYczR5S9rqyDREa7ZsYI5pFGA,19885
 dtflow/streaming.py,sha256=dxpNd1-Wz_PTLTdvM5qn06_2TJr5NRlIIuw0LOSS2Iw,24755
-dtflow/tokenizers.py,sha256=7ZAelSmcDxLWH5kICgH9Q1ULH3_BfDZb9suHMjJJRZU,20589
+dtflow/tokenizers.py,sha256=GFQsuLSLn2GHn2kaXhJkP8G85lgsdLzYtJNbppQhYPE,23408
 dtflow/cli/__init__.py,sha256=QhZ-thgx9IBTFII7T_hdoWFUl0CCsdGQHN5ZEZw2XB0,423
 dtflow/cli/clean.py,sha256=BEQQlH2q6luCbx51M3oxxOwcnwlOA8vo9WX3Fp7I6AY,29498
 dtflow/cli/commands.py,sha256=LvyDQ_nWUM7UlPDEFQadRdw5O2ZKDLgF41_xAJRhYxI,1583
@@ -23,9 +24,9 @@ dtflow/cli/pipeline.py,sha256=QNEo-BJlaC1CVnVeRZr7TwfuZYloJ4TebIzJ5ALzry0,1426
 dtflow/cli/sample.py,sha256=etbro5I0pyNgn0Qfhp1M6Bh-95JN-AntDa5AwVe_oKY,18269
 dtflow/cli/skill.py,sha256=opiTEBejA7JHKrEMftMOPDQlOgZ4n59rwaHXGU1Nukk,2022
 dtflow/cli/split.py,sha256=96bhWnxHnjIqifoliLgciApkLbwQU8bWHovK8bcMk9g,3667
-dtflow/cli/stats.py,sha256=Jx3d4X0ftgpzU5q5RAWZEVJWwXviQTF4EAwBmz1IliA,31366
+dtflow/cli/stats.py,sha256=HkTZD80h4tzYXTtMnfpjLUMP6kl_es6ifcmExxzGdMU,31813
 dtflow/cli/transform.py,sha256=w6xqMOxPxQvL2u_BPCfpDHuPSC9gmcqMPVN8s-B6bbY,15052
-dtflow/cli/validate.py,sha256=65aGVlMS_Rq0Ch0YQ-TclVJ03RQP4CnG137wthzb8Ao,4384
+dtflow/cli/validate.py,sha256=Frs-jKcDHmYozpmIYZueDSX5o2i1Xn-WW81FGUyUrng,5796
 dtflow/storage/__init__.py,sha256=C0jpWNQU808Ezz7lWneddABal3wILy8ijFUNiSKbHV4,362
 dtflow/storage/io.py,sha256=ZH2aSE-S89gpy3z4oTqhcqWf4u10OdkDoyul7o_YBDI,23374
 dtflow/utils/__init__.py,sha256=Pn-ltwV04fBQmeZG7FxInDQmzH29LYOi90LgeLMEuQk,506
@@ -33,7 +34,7 @@ dtflow/utils/display.py,sha256=OeOdTh6mbDwSkDWlmkjfpTjy2QG8ZUaYU0NpHUWkpEQ,5881
 dtflow/utils/field_path.py,sha256=K8nU196RxTSJ1OoieTWGcYOWl9KjGq2iSxCAkfjECuM,7621
 dtflow/utils/helpers.py,sha256=JXN176_B2pm53GLVyZ1wj3wrmBJG52Tkw6AMQSdj7M8,791
 dtflow/utils/text_parser.py,sha256=0t2TMOSha4dTiDu9H4ygdb67cI20zhtBH1XavDspL_g,3727
-dtflow-0.5.9.dist-info/METADATA,sha256=Pu92Dz2vj7U_dki4A0e5xgka36BTT9K2PnN1LIeEhN0,25839
-dtflow-0.5.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dtflow-0.5.9.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
-dtflow-0.5.9.dist-info/RECORD,,
+dtflow-0.5.10.dist-info/METADATA,sha256=OGefMoe17by5IbxdxZgqoJ1Y6OWPt_iGEFM4KgltRZw,26023
+dtflow-0.5.10.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dtflow-0.5.10.dist-info/entry_points.txt,sha256=dadIDOK7Iu9pMxnMPBfpb4aAPe4hQbBOshpQYjVYpGc,44
+dtflow-0.5.10.dist-info/RECORD,,