dtflow 0.4.3__py3-none-any.whl → 0.5.2__py3-none-any.whl

dtflow/__init__.py

@@ -4,6 +4,7 @@ DataTransformer: 简洁的数据格式转换工具
 核心功能:
 - DataTransformer: 数据加载、转换、保存
 - presets: 预设转换模板 (openai_chat, alpaca, sharegpt, dpo_pair, simple_qa)
+- schema: 数据结构验证 (Schema, Field)
 - tokenizers: Token 统计和过滤
 - converters: HuggingFace/OpenAI 等格式转换
 """
@@ -26,6 +27,23 @@ from .converters import (  # LLaMA-Factory 扩展; ms-swift
 )
 from .core import DataTransformer, DictWrapper, TransformError, TransformErrors
 from .presets import get_preset, list_presets
+from .schema import (
+    Field,
+    Schema,
+    ValidationError,
+    ValidationResult,
+    alpaca_schema,
+    dpo_schema,
+    openai_chat_schema,
+    sharegpt_schema,
+    validate_data,
+)
+from .framework import (
+    CompatibilityResult,
+    check_compatibility,
+    detect_format,
+    export_for,
+)
 from .storage import load_data, sample_file, save_data
 from .streaming import StreamingTransformer, load_sharded, load_stream, process_shards
 from .tokenizers import (
@@ -42,7 +60,7 @@ from .tokenizers import (
     token_stats,
 )
 
-__version__ = "0.4.3"
+__version__ = "0.5.2"
 
 __all__ = [
     # core
@@ -53,6 +71,21 @@ __all__ = [
     # presets
     "get_preset",
     "list_presets",
+    # schema
+    "Schema",
+    "Field",
+    "ValidationResult",
+    "ValidationError",
+    "validate_data",
+    "openai_chat_schema",
+    "alpaca_schema",
+    "dpo_schema",
+    "sharegpt_schema",
+    # framework
+    "CompatibilityResult",
+    "check_compatibility",
+    "detect_format",
+    "export_for",
     # storage
     "save_data",
     "load_data",

dtflow/__main__.py

@@ -18,6 +18,7 @@ Commands:
     clean        数据清洗
     run          执行 Pipeline 配置文件
     history      显示数据血缘历史
+    validate     使用 Schema 验证数据格式
     mcp          MCP 服务管理（install/uninstall/status）
     logs         日志查看工具使用说明
 """
@@ -40,6 +41,7 @@ from .cli.commands import stats as _stats
 from .cli.commands import tail as _tail
 from .cli.commands import token_stats as _token_stats
 from .cli.commands import transform as _transform
+from .cli.commands import validate as _validate
 
 # 创建主应用
 app = typer.Typer(
@@ -211,6 +213,26 @@ def history(
     _history(filename, json)
 
 
+# ============ 验证命令 ============
+
+
+@app.command()
+def validate(
+    filename: str = typer.Argument(..., help="输入文件路径"),
+    preset: Optional[str] = typer.Option(
+        None, "--preset", "-p", help="预设 Schema: openai_chat, alpaca, dpo, sharegpt"
+    ),
+    output: Optional[str] = typer.Option(None, "--output", "-o", help="输出有效数据的文件路径"),
+    filter: bool = typer.Option(
+        False, "--filter", "-f", help="过滤无效数据并保存"
+    ),
+    max_errors: int = typer.Option(20, "--max-errors", help="最多显示的错误数量"),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="显示详细信息"),
+):
+    """使用预设 Schema 验证数据格式"""
+    _validate(filename, preset, output, filter, max_errors, verbose)
+
+
 # ============ 工具命令 ============
 
 

dtflow/cli/commands.py

@@ -33,6 +33,9 @@ from .pipeline import run
 # 血缘追踪命令
 from .lineage import history
 
+# 验证命令
+from .validate import validate
+
 __all__ = [
     # 采样
     "sample",
@@ -53,4 +56,6 @@ __all__ = [
     "run",
     # 血缘
     "history",
+    # 验证
+    "validate",
 ]

dtflow/cli/common.py

@@ -57,7 +57,7 @@ def _get_file_row_count(filepath: Path) -> Optional[int]:
     return None
 
 
-def _format_value(value: Any, max_len: int = 80) -> str:
+def _format_value(value: Any, max_len: int = 120) -> str:
     """格式化单个值，长文本截断。"""
     if value is None:
         return "[dim]null[/dim]"
@@ -66,18 +66,22 @@ def _format_value(value: Any, max_len: int = 80) -> str:
     if isinstance(value, (int, float)):
         return f"[cyan]{value}[/cyan]"
     if isinstance(value, str):
+        half_len = max_len // 2
         # 处理多行文本
         if "\n" in value:
             lines = value.split("\n")
-            if len(lines) > 3:
-                preview = lines[0][:max_len] + f"... [dim]({len(lines)} 行)[/dim]"
-            else:
-                preview = value.replace("\n", "\\n")
-                if len(preview) > max_len:
-                    preview = preview[:max_len] + "..."
+            preview = value.replace("\n", "\\n")
+            if len(preview) > max_len:
+                # 前半 + 省略标记 + 后半
+                head = preview[:half_len]
+                tail = preview[-half_len:]
+                return f'"{head} [yellow]<<<{len(lines)}行>>>[/yellow] {tail}"'
             return f'"{preview}"'
         if len(value) > max_len:
-            return f'"{value[:max_len]}..." [dim]({len(value)} 字符)[/dim]'
+            # 前半 + 省略标记 + 后半
+            head = value[:half_len]
+            tail = value[-half_len:]
+            return f'"{head} [yellow]<<<{len(value)}字符>>>[/yellow] {tail}"'
         return f'"{value}"'
     return str(value)
 
@@ -86,7 +90,7 @@ def _format_nested(
     value: Any,
     indent: str = "",
     is_last: bool = True,
-    max_len: int = 80,
+    max_len: int = 120,
 ) -> List[str]:
     """
     递归格式化嵌套结构，返回行列表。

dtflow/cli/stats.py

@@ -465,34 +465,65 @@ def token_stats(
         return
 
     total = len(data)
-    print(f"   共 {total} 条数据")
-    print(f"🔢 统计 Token (模型: {model}, 字段: {field})...")
+    print(f"   共 {total:,} 条数据")
 
     # 检查字段类型并选择合适的统计方法（支持嵌套路径）
     sample = data[0]
     field_value = get_field_with_spec(sample, field)
 
+    # 尝试使用 rich 进度条
     try:
-        if isinstance(field_value, list) and field_value and isinstance(field_value[0], dict):
-            # messages 格式
-            from ..tokenizers import messages_token_stats
-
-            stats_result = messages_token_stats(data, messages_field=field, model=model)
-            _print_messages_token_stats(stats_result, detailed)
-        else:
-            # 普通文本字段
-            from ..tokenizers import token_stats as compute_token_stats
-
-            stats_result = compute_token_stats(data, fields=field, model=model)
-            _print_text_token_stats(stats_result, detailed)
-    except ImportError as e:
-        print(f"错误: {e}")
-        return
-    except Exception as e:
-        print(f"错误: 统计失败 - {e}")
-        import traceback
+        from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
+
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[bold blue]统计 Token"),
+            BarColumn(),
+            TaskProgressColumn(),
+            TextColumn(f"(模型: {model})"),
+        ) as progress:
+            task = progress.add_task("", total=total)
+
+            def update_progress(current: int, total_count: int):
+                progress.update(task, completed=current)
+
+            if isinstance(field_value, list) and field_value and isinstance(field_value[0], dict):
+                from ..tokenizers import messages_token_stats
+
+                stats_result = messages_token_stats(
+                    data, messages_field=field, model=model, progress_callback=update_progress
+                )
+                _print_messages_token_stats(stats_result, detailed)
+            else:
+                from ..tokenizers import token_stats as compute_token_stats
+
+                stats_result = compute_token_stats(
+                    data, fields=field, model=model, progress_callback=update_progress
+                )
+                _print_text_token_stats(stats_result, detailed)
+
+    except ImportError:
+        # 没有 rich，显示简单进度
+        print(f"🔢 统计 Token (模型: {model}, 字段: {field})...")
+        try:
+            if isinstance(field_value, list) and field_value and isinstance(field_value[0], dict):
+                from ..tokenizers import messages_token_stats
+
+                stats_result = messages_token_stats(data, messages_field=field, model=model)
+                _print_messages_token_stats(stats_result, detailed)
+            else:
+                from ..tokenizers import token_stats as compute_token_stats
 
-        traceback.print_exc()
+                stats_result = compute_token_stats(data, fields=field, model=model)
+                _print_text_token_stats(stats_result, detailed)
+        except ImportError as e:
+            print(f"错误: {e}")
+            return
+        except Exception as e:
+            print(f"错误: 统计失败 - {e}")
+            import traceback
+
+            traceback.print_exc()
 
 
 def _print_messages_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
@@ -505,21 +536,39 @@ def _print_messages_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
         console = Console()
 
         # 概览
+        std = stats.get("std_tokens", 0)
         overview = (
             f"[bold]总样本数:[/bold] {stats['count']:,}\n"
             f"[bold]总 Token:[/bold] {stats['total_tokens']:,}\n"
-            f"[bold]平均 Token:[/bold] {stats['avg_tokens']:,}\n"
-            f"[bold]中位数:[/bold] {stats['median_tokens']:,}\n"
+            f"[bold]平均 Token:[/bold] {stats['avg_tokens']:,} (std: {std:.1f})\n"
             f"[bold]范围:[/bold] {stats['min_tokens']:,} - {stats['max_tokens']:,}"
         )
         console.print(Panel(overview, title="📊 Token 统计概览", expand=False))
 
+        # 百分位数表格
+        table = Table(title="📈 分布统计")
+        table.add_column("百分位", style="cyan", justify="center")
+        table.add_column("Token 数", justify="right")
+        percentiles = [
+            ("Min", stats["min_tokens"]),
+            ("P25", stats.get("p25", "-")),
+            ("P50 (中位数)", stats.get("median_tokens", "-")),
+            ("P75", stats.get("p75", "-")),
+            ("P90", stats.get("p90", "-")),
+            ("P95", stats.get("p95", "-")),
+            ("P99", stats.get("p99", "-")),
+            ("Max", stats["max_tokens"]),
+        ]
+        for name, val in percentiles:
+            table.add_row(name, f"{val:,}" if isinstance(val, int) else str(val))
+        console.print(table)
+
         if detailed:
-            # 详细统计
-            table = Table(title="📋 分角色统计")
-            table.add_column("角色", style="cyan")
-            table.add_column("Token 数", justify="right")
-            table.add_column("占比", justify="right")
+            # 分角色统计
+            role_table = Table(title="📋 分角色统计")
+            role_table.add_column("角色", style="cyan")
+            role_table.add_column("Token 数", justify="right")
+            role_table.add_column("占比", justify="right")
 
             total = stats["total_tokens"]
             for role, key in [
@@ -529,22 +578,27 @@ def _print_messages_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
             ]:
                 tokens = stats.get(key, 0)
                 pct = tokens / total * 100 if total > 0 else 0
-                table.add_row(role, f"{tokens:,}", f"{pct:.1f}%")
+                role_table.add_row(role, f"{tokens:,}", f"{pct:.1f}%")
 
-            console.print(table)
+            console.print(role_table)
             console.print(f"\n平均对话轮数: {stats.get('avg_turns', 0)}")
 
     except ImportError:
         # 没有 rich，使用普通打印
+        std = stats.get("std_tokens", 0)
         print(f"\n{'=' * 40}")
         print("📊 Token 统计概览")
         print(f"{'=' * 40}")
         print(f"总样本数: {stats['count']:,}")
         print(f"总 Token: {stats['total_tokens']:,}")
-        print(f"平均 Token: {stats['avg_tokens']:,}")
-        print(f"中位数: {stats['median_tokens']:,}")
+        print(f"平均 Token: {stats['avg_tokens']:,} (std: {std:.1f})")
         print(f"范围: {stats['min_tokens']:,} - {stats['max_tokens']:,}")
 
+        print(f"\n📈 百分位分布:")
+        print(f"  P25: {stats.get('p25', '-'):,}  P50: {stats.get('median_tokens', '-'):,}")
+        print(f"  P75: {stats.get('p75', '-'):,}  P90: {stats.get('p90', '-'):,}")
+        print(f"  P95: {stats.get('p95', '-'):,}  P99: {stats.get('p99', '-'):,}")
+
         if detailed:
             print(f"\n{'=' * 40}")
             print("📋 分角色统计")
@@ -566,24 +620,48 @@ def _print_text_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
     try:
         from rich.console import Console
         from rich.panel import Panel
+        from rich.table import Table
 
         console = Console()
 
+        std = stats.get("std_tokens", 0)
         overview = (
             f"[bold]总样本数:[/bold] {stats['count']:,}\n"
             f"[bold]总 Token:[/bold] {stats['total_tokens']:,}\n"
-            f"[bold]平均 Token:[/bold] {stats['avg_tokens']:.1f}\n"
-            f"[bold]中位数:[/bold] {stats['median_tokens']:,}\n"
+            f"[bold]平均 Token:[/bold] {stats['avg_tokens']:.1f} (std: {std:.1f})\n"
             f"[bold]范围:[/bold] {stats['min_tokens']:,} - {stats['max_tokens']:,}"
         )
         console.print(Panel(overview, title="📊 Token 统计", expand=False))
 
+        # 百分位数表格
+        table = Table(title="📈 分布统计")
+        table.add_column("百分位", style="cyan", justify="center")
+        table.add_column("Token 数", justify="right")
+        percentiles = [
+            ("Min", stats["min_tokens"]),
+            ("P25", stats.get("p25", "-")),
+            ("P50 (中位数)", stats.get("median_tokens", "-")),
+            ("P75", stats.get("p75", "-")),
+            ("P90", stats.get("p90", "-")),
+            ("P95", stats.get("p95", "-")),
+            ("P99", stats.get("p99", "-")),
+            ("Max", stats["max_tokens"]),
+        ]
+        for name, val in percentiles:
+            table.add_row(name, f"{val:,}" if isinstance(val, int) else str(val))
+        console.print(table)
+
     except ImportError:
+        std = stats.get("std_tokens", 0)
         print(f"\n{'=' * 40}")
         print("📊 Token 统计")
         print(f"{'=' * 40}")
         print(f"总样本数: {stats['count']:,}")
         print(f"总 Token: {stats['total_tokens']:,}")
-        print(f"平均 Token: {stats['avg_tokens']:.1f}")
-        print(f"中位数: {stats['median_tokens']:,}")
+        print(f"平均 Token: {stats['avg_tokens']:.1f} (std: {std:.1f})")
         print(f"范围: {stats['min_tokens']:,} - {stats['max_tokens']:,}")
+
+        print(f"\n📈 百分位分布:")
+        print(f"  P25: {stats.get('p25', '-'):,}  P50: {stats.get('median_tokens', '-'):,}")
+        print(f"  P75: {stats.get('p75', '-'):,}  P90: {stats.get('p90', '-'):,}")
+        print(f"  P95: {stats.get('p95', '-'):,}  P99: {stats.get('p99', '-'):,}")

dtflow/cli/validate.py

@@ -0,0 +1,152 @@
+"""
+CLI Schema 验证命令
+"""
+
+from pathlib import Path
+from typing import Optional
+
+from ..schema import (
+    Schema,
+    Field,
+    alpaca_schema,
+    dpo_schema,
+    openai_chat_schema,
+    sharegpt_schema,
+)
+from ..storage.io import load_data, save_data
+from .common import _check_file_format
+
+
+# 预设 Schema 映射
+PRESET_SCHEMAS = {
+    "openai_chat": openai_chat_schema,
+    "openai-chat": openai_chat_schema,
+    "chat": openai_chat_schema,
+    "alpaca": alpaca_schema,
+    "dpo": dpo_schema,
+    "dpo_pair": dpo_schema,
+    "sharegpt": sharegpt_schema,
+}
+
+
+def validate(
+    filename: str,
+    preset: Optional[str] = None,
+    output: Optional[str] = None,
+    filter_invalid: bool = False,
+    max_errors: int = 20,
+    verbose: bool = False,
+) -> None:
+    """
+    使用 Schema 验证数据文件。
+
+    Args:
+        filename: 输入文件路径
+        preset: 预设 Schema 名称 (openai_chat, alpaca, dpo, sharegpt)
+        output: 输出文件路径（保存有效数据）
+        filter_invalid: 过滤无效数据并保存
+        max_errors: 最多显示的错误数量
+        verbose: 显示详细信息
+
+    Examples:
+        dt validate data.jsonl --preset=openai_chat
+        dt validate data.jsonl --preset=alpaca -o valid.jsonl
+        dt validate data.jsonl --preset=chat --filter
+    """
+    filepath = Path(filename)
+
+    if not filepath.exists():
+        print(f"错误: 文件不存在 - {filename}")
+        return
+
+    if not _check_file_format(filepath):
+        return
+
+    # 确定 Schema
+    if preset is None:
+        # 列出可用的预设
+        print("请指定预设 Schema (--preset):")
+        print()
+        for name in ["openai_chat", "alpaca", "dpo", "sharegpt"]:
+            print(f"  --preset={name}")
+        print()
+        print("示例:")
+        print(f"  dt validate {filename} --preset=openai_chat")
+        return
+
+    preset_lower = preset.lower().replace("-", "_")
+    if preset_lower not in PRESET_SCHEMAS:
+        print(f"错误: 未知的预设 Schema '{preset}'")
+        print(f"可用预设: {', '.join(['openai_chat', 'alpaca', 'dpo', 'sharegpt'])}")
+        return
+
+    schema = PRESET_SCHEMAS[preset_lower]()
+
+    # 加载数据
+    try:
+        data = load_data(str(filepath))
+    except Exception as e:
+        print(f"错误: 无法读取文件 - {e}")
+        return
+
+    if not data:
+        print("文件为空")
+        return
+
+    total = len(data)
+    print(f"验证文件: {filepath.name}")
+    print(f"预设 Schema: {preset}")
+    print(f"总记录数: {total}")
+    print()
+
+    # 验证
+    valid_data = []
+    invalid_count = 0
+    error_samples = []
+
+    for i, item in enumerate(data):
+        result = schema.validate(item)
+        if result.valid:
+            valid_data.append(item)
+        else:
+            invalid_count += 1
+            if len(error_samples) < max_errors:
+                error_samples.append((i, result))
+
+    valid_count = len(valid_data)
+    valid_ratio = valid_count / total * 100 if total > 0 else 0
+
+    # 输出结果
+    if invalid_count == 0:
+        print(f"✅ 全部通过! {valid_count}/{total} 条记录有效 (100%)")
+    else:
+        print(f"⚠️ 验证结果: {valid_count}/{total} 条有效 ({valid_ratio:.1f}%)")
+        print(f"   无效记录: {invalid_count} 条")
+        print()
+
+        # 显示错误示例
+        print(f"错误示例 (最多显示 {max_errors} 条):")
+        print("-" * 60)
+
+        for idx, result in error_samples:
+            print(f"[第 {idx} 行]")
+            for err in result.errors[:3]:  # 每条记录最多显示 3 个错误
+                print(f"  - {err}")
+            if len(result.errors) > 3:
+                print(f"  ... 还有 {len(result.errors) - 3} 个错误")
+            print()
+
+    # 保存有效数据
+    if output or filter_invalid:
+        output_path = output or str(filepath).replace(
+            filepath.suffix, f"_valid{filepath.suffix}"
+        )
+        save_data(valid_data, output_path)
+        print(f"✅ 有效数据已保存: {output_path} ({valid_count} 条)")
+
+    # 详细模式：显示 Schema 定义
+    if verbose:
+        print()
+        print("Schema 定义:")
+        print("-" * 40)
+        print(schema)