dtflow 0.5.8__py3-none-any.whl → 0.5.9__py3-none-any.whl

dtflow/cli/stats.py

@@ -209,7 +209,7 @@ def _quick_stats(filepath: Path) -> None:
         print(f"字段: {len(fields)} 个")
 
         if fields:
-            print("\n📋 字段结构:")
+            print(f"\n📋 字段结构:")
             for i, f in enumerate(fields, 1):
                 print(f"  {i}. {f['field']} ({f['type']})")
 
@@ -597,14 +597,14 @@ def _print_stats(filename: str, total: int, field_stats: List[Dict[str, Any]]) -
     except ImportError:
         # 没有 rich，使用普通打印
         print(f"\n{'=' * 50}")
-        print("📊 数据概览")
+        print(f"📊 数据概览")
         print(f"{'=' * 50}")
         print(f"文件: {filename}")
         print(f"总数: {total:,} 条")
         print(f"字段: {len(field_stats)} 个")
 
         print(f"\n{'=' * 50}")
-        print("📋 字段统计")
+        print(f"📋 字段统计")
         print(f"{'=' * 50}")
         print(f"{'字段':<20} {'类型':<8} {'非空率':<8} {'唯一值':<8}")
         print("-" * 50)
@@ -620,7 +620,6 @@ def token_stats(
     field: str = "messages",
     model: str = "cl100k_base",
     detailed: bool = False,
-    workers: Optional[int] = None,
 ) -> None:
     """
     统计数据集的 Token 信息。
@@ -630,7 +629,6 @@ def token_stats(
         field: 要统计的字段（默认 messages），支持嵌套路径语法
         model: 分词器: cl100k_base (默认), qwen2.5, llama3, gpt-4 等
         detailed: 是否显示详细统计
-        workers: 并行进程数，None 自动检测，1 禁用并行
 
     Examples:
         dt token-stats data.jsonl
@@ -638,7 +636,6 @@ def token_stats(
         dt token-stats data.jsonl --field=conversation.messages
         dt token-stats data.jsonl --field=messages[-1].content   # 统计最后一条消息
         dt token-stats data.jsonl --detailed
-        dt token-stats data.jsonl --workers=4   # 使用 4 进程
     """
     filepath = Path(filename)
 
@@ -670,7 +667,7 @@ def token_stats(
 
     # 尝试使用 rich 进度条
     try:
-        from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
+        from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
 
         with Progress(
             SpinnerColumn(),
@@ -688,22 +685,14 @@ def token_stats(
                 from ..tokenizers import messages_token_stats
 
                 stats_result = messages_token_stats(
-                    data,
-                    messages_field=field,
-                    model=model,
-                    progress_callback=update_progress,
-                    workers=workers,
+                    data, messages_field=field, model=model, progress_callback=update_progress
                 )
                 _print_messages_token_stats(stats_result, detailed)
             else:
                 from ..tokenizers import token_stats as compute_token_stats
 
                 stats_result = compute_token_stats(
-                    data,
-                    fields=field,
-                    model=model,
-                    progress_callback=update_progress,
-                    workers=workers,
+                    data, fields=field, model=model, progress_callback=update_progress
                 )
                 _print_text_token_stats(stats_result, detailed)
 
@@ -714,14 +703,12 @@ def token_stats(
             if isinstance(field_value, list) and field_value and isinstance(field_value[0], dict):
                 from ..tokenizers import messages_token_stats
 
-                stats_result = messages_token_stats(
-                    data, messages_field=field, model=model, workers=workers
-                )
+                stats_result = messages_token_stats(data, messages_field=field, model=model)
                 _print_messages_token_stats(stats_result, detailed)
             else:
                 from ..tokenizers import token_stats as compute_token_stats
 
-                stats_result = compute_token_stats(data, fields=field, model=model, workers=workers)
+                stats_result = compute_token_stats(data, fields=field, model=model)
                 _print_text_token_stats(stats_result, detailed)
         except ImportError as e:
             print(f"错误: {e}")
@@ -801,7 +788,7 @@ def _print_messages_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
         print(f"平均 Token: {stats['avg_tokens']:,} (std: {std:.1f})")
         print(f"范围: {stats['min_tokens']:,} - {stats['max_tokens']:,}")
 
-        print("\n📈 百分位分布:")
+        print(f"\n📈 百分位分布:")
         print(f"  P25: {stats.get('p25', '-'):,}  P50: {stats.get('median_tokens', '-'):,}")
         print(f"  P75: {stats.get('p75', '-'):,}  P90: {stats.get('p90', '-'):,}")
         print(f"  P95: {stats.get('p95', '-'):,}  P99: {stats.get('p99', '-'):,}")
@@ -868,7 +855,7 @@ def _print_text_token_stats(stats: Dict[str, Any], detailed: bool) -> None:
         print(f"平均 Token: {stats['avg_tokens']:.1f} (std: {std:.1f})")
         print(f"范围: {stats['min_tokens']:,} - {stats['max_tokens']:,}")
 
-        print("\n📈 百分位分布:")
+        print(f"\n📈 百分位分布:")
         print(f"  P25: {stats.get('p25', '-'):,}  P50: {stats.get('median_tokens', '-'):,}")
         print(f"  P75: {stats.get('p75', '-'):,}  P90: {stats.get('p90', '-'):,}")
         print(f"  P95: {stats.get('p95', '-'):,}  P99: {stats.get('p99', '-'):,}")

dtflow/cli/validate.py

@@ -6,6 +6,8 @@ from pathlib import Path
 from typing import Optional
 
 from ..schema import (
+    Schema,
+    Field,
     alpaca_schema,
     dpo_schema,
     openai_chat_schema,
@@ -14,6 +16,7 @@ from ..schema import (
 from ..storage.io import load_data, save_data
 from .common import _check_file_format
 
+
 # 预设 Schema 映射
 PRESET_SCHEMAS = {
     "openai_chat": openai_chat_schema,
@@ -33,7 +36,6 @@ def validate(
     filter_invalid: bool = False,
     max_errors: int = 20,
     verbose: bool = False,
-    workers: Optional[int] = None,
 ) -> None:
     """
     使用 Schema 验证数据文件。
@@ -45,13 +47,11 @@ def validate(
         filter_invalid: 过滤无效数据并保存
         max_errors: 最多显示的错误数量
         verbose: 显示详细信息
-        workers: 并行进程数，None 自动检测，1 禁用并行
 
     Examples:
         dt validate data.jsonl --preset=openai_chat
         dt validate data.jsonl --preset=alpaca -o valid.jsonl
         dt validate data.jsonl --preset=chat --filter
-        dt validate data.jsonl --preset=chat --workers=4
     """
     filepath = Path(filename)
 
@@ -99,54 +99,19 @@ def validate(
     print(f"总记录数: {total}")
     print()
 
-    # 验证（使用并行或串行）
-    use_parallel = workers != 1 and total >= 1000
-
-    if use_parallel:
-        # 使用进度条（如果有 rich）
-        try:
-            from rich.progress import (
-                BarColumn,
-                Progress,
-                SpinnerColumn,
-                TaskProgressColumn,
-                TextColumn,
-            )
-
-            with Progress(
-                SpinnerColumn(),
-                TextColumn("[bold blue]验证数据"),
-                BarColumn(),
-                TaskProgressColumn(),
-            ) as progress:
-                task = progress.add_task("", total=total)
-
-                def update_progress(current: int, total_count: int):
-                    progress.update(task, completed=current)
-
-                valid_data, invalid_results = schema.validate_parallel(
-                    data, workers=workers, progress_callback=update_progress
-                )
-        except ImportError:
-            print("🔍 验证数据...")
-            valid_data, invalid_results = schema.validate_parallel(data, workers=workers)
-
-        invalid_count = len(invalid_results)
-        error_samples = invalid_results[:max_errors]
-    else:
-        # 串行验证
-        valid_data = []
-        invalid_count = 0
-        error_samples = []
-
-        for i, item in enumerate(data):
-            result = schema.validate(item)
-            if result.valid:
-                valid_data.append(item)
-            else:
-                invalid_count += 1
-                if len(error_samples) < max_errors:
-                    error_samples.append((i, result))
+    # 验证
+    valid_data = []
+    invalid_count = 0
+    error_samples = []
+
+    for i, item in enumerate(data):
+        result = schema.validate(item)
+        if result.valid:
+            valid_data.append(item)
+        else:
+            invalid_count += 1
+            if len(error_samples) < max_errors:
+                error_samples.append((i, result))
 
     valid_count = len(valid_data)
     valid_ratio = valid_count / total * 100 if total > 0 else 0
@@ -173,7 +138,9 @@ def validate(
 
     # 保存有效数据
     if output or filter_invalid:
-        output_path = output or str(filepath).replace(filepath.suffix, f"_valid{filepath.suffix}")
+        output_path = output or str(filepath).replace(
+            filepath.suffix, f"_valid{filepath.suffix}"
+        )
         save_data(valid_data, output_path)
         print(f"✅ 有效数据已保存: {output_path} ({valid_count} 条)")
 

dtflow/eval.py

@@ -0,0 +1,276 @@
+"""
+评估指标计算模块
+
+提供分类任务的指标计算和评估报告导出：
+- MetricsCalculator: 计算 accuracy/precision/recall/F1/混淆矩阵
+- export_eval_report: 生成 metrics.md + result.jsonl + bad_case.jsonl
+
+依赖: scikit-learn, pandas
+安装: pip install dtflow[eval]
+"""
+
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
+
+def _check_eval_deps():
+    """检查 eval 依赖是否已安装"""
+    try:
+        import pandas  # noqa: F401
+        import sklearn  # noqa: F401
+    except ImportError as e:
+        missing = str(e).split("'")[1] if "'" in str(e) else str(e)
+        raise ImportError(
+            f"eval 功能需要额外依赖: {missing}\n" f"请运行: pip install dtflow[eval]"
+        ) from e
+
+
+class MetricsCalculator:
+    """分类指标计算器
+
+    基于 sklearn 计算 accuracy/precision/recall/F1/混淆矩阵/分类报告。
+
+    Args:
+        df: 包含预测列和标签列的 DataFrame
+        pred_col: 预测值列名
+        label_col: 标签值列名
+        include_macro_micro_avg: 是否在报告中包含 macro/micro 平均
+        remove_matrix_zero_row: 是否移除混淆矩阵中 support=0 的行
+    """
+
+    def __init__(
+        self,
+        df: "DataFrame",
+        pred_col: str = "predict",
+        label_col: str = "label",
+        include_macro_micro_avg: bool = False,
+        remove_matrix_zero_row: bool = False,
+    ):
+        _check_eval_deps()
+        self.df = df
+        self.y_pred = df[pred_col]
+        self.y_true = df[label_col]
+        self.all_labels = sorted(set(self.y_true.unique()).union(set(self.y_pred.unique())))
+        self.needed_labels = None
+        self.remove_matrix_zero_row = remove_matrix_zero_row
+        self.include_macro_micro_avg = include_macro_micro_avg
+        self.metrics = self._calculate_metrics()
+
+    def _calculate_metrics(self):
+        from sklearn.metrics import (
+            accuracy_score,
+            classification_report,
+            confusion_matrix,
+            precision_score,
+            recall_score,
+        )
+
+        accuracy = accuracy_score(self.y_true, self.y_pred)
+        precision = precision_score(
+            self.y_true, self.y_pred, labels=self.all_labels, average="weighted", zero_division=0
+        )
+        recall = recall_score(
+            self.y_true, self.y_pred, labels=self.all_labels, average="weighted", zero_division=0
+        )
+        conf_matrix = confusion_matrix(self.y_true, self.y_pred, labels=self.all_labels)
+        report = classification_report(
+            self.y_true, self.y_pred, labels=self.all_labels, output_dict=True, zero_division=0
+        )
+
+        # 默认只保留加权平均
+        if not self.include_macro_micro_avg:
+            report = {
+                label: metrics
+                for label, metrics in report.items()
+                if label in self.all_labels or label == "weighted avg"
+            }
+
+        # 去除 support=0 的类别（注意 accuracy 是 float 不是 dict）
+        report = {
+            label: metrics
+            for label, metrics in report.items()
+            if isinstance(metrics, dict) and metrics.get("support", 0) > 0
+        }
+
+        self.needed_labels = [label for label in report.keys() if label in self.all_labels]
+
+        # 可选移除混淆矩阵中不需要的行
+        needed_idx_list = [self.all_labels.index(label) for label in self.needed_labels]
+        if self.remove_matrix_zero_row:
+            conf_matrix = conf_matrix[needed_idx_list]
+
+        return {
+            "accuracy": accuracy,
+            "precision": precision,
+            "recall": recall,
+            "confusion_matrix": conf_matrix,
+            "classification_report": report,
+        }
+
+    def get_metrics(self):
+        return self.metrics
+
+    def format_classification_report_as_markdown(self):
+        """将分类报告格式化为 Markdown 表格"""
+        report = self.metrics["classification_report"]
+        header = "| Label | Precision | Recall | F1-score | Support |\n"
+        separator = "|-------|-----------|--------|----------|---------|\n"
+        rows = []
+        for label, metrics in report.items():
+            if isinstance(metrics, dict):
+                rows.append(
+                    f"| {label} | {metrics['precision']:.2f} | {metrics['recall']:.2f} "
+                    f"| {metrics['f1-score']:.2f} | {metrics['support']:.0f} |"
+                )
+        return header + separator + "\n".join(rows)
+
+    def _clean_label_for_markdown(self, label, max_length=20):
+        """清理标签文本，使其适合 Markdown 表格显示"""
+        label = str(label).replace("\n", " ")
+        label = label.replace("|", "\\|")
+        label = label.replace("-", "\\-")
+        label = label.replace("<", "&lt;")
+        label = label.replace(">", "&gt;")
+        if len(label) > max_length:
+            label = label[:max_length] + "..."
+        label = label.strip()
+        if not label:
+            label = "(empty)"
+        return label
+
+    def format_confusion_matrix_as_markdown(self, max_label_length=20):
+        """将混淆矩阵格式化为 Markdown 表格"""
+        matrix = self.metrics["confusion_matrix"]
+
+        if self.remove_matrix_zero_row:
+            labels = self.needed_labels
+        else:
+            labels = self.all_labels
+
+        processed_labels = [self._clean_label_for_markdown(lb, max_label_length) for lb in labels]
+
+        header = "| 真实值/预测值 | " + " | ".join(processed_labels) + " |\n"
+        separator_parts = [":---:"] * (len(processed_labels) + 1)
+        separator = "| " + " | ".join(separator_parts) + " |\n"
+
+        rows = []
+        for i, row in enumerate(matrix):
+            row_label = self._clean_label_for_markdown(labels[i], max_label_length)
+            formatted_row = [f"{num:,}" for num in row]
+            rows.append(f"| {row_label} | " + " | ".join(formatted_row) + " |")
+
+        return header + separator + "\n".join(rows)
+
+
+def export_eval_report(
+    df: "DataFrame",
+    pred_col: str,
+    label_col: str,
+    record_folder: str = "record",
+    input_name: Optional[str] = None,
+):
+    """生成评估报告并保存到指定目录
+
+    输出文件：
+    - metrics.md: 指标概览 + 分类报告 + 混淆矩阵
+    - result.jsonl: 完整预测结果
+    - bad_case.jsonl: 预测错误样本
+
+    Args:
+        df: 包含预测和标签的 DataFrame
+        pred_col: 预测值列名
+        label_col: 标签值列名
+        record_folder: 输出根目录
+        input_name: 输入文件名（用于子目录命名）
+    """
+    from rich.console import Console
+    from rich.markdown import Markdown
+
+    calculator = MetricsCalculator(df, pred_col=pred_col, label_col=label_col)
+    metrics = calculator.get_metrics()
+
+    # 用 Rich Table 构建指标概览（替代 tabulate）
+    from rich.table import Table
+
+    overview_table = Table(title="指标概览", show_header=True)
+    overview_table.add_column("Accuracy", justify="center")
+    overview_table.add_column("Precision", justify="center")
+    overview_table.add_column("Recall", justify="center")
+    overview_table.add_row(
+        f"{metrics['accuracy']:.4f}",
+        f"{metrics['precision']:.4f}",
+        f"{metrics['recall']:.4f}",
+    )
+
+    # 构建 Markdown 报告内容
+    md = (
+        f"\n\n### 指标概览\n\n"
+        f"| Accuracy | Precision | Recall |\n"
+        f"|----------|-----------|--------|\n"
+        f"| {metrics['accuracy']:.4f} | {metrics['precision']:.4f} | {metrics['recall']:.4f} |"
+    )
+    metrics_md = calculator.format_classification_report_as_markdown()
+    confusion_md = calculator.format_confusion_matrix_as_markdown()
+    md += f"\n\n### Classification Report\n{metrics_md}\n" f"\n### Confusion Matrix\n{confusion_md}"
+
+    # 创建输出目录（带序号和时间戳）
+    now = datetime.now().strftime("%Y%m%d-%H-%M-%S")
+    record_path = Path(record_folder)
+    if input_name:
+        record_path = record_path / input_name
+
+    if record_path.exists():
+        existing = [d.name for d in record_path.iterdir() if d.is_dir()]
+        max_idx = 0
+        for name in existing:
+            parts = name.split("-", 1)
+            if parts[0].isdigit():
+                max_idx = max(max_idx, int(parts[0]))
+        idx = max_idx + 1
+    else:
+        idx = 1
+
+    record_path = record_path / f"{idx}-{now}"
+    record_path.mkdir(parents=True, exist_ok=True)
+
+    # 终端输出
+    console = Console()
+    console.print(overview_table)
+    console.print(Markdown(md))
+
+    # 保存文件
+    with open(os.path.join(record_path, "metrics.md"), "w", encoding="utf-8") as f:
+        f.write(md)
+
+    bad_case_df = df[df[pred_col] != df[label_col]]
+
+    # 保存 JSONL
+    df.to_json(
+        os.path.join(record_path, "result.jsonl"),
+        orient="records",
+        lines=True,
+        force_ascii=False,
+    )
+    bad_case_df.to_json(
+        os.path.join(record_path, "bad_case.jsonl"),
+        orient="records",
+        lines=True,
+        force_ascii=False,
+    )
+
+    # 尝试保存 CSV
+    try:
+        df.to_csv(os.path.join(record_path, "result.csv"), index=False)
+        bad_case_df.to_csv(os.path.join(record_path, "bad_case.csv"), index=False)
+    except Exception:
+        pass
+
+    console.print(f"\n[green]报告已保存到: {record_path}[/green]")
+    console.print(f"[dim]  - metrics.md ({len(df)} 条数据, {len(bad_case_df)} 条错误)[/dim]")
+
+    return record_path

dtflow/schema.py

@@ -26,35 +26,10 @@ Schema 验证模块
     results = dt.validate_schema(schema)
 """
 
-from dataclasses import dataclass
-from dataclasses import field as dataclass_field
-from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
-
-from .utils.field_path import _parse_path, get_field
-
-
-def _validate_item_wrapper(args: tuple) -> Tuple[int, bool, list]:
-    """
-    验证单条数据（用于多进程）。
-
-    Args:
-        args: (index, item, schema_fields) 元组
-
-    Returns:
-        (index, is_valid, errors_as_dicts) - 返回字典列表而非对象（pickle 兼容）
-    """
-    idx, item, fields = args
-    # 在子进程中重建 Schema
-    schema = Schema(fields)
-    result = schema.validate(item)
-
-    if result.valid:
-        return (idx, True, [])
-    else:
-        # 将错误转换为字典（pickle 兼容）
-        errors = [{"path": e.path, "message": e.message, "value": e.value} for e in result.errors]
-        return (idx, False, errors)
+from dataclasses import dataclass, field as dataclass_field
+from typing import Any, Callable, Dict, List, Literal, Optional, Set, Union
 
+from .utils.field_path import get_field, _parse_path, _get_value_by_segments
 
 # 支持的类型
 FieldType = Literal["str", "int", "float", "bool", "list", "dict", "any"]
@@ -187,7 +162,9 @@ class Field:
 
         # 选项检查
         if self.choices is not None and value not in self.choices:
-            errors.append(ValidationError(path, f"值必须是 {self.choices} 之一", value))
+            errors.append(
+                ValidationError(path, f"值必须是 {self.choices} 之一", value)
+            )
 
         # 正则表达式检查
         if self.pattern is not None and isinstance(value, str):
@@ -347,7 +324,9 @@ class Schema:
 
         return errors
 
-    def validate_batch(self, data: List[dict], max_errors: int = 100) -> List[tuple]:
+    def validate_batch(
+        self, data: List[dict], max_errors: int = 100
+    ) -> List[tuple]:
         """
         批量验证数据
 
@@ -371,76 +350,9 @@ class Schema:
 
         return failed
 
-    def validate_parallel(
-        self,
-        data: List[dict],
-        workers: Optional[int] = None,
-        progress_callback: Optional[Callable[[int, int], None]] = None,
-    ) -> tuple:
-        """
-        并行验证数据列表。
-
-        Args:
-            data: 数据列表
-            workers: 进程数，None 自动检测，1 禁用并行
-            progress_callback: 进度回调函数
-
-        Returns:
-            (valid_data, invalid_indices_results) 元组
-            - valid_data: 有效数据列表
-            - invalid_indices_results: [(index, ValidationResult), ...] 无效数据
-        """
-        if not data:
-            return [], []
-
-        total = len(data)
-        use_parallel = workers != 1 and total >= 1000
-
-        valid_data = []
-        invalid_results = []
-
-        if use_parallel:
-            from .parallel import get_optimal_workers, parallel_imap
-
-            actual_workers = get_optimal_workers(total, workers)
-            # 准备参数：(index, item, schema_fields)
-            args_list = [(i, item, self._fields) for i, item in enumerate(data)]
-
-            for i, (idx, is_valid, result_data) in enumerate(
-                parallel_imap(
-                    _validate_item_wrapper,
-                    args_list,
-                    workers=actual_workers,
-                    threshold=1000,
-                )
-            ):
-                if is_valid:
-                    valid_data.append(data[idx])
-                else:
-                    # 重建 ValidationResult（因为不能直接 pickle）
-                    errors = [
-                        ValidationError(path=e["path"], message=e["message"], value=e.get("value"))
-                        for e in result_data
-                    ]
-                    invalid_results.append((idx, ValidationResult(valid=False, errors=errors)))
-                if progress_callback:
-                    progress_callback(i + 1, total)
-        else:
-            # 串行处理
-            for i, item in enumerate(data):
-                result = self.validate(item)
-                if result.valid:
-                    valid_data.append(item)
-                else:
-                    invalid_results.append((i, result))
-                if progress_callback:
-                    progress_callback(i + 1, total)
-
-        return valid_data, invalid_results
-
     def __repr__(self) -> str:
         field_strs = [f"  {path}: {field_def}" for path, field_def in self._fields.items()]
-        return "Schema({\n" + ",\n".join(field_strs) + "\n}})"
+        return f"Schema({{\n" + ",\n".join(field_strs) + "\n}})"
 
 
 # ============================================================================
@@ -549,7 +461,9 @@ def sharegpt_schema(
     """
     return Schema(
         {
-            "conversations": Field(type="list", required=True, min_length=min_conversations),
+            "conversations": Field(
+                type="list", required=True, min_length=min_conversations
+            ),
             "conversations[*].from": Field(
                 type="str", required=True, choices=[human_role, gpt_role]
             ),