algobench-sdk 1.0.0__tar.gz

algobench_sdk-1.0.0/PKG-INFO

@@ -0,0 +1,114 @@
+Metadata-Version: 2.4
+Name: algobench-sdk
+Version: 1.0.0
+Summary: AlgoBench - 算法基准测试分析工具的 Python SDK 与 CLI
+Author: DuanxiangLiu
+License-Expression: AGPL-3.0-only
+Project-URL: Homepage, https://github.com/DuanxiangLiu/AlgoBench
+Project-URL: Documentation, https://duanxiangliu.github.io/AlgoBench-pages/
+Project-URL: Repository, https://github.com/DuanxiangLiu/AlgoBench
+Keywords: benchmark,statistics,analysis,decision,eda
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=1.24
+Requires-Dist: scipy>=1.10
+Requires-Dist: pandas>=2.0
+
+# AlgoBench Python SDK
+
+AlgoBench 是一个算法基准测试分析工具，提供完整的统计分析和决策评估流水线。
+
+## 安装
+
+```bash
+pip install algobench
+```
+
+## 快速开始
+
+### Python 库
+
+```python
+from algobench import analyze
+
+result = analyze("data.csv", baseline="V1", compare="V2", metrics=["HPWL", "runtime"])
+
+print(result.decision.label)        # "建议上线"
+print(result.decision.status)       # "yes"
+print(result.statistics["HPWL"].mean_imp)  # 5.07
+print(result.statistics["HPWL"].p_value)   # 6.13e-13
+```
+
+### 命令行
+
+```bash
+# 完整分析
+algobench analyze data.csv -b V1 -c V2 -m HPWL,runtime
+
+# 单指标统计
+algobench stats data.csv -b V1 -c V2 -m HPWL
+
+# 数据质量检查
+algobench quality data.csv
+
+# 数据诊断
+algobench diagnose data.csv
+
+# CSV 格式验证
+algobench validate data.csv
+```
+
+## CSV 数据格式
+
+```csv
+Case,Baseline/HPWL,New/HPWL,Baseline/runtime,New/runtime
+case1,1200,1100,50,45
+case2,800,750,30,28
+```
+
+- 用例列：`Case`、`Benchmark`、`Test`、`Instance` 等
+- 指标列格式：`算法名/指标名`（使用 `/` 分隔符）
+- 元数据列：`#` 前缀（如 `#Size`）
+- 参数列：`p_` 前缀（如 `p_mode`）
+
+## 分析标准
+
+| 模式 | 说明 |
+|------|------|
+| `exploratory` | 探索模式，识别更多潜在改进 |
+| `standard` | 标准模式，平衡灵敏度和可靠性 |
+| `strict` | 严格模式，用于发布或关键决策 |
+
+```python
+result = analyze("data.csv", baseline="V1", compare="V2", criteria_id="strict")
+```
+
+## 决策状态
+
+| 状态 | 含义 |
+|------|------|
+| `yes` | 建议上线 |
+| `watch` | 观察期，需要更多数据 |
+| `no` | 不建议上线 |
+| `insufficient` | 数据不足 |
+
+## 依赖
+
+- Python >= 3.10
+- numpy >= 1.24
+- scipy >= 1.10
+- pandas >= 2.0
+
+## 许可证
+
+AGPL-3.0-only

algobench_sdk-1.0.0/README.md

@@ -0,0 +1,88 @@
+# AlgoBench Python SDK
+
+AlgoBench 是一个算法基准测试分析工具，提供完整的统计分析和决策评估流水线。
+
+## 安装
+
+```bash
+pip install algobench
+```
+
+## 快速开始
+
+### Python 库
+
+```python
+from algobench import analyze
+
+result = analyze("data.csv", baseline="V1", compare="V2", metrics=["HPWL", "runtime"])
+
+print(result.decision.label)        # "建议上线"
+print(result.decision.status)       # "yes"
+print(result.statistics["HPWL"].mean_imp)  # 5.07
+print(result.statistics["HPWL"].p_value)   # 6.13e-13
+```
+
+### 命令行
+
+```bash
+# 完整分析
+algobench analyze data.csv -b V1 -c V2 -m HPWL,runtime
+
+# 单指标统计
+algobench stats data.csv -b V1 -c V2 -m HPWL
+
+# 数据质量检查
+algobench quality data.csv
+
+# 数据诊断
+algobench diagnose data.csv
+
+# CSV 格式验证
+algobench validate data.csv
+```
+
+## CSV 数据格式
+
+```csv
+Case,Baseline/HPWL,New/HPWL,Baseline/runtime,New/runtime
+case1,1200,1100,50,45
+case2,800,750,30,28
+```
+
+- 用例列：`Case`、`Benchmark`、`Test`、`Instance` 等
+- 指标列格式：`算法名/指标名`（使用 `/` 分隔符）
+- 元数据列：`#` 前缀（如 `#Size`）
+- 参数列：`p_` 前缀（如 `p_mode`）
+
+## 分析标准
+
+| 模式 | 说明 |
+|------|------|
+| `exploratory` | 探索模式，识别更多潜在改进 |
+| `standard` | 标准模式，平衡灵敏度和可靠性 |
+| `strict` | 严格模式，用于发布或关键决策 |
+
+```python
+result = analyze("data.csv", baseline="V1", compare="V2", criteria_id="strict")
+```
+
+## 决策状态
+
+| 状态 | 含义 |
+|------|------|
+| `yes` | 建议上线 |
+| `watch` | 观察期，需要更多数据 |
+| `no` | 不建议上线 |
+| `insufficient` | 数据不足 |
+
+## 依赖
+
+- Python >= 3.10
+- numpy >= 1.24
+- scipy >= 1.10
+- pandas >= 2.0
+
+## 许可证
+
+AGPL-3.0-only

algobench_sdk-1.0.0/algobench/__init__.py

@@ -0,0 +1,25 @@
+"""AlgoBench - 算法基准测试分析工具的 Python SDK 与 CLI。"""
+
+from __future__ import annotations
+
+__version__ = "1.0.0"
+
+from algobench.api import analyze, compute_statistics, make_decision
+from algobench.models import (
+    AnalysisResult,
+    Decision,
+    EvaluationSummary,
+    MetricEvaluation,
+    StatisticsResult,
+)
+
+__all__ = [
+    "analyze",
+    "compute_statistics",
+    "make_decision",
+    "AnalysisResult",
+    "Decision",
+    "EvaluationSummary",
+    "MetricEvaluation",
+    "StatisticsResult",
+]

algobench_sdk-1.0.0/algobench/api.py

@@ -0,0 +1,123 @@
+"""AlgoBench 顶层 API，对应前端的一站式分析流程。"""
+
+from __future__ import annotations
+
+from algobench.models import AnalysisResult, StatisticsResult, EvaluationSummary, Decision, MetricEvaluation
+from algobench.parsers.csv_parser import parse_csv, parse_csv_file
+from algobench.config.metrics import get_metric_config, get_better_direction
+from algobench.stats.core import compute_statistics
+from algobench.stats.diagnosis import check_data_quality
+from algobench.decision.engine import build_evaluation_summary
+
+
+def analyze(
+    input_source: str,
+    baseline: str = "Baseline",
+    compare: str = "New",
+    metrics: list[str] | None = None,
+    selected_cases: list[str] | None = None,
+    criteria_id: str = "standard",
+    custom_thresholds: dict | None = None,
+    metric_configs: dict | None = None,
+) -> AnalysisResult:
+    """一站式分析：解析 → 统计 → 决策。
+
+    Args:
+        input_source: CSV 文件路径或 CSV 字符串
+        baseline: 基线算法名
+        compare: 对比算法名
+        metrics: 要分析的指标列表，None 表示全部
+        selected_cases: 要分析的用例列表，None 表示全部
+        criteria_id: 分析标准预设 ID
+        custom_thresholds: 自定义阈值
+        metric_configs: 用户指标配置
+
+    Returns:
+        AnalysisResult 包含完整分析结果
+    """
+    # 1. 解析数据
+    if input_source.endswith(".csv"):
+        parsed = parse_csv_file(input_source, selected_cases)
+    else:
+        parsed = parse_csv(input_source, selected_cases)
+
+    data = parsed["data"]
+    algos = parsed["algos"]
+    available_metrics = parsed["metrics"]
+
+    if not data or len(algos) < 2:
+        return AnalysisResult(parsed=parsed)
+
+    # 确定要分析的指标
+    target_metrics = [m for m in (metrics or available_metrics) if m in available_metrics]
+
+    # 2. 数据质量检查
+    quality = check_data_quality(data, algos, target_metrics)
+
+    # 3. 逐指标统计计算
+    statistics_results: dict[str, StatisticsResult] = {}
+    for metric in target_metrics:
+        direction = get_better_direction(metric, metric_configs)
+        stats = compute_statistics(data, metric, baseline, compare, direction, selected_cases)
+        statistics_results[metric] = stats
+
+    # 4. 构建评估摘要（包含决策）
+    dataset_rows = parsed.get("filteredCases", 0) + len(data)
+    evaluation = build_evaluation_summary(
+        stats_results=statistics_results,
+        metric_configs=metric_configs,
+        criteria_id=criteria_id,
+        custom_thresholds=custom_thresholds,
+        n_selected_cases=len(data),
+        dataset_rows=dataset_rows,
+    )
+
+    # 5. 提取决策
+    decision = Decision(
+        status=evaluation.decision_status,
+        label=evaluation.decision_label,
+        reason=evaluation.decision_reason,
+        blockers=evaluation.blockers,
+        recommendations=evaluation.recommendations,
+    )
+
+    return AnalysisResult(
+        parsed=parsed,
+        statistics=statistics_results,
+        quality=quality,
+        evaluation=evaluation,
+        decision=decision,
+    )
+
+
+def compute_statistics_only(
+    input_source: str,
+    metric: str,
+    baseline: str = "Baseline",
+    compare: str = "New",
+    selected_cases: list[str] | None = None,
+    metric_configs: dict | None = None,
+) -> StatisticsResult:
+    """仅计算单指标统计。"""
+    if input_source.endswith(".csv"):
+        parsed = parse_csv_file(input_source, selected_cases)
+    else:
+        parsed = parse_csv(input_source, selected_cases)
+
+    direction = get_better_direction(metric, metric_configs)
+    return compute_statistics(parsed["data"], metric, baseline, compare, direction, selected_cases)
+
+
+def make_decision(
+    stats_results: dict[str, StatisticsResult],
+    metric_configs: dict | None = None,
+    criteria_id: str = "standard",
+    custom_thresholds: dict | None = None,
+) -> EvaluationSummary:
+    """基于已有统计结果生成决策。"""
+    return build_evaluation_summary(
+        stats_results=stats_results,
+        metric_configs=metric_configs,
+        criteria_id=criteria_id,
+        custom_thresholds=custom_thresholds,
+    )

algobench_sdk-1.0.0/algobench/cli.py

@@ -0,0 +1,181 @@
+"""AlgoBench CLI 入口。"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import asdict
+from pathlib import Path
+
+
+def _serialize(obj):
+    """递归序列化 dataclass 为 dict。"""
+    if hasattr(obj, "__dataclass_fields__"):
+        return {k: _serialize(v) for k, v in asdict(obj).items()}
+    if isinstance(obj, list):
+        return [_serialize(item) for item in obj]
+    if isinstance(obj, dict):
+        return {k: _serialize(v) for k, v in obj.items()}
+    return obj
+
+
+def cmd_analyze(args):
+    from algobench.api import analyze
+    metrics = args.metrics.split(",") if args.metrics else None
+    result = analyze(
+        input_source=args.input,
+        baseline=args.baseline,
+        compare=args.compare,
+        metrics=metrics,
+        criteria_id=args.criteria,
+    )
+    output = {
+        "decision": _serialize(result.decision),
+        "statistics": {k: _serialize(v) for k, v in result.statistics.items()},
+        "quality": result.quality,
+        "evaluation": _serialize(result.evaluation),
+    }
+    _print_output(output, args.format)
+
+
+def cmd_stats(args):
+    from algobench.api import compute_statistics_only
+    result = compute_statistics_only(
+        input_source=args.input,
+        metric=args.metric,
+        baseline=args.baseline,
+        compare=args.compare,
+    )
+    _print_output(_serialize(result), args.format)
+
+
+def cmd_quality(args):
+    from algobench.parsers.csv_parser import parse_csv_file
+    from algobench.stats.diagnosis import check_data_quality
+    parsed = parse_csv_file(args.input)
+    result = check_data_quality(parsed["data"], parsed["algos"], parsed["metrics"])
+    _print_output(result, args.format)
+
+
+def cmd_diagnose(args):
+    from algobench.parsers.csv_parser import parse_csv_file
+    from algobench.stats.diagnosis import diagnose_data_issues
+    parsed = parse_csv_file(args.input)
+    issues = diagnose_data_issues(parsed["data"], parsed["algos"], parsed["metrics"])
+    _print_output(issues, args.format)
+
+
+def cmd_validate(args):
+    from algobench.parsers.csv_parser import parse_csv_file
+    parsed = parse_csv_file(args.input)
+    result = {
+        "valid": bool(parsed["data"] and len(parsed["algos"]) >= 2),
+        "total_cases": len(parsed["data"]),
+        "algos": parsed["algos"],
+        "metrics": parsed["metrics"],
+        "meta_columns": parsed["metaColumns"],
+        "param_columns": parsed["paramColumns"],
+    }
+    _print_output(result, args.format)
+
+
+def _print_output(data, fmt: str = "json"):
+    if fmt == "json":
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+    elif fmt == "table":
+        _print_table(data)
+    else:
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+
+
+def _print_table(data):
+    if isinstance(data, dict) and "decision" in data:
+        d = data["decision"]
+        print(f"决策: {d['label']} ({d['status']})")
+        print(f"原因: {d['reason']}")
+        if d.get("blockers"):
+            print("阻塞项:")
+            for b in d["blockers"]:
+                print(f"  - {b}")
+        if d.get("recommendations"):
+            print("建议:")
+            for r in d["recommendations"]:
+                print(f"  - {r}")
+        if "statistics" in data:
+            print("\n统计结果:")
+            for metric, stats in data["statistics"].items():
+                imp = stats.get("mean_imp")
+                p = stats.get("p_value")
+                n = stats.get("n_valid")
+                print(f"  {metric}: 改进率={imp}%, p值={p}, 样本量={n}")
+    elif isinstance(data, dict) and "score" in data:
+        print(f"数据质量: {data['score']}分 ({data['level']})")
+        if data.get("issues"):
+            for issue in data["issues"]:
+                print(f"  [{issue['severity']}] {issue['message']}")
+    else:
+        print(json.dumps(data, indent=2, ensure_ascii=False))
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="algobench", description="AlgoBench - 算法基准测试分析工具")
+    sub = parser.add_subparsers(dest="command", help="可用命令")
+
+    # analyze
+    p_analyze = sub.add_parser("analyze", help="完整分析流程")
+    p_analyze.add_argument("input", help="CSV 文件路径")
+    p_analyze.add_argument("-b", "--baseline", default="Baseline", help="基线算法名")
+    p_analyze.add_argument("-c", "--compare", default="New", help="对比算法名")
+    p_analyze.add_argument("-m", "--metrics", default=None, help="指标列表（逗号分隔）")
+    p_analyze.add_argument("--criteria", default="standard", choices=["exploratory", "standard", "strict"], help="分析标准")
+    p_analyze.add_argument("-f", "--format", default="table", choices=["json", "table"], help="输出格式")
+
+    # stats
+    p_stats = sub.add_parser("stats", help="单指标统计计算")
+    p_stats.add_argument("input", help="CSV 文件路径")
+    p_stats.add_argument("-b", "--baseline", default="Baseline", help="基线算法名")
+    p_stats.add_argument("-c", "--compare", default="New", help="对比算法名")
+    p_stats.add_argument("-m", "--metric", required=True, help="指标名")
+    p_stats.add_argument("-f", "--format", default="json", choices=["json", "table"], help="输出格式")
+
+    # quality
+    p_quality = sub.add_parser("quality", help="数据质量检查")
+    p_quality.add_argument("input", help="CSV 文件路径")
+    p_quality.add_argument("-f", "--format", default="table", choices=["json", "table"], help="输出格式")
+
+    # diagnose
+    p_diagnose = sub.add_parser("diagnose", help="数据诊断")
+    p_diagnose.add_argument("input", help="CSV 文件路径")
+    p_diagnose.add_argument("-f", "--format", default="json", choices=["json", "table"], help="输出格式")
+
+    # validate
+    p_validate = sub.add_parser("validate", help="CSV 格式验证")
+    p_validate.add_argument("input", help="CSV 文件路径")
+    p_validate.add_argument("-f", "--format", default="json", choices=["json", "table"], help="输出格式")
+
+    args = parser.parse_args()
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    commands = {
+        "analyze": cmd_analyze,
+        "stats": cmd_stats,
+        "quality": cmd_quality,
+        "diagnose": cmd_diagnose,
+        "validate": cmd_validate,
+    }
+
+    try:
+        commands[args.command](args)
+    except FileNotFoundError:
+        print(f"错误: 文件不存在 - {args.input}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"错误: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

algobench_sdk-1.0.0/algobench/config/analysis_criteria.py

@@ -0,0 +1,79 @@
+"""分析标准预设，对应 JS 的 src/config/analysisCriteria.js。"""
+
+from __future__ import annotations
+
+DEFAULT_CRITERIA_ID = "standard"
+
+ANALYSIS_CRITERIA_PRESETS = {
+    "exploratory": {
+        "id": "exploratory",
+        "label": "探索模式",
+        "description": "识别更多潜在的改进机会",
+        "thresholds": {
+            "minMeaningfulImprovement": 2,
+            "degradationWarning": 0.25,
+            "degradationCritical": 0.35,
+            "effectSize": {"NEGLIGIBLE": 0.15, "SMALL": 0.35, "MEDIUM": 0.55},
+            "reliabilityFactors": {
+                "significance": {"high": 0.05, "medium": 0.1, "marginal": 0.15},
+                "sample": {"large": 15, "medium": 8, "small": 5},
+            },
+        },
+    },
+    "standard": {
+        "id": "standard",
+        "label": "标准模式",
+        "description": "获取可靠结论以支持决策",
+        "thresholds": {
+            "minMeaningfulImprovement": 3,
+            "degradationWarning": 0.2,
+            "degradationCritical": 0.3,
+            "effectSize": {"NEGLIGIBLE": 0.2, "SMALL": 0.5, "MEDIUM": 0.8},
+            "reliabilityFactors": {
+                "significance": {"high": 0.01, "medium": 0.05, "marginal": 0.1},
+                "sample": {"large": 30, "medium": 15, "small": 8},
+            },
+        },
+    },
+    "strict": {
+        "id": "strict",
+        "label": "严格模式",
+        "description": "发布结果或制定关键决策",
+        "thresholds": {
+            "minMeaningfulImprovement": 4,
+            "degradationWarning": 0.15,
+            "degradationCritical": 0.25,
+            "effectSize": {"NEGLIGIBLE": 0.25, "SMALL": 0.6, "MEDIUM": 0.9},
+            "reliabilityFactors": {
+                "significance": {"high": 0.005, "medium": 0.01, "marginal": 0.05},
+                "sample": {"large": 40, "medium": 25, "small": 15},
+            },
+        },
+    },
+}
+
+
+def get_criteria_preset(criteria_id: str) -> dict:
+    return ANALYSIS_CRITERIA_PRESETS.get(criteria_id, ANALYSIS_CRITERIA_PRESETS[DEFAULT_CRITERIA_ID])
+
+
+def get_effective_thresholds(
+    criteria_id: str = DEFAULT_CRITERIA_ID,
+    custom_thresholds: dict | None = None,
+) -> dict:
+    """获取生效的阈值配置。本地自定义 > 预设。"""
+    base = get_criteria_preset(criteria_id)["thresholds"]
+    if not custom_thresholds:
+        return base
+
+    merged = {**base, **{k: v for k, v in custom_thresholds.items() if k not in ("effectSize", "reliabilityFactors")}}
+    merged["effectSize"] = {**base.get("effectSize", {}), **(custom_thresholds.get("effectSize") or {})}
+    base_rf = base.get("reliabilityFactors", {})
+    custom_rf = custom_thresholds.get("reliabilityFactors") or {}
+    merged["reliabilityFactors"] = {
+        **base_rf,
+        **{k: v for k, v in custom_rf.items() if k not in ("sample", "significance")},
+        "sample": {**base_rf.get("sample", {}), **(custom_rf.get("sample") or {})},
+        "significance": {**base_rf.get("significance", {}), **(custom_rf.get("significance") or {})},
+    }
+    return merged

algobench_sdk-1.0.0/algobench/config/business.py

@@ -0,0 +1,37 @@
+"""业务规则配置，对应 JS 的 src/config/business.js。"""
+
+METRIC_SEPARATOR = "/"
+PARAM_PREFIX = "p_"
+META_PREFIX = "#"
+
+# 用例列名候选（不区分大小写匹配）
+CASE_COLUMN_NAMES = [
+    "case", "benchmark", "test", "circuit", "design",
+    "instance", "sample", "experiment", "dataset", "workload",
+]
+
+
+def is_case_column(header: str) -> bool:
+    return header.strip().lower() in CASE_COLUMN_NAMES
+
+
+def is_meta_column(header: str) -> bool:
+    return header.startswith(META_PREFIX)
+
+
+def is_param_column(header: str) -> bool:
+    return header.startswith(PARAM_PREFIX)
+
+
+def is_metric_column(header: str) -> bool:
+    return METRIC_SEPARATOR in header
+
+
+def parse_metric_column(header: str) -> tuple[str, str] | None:
+    """解析指标列名，返回 (算法名, 指标名)。"""
+    if not is_metric_column(header):
+        return None
+    parts = header.split(METRIC_SEPARATOR, 1)
+    if len(parts) == 2 and parts[0].strip() and parts[1].strip():
+        return parts[0].strip(), parts[1].strip()
+    return None