fusionkit-evals 0.1.1__tar.gz

fusionkit_evals-0.1.1/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.3
+Name: fusionkit-evals
+Version: 0.1.1
+Summary: Evaluation and Pareto analysis tools for fusionkit.
+Requires-Dist: fusionkit-core==0.1.1
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: pandas>=2.3.3 ; extra == 'evals'
+Requires-Python: >=3.11
+Provides-Extra: evals

fusionkit_evals-0.1.1/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "fusionkit-evals"
+version = "0.1.1"
+description = "Evaluation and Pareto analysis tools for fusionkit."
+requires-python = ">=3.11"
+dependencies = [
+  "fusionkit-core==0.1.1",
+  "pydantic>=2.12.5",
+]
+
+[project.optional-dependencies]
+evals = [
+  "pandas>=2.3.3",
+]
+
+[build-system]
+requires = ["uv_build>=0.11.21,<0.12.0"]
+build-backend = "uv_build"

fusionkit_evals-0.1.1/src/fusionkit_evals/__init__.py

@@ -0,0 +1,119 @@
+from fusionkit_evals.benchmark import BenchmarkRunner
+from fusionkit_evals.dirty_dozen import (
+    DIRTY_DOZEN_REPOS,
+    DIRTY_DOZEN_ROOT,
+    DIRTY_DOZEN_TASK_COUNT,
+    DirtyDozenRepo,
+    assert_dirty_dozen_manifest,
+    load_dirty_dozen_tasks,
+)
+from fusionkit_evals.fusion_bench import (
+    FUSION_BENCH_DISCLAIMER,
+    CommandHandoffKitExecutor,
+    FusionBenchAggregateMetrics,
+    FusionBenchAttemptRow,
+    FusionBenchFailure,
+    FusionBenchFailureCorrelation,
+    FusionBenchParetoPoint,
+    FusionBenchReport,
+    FusionBenchReproducibilityMetadata,
+    FusionBenchRunner,
+    FusionBenchTask,
+    FusionBenchTaskMetrics,
+    HandoffKitExecutor,
+    HandoffKitExecutorError,
+    HandoffKitExecutorUnavailable,
+    build_fusion_bench_report,
+    format_fusion_bench_html_report,
+    format_fusion_bench_markdown_report,
+    join_handoffkit_records,
+    load_benchmark_tasks,
+    load_fusion_bench_jsonl,
+    parse_handoffkit_records,
+    score_fusion_bench_row,
+    write_fusion_bench_html_report,
+    write_fusion_bench_jsonl,
+    write_fusion_bench_markdown_report,
+    write_fusion_bench_report_jsonl,
+)
+from fusionkit_evals.pareto import ParetoPoint, find_pareto_front, format_pareto_markdown
+from fusionkit_evals.public_smoke import (
+    PUBLIC_SMOKE_DISCLAIMER,
+    PUBLIC_SMOKE_FIXTURE_ROOT,
+    PUBLIC_SMOKE_SUITE_INFO,
+    PUBLIC_SMOKE_SUITES,
+    PublicSmokeSuite,
+    PublicSmokeSuiteInfo,
+    assert_public_smoke_matrix,
+    load_public_smoke_tasks,
+)
+from fusionkit_evals.schema import EvalResult, EvalSample
+from fusionkit_evals.scorers import contains_expected, exact_match
+from fusionkit_evals.tiny import (
+    TinyBenchmarkResult,
+    TinyBenchmarkTask,
+    format_tiny_benchmark_report,
+    load_tiny_tasks,
+    run_tiny_benchmark,
+    write_tiny_benchmark_report,
+    write_tiny_jsonl,
+)
+
+__all__ = [
+    "BenchmarkRunner",
+    "CommandHandoffKitExecutor",
+    "DIRTY_DOZEN_REPOS",
+    "DIRTY_DOZEN_ROOT",
+    "DIRTY_DOZEN_TASK_COUNT",
+    "DirtyDozenRepo",
+    "EvalResult",
+    "EvalSample",
+    "FUSION_BENCH_DISCLAIMER",
+    "FusionBenchAggregateMetrics",
+    "FusionBenchAttemptRow",
+    "FusionBenchFailure",
+    "FusionBenchFailureCorrelation",
+    "FusionBenchParetoPoint",
+    "FusionBenchReport",
+    "FusionBenchReproducibilityMetadata",
+    "FusionBenchRunner",
+    "FusionBenchTask",
+    "FusionBenchTaskMetrics",
+    "HandoffKitExecutor",
+    "HandoffKitExecutorError",
+    "HandoffKitExecutorUnavailable",
+    "PUBLIC_SMOKE_DISCLAIMER",
+    "PUBLIC_SMOKE_FIXTURE_ROOT",
+    "PUBLIC_SMOKE_SUITES",
+    "PUBLIC_SMOKE_SUITE_INFO",
+    "ParetoPoint",
+    "PublicSmokeSuite",
+    "PublicSmokeSuiteInfo",
+    "TinyBenchmarkResult",
+    "TinyBenchmarkTask",
+    "assert_dirty_dozen_manifest",
+    "assert_public_smoke_matrix",
+    "build_fusion_bench_report",
+    "contains_expected",
+    "exact_match",
+    "find_pareto_front",
+    "format_fusion_bench_html_report",
+    "format_fusion_bench_markdown_report",
+    "format_pareto_markdown",
+    "format_tiny_benchmark_report",
+    "join_handoffkit_records",
+    "load_benchmark_tasks",
+    "load_dirty_dozen_tasks",
+    "load_fusion_bench_jsonl",
+    "load_public_smoke_tasks",
+    "load_tiny_tasks",
+    "parse_handoffkit_records",
+    "run_tiny_benchmark",
+    "score_fusion_bench_row",
+    "write_tiny_benchmark_report",
+    "write_fusion_bench_html_report",
+    "write_fusion_bench_jsonl",
+    "write_fusion_bench_markdown_report",
+    "write_fusion_bench_report_jsonl",
+    "write_tiny_jsonl",
+]

fusionkit_evals-0.1.1/src/fusionkit_evals/benchmark.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Callable, Iterable
+from pathlib import Path
+
+from fusionkit_core.config import FusionMode
+from fusionkit_core.fusion import FusionEngine
+from fusionkit_core.types import ChatMessage
+
+from fusionkit_evals.schema import EvalResult, EvalSample
+from fusionkit_evals.scorers import contains_expected
+
+Scorer = Callable[[str, str | None], float | None]
+
+
+class BenchmarkRunner:
+    def __init__(self, engine: FusionEngine, scorer: Scorer = contains_expected) -> None:
+        self.engine = engine
+        self.scorer = scorer
+
+    async def run_samples(
+        self,
+        samples: Iterable[EvalSample],
+        config_id: str,
+        mode: FusionMode,
+    ) -> list[EvalResult]:
+        results = []
+        for sample in samples:
+            started = time.perf_counter()
+            fusion_result = await self.engine.run(
+                [ChatMessage(role="user", content=sample.prompt)],
+                mode=mode,
+            )
+            latency_s = time.perf_counter() - started
+            results.append(
+                EvalResult(
+                    sample_id=sample.id,
+                    config_id=config_id,
+                    mode=mode,
+                    output=fusion_result.content,
+                    score=self.scorer(fusion_result.content, sample.expected),
+                    latency_s=latency_s,
+                    metadata={
+                        "fusion_mode": fusion_result.mode,
+                        "route": fusion_result.route,
+                    },
+                )
+            )
+        return results
+
+
+def load_jsonl_samples(path: str | Path) -> list[EvalSample]:
+    samples = []
+    with Path(path).open(encoding="utf-8") as handle:
+        for line in handle:
+            if line.strip():
+                samples.append(EvalSample.model_validate_json(line))
+    return samples
+
+
+def write_jsonl_results(path: str | Path, results: Iterable[EvalResult]) -> None:
+    output_path = Path(path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with output_path.open("w", encoding="utf-8") as handle:
+        for result in results:
+            handle.write(json.dumps(result.model_dump(mode="json")) + "\n")

fusionkit_evals-0.1.1/src/fusionkit_evals/dirty_dozen.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Literal
+
+from fusionkit_evals.fusion_bench import FusionBenchTask, load_benchmark_tasks
+
+DIRTY_DOZEN_ROOT = Path(__file__).resolve().parents[2] / "benchmarks" / "dirty-dozen"
+DIRTY_DOZEN_TASK_COUNT = 12
+DirtyDozenRepo = Literal["fusionkit", "handoffkit", "cursorkit", "mlx-lm"]
+DIRTY_DOZEN_REPOS: tuple[DirtyDozenRepo, ...] = (
+    "fusionkit",
+    "handoffkit",
+    "cursorkit",
+    "mlx-lm",
+)
+
+
+def load_dirty_dozen_tasks(
+    root: str | Path = DIRTY_DOZEN_ROOT,
+    repos: Iterable[DirtyDozenRepo] | None = None,
+) -> list[FusionBenchTask]:
+    tasks = load_benchmark_tasks(root)
+    if repos is None:
+        return tasks
+    selected = set(repos)
+    unknown = selected - set(DIRTY_DOZEN_REPOS)
+    if unknown:
+        raise ValueError(f"Unknown dirty-dozen repos: {sorted(unknown)}")
+    return [task for task in tasks if task.record.source_repo in selected]
+
+
+def assert_dirty_dozen_manifest(tasks: Iterable[FusionBenchTask]) -> None:
+    task_list = list(tasks)
+    if len(task_list) != DIRTY_DOZEN_TASK_COUNT:
+        raise ValueError(f"Expected 12 dirty-dozen tasks, got {len(task_list)}")
+    task_ids = [task.record.task_id for task in task_list]
+    duplicate_ids = sorted(
+        task_id for task_id, count in Counter(task_ids).items() if count > 1
+    )
+    if duplicate_ids:
+        raise ValueError(f"Dirty-dozen task IDs must be unique: {duplicate_ids}")
+    repo_counts = Counter(task.record.source_repo for task in task_list)
+    missing_repos = [repo for repo in DIRTY_DOZEN_REPOS if repo_counts[repo] == 0]
+    underrepresented = {
+        repo: count
+        for repo, count in sorted(repo_counts.items())
+        if repo in DIRTY_DOZEN_REPOS and count < 2
+    }
+    unexpected_repos = sorted(set(repo_counts) - set(DIRTY_DOZEN_REPOS))
+    if missing_repos:
+        raise ValueError(f"Missing dirty-dozen source repos: {missing_repos}")
+    if underrepresented:
+        raise ValueError(f"Expected at least two tasks per repo, got {underrepresented}")
+    if unexpected_repos:
+        raise ValueError(f"Unexpected dirty-dozen source repos: {unexpected_repos}")
+    task_kinds = {task.record.task_kind for task in task_list}
+    if not {"model_fusion", "harness_coding"}.issubset(task_kinds):
+        raise ValueError(f"Dirty-dozen must include both task kinds, got {task_kinds}")
+    for task in task_list:
+        _assert_task_policy(task)
+
+
+def _assert_task_policy(task: FusionBenchTask) -> None:
+    record = task.record
+    if record.source_repo != task.category:
+        raise ValueError(
+            f"Dirty-dozen category {task.category!r} must match source repo "
+            f"{record.source_repo!r} for {record.task_id}"
+        )
+    if not record.expected_evidence:
+        raise ValueError(f"Dirty-dozen task must include expected evidence: {record.task_id}")
+    if not record.contamination_notes:
+        raise ValueError(f"Dirty-dozen task must include contamination notes: {record.task_id}")
+    if "solution" not in record.contamination_notes:
+        raise ValueError(
+            f"Dirty-dozen contamination notes must state no solution is included: "
+            f"{record.task_id}"
+        )
+    if not record.prompt_hash.startswith("sha256:"):
+        raise ValueError(f"Dirty-dozen task must include prompt hash: {record.task_id}")
+    if not record.setup_hash.startswith("sha256:"):
+        raise ValueError(f"Dirty-dozen task must include setup hash: {record.task_id}")
+    if not record.allowed_tools:
+        raise ValueError(f"Dirty-dozen task must include allowed tools: {record.task_id}")
+    params = record.scorer.params or {}
+    if params.get("dirty_dozen") is not True:
+        raise ValueError(f"Dirty-dozen scorer params must set dirty_dozen: {record.task_id}")
+
+
+__all__ = [
+    "DIRTY_DOZEN_REPOS",
+    "DIRTY_DOZEN_ROOT",
+    "DIRTY_DOZEN_TASK_COUNT",
+    "DirtyDozenRepo",
+    "assert_dirty_dozen_manifest",
+    "load_dirty_dozen_tasks",
+]