diffbio 0.1.0__py3-none-any.whl

diffbio/evaluation/problem.py

@@ -0,0 +1,157 @@
+"""Benchmark problem definition and JSON loading.
+
+Defines the ``BenchmarkProblem`` dataclass representing a single evaluation
+problem from scBench or SpatialBench, and provides utilities for loading
+problem sets from JSON files.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class _TaskSpec:
+    """Normalized task specification for a benchmark problem."""
+
+    type: str
+    config: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class _GraderSpec:
+    """Normalized grader specification for a benchmark problem."""
+
+    type: str
+    config: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True, init=False)
+class BenchmarkProblem:
+    """A single benchmark evaluation problem.
+
+    Public properties:
+        problem_id: Unique identifier for the problem.
+        task_type: Category of the task (e.g., ``"qc_filtering"``,
+            ``"clustering"``, ``"differential_expression"``).
+        grader_type: Which grading algorithm to use (e.g.,
+            ``"numeric_tolerance"``, ``"multiple_choice"``).
+        expected_answer: The ground-truth answer in the format expected
+            by the corresponding grader.
+        grader_config: Extra parameters for the grader (e.g.,
+            ``{"tolerance": 0.05, "mode": "relative"}``).
+        task_config: Parameters for the DiffBio operator invocation
+            (e.g., ``{"n_clusters": 5, "temperature": 1.0}``).
+        data_path: Optional path to the h5ad or other data file.
+        description: Human-readable description of the problem.
+        source: Origin benchmark suite (``"scbench"`` or ``"spatialbench"``).
+    """
+
+    problem_id: str
+    expected_answer: Any
+    data_path: str | None
+    description: str
+    source: str
+    _task: _TaskSpec = field(init=False, repr=False)
+    _grader: _GraderSpec = field(init=False, repr=False)
+
+    def __init__(
+        self,
+        *,
+        problem_id: str,
+        task_type: str,
+        grader_type: str,
+        expected_answer: Any,
+        grader_config: dict[str, Any] | None = None,
+        task_config: dict[str, Any] | None = None,
+        data_path: str | None = None,
+        description: str = "",
+        source: str = "",
+    ) -> None:
+        """Normalize task and grader state into grouped specs."""
+        object.__setattr__(self, "problem_id", problem_id)
+        object.__setattr__(self, "expected_answer", expected_answer)
+        object.__setattr__(self, "data_path", data_path)
+        object.__setattr__(self, "description", description)
+        object.__setattr__(self, "source", source)
+        object.__setattr__(
+            self,
+            "_task",
+            _TaskSpec(type=task_type, config=dict(task_config or {})),
+        )
+        object.__setattr__(
+            self,
+            "_grader",
+            _GraderSpec(type=grader_type, config=dict(grader_config or {})),
+        )
+
+    @property
+    def task_type(self) -> str:
+        """Return the task category."""
+        return self._task.type
+
+    @property
+    def task_config(self) -> dict[str, Any]:
+        """Return task-specific operator configuration."""
+        return self._task.config
+
+    @property
+    def grader_type(self) -> str:
+        """Return the grader category."""
+        return self._grader.type
+
+    @property
+    def grader_config(self) -> dict[str, Any]:
+        """Return grader-specific configuration."""
+        return self._grader.config
+
+
+def load_problems(path: Path | str) -> list[BenchmarkProblem]:
+    """Load benchmark problems from a JSON file.
+
+    The JSON file should contain a list of objects, each with fields matching
+    ``BenchmarkProblem`` attributes.
+
+    Args:
+        path: Path to the JSON file.
+
+    Returns:
+        List of parsed BenchmarkProblem instances.
+
+    Raises:
+        FileNotFoundError: If the file does not exist.
+        json.JSONDecodeError: If the file is not valid JSON.
+        KeyError: If a required field is missing from a problem entry.
+    """
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"Benchmark problems file not found: {path}")
+
+    raw = json.loads(path.read_text(encoding="utf-8"))
+
+    if isinstance(raw, dict) and "problems" in raw:
+        raw = raw["problems"]
+
+    problems: list[BenchmarkProblem] = []
+    for entry in raw:
+        problem = BenchmarkProblem(
+            problem_id=entry["problem_id"],
+            task_type=entry["task_type"],
+            grader_type=entry["grader_type"],
+            expected_answer=entry["expected_answer"],
+            grader_config=entry.get("grader_config", {}),
+            task_config=entry.get("task_config", {}),
+            data_path=entry.get("data_path"),
+            description=entry.get("description", ""),
+            source=entry.get("source", ""),
+        )
+        problems.append(problem)
+
+    logger.info("Loaded %d benchmark problems from %s", len(problems), path)
+    return problems

diffbio/evaluation/runner.py

@@ -0,0 +1,277 @@
+"""Evaluation runner for executing benchmark problems and collecting results.
+
+Provides ``run_problem`` for single-problem execution, ``run_benchmark``
+for batch execution, and ``summarize`` for aggregating results.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Any, cast
+
+from diffbio.evaluation.adapters import TaskAdapter
+from diffbio.evaluation.graders import (
+    GradeResult,
+    grade_distribution_comparison,
+    grade_label_set_jaccard,
+    grade_marker_gene_precision_recall,
+    grade_multiple_choice,
+    grade_numeric_tolerance,
+)
+from diffbio.evaluation.problem import BenchmarkProblem
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class EvalResult:
+    """Result of evaluating a single benchmark problem.
+
+    Attributes:
+        problem_id: Identifier of the evaluated problem.
+        task_type: Task category of the problem.
+        grade: The grading result from the appropriate grader.
+        predicted_answer: The raw answer produced by the adapter.
+        quality_metrics: Calibrax quality metrics on operator output
+            (e.g., silhouette score for clustering, MMD for batch correction).
+        elapsed_seconds: Wall-clock time for the solve step.
+        error: Error message if the solve step failed, else empty string.
+    """
+
+    problem_id: str
+    task_type: str
+    grade: GradeResult
+    predicted_answer: Any = None
+    quality_metrics: dict[str, float] = field(default_factory=dict)
+    elapsed_seconds: float = 0.0
+    error: str = ""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class BenchmarkSummary:
+    """Aggregated summary of a benchmark run.
+
+    Attributes:
+        total: Total number of problems evaluated.
+        passed: Number of problems that passed.
+        failed: Number of problems that failed.
+        errored: Number of problems that encountered errors.
+        pass_rate: Fraction of problems that passed (0.0 to 1.0).
+        mean_score: Mean score across all evaluated problems.
+        by_task_type: Pass rate broken down by task_type.
+    """
+
+    total: int
+    passed: int
+    failed: int
+    errored: int
+    pass_rate: float
+    mean_score: float
+    by_task_type: dict[str, float] = field(default_factory=dict)
+
+
+def _grade_answer(
+    predicted: Any,
+    problem: BenchmarkProblem,
+) -> GradeResult:
+    """Apply the appropriate grader to a predicted answer.
+
+    Args:
+        predicted: The answer produced by the TaskAdapter.
+        problem: The benchmark problem (contains expected answer and grader config).
+
+    Returns:
+        GradeResult from the matching grader algorithm.
+
+    Raises:
+        ValueError: If grader_type is not recognised.
+    """
+    grader_type = problem.grader_type
+    expected = problem.expected_answer
+    config = problem.grader_config
+
+    if grader_type == "numeric_tolerance":
+        return grade_numeric_tolerance(
+            float(predicted),
+            float(expected),
+            tolerance=config.get("tolerance", 0.1),
+            mode=config.get("mode", "absolute"),
+        )
+
+    if grader_type == "multiple_choice":
+        return grade_multiple_choice(str(predicted), str(expected))
+
+    if grader_type == "marker_gene_precision_recall":
+        return grade_marker_gene_precision_recall(
+            list(predicted),
+            list(expected),
+            k=config.get("k"),
+        )
+
+    if grader_type == "distribution_comparison":
+        return grade_distribution_comparison(
+            dict(predicted),
+            dict(expected),
+            tolerance=config.get("tolerance", 0.1),
+        )
+
+    if grader_type == "label_set_jaccard":
+        return grade_label_set_jaccard(
+            set(predicted) if not isinstance(predicted, set) else predicted,
+            set(expected) if not isinstance(expected, set) else expected,
+            threshold=config.get("threshold", 0.5),
+        )
+
+    raise ValueError(
+        f"Unknown grader_type {grader_type!r}. Supported: "
+        "numeric_tolerance, multiple_choice, marker_gene_precision_recall, "
+        "distribution_comparison, label_set_jaccard"
+    )
+
+
+def run_problem(
+    problem: BenchmarkProblem,
+    data_dict: dict[str, Any],
+    *,
+    adapter: TaskAdapter | None = None,
+    collect_metrics: bool = True,
+) -> EvalResult:
+    """Run a single benchmark problem end-to-end.
+
+    Args:
+        problem: The benchmark problem to evaluate.
+        data_dict: Operator-ready input data.
+        adapter: Optional TaskAdapter instance. Created with defaults if None.
+        collect_metrics: Whether to compute calibrax quality metrics.
+
+    Returns:
+        EvalResult with grading outcome and optional quality metrics.
+    """
+    if adapter is None:
+        adapter = TaskAdapter()
+
+    quality_metrics: dict[str, float] = {}
+    try:
+        t0 = time.monotonic()
+        if collect_metrics:
+            predicted, quality_metrics = adapter.solve_with_metrics(problem, data_dict)
+        else:
+            predicted = adapter.solve(problem, data_dict)
+        elapsed = time.monotonic() - t0
+    except Exception as exc:
+        logger.warning("Problem %s failed: %s", problem.problem_id, exc)
+        return EvalResult(
+            problem_id=problem.problem_id,
+            task_type=problem.task_type,
+            grade=GradeResult(passed=False, score=0.0, detail=f"solve error: {exc}"),
+            error=str(exc),
+        )
+
+    try:
+        grade = _grade_answer(predicted, problem)
+    except Exception as exc:
+        logger.warning("Grading %s failed: %s", problem.problem_id, exc)
+        return EvalResult(
+            problem_id=problem.problem_id,
+            task_type=problem.task_type,
+            grade=GradeResult(passed=False, score=0.0, detail=f"grading error: {exc}"),
+            predicted_answer=predicted,
+            quality_metrics=quality_metrics,
+            elapsed_seconds=elapsed,
+            error=str(exc),
+        )
+
+    return EvalResult(
+        problem_id=problem.problem_id,
+        task_type=problem.task_type,
+        grade=grade,
+        predicted_answer=predicted,
+        quality_metrics=quality_metrics,
+        elapsed_seconds=elapsed,
+    )
+
+
+def run_benchmark(
+    problems: list[BenchmarkProblem],
+    data_loader: Any,
+    *,
+    adapter: TaskAdapter | None = None,
+) -> list[EvalResult]:
+    """Run a batch of benchmark problems.
+
+    Args:
+        problems: List of problems to evaluate.
+        data_loader: Callable ``(problem) -> dict`` that provides operator-ready
+            data for each problem. Can also be a dict mapping problem_id to data.
+        adapter: Optional shared TaskAdapter.
+
+    Returns:
+        List of EvalResult, one per problem.
+    """
+    if adapter is None:
+        adapter = TaskAdapter()
+
+    results: list[EvalResult] = []
+    for problem in problems:
+        data_dict: dict[str, Any]
+        if callable(data_loader):
+            data_dict = data_loader(problem)
+        else:
+            data_dict = cast(dict[str, Any], data_loader[problem.problem_id])
+        result = run_problem(problem, data_dict, adapter=adapter)
+        results.append(result)
+        logger.info(
+            "Problem %s: %s (score=%.3f, %.2fs)",
+            problem.problem_id,
+            "PASS" if result.grade.passed else "FAIL",
+            result.grade.score,
+            result.elapsed_seconds,
+        )
+
+    return results
+
+
+def summarize(results: list[EvalResult]) -> BenchmarkSummary:
+    """Aggregate evaluation results into a summary.
+
+    Args:
+        results: List of EvalResult from a benchmark run.
+
+    Returns:
+        BenchmarkSummary with pass rates and score statistics.
+    """
+    if not results:
+        return BenchmarkSummary(
+            total=0,
+            passed=0,
+            failed=0,
+            errored=0,
+            pass_rate=0.0,
+            mean_score=0.0,
+        )
+
+    passed = sum(1 for r in results if r.grade.passed)
+    errored = sum(1 for r in results if r.error)
+    failed = len(results) - passed - errored
+
+    scores = [r.grade.score for r in results]
+    mean_score = sum(scores) / len(scores) if scores else 0.0
+
+    # Per-task-type pass rate
+    by_task: dict[str, list[bool]] = {}
+    for r in results:
+        by_task.setdefault(r.task_type, []).append(r.grade.passed)
+
+    by_task_rate = {task: sum(passes) / len(passes) for task, passes in by_task.items()}
+
+    return BenchmarkSummary(
+        total=len(results),
+        passed=passed,
+        failed=failed,
+        errored=errored,
+        pass_rate=passed / len(results),
+        mean_score=mean_score,
+        by_task_type=by_task_rate,
+    )

diffbio/losses/__init__.py

@@ -0,0 +1,59 @@
+"""Loss functions and regularization for bioinformatics pipelines.
+
+This module provides loss functions for training differentiable bioinformatics
+pipelines, including biological regularization to prevent adversarial optimization.
+"""
+
+from diffbio.losses.alignment_losses import (
+    AlignmentConsistencyLoss,
+    AlignmentScoreLoss,
+    SoftEditDistanceLoss,
+)
+from diffbio.losses.biological_regularization import (
+    BiologicalPlausibilityLoss,
+    BiologicalRegularizationConfig,
+    GapPatternRegularization,
+    GCContentRegularization,
+    SequenceComplexityLoss,
+)
+from diffbio.losses.singlecell_losses import (
+    BatchMixingLoss,
+    ClusteringCompactnessLoss,
+    ShannonDiversityLoss,
+    SimpsonDiversityLoss,
+    VelocityConsistencyLoss,
+)
+from diffbio.losses.metric_losses import DifferentiableAUROC, ExactAUROC
+from diffbio.losses.statistical_losses import (
+    HMMLikelihoodLoss,
+    NegativeBinomialLoss,
+    VAELoss,
+    zinb_negative_log_likelihood,
+)
+
+__all__ = [
+    # Alignment losses
+    "AlignmentConsistencyLoss",
+    "AlignmentScoreLoss",
+    "SoftEditDistanceLoss",
+    # Biological regularization
+    "BiologicalPlausibilityLoss",
+    "BiologicalRegularizationConfig",
+    "GapPatternRegularization",
+    "GCContentRegularization",
+    "SequenceComplexityLoss",
+    # Metric losses
+    "DifferentiableAUROC",
+    "ExactAUROC",
+    # Single-cell losses
+    "BatchMixingLoss",
+    "ClusteringCompactnessLoss",
+    "ShannonDiversityLoss",
+    "SimpsonDiversityLoss",
+    "VelocityConsistencyLoss",
+    # Statistical losses
+    "HMMLikelihoodLoss",
+    "NegativeBinomialLoss",
+    "VAELoss",
+    "zinb_negative_log_likelihood",
+]