evalrun 0.4.0__py3-none-any.whl

framework/profiles/travel.py

@@ -0,0 +1,78 @@
+"""Evaluation profiles for travel planning agents."""
+
+from framework.models import EvaluationProfile
+from framework.evaluation.dimensions import (
+    CONSTRAINT_SATISFACTION,
+    PLANNING_QUALITY,
+    INFORMATION_ACCURACY,
+    PERSONALIZATION,
+    ADAPTABILITY,
+)
+
+TRAVEL_PROFILE = EvaluationProfile(
+    name="travel-agent",
+    weights={
+        CONSTRAINT_SATISFACTION: 25.0,
+        PLANNING_QUALITY: 20.0,
+        INFORMATION_ACCURACY: 20.0,
+        PERSONALIZATION: 20.0,
+        ADAPTABILITY: 15.0,
+    },
+    pass_threshold=75.0,
+)
+
+TRAVEL_ROUTE_OPTIMIZATION_PROFILE = EvaluationProfile(
+    name="travel-route-optimization",
+    weights={
+        CONSTRAINT_SATISFACTION: 25.0,
+        PLANNING_QUALITY: 55.0,
+        INFORMATION_ACCURACY: 10.0,
+        PERSONALIZATION: 5.0,
+        ADAPTABILITY: 5.0,
+    },
+    pass_threshold=70.0,
+)
+
+TRAVEL_REMOTE_WORKER_TIMEZONES_PROFILE = EvaluationProfile(
+    name="travel-remote-worker-timezones",
+    weights={
+        CONSTRAINT_SATISFACTION: 35.0,
+        PLANNING_QUALITY: 20.0,
+        INFORMATION_ACCURACY: 10.0,
+        PERSONALIZATION: 30.0,
+        ADAPTABILITY: 5.0,
+    },
+    pass_threshold=75.0,
+)
+
+TRAVEL_MID_TRIP_REPLANNING_PROFILE = EvaluationProfile(
+    name="travel-mid-trip-replanning",
+    weights={
+        ADAPTABILITY: 55.0,
+        CONSTRAINT_SATISFACTION: 20.0,
+        PLANNING_QUALITY: 15.0,
+        INFORMATION_ACCURACY: 5.0,
+        PERSONALIZATION: 5.0,
+    },
+    pass_threshold=75.0,
+)
+
+TRAVEL_INFORMATION_GATHERING_UNCERTAINTY_PROFILE = EvaluationProfile(
+    name="travel-information-gathering-uncertainty",
+    weights={
+        CONSTRAINT_SATISFACTION: 50.0,
+        INFORMATION_ACCURACY: 25.0,
+        PLANNING_QUALITY: 15.0,
+        PERSONALIZATION: 5.0,
+        ADAPTABILITY: 5.0,
+    },
+    pass_threshold=75.0,
+)
+
+PROFILE_REGISTRY = {
+    TRAVEL_PROFILE.name: TRAVEL_PROFILE,
+    TRAVEL_ROUTE_OPTIMIZATION_PROFILE.name: TRAVEL_ROUTE_OPTIMIZATION_PROFILE,
+    TRAVEL_REMOTE_WORKER_TIMEZONES_PROFILE.name: TRAVEL_REMOTE_WORKER_TIMEZONES_PROFILE,
+    TRAVEL_MID_TRIP_REPLANNING_PROFILE.name: TRAVEL_MID_TRIP_REPLANNING_PROFILE,
+    TRAVEL_INFORMATION_GATHERING_UNCERTAINTY_PROFILE.name: TRAVEL_INFORMATION_GATHERING_UNCERTAINTY_PROFILE,
+}

framework/regression/__init__.py

@@ -0,0 +1,17 @@
+"""Regression analysis and baseline comparison package."""
+
+from framework.regression.loader import load_baseline_manifest
+from framework.regression.comparator import (
+    compare_runs,
+    DimensionDelta,
+    ScenarioComparison,
+    RegressionReport,
+)
+
+__all__ = [
+    "load_baseline_manifest",
+    "compare_runs",
+    "DimensionDelta",
+    "ScenarioComparison",
+    "RegressionReport",
+]

framework/regression/comparator.py

@@ -0,0 +1,273 @@
+"""Baseline comparator engine for score delta calculation and release gate enforcement."""
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+from framework.models import EvaluationResult
+
+
+@dataclass
+class DimensionDelta:
+    """Stores score delta for a single evaluation dimension."""
+
+    dimension: str
+    baseline_score: float
+    candidate_score: float
+    delta: float
+    is_regression: bool
+
+
+@dataclass
+class ScenarioComparison:
+    """Stores regression comparison metrics for a single scenario."""
+
+    scenario_id: str
+    scenario_name: str
+    status: str  # PASSED, FAILED, BLOCKED, REGRESSED, MISSING_IN_CANDIDATE, NEW
+    baseline_score: Optional[float]
+    candidate_score: Optional[float]
+    overall_delta: Optional[float]
+    evaluator_passed: bool
+    auditor_gate: str
+    is_regression: bool
+    dimension_deltas: List[DimensionDelta] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "scenario_id": self.scenario_id,
+            "scenario_name": self.scenario_name,
+            "status": self.status,
+            "baseline_score": round(self.baseline_score, 2) if self.baseline_score is not None else None,
+            "candidate_score": round(self.candidate_score, 2) if self.candidate_score is not None else None,
+            "overall_delta": round(self.overall_delta, 2) if self.overall_delta is not None else None,
+            "evaluator_passed": self.evaluator_passed,
+            "auditor_gate": self.auditor_gate,
+            "is_regression": self.is_regression,
+            "dimension_deltas": [
+                {
+                    "dimension": d.dimension,
+                    "baseline_score": round(d.baseline_score, 2),
+                    "candidate_score": round(d.candidate_score, 2),
+                    "delta": round(d.delta, 2),
+                    "is_regression": d.is_regression,
+                }
+                for d in self.dimension_deltas
+            ],
+        }
+
+
+@dataclass
+class RegressionReport:
+    """Encapsulates full run baseline comparison report."""
+
+    run_id: str
+    baseline_run_id: Optional[str]
+    regression_detected: bool
+    release_blocked: bool
+    max_allowed_overall_regression: float
+    max_allowed_dimension_regression: float
+    scenario_comparisons: List[ScenarioComparison]
+    summary: Dict[str, Any]
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "baseline_run_id": self.baseline_run_id,
+            "regression_detected": self.regression_detected,
+            "release_blocked": self.release_blocked,
+            "max_allowed_overall_regression": self.max_allowed_overall_regression,
+            "max_allowed_dimension_regression": self.max_allowed_dimension_regression,
+            "summary": self.summary,
+            "scenarios": [sc.to_dict() for sc in self.scenario_comparisons],
+        }
+
+
+def compare_runs(
+    candidate_results: List[EvaluationResult],
+    baseline_data: Dict[str, Any],
+    max_overall_drop: float = 5.0,
+    max_dim_drop: float = 10.0,
+    candidate_run_id: str = "candidate-run",
+) -> RegressionReport:
+    """Compares candidate evaluation results against baseline data.
+
+    Calculates overall and dimension score deltas (candidate - baseline).
+    Enforces non-short-circuiting 3-tier release gates:
+    1. Evaluator Threshold Gate
+    2. Independent Auditor Gate (not BLOCK)
+    3. Regression Delta Gate (overall delta >= -max_overall_drop AND dim deltas >= -max_dim_drop)
+
+    Args:
+        candidate_results: List of candidate EvaluationResult objects.
+        baseline_data: Dictionary returned by load_baseline_manifest().
+        max_overall_drop: Maximum allowed overall score drop before regressed.
+        max_dim_drop: Maximum allowed per-dimension score drop before regressed.
+        candidate_run_id: Identifier for candidate run.
+
+    Returns:
+        A RegressionReport containing comparison details and release block status.
+    """
+    baseline_manifest = baseline_data.get("manifest", {})
+    baseline_scenarios = baseline_data.get("scenarios", {})
+    baseline_run_id = baseline_manifest.get("run_id")
+
+    candidate_by_id = {res.benchmark_id: res for res in candidate_results}
+
+    comparisons: List[ScenarioComparison] = []
+    regression_detected = False
+    release_blocked = False
+
+    # 1. Check for baseline scenarios missing from candidate run
+    for b_id, b_info in baseline_scenarios.items():
+        if b_id not in candidate_by_id:
+            regression_detected = True
+            release_blocked = True
+            comparisons.append(
+                ScenarioComparison(
+                    scenario_id=b_id,
+                    scenario_name=b_info.get("scenario_name", b_id),
+                    status="MISSING_IN_CANDIDATE",
+                    baseline_score=b_info.get("overall_score"),
+                    candidate_score=None,
+                    overall_delta=None,
+                    evaluator_passed=False,
+                    auditor_gate="UNKNOWN",
+                    is_regression=True,
+                )
+            )
+
+    # 2. Evaluate all candidate scenarios
+    for cand_res in candidate_results:
+        s_id = cand_res.benchmark_id
+        cand_score = cand_res.overall_score
+        eval_passed = cand_res.passed
+
+        meta = getattr(cand_res, "agent_metadata", {})
+        auditor_gate = meta.get("audit_gate_decision", "PASS")
+
+        if auditor_gate != "PASS":
+            release_blocked = True
+
+        if not eval_passed:
+            release_blocked = True
+
+        if s_id in baseline_scenarios:
+            b_info = baseline_scenarios[s_id]
+            base_score = b_info["overall_score"]
+            overall_delta = cand_score - base_score
+
+            # Formula: delta < -max_overall_drop
+            overall_regressed = overall_delta < (-abs(max_overall_drop))
+
+            # Per-dimension deltas
+            base_dims = b_info.get("dimension_scores", {})
+            cand_dims = {ds.dimension: ds.score for ds in cand_res.dimension_scores}
+
+            dimension_deltas: List[DimensionDelta] = []
+            dim_regressed = False
+            all_dimensions = set(base_dims.keys()).union(cand_dims.keys())
+
+            for d_name in sorted(all_dimensions):
+                if d_name in base_dims and d_name in cand_dims:
+                    b_dscore = base_dims[d_name]
+                    c_dscore = cand_dims[d_name]
+                    ddelta = c_dscore - b_dscore
+                    d_is_regressed = ddelta < (-abs(max_dim_drop))
+                    if d_is_regressed:
+                        dim_regressed = True
+                    dimension_deltas.append(
+                        DimensionDelta(
+                            dimension=d_name,
+                            baseline_score=b_dscore,
+                            candidate_score=c_dscore,
+                            delta=ddelta,
+                            is_regression=d_is_regressed,
+                        )
+                    )
+                elif d_name in base_dims and d_name not in cand_dims:
+                    b_dscore = base_dims[d_name]
+                    dim_regressed = True
+                    dimension_deltas.append(
+                        DimensionDelta(
+                            dimension=f"{d_name} (Missing in Candidate)",
+                            baseline_score=b_dscore,
+                            candidate_score=0.0,
+                            delta=-b_dscore,
+                            is_regression=True,
+                        )
+                    )
+                else:
+                    c_dscore = cand_dims[d_name]
+                    dimension_deltas.append(
+                        DimensionDelta(
+                            dimension=f"{d_name} (New Dimension)",
+                            baseline_score=0.0,
+                            candidate_score=c_dscore,
+                            delta=c_dscore,
+                            is_regression=False,
+                        )
+                    )
+
+            is_scenario_regression = overall_regressed or dim_regressed
+            if is_scenario_regression:
+                regression_detected = True
+                release_blocked = True
+                status_str = "REGRESSED"
+            elif not eval_passed:
+                status_str = "FAILED"
+            elif auditor_gate != "PASS":
+                status_str = "BLOCKED"
+            else:
+                status_str = "PASSED"
+
+            comparisons.append(
+                ScenarioComparison(
+                    scenario_id=s_id,
+                    scenario_name=cand_res.benchmark_name,
+                    status=status_str,
+                    baseline_score=base_score,
+                    candidate_score=cand_score,
+                    overall_delta=overall_delta,
+                    evaluator_passed=eval_passed,
+                    auditor_gate=auditor_gate,
+                    is_regression=is_scenario_regression,
+                    dimension_deltas=dimension_deltas,
+                )
+            )
+        else:
+            # Candidate scenario is new
+            if not eval_passed or auditor_gate != "PASS":
+                release_blocked = True
+
+            status_str = "PASSED" if (eval_passed and auditor_gate == "PASS") else ("BLOCKED" if auditor_gate != "PASS" else "FAILED")
+            comparisons.append(
+                ScenarioComparison(
+                    scenario_id=s_id,
+                    scenario_name=cand_res.benchmark_name,
+                    status=f"NEW ({status_str})",
+                    baseline_score=None,
+                    candidate_score=cand_score,
+                    overall_delta=None,
+                    evaluator_passed=eval_passed,
+                    auditor_gate=auditor_gate,
+                    is_regression=False,
+                )
+            )
+
+    summary = {
+        "total_scenarios": len(comparisons),
+        "passed": sum(1 for c in comparisons if c.status == "PASSED"),
+        "failed": sum(1 for c in comparisons if not c.evaluator_passed),
+        "blocked": sum(1 for c in comparisons if c.auditor_gate != "PASS"),
+        "regressed": sum(1 for c in comparisons if c.is_regression),
+    }
+
+    return RegressionReport(
+        run_id=candidate_run_id,
+        baseline_run_id=baseline_run_id,
+        regression_detected=regression_detected,
+        release_blocked=release_blocked,
+        max_allowed_overall_regression=max_overall_drop,
+        max_allowed_dimension_regression=max_dim_drop,
+        scenario_comparisons=comparisons,
+        summary=summary,
+    )

framework/regression/loader.py

@@ -0,0 +1,145 @@
+"""Baseline manifest and report loader for regression comparison."""
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+def load_baseline_manifest(path_str: str) -> Dict[str, Any]:
+    """Loads baseline manifest and per-scenario evaluation reports.
+
+    Args:
+        path_str: Path to a baseline directory containing manifest.json or path to manifest.json file.
+
+    Returns:
+        A dictionary structured as:
+        {
+            "manifest": dict,
+            "scenarios": {
+                "<scenario_id>": {
+                    "scenario_id": str,
+                    "scenario_name": str,
+                    "overall_score": float,
+                    "passed": bool,
+                    "audit_gate_decision": str,
+                    "dimension_scores": { "<dim_name>": float },
+                    "report_path": str,
+                    "itinerary_path": str
+                }
+            }
+        }
+
+    Raises:
+        FileNotFoundError: If baseline manifest file or directory does not exist.
+        ValueError: If manifest format is invalid.
+    """
+    path = Path(path_str)
+    if not path.exists():
+        raise FileNotFoundError(f"Baseline path '{path_str}' does not exist.")
+
+    is_dedicated_dir = path.is_dir()
+    if is_dedicated_dir:
+        manifest_file = path / "manifest.json"
+        base_dir = path
+    else:
+        manifest_file = path
+        base_dir = path.parent
+
+    if not manifest_file.exists():
+        raise FileNotFoundError(f"Baseline manifest file '{manifest_file}' not found.")
+
+    with open(manifest_file, "r", encoding="utf-8") as f:
+        manifest_data = json.load(f)
+
+    scenarios_by_id: Dict[str, Dict[str, Any]] = {}
+
+    # 1. Parse scenarios array if present in manifest.json
+    if "scenarios" in manifest_data and isinstance(manifest_data["scenarios"], list):
+        for item in manifest_data["scenarios"]:
+            s_id = item.get("scenario_id") or item.get("benchmark_id")
+            if not s_id:
+                continue
+
+            dim_scores = item.get("dimension_scores", {})
+            if isinstance(dim_scores, list):
+                dim_scores = {d["dimension"]: d["score"] for d in dim_scores if "dimension" in d and "score" in d}
+
+            rep_path = item.get("report_path", "")
+
+            # If report_path is specified and file exists, load detailed report file
+            if rep_path and not dim_scores:
+                full_rep = base_dir / rep_path
+                # Older manifests used the adapter class name in report_path.
+                # Fall back to the actual scenario-id filename so those
+                # baselines remain usable after report naming was corrected.
+                if not full_rep.exists() and base_dir.exists():
+                    matches = sorted(base_dir.glob(f"*_{s_id}_report.json"))
+                    if matches:
+                        full_rep = matches[0]
+                if full_rep.exists():
+                    try:
+                        with open(full_rep, "r", encoding="utf-8") as rf:
+                            rep = json.load(rf)
+                        dim_scores = {
+                            ds["dimension"]: float(ds["score"])
+                            for ds in rep.get("dimension_scores", [])
+                            if isinstance(ds, dict) and "dimension" in ds and "score" in ds
+                        }
+                    except Exception:
+                        pass
+
+            scenarios_by_id[s_id] = {
+                "scenario_id": s_id,
+                "scenario_name": item.get("scenario_name") or item.get("name") or s_id,
+                "overall_score": float(item.get("overall_score", 0.0)),
+                "passed": bool(item.get("passed", False)),
+                "audit_gate_decision": item.get("audit_gate_decision") or item.get("auditor_gate") or "PASS",
+                "dimension_scores": dim_scores,
+                "report_path": rep_path,
+                "itinerary_path": item.get("itinerary_path", ""),
+            }
+
+    # 2. Only scan directory for report files if path_str was a dedicated directory
+    if is_dedicated_dir and base_dir.exists():
+        for fname in os.listdir(base_dir):
+            if fname.endswith("_report.json") and fname not in ("manifest.json", "regression_report.json"):
+                report_filepath = base_dir / fname
+                try:
+                    with open(report_filepath, "r", encoding="utf-8") as f:
+                        rep = json.load(f)
+                    s_id = rep.get("benchmark_id") or rep.get("scenario_id")
+                    if not s_id or s_id in scenarios_by_id:
+                        continue
+
+                    dim_scores = {
+                        ds["dimension"]: float(ds["score"])
+                        for ds in rep.get("dimension_scores", [])
+                        if isinstance(ds, dict) and "dimension" in ds and "score" in ds
+                    }
+                    agent_meta = rep.get("agent_metadata", {})
+                    audit_gate = agent_meta.get("audit_gate_decision", "PASS")
+
+                    scenarios_by_id[s_id] = {
+                        "scenario_id": s_id,
+                        "scenario_name": rep.get("benchmark_name") or rep.get("scenario_name") or s_id,
+                        "overall_score": float(rep.get("overall_score", 0.0)),
+                        "passed": bool(rep.get("passed", False)),
+                        "audit_gate_decision": audit_gate,
+                        "dimension_scores": dim_scores,
+                        "report_path": fname,
+                        "itinerary_path": rep.get("itinerary_path", ""),
+                    }
+                except Exception:
+                    pass
+
+    if not scenarios_by_id:
+        raise ValueError(
+            f"Baseline manifest '{manifest_file}' contains no scenario results. "
+            "Pass a dedicated run directory containing scenario reports or a manifest with scenario/report entries."
+        )
+
+    return {
+        "manifest": manifest_data,
+        "scenarios": scenarios_by_id,
+    }

framework/sdk.py

@@ -0,0 +1,151 @@
+"""Public Python SDK API for programmatic agent evaluation and regression testing."""
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from framework.core.adapters import PythonAgentAdapter, HttpAgentAdapter, CliAgentAdapter
+from framework.evaluation.runner import BenchmarkRunner
+from framework.llms.openai_compatible import OpenAICompatibleLLM
+from framework.models import EvaluationResult
+from framework.regression import load_baseline_manifest, compare_runs, RegressionReport
+from cli.resolver import resolve_agent
+
+
+def evaluate(
+    scenario: str,
+    agent: str,
+    model: str,
+    base_url: str = "https://api.openai.com/v1",
+    api_key: Optional[str] = None,
+    judge_model: Optional[str] = None,
+    judge_base_url: Optional[str] = None,
+    judge_api_key: Optional[str] = None,
+    ground_truth: str = "ground_truth/japan_demo.json",
+    output_dir: str = "./eval_results",
+) -> List[EvaluationResult]:
+    """Programmatically runs agent evaluation on a scenario or suite directory.
+
+    Args:
+        scenario: Path to a scenario file (.md) or suite directory.
+        agent: Agent specifier string ('module:Class' or 'module:factory').
+        model: Target agent model name.
+        base_url: Target agent model OpenAI-compatible base URL.
+        api_key: Target agent API key.
+        judge_model: Judge model name (defaults to target model).
+        judge_base_url: Judge base URL (defaults to target base_url).
+        judge_api_key: Judge API key (defaults to target api_key).
+        ground_truth: Ground truth knowledge base path.
+        output_dir: Target output directory for evaluation artifacts.
+
+    Returns:
+        List of EvaluationResult objects.
+    """
+    target_api_key = api_key or os.getenv("OPENAI_API_KEY", "EMPTY")
+    judge_model_name = judge_model or model
+    j_base_url = judge_base_url or base_url
+    j_api_key = judge_api_key or target_api_key
+
+    # Initialize Target LLM
+    target_llm = OpenAICompatibleLLM(
+        model_name=model,
+        base_url=base_url,
+        api_key=target_api_key,
+    )
+
+    # Initialize Judge LLM
+    judge_llm = OpenAICompatibleLLM(
+        model_name=judge_model_name,
+        base_url=j_base_url,
+        api_key=j_api_key,
+    )
+
+    # Resolve target agent
+    resolved_agent = resolve_agent(agent, target_llm)
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    # Instantiate BenchmarkRunner with output_dir
+    runner = BenchmarkRunner(
+        agent=resolved_agent,
+        judge_llm=judge_llm,
+        local_verifier_path=ground_truth,
+        output_dir=str(out_path),
+    )
+
+    scenario_path = Path(scenario)
+    results: List[EvaluationResult] = []
+
+    if scenario_path.is_dir():
+        for fname in os.listdir(scenario_path):
+            if fname.endswith(".md") or fname.endswith(".json"):
+                s_file = scenario_path / fname
+                try:
+                    res = runner.run(str(s_file))
+                    results.append(res)
+                except Exception as e:
+                    print(f"Warning: Failed to evaluate scenario '{s_file}': {e}")
+    else:
+        res = runner.run(str(scenario_path))
+        results.append(res)
+
+    # Write SDK evaluation artifacts
+    try:
+        from datetime import datetime, timezone
+        import uuid
+        from cli.formatter import redact_credentials
+        from cli.html_reporter import generate_html_report
+
+        run_id = f"evalrun-sdk-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}-{uuid.uuid4().hex[:6]}"
+        manifest = {
+            "run_id": run_id,
+            "timestamp_utc": datetime.now(timezone.utc).isoformat(),
+            "target_agent_spec": agent,
+            # Never place the actual credential in the artifact structure.
+            # The key is used only by the in-process model client.
+            "target_model": {"model_name": model, "base_url": base_url, "api_key": "[REDACTED]"},
+            "judge_model": {"model_name": judge_model_name, "base_url": j_base_url, "api_key": "[REDACTED]"},
+            "ground_truth_path": ground_truth,
+            "output_dir": str(out_path),
+            "total_scenarios": len(results),
+            "overall_passed": all(r.passed for r in results),
+        }
+        with open(out_path / "manifest.json", "w", encoding="utf-8") as f:
+            json.dump(redact_credentials(manifest), f, indent=2)
+
+        generate_html_report(results, redact_credentials(manifest), str(out_path))
+    except Exception as e:
+        print(f"Warning: Failed to generate SDK report artifacts: {e}")
+
+    return results
+
+
+def compare(
+    candidate_results: List[EvaluationResult],
+    baseline: str,
+    max_regression: float = 5.0,
+    max_dimension_regression: float = 10.0,
+    candidate_run_id: str = "candidate-run",
+) -> RegressionReport:
+    """Programmatically compares candidate evaluation results against a baseline.
+
+    Args:
+        candidate_results: List of candidate EvaluationResult objects.
+        baseline: Path to baseline directory or manifest.json file.
+        max_regression: Maximum allowed overall score drop before release blocked.
+        max_dimension_regression: Maximum allowed per-dimension score drop before release blocked.
+        candidate_run_id: Identifier for candidate run.
+
+    Returns:
+        A RegressionReport object.
+    """
+    baseline_data = load_baseline_manifest(baseline)
+    return compare_runs(
+        candidate_results=candidate_results,
+        baseline_data=baseline_data,
+        max_overall_drop=max_regression,
+        max_dim_drop=max_dimension_regression,
+        candidate_run_id=candidate_run_id,
+    )