gatewaysdk 0.3.2__py3-none-any.whl

gatewaysdk/benchmark_hub/harbor_adapter.py

@@ -0,0 +1,673 @@
+"""Harbor → Gateway evals adapter.
+
+A harbor-kind container version is a directory of Harbor task dirs
+(``task.toml`` + ``instruction.md`` + ``tests/``). The first-party ``harbor``
+engine executes them (agent in a sandbox, then the task's own verifier script,
+which writes ``/logs/verifier/reward.json`` / ``reward.txt``) and leaves a jobs
+directory of trials. This module maps that jobs directory into the SAME
+version-pinned evaluation shape the verifiers adapter produces — so the
+Environment page, dispatcher back-link, and sub-reward drill-down are identical
+across env kinds.
+
+Trial dir layout (harbor >= 0.15, same shape Pier emits):
+  trial_dir/
+    result.json                  # TrialResult: task_name, agent_info, verifier_result.rewards, timing
+    verifier/reward.json         # named numeric metrics (fallback / sub-rewards)
+    agent/**                     # agent logs
+
+Reward precedence mirrors deploy/benchmark-runner/harvest.py:_reward — the
+``result.json`` ``verifier_result.rewards`` dict first, then
+``verifier/reward.json``. The scalar reward is the dict's ``reward`` key (or
+its first numeric value); ALL other numeric keys become named sub-rewards.
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import shutil
+import subprocess
+from typing import Any, Dict, List, Optional
+
+from .evals import EvalHarvestError, GatewayEvalsClient
+
+# --- verifiers v1 detection (kept in sync with the runner, engines/hub_bundle.py).
+# The `verifiers.v1` namespace first shipped in verifiers 0.2.0 (PyPI), so a v1
+# Taskset package declares `verifiers>=0.2` (or `~=0.2` / `==0.2.*` / a future
+# `>=1`) in pyproject and/or imports the `verifiers.v1` namespace / subclasses
+# `Taskset` (with the v1 convention of an `__all__` export) in its package
+# __init__. A bare unversioned `verifiers` dep, a pre-0.2 spec (`==0.1.5`,
+# `>=0.1,<0.2`), or the classic v0 `verifiers<1.0.0` pin never matches.
+# Version alternation: minor >= 2 on the 0.x line (`0.2`, `0.10`, ...) or any
+# major >= 1 — never `0.0`/`0.1[.z]`.
+_V1_DEP_RE = re.compile(
+    r"""["']verifiers\s*(?:\[[^\]]*\])?\s*(?:>=|~=|==|>)\s*"""
+    r"""(?:0\.(?:[2-9]\d*|[1-9]\d+)|[1-9]\d*)"""
+)
+_V1_IMPORT_RE = re.compile(
+    r"^\s*(?:from\s+verifiers\.v1\s+import\b"
+    r"|from\s+verifiers\s+import\s+v1\b"
+    r"|import\s+verifiers\.v1\b)",
+    re.MULTILINE,
+)
+_V1_TASKSET_RE = re.compile(
+    r"class\s+\w+\s*\(\s*(?:(?:vf|v1|verifiers(?:\.v1)?)\.)?Taskset\s*[,)]"
+)
+
+
+def _read_text(path: str) -> str:
+    try:
+        with open(path, encoding="utf-8", errors="replace") as f:
+            return f.read()
+    except OSError:
+        return ""
+
+
+def _is_verifiers_v1(src: str) -> bool:
+    pyproject = os.path.join(src, "pyproject.toml")
+    if os.path.isfile(pyproject) and _V1_DEP_RE.search(_read_text(pyproject)):
+        return True
+    inits = glob.glob(os.path.join(src, "*", "__init__.py")) + glob.glob(
+        os.path.join(src, "src", "*", "__init__.py")
+    )
+    for init in inits:
+        text = _read_text(init)
+        if not text:
+            continue
+        if _V1_IMPORT_RE.search(text):
+            return True
+        if "__all__" in text and _V1_TASKSET_RE.search(text):
+            return True
+    return False
+
+
+def detect_env_kind(src: str) -> str:
+    """"mock-tools" iff a ``gateway-env.toml`` sits at the source root (the
+    environment builder's declarative bundle: manifest + ``db/`` + ``tools/`` +
+    ``verifiers/``); else "harbor" iff the source contains Harbor task dirs
+    (``tasks/*/task.toml`` or ``*/task.toml``); else "verifiers-v1" for verifiers
+    v1 Taskset packages; else "verifiers" (legacy v0).
+
+    ``mock-tools`` is checked FIRST because a root ``gateway-env.toml`` is
+    unambiguous — such a bundle is neither a Python package (no ``load_environment``)
+    nor a Harbor task tree, so classifying it as ``verifiers`` would drive the
+    runner into ``pip install -e`` and crash.
+
+    Kept in sync with the runner (engines/hub_bundle.py), the local runtime's
+    ``gatewaysdk.environment._bundle.detect_kind``, and the web read router
+    (deriveEnvKind — where ``mock-tools`` maps to the cosmetic ``verifiers``
+    badge, since the badge is binary)."""
+    if os.path.isfile(os.path.join(src, "gateway-api-world.toml")):
+        return "api"
+    if os.path.isfile(os.path.join(src, "gateway-env.toml")):
+        return "mock-tools"
+    if glob.glob(os.path.join(src, "tasks", "*", "task.toml")) or glob.glob(
+        os.path.join(src, "*", "task.toml")
+    ):
+        return "harbor"
+    if _is_verifiers_v1(src):
+        return "verifiers-v1"
+    return "verifiers"
+
+
+def tasks_root(src: str) -> str:
+    """The dir whose CHILDREN are task dirs: <src>/tasks if present, else <src>."""
+    tasks = os.path.join(src, "tasks")
+    return tasks if glob.glob(os.path.join(tasks, "*", "task.toml")) else src
+
+
+def _read_json(path: str) -> Optional[Any]:
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except (OSError, json.JSONDecodeError):
+        return None
+
+
+def find_trials(job_dir: str) -> List[str]:
+    """Trial dirs = dirs holding a per-trial ``result.json`` AND a ``verifier/``
+    subdir (excludes the job-level result.json at the run root)."""
+    out = []
+    for p in glob.glob(os.path.join(job_dir, "**", "result.json"), recursive=True):
+        d = os.path.dirname(p)
+        if os.path.isdir(os.path.join(d, "verifier")):
+            out.append(d)
+    return sorted(out)
+
+
+def trial_rewards(trial_dir: str) -> Dict[str, float]:
+    """All numeric reward metrics for a trial (``{}`` when the verifier wrote
+    nothing — the trial is then UNGRADED and its reward stays null)."""
+    results = _read_json(os.path.join(trial_dir, "result.json")) or {}
+    rewards = (results.get("verifier_result") or {}).get("rewards") or _read_json(
+        os.path.join(trial_dir, "verifier", "reward.json")
+    )
+    if isinstance(rewards, (int, float)):
+        return {"reward": float(rewards)}
+    if isinstance(rewards, dict):
+        return {
+            k: float(v) for k, v in rewards.items() if isinstance(v, (int, float))
+        }
+    return {}
+
+
+def scalar_reward(rewards: Dict[str, float]) -> Optional[float]:
+    """The trial's headline reward, or None when NOTHING was graded.
+
+    An ungraded trial must stay null. Coercing it to 0.0 made "the grader
+    never ran" indistinguishable from "the agent scored zero" — the exact
+    confusion behind a prod zero that was actually a RewardFileNotFoundError.
+    Null is the platform's own contract (null scores are their own category,
+    never coerced), and it is what lets an agent act on "nothing was graded —
+    a failure to fix, not a pass".
+    """
+    if "reward" in rewards:
+        return rewards["reward"]
+    return next(iter(rewards.values()), None)
+
+
+def samples_from_job_dir(job_dir: str) -> List[Dict[str, Any]]:
+    """Map a harbor jobs directory → eval-sample dicts (one per trial).
+
+    ``exampleId`` = the task name; ``rolloutIndex`` = the occurrence of that
+    task within the job (n-attempts > 1 produces one trial per attempt).
+    Trajectories stay in the jobs dir — samples carry compact info only.
+    """
+    seen: Dict[str, int] = {}
+    samples: List[Dict[str, Any]] = []
+    for trial_dir in find_trials(job_dir):
+        results = _read_json(os.path.join(trial_dir, "result.json")) or {}
+        task_id = (results.get("task_name") or "").split("/")[-1] or os.path.basename(
+            trial_dir.rstrip("/")
+        )
+        idx = seen.get(task_id, 0)
+        seen[task_id] = idx + 1
+
+        rewards = trial_rewards(trial_dir)
+        reward = scalar_reward(rewards)
+        agent_info = results.get("agent_info") or {}
+        agent_result = results.get("agent_result") or {}
+        samples.append(
+            {
+                "exampleId": task_id,
+                # `task` is what run tables and reward breakdowns display —
+                # without it every harbor sample rendered as "?".
+                "task": task_id,
+                "rolloutIndex": idx,
+                "reward": reward,
+                "score": reward,
+                # Ungraded is not incorrect — correctness is unknowable when
+                # nothing scored the trial.
+                "correct": (reward >= 1.0) if reward is not None else None,
+                "rewards": {k: v for k, v in rewards.items() if k != "reward"},
+                "info": {
+                    "trialName": os.path.basename(trial_dir.rstrip("/")),
+                    "agent": agent_info.get("name"),
+                    "model": (agent_info.get("model_info") or {}).get("name"),
+                    "costUsd": agent_result.get("cost_usd"),
+                    "startedAt": results.get("started_at"),
+                    "finishedAt": results.get("finished_at"),
+                },
+            }
+        )
+    return samples
+
+
+def metrics_from_samples(samples: List[Dict[str, Any]]) -> Dict[str, Any]:
+    """Aggregate metrics: average reward + average of each named sub-reward
+    (mirrors verifiers_adapter.metrics_from_generate_outputs)."""
+    rewards = [s["reward"] for s in samples if isinstance(s.get("reward"), (int, float))]
+    by_metric: Dict[str, List[float]] = {}
+    for s in samples:
+        for name, value in (s.get("rewards") or {}).items():
+            if isinstance(value, (int, float)):
+                by_metric.setdefault(name, []).append(float(value))
+
+    def _mean(xs: List[float]) -> Optional[float]:
+        return sum(xs) / len(xs) if xs else None
+
+    return {
+        "n": len(samples),
+        "avg_reward": _mean(rewards),
+        "metrics": {name: _mean(vals) for name, vals in by_metric.items()},
+    }
+
+
+def emit_trial_traces(job_dir: str, evaluation_id: str) -> int:
+    """Emit one rich OTLP trace per trial so platform traces are not shells.
+
+    Reads the verifier's full-state export (``verifier/state.json``, written by
+    task verifiers that support it) and maps it to an AGENT root span with one
+    TOOL child span per audited tool call, joined to the platform's harvest
+    session (``<evaluationId>:<task>:<rolloutIndex>``). Requires
+    ``GATEWAY_OTLP_ENDPOINT`` + key envs and the opentelemetry sdk; silently
+    returns 0 when either is absent (tracing is additive, never load-bearing).
+    """
+    endpoint = os.environ.get("GATEWAY_OTLP_ENDPOINT")
+    public_key = os.environ.get("GATEWAY_PUBLIC_KEY")
+    secret_key = os.environ.get("GATEWAY_SECRET_KEY")
+    if not (endpoint and public_key and secret_key):
+        return 0
+    try:
+        from opentelemetry import trace as otel_trace
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+            OTLPSpanExporter,
+        )
+        from opentelemetry.sdk.resources import Resource
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    except ImportError:
+        return 0
+    import base64
+
+    auth = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode()
+    provider = TracerProvider(
+        resource=Resource.create({"service.name": "gateway-bench-harvest"})
+    )
+    provider.add_span_processor(
+        BatchSpanProcessor(
+            OTLPSpanExporter(
+                endpoint=endpoint, headers={"Authorization": f"Basic {auth}"}
+            )
+        )
+    )
+    tracer = provider.get_tracer("bench-harvest")
+    emitted = 0
+    seen: Dict[str, int] = {}
+    for trial_dir in find_trials(job_dir):
+        results = _read_json(os.path.join(trial_dir, "result.json")) or {}
+        task_id = (results.get("task_name") or "").split("/")[-1] or os.path.basename(
+            trial_dir.rstrip("/")
+        )
+        idx = seen.get(task_id, 0)
+        seen[task_id] = idx + 1
+        # No full-state export is NOT "no session": every rollout emits at
+        # least its graded outline (instruction context, reward, sub-reward
+        # gates, agent/model/cost). state.json, when a world exports it,
+        # upgrades the same trace with one TOOL span per audited call.
+        state = _read_json(os.path.join(trial_dir, "verifier", "state.json")) or {}
+        rewards = trial_rewards(trial_dir)
+        report = state.get("report") or {}
+        case = state.get("case_state") or {}
+        with tracer.start_as_current_span(f"episode {task_id}") as root:
+            root.set_attribute("openinference.span.kind", "AGENT")
+            root.set_attribute("session.id", f"{evaluation_id}:{task_id}:{idx}")
+            root.set_attribute("langfuse.session.id", f"{evaluation_id}:{task_id}:{idx}")
+            agent_info = results.get("agent_info") or {}
+            root.set_attribute(
+                "input.value",
+                json.dumps(
+                    {
+                        "task": task_id,
+                        **({"case": case.get("title")} if case else {}),
+                        **(
+                            {"agent": agent_info.get("name")}
+                            if agent_info.get("name")
+                            else {}
+                        ),
+                    }
+                ),
+            )
+            root.set_attribute(
+                "output.value",
+                json.dumps(
+                    {
+                        "reward": scalar_reward(rewards),
+                        "report": report,
+                        "failing_gates": [
+                            k for k, v in rewards.items() if v not in (1.0, None)
+                        ] or "none",
+                    },
+                    default=str,
+                )[:4000],
+            )
+            audit_by_seq = {
+                entry.get("seq"): entry for entry in state.get("audit_entries") or []
+            }
+            call_log = state.get("call_log") or []
+            per_call = call_log or [
+                {
+                    "seq": entry.get("seq"),
+                    "tool": entry.get("tool"),
+                    "arguments": None,
+                    "result_preview": None,
+                    "error": None,
+                }
+                for entry in state.get("audit_entries") or []
+            ]
+            for call in per_call:
+                audit = audit_by_seq.get(call.get("seq"), {})
+                with tracer.start_as_current_span(
+                    f"{call.get('seq', 0):02d} {call.get('tool', '?')}"
+                ) as span:
+                    span.set_attribute("openinference.span.kind", "TOOL")
+                    span.set_attribute(
+                        "input.value",
+                        call.get("arguments")
+                        or json.dumps({"args_hash": audit.get("args_hash")}),
+                    )
+                    span.set_attribute(
+                        "output.value",
+                        (
+                            json.dumps({"error": call["error"]})
+                            if call.get("error")
+                            else call.get("result_preview")
+                        )
+                        or json.dumps({"result_hash": audit.get("result_hash")}),
+                    )
+                    if audit.get("args_hash"):
+                        span.set_attribute("audit.args_hash", audit["args_hash"])
+                        span.set_attribute(
+                            "audit.result_hash", audit.get("result_hash") or ""
+                        )
+                        span.set_attribute("audit.at", audit.get("timestamp") or "")
+
+            # THE AGENT'S OWN DIALOGUE, when the trial recorded one. Installed
+            # agents (claude-code and kin) never touch state.json, so without
+            # this the rollout page showed a graded outline over an empty
+            # conversation — a run that traced "nothing" while the agent had
+            # written a sixteen-point redline. Capped, never load-bearing.
+            transcript = agent_transcript(trial_dir)
+            if transcript and transcript.get("kind") == "claude-code":
+                messages = transcript["messages"][:80]
+                first_user = next(
+                    (m["text"] for m in messages if m["role"] == "user" and m["text"]),
+                    "",
+                )
+                if first_user:
+                    root.set_attribute(
+                        "input.value",
+                        json.dumps(
+                            {"task": task_id, "instruction": first_user[:3000]}
+                        ),
+                    )
+                for step, message in enumerate(messages):
+                    if message["role"] == "assistant":
+                        with tracer.start_as_current_span(
+                            f"{step:02d} assistant"
+                        ) as span:
+                            span.set_attribute("openinference.span.kind", "LLM")
+                            span.set_attribute(
+                                "output.value",
+                                json.dumps(
+                                    {
+                                        "text": message["text"],
+                                        **(
+                                            {"tool_calls": message["tool_uses"]}
+                                            if message["tool_uses"]
+                                            else {}
+                                        ),
+                                    },
+                                    default=str,
+                                )[:4000],
+                            )
+                    elif message["tool_results"]:
+                        with tracer.start_as_current_span(
+                            f"{step:02d} tool result"
+                        ) as span:
+                            span.set_attribute("openinference.span.kind", "TOOL")
+                            span.set_attribute(
+                                "output.value",
+                                "\n".join(message["tool_results"])[:4000],
+                            )
+            elif transcript and transcript.get("kind") == "trajectory":
+                for step, entry in enumerate(transcript["steps"][:80]):
+                    if not isinstance(entry, dict):
+                        continue
+                    with tracer.start_as_current_span(
+                        f"{step:02d} {entry.get('type') or entry.get('role') or 'step'}"
+                    ) as span:
+                        span.set_attribute("openinference.span.kind", "LLM")
+                        span.set_attribute(
+                            "output.value", json.dumps(entry, default=str)[:4000]
+                        )
+        emitted += 1
+    provider.shutdown()
+    return emitted
+
+
+def agent_transcript(trial_dir: str) -> Optional[Dict[str, Any]]:
+    """The best agent transcript a trial left behind.
+
+    Installed agents record their dialogue in agent-specific shapes; the
+    state.json convention only covers worlds that export audited tool calls.
+    Preference order:
+      1. claude-code's session JSONL (agent/sessions/projects/*/<id>.jsonl) —
+         the full message stream, one JSON object per line;
+      2. harbor's normalized agent/trajectory.json, shape-tolerantly.
+    Returns None when a trial recorded neither (e.g. mini-swe, whose
+    transcript already flows through state.json-style worlds)."""
+    import glob as _glob
+
+    jsonls = sorted(
+        _glob.glob(
+            os.path.join(trial_dir, "agent", "sessions", "projects", "*", "*.jsonl")
+        ),
+        key=lambda path: os.path.getmtime(path),
+    )
+    if jsonls:
+        messages: List[Dict[str, Any]] = []
+        with open(jsonls[-1], errors="replace") as fh:
+            for line in fh:
+                try:
+                    event = json.loads(line)
+                except (ValueError, TypeError):
+                    continue
+                message = event.get("message") if isinstance(event, dict) else None
+                if not isinstance(message, dict):
+                    continue
+                role = message.get("role")
+                if role not in ("user", "assistant"):
+                    continue
+                content = message.get("content")
+                text = ""
+                tool_uses: List[Dict[str, Any]] = []
+                tool_results: List[str] = []
+                if isinstance(content, str):
+                    text = content
+                elif isinstance(content, list):
+                    for block in content:
+                        if not isinstance(block, dict):
+                            continue
+                        if block.get("type") == "text":
+                            text += str(block.get("text") or "")
+                        elif block.get("type") == "tool_use":
+                            tool_uses.append(
+                                {
+                                    "name": block.get("name"),
+                                    "input": json.dumps(
+                                        block.get("input"), default=str
+                                    )[:600],
+                                }
+                            )
+                        elif block.get("type") == "tool_result":
+                            raw = block.get("content")
+                            if isinstance(raw, list):
+                                raw = "".join(
+                                    str(piece.get("text") or "")
+                                    for piece in raw
+                                    if isinstance(piece, dict)
+                                )
+                            tool_results.append(str(raw)[:800])
+                if text or tool_uses or tool_results:
+                    messages.append(
+                        {
+                            "role": role,
+                            "text": text[:2000],
+                            "tool_uses": tool_uses,
+                            "tool_results": tool_results,
+                        }
+                    )
+        if messages:
+            return {"kind": "claude-code", "messages": messages}
+
+    trajectory = _read_json(os.path.join(trial_dir, "agent", "trajectory.json"))
+    if trajectory:
+        steps = (
+            trajectory.get("steps") if isinstance(trajectory, dict) else trajectory
+        )
+        if isinstance(steps, list) and steps:
+            return {"kind": "trajectory", "steps": steps}
+    return None
+
+
+def harvest_job_dir(
+    job_dir: str,
+    version_id: str,
+    evals_client: GatewayEvalsClient,
+    *,
+    model: str,
+    run_id: Optional[str] = None,
+    dispatch_shard: Optional[Dict[str, Any]] = None,
+    rollouts_per_example: int = 1,
+) -> Dict[str, Any]:
+    """Harvest a completed harbor jobs directory into a version-pinned
+    evaluation: create → push samples → finalize with aggregate metrics."""
+    samples = samples_from_job_dir(job_dir)
+    if not samples:
+        raise EvalHarvestError(f"no harbor trials found under {job_dir}")
+    metrics = metrics_from_samples(samples)
+
+    evaluation_id = evals_client.create_evaluation(
+        version_id,
+        model=model,
+        run_id=run_id,
+        **({"dispatch_shard": dispatch_shard} if dispatch_shard is not None else {}),
+        num_examples=len({s["exampleId"] for s in samples}),
+        rollouts_per_example=rollouts_per_example,
+    )
+    inserted = evals_client.push_samples(evaluation_id, samples)
+    evals_client.finalize_evaluation(evaluation_id, metrics=metrics)
+    traced = emit_trial_traces(job_dir, evaluation_id)
+    return {
+        "evaluationId": evaluation_id,
+        "samples": inserted,
+        "metrics": metrics,
+        "richTraces": traced,
+    }
+
+
+def harbor_bin() -> str:
+    """The harbor CLI: on PATH, or fall back to the Pier tool venv (Pier depends
+    on the same harbor package)."""
+    found = shutil.which("harbor")
+    if found:
+        return found
+    pier_sibling = os.path.expanduser("~/.local/share/uv/tools/datacurve-pier/bin/harbor")
+    if os.path.exists(pier_sibling):
+        return pier_sibling
+    raise EvalHarvestError(
+        "harbor CLI not found — `uv tool install harbor` (or install datacurve-pier)."
+    )
+
+
+DEFAULT_HARBOR_AGENT = "mini-swe-agent"
+
+# Env vars forwarded into the agent sandbox when present — installed agents
+# authenticate their own model calls (mini-swe-agent via OpenRouter/litellm,
+# claude-code via the Anthropic API, goose via provider envs). OPENAI_HOST
+# lets goose's openai provider point at any OpenAI-compatible endpoint
+# (e.g. OPENAI_API_KEY=$OPENROUTER_API_KEY + OPENAI_HOST=https://openrouter.ai/api).
+AGENT_ENV_PASSTHROUGH = (
+    "OPENROUTER_API_KEY",
+    "ANTHROPIC_API_KEY",
+    "OPENAI_API_KEY",
+    "OPENAI_HOST",
+)
+
+
+def find_mcp_config(src: str) -> Optional[str]:
+    """An env-bundled Claude-style MCP config: <src>/mcp.json (local source
+    dirs) or <src>/tasks/mcp.json (what survives the push archiver)."""
+    for candidate in (os.path.join(src, "mcp.json"), os.path.join(src, "tasks", "mcp.json")):
+        if os.path.isfile(candidate):
+            return candidate
+    return None
+
+
+def resolve_harbor_model(model: str, agent: str) -> str:
+    """mini-swe-agent speaks litellm ids — default to the OpenRouter provider
+    prefix. Other agents (claude-code, codex, …) take their provider's native
+    ids verbatim."""
+    if agent == DEFAULT_HARBOR_AGENT and not model.startswith("openrouter/"):
+        return f"openrouter/{model}"
+    return model
+
+
+def run_harbor_job(
+    tasks_dir: str,
+    jobs_dir: str,
+    *,
+    model: str,
+    agent: str = DEFAULT_HARBOR_AGENT,
+    sandbox_env: Optional[str] = None,
+    num_examples: int = -1,
+    rollouts_per_example: int = 1,
+    n_concurrent: int = 4,
+    extra_env: Optional[Dict[str, str]] = None,
+    mcp_config: Optional[str] = None,
+) -> str:
+    """Run the first-party harbor engine over a Harbor task dir. Returns the
+    job output dir. Exit-code tolerant: harbor exits non-zero when trials fail;
+    the reward files still land and the harvest reflects them honestly."""
+    resolved_sandbox = sandbox_env or os.environ.get("HARBOR_SANDBOX_ENV", "docker")
+    # Trial concurrency is a HOST-CAPACITY knob, not a task knob: each harbor
+    # task composes several docker networks, and a host whose address pools
+    # are exhausted fails every subsequent trial in ~1s with reward null
+    # (run cmt6a2bzc: 2 shards x -n 4 on one instance -> 42/50 trials never
+    # ran). HARBOR_N_CONCURRENT lets the dispatcher size runs to the host
+    # without a code change.
+    env_concurrency = os.environ.get("HARBOR_N_CONCURRENT", "").strip()
+    resolved_concurrency = (
+        int(env_concurrency) if env_concurrency.isdigit() else n_concurrent
+    )
+    if resolved_sandbox == "docker":
+        # Leftover networks from crashed prior trials shrink the same address
+        # pools; a run should never inherit a previous run's leak. Unused
+        # networks only — running trials are untouched.
+        subprocess.run(
+            ["docker", "network", "prune", "-f"],
+            check=False,
+            capture_output=True,
+        )
+    cmd = [
+        harbor_bin(),
+        "run",
+        "-p", tasks_dir,
+        "-a", agent,
+        "-m", model,
+        "-e", resolved_sandbox,
+        "-o", jobs_dir,
+        "-k", str(max(1, rollouts_per_example)),
+        "-n", str(max(1, resolved_concurrency)),
+        "-y",
+        "-q",
+    ]
+    if num_examples and num_examples > 0:
+        cmd += ["-l", str(num_examples)]
+    if mcp_config:
+        cmd += ["--mcp-config", mcp_config]
+
+    env_pairs: Dict[str, str] = {
+        key: os.environ[key]
+        for key in AGENT_ENV_PASSTHROUGH
+        if os.environ.get(key)
+    }
+    env_pairs.update(extra_env or {})
+    for key, value in env_pairs.items():
+        if value:
+            cmd += ["--ae", f"{key}={value}"]
+
+    # Redact env values from the printed command line.
+    printable = " ".join(
+        "--ae " + arg.split("=", 1)[0] + "=***" if i and cmd[i - 1] == "--ae" else arg
+        for i, arg in enumerate(cmd)
+        if arg != "--ae"
+    )
+    print(f"[harbor] {printable}", flush=True)
+    proc = subprocess.run(cmd)
+    if proc.returncode != 0:
+        print(f"[harbor] exited {proc.returncode} (harvesting whatever ran)", flush=True)
+    return jobs_dir