benchflow 0.7.4__tar.gz → 0.7.6.dev2055__tar.gz

{benchflow-0.7.4 → benchflow-0.7.6.dev2055}/CHANGELOG.md

@@ -2,6 +2,27 @@
 
 ## [Unreleased]
 
+## 0.7.5 — 2026-08-19
+
+### Added
+
+- **Weighted rubric-review contract (v0.2).** `bench review` now accepts the
+  versionless `rubric.json` shape introduced by FrontierPhysics PR #109, where
+  every criterion adds strict `blocker` (`0` or `1`) and `weight` (`1` through
+  `10`) fields. Binary blockers gate publication; non-blockers receive weighted
+  `0` / `1` / `2` scores with raw and gated quality plus publication bands in
+  the report. Blocker weights are excluded from quality, and the wrapper reward
+  remains a structural-validity signal. Existing three-field v0.1 rubrics keep
+  their `pass` / `fail` / `not_applicable` behavior unchanged. Docker runs now
+  probe whether verifier-log bind mounts are genuinely visible to the container
+  and fall back to explicit copy-out when path translation is unavailable.
+
+### Fixed
+
+- **Linked-worktree `.git` pointer files stay out of workspace attachments.**
+  Workspace capture now excludes `.git` files as well as directories, preventing
+  local absolute worktree metadata from entering uploaded archives. (#1032)
+
 ## 0.7.4 — 2026-08-16
 
 ### Added

{benchflow-0.7.4 → benchflow-0.7.6.dev2055}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.4
+Version: 0.7.6.dev2055
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.4 → benchflow-0.7.6.dev2055}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.4"
+version = "0.7.6.dev2055"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.4 → benchflow-0.7.6.dev2055}/src/benchflow/__init__.py

@@ -61,9 +61,12 @@ from benchflow.monitor import (
     MonitorResult,
 )
 from benchflow.review import (
+    PublicationDecision,
     ReviewReport,
     ReviewRubricError,
+    ReviewScoring,
     run_reviews,
+    score_weighted_review,
 )
 from benchflow.review import Rubric as ReviewRubric
 from benchflow.review import RubricCriterion as ReviewRubricCriterion
@@ -169,11 +172,14 @@ __all__ = [
     "load_rubric_json",
     "load_rubric_toml",
     "ReviewReport",
+    "PublicationDecision",
     "ReviewRubric",
     "ReviewRubricCriterion",
     "ReviewRubricError",
+    "ReviewScoring",
     "load_review_rubric",
     "run_reviews",
+    "score_weighted_review",
     "Sandbox",
     "SandboxExecResult",
     "SandboxImage",

{benchflow-0.7.4 → benchflow-0.7.6.dev2055}/src/benchflow/_utils/task_authoring/__init__.py

@@ -146,10 +146,10 @@ def task_digest(task_dir: Path) -> str:
 def _check_review_rubric(verifier_dir: Path, *, verifier_label: str) -> list[str]:
     """Validate a review ``rubric.json`` when the task ships one.
 
-    Review rubrics are ``{"criteria": [{name, description, guidance}]}``
-    JSON documents. A ``rubric.json`` that is not shaped like that (for
-    example an llm-judge rubric owned by the verifier-strategy machinery)
-    is not checked here.
+    Review rubrics use either the legacy ``{name, description, guidance}``
+    criterion shape or the weighted shape that adds ``blocker`` and
+    ``weight``. A ``rubric.json`` in the full llm-judge dialect is owned by
+    the verifier-strategy machinery and is not checked here.
     """
     from benchflow.review.config import (
         ReviewRubricError,

{benchflow-0.7.4 → benchflow-0.7.6.dev2055}/src/benchflow/cli/review.py

@@ -25,43 +25,88 @@ _REVIEW_OUTCOME_STYLES = {
     "not_applicable": "grey50",
 }
 
+_REVIEW_SCORE_STYLES = {
+    0: "red",
+    1: "yellow",
+    2: "green",
+}
+
 
 def _render_trial_review(trial) -> None:
     table = Table(title=f"Review: {escape(str(trial.trial_name))}", show_lines=True)
     table.add_column("Criterion")
-    table.add_column("Outcome")
+    table.add_column("Result")
     table.add_column("Explanation")
     for name, check in (trial.checks or {}).items():
-        outcome = str(check.get("outcome", ""))
+        if "score" in check:
+            raw_score = check.get("score")
+            result = str(raw_score)
+            style = (
+                _REVIEW_SCORE_STYLES.get(raw_score, "white")
+                if type(raw_score) is int
+                else "white"
+            )
+        else:
+            result = str(check.get("outcome", ""))
+            style = _REVIEW_OUTCOME_STYLES.get(result, "white")
         table.add_row(
             escape(str(name).replace("_", " ").title()),
-            escape(outcome),
+            escape(result),
             escape(str(check.get("explanation", ""))),
-            style=_REVIEW_OUTCOME_STYLES.get(outcome, "white"),
+            style=style,
         )
     if trial.summary:
         console.print(f"\n[bold]Summary:[/bold] {escape(str(trial.summary))}\n")
+    scoring = getattr(trial, "scoring", None)
+    if scoring is not None:
+        console.print(
+            "[bold]Weighted quality:[/bold] "
+            f"{scoring.raw_quality:.3f} raw / {scoring.gated_quality:.3f} gated "
+            f"— {escape(scoring.decision.value)}\n"
+        )
     console.print(table)
 
 
 def _render_review_overview(trials) -> None:
     table = Table(title="Rubric Reviews", show_lines=True)
-    for column in ("Run", "Pass", "Fail", "N/A", "Valid"):
+    for column in ("Run", "Pass", "Fail", "N/A", "Quality", "Decision", "Valid"):
         table.add_column(column)
     for trial in trials:
         if trial.error and not trial.checks:
             table.add_row(
-                escape(str(trial.trial_name)), "-", "-", "-", "no", style="red"
+                escape(str(trial.trial_name)),
+                "-",
+                "-",
+                "-",
+                "-",
+                "-",
+                "no",
+                style="red",
             )
             continue
         counts = trial.outcome_counts()
+        scoring = getattr(trial, "scoring", None)
+        quality = f"{scoring.raw_quality:.3f}" if scoring is not None else "-"
+        decision = scoring.decision.value if scoring is not None else "-"
+        if (
+            not trial.review_valid
+            or counts["fail"]
+            or (scoring is not None and decision == "not_publishable")
+        ):
+            style = "red"
+        elif scoring is not None and decision == "presentable_with_revisions":
+            style = "yellow"
+        else:
+            style = "white"
         table.add_row(
             escape(str(trial.trial_name)),
             str(counts["pass"]),
             str(counts["fail"]),
             str(counts["not_applicable"]),
+            quality,
+            escape(decision),
             "yes" if trial.review_valid else "no",
-            style="red" if counts["fail"] or not trial.review_valid else "white",
+            style=style,
         )
     console.print(table)
     for trial in trials:

{benchflow-0.7.4 → benchflow-0.7.6.dev2055}/src/benchflow/publish/traj_capture.py

@@ -72,6 +72,9 @@ WORKSPACE_EXCLUDED_DIRS = frozenset(
 # are uploaded as-is (no content redaction), so obvious secret carriers stay
 # on the contributor's machine.
 WORKSPACE_EXCLUDED_FILES = (
+    # A linked git worktree's ``.git`` is a pointer FILE (not a directory)
+    # carrying a local absolute path; the directory case is excluded above.
+    re.compile(r"^\.git$", re.IGNORECASE),
     re.compile(r"^\.env(\..+)?$", re.IGNORECASE),
     re.compile(r".*\.(pem|key|p12|pfx|keystore)$", re.IGNORECASE),
     re.compile(r"^id_(rsa|dsa|ecdsa|ed25519)(\..+)?$", re.IGNORECASE),

{benchflow-0.7.4 → benchflow-0.7.6.dev2055}/src/benchflow/review/__init__.py

@@ -1,10 +1,10 @@
 """Rubric review — detached agentic grading of finished rollouts.
 
-A rubric (contract **v0.1**) is a JSON object containing a ``criteria`` list;
-each criterion has ``name``, ``description``, and ``guidance``. A reviewer
-agent reads a finished rollout's records inside its own sandbox and answers
-each criterion with ``pass`` / ``fail`` / ``not_applicable`` plus an
-explanation.
+A rubric is a JSON object containing a ``criteria`` list. Legacy v0.1
+criteria have ``name``, ``description``, and ``guidance`` and receive
+pass/fail/not-applicable judgments. Weighted v0.2 criteria additionally have
+strict ``blocker`` and ``weight`` integers: blockers receive pass/fail gates,
+while non-blockers receive 0/1/2 scores that Benchflow aggregates host-side.
 
 Reviews run *after* rollouts, from their host-side directories, as ordinary
 rollouts of throwaway wrapper tasks (:mod:`benchflow.review.wrapper`), so
@@ -23,14 +23,19 @@ Public surface:
 
 from benchflow.review.config import (
     DEFAULT_RUBRIC_PATH,
+    LEGACY_REVIEW_RUBRIC_CONTRACT,
     REVIEW_RESULT_FILENAME,
     REVIEW_RUBRIC_CONTRACT,
     REVIEW_RUBRIC_FILENAME,
+    WEIGHTED_REVIEW_RUBRIC_CONTRACT,
+    BlockerCriterionCheck,
+    BlockerOutcomeValue,
     CriterionCheck,
     ReviewOutcomeValue,
     ReviewRubricError,
     Rubric,
     RubricCriterion,
+    ScoredCriterionCheck,
     build_criteria_guidance,
     build_review_response_model,
     find_task_rubric,
@@ -44,21 +49,37 @@ from benchflow.review.runner import (
     discover_rollouts,
     run_reviews,
 )
+from benchflow.review.scoring import (
+    PUBLISHABLE_QUALITY,
+    REVISIONS_QUALITY,
+    PublicationDecision,
+    ReviewScoring,
+    score_weighted_review,
+)
 from benchflow.review.wrapper import assemble_review_task
 
 __all__ = [
     "DEFAULT_RUBRIC_PATH",
+    "LEGACY_REVIEW_RUBRIC_CONTRACT",
+    "PUBLISHABLE_QUALITY",
     "REVIEW_RUBRIC_CONTRACT",
     "REVIEW_REPORT_FILENAME",
     "REVIEW_RESULT_FILENAME",
     "REVIEW_RUBRIC_FILENAME",
+    "REVISIONS_QUALITY",
+    "WEIGHTED_REVIEW_RUBRIC_CONTRACT",
+    "BlockerCriterionCheck",
+    "BlockerOutcomeValue",
     "CriterionCheck",
+    "PublicationDecision",
     "ReviewOutcomeValue",
     "ReviewReport",
     "ReviewRubricError",
     "ReviewRunError",
+    "ReviewScoring",
     "Rubric",
     "RubricCriterion",
+    "ScoredCriterionCheck",
     "TrialReview",
     "assemble_review_task",
     "build_criteria_guidance",
@@ -67,4 +88,5 @@ __all__ = [
     "find_task_rubric",
     "load_rubric",
     "run_reviews",
+    "score_weighted_review",
 ]

benchflow-0.7.6.dev2055/src/benchflow/review/config.py

@@ -0,0 +1,359 @@
+"""Rubric schemas and loading for detached post-run review.
+
+The current weighted contract (v0.2) extends each criterion with two strict
+integer fields::
+
+    {
+      "criteria": [
+        {
+          "name": "method_soundness",
+          "blocker": 0,
+          "weight": 5,
+          "description": "Author-facing note; never shown to the reviewer.",
+          "guidance": "Score 2 when ...; 1 when ...; 0 when ..."
+        }
+      ]
+    }
+
+``blocker: 1`` criteria receive a binary pass/fail judgment. ``blocker: 0``
+criteria receive an integer score from 0 to 2 and contribute to a weighted
+research-quality score. ``weight`` is an integer from 1 to 10.
+
+Versionless v0.1 rubrics remain supported for compatibility. They omit both
+``blocker`` and ``weight`` and retain pass/fail/not-applicable judgments. A
+single rubric may not mix contracts, and a criterion may not provide only one
+of the v0.2 fields; both cases fail closed instead of guessing intent.
+"""
+
+from __future__ import annotations
+
+import json
+import keyword
+from enum import StrEnum
+from pathlib import Path
+from typing import Annotated, Any, Self
+
+from pydantic import (
+    AfterValidator,
+    BaseModel,
+    ConfigDict,
+    Field,
+    ValidationError,
+    create_model,
+    field_validator,
+    model_validator,
+)
+
+LEGACY_REVIEW_RUBRIC_CONTRACT = "v0.1"
+WEIGHTED_REVIEW_RUBRIC_CONTRACT = "v0.2"
+# Backward-compatible public alias. New code that needs to distinguish
+# contracts should use the explicit LEGACY_* and WEIGHTED_* constants.
+REVIEW_RUBRIC_CONTRACT = LEGACY_REVIEW_RUBRIC_CONTRACT
+REVIEW_RUBRIC_FILENAME = "rubric.json"
+REVIEW_RESULT_FILENAME = "review-result.json"
+
+DEFAULT_RUBRIC_PATH = Path(__file__).parent / "default-rubric.json"
+
+
+def _non_blank(value: str) -> str:
+    if not value.strip():
+        raise ValueError("must contain non-whitespace text")
+    return value
+
+
+NonBlankText = Annotated[
+    str,
+    Field(min_length=1),
+    AfterValidator(_non_blank),
+]
+
+
+class ReviewRubricError(ValueError):
+    """Raised when a rubric file cannot be loaded or is not a valid rubric."""
+
+
+class RubricCriterion(BaseModel):
+    """One criterion the reviewer grades."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    description: str
+    guidance: str
+    blocker: int | None = Field(default=None, strict=True, ge=0, le=1)
+    weight: int | None = Field(default=None, strict=True, ge=1, le=10)
+
+    @field_validator("name")
+    @classmethod
+    def _name_is_usable_field(cls, value: str) -> str:
+        # The name becomes a dynamically created model field, so anything the
+        # schema library reserves either crashes model construction
+        # (``model_config``), trips protected-namespace rules
+        # (``model_dump``), or is silently dropped from the generated schema
+        # (private/dunder names) -- yielding an impossible reviewer/verifier
+        # contract instead of a loud failure.
+        if not value.isidentifier() or keyword.iskeyword(value):
+            raise ValueError(
+                f"criterion name {value!r} must be a valid, non-keyword "
+                "Python identifier (it becomes a structured-output field)"
+            )
+        if value.startswith("_"):
+            raise ValueError(
+                f"criterion name {value!r} must not start with '_': private "
+                "and dunder names are dropped from the generated schema"
+            )
+        if value.startswith("model_"):
+            raise ValueError(
+                f"criterion name {value!r} must not start with 'model_': "
+                "that namespace is reserved by the schema library"
+            )
+        if hasattr(BaseModel, value):
+            raise ValueError(
+                f"criterion name {value!r} collides with reserved schema "
+                f"attribute {value!r}; choose another name"
+            )
+        return value
+
+    @model_validator(mode="after")
+    def _scoring_fields_are_paired(self) -> Self:
+        blocker_supplied = "blocker" in self.model_fields_set
+        weight_supplied = "weight" in self.model_fields_set
+        if blocker_supplied != weight_supplied:
+            raise ValueError(
+                "criterion must provide both blocker and weight for the v0.2 "
+                "contract, or omit both for legacy v0.1"
+            )
+        if blocker_supplied and (self.blocker is None or self.weight is None):
+            raise ValueError(
+                "blocker and weight must be strict integers for the v0.2 "
+                "contract; null is not allowed"
+            )
+        return self
+
+    @property
+    def is_legacy(self) -> bool:
+        return self.blocker is None
+
+    @property
+    def is_blocker(self) -> bool:
+        return self.blocker == 1
+
+    def metadata(self) -> dict[str, str | int | None]:
+        """Return the non-prompt criterion contract recorded in artifacts."""
+
+        return {
+            "name": self.name,
+            "blocker": self.blocker,
+            "weight": self.weight,
+        }
+
+
+class Rubric(BaseModel):
+    """A parsed review rubric."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    criteria: list[RubricCriterion] = Field(min_length=1)
+
+    @field_validator("criteria")
+    @classmethod
+    def _criteria_are_coherent(
+        cls, value: list[RubricCriterion]
+    ) -> list[RubricCriterion]:
+        seen: set[str] = set()
+        duplicates: set[str] = set()
+        for criterion in value:
+            if criterion.name in seen:
+                duplicates.add(criterion.name)
+            seen.add(criterion.name)
+        if duplicates:
+            raise ValueError(
+                f"criterion names must be unique; duplicated: {sorted(duplicates)} "
+                "(duplicate names would silently collapse into one "
+                "structured-output field)"
+            )
+        contracts = {criterion.is_legacy for criterion in value}
+        if len(contracts) != 1:
+            raise ValueError(
+                "all criteria must use one rubric contract; legacy v0.1 and "
+                "weighted v0.2 criteria cannot be mixed"
+            )
+        if not value[0].is_legacy and not any(
+            not criterion.is_blocker for criterion in value
+        ):
+            raise ValueError(
+                "weighted v0.2 rubrics require at least one scored "
+                "(blocker: 0) criterion"
+            )
+        return value
+
+    @property
+    def contract(self) -> str:
+        return (
+            LEGACY_REVIEW_RUBRIC_CONTRACT
+            if self.criteria[0].is_legacy
+            else WEIGHTED_REVIEW_RUBRIC_CONTRACT
+        )
+
+    @property
+    def is_weighted(self) -> bool:
+        return self.contract == WEIGHTED_REVIEW_RUBRIC_CONTRACT
+
+    def metadata(self) -> dict[str, Any]:
+        """Return the rubric contract carried into wrapper/report artifacts."""
+
+        return {
+            "contract": self.contract,
+            "criteria": [criterion.metadata() for criterion in self.criteria],
+        }
+
+
+class ReviewOutcomeValue(StrEnum):
+    """Closed outcome vocabulary for one legacy v0.1 criterion."""
+
+    PASS = "pass"
+    FAIL = "fail"
+    NOT_APPLICABLE = "not_applicable"
+
+
+class CriterionCheck(BaseModel):
+    """The reviewer's answer for one legacy v0.1 criterion."""
+
+    explanation: NonBlankText
+    outcome: ReviewOutcomeValue
+
+
+class BlockerOutcomeValue(StrEnum):
+    """Closed outcome vocabulary for a v0.2 blocker."""
+
+    PASS = "pass"
+    FAIL = "fail"
+
+
+class BlockerCriterionCheck(BaseModel):
+    """The reviewer's binary answer for a v0.2 blocker."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    explanation: NonBlankText
+    outcome: BlockerOutcomeValue
+
+
+class ScoredCriterionCheck(BaseModel):
+    """The reviewer's 0-2 answer for a weighted v0.2 criterion."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    explanation: NonBlankText
+    score: int = Field(strict=True, ge=0, le=2)
+
+
+def load_rubric(path: Path | None = None) -> Rubric:
+    """Load a rubric from a JSON file, or the built-in default rubric.
+
+    Only JSON is accepted. Every criterion must consistently use either the
+    legacy v0.1 three-field shape or the weighted v0.2 five-field shape.
+    """
+
+    rubric_path = path if path is not None else DEFAULT_RUBRIC_PATH
+    if rubric_path.suffix.lower() != ".json":
+        raise ReviewRubricError(
+            f"unsupported rubric format {rubric_path.suffix!r}: rubrics are JSON files"
+        )
+    try:
+        text = rubric_path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ReviewRubricError(f"cannot read {rubric_path}: {exc}") from exc
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise ReviewRubricError(f"{rubric_path} is not valid JSON: {exc}") from exc
+    try:
+        return Rubric.model_validate(data)
+    except ValidationError as exc:
+        raise ReviewRubricError(f"{rubric_path} is not a valid rubric: {exc}") from exc
+
+
+def is_review_rubric_file(path: Path) -> bool:
+    """Whether ``path`` claims either detached-review rubric contract.
+
+    ``rubric.json`` is an overloaded filename: llm-judge verifier rubrics
+    use entries carrying the full ``{id, match_criteria}`` shape. Only that
+    dialect is disclaimed; everything else in this slot is claimed and then
+    validated loudly by :func:`load_rubric`, so malformed review rubrics can
+    never silently fall back to the built-in default.
+    """
+
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+        return True
+    if not isinstance(data, dict):
+        return True
+    criteria = data.get("criteria")
+    if not isinstance(criteria, list):
+        return True
+
+    def is_judge_entry(entry: object) -> bool:
+        return isinstance(entry, dict) and {"id", "match_criteria"} <= set(entry)
+
+    return not (criteria and all(is_judge_entry(entry) for entry in criteria))
+
+
+def find_task_rubric(task_path: Path) -> Path | None:
+    """Return the detached-review rubric a task ships, if any."""
+
+    for tests_dir_name in ("verifier", "tests"):
+        candidate = task_path / tests_dir_name / REVIEW_RUBRIC_FILENAME
+        if candidate.is_file() and is_review_rubric_file(candidate):
+            return candidate
+    return None
+
+
+def build_criteria_guidance(rubric: Rubric) -> str:
+    """Render the criterion guidance lines included in the reviewer prompt."""
+
+    if not rubric.is_weighted:
+        return "\n".join(
+            f"- {criterion.name}: {criterion.guidance}" for criterion in rubric.criteria
+        )
+
+    return "\n".join(
+        (
+            f"- {criterion.name} [BLOCKER; answer pass or fail; weight "
+            f"{criterion.weight} does not enter weighted quality]: "
+            f"{criterion.guidance}"
+            if criterion.is_blocker
+            else f"- {criterion.name} [SCORED; weight {criterion.weight}; "
+            f"answer 0, 1, or 2]: {criterion.guidance}"
+        )
+        for criterion in rubric.criteria
+    )
+
+
+def build_review_response_model(rubric: Rubric) -> type[BaseModel]:
+    """Build the structured-output model for one rollout review."""
+
+    checks_fields: dict[str, Any] = {}
+    for criterion in rubric.criteria:
+        check_model: type[BaseModel]
+        if criterion.is_legacy:
+            check_model = CriterionCheck
+        elif criterion.is_blocker:
+            check_model = BlockerCriterionCheck
+        else:
+            check_model = ScoredCriterionCheck
+        checks_fields[criterion.name] = (check_model, ...)
+    extra_policy = "forbid" if rubric.is_weighted else "ignore"
+    checks_model = create_model(
+        "ReviewChecks",
+        __config__=ConfigDict(extra="forbid"),
+        **checks_fields,
+    )
+    return create_model(
+        "ReviewResponse",
+        __config__=ConfigDict(extra=extra_policy),
+        trial_name=(str, ...),
+        summary=(NonBlankText, ...),
+        checks=(checks_model, ...),
+    )

{benchflow-0.7.4 → benchflow-0.7.6.dev2055}/src/benchflow/review/prompts.py

@@ -62,7 +62,21 @@ OUTPUT_TEMPLATE = """When you are done, write your answer as JSON to {result_pat
 
 {output_schema}
 
-"trial_name" must be exactly {trial_name_json}. Every criterion listed in the schema must appear in "checks" with an "outcome" of "pass", "fail", or "not_applicable" and a non-empty "explanation". Write the file and nothing else; do not print the JSON instead of writing it."""
+"trial_name" must be exactly {trial_name_json}. {judgment_contract} Write the file and nothing else; do not print the JSON instead of writing it."""
+
+LEGACY_JUDGMENT_CONTRACT = (
+    'Every criterion listed in the schema must appear in "checks" with an '
+    '"outcome" of "pass", "fail", or "not_applicable" and a non-empty '
+    '"explanation".'
+)
+
+WEIGHTED_JUDGMENT_CONTRACT = (
+    'Every criterion listed in the schema must appear in "checks" with a '
+    'non-empty "explanation". BLOCKER criteria require an "outcome" of '
+    '"pass" or "fail". SCORED criteria require an integer "score" of 0, 1, '
+    "or 2. Do not calculate an aggregate score; Benchflow derives it "
+    "deterministically from the individual judgments and rubric weights."
+)
 
 
 def render_task_section(task_path: str | None) -> str:
@@ -97,6 +111,11 @@ def render_review_instruction(
             "result_path": result_path,
             "trial_name_json": json.dumps(trial_name),
             "output_schema": json.dumps(output_schema or {}, indent=2),
+            "judgment_contract": (
+                WEIGHTED_JUDGMENT_CONTRACT
+                if rubric.is_weighted
+                else LEGACY_JUDGMENT_CONTRACT
+            ),
         }
     )
     return f"{body.rstrip()}\n\n{output.strip()}\n"