harbor-rewardkit 0.1.dev1__tar.gz → 0.1.dev4__tar.gz

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: harbor-rewardkit
-Version: 0.1.dev1
+Version: 0.1.dev4
 Summary: Lightweight grading toolkit for environment-based tasks.
 Keywords: grading,evaluation,rewards,llm,agents,benchmarks
 Author: benediktstroebl

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "harbor-rewardkit"
-version = "0.1.dev1"
+version = "0.1.dev4"
 description = "Lightweight grading toolkit for environment-based tasks."
 readme = "README.md"
 license = "Apache-2.0"

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/csv_cell_equals.py

@@ -1,6 +1,7 @@
 """Criterion: check that a specific CSV cell has the expected value."""
 
 import csv
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -29,5 +30,12 @@ def csv_cell_equals(
                             return r[col] == expected
                         return False
                 return False
-    except (FileNotFoundError, OSError, csv.Error, KeyError, IndexError):
+    except (FileNotFoundError, OSError) as e:
+        if isinstance(e, FileNotFoundError):
+            warnings.warn(
+                f"csv_cell_equals: '{path}' not found in workspace, assigning reward 0",
+                stacklevel=2,
+            )
+        return False
+    except (csv.Error, KeyError, IndexError):
         return False

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/diff_ratio.py

@@ -1,5 +1,6 @@
 """Criterion: fuzzy text comparison returning a similarity ratio [0, 1]."""
 
+import warnings
 from difflib import SequenceMatcher
 from pathlib import Path
 
@@ -11,5 +12,9 @@ def diff_ratio(workspace: Path, path: str, expected: str) -> float:
     try:
         content = (workspace / path).read_text()
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"diff_ratio: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return 0.0
     return SequenceMatcher(None, content.strip(), expected.strip()).ratio()

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/file_contains.py

@@ -1,5 +1,6 @@
 """Criterion: check that a file contains a given string."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -10,4 +11,8 @@ def file_contains(workspace: Path, path: str, text: str) -> bool:
     try:
         return text in (workspace / path).read_text()
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"file_contains: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/file_contains_regex.py

@@ -1,6 +1,7 @@
 """Criterion: check that a file contains text matching a regex pattern."""
 
 import re
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -11,4 +12,8 @@ def file_contains_regex(workspace: Path, path: str, pattern: str) -> bool:
     try:
         return re.search(pattern, (workspace / path).read_text()) is not None
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"file_contains_regex: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/file_matches.py

@@ -1,5 +1,6 @@
 """Criterion: check that a file's content matches an expected string."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,4 +13,8 @@ def file_matches(workspace: Path, path: str, expected: str) -> bool:
     try:
         return (workspace / path).read_text().strip() == expected.strip()
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"file_matches: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/files_equal.py

@@ -1,5 +1,6 @@
 """Criterion: check that two workspace files have equal content."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,4 +13,8 @@ def files_equal(workspace: Path, path1: str, path2: str) -> bool:
             workspace / path2
         ).read_text().strip()
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"files_equal: '{path1}' or '{path2}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/image_similarity.py

@@ -1,5 +1,6 @@
 """Criterion: compare two images pixel-by-pixel and return similarity ratio."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,7 +13,7 @@ def image_similarity(workspace: Path, path1: str, path2: str) -> float:
     except ImportError:
         raise ImportError(
             "image_similarity requires Pillow. "
-            "Install with: pip install rewardkit[image]"
+            "Install with: uv add harbor-rewardkit[image]"
         )
     try:
         with (
@@ -25,8 +26,6 @@ def image_similarity(workspace: Path, path1: str, path2: str) -> float:
             rgba2 = img2.convert("RGBA")
             diff = ImageChops.difference(rgba1, rgba2)
             pixel_count = rgba1.size[0] * rgba1.size[1]
-            # A pixel matches iff all 4 RGBA channels have zero difference.
-            # Use per-channel point() to build a mask in C rather than Python loops.
             r, g, b, a = diff.split()
 
             def zero_fn(x: int) -> int:
@@ -38,5 +37,9 @@ def image_similarity(workspace: Path, path1: str, path2: str) -> float:
             )
             matching = mask.tobytes().count(b"\xff")
             return matching / pixel_count
-    except Exception:
+    except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"image_similarity: '{path1}' or '{path2}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return 0.0

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/image_size_equals.py

@@ -1,5 +1,6 @@
 """Criterion: check that an image has the expected dimensions."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,10 +13,14 @@ def image_size_equals(workspace: Path, path: str, width: int, height: int) -> bo
     except ImportError:
         raise ImportError(
             "image_size_equals requires Pillow. "
-            "Install with: pip install rewardkit[image]"
+            "Install with: uv add harbor-rewardkit[image]"
         )
     try:
         with Image.open(workspace / path) as img:
             return img.size == (width, height)
-    except Exception:
+    except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"image_size_equals: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/json_key_equals.py

@@ -1,6 +1,7 @@
 """Criterion: check that a JSON file has a specific key-value pair."""
 
 import json as _json
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -13,5 +14,11 @@ def json_key_equals(workspace: Path, path: str, key: str, expected: object) -> b
         if not isinstance(data, dict):
             return False
         return data.get(key) == expected
-    except (FileNotFoundError, OSError, ValueError, KeyError):
+    except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"json_key_equals: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
+        return False
+    except (ValueError, KeyError):
         return False

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/json_path_equals.py

@@ -1,6 +1,7 @@
 """Criterion: check a nested JSON path against an expected value."""
 
 import json as _json
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,6 +13,15 @@ def json_path_equals(
 ) -> bool:
     try:
         data = _json.loads((workspace / path).read_text())
+    except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"json_path_equals: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
+        return False
+    except ValueError:
+        return False
+    try:
         for segment in json_path.split("."):
             if isinstance(data, dict):
                 data = data[segment]
@@ -20,5 +30,5 @@ def json_path_equals(
             else:
                 return False
         return data == expected
-    except (FileNotFoundError, OSError, ValueError, KeyError, IndexError, TypeError):
+    except (KeyError, IndexError, TypeError, ValueError):
         return False

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/sqlite_query_equals.py

@@ -1,6 +1,7 @@
 """Criterion: run a SQL query against a SQLite database and check the result."""
 
 import sqlite3
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,6 +13,10 @@ def sqlite_query_equals(
 ) -> bool:
     target = workspace / db_path
     if not target.exists():
+        warnings.warn(
+            f"sqlite_query_equals: '{db_path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False
     try:
         with sqlite3.connect(str(target)) as conn:

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/criteria/xlsx_cell_equals.py

@@ -1,5 +1,6 @@
 """Criterion: check that a specific cell in an xlsx file has the expected value."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -14,15 +15,24 @@ def xlsx_cell_equals(
     except ImportError:
         raise ImportError(
             "xlsx_cell_equals requires openpyxl. "
-            "Install with: pip install rewardkit[office]"
+            "Install with: uv add harbor-rewardkit[office]"
         )
     try:
         wb = openpyxl.load_workbook(
             str(workspace / path), read_only=True, data_only=True
         )
+    except (FileNotFoundError, OSError) as e:
+        if isinstance(e, FileNotFoundError):
+            warnings.warn(
+                f"xlsx_cell_equals: '{path}' not found in workspace, assigning reward 0",
+                stacklevel=2,
+            )
+        return False
+    try:
         ws = wb[sheet] if sheet else wb.active
         value = ws[cell].value
-        wb.close()
         return value == expected
-    except (FileNotFoundError, OSError, KeyError, ValueError):
+    except (KeyError, ValueError):
         return False
+    finally:
+        wb.close()

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/isolation.py

@@ -112,8 +112,8 @@ class _Overlay:
 def isolate(path: Path) -> Generator[Path, None, None]:
     """Yield an overlayfs view of *path*. Writes go to a tmpdir; *path* is untouched."""
     ov = _Overlay(path)
-    ov.mount()
     try:
+        ov.mount()
         yield ov._merged
     finally:
         ov.cleanup()

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/judges.py

@@ -10,6 +10,7 @@ import os
 import re
 import shutil
 import subprocess
+import tempfile
 from importlib import resources
 from pathlib import Path
 from typing import Any
@@ -43,6 +44,28 @@ def _build_criteria_block(criteria: list[Criterion]) -> str:
     return "\n".join(lines)
 
 
+def _build_response_schema(criteria: list[Criterion]) -> dict[str, Any]:
+    """Build a JSON Schema that enforces the expected judge response structure."""
+    props: dict[str, Any] = {}
+    for c in criteria:
+        name = c.name or "criterion"
+        props[name] = {
+            "type": "object",
+            "properties": {
+                "score": c.output_format.json_schema(),
+                "reasoning": {"type": "string"},
+            },
+            "required": ["score", "reasoning"],
+            "additionalProperties": False,
+        }
+    return {
+        "type": "object",
+        "properties": props,
+        "required": list(props.keys()),
+        "additionalProperties": False,
+    }
+
+
 def build_prompt(
     criteria: list[Criterion],
     template: str | None = None,
@@ -129,6 +152,9 @@ def _text_from_blocks(blocks: list[ContentBlock]) -> str:
     return "\n\n".join(b["text"] for b in blocks if b.get("type") == "text")
 
 
+_MAX_JUDGE_RETRIES = 3
+
+
 def parse_judge_response(
     text: str,
     criteria: list[Criterion],
@@ -147,8 +173,13 @@ def parse_judge_response(
     scores: list[Score] = []
     for i, c in enumerate(criteria):
         cname = c.name or f"criterion_{i}"
-        entry = data.get(cname, {})
-        raw_score = entry.get("score", 0)
+        entry = data.get(cname)
+        if not isinstance(entry, dict) or "score" not in entry:
+            raise ValueError(
+                f"Criterion {cname!r}: expected dict with 'score' and 'reasoning', "
+                f"got {type(entry).__name__}: {str(entry)[:100]}"
+            )
+        raw_score = entry["score"]
         reasoning = entry.get("reasoning", "")
         value = c.output_format.normalize(raw_score)
         weight = weights[i] if weights else 1.0
@@ -165,24 +196,6 @@ def parse_judge_response(
     return scores
 
 
-def _fallback_scores(
-    criteria: list[Criterion],
-    weights: list[float] | None,
-    error: str,
-) -> list[Score]:
-    return [
-        Score(
-            name=c.name or f"criterion_{i}",
-            value=0.0,
-            raw=False,
-            weight=weights[i] if weights else 1.0,
-            error=error,
-            description=c.description,
-        )
-        for i, c in enumerate(criteria)
-    ]
-
-
 async def arun_llm(
     judge: LLMJudge,
     criteria: list[Criterion],
@@ -220,6 +233,14 @@ async def arun_llm(
         )
         available_tokens = max_input_tokens - prompt_tokens - user_tokens - 32_000
 
+        if available_tokens <= 0:
+            raise ValueError(
+                f"Trajectory too large to include in judge prompt: "
+                f"no token budget remaining "
+                f"(prompt={prompt_tokens}, user={user_tokens}, "
+                f"limit={max_input_tokens})."
+            )
+
         traj_text = format_trajectory(
             judge.atif_trajectory,
             max_tokens=available_tokens,
@@ -231,23 +252,34 @@ async def arun_llm(
     messages: list[dict[str, Any]] = [{"role": "system", "content": prompt}]
     if user_blocks:
         messages.append({"role": "user", "content": user_blocks})
-    resp = await litellm.acompletion(
-        model=judge.model,
-        messages=messages,
-        response_format={"type": "json_object"},
-        max_tokens=4096,
-        timeout=judge.timeout,
-        reasoning_effort=judge.reasoning_effort,
-    )
-    raw_output = resp.choices[0].message.content
-    try:
-        scores = parse_judge_response(raw_output, criteria, weights)
-    except (ValueError, json.JSONDecodeError) as e:
-        warn_list.append(f"Judge response parse error: {e}")
-        scores = _fallback_scores(
-            criteria, weights, f"Failed to parse judge response: {e}"
+    for attempt in range(_MAX_JUDGE_RETRIES):
+        resp = await litellm.acompletion(
+            model=judge.model,
+            messages=messages,
+            response_format={
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "judge_response",
+                    "schema": _build_response_schema(criteria),
+                    "strict": True,
+                },
+            },
+            timeout=judge.timeout,
+            reasoning_effort=judge.reasoning_effort,
         )
-    return scores, raw_output, warn_list
+        raw_output = resp.choices[0].message.content
+        try:
+            scores = parse_judge_response(raw_output, criteria, weights)
+            return scores, raw_output, warn_list
+        except ValueError:
+            if attempt == _MAX_JUDGE_RETRIES - 1:
+                raise
+            logger.debug(
+                "Judge response did not match schema, retrying (%d/%d)",
+                attempt + 1,
+                _MAX_JUDGE_RETRIES,
+            )
+    raise RuntimeError("Unreachable")
 
 
 def _is_alpine() -> bool:
@@ -315,51 +347,89 @@ async def arun_agent(
         prompt = build_prompt(criteria, kind="agent")
     if judge.atif_trajectory:
         prompt += f"\n\nThe agent's trajectory is stored at: {judge.atif_trajectory}"
+    schema = _build_response_schema(criteria)
+    schema_path: str | None = None
     if judge.agent == "claude-code":
-        cmd = ["claude", "-p", prompt, "--output-format", "json"]
+        cmd = [
+            "claude",
+            "-p",
+            prompt,
+            "--output-format",
+            "json",
+            "--json-schema",
+            json.dumps(schema),
+        ]
         cmd_name = "claude"
     else:
-        cmd = ["codex", "exec", prompt]
+        fd, schema_path = tempfile.mkstemp(suffix=".json")
+        with os.fdopen(fd, "w") as f:
+            json.dump(schema, f)
+        cmd = ["codex", "exec", prompt, "--output-schema", schema_path]
         cmd_name = "codex"
 
     if judge.model:
+        model_name = judge.model
+        # Claude CLI uses bare model names (e.g. "claude-haiku-4-5"),
+        # not provider-prefixed ones (e.g. "anthropic/claude-haiku-4-5").
+        if judge.agent == "claude-code" and model_name.startswith("anthropic/"):
+            model_name = model_name.removeprefix("anthropic/")
         flag = "-m" if judge.agent == "codex" else "--model"
-        cmd.extend([flag, judge.model])
+        cmd.extend([flag, model_name])
 
     _ensure_cli(cmd_name)
     cwd = judge.cwd or (
         str(workspace) if workspace and Path(workspace).is_dir() else None
     )
-    proc = await asyncio.create_subprocess_exec(
-        *cmd,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-        cwd=cwd,
-    )
-    try:
-        stdout, _stderr = await asyncio.wait_for(
-            proc.communicate(), timeout=judge.timeout
-        )
-    except asyncio.TimeoutError:
-        proc.kill()
-        await proc.communicate()
-        raise
-    raw_output = stdout.decode()
-    # Claude CLI with --output-format json wraps the actual response in a
-    # JSON envelope with a "result" field. Extract the inner text so
-    # parse_judge_response finds the scoring JSON, not the wrapper.
-    if judge.agent == "claude-code":
-        try:
-            envelope = json.loads(raw_output)
-            if isinstance(envelope, dict) and "result" in envelope:
-                raw_output = envelope["result"]
-        except (json.JSONDecodeError, TypeError):
-            pass
+
     try:
-        scores = parse_judge_response(raw_output, criteria, weights)
-    except (ValueError, json.JSONDecodeError) as e:
-        warn_list.append(f"Agent judge response parse error: {e}")
-        scores = _fallback_scores(
-            criteria, weights, f"Failed to parse agent judge response: {e}"
-        )
-    return scores, raw_output, warn_list
+        for attempt in range(_MAX_JUDGE_RETRIES):
+            proc = await asyncio.create_subprocess_exec(
+                *cmd,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                cwd=cwd,
+            )
+            try:
+                stdout, _stderr = await asyncio.wait_for(
+                    proc.communicate(), timeout=judge.timeout
+                )
+            except asyncio.TimeoutError:
+                proc.kill()
+                await proc.communicate()
+                raise
+            raw_output = stdout.decode()
+            if proc.returncode != 0:
+                stderr_text = _stderr.decode().strip() if _stderr else ""
+                raise ValueError(
+                    f"Agent CLI '{cmd_name}' exited with code {proc.returncode}: "
+                    f"{stderr_text or raw_output[:200]}"
+                )
+            # Claude CLI with --output-format json and --json-schema wraps the
+            # structured response in an envelope with a "structured_output" field.
+            if judge.agent == "claude-code":
+                try:
+                    envelope = json.loads(raw_output)
+                    if isinstance(envelope, dict):
+                        if envelope.get("is_error"):
+                            raise ValueError(
+                                f"Claude CLI returned an error: {envelope.get('result', raw_output[:200])}"
+                            )
+                        if "structured_output" in envelope:
+                            raw_output = json.dumps(envelope["structured_output"])
+                except (json.JSONDecodeError, TypeError):
+                    pass
+            try:
+                scores = parse_judge_response(raw_output, criteria, weights)
+                return scores, raw_output, warn_list
+            except ValueError:
+                if attempt == _MAX_JUDGE_RETRIES - 1:
+                    raise
+                logger.debug(
+                    "Agent judge response did not match schema, retrying (%d/%d)",
+                    attempt + 1,
+                    _MAX_JUDGE_RETRIES,
+                )
+        raise RuntimeError("Unreachable")
+    finally:
+        if schema_path:
+            Path(schema_path).unlink(missing_ok=True)

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/models.py

@@ -14,6 +14,7 @@ Aggregation = Literal["weighted_mean", "all_pass", "any_pass", "threshold"]
 class OutputFormat(Protocol):
     def normalize(self, raw: float | bool | str) -> float: ...
     def prompt_fragment(self) -> str: ...
+    def json_schema(self) -> dict[str, Any]: ...
 
 
 class Binary(BaseModel):
@@ -29,6 +30,9 @@ class Binary(BaseModel):
     def prompt_fragment(self) -> str:
         return '"yes" or "no"'
 
+    def json_schema(self) -> dict[str, Any]:
+        return {"type": "string", "enum": ["yes", "no"]}
+
 
 class Likert(BaseModel):
     model_config = ConfigDict(frozen=True)
@@ -43,6 +47,9 @@ class Likert(BaseModel):
     def prompt_fragment(self) -> str:
         return f"an integer from 1 to {self.points}"
 
+    def json_schema(self) -> dict[str, Any]:
+        return {"type": "integer"}
+
 
 class Numeric(BaseModel):
     model_config = ConfigDict(frozen=True)
@@ -59,6 +66,9 @@ class Numeric(BaseModel):
     def prompt_fragment(self) -> str:
         return f"a number from {self.min} to {self.max}"
 
+    def json_schema(self) -> dict[str, Any]:
+        return {"type": "number"}
+
 
 def _slugify(text: str) -> str:
     slug = re.sub(r"[^a-z0-9]+", "_", text[:40].lower())

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/reward.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import inspect
+import warnings
 from contextlib import nullcontext
 from pathlib import Path
 from typing import Any, Awaitable, TypeVar
@@ -102,9 +103,24 @@ class Reward:
             if isinstance(raw, bool):
                 value = 1.0 if raw else 0.0
             elif isinstance(raw, (int, float)):
-                value = max(0.0, min(1.0, float(raw)))
+                value = float(raw)
+                if value > 1.0:
+                    warnings.warn(
+                        f"Criterion {fn_name!r} returned {value:.4f} which exceeds 1.0; "
+                        f"score will not be clamped — verify your criterion logic.",
+                        stacklevel=2,
+                    )
+                elif value < 0.0:
+                    warnings.warn(
+                        f"Criterion {fn_name!r} returned {value:.4f} which is below 0.0; "
+                        f"score will not be clamped — verify your criterion logic.",
+                        stacklevel=2,
+                    )
             else:
-                value = 1.0 if raw else 0.0
+                raise TypeError(
+                    f"Criterion {fn_name!r} returned {type(raw).__name__}, "
+                    f"expected bool, int, or float."
+                )
 
             return Score(
                 name=fn_name,
@@ -113,15 +129,13 @@ class Reward:
                 weight=weight,
                 description=description,
             )
+        except ImportError as e:
+            raise ImportError(
+                f"Criterion {fn_name!r} failed due to missing dependency: {e}. "
+                f"Install the required extra (e.g. uv add harbor-rewardkit[all])."
+            ) from e
         except Exception as e:
-            return Score(
-                name=fn_name,
-                value=0.0,
-                raw=False,
-                weight=weight,
-                error=str(e),
-                description=description,
-            )
+            raise RuntimeError(f"Criterion {fn_name!r} failed: {e}") from e
 
     def _run_one(self, i: int, fn: Any) -> Score:
         """Run a single criterion, with isolation if configured."""
@@ -132,6 +146,16 @@ class Reward:
         return self._eval_criterion(i, fn, self.workspace)
 
     async def arun(self, sem: asyncio.Semaphore | None = None) -> list[Score]:
+        try:
+            return await self._arun_inner(sem)
+        except ExceptionGroup as eg:
+            # Unwrap TaskGroup ExceptionGroup to surface the first real error.
+            first = eg.exceptions[0]
+            raise RuntimeError(f"Reward {self.name!r} failed: {first}") from first
+        except Exception as e:
+            raise RuntimeError(f"Reward {self.name!r} failed: {e}") from e
+
+    async def _arun_inner(self, sem: asyncio.Semaphore | None) -> list[Score]:
         if self.judge is None:
             tasks: list[asyncio.Task[Score]] = []
             async with asyncio.TaskGroup() as tg:

{harbor_rewardkit-0.1.dev1 → harbor_rewardkit-0.1.dev4}/src/rewardkit/runner.py

@@ -18,6 +18,7 @@ from rewardkit.models import (
     Criterion,
     LLMJudge,
     Likert,
+    Numeric,
 )
 from rewardkit.reward import Reward
 from rewardkit.session import Session, _builtin_names, _factory_registry, set_current
@@ -28,6 +29,13 @@ def _load_toml(path: Path) -> dict[str, Any]:
 
 
 def _import_py_file(path: Path) -> None:
+    """Import a Python file as a module, caching by file-path hash.
+
+    Once imported, subsequent calls with the same resolved path are
+    no-ops.  This is intentional for the primary single-run container
+    use case but means repeated ``discover()`` or ``run()`` calls in a
+    REPL or notebook will not re-execute already-loaded criterion files.
+    """
     import hashlib
 
     digest = hashlib.sha1(str(path.resolve()).encode()).hexdigest()[:12]
@@ -48,6 +56,11 @@ def _build_criteria_from_toml(toml_criteria: list[dict[str, Any]]) -> list[Crite
         fmt_name = c.get("type", "binary")
         if fmt_name == "likert":
             output_format = Likert(points=c.get("points", 5))
+        elif fmt_name == "numeric":
+            output_format = Numeric(
+                min=c.get("min", 0.0),
+                max=c.get("max", 1.0),
+            )
         else:
             output_format = Binary()
         criteria.append(
@@ -372,12 +385,23 @@ def run_multi(
     to stdout for overlapping reward names.
     """
     all_rewards: list[Reward] = []
-    dir_labels: list[str] = []
+    dir_labels = [Path(d).name for d in tests_dirs]
+    if len(dir_labels) != len(set(dir_labels)):
+        dupes = {name for name in dir_labels if dir_labels.count(name) > 1}
+        paths_by_label = {
+            name: [str(d) for d, n in zip(tests_dirs, dir_labels) if n == name]
+            for name in dupes
+        }
+        raise ValueError(
+            "Duplicate test directory basenames: "
+            + ", ".join(
+                f"{name!r} ({', '.join(ps)})" for name, ps in paths_by_label.items()
+            )
+            + ". Use directories with distinct basenames."
+        )
     dir_reward_ranges: list[tuple[int, int]] = []  # (start, end) indices
 
     for tests_dir in tests_dirs:
-        label = Path(tests_dir).name
-        dir_labels.append(label)
         rewards = discover(tests_dir, workspace=workspace)
         start = len(all_rewards)
         all_rewards.extend(rewards)