harbor-rewardkit 0.1.dev0__tar.gz → 0.1.dev3__tar.gz

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: harbor-rewardkit
-Version: 0.1.dev0
+Version: 0.1.dev3
 Summary: Lightweight grading toolkit for environment-based tasks.
 Keywords: grading,evaluation,rewards,llm,agents,benchmarks
 Author: benediktstroebl
@@ -35,7 +35,7 @@ The Harbor Rewardkit is a lightweight package to define and run verifiers. Rewar
 ## Installation
 
 ```bash
-uv tool install harbor-reward-kit
+uv tool install harbor-rewardkit
 ```
 
 ## Example: Programmatic criteria
@@ -67,7 +67,7 @@ Add rewardkit to your `test.sh` file:
 
 ```bash
 # tests/test.sh
-uvx harbor-reward-kit@0.1 /tests
+uvx harbor-rewardkit@0.1 /tests
 ```
 
 See the [documentation](https://harborframework.com/docs/rewardkit) and a full [working example](https://github.com/harbor-framework/harbor/tree/main/examples/tasks/reward-kit-example).

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/README.md

@@ -8,7 +8,7 @@ The Harbor Rewardkit is a lightweight package to define and run verifiers. Rewar
 ## Installation
 
 ```bash
-uv tool install harbor-reward-kit
+uv tool install harbor-rewardkit
 ```
 
 ## Example: Programmatic criteria
@@ -40,7 +40,7 @@ Add rewardkit to your `test.sh` file:
 
 ```bash
 # tests/test.sh
-uvx harbor-reward-kit@0.1 /tests
+uvx harbor-rewardkit@0.1 /tests
 ```
 
 See the [documentation](https://harborframework.com/docs/rewardkit) and a full [working example](https://github.com/harbor-framework/harbor/tree/main/examples/tasks/reward-kit-example).

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "harbor-rewardkit"
-version = "0.1.dev0"
+version = "0.1.dev3"
 description = "Lightweight grading toolkit for environment-based tasks."
 readme = "README.md"
 license = "Apache-2.0"

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/csv_cell_equals.py

@@ -1,6 +1,7 @@
 """Criterion: check that a specific CSV cell has the expected value."""
 
 import csv
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -29,5 +30,12 @@ def csv_cell_equals(
                             return r[col] == expected
                         return False
                 return False
-    except (FileNotFoundError, OSError, csv.Error, KeyError, IndexError):
+    except (FileNotFoundError, OSError) as e:
+        if isinstance(e, FileNotFoundError):
+            warnings.warn(
+                f"csv_cell_equals: '{path}' not found in workspace, assigning reward 0",
+                stacklevel=2,
+            )
+        return False
+    except (csv.Error, KeyError, IndexError):
         return False

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/diff_ratio.py

@@ -1,5 +1,6 @@
 """Criterion: fuzzy text comparison returning a similarity ratio [0, 1]."""
 
+import warnings
 from difflib import SequenceMatcher
 from pathlib import Path
 
@@ -11,5 +12,9 @@ def diff_ratio(workspace: Path, path: str, expected: str) -> float:
     try:
         content = (workspace / path).read_text()
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"diff_ratio: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return 0.0
     return SequenceMatcher(None, content.strip(), expected.strip()).ratio()

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/file_contains.py

@@ -1,5 +1,6 @@
 """Criterion: check that a file contains a given string."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -10,4 +11,8 @@ def file_contains(workspace: Path, path: str, text: str) -> bool:
     try:
         return text in (workspace / path).read_text()
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"file_contains: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/file_contains_regex.py

@@ -1,6 +1,7 @@
 """Criterion: check that a file contains text matching a regex pattern."""
 
 import re
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -11,4 +12,8 @@ def file_contains_regex(workspace: Path, path: str, pattern: str) -> bool:
     try:
         return re.search(pattern, (workspace / path).read_text()) is not None
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"file_contains_regex: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/file_matches.py

@@ -1,5 +1,6 @@
 """Criterion: check that a file's content matches an expected string."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,4 +13,8 @@ def file_matches(workspace: Path, path: str, expected: str) -> bool:
     try:
         return (workspace / path).read_text().strip() == expected.strip()
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"file_matches: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/files_equal.py

@@ -1,5 +1,6 @@
 """Criterion: check that two workspace files have equal content."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,4 +13,8 @@ def files_equal(workspace: Path, path1: str, path2: str) -> bool:
             workspace / path2
         ).read_text().strip()
     except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"files_equal: '{path1}' or '{path2}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/image_similarity.py

@@ -1,5 +1,6 @@
 """Criterion: compare two images pixel-by-pixel and return similarity ratio."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,7 +13,7 @@ def image_similarity(workspace: Path, path1: str, path2: str) -> float:
     except ImportError:
         raise ImportError(
             "image_similarity requires Pillow. "
-            "Install with: pip install rewardkit[image]"
+            "Install with: uv add harbor-rewardkit[image]"
         )
     try:
         with (
@@ -25,8 +26,6 @@ def image_similarity(workspace: Path, path1: str, path2: str) -> float:
             rgba2 = img2.convert("RGBA")
             diff = ImageChops.difference(rgba1, rgba2)
             pixel_count = rgba1.size[0] * rgba1.size[1]
-            # A pixel matches iff all 4 RGBA channels have zero difference.
-            # Use per-channel point() to build a mask in C rather than Python loops.
             r, g, b, a = diff.split()
 
             def zero_fn(x: int) -> int:
@@ -38,5 +37,9 @@ def image_similarity(workspace: Path, path1: str, path2: str) -> float:
             )
             matching = mask.tobytes().count(b"\xff")
             return matching / pixel_count
-    except Exception:
+    except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"image_similarity: '{path1}' or '{path2}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return 0.0

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/image_size_equals.py

@@ -1,5 +1,6 @@
 """Criterion: check that an image has the expected dimensions."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,10 +13,14 @@ def image_size_equals(workspace: Path, path: str, width: int, height: int) -> bo
     except ImportError:
         raise ImportError(
             "image_size_equals requires Pillow. "
-            "Install with: pip install rewardkit[image]"
+            "Install with: uv add harbor-rewardkit[image]"
         )
     try:
         with Image.open(workspace / path) as img:
             return img.size == (width, height)
-    except Exception:
+    except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"image_size_equals: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/json_key_equals.py

@@ -1,6 +1,7 @@
 """Criterion: check that a JSON file has a specific key-value pair."""
 
 import json as _json
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -13,5 +14,11 @@ def json_key_equals(workspace: Path, path: str, key: str, expected: object) -> b
         if not isinstance(data, dict):
             return False
         return data.get(key) == expected
-    except (FileNotFoundError, OSError, ValueError, KeyError):
+    except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"json_key_equals: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
+        return False
+    except (ValueError, KeyError):
         return False

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/json_path_equals.py

@@ -1,6 +1,7 @@
 """Criterion: check a nested JSON path against an expected value."""
 
 import json as _json
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,6 +13,15 @@ def json_path_equals(
 ) -> bool:
     try:
         data = _json.loads((workspace / path).read_text())
+    except (FileNotFoundError, OSError):
+        warnings.warn(
+            f"json_path_equals: '{path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
+        return False
+    except ValueError:
+        return False
+    try:
         for segment in json_path.split("."):
             if isinstance(data, dict):
                 data = data[segment]
@@ -20,5 +30,5 @@ def json_path_equals(
             else:
                 return False
         return data == expected
-    except (FileNotFoundError, OSError, ValueError, KeyError, IndexError, TypeError):
+    except (KeyError, IndexError, TypeError, ValueError):
         return False

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/sqlite_query_equals.py

@@ -1,6 +1,7 @@
 """Criterion: run a SQL query against a SQLite database and check the result."""
 
 import sqlite3
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -12,6 +13,10 @@ def sqlite_query_equals(
 ) -> bool:
     target = workspace / db_path
     if not target.exists():
+        warnings.warn(
+            f"sqlite_query_equals: '{db_path}' not found in workspace, assigning reward 0",
+            stacklevel=2,
+        )
         return False
     try:
         with sqlite3.connect(str(target)) as conn:

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/criteria/xlsx_cell_equals.py

@@ -1,5 +1,6 @@
 """Criterion: check that a specific cell in an xlsx file has the expected value."""
 
+import warnings
 from pathlib import Path
 
 from rewardkit.session import criterion
@@ -14,7 +15,7 @@ def xlsx_cell_equals(
     except ImportError:
         raise ImportError(
             "xlsx_cell_equals requires openpyxl. "
-            "Install with: pip install rewardkit[office]"
+            "Install with: uv add harbor-rewardkit[office]"
         )
     try:
         wb = openpyxl.load_workbook(
@@ -24,5 +25,12 @@ def xlsx_cell_equals(
         value = ws[cell].value
         wb.close()
         return value == expected
-    except (FileNotFoundError, OSError, KeyError, ValueError):
+    except (FileNotFoundError, OSError) as e:
+        if isinstance(e, FileNotFoundError):
+            warnings.warn(
+                f"xlsx_cell_equals: '{path}' not found in workspace, assigning reward 0",
+                stacklevel=2,
+            )
+        return False
+    except (KeyError, ValueError):
         return False

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/judges.py

@@ -165,24 +165,6 @@ def parse_judge_response(
     return scores
 
 
-def _fallback_scores(
-    criteria: list[Criterion],
-    weights: list[float] | None,
-    error: str,
-) -> list[Score]:
-    return [
-        Score(
-            name=c.name or f"criterion_{i}",
-            value=0.0,
-            raw=False,
-            weight=weights[i] if weights else 1.0,
-            error=error,
-            description=c.description,
-        )
-        for i, c in enumerate(criteria)
-    ]
-
-
 async def arun_llm(
     judge: LLMJudge,
     criteria: list[Criterion],
@@ -240,13 +222,7 @@ async def arun_llm(
         reasoning_effort=judge.reasoning_effort,
     )
     raw_output = resp.choices[0].message.content
-    try:
-        scores = parse_judge_response(raw_output, criteria, weights)
-    except (ValueError, json.JSONDecodeError) as e:
-        warn_list.append(f"Judge response parse error: {e}")
-        scores = _fallback_scores(
-            criteria, weights, f"Failed to parse judge response: {e}"
-        )
+    scores = parse_judge_response(raw_output, criteria, weights)
     return scores, raw_output, warn_list
 
 
@@ -319,11 +295,12 @@ async def arun_agent(
         cmd = ["claude", "-p", prompt, "--output-format", "json"]
         cmd_name = "claude"
     else:
-        cmd = ["codex", "-q", prompt]
+        cmd = ["codex", "exec", prompt]
         cmd_name = "codex"
 
     if judge.model:
-        cmd.extend(["--model", judge.model])
+        flag = "-m" if judge.agent == "codex" else "--model"
+        cmd.extend([flag, judge.model])
 
     _ensure_cli(cmd_name)
     cwd = judge.cwd or (
@@ -344,11 +321,15 @@ async def arun_agent(
         await proc.communicate()
         raise
     raw_output = stdout.decode()
-    try:
-        scores = parse_judge_response(raw_output, criteria, weights)
-    except (ValueError, json.JSONDecodeError) as e:
-        warn_list.append(f"Agent judge response parse error: {e}")
-        scores = _fallback_scores(
-            criteria, weights, f"Failed to parse agent judge response: {e}"
-        )
+    # Claude CLI with --output-format json wraps the actual response in a
+    # JSON envelope with a "result" field. Extract the inner text so
+    # parse_judge_response finds the scoring JSON, not the wrapper.
+    if judge.agent == "claude-code":
+        try:
+            envelope = json.loads(raw_output)
+            if isinstance(envelope, dict) and "result" in envelope:
+                raw_output = envelope["result"]
+        except (json.JSONDecodeError, TypeError):
+            pass
+    scores = parse_judge_response(raw_output, criteria, weights)
     return scores, raw_output, warn_list

{harbor_rewardkit-0.1.dev0 → harbor_rewardkit-0.1.dev3}/src/rewardkit/reward.py

@@ -113,15 +113,13 @@ class Reward:
                 weight=weight,
                 description=description,
             )
+        except ImportError as e:
+            raise ImportError(
+                f"Criterion {fn_name!r} failed due to missing dependency: {e}. "
+                f"Install the required extra (e.g. uv add harbor-rewardkit[all])."
+            ) from e
         except Exception as e:
-            return Score(
-                name=fn_name,
-                value=0.0,
-                raw=False,
-                weight=weight,
-                error=str(e),
-                description=description,
-            )
+            raise RuntimeError(f"Criterion {fn_name!r} failed: {e}") from e
 
     def _run_one(self, i: int, fn: Any) -> Score:
         """Run a single criterion, with isolation if configured."""
@@ -132,6 +130,16 @@ class Reward:
         return self._eval_criterion(i, fn, self.workspace)
 
     async def arun(self, sem: asyncio.Semaphore | None = None) -> list[Score]:
+        try:
+            return await self._arun_inner(sem)
+        except ExceptionGroup as eg:
+            # Unwrap TaskGroup ExceptionGroup to surface the first real error.
+            first = eg.exceptions[0]
+            raise RuntimeError(f"Reward {self.name!r} failed: {first}") from first
+        except Exception as e:
+            raise RuntimeError(f"Reward {self.name!r} failed: {e}") from e
+
+    async def _arun_inner(self, sem: asyncio.Semaphore | None) -> list[Score]:
         if self.judge is None:
             tasks: list[asyncio.Task[Score]] = []
             async with asyncio.TaskGroup() as tg: