graded 1.0.0__py3-none-any.whl

graded/__init__.py

@@ -0,0 +1,4 @@
+from graded.evaluator import Evaluator
+from graded.types import Criterion, Trajectory, ToolCall, Step
+
+__all__ = ["Evaluator", "Criterion", "Trajectory", "ToolCall", "Step"]

graded/evaluator.py

@@ -0,0 +1,329 @@
+import os
+import json
+import shutil
+import logging
+from pathlib import Path
+from typing import Callable, Any, Dict, List, Optional, Type, Union
+from pydantic import BaseModel
+
+from graded.types import Criterion, Trajectory
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s"
+)
+
+
+class Evaluator:
+    def __init__(
+        self,
+        workspace: Union[str, Path] = "/workspace",
+        output_path: Union[str, Path] = "/logs/verifier/reward.json",
+        auto_save_artifacts: bool = True,
+        artifacts_dir: Optional[Union[str, Path]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ):
+        self.workspace = Path(workspace)
+        self.output_path = Path(output_path)
+        self.auto_save_artifacts = auto_save_artifacts
+        self.artifacts_dir = (
+            Path(artifacts_dir)
+            if artifacts_dir
+            else self.output_path.parent / "artifacts"
+        )
+        self.metadata: Dict[str, Any] = metadata or {}
+        self.criteria: List[Criterion] = []
+        self.scores: Dict[str, float] = {}
+        self.traces: List[Dict[str, Any]] = []
+
+    def criterion(self, name: str, weight: float = 1.0, fatal: bool = False):
+        """Decorator to declare a grading criterion.
+
+        Args:
+            name: Name of the criterion.
+            weight: Relative weight for scoring.
+            fatal: If True, a score of 0.0 short-circuits the entire evaluation to 0.0.
+        """
+
+        def decorator(func: Callable[[Path], Any]):
+            if any(c.name == name for c in self.criteria):
+                raise ValueError(f"Duplicate criterion name: '{name}'")
+            self.criteria.append(
+                Criterion(name=name, weight=weight, fatal=fatal, func=func)
+            )
+            return func
+
+        return decorator
+
+    def _save_artifact(self, filename: str, content: str) -> None:
+        """Internal helper to save content to the artifacts directory."""
+        try:
+            dest = self.artifacts_dir / filename
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            dest.write_text(content, encoding="utf-8")
+        except Exception as e:
+            logging.error(f"Failed to save artifact {filename}: {e}")
+
+    def save_file(self, filename: str, content: str) -> None:
+        """Explicitly save content to the artifacts directory."""
+        self._save_artifact(filename, content)
+
+    def save_dir(self, dirname: str) -> None:
+        """Copy an entire directory from the workspace to the artifacts directory."""
+        src = self.workspace / dirname
+        dest = self.artifacts_dir / dirname
+        if not src.is_dir():
+            logging.warning(f"Directory {dirname} not found in workspace.")
+            return
+        try:
+            if dest.exists():
+                shutil.rmtree(dest)
+            shutil.copytree(src, dest)
+        except Exception as e:
+            logging.error(f"Failed to save directory artifact {dirname}: {e}")
+
+    def load_json(
+        self, filename: str, save_artifact: Optional[bool] = None
+    ) -> Optional[Any]:
+        """Safely loads and parses JSON from the workspace.
+
+        Args:
+            filename: Path relative to the workspace.
+            save_artifact: Whether to save a copy to the artifacts directory.
+                Defaults to the instance-level auto_save_artifacts setting.
+        """
+        path = self.workspace / filename
+        if not path.exists():
+            logging.warning(f"File {filename} not found in workspace.")
+            return None
+        try:
+            raw = path.read_text(encoding="utf-8")
+            should_save = (
+                save_artifact if save_artifact is not None else self.auto_save_artifacts
+            )
+            if should_save:
+                self._save_artifact(filename, raw)
+            return json.loads(raw)
+        except Exception as e:
+            logging.error(f"Error parsing JSON file {filename}: {e}")
+            return None
+
+    def read_file(
+        self, filename: str, save_artifact: Optional[bool] = None
+    ) -> Optional[str]:
+        """Safely reads file content from the workspace.
+
+        Args:
+            filename: Path relative to the workspace.
+            save_artifact: Whether to save a copy to the artifacts directory.
+                Defaults to the instance-level auto_save_artifacts setting.
+        """
+        path = self.workspace / filename
+        if not path.exists():
+            logging.warning(f"File {filename} not found in workspace.")
+            return None
+        try:
+            content = path.read_text(encoding="utf-8")
+            should_save = (
+                save_artifact if save_artifact is not None else self.auto_save_artifacts
+            )
+            if should_save:
+                self._save_artifact(filename, content)
+            return content
+        except Exception as e:
+            logging.error(f"Error reading file {filename}: {e}")
+            return None
+
+    def load_trajectory(
+        self, path: str = "/logs/agent/trajectory.json"
+    ) -> Optional[Trajectory]:
+        """Load and parse the ATIF trajectory written by the agent.
+
+        Args:
+            path: Absolute path to the trajectory JSON file.
+                  Defaults to ``/logs/agent/trajectory.json``.
+
+        Returns:
+            A typed :class:`Trajectory` on success, or ``None`` if the file is
+            missing or unparseable (a warning is logged in either case).
+        """
+        traj_path = Path(path)
+        if not traj_path.exists():
+            logging.warning(f"Trajectory file not found: {path}")
+            return None
+        try:
+            return Trajectory.model_validate_json(traj_path.read_text(encoding="utf-8"))
+        except Exception as e:
+            logging.error(f"Failed to parse trajectory at {path}: {e}")
+            return None
+
+    def file_exists(self, filename: str) -> bool:
+        """Checks if a file exists in the workspace."""
+        path = self.workspace / filename
+        return path.is_file()
+
+    def dir_exists(self, dirname: str) -> bool:
+        """Checks if a directory exists in the workspace."""
+        path = self.workspace / dirname
+        return path.is_dir()
+
+    def llm_judge(
+        self,
+        response_model: Type[BaseModel],
+        system: str,
+        prompt: str,
+        model: str,
+        client: Optional[Any] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ) -> Any:
+        """Call instructor LLM judge with structured responses and trace the call.
+
+        Args:
+            response_model: Pydantic model for structured output.
+            system: System prompt text.
+            prompt: User prompt text.
+            model: Model identifier (e.g. "google/gemini-3.1-flash-lite").
+            client: Optional pre-configured instructor client.
+            metadata: Optional per-call metadata dict. Merged with evaluator-level
+                metadata for experiment tracking.
+            **kwargs: Additional arguments passed to client.create().
+        """
+        import instructor
+
+        call_metadata = metadata or {}
+
+        # Merge metadata: evaluator-level -> per-call
+        merged_metadata = {
+            **self.metadata,
+            **call_metadata,
+        }
+
+        trace_data = {
+            "model": model,
+            "system": system,
+            "prompt": prompt,
+            "kwargs": {k: repr(v) for k, v in kwargs.items()},
+            "response_model_schema": response_model.model_json_schema(),
+            "metadata": merged_metadata,
+        }
+
+        try:
+            if client is None:
+                client = instructor.from_provider(model=model)
+
+            model_name = model.split("/")[-1]
+            result = client.create(
+                model=model_name,
+                response_model=response_model,
+                messages=[
+                    {"role": "system", "content": system},
+                    {"role": "user", "content": prompt},
+                ],
+                **kwargs,
+            )
+            # Record trace on success
+            success_trace = {
+                **trace_data,
+                "response": result.model_dump(),
+                "status": "success",
+            }
+            self.traces.append(success_trace)
+            return result
+        except Exception as e:
+            # Record trace on failure
+            failed_trace = {
+                **trace_data,
+                "error": str(e),
+                "status": "failed",
+            }
+            self.traces.append(failed_trace)
+            raise e
+
+    def _score_criterion(self, crit: Criterion) -> float:
+        """Run a single criterion and coerce its result to a float score.
+
+        A crash inside the criterion is caught and scored 0.0. A return value
+        that is not ``bool | int | float`` raises ``ValueError`` (a likely
+        forgotten ``return``).
+        """
+        try:
+            res = crit.func(self.workspace)
+        except Exception as e:
+            logging.error(
+                f"Failed executing criterion '{crit.name}': {e}", exc_info=True
+            )
+            print(
+                f"CRITERION: {crit.name} (weight={crit.weight}) -> FAILED (Score: 0.0)"
+            )
+            return 0.0
+
+        if not isinstance(res, (bool, int, float)):
+            raise ValueError(
+                f"Criterion '{crit.name}' must return bool | int | float, "
+                f"got {type(res).__name__}. Did you forget a return?"
+            )
+
+        score = float(res)  # float(True) == 1.0, float(False) == 0.0
+        print(f"CRITERION: {crit.name} (weight={crit.weight}) -> Score: {score}")
+        return score
+
+    def run(self):
+        """Executes all criteria, aggregates weighted scores, and writes outputs."""
+        total_weight = 0.0
+        weighted_score = 0.0
+
+        print("=== Start Evaluation ===")
+        for crit in self.criteria:
+            total_weight += crit.weight
+            score = self._score_criterion(crit)
+            self.scores[crit.name] = score
+            weighted_score += score * crit.weight
+
+            if crit.fatal and score == 0.0:
+                print(
+                    f"FATAL: Criterion '{crit.name}' failed. Short-circuiting to 0.0."
+                )
+                self._write_outputs(0.0)
+                return
+
+        final_reward = (weighted_score / total_weight) if total_weight > 0 else 0.0
+        self._write_outputs(final_reward)
+
+    def _write_outputs(self, final_reward: float):
+        """Write all output files (reward, traces)."""
+        print(f"Final Computed Reward: {final_reward:.4f}")
+
+        # Ensure output directories exist
+        self.output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write legacy reward.txt format
+        reward_txt = self.output_path.with_name("reward.txt")
+        try:
+            reward_txt.write_text(f"{final_reward:.4f}\n")
+        except Exception as e:
+            logging.error(f"Failed to write reward.txt: {e}")
+
+        # Write structured reward.json — flat dict[str, float|int] for Harbor compatibility
+        # Each criterion score is a top-level key alongside 'reward'
+        output_data = {"reward": final_reward, **self.scores}
+        try:
+            self.output_path.write_text(json.dumps(output_data, indent=2))
+        except Exception as e:
+            logging.error(f"Failed to write reward.json: {e}")
+
+        # Write metadata.json if metadata was provided
+        if self.metadata:
+            metadata_path = self.output_path.parent / "metadata.json"
+            try:
+                metadata_path.write_text(json.dumps(self.metadata, indent=2))
+            except Exception as e:
+                logging.error(f"Failed to write metadata.json: {e}")
+
+        # Write LLM judge traces
+        traces_path = self.output_path.parent / "traces.json"
+        try:
+            traces_path.write_text(json.dumps(self.traces, indent=2))
+        except Exception as e:
+            logging.error(f"Failed to write traces.json: {e}")
+
+        print("=== Evaluation Finished ===")

graded/types.py

@@ -0,0 +1,145 @@
+"""Shared types for graded verifiers.
+
+Contains the grading :class:`Criterion` and the lightweight ATIF trajectory
+types (:class:`Trajectory`, :class:`Step`, :class:`ToolCall`). The trajectory
+types are intentionally minimal read-only mirrors of the ATIF types defined in
+``harbor.models.trajectories``. They use ``extra="ignore"`` throughout so that
+forward-compatible ATIF schema additions (new fields, new versions) never cause
+parse failures in verifier scripts.
+
+Use ``Evaluator.load_trajectory()`` to get a typed ``Trajectory`` from the
+agent log written during a trial.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Callable
+
+from pydantic import BaseModel, Field
+
+
+class Criterion(BaseModel):
+    """A single grading criterion registered via ``Evaluator.criterion``."""
+
+    name: str
+    weight: float = 1.0
+    fatal: bool = False
+    func: Callable[[Path], Any]
+
+    model_config = {"arbitrary_types_allowed": True}
+
+
+class ToolCall(BaseModel):
+    """A single tool invocation within an agent step."""
+
+    tool_call_id: str
+    function_name: str
+    arguments: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = {"extra": "ignore"}
+
+    def arg(self, key: str, default: Any = None) -> Any:
+        """Convenience accessor for a single argument value."""
+        return self.arguments.get(key, default)
+
+
+class Step(BaseModel):
+    """One turn in the agent trajectory."""
+
+    step_id: int
+    source: str  # "user" | "agent" | "system"
+    message: Any = ""  # str or list[ContentPart] — we accept either
+    tool_calls: list[ToolCall] | None = None
+
+    model_config = {"extra": "ignore"}
+
+
+class Trajectory(BaseModel):
+    """Parsed ATIF trajectory.  Exposes a small query API for verifier use."""
+
+    schema_version: str = ""
+    session_id: str | None = None
+    steps: list[Step] = Field(default_factory=list)
+
+    model_config = {"extra": "ignore"}
+
+    # ------------------------------------------------------------------
+    # Query primitives
+    # ------------------------------------------------------------------
+
+    def all_tool_calls(self) -> list[ToolCall]:
+        """Flat list of every tool call made across all agent steps."""
+        return [
+            tc
+            for step in self.steps
+            if step.tool_calls
+            for tc in step.tool_calls
+        ]
+
+    def tool_calls_for(self, function_name: str) -> list[ToolCall]:
+        """All tool calls whose ``function_name`` matches exactly.
+
+        Equivalent to ``find_all(function_name)``.
+        """
+        return self.find_all(function_name)
+
+    def exists(
+        self,
+        function_name: str,
+        predicate: Callable[[ToolCall], bool] | None = None,
+    ) -> bool:
+        """Return ``True`` if any tool call matches ``function_name`` and,
+        optionally, satisfies ``predicate``.
+
+        Examples::
+
+            trajectory.exists("write_file")
+            trajectory.exists(
+                "write_file",
+                lambda tc: PurePosixPath(tc.arg("path", "")).name == "blog_post.md",
+            )
+            trajectory.exists("terminal", lambda tc: "pytest" in tc.arg("command", ""))
+        """
+        return self.find(function_name, predicate) is not None
+
+    def find(
+        self,
+        function_name: str,
+        predicate: Callable[[ToolCall], bool] | None = None,
+    ) -> ToolCall | None:
+        """Return the first :class:`ToolCall` matching ``function_name`` (and
+        ``predicate`` if given), or ``None`` if no match is found.
+
+        Example::
+
+            tc = trajectory.find("write_file")
+            if tc:
+                print(tc.arg("path"))
+        """
+        for tc in self.all_tool_calls():
+            if tc.function_name != function_name:
+                continue
+            if predicate is None or predicate(tc):
+                return tc
+        return None
+
+    def find_all(
+        self,
+        function_name: str,
+        predicate: Callable[[ToolCall], bool] | None = None,
+    ) -> list[ToolCall]:
+        """Return all :class:`ToolCall` objects matching ``function_name`` (and
+        ``predicate`` if given).
+
+        Example::
+
+            calls = trajectory.find_all("write_file")
+            paths = [tc.arg("path") for tc in calls]
+        """
+        return [
+            tc
+            for tc in self.all_tool_calls()
+            if tc.function_name == function_name
+            and (predicate is None or predicate(tc))
+        ]

graded-1.0.0.dist-info/METADATA

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: graded
+Version: 1.0.0
+Summary: Defensive verifier framework and helpers for Harbor evaluations
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic>=2.0
+Requires-Dist: instructor>=1.0.0
+Requires-Dist: jsonref>=1.1.0
+Requires-Dist: google-genai>=1.47.0
+
+# graded 🍳
+
+`graded` is a defensive verifier and grading framework designed for agent evaluations (particularly for Harbor agent evaluations). It allows you to declare structured grading criteria, leverage LLM judges with automatic tracing, and safely manage evaluation artifacts.
+
+---
+
+## Installation
+
+Install `graded` directly from PyPI (or your internal registry):
+
+```bash
+pip install graded
+```
+
+Or with `uv`:
+
+```bash
+uv pip install graded
+```
+
+---
+
+## Quick Start
+
+Create an evaluation script (e.g. `verify.py`) to grade a task workspace:
+
+```python
+from pathlib import Path
+from graded import Evaluator
+
+# Initialize the evaluator
+ev = Evaluator(
+    workspace="/workspace",
+    output_path="/logs/verifier/reward.json",
+    auto_save_artifacts=True
+)
+
+# 1. Declare a standard criterion
+@ev.criterion(name="has_output_file", weight=1.0)
+def check_output(workspace: Path) -> bool:
+    return (workspace / "output.txt").is_file()
+
+# 2. Declare a fatal criterion (short-circuits score to 0.0 if failed)
+@ev.criterion(name="no_syntax_errors", weight=2.0, fatal=True)
+def check_syntax(workspace: Path) -> bool:
+    # return True or False (or float 0.0 - 1.0)
+    return True
+
+# 3. Declare a fractional scoring criterion
+@ev.criterion(name="test_pass_rate", weight=3.0)
+def check_tests(workspace: Path) -> float:
+    # Returns a score between 0.0 and 1.0
+    return 0.8  # e.g., 80% of tests passed
+
+# Run the evaluation and write outputs
+if __name__ == "__main__":
+    ev.run()
+```
+
+---
+
+## Core Features
+
+### 1. Criteria Declarations (`@ev.criterion`)
+Define check functions using the `@ev.criterion` decorator.
+- **`name`**: The unique identifier for the criterion.
+- **`weight`**: Relative weight of the score in the final weighted average calculation.
+- **`fatal`**: If set to `True`, any score of `0.0` or `False` immediately short-circuits the final score to `0.0`.
+- **Return Value**: Must return a `bool`, `int`, or `float`. Anything else raises a `ValueError`.
+
+### 2. LLM Judge with Automatic Tracing
+`graded` integrates with `instructor` to run structured, schema-validated LLM grading prompts, automatically logging prompt, parameters, response schema, and LLM responses to `traces.json`.
+
+```python
+from pydantic import BaseModel, Field
+
+class Rubric(BaseModel):
+    score: float = Field(description="Score between 0.0 and 1.0 based on correctness.")
+    reasoning: str = Field(description="Detailed reasoning for the score.")
+
+# In your criterion:
+result = ev.llm_judge(
+    model="google/gemini-3.5-flash",
+    response_model=Rubric,
+    system="You are a strict code correctness evaluator.",
+    prompt="Compare the student's solution in code.py with the requirements...",
+)
+
+print(f"LLM Score: {result.score}")
+print(f"Reasoning: {result.reasoning}")
+```
+
+### 3. File & Artifact Management
+Safely access files and copy evaluation artifacts to the logs directory for post-evaluation review.
+
+- **`ev.read_file(filename)`**: Safely reads content as a string. Auto-saves a copy to artifacts.
+- **`ev.load_json(filename)`**: Safely parses JSON file content. Auto-saves a copy to artifacts.
+- **`ev.save_file(filename, content)`**: Save arbitrary text/data to the artifacts directory.
+- **`ev.save_dir(dirname)`**: Copy an entire directory from the workspace to the artifacts directory.
+- **`ev.load_trajectory(path)`**: Load and parse an agent's ATIF `trajectory.json` file into a typed `Trajectory` object.
+
+---
+
+## Outputs
+
+When `ev.run()` completes, the following files are written to the directory containing your configured `output_path`:
+
+1. **`reward.json`**: A flat JSON dictionary containing the final calculated `reward` and the individual scores for each criterion:
+   ```json
+   {
+     "reward": 0.75,
+     "has_output_file": 1.0,
+     "no_syntax_errors": 1.0,
+     "test_pass_rate": 0.8
+   }
+   ```
+2. **`reward.txt`**: A text file containing just the final reward float value (e.g. `0.7500\n`).
+3. **`traces.json`**: A list of structured LLM calls made via `ev.llm_judge`, detailing inputs, responses, latencies, and metadata.
+4. **`metadata.json`**: (Optional) Contains evaluator-level and run-level metadata.
+5. **`artifacts/`**: Subfolder containing copy-back files preserved during the evaluation run.

graded-1.0.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+graded/__init__.py,sha256=Kfb66m1tMq79DHJAqNnIPZgAyE8pcVVk16VcM0D2kF8,174
+graded/evaluator.py,sha256=Mt0tJLe_fHVgnq6c1mDCtvSiwn8yhQrisYRTl7T5nWw,12141
+graded/types.py,sha256=sYQBQmnwaLPQK1CYiei0T4rtMfuEMBjRWgl-ETe2WxI,4478
+graded-1.0.0.dist-info/METADATA,sha256=sSKpwxQX3tJoIgZ8TWOQT-saTLEqfdikj3Xy1IZrE7Q,4650
+graded-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+graded-1.0.0.dist-info/top_level.txt,sha256=I4zw-dv35tCOLJzWEziFD8e8LehVwqPoCwaPzUzy8R0,7
+graded-1.0.0.dist-info/RECORD,,

graded-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

graded-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+graded