verifiers 0.3.2.dev184__tar.gz → 0.3.2.dev185__tar.gz

{verifiers-0.3.2.dev184 → verifiers-0.3.2.dev185}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev184
+Version: 0.3.2.dev185
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev184 → verifiers-0.3.2.dev185}/verifiers/v1/__init__.py

@@ -66,6 +66,8 @@ from verifiers.v1.harness import Harness, HarnessSession
 from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
 from verifiers.v1.judges import (
     Criterion,
+    DecisionJudge,
+    DecisionJudgeConfig,
     ReferenceJudge,
     ReferenceJudgeConfig,
     RubricJudge,
@@ -358,6 +360,8 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Judges",
     "JudgeResponse",
     "JudgeView",
+    "DecisionJudge",
+    "DecisionJudgeConfig",
     "ReferenceJudge",
     "ReferenceJudgeConfig",
     "RubricJudge",

{verifiers-0.3.2.dev184 → verifiers-0.3.2.dev185}/verifiers/v1/configs/client.py

@@ -60,7 +60,10 @@ class BaseClientConfig(BaseConfig):
             or prime_config.get("inference_url")
             or DEFAULT_PRIME_INFERENCE_URL
         )
-        if "base_url" not in self.model_fields_set:
+        if (
+            "base_url" not in self.model_fields_set
+            and self.base_url == DEFAULT_PRIME_INFERENCE_URL
+        ):
             self.base_url = prime_base_url
         host = urlparse(self.base_url).hostname or ""
         if host != PRIME_INFERENCE_HOST and not host.endswith(

{verifiers-0.3.2.dev184 → verifiers-0.3.2.dev185}/verifiers/v1/configs/judge.py

@@ -40,7 +40,7 @@ def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
         if not raw.get("id"):
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, or an installed package)"
+                "`rubric`, `decision`, or an installed package)"
             )
         resolved.append(judge_config_type(raw["id"]).model_validate(raw))
     return resolved
@@ -51,7 +51,7 @@ def check_judges(entries: Sequence[JudgeConfig]) -> None:
         if not entry.id:
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, or an installed package)"
+                "`rubric`, `decision`, or an installed package)"
             )
     keys = [judge_key(entry) for entry in entries]
     if duplicates := {key for key in keys if keys.count(key) > 1}:

{verifiers-0.3.2.dev184 → verifiers-0.3.2.dev185}/verifiers/v1/judges/__init__.py

@@ -1,8 +1,11 @@
+from verifiers.v1.judges.decision import DecisionJudge, DecisionJudgeConfig
 from verifiers.v1.judges.reference import ReferenceJudge, ReferenceJudgeConfig
 from verifiers.v1.judges.rubric import Criterion, RubricJudge, RubricJudgeConfig
 
 __all__ = [
     "Criterion",
+    "DecisionJudge",
+    "DecisionJudgeConfig",
     "ReferenceJudge",
     "ReferenceJudgeConfig",
     "RubricJudge",

verifiers-0.3.2.dev185/verifiers/v1/judges/decision.py

@@ -0,0 +1,105 @@
+"""Decision-model judge using Prime's evaluations API by default."""
+
+import json
+from typing import Any, cast
+
+from pydantic import Field
+
+from verifiers.v1.clients.base import build_async_openai
+from verifiers.v1.configs.judge import JudgeConfig
+from verifiers.v1.judge import (
+    Judge,
+    JudgeResponse,
+    JudgeView,
+    judge_question,
+    judge_response,
+)
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+from verifiers.v1.types import ID, Usage
+
+
+class DecisionJudgeConfig(JudgeConfig):
+    id: ID = "decision"
+    model: str = "typesafe-ai/jev"
+    base_url: str = "https://api.pinference.ai/api/v1/evaluations"
+    """Full URL to POST evaluation requests to."""
+    levels: list[str] = Field(
+        default_factory=lambda: ["Incorrect", "Correct"], min_length=2, max_length=10
+    )
+    """Grading levels, worst to best. The model's probability-weighted level is
+    normalized to [0, 1], so a two-level grade is the probability of correctness."""
+    question_field: str = ""
+    """Task field containing the question; empty uses the task's prompt text."""
+    answer_field: str = ""
+    """Optional task field containing a reference answer to include in the state."""
+    view: JudgeView = "last_reply"
+
+
+class DecisionJudge(Judge[float, DecisionJudgeConfig]):
+    prompt = (
+        "Grade how well the response satisfies the task in question. "
+        "Use the reference answer when provided; equivalent answers are acceptable."
+    )
+
+    async def evaluate(
+        self, *, trace: Trace | None = None, **fields: Any
+    ) -> JudgeResponse[float]:
+        request = {
+            "model": self.config.model,
+            "state": fields,
+            "questions": {
+                "grade": {
+                    "type": "score",
+                    "instructions": self.build_messages(**fields),
+                    "criteria": self.config.levels,
+                }
+            },
+        }
+        async with build_async_openai(self.config) as client:
+            raw = await client.post(
+                self.config.base_url, cast_to=dict[str, Any], body=request
+            )
+        result = raw.get("result", raw)  # Workers AI wraps its response in `result`.
+        response = JudgeResponse[float](text=json.dumps(raw))
+        try:
+            if usage := result.get("usage"):
+                response.usage = Usage(
+                    prompt_tokens=usage.get("inputTokens", usage.get("input_tokens")),
+                    completion_tokens=usage.get(
+                        "outputTokens", usage.get("output_tokens")
+                    ),
+                    cost=usage.get("cost"),
+                )
+            score = result["answers"]["grade"]["score"] / (len(self.config.levels) - 1)
+            if not 0 <= score <= 1:
+                raise ValueError(f"decision judge score {score!r} is outside [0, 1]")
+            response.parsed = score
+            return response
+        finally:
+            # The call was billed even if the score could not be parsed.
+            if trace is not None:
+                trace.record_judge_call(
+                    name=self.reward_name, request=request, response=response
+                )
+
+    async def score(self, task: TaskData, trace: Trace) -> float:
+        response = judge_response(trace, self.config.view)
+        if not response.strip():
+            return 0.0
+        fields = {
+            "question": judge_question(task, self.config.question_field),
+            "response": response,
+        }
+        if self.config.answer_field:
+            answer = getattr(task, self.config.answer_field, None)
+            if answer is None:
+                raise ValueError(
+                    f"decision judge found no {self.config.answer_field!r} field on the task"
+                )
+            fields["answer"] = answer
+        result = await self.evaluate(trace=trace, **fields)
+        return cast(float, result.parsed)
+
+
+__all__ = ["DecisionJudge", "DecisionJudgeConfig"]