verifiers 0.3.2.dev184__py3-none-any.whl → 0.3.2.dev185__py3-none-any.whl

verifiers/v1/__init__.py

@@ -66,6 +66,8 @@ from verifiers.v1.harness import Harness, HarnessSession
 from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
 from verifiers.v1.judges import (
     Criterion,
+    DecisionJudge,
+    DecisionJudgeConfig,
     ReferenceJudge,
     ReferenceJudgeConfig,
     RubricJudge,
@@ -358,6 +360,8 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Judges",
     "JudgeResponse",
     "JudgeView",
+    "DecisionJudge",
+    "DecisionJudgeConfig",
     "ReferenceJudge",
     "ReferenceJudgeConfig",
     "RubricJudge",

verifiers/v1/configs/client.py

@@ -60,7 +60,10 @@ class BaseClientConfig(BaseConfig):
             or prime_config.get("inference_url")
             or DEFAULT_PRIME_INFERENCE_URL
         )
-        if "base_url" not in self.model_fields_set:
+        if (
+            "base_url" not in self.model_fields_set
+            and self.base_url == DEFAULT_PRIME_INFERENCE_URL
+        ):
             self.base_url = prime_base_url
         host = urlparse(self.base_url).hostname or ""
         if host != PRIME_INFERENCE_HOST and not host.endswith(

verifiers/v1/configs/judge.py

@@ -40,7 +40,7 @@ def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
         if not raw.get("id"):
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, or an installed package)"
+                "`rubric`, `decision`, or an installed package)"
             )
         resolved.append(judge_config_type(raw["id"]).model_validate(raw))
     return resolved
@@ -51,7 +51,7 @@ def check_judges(entries: Sequence[JudgeConfig]) -> None:
         if not entry.id:
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, or an installed package)"
+                "`rubric`, `decision`, or an installed package)"
             )
     keys = [judge_key(entry) for entry in entries]
     if duplicates := {key for key in keys if keys.count(key) > 1}:

verifiers/v1/judges/__init__.py

@@ -1,8 +1,11 @@
+from verifiers.v1.judges.decision import DecisionJudge, DecisionJudgeConfig
 from verifiers.v1.judges.reference import ReferenceJudge, ReferenceJudgeConfig
 from verifiers.v1.judges.rubric import Criterion, RubricJudge, RubricJudgeConfig
 
 __all__ = [
     "Criterion",
+    "DecisionJudge",
+    "DecisionJudgeConfig",
     "ReferenceJudge",
     "ReferenceJudgeConfig",
     "RubricJudge",

verifiers/v1/judges/decision.py

@@ -0,0 +1,105 @@
+"""Decision-model judge using Prime's evaluations API by default."""
+
+import json
+from typing import Any, cast
+
+from pydantic import Field
+
+from verifiers.v1.clients.base import build_async_openai
+from verifiers.v1.configs.judge import JudgeConfig
+from verifiers.v1.judge import (
+    Judge,
+    JudgeResponse,
+    JudgeView,
+    judge_question,
+    judge_response,
+)
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+from verifiers.v1.types import ID, Usage
+
+
+class DecisionJudgeConfig(JudgeConfig):
+    id: ID = "decision"
+    model: str = "typesafe-ai/jev"
+    base_url: str = "https://api.pinference.ai/api/v1/evaluations"
+    """Full URL to POST evaluation requests to."""
+    levels: list[str] = Field(
+        default_factory=lambda: ["Incorrect", "Correct"], min_length=2, max_length=10
+    )
+    """Grading levels, worst to best. The model's probability-weighted level is
+    normalized to [0, 1], so a two-level grade is the probability of correctness."""
+    question_field: str = ""
+    """Task field containing the question; empty uses the task's prompt text."""
+    answer_field: str = ""
+    """Optional task field containing a reference answer to include in the state."""
+    view: JudgeView = "last_reply"
+
+
+class DecisionJudge(Judge[float, DecisionJudgeConfig]):
+    prompt = (
+        "Grade how well the response satisfies the task in question. "
+        "Use the reference answer when provided; equivalent answers are acceptable."
+    )
+
+    async def evaluate(
+        self, *, trace: Trace | None = None, **fields: Any
+    ) -> JudgeResponse[float]:
+        request = {
+            "model": self.config.model,
+            "state": fields,
+            "questions": {
+                "grade": {
+                    "type": "score",
+                    "instructions": self.build_messages(**fields),
+                    "criteria": self.config.levels,
+                }
+            },
+        }
+        async with build_async_openai(self.config) as client:
+            raw = await client.post(
+                self.config.base_url, cast_to=dict[str, Any], body=request
+            )
+        result = raw.get("result", raw)  # Workers AI wraps its response in `result`.
+        response = JudgeResponse[float](text=json.dumps(raw))
+        try:
+            if usage := result.get("usage"):
+                response.usage = Usage(
+                    prompt_tokens=usage.get("inputTokens", usage.get("input_tokens")),
+                    completion_tokens=usage.get(
+                        "outputTokens", usage.get("output_tokens")
+                    ),
+                    cost=usage.get("cost"),
+                )
+            score = result["answers"]["grade"]["score"] / (len(self.config.levels) - 1)
+            if not 0 <= score <= 1:
+                raise ValueError(f"decision judge score {score!r} is outside [0, 1]")
+            response.parsed = score
+            return response
+        finally:
+            # The call was billed even if the score could not be parsed.
+            if trace is not None:
+                trace.record_judge_call(
+                    name=self.reward_name, request=request, response=response
+                )
+
+    async def score(self, task: TaskData, trace: Trace) -> float:
+        response = judge_response(trace, self.config.view)
+        if not response.strip():
+            return 0.0
+        fields = {
+            "question": judge_question(task, self.config.question_field),
+            "response": response,
+        }
+        if self.config.answer_field:
+            answer = getattr(task, self.config.answer_field, None)
+            if answer is None:
+                raise ValueError(
+                    f"decision judge found no {self.config.answer_field!r} field on the task"
+                )
+            fields["answer"] = answer
+        result = await self.evaluate(trace=trace, **fields)
+        return cast(float, result.parsed)
+
+
+__all__ = ["DecisionJudge", "DecisionJudgeConfig"]

{verifiers-0.3.2.dev184.dist-info → verifiers-0.3.2.dev185.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev184
+Version: 0.3.2.dev185
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev184.dist-info → verifiers-0.3.2.dev185.dist-info}/RECORD

@@ -1,5 +1,5 @@
 verifiers/__init__.py,sha256=F5IVMw6mhGaPBMCw7QKk0la_r5mMwydP2hdTTdQucik,352
-verifiers/v1/__init__.py,sha256=ybCeLKoVNDU9nwla4BG8HrcZ2oXOuXquRBhaorGevxI,8792
+verifiers/v1/__init__.py,sha256=xvS_VfN7QwTethhEJsZCi7lHicKvQdQKh2nhB2h9Lu8,8884
 verifiers/v1/agent.py,sha256=zBqbHDeqdPghg_QOkcyFplIDY3f34qCOtwcxJnl5N-U,45088
 verifiers/v1/env.py,sha256=fxXBjVyi0milYViHt6O0vBK3JdM2TYSOoQpXd0P0rFE,18110
 verifiers/v1/episode.py,sha256=UPtA02aGWA-fY8cJKEuYCMtQVnPN1C0BSN13LggNT8U,5986
@@ -43,10 +43,10 @@ verifiers/v1/clients/eval.py,sha256=x_5E8XnhFu4v-k4yI8WWHd39rDkvB7mSEPCVAWL4AiE,
 verifiers/v1/clients/train.py,sha256=lSJTwHjwOET3D_CI_eYZSoXSQcv2H0K5sYJ7kD_kdw4,19489
 verifiers/v1/configs/__init__.py,sha256=X7u6X7B3ieD1HZl79Tv1kg-yPHbT0QLvAebaGqxVtvA,317
 verifiers/v1/configs/agent.py,sha256=Vut73R2EW8QuL5c0iTnNFC_g-Obvl8YUKsjlvkABuL0,5311
-verifiers/v1/configs/client.py,sha256=vl46nrayZa84zrPBuEVpJIjDjbmYUIrDIuR0vZvvugw,5212
+verifiers/v1/configs/client.py,sha256=i45QOop2Wb-LQRpNgUZGTCT1_HpNmdcG41eQrGMgJKQ,5297
 verifiers/v1/configs/env.py,sha256=k47XBYYWLOf-8P1dbek3x_s0ucNb726xxoryG69DIr4,7256
 verifiers/v1/configs/harness.py,sha256=ZrR-Egzfft95oMoqQ78bAq9TqP882Q68M_47Ajw9lE8,2918
-verifiers/v1/configs/judge.py,sha256=WEwA8D4X5JSaIP3c-E2wPnvgByWeT6JRAn1d8gfiQ40,2142
+verifiers/v1/configs/judge.py,sha256=3ZvxcTf-3clj8bDLyaqsMSKCRclTQCvbyb2qp3QSRD8,2166
 verifiers/v1/configs/retries.py,sha256=BgpGbdMBmjyjcIGnvb5VTJzKMDA6TdKil-PBQaRwGZM,1770
 verifiers/v1/configs/runtime.py,sha256=3LDAnKy-BotE6MqyT1l07Y_bAq9Wye1OU8JZ7lyUVvY,5474
 verifiers/v1/configs/select.py,sha256=_HdcepQaJ1sDCYcs7Vqx72npDwHGOgUjsLDZmcgnonU,4743
@@ -131,7 +131,8 @@ verifiers/v1/interception/tunnel/__init__.py,sha256=eVNZJszj6myrKhx9jmJz7VRd9qlf
 verifiers/v1/interception/tunnel/base.py,sha256=DZB6uPLwM4Qy7n7m0vKeyg-Px87MhsmpYTpe2vpPEng,1998
 verifiers/v1/interception/tunnel/custom.py,sha256=yL4UbGf4bAuMsxk42mEP-t1wITQCT8BE9FHQIz0qRMA,1708
 verifiers/v1/interception/tunnel/prime.py,sha256=jVk_6xyrxXiHtsVFErcRNd4MoENdFbqzfC0i20nlizU,2803
-verifiers/v1/judges/__init__.py,sha256=MUIBWcx6c70BykrTDHhB0ITIyJPxXe6xDBou4VM7jDQ,286
+verifiers/v1/judges/__init__.py,sha256=z-16awBqEZJrlVYIb9HD-rk12RaABh6jodJgwX4blnA,410
+verifiers/v1/judges/decision.py,sha256=U8BKuzJVMWfX76fa6jaG_rmcOhzMkZoXSnkUuHKpoV4,3991
 verifiers/v1/judges/reference.py,sha256=iEVHw-iJ6dbwLOOqrvWrEMJOcUEYJ0YspT04rZz7Sd4,3868
 verifiers/v1/judges/reference.txt,sha256=Ej35kGXiT2uJ0LcHIeez49rCS_9uHmCQKoVdn93V6A8,353
 verifiers/v1/judges/rubric.py,sha256=HGQYI3xdq8I84Ce0_sDeNgk5JM597es8rOVPipjCyxU,12025
@@ -194,8 +195,8 @@ verifiers/v1/utils/scope.py,sha256=bzUEiWDOVdbj7dXOwGOlzsQ_-Xu0j9NR5TalQB-8TS8,8
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/trace_store.py,sha256=RsCDonGR-bs-tvEstcppiJm3jFmAdIiG83fA5KtCbAw,2801
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev184.dist-info/METADATA,sha256=2jIg7eRwvlbVGsPE_AYQvvGKAPnBCZiY1E7GXeE66Ck,4227
-verifiers-0.3.2.dev184.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-verifiers-0.3.2.dev184.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
-verifiers-0.3.2.dev184.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev184.dist-info/RECORD,,
+verifiers-0.3.2.dev185.dist-info/METADATA,sha256=DF9K7iZDqE_b9Sf8WSNbMwPW1mZlDdTxqrUZtxUQNrY,4227
+verifiers-0.3.2.dev185.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+verifiers-0.3.2.dev185.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
+verifiers-0.3.2.dev185.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev185.dist-info/RECORD,,