dreadnode 1.12.0__tar.gz → 1.12.2__tar.gz

{dreadnode-1.12.0 → dreadnode-1.12.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dreadnode
-Version: 1.12.0
+Version: 1.12.2
 Summary: Dreadnode SDK
 Author: Nick Landers
 Author-email: monoxgas@gmail.com
@@ -22,7 +22,7 @@ Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: pillow (>=11.2.1,<12.0.0) ; extra == "multimodal"
 Requires-Dist: pydantic (>=2.9.2,<3.0.0)
 Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
-Requires-Dist: rigging (>=3.1.1,<4.0.0)
+Requires-Dist: rigging (>=3.2.1,<4.0.0)
 Requires-Dist: soundfile (>=0.13.1,<0.14.0) ; extra == "multimodal"
 Requires-Dist: transformers (>=4.41.0,<5.0.0) ; extra == "training"
 Project-URL: Repository, https://github.com/dreadnode/sdk

{dreadnode-1.12.0 → dreadnode-1.12.2}/dreadnode/object.py

@@ -1,6 +1,7 @@
 import typing as t
 from dataclasses import dataclass
 
+from litellm import ConfigDict
 from pydantic import BaseModel, Field
 
 from dreadnode.types import AnyDict
@@ -31,6 +32,8 @@ class ObjectUri(BaseModel):
 
 
 class ObjectVal(BaseModel):
+    model_config = ConfigDict(serialize_by_alias=True)
+
     hash: str
     schema_hash: str
     value_: t.Any = Field(alias="value")

{dreadnode-1.12.0 → dreadnode-1.12.2}/dreadnode/scorers/__init__.py

@@ -7,6 +7,7 @@ from dreadnode.scorers.contains import (
     detect_unsafe_shell_content,
 )
 from dreadnode.scorers.length import length_in_range, length_ratio, length_target
+from dreadnode.scorers.llm_judge import llm_judge
 from dreadnode.scorers.pii import detect_pii, detect_pii_with_presidio
 from dreadnode.scorers.readability import readability
 from dreadnode.scorers.rigging import wrap_chat
@@ -26,6 +27,7 @@ __all__ = [
     "length_in_range",
     "length_ratio",
     "length_target",
+    "llm_judge",
     "readability",
     "semantic_similarity",
     "sentiment",

dreadnode-1.12.2/dreadnode/scorers/llm_judge.py

@@ -0,0 +1,102 @@
+import typing as t
+
+from rigging import GenerateParams, get_generator
+from rigging.generator import Generator
+from rigging.model import Model, element
+from rigging.prompt import prompt
+
+from dreadnode.metric import Metric, Scorer
+from dreadnode.task import TaskInput
+
+
+class JudgeInput(Model):
+    input: str | None = element(default=None)
+    expected_output: str | None = element(default=None)
+    output: str = element()
+    rubric: str = element()
+
+
+class Judgement(Model):
+    reason: str = element()
+    pass_: bool = element(alias="pass")
+    score: float = element()
+
+
+@prompt()
+def judge(input: JudgeInput) -> Judgement:  # type: ignore [empty-body]
+    """
+    You are grading output according to a user-specified rubric. \
+    If the statement in the rubric is true for the provided input and output, then the output passes the test.
+    Assign a score based on the rubric, where applicable, otherwise 1.0 for passing and 0.0 for failing.
+    """
+
+
+def llm_judge(
+    model: "str | Generator | TaskInput",
+    rubric: str | TaskInput,
+    *,
+    expected_output: str | TaskInput | None = None,
+    params: "GenerateParams | None" = None,
+    passing: t.Callable[[float], bool] | None = None,
+    min_score: float | None = None,
+    max_score: float | None = None,
+    name: str = "llm_judge",
+) -> "Scorer[t.Any]":
+    """
+    Score the output of a task using an LLM to judge it against a rubric.
+
+    Args:
+        model: The model to use for judging. Can be a string identifier (rigging), a Generator instance
+            or a TaskInput that resolves to a string identifier.
+        rubric: The rubric to use for judging. Can be a string or a TaskInput that resolves to a string.
+        expected_output: The expected output to compare against, if applicable. Can be a string or a TaskInput that resolves to a string.
+        params: Optional parameters for the generator.
+        passing: Optional callback to determine if the score is passing based on the score value - overrides any model-specified value.
+        min_score: Optional minimum score for the judgement - if provided, the score will be clamped to this value.
+        max_score: Optional maximum score for the judgement - if provided, the score will be clamped to this value.
+        name: The name of the scorer.
+    """
+
+    async def evaluate(data: t.Any) -> Metric:
+        _model = model.resolve() if isinstance(model, TaskInput) else model
+        _rubric = rubric.resolve(cast_as=str) if isinstance(rubric, TaskInput) else rubric
+        _expected_output = (
+            expected_output.resolve(cast_as=str)
+            if isinstance(expected_output, TaskInput)
+            else expected_output
+        )
+
+        generator: Generator
+        if isinstance(_model, str):
+            generator = get_generator(_model, params=params or GenerateParams())
+        elif isinstance(_model, Generator):
+            generator = _model
+        else:
+            raise TypeError("Model must be a string identifier or a Generator instance.")
+
+        input_data = JudgeInput(
+            input=str(data),
+            expected_output=_expected_output,
+            output=str(data),
+            rubric=_rubric,
+        )
+
+        judgement = await judge.bind(generator)(input_data)
+
+        if min_score is not None:
+            judgement.score = max(min_score, judgement.score)
+        if max_score is not None:
+            judgement.score = min(max_score, judgement.score)
+
+        if passing is not None:
+            judgement.pass_ = passing(judgement.score)
+
+        return Metric(
+            value=judgement.score,
+            attributes={
+                "reason": judgement.reason,
+                "pass": judgement.pass_,
+            },
+        )
+
+    return Scorer.from_callable(evaluate, name=name, catch=True)

{dreadnode-1.12.0 → dreadnode-1.12.2}/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "dreadnode"
-version = "1.12.0"
+version = "1.12.2"
 description = "Dreadnode SDK"
 requires-python = ">=3.10,<3.14"
 
 [tool.poetry]
 name = "dreadnode"
-version = "1.12.0"
+version = "1.12.2"
 description = "Dreadnode SDK"
 authors = ["Nick Landers <monoxgas@gmail.com>"]
 repository = "https://github.com/dreadnode/sdk"
@@ -23,12 +23,12 @@ pandas = "^2.2.3"
 fsspec = { version = ">=2023.1.0,<=2025.3.0", extras = [
     "s3",
 ] } # Pinned for datasets compatibility
-rigging = "^3.1.1"
 
 transformers = { version = "^4.41.0", optional = true }
 soundfile = { version = "^0.13.1", optional = true }
 moviepy = { version = "^2.1.2", optional = true }
 pillow = { version = "^11.2.1", optional = true }
+rigging = "^3.2.1"
 
 [tool.poetry.extras]
 training = ["transformers"]