pydantic-evals 0.2.12__tar.gz → 0.2.13__tar.gz

{pydantic_evals-0.2.12 → pydantic_evals-0.2.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydantic-evals
-Version: 0.2.12
+Version: 0.2.13
 Summary: Framework for evaluating stochastic code execution, especially code making use of LLMs
 Project-URL: Homepage, https://ai.pydantic.dev/evals
 Project-URL: Source, https://github.com/pydantic/pydantic-ai
@@ -32,7 +32,7 @@ Requires-Python: >=3.9
 Requires-Dist: anyio>=0
 Requires-Dist: eval-type-backport>=0; python_version < '3.11'
 Requires-Dist: logfire-api>=1.2.0
-Requires-Dist: pydantic-ai-slim==0.2.12
+Requires-Dist: pydantic-ai-slim==0.2.13
 Requires-Dist: pydantic>=2.10
 Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: rich>=13.9.4

{pydantic_evals-0.2.12 → pydantic_evals-0.2.13}/pydantic_evals/evaluators/common.py

@@ -194,6 +194,7 @@ class LLMJudge(Evaluator[object, object, object]):
     rubric: str
     model: models.Model | models.KnownModelName | None = None
     include_input: bool = False
+    include_expected_output: bool = False
     model_settings: ModelSettings | None = None
     score: OutputConfig | Literal[False] = False
     assertion: OutputConfig | Literal[False] = field(default_factory=lambda: OutputConfig(include_reason=True))
@@ -203,15 +204,29 @@ class LLMJudge(Evaluator[object, object, object]):
         ctx: EvaluatorContext[object, object, object],
     ) -> EvaluatorOutput:
         if self.include_input:
-            from .llm_as_a_judge import judge_input_output
-
-            grading_output = await judge_input_output(
-                ctx.inputs, ctx.output, self.rubric, self.model, self.model_settings
-            )
+            if self.include_expected_output:
+                from .llm_as_a_judge import judge_input_output_expected
+
+                grading_output = await judge_input_output_expected(
+                    ctx.inputs, ctx.output, ctx.expected_output, self.rubric, self.model, self.model_settings
+                )
+            else:
+                from .llm_as_a_judge import judge_input_output
+
+                grading_output = await judge_input_output(
+                    ctx.inputs, ctx.output, self.rubric, self.model, self.model_settings
+                )
         else:
-            from .llm_as_a_judge import judge_output
+            if self.include_expected_output:
+                from .llm_as_a_judge import judge_output_expected
+
+                grading_output = await judge_output_expected(
+                    ctx.output, ctx.expected_output, self.rubric, self.model, self.model_settings
+                )
+            else:
+                from .llm_as_a_judge import judge_output
 
-            grading_output = await judge_output(ctx.output, self.rubric, self.model, self.model_settings)
+                grading_output = await judge_output(ctx.output, self.rubric, self.model, self.model_settings)
 
         output: dict[str, EvaluationScalar | EvaluationReason] = {}
         include_both = self.score is not False and self.assertion is not False

pydantic_evals-0.2.13/pydantic_evals/evaluators/llm_as_a_judge.py

@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+from textwrap import dedent
+from typing import Any
+
+from pydantic import BaseModel, Field
+from pydantic_core import to_json
+
+from pydantic_ai import Agent, models
+from pydantic_ai.settings import ModelSettings
+
+__all__ = (
+    'GradingOutput',
+    'judge_input_output',
+    'judge_input_output_expected',
+    'judge_output',
+    'judge_output_expected',
+    'set_default_judge_model',
+)
+
+
+_default_model: models.Model | models.KnownModelName = 'openai:gpt-4o'
+
+
+class GradingOutput(BaseModel, populate_by_name=True):
+    """The output of a grading operation."""
+
+    reason: str
+    pass_: bool = Field(validation_alias='pass', serialization_alias='pass')
+    score: float
+
+
+_judge_output_agent = Agent(
+    name='judge_output',
+    system_prompt=dedent(
+        """
+        You are grading output according to a user-specified rubric. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+        Examples:
+
+        <Output>Hello world</Output>
+        <Rubric>Content contains a greeting</Rubric>
+        {"reason": "the content contains the word 'Hello'", "pass": true, "score": 1.0}
+
+        <Output>Avast ye swabs, repel the invaders!</Output>
+        <Rubric>Does not speak like a pirate</Rubric>
+        {"reason": "'avast ye' is a common pirate term", "pass": false, "score": 0.0}
+        """
+    ),
+    output_type=GradingOutput,
+)
+
+
+async def judge_output(
+    output: Any,
+    rubric: str,
+    model: models.Model | models.KnownModelName | None = None,
+    model_settings: ModelSettings | None = None,
+) -> GradingOutput:
+    """Judge the output of a model based on a rubric.
+
+    If the model is not specified, a default model is used. The default model starts as 'openai:gpt-4o',
+    but this can be changed using the `set_default_judge_model` function.
+    """
+    user_prompt = dedent(
+        f"""
+        <Output>
+        {_stringify(output)}
+        </Output>
+        <Rubric>
+        {rubric}
+        </Rubric>
+        """
+    )
+    return (
+        await _judge_output_agent.run(user_prompt, model=model or _default_model, model_settings=model_settings)
+    ).output
+
+
+_judge_input_output_agent = Agent(
+    name='judge_input_output',
+    system_prompt=dedent(
+        """
+        You are grading output according to a user-specified rubric. If the statement in the rubric is true for the provided input and output, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+        Examples:
+
+        <Input>Hello world</Input>
+        <Output>Hello</Output>
+        <Rubric>Content contains a greeting word which is present in the input</Rubric>
+        {"reason": "the content contains the word 'Hello'", "pass": true, "score": 1.0}
+
+        <Input>Pirate</Input>
+        <Output>Avast ye swabs, repel the invaders!</Output>
+        <Rubric>Does not speak in the style described by the input</Rubric>
+        {"reason": "'avast ye' is a common pirate term", "pass": false, "score": 0.0}
+        """
+    ),
+    output_type=GradingOutput,
+)
+
+
+async def judge_input_output(
+    inputs: Any,
+    output: Any,
+    rubric: str,
+    model: models.Model | models.KnownModelName | None = None,
+    model_settings: ModelSettings | None = None,
+) -> GradingOutput:
+    """Judge the output of a model based on the inputs and a rubric.
+
+    If the model is not specified, a default model is used. The default model starts as 'openai:gpt-4o',
+    but this can be changed using the `set_default_judge_model` function.
+    """
+    user_prompt = dedent(
+        f"""
+        <Input>
+        {_stringify(inputs)}
+        </Input>
+        <Output>
+        {_stringify(output)}
+        </Output>
+        <Rubric>
+        {rubric}
+        </Rubric>
+        """
+    )
+    return (
+        await _judge_input_output_agent.run(user_prompt, model=model or _default_model, model_settings=model_settings)
+    ).output
+
+
+_judge_input_output_expected_agent = Agent(
+    name='judge_input_output_expected',
+    system_prompt=dedent(
+        """
+        You are grading output according to a user-specified rubric. If the statement in the rubric is true for the provided input, expected output, and output, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+        Examples:
+
+        <Input>What color is the sky?</Input>
+        <ExpectedOutput>Blue</ExpectedOutput>
+        <Output>Cerulean</Output>
+        <Rubric>The output is consistent with the expected output but doesn't have to match exactly</Rubric>
+        {"reason": "'Cerulean' is a shade of blue", "pass": true, "score": 1.0}
+
+        <Input>How many legs does a spider have?</Input>
+        <ExpectedOutput>8</ExpectedOutput>
+        <Output>Six</Output>
+        <Rubric>The output is factually consistent with the expected output</Rubric>
+        {"reason": "Spiders have 8 legs", "pass": false, "score": 0.0}
+        """
+    ),
+    output_type=GradingOutput,
+)
+
+
+async def judge_input_output_expected(
+    inputs: Any,
+    output: Any,
+    expected_output: Any,
+    rubric: str,
+    model: models.Model | models.KnownModelName | None = None,
+    model_settings: ModelSettings | None = None,
+) -> GradingOutput:
+    """Judge the output of a model based on the inputs and a rubric.
+
+    If the model is not specified, a default model is used. The default model starts as 'openai:gpt-4o',
+    but this can be changed using the `set_default_judge_model` function.
+    """
+    user_prompt = dedent(
+        f"""
+        <Input>
+        {_stringify(inputs)}
+        </Input>
+        <ExpectedOutput>
+        {_stringify(expected_output)}
+        </ExpectedOutput>
+        <Output>
+        {_stringify(output)}
+        </Output>
+        <Rubric>
+        {rubric}
+        </Rubric>
+        """
+    )
+
+    return (
+        await _judge_input_output_expected_agent.run(
+            user_prompt, model=model or _default_model, model_settings=model_settings
+        )
+    ).output
+
+
+_judge_output_expected_agent = Agent(
+    name='judge_output_expected',
+    system_prompt=dedent(
+        """
+        You are grading output according to a user-specified rubric. If the statement in the rubric is true for the provided expected output and output, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+        Examples:
+
+        <ExpectedOutput>Blue</ExpectedOutput>
+        <Output>Cerulean</Output>
+        <Rubric>The output should be a shade of the expected output color</Rubric>
+        {"reason": "'Cerulean' is a shade of blue", "pass": true, "score": 1.0}
+
+        <ExpectedOutput>8</ExpectedOutput>
+        <Output>Six</Output>
+        <Rubric>The output should be a number written in words which matches the number written in digits in the expected output</Rubric>
+        {"reason": "The output is 'Six' which is a different number than 8", "pass": false, "score": 0.0}
+        """
+    ),
+    output_type=GradingOutput,
+)
+
+
+async def judge_output_expected(
+    output: Any,
+    expected_output: Any,
+    rubric: str,
+    model: models.Model | models.KnownModelName | None = None,
+    model_settings: ModelSettings | None = None,
+) -> GradingOutput:
+    """Judge the output of a model based on the expected output, output, and a rubric.
+
+    If the model is not specified, a default model is used. The default model starts as 'openai:gpt-4o',
+    but this can be changed using the `set_default_judge_model` function.
+    """
+    user_prompt = dedent(
+        f"""
+        <ExpectedOutput>
+        {_stringify(expected_output)}
+        </ExpectedOutput>
+        <Output>
+        {_stringify(output)}
+        </Output>
+        <Rubric>
+        {rubric}
+        </Rubric>
+        """
+    )
+    return (
+        await _judge_output_expected_agent.run(
+            user_prompt, model=model or _default_model, model_settings=model_settings
+        )
+    ).output
+
+
+def set_default_judge_model(model: models.Model | models.KnownModelName) -> None:  # pragma: no cover
+    """Set the default model used for judging.
+
+    This model is used if `None` is passed to the `model` argument of `judge_output` and `judge_input_output`.
+    """
+    global _default_model
+    _default_model = model
+
+
+def _stringify(value: Any) -> str:
+    if isinstance(value, str):
+        return value
+    try:
+        # If the value can be serialized to JSON, use that.
+        # If that behavior is undesirable, the user could manually call repr on the arguments to the judge_* functions
+        return to_json(value).decode()
+    except Exception:
+        return repr(value)

pydantic_evals-0.2.12/pydantic_evals/evaluators/llm_as_a_judge.py

@@ -1,122 +0,0 @@
-from __future__ import annotations
-
-from textwrap import dedent
-from typing import Any
-
-from pydantic import BaseModel, Field
-from pydantic_core import to_json
-
-from pydantic_ai import Agent, models
-from pydantic_ai.settings import ModelSettings
-
-__all__ = ('GradingOutput', 'judge_input_output', 'judge_output', 'set_default_judge_model')
-
-
-_default_model: models.Model | models.KnownModelName = 'openai:gpt-4o'
-
-
-class GradingOutput(BaseModel, populate_by_name=True):
-    """The output of a grading operation."""
-
-    reason: str
-    pass_: bool = Field(validation_alias='pass', serialization_alias='pass')
-    score: float
-
-
-_judge_output_agent = Agent(
-    name='judge_output',
-    system_prompt=dedent(
-        """
-        You are grading output according to a user-specified rubric. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
-
-        Examples:
-
-        <Output>Hello world</Output>
-        <Rubric>Content contains a greeting</Rubric>
-        {"reason": "the content contains the word 'Hello'", "pass": true, "score": 1.0}
-
-        <Output>Avast ye swabs, repel the invaders!</Output>
-        <Rubric>Does not speak like a pirate</Rubric>
-        {"reason": "'avast ye' is a common pirate term", "pass": false, "score": 0.0}
-        """
-    ),
-    output_type=GradingOutput,
-)
-
-
-async def judge_output(
-    output: Any,
-    rubric: str,
-    model: models.Model | models.KnownModelName | None = None,
-    model_settings: ModelSettings | None = None,
-) -> GradingOutput:
-    """Judge the output of a model based on a rubric.
-
-    If the model is not specified, a default model is used. The default model starts as 'openai:gpt-4o',
-    but this can be changed using the `set_default_judge_model` function.
-    """
-    user_prompt = f'<Output>\n{_stringify(output)}\n</Output>\n<Rubric>\n{rubric}\n</Rubric>'
-    return (
-        await _judge_output_agent.run(user_prompt, model=model or _default_model, model_settings=model_settings)
-    ).output
-
-
-_judge_input_output_agent = Agent(
-    name='judge_input_output',
-    system_prompt=dedent(
-        """
-        You are grading output according to a user-specified rubric. If the statement in the rubric is true for the provided input and output, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
-
-        Examples:
-
-        <Input>Hello world</Input>
-        <Output>Hello</Output>
-        <Rubric>Content contains a greeting word which is present in the input</Rubric>
-        {"reason": "the content contains the word 'Hello'", "pass": true, "score": 1.0}
-
-        <Input>Pirate</Input>
-        <Output>Avast ye swabs, repel the invaders!</Output>
-        <Rubric>Does not speak in the style described by the input</Rubric>
-        {"reason": "'avast ye' is a common pirate term", "pass": false, "score": 0.0}
-        """
-    ),
-    output_type=GradingOutput,
-)
-
-
-async def judge_input_output(
-    inputs: Any,
-    output: Any,
-    rubric: str,
-    model: models.Model | models.KnownModelName | None = None,
-    model_settings: ModelSettings | None = None,
-) -> GradingOutput:
-    """Judge the output of a model based on the inputs and a rubric.
-
-    If the model is not specified, a default model is used. The default model starts as 'openai:gpt-4o',
-    but this can be changed using the `set_default_judge_model` function.
-    """
-    user_prompt = f'<Input>\n{_stringify(inputs)}\n</Input>\n<Output>\n{_stringify(output)}\n</Output>\n<Rubric>\n{rubric}\n</Rubric>'
-    return (
-        await _judge_input_output_agent.run(user_prompt, model=model or _default_model, model_settings=model_settings)
-    ).output
-
-
-def set_default_judge_model(model: models.Model | models.KnownModelName) -> None:  # pragma: no cover
-    """Set the default model used for judging.
-
-    This model is used if `None` is passed to the `model` argument of `judge_output` and `judge_input_output`.
-    """
-    global _default_model
-    _default_model = model
-
-
-def _stringify(value: Any) -> str:
-    if isinstance(value, str):
-        return value
-    try:
-        # If the value can be serialized to JSON, use that.
-        # If that behavior is undesirable, the user could manually call repr on the arguments to the judge_* functions
-        return to_json(value).decode()
-    except Exception:
-        return repr(value)