pydantic-ai 0.2.3__tar.gz → 0.2.5__tar.gz

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/.gitignore

@@ -17,3 +17,5 @@ examples/pydantic_ai_examples/.chat_app_messages.sqlite
 /docs-site/.wrangler/
 /CLAUDE.md
 node_modules/
+**.idea/
+.coverage*

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/Makefile

@@ -63,6 +63,10 @@ test: ## Run tests and collect coverage data
 	uv run coverage run -m pytest
 	@uv run coverage report
 
+.PHONY: test-fast
+test-fast: ## Same as test except no coverage. ~1/4th the time depending on hardware.
+	uv run pytest -n auto --dist=loadgroup
+
 .PHONY: test-all-python
 test-all-python: ## Run tests on Python 3.9 to 3.13
 	UV_PROJECT_ENVIRONMENT=.venv39 uv run --python 3.9 --all-extras --all-packages coverage run -p -m pytest

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydantic-ai
-Version: 0.2.3
+Version: 0.2.5
 Summary: Agent Framework / shim to use Pydantic with LLMs
 Project-URL: Homepage, https://ai.pydantic.dev
 Project-URL: Source, https://github.com/pydantic/pydantic-ai
@@ -28,9 +28,9 @@ Classifier: Topic :: Internet
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.9
-Requires-Dist: pydantic-ai-slim[a2a,anthropic,bedrock,cli,cohere,evals,groq,mcp,mistral,openai,vertexai]==0.2.3
+Requires-Dist: pydantic-ai-slim[a2a,anthropic,bedrock,cli,cohere,evals,google,groq,mcp,mistral,openai,vertexai]==0.2.5
 Provides-Extra: examples
-Requires-Dist: pydantic-ai-examples==0.2.3; extra == 'examples'
+Requires-Dist: pydantic-ai-examples==0.2.5; extra == 'examples'
 Provides-Extra: logfire
 Requires-Dist: logfire>=3.11.0; extra == 'logfire'
 Description-Content-Type: text/markdown

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/pyproject.toml

@@ -46,7 +46,7 @@ requires-python = ">=3.9"
 
 [tool.hatch.metadata.hooks.uv-dynamic-versioning]
 dependencies = [
-    "pydantic-ai-slim[openai,vertexai,groq,anthropic,mistral,cohere,bedrock,cli,mcp,evals,a2a]=={{ version }}",
+    "pydantic-ai-slim[openai,vertexai,google,groq,anthropic,mistral,cohere,bedrock,cli,mcp,evals,a2a]=={{ version }}",
 ]
 
 [tool.hatch.metadata.hooks.uv-dynamic-versioning.optional-dependencies]
@@ -60,7 +60,7 @@ Documentation = "https://ai.pydantic.dev"
 Changelog = "https://github.com/pydantic/pydantic-ai/releases"
 
 [project.scripts]
-pai = "pydantic_ai._cli:cli_exit"  # TODO remove this when clai has been out for a while
+pai = "pydantic_ai._cli:cli_exit" # TODO remove this when clai has been out for a while
 
 [tool.uv.sources]
 pydantic-ai-slim = { workspace = true }
@@ -170,7 +170,8 @@ include = [
     "examples",
     "clai",
 ]
-venvPath = ".venv"
+venvPath = '.'
+venv = ".venv"
 # see https://github.com/microsoft/pyright/issues/7771 - we don't want to error on decorated functions in tests
 # which are not otherwise used
 executionEnvironments = [

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/tests/conftest.py

@@ -65,7 +65,7 @@ class TestEnv:
             if value is None:
                 os.environ.pop(name, None)
             else:
-                os.environ[name] = value
+                os.environ[name] = value  # pragma: lax no cover
 
 
 @pytest.fixture
@@ -101,7 +101,7 @@ async def client_with_handler() -> AsyncIterator[ClientWithHandler]:
     try:
         yield create_client
     finally:
-        if client:
+        if client:  # pragma: no branch
             await client.aclose()
 
 
@@ -276,6 +276,11 @@ def mistral_api_key() -> str:
     return os.getenv('MISTRAL_API_KEY', 'mock-api-key')
 
 
+@pytest.fixture(scope='session')
+def openrouter_api_key() -> str:
+    return os.getenv('OPENROUTER_API_KEY', 'mock-api-key')
+
+
 @pytest.fixture(scope='session')
 def bedrock_provider():
     try:
@@ -291,7 +296,7 @@ def bedrock_provider():
         )
         yield BedrockProvider(bedrock_client=bedrock_client)
         bedrock_client.close()
-    except ImportError:
+    except ImportError:  # pragma: lax no cover
         pytest.skip('boto3 is not installed')
 
 

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/tests/evals/test_dataset.py

@@ -39,9 +39,9 @@ pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-eval
 
 
 if sys.version_info < (3, 11):
-    from exceptiongroup import ExceptionGroup
+    from exceptiongroup import ExceptionGroup  # pragma: lax no cover
 else:
-    ExceptionGroup = ExceptionGroup
+    ExceptionGroup = ExceptionGroup  # pragma: lax no cover
 
 
 @pytest.fixture(autouse=True)
@@ -636,7 +636,7 @@ async def test_from_text_failure():
     with pytest.raises(ExceptionGroup) as exc_info:
         Dataset[TaskInput, TaskOutput, TaskMetadata].from_text(json.dumps(dataset_dict))
     if sys.version_info >= (3, 10):
-        assert exc_info.value == HasRepr(
+        assert exc_info.value == HasRepr(  # pragma: lax no cover
             repr(
                 ExceptionGroup(
                     '2 error(s) loading evaluators from registry',
@@ -652,7 +652,7 @@ async def test_from_text_failure():
             )
         )
     else:
-        assert exc_info.value == HasRepr(
+        assert exc_info.value == HasRepr(  # pragma: lax no cover
             repr(
                 ExceptionGroup(
                     '2 error(s) loading evaluators from registry',

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/tests/evals/test_evaluator_base.py

@@ -86,7 +86,7 @@ def test_strict_abc_meta():
     assert 'evaluate' in str(exc_info.value)
 
 
-if TYPE_CHECKING or imports_successful():
+if TYPE_CHECKING or imports_successful():  # pragma: no branch
 
     @dataclass
     class SimpleEvaluator(Evaluator[Any, Any, Any]):
@@ -172,10 +172,11 @@ async def test_evaluator_async():
     assert result is True
 
 
-async def test_evaluator_name():
+async def test_evaluation_name():
     """Test evaluator name method."""
     evaluator = SimpleEvaluator()
-    assert evaluator.name() == 'SimpleEvaluator'
+    assert evaluator.get_serialization_name() == 'SimpleEvaluator'
+    assert evaluator.get_default_evaluation_name() == 'SimpleEvaluator'
 
 
 async def test_evaluator_serialization():

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/tests/evals/test_evaluator_common.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any
 
 import pytest
 from inline_snapshot import snapshot
+from pydantic_core import to_jsonable_python
 from pytest_mock import MockerFixture
 
 from pydantic_ai.settings import ModelSettings
@@ -25,6 +26,7 @@ with try_import() as imports_successful:
         IsInstance,
         LLMJudge,
         MaxDuration,
+        OutputConfig,
         Python,
     )
     from pydantic_evals.otel._context_in_memory_span_exporter import context_subtree
@@ -43,7 +45,7 @@ if TYPE_CHECKING or imports_successful():
             self.inputs = inputs
             self.duration = duration
 else:
-    MockContext = object
+    MockContext = object  # pragma: lax no cover
 
 
 async def test_equals():
@@ -194,6 +196,7 @@ async def test_llm_judge_evaluator(mocker: MockerFixture):
     """Test LLMJudge evaluator."""
     # Create a mock GradingOutput
     mock_grading_output = mocker.MagicMock()
+    mock_grading_output.score = 1.0
     mock_grading_output.pass_ = True
     mock_grading_output.reason = 'Test passed'
 
@@ -219,31 +222,42 @@ async def test_llm_judge_evaluator(mocker: MockerFixture):
 
     # Test without input
     evaluator = LLMJudge(rubric='Content contains a greeting')
-    result = await evaluator.evaluate(ctx)
-    assert isinstance(result, EvaluationReason)
-    assert result.value is True
-    assert result.reason == 'Test passed'
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': True, 'reason': 'Test passed'}}
+    )
 
     mock_judge_output.assert_called_once_with('Hello world', 'Content contains a greeting', None, None)
 
     # Test with input
     evaluator = LLMJudge(rubric='Output contains input', include_input=True, model='openai:gpt-4o')
-    result = await evaluator.evaluate(ctx)
-    assert isinstance(result, EvaluationReason)
-    assert result.value is True
-    assert result.reason == 'Test passed'
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': True, 'reason': 'Test passed'}}
+    )
 
     mock_judge_input_output.assert_called_once_with(
         {'prompt': 'Hello'}, 'Hello world', 'Output contains input', 'openai:gpt-4o', None
     )
 
     # Test with failing result
+    mock_grading_output.score = 0.0
     mock_grading_output.pass_ = False
     mock_grading_output.reason = 'Test failed'
-    result = await evaluator.evaluate(ctx)
-    assert isinstance(result, EvaluationReason)
-    assert result.value is False
-    assert result.reason == 'Test failed'
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': False, 'reason': 'Test failed'}}
+    )
+
+    # Test with overridden configs
+    evaluator = LLMJudge(rubric='Mock rubric', assertion=False)
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot({})
+
+    evaluator = LLMJudge(
+        rubric='Mock rubric',
+        score=OutputConfig(evaluation_name='my_score', include_reason=True),
+        assertion=OutputConfig(evaluation_name='my_assertion'),
+    )
+    assert to_jsonable_python(await evaluator.evaluate(ctx)) == snapshot(
+        {'my_assertion': False, 'my_score': {'reason': 'Test failed', 'value': 0.0}}
+    )
 
 
 @pytest.mark.anyio
@@ -275,9 +289,9 @@ async def test_llm_judge_evaluator_with_model_settings(mocker: MockerFixture):
 
     # Test without input, with custom model_settings
     evaluator_no_input = LLMJudge(rubric='Greeting with custom settings', model_settings=custom_model_settings)
-    result_no_input = await evaluator_no_input.evaluate(ctx)
-    assert result_no_input.value is True
-    assert result_no_input.reason == 'Test passed with settings'
+    assert to_jsonable_python(await evaluator_no_input.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': True, 'reason': 'Test passed with settings'}}
+    )
     mock_judge_output.assert_called_once_with(
         'Hello world custom settings', 'Greeting with custom settings', None, custom_model_settings
     )
@@ -289,9 +303,9 @@ async def test_llm_judge_evaluator_with_model_settings(mocker: MockerFixture):
         model='openai:gpt-3.5-turbo',
         model_settings=custom_model_settings,
     )
-    result_with_input = await evaluator_with_input.evaluate(ctx)
-    assert result_with_input.value is True
-    assert result_with_input.reason == 'Test passed with settings'
+    assert to_jsonable_python(await evaluator_with_input.evaluate(ctx)) == snapshot(
+        {'LLMJudge': {'value': True, 'reason': 'Test passed with settings'}}
+    )
     mock_judge_input_output.assert_called_once_with(
         {'prompt': 'Hello Custom'},
         'Hello world custom settings',

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/tests/evals/test_evaluators.py

@@ -6,6 +6,7 @@ from typing import Any, cast
 import pytest
 from inline_snapshot import snapshot
 from pydantic import BaseModel, TypeAdapter
+from pydantic_core import to_jsonable_python
 
 from pydantic_ai.messages import ModelMessage, ModelResponse
 from pydantic_ai.models import Model, ModelRequestParameters
@@ -207,12 +208,6 @@ async def test_is_instance_evaluator():
     assert result.value is False
 
 
-async def test_llm_judge_evaluator():
-    """Test the LLMJudge evaluator."""
-    # We can't easily test this without mocking the LLM, so we'll just check that it's importable
-    assert LLMJudge
-
-
 async def test_custom_evaluator(test_context: EvaluatorContext[TaskInput, TaskOutput, TaskMetadata]):
     """Test a custom evaluator."""
 
@@ -232,9 +227,41 @@ async def test_custom_evaluator(test_context: EvaluatorContext[TaskInput, TaskOu
 
     evaluator = CustomEvaluator()
     result = evaluator.evaluate(test_context)
-    assert isinstance(result, dict)
-    assert result['is_correct'] is True
-    assert result['difficulty'] == 'easy'
+    assert result == snapshot({'difficulty': 'easy', 'is_correct': True})
+
+
+async def test_custom_evaluator_name(test_context: EvaluatorContext[TaskInput, TaskOutput, TaskMetadata]):
+    @dataclass
+    class CustomNameFieldEvaluator(Evaluator[TaskInput, TaskOutput, TaskMetadata]):
+        result: int
+        evaluation_name: str
+
+        def evaluate(self, ctx: EvaluatorContext[TaskInput, TaskOutput, TaskMetadata]) -> EvaluatorOutput:
+            return self.result
+
+    evaluator = CustomNameFieldEvaluator(result=123, evaluation_name='abc')
+
+    assert to_jsonable_python(await run_evaluator(evaluator, test_context)) == snapshot(
+        [{'name': 'abc', 'reason': None, 'source': {'evaluation_name': 'abc', 'result': 123}, 'value': 123}]
+    )
+
+    @dataclass
+    class CustomNamePropertyEvaluator(Evaluator[TaskInput, TaskOutput, TaskMetadata]):
+        result: int
+        my_name: str
+
+        @property
+        def evaluation_name(self) -> str:
+            return f'hello {self.my_name}'
+
+        def evaluate(self, ctx: EvaluatorContext[TaskInput, TaskOutput, TaskMetadata]) -> EvaluatorOutput:
+            return self.result
+
+    evaluator = CustomNamePropertyEvaluator(result=123, my_name='marcelo')
+
+    assert to_jsonable_python(await run_evaluator(evaluator, test_context)) == snapshot(
+        [{'name': 'hello marcelo', 'reason': None, 'source': {'my_name': 'marcelo', 'result': 123}, 'value': 123}]
+    )
 
 
 async def test_evaluator_error_handling(test_context: EvaluatorContext[TaskInput, TaskOutput, TaskMetadata]):

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/tests/evals/test_render_numbers.py

@@ -21,8 +21,8 @@ pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-eval
     [
         (0, snapshot('0')),
         (0.0, snapshot('0.000')),
-        (17348, snapshot('17348')),
-        (-17348, snapshot('-17348')),
+        (17348, snapshot('17,348')),
+        (-17348, snapshot('-17,348')),
         (17347.0, snapshot('17,347.0')),
         (-17347.0, snapshot('-17,347.0')),
         (0.1234, snapshot('0.123')),

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/tests/evals/test_utils.py

@@ -11,9 +11,9 @@ from dirty_equals import HasRepr
 from ..conftest import try_import
 
 if sys.version_info < (3, 11):
-    from exceptiongroup import ExceptionGroup
+    from exceptiongroup import ExceptionGroup  # pragma: lax no cover
 else:
-    ExceptionGroup = ExceptionGroup
+    ExceptionGroup = ExceptionGroup  # pragma: lax no cover
 
 
 with try_import() as imports_successful:

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/tests/graph/test_graph.py

@@ -410,7 +410,7 @@ async def test_next(mock_snapshot_id: object):
     @dataclass
     class Bar(BaseNode):
         async def run(self, ctx: GraphRunContext) -> Foo:
-            return Foo()
+            return Foo()  # pragma: no cover
 
     g = Graph(nodes=(Foo, Bar))
     assert g.name is None

{pydantic_ai-0.2.3 → pydantic_ai-0.2.5}/tests/graph/test_mermaid.py

@@ -259,7 +259,7 @@ def httpx_with_handler() -> Iterator[HttpxWithHandler]:
     try:
         yield create_client
     finally:
-        if client:
+        if client:  # pragma: no branch
             client.close()
 
 

pydantic_ai-0.2.5/tests/models/cassettes/test_google/test_google_model.yaml

@@ -0,0 +1,67 @@
+interactions:
+- request:
+    headers:
+      accept:
+      - '*/*'
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '169'
+      content-type:
+      - application/json
+      host:
+      - generativelanguage.googleapis.com
+    method: POST
+    parsed_body:
+      contents:
+      - parts:
+        - text: Hello!
+        role: user
+      generationConfig: {}
+      systemInstruction:
+        parts:
+        - text: You are a chatbot.
+        role: user
+    uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash:generateContent
+  response:
+    headers:
+      alt-svc:
+      - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
+      content-length:
+      - '644'
+      content-type:
+      - application/json; charset=UTF-8
+      server-timing:
+      - gfet4t7; dur=322
+      transfer-encoding:
+      - chunked
+      vary:
+      - Origin
+      - X-Origin
+      - Referer
+    parsed_body:
+      candidates:
+      - avgLogprobs: -0.0009223055941137401
+        content:
+          parts:
+          - text: |
+              Hello there! How can I help you today?
+          role: model
+        finishReason: STOP
+      modelVersion: gemini-1.5-flash
+      usageMetadata:
+        candidatesTokenCount: 11
+        candidatesTokensDetails:
+        - modality: TEXT
+          tokenCount: 11
+        promptTokenCount: 7
+        promptTokensDetails:
+        - modality: TEXT
+          tokenCount: 7
+        totalTokenCount: 18
+    status:
+      code: 200
+      message: OK
+version: 1