pydantic-ai 0.4.11__tar.gz → 0.5.1__tar.gz

{pydantic_ai-0.4.11 → pydantic_ai-0.5.1}/Makefile

@@ -60,13 +60,10 @@ typecheck-both: typecheck-pyright typecheck-mypy
 
 .PHONY: test
 test: ## Run tests and collect coverage data
-	uv run coverage run -m pytest
+	COVERAGE_PROCESS_START=./pyproject.toml uv run coverage run -m pytest -n auto --dist=loadgroup
+	@uv run coverage combine
 	@uv run coverage report
 
-.PHONY: test-fast
-test-fast: ## Same as test except no coverage and 4x faster depending on hardware
-	uv run pytest -n auto --dist=loadgroup
-
 .PHONY: test-all-python
 test-all-python: ## Run tests on Python 3.9 to 3.13
 	UV_PROJECT_ENVIRONMENT=.venv39 uv run --python 3.9 --all-extras --all-packages coverage run -p -m pytest

{pydantic_ai-0.4.11 → pydantic_ai-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydantic-ai
-Version: 0.4.11
+Version: 0.5.1
 Summary: Agent Framework / shim to use Pydantic with LLMs
 Project-URL: Homepage, https://ai.pydantic.dev
 Project-URL: Source, https://github.com/pydantic/pydantic-ai
@@ -28,11 +28,11 @@ Classifier: Topic :: Internet
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.9
-Requires-Dist: pydantic-ai-slim[ag-ui,anthropic,bedrock,cli,cohere,evals,google,groq,huggingface,mcp,mistral,openai,retries,vertexai]==0.4.11
+Requires-Dist: pydantic-ai-slim[ag-ui,anthropic,bedrock,cli,cohere,evals,google,groq,huggingface,mcp,mistral,openai,retries,vertexai]==0.5.1
 Provides-Extra: a2a
 Requires-Dist: fasta2a>=0.4.1; extra == 'a2a'
 Provides-Extra: examples
-Requires-Dist: pydantic-ai-examples==0.4.11; extra == 'examples'
+Requires-Dist: pydantic-ai-examples==0.5.1; extra == 'examples'
 Provides-Extra: logfire
 Requires-Dist: logfire>=3.11.0; extra == 'logfire'
 Description-Content-Type: text/markdown

{pydantic_ai-0.4.11 → pydantic_ai-0.5.1}/pyproject.toml

@@ -215,6 +215,10 @@ filterwarnings = [
 
 # https://coverage.readthedocs.io/en/latest/config.html#run
 [tool.coverage.run]
+parallel = true
+concurrency = ["multiprocessing", "thread"]
+# We use a subdirectory for coverage data to avoid noisy coverage data files.
+data_file = ".coverage/.coverage"
 # required to avoid warnings about files created by create_module fixture
 include = [
     "pydantic_ai_slim/**/*.py",
@@ -245,6 +249,7 @@ source = [
 
 # https://coverage.readthedocs.io/en/latest/config.html#report
 [tool.coverage.report]
+fail_under = 100
 skip_covered = true
 show_missing = true
 ignore_errors = true

{pydantic_ai-0.4.11 → pydantic_ai-0.5.1}/tests/conftest.py

@@ -251,7 +251,7 @@ def pytest_recording_configure(config: Any, vcr: VCR):
 def mock_vcr_aiohttp_content(mocker: MockerFixture):
     try:
         from vcr.stubs import aiohttp_stubs
-    except ImportError:
+    except ImportError:  # pragma: lax no cover
         return
 
     # google-genai calls `self.response_stream.content.readline()` where `self.response_stream` is a `MockClientResponse`,
@@ -416,9 +416,9 @@ def vertex_provider_auth(mocker: MockerFixture) -> None:  # pragma: lax no cover
 
 
 @pytest.fixture()
-async def vertex_provider():
+async def vertex_provider():  # pragma: lax no cover
     # NOTE: You need to comment out this line to rewrite the cassettes locally.
-    if not os.getenv('CI', False):  # pragma: lax no cover
+    if not os.getenv('CI', False):
         pytest.skip('Requires properly configured local google vertex config to pass')
 
     try:
@@ -478,10 +478,10 @@ def model(
 
             return CohereModel('command-r-plus', provider=CohereProvider(api_key=co_api_key))
         elif request.param == 'gemini':
-            from pydantic_ai.models.gemini import GeminiModel
+            from pydantic_ai.models.gemini import GeminiModel  # type: ignore[reportDeprecated]
             from pydantic_ai.providers.google_gla import GoogleGLAProvider
 
-            return GeminiModel('gemini-1.5-flash', provider=GoogleGLAProvider(api_key=gemini_api_key))
+            return GeminiModel('gemini-1.5-flash', provider=GoogleGLAProvider(api_key=gemini_api_key))  # type: ignore[reportDeprecated]
         elif request.param == 'google':
             from pydantic_ai.models.google import GoogleModel
             from pydantic_ai.providers.google import GoogleProvider

{pydantic_ai-0.4.11 → pydantic_ai-0.5.1}/tests/evals/test_dataset.py

@@ -7,11 +7,11 @@ from pathlib import Path
 from typing import Any
 
 import pytest
-from dirty_equals import HasRepr, IsNumber
+from dirty_equals import HasRepr, IsNumber, IsPartialDict
 from inline_snapshot import snapshot
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 
-from ..conftest import IsStr, try_import
+from ..conftest import try_import
 from .utils import render_table
 
 with try_import() as imports_successful:
@@ -20,7 +20,7 @@ with try_import() as imports_successful:
 
     from pydantic_evals import Case, Dataset
     from pydantic_evals.dataset import increment_eval_metric, set_eval_attribute
-    from pydantic_evals.evaluators import EvaluationResult, Evaluator, EvaluatorOutput, LLMJudge, Python
+    from pydantic_evals.evaluators import EvaluationResult, Evaluator, EvaluatorOutput, EvaluatorSpec, LLMJudge, Python
     from pydantic_evals.evaluators.context import EvaluatorContext
 
     @dataclass
@@ -32,7 +32,7 @@ with try_import() as imports_successful:
         def evaluate(self, ctx: EvaluatorContext[object, object, object]) -> EvaluatorOutput:
             return self.output
 
-    from pydantic_evals.reporting import ReportCase, ReportCaseAdapter
+    from pydantic_evals.reporting import EvaluationReport, ReportCase, ReportCaseAdapter
 
 pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
 
@@ -456,13 +456,13 @@ async def test_repeated_name_outputs(example_dataset: Dataset[TaskInput, TaskOut
                 scores={},
                 labels={
                     'output': EvaluationResult(
-                        name='output', value='a', reason=None, source=MockEvaluator(output={'output': 'a'})
+                        name='output', value='a', reason=None, source=MockEvaluator(output={'output': 'a'}).as_spec()
                     ),
                     'output_2': EvaluationResult(
-                        name='output', value='b', reason=None, source=MockEvaluator(output={'output': 'b'})
+                        name='output', value='b', reason=None, source=MockEvaluator(output={'output': 'b'}).as_spec()
                     ),
                     'output_3': EvaluationResult(
-                        name='output', value='c', reason=None, source=MockEvaluator(output={'output': 'c'})
+                        name='output', value='c', reason=None, source=MockEvaluator(output={'output': 'c'}).as_spec()
                     ),
                 },
                 assertions={},
@@ -482,13 +482,13 @@ async def test_repeated_name_outputs(example_dataset: Dataset[TaskInput, TaskOut
                 scores={},
                 labels={
                     'output': EvaluationResult(
-                        name='output', value='a', reason=None, source=MockEvaluator(output={'output': 'a'})
+                        name='output', value='a', reason=None, source=MockEvaluator(output={'output': 'a'}).as_spec()
                     ),
                     'output_2': EvaluationResult(
-                        name='output', value='b', reason=None, source=MockEvaluator(output={'output': 'b'})
+                        name='output', value='b', reason=None, source=MockEvaluator(output={'output': 'b'}).as_spec()
                     ),
                     'output_3': EvaluationResult(
-                        name='output', value='c', reason=None, source=MockEvaluator(output={'output': 'c'})
+                        name='output', value='c', reason=None, source=MockEvaluator(output={'output': 'c'}).as_spec()
                     ),
                 },
                 assertions={},
@@ -501,6 +501,73 @@ async def test_repeated_name_outputs(example_dataset: Dataset[TaskInput, TaskOut
     )
 
 
+async def test_report_round_trip_serialization(example_dataset: Dataset[TaskInput, TaskOutput, TaskMetadata]):
+    """Test the increment_eval_metric function."""
+
+    async def my_task(inputs: TaskInput) -> TaskOutput:
+        return TaskOutput(answer=f'answer to {inputs.query}')
+
+    example_dataset.add_evaluator(MockEvaluator({'output': 'a'}))
+
+    report = await example_dataset.evaluate(my_task)
+    assert report == snapshot(
+        EvaluationReport(
+            name='my_task',
+            cases=[
+                ReportCase(
+                    name='case1',
+                    inputs=TaskInput(query='What is 2+2?'),
+                    metadata=TaskMetadata(difficulty='easy', category='general'),
+                    expected_output=TaskOutput(answer='4', confidence=1.0),
+                    output=TaskOutput(answer='answer to What is 2+2?', confidence=1.0),
+                    metrics={},
+                    attributes={},
+                    scores={},
+                    labels={
+                        'output': EvaluationResult(
+                            name='output',
+                            value='a',
+                            reason=None,
+                            source=EvaluatorSpec(name='MockEvaluator', arguments=({'output': 'a'},)),
+                        )
+                    },
+                    assertions={},
+                    task_duration=1.0,
+                    total_duration=6.0,
+                    trace_id='00000000000000000000000000000001',
+                    span_id='0000000000000003',
+                ),
+                ReportCase(
+                    name='case2',
+                    inputs=TaskInput(query='What is the capital of France?'),
+                    metadata=TaskMetadata(difficulty='medium', category='geography'),
+                    expected_output=TaskOutput(answer='Paris', confidence=1.0),
+                    output=TaskOutput(answer='answer to What is the capital of France?', confidence=1.0),
+                    metrics={},
+                    attributes={},
+                    scores={},
+                    labels={
+                        'output': EvaluationResult(
+                            name='output',
+                            value='a',
+                            reason=None,
+                            source=EvaluatorSpec(name='MockEvaluator', arguments=({'output': 'a'},)),
+                        )
+                    },
+                    assertions={},
+                    task_duration=1.0,
+                    total_duration=4.0,
+                    trace_id='00000000000000000000000000000001',
+                    span_id='0000000000000007',
+                ),
+            ],
+        )
+    )
+
+    report_adapter = TypeAdapter(EvaluationReport[TaskInput, TaskOutput, TaskMetadata])
+    assert report == report_adapter.validate_json(report_adapter.dump_json(report, indent=2))
+
+
 async def test_genai_attribute_collection(example_dataset: Dataset[TaskInput, TaskOutput, TaskMetadata]):
     async def my_task(inputs: TaskInput) -> TaskOutput:
         with logfire.span(
@@ -1092,7 +1159,7 @@ async def test_evaluate_async_logfire(
     example_dataset: Dataset[TaskInput, TaskOutput, TaskMetadata],
     simple_evaluator: type[Evaluator[TaskInput, TaskOutput, TaskMetadata]],
     capfire: CaptureLogfire,
-):
+):  # pragma: lax no cover
     """Test evaluating a dataset."""
     example_dataset.add_evaluator(simple_evaluator())
 
@@ -1130,90 +1197,90 @@ async def test_evaluate_async_logfire(
             'parent': None,
             'start_time': 1000000000,
         },
-        {
-            'attributes': {
-                'assertions': '{"correct":{"name":"correct","value":true,"reason":null,"source":{"name":"SimpleEvaluator","arguments":null}}}',
-                'attributes': '{}',
-                'case_name': 'case1',
-                'code.filepath': IsStr(),
-                'code.lineno': 123,
-                'expected_output': '{"answer":"4","confidence":1.0}',
-                'inputs': '{"query":"What is 2+2?"}',
-                'labels': '{}',
-                'logfire.json_schema': '{"type":"object","properties":{"task_name":{},"case_name":{},"inputs":{"type":"object","title":"TaskInput","x-python-datatype":"PydanticModel"},"metadata":{"type":"object","title":"TaskMetadata","x-python-datatype":"PydanticModel"},"expected_output":{"type":"object","title":"TaskOutput","x-python-datatype":"PydanticModel"},"output":{"type":"object","title":"TaskOutput","x-python-datatype":"PydanticModel"},"task_duration":{},"metrics":{"type":"object"},"attributes":{"type":"object"},"assertions":{"type":"object"},"scores":{"type":"object"},"labels":{"type":"object"}}}',
-                'logfire.msg': 'case: case1',
-                'logfire.msg_template': 'case: {case_name}',
-                'logfire.span_type': 'span',
-                'metadata': '{"difficulty":"easy","category":"general"}',
-                'metrics': '{}',
-                'output': '{"answer":"4","confidence":1.0}',
-                'scores': '{"confidence":{"name":"confidence","value":1.0,"reason":null,"source":{"name":"SimpleEvaluator","arguments":null}}}',
-                'task_duration': 1.0,
-                'task_name': 'mock_async_task',
-            },
-            'context': {'is_remote': False, 'span_id': 3, 'trace_id': 1},
-            'end_time': 8000000000,
-            'name': 'case: {case_name}',
-            'parent': {'is_remote': False, 'span_id': 1, 'trace_id': 1},
-            'start_time': 2000000000,
-        },
-        {
-            'attributes': {
-                'code.filepath': IsStr(),
-                'code.lineno': 123,
-                'logfire.json_schema': '{"type":"object","properties":{"task":{}}}',
-                'logfire.msg': 'execute mock_async_task',
-                'logfire.msg_template': 'execute {task}',
-                'logfire.span_type': 'span',
-                'task': 'mock_async_task',
-            },
-            'context': {'is_remote': False, 'span_id': 5, 'trace_id': 1},
-            'end_time': 4000000000,
-            'name': 'execute {task}',
-            'parent': {'is_remote': False, 'span_id': 3, 'trace_id': 1},
-            'start_time': 3000000000,
-        },
-        {
-            'attributes': {
-                'assertions': '{"correct":{"name":"correct","value":true,"reason":null,"source":{"name":"SimpleEvaluator","arguments":null}}}',
-                'attributes': '{}',
-                'case_name': 'case2',
-                'code.filepath': IsStr(),
-                'code.lineno': 123,
-                'expected_output': '{"answer":"Paris","confidence":1.0}',
-                'inputs': '{"query":"What is the capital of France?"}',
-                'labels': '{}',
-                'logfire.json_schema': '{"type":"object","properties":{"task_name":{},"case_name":{},"inputs":{"type":"object","title":"TaskInput","x-python-datatype":"PydanticModel"},"metadata":{"type":"object","title":"TaskMetadata","x-python-datatype":"PydanticModel"},"expected_output":{"type":"object","title":"TaskOutput","x-python-datatype":"PydanticModel"},"output":{"type":"object","title":"TaskOutput","x-python-datatype":"PydanticModel"},"task_duration":{},"metrics":{"type":"object"},"attributes":{"type":"object"},"assertions":{"type":"object"},"scores":{"type":"object"},"labels":{"type":"object"}}}',
-                'logfire.msg': 'case: case2',
-                'logfire.msg_template': 'case: {case_name}',
-                'logfire.span_type': 'span',
-                'metadata': '{"difficulty":"medium","category":"geography"}',
-                'metrics': '{}',
-                'output': '{"answer":"Paris","confidence":1.0}',
-                'scores': '{"confidence":{"name":"confidence","value":1.0,"reason":null,"source":{"name":"SimpleEvaluator","arguments":null}}}',
-                'task_duration': 1.0,
-                'task_name': 'mock_async_task',
-            },
-            'context': {'is_remote': False, 'span_id': 7, 'trace_id': 1},
-            'end_time': 9000000000,
-            'name': 'case: {case_name}',
-            'parent': {'is_remote': False, 'span_id': 1, 'trace_id': 1},
-            'start_time': 5000000000,
-        },
-        {
-            'attributes': {
-                'code.filepath': IsStr(),
-                'code.lineno': 123,
-                'logfire.json_schema': '{"type":"object","properties":{"task":{}}}',
-                'logfire.msg': 'execute mock_async_task',
-                'logfire.msg_template': 'execute {task}',
-                'logfire.span_type': 'span',
-                'task': 'mock_async_task',
-            },
-            'context': {'is_remote': False, 'span_id': 9, 'trace_id': 1},
-            'end_time': 7000000000,
-            'name': 'execute {task}',
-            'parent': {'is_remote': False, 'span_id': 7, 'trace_id': 1},
-            'start_time': 6000000000,
-        },
+        IsPartialDict(
+            {
+                'attributes': {
+                    'assertions': '{"correct":{"name":"correct","value":true,"reason":null,"source":{"name":"SimpleEvaluator","arguments":null}}}',
+                    'attributes': '{}',
+                    'case_name': 'case1',
+                    'expected_output': '{"answer":"4","confidence":1.0}',
+                    'inputs': '{"query":"What is 2+2?"}',
+                    'labels': '{}',
+                    'logfire.json_schema': '{"type":"object","properties":{"task_name":{},"case_name":{},"inputs":{"type":"object","title":"TaskInput","x-python-datatype":"PydanticModel"},"metadata":{"type":"object","title":"TaskMetadata","x-python-datatype":"PydanticModel"},"expected_output":{"type":"object","title":"TaskOutput","x-python-datatype":"PydanticModel"},"output":{"type":"object","title":"TaskOutput","x-python-datatype":"PydanticModel"},"task_duration":{},"metrics":{"type":"object"},"attributes":{"type":"object"},"assertions":{"type":"object"},"scores":{"type":"object"},"labels":{"type":"object"}}}',
+                    'logfire.msg': 'case: case1',
+                    'logfire.msg_template': 'case: {case_name}',
+                    'logfire.span_type': 'span',
+                    'metadata': '{"difficulty":"easy","category":"general"}',
+                    'metrics': '{}',
+                    'output': '{"answer":"4","confidence":1.0}',
+                    'scores': '{"confidence":{"name":"confidence","value":1.0,"reason":null,"source":{"name":"SimpleEvaluator","arguments":null}}}',
+                    'task_duration': 1.0,
+                    'task_name': 'mock_async_task',
+                },
+                'context': {'is_remote': False, 'span_id': 3, 'trace_id': 1},
+                'end_time': 8000000000,
+                'name': 'case: {case_name}',
+                'parent': {'is_remote': False, 'span_id': 1, 'trace_id': 1},
+                'start_time': 2000000000,
+            }
+        ),
+        IsPartialDict(
+            {
+                'attributes': {
+                    'logfire.json_schema': '{"type":"object","properties":{"task":{}}}',
+                    'logfire.msg': 'execute mock_async_task',
+                    'logfire.msg_template': 'execute {task}',
+                    'logfire.span_type': 'span',
+                    'task': 'mock_async_task',
+                },
+                'context': {'is_remote': False, 'span_id': 5, 'trace_id': 1},
+                'end_time': 4000000000,
+                'name': 'execute {task}',
+                'parent': {'is_remote': False, 'span_id': 3, 'trace_id': 1},
+                'start_time': 3000000000,
+            }
+        ),
+        IsPartialDict(
+            {
+                'attributes': {
+                    'assertions': '{"correct":{"name":"correct","value":true,"reason":null,"source":{"name":"SimpleEvaluator","arguments":null}}}',
+                    'attributes': '{}',
+                    'case_name': 'case2',
+                    'expected_output': '{"answer":"Paris","confidence":1.0}',
+                    'inputs': '{"query":"What is the capital of France?"}',
+                    'labels': '{}',
+                    'logfire.json_schema': '{"type":"object","properties":{"task_name":{},"case_name":{},"inputs":{"type":"object","title":"TaskInput","x-python-datatype":"PydanticModel"},"metadata":{"type":"object","title":"TaskMetadata","x-python-datatype":"PydanticModel"},"expected_output":{"type":"object","title":"TaskOutput","x-python-datatype":"PydanticModel"},"output":{"type":"object","title":"TaskOutput","x-python-datatype":"PydanticModel"},"task_duration":{},"metrics":{"type":"object"},"attributes":{"type":"object"},"assertions":{"type":"object"},"scores":{"type":"object"},"labels":{"type":"object"}}}',
+                    'logfire.msg': 'case: case2',
+                    'logfire.msg_template': 'case: {case_name}',
+                    'logfire.span_type': 'span',
+                    'metadata': '{"difficulty":"medium","category":"geography"}',
+                    'metrics': '{}',
+                    'output': '{"answer":"Paris","confidence":1.0}',
+                    'scores': '{"confidence":{"name":"confidence","value":1.0,"reason":null,"source":{"name":"SimpleEvaluator","arguments":null}}}',
+                    'task_duration': 1.0,
+                    'task_name': 'mock_async_task',
+                },
+                'context': {'is_remote': False, 'span_id': 7, 'trace_id': 1},
+                'end_time': 9000000000,
+                'name': 'case: {case_name}',
+                'parent': {'is_remote': False, 'span_id': 1, 'trace_id': 1},
+                'start_time': 5000000000,
+            }
+        ),
+        IsPartialDict(
+            {
+                'attributes': {
+                    'logfire.json_schema': '{"type":"object","properties":{"task":{}}}',
+                    'logfire.msg': 'execute mock_async_task',
+                    'logfire.msg_template': 'execute {task}',
+                    'logfire.span_type': 'span',
+                    'task': 'mock_async_task',
+                },
+                'context': {'is_remote': False, 'span_id': 9, 'trace_id': 1},
+                'end_time': 7000000000,
+                'name': 'execute {task}',
+                'parent': {'is_remote': False, 'span_id': 7, 'trace_id': 1},
+                'start_time': 6000000000,
+            }
+        ),
     ]

{pydantic_ai-0.4.11 → pydantic_ai-0.5.1}/tests/evals/test_evaluator_base.py

@@ -52,11 +52,11 @@ def test_evaluation_result():
     evaluator = DummyEvaluator()
 
     # Test basic result
-    result = EvaluationResult(name='test', value=True, reason='Success', source=evaluator)
+    result = EvaluationResult(name='test', value=True, reason='Success', source=evaluator.as_spec())
     assert result.name == 'test'
     assert result.value is True
     assert result.reason == 'Success'
-    assert result.source == evaluator
+    assert result.source == evaluator.as_spec()
 
     # Test downcast with matching type
     downcast = result.downcast(bool)

{pydantic_ai-0.4.11 → pydantic_ai-0.5.1}/tests/evals/test_evaluator_spec.py

@@ -6,7 +6,7 @@ from pydantic import ValidationError
 from ..conftest import try_import
 
 with try_import() as imports_successful:
-    from pydantic_evals.evaluators._spec import (
+    from pydantic_evals.evaluators.spec import (
         EvaluatorSpec,
         _SerializedEvaluatorSpec,  # pyright: ignore[reportPrivateUsage]
     )

{pydantic_ai-0.4.11 → pydantic_ai-0.5.1}/tests/evals/test_evaluators.py

@@ -19,7 +19,6 @@ with try_import() as imports_successful:
     from logfire.testing import CaptureLogfire
 
     from pydantic_evals.evaluators._run_evaluator import run_evaluator
-    from pydantic_evals.evaluators._spec import EvaluatorSpec
     from pydantic_evals.evaluators.common import (
         Contains,
         Equals,
@@ -36,6 +35,7 @@ with try_import() as imports_successful:
         Evaluator,
         EvaluatorOutput,
     )
+    from pydantic_evals.evaluators.spec import EvaluatorSpec
     from pydantic_evals.otel._context_in_memory_span_exporter import context_subtree
     from pydantic_evals.otel.span_tree import SpanQuery, SpanTree
 
@@ -162,7 +162,7 @@ async def test_evaluator_call(test_context: EvaluatorContext[TaskInput, TaskOutp
     assert results[0].name == 'result'
     assert results[0].value == 'passed'
     assert results[0].reason is None
-    assert results[0].source is evaluator
+    assert results[0].source == EvaluatorSpec(name='ExampleEvaluator', arguments=None)
 
 
 async def test_is_instance_evaluator():
@@ -242,7 +242,14 @@ async def test_custom_evaluator_name(test_context: EvaluatorContext[TaskInput, T
     evaluator = CustomNameFieldEvaluator(result=123, evaluation_name='abc')
 
     assert to_jsonable_python(await run_evaluator(evaluator, test_context)) == snapshot(
-        [{'name': 'abc', 'reason': None, 'source': {'evaluation_name': 'abc', 'result': 123}, 'value': 123}]
+        [
+            {
+                'name': 'abc',
+                'reason': None,
+                'source': {'arguments': {'evaluation_name': 'abc', 'result': 123}, 'name': 'CustomNameFieldEvaluator'},
+                'value': 123,
+            }
+        ]
     )
 
     @dataclass
@@ -260,7 +267,14 @@ async def test_custom_evaluator_name(test_context: EvaluatorContext[TaskInput, T
     evaluator = CustomNamePropertyEvaluator(result=123, my_name='marcelo')
 
     assert to_jsonable_python(await run_evaluator(evaluator, test_context)) == snapshot(
-        [{'name': 'hello marcelo', 'reason': None, 'source': {'my_name': 'marcelo', 'result': 123}, 'value': 123}]
+        [
+            {
+                'name': 'hello marcelo',
+                'reason': None,
+                'source': {'arguments': {'my_name': 'marcelo', 'result': 123}, 'name': 'CustomNamePropertyEvaluator'},
+                'value': 123,
+            }
+        ]
     )