kiln-ai 0.0.4__py3-none-any.whl → 0.5.1__py3-none-any.whl

kiln_ai/adapters/test_prompt_builders.py

@@ -0,0 +1,315 @@
+import json
+
+import pytest
+from kiln_ai.adapters.base_adapter import AdapterInfo, BaseAdapter
+from kiln_ai.adapters.prompt_builders import (
+    FewShotPromptBuilder,
+    MultiShotPromptBuilder,
+    SimplePromptBuilder,
+    prompt_builder_from_ui_name,
+)
+from kiln_ai.adapters.test_prompt_adaptors import build_test_task
+from kiln_ai.adapters.test_structured_output import build_structured_output_test_task
+from kiln_ai.datamodel import (
+    DataSource,
+    DataSourceType,
+    Project,
+    Task,
+    TaskOutput,
+    TaskOutputRating,
+    TaskRun,
+)
+
+
+def test_simple_prompt_builder(tmp_path):
+    task = build_test_task(tmp_path)
+    builder = SimplePromptBuilder(task=task)
+    input = "two plus two"
+    prompt = builder.build_prompt()
+    assert (
+        "You are an assistant which performs math tasks provided in plain text."
+        in prompt
+    )
+
+    assert "1) " + task.requirements[0].instruction in prompt
+    assert "2) " + task.requirements[1].instruction in prompt
+    assert "3) " + task.requirements[2].instruction in prompt
+
+    user_msg = builder.build_user_message(input)
+    assert input in user_msg
+    assert input not in prompt
+
+
+class MockAdapter(BaseAdapter):
+    def adapter_specific_instructions(self) -> str | None:
+        return "You are a mock, send me the response!"
+
+    def _run(self, input: str) -> str:
+        return "mock response"
+
+    def adapter_info(self) -> AdapterInfo:
+        return AdapterInfo(
+            adapter_name="mock_adapter",
+            model_name="mock_model",
+            model_provider="mock_provider",
+        )
+
+
+def test_simple_prompt_builder_structured_output(tmp_path):
+    task = build_structured_output_test_task(tmp_path)
+    builder = SimplePromptBuilder(task=task)
+    input = "Cows"
+    prompt = builder.build_prompt()
+    assert "You are an assistant which tells a joke, given a subject." in prompt
+
+    # check adapter instructions are included
+    run_adapter = MockAdapter(task, prompt_builder=builder)
+    assert "You are a mock, send me the response!" in run_adapter.build_prompt()
+
+    user_msg = builder.build_user_message(input)
+    assert input in user_msg
+    assert input not in prompt
+
+
+def test_multi_shot_prompt_builder(tmp_path):
+    # Create a project and task hierarchy
+    project = Project(name="Test Project", path=(tmp_path / "test_project.kiln"))
+    project.save_to_file()
+    task = Task(
+        name="Test Task",
+        instruction="You are an assistant which tells a joke, given a subject.",
+        parent=project,
+        input_json_schema=json.dumps(
+            {
+                "type": "object",
+                "properties": {
+                    "subject": {"type": "string"},
+                },
+                "required": ["subject"],
+            }
+        ),
+        output_json_schema=json.dumps(
+            {
+                "type": "object",
+                "properties": {"joke": {"type": "string"}},
+                "required": ["joke"],
+            }
+        ),
+    )
+    task.save_to_file()
+
+    check_example_outputs(task, 0)
+
+    # Create an task input, but with no output
+    e1 = TaskRun(
+        input='{"subject": "Cows"}',
+        input_source=DataSource(
+            type=DataSourceType.human,
+            properties={"created_by": "john_doe"},
+        ),
+        parent=task,
+        output=TaskOutput(
+            output='{"joke": "Moo I am a cow joke."}',
+            source=DataSource(
+                type=DataSourceType.human,
+                properties={"created_by": "john_doe"},
+            ),
+        ),
+    )
+    e1.save_to_file()
+
+    ## still zero since not fixed and not rated highly
+    check_example_outputs(task, 0)
+
+    e1.output.rating = TaskOutputRating(value=4)
+    e1.save_to_file()
+    # Now that it's highly rated, it should be included
+    check_example_outputs(task, 1)
+
+    # Test with repaired output (highest priority)
+    e1 = TaskRun(
+        input='{"subject": "Cows"}',
+        input_source=DataSource(
+            type=DataSourceType.human,
+            properties={"created_by": "john_doe"},
+        ),
+        parent=task,
+        output=TaskOutput(
+            output='{"joke": "Moo I am a cow joke."}',
+            source=DataSource(
+                type=DataSourceType.human,
+                properties={"created_by": "john_doe"},
+            ),
+        ),
+        repair_instructions="Fix the joke",
+        repaired_output=TaskOutput(
+            output='{"joke": "Why did the cow cross the road? To get to the udder side!"}',
+            source=DataSource(
+                type=DataSourceType.human,
+                properties={"created_by": "jane_doe"},
+            ),
+        ),
+    )
+    e1.save_to_file()
+    check_example_outputs(task, 1)
+
+    # Test with high-quality output (second priority)
+    e2 = TaskRun(
+        input='{"subject": "Dogs"}',
+        input_source=DataSource(
+            type=DataSourceType.human,
+            properties={"created_by": "john_doe"},
+        ),
+        parent=task,
+        output=TaskOutput(
+            output='{"joke": "Why did the dog get a job? He wanted to be a collar-ary!"}',
+            source=DataSource(
+                type=DataSourceType.human,
+                properties={"created_by": "john_doe"},
+            ),
+            rating=TaskOutputRating(value=4, reason="Good pun"),
+        ),
+    )
+    e2.save_to_file()
+    check_example_outputs(task, 2)
+
+    # Test sorting by rating value
+    e3 = TaskRun(
+        input='{"subject": "Cats"}',
+        input_source=DataSource(
+            type=DataSourceType.human,
+            properties={"created_by": "john_doe"},
+        ),
+        parent=task,
+        output=TaskOutput(
+            output='{"joke": "Why don\'t cats play poker in the jungle? Too many cheetahs!"}',
+            source=DataSource(
+                type=DataSourceType.human,
+                properties={"created_by": "john_doe"},
+            ),
+            rating=TaskOutputRating(value=5, reason="Excellent joke"),
+        ),
+    )
+    e3.save_to_file()
+    check_example_outputs(task, 3)
+
+    # Verify the order of examples
+    prompt_builder = MultiShotPromptBuilder(task=task)
+    prompt = prompt_builder.build_prompt()
+    assert "Why did the cow cross the road?" in prompt
+    assert prompt.index("Why did the cow cross the road?") < prompt.index(
+        "Why don't cats play poker in the jungle?"
+    )
+    assert prompt.index("Why don't cats play poker in the jungle?") < prompt.index(
+        "Why did the dog get a job?"
+    )
+
+
+# Add a new test for the FewShotPromptBuilder
+def test_few_shot_prompt_builder(tmp_path):
+    # Create a project and task hierarchy (similar to test_multi_shot_prompt_builder)
+    project = Project(name="Test Project", path=(tmp_path / "test_project.kiln"))
+    project.save_to_file()
+    task = Task(
+        name="Test Task",
+        instruction="You are an assistant which tells a joke, given a subject.",
+        parent=project,
+        input_json_schema=json.dumps(
+            {
+                "type": "object",
+                "properties": {
+                    "subject": {"type": "string"},
+                },
+                "required": ["subject"],
+            }
+        ),
+        output_json_schema=json.dumps(
+            {
+                "type": "object",
+                "properties": {"joke": {"type": "string"}},
+                "required": ["joke"],
+            }
+        ),
+    )
+    task.save_to_file()
+
+    # Create 6 examples (2 repaired, 4 high-quality)
+    for i in range(6):
+        run = TaskRun(
+            input=f'{{"subject": "Subject {i+1}"}}',
+            input_source=DataSource(
+                type=DataSourceType.human,
+                properties={"created_by": "john_doe"},
+            ),
+            parent=task,
+            output=TaskOutput(
+                output=f'{{"joke": "Joke Initial Output {i+1}"}}',
+                source=DataSource(
+                    type=DataSourceType.human,
+                    properties={"created_by": "john_doe"},
+                ),
+                rating=TaskOutputRating(value=4 + (i % 2), reason="Good joke"),
+            ),
+        )
+        print("RATING", "Joke Initial Output ", i + 1, " - RATED:", 4 + (i % 2), "\n")
+        if i < 2:
+            run = run.model_copy(
+                update={
+                    "repair_instructions": "Fix the joke",
+                    "repaired_output": TaskOutput(
+                        output=f'{{"joke": "Repaired Joke {i+1}"}}',
+                        source=DataSource(
+                            type=DataSourceType.human,
+                            properties={"created_by": "jane_doe"},
+                        ),
+                    ),
+                }
+            )
+        run.save_to_file()
+
+    # Check that only 4 examples are included
+    prompt_builder = FewShotPromptBuilder(task=task)
+    prompt = prompt_builder.build_prompt()
+    assert prompt.count("## Example") == 4
+
+    print("PROMPT", prompt)
+    # Verify the order of examples (2 repaired, then 2 highest-rated)
+    assert "Repaired Joke 1" in prompt
+    assert "Repaired Joke 2" in prompt
+    assert "Joke Initial Output 6" in prompt  # Rating 5
+    assert "Joke Initial Output 4" in prompt  # Rating 5
+    assert "Joke Initial Output 5" not in prompt  # Rating 4, not included
+    assert "Joke Initial Output 3" not in prompt  # Rating 4, not included
+    assert "Joke Initial Output 1" not in prompt  # Repaired, so using that
+    assert "Joke Initial Output 2" not in prompt  # Repaired, so using that
+
+
+def check_example_outputs(task: Task, count: int):
+    prompt_builder = MultiShotPromptBuilder(task=task)
+    prompt = prompt_builder.build_prompt()
+    assert "# Instruction" in prompt
+    assert task.instruction in prompt
+    if count == 0:
+        assert "# Example Outputs" not in prompt
+    else:
+        assert "# Example Outputs" in prompt
+        assert f"## Example {count}" in prompt
+
+
+def test_prompt_builder_name():
+    assert SimplePromptBuilder.prompt_builder_name() == "simple_prompt_builder"
+    assert MultiShotPromptBuilder.prompt_builder_name() == "multi_shot_prompt_builder"
+
+
+def test_prompt_builder_from_ui_name():
+    assert prompt_builder_from_ui_name("basic") == SimplePromptBuilder
+    assert prompt_builder_from_ui_name("few_shot") == FewShotPromptBuilder
+    assert prompt_builder_from_ui_name("many_shot") == MultiShotPromptBuilder
+
+    with pytest.raises(ValueError, match="Unknown prompt builder: invalid_name"):
+        prompt_builder_from_ui_name("invalid_name")
+
+
+def test_example_count():
+    assert FewShotPromptBuilder.example_count() == 4
+    assert MultiShotPromptBuilder.example_count() == 25

kiln_ai/adapters/test_saving_adapter_results.py

@@ -0,0 +1,168 @@
+from unittest.mock import patch
+
+import pytest
+from kiln_ai.adapters.base_adapter import AdapterInfo, BaseAdapter
+from kiln_ai.datamodel import (
+    DataSource,
+    DataSourceType,
+    Project,
+    Task,
+)
+from kiln_ai.utils.config import Config
+
+
+class MockAdapter(BaseAdapter):
+    async def _run(self, input: dict | str) -> dict | str:
+        return "Test output"
+
+    def adapter_info(self) -> AdapterInfo:
+        return AdapterInfo(
+            adapter_name="mock_adapter",
+            model_name="mock_model",
+            model_provider="mock_provider",
+            prompt_builder_name="mock_prompt_builder",
+        )
+
+
+@pytest.fixture
+def test_task(tmp_path):
+    project = Project(name="test_project", path=tmp_path / "test_project.kiln")
+    project.save_to_file()
+    task = Task(
+        parent=project,
+        name="test_task",
+        instruction="Task instruction",
+    )
+    task.save_to_file()
+    return task
+
+
+def test_save_run_isolation(test_task):
+    adapter = MockAdapter(test_task)
+    input_data = "Test input"
+    output_data = "Test output"
+
+    task_run = adapter.generate_run(
+        input=input_data, input_source=None, output=output_data
+    )
+    task_run.save_to_file()
+
+    # Check that the task input was saved correctly
+    assert task_run.parent == test_task
+    assert task_run.input == input_data
+    assert task_run.input_source.type == DataSourceType.human
+    created_by = Config.shared().user_id
+    if created_by and created_by != "":
+        assert task_run.input_source.properties["created_by"] == created_by
+    else:
+        assert "created_by" not in task_run.input_source.properties
+
+    # Check that the task output was saved correctly
+    saved_output = task_run.output
+    assert saved_output.output == output_data
+    assert saved_output.source.type == DataSourceType.synthetic
+    assert saved_output.rating is None
+
+    # Verify that the data can be read back from disk
+    reloaded_task = Task.load_from_file(test_task.path)
+    reloaded_runs = reloaded_task.runs()
+    assert len(reloaded_runs) == 1
+    reloaded_run = reloaded_runs[0]
+    assert reloaded_run.input == input_data
+    assert reloaded_run.input_source.type == DataSourceType.human
+    reloaded_output = reloaded_run.output
+
+    reloaded_output = reloaded_run.output
+    assert reloaded_output.output == output_data
+    assert reloaded_output.source.type == DataSourceType.synthetic
+    assert reloaded_output.rating is None
+    assert reloaded_output.source.properties["adapter_name"] == "mock_adapter"
+    assert reloaded_output.source.properties["model_name"] == "mock_model"
+    assert reloaded_output.source.properties["model_provider"] == "mock_provider"
+    assert (
+        reloaded_output.source.properties["prompt_builder_name"]
+        == "mock_prompt_builder"
+    )
+
+    # Run again, with same input and different output. Should create a new TaskRun.
+    task_output = adapter.generate_run(input_data, None, "Different output")
+    task_output.save_to_file()
+    assert len(test_task.runs()) == 2
+    assert "Different output" in set(run.output.output for run in test_task.runs())
+
+    # run again with same input and same output. Should not create a new TaskRun.
+    task_output = adapter.generate_run(input_data, None, output_data)
+    task_output.save_to_file()
+    assert len(test_task.runs()) == 2
+    assert "Different output" in set(run.output.output for run in test_task.runs())
+    assert output_data in set(run.output.output for run in test_task.runs())
+
+    # run again with input of different type. Should create a new TaskRun and TaskOutput.
+    task_output = adapter.generate_run(
+        input_data,
+        DataSource(
+            type=DataSourceType.synthetic,
+            properties={
+                "model_name": "mock_model",
+                "model_provider": "mock_provider",
+                "prompt_builder_name": "mock_prompt_builder",
+                "adapter_name": "mock_adapter",
+            },
+        ),
+        output_data,
+    )
+    task_output.save_to_file()
+    assert len(test_task.runs()) == 3
+    assert task_output.input == input_data
+    assert task_output.input_source.type == DataSourceType.synthetic
+    assert "Different output" in set(run.output.output for run in test_task.runs())
+    assert output_data in set(run.output.output for run in test_task.runs())
+
+
+@pytest.mark.asyncio
+async def test_autosave_false(test_task):
+    with patch("kiln_ai.utils.config.Config.shared") as mock_shared:
+        mock_config = mock_shared.return_value
+        mock_config.autosave_runs = False
+        mock_config.user_id = "test_user"
+
+        adapter = MockAdapter(test_task)
+        input_data = "Test input"
+
+        run = await adapter.invoke(input_data)
+
+        # Check that no runs were saved
+        assert len(test_task.runs()) == 0
+
+        # Check that the run ID is not set
+        assert run.id is None
+
+
+@pytest.mark.asyncio
+async def test_autosave_true(test_task):
+    with patch("kiln_ai.utils.config.Config.shared") as mock_shared:
+        mock_config = mock_shared.return_value
+        mock_config.autosave_runs = True
+        mock_config.user_id = "test_user"
+
+        adapter = MockAdapter(test_task)
+        input_data = "Test input"
+
+        run = await adapter.invoke(input_data)
+
+        # Check that the run ID is set
+        assert run.id is not None
+
+        # Check that an task input was saved
+        task_runs = test_task.runs()
+        assert len(task_runs) == 1
+        assert task_runs[0].input == input_data
+        assert task_runs[0].input_source.type == DataSourceType.human
+
+        output = task_runs[0].output
+        assert output.output == "Test output"
+        assert output.source.type == DataSourceType.synthetic
+        assert output.source.properties["adapter_name"] == "mock_adapter"
+        assert output.source.properties["model_name"] == "mock_model"
+        assert output.source.properties["model_provider"] == "mock_provider"
+        assert output.source.properties["prompt_builder_name"] == "mock_prompt_builder"

kiln_ai/adapters/test_structured_output.py

@@ -0,0 +1,218 @@
+from pathlib import Path
+from typing import Dict
+
+import jsonschema
+import jsonschema.exceptions
+import kiln_ai.datamodel as datamodel
+import pytest
+from kiln_ai.adapters.base_adapter import AdapterInfo, BaseAdapter
+from kiln_ai.adapters.langchain_adapters import LangChainPromptAdapter
+from kiln_ai.adapters.ml_model_list import (
+    built_in_models,
+    ollama_online,
+)
+from kiln_ai.datamodel.test_json_schema import json_joke_schema, json_triangle_schema
+
+
+@pytest.mark.paid
+async def test_structured_output_groq(tmp_path):
+    await run_structured_output_test(tmp_path, "llama_3_1_8b", "groq")
+
+
+@pytest.mark.paid
+async def test_structured_output_openrouter(tmp_path):
+    await run_structured_output_test(tmp_path, "mistral_nemo", "openrouter")
+
+
+@pytest.mark.paid
+async def test_structured_output_bedrock(tmp_path):
+    await run_structured_output_test(tmp_path, "llama_3_1_70b", "amazon_bedrock")
+
+
+@pytest.mark.ollama
+async def test_structured_output_ollama_phi(tmp_path):
+    # https://python.langchain.com/v0.2/docs/how_to/structured_output/#advanced-specifying-the-method-for-structuring-outputs
+    pytest.skip(
+        "not working yet - phi3.5 does not support tools. Need json_mode + format in prompt"
+    )
+    await run_structured_output_test(tmp_path, "phi_3_5", "ollama")
+
+
+@pytest.mark.ollama
+async def test_structured_output_gpt_4o_mini(tmp_path):
+    await run_structured_output_test(tmp_path, "gpt_4o_mini", "openai")
+
+
+@pytest.mark.ollama
+async def test_structured_output_ollama_llama(tmp_path):
+    if not await ollama_online():
+        pytest.skip("Ollama API not running. Expect it running on localhost:11434")
+    await run_structured_output_test(tmp_path, "llama_3_1_8b", "ollama")
+
+
+class MockAdapter(BaseAdapter):
+    def __init__(self, kiln_task: datamodel.Task, response: Dict | str | None):
+        super().__init__(kiln_task)
+        self.response = response
+
+    async def _run(self, input: str) -> Dict | str:
+        return self.response
+
+    def adapter_info(self) -> AdapterInfo:
+        return AdapterInfo(
+            adapter_name="mock_adapter",
+            model_name="mock_model",
+            model_provider="mock_provider",
+            prompt_builder_name="mock_prompt_builder",
+        )
+
+
+async def test_mock_unstructred_response(tmp_path):
+    task = build_structured_output_test_task(tmp_path)
+
+    # don't error on valid response
+    adapter = MockAdapter(task, response={"setup": "asdf", "punchline": "asdf"})
+    answer = await adapter.invoke_returning_raw("You are a mock, send me the response!")
+    assert answer["setup"] == "asdf"
+    assert answer["punchline"] == "asdf"
+
+    # error on response that doesn't match schema
+    adapter = MockAdapter(task, response={"setup": "asdf"})
+    with pytest.raises(Exception):
+        answer = await adapter.invoke("You are a mock, send me the response!")
+
+    adapter = MockAdapter(task, response="string instead of dict")
+    with pytest.raises(RuntimeError):
+        # Not a structed response so should error
+        answer = await adapter.invoke("You are a mock, send me the response!")
+
+    # Should error, expecting a string, not a dict
+    project = datamodel.Project(name="test", path=tmp_path / "test.kiln")
+    task = datamodel.Task(
+        parent=project,
+        name="test task",
+        instruction="You are an assistant which performs math tasks provided in plain text.",
+    )
+    task.instruction = (
+        "You are an assistant which performs math tasks provided in plain text."
+    )
+    adapter = MockAdapter(task, response={"dict": "value"})
+    with pytest.raises(RuntimeError):
+        answer = await adapter.invoke("You are a mock, send me the response!")
+
+
+@pytest.mark.paid
+@pytest.mark.ollama
+async def test_all_built_in_models_structured_output(tmp_path):
+    for model in built_in_models:
+        if not model.supports_structured_output:
+            print(
+                f"Skipping {model.name} because it does not support structured output"
+            )
+            continue
+        for provider in model.providers:
+            if not provider.supports_structured_output:
+                print(
+                    f"Skipping {model.name} {provider.name} because it does not support structured output"
+                )
+                continue
+            try:
+                print(f"Running {model.name} {provider.name}")
+                await run_structured_output_test(tmp_path, model.name, provider.name)
+            except Exception as e:
+                raise RuntimeError(f"Error running {model.name} {provider}") from e
+
+
+def build_structured_output_test_task(tmp_path: Path):
+    project = datamodel.Project(name="test", path=tmp_path / "test.kiln")
+    project.save_to_file()
+    task = datamodel.Task(
+        parent=project,
+        name="test task",
+        instruction="You are an assistant which tells a joke, given a subject.",
+    )
+    task.output_json_schema = json_joke_schema
+    schema = task.output_schema()
+    assert schema is not None
+    assert schema["properties"]["setup"]["type"] == "string"
+    assert schema["properties"]["punchline"]["type"] == "string"
+    task.save_to_file()
+    assert task.name == "test task"
+    assert len(task.requirements) == 0
+    return task
+
+
+async def run_structured_output_test(tmp_path: Path, model_name: str, provider: str):
+    task = build_structured_output_test_task(tmp_path)
+    a = LangChainPromptAdapter(task, model_name=model_name, provider=provider)
+    parsed = await a.invoke_returning_raw("Cows")  # a joke about cows
+    if parsed is None or not isinstance(parsed, Dict):
+        raise RuntimeError(f"structured response is not a dict: {parsed}")
+    assert parsed["setup"] is not None
+    assert parsed["punchline"] is not None
+    if "rating" in parsed and parsed["rating"] is not None:
+        rating = parsed["rating"]
+        # Note: really should be an int according to json schema, but mistral returns a string
+        if isinstance(rating, str):
+            rating = int(rating)
+        assert rating >= 0
+        assert rating <= 10
+
+
+def build_structured_input_test_task(tmp_path: Path):
+    project = datamodel.Project(name="test", path=tmp_path / "test.kiln")
+    project.save_to_file()
+    task = datamodel.Task(
+        parent=project,
+        name="test task",
+        instruction="You are an assistant which classifies a triangle given the lengths of its sides. If all sides are of equal length, the triangle is equilateral. If two sides are equal, the triangle is isosceles. Otherwise, it is scalene.\n\nAt the end of your response return the result in double square brackets. It should be plain text. It should be exactly one of the three following strings: '[[equilateral]]', or '[[isosceles]]', or '[[scalene]]'.",
+    )
+    task.input_json_schema = json_triangle_schema
+    schema = task.input_schema()
+    assert schema is not None
+    assert schema["properties"]["a"]["type"] == "integer"
+    assert schema["properties"]["b"]["type"] == "integer"
+    assert schema["properties"]["c"]["type"] == "integer"
+    assert schema["required"] == ["a", "b", "c"]
+    task.save_to_file()
+    assert task.name == "test task"
+    assert len(task.requirements) == 0
+    return task
+
+
+async def run_structured_input_test(tmp_path: Path, model_name: str, provider: str):
+    task = build_structured_input_test_task(tmp_path)
+    a = LangChainPromptAdapter(task, model_name=model_name, provider=provider)
+    with pytest.raises(ValueError):
+        # not structured input in dictionary
+        await a.invoke("a=1, b=2, c=3")
+    with pytest.raises(jsonschema.exceptions.ValidationError):
+        # invalid structured input
+        await a.invoke({"a": 1, "b": 2, "d": 3})
+
+    response = await a.invoke_returning_raw({"a": 2, "b": 2, "c": 2})
+    assert response is not None
+    assert isinstance(response, str)
+    assert "[[equilateral]]" in response
+    adapter_info = a.adapter_info()
+    assert adapter_info.prompt_builder_name == "SimplePromptBuilder"
+    assert adapter_info.model_name == model_name
+    assert adapter_info.model_provider == provider
+    assert adapter_info.adapter_name == "kiln_langchain_adapter"
+
+
+@pytest.mark.paid
+async def test_structured_input_gpt_4o_mini(tmp_path):
+    await run_structured_input_test(tmp_path, "llama_3_1_8b", "groq")
+
+
+@pytest.mark.paid
+@pytest.mark.ollama
+async def test_all_built_in_models_structured_input(tmp_path):
+    for model in built_in_models:
+        for provider in model.providers:
+            try:
+                print(f"Running {model.name} {provider.name}")
+                await run_structured_input_test(tmp_path, model.name, provider.name)
+            except Exception as e:
+                raise RuntimeError(f"Error running {model.name} {provider}") from e