kiln-ai 0.11.1__py3-none-any.whl → 0.12.0__py3-none-any.whl

kiln_ai/adapters/model_adapters/openai_model_adapter.py

@@ -9,18 +9,23 @@ from openai.types.chat import (
 )
 
 import kiln_ai.datamodel as datamodel
-from kiln_ai.adapters.ml_model_list import StructuredOutputMode
+from kiln_ai.adapters.ml_model_list import (
+    KilnModelProvider,
+    ModelProviderName,
+    StructuredOutputMode,
+)
 from kiln_ai.adapters.model_adapters.base_adapter import (
     COT_FINAL_ANSWER_PROMPT,
-    AdapterInfo,
+    AdapterConfig,
     BaseAdapter,
-    BasePromptBuilder,
     RunOutput,
 )
 from kiln_ai.adapters.model_adapters.openai_compatible_config import (
     OpenAICompatibleConfig,
 )
 from kiln_ai.adapters.parsers.json_parser import parse_json_string
+from kiln_ai.datamodel import PromptGenerators, PromptId
+from kiln_ai.datamodel.task import RunConfig
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
 
@@ -29,8 +34,8 @@ class OpenAICompatibleAdapter(BaseAdapter):
         self,
         config: OpenAICompatibleConfig,
         kiln_task: datamodel.Task,
-        prompt_builder: BasePromptBuilder | None = None,
-        tags: list[str] | None = None,
+        prompt_id: PromptId | None = None,
+        base_adapter_config: AdapterConfig | None = None,
     ):
         self.config = config
         self.client = AsyncOpenAI(
@@ -39,12 +44,16 @@ class OpenAICompatibleAdapter(BaseAdapter):
             default_headers=config.default_headers,
         )
 
-        super().__init__(
-            kiln_task,
+        run_config = RunConfig(
+            task=kiln_task,
             model_name=config.model_name,
             model_provider_name=config.provider_name,
-            prompt_builder=prompt_builder,
-            tags=tags,
+            prompt_id=prompt_id or PromptGenerators.SIMPLE,
+        )
+
+        super().__init__(
+            run_config=run_config,
+            config=base_adapter_config,
         )
 
     async def _run(self, input: Dict | str) -> RunOutput:
@@ -93,21 +102,8 @@ class OpenAICompatibleAdapter(BaseAdapter):
                 ]
             )
 
-        # OpenRouter specific options for reasoning models
-        extra_body = {}
-        require_or_reasoning = (
-            self.config.openrouter_style_reasoning and provider.reasoning_capable
-        )
-        if require_or_reasoning:
-            extra_body["include_reasoning"] = True
-            # Filter to providers that support the reasoning parameter
-            extra_body["provider"] = {
-                "require_parameters": True,
-                # Ugly to have these here, but big range of quality of R1 providers
-                "order": ["Fireworks", "Together"],
-                # fp8 quants are awful
-                "ignore": ["DeepInfra"],
-            }
+        # Build custom request params based on model provider
+        extra_body = self.build_extra_body(provider)
 
         # Main completion call
         response_format_options = await self.response_format_options()
@@ -115,6 +111,8 @@ class OpenAICompatibleAdapter(BaseAdapter):
             model=provider.provider_options["model"],
             messages=messages,
             extra_body=extra_body,
+            logprobs=self.base_adapter_config.top_logprobs is not None,
+            top_logprobs=self.base_adapter_config.top_logprobs,
             **response_format_options,
         )
 
@@ -133,9 +131,14 @@ class OpenAICompatibleAdapter(BaseAdapter):
             )
 
         message = response.choices[0].message
+        logprobs = response.choices[0].logprobs
+
+        # Check logprobs worked, if requested
+        if self.base_adapter_config.top_logprobs is not None and logprobs is None:
+            raise RuntimeError("Logprobs were required, but no logprobs were returned.")
 
-        # Save reasoning if it exists (OpenRouter specific format)
-        if require_or_reasoning:
+        # Save reasoning if it exists (OpenRouter specific api response field)
+        if provider.require_openrouter_reasoning:
             if (
                 hasattr(message, "reasoning") and message.reasoning  # pyright: ignore
             ):
@@ -164,26 +167,19 @@ class OpenAICompatibleAdapter(BaseAdapter):
         if not isinstance(response_content, str):
             raise RuntimeError(f"response is not a string: {response_content}")
 
+        # Parse to dict if we have structured output
+        output: Dict | str = response_content
         if self.has_structured_output():
-            structured_response = parse_json_string(response_content)
-            return RunOutput(
-                output=structured_response,
-                intermediate_outputs=intermediate_outputs,
-            )
+            output = parse_json_string(response_content)
 
         return RunOutput(
-            output=response_content,
+            output=output,
             intermediate_outputs=intermediate_outputs,
+            output_logprobs=logprobs,
         )
 
-    def adapter_info(self) -> AdapterInfo:
-        return AdapterInfo(
-            model_name=self.model_name,
-            model_provider=self.model_provider_name,
-            adapter_name="kiln_openai_compatible_adapter",
-            prompt_builder_name=self.prompt_builder.__class__.prompt_builder_name(),
-            prompt_id=self.prompt_builder.prompt_id(),
-        )
+    def adapter_name(self) -> str:
+        return "kiln_openai_compatible_adapter"
 
     async def response_format_options(self) -> dict[str, Any]:
         # Unstructured if task isn't structured
@@ -195,7 +191,7 @@ class OpenAICompatibleAdapter(BaseAdapter):
             case StructuredOutputMode.json_mode:
                 return {"response_format": {"type": "json_object"}}
             case StructuredOutputMode.json_schema:
-                output_schema = self.kiln_task.output_schema()
+                output_schema = self.task().output_schema()
                 return {
                     "response_format": {
                         "type": "json_schema",
@@ -205,8 +201,10 @@ class OpenAICompatibleAdapter(BaseAdapter):
                         },
                     }
                 }
+            case StructuredOutputMode.function_calling_weak:
+                return self.tool_call_params(strict=False)
             case StructuredOutputMode.function_calling:
-                return self.tool_call_params()
+                return self.tool_call_params(strict=True)
             case StructuredOutputMode.json_instructions:
                 # JSON done via instructions in prompt, not the API response format. Do not ask for json_object (see option below).
                 return {}
@@ -215,28 +213,32 @@ class OpenAICompatibleAdapter(BaseAdapter):
                 return {"response_format": {"type": "json_object"}}
             case StructuredOutputMode.default:
                 # Default to function calling -- it's older than the other modes. Higher compatibility.
-                return self.tool_call_params()
+                return self.tool_call_params(strict=True)
             case _:
                 raise_exhaustive_enum_error(provider.structured_output_mode)
 
-    def tool_call_params(self) -> dict[str, Any]:
+    def tool_call_params(self, strict: bool) -> dict[str, Any]:
         # Add additional_properties: false to the schema (OpenAI requires this for some models)
-        output_schema = self.kiln_task.output_schema()
+        output_schema = self.task().output_schema()
         if not isinstance(output_schema, dict):
             raise ValueError(
                 "Invalid output schema for this task. Can not use tool calls."
             )
         output_schema["additionalProperties"] = False
 
+        function_params = {
+            "name": "task_response",
+            "parameters": output_schema,
+        }
+        # This should be on, but we allow setting function_calling_weak for APIs that don't support it.
+        if strict:
+            function_params["strict"] = True
+
         return {
             "tools": [
                 {
                     "type": "function",
-                    "function": {
-                        "name": "task_response",
-                        "parameters": output_schema,
-                        "strict": True,
-                    },
+                    "function": function_params,
                 }
             ],
             "tool_choice": {
@@ -244,3 +246,44 @@ class OpenAICompatibleAdapter(BaseAdapter):
                 "function": {"name": "task_response"},
             },
         }
+
+    def build_extra_body(self, provider: KilnModelProvider) -> dict[str, Any]:
+        # TODO P1: Don't love having this logic here. But it's a usability improvement
+        # so better to keep it than exclude it. Should figure out how I want to isolate
+        # this sort of logic so it's config driven and can be overridden
+
+        extra_body = {}
+        provider_options = {}
+
+        if provider.require_openrouter_reasoning:
+            # https://openrouter.ai/docs/use-cases/reasoning-tokens
+            extra_body["reasoning"] = {
+                "exclude": False,
+            }
+
+        if provider.r1_openrouter_options:
+            # Require providers that support the reasoning parameter
+            provider_options["require_parameters"] = True
+            # Prefer R1 providers with reasonable perf/quants
+            provider_options["order"] = ["Fireworks", "Together"]
+            # R1 providers with unreasonable quants
+            provider_options["ignore"] = ["DeepInfra"]
+
+        # Only set of this request is to get logprobs.
+        if (
+            provider.logprobs_openrouter_options
+            and self.base_adapter_config.top_logprobs is not None
+        ):
+            # Don't let OpenRouter choose a provider that doesn't support logprobs.
+            provider_options["require_parameters"] = True
+            # DeepInfra silently fails to return logprobs consistently.
+            provider_options["ignore"] = ["DeepInfra"]
+
+        if provider.openrouter_skip_required_parameters:
+            # Oddball case, R1 14/8/1.5B fail with this param, even though they support thinking params.
+            provider_options["require_parameters"] = False
+
+        if len(provider_options) > 0:
+            extra_body["provider"] = provider_options
+
+        return extra_body

kiln_ai/adapters/model_adapters/test_base_adapter.py

@@ -3,8 +3,9 @@ from unittest.mock import MagicMock, patch
 import pytest
 
 from kiln_ai.adapters.ml_model_list import KilnModelProvider, StructuredOutputMode
-from kiln_ai.adapters.model_adapters.base_adapter import AdapterInfo, BaseAdapter
+from kiln_ai.adapters.model_adapters.base_adapter import BaseAdapter
 from kiln_ai.datamodel import Task
+from kiln_ai.datamodel.task import RunConfig
 
 
 class MockAdapter(BaseAdapter):
@@ -13,13 +14,8 @@ class MockAdapter(BaseAdapter):
     async def _run(self, input):
         return None
 
-    def adapter_info(self) -> AdapterInfo:
-        return AdapterInfo(
-            adapter_name="test",
-            model_name=self.model_name,
-            model_provider=self.model_provider_name,
-            prompt_builder_name="test",
-        )
+    def adapter_name(self) -> str:
+        return "test"
 
 
 @pytest.fixture
@@ -37,9 +33,12 @@ def base_task():
 @pytest.fixture
 def adapter(base_task):
     return MockAdapter(
-        kiln_task=base_task,
-        model_name="test_model",
-        model_provider_name="test_provider",
+        run_config=RunConfig(
+            task=base_task,
+            model_name="test_model",
+            model_provider_name="test_provider",
+            prompt_id="simple_prompt_builder",
+        ),
     )
 
 
@@ -85,7 +84,12 @@ async def test_model_provider_missing_names(base_task):
     """Test error when model or provider name is missing"""
     # Test with missing model name
     adapter = MockAdapter(
-        kiln_task=base_task, model_name="", model_provider_name="test_provider"
+        run_config=RunConfig(
+            task=base_task,
+            model_name="",
+            model_provider_name="",
+            prompt_id="simple_prompt_builder",
+        ),
     )
     with pytest.raises(
         ValueError, match="model_name and model_provider_name must be provided"
@@ -94,7 +98,12 @@ async def test_model_provider_missing_names(base_task):
 
     # Test with missing provider name
     adapter = MockAdapter(
-        kiln_task=base_task, model_name="test_model", model_provider_name=""
+        run_config=RunConfig(
+            task=base_task,
+            model_name="test_model",
+            model_provider_name="",
+            prompt_id="simple_prompt_builder",
+        ),
     )
     with pytest.raises(
         ValueError, match="model_name and model_provider_name must be provided"

kiln_ai/adapters/model_adapters/test_langchain_adapter.py

@@ -18,8 +18,8 @@ from kiln_ai.adapters.model_adapters.langchain_adapters import (
     LangchainAdapter,
     langchain_model_from_provider,
 )
-from kiln_ai.adapters.prompt_builders import SimpleChainOfThoughtPromptBuilder
 from kiln_ai.adapters.test_prompt_adaptors import build_test_task
+from kiln_ai.datamodel.task import RunConfig
 
 
 @pytest.fixture
@@ -56,9 +56,8 @@ def test_langchain_adapter_infer_model_name(tmp_path):
 
     lca = LangchainAdapter(kiln_task=task, custom_model=custom)
 
-    model_info = lca.adapter_info()
-    assert model_info.model_name == "custom.langchain:llama-3.1-8b-instant"
-    assert model_info.model_provider == "custom.langchain:ChatGroq"
+    assert lca.run_config.model_name == "custom.langchain:llama-3.1-8b-instant"
+    assert lca.run_config.model_provider_name == "custom.langchain:ChatGroq"
 
 
 def test_langchain_adapter_info(tmp_path):
@@ -66,10 +65,9 @@ def test_langchain_adapter_info(tmp_path):
 
     lca = LangchainAdapter(kiln_task=task, model_name="llama_3_1_8b", provider="ollama")
 
-    model_info = lca.adapter_info()
-    assert model_info.adapter_name == "kiln_langchain_adapter"
-    assert model_info.model_name == "llama_3_1_8b"
-    assert model_info.model_provider == "ollama"
+    assert lca.adapter_name() == "kiln_langchain_adapter"
+    assert lca.run_config.model_name == "llama_3_1_8b"
+    assert lca.run_config.model_provider_name == "ollama"
 
 
 async def test_langchain_adapter_with_cot(tmp_path):
@@ -81,7 +79,7 @@ async def test_langchain_adapter_with_cot(tmp_path):
         kiln_task=task,
         model_name="llama_3_1_8b",
         provider="ollama",
-        prompt_builder=SimpleChainOfThoughtPromptBuilder(task),
+        prompt_id="simple_chain_of_thought_prompt_builder",
     )
 
     # Mock the base model and its invoke method
@@ -324,11 +322,6 @@ async def test_langchain_adapter_model_no_structured_output_support(tmp_path):
 
 import pytest
 
-from kiln_ai.adapters.ml_model_list import KilnModelProvider, ModelProviderName
-from kiln_ai.adapters.model_adapters.langchain_adapters import (
-    langchain_model_from_provider,
-)
-
 
 @pytest.mark.parametrize(
     "provider_name",

kiln_ai/adapters/model_adapters/test_openai_model_adapter.py

@@ -5,7 +5,7 @@ import pytest
 from openai import AsyncOpenAI
 
 from kiln_ai.adapters.ml_model_list import StructuredOutputMode
-from kiln_ai.adapters.model_adapters.base_adapter import AdapterInfo, BasePromptBuilder
+from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig
 from kiln_ai.adapters.model_adapters.openai_compatible_config import (
     OpenAICompatibleConfig,
 )
@@ -37,63 +37,47 @@ def mock_task(tmp_path):
     return task
 
 
-@pytest.fixture
-def mock_prompt_builder():
-    builder = Mock(spec=BasePromptBuilder)
-    type(builder).prompt_builder_name = Mock(return_value="test_prompt_builder")
-    builder.prompt_id = Mock(return_value="test_prompt_id")
-    return builder
-
-
 @pytest.fixture
 def config():
     return OpenAICompatibleConfig(
         api_key="test_key",
         base_url="https://api.test.com",
         model_name="test-model",
-        provider_name="test-provider",
+        provider_name="openrouter",
         default_headers={"X-Test": "test"},
     )
 
 
-def test_initialization(config, mock_task, mock_prompt_builder):
+def test_initialization(config, mock_task):
     adapter = OpenAICompatibleAdapter(
         config=config,
         kiln_task=mock_task,
-        prompt_builder=mock_prompt_builder,
-        tags=["test-tag"],
+        prompt_id="simple_prompt_builder",
+        base_adapter_config=AdapterConfig(default_tags=["test-tag"]),
     )
 
     assert isinstance(adapter.client, AsyncOpenAI)
     assert adapter.config == config
-    assert adapter.kiln_task == mock_task
-    assert adapter.prompt_builder == mock_prompt_builder
-    assert adapter.default_tags == ["test-tag"]
-    assert adapter.model_name == config.model_name
-    assert adapter.model_provider_name == config.provider_name
+    assert adapter.run_config.task == mock_task
+    assert adapter.run_config.prompt_id == "simple_prompt_builder"
+    assert adapter.base_adapter_config.default_tags == ["test-tag"]
+    assert adapter.run_config.model_name == config.model_name
+    assert adapter.run_config.model_provider_name == config.provider_name
 
 
-def test_adapter_info(config, mock_task, mock_prompt_builder):
-    adapter = OpenAICompatibleAdapter(
-        config=config, kiln_task=mock_task, prompt_builder=mock_prompt_builder
-    )
+def test_adapter_info(config, mock_task):
+    adapter = OpenAICompatibleAdapter(config=config, kiln_task=mock_task)
 
-    info = adapter.adapter_info()
-    assert isinstance(info, AdapterInfo)
-    assert info.model_name == config.model_name
-    assert info.model_provider == config.provider_name
-    assert info.adapter_name == "kiln_openai_compatible_adapter"
-    assert info.prompt_builder_name == "base_prompt_builder"
-    assert info.prompt_id == "test_prompt_id"
+    assert adapter.adapter_name() == "kiln_openai_compatible_adapter"
+
+    assert adapter.run_config.model_name == config.model_name
+    assert adapter.run_config.model_provider_name == config.provider_name
+    assert adapter.run_config.prompt_id == "simple_prompt_builder"
 
 
 @pytest.mark.asyncio
-async def test_response_format_options_unstructured(
-    config, mock_task, mock_prompt_builder
-):
-    adapter = OpenAICompatibleAdapter(
-        config=config, kiln_task=mock_task, prompt_builder=mock_prompt_builder
-    )
+async def test_response_format_options_unstructured(config, mock_task):
+    adapter = OpenAICompatibleAdapter(config=config, kiln_task=mock_task)
 
     # Mock has_structured_output to return False
     with patch.object(adapter, "has_structured_output", return_value=False):
@@ -109,12 +93,8 @@ async def test_response_format_options_unstructured(
     ],
 )
 @pytest.mark.asyncio
-async def test_response_format_options_json_mode(
-    config, mock_task, mock_prompt_builder, mode
-):
-    adapter = OpenAICompatibleAdapter(
-        config=config, kiln_task=mock_task, prompt_builder=mock_prompt_builder
-    )
+async def test_response_format_options_json_mode(config, mock_task, mode):
+    adapter = OpenAICompatibleAdapter(config=config, kiln_task=mock_task)
 
     with (
         patch.object(adapter, "has_structured_output", return_value=True),
@@ -134,12 +114,8 @@ async def test_response_format_options_json_mode(
     ],
 )
 @pytest.mark.asyncio
-async def test_response_format_options_function_calling(
-    config, mock_task, mock_prompt_builder, mode
-):
-    adapter = OpenAICompatibleAdapter(
-        config=config, kiln_task=mock_task, prompt_builder=mock_prompt_builder
-    )
+async def test_response_format_options_function_calling(config, mock_task, mode):
+    adapter = OpenAICompatibleAdapter(config=config, kiln_task=mock_task)
 
     with (
         patch.object(adapter, "has_structured_output", return_value=True),
@@ -153,12 +129,8 @@ async def test_response_format_options_function_calling(
 
 
 @pytest.mark.asyncio
-async def test_response_format_options_json_instructions(
-    config, mock_task, mock_prompt_builder
-):
-    adapter = OpenAICompatibleAdapter(
-        config=config, kiln_task=mock_task, prompt_builder=mock_prompt_builder
-    )
+async def test_response_format_options_json_instructions(config, mock_task):
+    adapter = OpenAICompatibleAdapter(config=config, kiln_task=mock_task)
 
     with (
         patch.object(adapter, "has_structured_output", return_value=True),
@@ -172,12 +144,8 @@ async def test_response_format_options_json_instructions(
 
 
 @pytest.mark.asyncio
-async def test_response_format_options_json_schema(
-    config, mock_task, mock_prompt_builder
-):
-    adapter = OpenAICompatibleAdapter(
-        config=config, kiln_task=mock_task, prompt_builder=mock_prompt_builder
-    )
+async def test_response_format_options_json_schema(config, mock_task):
+    adapter = OpenAICompatibleAdapter(config=config, kiln_task=mock_task)
 
     with (
         patch.object(adapter, "has_structured_output", return_value=True),
@@ -198,12 +166,35 @@ async def test_response_format_options_json_schema(
         }
 
 
-def test_tool_call_params(config, mock_task, mock_prompt_builder):
-    adapter = OpenAICompatibleAdapter(
-        config=config, kiln_task=mock_task, prompt_builder=mock_prompt_builder
-    )
+def test_tool_call_params_weak(config, mock_task):
+    adapter = OpenAICompatibleAdapter(config=config, kiln_task=mock_task)
+
+    params = adapter.tool_call_params(strict=False)
+    expected_schema = mock_task.output_schema()
+    expected_schema["additionalProperties"] = False
+
+    assert params == {
+        "tools": [
+            {
+                "type": "function",
+                "function": {
+                    "name": "task_response",
+                    "parameters": expected_schema,
+                },
+            }
+        ],
+        "tool_choice": {
+            "type": "function",
+            "function": {"name": "task_response"},
+        },
+    }
+
+
+def test_tool_call_params_strict(config, mock_task):
+    config.provider_name = "openai"
+    adapter = OpenAICompatibleAdapter(config=config, kiln_task=mock_task)
 
-    params = adapter.tool_call_params()
+    params = adapter.tool_call_params(strict=True)
     expected_schema = mock_task.output_schema()
     expected_schema["additionalProperties"] = False
 

kiln_ai/adapters/model_adapters/test_saving_adapter_results.py

@@ -3,7 +3,6 @@ from unittest.mock import patch
 import pytest
 
 from kiln_ai.adapters.model_adapters.base_adapter import (
-    AdapterInfo,
     BaseAdapter,
     RunOutput,
 )
@@ -13,6 +12,7 @@ from kiln_ai.datamodel import (
     Project,
     Task,
 )
+from kiln_ai.datamodel.task import RunConfig
 from kiln_ai.utils.config import Config
 
 
@@ -20,14 +20,8 @@ class MockAdapter(BaseAdapter):
     async def _run(self, input: dict | str) -> dict | str:
         return RunOutput(output="Test output", intermediate_outputs=None)
 
-    def adapter_info(self) -> AdapterInfo:
-        return AdapterInfo(
-            adapter_name="mock_adapter",
-            model_name="mock_model",
-            model_provider="mock_provider",
-            prompt_builder_name="mock_prompt_builder",
-            prompt_id="mock_prompt_id",
-        )
+    def adapter_name(self) -> str:
+        return "mock_adapter"
 
 
 @pytest.fixture
@@ -45,7 +39,14 @@ def test_task(tmp_path):
 
 @pytest.fixture
 def adapter(test_task):
-    return MockAdapter(test_task, model_name="phi_3_5", model_provider_name="ollama")
+    return MockAdapter(
+        run_config=RunConfig(
+            task=test_task,
+            model_name="phi_3_5",
+            model_provider_name="ollama",
+            prompt_id="simple_chain_of_thought_prompt_builder",
+        ),
+    )
 
 
 def test_save_run_isolation(test_task, adapter):
@@ -94,13 +95,12 @@ def test_save_run_isolation(test_task, adapter):
     assert reloaded_output.source.type == DataSourceType.synthetic
     assert reloaded_output.rating is None
     assert reloaded_output.source.properties["adapter_name"] == "mock_adapter"
-    assert reloaded_output.source.properties["model_name"] == "mock_model"
-    assert reloaded_output.source.properties["model_provider"] == "mock_provider"
+    assert reloaded_output.source.properties["model_name"] == "phi_3_5"
+    assert reloaded_output.source.properties["model_provider"] == "ollama"
     assert (
-        reloaded_output.source.properties["prompt_builder_name"]
-        == "mock_prompt_builder"
+        reloaded_output.source.properties["prompt_id"]
+        == "simple_chain_of_thought_prompt_builder"
     )
-    assert reloaded_output.source.properties["prompt_id"] == "mock_prompt_id"
     # Run again, with same input and different output. Should create a new TaskRun.
     different_run_output = RunOutput(
         output="Different output", intermediate_outputs=None
@@ -118,7 +118,7 @@ def test_save_run_isolation(test_task, adapter):
             properties={
                 "model_name": "mock_model",
                 "model_provider": "mock_provider",
-                "prompt_builder_name": "mock_prompt_builder",
+                "prompt_id": "mock_prompt_builder",
                 "adapter_name": "mock_adapter",
             },
         ),
@@ -178,6 +178,25 @@ async def test_autosave_false(test_task, adapter):
         assert run.id is None
 
 
+@pytest.mark.asyncio
+async def test_autosave_true_with_disabled(test_task, adapter):
+    with patch("kiln_ai.utils.config.Config.shared") as mock_shared:
+        mock_config = mock_shared.return_value
+        mock_config.autosave_runs = True
+        mock_config.user_id = "test_user"
+
+        input_data = "Test input"
+
+        adapter.base_adapter_config.allow_saving = False
+        run = await adapter.invoke(input_data)
+
+        # Check that no runs were saved
+        assert len(test_task.runs()) == 0
+
+        # Check that the run ID is not set
+        assert run.id is None
+
+
 @pytest.mark.asyncio
 async def test_autosave_true(test_task, adapter):
     with patch("kiln_ai.utils.config.Config.shared") as mock_shared:
@@ -202,6 +221,9 @@ async def test_autosave_true(test_task, adapter):
         assert output.output == "Test output"
         assert output.source.type == DataSourceType.synthetic
         assert output.source.properties["adapter_name"] == "mock_adapter"
-        assert output.source.properties["model_name"] == "mock_model"
-        assert output.source.properties["model_provider"] == "mock_provider"
-        assert output.source.properties["prompt_builder_name"] == "mock_prompt_builder"
+        assert output.source.properties["model_name"] == "phi_3_5"
+        assert output.source.properties["model_provider"] == "ollama"
+        assert (
+            output.source.properties["prompt_id"]
+            == "simple_chain_of_thought_prompt_builder"
+        )