kiln-ai 0.11.1__py3-none-any.whl → 0.13.0__py3-none-any.whl

kiln_ai/adapters/repair/repair_task.py

@@ -6,7 +6,7 @@ from pydantic import BaseModel, Field
 from kiln_ai.adapters.prompt_builders import (
     BasePromptBuilder,
     SavedPromptBuilder,
-    prompt_builder_registry,
+    prompt_builder_from_id,
 )
 from kiln_ai.datamodel import Priority, Project, Task, TaskRequirement, TaskRun
 
@@ -49,28 +49,16 @@ feedback describing what should be improved. Your job is to understand the evalu
         if run.output.source is None or run.output.source.properties is None:
             raise ValueError("No source properties found")
 
-        # Try ID first, then builder name
-        prompt_id = run.output.source.properties.get("prompt_id", None)
+        # Get the prompt builder id. Need the second check because we used to store this in a prompt_builder_name field, so loading legacy runs will need this.
+        prompt_id = run.output.source.properties.get(
+            "prompt_id"
+        ) or run.output.source.properties.get("prompt_builder_name", None)
         if prompt_id is not None and isinstance(prompt_id, str):
-            static_prompt_builder = SavedPromptBuilder(task, prompt_id)
-            return static_prompt_builder.build_prompt(include_json_instructions=False)
+            prompt_builder = prompt_builder_from_id(prompt_id, task)
+            if isinstance(prompt_builder, BasePromptBuilder):
+                return prompt_builder.build_prompt(include_json_instructions=False)
 
-        prompt_builder_class: Type[BasePromptBuilder] | None = None
-        prompt_builder_name = run.output.source.properties.get(
-            "prompt_builder_name", None
-        )
-        if prompt_builder_name is not None and isinstance(prompt_builder_name, str):
-            prompt_builder_class = prompt_builder_registry.get(
-                prompt_builder_name, None
-            )
-        if prompt_builder_class is None:
-            raise ValueError(f"No prompt builder found for name: {prompt_builder_name}")
-        prompt_builder = prompt_builder_class(task=task)
-        if not isinstance(prompt_builder, BasePromptBuilder):
-            raise ValueError(
-                f"Prompt builder {prompt_builder_name} is not a valid prompt builder"
-            )
-        return prompt_builder.build_prompt(include_json_instructions=False)
+        raise ValueError(f"Prompt builder '{prompt_id}' is not a valid prompt builder")
 
     @classmethod
     def build_repair_task_input(

kiln_ai/adapters/repair/test_repair_task.py

@@ -7,7 +7,7 @@ from pydantic import ValidationError
 
 from kiln_ai.adapters.adapter_registry import adapter_for_task
 from kiln_ai.adapters.model_adapters.base_adapter import RunOutput
-from kiln_ai.adapters.model_adapters.langchain_adapters import LangchainAdapter
+from kiln_ai.adapters.model_adapters.litellm_adapter import LiteLlmAdapter
 from kiln_ai.adapters.repair.repair_task import (
     RepairTaskInput,
     RepairTaskRun,
@@ -95,7 +95,7 @@ def sample_task_run(sample_task):
                     "model_name": "gpt_4o",
                     "model_provider": "openai",
                     "adapter_name": "langchain_adapter",
-                    "prompt_builder_name": "simple_prompt_builder",
+                    "prompt_id": "simple_prompt_builder",
                 },
             ),
         ),
@@ -201,7 +201,7 @@ async def test_live_run(sample_task, sample_task_run, sample_repair_data):
         "adapter_name": "kiln_langchain_adapter",
         "model_name": "llama_3_1_8b",
         "model_provider": "groq",
-        "prompt_builder_name": "simple_prompt_builder",
+        "prompt_id": "simple_prompt_builder",
     }
 
 
@@ -217,7 +217,7 @@ async def test_mocked_repair_task_run(sample_task, sample_task_run, sample_repai
         "rating": 8,
     }
 
-    with patch.object(LangchainAdapter, "_run", new_callable=AsyncMock) as mock_run:
+    with patch.object(LiteLlmAdapter, "_run", new_callable=AsyncMock) as mock_run:
         mock_run.return_value = RunOutput(
             output=mocked_output, intermediate_outputs=None
         )
@@ -235,10 +235,10 @@ async def test_mocked_repair_task_run(sample_task, sample_task_run, sample_repai
     parsed_output = json.loads(run.output.output)
     assert parsed_output == mocked_output
     assert run.output.source.properties == {
-        "adapter_name": "kiln_langchain_adapter",
+        "adapter_name": "kiln_openai_compatible_adapter",
         "model_name": "llama_3_1_8b",
         "model_provider": "ollama",
-        "prompt_builder_name": "simple_prompt_builder",
+        "prompt_id": "simple_prompt_builder",
     }
     assert run.input_source.type == DataSourceType.human
     assert "created_by" in run.input_source.properties

kiln_ai/adapters/run_output.py

@@ -1,8 +1,11 @@
 from dataclasses import dataclass
 from typing import Dict
 
+from litellm.types.utils import ChoiceLogprobs
+
 
 @dataclass
 class RunOutput:
     output: Dict | str
     intermediate_outputs: Dict[str, str] | None
+    output_logprobs: ChoiceLogprobs | None = None

kiln_ai/adapters/test_adapter_registry.py

@@ -5,8 +5,8 @@ import pytest
 from kiln_ai import datamodel
 from kiln_ai.adapters.adapter_registry import adapter_for_task
 from kiln_ai.adapters.ml_model_list import ModelProviderName
-from kiln_ai.adapters.model_adapters.langchain_adapters import LangchainAdapter
-from kiln_ai.adapters.model_adapters.openai_model_adapter import OpenAICompatibleAdapter
+from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig
+from kiln_ai.adapters.model_adapters.litellm_adapter import LiteLlmAdapter
 from kiln_ai.adapters.prompt_builders import BasePromptBuilder
 from kiln_ai.adapters.provider_tools import kiln_model_provider_from
 
@@ -43,9 +43,9 @@ def test_openai_adapter_creation(mock_config, basic_task):
         kiln_task=basic_task, model_name="gpt-4", provider=ModelProviderName.openai
     )
 
-    assert isinstance(adapter, OpenAICompatibleAdapter)
+    assert isinstance(adapter, LiteLlmAdapter)
     assert adapter.config.model_name == "gpt-4"
-    assert adapter.config.api_key == "test-openai-key"
+    assert adapter.config.additional_body_options == {"api_key": "test-openai-key"}
     assert adapter.config.provider_name == ModelProviderName.openai
     assert adapter.config.base_url is None  # OpenAI url is default
     assert adapter.config.default_headers is None
@@ -58,11 +58,10 @@ def test_openrouter_adapter_creation(mock_config, basic_task):
         provider=ModelProviderName.openrouter,
     )
 
-    assert isinstance(adapter, OpenAICompatibleAdapter)
+    assert isinstance(adapter, LiteLlmAdapter)
     assert adapter.config.model_name == "anthropic/claude-3-opus"
-    assert adapter.config.api_key == "test-openrouter-key"
+    assert adapter.config.additional_body_options == {"api_key": "test-openrouter-key"}
     assert adapter.config.provider_name == ModelProviderName.openrouter
-    assert adapter.config.base_url == "https://openrouter.ai/api/v1"
     assert adapter.config.default_headers == {
         "HTTP-Referer": "https://getkiln.ai/openrouter",
         "X-Title": "KilnAI",
@@ -78,30 +77,25 @@ def test_openrouter_adapter_creation(mock_config, basic_task):
         ModelProviderName.fireworks_ai,
     ],
 )
-def test_langchain_adapter_creation(mock_config, basic_task, provider):
+def test_openai_compatible_adapter_creation(mock_config, basic_task, provider):
     adapter = adapter_for_task(
         kiln_task=basic_task, model_name="test-model", provider=provider
     )
 
-    assert isinstance(adapter, LangchainAdapter)
-    assert adapter.model_name == "test-model"
+    assert isinstance(adapter, LiteLlmAdapter)
+    assert adapter.run_config.model_name == "test-model"
 
 
 # TODO should run for all cases
 def test_custom_prompt_builder(mock_config, basic_task):
-    class TestPromptBuilder(BasePromptBuilder):
-        def build_base_prompt(self, kiln_task) -> str:
-            return "test-prompt"
-
-    prompt_builder = TestPromptBuilder(basic_task)
     adapter = adapter_for_task(
         kiln_task=basic_task,
         model_name="gpt-4",
         provider=ModelProviderName.openai,
-        prompt_builder=prompt_builder,
+        prompt_id="simple_chain_of_thought_prompt_builder",
     )
 
-    assert adapter.prompt_builder == prompt_builder
+    assert adapter.run_config.prompt_id == "simple_chain_of_thought_prompt_builder"
 
 
 # TODO should run for all cases
@@ -111,10 +105,12 @@ def test_tags_passed_through(mock_config, basic_task):
         kiln_task=basic_task,
         model_name="gpt-4",
         provider=ModelProviderName.openai,
-        tags=tags,
+        base_adapter_config=AdapterConfig(
+            default_tags=tags,
+        ),
     )
 
-    assert adapter.default_tags == tags
+    assert adapter.base_adapter_config.default_tags == tags
 
 
 def test_invalid_provider(mock_config, basic_task):
@@ -124,11 +120,14 @@ def test_invalid_provider(mock_config, basic_task):
         )
 
 
-@patch("kiln_ai.adapters.adapter_registry.openai_compatible_config")
+@patch("kiln_ai.adapters.adapter_registry.lite_llm_config")
 def test_openai_compatible_adapter(mock_compatible_config, mock_config, basic_task):
     mock_compatible_config.return_value.model_name = "test-model"
-    mock_compatible_config.return_value.api_key = "test-key"
+    mock_compatible_config.return_value.additional_body_options = {
+        "api_key": "test-key"
+    }
     mock_compatible_config.return_value.base_url = "https://test.com/v1"
+    mock_compatible_config.return_value.provider_name = "CustomProvider99"
 
     adapter = adapter_for_task(
         kiln_task=basic_task,
@@ -136,11 +135,9 @@ def test_openai_compatible_adapter(mock_compatible_config, mock_config, basic_ta
         provider=ModelProviderName.openai_compatible,
     )
 
-    assert isinstance(adapter, OpenAICompatibleAdapter)
+    assert isinstance(adapter, LiteLlmAdapter)
     mock_compatible_config.assert_called_once_with("provider::test-model")
-    assert adapter.config.model_name == "test-model"
-    assert adapter.config.api_key == "test-key"
-    assert adapter.config.base_url == "https://test.com/v1"
+    assert adapter.config == mock_compatible_config.return_value
 
 
 def test_custom_openai_compatible_provider(mock_config, basic_task):
@@ -150,9 +147,9 @@ def test_custom_openai_compatible_provider(mock_config, basic_task):
         provider=ModelProviderName.kiln_custom_registry,
     )
 
-    assert isinstance(adapter, OpenAICompatibleAdapter)
+    assert isinstance(adapter, LiteLlmAdapter)
     assert adapter.config.model_name == "openai::test-model"
-    assert adapter.config.api_key == "test-openai-key"
+    assert adapter.config.additional_body_options == {"api_key": "test-openai-key"}
     assert adapter.config.base_url is None  # openai is none
     assert adapter.config.provider_name == ModelProviderName.kiln_custom_registry
 
@@ -165,7 +162,7 @@ async def test_fine_tune_provider(mock_config, basic_task, mock_finetune_from_id
     )
 
     mock_finetune_from_id.assert_called_once_with("proj::task::tune")
-    assert isinstance(adapter, OpenAICompatibleAdapter)
+    assert isinstance(adapter, LiteLlmAdapter)
     assert adapter.config.provider_name == ModelProviderName.kiln_fine_tune
     # Kiln model name here, but the underlying openai model id below
     assert adapter.config.model_name == "proj::task::tune"
@@ -174,4 +171,4 @@ async def test_fine_tune_provider(mock_config, basic_task, mock_finetune_from_id
         "proj::task::tune", provider_name=ModelProviderName.kiln_fine_tune
     )
     # The actual model name from the fine tune object
-    assert provider.provider_options["model"] == "test-model"
+    assert provider.model_id == "test-model"

kiln_ai/adapters/test_generate_docs.py

@@ -1,13 +1,13 @@
+import logging
 from typing import List
 
 import pytest
 
-from libs.core.kiln_ai.adapters.ml_model_list import (
-    KilnModelProvider,
-    built_in_models,
-)
+from libs.core.kiln_ai.adapters.ml_model_list import KilnModelProvider, built_in_models
 from libs.core.kiln_ai.adapters.provider_tools import provider_name_from_id
 
+logger = logging.getLogger(__name__)
+
 
 def _all_providers_support(providers: List[KilnModelProvider], attribute: str) -> bool:
     """Check if all providers support a given feature"""

kiln_ai/adapters/test_ollama_tools.py

@@ -10,7 +10,6 @@ from kiln_ai.adapters.ollama_tools import (
 def test_parse_ollama_tags_no_models():
     json_response = '{"models":[{"name":"scosman_net","model":"scosman_net:latest"},{"name":"phi3.5:latest","model":"phi3.5:latest","modified_at":"2024-10-02T12:04:35.191519822-04:00","size":2176178843,"digest":"61819fb370a3c1a9be6694869331e5f85f867a079e9271d66cb223acb81d04ba","details":{"parent_model":"","format":"gguf","family":"phi3","families":["phi3"],"parameter_size":"3.8B","quantization_level":"Q4_0"}},{"name":"gemma2:2b","model":"gemma2:2b","modified_at":"2024-09-09T16:46:38.64348929-04:00","size":1629518495,"digest":"8ccf136fdd5298f3ffe2d69862750ea7fb56555fa4d5b18c04e3fa4d82ee09d7","details":{"parent_model":"","format":"gguf","family":"gemma2","families":["gemma2"],"parameter_size":"2.6B","quantization_level":"Q4_0"}},{"name":"llama3.1:latest","model":"llama3.1:latest","modified_at":"2024-09-01T17:19:43.481523695-04:00","size":4661230720,"digest":"f66fc8dc39ea206e03ff6764fcc696b1b4dfb693f0b6ef751731dd4e6269046e","details":{"parent_model":"","format":"gguf","family":"llama","families":["llama"],"parameter_size":"8.0B","quantization_level":"Q4_0"}}]}'
     tags = json.loads(json_response)
-    print(json.dumps(tags, indent=2))
     conn = parse_ollama_tags(tags)
     assert "phi3.5:latest" in conn.supported_models
     assert "gemma2:2b" in conn.supported_models

kiln_ai/adapters/test_prompt_adaptors.py

@@ -1,24 +1,32 @@
 import os
 from pathlib import Path
+from unittest.mock import patch
 
 import pytest
-from langchain_core.language_models.fake_chat_models import FakeListChatModel
+from litellm.utils import ModelResponse
 
 import kiln_ai.datamodel as datamodel
 from kiln_ai.adapters.adapter_registry import adapter_for_task
 from kiln_ai.adapters.ml_model_list import built_in_models
-from kiln_ai.adapters.model_adapters.langchain_adapters import LangchainAdapter
+from kiln_ai.adapters.model_adapters.litellm_adapter import (
+    LiteLlmAdapter,
+    LiteLlmConfig,
+)
 from kiln_ai.adapters.ollama_tools import ollama_online
 from kiln_ai.adapters.prompt_builders import (
     BasePromptBuilder,
     SimpleChainOfThoughtPromptBuilder,
 )
+from kiln_ai.datamodel import PromptId
 
 
 def get_all_models_and_providers():
     model_provider_pairs = []
     for model in built_in_models:
         for provider in model.providers:
+            if not provider.model_id:
+                # it's possible for models to not have an ID (fine-tune only model)
+                continue
             model_provider_pairs.append((model.name, provider.name))
     return model_provider_pairs
 
@@ -105,23 +113,27 @@ async def test_amazon_bedrock(tmp_path):
     await run_simple_test(tmp_path, "llama_3_1_8b", "amazon_bedrock")
 
 
-async def test_mock(tmp_path):
-    task = build_test_task(tmp_path)
-    mockChatModel = FakeListChatModel(responses=["mock response"])
-    adapter = LangchainAdapter(
-        task,
-        custom_model=mockChatModel,
-        provider="ollama",
-    )
-    run = await adapter.invoke("You are a mock, send me the response!")
-    assert "mock response" in run.output.output
-
-
 async def test_mock_returning_run(tmp_path):
     task = build_test_task(tmp_path)
-    mockChatModel = FakeListChatModel(responses=["mock response"])
-    adapter = LangchainAdapter(task, custom_model=mockChatModel, provider="ollama")
-    run = await adapter.invoke("You are a mock, send me the response!")
+    with patch("litellm.acompletion") as mock_acompletion:
+        # Configure the mock to return a properly structured response
+        mock_acompletion.return_value = ModelResponse(
+            model="custom_model",
+            choices=[{"message": {"content": "mock response"}}],
+        )
+
+        adapter = LiteLlmAdapter(
+            config=LiteLlmConfig(
+                model_name="custom_model",
+                provider_name="ollama",
+                base_url="http://localhost:11434",
+                additional_body_options={"api_key": "test_key"},
+            ),
+            kiln_task=task,
+        )
+
+        run = await adapter.invoke("You are a mock, send me the response!")
+
     assert run.output.output == "mock response"
     assert run is not None
     assert run.id is not None
@@ -129,10 +141,10 @@ async def test_mock_returning_run(tmp_path):
     assert run.output.output == "mock response"
     assert "created_by" in run.input_source.properties
     assert run.output.source.properties == {
-        "adapter_name": "kiln_langchain_adapter",
-        "model_name": "custom.langchain:unknown_model",
+        "adapter_name": "kiln_openai_compatible_adapter",
+        "model_name": "custom_model",
         "model_provider": "ollama",
-        "prompt_builder_name": "simple_prompt_builder",
+        "prompt_id": "simple_prompt_builder",
     }
 
 
@@ -149,8 +161,9 @@ async def test_all_models_providers_plaintext(tmp_path, model_name, provider_nam
 @pytest.mark.parametrize("model_name,provider_name", get_all_models_and_providers())
 async def test_cot_prompt_builder(tmp_path, model_name, provider_name):
     task = build_test_task(tmp_path)
-    pb = SimpleChainOfThoughtPromptBuilder(task)
-    await run_simple_task(task, model_name, provider_name, pb)
+    await run_simple_task(
+        task, model_name, provider_name, "simple_chain_of_thought_prompt_builder"
+    )
 
 
 def build_test_task(tmp_path: Path):
@@ -186,20 +199,20 @@ async def run_simple_test(
     tmp_path: Path,
     model_name: str,
     provider: str | None = None,
-    prompt_builder: BasePromptBuilder | None = None,
+    prompt_id: PromptId | None = None,
 ):
     task = build_test_task(tmp_path)
-    return await run_simple_task(task, model_name, provider, prompt_builder)
+    return await run_simple_task(task, model_name, provider, prompt_id)
 
 
 async def run_simple_task(
     task: datamodel.Task,
     model_name: str,
     provider: str,
-    prompt_builder: BasePromptBuilder | None = None,
+    prompt_id: PromptId | None = None,
 ) -> datamodel.TaskRun:
     adapter = adapter_for_task(
-        task, model_name=model_name, provider=provider, prompt_builder=prompt_builder
+        task, model_name=model_name, provider=provider, prompt_id=prompt_id
     )
 
     run = await adapter.invoke(
@@ -212,13 +225,14 @@ async def run_simple_task(
     )
     assert "64" in run.output.output
     source_props = run.output.source.properties
-    assert source_props["adapter_name"] == "kiln_langchain_adapter"
+    assert source_props["adapter_name"] in [
+        "kiln_langchain_adapter",
+        "kiln_openai_compatible_adapter",
+    ]
     assert source_props["model_name"] == model_name
     assert source_props["model_provider"] == provider
-    expected_prompt_builder_name = (
-        prompt_builder.__class__.prompt_builder_name()
-        if prompt_builder
-        else "simple_prompt_builder"
-    )
-    assert source_props["prompt_builder_name"] == expected_prompt_builder_name
+    if prompt_id is None:
+        assert source_props["prompt_id"] == "simple_prompt_builder"
+    else:
+        assert source_props["prompt_id"] == prompt_id
     return run

kiln_ai/adapters/test_prompt_builders.py

@@ -1,8 +1,9 @@
 import json
+import logging
 
 import pytest
 
-from kiln_ai.adapters.model_adapters.base_adapter import AdapterInfo, BaseAdapter
+from kiln_ai.adapters.model_adapters.base_adapter import BaseAdapter
 from kiln_ai.adapters.model_adapters.test_structured_output import (
     build_structured_output_test_task,
 )
@@ -16,8 +17,9 @@ from kiln_ai.adapters.prompt_builders import (
     SavedPromptBuilder,
     SimpleChainOfThoughtPromptBuilder,
     SimplePromptBuilder,
+    TaskRunConfigPromptBuilder,
     chain_of_thought_prompt,
-    prompt_builder_from_ui_name,
+    prompt_builder_from_id,
 )
 from kiln_ai.adapters.test_prompt_adaptors import build_test_task
 from kiln_ai.datamodel import (
@@ -32,6 +34,9 @@ from kiln_ai.datamodel import (
     TaskOutputRating,
     TaskRun,
 )
+from kiln_ai.datamodel.task import RunConfigProperties, TaskRunConfig
+
+logger = logging.getLogger(__name__)
 
 
 def test_simple_prompt_builder(tmp_path):
@@ -57,12 +62,8 @@ class MockAdapter(BaseAdapter):
     def _run(self, input: str) -> str:
         return "mock response"
 
-    def adapter_info(self) -> AdapterInfo:
-        return AdapterInfo(
-            adapter_name="mock_adapter",
-            model_name="mock_model",
-            model_provider="mock_provider",
-        )
+    def adapter_name(self) -> str:
+        return "mock_adapter"
 
 
 def test_simple_prompt_builder_structured_output(tmp_path):
@@ -269,7 +270,6 @@ def test_few_shot_prompt_builder(tmp_path):
                 rating=TaskOutputRating(value=4 + (i % 2), reason="Good joke"),
             ),
         )
-        print("RATING", "Joke Initial Output ", i + 1, " - RATED:", 4 + (i % 2), "\n")
         if i < 2:
             run = run.model_copy(
                 update={
@@ -290,7 +290,7 @@ def test_few_shot_prompt_builder(tmp_path):
     prompt = prompt_builder.build_prompt(include_json_instructions=False)
     assert prompt.count("## Example") == 4
 
-    print("PROMPT", prompt)
+    logger.info("PROMPT: %s", prompt)
     # Verify the order of examples (2 repaired, then 2 highest-rated)
     assert "Repaired Joke 1" in prompt
     assert "Repaired Joke 2" in prompt
@@ -314,54 +314,53 @@ def check_example_outputs(task: Task, count: int):
         assert f"## Example {count}" in prompt
 
 
-def test_prompt_builder_name():
-    assert SimplePromptBuilder.prompt_builder_name() == "simple_prompt_builder"
-    assert MultiShotPromptBuilder.prompt_builder_name() == "multi_shot_prompt_builder"
-    assert RepairsPromptBuilder.prompt_builder_name() == "repairs_prompt_builder"
-
-
-def test_prompt_builder_from_ui_name(task_with_examples):
+def test_prompt_builder_from_id(task_with_examples):
     task = task_with_examples
-    assert isinstance(prompt_builder_from_ui_name("basic", task), SimplePromptBuilder)
     assert isinstance(
-        prompt_builder_from_ui_name("few_shot", task), FewShotPromptBuilder
+        prompt_builder_from_id("simple_prompt_builder", task), SimplePromptBuilder
     )
     assert isinstance(
-        prompt_builder_from_ui_name("many_shot", task), MultiShotPromptBuilder
+        prompt_builder_from_id("few_shot_prompt_builder", task),
+        FewShotPromptBuilder,
     )
     assert isinstance(
-        prompt_builder_from_ui_name("repairs", task), RepairsPromptBuilder
+        prompt_builder_from_id("multi_shot_prompt_builder", task),
+        MultiShotPromptBuilder,
     )
     assert isinstance(
-        prompt_builder_from_ui_name("simple_chain_of_thought", task),
+        prompt_builder_from_id("repairs_prompt_builder", task),
+        RepairsPromptBuilder,
+    )
+    assert isinstance(
+        prompt_builder_from_id("simple_chain_of_thought_prompt_builder", task),
         SimpleChainOfThoughtPromptBuilder,
     )
     assert isinstance(
-        prompt_builder_from_ui_name("few_shot_chain_of_thought", task),
+        prompt_builder_from_id("few_shot_chain_of_thought_prompt_builder", task),
         FewShotChainOfThoughtPromptBuilder,
     )
     assert isinstance(
-        prompt_builder_from_ui_name("multi_shot_chain_of_thought", task),
+        prompt_builder_from_id("multi_shot_chain_of_thought_prompt_builder", task),
         MultiShotChainOfThoughtPromptBuilder,
     )
 
-    with pytest.raises(ValueError, match="Unknown prompt builder: invalid_name"):
-        prompt_builder_from_ui_name("invalid_name", task)
+    with pytest.raises(ValueError, match="Unknown prompt generator: invalid_name"):
+        prompt_builder_from_id("invalid_name", task)
 
     with pytest.raises(ValueError, match="Prompt ID not found: 123"):
-        prompt_builder_from_ui_name("id::123", task)
+        prompt_builder_from_id("id::123", task)
 
     with pytest.raises(
         ValueError,
         match="Invalid fine-tune ID format. Expected 'project_id::task_id::fine_tune_id'",
     ):
-        prompt_builder_from_ui_name("fine_tune_prompt::123", task)
+        prompt_builder_from_id("fine_tune_prompt::123", task)
 
     with pytest.raises(
         ValueError,
         match="Fine-tune ID not found",
     ):
-        prompt_builder_from_ui_name("fine_tune_prompt::123::456::789", task)
+        prompt_builder_from_id("fine_tune_prompt::123::456::789", task)
 
     prompt = Prompt(
         name="test_prompt_name",
@@ -370,7 +369,7 @@ def test_prompt_builder_from_ui_name(task_with_examples):
         parent=task,
     )
     prompt.save_to_file()
-    pb = prompt_builder_from_ui_name("id::" + prompt.id, task)
+    pb = prompt_builder_from_id("id::" + prompt.id, task)
     assert isinstance(pb, SavedPromptBuilder)
     assert pb.prompt_id() == prompt.id
     assert pb.build_prompt(include_json_instructions=False) == "test_prompt"
@@ -390,7 +389,7 @@ def test_prompt_builder_from_ui_name(task_with_examples):
     nested_fine_tune_id = (
         task_with_examples.parent.id + "::" + task_with_examples.id + "::" + finetune.id
     )
-    pb = prompt_builder_from_ui_name(
+    pb = prompt_builder_from_id(
         "fine_tune_prompt::" + nested_fine_tune_id,
         task_with_examples,
     )
@@ -587,3 +586,64 @@ def test_build_prompt_with_json_instructions(tmp_path):
     assert task.instruction in prompt_with_json
     for requirement in task.requirements:
         assert requirement.instruction in prompt_with_json
+
+
+def test_task_run_config_prompt_builder(tmp_path):
+    task = build_test_task(tmp_path)
+
+    run_config = TaskRunConfig(
+        name="test_run_config",
+        parent=task,
+        run_config_properties=RunConfigProperties(
+            model_name="gpt-4",
+            model_provider_name="openai",
+            prompt_id="simple_prompt_builder",
+        ),
+        prompt=Prompt(
+            name="test prompt name",
+            prompt="test prompt content",
+            chain_of_thought_instructions="test step by step",
+        ),
+    )
+    run_config.save_to_file()
+
+    # Construct the eval prompt ID
+    run_config_prompt_id = (
+        f"task_run_config::{task.parent.id}::{task.id}::{run_config.id}"
+    )
+
+    # Test successful creation 2 ways: constructor and ID creation
+    builders = [
+        TaskRunConfigPromptBuilder(
+            task=task, run_config_prompt_id=run_config_prompt_id
+        ),
+        prompt_builder_from_id(run_config_prompt_id, task),
+    ]
+
+    for builder in builders:
+        assert (
+            builder.build_prompt(include_json_instructions=False)
+            == "test prompt content"
+        )
+        assert builder.chain_of_thought_prompt() == "test step by step"
+        assert builder.prompt_id() == run_config_prompt_id
+
+
+def test_task_run_config_prompt_builder_validation_errors(tmp_path):
+    task = build_test_task(tmp_path)
+
+    # Test invalid format
+    with pytest.raises(ValueError, match="Invalid task run config prompt ID"):
+        TaskRunConfigPromptBuilder(
+            task=task, run_config_prompt_id="task_run_config::wrong::format"
+        )
+
+    # Test task ID mismatch
+    wrong_task_id = f"task_run_config::{task.parent.id}::wrong_task_id::config_id"
+    with pytest.raises(ValueError, match="Task ID mismatch"):
+        TaskRunConfigPromptBuilder(task=task, run_config_prompt_id=wrong_task_id)
+
+    # Test eval not found
+    nonexistent_eval = f"task_run_config::{task.parent.id}::{task.id}::nonexistent_id"
+    with pytest.raises(ValueError, match="Task run config ID not found"):
+        TaskRunConfigPromptBuilder(task=task, run_config_prompt_id=nonexistent_eval)