kiln-ai 0.19.0__py3-none-any.whl → 0.20.1__py3-none-any.whl

kiln_ai/adapters/__init__.py

@@ -28,12 +28,12 @@ from . import (
 )
 
 __all__ = [
-    "model_adapters",
     "chat",
     "data_gen",
+    "eval",
     "fine_tune",
     "ml_model_list",
+    "model_adapters",
     "prompt_builders",
     "repair",
-    "eval",
 ]

kiln_ai/adapters/adapter_registry.py

@@ -52,7 +52,7 @@ def adapter_for_task(
                     base_url=getenv("SILICONFLOW_BASE_URL")
                     or "https://api.siliconflow.cn/v1",
                     default_headers={
-                        "HTTP-Referer": "https://getkiln.ai/siliconflow",
+                        "HTTP-Referer": "https://kiln.tech/siliconflow",
                         "X-Title": "KilnAI",
                     },
                     additional_body_options={
@@ -123,6 +123,24 @@ def adapter_for_task(
                     },
                 ),
             )
+        case ModelProviderName.docker_model_runner:
+            docker_base_url = (
+                Config.shared().docker_model_runner_base_url
+                or "http://localhost:12434/engines/llama.cpp"
+            )
+            return LiteLlmAdapter(
+                kiln_task=kiln_task,
+                base_adapter_config=base_adapter_config,
+                config=LiteLlmConfig(
+                    run_config_properties=run_config_properties,
+                    # Docker Model Runner uses OpenAI-compatible API at /v1 endpoint
+                    base_url=docker_base_url + "/v1",
+                    additional_body_options={
+                        # LiteLLM errors without an api_key, even though Docker Model Runner doesn't require one.
+                        "api_key": "DMR",
+                    },
+                ),
+            )
         case ModelProviderName.fireworks_ai:
             return LiteLlmAdapter(
                 kiln_task=kiln_task,

kiln_ai/adapters/chat/chat_formatter.py

@@ -106,14 +106,12 @@ class TwoMessageCotLegacyFormatter(ChatFormatter):
         if self._state == "awaiting_thinking":
             if previous_output is None:
                 raise ValueError("previous_output required for thinking step")
-            msgs = [
-                ChatMessage("assistant", previous_output),
-                ChatMessage("user", COT_FINAL_ANSWER_PROMPT),
-            ]
             self._intermediate_outputs["chain_of_thought"] = previous_output
             self._state = "awaiting_final"
-            self._messages.extend(msgs)
-            return ChatTurn(messages=msgs, final_call=True)
+            cot_message = ChatMessage("user", COT_FINAL_ANSWER_PROMPT)
+            self._messages.append(ChatMessage("assistant", previous_output))
+            self._messages.append(cot_message)
+            return ChatTurn(messages=[cot_message], final_call=True)
 
         if self._state == "awaiting_final":
             if previous_output is None:
@@ -155,14 +153,12 @@ class TwoMessageCotFormatter(ChatFormatter):
         if self._state == "awaiting_thinking":
             if previous_output is None:
                 raise ValueError("previous_output required for thinking step")
-            msgs = [
-                ChatMessage("assistant", previous_output),
-                ChatMessage("user", COT_FINAL_ANSWER_PROMPT),
-            ]
             self._intermediate_outputs["chain_of_thought"] = previous_output
             self._state = "awaiting_final"
-            self._messages.extend(msgs)
-            return ChatTurn(messages=msgs, final_call=True)
+            self._messages.append(ChatMessage("assistant", previous_output))
+            cot_message = ChatMessage("user", COT_FINAL_ANSWER_PROMPT)
+            self._messages.append(cot_message)
+            return ChatTurn(messages=[cot_message], final_call=True)
 
         if self._state == "awaiting_final":
             if previous_output is None:

kiln_ai/adapters/chat/test_chat_formatter.py

@@ -46,12 +46,14 @@ def test_chat_formatter_final_and_intermediate():
     )
 
     first = formatter.next_turn()
+    assert first is not None
     assert [m.__dict__ for m in first.messages] == expected[:3]
     assert not first.final_call
     assert formatter.intermediate_outputs() == {}
 
     second = formatter.next_turn("thinking output")
-    assert [m.__dict__ for m in second.messages] == expected[3:5]
+    assert second is not None
+    assert [m.__dict__ for m in second.messages] == expected[4:5]
     assert second.final_call
     assert formatter.intermediate_outputs() == {"chain_of_thought": "thinking output"}
 
@@ -78,12 +80,14 @@ def test_chat_formatter_two_message_cot():
     )
 
     first = formatter.next_turn()
+    assert first is not None
     assert [m.__dict__ for m in first.messages] == expected[:2]
     assert not first.final_call
     assert formatter.intermediate_outputs() == {}
 
     second = formatter.next_turn("thinking output")
-    assert [m.__dict__ for m in second.messages] == expected[2:4]
+    assert second is not None
+    assert [m.__dict__ for m in second.messages] == expected[3:4]
     assert second.final_call
     assert formatter.intermediate_outputs() == {"chain_of_thought": "thinking output"}
 

kiln_ai/adapters/docker_model_runner_tools.py

@@ -0,0 +1,119 @@
+from typing import List
+
+import httpx
+import openai
+from pydantic import BaseModel, Field
+
+from kiln_ai.adapters.ml_model_list import ModelProviderName, built_in_models
+from kiln_ai.utils.config import Config
+
+
+def docker_model_runner_base_url() -> str:
+    """
+    Gets the base URL for Docker Model Runner API connections.
+
+    Returns:
+        The base URL to use for Docker Model Runner API calls, using environment variable if set
+        or falling back to localhost default
+    """
+    config_base_url = Config.shared().docker_model_runner_base_url
+    if config_base_url:
+        return config_base_url
+    return "http://localhost:12434/engines/llama.cpp"
+
+
+async def docker_model_runner_online() -> bool:
+    """
+    Checks if the Docker Model Runner service is available and responding.
+
+    Returns:
+        True if Docker Model Runner is available and responding, False otherwise
+    """
+    try:
+        base_url = docker_model_runner_base_url()
+        # Docker Model Runner uses OpenAI-compatible endpoints
+        async with httpx.AsyncClient() as client:
+            response = await client.get(f"{base_url}/v1/models", timeout=5.0)
+            response.raise_for_status()
+    except httpx.RequestError:
+        return False
+    return True
+
+
+class DockerModelRunnerConnection(BaseModel):
+    message: str
+    version: str | None = None
+    supported_models: List[str]
+    untested_models: List[str] = Field(default_factory=list)
+
+    def all_models(self) -> List[str]:
+        return self.supported_models + self.untested_models
+
+
+# Parse the Docker Model Runner /v1/models response
+def parse_docker_model_runner_models(
+    models: List[openai.types.Model],
+) -> DockerModelRunnerConnection | None:
+    # Build a list of models we support for Docker Model Runner from the built-in model list
+    supported_docker_models = [
+        provider.model_id
+        for model in built_in_models
+        for provider in model.providers
+        if provider.name == ModelProviderName.docker_model_runner
+    ]
+    # Note: Docker Model Runner aliases will be added when we configure models
+
+    model_names = [model.id for model in models]
+    available_supported_models = []
+    untested_models = []
+
+    for model_name in model_names:
+        if model_name in supported_docker_models:
+            available_supported_models.append(model_name)
+        else:
+            untested_models.append(model_name)
+
+    if available_supported_models or untested_models:
+        return DockerModelRunnerConnection(
+            message="Docker Model Runner connected",
+            supported_models=available_supported_models,
+            untested_models=untested_models,
+        )
+
+    return DockerModelRunnerConnection(
+        message="Docker Model Runner is running, but no supported models are available. Ensure models like 'ai/llama3.2:3B-Q4_K_M', 'ai/qwen3:8B-Q4_K_M', or 'ai/gemma3n:4B-Q4_K_M' are loaded.",
+        supported_models=[],
+        untested_models=[],
+    )
+
+
+async def get_docker_model_runner_connection(
+    custom_url: str | None = None,
+) -> DockerModelRunnerConnection | None:
+    """
+    Gets the connection status for Docker Model Runner.
+
+    Args:
+        custom_url: Optional custom URL to use instead of the configured one
+    """
+    try:
+        base_url = custom_url or docker_model_runner_base_url()
+        # Use OpenAI client to get models list
+        client = openai.OpenAI(
+            api_key="dummy",  # Docker Model Runner doesn't require API key
+            base_url=f"{base_url}/v1",
+            max_retries=0,
+        )
+        models_response = client.models.list()
+
+    except (openai.APIConnectionError, openai.APIError, httpx.RequestError):
+        return None
+
+    return parse_docker_model_runner_models(list(models_response))
+
+
+def docker_model_runner_model_installed(
+    conn: DockerModelRunnerConnection, model_name: str
+) -> bool:
+    all_models = conn.all_models()
+    return model_name in all_models

kiln_ai/adapters/eval/base_eval.py

@@ -7,7 +7,7 @@ from kiln_ai.adapters.ml_model_list import ModelProviderName
 from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig
 from kiln_ai.datamodel.eval import Eval, EvalConfig, EvalScores
 from kiln_ai.datamodel.json_schema import validate_schema_with_value_error
-from kiln_ai.datamodel.task import RunConfig, TaskOutputRatingType, TaskRun
+from kiln_ai.datamodel.task import RunConfigProperties, TaskOutputRatingType, TaskRun
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
 
@@ -18,7 +18,7 @@ class BaseEval:
     Should be subclassed, and the run_eval method implemented.
     """
 
-    def __init__(self, eval_config: EvalConfig, run_config: RunConfig | None):
+    def __init__(self, eval_config: EvalConfig, run_config: RunConfigProperties | None):
         self.eval_config = eval_config
         eval = eval_config.parent_eval()
         if not eval:

kiln_ai/adapters/eval/eval_runner.py

@@ -169,7 +169,9 @@ class EvalRunner:
             # Create the evaluator for this eval config/run config pair
             evaluator = eval_adapter_from_type(job.eval_config.config_type)(
                 job.eval_config,
-                job.task_run_config.run_config() if job.task_run_config else None,
+                job.task_run_config.run_config_properties
+                if job.task_run_config
+                else None,
             )
             if not isinstance(evaluator, BaseEval):
                 raise ValueError("Not able to create evaluator from eval config")

kiln_ai/adapters/eval/g_eval.py

@@ -12,7 +12,7 @@ from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig, RunOutpu
 from kiln_ai.adapters.prompt_builders import PromptGenerators
 from kiln_ai.datamodel import Project, Task, TaskRun
 from kiln_ai.datamodel.eval import EvalConfig, EvalConfigType, EvalScores
-from kiln_ai.datamodel.task import RunConfig, RunConfigProperties, StructuredOutputMode
+from kiln_ai.datamodel.task import RunConfigProperties, StructuredOutputMode
 
 # all the tokens we score for, and their float scores.
 TOKEN_TO_SCORE_MAP: Dict[str, float] = {
@@ -89,7 +89,7 @@ class GEval(BaseEval):
     }
     """
 
-    def __init__(self, eval_config: EvalConfig, run_config: RunConfig | None):
+    def __init__(self, eval_config: EvalConfig, run_config: RunConfigProperties | None):
         if (
             eval_config.config_type != EvalConfigType.g_eval
             and eval_config.config_type != EvalConfigType.llm_as_judge

kiln_ai/adapters/eval/test_base_eval.py

@@ -380,7 +380,7 @@ async def test_run_task_and_eval():
         async def run_eval(self, task_run):
             return {"overall_rating": 5, "quality": 4}, {"thinking": "test thinking"}
 
-    evaluator = MockEval(eval_config, run_config.run_config())
+    evaluator = MockEval(eval_config, run_config.run_config_properties)
 
     # Mock dependencies
     mock_adapter = AsyncMock()

kiln_ai/adapters/eval/test_g_eval.py

@@ -19,7 +19,7 @@ from kiln_ai.datamodel import (
     TaskRun,
 )
 from kiln_ai.datamodel.eval import Eval, EvalConfig, EvalConfigType, EvalOutputScore
-from kiln_ai.datamodel.task import RunConfig
+from kiln_ai.datamodel.task import RunConfigProperties
 
 
 @pytest.fixture
@@ -93,11 +93,10 @@ def test_eval_config(test_task):
 
 
 @pytest.fixture
-def test_run_config(test_task):
-    return RunConfig(
+def test_run_config():
+    return RunConfigProperties(
         model_name="llama_3_1_8b",
         model_provider_name="groq",
-        task=test_task,
         prompt_id="simple_prompt_builder",
         structured_output_mode="json_schema",
     )

kiln_ai/adapters/fine_tune/__init__.py

@@ -8,7 +8,7 @@ from . import base_finetune, dataset_formatter, finetune_registry, openai_finetu
 
 __all__ = [
     "base_finetune",
-    "openai_finetune",
     "dataset_formatter",
     "finetune_registry",
+    "openai_finetune",
 ]

kiln_ai/adapters/fine_tune/openai_finetune.py

@@ -13,9 +13,16 @@ from kiln_ai.adapters.fine_tune.dataset_formatter import DatasetFormat, DatasetF
 from kiln_ai.datamodel import DatasetSplit, StructuredOutputMode, Task
 from kiln_ai.utils.config import Config
 
-oai_client = openai.AsyncOpenAI(
-    api_key=Config.shared().open_ai_api_key or "",
-)
+
+def _get_openai_client():
+    key = Config.shared().open_ai_api_key
+    if not key:
+        raise RuntimeError(
+            "OpenAI API key not set. You must connect OpenAI in settings."
+        )
+    return openai.AsyncOpenAI(
+        api_key=key,
+    )
 
 
 class OpenAIFinetune(BaseFinetuneAdapter):
@@ -45,6 +52,7 @@ class OpenAIFinetune(BaseFinetuneAdapter):
 
         try:
             # Will raise an error if the job is not found, or for other issues
+            oai_client = _get_openai_client()
             response = await oai_client.fine_tuning.jobs.retrieve(
                 self.datamodel.provider_id
             )
@@ -79,7 +87,7 @@ class OpenAIFinetune(BaseFinetuneAdapter):
                 )
             return FineTuneStatus(
                 status=FineTuneStatusType.unknown,
-                message=f"Unknown error: [{str(e)}]",
+                message=f"Unknown error: [{e!s}]",
             )
 
         if not response or not isinstance(response, FineTuningJob):
@@ -145,6 +153,7 @@ class OpenAIFinetune(BaseFinetuneAdapter):
             if k in ["n_epochs", "learning_rate_multiplier", "batch_size"]
         }
 
+        oai_client = _get_openai_client()
         ft = await oai_client.fine_tuning.jobs.create(
             training_file=train_file_id,
             model=self.datamodel.base_model_id,
@@ -168,6 +177,7 @@ class OpenAIFinetune(BaseFinetuneAdapter):
         )
         path = formatter.dump_to_file(split_name, format, self.datamodel.data_strategy)
 
+        oai_client = _get_openai_client()
         response = await oai_client.files.create(
             file=open(path, "rb"),
             purpose="fine-tune",