kiln-ai 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

kiln_ai/adapters/langchain_adapters.py

@@ -1,16 +1,23 @@
 from typing import Dict
 
+from langchain_core.language_models import LanguageModelInput
 from langchain_core.language_models.chat_models import BaseChatModel
 from langchain_core.messages import HumanMessage, SystemMessage
 from langchain_core.messages.base import BaseMessage
+from langchain_core.runnables import Runnable
+from pydantic import BaseModel
 
 import kiln_ai.datamodel as datamodel
 
 from .base_adapter import AdapterInfo, BaseAdapter, BasePromptBuilder
 from .ml_model_list import langchain_model_from
 
+LangChainModelType = BaseChatModel | Runnable[LanguageModelInput, Dict | BaseModel]
+
 
 class LangChainPromptAdapter(BaseAdapter):
+    _model: LangChainModelType | None = None
+
     def __init__(
         self,
         kiln_task: datamodel.Task,
@@ -21,7 +28,7 @@ class LangChainPromptAdapter(BaseAdapter):
     ):
         super().__init__(kiln_task, prompt_builder=prompt_builder)
         if custom_model is not None:
-            self.model = custom_model
+            self._model = custom_model
 
             # Attempt to infer model provider and name from custom model
             self.model_provider = "custom.langchain:" + custom_model.__class__.__name__
@@ -37,19 +44,32 @@ class LangChainPromptAdapter(BaseAdapter):
             ):
                 self.model_name = "custom.langchain:" + getattr(custom_model, "model")
         elif model_name is not None:
-            self.model = langchain_model_from(model_name, provider)
             self.model_name = model_name
             self.model_provider = provider or "custom.langchain.default_provider"
         else:
             raise ValueError(
                 "model_name and provider must be provided if custom_model is not provided"
             )
+
+    def adapter_specific_instructions(self) -> str | None:
+        # TODO: would be better to explicitly use bind_tools:tool_choice="task_response" here
         if self.has_structured_output():
-            if not hasattr(self.model, "with_structured_output") or not callable(
-                getattr(self.model, "with_structured_output")
+            return "Always respond with a tool call. Never respond with a human readable message."
+        return None
+
+    async def model(self) -> LangChainModelType:
+        # cached model
+        if self._model:
+            return self._model
+
+        self._model = await langchain_model_from(self.model_name, self.model_provider)
+
+        if self.has_structured_output():
+            if not hasattr(self._model, "with_structured_output") or not callable(
+                getattr(self._model, "with_structured_output")
             ):
                 raise ValueError(
-                    f"model {self.model} does not support structured output, cannot use output_json_schema"
+                    f"model {self._model} does not support structured output, cannot use output_json_schema"
                 )
             # Langchain expects title/description to be at top level, on top of json schema
             output_schema = self.kiln_task.output_schema()
@@ -59,15 +79,10 @@ class LangChainPromptAdapter(BaseAdapter):
                 )
             output_schema["title"] = "task_response"
             output_schema["description"] = "A response from the task"
-            self.model = self.model.with_structured_output(
+            self._model = self._model.with_structured_output(
                 output_schema, include_raw=True
             )
-
-    def adapter_specific_instructions(self) -> str | None:
-        # TODO: would be better to explicitly use bind_tools:tool_choice="task_response" here
-        if self.has_structured_output():
-            return "Always respond with a tool call. Never respond with a human readable message."
-        return None
+        return self._model
 
     async def _run(self, input: Dict | str) -> Dict | str:
         prompt = self.build_prompt()
@@ -76,7 +91,8 @@ class LangChainPromptAdapter(BaseAdapter):
             SystemMessage(content=prompt),
             HumanMessage(content=user_msg),
         ]
-        response = self.model.invoke(messages)
+        model = await self.model()
+        response = model.invoke(messages)
 
         if self.has_structured_output():
             if (

kiln_ai/adapters/ml_model_list.py

@@ -2,9 +2,10 @@ import os
 from dataclasses import dataclass
 from enum import Enum
 from os import getenv
-from typing import Dict, List, NoReturn
+from typing import Any, Dict, List, NoReturn
 
 import httpx
+import requests
 from langchain_aws import ChatBedrockConverse
 from langchain_core.language_models.chat_models import BaseChatModel
 from langchain_groq import ChatGroq
@@ -43,6 +44,8 @@ class ModelFamily(str, Enum):
     phi = "phi"
     mistral = "mistral"
     gemma = "gemma"
+    gemini = "gemini"
+    claude = "claude"
 
 
 # Where models have instruct and raw versions, instruct is default and raw is specified
@@ -55,6 +58,9 @@ class ModelName(str, Enum):
     llama_3_1_8b = "llama_3_1_8b"
     llama_3_1_70b = "llama_3_1_70b"
     llama_3_1_405b = "llama_3_1_405b"
+    llama_3_2_3b = "llama_3_2_3b"
+    llama_3_2_11b = "llama_3_2_11b"
+    llama_3_2_90b = "llama_3_2_90b"
     gpt_4o_mini = "gpt_4o_mini"
     gpt_4o = "gpt_4o"
     phi_3_5 = "phi_3_5"
@@ -63,6 +69,11 @@ class ModelName(str, Enum):
     gemma_2_2b = "gemma_2_2b"
     gemma_2_9b = "gemma_2_9b"
     gemma_2_27b = "gemma_2_27b"
+    claude_3_5_sonnet = "claude_3_5_sonnet"
+    gemini_1_5_flash = "gemini_1_5_flash"
+    gemini_1_5_flash_8b = "gemini_1_5_flash_8b"
+    gemini_1_5_pro = "gemini_1_5_pro"
+    nemotron_70b = "nemotron_70b"
 
 
 class KilnModelProvider(BaseModel):
@@ -132,6 +143,67 @@ built_in_models: List[KilnModel] = [
             ),
         ],
     ),
+    # Claude 3.5 Sonnet
+    KilnModel(
+        family=ModelFamily.claude,
+        name=ModelName.claude_3_5_sonnet,
+        friendly_name="Claude 3.5 Sonnet",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                provider_options={"model": "anthropic/claude-3.5-sonnet"},
+            ),
+        ],
+    ),
+    # Gemini 1.5 Pro
+    KilnModel(
+        family=ModelFamily.gemini,
+        name=ModelName.gemini_1_5_pro,
+        friendly_name="Gemini 1.5 Pro",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                provider_options={"model": "google/gemini-pro-1.5"},
+            ),
+        ],
+    ),
+    # Gemini 1.5 Flash
+    KilnModel(
+        family=ModelFamily.gemini,
+        name=ModelName.gemini_1_5_flash,
+        friendly_name="Gemini 1.5 Flash",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                provider_options={"model": "google/gemini-flash-1.5"},
+            ),
+        ],
+    ),
+    # Gemini 1.5 Flash 8B
+    KilnModel(
+        family=ModelFamily.gemini,
+        name=ModelName.gemini_1_5_flash_8b,
+        friendly_name="Gemini 1.5 Flash 8B",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                provider_options={"model": "google/gemini-flash-1.5-8b"},
+            ),
+        ],
+    ),
+    # Nemotron 70B
+    KilnModel(
+        family=ModelFamily.llama,
+        name=ModelName.nemotron_70b,
+        friendly_name="Nemotron 70B",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                supports_structured_output=False,
+                provider_options={"model": "nvidia/llama-3.1-nemotron-70b-instruct"},
+            ),
+        ],
+    ),
     # Llama 3.1-8b
     KilnModel(
         family=ModelFamily.llama,
@@ -144,6 +216,7 @@ built_in_models: List[KilnModel] = [
             ),
             KilnModelProvider(
                 name=ModelProviderName.amazon_bedrock,
+                supports_structured_output=False,
                 provider_options={
                     "model": "meta.llama3-1-8b-instruct-v1:0",
                     "region_name": "us-west-2",  # Llama 3.1 only in west-2
@@ -151,10 +224,14 @@ built_in_models: List[KilnModel] = [
             ),
             KilnModelProvider(
                 name=ModelProviderName.ollama,
-                provider_options={"model": "llama3.1"},  # 8b is default
+                provider_options={
+                    "model": "llama3.1:8b",
+                    "model_aliases": ["llama3.1"],  # 8b is default
+                },
             ),
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
+                supports_structured_output=False,
                 provider_options={"model": "meta-llama/llama-3.1-8b-instruct"},
             ),
         ],
@@ -171,7 +248,6 @@ built_in_models: List[KilnModel] = [
             ),
             KilnModelProvider(
                 name=ModelProviderName.amazon_bedrock,
-                # TODO: this should work but a bug in the bedrock response schema
                 supports_structured_output=False,
                 provider_options={
                     "model": "meta.llama3-1-70b-instruct-v1:0",
@@ -182,11 +258,10 @@ built_in_models: List[KilnModel] = [
                 name=ModelProviderName.openrouter,
                 provider_options={"model": "meta-llama/llama-3.1-70b-instruct"},
             ),
-            # TODO: enable once tests update to check if model is available
-            # KilnModelProvider(
-            #     provider=ModelProviders.ollama,
-            #     provider_options={"model": "llama3.1:70b"},
-            # ),
+            KilnModelProvider(
+                name=ModelProviderName.ollama,
+                provider_options={"model": "llama3.1:70b"},
+            ),
         ],
     ),
     # Llama 3.1 405b
@@ -195,11 +270,6 @@ built_in_models: List[KilnModel] = [
         name=ModelName.llama_3_1_405b,
         friendly_name="Llama 3.1 405B",
         providers=[
-            # TODO: bring back when groq does: https://console.groq.com/docs/models
-            # KilnModelProvider(
-            #     name=ModelProviderName.groq,
-            #     provider_options={"model": "llama-3.1-405b-instruct-v1:0"},
-            # ),
             KilnModelProvider(
                 name=ModelProviderName.amazon_bedrock,
                 provider_options={
@@ -207,11 +277,10 @@ built_in_models: List[KilnModel] = [
                     "region_name": "us-west-2",  # Llama 3.1 only in west-2
                 },
             ),
-            # TODO: enable once tests update to check if model is available
-            # KilnModelProvider(
-            #     name=ModelProviderName.ollama,
-            #     provider_options={"model": "llama3.1:405b"},
-            # ),
+            KilnModelProvider(
+                name=ModelProviderName.ollama,
+                provider_options={"model": "llama3.1:405b"},
+            ),
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
                 provider_options={"model": "meta-llama/llama-3.1-405b-instruct"},
@@ -247,11 +316,49 @@ built_in_models: List[KilnModel] = [
                 name=ModelProviderName.openrouter,
                 provider_options={"model": "mistralai/mistral-large"},
             ),
-            # TODO: enable once tests update to check if model is available
-            # KilnModelProvider(
-            #     provider=ModelProviders.ollama,
-            #     provider_options={"model": "mistral-large"},
-            # ),
+            KilnModelProvider(
+                name=ModelProviderName.ollama,
+                provider_options={"model": "mistral-large"},
+            ),
+        ],
+    ),
+    # Llama 3.2 3B
+    KilnModel(
+        family=ModelFamily.llama,
+        name=ModelName.llama_3_2_3b,
+        friendly_name="Llama 3.2 3B",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                supports_structured_output=False,
+                provider_options={"model": "meta-llama/llama-3.2-3b-instruct"},
+            ),
+        ],
+    ),
+    # Llama 3.2 11B
+    KilnModel(
+        family=ModelFamily.llama,
+        name=ModelName.llama_3_2_11b,
+        friendly_name="Llama 3.2 11B",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                supports_structured_output=False,
+                provider_options={"model": "meta-llama/llama-3.2-11b-vision-instruct"},
+            ),
+        ],
+    ),
+    # Llama 3.2 90B
+    KilnModel(
+        family=ModelFamily.llama,
+        name=ModelName.llama_3_2_90b,
+        friendly_name="Llama 3.2 90B",
+        providers=[
+            KilnModelProvider(
+                name=ModelProviderName.openrouter,
+                supports_structured_output=False,
+                provider_options={"model": "meta-llama/llama-3.2-90b-vision-instruct"},
+            ),
         ],
     ),
     # Phi 3.5
@@ -263,6 +370,7 @@ built_in_models: List[KilnModel] = [
         providers=[
             KilnModelProvider(
                 name=ModelProviderName.ollama,
+                supports_structured_output=False,
                 provider_options={"model": "phi3.5"},
             ),
             KilnModelProvider(
@@ -280,6 +388,7 @@ built_in_models: List[KilnModel] = [
         providers=[
             KilnModelProvider(
                 name=ModelProviderName.ollama,
+                supports_structured_output=False,
                 provider_options={
                     "model": "gemma2:2b",
                 },
@@ -293,13 +402,12 @@ built_in_models: List[KilnModel] = [
         friendly_name="Gemma 2 9B",
         supports_structured_output=False,
         providers=[
-            # TODO: enable once tests update to check if model is available
-            # KilnModelProvider(
-            #     name=ModelProviderName.ollama,
-            #     provider_options={
-            #         "model": "gemma2:9b",
-            #     },
-            # ),
+            KilnModelProvider(
+                name=ModelProviderName.ollama,
+                provider_options={
+                    "model": "gemma2:9b",
+                },
+            ),
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
                 provider_options={"model": "google/gemma-2-9b-it"},
@@ -313,13 +421,12 @@ built_in_models: List[KilnModel] = [
         friendly_name="Gemma 2 27B",
         supports_structured_output=False,
         providers=[
-            # TODO: enable once tests update to check if model is available
-            # KilnModelProvider(
-            #     name=ModelProviderName.ollama,
-            #     provider_options={
-            #         "model": "gemma2:27b",
-            #     },
-            # ),
+            KilnModelProvider(
+                name=ModelProviderName.ollama,
+                provider_options={
+                    "model": "gemma2:27b",
+                },
+            ),
             KilnModelProvider(
                 name=ModelProviderName.openrouter,
                 provider_options={"model": "google/gemma-2-27b-it"},
@@ -417,7 +524,9 @@ def check_provider_warnings(provider_name: ModelProviderName):
             raise ValueError(warning_check.message)
 
 
-def langchain_model_from(name: str, provider_name: str | None = None) -> BaseChatModel:
+async def langchain_model_from(
+    name: str, provider_name: str | None = None
+) -> BaseChatModel:
     """
     Creates a LangChain chat model instance for the specified model and provider.
 
@@ -476,7 +585,23 @@ def langchain_model_from(name: str, provider_name: str | None = None) -> BaseCha
             **provider.provider_options,
         )
     elif provider.name == ModelProviderName.ollama:
-        return ChatOllama(**provider.provider_options, base_url=ollama_base_url())
+        # Ollama model naming is pretty flexible. We try a few versions of the model name
+        potential_model_names = []
+        if "model" in provider.provider_options:
+            potential_model_names.append(provider.provider_options["model"])
+        if "model_aliases" in provider.provider_options:
+            potential_model_names.extend(provider.provider_options["model_aliases"])
+
+        # Get the list of models Ollama supports
+        ollama_connection = await get_ollama_connection()
+        if ollama_connection is None:
+            raise ValueError("Failed to connect to Ollama. Ensure Ollama is running.")
+
+        for model_name in potential_model_names:
+            if ollama_model_supported(ollama_connection, model_name):
+                return ChatOllama(model=model_name, base_url=ollama_base_url())
+
+        raise ValueError(f"Model {name} not installed on Ollama")
     elif provider.name == ModelProviderName.openrouter:
         api_key = Config.shared().open_router_api_key
         base_url = getenv("OPENROUTER_BASE_URL") or "https://openrouter.ai/api/v1"
@@ -519,3 +644,67 @@ async def ollama_online() -> bool:
     except httpx.RequestError:
         return False
     return True
+
+
+class OllamaConnection(BaseModel):
+    message: str
+    models: List[str]
+
+
+# Parse the Ollama /api/tags response
+def parse_ollama_tags(tags: Any) -> OllamaConnection | None:
+    # Build a list of models we support for Ollama from the built-in model list
+    supported_ollama_models = [
+        provider.provider_options["model"]
+        for model in built_in_models
+        for provider in model.providers
+        if provider.name == ModelProviderName.ollama
+    ]
+    # Append model_aliases to supported_ollama_models
+    supported_ollama_models.extend(
+        [
+            alias
+            for model in built_in_models
+            for provider in model.providers
+            for alias in provider.provider_options.get("model_aliases", [])
+        ]
+    )
+
+    if "models" in tags:
+        models = tags["models"]
+        if isinstance(models, list):
+            model_names = [model["model"] for model in models]
+            print(f"model_names: {model_names}")
+            available_supported_models = [
+                model
+                for model in model_names
+                if model in supported_ollama_models
+                or model in [f"{m}:latest" for m in supported_ollama_models]
+            ]
+            if available_supported_models:
+                return OllamaConnection(
+                    message="Ollama connected",
+                    models=available_supported_models,
+                )
+
+    return OllamaConnection(
+        message="Ollama is running, but no supported models are installed. Install one or more supported model, like 'ollama pull phi3.5'.",
+        models=[],
+    )
+
+
+async def get_ollama_connection() -> OllamaConnection | None:
+    """
+    Gets the connection status for Ollama.
+    """
+    try:
+        tags = requests.get(ollama_base_url() + "/api/tags", timeout=5).json()
+
+    except Exception:
+        return None
+
+    return parse_ollama_tags(tags)
+
+
+def ollama_model_supported(conn: OllamaConnection, model_name: str) -> bool:
+    return model_name in conn.models or f"{model_name}:latest" in conn.models

kiln_ai/adapters/test_ml_model_list.py

@@ -1,10 +1,14 @@
+import json
 from unittest.mock import patch
 
 import pytest
 
 from kiln_ai.adapters.ml_model_list import (
     ModelProviderName,
+    OllamaConnection,
     check_provider_warnings,
+    ollama_model_supported,
+    parse_ollama_tags,
     provider_name_from_id,
     provider_warnings,
 )
@@ -97,3 +101,25 @@ def test_provider_name_from_id_case_sensitivity():
 )
 def test_provider_name_from_id_parametrized(provider_id, expected_name):
     assert provider_name_from_id(provider_id) == expected_name
+
+
+def test_parse_ollama_tags_no_models():
+    json_response = '{"models":[{"name":"phi3.5:latest","model":"phi3.5:latest","modified_at":"2024-10-02T12:04:35.191519822-04:00","size":2176178843,"digest":"61819fb370a3c1a9be6694869331e5f85f867a079e9271d66cb223acb81d04ba","details":{"parent_model":"","format":"gguf","family":"phi3","families":["phi3"],"parameter_size":"3.8B","quantization_level":"Q4_0"}},{"name":"gemma2:2b","model":"gemma2:2b","modified_at":"2024-09-09T16:46:38.64348929-04:00","size":1629518495,"digest":"8ccf136fdd5298f3ffe2d69862750ea7fb56555fa4d5b18c04e3fa4d82ee09d7","details":{"parent_model":"","format":"gguf","family":"gemma2","families":["gemma2"],"parameter_size":"2.6B","quantization_level":"Q4_0"}},{"name":"llama3.1:latest","model":"llama3.1:latest","modified_at":"2024-09-01T17:19:43.481523695-04:00","size":4661230720,"digest":"f66fc8dc39ea206e03ff6764fcc696b1b4dfb693f0b6ef751731dd4e6269046e","details":{"parent_model":"","format":"gguf","family":"llama","families":["llama"],"parameter_size":"8.0B","quantization_level":"Q4_0"}}]}'
+    tags = json.loads(json_response)
+    print(json.dumps(tags, indent=2))
+    conn = parse_ollama_tags(tags)
+    assert "phi3.5:latest" in conn.models
+    assert "gemma2:2b" in conn.models
+    assert "llama3.1:latest" in conn.models
+
+
+def test_ollama_model_supported():
+    conn = OllamaConnection(
+        models=["phi3.5:latest", "gemma2:2b", "llama3.1:latest"], message="Connected"
+    )
+    assert ollama_model_supported(conn, "phi3.5:latest")
+    assert ollama_model_supported(conn, "phi3.5")
+    assert ollama_model_supported(conn, "gemma2:2b")
+    assert ollama_model_supported(conn, "llama3.1:latest")
+    assert ollama_model_supported(conn, "llama3.1")
+    assert not ollama_model_supported(conn, "unknown_model")

kiln_ai/adapters/test_prompt_adaptors.py

@@ -16,9 +16,23 @@ async def test_groq(tmp_path):
     await run_simple_test(tmp_path, "llama_3_1_8b", "groq")
 
 
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        "llama_3_1_8b",
+        "llama_3_1_70b",
+        "gemini_1_5_pro",
+        "gemini_1_5_flash",
+        "gemini_1_5_flash_8b",
+        "nemotron_70b",
+        "llama_3_2_3b",
+        "llama_3_2_11b",
+        "llama_3_2_90b",
+    ],
+)
 @pytest.mark.paid
-async def test_openrouter(tmp_path):
-    await run_simple_test(tmp_path, "llama_3_1_8b", "openrouter")
+async def test_openrouter(tmp_path, model_name):
+    await run_simple_test(tmp_path, model_name, "openrouter")
 
 
 @pytest.mark.ollama

kiln_ai/adapters/test_structured_output.py

@@ -15,19 +15,21 @@ from kiln_ai.adapters.ml_model_list import (
 from kiln_ai.datamodel.test_json_schema import json_joke_schema, json_triangle_schema
 
 
+@pytest.mark.parametrize(
+    "model_name,provider",
+    [
+        ("llama_3_1_8b", "groq"),
+        ("mistral_nemo", "openrouter"),
+        ("llama_3_1_70b", "amazon_bedrock"),
+        ("claude_3_5_sonnet", "openrouter"),
+        ("gemini_1_5_pro", "openrouter"),
+        ("gemini_1_5_flash", "openrouter"),
+        ("gemini_1_5_flash_8b", "openrouter"),
+    ],
+)
 @pytest.mark.paid
-async def test_structured_output_groq(tmp_path):
-    await run_structured_output_test(tmp_path, "llama_3_1_8b", "groq")
-
-
-@pytest.mark.paid
-async def test_structured_output_openrouter(tmp_path):
-    await run_structured_output_test(tmp_path, "mistral_nemo", "openrouter")
-
-
-@pytest.mark.paid
-async def test_structured_output_bedrock(tmp_path):
-    await run_structured_output_test(tmp_path, "llama_3_1_70b", "amazon_bedrock")
+async def test_structured_output(tmp_path, model_name, provider):
+    await run_structured_output_test(tmp_path, model_name, provider)
 
 
 @pytest.mark.ollama
@@ -39,16 +41,17 @@ async def test_structured_output_ollama_phi(tmp_path):
     await run_structured_output_test(tmp_path, "phi_3_5", "ollama")
 
 
-@pytest.mark.ollama
+@pytest.mark.paid
 async def test_structured_output_gpt_4o_mini(tmp_path):
     await run_structured_output_test(tmp_path, "gpt_4o_mini", "openai")
 
 
+@pytest.mark.parametrize("model_name", ["llama_3_1_8b"])
 @pytest.mark.ollama
-async def test_structured_output_ollama_llama(tmp_path):
+async def test_structured_output_ollama_llama(tmp_path, model_name):
     if not await ollama_online():
         pytest.skip("Ollama API not running. Expect it running on localhost:11434")
-    await run_structured_output_test(tmp_path, "llama_3_1_8b", "ollama")
+    await run_structured_output_test(tmp_path, model_name, "ollama")
 
 
 class MockAdapter(BaseAdapter):
@@ -105,6 +108,7 @@ async def test_mock_unstructred_response(tmp_path):
 @pytest.mark.paid
 @pytest.mark.ollama
 async def test_all_built_in_models_structured_output(tmp_path):
+    errors = []
     for model in built_in_models:
         if not model.supports_structured_output:
             print(
@@ -121,7 +125,10 @@ async def test_all_built_in_models_structured_output(tmp_path):
                 print(f"Running {model.name} {provider.name}")
                 await run_structured_output_test(tmp_path, model.name, provider.name)
             except Exception as e:
-                raise RuntimeError(f"Error running {model.name} {provider}") from e
+                print(f"Error running {model.name} {provider.name}")
+                errors.append(f"{model.name} {provider.name}: {e}")
+    if len(errors) > 0:
+        raise RuntimeError(f"Errors: {errors}")
 
 
 def build_structured_output_test_task(tmp_path: Path):

{kiln_ai-0.5.2.dist-info → kiln_ai-0.5.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: kiln-ai
-Version: 0.5.2
+Version: 0.5.4
 Summary: Kiln AI
 Project-URL: Homepage, https://getkiln.ai
 Project-URL: Repository, https://github.com/Kiln-AI/kiln
@@ -8,6 +8,12 @@ Project-URL: Documentation, https://kiln-ai.github.io/Kiln/kiln_core_docs/kiln_a
 Project-URL: Issues, https://github.com/Kiln-AI/kiln/issues
 Author-email: "Steve Cosman, Chesterfield Laboratories Inc" <scosman@users.noreply.github.com>
 License-File: LICENSE.txt
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
 Requires-Dist: coverage>=7.6.4
 Requires-Dist: jsonschema>=4.23.0

{kiln_ai-0.5.2.dist-info → kiln_ai-0.5.4.dist-info}/RECORD

@@ -1,15 +1,15 @@
 kiln_ai/__init__.py,sha256=Sc4z8LRVFMwJUoc_DPVUriSXTZ6PO9MaJ80PhRbKyB8,34
 kiln_ai/adapters/__init__.py,sha256=3NC1lE_Sg1bF4IsKCoUgje2GL0IwTd1atw1BcDLI8IA,883
 kiln_ai/adapters/base_adapter.py,sha256=xXCISAJHaPCYHad28CS0wZEUlx711FZ_6AwW4rJx4jk,6688
-kiln_ai/adapters/langchain_adapters.py,sha256=Fo7w7hWdkxuuvxoNZhcGE25tOS6ObzhKEUKGszzPdtk,4929
-kiln_ai/adapters/ml_model_list.py,sha256=J13pDFp6UwTd7sa_kZyesRaqR1rin4U1iYMc5NYF05Q,17507
+kiln_ai/adapters/langchain_adapters.py,sha256=WNxhuTdjGCsCyqmXJNLe7HJ-MzJ08yagGV-eAHPZF-E,5411
+kiln_ai/adapters/ml_model_list.py,sha256=mqtFCav-m4m4DzxtoWHBktxrTB7haj0vUPx3xa8CL2U,23414
 kiln_ai/adapters/prompt_builders.py,sha256=nfZnEr1E30ZweQhEzIP21rNrL2Or1ILajyX8gU3B7w0,7796
 kiln_ai/adapters/test_langchain_adapter.py,sha256=_xHpVAkkoGh0PRO3BFFqvVj95SVtYZPOdFbYGYfzvQ0,1876
-kiln_ai/adapters/test_ml_model_list.py,sha256=BNuJSIegMMLzcICDR49qLFm7ezSl188LE4-W98c73tA,2786
-kiln_ai/adapters/test_prompt_adaptors.py,sha256=TXfSLfOHcg9EJINLfyJDQ-WcMw4He8ab4k-fGeryJcY,6033
+kiln_ai/adapters/test_ml_model_list.py,sha256=XHbwEFFb7WvZ6UkArqIiQ_yhS_urezHtgvJOSnaricY,4660
+kiln_ai/adapters/test_prompt_adaptors.py,sha256=z_X-REnWmTai23Ay_xSEB1qRKpp3HAeqgs4K6TQneb0,6332
 kiln_ai/adapters/test_prompt_builders.py,sha256=WmTR59tnKnKQ5gnX1X9EqvEUdQr0PQ8OvadYtRQR5sQ,11483
 kiln_ai/adapters/test_saving_adapter_results.py,sha256=tQvpLawo8mR2scPwmRCIz9Sp0ZkerS3kVJKBzlcjwRE,6041
-kiln_ai/adapters/test_structured_output.py,sha256=Okl6kLaAEKOuy1UBvQuiM5LGmJJi2aPB8sQR4bzIyIA,8755
+kiln_ai/adapters/test_structured_output.py,sha256=Z9A2R-TC-2atsdr8sGVGDlJhfa7uytW8Xi8PKBdEEAw,9033
 kiln_ai/adapters/repair/__init__.py,sha256=dOO9MEpEhjiwzDVFg3MNfA2bKMPlax9iekDatpTkX8E,217
 kiln_ai/adapters/repair/repair_task.py,sha256=VXvX1l9AYDE_GV0i3S_vPThltJoQlCFVCCHV9m-QA7k,3297
 kiln_ai/adapters/repair/test_repair_task.py,sha256=12PHb4SgBvVdLUzjZz31M8OTa8D8QjHD0Du4s7ij-i8,7819
@@ -27,7 +27,7 @@ kiln_ai/utils/__init__.py,sha256=PTD0MwBCKAMIOGsTAwsFaJOusTJJoRFTfOGqRvCaU-E,142
 kiln_ai/utils/config.py,sha256=jXUB8lwFkxLNEaizwIsoeFLg1BwjWr39-5KdEGF37Bg,5424
 kiln_ai/utils/formatting.py,sha256=VtB9oag0lOGv17dwT7OPX_3HzBfaU9GsLH-iLete0yM,97
 kiln_ai/utils/test_config.py,sha256=lbN0NhgKPEZ0idaS-zTn6mWsSAV6omo32JcIy05h2-M,7411
-kiln_ai-0.5.2.dist-info/METADATA,sha256=yClipqFUcwHfRWVBH6oJlE2D0S2ENBk58gFTapZBCjk,1718
-kiln_ai-0.5.2.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-kiln_ai-0.5.2.dist-info/licenses/LICENSE.txt,sha256=-AhuIX-CMdNGJNj74C29e9cKKmsh-1PBPINCsNvwAeg,82
-kiln_ai-0.5.2.dist-info/RECORD,,
+kiln_ai-0.5.4.dist-info/METADATA,sha256=UGIFG4gewCNOmUQpxYq_3g_dzCfq3tLuLLTVCnrhbOY,2017
+kiln_ai-0.5.4.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+kiln_ai-0.5.4.dist-info/licenses/LICENSE.txt,sha256=_NA5pnTYgRRr4qH6lE3X-TuZJ8iRcMUi5ASoGr-lEx8,1209
+kiln_ai-0.5.4.dist-info/RECORD,,

kiln_ai-0.5.4.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,13 @@
+
+
+This license applies only to the software in the libs/core directory.
+
+=======================================================
+
+Copyright 2024 - Chesterfield Laboratories Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

kiln_ai-0.5.2.dist-info/licenses/LICENSE.txt

@@ -1,4 +0,0 @@
-
-All rights reserved.
-
-Copyright (c) Steve Cosman, Chesterfield Laboratories Inc.