kiln-ai 0.16.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

kiln_ai/datamodel/test_task.py

@@ -1,7 +1,7 @@
 import pytest
 from pydantic import ValidationError
 
-from kiln_ai.datamodel.datamodel_enums import TaskOutputRatingType
+from kiln_ai.datamodel.datamodel_enums import StructuredOutputMode, TaskOutputRatingType
 from kiln_ai.datamodel.prompt_id import PromptGenerators
 from kiln_ai.datamodel.task import RunConfig, RunConfigProperties, Task, TaskRunConfig
 from kiln_ai.datamodel.task_output import normalize_rating
@@ -15,6 +15,7 @@ def test_runconfig_valid_creation():
         model_name="gpt-4",
         model_provider_name="openai",
         prompt_id=PromptGenerators.SIMPLE,
+        structured_output_mode="json_schema",
     )
 
     assert config.task == task
@@ -29,12 +30,13 @@ def test_runconfig_missing_required_fields():
 
     errors = exc_info.value.errors()
     assert (
-        len(errors) == 4
+        len(errors) == 5
     )  # task, model_name, model_provider_name, and prompt_id are required
     assert any(error["loc"][0] == "task" for error in errors)
     assert any(error["loc"][0] == "model_name" for error in errors)
     assert any(error["loc"][0] == "model_provider_name" for error in errors)
     assert any(error["loc"][0] == "prompt_id" for error in errors)
+    assert any(error["loc"][0] == "structured_output_mode" for error in errors)
 
 
 def test_runconfig_custom_prompt_id():
@@ -45,6 +47,7 @@ def test_runconfig_custom_prompt_id():
         model_name="gpt-4",
         model_provider_name="openai",
         prompt_id=PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT,
+        structured_output_mode="json_schema",
     )
 
     assert config.prompt_id == PromptGenerators.SIMPLE_CHAIN_OF_THOUGHT
@@ -61,6 +64,7 @@ def sample_run_config_props(sample_task):
         model_name="gpt-4",
         model_provider_name="openai",
         prompt_id=PromptGenerators.SIMPLE,
+        structured_output_mode="json_schema",
     )
 
 
@@ -157,3 +161,165 @@ def test_normalize_rating(rating_type, rating, expected):
 def test_normalize_rating_errors(rating_type, rating):
     with pytest.raises(ValueError):
         normalize_rating(rating, rating_type)
+
+
+def test_run_config_defaults():
+    """RunConfig should require top_p, temperature, and structured_output_mode to be set."""
+    task = Task(id="task1", name="Test Task", instruction="Do something")
+
+    config = RunConfig(
+        task=task,
+        model_name="gpt-4",
+        model_provider_name="openai",
+        prompt_id=PromptGenerators.SIMPLE,
+        structured_output_mode="json_schema",
+    )
+    assert config.top_p == 1.0
+    assert config.temperature == 1.0
+
+
+def test_run_config_valid_ranges():
+    """RunConfig should accept valid ranges for top_p and temperature."""
+    task = Task(id="task1", name="Test Task", instruction="Do something")
+
+    # Test valid values
+    config = RunConfig(
+        task=task,
+        model_name="gpt-4",
+        model_provider_name="openai",
+        prompt_id=PromptGenerators.SIMPLE,
+        top_p=0.9,
+        temperature=0.7,
+        structured_output_mode=StructuredOutputMode.json_schema,
+    )
+
+    assert config.top_p == 0.9
+    assert config.temperature == 0.7
+    assert config.structured_output_mode == StructuredOutputMode.json_schema
+
+
+@pytest.mark.parametrize("top_p", [0.0, 0.5, 1.0])
+def test_run_config_valid_top_p(top_p):
+    """Test that RunConfig accepts valid top_p values (0-1)."""
+    task = Task(id="task1", name="Test Task", instruction="Do something")
+
+    config = RunConfig(
+        task=task,
+        model_name="gpt-4",
+        model_provider_name="openai",
+        prompt_id=PromptGenerators.SIMPLE,
+        top_p=top_p,
+        temperature=1.0,
+        structured_output_mode=StructuredOutputMode.json_schema,
+    )
+
+    assert config.top_p == top_p
+
+
+@pytest.mark.parametrize("top_p", [-0.1, 1.1, 2.0])
+def test_run_config_invalid_top_p(top_p):
+    """Test that RunConfig rejects invalid top_p values."""
+    task = Task(id="task1", name="Test Task", instruction="Do something")
+
+    with pytest.raises(ValueError, match="top_p must be between 0 and 1"):
+        RunConfig(
+            task=task,
+            model_name="gpt-4",
+            model_provider_name="openai",
+            prompt_id=PromptGenerators.SIMPLE,
+            top_p=top_p,
+            temperature=1.0,
+            structured_output_mode=StructuredOutputMode.json_schema,
+        )
+
+
+@pytest.mark.parametrize("temperature", [0.0, 1.0, 2.0])
+def test_run_config_valid_temperature(temperature):
+    """Test that RunConfig accepts valid temperature values (0-2)."""
+    task = Task(id="task1", name="Test Task", instruction="Do something")
+
+    config = RunConfig(
+        task=task,
+        model_name="gpt-4",
+        model_provider_name="openai",
+        prompt_id=PromptGenerators.SIMPLE,
+        top_p=0.9,
+        temperature=temperature,
+        structured_output_mode=StructuredOutputMode.json_schema,
+    )
+
+    assert config.temperature == temperature
+
+
+@pytest.mark.parametrize("temperature", [-0.1, 2.1, 3.0])
+def test_run_config_invalid_temperature(temperature):
+    """Test that RunConfig rejects invalid temperature values."""
+    task = Task(id="task1", name="Test Task", instruction="Do something")
+
+    with pytest.raises(ValueError, match="temperature must be between 0 and 2"):
+        RunConfig(
+            task=task,
+            model_name="gpt-4",
+            model_provider_name="openai",
+            prompt_id=PromptGenerators.SIMPLE,
+            top_p=0.9,
+            temperature=temperature,
+            structured_output_mode=StructuredOutputMode.json_schema,
+        )
+
+
+def test_run_config_upgrade_old_entries():
+    """Test that TaskRunConfig parses old entries correctly with nested objects, filling in defaults where needed."""
+
+    data = {
+        "v": 1,
+        "name": "test name",
+        "created_at": "2025-06-09T13:33:35.276927",
+        "created_by": "scosman",
+        "run_config_properties": {
+            "model_name": "gpt_4_1_nano",
+            "model_provider_name": "openai",
+            "prompt_id": "task_run_config::189194447826::228174773209::244130257039",
+            "top_p": 0.77,
+            "temperature": 0.77,
+            "structured_output_mode": "json_instruction_and_object",
+        },
+        "prompt": {
+            "name": "Dazzling Unicorn",
+            "description": "Frozen copy of prompt 'simple_prompt_builder', created for evaluations.",
+            "generator_id": "simple_prompt_builder",
+            "prompt": "Generate a joke, given a theme. The theme will be provided as a word or phrase as the input to the model. The assistant should output a joke that is funny and relevant to the theme. If a style is provided, the joke should be in that style. The output should include a setup and punchline.\n\nYour response should respect the following requirements:\n1) Keep the joke on topic. If the user specifies a theme, the joke must be related to that theme.\n2) Avoid any jokes that are offensive or inappropriate. Keep the joke clean and appropriate for all audiences.\n3) Make the joke funny and engaging. It should be something that someone would want to tell to their friends. Something clever, not just a simple pun.\n",
+            "chain_of_thought_instructions": None,
+        },
+        "model_type": "task_run_config",
+    }
+
+    # Parse the data - this should be TaskRunConfig, not RunConfig
+    parsed = TaskRunConfig.model_validate(data)
+    assert parsed.name == "test name"
+    assert parsed.created_by == "scosman"
+    assert (
+        parsed.run_config_properties.structured_output_mode
+        == "json_instruction_and_object"
+    )
+
+    # should still work if loading from file
+    parsed = TaskRunConfig.model_validate(data, context={"loading_from_file": True})
+    assert parsed.name == "test name"
+    assert parsed.created_by == "scosman"
+    assert (
+        parsed.run_config_properties.structured_output_mode
+        == "json_instruction_and_object"
+    )
+
+    # Remove structured_output_mode from run_config_properties and parse again
+    del data["run_config_properties"]["structured_output_mode"]
+
+    with pytest.raises(ValidationError):
+        # should error if not loading from file
+        parsed = TaskRunConfig.model_validate(data)
+
+    parsed = TaskRunConfig.model_validate(data, context={"loading_from_file": True})
+    assert parsed.name == "test name"
+    assert parsed.created_by == "scosman"
+    assert parsed.run_config_properties.structured_output_mode == "unknown"

kiln_ai/utils/config.py

@@ -138,6 +138,7 @@ class Config:
                 sensitive_keys=["api_key"],
             ),
         }
+        self._lock = threading.Lock()
         self._settings = self.load_settings()
 
     @classmethod
@@ -180,7 +181,7 @@ class Config:
         return None if value is None else property_config.type(value)
 
     def __setattr__(self, name, value):
-        if name in ("_properties", "_settings"):
+        if name in ("_properties", "_settings", "_lock"):
             super().__setattr__(name, value)
         elif name in self._properties:
             self.update_settings({name: value})
@@ -234,7 +235,7 @@ class Config:
 
     def update_settings(self, new_settings: Dict[str, Any]):
         # Lock to prevent race conditions in multi-threaded scenarios
-        with threading.Lock():
+        with self._lock:
             # Fresh load to avoid clobbering changes from other instances
             current_settings = self.load_settings()
             current_settings.update(new_settings)

kiln_ai/utils/dataset_import.py

@@ -221,7 +221,7 @@ def import_csv(
     optional_headers = {"reasoning", "tags", "chain_of_thought"}  # optional headers
 
     rows: list[TaskRun] = []
-    with open(dataset_path, "r", newline="") as csvfile:
+    with open(dataset_path, "r", newline="", encoding="utf-8") as csvfile:
         reader = csv.DictReader(csvfile)
 
         # Check if we have headers

kiln_ai/utils/logging.py

@@ -0,0 +1,165 @@
+import datetime
+import json
+import logging
+import logging.handlers
+import os
+
+import litellm
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.litellm_logging import Logging
+
+from kiln_ai.utils.config import Config
+
+
+def get_default_formatter() -> str:
+    return "%(asctime)s.%(msecs)03d - %(levelname)s - %(name)s - %(message)s"
+
+
+def get_log_file_path(filename: str) -> str:
+    """Get the path to the log file, using environment override if specified.
+
+    Returns:
+        str: The path to the log file
+    """
+    log_path_default = os.path.join(Config.settings_dir(), "logs", filename)
+    log_path = os.getenv("KILN_LOG_FILE", log_path_default)
+
+    # Ensure the log directory exists
+    os.makedirs(os.path.dirname(log_path), exist_ok=True)
+    return log_path
+
+
+class CustomLiteLLMLogger(CustomLogger):
+    def __init__(self, logger: logging.Logger):
+        self.logger = logger
+
+    def log_pre_api_call(self, model, messages, kwargs):
+        api_base = kwargs.get("litellm_params", {}).get("api_base", "")
+        headers = kwargs.get("additional_args", {}).get("headers", {})
+        data = kwargs.get("additional_args", {}).get("complete_input_dict", {})
+
+        try:
+            # Print the curl command for the request
+            logger = Logging(
+                model=model,
+                messages=messages,
+                stream=False,
+                call_type="completion",
+                start_time=datetime.datetime.now(),
+                litellm_call_id="",
+                function_id="na",
+                kwargs=kwargs,
+            )
+            curl_command = logger._get_request_curl_command(
+                api_base=api_base,
+                headers=headers,
+                additional_args=kwargs,
+                data=data,
+            )
+            self.logger.info(f"{curl_command}")
+        except Exception as e:
+            self.logger.info(f"Curl Command: Could not print {e}")
+
+        # Print the formatted input data for the request in API format, pretty print
+        try:
+            self.logger.info(
+                f"Formatted Input Data (API):\n{json.dumps(data, indent=2)}"
+            )
+        except Exception as e:
+            self.logger.info(f"Formatted Input Data (API): Could not print {e}")
+
+        # Print the messages for the request in LiteLLM Message list, pretty print
+        try:
+            json_messages = json.dumps(messages, indent=2)
+            self.logger.info(f"Messages:\n{json_messages}")
+        except Exception as e:
+            self.logger.info(f"Messages: Could not print {e}")
+
+    def log_post_api_call(self, kwargs, response_obj, start_time, end_time):
+        # No op
+        pass
+
+    def log_success_event(self, kwargs, response_obj, start_time, end_time):
+        litellm_logger = logging.getLogger("LiteLLM")
+        litellm_logger.error(
+            "Used a sync call in Litellm. Kiln should use async calls."
+        )
+
+    def log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        # This logging method is supposed to be called by Litellm in synchronous error cases (Kiln should use async calls instead)
+        # but it appears to also be getting called in async calls that fail early (e.g. UnsupportedParamsError).
+        litellm_logger = logging.getLogger("LiteLLM")
+        litellm_logger.error(
+            "LiteLLM logged a synchronous failure event. This may result from a sync call, or from an async call failing early (e.g. invalid parameters). Make sure you are using async calls.",
+        )
+
+    #### ASYNC #### - for acompletion/aembeddings
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        try:
+            if len(response_obj.choices) == 1:
+                if response_obj.choices[0].message.tool_calls:
+                    for tool_call in response_obj.choices[0].message.tool_calls:
+                        try:
+                            args = tool_call.function.arguments
+                            function_name = tool_call.function.name
+                            self.logger.info(
+                                f"Model Response Tool Call Arguments [{function_name}]:\n{args}"
+                            )
+                        except Exception:
+                            self.logger.info(f"Model Response Tool Call:\n{tool_call}")
+
+                content = response_obj.choices[0].message.content
+                if content:
+                    try:
+                        # JSON format logs if possible
+                        json_content = json.loads(content)
+                        self.logger.info(
+                            f"Model Response Content:\n{json.dumps(json_content, indent=2)}"
+                        )
+                    except Exception:
+                        self.logger.info(f"Model Response Content:\n{content}")
+            elif len(response_obj.choices) > 1:
+                self.logger.info(
+                    f"Model Response (multiple choices):\n{response_obj.choices}"
+                )
+            else:
+                self.logger.info("Model Response: No choices returned")
+
+        except Exception as e:
+            self.logger.info(f"Model Response: Could not print {e}")
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        self.logger.info(f"LiteLLM Failure: {response_obj}")
+
+
+def setup_litellm_logging(filename: str = "model_calls.log"):
+    # Check if we already have a custom litellm logger
+    for callback in litellm.callbacks or []:
+        if isinstance(callback, CustomLiteLLMLogger):
+            return  # We already have a custom litellm logger
+
+    # If we don't have a custom litellm logger, create one
+    # Disable the default litellm logger except for errors. It's ugly, hard to use, and we don't want it to mix with kiln logs.
+    litellm_logger = logging.getLogger("LiteLLM")
+    litellm_logger.setLevel(logging.ERROR)
+
+    # Create a logger that logs to files, with a max size of 5MB and 3 backup files
+    handler = logging.handlers.RotatingFileHandler(
+        get_log_file_path(filename),
+        maxBytes=5 * 1024 * 1024,  # 5MB
+        backupCount=3,
+    )
+
+    # Set formatter to match the default formatting
+    formatter = logging.Formatter(get_default_formatter())
+    handler.setFormatter(formatter)
+
+    # Create a new logger for model calls
+    model_calls_logger = logging.getLogger("ModelCalls")
+    model_calls_logger.setLevel(logging.INFO)
+    model_calls_logger.propagate = False  # Only log to file
+    model_calls_logger.addHandler(handler)
+
+    # Tell litellm to use our custom logger
+    litellm.callbacks = [CustomLiteLLMLogger(model_calls_logger)]

kiln_ai/utils/test_config.py

@@ -1,5 +1,6 @@
 import getpass
 import os
+import threading
 from unittest.mock import patch
 
 import pytest
@@ -299,3 +300,25 @@ def test_yaml_persistence_structured_data(config_with_yaml, mock_yaml_file):
     with open(mock_yaml_file, "r") as f:
         saved_settings = yaml.safe_load(f)
     assert saved_settings["list_of_objects"] == new_settings
+
+
+def test_update_settings_thread_safety(config_with_yaml):
+    config = config_with_yaml
+
+    exceptions = []
+
+    def update(val):
+        try:
+            config.update_settings({"int_property": val})
+        except Exception as e:
+            exceptions.append(e)
+
+    threads = [threading.Thread(target=update, args=(i,)) for i in range(5)]
+
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+
+    assert not exceptions
+    assert config.int_property in range(5)

kiln_ai/utils/test_dataset_import.py

@@ -261,6 +261,36 @@ def test_import_csv_plain_text_missing_output(base_task: Task, tmp_path):
     assert "Missing required headers" in str(e.value)
 
 
+def test_import_csv_utf8_encoding(base_task: Task, tmp_path):
+    """Ensure UTF-8 encoded files are read correctly."""
+
+    row_data = [
+        {
+            "input": "Español entrada 你好👋",
+            "output": "salida áéí 你好👋",
+            "tags": "",
+        },
+    ]
+
+    file_path = dicts_to_file_as_csv(row_data, "utf8.csv", tmp_path)
+
+    importer = DatasetFileImporter(
+        base_task,
+        ImportConfig(
+            dataset_type=DatasetImportFormat.CSV,
+            dataset_path=file_path,
+            dataset_name="utf8.csv",
+        ),
+    )
+
+    importer.create_runs_from_file()
+
+    assert len(base_task.runs()) == 1
+    run = base_task.runs()[0]
+    assert run.input == "Español entrada 你好👋"
+    assert run.output.output == "salida áéí 你好👋"
+
+
 def test_import_csv_structured_output(task_with_structured_output: Task, tmp_path):
     row_data = [
         {

{kiln_ai-0.16.0.dist-info → kiln_ai-0.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiln-ai
-Version: 0.16.0
+Version: 0.17.0
 Summary: Kiln AI
 Project-URL: Homepage, https://getkiln.ai
 Project-URL: Repository, https://github.com/Kiln-AI/kiln

{kiln_ai-0.16.0.dist-info → kiln_ai-0.17.0.dist-info}/RECORD

@@ -1,51 +1,55 @@
 kiln_ai/__init__.py,sha256=Sc4z8LRVFMwJUoc_DPVUriSXTZ6PO9MaJ80PhRbKyB8,34
-kiln_ai/adapters/__init__.py,sha256=XjGmWagEyOEVwVIAxjN5rYNsQWIEACT5DB7MMTxdPss,1005
-kiln_ai/adapters/adapter_registry.py,sha256=KmMHYQ3mxpjVLE6D-hMNWCGt6Cw9JvnFn6nMb48GE8Y,9166
-kiln_ai/adapters/ml_model_list.py,sha256=DOl3kULrWOUwqEdOaKHk_LEPgnFhxLjcubpLk9Norek,87307
+kiln_ai/adapters/__init__.py,sha256=5GTN1bnEWz1NxWrlzsI6CfUio-D1jG74_-fx_PXJkY8,1027
+kiln_ai/adapters/adapter_registry.py,sha256=oEY6Zy6rtzs5mYLSZVhCzNWE4UJiUzIkVmmE_UYnBbE,8529
+kiln_ai/adapters/ml_model_list.py,sha256=kXt2eJ68HCUZ2dzfl9t-9pUv_s8149yMjRjmDi3GWuA,88641
 kiln_ai/adapters/ollama_tools.py,sha256=uObtLWfqKb9RXHN-TGGw2Y1FQlEMe0u8FgszI0zQn6U,3550
-kiln_ai/adapters/prompt_builders.py,sha256=Af4u0Oco0SfErBUGP7ujIu_qTK27d9VnOLumz2eqCmU,15639
-kiln_ai/adapters/provider_tools.py,sha256=7NjkxGD58P9s0NEbWDY2yToE2dTthlObU7qS_4oA4b4,15900
+kiln_ai/adapters/prompt_builders.py,sha256=R5IgZ7I2Ftx7i67xQb9UwcHE5gbPxgvgys_UAs3fc6A,15165
+kiln_ai/adapters/provider_tools.py,sha256=HIGsU38cAWKGONi8-Rh66WNOved3O0w9OYbj9tU_HU4,16794
 kiln_ai/adapters/run_output.py,sha256=RAi2Qp6dmqJVNm3CxbNTdAuhitHfH5NiUGbf6ygUP-k,257
-kiln_ai/adapters/test_adapter_registry.py,sha256=eDLHqv9mwgdde221pa47bTV87vCXwkUyjqsas-iFUrY,6123
+kiln_ai/adapters/test_adapter_registry.py,sha256=W4KoZ5cFTZwmW3i19cCgwzmcG802tpFC0GrUnZcD8_4,8378
+kiln_ai/adapters/test_ml_model_list.py,sha256=L5XdRz6c2vbfSYe0rWfsNOM73lZbuGsnG1gzgVPEUA8,6726
 kiln_ai/adapters/test_ollama_tools.py,sha256=xAUzL0IVmmXadVehJu1WjqbhpKEYGAgGt3pWx7hrubc,2514
-kiln_ai/adapters/test_prompt_adaptors.py,sha256=J1ZGZ8GG7SxP3_J3Zw0e6XmZY4NyPmUGX3IPgjh2LD8,7767
-kiln_ai/adapters/test_prompt_builders.py,sha256=ZwSpKjI0DFVZwmwdbBkooWVCwmwo6UAXsz4X9TyFtyw,22965
-kiln_ai/adapters/test_provider_tools.py,sha256=wdr9SUKt2dwapiYkIrbxbhtQ-vJLUP0PBn_ihIWXevc,30099
+kiln_ai/adapters/test_prompt_adaptors.py,sha256=nGq6H84rDMl72WUP2xqorQYW5yot0Z6sTQPXRL0mh5Y,8291
+kiln_ai/adapters/test_prompt_builders.py,sha256=SqGkluKZd7m7SPuq64NAprG0db11RDqoYai1m-1QgeQ,22585
+kiln_ai/adapters/test_provider_tools.py,sha256=QCknT944HN7jKLeSi9qS0cH_O5jKMMLThX7kWfAcJqg,32090
+kiln_ai/adapters/chat/__init__.py,sha256=ELydKUGeCcTiBJogzZUT8VXqr5kDtmoB8_GoyO28qR0,192
+kiln_ai/adapters/chat/chat_formatter.py,sha256=aBGpMUUBx0V_-g5GXkAhmhIqIyfMwd7kphFu6E8ivTM,8343
+kiln_ai/adapters/chat/test_chat_formatter.py,sha256=MVEZTSIFBwLvplOmit-4TDdcmPXsaMZMQEwoXWmq1FI,4603
 kiln_ai/adapters/data_gen/__init__.py,sha256=QTZWaf7kq5BorhPvexJfwDEKmjRmIbhwW9ei8LW2SIs,276
 kiln_ai/adapters/data_gen/data_gen_prompts.py,sha256=kudjHnAz7L3q0k_NLyTlaIV7M0uRFrxXNcfcnjOE2uc,5810
 kiln_ai/adapters/data_gen/data_gen_task.py,sha256=0PuYCcj09BtpgNj23mKj_L45mKZBdV5VreUeZ-Tj_xM,6642
-kiln_ai/adapters/data_gen/test_data_gen_task.py,sha256=cRKUKMvC0uVompbmPTKwbnQ_N3c0cQDm4J_9H4Y5U18,10129
+kiln_ai/adapters/data_gen/test_data_gen_task.py,sha256=_v80CcPpTRQRBGgO9U-tj8PNg5Ixzc-7xCgSocCRai0,10706
 kiln_ai/adapters/eval/__init__.py,sha256=0ptbK0ZxWuraxGn_WMgmE1tcaq0k5t-g-52kVohvWCg,693
-kiln_ai/adapters/eval/base_eval.py,sha256=IF4kYGt93bqJqSfj8UUaTng38fwPmi3cFKRSKUZhXJs,7381
-kiln_ai/adapters/eval/eval_runner.py,sha256=Wulelqb-qqcpjI33fgYhWiZhqFl7SqD0Q2o5xLEQQXQ,8729
-kiln_ai/adapters/eval/g_eval.py,sha256=4r2J091Q67EvxZ0LA0iEqrGsKMF54Am071tbW05nBrA,15239
+kiln_ai/adapters/eval/base_eval.py,sha256=Ms-OPfOmp4_7jIgAxi9QbwEPQYHkkHA-F2ZW11S6MLg,7324
+kiln_ai/adapters/eval/eval_runner.py,sha256=EjgAM88RhV51aIM1tNwmtnOU8a7uyaSM95ZYu0JZ-Pc,8791
+kiln_ai/adapters/eval/g_eval.py,sha256=yDsyepDhvsi67F2gi85KZQO_0UoWKNJSpxc43ufvJOU,16098
 kiln_ai/adapters/eval/registry.py,sha256=gZ_s0VgEx79Fswkgi1tS4yOl7lzpkvUBJZ62RldhM_w,626
-kiln_ai/adapters/eval/test_base_eval.py,sha256=_1CiOUOiBt1R_gGYMcRblrPkHf-H4uIlvfcHj5-Wh7o,10724
-kiln_ai/adapters/eval/test_eval_runner.py,sha256=82WPE_frNRTSQ2lylqT0inkqcDgM72nWt8GEuoDkJ7w,18568
-kiln_ai/adapters/eval/test_g_eval.py,sha256=-Stx7E0D-WAH1HWrRSp48CiGsf-no1SHeFF9IqVXeMI,16433
+kiln_ai/adapters/eval/test_base_eval.py,sha256=zNoIrFqImgccOazC4yNof11ZObHkvh_DVnaaBBNw_jU,15776
+kiln_ai/adapters/eval/test_eval_runner.py,sha256=37cmZPjvZaNlNDzmstmOEUwQEFhpafD9jRhDD1UH6PE,18718
+kiln_ai/adapters/eval/test_g_eval.py,sha256=KaVjG6aHQQzGlOd4J_7S8YKR3EmA2aFpY8VOEDdrRHY,16479
 kiln_ai/adapters/eval/test_g_eval_data.py,sha256=8caiZfLWnXVX8alrBPrH7L7gqqSS9vO7u6PzcHurQcA,27769
 kiln_ai/adapters/fine_tune/__init__.py,sha256=DxdTR60chwgck1aEoVYWyfWi6Ed2ZkdJj0lar-SEAj4,257
-kiln_ai/adapters/fine_tune/base_finetune.py,sha256=g-lWuZMkOj2djcczuHke_Ai7Z7RPg41AFSgoxHgsw3U,5889
-kiln_ai/adapters/fine_tune/dataset_formatter.py,sha256=OCfU-hVR_pM9Y_rQCPjyq7sMR2f20A8gKfyh45AQN28,17931
+kiln_ai/adapters/fine_tune/base_finetune.py,sha256=CGho356kKnxQsvkHws9XfWuBYV0nmbCvEkff21D4e48,5914
+kiln_ai/adapters/fine_tune/dataset_formatter.py,sha256=yOVefk2qXwX29Y2nKU0XWtQgf1kNNMY5M0NhzsaatZ8,12975
 kiln_ai/adapters/fine_tune/finetune_registry.py,sha256=9RJLjviSoN3dQnKJE9Ss7df7dtdJgbuShB8IUcI-q9k,726
 kiln_ai/adapters/fine_tune/fireworks_finetune.py,sha256=ze0QxghpHAqwO9nXOTkDEC9irmqduX5bjIhZDU0DCZQ,20101
 kiln_ai/adapters/fine_tune/openai_finetune.py,sha256=Dz9E_0BWfrIkvv8ArZe-RKPwbIKPZ3v8rfbc3JELyTY,8571
-kiln_ai/adapters/fine_tune/test_base_finetune.py,sha256=Tq0Klw7ou5_6H_bouTbI3PxYw7H30K32wlgWJE_luYk,10751
-kiln_ai/adapters/fine_tune/test_dataset_formatter.py,sha256=cHeiUts5rNJooslAy8SlnUgSejT0WGGwgNl_3WYj7lI,32949
-kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py,sha256=NCl2U6ZqqJ8dnysGGmfir9RGcV-StPtoi5cetRjW6Zc,36754
-kiln_ai/adapters/fine_tune/test_openai_finetune.py,sha256=H63Xk2PNHbt5Ev5IQpdR9JZ4uz-Huo2gfuC4mHHqe0w,20011
-kiln_ai/adapters/fine_tune/test_together_finetune.py,sha256=BUJFsyq_g77gU0JN3hg6FMBvqb0DIyTeAek-wxomKIg,18090
-kiln_ai/adapters/fine_tune/test_vertex_finetune.py,sha256=rAmcQJNPXqRacxg6RzjEQ8FNLKCp9qZRHToH7fm-7W0,19214
-kiln_ai/adapters/fine_tune/together_finetune.py,sha256=EbMPsTyKMubfwOalkFLiNFlMFIRKxLibzMTyLeUkle4,14010
+kiln_ai/adapters/fine_tune/test_base_finetune.py,sha256=atZI54SC6mVili5dEIuqwv2groLCZWYKSMzDZYjbUxQ,10705
+kiln_ai/adapters/fine_tune/test_dataset_formatter.py,sha256=7-axt5WPk1cEmsojr9xG6o1PHu7EvEn_YWIg1ZmqRK0,29907
+kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py,sha256=dzOOFIzgRUGGNlfA7-L1URI4qwMVHg1fuUc8RBqrDNI,36765
+kiln_ai/adapters/fine_tune/test_openai_finetune.py,sha256=NtMFo3LZjpAsNHFvIMHsFmWRrHrOKjniRvrZTx4nXhg,20001
+kiln_ai/adapters/fine_tune/test_together_finetune.py,sha256=ZwYmT4pPkhU-45Q-5fs4WL16dQ4OyvI7KXPevsrA24E,18122
+kiln_ai/adapters/fine_tune/test_vertex_finetune.py,sha256=W7oz6ZJktde1WludBb2lFpTXmU7VQN1FkeJpyN1dPbI,19218
+kiln_ai/adapters/fine_tune/together_finetune.py,sha256=rsJETfwkx21UzJH1CUd_hP-v1zQNBxK5b2uJIYSTpS4,14775
 kiln_ai/adapters/fine_tune/vertex_finetune.py,sha256=Ik6Ov711-oruJnMHpVZTPimWJY2W_JnfdKIdR2djGrc,8545
 kiln_ai/adapters/model_adapters/__init__.py,sha256=m5GRtOHwVVvp_XDOss8c1X3NFf1wQQlC2eBgI4tXQhM,212
-kiln_ai/adapters/model_adapters/base_adapter.py,sha256=2CVGoKVfPSqiC9j0SdAPUj4HiFb8-io5zNG0IIKQtBc,10965
-kiln_ai/adapters/model_adapters/litellm_adapter.py,sha256=ic4eLbmFZDcUNDTRNT6WQ1Qw_haf3ciu10cvzqcsg18,17791
-kiln_ai/adapters/model_adapters/litellm_config.py,sha256=7-tIh5cuVu23Uy2Sd6q7UCT_4VgevBsAzVhQMj6Svgw,425
-kiln_ai/adapters/model_adapters/test_base_adapter.py,sha256=E9XnCOJm9DNEPpt4otyMYn_AzK3WU4xVDRwxlSvPBI4,9007
-kiln_ai/adapters/model_adapters/test_litellm_adapter.py,sha256=8koRwW3yxmECJamYeRc53g4dq0SwwCa0g2i6nt9xNXA,15896
-kiln_ai/adapters/model_adapters/test_saving_adapter_results.py,sha256=XGainAeG3PkvmKhlHZG56POfiOjZ1Fl8pHvci_ouFiY,7907
-kiln_ai/adapters/model_adapters/test_structured_output.py,sha256=Dqoy4zKi8rG0S3vbnpx1XBDHHj3oj-ayPY3xLpuVyU4,12457
+kiln_ai/adapters/model_adapters/base_adapter.py,sha256=mGF3ozrFp14_elYXghgzZiWrEKo6EGY9lSdo3aRaJ-A,13101
+kiln_ai/adapters/model_adapters/litellm_adapter.py,sha256=G9SBxw-B6uB8I8LyKaZ6WDAq2OaUOCmvoVNYKbNdVu4,17074
+kiln_ai/adapters/model_adapters/litellm_config.py,sha256=zOQEkYKeoQ0FIbxTkyyoaGtaQiB9eYK3IuyUgqSwzLE,485
+kiln_ai/adapters/model_adapters/test_base_adapter.py,sha256=CWvVCr3Y_m8iQ1jU60PDG7qPpIP3gOSAXUfOUsQVAfI,16050
+kiln_ai/adapters/model_adapters/test_litellm_adapter.py,sha256=NZ9rXUHCDjNNuvzMjOHmmT5U02HXCm7WkuyNOXYaLNU,17301
+kiln_ai/adapters/model_adapters/test_saving_adapter_results.py,sha256=xkd_b9GS-6ybst97FsS_XrvyTXMlEufYF1Q8CGpE1V8,9697
+kiln_ai/adapters/model_adapters/test_structured_output.py,sha256=-4NWrDTYJXC53UKoKfYzAg-uyD73I-iApqN-9FNB2qM,13463
 kiln_ai/adapters/parsers/__init__.py,sha256=TGJS_8JhjUwg5Bnq4cDmwt5eIRo4vowmcL2A72L1Hzk,202
 kiln_ai/adapters/parsers/base_parser.py,sha256=AE8UYCttmVXbilpICotnDdFYTFhGMiBJHrxIUgPTAWM,280
 kiln_ai/adapters/parsers/json_parser.py,sha256=IszrBrhIFrrVr76UZsuejkBdqpZG27mU72264HVgVzE,1274
@@ -58,14 +62,14 @@ kiln_ai/adapters/parsers/test_r1_parser.py,sha256=XbB0saThkcgOvoYNq3TxDgILUu5Me3
 kiln_ai/adapters/parsers/test_request_formatters.py,sha256=t3FlKT_Tte2u8zXJTMl8VaE8IrSzumuBysahbGesrbU,2090
 kiln_ai/adapters/repair/__init__.py,sha256=dOO9MEpEhjiwzDVFg3MNfA2bKMPlax9iekDatpTkX8E,217
 kiln_ai/adapters/repair/repair_task.py,sha256=iW0bHWQq6Tir6ULTATWFS0zpwNji8Tbwhm2lZu52RsM,3342
-kiln_ai/adapters/repair/test_repair_task.py,sha256=ZMoQcIpD_-O9lhtlMD6pN-VgzJURpMIZ-faVz1-taXs,7966
-kiln_ai/datamodel/__init__.py,sha256=MCEPLNpNFNP0zCwQfDaM8yKs7stdtY8c7scf0pEtfdQ,1981
+kiln_ai/adapters/repair/test_repair_task.py,sha256=fMlad29edA2tTt4t7cv6cXVWhuWOJ9x1Mpb3wJFTD1I,8603
+kiln_ai/datamodel/__init__.py,sha256=eHDUB9ltKmnsx0TAX310o5OMLnZbSznMJLFRnNk6HlA,1927
 kiln_ai/datamodel/basemodel.py,sha256=Qsr3Kxq5B5rfPZxQFrfOMTGwInnfYy5srd8nK0nwOs0,22204
-kiln_ai/datamodel/datamodel_enums.py,sha256=5ktCfKzNM2lHUFv-BUjrHj2GOtDoB4kd2E8Lh1giirk,2877
+kiln_ai/datamodel/datamodel_enums.py,sha256=7BpyiXEtNsz43el7zTAeaaozDVUZUJCFKHsoaXoTomQ,3796
 kiln_ai/datamodel/dataset_filters.py,sha256=zXS5QeCqWa1vAxs3Pyy5FJtySx9yYJzxsCMANd23-3c,5502
 kiln_ai/datamodel/dataset_split.py,sha256=GDF3Pj3NLG42X8kjR606R_aN14rpqx8Ig56FG8NCn9k,5790
 kiln_ai/datamodel/eval.py,sha256=larT3emjpLVYOSB2ga5GHTS4O7l9mJwtIO4EqCyYF7g,14214
-kiln_ai/datamodel/finetune.py,sha256=XvwpYDNiqZEJeMzmc81jqG5ZSXR2hWcBOS6hXiZASu8,4599
+kiln_ai/datamodel/finetune.py,sha256=mOTM0KEbRNZ5fvzgpUtLRugaGkhWQKFwdsQWggIzMls,4813
 kiln_ai/datamodel/json_schema.py,sha256=o50wSp8frRXjT-NZjml4-Is7LNoF7DQP4g3AaaYzBfI,3379
 kiln_ai/datamodel/model_cache.py,sha256=9X4aAigbkFdytckgw8InCMh86uBna0ME_1HJSeMPEn0,4495
 kiln_ai/datamodel/project.py,sha256=uVH2_3TDFtsG_tpts81A-zbd9uPDFxAwMCKZt_km3IE,727
@@ -73,36 +77,37 @@ kiln_ai/datamodel/prompt.py,sha256=70JPYHfgyX18cHW_DXoMzIOA28Jbaz6gyabElmpycyc,1
 kiln_ai/datamodel/prompt_id.py,sha256=S4Wcrt05USN-JrO8BeDlNYGrcoTOVocR3iUxBbgoq1c,2566
 kiln_ai/datamodel/registry.py,sha256=XwGFXJFKZtOpR1Z9ven6SftggfADdZRm8TFxCEVtfUQ,957
 kiln_ai/datamodel/strict_mode.py,sha256=sm4Xka8mnJHCShtbh6MMU5dDQv-cLj8lHgHkmFKpsl0,849
-kiln_ai/datamodel/task.py,sha256=r-_zgrQCIiIkN8gvBISdU449Z9oKp7E1XL0lkik_rVI,7036
+kiln_ai/datamodel/task.py,sha256=ZjTfOxk4Op83CAVp1KFsuHeaeqBNAvdVS-QQCKCNIug,9253
 kiln_ai/datamodel/task_output.py,sha256=P7bRI-Qn2n7f8XpR7hwgSTKhx3gzvsZ1aJOQMl-YW4s,13245
 kiln_ai/datamodel/task_run.py,sha256=0Lwc7zQoeWzuvHYv00lUGfsNQp8gCKmCVKAq3FDrruY,8806
-kiln_ai/datamodel/test_basemodel.py,sha256=arzlXwtPi0H3BRTCILn7WkcBV9bFFAI87iiEs5BsWVA,17978
+kiln_ai/datamodel/test_basemodel.py,sha256=1__3dVyxCVMQH7jhBYYEYgaCgtc08faSIlVOLfPdMZ0,18021
 kiln_ai/datamodel/test_dataset_filters.py,sha256=TFHQZLB0rJxnxsFjD546eXVFxZNAJi0sPZ8j24wYd1w,5322
-kiln_ai/datamodel/test_dataset_split.py,sha256=NdJ042JNq9A6JGaMt6LS19DMz1IiwcVCZpUHw0AqJrE,11253
+kiln_ai/datamodel/test_dataset_split.py,sha256=mXB8udml_6U4BXR0xAGHsFINnhdcTDB1qhuZbQemk-w,11055
 kiln_ai/datamodel/test_datasource.py,sha256=H4Kc-Im9eM7WnADWZXdoiOIrOl05RtkyuhTCKiRimyU,3905
 kiln_ai/datamodel/test_eval_model.py,sha256=J7MqwWBgPpeXGqh3IacVUUHdZFJSZ2MgTsUNu-hNOJw,19528
 kiln_ai/datamodel/test_example_models.py,sha256=dwLAAOXLvdKupE5Q1m6VNcHtfdhpi2qWtoEbC0nfJg8,26156
 kiln_ai/datamodel/test_json_schema.py,sha256=R0Cfc9WbieMslgvYsj2HFx8RHIq2fF9NcT5jH-kEqh4,4793
 kiln_ai/datamodel/test_model_cache.py,sha256=Fy-ucYNzS5JEG-8SFY4nVHA8iRbXXxai20f8_oGl97o,8184
 kiln_ai/datamodel/test_model_perf.py,sha256=9_76f__5XtZSHSjuaoiHRj2t-z3OWn-sSA4S9kH1jpY,3306
-kiln_ai/datamodel/test_models.py,sha256=ZZZdhN6-uWDBfyVr7b7LZhDA7B2NIcPlNGfnb3HI7II,21084
+kiln_ai/datamodel/test_models.py,sha256=wENuBiZ2Y-N_puFkitNZ1T-lBpVgQWotfhGug6k4AMY,21674
 kiln_ai/datamodel/test_nested_save.py,sha256=xciCddqvPyKyoyjC5Lx_3Kh1t4LJv1xYRAPazR3SRcs,5588
 kiln_ai/datamodel/test_output_rating.py,sha256=zvPIp2shAgCs2RQBgwYoL09fRA3krHvgAqUa91RlWR0,15125
 kiln_ai/datamodel/test_prompt_id.py,sha256=ihyXVPQi0dSLGnBM7rTXRnVaiWXhh7HJmSy4nZZKmso,4225
 kiln_ai/datamodel/test_registry.py,sha256=PhS4anLi5Bf_023obuTlO5DALhtPB8WIc_bX12Yg6Po,2705
-kiln_ai/datamodel/test_task.py,sha256=FYyoEqJXQIy8rcBsLTdki4-1z9COnZQk1-aoS3ZoNuU,5307
+kiln_ai/datamodel/test_task.py,sha256=X85YgGt7Y9kuv6-jE9kl5b8mPz3cgrEFqiZAZFWfpO0,11890
 kiln_ai/utils/__init__.py,sha256=PTD0MwBCKAMIOGsTAwsFaJOusTJJoRFTfOGqRvCaU-E,142
 kiln_ai/utils/async_job_runner.py,sha256=1gjoEq5yc2MOVjDo05O1wztguEuMC6l3haDZsltlvuw,3457
-kiln_ai/utils/config.py,sha256=kAgb_4nSnb-IWbLVS4FBygYmnHypJADjDEk_Fh-eLeg,8479
-kiln_ai/utils/dataset_import.py,sha256=zz8O4RniellZ0vPipeTUltWU7J7aCO018beD_cNHMb4,8993
+kiln_ai/utils/config.py,sha256=Pk9w7C985jinE9_SDJzgkG1CkPtIaitLZqia_dtjhwQ,8520
+kiln_ai/utils/dataset_import.py,sha256=EqBBBopCEUy1JH4-EAsBETwGp4MFjzZGfUUBZ6FLfGY,9011
 kiln_ai/utils/exhaustive_error.py,sha256=TkkRixIAR3CPEKHeAJzyv0mtxp6BxUBKMvobA3vzQug,262
 kiln_ai/utils/formatting.py,sha256=VtB9oag0lOGv17dwT7OPX_3HzBfaU9GsLH-iLete0yM,97
+kiln_ai/utils/logging.py,sha256=LY7EnxZIsSuVxKP53JMLp-TdFomOMkxN793trN_kAws,6729
 kiln_ai/utils/name_generator.py,sha256=v26TgpCwQbhQFcZvzgjZvURinjrOyyFhxpsI6NQrHKc,1914
 kiln_ai/utils/test_async_job_runner.py,sha256=8AwPfOlR09qzfhVm-djpkmuoyHxJJ19QEzs0WV5KFSQ,6813
-kiln_ai/utils/test_config.py,sha256=Jw3nMFeIgZUsZDRJJY2HpB-2EkR2NoZ-rDe_o9oA7ws,9174
-kiln_ai/utils/test_dataset_import.py,sha256=WxyjquKod8szKNA2HYs7tNTn71Apg5G9MNI8GRWhAlI,24965
+kiln_ai/utils/test_config.py,sha256=8goGdVLOO1OiSPnuDLUHbEJWJ8hAjaAIDg0myhR2A00,9687
+kiln_ai/utils/test_dataset_import.py,sha256=BEl38D95HQYpc7_jeB1N-ocOnKM1DLutp669cNrVOuE,25765
 kiln_ai/utils/test_name_geneator.py,sha256=9-hSTBshyakqlPbFnNcggwLrL7lcPTitauBYHg9jFWI,1513
-kiln_ai-0.16.0.dist-info/METADATA,sha256=9BCNdJP6PBasSKgtPdVZGdhQM-IGpJFHgJhezj9Lplw,12263
-kiln_ai-0.16.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kiln_ai-0.16.0.dist-info/licenses/LICENSE.txt,sha256=_NA5pnTYgRRr4qH6lE3X-TuZJ8iRcMUi5ASoGr-lEx8,1209
-kiln_ai-0.16.0.dist-info/RECORD,,
+kiln_ai-0.17.0.dist-info/METADATA,sha256=W9UIYyQGlhO3fcDr9IBaxUTST_lF_-DHKGm3cCUPcH0,12263
+kiln_ai-0.17.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kiln_ai-0.17.0.dist-info/licenses/LICENSE.txt,sha256=_NA5pnTYgRRr4qH6lE3X-TuZJ8iRcMUi5ASoGr-lEx8,1209
+kiln_ai-0.17.0.dist-info/RECORD,,