kiln-ai 0.17.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

kiln_ai/adapters/model_adapters/test_structured_output.py

@@ -245,7 +245,7 @@ async def run_structured_input_task_no_validation(
     try:
         run = await a.invoke({"a": 2, "b": 2, "c": 2})
         response = run.output.output
-        return response, a
+        return response, a, run
     except ValueError as e:
         if str(e) == "Failed to connect to Ollama. Ensure Ollama is running.":
             pytest.skip(
@@ -260,7 +260,7 @@ async def run_structured_input_task(
     provider: str,
     prompt_id: PromptId,
 ):
-    response, a = await run_structured_input_task_no_validation(
+    response, a, run = await run_structured_input_task_no_validation(
         task, model_name, provider, prompt_id
     )
     assert response is not None
@@ -350,7 +350,7 @@ When asked for a final result, this is the format (for an equilateral example):
 """
     task.output_json_schema = json.dumps(triangle_schema)
     task.save_to_file()
-    response, adapter = await run_structured_input_task_no_validation(
+    response, adapter, _ = await run_structured_input_task_no_validation(
         task, model_name, provider_name, "simple_chain_of_thought_prompt_builder"
     )
 

kiln_ai/adapters/parsers/parser_registry.py

@@ -1,5 +1,3 @@
-from typing import Type
-
 from kiln_ai.adapters.ml_model_list import ModelParserID
 from kiln_ai.adapters.parsers.base_parser import BaseParser
 from kiln_ai.adapters.parsers.r1_parser import R1ThinkingParser

kiln_ai/adapters/parsers/r1_parser.py

@@ -1,5 +1,4 @@
 from kiln_ai.adapters.parsers.base_parser import BaseParser
-from kiln_ai.adapters.parsers.json_parser import parse_json_string
 from kiln_ai.adapters.run_output import RunOutput
 
 

kiln_ai/adapters/remote_config.py

@@ -0,0 +1,66 @@
+import argparse
+import json
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import List
+
+import requests
+
+from .ml_model_list import KilnModel, built_in_models
+
+logger = logging.getLogger(__name__)
+
+
+def serialize_config(models: List[KilnModel], path: str | Path) -> None:
+    data = {"model_list": [m.model_dump(mode="json") for m in models]}
+    Path(path).write_text(json.dumps(data, indent=2, sort_keys=True))
+
+
+def deserialize_config(path: str | Path) -> List[KilnModel]:
+    raw = json.loads(Path(path).read_text())
+    model_data = raw.get("model_list", raw if isinstance(raw, list) else [])
+    return [KilnModel.model_validate(item) for item in model_data]
+
+
+def load_from_url(url: str) -> List[KilnModel]:
+    response = requests.get(url, timeout=10)
+    response.raise_for_status()
+    data = response.json()
+    if isinstance(data, list):
+        model_data = data
+    else:
+        model_data = data.get("model_list", [])
+    return [KilnModel.model_validate(item) for item in model_data]
+
+
+def dump_builtin_config(path: str | Path) -> None:
+    serialize_config(built_in_models, path)
+
+
+def load_remote_models(url: str) -> None:
+    if os.environ.get("KILN_SKIP_REMOTE_MODEL_LIST") == "true":
+        return
+
+    def fetch_and_replace() -> None:
+        try:
+            models = load_from_url(url)
+            built_in_models[:] = models
+        except Exception as exc:
+            # Do not crash startup, but surface the issue
+            logger.warning("Failed to fetch remote model list from %s: %s", url, exc)
+
+    thread = threading.Thread(target=fetch_and_replace, daemon=True)
+    thread.start()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="output path")
+    args = parser.parse_args()
+    dump_builtin_config(args.path)
+
+
+if __name__ == "__main__":
+    main()

kiln_ai/adapters/repair/repair_task.py

@@ -1,13 +1,8 @@
 import json
-from typing import Type
 
 from pydantic import BaseModel, Field
 
-from kiln_ai.adapters.prompt_builders import (
-    BasePromptBuilder,
-    SavedPromptBuilder,
-    prompt_builder_from_id,
-)
+from kiln_ai.adapters.prompt_builders import BasePromptBuilder, prompt_builder_from_id
 from kiln_ai.datamodel import Priority, Project, Task, TaskRequirement, TaskRun
 
 

kiln_ai/adapters/test_ml_model_list.py

@@ -156,3 +156,21 @@ class TestDefaultStructuredOutputModeForModelProvider:
             provider=first_provider.name,
         )
         assert result == first_provider.structured_output_mode
+
+
+def test_uncensored():
+    """Test that uncensored is set correctly"""
+    model = get_model_by_name(ModelName.grok_3_mini)
+    for provider in model.providers:
+        assert provider.uncensored
+        assert not provider.suggested_for_uncensored_data_gen
+
+    model = get_model_by_name(ModelName.gpt_4_1_nano)
+    for provider in model.providers:
+        assert not provider.uncensored
+        assert not provider.suggested_for_uncensored_data_gen
+
+    model = get_model_by_name(ModelName.grok_4)
+    for provider in model.providers:
+        assert provider.uncensored
+        assert provider.suggested_for_uncensored_data_gen

kiln_ai/adapters/test_prompt_adaptors.py

@@ -13,10 +13,6 @@ from kiln_ai.adapters.model_adapters.litellm_adapter import (
     LiteLlmConfig,
 )
 from kiln_ai.adapters.ollama_tools import ollama_online
-from kiln_ai.adapters.prompt_builders import (
-    BasePromptBuilder,
-    SimpleChainOfThoughtPromptBuilder,
-)
 from kiln_ai.datamodel import PromptId
 from kiln_ai.datamodel.task import RunConfigProperties
 

kiln_ai/adapters/test_remote_config.py

@@ -0,0 +1,100 @@
+import asyncio
+import os
+from unittest.mock import patch
+
+import pytest
+
+from kiln_ai.adapters.ml_model_list import built_in_models
+from kiln_ai.adapters.remote_config import (
+    deserialize_config,
+    dump_builtin_config,
+    load_from_url,
+    load_remote_models,
+    serialize_config,
+)
+
+
+def test_round_trip(tmp_path):
+    path = tmp_path / "models.json"
+    serialize_config(built_in_models, path)
+    loaded = deserialize_config(path)
+    assert [m.model_dump(mode="json") for m in loaded] == [
+        m.model_dump(mode="json") for m in built_in_models
+    ]
+
+
+def test_load_from_url():
+    sample = [built_in_models[0].model_dump(mode="json")]
+
+    class FakeResponse:
+        def raise_for_status(self):
+            pass
+
+        def json(self):
+            return {"model_list": sample}
+
+    with patch(
+        "kiln_ai.adapters.remote_config.requests.get", return_value=FakeResponse()
+    ):
+        models = load_from_url("http://example.com/models.json")
+    assert [m.model_dump(mode="json") for m in models] == sample
+
+
+def test_dump_builtin_config(tmp_path):
+    path = tmp_path / "out.json"
+    dump_builtin_config(path)
+    loaded = deserialize_config(path)
+    assert [m.model_dump(mode="json") for m in loaded] == [
+        m.model_dump(mode="json") for m in built_in_models
+    ]
+
+
+@pytest.mark.asyncio
+async def test_load_remote_models_success(monkeypatch):
+    del os.environ["KILN_SKIP_REMOTE_MODEL_LIST"]
+    original = built_in_models.copy()
+    sample_models = [built_in_models[0]]
+
+    def fake_fetch(url):
+        return sample_models
+
+    monkeypatch.setattr("kiln_ai.adapters.remote_config.load_from_url", fake_fetch)
+
+    load_remote_models("http://example.com/models.json")
+    await asyncio.sleep(0.01)
+    assert built_in_models == sample_models
+    built_in_models[:] = original
+
+
+@pytest.mark.asyncio
+async def test_load_remote_models_failure(monkeypatch):
+    original = built_in_models.copy()
+
+    def fake_fetch(url):
+        raise RuntimeError("fail")
+
+    monkeypatch.setattr("kiln_ai.adapters.remote_config.load_from_url", fake_fetch)
+
+    load_remote_models("http://example.com/models.json")
+    await asyncio.sleep(0.01)
+    assert built_in_models == original
+
+
+def test_deserialize_config_with_extra_keys(tmp_path):
+    # Take a valid model and add an extra key, ensure it is ignored and still loads
+    import json
+
+    from kiln_ai.adapters.ml_model_list import built_in_models
+
+    model_dict = built_in_models[0].model_dump(mode="json")
+    model_dict["extra_key"] = "should be ignored or error"
+    model_dict["providers"][0]["extra_key"] = "should be ignored or error"
+    data = {"model_list": [model_dict]}
+    path = tmp_path / "extra.json"
+    path.write_text(json.dumps(data))
+    # Should NOT raise, and extra key should be ignored
+    models = deserialize_config(path)
+    assert hasattr(models[0], "family")
+    assert not hasattr(models[0], "extra_key")
+    assert hasattr(models[0], "providers")
+    assert not hasattr(models[0].providers[0], "extra_key")

kiln_ai/datamodel/eval.py

@@ -14,6 +14,7 @@ from kiln_ai.datamodel.basemodel import (
 from kiln_ai.datamodel.datamodel_enums import TaskOutputRatingType
 from kiln_ai.datamodel.dataset_filters import DatasetFilterId
 from kiln_ai.datamodel.json_schema import string_to_json_key
+from kiln_ai.datamodel.task_run import Usage
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
 if TYPE_CHECKING:
@@ -28,6 +29,7 @@ class EvalTemplateId(str, Enum):
     """
 
     kiln_requirements = "kiln_requirements"
+    issue = "kiln_issue"
     toxicity = "toxicity"
     bias = "bias"
     maliciousness = "maliciousness"
@@ -110,6 +112,10 @@ class EvalRun(KilnParentedModel):
     scores: EvalScores = Field(
         description="The output scores of the evaluator (aligning to those required by the grand-parent Eval this object is a child of)."
     )
+    task_run_usage: Usage | None = Field(
+        default=None,
+        description="The usage of the task run that produced this eval run output (not the usage by the evaluation model).",
+    )
 
     def parent_eval_config(self) -> Union["EvalConfig", None]:
         if self.parent is not None and self.parent.__class__.__name__ != "EvalConfig":
@@ -280,6 +286,10 @@ class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}
         default=False,
         description="Whether this eval is a favourite of the user. Rendered as a star icon in the UI.",
     )
+    template_properties: dict[str, str | int | bool | float] = Field(
+        default={},
+        description="Properties to be used to execute the eval. This is template_type specific and should serialize to a json dict.",
+    )
 
     # Workaround to return typed parent without importing Task
     def parent_task(self) -> Union["Task", None]:
@@ -304,3 +314,25 @@ class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}
                 f"output_scores must have unique names (once transformed to JSON keys). Got: [{', '.join(output_score_keys)}]"
             )
         return self
+
+    @model_validator(mode="after")
+    def validate_template_properties(self) -> Self:
+        # Check for properties that are required for the issue template
+        if self.template == EvalTemplateId.issue:
+            if "issue_prompt" not in self.template_properties or not isinstance(
+                self.template_properties["issue_prompt"], str
+            ):
+                raise ValueError("issue_prompt is required for issue template")
+            if "failure_example" in self.template_properties and not isinstance(
+                self.template_properties["failure_example"], str
+            ):
+                raise ValueError(
+                    "failure_example is optional for issue template, but if provided must be a string"
+                )
+            if "pass_example" in self.template_properties and not isinstance(
+                self.template_properties["pass_example"], str
+            ):
+                raise ValueError(
+                    "pass_example is optional for issue template, but if provided must be a string"
+                )
+        return self

kiln_ai/datamodel/finetune.py

@@ -5,7 +5,6 @@ from typing_extensions import Self
 
 from kiln_ai.datamodel.basemodel import NAME_FIELD, KilnParentedModel
 from kiln_ai.datamodel.datamodel_enums import (
-    THINKING_DATA_STRATEGIES,
     ChatStrategy,
     FineTuneStatusType,
     StructuredOutputMode,

kiln_ai/datamodel/task_output.py

@@ -2,8 +2,6 @@ import json
 from enum import Enum
 from typing import TYPE_CHECKING, Dict, List, Type, Union
 
-import jsonschema
-import jsonschema.exceptions
 from pydantic import BaseModel, Field, ValidationInfo, model_validator
 from typing_extensions import Self
 

kiln_ai/datamodel/task_run.py

@@ -1,8 +1,6 @@
 import json
 from typing import TYPE_CHECKING, Dict, List, Union
 
-import jsonschema
-import jsonschema.exceptions
 from pydantic import BaseModel, Field, ValidationInfo, model_validator
 from typing_extensions import Self
 

kiln_ai/datamodel/test_eval_model.py

@@ -1,7 +1,6 @@
 import pytest
 from pydantic import ValidationError
 
-from kiln_ai.datamodel import BasePrompt
 from kiln_ai.datamodel.basemodel import KilnParentModel
 from kiln_ai.datamodel.eval import (
     Eval,
@@ -9,11 +8,10 @@ from kiln_ai.datamodel.eval import (
     EvalConfigType,
     EvalOutputScore,
     EvalRun,
+    EvalTemplateId,
 )
 from kiln_ai.datamodel.task import Task
-from kiln_ai.datamodel.task_output import (
-    TaskOutputRatingType,
-)
+from kiln_ai.datamodel.task_output import TaskOutputRatingType
 
 
 @pytest.fixture
@@ -633,3 +631,147 @@ def test_eval_run_eval_config_eval_validation():
             output="test output",
             scores={"score": 1.0},
         )
+
+
+@pytest.mark.parametrize(
+    "template_properties,should_raise,expected_error",
+    [
+        # Valid cases
+        (
+            {"issue_prompt": "Test issue prompt"},
+            False,
+            None,
+        ),
+        (
+            {
+                "issue_prompt": "Test issue prompt",
+                "failure_example": "Test failure example",
+            },
+            False,
+            None,
+        ),
+        (
+            {
+                "issue_prompt": "Test issue prompt",
+                "failure_example": "Test failure example",
+                "pass_example": "Test pass example",
+            },
+            False,
+            None,
+        ),
+        (
+            {
+                "issue_prompt": "",
+                "failure_example": "",
+                "pass_example": "",
+            },
+            False,
+            None,
+        ),
+        # Invalid cases
+        (
+            {},
+            True,
+            "issue_prompt is required for issue template",
+        ),
+        (
+            {"failure_example": "Test failure example"},
+            True,
+            "issue_prompt is required for issue template",
+        ),
+        (
+            {"issue_prompt": 123},
+            True,
+            "issue_prompt is required for issue template",
+        ),
+        (
+            {
+                "issue_prompt": "Test issue prompt",
+                "failure_example": 456,
+            },
+            True,
+            "failure_example is optional for issue template, but if provided must be a string",
+        ),
+        (
+            {
+                "issue_prompt": "Test issue prompt",
+                "failure_example": "Test failure example",
+                "pass_example": 789,
+            },
+            True,
+            "pass_example is optional for issue template, but if provided must be a string",
+        ),
+    ],
+)
+def test_eval_template_properties_issue_template_validation(
+    template_properties, should_raise, expected_error
+):
+    """Test issue template validation with various property combinations"""
+    if should_raise:
+        with pytest.raises(ValueError, match=expected_error):
+            Eval(
+                name="Test Eval",
+                template=EvalTemplateId.issue,
+                eval_set_filter_id="tag::tag1",
+                eval_configs_filter_id="tag::tag2",
+                output_scores=[
+                    EvalOutputScore(
+                        name="score",
+                        type=TaskOutputRatingType.pass_fail,
+                    )
+                ],
+                template_properties=template_properties,
+            )
+    else:
+        eval = Eval(
+            name="Test Eval",
+            template=EvalTemplateId.issue,
+            eval_set_filter_id="tag::tag1",
+            eval_configs_filter_id="tag::tag2",
+            output_scores=[
+                EvalOutputScore(
+                    name="score",
+                    type=TaskOutputRatingType.pass_fail,
+                )
+            ],
+            template_properties=template_properties,
+        )
+        assert eval.template == EvalTemplateId.issue
+        for key, value in template_properties.items():
+            assert eval.template_properties[key] == value
+
+
+@pytest.mark.parametrize(
+    "template,template_properties",
+    [
+        (EvalTemplateId.kiln_requirements, {"random_property": "random_value"}),
+        (EvalTemplateId.toxicity, {}),
+        (EvalTemplateId.bias, {"some_property": 123}),
+        (EvalTemplateId.maliciousness, {"test": True}),
+        (EvalTemplateId.factual_correctness, {"score": 4.5}),
+        (EvalTemplateId.jailbreak, {"prompt": "test"}),
+        (
+            None,
+            {"issue_prompt": "This should not be validated", "failure_example": 123},
+        ),
+    ],
+)
+def test_eval_template_properties_non_issue_templates(template, template_properties):
+    """Test that non-issue templates pass validation regardless of template_properties"""
+    eval = Eval(
+        name="Test Eval",
+        template=template,
+        eval_set_filter_id="tag::tag1",
+        eval_configs_filter_id="tag::tag2",
+        output_scores=[
+            EvalOutputScore(
+                name="score",
+                type=TaskOutputRatingType.pass_fail,
+            )
+        ],
+        template_properties=template_properties,
+    )
+
+    assert eval.template == template
+    for key, value in template_properties.items():
+        assert eval.template_properties[key] == value

kiln_ai/utils/logging.py

@@ -63,14 +63,14 @@ class CustomLiteLLMLogger(CustomLogger):
         # Print the formatted input data for the request in API format, pretty print
         try:
             self.logger.info(
-                f"Formatted Input Data (API):\n{json.dumps(data, indent=2)}"
+                f"Formatted Input Data (API):\n{json.dumps(data, indent=2, ensure_ascii=False)}"
             )
         except Exception as e:
             self.logger.info(f"Formatted Input Data (API): Could not print {e}")
 
         # Print the messages for the request in LiteLLM Message list, pretty print
         try:
-            json_messages = json.dumps(messages, indent=2)
+            json_messages = json.dumps(messages, indent=2, ensure_ascii=False)
             self.logger.info(f"Messages:\n{json_messages}")
         except Exception as e:
             self.logger.info(f"Messages: Could not print {e}")
@@ -115,7 +115,7 @@ class CustomLiteLLMLogger(CustomLogger):
                         # JSON format logs if possible
                         json_content = json.loads(content)
                         self.logger.info(
-                            f"Model Response Content:\n{json.dumps(json_content, indent=2)}"
+                            f"Model Response Content:\n{json.dumps(json_content, indent=2, ensure_ascii=False)}"
                         )
                     except Exception:
                         self.logger.info(f"Model Response Content:\n{content}")
@@ -149,6 +149,7 @@ def setup_litellm_logging(filename: str = "model_calls.log"):
         get_log_file_path(filename),
         maxBytes=5 * 1024 * 1024,  # 5MB
         backupCount=3,
+        encoding="utf-8",
     )
 
     # Set formatter to match the default formatting

{kiln_ai-0.17.0.dist-info → kiln_ai-0.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiln-ai
-Version: 0.17.0
+Version: 0.18.0
 Summary: Kiln AI
 Project-URL: Homepage, https://getkiln.ai
 Project-URL: Repository, https://github.com/Kiln-AI/kiln
@@ -19,7 +19,7 @@ Requires-Dist: boto3>=1.37.10
 Requires-Dist: coverage>=7.6.4
 Requires-Dist: google-cloud-aiplatform>=1.84.0
 Requires-Dist: jsonschema>=4.23.0
-Requires-Dist: litellm>=1.67.0
+Requires-Dist: litellm>=1.72.6
 Requires-Dist: openai>=1.53.0
 Requires-Dist: pdoc>=15.0.0
 Requires-Dist: pydantic>=2.9.2