kiln-ai 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

kiln_ai/datamodel/dataset_split.py

@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
 
 from pydantic import BaseModel, Field, model_validator
 
-from kiln_ai.datamodel.basemodel import NAME_FIELD, KilnParentedModel
+from kiln_ai.datamodel.basemodel import FilenameString, KilnParentedModel
 from kiln_ai.datamodel.dataset_filters import (
     DatasetFilter,
     DatasetFilterId,
@@ -26,7 +26,9 @@ class DatasetSplitDefinition(BaseModel):
     Example: name="train", description="The training set", percentage=0.8 (80% of the dataset)
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(
+        description="The name of the dataset split definition."
+    )
     description: str | None = Field(
         default=None,
         description="A description of the dataset for you and your team. Not used in training.",
@@ -70,7 +72,7 @@ class DatasetSplit(KilnParentedModel):
     Maintains a list of IDs for each split, to avoid data duplication.
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the dataset split.")
     description: str | None = Field(
         default=None,
         description="A description of the dataset for you and your team. Not used in training.",

kiln_ai/datamodel/eval.py

@@ -7,13 +7,14 @@ from typing_extensions import Self
 
 from kiln_ai.datamodel.basemodel import (
     ID_TYPE,
-    NAME_FIELD,
+    FilenameString,
     KilnParentedModel,
     KilnParentModel,
 )
 from kiln_ai.datamodel.datamodel_enums import TaskOutputRatingType
 from kiln_ai.datamodel.dataset_filters import DatasetFilterId
 from kiln_ai.datamodel.json_schema import string_to_json_key
+from kiln_ai.datamodel.task_run import Usage
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
 if TYPE_CHECKING:
@@ -28,6 +29,7 @@ class EvalTemplateId(str, Enum):
     """
 
     kiln_requirements = "kiln_requirements"
+    issue = "kiln_issue"
     toxicity = "toxicity"
     bias = "bias"
     maliciousness = "maliciousness"
@@ -110,6 +112,10 @@ class EvalRun(KilnParentedModel):
     scores: EvalScores = Field(
         description="The output scores of the evaluator (aligning to those required by the grand-parent Eval this object is a child of)."
     )
+    task_run_usage: Usage | None = Field(
+        default=None,
+        description="The usage of the task run that produced this eval run output (not the usage by the evaluation model).",
+    )
 
     def parent_eval_config(self) -> Union["EvalConfig", None]:
         if self.parent is not None and self.parent.__class__.__name__ != "EvalConfig":
@@ -196,7 +202,7 @@ class EvalConfig(KilnParentedModel, KilnParentModel, parent_of={"runs": EvalRun}
     A eval might have many configs, example running the same eval with 2 different models. Comparing eval results is only valid within the scope of the same config.
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the eval config.")
     model_name: str = Field(
         description="The name of the model to use for this eval config. ",
     )
@@ -251,7 +257,7 @@ class EvalConfig(KilnParentedModel, KilnParentModel, parent_of={"runs": EvalRun}
 
 
 class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}):
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the eval.")
     description: str | None = Field(
         default=None, description="The description of the eval"
     )
@@ -280,6 +286,10 @@ class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}
         default=False,
         description="Whether this eval is a favourite of the user. Rendered as a star icon in the UI.",
     )
+    template_properties: dict[str, str | int | bool | float] = Field(
+        default={},
+        description="Properties to be used to execute the eval. This is template_type specific and should serialize to a json dict.",
+    )
 
     # Workaround to return typed parent without importing Task
     def parent_task(self) -> Union["Task", None]:
@@ -304,3 +314,25 @@ class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}
                 f"output_scores must have unique names (once transformed to JSON keys). Got: [{', '.join(output_score_keys)}]"
             )
         return self
+
+    @model_validator(mode="after")
+    def validate_template_properties(self) -> Self:
+        # Check for properties that are required for the issue template
+        if self.template == EvalTemplateId.issue:
+            if "issue_prompt" not in self.template_properties or not isinstance(
+                self.template_properties["issue_prompt"], str
+            ):
+                raise ValueError("issue_prompt is required for issue template")
+            if "failure_example" in self.template_properties and not isinstance(
+                self.template_properties["failure_example"], str
+            ):
+                raise ValueError(
+                    "failure_example is optional for issue template, but if provided must be a string"
+                )
+            if "pass_example" in self.template_properties and not isinstance(
+                self.template_properties["pass_example"], str
+            ):
+                raise ValueError(
+                    "pass_example is optional for issue template, but if provided must be a string"
+                )
+        return self

kiln_ai/datamodel/finetune.py

@@ -3,9 +3,8 @@ from typing import TYPE_CHECKING, Dict, Union
 from pydantic import Field, model_validator
 from typing_extensions import Self
 
-from kiln_ai.datamodel.basemodel import NAME_FIELD, KilnParentedModel
+from kiln_ai.datamodel.basemodel import FilenameString, KilnParentedModel
 from kiln_ai.datamodel.datamodel_enums import (
-    THINKING_DATA_STRATEGIES,
     ChatStrategy,
     FineTuneStatusType,
     StructuredOutputMode,
@@ -27,7 +26,7 @@ class Finetune(KilnParentedModel):
     Initially holds a reference to a training job, with needed identifiers to update the status. When complete, contains the new model ID.
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the fine-tune.")
     description: str | None = Field(
         default=None,
         description="A description of the fine-tune for you and your team. Not used in training.",

kiln_ai/datamodel/project.py

@@ -1,6 +1,6 @@
 from pydantic import Field
 
-from kiln_ai.datamodel.basemodel import NAME_FIELD, KilnParentModel
+from kiln_ai.datamodel.basemodel import FilenameString, KilnParentModel
 from kiln_ai.datamodel.task import Task
 
 
@@ -12,12 +12,12 @@ class Project(KilnParentModel, parent_of={"tasks": Task}):
     of the overall goals.
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the project.")
     description: str | None = Field(
         default=None,
         description="A description of the project for you and your team. Will not be used in prompts/training/validation.",
     )
 
-    # Needed for typechecking. TODO P2: fix this in KilnParentModel
+    # Needed for typechecking. We should fix this in KilnParentModel
     def tasks(self) -> list[Task]:
         return super().tasks()  # type: ignore

kiln_ai/datamodel/prompt.py

@@ -1,6 +1,6 @@
 from pydantic import BaseModel, Field
 
-from kiln_ai.datamodel.basemodel import NAME_FIELD, KilnParentedModel
+from kiln_ai.datamodel.basemodel import FilenameString, KilnParentedModel
 
 
 class BasePrompt(BaseModel):
@@ -10,7 +10,7 @@ class BasePrompt(BaseModel):
     The "Prompt" model name is reserved for the custom prompts parented by a task.
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the prompt.")
     description: str | None = Field(
         default=None,
         description="A more detailed description of the prompt.",

kiln_ai/datamodel/prompt_id.py

@@ -60,11 +60,11 @@ def _check_prompt_id(id: str) -> str:
         return id
 
     if id.startswith("fine_tune_prompt::"):
-        # check it had a fine_tune_id after the :: -- 'fine_tune_prompt::fine_tune_id'
-        fine_tune_id = id[18:]
-        if len(fine_tune_id) == 0:
+        # check it had a fine_tune_id after the :: -- 'fine_tune_prompt::[project_id]::[task_id]::fine_tune_id'
+        parts = id.split("::")
+        if len(parts) != 4 or len(parts[3]) == 0:
             raise ValueError(
-                f"Invalid fine-tune prompt ID: {id}. Expected format: 'fine_tune_prompt::[fine_tune_id]'."
+                f"Invalid fine-tune prompt ID: {id}. Expected format: 'fine_tune_prompt::[project_id]::[task_id]::[fine_tune_id]'."
             )
         return id
 

kiln_ai/datamodel/task.py

@@ -7,8 +7,8 @@ from kiln_ai.datamodel import Finetune
 from kiln_ai.datamodel.basemodel import (
     ID_FIELD,
     ID_TYPE,
-    NAME_FIELD,
-    SHORT_NAME_FIELD,
+    FilenameString,
+    FilenameStringShort,
     KilnParentedModel,
     KilnParentModel,
 )
@@ -38,7 +38,7 @@ class TaskRequirement(BaseModel):
     """
 
     id: ID_TYPE = ID_FIELD
-    name: str = SHORT_NAME_FIELD
+    name: FilenameStringShort = Field(description="The name of the task requirement.")
     description: str | None = Field(default=None)
     instruction: str = Field(min_length=1)
     priority: Priority = Field(default=Priority.p2)
@@ -103,7 +103,7 @@ class TaskRunConfig(KilnParentedModel):
     A run config includes everything needed to run a task, except the input. Running the same RunConfig with the same input should make identical calls to the model (output may vary as models are non-deterministic).
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the task run config.")
     description: str | None = Field(
         default=None, description="The description of the task run config."
     )
@@ -189,7 +189,7 @@ class Task(
     a collection of task runs.
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the task.")
     description: str | None = Field(
         default=None,
         description="A description of the task for you and your team. Will not be used in prompts/training/validation.",
@@ -216,7 +216,7 @@ class Task(
             return None
         return schema_from_json_str(self.input_json_schema)
 
-    # These wrappers help for typechecking. TODO P2: fix this in KilnParentModel
+    # These wrappers help for typechecking. We should fix this in KilnParentModel
     def runs(self, readonly: bool = False) -> list[TaskRun]:
         return super().runs(readonly=readonly)  # type: ignore
 

kiln_ai/datamodel/task_output.py

@@ -2,8 +2,6 @@ import json
 from enum import Enum
 from typing import TYPE_CHECKING, Dict, List, Type, Union
 
-import jsonschema
-import jsonschema.exceptions
 from pydantic import BaseModel, Field, ValidationInfo, model_validator
 from typing_extensions import Self
 
@@ -309,7 +307,7 @@ class TaskOutput(KilnBaseModel):
         if task.output_json_schema is not None:
             try:
                 output_parsed = json.loads(self.output)
-            except json.JSONDecodeError as e:
+            except json.JSONDecodeError:
                 raise ValueError("Output is not a valid JSON object")
 
             validate_schema_with_value_error(

kiln_ai/datamodel/task_run.py

@@ -1,8 +1,6 @@
 import json
 from typing import TYPE_CHECKING, Dict, List, Union
 
-import jsonschema
-import jsonschema.exceptions
 from pydantic import BaseModel, Field, ValidationInfo, model_validator
 from typing_extensions import Self
 

kiln_ai/datamodel/test_basemodel.py

@@ -1,5 +1,6 @@
 import datetime
 import json
+import uuid
 from pathlib import Path
 from typing import Optional
 from unittest.mock import MagicMock, patch
@@ -12,6 +13,7 @@ from kiln_ai.datamodel import Task, TaskRun
 from kiln_ai.datamodel.basemodel import (
     KilnBaseModel,
     KilnParentedModel,
+    name_validator,
     string_to_valid_name,
 )
 from kiln_ai.datamodel.model_cache import ModelCache
@@ -328,28 +330,81 @@ def test_delete_no_path():
         model.delete()
 
 
-def test_string_to_valid_name():
-    # Test basic valid strings remain unchanged
-    assert string_to_valid_name("Hello World") == "Hello World"
-    assert string_to_valid_name("Test-123") == "Test-123"
-    assert string_to_valid_name("my_file_name") == "my_file_name"
+@pytest.mark.parametrize(
+    "name,expected",
+    [
+        # Basic valid strings remain unchanged
+        ("Hello World", "Hello World"),
+        ("Test-123", "Test-123"),
+        ("my_file_name", "my_file_name"),
+        ("multiple!!!symbols", "multiple!!!symbols"),
+        # Emoji
+        ("Hello 👍", "Hello 👍"),
+        # Invalid characters are replaced
+        ("Hello@World!", "Hello@World!"),
+        ("File.name.txt", "File_name_txt"),
+        ("Special%%%Chars", "Special_Chars"),
+        ("Special#$%Chars", "Special#$_Chars"),
+        # Consecutive invalid characters are replaced
+        ("Special%%%Chars", "Special_Chars"),
+        ("path/to/file", "path_to_file"),
+        # Leading/trailing special characters are removed
+        ("__test__", "test"),
+        ("...test...", "test"),
+        # Whitespace is replaced
+        ("", ""),
+        ("   ", ""),
+        ("Hello   World", "Hello World"),
+        # Unicode characters are replaced
+        ("你好", "你好"),
+        ("你好_世界", "你好_世界"),
+        ("你好_世界_你好", "你好_世界_你好"),
+        # Newlines, tabs, and other control characters are replaced
+        ("Hello\nworld", "Hello_world"),
+        ("Hello\tworld", "Hello_world"),
+        ("Hello\rworld", "Hello_world"),
+        ("Hello\fworld", "Hello_world"),
+        ("Hello\bworld", "Hello_world"),
+        ("Hello\vworld", "Hello_world"),
+        ("Hello\0world", "Hello_world"),
+        ("Hello\x00world", "Hello_world"),
+    ],
+)
+def test_string_to_valid_name(tmp_path, name, expected):
+    assert string_to_valid_name(name) == expected
 
-    # Test invalid characters are replaced
-    assert string_to_valid_name("Hello@World!") == "Hello_World"
-    assert string_to_valid_name("File.name.txt") == "File_name_txt"
-    assert string_to_valid_name("Special#$%Chars") == "Special_Chars"
+    # check we can create a folder with the valid name
+    dir_path = tmp_path / str(uuid.uuid4()) / expected
+    dir_path.mkdir(parents=True)
 
-    # Test consecutive invalid characters
-    assert string_to_valid_name("multiple!!!symbols") == "multiple_symbols"
-    assert string_to_valid_name("path/to/file") == "path_to_file"
 
-    # Test leading/trailing special characters
-    assert string_to_valid_name("__test__") == "test"
-    assert string_to_valid_name("...test...") == "test"
+@pytest.mark.parametrize(
+    "name,min_length,max_length,should_pass",
+    [
+        # Valid cases
+        ("ValidName", 5, 20, True),
+        ("Short", 1, 10, True),
+        ("LongerValidName", 5, 20, True),
+        # None case (line 53)
+        (None, 5, 20, False),
+        # Too short cases (lines 57-59)
+        ("Hi", 5, 20, False),
+        ("", 1, 20, False),
+        ("a", 2, 20, False),
+        # Too long cases (lines 61-63)
+        ("ThisNameIsTooLong", 5, 10, False),
+        ("VeryVeryVeryLongName", 1, 15, False),
+    ],
+)
+def test_name_validator_error_conditions(name, min_length, max_length, should_pass):
+    validator = name_validator(min_length=min_length, max_length=max_length)
 
-    # Test empty string and whitespace
-    assert string_to_valid_name("") == ""
-    assert string_to_valid_name("   ") == ""
+    if should_pass:
+        result = validator(name)
+        assert result == name
+    else:
+        with pytest.raises(ValueError):
+            validator(name)
 
 
 def test_load_from_file_with_cache(test_base_file, tmp_model_cache):
@@ -553,3 +608,140 @@ async def test_invoke_parsing_flow(adapter):
             match="Reasoning is required for this model, but no reasoning was returned.",
         ):
             await adapter.invoke("test input")
+
+
+async def test_invoke_parsing_flow_basic_no_reasoning(adapter):
+    """Test for reasoning_optional_for_structured_output
+    when reasoning is not required.
+    This is a special case where we want to return the output as is.
+    """
+    # Mock dependencies
+    mock_provider = MagicMock()
+    mock_provider.parser = "test_parser"
+    mock_provider.formatter = None
+    mock_provider.reasoning_capable = False
+    mock_provider.reasoning_optional_for_structured_output = True
+
+    mock_parser = MagicMock()
+    mock_parser.parse_output.return_value = RunOutput(
+        output="parsed test output", intermediate_outputs={"key": "value"}
+    )
+
+    with (
+        patch.object(adapter, "model_provider", return_value=mock_provider),
+        patch(
+            "kiln_ai.adapters.model_adapters.base_adapter.model_parser_from_id",
+            return_value=mock_parser,
+        ),
+        patch("kiln_ai.adapters.model_adapters.base_adapter.Config") as mock_config,
+    ):
+        # Disable autosaving for this test
+        mock_config.shared.return_value.autosave_runs = False
+        mock_config.shared.return_value.user_id = "test_user_id"
+
+        # Execute
+        result = await adapter.invoke("test input")
+
+        # Verify parsing occurred
+        mock_parser.parse_output.assert_called_once()
+        parsed_args = mock_parser.parse_output.call_args[1]
+        assert isinstance(parsed_args["original_output"], RunOutput)
+        assert parsed_args["original_output"].output == "test output"
+
+        # Verify result contains parsed output
+        assert isinstance(result, TaskRun)
+        assert result.output.output == "parsed test output"
+        assert result.intermediate_outputs == {"key": "value"}
+        assert result.input == "test input"
+
+
+async def test_invoke_parsing_flow_no_reasoning_with_structured_output(adapter):
+    """Test for reasoning_optional_for_structured_output
+    when reasoning is required but not provided, with structured output enabled.
+    This is a special case where we don't want to error, but we want to return the output as is.
+    """
+    # Mock dependencies
+    mock_provider = MagicMock()
+    mock_provider.parser = "test_parser"
+    mock_provider.formatter = None
+    mock_provider.reasoning_capable = True
+    mock_provider.reasoning_optional_for_structured_output = True
+
+    mock_parser = MagicMock()
+    mock_parser.parse_output.return_value = RunOutput(
+        output="parsed test output", intermediate_outputs={"key": "value"}
+    )
+
+    with (
+        patch.object(adapter, "model_provider", return_value=mock_provider),
+        patch(
+            "kiln_ai.adapters.model_adapters.base_adapter.model_parser_from_id",
+            return_value=mock_parser,
+        ),
+        patch("kiln_ai.adapters.model_adapters.base_adapter.Config") as mock_config,
+        patch.object(adapter, "has_structured_output", return_value=True),
+    ):
+        # Disable autosaving for this test
+        mock_config.shared.return_value.autosave_runs = False
+        mock_config.shared.return_value.user_id = "test_user_id"
+
+        # Execute
+        result = await adapter.invoke("test input")
+
+        # Verify parsing occurred
+        mock_parser.parse_output.assert_called_once()
+        parsed_args = mock_parser.parse_output.call_args[1]
+        assert isinstance(parsed_args["original_output"], RunOutput)
+        assert parsed_args["original_output"].output == "test output"
+
+        # Verify result contains parsed output
+        assert isinstance(result, TaskRun)
+        assert result.output.output == "parsed test output"
+        assert result.intermediate_outputs == {"key": "value"}
+        assert result.input == "test input"
+
+
+async def test_invoke_parsing_flow_with_reasoning_and_structured_output(adapter):
+    """Test for reasoning_optional_for_structured_output
+    when reasoning is provided with structured output enabled.
+    This is a special case where we want to return the output as is.
+    """
+    # Mock dependencies
+    mock_provider = MagicMock()
+    mock_provider.parser = "test_parser"
+    mock_provider.formatter = None
+    mock_provider.reasoning_capable = True
+    mock_provider.reasoning_optional_for_structured_output = True
+
+    mock_parser = MagicMock()
+    mock_parser.parse_output.return_value = RunOutput(
+        output="parsed test output", intermediate_outputs={"reasoning": "value"}
+    )
+
+    with (
+        patch.object(adapter, "model_provider", return_value=mock_provider),
+        patch(
+            "kiln_ai.adapters.model_adapters.base_adapter.model_parser_from_id",
+            return_value=mock_parser,
+        ),
+        patch("kiln_ai.adapters.model_adapters.base_adapter.Config") as mock_config,
+        patch.object(adapter, "has_structured_output", return_value=True),
+    ):
+        # Disable autosaving for this test
+        mock_config.shared.return_value.autosave_runs = False
+        mock_config.shared.return_value.user_id = "test_user_id"
+
+        # Execute
+        result = await adapter.invoke("test input")
+
+        # Verify parsing occurred
+        mock_parser.parse_output.assert_called_once()
+        parsed_args = mock_parser.parse_output.call_args[1]
+        assert isinstance(parsed_args["original_output"], RunOutput)
+        assert parsed_args["original_output"].output == "test output"
+
+        # Verify result contains parsed output
+        assert isinstance(result, TaskRun)
+        assert result.output.output == "parsed test output"
+        assert result.intermediate_outputs == {"reasoning": "value"}
+        assert result.input == "test input"