kiln-ai 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

kiln_ai/datamodel/prompt.py

@@ -1,6 +1,6 @@
 from pydantic import BaseModel, Field
 
-from kiln_ai.datamodel.basemodel import NAME_FIELD, KilnParentedModel
+from kiln_ai.datamodel.basemodel import FilenameString, KilnParentedModel
 
 
 class BasePrompt(BaseModel):
@@ -10,7 +10,7 @@ class BasePrompt(BaseModel):
     The "Prompt" model name is reserved for the custom prompts parented by a task.
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the prompt.")
     description: str | None = Field(
         default=None,
         description="A more detailed description of the prompt.",

kiln_ai/datamodel/prompt_id.py

@@ -60,11 +60,11 @@ def _check_prompt_id(id: str) -> str:
         return id
 
     if id.startswith("fine_tune_prompt::"):
-        # check it had a fine_tune_id after the :: -- 'fine_tune_prompt::fine_tune_id'
-        fine_tune_id = id[18:]
-        if len(fine_tune_id) == 0:
+        # check it had a fine_tune_id after the :: -- 'fine_tune_prompt::[project_id]::[task_id]::fine_tune_id'
+        parts = id.split("::")
+        if len(parts) != 4 or len(parts[3]) == 0:
             raise ValueError(
-                f"Invalid fine-tune prompt ID: {id}. Expected format: 'fine_tune_prompt::[fine_tune_id]'."
+                f"Invalid fine-tune prompt ID: {id}. Expected format: 'fine_tune_prompt::[project_id]::[task_id]::[fine_tune_id]'."
             )
         return id
 

kiln_ai/datamodel/task.py

@@ -7,8 +7,8 @@ from kiln_ai.datamodel import Finetune
 from kiln_ai.datamodel.basemodel import (
     ID_FIELD,
     ID_TYPE,
-    NAME_FIELD,
-    SHORT_NAME_FIELD,
+    FilenameString,
+    FilenameStringShort,
     KilnParentedModel,
     KilnParentModel,
 )
@@ -38,7 +38,7 @@ class TaskRequirement(BaseModel):
     """
 
     id: ID_TYPE = ID_FIELD
-    name: str = SHORT_NAME_FIELD
+    name: FilenameStringShort = Field(description="The name of the task requirement.")
     description: str | None = Field(default=None)
     instruction: str = Field(min_length=1)
     priority: Priority = Field(default=Priority.p2)
@@ -103,7 +103,7 @@ class TaskRunConfig(KilnParentedModel):
     A run config includes everything needed to run a task, except the input. Running the same RunConfig with the same input should make identical calls to the model (output may vary as models are non-deterministic).
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the task run config.")
     description: str | None = Field(
         default=None, description="The description of the task run config."
     )
@@ -189,7 +189,7 @@ class Task(
     a collection of task runs.
     """
 
-    name: str = NAME_FIELD
+    name: FilenameString = Field(description="The name of the task.")
     description: str | None = Field(
         default=None,
         description="A description of the task for you and your team. Will not be used in prompts/training/validation.",
@@ -216,7 +216,7 @@ class Task(
             return None
         return schema_from_json_str(self.input_json_schema)
 
-    # These wrappers help for typechecking. TODO P2: fix this in KilnParentModel
+    # These wrappers help for typechecking. We should fix this in KilnParentModel
     def runs(self, readonly: bool = False) -> list[TaskRun]:
         return super().runs(readonly=readonly)  # type: ignore
 

kiln_ai/datamodel/task_output.py

@@ -307,7 +307,7 @@ class TaskOutput(KilnBaseModel):
         if task.output_json_schema is not None:
             try:
                 output_parsed = json.loads(self.output)
-            except json.JSONDecodeError as e:
+            except json.JSONDecodeError:
                 raise ValueError("Output is not a valid JSON object")
 
             validate_schema_with_value_error(

kiln_ai/datamodel/test_basemodel.py

@@ -1,5 +1,6 @@
 import datetime
 import json
+import uuid
 from pathlib import Path
 from typing import Optional
 from unittest.mock import MagicMock, patch
@@ -12,6 +13,7 @@ from kiln_ai.datamodel import Task, TaskRun
 from kiln_ai.datamodel.basemodel import (
     KilnBaseModel,
     KilnParentedModel,
+    name_validator,
     string_to_valid_name,
 )
 from kiln_ai.datamodel.model_cache import ModelCache
@@ -328,28 +330,81 @@ def test_delete_no_path():
         model.delete()
 
 
-def test_string_to_valid_name():
-    # Test basic valid strings remain unchanged
-    assert string_to_valid_name("Hello World") == "Hello World"
-    assert string_to_valid_name("Test-123") == "Test-123"
-    assert string_to_valid_name("my_file_name") == "my_file_name"
+@pytest.mark.parametrize(
+    "name,expected",
+    [
+        # Basic valid strings remain unchanged
+        ("Hello World", "Hello World"),
+        ("Test-123", "Test-123"),
+        ("my_file_name", "my_file_name"),
+        ("multiple!!!symbols", "multiple!!!symbols"),
+        # Emoji
+        ("Hello 👍", "Hello 👍"),
+        # Invalid characters are replaced
+        ("Hello@World!", "Hello@World!"),
+        ("File.name.txt", "File_name_txt"),
+        ("Special%%%Chars", "Special_Chars"),
+        ("Special#$%Chars", "Special#$_Chars"),
+        # Consecutive invalid characters are replaced
+        ("Special%%%Chars", "Special_Chars"),
+        ("path/to/file", "path_to_file"),
+        # Leading/trailing special characters are removed
+        ("__test__", "test"),
+        ("...test...", "test"),
+        # Whitespace is replaced
+        ("", ""),
+        ("   ", ""),
+        ("Hello   World", "Hello World"),
+        # Unicode characters are replaced
+        ("你好", "你好"),
+        ("你好_世界", "你好_世界"),
+        ("你好_世界_你好", "你好_世界_你好"),
+        # Newlines, tabs, and other control characters are replaced
+        ("Hello\nworld", "Hello_world"),
+        ("Hello\tworld", "Hello_world"),
+        ("Hello\rworld", "Hello_world"),
+        ("Hello\fworld", "Hello_world"),
+        ("Hello\bworld", "Hello_world"),
+        ("Hello\vworld", "Hello_world"),
+        ("Hello\0world", "Hello_world"),
+        ("Hello\x00world", "Hello_world"),
+    ],
+)
+def test_string_to_valid_name(tmp_path, name, expected):
+    assert string_to_valid_name(name) == expected
 
-    # Test invalid characters are replaced
-    assert string_to_valid_name("Hello@World!") == "Hello_World"
-    assert string_to_valid_name("File.name.txt") == "File_name_txt"
-    assert string_to_valid_name("Special#$%Chars") == "Special_Chars"
+    # check we can create a folder with the valid name
+    dir_path = tmp_path / str(uuid.uuid4()) / expected
+    dir_path.mkdir(parents=True)
 
-    # Test consecutive invalid characters
-    assert string_to_valid_name("multiple!!!symbols") == "multiple_symbols"
-    assert string_to_valid_name("path/to/file") == "path_to_file"
 
-    # Test leading/trailing special characters
-    assert string_to_valid_name("__test__") == "test"
-    assert string_to_valid_name("...test...") == "test"
+@pytest.mark.parametrize(
+    "name,min_length,max_length,should_pass",
+    [
+        # Valid cases
+        ("ValidName", 5, 20, True),
+        ("Short", 1, 10, True),
+        ("LongerValidName", 5, 20, True),
+        # None case (line 53)
+        (None, 5, 20, False),
+        # Too short cases (lines 57-59)
+        ("Hi", 5, 20, False),
+        ("", 1, 20, False),
+        ("a", 2, 20, False),
+        # Too long cases (lines 61-63)
+        ("ThisNameIsTooLong", 5, 10, False),
+        ("VeryVeryVeryLongName", 1, 15, False),
+    ],
+)
+def test_name_validator_error_conditions(name, min_length, max_length, should_pass):
+    validator = name_validator(min_length=min_length, max_length=max_length)
 
-    # Test empty string and whitespace
-    assert string_to_valid_name("") == ""
-    assert string_to_valid_name("   ") == ""
+    if should_pass:
+        result = validator(name)
+        assert result == name
+    else:
+        with pytest.raises(ValueError):
+            validator(name)
 
 
 def test_load_from_file_with_cache(test_base_file, tmp_model_cache):
@@ -553,3 +608,140 @@ async def test_invoke_parsing_flow(adapter):
             match="Reasoning is required for this model, but no reasoning was returned.",
         ):
             await adapter.invoke("test input")
+
+
+async def test_invoke_parsing_flow_basic_no_reasoning(adapter):
+    """Test for reasoning_optional_for_structured_output
+    when reasoning is not required.
+    This is a special case where we want to return the output as is.
+    """
+    # Mock dependencies
+    mock_provider = MagicMock()
+    mock_provider.parser = "test_parser"
+    mock_provider.formatter = None
+    mock_provider.reasoning_capable = False
+    mock_provider.reasoning_optional_for_structured_output = True
+
+    mock_parser = MagicMock()
+    mock_parser.parse_output.return_value = RunOutput(
+        output="parsed test output", intermediate_outputs={"key": "value"}
+    )
+
+    with (
+        patch.object(adapter, "model_provider", return_value=mock_provider),
+        patch(
+            "kiln_ai.adapters.model_adapters.base_adapter.model_parser_from_id",
+            return_value=mock_parser,
+        ),
+        patch("kiln_ai.adapters.model_adapters.base_adapter.Config") as mock_config,
+    ):
+        # Disable autosaving for this test
+        mock_config.shared.return_value.autosave_runs = False
+        mock_config.shared.return_value.user_id = "test_user_id"
+
+        # Execute
+        result = await adapter.invoke("test input")
+
+        # Verify parsing occurred
+        mock_parser.parse_output.assert_called_once()
+        parsed_args = mock_parser.parse_output.call_args[1]
+        assert isinstance(parsed_args["original_output"], RunOutput)
+        assert parsed_args["original_output"].output == "test output"
+
+        # Verify result contains parsed output
+        assert isinstance(result, TaskRun)
+        assert result.output.output == "parsed test output"
+        assert result.intermediate_outputs == {"key": "value"}
+        assert result.input == "test input"
+
+
+async def test_invoke_parsing_flow_no_reasoning_with_structured_output(adapter):
+    """Test for reasoning_optional_for_structured_output
+    when reasoning is required but not provided, with structured output enabled.
+    This is a special case where we don't want to error, but we want to return the output as is.
+    """
+    # Mock dependencies
+    mock_provider = MagicMock()
+    mock_provider.parser = "test_parser"
+    mock_provider.formatter = None
+    mock_provider.reasoning_capable = True
+    mock_provider.reasoning_optional_for_structured_output = True
+
+    mock_parser = MagicMock()
+    mock_parser.parse_output.return_value = RunOutput(
+        output="parsed test output", intermediate_outputs={"key": "value"}
+    )
+
+    with (
+        patch.object(adapter, "model_provider", return_value=mock_provider),
+        patch(
+            "kiln_ai.adapters.model_adapters.base_adapter.model_parser_from_id",
+            return_value=mock_parser,
+        ),
+        patch("kiln_ai.adapters.model_adapters.base_adapter.Config") as mock_config,
+        patch.object(adapter, "has_structured_output", return_value=True),
+    ):
+        # Disable autosaving for this test
+        mock_config.shared.return_value.autosave_runs = False
+        mock_config.shared.return_value.user_id = "test_user_id"
+
+        # Execute
+        result = await adapter.invoke("test input")
+
+        # Verify parsing occurred
+        mock_parser.parse_output.assert_called_once()
+        parsed_args = mock_parser.parse_output.call_args[1]
+        assert isinstance(parsed_args["original_output"], RunOutput)
+        assert parsed_args["original_output"].output == "test output"
+
+        # Verify result contains parsed output
+        assert isinstance(result, TaskRun)
+        assert result.output.output == "parsed test output"
+        assert result.intermediate_outputs == {"key": "value"}
+        assert result.input == "test input"
+
+
+async def test_invoke_parsing_flow_with_reasoning_and_structured_output(adapter):
+    """Test for reasoning_optional_for_structured_output
+    when reasoning is provided with structured output enabled.
+    This is a special case where we want to return the output as is.
+    """
+    # Mock dependencies
+    mock_provider = MagicMock()
+    mock_provider.parser = "test_parser"
+    mock_provider.formatter = None
+    mock_provider.reasoning_capable = True
+    mock_provider.reasoning_optional_for_structured_output = True
+
+    mock_parser = MagicMock()
+    mock_parser.parse_output.return_value = RunOutput(
+        output="parsed test output", intermediate_outputs={"reasoning": "value"}
+    )
+
+    with (
+        patch.object(adapter, "model_provider", return_value=mock_provider),
+        patch(
+            "kiln_ai.adapters.model_adapters.base_adapter.model_parser_from_id",
+            return_value=mock_parser,
+        ),
+        patch("kiln_ai.adapters.model_adapters.base_adapter.Config") as mock_config,
+        patch.object(adapter, "has_structured_output", return_value=True),
+    ):
+        # Disable autosaving for this test
+        mock_config.shared.return_value.autosave_runs = False
+        mock_config.shared.return_value.user_id = "test_user_id"
+
+        # Execute
+        result = await adapter.invoke("test input")
+
+        # Verify parsing occurred
+        mock_parser.parse_output.assert_called_once()
+        parsed_args = mock_parser.parse_output.call_args[1]
+        assert isinstance(parsed_args["original_output"], RunOutput)
+        assert parsed_args["original_output"].output == "test output"
+
+        # Verify result contains parsed output
+        assert isinstance(result, TaskRun)
+        assert result.output.output == "parsed test output"
+        assert result.intermediate_outputs == {"reasoning": "value"}
+        assert result.input == "test input"

kiln_ai/datamodel/test_eval_model.py

@@ -517,13 +517,13 @@ def test_eval_run_score_keys_must_match(valid_eval_config, valid_eval_run_data):
     valid_eval_config.parent = eval
 
     # Correct
-    run = EvalRun(
+    EvalRun(
         parent=valid_eval_config,
         **{**valid_eval_run_data, "scores": {"accuracy": 4.5, "critical": 1.0}},
     )
 
     # Correct but wrong order still okay
-    run = EvalRun(
+    EvalRun(
         parent=valid_eval_config,
         **{**valid_eval_run_data, "scores": {"critical": 1.0, "accuracy": 4.5}},
     )
@@ -533,7 +533,7 @@ def test_eval_run_score_keys_must_match(valid_eval_config, valid_eval_run_data):
         ValueError,
         match="The scores produced by the evaluator must match the scores expected by the eval",
     ):
-        run = EvalRun(
+        EvalRun(
             parent=valid_eval_config,
             **{**valid_eval_run_data, "scores": {"accuracy": 4.5}},
         )
@@ -543,7 +543,7 @@ def test_eval_run_score_keys_must_match(valid_eval_config, valid_eval_run_data):
         ValueError,
         match="The scores produced by the evaluator must match the scores expected by the eval",
     ):
-        run = EvalRun(
+        EvalRun(
             parent=valid_eval_config,
             **{
                 **valid_eval_run_data,
@@ -556,7 +556,7 @@ def test_eval_run_score_keys_must_match(valid_eval_config, valid_eval_run_data):
         ValueError,
         match="The scores produced by the evaluator must match the scores expected by the eval",
     ):
-        run = EvalRun(
+        EvalRun(
             parent=valid_eval_config,
             **{**valid_eval_run_data, "scores": {"accuracy": 4.5, "wrong": 1.0}},
         )
@@ -566,7 +566,7 @@ def test_eval_run_custom_scores_not_allowed(valid_eval_config, valid_eval_run_da
     with pytest.raises(
         ValueError, match="Custom scores are not supported in evaluators"
     ):
-        eval = Eval(
+        Eval(
             name="Test Eval",
             eval_set_filter_id="tag::tag1",
             eval_configs_filter_id="tag::tag2",

kiln_ai/datamodel/test_model_perf.py

@@ -121,6 +121,6 @@ def test_benchmark_load_from_file(benchmark, task_run):
 
     # I get 8k ops per second on my MBP. Lower value here for CI and parallel testing.
     # Prior to optimization was 290 ops per second.
-    print(f"Ops per second: {ops_per_second:.6f}")
+    # sys.stdout.write(f"Ops per second: {ops_per_second:.6f}")
     if ops_per_second < 500:
         pytest.fail(f"Ops per second: {ops_per_second:.6f}, expected more than 1k ops")

kiln_ai/datamodel/test_prompt_id.py

@@ -29,7 +29,7 @@ def test_valid_saved_prompt_id():
 
 def test_valid_fine_tune_prompt_id():
     """Test that valid fine-tune prompt IDs are accepted"""
-    valid_id = "fine_tune_prompt::ft_123456"
+    valid_id = "fine_tune_prompt::project_123::task_456::ft_123456"
     model = ModelTester(prompt_id=valid_id)
     assert model.prompt_id == valid_id
 
@@ -53,6 +53,10 @@ def test_invalid_saved_prompt_id_format(invalid_id):
     [
         ("fine_tune_prompt::", "Invalid fine-tune prompt ID: fine_tune_prompt::"),
         ("fine_tune_prompt", "Invalid prompt ID: fine_tune_prompt"),
+        (
+            "fine_tune_prompt::ft_123456",
+            "Invalid fine-tune prompt ID: fine_tune_prompt::ft_123456",
+        ),
     ],
 )
 def test_invalid_fine_tune_prompt_id_format(invalid_id, expected_error):

kiln_ai/datamodel/test_task.py

@@ -323,3 +323,8 @@ def test_run_config_upgrade_old_entries():
     assert parsed.name == "test name"
     assert parsed.created_by == "scosman"
     assert parsed.run_config_properties.structured_output_mode == "unknown"
+
+
+def test_task_name_unicode_name():
+    task = Task(name="你好", instruction="Do something")
+    assert task.name == "你好"

kiln_ai/utils/config.py

@@ -124,6 +124,11 @@ class Config:
                 env_var="WANDB_API_KEY",
                 sensitive=True,
             ),
+            "siliconflow_cn_api_key": ConfigProperty(
+                str,
+                env_var="SILICONFLOW_CN_API_KEY",
+                sensitive=True,
+            ),
             "wandb_base_url": ConfigProperty(
                 str,
                 env_var="WANDB_BASE_URL",
@@ -137,6 +142,11 @@ class Config:
                 default_lambda=lambda: [],
                 sensitive_keys=["api_key"],
             ),
+            "cerebras_api_key": ConfigProperty(
+                str,
+                env_var="CEREBRAS_API_KEY",
+                sensitive=True,
+            ),
         }
         self._lock = threading.Lock()
         self._settings = self.load_settings()

{kiln_ai-0.18.0.dist-info → kiln_ai-0.19.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiln-ai
-Version: 0.18.0
+Version: 0.19.0
 Summary: Kiln AI
 Project-URL: Homepage, https://getkiln.ai
 Project-URL: Repository, https://github.com/Kiln-AI/kiln
@@ -65,6 +65,7 @@ The library has a [comprehensive set of docs](https://kiln-ai.github.io/Kiln/kil
 
 ## Table of Contents
 
+- [Connecting AI Providers](#connecting-ai-providers-openai-openrouter-ollama-etc)
 - [Using the Kiln Data Model](#using-the-kiln-data-model)
   - [Understanding the Kiln Data Model](#understanding-the-kiln-data-model)
   - [Datamodel Overview](#datamodel-overview)
@@ -73,6 +74,7 @@ The library has a [comprehensive set of docs](https://kiln-ai.github.io/Kiln/kil
   - [Using your Kiln Dataset in a Notebook or Project](#using-your-kiln-dataset-in-a-notebook-or-project)
   - [Using Kiln Dataset in Pandas](#using-kiln-dataset-in-pandas)
   - [Building and Running a Kiln Task from Code](#building-and-running-a-kiln-task-from-code)
+  - [Tagging Task Runs Programmatically](#tagging-task-runs-programmatically)
   - [Adding Custom Model or AI Provider from Code](#adding-custom-model-or-ai-provider-from-code)
 - [Full API Reference](#full-api-reference)
 
@@ -82,6 +84,12 @@ The library has a [comprehensive set of docs](https://kiln-ai.github.io/Kiln/kil
 pip install kiln-ai
 ```
 
+## Connecting AI Providers (OpenAI, OpenRouter, Ollama, etc)
+
+The easiest way to connect AI providers is to use the Kiln app UI. Once connected in the UI, credentials will be stored to `~/.kiln_ai/settings.yml`, which will be available to the library.
+
+For configuring credentials from code or connecting custom servers/model, see [Adding Custom Model or AI Provider from Code](#adding-custom-model-or-ai-provider-from-code).
+
 ## Using the Kiln Data Model
 
 ### Understanding the Kiln Data Model
@@ -179,7 +187,10 @@ item = kiln_ai.datamodel.TaskRun(
             type=kiln_ai.datamodel.DataSourceType.human,
             properties={"created_by": "Jane Doe"},
         ),
-        rating=kiln_ai.datamodel.TaskOutputRating(score=5,type="five_star"),
+        rating=kiln_ai.datamodel.TaskOutputRating(
+            value=5,
+            type=kiln_ai.datamodel.datamodel_enums.five_star,
+        ),
     ),
 )
 item.save_to_file()
@@ -270,6 +281,25 @@ for run in task.runs():
 
 ```
 
+## Tagging Task Runs Programmatically
+
+You can also tag your Kiln Task runs programmatically:
+
+```py
+# Load your Kiln Task from disk
+task_path = "/Users/youruser/Kiln Projects/test project/tasks/632780983478 - Joke Generator/task.kiln"
+task = kiln_ai.datamodel.Task.load_from_file(task_path)
+
+for run in task.runs():
+    # Parse the task output from JSON
+    output = json.loads(run.output.output)
+
+    # Add a tag if the punchline is unusually short
+    if len(output["punchline"]) < 100:
+        run.tags.append("very_short")
+        run.save_to_file()  # Persist the updated tags
+```
+
 ### Adding Custom Model or AI Provider from Code
 
 You can add additional AI models and providers to Kiln.