kiln-ai 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

kiln_ai/adapters/parsers/test_request_formatters.py

@@ -0,0 +1,76 @@
+import pytest
+
+from kiln_ai.adapters.ml_model_list import ModelFormatterID
+from kiln_ai.adapters.parsers.request_formatters import (
+    Qwen3StyleNoThinkFormatter,
+    request_formatter_from_id,
+)
+
+
+@pytest.fixture
+def qwen_formatter():
+    return Qwen3StyleNoThinkFormatter()
+
+
+def test_qwen_formatter_string_input(qwen_formatter):
+    input_text = "Hello world"
+    formatted = qwen_formatter.format_input(input_text)
+    assert formatted == "Hello world\n\n/no_think"
+
+
+def test_qwen_formatter_dict_input(qwen_formatter):
+    input_dict = {"key": "value", "nested": {"inner": "data"}}
+    formatted = qwen_formatter.format_input(input_dict)
+    expected = """{
+  "key": "value",
+  "nested": {
+    "inner": "data"
+  }
+}
+
+/no_think"""
+    assert formatted == expected
+
+
+def test_qwen_formatter_empty_input(qwen_formatter):
+    # Test empty string
+    assert qwen_formatter.format_input("") == "\n\n/no_think"
+
+    # Test empty dict
+    assert qwen_formatter.format_input({}) == "{}\n\n/no_think"
+
+
+def test_qwen_formatter_special_characters(qwen_formatter):
+    input_text = "Special chars: !@#$%^&*()_+思"
+    formatted = qwen_formatter.format_input(input_text)
+    assert formatted == "Special chars: !@#$%^&*()_+思\n\n/no_think"
+
+
+def test_qwen_formatter_multiline_string(qwen_formatter):
+    input_text = """Line 1
+    Line 2
+    Line 3"""
+    formatted = qwen_formatter.format_input(input_text)
+    assert (
+        formatted
+        == """Line 1
+    Line 2
+    Line 3
+
+/no_think"""
+    )
+
+
+def test_request_formatter_factory():
+    # Test valid formatter ID
+    formatter = request_formatter_from_id(ModelFormatterID.qwen3_style_no_think)
+    assert isinstance(formatter, Qwen3StyleNoThinkFormatter)
+
+    # Test that the formatter works
+    assert formatter.format_input("test") == "test\n\n/no_think"
+
+
+def test_request_formatter_factory_invalid_id():
+    # Test with an invalid enum value by using a string that doesn't exist in the enum
+    with pytest.raises(ValueError, match="Unhandled enum value"):
+        request_formatter_from_id("invalid_formatter_id")  # type: ignore

kiln_ai/adapters/prompt_builders.py

@@ -101,7 +101,6 @@ class SimplePromptBuilder(BasePromptBuilder):
         """
         base_prompt = self.task.instruction
 
-        # TODO: this is just a quick version. Formatting and best practices TBD
         if len(self.task.requirements) > 0:
             base_prompt += (
                 "\n\nYour response should respect the following requirements:\n"
@@ -113,6 +112,18 @@ class SimplePromptBuilder(BasePromptBuilder):
         return base_prompt
 
 
+class ShortPromptBuilder(BasePromptBuilder):
+    """A prompt builder that includes a the base prompt but excludes the requirements."""
+
+    def build_base_prompt(self) -> str:
+        """Build a short prompt with just the base prompt, no requirements.
+
+        Returns:
+            str: The constructed prompt string.
+        """
+        return self.task.instruction
+
+
 class MultiShotPromptBuilder(BasePromptBuilder):
     """A prompt builder that includes multiple examples in the prompt."""
 
@@ -414,6 +425,8 @@ def prompt_builder_from_id(prompt_id: PromptId, task: Task) -> BasePromptBuilder
     match typed_prompt_generator:
         case PromptGenerators.SIMPLE:
             return SimplePromptBuilder(task)
+        case PromptGenerators.SHORT:
+            return ShortPromptBuilder(task)
         case PromptGenerators.FEW_SHOT:
             return FewShotPromptBuilder(task)
         case PromptGenerators.MULTI_SHOT:

kiln_ai/adapters/provider_tools.py

@@ -5,6 +5,7 @@ from kiln_ai.adapters.ml_model_list import (
     KilnModel,
     KilnModelProvider,
     ModelName,
+    ModelParserID,
     ModelProviderName,
     StructuredOutputMode,
     built_in_models,
@@ -15,7 +16,7 @@ from kiln_ai.adapters.model_adapters.litellm_config import (
 from kiln_ai.adapters.ollama_tools import (
     get_ollama_connection,
 )
-from kiln_ai.datamodel import Finetune, Task
+from kiln_ai.datamodel import Finetune, FinetuneDataStrategy, Task
 from kiln_ai.datamodel.registry import project_from_id
 from kiln_ai.utils.config import Config
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
@@ -257,6 +258,14 @@ def finetune_from_id(model_id: str) -> Finetune:
     return fine_tune
 
 
+def parser_from_data_strategy(
+    data_strategy: FinetuneDataStrategy,
+) -> ModelParserID | None:
+    if data_strategy == FinetuneDataStrategy.final_and_intermediate_r1_compatible:
+        return ModelParserID.r1_thinking
+    return None
+
+
 def finetune_provider_model(
     model_id: str,
 ) -> KilnModelProvider:
@@ -266,8 +275,23 @@ def finetune_provider_model(
     model_provider = KilnModelProvider(
         name=provider,
         model_id=fine_tune.fine_tune_model_id,
+        parser=parser_from_data_strategy(fine_tune.data_strategy),
+        reasoning_capable=(
+            fine_tune.data_strategy
+            in [
+                FinetuneDataStrategy.final_and_intermediate,
+                FinetuneDataStrategy.final_and_intermediate_r1_compatible,
+            ]
+        ),
     )
 
+    if provider == ModelProviderName.vertex and fine_tune.fine_tune_model_id:
+        # Vertex AI trick: use the model_id "openai/endpoint_id". OpenAI calls the openai compatible API, which supports endpoint.
+        # Context: vertex has at least 3 APIS: vertex, openai compatible, and gemini. LiteLLM tries to infer which to use. This works
+        # on current LiteLLM version. Could also set base_model to gemini to tell it which to use, but same result.
+        endpoint_id = fine_tune.fine_tune_model_id.split("/")[-1]
+        model_provider.model_id = f"openai/{endpoint_id}"
+
     if fine_tune.structured_output_mode is not None:
         # If we know the model was trained with specific output mode, set it
         model_provider.structured_output_mode = fine_tune.structured_output_mode

kiln_ai/adapters/repair/test_repair_task.py

@@ -218,8 +218,9 @@ async def test_mocked_repair_task_run(sample_task, sample_task_run, sample_repai
     }
 
     with patch.object(LiteLlmAdapter, "_run", new_callable=AsyncMock) as mock_run:
-        mock_run.return_value = RunOutput(
-            output=mocked_output, intermediate_outputs=None
+        mock_run.return_value = (
+            RunOutput(output=mocked_output, intermediate_outputs=None),
+            None,
         )
 
         adapter = adapter_for_task(

kiln_ai/adapters/test_prompt_builders.py

@@ -3,7 +3,7 @@ import logging
 
 import pytest
 
-from kiln_ai.adapters.model_adapters.base_adapter import BaseAdapter
+from kiln_ai.adapters.model_adapters.base_adapter import BaseAdapter, RunOutput
 from kiln_ai.adapters.model_adapters.test_structured_output import (
     build_structured_output_test_task,
 )
@@ -15,6 +15,7 @@ from kiln_ai.adapters.prompt_builders import (
     MultiShotPromptBuilder,
     RepairsPromptBuilder,
     SavedPromptBuilder,
+    ShortPromptBuilder,
     SimpleChainOfThoughtPromptBuilder,
     SimplePromptBuilder,
     TaskRunConfigPromptBuilder,
@@ -33,6 +34,7 @@ from kiln_ai.datamodel import (
     TaskOutput,
     TaskOutputRating,
     TaskRun,
+    Usage,
 )
 from kiln_ai.datamodel.task import RunConfigProperties, TaskRunConfig
 
@@ -58,9 +60,28 @@ def test_simple_prompt_builder(tmp_path):
     assert input not in prompt
 
 
+def test_short_prompt_builder(tmp_path):
+    task = build_test_task(tmp_path)
+    builder = ShortPromptBuilder(task=task)
+    prompt = builder.build_prompt(include_json_instructions=False)
+
+    # Should only include the instruction, not requirements
+    assert task.instruction == prompt
+    assert task.requirements[0].instruction not in prompt
+    assert task.requirements[1].instruction not in prompt
+    assert task.requirements[2].instruction not in prompt
+
+    # Should handle JSON instructions correctly
+    prompt_with_json = builder.build_prompt(include_json_instructions=True)
+    assert task.instruction in prompt_with_json
+    if task.output_schema():
+        assert "# Format Instructions" in prompt_with_json
+        assert task.output_schema() in prompt_with_json
+
+
 class MockAdapter(BaseAdapter):
-    def _run(self, input: str) -> str:
-        return "mock response"
+    async def _run(self, input: str) -> tuple[RunOutput, Usage | None]:
+        return RunOutput(output="mock response", intermediate_outputs=None), None
 
     def adapter_name(self) -> str:
         return "mock_adapter"

kiln_ai/adapters/test_provider_tools.py

@@ -5,6 +5,7 @@ import pytest
 from kiln_ai.adapters.ml_model_list import (
     KilnModel,
     ModelName,
+    ModelParserID,
     ModelProviderName,
 )
 from kiln_ai.adapters.ollama_tools import OllamaConnection
@@ -24,7 +25,12 @@ from kiln_ai.adapters.provider_tools import (
     provider_name_from_id,
     provider_warnings,
 )
-from kiln_ai.datamodel import Finetune, StructuredOutputMode, Task
+from kiln_ai.datamodel import (
+    Finetune,
+    FinetuneDataStrategy,
+    StructuredOutputMode,
+    Task,
+)
 
 
 @pytest.fixture(autouse=True)
@@ -65,6 +71,33 @@ def mock_finetune():
         finetune.provider = ModelProviderName.openai
         finetune.fine_tune_model_id = "ft:gpt-3.5-turbo:custom:model-123"
         finetune.structured_output_mode = StructuredOutputMode.json_schema
+        finetune.data_strategy = FinetuneDataStrategy.final_only
+        mock.return_value = finetune
+        yield mock
+
+
+@pytest.fixture
+def mock_finetune_final_and_intermediate():
+    with patch("kiln_ai.datamodel.Finetune.from_id_and_parent_path") as mock:
+        finetune = Mock(spec=Finetune)
+        finetune.provider = ModelProviderName.openai
+        finetune.fine_tune_model_id = "ft:gpt-3.5-turbo:custom:model-123"
+        finetune.structured_output_mode = StructuredOutputMode.json_schema
+        finetune.data_strategy = FinetuneDataStrategy.final_and_intermediate
+        mock.return_value = finetune
+        yield mock
+
+
+@pytest.fixture
+def mock_finetune_r1_compatible():
+    with patch("kiln_ai.datamodel.Finetune.from_id_and_parent_path") as mock:
+        finetune = Mock(spec=Finetune)
+        finetune.provider = ModelProviderName.ollama
+        finetune.fine_tune_model_id = "ft:deepseek-r1:671b:custom:model-123"
+        finetune.structured_output_mode = StructuredOutputMode.json_schema
+        finetune.data_strategy = (
+            FinetuneDataStrategy.final_and_intermediate_r1_compatible
+        )
         mock.return_value = finetune
         yield mock
 
@@ -426,6 +459,38 @@ def test_finetune_provider_model_success(mock_project, mock_task, mock_finetune)
     assert provider.name == ModelProviderName.openai
     assert provider.model_id == "ft:gpt-3.5-turbo:custom:model-123"
     assert provider.structured_output_mode == StructuredOutputMode.json_schema
+    assert provider.reasoning_capable is False
+    assert provider.parser == None
+
+
+def test_finetune_provider_model_success_final_and_intermediate(
+    mock_project, mock_task, mock_finetune_final_and_intermediate
+):
+    """Test successful creation of a fine-tuned model provider"""
+    model_id = "project-123::task-456::finetune-789"
+
+    provider = finetune_provider_model(model_id)
+
+    assert provider.name == ModelProviderName.openai
+    assert provider.model_id == "ft:gpt-3.5-turbo:custom:model-123"
+    assert provider.structured_output_mode == StructuredOutputMode.json_schema
+    assert provider.reasoning_capable is True
+    assert provider.parser == None
+
+
+def test_finetune_provider_model_success_r1_compatible(
+    mock_project, mock_task, mock_finetune_r1_compatible
+):
+    """Test successful creation of a fine-tuned model provider"""
+    model_id = "project-123::task-456::finetune-789"
+
+    provider = finetune_provider_model(model_id)
+
+    assert provider.name == ModelProviderName.ollama
+    assert provider.model_id == "ft:deepseek-r1:671b:custom:model-123"
+    assert provider.structured_output_mode == StructuredOutputMode.json_schema
+    assert provider.reasoning_capable is True
+    assert provider.parser == ModelParserID.r1_thinking
 
 
 def test_finetune_provider_model_invalid_id():
@@ -515,6 +580,7 @@ def test_finetune_provider_model_structured_mode(
     finetune.provider = provider_name
     finetune.fine_tune_model_id = "fireworks-model-123"
     finetune.structured_output_mode = structured_output_mode
+    finetune.data_strategy = FinetuneDataStrategy.final_only
     mock_finetune.return_value = finetune
 
     provider = finetune_provider_model("project-123::task-456::finetune-789")
@@ -522,6 +588,8 @@ def test_finetune_provider_model_structured_mode(
     assert provider.name == provider_name
     assert provider.model_id == "fireworks-model-123"
     assert provider.structured_output_mode == expected_mode
+    assert provider.reasoning_capable is False
+    assert provider.parser == None
 
 
 def test_openai_compatible_provider_config(mock_shared_config):
@@ -791,3 +859,20 @@ def test_finetune_from_id_cache_hit(mock_project, mock_task, mock_finetune):
     mock_project.assert_not_called()
     mock_task.assert_not_called()
     mock_finetune.assert_not_called()
+
+
+def test_finetune_provider_model_vertex_ai(mock_project, mock_task, mock_finetune):
+    """Test creation of provider for Vertex AI with endpoint ID transformation"""
+    finetune = Mock(spec=Finetune)
+    finetune.provider = ModelProviderName.vertex
+    finetune.fine_tune_model_id = "projects/123/locations/us-central1/endpoints/456"
+    finetune.structured_output_mode = StructuredOutputMode.json_mode
+    finetune.data_strategy = FinetuneDataStrategy.final_only
+    mock_finetune.return_value = finetune
+
+    provider = finetune_provider_model("project-123::task-456::finetune-789")
+
+    assert provider.name == ModelProviderName.vertex
+    # Verify the model_id is transformed into openai/endpoint_id format
+    assert provider.model_id == "openai/456"
+    assert provider.structured_output_mode == StructuredOutputMode.json_mode

kiln_ai/datamodel/__init__.py

@@ -44,6 +44,7 @@ from kiln_ai.datamodel.task_output import (
 )
 from kiln_ai.datamodel.task_run import (
     TaskRun,
+    Usage,
 )
 
 __all__ = [
@@ -74,4 +75,5 @@ __all__ = [
     "PromptId",
     "PromptGenerators",
     "prompt_generator_values",
+    "Usage",
 ]

kiln_ai/datamodel/datamodel_enums.py

@@ -56,5 +56,19 @@ class FineTuneStatusType(str, Enum):
 
 
 class FinetuneDataStrategy(str, Enum):
+    """Strategy for what data to include when fine-tuning a model."""
+
+    # Only train on the final response, ignoring any intermediate steps or chain of thought
     final_only = "final_only"
+
+    # Train on both the final response and any intermediate steps/chain of thought
     final_and_intermediate = "final_and_intermediate"
+
+    # Train using R1-style thinking format, which includes the reasoning in <think> tags in the message
+    final_and_intermediate_r1_compatible = "final_and_intermediate_r1_compatible"
+
+
+THINKING_DATA_STRATEGIES: list[FinetuneDataStrategy] = [
+    FinetuneDataStrategy.final_and_intermediate,
+    FinetuneDataStrategy.final_and_intermediate_r1_compatible,
+]

kiln_ai/datamodel/dataset_filters.py

@@ -1,5 +1,6 @@
+import re
 from enum import Enum
-from typing import Annotated, Protocol
+from typing import Annotated, ClassVar, List, Protocol
 
 from pydantic import AfterValidator
 
@@ -59,6 +60,65 @@ class TagFilter:
         return self.tag in task_run.tags
 
 
+class MultiDatasetFilter:
+    """
+    A filter that combines multiple filters using AND logic.
+    The filters are specified in a query string format after 'multi_filter::'
+    Example: multi_filter::high_rating&thinking_model&tag::tag_name
+
+    Ampersands in filter IDs can be escaped with a backslash.
+    """
+
+    PREFIX: ClassVar[str] = "multi_filter::"
+    ESCAPED_AMPERSAND: ClassVar[str] = r"\&"
+    UNESCAPED_AMPERSAND: ClassVar[str] = "&"
+
+    @classmethod
+    def parse_filter_string(cls, filter_string: str) -> List[str]:
+        """
+        Parse a filter string into individual filter IDs, handling escaped ampersands.
+        """
+        if not filter_string.startswith(cls.PREFIX):
+            raise ValueError(f"Filter string must start with {cls.PREFIX}")
+
+        # Remove the prefix
+        content = filter_string[len(cls.PREFIX) :]
+        if not content:
+            raise ValueError("No filters specified after prefix")
+
+        # Split on unescaped ampersands
+        # This regex matches & that are not preceded by a backslash
+        parts = re.split(r"(?<!\\)&", content)
+
+        # Unescape ampersands in each part
+        filter_ids = [
+            part.replace(cls.ESCAPED_AMPERSAND, cls.UNESCAPED_AMPERSAND)
+            for part in parts
+        ]
+
+        # Validate each filter ID using the existing validation
+        for fid in filter_ids:
+            _check_dataset_filter_id(fid)
+
+        return filter_ids
+
+    @classmethod
+    def is_valid_filter_string(cls, filter_string: str) -> bool:
+        """Check if a filter string is valid."""
+        try:
+            cls.parse_filter_string(filter_string)
+            return True
+        except ValueError:
+            return False
+
+    def __init__(self, filter_id: str):
+        filter_ids = MultiDatasetFilter.parse_filter_string(filter_id)
+        self.filters = [dataset_filter_from_id(fid) for fid in filter_ids]
+
+    def __call__(self, task_run: TaskRun) -> bool:
+        return all(f(task_run) for f in self.filters)
+
+
 class StaticDatasetFilters(str, Enum):
     """Dataset filter names."""
 
@@ -98,6 +158,11 @@ def _check_dataset_filter_id(id: str) -> str:
     if id.startswith("tag::") and len(id) > 5:
         return id
 
+    if id.startswith(MultiDatasetFilter.PREFIX):
+        if not MultiDatasetFilter.is_valid_filter_string(id):
+            raise ValueError(f"Invalid multi-filter string: {id}")
+        return id
+
     raise ValueError(f"Invalid dataset filter ID: {id}")
 
 
@@ -108,6 +173,9 @@ def dataset_filter_from_id(id: DatasetFilterId) -> DatasetFilter:
     if id.startswith("tag::") and len(id) > 5:
         return TagFilter(id[5:])
 
+    if id.startswith(MultiDatasetFilter.PREFIX):
+        return MultiDatasetFilter(id)
+
     if id in static_dataset_filters:
         return static_dataset_filters[id]
 

kiln_ai/datamodel/dataset_split.py

@@ -45,6 +45,10 @@ Train80Test20SplitDefinition: list[DatasetSplitDefinition] = [
     DatasetSplitDefinition(name="train", percentage=0.8),
     DatasetSplitDefinition(name="test", percentage=0.2),
 ]
+Train80Val20SplitDefinition: list[DatasetSplitDefinition] = [
+    DatasetSplitDefinition(name="train", percentage=0.8),
+    DatasetSplitDefinition(name="val", percentage=0.2),
+]
 Train60Test20Val20SplitDefinition: list[DatasetSplitDefinition] = [
     DatasetSplitDefinition(name="train", percentage=0.6),
     DatasetSplitDefinition(name="test", percentage=0.2),

kiln_ai/datamodel/eval.py

@@ -263,6 +263,10 @@ class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}
         default=None,
         description="The id of the current config to use for this eval. This can be changed over time to run the same eval with different configs.",
     )
+    current_run_config_id: ID_TYPE = Field(
+        default=None,
+        description="The id of the a run config which was selected as the best run config for this eval. The run config must belong to the parent Task.",
+    )
     eval_set_filter_id: DatasetFilterId = Field(
         description="The id of the dataset filter which defines which dataset items are included when running this eval. Should be mutually exclusive with eval_configs_filter_id."
     )
@@ -272,6 +276,10 @@ class Eval(KilnParentedModel, KilnParentModel, parent_of={"configs": EvalConfig}
     output_scores: List[EvalOutputScore] = Field(
         description="The scores this evaluator should produce."
     )
+    favourite: bool = Field(
+        default=False,
+        description="Whether this eval is a favourite of the user. Rendered as a star icon in the UI.",
+    )
 
     # Workaround to return typed parent without importing Task
     def parent_task(self) -> Union["Task", None]:

kiln_ai/datamodel/finetune.py

@@ -5,6 +5,7 @@ from typing_extensions import Self
 
 from kiln_ai.datamodel.basemodel import NAME_FIELD, KilnParentedModel
 from kiln_ai.datamodel.datamodel_enums import (
+    THINKING_DATA_STRATEGIES,
     FinetuneDataStrategy,
     FineTuneStatusType,
     StructuredOutputMode,

kiln_ai/datamodel/json_schema.py

@@ -41,16 +41,33 @@ def validate_schema(instance: Dict, schema_str: str) -> None:
 
     Raises:
         jsonschema.exceptions.ValidationError: If validation fails
-        ValueError: If the schema is invalid
+    """
+    schema = schema_from_json_str(schema_str)
+    v = jsonschema.Draft202012Validator(schema)
+    v.validate(instance)
+
+
+def validate_schema_with_value_error(
+    instance: Dict, schema_str: str, error_prefix: str | None = None
+) -> None:
+    """Validate a dictionary against a JSON schema and raise a ValueError if the schema is invalid.
+
+    Args:
+        instance: Dictionary to validate
+        schema_str: JSON schema string to validate against
+        error_prefix: Error message prefix to include in the ValueError
+
+    Raises:
+        ValueError: If the instance does not match the schema
     """
     try:
-        schema = schema_from_json_str(schema_str)
-        v = jsonschema.Draft202012Validator(schema)
-        v.validate(instance)
+        validate_schema(instance, schema_str)
     except jsonschema.exceptions.ValidationError as e:
-        raise ValueError(
-            f"This task requires a specific output schema. While the model produced JSON, that JSON didn't meet the schema. Search 'Troubleshooting Structured Data Issues' in our docs for more information. The error from the schema check was: {e.message}. The JSON was: \n```json\n{instance}\n```"
-        ) from e
+        msg = f"The error from the schema check was: {e.message}. The JSON was: \n```json\n{instance}\n```"
+        if error_prefix:
+            msg = f"{error_prefix} {msg}"
+
+        raise ValueError(msg) from e
 
 
 def schema_from_json_str(v: str) -> Dict:

kiln_ai/datamodel/prompt_id.py

@@ -13,6 +13,7 @@ class PromptGenerators(str, Enum):
     SIMPLE_CHAIN_OF_THOUGHT = "simple_chain_of_thought_prompt_builder"
     FEW_SHOT_CHAIN_OF_THOUGHT = "few_shot_chain_of_thought_prompt_builder"
     MULTI_SHOT_CHAIN_OF_THOUGHT = "multi_shot_chain_of_thought_prompt_builder"
+    SHORT = "short_prompt_builder"
 
 
 prompt_generator_values = [pg.value for pg in PromptGenerators]

kiln_ai/datamodel/task_output.py

@@ -9,7 +9,7 @@ from typing_extensions import Self
 
 from kiln_ai.datamodel.basemodel import ID_TYPE, KilnBaseModel
 from kiln_ai.datamodel.datamodel_enums import TaskOutputRatingType
-from kiln_ai.datamodel.json_schema import validate_schema
+from kiln_ai.datamodel.json_schema import validate_schema_with_value_error
 from kiln_ai.datamodel.strict_mode import strict_mode
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
@@ -64,7 +64,7 @@ class TaskOutputRating(KilnBaseModel):
     )
     requirement_ratings: Dict[ID_TYPE, RequirementRating] = Field(
         default={},
-        description="The ratings of the requirements of the task.",
+        description="The ratings of the requirements of the task. The ID can be either a task_requirement_id or a named rating for an eval_output_score name (in format 'named::<name>').",
     )
 
     # Previously we stored rating values as a dict of floats, but now we store them as RequirementRating objects.
@@ -308,11 +308,15 @@ class TaskOutput(KilnBaseModel):
         # validate output
         if task.output_json_schema is not None:
             try:
-                validate_schema(json.loads(self.output), task.output_json_schema)
-            except json.JSONDecodeError:
+                output_parsed = json.loads(self.output)
+            except json.JSONDecodeError as e:
                 raise ValueError("Output is not a valid JSON object")
-            except jsonschema.exceptions.ValidationError as e:
-                raise ValueError(f"Output does not match task output schema: {e}")
+
+            validate_schema_with_value_error(
+                output_parsed,
+                task.output_json_schema,
+                "This task requires a specific output schema. While the model produced JSON, that JSON didn't meet the schema. Search 'Troubleshooting Structured Data Issues' in our docs for more information.",
+            )
         return self
 
     @model_validator(mode="after")