kiln-ai 0.14.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

kiln_ai/adapters/eval/base_eval.py

@@ -2,11 +2,13 @@ import json
 from abc import abstractmethod
 from typing import Dict
 
+import jsonschema
+
 from kiln_ai.adapters.adapter_registry import adapter_for_task
 from kiln_ai.adapters.ml_model_list import ModelProviderName
 from kiln_ai.adapters.model_adapters.base_adapter import AdapterConfig
 from kiln_ai.datamodel.eval import Eval, EvalConfig, EvalScores
-from kiln_ai.datamodel.json_schema import validate_schema
+from kiln_ai.datamodel.json_schema import validate_schema_with_value_error
 from kiln_ai.datamodel.task import RunConfig, TaskOutputRatingType, TaskRun
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
@@ -72,7 +74,10 @@ class BaseEval:
         run_output = await run_adapter.invoke(parsed_input)
 
         eval_output, intermediate_outputs = await self.run_eval(run_output)
-        validate_schema(eval_output, self.score_schema)
+
+        validate_schema_with_value_error(
+            eval_output, self.score_schema, "Eval output does not match score schema."
+        )
 
         return run_output, eval_output, intermediate_outputs
 

kiln_ai/adapters/fine_tune/base_finetune.py

@@ -166,9 +166,12 @@ class BaseFinetuneAdapter(ABC):
 
                 # Strict type checking for numeric types
                 if expected_type is float and not isinstance(value, float):
-                    raise ValueError(
-                        f"Parameter {parameter.name} must be a float, got {type(value)}"
-                    )
+                    if isinstance(value, int):
+                        value = float(value)
+                    else:
+                        raise ValueError(
+                            f"Parameter {parameter.name} must be a float, got {type(value)}"
+                        )
                 elif expected_type is int and not isinstance(value, int):
                     raise ValueError(
                         f"Parameter {parameter.name} must be an integer, got {type(value)}"

kiln_ai/adapters/fine_tune/dataset_formatter.py

@@ -30,8 +30,8 @@ class DatasetFormat(str, Enum):
         "huggingface_chat_template_toolcall_jsonl"
     )
 
-    """Vertex Gemini 1.5 format (flash and pro)"""
-    VERTEX_GEMINI_1_5 = "vertex_gemini_1_5"
+    """Vertex Gemini format"""
+    VERTEX_GEMINI = "vertex_gemini"
 
 
 @dataclass
@@ -288,7 +288,7 @@ def generate_huggingface_chat_template_toolcall(
     return {"conversations": conversations}
 
 
-def generate_vertex_gemini_1_5(
+def generate_vertex_gemini(
     training_data: ModelTrainingData,
 ) -> Dict[str, Any]:
     """Generate Vertex Gemini 1.5 format (flash and pro)"""
@@ -346,7 +346,7 @@ FORMAT_GENERATORS: Dict[DatasetFormat, FormatGenerator] = {
     DatasetFormat.OPENAI_CHAT_TOOLCALL_JSONL: generate_chat_message_toolcall,
     DatasetFormat.HUGGINGFACE_CHAT_TEMPLATE_JSONL: generate_huggingface_chat_template,
     DatasetFormat.HUGGINGFACE_CHAT_TEMPLATE_TOOLCALL_JSONL: generate_huggingface_chat_template_toolcall,
-    DatasetFormat.VERTEX_GEMINI_1_5: generate_vertex_gemini_1_5,
+    DatasetFormat.VERTEX_GEMINI: generate_vertex_gemini,
 }
 
 

kiln_ai/adapters/fine_tune/finetune_registry.py

@@ -4,10 +4,12 @@ from kiln_ai.adapters.fine_tune.base_finetune import BaseFinetuneAdapter
 from kiln_ai.adapters.fine_tune.fireworks_finetune import FireworksFinetune
 from kiln_ai.adapters.fine_tune.openai_finetune import OpenAIFinetune
 from kiln_ai.adapters.fine_tune.together_finetune import TogetherFinetune
+from kiln_ai.adapters.fine_tune.vertex_finetune import VertexFinetune
 from kiln_ai.adapters.ml_model_list import ModelProviderName
 
 finetune_registry: dict[ModelProviderName, Type[BaseFinetuneAdapter]] = {
     ModelProviderName.openai: OpenAIFinetune,
     ModelProviderName.fireworks_ai: FireworksFinetune,
     ModelProviderName.together_ai: TogetherFinetune,
+    ModelProviderName.vertex: VertexFinetune,
 }

kiln_ai/adapters/fine_tune/fireworks_finetune.py

@@ -198,7 +198,8 @@ class FireworksFinetune(BaseFinetuneAdapter):
         if not api_key or not account_id:
             raise ValueError("Fireworks API key or account ID not set")
         url = f"https://api.fireworks.ai/v1/accounts/{account_id}/datasets"
-        dataset_id = str(uuid4())
+        # First char can't be a digit: https://discord.com/channels/1137072072808472616/1363214412395184350/1363214412395184350
+        dataset_id = "kiln-" + str(uuid4())
         payload = {
             "datasetId": dataset_id,
             "dataset": {

kiln_ai/adapters/fine_tune/test_base_finetune.py

@@ -98,6 +98,13 @@ def test_validate_parameters_valid():
     }
     MockFinetune.validate_parameters(valid_params)  # Should not raise
 
+    # Test valid parameters (float as int)
+    valid_params = {
+        "learning_rate": 1,
+        "epochs": 10,
+    }
+    MockFinetune.validate_parameters(valid_params)  # Should not raise
+
 
 def test_validate_parameters_missing_required():
     # Test missing required parameter

kiln_ai/adapters/fine_tune/test_dataset_formatter.py

@@ -15,7 +15,7 @@ from kiln_ai.adapters.fine_tune.dataset_formatter import (
     generate_chat_message_toolcall,
     generate_huggingface_chat_template,
     generate_huggingface_chat_template_toolcall,
-    generate_vertex_gemini_1_5,
+    generate_vertex_gemini,
 )
 from kiln_ai.adapters.model_adapters.base_adapter import COT_FINAL_ANSWER_PROMPT
 from kiln_ai.datamodel import (
@@ -447,7 +447,7 @@ def test_generate_vertex_template():
         final_output="test output",
     )
 
-    result = generate_vertex_gemini_1_5(training_data)
+    result = generate_vertex_gemini(training_data)
 
     assert result == {
         "systemInstruction": {
@@ -475,7 +475,7 @@ def test_generate_vertex_template_thinking():
         thinking_final_answer_prompt="thinking final answer prompt",
     )
 
-    result = generate_vertex_gemini_1_5(training_data)
+    result = generate_vertex_gemini(training_data)
 
     logger.info(result)
 

kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py

@@ -315,7 +315,7 @@ async def test_generate_and_upload_jsonl_success(
             "thinking_instructions": thinking_instructions,
         }
 
-        assert result == mock_dataset_id
+        assert result == "kiln-" + mock_dataset_id
         assert mock_client.post.call_count == 2
         assert mock_client.get.call_count == 1