kiln-ai 0.14.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

kiln_ai/adapters/model_adapters/litellm_adapter.py

@@ -65,6 +65,7 @@ class LiteLlmAdapter(BaseAdapter):
         run_strategy, cot_prompt = self.run_strategy()
 
         if run_strategy == "cot_as_message":
+            # Used for reasoning-capable models that can output thinking and structured format
             if not cot_prompt:
                 raise ValueError("cot_prompt is required for cot_as_message strategy")
             messages.append({"role": "system", "content": cot_prompt})
@@ -73,9 +74,11 @@ class LiteLlmAdapter(BaseAdapter):
                 raise ValueError("cot_prompt is required for cot_two_call strategy")
             messages.append({"role": "system", "content": cot_prompt})
 
-            # First call for chain of thought - No logprobs as only needed for final answer
+            # First call for chain of thought
+            # No response format as this request is for "thinking" in plain text
+            # No logprobs as only needed for final answer
             completion_kwargs = await self.build_completion_kwargs(
-                provider, messages, None
+                provider, messages, None, skip_response_format=True
             )
             cot_response = await litellm.acompletion(**completion_kwargs)
             if (
@@ -367,6 +370,7 @@ class LiteLlmAdapter(BaseAdapter):
         provider: KilnModelProvider,
         messages: list[dict[str, Any]],
         top_logprobs: int | None,
+        skip_response_format: bool = False,
     ) -> dict[str, Any]:
         extra_body = self.build_extra_body(provider)
 
@@ -380,9 +384,10 @@ class LiteLlmAdapter(BaseAdapter):
             **self._additional_body_options,
         }
 
-        # Response format: json_schema, json_instructions, json_mode, function_calling, etc
-        response_format_options = await self.response_format_options()
-        completion_kwargs.update(response_format_options)
+        if not skip_response_format:
+            # Response format: json_schema, json_instructions, json_mode, function_calling, etc
+            response_format_options = await self.response_format_options()
+            completion_kwargs.update(response_format_options)
 
         if top_logprobs is not None:
             completion_kwargs["logprobs"] = True

kiln_ai/adapters/provider_tools.py

@@ -268,6 +268,13 @@ def finetune_provider_model(
         model_id=fine_tune.fine_tune_model_id,
     )
 
+    if provider == ModelProviderName.vertex and fine_tune.fine_tune_model_id:
+        # Vertex AI trick: use the model_id "openai/endpoint_id". OpenAI calls the openai compatible API, which supports endpoint.
+        # Context: vertex has at least 3 APIS: vertex, openai compatible, and gemini. LiteLLM tries to infer which to use. This works
+        # on current LiteLLM version. Could also set base_model to gemini to tell it which to use, but same result.
+        endpoint_id = fine_tune.fine_tune_model_id.split("/")[-1]
+        model_provider.model_id = f"openai/{endpoint_id}"
+
     if fine_tune.structured_output_mode is not None:
         # If we know the model was trained with specific output mode, set it
         model_provider.structured_output_mode = fine_tune.structured_output_mode

kiln_ai/adapters/test_provider_tools.py

@@ -791,3 +791,19 @@ def test_finetune_from_id_cache_hit(mock_project, mock_task, mock_finetune):
     mock_project.assert_not_called()
     mock_task.assert_not_called()
     mock_finetune.assert_not_called()
+
+
+def test_finetune_provider_model_vertex_ai(mock_project, mock_task, mock_finetune):
+    """Test creation of provider for Vertex AI with endpoint ID transformation"""
+    finetune = Mock(spec=Finetune)
+    finetune.provider = ModelProviderName.vertex
+    finetune.fine_tune_model_id = "projects/123/locations/us-central1/endpoints/456"
+    finetune.structured_output_mode = StructuredOutputMode.json_mode
+    mock_finetune.return_value = finetune
+
+    provider = finetune_provider_model("project-123::task-456::finetune-789")
+
+    assert provider.name == ModelProviderName.vertex
+    # Verify the model_id is transformed into openai/endpoint_id format
+    assert provider.model_id == "openai/456"
+    assert provider.structured_output_mode == StructuredOutputMode.json_mode

kiln_ai/datamodel/json_schema.py

@@ -41,16 +41,33 @@ def validate_schema(instance: Dict, schema_str: str) -> None:
 
     Raises:
         jsonschema.exceptions.ValidationError: If validation fails
-        ValueError: If the schema is invalid
+    """
+    schema = schema_from_json_str(schema_str)
+    v = jsonschema.Draft202012Validator(schema)
+    v.validate(instance)
+
+
+def validate_schema_with_value_error(
+    instance: Dict, schema_str: str, error_prefix: str | None = None
+) -> None:
+    """Validate a dictionary against a JSON schema and raise a ValueError if the schema is invalid.
+
+    Args:
+        instance: Dictionary to validate
+        schema_str: JSON schema string to validate against
+        error_prefix: Error message prefix to include in the ValueError
+
+    Raises:
+        ValueError: If the instance does not match the schema
     """
     try:
-        schema = schema_from_json_str(schema_str)
-        v = jsonschema.Draft202012Validator(schema)
-        v.validate(instance)
+        validate_schema(instance, schema_str)
     except jsonschema.exceptions.ValidationError as e:
-        raise ValueError(
-            f"This task requires a specific output schema. While the model produced JSON, that JSON didn't meet the schema. Search 'Troubleshooting Structured Data Issues' in our docs for more information. The error from the schema check was: {e.message}. The JSON was: \n```json\n{instance}\n```"
-        ) from e
+        msg = f"The error from the schema check was: {e.message}. The JSON was: \n```json\n{instance}\n```"
+        if error_prefix:
+            msg = f"{error_prefix} {msg}"
+
+        raise ValueError(msg) from e
 
 
 def schema_from_json_str(v: str) -> Dict:

kiln_ai/datamodel/task_output.py

@@ -9,7 +9,7 @@ from typing_extensions import Self
 
 from kiln_ai.datamodel.basemodel import ID_TYPE, KilnBaseModel
 from kiln_ai.datamodel.datamodel_enums import TaskOutputRatingType
-from kiln_ai.datamodel.json_schema import validate_schema
+from kiln_ai.datamodel.json_schema import validate_schema_with_value_error
 from kiln_ai.datamodel.strict_mode import strict_mode
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
@@ -308,11 +308,15 @@ class TaskOutput(KilnBaseModel):
         # validate output
         if task.output_json_schema is not None:
             try:
-                validate_schema(json.loads(self.output), task.output_json_schema)
-            except json.JSONDecodeError:
+                output_parsed = json.loads(self.output)
+            except json.JSONDecodeError as e:
                 raise ValueError("Output is not a valid JSON object")
-            except jsonschema.exceptions.ValidationError as e:
-                raise ValueError(f"Output does not match task output schema: {e}")
+
+            validate_schema_with_value_error(
+                output_parsed,
+                task.output_json_schema,
+                "This task requires a specific output schema. While the model produced JSON, that JSON didn't meet the schema. Search 'Troubleshooting Structured Data Issues' in our docs for more information.",
+            )
         return self
 
     @model_validator(mode="after")

kiln_ai/datamodel/task_run.py

@@ -7,7 +7,7 @@ from pydantic import Field, ValidationInfo, model_validator
 from typing_extensions import Self
 
 from kiln_ai.datamodel.basemodel import KilnParentedModel
-from kiln_ai.datamodel.json_schema import validate_schema
+from kiln_ai.datamodel.json_schema import validate_schema_with_value_error
 from kiln_ai.datamodel.strict_mode import strict_mode
 from kiln_ai.datamodel.task_output import DataSource, TaskOutput
 
@@ -87,14 +87,19 @@ class TaskRun(KilnParentedModel):
             # don't validate this relationship until we have a path or parent. Give them time to build it (but will catch it before saving)
             return self
 
-        # validate output
+        # validate input
         if task.input_json_schema is not None:
             try:
-                validate_schema(json.loads(self.input), task.input_json_schema)
+                input_parsed = json.loads(self.input)
             except json.JSONDecodeError:
                 raise ValueError("Input is not a valid JSON object")
-            except jsonschema.exceptions.ValidationError as e:
-                raise ValueError(f"Input does not match task input schema: {e}")
+
+            validate_schema_with_value_error(
+                input_parsed,
+                task.input_json_schema,
+                "Input does not match task input schema.",
+            )
+
         self._last_validated_input = self.input
         return self
 
@@ -131,6 +136,24 @@ class TaskRun(KilnParentedModel):
                 raise ValueError(
                     "Repaired output rating must be None. Repaired outputs are assumed to have a perfect rating, as they have been fixed."
                 )
+
+            task = self.parent_task()
+            if (
+                task is not None
+                and self.repaired_output.output is not None
+                and task.output_json_schema is not None
+            ):
+                try:
+                    output_parsed = json.loads(self.repaired_output.output)
+                except json.JSONDecodeError:
+                    raise ValueError("Repaired output is not a valid JSON object")
+
+                validate_schema_with_value_error(
+                    output_parsed,
+                    task.output_json_schema,
+                    "Repaired output does not match task output schema.",
+                )
+
         if self.repair_instructions is None and self.repaired_output is not None:
             raise ValueError(
                 "Repair instructions are required if providing a repaired output."
@@ -139,6 +162,7 @@ class TaskRun(KilnParentedModel):
             raise ValueError(
                 "A repaired output is required if providing repair instructions."
             )
+
         return self
 
     @model_validator(mode="after")

kiln_ai/datamodel/test_example_models.py

@@ -358,6 +358,9 @@ def test_task_output_schema_validation(tmp_path):
         task_output.save_to_file()
 
 
+_input_schema_match = "Input does not match task input schema"
+
+
 def test_task_input_schema_validation(tmp_path):
     # Create a project and task hierarchy
     project = Project(name="Test Project", path=(tmp_path / "test_project"))
@@ -395,18 +398,18 @@ def test_task_input_schema_validation(tmp_path):
     valid_task_output.save_to_file()
 
     # Changing to invalid input
-    with pytest.raises(ValueError, match=_schema_match):
+    with pytest.raises(ValueError, match=_input_schema_match):
         valid_task_output.input = '{"name": "John Doe", "age": "thirty"}'
         valid_task_output.save_to_file()
 
     # loading from file, then changing to invalid input
     loaded_task_output = TaskRun.load_from_file(valid_task_output.path)
-    with pytest.raises(ValueError, match=_schema_match):
+    with pytest.raises(ValueError, match=_input_schema_match):
         loaded_task_output.input = '{"name": "John Doe", "age": "thirty"}'
         loaded_task_output.save_to_file()
 
     # Invalid case: input does not match task input schema
-    with pytest.raises(ValueError, match=_schema_match):
+    with pytest.raises(ValueError, match=_input_schema_match):
         task_output = TaskRun(
             input='{"name": "John Doe", "age": "thirty"}',
             input_source=DataSource(
@@ -642,3 +645,101 @@ def test_task_run_validate_repaired_output():
         )
 
     assert "Repaired output rating must be None" in str(exc_info.value)
+
+
+def test_task_run_validate_repaired_output_structured(tmp_path):
+    # Create a project, task, and example hierarchy
+    project = Project(name="Test Project", path=(tmp_path / "test_project"))
+    project.save_to_file()
+    task = Task(
+        name="Test Task",
+        instruction="test instruction",
+        parent=project,
+        output_json_schema=json.dumps(
+            {
+                "type": "object",
+                "properties": {"name": {"type": "string"}, "age": {"type": "integer"}},
+                "required": ["name", "age"],
+            }
+        ),
+    )
+    task.save_to_file()
+
+    # test valid repaired output schema
+    task_run = TaskRun(
+        parent=task,
+        input="test input",
+        input_source=DataSource(
+            type=DataSourceType.human,
+            properties={"created_by": "john_doe"},
+        ),
+        output=TaskOutput(
+            output='{"name": "John Doe", "age": 30}',
+            source=DataSource(
+                type=DataSourceType.human,
+                properties={"created_by": "john_doe"},
+            ),
+        ),
+        repair_instructions="Fix the output",
+        repaired_output=TaskOutput(
+            output='{"name": "John Doe", "age": 30}',
+            source=DataSource(
+                type=DataSourceType.human, properties={"created_by": "john_doe"}
+            ),
+        ),
+    )
+
+    assert task_run.repaired_output is not None
+    assert task_run.repaired_output.rating is None
+
+    # test invalid JSON
+    with pytest.raises(ValueError):
+        TaskRun(
+            parent=task,
+            input="test input",
+            input_source=DataSource(
+                type=DataSourceType.human,
+                properties={"created_by": "john_doe"},
+            ),
+            output=TaskOutput(
+                output='{"name": "John Doe", "age": 30}',
+                source=DataSource(
+                    type=DataSourceType.human,
+                    properties={"created_by": "john_doe"},
+                ),
+            ),
+            repair_instructions="Fix the output",
+            repaired_output=TaskOutput(
+                output='{"name": "John Doe", "age": 30',  # missing closing brace
+                source=DataSource(
+                    type=DataSourceType.human,
+                    properties={"created_by": "john_doe"},
+                ),
+            ),
+        )
+
+    # test invalid repaired output schema
+    with pytest.raises(ValueError):
+        TaskRun(
+            parent=task,
+            input="test input",
+            input_source=DataSource(
+                type=DataSourceType.human,
+                properties={"created_by": "john_doe"},
+            ),
+            output=TaskOutput(
+                output='{"name": "John Doe", "age": 30}',
+                source=DataSource(
+                    type=DataSourceType.human,
+                    properties={"created_by": "john_doe"},
+                ),
+            ),
+            repair_instructions="Fix the output",
+            repaired_output=TaskOutput(
+                output='{"name": "John Doe", "age": "thirty"}',  # invalid schema
+                source=DataSource(
+                    type=DataSourceType.human,
+                    properties={"created_by": "john_doe"},
+                ),
+            ),
+        )

kiln_ai/datamodel/test_json_schema.py

@@ -1,3 +1,4 @@
+import jsonschema
 import pytest
 from pydantic import BaseModel
 
@@ -6,6 +7,7 @@ from kiln_ai.datamodel.json_schema import (
     schema_from_json_str,
     string_to_json_key,
     validate_schema,
+    validate_schema_with_value_error,
 )
 
 
@@ -71,15 +73,32 @@ def test_validate_schema_content():
     o = {"setup": "asdf", "punchline": "asdf", "rating": 1}
     validate_schema(o, json_joke_schema)
     o = {"setup": "asdf"}
-    with pytest.raises(Exception):
+    with pytest.raises(jsonschema.exceptions.ValidationError):
         validate_schema(0, json_joke_schema)
     o = {"setup": "asdf", "punchline": "asdf"}
     validate_schema(o, json_joke_schema)
     o = {"setup": "asdf", "punchline": "asdf", "rating": "1"}
-    with pytest.raises(Exception):
+    with pytest.raises(jsonschema.exceptions.ValidationError):
         validate_schema(o, json_joke_schema)
 
 
+def test_validate_schema_content_with_value_error():
+    o = {"setup": "asdf", "punchline": "asdf", "rating": 1}
+    validate_schema_with_value_error(o, json_joke_schema, "PREFIX")
+    o = {"setup": "asdf"}
+    with pytest.raises(
+        ValueError, match="PREFIX The error from the schema check was: "
+    ):
+        validate_schema_with_value_error(0, json_joke_schema, "PREFIX")
+    o = {"setup": "asdf", "punchline": "asdf"}
+    validate_schema_with_value_error(o, json_joke_schema, "PREFIX")
+    o = {"setup": "asdf", "punchline": "asdf", "rating": "1"}
+    with pytest.raises(
+        ValueError, match="PREFIX The error from the schema check was: "
+    ):
+        validate_schema_with_value_error(o, json_joke_schema, "PREFIX")
+
+
 json_triangle_schema = """{
   "type": "object",
   "properties": {
@@ -122,7 +141,7 @@ def test_triangle_schema():
     assert schema["properties"]["c"]["type"] == "integer"
     assert schema["required"] == ["a", "b", "c"]
     validate_schema({"a": 1, "b": 2, "c": 3}, json_triangle_schema)
-    with pytest.raises(Exception):
+    with pytest.raises(jsonschema.exceptions.ValidationError):
         validate_schema({"a": 1, "b": 2, "c": "3"}, json_triangle_schema)
 
 

kiln_ai/datamodel/test_model_perf.py

@@ -119,7 +119,8 @@ def test_benchmark_load_from_file(benchmark, task_run):
     avg_time_per_iteration = total_time / iterations
     ops_per_second = 1.0 / avg_time_per_iteration
 
-    # I get 8k ops per second on my MBP. Lower value here for CI.
+    # I get 8k ops per second on my MBP. Lower value here for CI and parallel testing.
     # Prior to optimization was 290 ops per second.
-    if ops_per_second < 1000:
+    print(f"Ops per second: {ops_per_second:.6f}")
+    if ops_per_second < 500:
         pytest.fail(f"Ops per second: {ops_per_second:.6f}, expected more than 1k ops")

{kiln_ai-0.14.0.dist-info → kiln_ai-0.15.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiln-ai
-Version: 0.14.0
+Version: 0.15.0
 Summary: Kiln AI
 Project-URL: Homepage, https://getkiln.ai
 Project-URL: Repository, https://github.com/Kiln-AI/kiln
@@ -19,7 +19,7 @@ Requires-Dist: boto3>=1.37.10
 Requires-Dist: coverage>=7.6.4
 Requires-Dist: google-cloud-aiplatform>=1.84.0
 Requires-Dist: jsonschema>=4.23.0
-Requires-Dist: litellm>=1.63.5
+Requires-Dist: litellm>=1.67.0
 Requires-Dist: openai>=1.53.0
 Requires-Dist: pdoc>=15.0.0
 Requires-Dist: pydantic>=2.9.2
@@ -28,6 +28,7 @@ Requires-Dist: pytest-cov>=6.0.0
 Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: together
 Requires-Dist: typing-extensions>=4.12.2
+Requires-Dist: vertexai>=1.43.0
 Description-Content-Type: text/markdown
 
 # Kiln AI Core Library

{kiln_ai-0.14.0.dist-info → kiln_ai-0.15.0.dist-info}/RECORD

@@ -1,23 +1,22 @@
 kiln_ai/__init__.py,sha256=Sc4z8LRVFMwJUoc_DPVUriSXTZ6PO9MaJ80PhRbKyB8,34
 kiln_ai/adapters/__init__.py,sha256=XjGmWagEyOEVwVIAxjN5rYNsQWIEACT5DB7MMTxdPss,1005
 kiln_ai/adapters/adapter_registry.py,sha256=KmMHYQ3mxpjVLE6D-hMNWCGt6Cw9JvnFn6nMb48GE8Y,9166
-kiln_ai/adapters/ml_model_list.py,sha256=f_z1daFR_w4-ccJ4OWwqlIMY0ILFJt4X5LdQb3AMt_c,58592
+kiln_ai/adapters/ml_model_list.py,sha256=RyRvPStx2TNGDjmRKSE02bOZjSWSWuJ030Ythu4Fgh4,68593
 kiln_ai/adapters/ollama_tools.py,sha256=uObtLWfqKb9RXHN-TGGw2Y1FQlEMe0u8FgszI0zQn6U,3550
 kiln_ai/adapters/prompt_builders.py,sha256=LYHTIaisQMBFtWDRIGo1QJgOsmQ-NBpQ8fI4eImHxaQ,15269
-kiln_ai/adapters/provider_tools.py,sha256=UL3XEnnxs1TrbqPPxxHSvnL7aBd84ggh38lI0yEsX6A,14725
+kiln_ai/adapters/provider_tools.py,sha256=ciFQfGJuTuHDj3FARY-sUqbSb-7oAT9lMGJGCBJoF4I,15309
 kiln_ai/adapters/run_output.py,sha256=RAi2Qp6dmqJVNm3CxbNTdAuhitHfH5NiUGbf6ygUP-k,257
 kiln_ai/adapters/test_adapter_registry.py,sha256=eDLHqv9mwgdde221pa47bTV87vCXwkUyjqsas-iFUrY,6123
-kiln_ai/adapters/test_generate_docs.py,sha256=M-uKcgF3hQmlEFOJ0o7DyL-9RgitGzkfROV-Dxtooec,2770
 kiln_ai/adapters/test_ollama_tools.py,sha256=xAUzL0IVmmXadVehJu1WjqbhpKEYGAgGt3pWx7hrubc,2514
 kiln_ai/adapters/test_prompt_adaptors.py,sha256=J1ZGZ8GG7SxP3_J3Zw0e6XmZY4NyPmUGX3IPgjh2LD8,7767
 kiln_ai/adapters/test_prompt_builders.py,sha256=5Xvfr-oQg_LLrle6UqfpRHWcPUYa8ywG3aL1rM7q1Jw,22054
-kiln_ai/adapters/test_provider_tools.py,sha256=mzMubpUupQu8pXhjDTj0_Kgrr-xcu_crj9xpcgcAzzA,26671
+kiln_ai/adapters/test_provider_tools.py,sha256=7s-njUBm_TJCTeNOh4TrP7R-Q6TXILAxbv-GK0p3YPU,27446
 kiln_ai/adapters/data_gen/__init__.py,sha256=QTZWaf7kq5BorhPvexJfwDEKmjRmIbhwW9ei8LW2SIs,276
 kiln_ai/adapters/data_gen/data_gen_prompts.py,sha256=kudjHnAz7L3q0k_NLyTlaIV7M0uRFrxXNcfcnjOE2uc,5810
 kiln_ai/adapters/data_gen/data_gen_task.py,sha256=0PuYCcj09BtpgNj23mKj_L45mKZBdV5VreUeZ-Tj_xM,6642
 kiln_ai/adapters/data_gen/test_data_gen_task.py,sha256=cRKUKMvC0uVompbmPTKwbnQ_N3c0cQDm4J_9H4Y5U18,10129
 kiln_ai/adapters/eval/__init__.py,sha256=0ptbK0ZxWuraxGn_WMgmE1tcaq0k5t-g-52kVohvWCg,693
-kiln_ai/adapters/eval/base_eval.py,sha256=jVXMiVBC07ZnLEuZVAjUAYewsnuV99put39n_GZcG1M,7261
+kiln_ai/adapters/eval/base_eval.py,sha256=IF4kYGt93bqJqSfj8UUaTng38fwPmi3cFKRSKUZhXJs,7381
 kiln_ai/adapters/eval/eval_runner.py,sha256=h3DvRFM5J5LDJqaLzNJ-q9i5LRycv2J9Ev5nw1mUDUQ,10806
 kiln_ai/adapters/eval/g_eval.py,sha256=d3UcBsZWeDt7cWp4uvDcfG7qdGLsGaZEBsIEqkpiWh4,15253
 kiln_ai/adapters/eval/registry.py,sha256=gZ_s0VgEx79Fswkgi1tS4yOl7lzpkvUBJZ62RldhM_w,626
@@ -26,20 +25,22 @@ kiln_ai/adapters/eval/test_eval_runner.py,sha256=82WPE_frNRTSQ2lylqT0inkqcDgM72n
 kiln_ai/adapters/eval/test_g_eval.py,sha256=-Stx7E0D-WAH1HWrRSp48CiGsf-no1SHeFF9IqVXeMI,16433
 kiln_ai/adapters/eval/test_g_eval_data.py,sha256=8caiZfLWnXVX8alrBPrH7L7gqqSS9vO7u6PzcHurQcA,27769
 kiln_ai/adapters/fine_tune/__init__.py,sha256=DxdTR60chwgck1aEoVYWyfWi6Ed2ZkdJj0lar-SEAj4,257
-kiln_ai/adapters/fine_tune/base_finetune.py,sha256=ORTclQTQYksMWPu7vNoD7wBzOIqNVK0YOwFEnvsKPWA,5759
-kiln_ai/adapters/fine_tune/dataset_formatter.py,sha256=qRhSSkMhTWn13OMb6LKPVwAU7uY4bB49GDiVSuhDkNg,14449
-kiln_ai/adapters/fine_tune/finetune_registry.py,sha256=CvcEVxtKwjgCMA-oYH9Tpjn1DVWmMzgHpXJOZ0YQA8k,610
-kiln_ai/adapters/fine_tune/fireworks_finetune.py,sha256=OlXp8j6Afwvk6-ySwA3Q7iuqBlKO7VLeAfNCnB3pZPI,19963
+kiln_ai/adapters/fine_tune/base_finetune.py,sha256=g-lWuZMkOj2djcczuHke_Ai7Z7RPg41AFSgoxHgsw3U,5889
+kiln_ai/adapters/fine_tune/dataset_formatter.py,sha256=ky48er7lMIS3Kv5WflaLpUDvWiVGYgl8QlI0M_wy6Vo,14409
+kiln_ai/adapters/fine_tune/finetune_registry.py,sha256=9RJLjviSoN3dQnKJE9Ss7df7dtdJgbuShB8IUcI-q9k,726
+kiln_ai/adapters/fine_tune/fireworks_finetune.py,sha256=ze0QxghpHAqwO9nXOTkDEC9irmqduX5bjIhZDU0DCZQ,20101
 kiln_ai/adapters/fine_tune/openai_finetune.py,sha256=Dz9E_0BWfrIkvv8ArZe-RKPwbIKPZ3v8rfbc3JELyTY,8571
-kiln_ai/adapters/fine_tune/test_base_finetune.py,sha256=sjuDgJDA_dynGRelx9_wXdssaxAYIuEG-Z8NzRx9Hl0,10559
-kiln_ai/adapters/fine_tune/test_dataset_formatter.py,sha256=T3jbFZooLVBaGCE0LUVxwPxzM3l8IY41zUj3jPk-Zi8,24027
-kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py,sha256=oLyLEG4TwW452lV2mvUo-wImLxzSwOuoKKeYFuGh3k8,36744
+kiln_ai/adapters/fine_tune/test_base_finetune.py,sha256=Tq0Klw7ou5_6H_bouTbI3PxYw7H30K32wlgWJE_luYk,10751
+kiln_ai/adapters/fine_tune/test_dataset_formatter.py,sha256=kUGn2kv2jwosuabuhYgA3oXJXAdqK1AAaJI496ScOGY,24015
+kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py,sha256=NCl2U6ZqqJ8dnysGGmfir9RGcV-StPtoi5cetRjW6Zc,36754
 kiln_ai/adapters/fine_tune/test_openai_finetune.py,sha256=H63Xk2PNHbt5Ev5IQpdR9JZ4uz-Huo2gfuC4mHHqe0w,20011
 kiln_ai/adapters/fine_tune/test_together_finetune.py,sha256=BUJFsyq_g77gU0JN3hg6FMBvqb0DIyTeAek-wxomKIg,18090
+kiln_ai/adapters/fine_tune/test_vertex_finetune.py,sha256=rAmcQJNPXqRacxg6RzjEQ8FNLKCp9qZRHToH7fm-7W0,19214
 kiln_ai/adapters/fine_tune/together_finetune.py,sha256=EbMPsTyKMubfwOalkFLiNFlMFIRKxLibzMTyLeUkle4,14010
+kiln_ai/adapters/fine_tune/vertex_finetune.py,sha256=Ik6Ov711-oruJnMHpVZTPimWJY2W_JnfdKIdR2djGrc,8545
 kiln_ai/adapters/model_adapters/__init__.py,sha256=m5GRtOHwVVvp_XDOss8c1X3NFf1wQQlC2eBgI4tXQhM,212
-kiln_ai/adapters/model_adapters/base_adapter.py,sha256=ifPJMg0nEKamfOSmBIsnp_MRFfBs47FLeQrLbav34yA,9872
-kiln_ai/adapters/model_adapters/litellm_adapter.py,sha256=c4J_tIpM96KWS2qzoPaQmBj7X7mHyRMShdkmEh7_EHM,16129
+kiln_ai/adapters/model_adapters/base_adapter.py,sha256=ishm_oVTNxSDC0GPrydHnyOPqp_U4XiTOx0-iI2fEiU,10433
+kiln_ai/adapters/model_adapters/litellm_adapter.py,sha256=pbXFfJckyvptp577-YXGMG2hltYMFQrUT97PsSWa2KQ,16437
 kiln_ai/adapters/model_adapters/litellm_config.py,sha256=7-tIh5cuVu23Uy2Sd6q7UCT_4VgevBsAzVhQMj6Svgw,425
 kiln_ai/adapters/model_adapters/test_base_adapter.py,sha256=uQyKrHLN3Jha6R-6SWkEME6brQecVFdPTSXogo-xpt0,6556
 kiln_ai/adapters/model_adapters/test_litellm_adapter.py,sha256=QpnzuReNeBzvvRYnNj_5c8l1PS7NyrDDUQx_o21IIH4,13731
@@ -63,7 +64,7 @@ kiln_ai/datamodel/dataset_filters.py,sha256=hWKxGJ-mSl4y0igyNcpmRoRYCiGrf0_uN4MM
 kiln_ai/datamodel/dataset_split.py,sha256=q4l4SlUvjLV547bzk7Z-fbmj_o26GDcYOZ2rA5RPh3c,5612
 kiln_ai/datamodel/eval.py,sha256=kio2LqQ87MsP75DJTiIVdVfopTZXH4xjGN9g11V1mUU,13826
 kiln_ai/datamodel/finetune.py,sha256=TYoNVRAfbjqvrY-1YmHwG6xSoDljiJWuuVcTbvQAJL4,4569
-kiln_ai/datamodel/json_schema.py,sha256=qIlR8btXhN-8Yj5GhwebzPLUHPw2sJC3uM1axV2xV7w,3032
+kiln_ai/datamodel/json_schema.py,sha256=o50wSp8frRXjT-NZjml4-Is7LNoF7DQP4g3AaaYzBfI,3379
 kiln_ai/datamodel/model_cache.py,sha256=9X4aAigbkFdytckgw8InCMh86uBna0ME_1HJSeMPEn0,4495
 kiln_ai/datamodel/project.py,sha256=uVH2_3TDFtsG_tpts81A-zbd9uPDFxAwMCKZt_km3IE,727
 kiln_ai/datamodel/prompt.py,sha256=70JPYHfgyX18cHW_DXoMzIOA28Jbaz6gyabElmpycyc,1161
@@ -71,17 +72,17 @@ kiln_ai/datamodel/prompt_id.py,sha256=eU2TV0RZapn-BgnZ4sOSNOOVEQ3aPaLzW4YSYCd3OB
 kiln_ai/datamodel/registry.py,sha256=XwGFXJFKZtOpR1Z9ven6SftggfADdZRm8TFxCEVtfUQ,957
 kiln_ai/datamodel/strict_mode.py,sha256=sm4Xka8mnJHCShtbh6MMU5dDQv-cLj8lHgHkmFKpsl0,849
 kiln_ai/datamodel/task.py,sha256=r-_zgrQCIiIkN8gvBISdU449Z9oKp7E1XL0lkik_rVI,7036
-kiln_ai/datamodel/task_output.py,sha256=uIYR8EyWv8Bbl60gPRuTIUSvfGGzP9Ltc5P280HyTpY,12931
-kiln_ai/datamodel/task_run.py,sha256=yquE0jyr_9WzcvrMsEmZfXUnn8zZDEZIXZhVcVBMrT8,7038
+kiln_ai/datamodel/task_output.py,sha256=PqI7Lyeox5lh9mItMOtpqP9Rk_K9dyMltKYu1c2p7A4,13125
+kiln_ai/datamodel/task_run.py,sha256=mVKmHn90iPmwXGja7TNgDA3iFzXBlamJ_6KndRPkhRA,7745
 kiln_ai/datamodel/test_basemodel.py,sha256=sJ8wXGef2WxzbrbMTYgrOwmkd5J6sHkly-cQBO2IZh4,18126
 kiln_ai/datamodel/test_dataset_filters.py,sha256=v88QPkIsq4diUmoUF3-qj5KAW2rLRp0KDAm_pexbFy4,1894
 kiln_ai/datamodel/test_dataset_split.py,sha256=5CHO1Lq4xQBB72tV2SPER7OZODJNvj15qxi_cYBV2Rs,11157
 kiln_ai/datamodel/test_datasource.py,sha256=H4Kc-Im9eM7WnADWZXdoiOIrOl05RtkyuhTCKiRimyU,3905
 kiln_ai/datamodel/test_eval_model.py,sha256=J7MqwWBgPpeXGqh3IacVUUHdZFJSZ2MgTsUNu-hNOJw,19528
-kiln_ai/datamodel/test_example_models.py,sha256=fpqh0u7zFhWHcRHgtxCjX8RD2oKHYOP_mJJymaUhEZU,20944
-kiln_ai/datamodel/test_json_schema.py,sha256=UgKwAFcdrJTq2byh7Yf-HoSAtiHiGAsNZxfkIvoMxIg,3915
+kiln_ai/datamodel/test_example_models.py,sha256=nrr13ZseFn-OVGa9bjCwoVHTVqydy0O0yJah4QiqqbU,24326
+kiln_ai/datamodel/test_json_schema.py,sha256=R0Cfc9WbieMslgvYsj2HFx8RHIq2fF9NcT5jH-kEqh4,4793
 kiln_ai/datamodel/test_model_cache.py,sha256=Fy-ucYNzS5JEG-8SFY4nVHA8iRbXXxai20f8_oGl97o,8184
-kiln_ai/datamodel/test_model_perf.py,sha256=NdD7L8XraGkunaEKGPsfYwdcbIgdjhFanOO3G6hU158,3235
+kiln_ai/datamodel/test_model_perf.py,sha256=9_76f__5XtZSHSjuaoiHRj2t-z3OWn-sSA4S9kH1jpY,3306
 kiln_ai/datamodel/test_models.py,sha256=hmV7sTbOamWJCwOY96w-g4PQRv4Uai-XaHtg0QKH-ak,19295
 kiln_ai/datamodel/test_nested_save.py,sha256=xciCddqvPyKyoyjC5Lx_3Kh1t4LJv1xYRAPazR3SRcs,5588
 kiln_ai/datamodel/test_output_rating.py,sha256=zvPIp2shAgCs2RQBgwYoL09fRA3krHvgAqUa91RlWR0,15125
@@ -97,7 +98,7 @@ kiln_ai/utils/name_generator.py,sha256=v26TgpCwQbhQFcZvzgjZvURinjrOyyFhxpsI6NQrH
 kiln_ai/utils/test_config.py,sha256=Jw3nMFeIgZUsZDRJJY2HpB-2EkR2NoZ-rDe_o9oA7ws,9174
 kiln_ai/utils/test_dataset_import.py,sha256=ZZOt7zqtaEIlMMx0VNXyRegDvnVqbWY2bcz-iMY_Oag,17427
 kiln_ai/utils/test_name_geneator.py,sha256=9-hSTBshyakqlPbFnNcggwLrL7lcPTitauBYHg9jFWI,1513
-kiln_ai-0.14.0.dist-info/METADATA,sha256=EjgZOnknE7P9uW5BsIFJZYQAN-aUQ817SAEXjtqtjK0,12231
-kiln_ai-0.14.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kiln_ai-0.14.0.dist-info/licenses/LICENSE.txt,sha256=_NA5pnTYgRRr4qH6lE3X-TuZJ8iRcMUi5ASoGr-lEx8,1209
-kiln_ai-0.14.0.dist-info/RECORD,,
+kiln_ai-0.15.0.dist-info/METADATA,sha256=80EooOjbu5b-7YgyfnOY9uYjFk8bo5czNWm3QgWaFys,12263
+kiln_ai-0.15.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kiln_ai-0.15.0.dist-info/licenses/LICENSE.txt,sha256=_NA5pnTYgRRr4qH6lE3X-TuZJ8iRcMUi5ASoGr-lEx8,1209
+kiln_ai-0.15.0.dist-info/RECORD,,

kiln_ai/adapters/test_generate_docs.py

@@ -1,69 +0,0 @@
-import logging
-from typing import List
-
-import pytest
-
-from libs.core.kiln_ai.adapters.ml_model_list import KilnModelProvider, built_in_models
-from libs.core.kiln_ai.adapters.provider_tools import provider_name_from_id
-
-logger = logging.getLogger(__name__)
-
-
-def _all_providers_support(providers: List[KilnModelProvider], attribute: str) -> bool:
-    """Check if all providers support a given feature"""
-    return all(getattr(provider, attribute) for provider in providers)
-
-
-def _any_providers_support(providers: List[KilnModelProvider], attribute: str) -> bool:
-    """Check if any providers support a given feature"""
-    return any(getattr(provider, attribute) for provider in providers)
-
-
-def _get_support_status(providers: List[KilnModelProvider], attribute: str) -> str:
-    """Get the support status for a feature"""
-    if _all_providers_support(providers, attribute):
-        return "✅︎"
-    elif _any_providers_support(providers, attribute):
-        return "✅︎ (some providers)"
-    return ""
-
-
-def _has_finetune_support(providers: List[KilnModelProvider]) -> str:
-    """Check if any provider supports fine-tuning"""
-    return "✅︎" if any(p.provider_finetune_id for p in providers) else ""
-
-
-@pytest.mark.paid(reason="Marking as paid so it isn't run by default")
-def test_generate_model_table():
-    """Generate a markdown table of all models and their capabilities"""
-
-    # Table header
-    table = [
-        "| Model Name | Providers | Structured Output | Reasoning | Synthetic Data | API Fine-Tuneable |",
-        "|------------|-----------|-------------------|-----------|----------------|-------------------|",
-    ]
-
-    for model in built_in_models:
-        provider_names = ", ".join(
-            sorted(provider_name_from_id(p.name.value) for p in model.providers)
-        )
-        structured_output = _get_support_status(
-            model.providers, "supports_structured_output"
-        )
-        reasoning = _get_support_status(model.providers, "reasoning_capable")
-        data_gen = _get_support_status(model.providers, "supports_data_gen")
-        finetune = _has_finetune_support(model.providers)
-
-        row = f"| {model.friendly_name} | {provider_names} | {structured_output} | {reasoning} | {data_gen} | {finetune} |"
-        table.append(row)
-
-    # Print the table (useful for documentation)
-    print("\nModel Capability Matrix:\n")
-    print("\n".join(table))
-
-    # Basic assertions to ensure the table is well-formed
-    assert len(table) > 2, "Table should have header and at least one row"
-    assert all("|" in row for row in table), "All rows should be properly formatted"
-    assert len(table[0].split("|")) == len(table[1].split("|")), (
-        "Header and separator should have same number of columns"
-    )