kiln-ai 0.13.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

kiln_ai/adapters/adapter_registry.py

@@ -108,6 +108,10 @@ def adapter_for_task(
                     # 1. To use the correct base URL
                     # 2. We use Ollama's OpenAI compatible API (/v1), and don't just let litellm use the Ollama API. We use more advanced features like json_schema.
                     base_url=ollama_base_url + "/v1",
+                    additional_body_options={
+                        # LiteLLM errors without an api_key, even though Ollama doesn't support one.
+                        "api_key": "NA",
+                    },
                 ),
             )
         case ModelProviderName.fireworks_ai:

kiln_ai/adapters/eval/g_eval.py

@@ -297,9 +297,12 @@ The model produced the following output for the task:
 
         total_score = 0.0
         total_probability = 0.0
+        top_logprobs_contains_primary_token = False
 
-        # Process all valid scoring tokens
+        # Process all valid scoring tokens from alternatives
         for top_logprob in token_logprob.top_logprobs:
+            if top_logprob.token == token_logprob.token:
+                top_logprobs_contains_primary_token = True
             token_score = self.score_from_token_string(top_logprob.token)
             if token_score is not None:
                 # Convert logprob to probability
@@ -307,9 +310,21 @@ The model produced the following output for the task:
                 total_score += token_score * probability
                 total_probability += probability
 
+        # Weird OpenAI 4o bug - sometimes the primary token is included in the top logprobs, sometimes not.
+        # Add the primary token back in if excluded
+        if not top_logprobs_contains_primary_token:
+            if token_logprob.logprob == -9999.0:
+                # Another "bug" - sometimes the logprob is -9999.0. This seems to happen when the rest of the logprobs are tiny probability.
+                total_score += primary_token_score * 1.0
+                total_probability += 1.0
+            else:
+                probability = math.exp(token_logprob.logprob)
+                total_score += primary_token_score * probability
+                total_probability += probability
+
         if total_probability <= 0.0:
             raise RuntimeError(
-                f"No valid scoring tokens found for {token_logprob.token}. This should never happen. Please file a bug if you see this."
+                f"No valid scoring tokens found for {token_logprob.token}. This should never happen as the token has a valid score (so it must be excluded from top logprobs). Please file a bug if you see this."
             )
 
         # Normalize by total probability of valid tokens (LLM may have wanted to generate other non-rating tokens, these shouldn't lower score of rating tokens)

kiln_ai/adapters/eval/test_g_eval.py

@@ -393,12 +393,13 @@ def test_rating_token_to_score(test_eval_config, test_run_config):
             self.logprob = logprob
 
     class MockTokenLogprob:
-        def __init__(self, token, top_logprobs):
+        def __init__(self, token, top_logprobs, logprob):
             self.token = token
             self.top_logprobs = [MockTopLogprob(t, lp) for t, lp in top_logprobs]
+            self.logprob = logprob
 
     # Test single token case
-    token_logprob = MockTokenLogprob("5", [("5", 0.0)])  # log(1) = 0
+    token_logprob = MockTokenLogprob("5", [("5", 0.0)], logprob=1e-8)  # log(1) = 0
     score = g_eval.rating_token_to_score(token_logprob)
     assert score == 5.0
 
@@ -409,18 +410,22 @@ def test_rating_token_to_score(test_eval_config, test_run_config):
             ("4", math.log(0.6)),  # 60% probability
             ("5", math.log(0.4)),  # 40% probability
         ],
+        logprob=math.log(0.6),
     )
     score = g_eval.rating_token_to_score(token_logprob)
     assert pytest.approx(score) == 4.4  # (4 * 0.6 + 5 * 0.4)
 
     # Test invalid token
-    token_logprob = MockTokenLogprob(":", [(":", 0.0)])
+    token_logprob = MockTokenLogprob(":", [(":", 0.0)], logprob=1e-8)
     assert g_eval.rating_token_to_score(token_logprob) is None
 
-    # Test no valid scoring tokens
-    token_logprob = MockTokenLogprob("5", [])
-    with pytest.raises(RuntimeError, match="No valid scoring tokens found"):
-        g_eval.rating_token_to_score(token_logprob)
+    # Test missing from top logprobs
+    token_logprob = MockTokenLogprob("5", [], logprob=1e-8)
+    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0
+
+    # Test missing from top logprobs, with special case logprob
+    token_logprob = MockTokenLogprob("5", [], logprob=-9999)
+    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0
 
 
 def test_g_eval_system_instruction():

kiln_ai/adapters/fine_tune/base_finetune.py

@@ -72,8 +72,6 @@ class BaseFinetuneAdapter(ABC):
         Create and start a fine-tune.
         """
 
-        cls.check_valid_provider_model(provider_id, provider_base_model_id)
-
         if not dataset.id:
             raise ValueError("Dataset must have an id")
 
@@ -184,21 +182,3 @@ class BaseFinetuneAdapter(ABC):
         for parameter_key in parameters:
             if parameter_key not in allowed_parameters:
                 raise ValueError(f"Parameter {parameter_key} is not available")
-
-    @classmethod
-    def check_valid_provider_model(
-        cls, provider_id: str, provider_base_model_id: str
-    ) -> None:
-        """
-        Check if the provider and base model are valid.
-        """
-        for model in built_in_models:
-            for provider in model.providers:
-                if (
-                    provider.name == provider_id
-                    and provider.provider_finetune_id == provider_base_model_id
-                ):
-                    return
-        raise ValueError(
-            f"Provider {provider_id} with base model {provider_base_model_id} is not available"
-        )

kiln_ai/adapters/fine_tune/fireworks_finetune.py

@@ -1,4 +1,5 @@
-from typing import Tuple
+import logging
+from typing import List, Tuple
 from uuid import uuid4
 
 import httpx
@@ -13,6 +14,14 @@ from kiln_ai.adapters.fine_tune.dataset_formatter import DatasetFormat, DatasetF
 from kiln_ai.datamodel import DatasetSplit, StructuredOutputMode, Task
 from kiln_ai.utils.config import Config
 
+logger = logging.getLogger(__name__)
+
+# https://docs.fireworks.ai/fine-tuning/fine-tuning-models#supported-base-models-loras-on-serverless
+serverless_models = [
+    "accounts/fireworks/models/llama-v3p1-8b-instruct",
+    "accounts/fireworks/models/llama-v3p1-70b-instruct",
+]
+
 
 class FireworksFinetune(BaseFinetuneAdapter):
     """
@@ -132,11 +141,18 @@ class FireworksFinetune(BaseFinetuneAdapter):
                 :60
             ]
         )
-        payload = {
+        payload: dict[str, str | dict[str, str | bool]] = {
             "dataset": f"accounts/{account_id}/datasets/{train_file_id}",
             "displayName": display_name,
             "baseModel": self.datamodel.base_model_id,
         }
+        # Add W&B config if API key is set
+        if Config.shared().wandb_api_key:
+            payload["wandbConfig"] = {
+                "enabled": True,
+                "project": "Kiln_AI",
+                "apiKey": Config.shared().wandb_api_key,
+            }
         hyperparameters = self.create_payload_parameters(self.datamodel.parameters)
         payload.update(hyperparameters)
         headers = {
@@ -276,32 +292,54 @@ class FireworksFinetune(BaseFinetuneAdapter):
         return {k: v for k, v in payload.items() if v is not None}
 
     async def _deploy(self) -> bool:
-        # Now we "deploy" the model using PEFT serverless.
-        # A bit complicated: most fireworks deploys are server based.
-        # However, a Lora can be serverless (PEFT).
-        # By calling the deploy endpoint WITHOUT first creating a deployment ID, it will only deploy if it can be done serverless.
-        # https://docs.fireworks.ai/models/deploying#deploying-to-serverless
-        # This endpoint will return 400 if already deployed with code 9, so we consider that a success.
+        if self.datamodel.base_model_id in serverless_models:
+            return await self._deploy_serverless()
+        else:
+            return await self._check_or_deploy_server()
 
+    def api_key_and_account_id(self) -> Tuple[str, str]:
         api_key = Config.shared().fireworks_api_key
         account_id = Config.shared().fireworks_account_id
         if not api_key or not account_id:
             raise ValueError("Fireworks API key or account ID not set")
+        return api_key, account_id
+
+    def deployment_display_name(self) -> str:
+        # Limit the display name to 60 characters
+        display_name = f"Kiln AI fine-tuned model [ID:{self.datamodel.id}][name:{self.datamodel.name}]"[
+            :60
+        ]
+        return display_name
 
+    async def model_id_checking_status(self) -> str | None:
         # Model ID != fine tune ID on Fireworks. Model is the result of the tune job. Call status to get it.
         status, model_id = await self._status()
         if status.status != FineTuneStatusType.completed:
-            return False
+            return None
         if not model_id or not isinstance(model_id, str):
-            return False
+            return None
+        return model_id
+
+    async def _deploy_serverless(self) -> bool:
+        # Now we "deploy" the model using PEFT serverless.
+        # A bit complicated: most fireworks deploys are server based.
+        # However, a Lora can be serverless (PEFT).
+        # By calling the deploy endpoint WITHOUT first creating a deployment ID, it will only deploy if it can be done serverless.
+        # https://docs.fireworks.ai/models/deploying#deploying-to-serverless
+        # This endpoint will return 400 if already deployed with code 9, so we consider that a success.
+
+        api_key, account_id = self.api_key_and_account_id()
 
         url = f"https://api.fireworks.ai/v1/accounts/{account_id}/deployedModels"
-        # Limit the display name to 60 characters
-        display_name = f"Kiln AI fine-tuned model [ID:{self.datamodel.id}][name:{self.datamodel.name}]"[
-            :60
-        ]
+        model_id = await self.model_id_checking_status()
+        if not model_id:
+            logger.error(
+                "Model ID not found - can't deploy model to Fireworks serverless"
+            )
+            return False
+
         payload = {
-            "displayName": display_name,
+            "displayName": self.deployment_display_name(),
             "model": model_id,
         }
         headers = {
@@ -320,4 +358,120 @@ class FireworksFinetune(BaseFinetuneAdapter):
                     self.datamodel.save_to_file()
             return True
 
+        logger.error(
+            f"Failed to deploy model to Fireworks serverless: [{response.status_code}] {response.text}"
+        )
         return False
+
+    async def _check_or_deploy_server(self) -> bool:
+        """
+        Check if the model is already deployed. If not, deploy it to a dedicated server.
+        """
+
+        # Check if the model is already deployed
+        # If it's fine_tune_model_id is set, it might be deployed. However, Fireworks deletes them over time so we need to check.
+        if self.datamodel.fine_tune_model_id:
+            deployments = await self._fetch_all_deployments()
+            for deployment in deployments:
+                if deployment[
+                    "baseModel"
+                ] == self.datamodel.fine_tune_model_id and deployment["state"] in [
+                    "READY",
+                    "CREATING",
+                ]:
+                    return True
+
+        # If the model is not deployed, deploy it
+        return await self._deploy_server()
+
+    async def _deploy_server(self) -> bool:
+        # For models that are not serverless, we just need to deploy the model to a server.
+        # We use a scale-to-zero on-demand deployment. If you stop using it, it
+        # will scale to zero and charges will stop.
+        model_id = await self.model_id_checking_status()
+        if not model_id:
+            logger.error("Model ID not found - can't deploy model to Fireworks server")
+            return False
+
+        api_key, account_id = self.api_key_and_account_id()
+        url = f"https://api.fireworks.ai/v1/accounts/{account_id}/deployments"
+
+        payload = {
+            "displayName": self.deployment_display_name(),
+            "description": "Deployed by Kiln AI",
+            # Allow scale to zero
+            "minReplicaCount": 0,
+            "autoscalingPolicy": {
+                "scaleUpWindow": "30s",
+                "scaleDownWindow": "300s",
+                # Scale to zero after 5 minutes of inactivity - this is the minimum allowed
+                "scaleToZeroWindow": "300s",
+            },
+            "baseModel": model_id,
+        }
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+
+        async with httpx.AsyncClient() as client:
+            response = await client.post(url, json=payload, headers=headers)
+
+        if response.status_code == 200:
+            basemodel = response.json().get("baseModel")
+            if basemodel is not None and isinstance(basemodel, str):
+                self.datamodel.fine_tune_model_id = basemodel
+                if self.datamodel.path:
+                    self.datamodel.save_to_file()
+                return True
+
+        logger.error(
+            f"Failed to deploy model to Fireworks server: [{response.status_code}] {response.text}"
+        )
+        return False
+
+    async def _fetch_all_deployments(self) -> List[dict]:
+        """
+        Fetch all deployments for an account.
+        """
+        api_key, account_id = self.api_key_and_account_id()
+
+        url = f"https://api.fireworks.ai/v1/accounts/{account_id}/deployments"
+
+        params = {
+            # Note: filter param does not work for baseModel, which would have been ideal, and ideally would have been documented. Instead we'll fetch all and filter.
+            # Max page size
+            "pageSize": 200,
+        }
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+        }
+
+        deployments = []
+
+        # Paginate through all deployments
+        async with httpx.AsyncClient() as client:
+            while True:
+                response = await client.get(url, params=params, headers=headers)
+                json = response.json()
+                if "deployments" not in json or not isinstance(
+                    json["deployments"], list
+                ):
+                    raise ValueError(
+                        f"Invalid response from Fireworks. Expected list of deployments in 'deployments' key: [{response.status_code}] {response.text}"
+                    )
+                deployments.extend(json["deployments"])
+                next_page_token = json.get("nextPageToken")
+                if (
+                    next_page_token
+                    and isinstance(next_page_token, str)
+                    and len(next_page_token) > 0
+                ):
+                    params = {
+                        "pageSize": 200,
+                        "pageToken": next_page_token,
+                    }
+                else:
+                    break
+
+        return deployments

kiln_ai/adapters/fine_tune/test_base_finetune.py

@@ -261,15 +261,6 @@ async def test_create_and_start_no_parent_task_path():
         )
 
 
-def test_check_valid_provider_model():
-    MockFinetune.check_valid_provider_model("openai", "gpt-4o-mini-2024-07-18")
-
-    with pytest.raises(
-        ValueError, match="Provider openai with base model gpt-99 is not available"
-    ):
-        MockFinetune.check_valid_provider_model("openai", "gpt-99")
-
-
 async def test_create_and_start_invalid_train_split(mock_dataset):
     # Test with an invalid train split name
     mock_dataset.split_contents = {"valid_train": [], "valid_test": []}

kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py

@@ -340,6 +340,7 @@ async def test_start_success(
     expected_mode,
     expected_format,
 ):
+    Config.shared().wandb_api_key = "test-api-key"
     mock_task.output_json_schema = output_schema
 
     fireworks_finetune.datamodel.parent = mock_task
@@ -378,6 +379,24 @@ async def test_start_success(
         assert fireworks_finetune.datamodel.structured_output_mode == expected_mode
         assert fireworks_finetune.datamodel.properties["endpoint_version"] == "v2"
 
+        # check mockclent.post call values
+        assert mock_client.post.call_count == 1
+        submit_call_values = mock_client.post.call_args[1]
+        assert submit_call_values["json"]["wandbConfig"] == {
+            "enabled": True,
+            "project": "Kiln_AI",
+            "apiKey": "test-api-key",
+        }
+        assert submit_call_values["json"]["baseModel"] == "llama-v2-7b"
+        assert (
+            submit_call_values["json"]["dataset"]
+            == f"accounts/{Config.shared().fireworks_account_id}/datasets/{mock_dataset_id}"
+        )
+        assert (
+            submit_call_values["json"]["displayName"]
+            == f"Kiln AI fine-tuning [ID:{fireworks_finetune.datamodel.id}][name:{fireworks_finetune.datamodel.name}]"
+        )
+
 
 async def test_start_api_error(
     fireworks_finetune, mock_dataset, mock_task, mock_api_key
@@ -429,7 +448,7 @@ def test_available_parameters(fireworks_finetune):
     assert payload_parameters == {"loraRank": 16, "epochs": 3}
 
 
-async def test_deploy_success(fireworks_finetune, mock_api_key):
+async def test_deploy_serverless_success(fireworks_finetune, mock_api_key):
     # Mock response for successful deployment
     success_response = MagicMock(spec=httpx.Response)
     success_response.status_code = 200
@@ -448,12 +467,12 @@ async def test_deploy_success(fireworks_finetune, mock_api_key):
         mock_client.post.return_value = success_response
         mock_client_class.return_value.__aenter__.return_value = mock_client
 
-        result = await fireworks_finetune._deploy()
+        result = await fireworks_finetune._deploy_serverless()
         assert result is True
         assert fireworks_finetune.datamodel.fine_tune_model_id == "ftm-123"
 
 
-async def test_deploy_already_deployed(fireworks_finetune, mock_api_key):
+async def test_deploy_serverless_already_deployed(fireworks_finetune, mock_api_key):
     # Mock response for already deployed model
     already_deployed_response = MagicMock(spec=httpx.Response)
     already_deployed_response.status_code = 400
@@ -475,12 +494,12 @@ async def test_deploy_already_deployed(fireworks_finetune, mock_api_key):
         mock_client.post.return_value = already_deployed_response
         mock_client_class.return_value.__aenter__.return_value = mock_client
 
-        result = await fireworks_finetune._deploy()
+        result = await fireworks_finetune._deploy_serverless()
         assert result is True
         assert fireworks_finetune.datamodel.fine_tune_model_id == "ftm-123"
 
 
-async def test_deploy_failure(fireworks_finetune, mock_api_key):
+async def test_deploy_serverless_failure(fireworks_finetune, mock_api_key):
     # Mock response for failed deployment
     failure_response = MagicMock(spec=httpx.Response)
     failure_response.status_code = 500
@@ -491,18 +510,28 @@ async def test_deploy_failure(fireworks_finetune, mock_api_key):
         mock_client.post.return_value = failure_response
         mock_client_class.return_value.__aenter__.return_value = mock_client
 
-        result = await fireworks_finetune._deploy()
+        result = await fireworks_finetune._deploy_serverless()
         assert result is False
 
 
-async def test_deploy_missing_credentials(fireworks_finetune):
+async def test_deploy_serverless_missing_credentials(fireworks_finetune):
     # Test missing API key or account ID
     with patch.object(Config, "shared") as mock_config:
         mock_config.return_value.fireworks_api_key = None
         mock_config.return_value.fireworks_account_id = None
 
         with pytest.raises(ValueError, match="Fireworks API key or account ID not set"):
-            await fireworks_finetune._deploy()
+            await fireworks_finetune._deploy_serverless()
+
+
+async def test_deploy_server_missing_credentials(fireworks_finetune):
+    # Test missing API key or account ID
+    with patch.object(Config, "shared") as mock_config:
+        mock_config.return_value.fireworks_api_key = None
+        mock_config.return_value.fireworks_account_id = None
+
+        response = await fireworks_finetune._check_or_deploy_server()
+        assert response is False
 
 
 async def test_deploy_missing_model_id(fireworks_finetune, mock_api_key):
@@ -545,3 +574,479 @@ async def test_status_with_deploy(fireworks_finetune, mock_api_key):
         # Verify message was updated due to failed deployment
         assert status.status == FineTuneStatusType.completed
         assert status.message == "Fine-tuning job completed but failed to deploy model."
+
+
+@pytest.mark.paid
+async def test_fetch_all_deployments(fireworks_finetune):
+    deployments = await fireworks_finetune._fetch_all_deployments()
+    assert isinstance(deployments, list)
+
+
+async def test_api_key_and_account_id(fireworks_finetune, mock_api_key):
+    # Test successful retrieval of API key and account ID
+    api_key, account_id = fireworks_finetune.api_key_and_account_id()
+    assert api_key == "test-api-key"
+    assert account_id == "test-account-id"
+
+
+async def test_api_key_and_account_id_missing_credentials(fireworks_finetune):
+    # Test missing API key or account ID
+    with patch.object(Config, "shared") as mock_config:
+        mock_config.return_value.fireworks_api_key = None
+        mock_config.return_value.fireworks_account_id = None
+
+        with pytest.raises(ValueError, match="Fireworks API key or account ID not set"):
+            fireworks_finetune.api_key_and_account_id()
+
+
+def test_deployment_display_name(fireworks_finetune):
+    # Test with default ID and name
+    display_name = fireworks_finetune.deployment_display_name()
+    expected = f"Kiln AI fine-tuned model [ID:{fireworks_finetune.datamodel.id}][name:test-finetune]"[
+        :60
+    ]
+    assert display_name == expected
+
+    # Test with a very long name to ensure 60 character limit
+    fireworks_finetune.datamodel.name = "x" * 100
+    display_name = fireworks_finetune.deployment_display_name()
+    assert len(display_name) == 60
+    assert display_name.startswith("Kiln AI fine-tuned model [ID:")
+
+
+async def test_model_id_checking_status_completed(fireworks_finetune):
+    # Test with completed status and valid model ID
+    status_response = (
+        FineTuneStatus(status=FineTuneStatusType.completed, message=""),
+        "model-123",
+    )
+
+    with patch.object(fireworks_finetune, "_status", return_value=status_response):
+        model_id = await fireworks_finetune.model_id_checking_status()
+        assert model_id == "model-123"
+
+
+async def test_model_id_checking_status_not_completed(fireworks_finetune):
+    # Test with non-completed status
+    status_response = (
+        FineTuneStatus(status=FineTuneStatusType.running, message=""),
+        "model-123",
+    )
+
+    with patch.object(fireworks_finetune, "_status", return_value=status_response):
+        model_id = await fireworks_finetune.model_id_checking_status()
+        assert model_id is None
+
+
+async def test_model_id_checking_status_invalid_model_id(fireworks_finetune):
+    # Test with completed status but invalid model ID
+    status_response = (
+        FineTuneStatus(status=FineTuneStatusType.completed, message=""),
+        None,
+    )
+
+    with patch.object(fireworks_finetune, "_status", return_value=status_response):
+        model_id = await fireworks_finetune.model_id_checking_status()
+        assert model_id is None
+
+    # Test with non-string model ID
+    status_response = (
+        FineTuneStatus(status=FineTuneStatusType.completed, message=""),
+        {"id": "model-123"},  # Not a string
+    )
+
+    with patch.object(fireworks_finetune, "_status", return_value=status_response):
+        model_id = await fireworks_finetune.model_id_checking_status()
+        assert model_id is None
+
+
+@pytest.mark.parametrize(
+    "base_model_id,expected_method",
+    [
+        ("accounts/fireworks/models/llama-v3p1-8b-instruct", "_deploy_serverless"),
+        ("accounts/fireworks/models/llama-v3p1-70b-instruct", "_deploy_serverless"),
+        ("some-other-model", "_check_or_deploy_server"),
+    ],
+)
+async def test_deploy_model_selection(
+    fireworks_finetune, base_model_id, expected_method, mock_api_key
+):
+    # Set the base model ID
+    fireworks_finetune.datamodel.base_model_id = base_model_id
+
+    # Mock the deployment methods
+    with (
+        patch.object(
+            fireworks_finetune, "_deploy_serverless", return_value=True
+        ) as mock_serverless,
+        patch.object(
+            fireworks_finetune, "_check_or_deploy_server", return_value=True
+        ) as mock_server,
+    ):
+        result = await fireworks_finetune._deploy()
+
+        # Verify the correct method was called based on the model
+        if expected_method == "_deploy_serverless":
+            mock_serverless.assert_called_once()
+            mock_server.assert_not_called()
+        else:
+            mock_serverless.assert_not_called()
+            mock_server.assert_called_once()
+
+        assert result is True
+
+
+async def test_fetch_all_deployments_request_error(fireworks_finetune, mock_api_key):
+    # Test with error response
+    error_response = MagicMock(spec=httpx.Response)
+    error_response.status_code = 500
+    error_response.text = "Internal Server Error"
+
+    with patch("httpx.AsyncClient") as mock_client_class:
+        mock_client = AsyncMock()
+        mock_client.get.side_effect = Exception("API request failed")
+        mock_client_class.return_value.__aenter__.return_value = mock_client
+
+        with pytest.raises(Exception, match="API request failed"):
+            await fireworks_finetune._fetch_all_deployments()
+
+        # Verify API was called with correct parameters
+        mock_client.get.assert_called_once()
+        call_args = mock_client.get.call_args[1]
+        assert "params" in call_args
+        assert call_args["params"]["pageSize"] == 200
+
+
+async def test_fetch_all_deployments_standard_case(fireworks_finetune, mock_api_key):
+    # Test with single page of results
+    mock_deployments = [
+        {"id": "deploy-1", "baseModel": "model-1", "state": "READY"},
+        {"id": "deploy-2", "baseModel": "model-2", "state": "READY"},
+    ]
+
+    success_response = MagicMock(spec=httpx.Response)
+    success_response.status_code = 200
+    success_response.json.return_value = {
+        "deployments": mock_deployments,
+        "nextPageToken": None,
+    }
+
+    with patch("httpx.AsyncClient") as mock_client_class:
+        mock_client = AsyncMock()
+        mock_client.get.return_value = success_response
+        mock_client_class.return_value.__aenter__.return_value = mock_client
+
+        deployments = await fireworks_finetune._fetch_all_deployments()
+
+        # Verify API was called correctly
+        mock_client.get.assert_called_once()
+
+        # Verify correct deployments were returned
+        assert deployments == mock_deployments
+        assert len(deployments) == 2
+        assert deployments[0]["id"] == "deploy-1"
+        assert deployments[1]["id"] == "deploy-2"
+
+
+async def test_fetch_all_deployments_paged_case(fireworks_finetune, mock_api_key):
+    # Test with multiple pages of results
+    mock_deployments_page1 = [
+        {"id": "deploy-1", "baseModel": "model-1", "state": "READY"},
+        {"id": "deploy-2", "baseModel": "model-2", "state": "READY"},
+    ]
+
+    mock_deployments_page2 = [
+        {"id": "deploy-3", "baseModel": "model-3", "state": "READY"},
+        {"id": "deploy-4", "baseModel": "model-4", "state": "READY"},
+    ]
+
+    page1_response = MagicMock(spec=httpx.Response)
+    page1_response.status_code = 200
+    page1_response.json.return_value = {
+        "deployments": mock_deployments_page1,
+        "nextPageToken": "page2token",
+    }
+
+    page2_response = MagicMock(spec=httpx.Response)
+    page2_response.status_code = 200
+    page2_response.json.return_value = {
+        "deployments": mock_deployments_page2,
+        "nextPageToken": None,
+    }
+
+    with patch("httpx.AsyncClient") as mock_client_class:
+        mock_client = AsyncMock()
+        mock_client.get.side_effect = [page1_response, page2_response]
+        mock_client_class.return_value.__aenter__.return_value = mock_client
+
+        deployments = await fireworks_finetune._fetch_all_deployments()
+
+        # Verify API was called twice (once for each page)
+        assert mock_client.get.call_count == 2
+
+        # Verify first call had no page token
+        first_call_args = mock_client.get.call_args_list[0][1]
+        assert "pageToken" not in first_call_args["params"]
+
+        # Verify second call included the page token
+        second_call_args = mock_client.get.call_args_list[1][1]
+        assert second_call_args["params"]["pageToken"] == "page2token"
+
+        # Verify all deployments from both pages were returned
+        assert len(deployments) == 4
+        assert deployments == mock_deployments_page1 + mock_deployments_page2
+        for deployment in deployments:
+            assert deployment["id"] in [
+                "deploy-1",
+                "deploy-2",
+                "deploy-3",
+                "deploy-4",
+            ]
+
+
+async def test_deploy_server_success(fireworks_finetune, mock_api_key):
+    # Mock response for successful deployment
+    success_response = MagicMock(spec=httpx.Response)
+    success_response.status_code = 200
+    success_response.json.return_value = {"baseModel": "model-123"}
+
+    status_response = (
+        FineTuneStatus(status=FineTuneStatusType.completed, message=""),
+        "model-123",
+    )
+
+    with (
+        patch("httpx.AsyncClient") as mock_client_class,
+        patch.object(
+            fireworks_finetune, "model_id_checking_status", return_value="model-123"
+        ),
+    ):
+        mock_client = AsyncMock()
+        mock_client.post.return_value = success_response
+        mock_client_class.return_value.__aenter__.return_value = mock_client
+
+        result = await fireworks_finetune._deploy_server()
+
+        # Verify result
+        assert result is True
+
+        # Verify fine_tune_model_id was updated
+        assert fireworks_finetune.datamodel.fine_tune_model_id == "model-123"
+
+        # Verify API was called with correct parameters
+        mock_client.post.assert_called_once()
+        call_args = mock_client.post.call_args[1]
+        assert "json" in call_args
+        assert call_args["json"]["baseModel"] == "model-123"
+        assert call_args["json"]["minReplicaCount"] == 0
+        assert "autoscalingPolicy" in call_args["json"]
+        assert call_args["json"]["autoscalingPolicy"]["scaleToZeroWindow"] == "300s"
+
+        # load the datamodel from the file and confirm the fine_tune_model_id was updated
+        loaded_datamodel = FinetuneModel.load_from_file(
+            fireworks_finetune.datamodel.path
+        )
+        assert loaded_datamodel.fine_tune_model_id == "model-123"
+
+
+async def test_deploy_server_failure(fireworks_finetune, mock_api_key):
+    # Mock response for failed deployment
+    failure_response = MagicMock(spec=httpx.Response)
+    failure_response.status_code = 500
+    failure_response.text = "Internal Server Error"
+
+    with (
+        patch("httpx.AsyncClient") as mock_client_class,
+        patch.object(
+            fireworks_finetune, "model_id_checking_status", return_value="model-123"
+        ),
+    ):
+        mock_client = AsyncMock()
+        mock_client.post.return_value = failure_response
+        mock_client_class.return_value.__aenter__.return_value = mock_client
+
+        result = await fireworks_finetune._deploy_server()
+
+        # Verify result
+        assert result is False
+
+        # Verify API was called
+        mock_client.post.assert_called_once()
+
+
+async def test_deploy_server_non_200_but_valid_response(
+    fireworks_finetune, mock_api_key
+):
+    # Mock response with non-200 status but valid JSON response
+    mixed_response = MagicMock(spec=httpx.Response)
+    mixed_response.status_code = 200
+    mixed_response.json.return_value = {"not_baseModel": "something-else"}
+
+    with (
+        patch("httpx.AsyncClient") as mock_client_class,
+        patch.object(
+            fireworks_finetune, "model_id_checking_status", return_value="model-123"
+        ),
+    ):
+        mock_client = AsyncMock()
+        mock_client.post.return_value = mixed_response
+        mock_client_class.return_value.__aenter__.return_value = mock_client
+
+        result = await fireworks_finetune._deploy_server()
+
+        # Verify result - should fail because baseModel is missing
+        assert result is False
+
+
+async def test_deploy_server_missing_model_id(fireworks_finetune, mock_api_key):
+    # Test when model_id_checking_status returns None
+    with patch.object(
+        fireworks_finetune, "model_id_checking_status", return_value=None
+    ):
+        result = await fireworks_finetune._deploy_server()
+
+        # Verify result - should fail because model ID is missing
+        assert result is False
+
+
+@pytest.mark.parametrize(
+    "state,expected_already_deployed",
+    [
+        ("READY", True),
+        ("CREATING", True),
+        ("FAILED", False),
+    ],
+)
+async def test_check_or_deploy_server_already_deployed(
+    fireworks_finetune, mock_api_key, state, expected_already_deployed
+):
+    # Test when model is already deployed (should return True without calling _deploy_server)
+
+    # Set a fine_tune_model_id so we search for deployments
+    fireworks_finetune.datamodel.fine_tune_model_id = "model-123"
+
+    # Mock deployments including one matching our model ID
+    mock_deployments = [
+        {"id": "deploy-1", "baseModel": "different-model", "state": "READY"},
+        {"id": "deploy-2", "baseModel": "model-123", "state": state},
+    ]
+
+    with (
+        patch.object(
+            fireworks_finetune, "_fetch_all_deployments", return_value=mock_deployments
+        ) as mock_fetch,
+        patch.object(fireworks_finetune, "_deploy_server") as mock_deploy,
+    ):
+        mock_deploy.return_value = True
+        result = await fireworks_finetune._check_or_deploy_server()
+        # Even true if the model is in a non-ready state, as we'll call deploy (checked below)
+        assert result is True
+
+        if expected_already_deployed:
+            assert mock_deploy.call_count == 0
+        else:
+            assert mock_deploy.call_count == 1
+
+        # Verify _fetch_all_deployments was called
+        mock_fetch.assert_called_once()
+
+
+async def test_check_or_deploy_server_not_deployed(fireworks_finetune, mock_api_key):
+    # Test when model exists but isn't deployed (should call _deploy_server)
+
+    # Set a fine_tune_model_id so we search for deployments
+    fireworks_finetune.datamodel.fine_tune_model_id = "model-123"
+
+    # Mock deployments without our model ID
+    mock_deployments = [
+        {"id": "deploy-1", "baseModel": "different-model-1", "state": "READY"},
+        {"id": "deploy-2", "baseModel": "different-model-2", "state": "READY"},
+    ]
+
+    with (
+        patch.object(
+            fireworks_finetune, "_fetch_all_deployments", return_value=mock_deployments
+        ) as mock_fetch,
+        patch.object(
+            fireworks_finetune, "_deploy_server", return_value=True
+        ) as mock_deploy,
+    ):
+        result = await fireworks_finetune._check_or_deploy_server()
+
+        # Verify method returned True (from _deploy_server)
+        assert result is True
+
+        # Verify _fetch_all_deployments was called
+        mock_fetch.assert_called_once()
+
+        # Verify _deploy_server was called since model is not deployed
+        mock_deploy.assert_called_once()
+
+
+async def test_check_or_deploy_server_no_model_id(fireworks_finetune, mock_api_key):
+    # Test when no fine_tune_model_id exists (should skip fetch and call _deploy_server directly)
+
+    # Ensure no fine_tune_model_id is set
+    fireworks_finetune.datamodel.fine_tune_model_id = None
+
+    with (
+        patch.object(fireworks_finetune, "_fetch_all_deployments") as mock_fetch,
+        patch.object(
+            fireworks_finetune, "_deploy_server", return_value=True
+        ) as mock_deploy,
+    ):
+        result = await fireworks_finetune._check_or_deploy_server()
+
+        # Verify method returned True (from _deploy_server)
+        assert result is True
+
+        # Verify _fetch_all_deployments was NOT called
+        mock_fetch.assert_not_called()
+
+        # Verify _deploy_server was called directly
+        mock_deploy.assert_called_once()
+
+
+async def test_check_or_deploy_server_deploy_fails(fireworks_finetune, mock_api_key):
+    # Test when deployment fails
+
+    # Ensure no fine_tune_model_id is set
+    fireworks_finetune.datamodel.fine_tune_model_id = None
+
+    with (
+        patch.object(
+            fireworks_finetune, "_deploy_server", return_value=False
+        ) as mock_deploy,
+    ):
+        result = await fireworks_finetune._check_or_deploy_server()
+
+        # Verify method returned False (from _deploy_server)
+        assert result is False
+
+        # Verify _deploy_server was called
+        mock_deploy.assert_called_once()
+
+
+async def test_fetch_all_deployments_invalid_json(fireworks_finetune, mock_api_key):
+    # Test with invalid JSON response (missing 'deployments' key)
+    invalid_response = MagicMock(spec=httpx.Response)
+    invalid_response.status_code = 200
+    invalid_response.json.return_value = {
+        "some_other_key": "value",
+        # No 'deployments' key
+    }
+    invalid_response.text = '{"some_other_key": "value"}'
+
+    with patch("httpx.AsyncClient") as mock_client_class:
+        mock_client = AsyncMock()
+        mock_client.get.return_value = invalid_response
+        mock_client_class.return_value.__aenter__.return_value = mock_client
+
+        with pytest.raises(
+            ValueError,
+            match="Invalid response from Fireworks. Expected list of deployments in 'deployments' key",
+        ):
+            await fireworks_finetune._fetch_all_deployments()
+
+        # Verify API was called
+        mock_client.get.assert_called_once()

kiln_ai/adapters/fine_tune/test_together_finetune.py

@@ -356,6 +356,8 @@ async def test_start_success(
             model=together_finetune.datamodel.base_model_id,
             lora=True,
             suffix=f"kiln_ai_{together_finetune.datamodel.id}"[:40],
+            wandb_api_key=Config.shared().wandb_api_key,
+            wandb_project_name="Kiln_AI",
         )
 
         # Check that datamodel was updated correctly

kiln_ai/adapters/fine_tune/together_finetune.py

@@ -130,6 +130,8 @@ class TogetherFinetune(BaseFinetuneAdapter):
             training_file=train_file_id,
             validation_file=validation_file_id,
             model=self.datamodel.base_model_id,
+            wandb_api_key=Config.shared().wandb_api_key,
+            wandb_project_name="Kiln_AI" if Config.shared().wandb_api_key else None,
             **self._build_finetune_parameters(),
         )
 

kiln_ai/adapters/ml_model_list.py

@@ -133,7 +133,7 @@ class KilnModelProvider(BaseModel):
         supports_structured_output: Whether the provider supports structured output formats
         supports_data_gen: Whether the provider supports data generation
         untested_model: Whether the model is untested (typically user added). The supports_ fields are not applicable.
-        provider_finetune_id: The finetune ID for the provider, if applicable
+        provider_finetune_id: The finetune ID for the provider, if applicable. Some providers like Fireworks load these from an API.
         structured_output_mode: The mode we should use to call the model for structured output, if it was trained with structured output.
         parser: A parser to use for the model, if applicable
         reasoning_capable: Whether the model is designed to output thinking in a structured format (eg <think></think>). If so we don't use COT across 2 calls, and ask for thinking and final response in the same call.
@@ -576,7 +576,6 @@ built_in_models: List[KilnModel] = [
                 # JSON mode not ideal (no schema), but tool calling doesn't work on 8b
                 structured_output_mode=StructuredOutputMode.json_instruction_and_object,
                 supports_data_gen=False,
-                provider_finetune_id="accounts/fireworks/models/llama-v3p1-8b-instruct",
                 model_id="accounts/fireworks/models/llama-v3p1-8b-instruct",
             ),
             KilnModelProvider(
@@ -618,7 +617,6 @@ built_in_models: List[KilnModel] = [
                 name=ModelProviderName.fireworks_ai,
                 # Tool calling forces schema -- fireworks doesn't support json_schema, just json_mode
                 structured_output_mode=StructuredOutputMode.function_calling_weak,
-                provider_finetune_id="accounts/fireworks/models/llama-v3p1-70b-instruct",
                 model_id="accounts/fireworks/models/llama-v3p1-70b-instruct",
             ),
             KilnModelProvider(
@@ -764,7 +762,6 @@ built_in_models: List[KilnModel] = [
             ),
             KilnModelProvider(
                 name=ModelProviderName.fireworks_ai,
-                provider_finetune_id="accounts/fireworks/models/llama-v3p2-3b-instruct",
                 supports_structured_output=False,
                 supports_data_gen=False,
                 model_id="accounts/fireworks/models/llama-v3p2-3b-instruct",
@@ -890,8 +887,6 @@ built_in_models: List[KilnModel] = [
             ),
             KilnModelProvider(
                 name=ModelProviderName.fireworks_ai,
-                # Finetuning not live yet
-                # provider_finetune_id="accounts/fireworks/models/llama-v3p3-70b-instruct",
                 # Tool calling forces schema -- fireworks doesn't support json_schema, just json_mode
                 structured_output_mode=StructuredOutputMode.function_calling_weak,
                 model_id="accounts/fireworks/models/llama-v3p3-70b-instruct",

kiln_ai/adapters/provider_tools.py

@@ -197,8 +197,8 @@ def lite_llm_config(
     if provider is None:
         raise ValueError(f"OpenAI compatible provider {openai_provider_name} not found")
 
-    # API key optional some providers don't use it
-    api_key = provider.get("api_key")
+    # API key optional - some providers like Ollama don't use it, but LiteLLM errors without one
+    api_key = provider.get("api_key") or "NA"
     base_url = provider.get("base_url")
     if base_url is None:
         raise ValueError(

kiln_ai/adapters/test_provider_tools.py

@@ -550,14 +550,14 @@ def test_litellm_provider_model_success(mock_shared_config):
 
 
 def test_lite_llm_config_no_api_key(mock_shared_config):
-    """Test provider creation without API key (should work as some providers don't require it)"""
+    """Test provider creation without API key (should work as some providers don't require it, but should pass NA to LiteLLM as it requires one)"""
     model_id = "no_key_provider::gpt-4"
 
     config = lite_llm_config(model_id)
 
     assert config.provider_name == ModelProviderName.openai_compatible
     assert config.model_name == "gpt-4"
-    assert config.additional_body_options == {"api_key": None}
+    assert config.additional_body_options == {"api_key": "NA"}
     assert config.base_url == "https://api.nokey.com"
 
 

kiln_ai/utils/config.py

@@ -119,6 +119,15 @@ class Config:
                 env_var="TOGETHERAI_API_KEY",
                 sensitive=True,
             ),
+            "wandb_api_key": ConfigProperty(
+                str,
+                env_var="WANDB_API_KEY",
+                sensitive=True,
+            ),
+            "wandb_base_url": ConfigProperty(
+                str,
+                env_var="WANDB_BASE_URL",
+            ),
             "custom_models": ConfigProperty(
                 list,
                 default_lambda=lambda: [],

{kiln_ai-0.13.0.dist-info → kiln_ai-0.14.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiln-ai
-Version: 0.13.0
+Version: 0.14.0
 Summary: Kiln AI
 Project-URL: Homepage, https://getkiln.ai
 Project-URL: Repository, https://github.com/Kiln-AI/kiln
@@ -26,7 +26,7 @@ Requires-Dist: pydantic>=2.9.2
 Requires-Dist: pytest-benchmark>=5.1.0
 Requires-Dist: pytest-cov>=6.0.0
 Requires-Dist: pyyaml>=6.0.2
-Requires-Dist: together>=1.4.6
+Requires-Dist: together
 Requires-Dist: typing-extensions>=4.12.2
 Description-Content-Type: text/markdown
 

{kiln_ai-0.13.0.dist-info → kiln_ai-0.14.0.dist-info}/RECORD

@@ -1,17 +1,17 @@
 kiln_ai/__init__.py,sha256=Sc4z8LRVFMwJUoc_DPVUriSXTZ6PO9MaJ80PhRbKyB8,34
 kiln_ai/adapters/__init__.py,sha256=XjGmWagEyOEVwVIAxjN5rYNsQWIEACT5DB7MMTxdPss,1005
-kiln_ai/adapters/adapter_registry.py,sha256=rOIL8xMm_REVO92iDphj_cBwhDOZVyS5ST-nHk4_6pA,8955
-kiln_ai/adapters/ml_model_list.py,sha256=u1nFkJm_UD1IZjBBoynmWnhx_aPkuvSuHVI69Thma3w,58939
+kiln_ai/adapters/adapter_registry.py,sha256=KmMHYQ3mxpjVLE6D-hMNWCGt6Cw9JvnFn6nMb48GE8Y,9166
+kiln_ai/adapters/ml_model_list.py,sha256=f_z1daFR_w4-ccJ4OWwqlIMY0ILFJt4X5LdQb3AMt_c,58592
 kiln_ai/adapters/ollama_tools.py,sha256=uObtLWfqKb9RXHN-TGGw2Y1FQlEMe0u8FgszI0zQn6U,3550
 kiln_ai/adapters/prompt_builders.py,sha256=LYHTIaisQMBFtWDRIGo1QJgOsmQ-NBpQ8fI4eImHxaQ,15269
-kiln_ai/adapters/provider_tools.py,sha256=pVLziYwHQxOZODil3iyn-BnW8CyCYnFsSdjerliWqvs,14671
+kiln_ai/adapters/provider_tools.py,sha256=UL3XEnnxs1TrbqPPxxHSvnL7aBd84ggh38lI0yEsX6A,14725
 kiln_ai/adapters/run_output.py,sha256=RAi2Qp6dmqJVNm3CxbNTdAuhitHfH5NiUGbf6ygUP-k,257
 kiln_ai/adapters/test_adapter_registry.py,sha256=eDLHqv9mwgdde221pa47bTV87vCXwkUyjqsas-iFUrY,6123
 kiln_ai/adapters/test_generate_docs.py,sha256=M-uKcgF3hQmlEFOJ0o7DyL-9RgitGzkfROV-Dxtooec,2770
 kiln_ai/adapters/test_ollama_tools.py,sha256=xAUzL0IVmmXadVehJu1WjqbhpKEYGAgGt3pWx7hrubc,2514
 kiln_ai/adapters/test_prompt_adaptors.py,sha256=J1ZGZ8GG7SxP3_J3Zw0e6XmZY4NyPmUGX3IPgjh2LD8,7767
 kiln_ai/adapters/test_prompt_builders.py,sha256=5Xvfr-oQg_LLrle6UqfpRHWcPUYa8ywG3aL1rM7q1Jw,22054
-kiln_ai/adapters/test_provider_tools.py,sha256=T2t99i4LnH4xilmyV4przff3MnTdDvAnTmaiYhFPShE,26621
+kiln_ai/adapters/test_provider_tools.py,sha256=mzMubpUupQu8pXhjDTj0_Kgrr-xcu_crj9xpcgcAzzA,26671
 kiln_ai/adapters/data_gen/__init__.py,sha256=QTZWaf7kq5BorhPvexJfwDEKmjRmIbhwW9ei8LW2SIs,276
 kiln_ai/adapters/data_gen/data_gen_prompts.py,sha256=kudjHnAz7L3q0k_NLyTlaIV7M0uRFrxXNcfcnjOE2uc,5810
 kiln_ai/adapters/data_gen/data_gen_task.py,sha256=0PuYCcj09BtpgNj23mKj_L45mKZBdV5VreUeZ-Tj_xM,6642
@@ -19,24 +19,24 @@ kiln_ai/adapters/data_gen/test_data_gen_task.py,sha256=cRKUKMvC0uVompbmPTKwbnQ_N
 kiln_ai/adapters/eval/__init__.py,sha256=0ptbK0ZxWuraxGn_WMgmE1tcaq0k5t-g-52kVohvWCg,693
 kiln_ai/adapters/eval/base_eval.py,sha256=jVXMiVBC07ZnLEuZVAjUAYewsnuV99put39n_GZcG1M,7261
 kiln_ai/adapters/eval/eval_runner.py,sha256=h3DvRFM5J5LDJqaLzNJ-q9i5LRycv2J9Ev5nw1mUDUQ,10806
-kiln_ai/adapters/eval/g_eval.py,sha256=muJuolDz3SSJNcruqkMmgo8P3sX5Pzm6XD7vtfiVeX4,14298
+kiln_ai/adapters/eval/g_eval.py,sha256=d3UcBsZWeDt7cWp4uvDcfG7qdGLsGaZEBsIEqkpiWh4,15253
 kiln_ai/adapters/eval/registry.py,sha256=gZ_s0VgEx79Fswkgi1tS4yOl7lzpkvUBJZ62RldhM_w,626
 kiln_ai/adapters/eval/test_base_eval.py,sha256=_1CiOUOiBt1R_gGYMcRblrPkHf-H4uIlvfcHj5-Wh7o,10724
 kiln_ai/adapters/eval/test_eval_runner.py,sha256=82WPE_frNRTSQ2lylqT0inkqcDgM72nWt8GEuoDkJ7w,18568
-kiln_ai/adapters/eval/test_g_eval.py,sha256=XSZ12BEqhzYLOfCg1VFZwu2DkbI9h4kVCqOFgzp9gX8,16163
+kiln_ai/adapters/eval/test_g_eval.py,sha256=-Stx7E0D-WAH1HWrRSp48CiGsf-no1SHeFF9IqVXeMI,16433
 kiln_ai/adapters/eval/test_g_eval_data.py,sha256=8caiZfLWnXVX8alrBPrH7L7gqqSS9vO7u6PzcHurQcA,27769
 kiln_ai/adapters/fine_tune/__init__.py,sha256=DxdTR60chwgck1aEoVYWyfWi6Ed2ZkdJj0lar-SEAj4,257
-kiln_ai/adapters/fine_tune/base_finetune.py,sha256=MxSnBiapWfZQw5UmkYAtC0QXj2zDeF9Ows0k0g3p1IA,6455
+kiln_ai/adapters/fine_tune/base_finetune.py,sha256=ORTclQTQYksMWPu7vNoD7wBzOIqNVK0YOwFEnvsKPWA,5759
 kiln_ai/adapters/fine_tune/dataset_formatter.py,sha256=qRhSSkMhTWn13OMb6LKPVwAU7uY4bB49GDiVSuhDkNg,14449
 kiln_ai/adapters/fine_tune/finetune_registry.py,sha256=CvcEVxtKwjgCMA-oYH9Tpjn1DVWmMzgHpXJOZ0YQA8k,610
-kiln_ai/adapters/fine_tune/fireworks_finetune.py,sha256=6IfTDn_8tg6PR0OFudRx6V7Wjvf4P7t0fm_xyRwII68,13978
+kiln_ai/adapters/fine_tune/fireworks_finetune.py,sha256=OlXp8j6Afwvk6-ySwA3Q7iuqBlKO7VLeAfNCnB3pZPI,19963
 kiln_ai/adapters/fine_tune/openai_finetune.py,sha256=Dz9E_0BWfrIkvv8ArZe-RKPwbIKPZ3v8rfbc3JELyTY,8571
-kiln_ai/adapters/fine_tune/test_base_finetune.py,sha256=0zWxFYrDGVuoQNQmi9vVUEkBc4mstfHnsUjQmiJA-sE,10864
+kiln_ai/adapters/fine_tune/test_base_finetune.py,sha256=sjuDgJDA_dynGRelx9_wXdssaxAYIuEG-Z8NzRx9Hl0,10559
 kiln_ai/adapters/fine_tune/test_dataset_formatter.py,sha256=T3jbFZooLVBaGCE0LUVxwPxzM3l8IY41zUj3jPk-Zi8,24027
-kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py,sha256=e88z5-KtU9Y7frXqCVy6r6iE6S-tInn5oMOqmSnbR2I,18144
+kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py,sha256=oLyLEG4TwW452lV2mvUo-wImLxzSwOuoKKeYFuGh3k8,36744
 kiln_ai/adapters/fine_tune/test_openai_finetune.py,sha256=H63Xk2PNHbt5Ev5IQpdR9JZ4uz-Huo2gfuC4mHHqe0w,20011
-kiln_ai/adapters/fine_tune/test_together_finetune.py,sha256=YHtOHi2bsZcUDVHY62DTRKF3ZQry3Dv-qeW8povyR1o,17991
-kiln_ai/adapters/fine_tune/together_finetune.py,sha256=kp-60ClwUimtjhRDsvPErS5whLDsBHGZhw8ITQ9JWCM,13868
+kiln_ai/adapters/fine_tune/test_together_finetune.py,sha256=BUJFsyq_g77gU0JN3hg6FMBvqb0DIyTeAek-wxomKIg,18090
+kiln_ai/adapters/fine_tune/together_finetune.py,sha256=EbMPsTyKMubfwOalkFLiNFlMFIRKxLibzMTyLeUkle4,14010
 kiln_ai/adapters/model_adapters/__init__.py,sha256=m5GRtOHwVVvp_XDOss8c1X3NFf1wQQlC2eBgI4tXQhM,212
 kiln_ai/adapters/model_adapters/base_adapter.py,sha256=ifPJMg0nEKamfOSmBIsnp_MRFfBs47FLeQrLbav34yA,9872
 kiln_ai/adapters/model_adapters/litellm_adapter.py,sha256=c4J_tIpM96KWS2qzoPaQmBj7X7mHyRMShdkmEh7_EHM,16129
@@ -89,7 +89,7 @@ kiln_ai/datamodel/test_prompt_id.py,sha256=ihyXVPQi0dSLGnBM7rTXRnVaiWXhh7HJmSy4n
 kiln_ai/datamodel/test_registry.py,sha256=PhS4anLi5Bf_023obuTlO5DALhtPB8WIc_bX12Yg6Po,2705
 kiln_ai/datamodel/test_task.py,sha256=FYyoEqJXQIy8rcBsLTdki4-1z9COnZQk1-aoS3ZoNuU,5307
 kiln_ai/utils/__init__.py,sha256=PTD0MwBCKAMIOGsTAwsFaJOusTJJoRFTfOGqRvCaU-E,142
-kiln_ai/utils/config.py,sha256=IKPk9dMecDbqoBkfj52GT9xeity_y_sGP-vdm9CcKe4,8201
+kiln_ai/utils/config.py,sha256=kAgb_4nSnb-IWbLVS4FBygYmnHypJADjDEk_Fh-eLeg,8479
 kiln_ai/utils/dataset_import.py,sha256=HvTCdK9OO8WE3Runn8_Vsks5KpCTckGIzAA7JKe-cWI,6956
 kiln_ai/utils/exhaustive_error.py,sha256=TkkRixIAR3CPEKHeAJzyv0mtxp6BxUBKMvobA3vzQug,262
 kiln_ai/utils/formatting.py,sha256=VtB9oag0lOGv17dwT7OPX_3HzBfaU9GsLH-iLete0yM,97
@@ -97,7 +97,7 @@ kiln_ai/utils/name_generator.py,sha256=v26TgpCwQbhQFcZvzgjZvURinjrOyyFhxpsI6NQrH
 kiln_ai/utils/test_config.py,sha256=Jw3nMFeIgZUsZDRJJY2HpB-2EkR2NoZ-rDe_o9oA7ws,9174
 kiln_ai/utils/test_dataset_import.py,sha256=ZZOt7zqtaEIlMMx0VNXyRegDvnVqbWY2bcz-iMY_Oag,17427
 kiln_ai/utils/test_name_geneator.py,sha256=9-hSTBshyakqlPbFnNcggwLrL7lcPTitauBYHg9jFWI,1513
-kiln_ai-0.13.0.dist-info/METADATA,sha256=csSjOcBVu_PrMLFU16V54GmCmibcfCoeCWV0fWydbEA,12238
-kiln_ai-0.13.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kiln_ai-0.13.0.dist-info/licenses/LICENSE.txt,sha256=_NA5pnTYgRRr4qH6lE3X-TuZJ8iRcMUi5ASoGr-lEx8,1209
-kiln_ai-0.13.0.dist-info/RECORD,,
+kiln_ai-0.14.0.dist-info/METADATA,sha256=EjgZOnknE7P9uW5BsIFJZYQAN-aUQ817SAEXjtqtjK0,12231
+kiln_ai-0.14.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kiln_ai-0.14.0.dist-info/licenses/LICENSE.txt,sha256=_NA5pnTYgRRr4qH6lE3X-TuZJ8iRcMUi5ASoGr-lEx8,1209
+kiln_ai-0.14.0.dist-info/RECORD,,