kiln-ai 0.13.0__tar.gz → 0.13.2__tar.gz

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kiln-ai
-Version: 0.13.0
+Version: 0.13.2
 Summary: Kiln AI
 Project-URL: Homepage, https://getkiln.ai
 Project-URL: Repository, https://github.com/Kiln-AI/kiln
@@ -26,7 +26,7 @@ Requires-Dist: pydantic>=2.9.2
 Requires-Dist: pytest-benchmark>=5.1.0
 Requires-Dist: pytest-cov>=6.0.0
 Requires-Dist: pyyaml>=6.0.2
-Requires-Dist: together>=1.4.6
+Requires-Dist: together
 Requires-Dist: typing-extensions>=4.12.2
 Description-Content-Type: text/markdown
 

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/adapters/adapter_registry.py

@@ -108,6 +108,10 @@ def adapter_for_task(
                     # 1. To use the correct base URL
                     # 2. We use Ollama's OpenAI compatible API (/v1), and don't just let litellm use the Ollama API. We use more advanced features like json_schema.
                     base_url=ollama_base_url + "/v1",
+                    additional_body_options={
+                        # LiteLLM errors without an api_key, even though Ollama doesn't support one.
+                        "api_key": "NA",
+                    },
                 ),
             )
         case ModelProviderName.fireworks_ai:

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/adapters/eval/g_eval.py

@@ -297,9 +297,12 @@ The model produced the following output for the task:
 
         total_score = 0.0
         total_probability = 0.0
+        top_logprobs_contains_primary_token = False
 
-        # Process all valid scoring tokens
+        # Process all valid scoring tokens from alternatives
         for top_logprob in token_logprob.top_logprobs:
+            if top_logprob.token == token_logprob.token:
+                top_logprobs_contains_primary_token = True
             token_score = self.score_from_token_string(top_logprob.token)
             if token_score is not None:
                 # Convert logprob to probability
@@ -307,9 +310,21 @@ The model produced the following output for the task:
                 total_score += token_score * probability
                 total_probability += probability
 
+        # Weird OpenAI 4o bug - sometimes the primary token is included in the top logprobs, sometimes not.
+        # Add the primary token back in if excluded
+        if not top_logprobs_contains_primary_token:
+            if token_logprob.logprob == -9999.0:
+                # Another "bug" - sometimes the logprob is -9999.0. This seems to happen when the rest of the logprobs are tiny probability.
+                total_score += primary_token_score * 1.0
+                total_probability += 1.0
+            else:
+                probability = math.exp(token_logprob.logprob)
+                total_score += primary_token_score * probability
+                total_probability += probability
+
         if total_probability <= 0.0:
             raise RuntimeError(
-                f"No valid scoring tokens found for {token_logprob.token}. This should never happen. Please file a bug if you see this."
+                f"No valid scoring tokens found for {token_logprob.token}. This should never happen as the token has a valid score (so it must be excluded from top logprobs). Please file a bug if you see this."
             )
 
         # Normalize by total probability of valid tokens (LLM may have wanted to generate other non-rating tokens, these shouldn't lower score of rating tokens)

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/adapters/eval/test_g_eval.py

@@ -393,12 +393,13 @@ def test_rating_token_to_score(test_eval_config, test_run_config):
             self.logprob = logprob
 
     class MockTokenLogprob:
-        def __init__(self, token, top_logprobs):
+        def __init__(self, token, top_logprobs, logprob):
             self.token = token
             self.top_logprobs = [MockTopLogprob(t, lp) for t, lp in top_logprobs]
+            self.logprob = logprob
 
     # Test single token case
-    token_logprob = MockTokenLogprob("5", [("5", 0.0)])  # log(1) = 0
+    token_logprob = MockTokenLogprob("5", [("5", 0.0)], logprob=1e-8)  # log(1) = 0
     score = g_eval.rating_token_to_score(token_logprob)
     assert score == 5.0
 
@@ -409,18 +410,22 @@ def test_rating_token_to_score(test_eval_config, test_run_config):
             ("4", math.log(0.6)),  # 60% probability
             ("5", math.log(0.4)),  # 40% probability
         ],
+        logprob=math.log(0.6),
     )
     score = g_eval.rating_token_to_score(token_logprob)
     assert pytest.approx(score) == 4.4  # (4 * 0.6 + 5 * 0.4)
 
     # Test invalid token
-    token_logprob = MockTokenLogprob(":", [(":", 0.0)])
+    token_logprob = MockTokenLogprob(":", [(":", 0.0)], logprob=1e-8)
     assert g_eval.rating_token_to_score(token_logprob) is None
 
-    # Test no valid scoring tokens
-    token_logprob = MockTokenLogprob("5", [])
-    with pytest.raises(RuntimeError, match="No valid scoring tokens found"):
-        g_eval.rating_token_to_score(token_logprob)
+    # Test missing from top logprobs
+    token_logprob = MockTokenLogprob("5", [], logprob=1e-8)
+    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0
+
+    # Test missing from top logprobs, with special case logprob
+    token_logprob = MockTokenLogprob("5", [], logprob=-9999)
+    assert pytest.approx(g_eval.rating_token_to_score(token_logprob)) == 5.0
 
 
 def test_g_eval_system_instruction():

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/adapters/fine_tune/fireworks_finetune.py

@@ -132,11 +132,18 @@ class FireworksFinetune(BaseFinetuneAdapter):
                 :60
             ]
         )
-        payload = {
+        payload: dict[str, str | dict[str, str | bool]] = {
             "dataset": f"accounts/{account_id}/datasets/{train_file_id}",
             "displayName": display_name,
             "baseModel": self.datamodel.base_model_id,
         }
+        # Add W&B config if API key is set
+        if Config.shared().wandb_api_key:
+            payload["wandbConfig"] = {
+                "enabled": True,
+                "project": "Kiln_AI",
+                "apiKey": Config.shared().wandb_api_key,
+            }
         hyperparameters = self.create_payload_parameters(self.datamodel.parameters)
         payload.update(hyperparameters)
         headers = {

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/adapters/fine_tune/test_fireworks_tinetune.py

@@ -340,6 +340,7 @@ async def test_start_success(
     expected_mode,
     expected_format,
 ):
+    Config.shared().wandb_api_key = "test-api-key"
     mock_task.output_json_schema = output_schema
 
     fireworks_finetune.datamodel.parent = mock_task
@@ -378,6 +379,24 @@ async def test_start_success(
         assert fireworks_finetune.datamodel.structured_output_mode == expected_mode
         assert fireworks_finetune.datamodel.properties["endpoint_version"] == "v2"
 
+        # check mockclent.post call values
+        assert mock_client.post.call_count == 1
+        submit_call_values = mock_client.post.call_args[1]
+        assert submit_call_values["json"]["wandbConfig"] == {
+            "enabled": True,
+            "project": "Kiln_AI",
+            "apiKey": "test-api-key",
+        }
+        assert submit_call_values["json"]["baseModel"] == "llama-v2-7b"
+        assert (
+            submit_call_values["json"]["dataset"]
+            == f"accounts/{Config.shared().fireworks_account_id}/datasets/{mock_dataset_id}"
+        )
+        assert (
+            submit_call_values["json"]["displayName"]
+            == f"Kiln AI fine-tuning [ID:{fireworks_finetune.datamodel.id}][name:{fireworks_finetune.datamodel.name}]"
+        )
+
 
 async def test_start_api_error(
     fireworks_finetune, mock_dataset, mock_task, mock_api_key

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/adapters/fine_tune/test_together_finetune.py

@@ -356,6 +356,8 @@ async def test_start_success(
             model=together_finetune.datamodel.base_model_id,
             lora=True,
             suffix=f"kiln_ai_{together_finetune.datamodel.id}"[:40],
+            wandb_api_key=Config.shared().wandb_api_key,
+            wandb_project_name="Kiln_AI",
         )
 
         # Check that datamodel was updated correctly

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/adapters/fine_tune/together_finetune.py

@@ -130,6 +130,8 @@ class TogetherFinetune(BaseFinetuneAdapter):
             training_file=train_file_id,
             validation_file=validation_file_id,
             model=self.datamodel.base_model_id,
+            wandb_api_key=Config.shared().wandb_api_key,
+            wandb_project_name="Kiln_AI" if Config.shared().wandb_api_key else None,
             **self._build_finetune_parameters(),
         )
 

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/adapters/provider_tools.py

@@ -197,8 +197,8 @@ def lite_llm_config(
     if provider is None:
         raise ValueError(f"OpenAI compatible provider {openai_provider_name} not found")
 
-    # API key optional some providers don't use it
-    api_key = provider.get("api_key")
+    # API key optional - some providers like Ollama don't use it, but LiteLLM errors without one
+    api_key = provider.get("api_key") or "NA"
     base_url = provider.get("base_url")
     if base_url is None:
         raise ValueError(

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/adapters/test_provider_tools.py

@@ -550,14 +550,14 @@ def test_litellm_provider_model_success(mock_shared_config):
 
 
 def test_lite_llm_config_no_api_key(mock_shared_config):
-    """Test provider creation without API key (should work as some providers don't require it)"""
+    """Test provider creation without API key (should work as some providers don't require it, but should pass NA to LiteLLM as it requires one)"""
     model_id = "no_key_provider::gpt-4"
 
     config = lite_llm_config(model_id)
 
     assert config.provider_name == ModelProviderName.openai_compatible
     assert config.model_name == "gpt-4"
-    assert config.additional_body_options == {"api_key": None}
+    assert config.additional_body_options == {"api_key": "NA"}
     assert config.base_url == "https://api.nokey.com"
 
 

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/kiln_ai/utils/config.py

@@ -119,6 +119,15 @@ class Config:
                 env_var="TOGETHERAI_API_KEY",
                 sensitive=True,
             ),
+            "wandb_api_key": ConfigProperty(
+                str,
+                env_var="WANDB_API_KEY",
+                sensitive=True,
+            ),
+            "wandb_base_url": ConfigProperty(
+                str,
+                env_var="WANDB_BASE_URL",
+            ),
             "custom_models": ConfigProperty(
                 list,
                 default_lambda=lambda: [],

{kiln_ai-0.13.0 → kiln_ai-0.13.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kiln-ai"
-version = "0.13.0"
+version = "0.13.2"
 requires-python = ">=3.10"
 readme = "README.md"
 description = 'Kiln AI'
@@ -29,7 +29,7 @@ dependencies = [
     "pytest-benchmark>=5.1.0",
     "pytest-cov>=6.0.0",
     "pyyaml>=6.0.2",
-    "together>=1.4.6",
+    "together",
     "typing-extensions>=4.12.2",
 ]
 
@@ -50,9 +50,13 @@ build-backend = "hatchling.build"
 [tool.hatch.metadata]
 requires-python = ">=3.10"
 
+[tool.uv.sources]
+together = { git = "https://github.com/scosman/together-python" }
+
 
 [project.urls]
 Homepage = "https://getkiln.ai"
 Repository = "https://github.com/Kiln-AI/kiln"
 Documentation = "https://kiln-ai.github.io/Kiln/kiln_core_docs/kiln_ai.html"
 Issues = "https://github.com/Kiln-AI/kiln/issues"
+