eval-studio-client 1.2.2a1__py3-none-any.whl → 1.2.2a2__py3-none-any.whl

eval_studio_client/api/models/v1_repeated_context.py

@@ -72,9 +72,9 @@ class V1RepeatedContext(BaseModel):
         # override the default output from pydantic by calling `to_dict()` of each item in content (list)
         _items = []
         if self.content:
-            for _item in self.content:
-                if _item:
-                    _items.append(_item.to_dict())
+            for _item_content in self.content:
+                if _item_content:
+                    _items.append(_item_content.to_dict())
             _dict['content'] = _items
         return _dict
 
@@ -85,11 +85,11 @@ class V1RepeatedContext(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "content": [V1Context.from_dict(_item) for _item in obj["content"]] if obj.get("content") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_repeated_string.py

@@ -77,11 +77,11 @@ class V1RepeatedString(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "content": obj.get("content")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_reset_workflow_node_response.py

@@ -81,11 +81,11 @@ class V1ResetWorkflowNodeResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "node": V1WorkflowNode.from_dict(obj["node"]) if obj.get("node") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_role_binding.py

@@ -80,13 +80,13 @@ class V1RoleBinding(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "subject": obj.get("subject"),
             "role": obj.get("role"),
             "resource": obj.get("resource")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_stats.py

@@ -94,13 +94,13 @@ class V1Stats(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "topicModelingPendingJobs": obj.get("topicModelingPendingJobs"),
             "testValidationPendingJobs": obj.get("testValidationPendingJobs"),
             "failureClusteringPendingJobs": obj.get("failureClusteringPendingJobs")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_test.py

@@ -107,7 +107,7 @@ class V1Test(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "name": obj.get("name"),
@@ -123,7 +123,7 @@ class V1Test(BaseModel):
             "tags": obj.get("tags"),
             "demo": obj.get("demo"),
             "type": obj.get("type")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_test_case.py

@@ -111,7 +111,7 @@ class V1TestCase(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "name": obj.get("name"),
@@ -129,7 +129,7 @@ class V1TestCase(BaseModel):
             "perturbedBy": obj.get("perturbedBy"),
             "topics": obj.get("topics"),
             "generator": obj.get("generator")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_test_case_relationship.py

@@ -79,13 +79,13 @@ class V1TestCaseRelationship(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "source": obj.get("source"),
             "target": obj.get("target"),
             "type": obj.get("type")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_test_class.py

@@ -105,7 +105,7 @@ class V1TestClass(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "name": obj.get("name"),
@@ -121,7 +121,7 @@ class V1TestClass(BaseModel):
             "recommendedTests": obj.get("recommendedTests"),
             "testClassType": obj.get("testClassType"),
             "tags": obj.get("tags")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_test_lab.py

@@ -115,7 +115,7 @@ class V1TestLab(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "name": obj.get("name"),
@@ -131,7 +131,7 @@ class V1TestLab(BaseModel):
             "llmModels": obj.get("llmModels"),
             "modelParameters": obj.get("modelParameters"),
             "h2ogpteCollection": obj.get("h2ogpteCollection")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_update_dashboard_response.py

@@ -81,11 +81,11 @@ class V1UpdateDashboardResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "dashboard": V1Dashboard.from_dict(obj["dashboard"]) if obj.get("dashboard") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_update_document_response.py

@@ -81,11 +81,11 @@ class V1UpdateDocumentResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "document": V1Document.from_dict(obj["document"]) if obj.get("document") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_update_leaderboard_response.py

@@ -81,11 +81,11 @@ class V1UpdateLeaderboardResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "leaderboard": V1Leaderboard.from_dict(obj["leaderboard"]) if obj.get("leaderboard") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_update_model_response.py

@@ -81,11 +81,11 @@ class V1UpdateModelResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "model": V1Model.from_dict(obj["model"]) if obj.get("model") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_update_operation_response.py

@@ -81,11 +81,11 @@ class V1UpdateOperationResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "operation": V1Operation.from_dict(obj["operation"]) if obj.get("operation") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_update_test_case_response.py

@@ -81,11 +81,11 @@ class V1UpdateTestCaseResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "testCase": V1TestCase.from_dict(obj["testCase"]) if obj.get("testCase") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_update_test_response.py

@@ -81,11 +81,11 @@ class V1UpdateTestResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "test": V1Test.from_dict(obj["test"]) if obj.get("test") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_update_workflow_node_response.py

@@ -81,11 +81,11 @@ class V1UpdateWorkflowNodeResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "node": V1WorkflowNode.from_dict(obj["node"]) if obj.get("node") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_update_workflow_response.py

@@ -81,11 +81,11 @@ class V1UpdateWorkflowResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "workflow": V1Workflow.from_dict(obj["workflow"]) if obj.get("workflow") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_who_am_i_response.py

@@ -79,13 +79,13 @@ class V1WhoAmIResponse(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "sub": obj.get("sub"),
             "preferredUsername": obj.get("preferredUsername"),
             "email": obj.get("email")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_workflow.py

@@ -117,9 +117,9 @@ class V1Workflow(BaseModel):
         # override the default output from pydantic by calling `to_dict()` of each value in output_artifacts (dict)
         _field_dict = {}
         if self.output_artifacts:
-            for _key in self.output_artifacts:
-                if self.output_artifacts[_key]:
-                    _field_dict[_key] = self.output_artifacts[_key].to_dict()
+            for _key_output_artifacts in self.output_artifacts:
+                if self.output_artifacts[_key_output_artifacts]:
+                    _field_dict[_key_output_artifacts] = self.output_artifacts[_key_output_artifacts].to_dict()
             _dict['outputArtifacts'] = _field_dict
         return _dict
 
@@ -130,7 +130,7 @@ class V1Workflow(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "name": obj.get("name"),
@@ -158,7 +158,7 @@ class V1Workflow(BaseModel):
             "document": obj.get("document"),
             "h2ogpteCollection": obj.get("h2ogpteCollection"),
             "clonedFromWorkflow": obj.get("clonedFromWorkflow")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_workflow_dependency.py

@@ -78,12 +78,12 @@ class V1WorkflowDependency(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "name": obj.get("name"),
             "optional": obj.get("optional")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_workflow_edge.py

@@ -103,7 +103,7 @@ class V1WorkflowEdge(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "name": obj.get("name"),
@@ -117,7 +117,7 @@ class V1WorkflowEdge(BaseModel):
             "type": obj.get("type"),
             "from": obj.get("from"),
             "to": obj.get("to")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_workflow_node.py

@@ -119,9 +119,9 @@ class V1WorkflowNode(BaseModel):
         # override the default output from pydantic by calling `to_dict()` of each item in output_artifacts (list)
         _items = []
         if self.output_artifacts:
-            for _item in self.output_artifacts:
-                if _item:
-                    _items.append(_item.to_dict())
+            for _item_output_artifacts in self.output_artifacts:
+                if _item_output_artifacts:
+                    _items.append(_item_output_artifacts.to_dict())
             _dict['outputArtifacts'] = _items
         # override the default output from pydantic by calling `to_dict()` of attributes
         if self.attributes:
@@ -135,7 +135,7 @@ class V1WorkflowNode(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "name": obj.get("name"),
@@ -157,7 +157,7 @@ class V1WorkflowNode(BaseModel):
             "processedByOperation": obj.get("processedByOperation"),
             "resultStatus": obj.get("resultStatus"),
             "stale": obj.get("stale")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_workflow_node_artifact.py

@@ -105,7 +105,7 @@ class V1WorkflowNodeArtifact(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "name": obj.get("name"),
@@ -120,7 +120,7 @@ class V1WorkflowNodeArtifact(BaseModel):
             "deleter": obj.get("deleter"),
             "mimeType": obj.get("mimeType"),
             "type": obj.get("type")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_workflow_node_artifacts.py

@@ -74,9 +74,9 @@ class V1WorkflowNodeArtifacts(BaseModel):
         # override the default output from pydantic by calling `to_dict()` of each item in artifacts (list)
         _items = []
         if self.artifacts:
-            for _item in self.artifacts:
-                if _item:
-                    _items.append(_item.to_dict())
+            for _item_artifacts in self.artifacts:
+                if _item_artifacts:
+                    _items.append(_item_artifacts.to_dict())
             _dict['artifacts'] = _items
         return _dict
 
@@ -87,11 +87,11 @@ class V1WorkflowNodeArtifacts(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "artifacts": [V1WorkflowNodeArtifact.from_dict(_item) for _item in obj["artifacts"]] if obj.get("artifacts") is not None else None
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/v1_workflow_node_attributes.py

@@ -77,11 +77,11 @@ class V1WorkflowNodeAttributes(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "canDelete": obj.get("canDelete")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/workflow_service_clone_workflow_request.py

@@ -81,7 +81,7 @@ class WorkflowServiceCloneWorkflowRequest(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "displayNameOverride": obj.get("displayNameOverride"),
@@ -89,7 +89,7 @@ class WorkflowServiceCloneWorkflowRequest(BaseModel):
             "modelOverride": obj.get("modelOverride"),
             "llmModelOverride": obj.get("llmModelOverride"),
             "modelParametersOverride": obj.get("modelParametersOverride")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/api/models/workflow_service_revoke_workflow_access_request.py

@@ -79,12 +79,12 @@ class WorkflowServiceRevokeWorkflowAccessRequest(BaseModel):
             return None
 
         if not isinstance(obj, dict):
-            return cls.model_validate(obj)
+            return cls.model_validate(obj, strict=False)
 
         _obj = cls.model_validate({
             "subject": obj.get("subject"),
             "role": obj.get("role")
-        })
+        }, strict=False)
         return _obj
 
 

eval_studio_client/leaderboards.py

@@ -36,6 +36,7 @@ class Leaderboard:
     insights: List[i6s.Insight] = dataclasses.field(default_factory=list)
     summary: Optional[str] = None
     existing_collection: Optional[str] = None
+    _model_parameters: Optional[str] = None
     _report: Optional[str] = None
     _leaderboard: Optional[str] = None
     _model_name: Optional[str] = None
@@ -83,6 +84,20 @@ class Leaderboard:
 
         return []
 
+    @property
+    def model_parameters(self) -> dict:
+        """Retrieves the model parameters (overrides) used in this leaderboard."""
+        if self._model_parameters:
+            if isinstance(self._model_parameters, str):
+                try:
+                    return json.loads(self._model_parameters)
+                except json.JSONDecodeError:
+                    print("Failed to parse model parameters")
+                    return {}
+            elif isinstance(self._model_parameters, dict):
+                return self._model_parameters
+        return {}
+
     def delete(self):
         """Deletes the leaderboard."""
         if self._client:
@@ -226,6 +241,7 @@ class Leaderboard:
             tests=self._test_names,
             model=self._model_name,
             h2ogpte_collection=self.existing_collection or None,
+            model_parameters=self._model_parameters,
         )
 
     def _update_result(self, api_leaderboard: models.V1Leaderboard):
@@ -264,6 +280,7 @@ class Leaderboard:
             _status=api_leaderboard.status,
             _client=client,
             _operation=api_leaderboard.create_operation or None,
+            _model_parameters=api_leaderboard.model_parameters,
         )
 
     @staticmethod