eval-studio-client 1.2.1a3__py3-none-any.whl → 1.2.2__py3-none-any.whl

eval_studio_client/models.py

@@ -67,6 +67,7 @@ class Model:
     is_rag: bool
     create_time: Optional[datetime.datetime] = None
     update_time: Optional[datetime.datetime] = None
+    _model_parameters: Optional[str] = None
     _client: Optional[api.ApiClient] = None
 
     def __post_init__(self):
@@ -180,6 +181,7 @@ class Model:
         description: Optional[str] = None,
         base_models: Optional[List[str]] = None,
         existing_collection: Optional[str] = None,
+        model_parameters: Optional[Union[dict, str]] = None,
     ) -> Optional[d8s.Dashboard]:
         """Runs a new evaluation for the model and creates a new dashboard.
 
@@ -191,6 +193,10 @@ class Model:
             existing_collection (str): ID or the resource name of the existing
                 collection, which will be used as a corpus for evaluation.
                 NOTE: This option works only for the H2OGPTe and Amazon Bedrock model hosts ATM.
+            model_parameters (optional): Optional override of the model parameters.
+                This can be either a JSON string or a dictionary.
+                Examples:
+                    model_parameters={"llm_args": {"temperature": 0.7, "use_agent": False}}
         """
         _evaluators = (
             [evaluators] if isinstance(evaluators, e8s.Evaluator) else evaluators
@@ -198,6 +204,21 @@ class Model:
         _test_suites = (
             [test_suites] if isinstance(test_suites, tests.Test) else test_suites
         )
+        _model_parameters = None
+        if model_parameters:
+            if isinstance(model_parameters, str):
+                _model_parameters = model_parameters
+            elif isinstance(model_parameters, dict):
+                try:
+                    _model_parameters = json.dumps(model_parameters)
+                except (TypeError, ValueError) as err:
+                    raise ValueError("Invalid dictionary for model_parameters") from err
+        else:
+            _model_parameters = (
+                self._model_parameters
+                if isinstance(self._model_parameters, str)
+                else None
+            )
 
         create_lb_reqs: List[models.V1CreateLeaderboardRequest] = []
         for evaluator in _evaluators:
@@ -207,6 +228,7 @@ class Model:
                 description=description or "",
                 base_models=base_models or [],
                 existing_collection=existing_collection,
+                _model_parameters=_model_parameters,
                 _model_name=self.key,
                 _evaluator_name=evaluator.key,
                 _test_names=[t.key for t in _test_suites],
@@ -261,6 +283,24 @@ class Model:
 
         return None
 
+    @property
+    def model_parameters(self) -> dict:
+        """Retrieves the model parameters as a dictionary.
+
+        Returns:
+            dict: The model parameters as a dictionary. Returns empty dict if no parameters are set.
+        """
+        if self._model_parameters:
+            if isinstance(self._model_parameters, str):
+                try:
+                    return json.loads(self._model_parameters)
+                except json.JSONDecodeError:
+                    print("Failed to parse model parameters")
+                    return {}
+            elif isinstance(self._model_parameters, dict):
+                return self._model_parameters
+        return {}
+
     def delete(self):
         """Deletes the model"""
         self._model_api.model_service_delete_model(self.key)
@@ -286,6 +326,7 @@ class Model:
             create_time=api_model.create_time,
             update_time=api_model.update_time,
             _client=client,
+            _model_parameters=api_model.parameters,
         )
 
     @staticmethod
@@ -342,7 +383,13 @@ class _Models:
         return self.get(DEFAULT_LLM_MODEL_KEY)
 
     def create_h2ogpte_model(
-        self, name: str, is_rag: bool, description: str, url: str, api_key: str
+        self,
+        name: str,
+        is_rag: bool,
+        description: str,
+        url: str,
+        api_key: str,
+        model_parameters: Optional[Union[dict, str]] = None,
     ) -> Model:
         """Creates a new H2OGPTe model in Eval Studio.
 
@@ -355,18 +402,24 @@ class _Models:
             description: Description of the model.
             url: URL of the model host system.
             api_key: API key for the model host system.
+            model_parameters (optional): Optional model parameters.
+                This can be either a JSON string or a dictionary.
+                Examples:
+                    model_parameters={"llm_args": {"temperature": 0.7}}
         """
         model_type = (
             models.V1ModelType.MODEL_TYPE_H2_OGPTE_RAG
             if is_rag
             else models.V1ModelType.MODEL_TYPE_H2_OGPTE_LLM
         )
+
         req = models.V1Model(
             display_name=name,
             description=description,
             url=url,
             api_key=api_key,
             type=model_type,
+            parameters=self._serialize_model_parameters(model_parameters),
         )
         res = self._api.model_service_create_model(req)
         if res and res.model:
@@ -375,7 +428,12 @@ class _Models:
         raise RuntimeError("Failed to create H2OGPTe model")
 
     def create_h2ogpt_model(
-        self, name: str, description: str, url: str, api_key: str
+        self,
+        name: str,
+        description: str,
+        url: str,
+        api_key: str,
+        model_parameters: Optional[Union[dict, str]] = None,
     ) -> Model:
         """Creates a new H2OGPT model in Eval Studio.
 
@@ -384,13 +442,19 @@ class _Models:
             description: Description of the model.
             url: URL of the model host system.
             api_key: API key for the model host system.
+            model_parameters (optional): Optional model parameters.
+                This can be either a JSON string or a dictionary.
+                Examples:
+                    model_parameters={"llm_args": {"temperature": 0.7}}
         """
+
         req = models.V1Model(
             display_name=name,
             description=description,
             url=url,
             api_key=api_key,
             type=models.V1ModelType.MODEL_TYPE_H2_OGPT_LLM,
+            parameters=self._serialize_model_parameters(model_parameters),
         )
         res = self._api.model_service_create_model(req)
         if res and res.model:
@@ -399,7 +463,12 @@ class _Models:
         raise RuntimeError("Failed to create H2OGPT model")
 
     def create_h2o_llmops_model(
-        self, name: str, description: str, url: str, api_key: str
+        self,
+        name: str,
+        description: str,
+        url: str,
+        api_key: str,
+        model_parameters: Optional[Union[dict, str]] = None,
     ) -> Model:
         """Creates a new H2O LLMOps Model.
 
@@ -408,6 +477,10 @@ class _Models:
             description: Description of the model.
             url: URL of the model host system.
             api_key: API key for the model host system.
+            model_parameters (optional): Optional model parameters.
+                This can be either a JSON string or a dictionary.
+                Examples:
+                    model_parameters={"llm_args": {"temperature": 0.7}}
         """
         req = models.V1Model(
             display_name=name,
@@ -415,6 +488,7 @@ class _Models:
             url=url,
             api_key=api_key,
             type=models.V1ModelType.MODEL_TYPE_H2_OLLMOPS,
+            parameters=self._serialize_model_parameters(model_parameters),
         )
         res = self._api.model_service_create_model(req)
         if res and res.model:
@@ -429,6 +503,7 @@ class _Models:
         api_key: str,
         url: str = "",
         is_rag: bool = True,
+        model_parameters: Optional[Union[dict, str]] = None,
     ) -> Model:
         """Creates a new OpenAI model in Eval Studio.
 
@@ -440,6 +515,10 @@ class _Models:
                 Otherwise can use custom OpenAI compatible API.
             is_rag (optional): If True, uses the OpenAI Assistants API for RAG.
                 If False, uses plain OpenAI Chat.
+            model_parameters (optional): Optional model parameters.
+                This can be either a JSON string or a dictionary.
+                Examples:
+                    model_parameters={"llm_args": {"temperature": 0.7}}
 
         """
         if url and is_rag:
@@ -458,6 +537,7 @@ class _Models:
             api_key=api_key,
             url=url or None,
             type=model_type,
+            parameters=self._serialize_model_parameters(model_parameters),
         )
         res = self._api.model_service_create_model(req)
         if res and res.model:
@@ -466,7 +546,13 @@ class _Models:
         raise RuntimeError("Failed to create OpenAI model")
 
     def create_azure_openai_model(
-        self, name: str, description: str, url: str, api_key: str, environmentID: str
+        self,
+        name: str,
+        description: str,
+        url: str,
+        api_key: str,
+        environmentID: str,
+        model_parameters: Optional[Union[dict, str]] = None,
     ) -> Model:
         """Creates a new Azure-hosted OpenAI model in Eval Studio.
 
@@ -476,15 +562,20 @@ class _Models:
             url: URL of the model host system.
             api_key: API key for the model host system.
             environmentID: Azure environment ID.
+            model_parameters (optional): Optional model parameters.
+                This can be either a JSON string or a dictionary.
+                Note: The environment_id parameter will be automatically added.
+                Examples:
+                    model_parameters={"llm_args": {"temperature": 0.7}}
         """
-        params = {_AZURE_ENV_ID_PARAM: environmentID}
+        extra_params = {_AZURE_ENV_ID_PARAM: environmentID}
         req = models.V1Model(
             display_name=name,
             description=description,
             url=url,
             api_key=api_key,
             type=models.V1ModelType.MODEL_TYPE_AZURE_OPENAI_CHAT,
-            parameters=json.dumps(params),
+            parameters=self._serialize_model_parameters(model_parameters, extra_params),
         )
         res = self._api.model_service_create_model(req)
         if res and res.model:
@@ -493,7 +584,12 @@ class _Models:
         raise RuntimeError("Failed to create Azure model")
 
     def create_ollama_model(
-        self, name: str, description: str, url: str, api_key: str
+        self,
+        name: str,
+        description: str,
+        url: str,
+        api_key: str,
+        model_parameters: Optional[Union[dict, str]] = None,
     ) -> Model:
         """Creates a new OLLAMA model in Eval Studio.
 
@@ -502,6 +598,10 @@ class _Models:
             description: Description of the model.
             url: URL of the model host system.
             api_key: API key for the model host system.
+            model_parameters (optional): Optional model parameters.
+                This can be either a JSON string or a dictionary.
+                Examples:
+                    model_parameters={"llm_args": {"temperature": 0.7}}
         """
         req = models.V1Model(
             display_name=name,
@@ -509,6 +609,7 @@ class _Models:
             url=url,
             api_key=api_key,
             type=models.V1ModelType.MODEL_TYPE_OLLAMA,
+            parameters=self._serialize_model_parameters(model_parameters),
         )
         res = self._api.model_service_create_model(req)
         if res and res.model:
@@ -524,6 +625,7 @@ class _Models:
         aws_secret_access_key: str,
         aws_session_token: str,
         aws_region: str,
+        model_parameters: Optional[Union[dict, str]] = None,
     ) -> Model:
         """Creates a new Amazon Bedrock model in Eval Studio.
 
@@ -534,18 +636,26 @@ class _Models:
             aws_secret_access_key: AWS secret access key.
             aws_session_token: AWS session token.
             aws_region: AWS region.
+            model_parameters (optional): Optional model parameters.
+                This can be either a JSON string or a dictionary.
+                Note: The region parameter will be automatically added.
+                Examples:
+                    model_parameters={"llm_args": {"temperature": 0.7}}
         """
         credentials = {
             "aws_access_key_id": aws_access_key_id,
             "aws_secret_access_key": aws_secret_access_key,
             "aws_session_token": aws_session_token,
         }
+
+        extra_params = {"region": aws_region}
+
         req = models.V1Model(
             display_name=name,
             description=description,
             type=models.V1ModelType.MODEL_TYPE_AMAZON_BEDROCK,
             api_key=json.dumps(credentials),
-            parameters=json.dumps({"region": aws_region}),
+            parameters=self._serialize_model_parameters(model_parameters, extra_params),
         )
         res = self._api.model_service_create_model(req)
         if res and res.model:
@@ -569,3 +679,38 @@ class _Models:
             return [Model._from_api_model(m, self._client) for m in res_models]
 
         return []
+
+    def _serialize_model_parameters(
+        self,
+        model_parameters: Optional[Union[dict, str]] = None,
+        additional_params: Optional[dict] = None,
+    ) -> Optional[str]:
+        """Helper method to serialize model parameters."""
+        if model_parameters:
+            if isinstance(model_parameters, str):
+                if additional_params:
+                    try:
+                        params_dict = json.loads(model_parameters)
+                        params_dict.update(additional_params)
+                        return json.dumps(params_dict)
+                    except json.JSONDecodeError as err:
+                        raise ValueError(
+                            "Invalid JSON string for model_parameters"
+                        ) from err
+
+                return model_parameters
+            elif isinstance(model_parameters, dict):
+                if additional_params:
+                    model_parameters.update(additional_params)
+
+                try:
+                    return json.dumps(model_parameters)
+                except (TypeError, ValueError) as err:
+                    raise ValueError("Invalid dictionary for model_parameters") from err
+        elif additional_params:
+            try:
+                return json.dumps(additional_params)
+            except (TypeError, ValueError):
+                print("Failed to serialize additional model parameters")
+
+        return None

{eval_studio_client-1.2.1a3.dist-info → eval_studio_client-1.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-studio-client
-Version: 1.2.1a3
+Version: 1.2.2
 Project-URL: Source, https://github.com/h2oai/eval-studio/tree/main/client-py/src/
 Project-URL: Issues, https://github.com/h2oai/eval-studio/issues
 Author-email: "H2O.ai" <support@h2o.ai>