datarobot-moderations 11.2.5__py3-none-any.whl → 11.2.7__py3-none-any.whl

datarobot_dome/async_http_client.py

@@ -232,9 +232,10 @@ class AsyncHTTPClient:
         try:
             response = await self.session.post(url, json=payload, headers=self.json_headers)
             if response.status != HTTPStatus.ACCEPTED:
+                response_text = await response.text()
                 raise Exception(
                     f"Error uploading custom metrics: Status Code: {response.status}"
-                    f"Message: {response.text}"
+                    f"Message: {response_text}"
                 )
             self._logger.info("Successfully uploaded custom metrics")
         except Exception as e:

datarobot_dome/drum_integration.py

@@ -774,11 +774,8 @@ def guard_chat_wrapper(
     # extract any fields mentioned in "datarobot_metrics" to send as custom metrics later
     completion_create_params, chat_extra_body_params = filter_extra_body(completion_create_params)
 
-    # register them for ID lookup later
-    pipeline.add_extra_body_custom_metric_definition(chat_extra_body_params)
-
     # define all pipeline-based and guard-based custom metrics (but not those from extra_body)
-    # sends single API request to look up IDs for all custom metrics (including extra_body)
+    # note: this is usually partially done at pipeline init; see delayed_custom_metric_creation
     pipeline.get_new_metrics_payload()
 
     # the chat request is not a dataframe, but we'll build a DF internally for moderation.
@@ -801,6 +798,13 @@ def guard_chat_wrapper(
         if association_id:
             data[association_id_column_name] = [association_id]
 
+    # DRUM initializes the pipeline (which reads the deployment's list of custom metrics)
+    # at start time.
+    # If there are no extra_body fields (meaning no user-defined custom metrics to report),
+    # then the list does not need to be reread.
+    if chat_extra_body_params:
+        pipeline.lookup_custom_metric_ids()
+
     # report any metrics from extra_body. They are not tied to a prompt or response phase.
     _logger.debug("Report extra_body params as custom metrics")
     pipeline.report_custom_metrics_from_extra_body(association_id, chat_extra_body_params)

datarobot_dome/pipeline/llm_pipeline.py

@@ -380,20 +380,13 @@ class LLMPipeline(Pipeline):
             buckets = self._add_guard_specific_custom_metrics(row, self.get_postscore_guards())
             payload["buckets"].extend(buckets)
 
-    def add_extra_body_custom_metric_definition(self, chat_extra_body_params: dict) -> None:
-        """
-        For each name found in extra_body earlier, add to the internal map for reporting later.
-        Custom metric IDs are looked up in create_custom_metrics().
-        :param chat_extra_body_params: dict of name=value pairs
-        """
-        for name in chat_extra_body_params.keys():
-            self.extra_body_custom_metric_map[name] = {"id": None}
-
     def report_custom_metrics_from_extra_body(
         self, association_id: str, extra_params: dict
     ) -> None:
         """
         Add any key-value pairs extracted from extra_body as custom metrics.
+        The custom metrics must be deployment-based, not model-specific.
+        (Bulk upload does not support heterogeneous model/non-model metrics.)
         :param association_id: Association ID of the chat request
         :param extra_params: a dict of {"name": value} for all extra_body parameters found
         """
@@ -412,19 +405,19 @@ class LLMPipeline(Pipeline):
                     f"will not be sent: {name}"
                 )
                 continue
-            if name not in self.extra_body_custom_metric_map:
+            if name not in self.custom_metric_names_to_ids:
                 self._logger.warning(f"extra_body custom metric ID not in map: {name}")
                 continue
-            metric_id = self.extra_body_custom_metric_map[name].get("id")
+            metric_id = self.custom_metric_names_to_ids.get(name)
             if not metric_id:
+                # this should not be possible, as the name/id information
+                # is taken directly from DataRobot API
                 self._logger.warning(f"extra_body custom metric has missing ID: {name}")
                 continue
             payload["buckets"].append(
-                {
-                    "customMetricId": metric_id,
-                    "value": value,
-                    "associationId": association_id,
-                }
+                self.custom_metric_individual_payload(
+                    metric_id=metric_id, value=value, association_id=association_id
+                )
             )
         self._logger.debug(f"Sending custom metrics payload from extra_body: {payload}")
         self.upload_custom_metrics(payload)

datarobot_dome/pipeline/pipeline.py

@@ -53,8 +53,7 @@ class Pipeline:
         self._custom_metrics_bulk_upload_url = None
         self.aggregate_custom_metric = None
         self.custom_metric_map = dict()
-        # even though only LLMPipeline really uses the next map, Pipeline still references it
-        self.extra_body_custom_metric_map = dict()
+        self.custom_metric_names_to_ids = dict()
         self.delayed_custom_metric_creation = False
         self.upload_custom_metrics_tasks = set()
 
@@ -198,6 +197,38 @@ class Pipeline:
             **kwargs,
         }
 
+    def lookup_custom_metric_ids(self):
+        """
+        The deployment's list of custom metrics is known when the pipeline is created.
+        The complete set of guard metrics is also known at that time.
+        However, the extra_body metrics needed are not known until guard_chat_wrapper
+        parses extra_body.
+        For that reason, refresh the list when a chat request arrives.
+        Side effect: updates self.custom_metric_names_to_ids
+        """
+        # set up DataRobot API client, only if not already set up
+        self.create_dr_client()
+
+        # Manually paginate; the dmm list_custom_metrics does not implement pagination
+        custom_metrics_list = []
+        offset, limit = 0, 50
+        while True:
+            response_list = self.dr_client.get(
+                f"deployments/{self._deployment_id}/customMetrics/?offset={offset}&limit={limit}"
+            ).json()
+            custom_metrics_list.extend(response_list["data"])
+            offset += response_list["count"]
+            if response_list["next"] is None:
+                break
+
+        if not custom_metrics_list:
+            self._logger.warning(
+                "lookup_custom_metric_ids: result list is empty; no custom metrics to process"
+            )
+
+        self.custom_metric_names_to_ids = {m["name"]: m["id"] for m in custom_metrics_list}
+        self._logger.debug(f"lookup_custom_metric_ids(): found {self.custom_metric_names_to_ids}")
+
     def create_custom_metrics(self):
         """
         Creates all the custom-metrics in the DR app for an active deployment.
@@ -268,35 +299,16 @@ class Pipeline:
         # Now query all the metrics and get their custom metric ids.  Specifically,
         # required in case a metric is duplicated, in which case, we don't have its
         # id in the loop above
-        #
-        # Note: user-defined custom metrics (such as for extra_body params) will also
-        # be included in the list. We will warn if an existing metric is not found
-        # in either list, which means this chat request does not have a value for it.
-        #
-        # We have to go through pagination - dmm list_custom_metrics does not implement
-        # pagination
-        custom_metrics_list = []
-        offset, limit = 0, 50
-        while True:
-            response_list = self.dr_client.get(
-                f"deployments/{self._deployment_id}/customMetrics/?offset={offset}&limit={limit}"
-            ).json()
-            custom_metrics_list.extend(response_list["data"])
-            offset += response_list["count"]
-            if response_list["next"] is None:
-                break
+        self.lookup_custom_metric_ids()
 
         # assign IDs to the "metric by name" maps so we can upload by ID later
-        for metric in custom_metrics_list:
-            metric_name = metric["name"]
+        for metric_name, metric_id in self.custom_metric_names_to_ids.items():
             if metric_name in self.custom_metric_map:
-                self.custom_metric_map[metric_name]["id"] = metric["id"]
-            elif metric_name in self.extra_body_custom_metric_map:
-                self.extra_body_custom_metric_map[metric_name]["id"] = metric["id"]
+                self.custom_metric_map[metric_name]["id"] = metric_id
             else:
                 self._logger.warning(
                     f"Metric '{metric_name}' exists at DR but not in moderation; "
-                    "no value will be reported"
+                    "no moderation value will be reported for it"
                 )
                 continue
 

{datarobot_moderations-11.2.5.dist-info → datarobot_moderations-11.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datarobot-moderations
-Version: 11.2.5
+Version: 11.2.7
 Summary: DataRobot Monitoring and Moderation framework
 License: DataRobot Tool and Utility Agreement
 Author: DataRobot

{datarobot_moderations-11.2.5.dist-info → datarobot_moderations-11.2.7.dist-info}/RECORD

@@ -1,8 +1,8 @@
 datarobot_dome/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
-datarobot_dome/async_http_client.py,sha256=cQFoSI2ovt0Kyk4XWQPXod5PAfA-ZPkjLYVWQZhDGDE,9809
+datarobot_dome/async_http_client.py,sha256=YVkzSSP7ItOEwEskjH0MAs1kJW0jXfbvJWkF_52K7CE,9863
 datarobot_dome/chat_helper.py,sha256=BzvtUyZSZxzOqq-5a2wQKhHhr2kMlcP1MFrHaDAeD_o,9671
 datarobot_dome/constants.py,sha256=jvgpHa3Wh_nZVZmfU-6ab8FHnKNW3KxOPYIIEb_oS6U,10662
-datarobot_dome/drum_integration.py,sha256=DIGeiw6fwKCbgi6mb-mJdjPEB6C1uMc-pZotpTnlGZQ,45770
+datarobot_dome/drum_integration.py,sha256=SLKpK-0pI9Dsi6iMBVhPv_w-SF0rqs2of-OYF3_KSPk,45983
 datarobot_dome/guard.py,sha256=xJds9hcbUaS-KD5nC1mn0GiPdBrileFUu6BuTAjDNuY,34668
 datarobot_dome/guard_executor.py,sha256=ox5_jOHcqMaxaaagIYJJHhCwEI7Wg-rUEiu5rutsfVU,35363
 datarobot_dome/guard_helpers.py,sha256=jfu8JTWCcxu4WD1MKxeP1n53DeebY3SSuP-t5sWyV1U,17187
@@ -14,11 +14,11 @@ datarobot_dome/metrics/citation_metrics.py,sha256=l2mnV1gz7nQeJ_yfaS4dcP3DFWf0p5
 datarobot_dome/metrics/factory.py,sha256=7caa8paI9LuFXDgguXdC4on28V7IwwIsKJT2Z-Aps8A,2187
 datarobot_dome/metrics/metric_scorer.py,sha256=uJ_IJRw7ZFHueg8xjsaXbt0ypO7JiydZ0WapCp96yng,2540
 datarobot_dome/pipeline/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
-datarobot_dome/pipeline/llm_pipeline.py,sha256=g8XZQjJPPs43GPqaBG4TtsCbvu1o7TWraOmzhtUGB_o,21172
-datarobot_dome/pipeline/pipeline.py,sha256=iDR6h25RDp4bnTW1eGWErG9W_ZXDdJ_sQGVD-tT_Ll0,18805
+datarobot_dome/pipeline/llm_pipeline.py,sha256=bindyorJq9VSPZt2X8w0-3REaXui7wIA-8c-zNOIdZU,20961
+datarobot_dome/pipeline/pipeline.py,sha256=vCcP8dgu4TE-af6wUt7RqD4Xg7MzfwfK8njjC4XLnIA,19249
 datarobot_dome/pipeline/vdb_pipeline.py,sha256=zt5d_41oJjdT8qOtvpgz-l5uvImwKE9f6pQsAU_TdR4,9866
 datarobot_dome/runtime.py,sha256=FD8wXOweqoQVzbZMh-mucL66xT2kGxPsJUGAcJBgwxw,1468
 datarobot_dome/streaming.py,sha256=DkvKEH0yN0aPEWMTAjMFJB3Kx4iLGdjUMQU1pAplbeg,17751
-datarobot_moderations-11.2.5.dist-info/METADATA,sha256=9Aj3fZnLvcgKpiAQwnvkkAzA3SxaBq2j45ctWEMCEaI,4741
-datarobot_moderations-11.2.5.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-datarobot_moderations-11.2.5.dist-info/RECORD,,
+datarobot_moderations-11.2.7.dist-info/METADATA,sha256=RqNHfInWAiYqI346wVcJEwZj2sTvJ3qvC9FXZEEKLAE,4741
+datarobot_moderations-11.2.7.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+datarobot_moderations-11.2.7.dist-info/RECORD,,