datarobot-moderations 11.2.4__py3-none-any.whl → 11.2.6__py3-none-any.whl

datarobot_dome/constants.py

@@ -74,8 +74,8 @@ PROMPT_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE = "prompt_token_count_from_usage"
 RESPONSE_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE = "response_token_count_from_usage"
 
 SPAN_PREFIX = "datarobot.guard"
-DATAROBOT_EXTRA_BODY_PREFIX = "datarobot_"
 DATAROBOT_ASSOCIATION_ID_FIELD_NAME = "datarobot_association_id"
+DATAROBOT_METRICS_DICT_FIELD_NAME = "datarobot_metrics"
 
 
 class TargetType(str, Enum):

datarobot_dome/drum_integration.py

@@ -42,7 +42,7 @@ from datarobot_dome.constants import AGENTIC_PIPELINE_INTERACTIONS_ATTR
 from datarobot_dome.constants import CHAT_COMPLETION_OBJECT
 from datarobot_dome.constants import CITATIONS_ATTR
 from datarobot_dome.constants import DATAROBOT_ASSOCIATION_ID_FIELD_NAME
-from datarobot_dome.constants import DATAROBOT_EXTRA_BODY_PREFIX
+from datarobot_dome.constants import DATAROBOT_METRICS_DICT_FIELD_NAME
 from datarobot_dome.constants import DATAROBOT_MODERATIONS_ATTR
 from datarobot_dome.constants import DISABLE_MODERATION_RUNTIME_PARAM_NAME
 from datarobot_dome.constants import LLM_BLUEPRINT_ID_ATTR
@@ -721,23 +721,24 @@ def report_otel_evaluation_set_metric(pipeline, result_df):
 
 def filter_extra_body(
     completion_create_params: CompletionCreateParams,
-) -> tuple[CompletionCreateParams, list]:
+) -> tuple[CompletionCreateParams, dict]:
     """
     completion_create_params is a typed dict of a few standard fields,
     and arbitrary fields from extra_body.
-    For all fields matching "datarobot_", copy them to a list for later use, and remove them
-    from completion_create_params.
+    If "datarobot_metrics" is in extra_body, process it here.
+    Save its value only if it is a dict as expected.
     :param completion_create_params: the chat completion params from OpenAI client via DRUM
-    :return: filtered completion_create_params and list of "datarobot_" fields
+    :return: filtered completion_create_params; dict of {name: value} for "datarobot_" fields
     """
-    datarobot_extra_body_params = []
-    our_param_names = [
-        p for p in completion_create_params if p.startswith(DATAROBOT_EXTRA_BODY_PREFIX)
-    ]
-    for name in our_param_names:
+    datarobot_extra_body_params = {}
+    name = DATAROBOT_METRICS_DICT_FIELD_NAME
+    if name in completion_create_params:
         value = completion_create_params[name]
-        datarobot_extra_body_params.append({name: value})
-        _logger.debug("found DataRobot parameter in extra_body: %s", f"{name}={value}")
+        _logger.debug("found DataRobot metrics in extra_body: %s", f"{name}={value}")
+        if isinstance(value, dict):
+            datarobot_extra_body_params = copy.deepcopy(value)
+        else:
+            _logger.warning("DataRobot metrics in extra_body is not a dict: %s", f"{name}={value}")
         completion_create_params.pop(name, None)
     return completion_create_params, datarobot_extra_body_params
 
@@ -770,9 +771,11 @@ def guard_chat_wrapper(
     # if association ID was included in extra_body, extract field name and value
     completion_create_params, eb_assoc_id_value = filter_association_id(completion_create_params)
 
-    # todo future: filter extra_body params here; pass to pipeline.report_custom_metrics
-    # completion_create_params, chat_extra_body_params = filter_extra_body(completion_create_params)
+    # extract any fields mentioned in "datarobot_metrics" to send as custom metrics later
+    completion_create_params, chat_extra_body_params = filter_extra_body(completion_create_params)
 
+    # define all pipeline-based and guard-based custom metrics (but not those from extra_body)
+    # note: this is usually partially done at pipeline init; see delayed_custom_metric_creation
     pipeline.get_new_metrics_payload()
 
     # the chat request is not a dataframe, but we'll build a DF internally for moderation.
@@ -795,6 +798,10 @@ def guard_chat_wrapper(
         if association_id:
             data[association_id_column_name] = [association_id]
 
+    # report any metrics from extra_body. They are not tied to a prompt or response phase.
+    _logger.debug("Report extra_body params as custom metrics")
+    pipeline.report_custom_metrics_from_extra_body(association_id, chat_extra_body_params)
+
     # ==================================================================
     # Step 1: Prescore Guards processing
     #
@@ -804,11 +811,6 @@ def guard_chat_wrapper(
     _logger.debug(filtered_df)
     _logger.debug(f"Pre Score Guard Latency: {prescore_latency} sec")
 
-    # todo future: add extra_body parameters to custom metrics reporting
-    # _logger.debug("Add extra_body params as custom metrics")
-    # for param in chat_extra_body_params:
-    #     _logger.debug(f"Future: add extra_body param: {param}")
-
     blocked_prompt_column_name = f"blocked_{prompt_column_name}"
     if prescore_df.loc[0, blocked_prompt_column_name]:
         pipeline.report_custom_metrics(prescore_df)
@@ -974,6 +976,8 @@ class ModerationPipeline:
     Base class to simplify interactions with DRUM.
     This class is not used outside of testing;
     moderation_pipeline_factory() will select the LLM or VDB subclass instead.
+    Also: Pipeline and ModerationPipeline are separate classes (not in samm hierarchy)
+    However, LlmModerationPipeline includes LLMPipeline by composition.
     """
 
     def score(self, input_df: pd.DataFrame, model, drum_score_fn, **kwargs):
@@ -1010,7 +1014,10 @@ class LlmModerationPipeline(ModerationPipeline):
         association_id=None,
         **kwargs,
     ):
-        """Calls the standard guard chat function."""
+        """
+        Calls the standard guard chat function.
+        See PythonModelAdapter.chat() in DRUM, which calls chat() here.
+        """
         return guard_chat_wrapper(
             completion_create_params,
             model,

datarobot_dome/pipeline/llm_pipeline.py

@@ -197,17 +197,17 @@ class LLMPipeline(Pipeline):
 
         if guard.has_average_score_custom_metric():
             metric_def = self._get_average_score_metric_definition(guard)
-            self.add_custom_metric(metric_def, True)
+            self.add_custom_metric_definition(metric_def, True)
 
         if guard.has_latency_custom_metric():
             metric_def = guard.get_latency_custom_metric()
-            self.add_custom_metric(metric_def, False)
+            self.add_custom_metric_definition(metric_def, False)
 
         if intervention_action:
             # Enforced metric for all kinds of guards, as long as they have intervention
             # action defined - even for token count
             metric_def = guard.get_enforced_custom_metric(guard_stage, intervention_action)
-            self.add_custom_metric(metric_def, True)
+            self.add_custom_metric_definition(metric_def, True)
 
     def _add_default_custom_metrics(self):
         """Default custom metrics"""
@@ -219,14 +219,14 @@ class LLMPipeline(Pipeline):
             postscore_guard_latency_custom_metric,
             score_latency,
         ]:
-            self.add_custom_metric(metric_def, False)
+            self.add_custom_metric_definition(metric_def, False)
 
         # These metrics report with an association-id
         for metric_def in [
             get_blocked_custom_metric(GuardStage.PROMPT),
             get_blocked_custom_metric(GuardStage.RESPONSE),
         ]:
-            self.add_custom_metric(metric_def, True)
+            self.add_custom_metric_definition(metric_def, True)
 
     def _add_guard_to_pipeline(self, guard):
         if guard.stage == GuardStage.PROMPT:
@@ -380,6 +380,46 @@ class LLMPipeline(Pipeline):
             buckets = self._add_guard_specific_custom_metrics(row, self.get_postscore_guards())
             payload["buckets"].extend(buckets)
 
+    def report_custom_metrics_from_extra_body(
+        self, association_id: str, extra_params: dict
+    ) -> None:
+        """
+        Add any key-value pairs extracted from extra_body as custom metrics.
+        :param association_id: Association ID of the chat request
+        :param extra_params: a dict of {"name": value} for all extra_body parameters found
+        """
+        # If no association ID is defined for deployment, custom metrics will not be processed
+        if self._association_id_column_name is None:
+            return
+        if not extra_params:
+            return  # nothing to send
+        payload = {"buckets": []}
+        for name, value in extra_params.items():
+            if name in self.custom_metric_map:
+                # In case of name collision:
+                # the extra_body metric will _not_ override the other moderation metric
+                self._logger.warning(
+                    "extra_body custom metric name is already in use in moderation; "
+                    f"will not be sent: {name}"
+                )
+                continue
+            if name not in self.custom_metric_names_to_ids:
+                self._logger.warning(f"extra_body custom metric ID not in map: {name}")
+                continue
+            metric_id = self.custom_metric_names_to_ids.get(name)
+            if not metric_id:
+                # this should not be possible, as the name/id information
+                # is taken directly from DataRobot API
+                self._logger.warning(f"extra_body custom metric has missing ID: {name}")
+                continue
+            payload["buckets"].append(
+                self.custom_metric_individual_payload(
+                    metric_id=metric_id, value=value, association_id=association_id
+                )
+            )
+        self._logger.debug(f"Sending custom metrics payload from extra_body: {payload}")
+        self.upload_custom_metrics(payload)
+
     def report_custom_metrics(self, result_df):
         if self.delayed_custom_metric_creation:
             # Flag is not set yet, so no point reporting custom metrics

datarobot_dome/pipeline/pipeline.py

@@ -41,7 +41,6 @@ class Pipeline:
 
     def __init__(self, async_http_timeout_sec=DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC):
         self._logger = logging.getLogger(LOGGER_NAME_PREFIX + "." + self.__class__.__name__)
-        self.custom_metric = {}
         self._deployment = None
         self._association_id_column_name = None
         self._datarobot_url = None
@@ -54,6 +53,7 @@ class Pipeline:
         self._custom_metrics_bulk_upload_url = None
         self.aggregate_custom_metric = None
         self.custom_metric_map = dict()
+        self.custom_metric_names_to_ids = dict()
         self.delayed_custom_metric_creation = False
         self.upload_custom_metrics_tasks = set()
 
@@ -77,6 +77,13 @@ class Pipeline:
             self._logger.warning(f"Missing DataRobot API Token, {self.common_message}")
             return
 
+        self._deployment_id = os.environ.get("MLOPS_DEPLOYMENT_ID", None)
+        if self._deployment_id is None:
+            self._logger.warning(
+                f"DataRobot deployment id not exported (MLOPS_DEPLOYMENT_ID), {self.common_message}"
+            )
+            return
+
         # This is regular / default DataRobot Client
         self.dr_client = dr.Client(endpoint=self._datarobot_url, token=self._datarobot_api_token)
         self._headers = {
@@ -91,7 +98,6 @@ class Pipeline:
         moderations for
         :return:
         """
-        self._deployment_id = os.environ.get("MLOPS_DEPLOYMENT_ID", None)
         if self._deployment_id is None:
             self._logger.warning(f'Custom Model workshop "test" mode?, {self.common_message}')
             return
@@ -172,13 +178,17 @@ class Pipeline:
             self.create_custom_metrics()
             self.delayed_custom_metric_creation = False
 
-    def add_custom_metric(
+    def add_custom_metric_definition(
         self, metric_definition: dict[str, Any], requires_association_id: bool, **kwargs
     ) -> None:
         """
         Adds an entry to the `custom_metric_map`.
+        Only 2 functions should write to this map:
+        * this function -- links the custom metric definition to its name
+        * create_custom_metrics() -- queries DR for the object ID and links it to the name
 
         NOTE: the kwargs allow implementations to add their own specialized values.
+        Currently only VDBPipeline calls this with kwargs.
         """
         name = metric_definition["name"]
         self.custom_metric_map[name] = {
@@ -187,12 +197,41 @@ class Pipeline:
             **kwargs,
         }
 
+    def lookup_custom_metric_ids(self):
+        """
+        The deployment's list of custom metrics is known when the pipeline is created.
+        The complete set of guard metrics is also known at that time.
+        However, the extra_body metrics needed are not known until guard_chat_wrapper
+        parses extra_body.
+        For that reason, read and cache all the metric name/id pairs at pipeline create time.
+        Side effect: updates self.custom_metric_names_to_ids
+        """
+        if self.custom_metric_names_to_ids:
+            self._logger.debug("lookup_custom_metric_ids(): IDs were already read; skipping")
+            return
+
+        # Manually paginate; the dmm list_custom_metrics does not implement pagination
+        custom_metrics_list = []
+        offset, limit = 0, 50
+        while True:
+            response_list = self.dr_client.get(
+                f"deployments/{self._deployment_id}/customMetrics/?offset={offset}&limit={limit}"
+            ).json()
+            custom_metrics_list.extend(response_list["data"])
+            offset += response_list["count"]
+            if response_list["next"] is None:
+                break
+
+        self.custom_metric_names_to_ids = {m["name"]: m["id"] for m in custom_metrics_list}
+
     def create_custom_metrics(self):
         """
         Creates all the custom-metrics in the DR app for an active deployment.
 
         Updates the `custom_metric_map` with id's to insure the appropriate data
         is put in place for reporting.
+
+        Every custom metric we want to use must already exist by name in the map.
         """
         cleanup_metrics_list = list()
         for index, (metric_name, custom_metric) in enumerate(self.custom_metric_map.items()):
@@ -255,26 +294,18 @@ class Pipeline:
         # Now query all the metrics and get their custom metric ids.  Specifically,
         # required in case a metric is duplicated, in which case, we don't have its
         # id in the loop above
-        #
-        # We have to go through pagination - dmm list_custom_metrics does not implement
-        # pagination
-        custom_metrics_list = []
-        offset, limit = 0, 50
-        while True:
-            response_list = self.dr_client.get(
-                f"deployments/{self._deployment_id}/customMetrics/?offset={offset}&limit={limit}"
-            ).json()
-            custom_metrics_list.extend(response_list["data"])
-            offset += response_list["count"]
-            if response_list["next"] is None:
-                break
-
-        for metric in custom_metrics_list:
-            metric_name = metric["name"]
-            if metric_name not in self.custom_metric_map:
-                self._logger.error(f"Metric '{metric_name}' exists at DR but not in moderation")
+        self.lookup_custom_metric_ids()
+
+        # assign IDs to the "metric by name" maps so we can upload by ID later
+        for metric_name, metric_id in self.custom_metric_names_to_ids.items():
+            if metric_name in self.custom_metric_map:
+                self.custom_metric_map[metric_name]["id"] = metric_id
+            else:
+                self._logger.warning(
+                    f"Metric '{metric_name}' exists at DR but not in moderation; "
+                    "no moderation value will be reported for it"
+                )
                 continue
-            self.custom_metric_map[metric_name]["id"] = metric["id"]
 
         # These are the metrics we couldn't create - so, don't track them
         for metric_name in cleanup_metrics_list:

datarobot_dome/pipeline/vdb_pipeline.py

@@ -70,7 +70,7 @@ class VDBPipeline(Pipeline):
 
         # Metric list so far does not need association id for reporting
         for metric_def, per_row, score_type in metric_list:
-            self.add_custom_metric(metric_def, per_row, scorer_type=score_type)
+            self.add_custom_metric_definition(metric_def, per_row, scorer_type=score_type)
 
     def create_scorers(self):
         """

{datarobot_moderations-11.2.4.dist-info → datarobot_moderations-11.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datarobot-moderations
-Version: 11.2.4
+Version: 11.2.6
 Summary: DataRobot Monitoring and Moderation framework
 License: DataRobot Tool and Utility Agreement
 Author: DataRobot
@@ -19,7 +19,7 @@ Requires-Dist: deepeval (>=3.3.5)
 Requires-Dist: langchain (>=0.1.12)
 Requires-Dist: langchain-nvidia-ai-endpoints (>=0.3.9)
 Requires-Dist: langchain-openai (>=0.1.7)
-Requires-Dist: llama-index (>=0.12.49)
+Requires-Dist: llama-index (>=0.13.0)
 Requires-Dist: llama-index-embeddings-azure-openai (>=0.1.6)
 Requires-Dist: llama-index-llms-bedrock-converse (>=0.1.6)
 Requires-Dist: llama-index-llms-langchain (>=0.1.3)

{datarobot_moderations-11.2.4.dist-info → datarobot_moderations-11.2.6.dist-info}/RECORD

@@ -1,8 +1,8 @@
 datarobot_dome/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
 datarobot_dome/async_http_client.py,sha256=cQFoSI2ovt0Kyk4XWQPXod5PAfA-ZPkjLYVWQZhDGDE,9809
 datarobot_dome/chat_helper.py,sha256=BzvtUyZSZxzOqq-5a2wQKhHhr2kMlcP1MFrHaDAeD_o,9671
-datarobot_dome/constants.py,sha256=EtdmYdEp9H2awbJVo2Xfmk5PFCJ0nymMSAPIAt8pQgE,10649
-datarobot_dome/drum_integration.py,sha256=nLENtjQEP4nwwyrtesQTj2844I-ap_HwHKvijfxz0Ng,45121
+datarobot_dome/constants.py,sha256=jvgpHa3Wh_nZVZmfU-6ab8FHnKNW3KxOPYIIEb_oS6U,10662
+datarobot_dome/drum_integration.py,sha256=NWTYJWGPM_z8RQYXteEOWQkCVBXzJ5XRZEWJYeUdHf8,45656
 datarobot_dome/guard.py,sha256=xJds9hcbUaS-KD5nC1mn0GiPdBrileFUu6BuTAjDNuY,34668
 datarobot_dome/guard_executor.py,sha256=ox5_jOHcqMaxaaagIYJJHhCwEI7Wg-rUEiu5rutsfVU,35363
 datarobot_dome/guard_helpers.py,sha256=jfu8JTWCcxu4WD1MKxeP1n53DeebY3SSuP-t5sWyV1U,17187
@@ -14,11 +14,11 @@ datarobot_dome/metrics/citation_metrics.py,sha256=l2mnV1gz7nQeJ_yfaS4dcP3DFWf0p5
 datarobot_dome/metrics/factory.py,sha256=7caa8paI9LuFXDgguXdC4on28V7IwwIsKJT2Z-Aps8A,2187
 datarobot_dome/metrics/metric_scorer.py,sha256=uJ_IJRw7ZFHueg8xjsaXbt0ypO7JiydZ0WapCp96yng,2540
 datarobot_dome/pipeline/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
-datarobot_dome/pipeline/llm_pipeline.py,sha256=4Q-DW8lzKdPBDTNgO-wI-Pyl53IRZNJcjJpfE3kiv08,18813
-datarobot_dome/pipeline/pipeline.py,sha256=GM1mmFtk4xm2xmHiFOefno4K38FNjdMfrynpsp6MLX0,17511
-datarobot_dome/pipeline/vdb_pipeline.py,sha256=q3c_Z-hGUqhH6j6n8VpS3wZiBIkWgpRDsBnyJyZhiw4,9855
+datarobot_dome/pipeline/llm_pipeline.py,sha256=PRJ7t5Bc8S2ZbD0yK8ztdTD1hPC7Yo8IvhFlWEkMkmU,20810
+datarobot_dome/pipeline/pipeline.py,sha256=m8m_QtoQNwJ5U-bbJGdtlmAbdI-UpobcMCuRz-U_H_Y,19052
+datarobot_dome/pipeline/vdb_pipeline.py,sha256=zt5d_41oJjdT8qOtvpgz-l5uvImwKE9f6pQsAU_TdR4,9866
 datarobot_dome/runtime.py,sha256=FD8wXOweqoQVzbZMh-mucL66xT2kGxPsJUGAcJBgwxw,1468
 datarobot_dome/streaming.py,sha256=DkvKEH0yN0aPEWMTAjMFJB3Kx4iLGdjUMQU1pAplbeg,17751
-datarobot_moderations-11.2.4.dist-info/METADATA,sha256=iksvFgFDIQZA7DF0vR6fICFawRl7xcdl0hy_E4QAakg,4742
-datarobot_moderations-11.2.4.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-datarobot_moderations-11.2.4.dist-info/RECORD,,
+datarobot_moderations-11.2.6.dist-info/METADATA,sha256=U20ej2pA1_79GwOl4f7qShgB9Q4F_zb-phsOBTITOF0,4741
+datarobot_moderations-11.2.6.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+datarobot_moderations-11.2.6.dist-info/RECORD,,