datarobot-moderations 11.2.1__py3-none-any.whl → 11.2.2__py3-none-any.whl

datarobot_dome/drum_integration.py

@@ -667,7 +667,7 @@ def __get_otel_values(guards_list, stage, result_df):
     for guard in guards_list:
         if not guard.has_average_score_custom_metric():
             continue
-        guard_metric_column_name = guard.get_metric_column_name(stage)
+        guard_metric_column_name = guard.metric_column_name
         if guard_metric_column_name not in result_df.columns:
             _logger.warning(f"Missing column: {guard_metric_column_name} in result_df")
             continue

datarobot_dome/guard.py

@@ -230,6 +230,52 @@ moderation_config_trafaret = t.Dict(
 )
 
 
+def get_metric_column_name(
+    guard_type: GuardType,
+    ootb_type: OOTBType | None,
+    stage: GuardStage,
+    model_guard_target_name: str | None = None,
+    metric_name: str | None = None,
+) -> str:
+    """Gets the metric column name. Note that this function gets used in buzok code. If you update
+    it, please also update the moderation library in the buzok worker image.
+    """
+    if guard_type == GuardType.MODEL:
+        if model_guard_target_name is None:
+            raise ValueError(
+                "For the model guard type, a valid model_guard_target_name has to be provided."
+            )
+        metric_result_key = Guard.get_stage_str(stage) + "_" + model_guard_target_name
+    elif guard_type == GuardType.OOTB:
+        if ootb_type is None:
+            raise ValueError("For the OOTB type, a valid OOTB guard type has to be provided.")
+        elif ootb_type == OOTBType.TOKEN_COUNT:
+            metric_result_key = Guard.get_stage_str(stage) + "_" + TOKEN_COUNT_COLUMN_NAME
+        elif ootb_type == OOTBType.ROUGE_1:
+            metric_result_key = Guard.get_stage_str(stage) + "_" + ROUGE_1_COLUMN_NAME
+        elif ootb_type == OOTBType.FAITHFULNESS:
+            metric_result_key = Guard.get_stage_str(stage) + "_" + FAITHFULLNESS_COLUMN_NAME
+        elif ootb_type == OOTBType.AGENT_GOAL_ACCURACY:
+            metric_result_key = AGENT_GOAL_ACCURACY_COLUMN_NAME
+        elif ootb_type == OOTBType.CUSTOM_METRIC:
+            if metric_name is None:
+                raise ValueError(
+                    "For the custom metric type, a valid metric_name has to be provided."
+                )
+            metric_result_key = Guard.get_stage_str(stage) + "_" + metric_name
+        elif ootb_type == OOTBType.COST:
+            metric_result_key = COST_COLUMN_NAME
+        elif ootb_type == OOTBType.TASK_ADHERENCE:
+            metric_result_key = TASK_ADHERENCE_SCORE_COLUMN_NAME
+        else:
+            raise ValueError("The provided OOTB type is not implemented.")
+    elif guard_type == GuardType.NEMO_GUARDRAILS:
+        metric_result_key = Guard.get_stage_str(stage) + "_" + NEMO_GUARD_COLUMN_NAME
+    else:
+        raise ValueError("The provided guard type is not implemented.")
+    return metric_result_key
+
+
 class Guard(ABC):
     def __init__(self, config: dict, stage=None):
         self._name = config["name"]
@@ -244,6 +290,13 @@ class Guard(ABC):
         self._faas_url = config.get("faas_url")
         self._copy_citations = config["copy_citations"]
         self.is_agentic = config.get("is_agentic", False)
+        self.metric_column_name = get_metric_column_name(
+            config["type"],
+            config.get("ootb_type"),
+            self._stage,
+            config.get("model_info", {}).get("target_name"),
+            config["name"],
+        )
 
         if config.get("intervention"):
             self.intervention = GuardIntervention(config["intervention"])
@@ -448,11 +501,6 @@ class ModelGuard(Guard):
     def model_info(self):
         return self._model_info
 
-    def get_metric_column_name(self, stage):
-        if self.model_info is None:
-            raise NotImplementedError("Missing model_info for model guard")
-        return self.get_stage_str(stage) + "_" + self._model_info.target_name
-
     def get_span_column_name(self, _):
         if self.model_info is None:
             raise NotImplementedError("Missing model_info for model guard")
@@ -580,9 +628,6 @@ class NeMoGuard(Guard, GuardLLMMixin):
         """No average score metrics for NemoGuard's"""
         return False
 
-    def get_metric_column_name(self, stage):
-        return self.get_stage_str(stage) + "_" + NEMO_GUARD_COLUMN_NAME
-
     @property
     def nemo_llm_rails(self):
         return self._nemo_llm_rails
@@ -601,16 +646,6 @@ class OOTBGuard(Guard):
         """Latency is not tracked for token counts guards"""
         return self._ootb_type != OOTBType.TOKEN_COUNT
 
-    def get_metric_column_name(self, stage):
-        if self._ootb_type == OOTBType.TOKEN_COUNT:
-            return self.get_stage_str(stage) + "_" + TOKEN_COUNT_COLUMN_NAME
-        elif self._ootb_type == OOTBType.ROUGE_1:
-            return self.get_stage_str(stage) + "_" + ROUGE_1_COLUMN_NAME
-        elif self._ootb_type == OOTBType.CUSTOM_METRIC:
-            return self.get_stage_str(stage) + "_" + self.name
-        else:
-            raise NotImplementedError(f"No metric column name defined for {self._ootb_type} guard")
-
     def get_span_column_name(self, _):
         if self._ootb_type == OOTBType.TOKEN_COUNT:
             return TOKEN_COUNT_COLUMN_NAME
@@ -640,9 +675,6 @@ class OOTBCostMetric(OOTBGuard):
         self.output_unit = cost_config["output_unit"]
         self.output_multiplier = self.output_price / self.output_unit
 
-    def get_metric_column_name(self, _):
-        return COST_COLUMN_NAME
-
     def get_average_score_custom_metric_name(self, _):
         return f"Total cost in {self.currency}"
 
@@ -681,9 +713,6 @@ class FaithfulnessGuard(OOTBGuard, GuardLLMMixin):
         Settings.embed_model = None
         self._evaluator = FaithfulnessEvaluator()
 
-    def get_metric_column_name(self, stage):
-        return self.get_stage_str(stage) + "_" + FAITHFULLNESS_COLUMN_NAME
-
     @property
     def faithfulness_evaluator(self):
         return self._evaluator
@@ -708,9 +737,6 @@ class AgentGoalAccuracyGuard(OOTBGuard, GuardLLMMixin):
         evaluator_llm = LangchainLLMWrapper(llm)
         self.scorer = AgentGoalAccuracyWithoutReference(llm=evaluator_llm)
 
-    def get_metric_column_name(self, _):
-        return AGENT_GOAL_ACCURACY_COLUMN_NAME
-
     @property
     def accuracy_scorer(self):
         return self.scorer
@@ -735,9 +761,6 @@ class TaskAdherenceGuard(OOTBGuard, GuardLLMMixin):
         deepeval_llm = ModerationDeepEvalLLM(llm)
         self.scorer = TaskCompletionMetric(model=deepeval_llm, include_reason=True)
 
-    def get_metric_column_name(self, _):
-        return TASK_ADHERENCE_SCORE_COLUMN_NAME
-
     @property
     def task_adherence_scorer(self):
         return self.scorer

datarobot_dome/guard_executor.py

@@ -113,7 +113,7 @@ class AsyncGuardExecutor:
             if guard.has_latency_custom_metric():
                 self.pipeline.report_guard_latency(guard, latency)
             if guard.has_average_score_custom_metric():
-                metric_column_name = guard.get_metric_column_name(stage)
+                metric_column_name = guard.metric_column_name
                 if metric_column_name in df.columns:
                     span.set_attribute(
                         guard.get_span_attribute_name(stage),
@@ -222,7 +222,7 @@ class AsyncGuardExecutor:
             # and "Response_toxicity_toxic_PREDICTION", if toxicity is configured for both
             # prompts and responses
             copy_df.rename(
-                columns={metric_column: guard.get_metric_column_name(stage)},
+                columns={metric_column: guard.metric_column_name},
                 inplace=True,
             )
         except Exception as ex:
@@ -359,7 +359,7 @@ class AsyncGuardExecutor:
         else:
             prompt_column_name = self.pipeline.get_input_column(GuardStage.PROMPT)
             response_column_name = self.pipeline.get_input_column(GuardStage.RESPONSE)
-            metric_column_name = guard.get_metric_column_name(stage)
+            metric_column_name = guard.metric_column_name
 
             try:
                 copy_df[metric_column_name] = copy_df.apply(
@@ -407,7 +407,7 @@ class AsyncGuardExecutor:
         else:
             prompt_column_name = self.pipeline.get_input_column(GuardStage.PROMPT)
             response_column_name = self.pipeline.get_input_column(GuardStage.RESPONSE)
-            metric_column_name = guard.get_metric_column_name(stage)
+            metric_column_name = guard.metric_column_name
 
             try:
                 copy_df[metric_column_name] = copy_df.apply(
@@ -454,7 +454,7 @@ class AsyncGuardExecutor:
         else:
             prompt_column_name = self.pipeline.get_input_column(GuardStage.PROMPT)
             response_column_name = self.pipeline.get_input_column(GuardStage.RESPONSE)
-            metric_column_name = guard.get_metric_column_name(stage)
+            metric_column_name = guard.metric_column_name
 
             try:
                 copy_df[metric_column_name] = copy_df.apply(
@@ -500,7 +500,7 @@ class AsyncGuardExecutor:
             intervene = False
         else:
             input_column = self.pipeline.get_input_column(stage)
-            metric_column_name = guard.get_metric_column_name(stage)
+            metric_column_name = guard.metric_column_name
             copy_df[metric_column_name] = copy_df.apply(
                 lambda x: get_rouge_1_score(
                     scorer=self.pipeline.rouge_scorer,
@@ -517,7 +517,7 @@ class AsyncGuardExecutor:
 
         prompt_column_name = self.pipeline.get_input_column(GuardStage.PROMPT)
         response_column_name = self.pipeline.get_input_column(GuardStage.RESPONSE)
-        metric_column_name = guard.get_metric_column_name(stage)
+        metric_column_name = guard.metric_column_name
         if (
             PROMPT_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE not in copy_df.columns
             or RESPONSE_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE not in copy_df.columns
@@ -539,7 +539,7 @@ class AsyncGuardExecutor:
         if not isinstance(guard, OOTBGuard):
             raise ValueError(f"Guard object should be of type OOTBGuard, got: {type(guard)}")
         input_column = self.pipeline.get_input_column(stage)
-        metric_column_name = guard.get_metric_column_name(stage)
+        metric_column_name = guard.metric_column_name
         intervene = self._should_intervene(guard)
 
         if guard.ootb_type == OOTBType.TOKEN_COUNT:
@@ -594,7 +594,7 @@ class AsyncGuardExecutor:
             raise ValueError(f"Guard object should be of type NeMoGuard, got: {type(guard)}")
 
         input_column = self.pipeline.get_input_column(stage)
-        metric_column_name = guard.get_metric_column_name(stage)
+        metric_column_name = guard.metric_column_name
         intervene = self._should_intervene(guard)
 
         try:

datarobot_dome/pipeline/llm_pipeline.py

@@ -304,7 +304,7 @@ class LLMPipeline(Pipeline):
             metric_list = [
                 (
                     guard.get_average_score_custom_metric_name(guard.stage),
-                    guard.get_metric_column_name(guard.stage),
+                    guard.metric_column_name,
                 )
             ]
         if intervention_action:

datarobot_dome/streaming.py

@@ -323,7 +323,7 @@ class ModerationIterator:
                 self.aggregated_metrics_df[column_name] = metrics_df[column_name]
             elif guard.type == GuardType.OOTB:
                 if guard.ootb_type == OOTBType.TOKEN_COUNT:
-                    column_name = guard.get_metric_column_name(GuardStage.RESPONSE)
+                    column_name = guard.metric_column_name
                     self.aggregated_metrics_df[column_name] += metrics_df[column_name]
                 else:
                     # Faithfulness, ROUGE-1 can't run on chunks so no merging
@@ -368,7 +368,7 @@ class ModerationIterator:
         postscore_df = postscore_df_chunk.copy(deep=True)
         for guard in self.pipeline.get_postscore_guards():
             if not self._guard_can_work_on_chunk(guard):
-                metric_column_name = guard.get_metric_column_name(GuardStage.RESPONSE)
+                metric_column_name = guard.metric_column_name
                 if metric_column_name in postscore_df_assembled.columns:
                     postscore_df[metric_column_name] = postscore_df_assembled[metric_column_name]
                 if guard.has_latency_custom_metric():

{datarobot_moderations-11.2.1.dist-info → datarobot_moderations-11.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datarobot-moderations
-Version: 11.2.1
+Version: 11.2.2
 Summary: DataRobot Monitoring and Moderation framework
 License: DataRobot Tool and Utility Agreement
 Author: DataRobot
@@ -15,7 +15,7 @@ Requires-Dist: aiohttp (>=3.9.5)
 Requires-Dist: backoff (>=2.2.1)
 Requires-Dist: datarobot (>=3.6.0)
 Requires-Dist: datarobot-predict (>=1.9.6)
-Requires-Dist: deepeval (==2.7.9)
+Requires-Dist: deepeval (>=3.3.5)
 Requires-Dist: langchain (>=0.1.12)
 Requires-Dist: langchain-nvidia-ai-endpoints (>=0.3.9)
 Requires-Dist: langchain-openai (>=0.1.7)

{datarobot_moderations-11.2.1.dist-info → datarobot_moderations-11.2.2.dist-info}/RECORD

@@ -2,9 +2,9 @@ datarobot_dome/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,58
 datarobot_dome/async_http_client.py,sha256=wkB4irwvnchNGzO1bk2C_HWM-GOSB3AUn5TXKl-X0ZI,9649
 datarobot_dome/chat_helper.py,sha256=BzvtUyZSZxzOqq-5a2wQKhHhr2kMlcP1MFrHaDAeD_o,9671
 datarobot_dome/constants.py,sha256=vM2_JkXbn4dkWARCqxNfLriSo0E05LDXVrwNktptpuc,10416
-datarobot_dome/drum_integration.py,sha256=HresblJwlCk_sRnWReWQWeZMg5rYzKTA2hjmy1Rcn6U,40553
-datarobot_dome/guard.py,sha256=1INYx17n9ToiB5bzI-jIReUUuqkK_ucxpOx4jQLts6g,33264
-datarobot_dome/guard_executor.py,sha256=AOI8MZeZETHMoFgBePe0wa2vE9d2975MYQnEDHLZL7s,35462
+datarobot_dome/drum_integration.py,sha256=BnhAP-D4AaEeh4ferZ-qXnORuWQzYzw9qKAZUTZZnJU,40542
+datarobot_dome/guard.py,sha256=ecpHnIOCEniTMgnVnqlYuU6HnR4baP9ruWaczv5q2lA,34496
+datarobot_dome/guard_executor.py,sha256=ox5_jOHcqMaxaaagIYJJHhCwEI7Wg-rUEiu5rutsfVU,35363
 datarobot_dome/guard_helpers.py,sha256=ajxm-w7MS7eN5DMMO-jbbzjcOYMZ-LvhO53n2NI5_Fk,16773
 datarobot_dome/guards/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
 datarobot_dome/guards/guard_llm_mixin.py,sha256=VovlpNZjWIGamF4SSvLF5lzOFyApH5IoOiB_qtCmRg0,12216
@@ -14,11 +14,11 @@ datarobot_dome/metrics/citation_metrics.py,sha256=l2mnV1gz7nQeJ_yfaS4dcP3DFWf0p5
 datarobot_dome/metrics/factory.py,sha256=7caa8paI9LuFXDgguXdC4on28V7IwwIsKJT2Z-Aps8A,2187
 datarobot_dome/metrics/metric_scorer.py,sha256=uJ_IJRw7ZFHueg8xjsaXbt0ypO7JiydZ0WapCp96yng,2540
 datarobot_dome/pipeline/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
-datarobot_dome/pipeline/llm_pipeline.py,sha256=g7PAiLOMADr2DQFrtg2NrUj4u_tcvnoiJXrBR8xWsmY,18789
+datarobot_dome/pipeline/llm_pipeline.py,sha256=DMZ4gu88MiSSEQtshDyHOzT3R2Seuf8UqZ7A36QHj3M,18772
 datarobot_dome/pipeline/pipeline.py,sha256=7UmvrZtNxTGewpgM4cf2oThHPoJSarEU1Dyp7xEsASU,17401
 datarobot_dome/pipeline/vdb_pipeline.py,sha256=q3c_Z-hGUqhH6j6n8VpS3wZiBIkWgpRDsBnyJyZhiw4,9855
 datarobot_dome/runtime.py,sha256=FD8wXOweqoQVzbZMh-mucL66xT2kGxPsJUGAcJBgwxw,1468
-datarobot_dome/streaming.py,sha256=6nYvh6SoxPRLfO6GGdEoHsQuyLP9oX1lDMe8IeGo4lw,17801
-datarobot_moderations-11.2.1.dist-info/METADATA,sha256=fEyM5I3z0qS9dT2ofZ3J8UAN_ybm2SBF_46mifEHIpA,4742
-datarobot_moderations-11.2.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datarobot_moderations-11.2.1.dist-info/RECORD,,
+datarobot_dome/streaming.py,sha256=DkvKEH0yN0aPEWMTAjMFJB3Kx4iLGdjUMQU1pAplbeg,17751
+datarobot_moderations-11.2.2.dist-info/METADATA,sha256=YLN3wvaq5UGP5hnJAD34DxbvACuGuH-RGUE_zCJbfBs,4742
+datarobot_moderations-11.2.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datarobot_moderations-11.2.2.dist-info/RECORD,,