datarobot-moderations 11.2.1__py3-none-any.whl → 11.2.3__py3-none-any.whl

datarobot_dome/drum_integration.py

@@ -667,7 +667,7 @@ def __get_otel_values(guards_list, stage, result_df):
     for guard in guards_list:
         if not guard.has_average_score_custom_metric():
             continue
-        guard_metric_column_name = guard.get_metric_column_name(stage)
+        guard_metric_column_name = guard.metric_column_name
         if guard_metric_column_name not in result_df.columns:
             _logger.warning(f"Missing column: {guard_metric_column_name} in result_df")
             continue

datarobot_dome/guard.py

@@ -23,6 +23,7 @@ from llama_index.core.evaluation import FaithfulnessEvaluator
 from nemoguardrails import LLMRails
 from nemoguardrails import RailsConfig
 from ragas.llms import LangchainLLMWrapper
+from ragas.llms import LlamaIndexLLMWrapper
 from ragas.metrics import AgentGoalAccuracyWithoutReference
 
 from datarobot_dome.constants import AGENT_GOAL_ACCURACY_COLUMN_NAME
@@ -230,6 +231,52 @@ moderation_config_trafaret = t.Dict(
 )
 
 
+def get_metric_column_name(
+    guard_type: GuardType,
+    ootb_type: OOTBType | None,
+    stage: GuardStage,
+    model_guard_target_name: str | None = None,
+    metric_name: str | None = None,
+) -> str:
+    """Gets the metric column name. Note that this function gets used in buzok code. If you update
+    it, please also update the moderation library in the buzok worker image.
+    """
+    if guard_type == GuardType.MODEL:
+        if model_guard_target_name is None:
+            raise ValueError(
+                "For the model guard type, a valid model_guard_target_name has to be provided."
+            )
+        metric_result_key = Guard.get_stage_str(stage) + "_" + model_guard_target_name
+    elif guard_type == GuardType.OOTB:
+        if ootb_type is None:
+            raise ValueError("For the OOTB type, a valid OOTB guard type has to be provided.")
+        elif ootb_type == OOTBType.TOKEN_COUNT:
+            metric_result_key = Guard.get_stage_str(stage) + "_" + TOKEN_COUNT_COLUMN_NAME
+        elif ootb_type == OOTBType.ROUGE_1:
+            metric_result_key = Guard.get_stage_str(stage) + "_" + ROUGE_1_COLUMN_NAME
+        elif ootb_type == OOTBType.FAITHFULNESS:
+            metric_result_key = Guard.get_stage_str(stage) + "_" + FAITHFULLNESS_COLUMN_NAME
+        elif ootb_type == OOTBType.AGENT_GOAL_ACCURACY:
+            metric_result_key = AGENT_GOAL_ACCURACY_COLUMN_NAME
+        elif ootb_type == OOTBType.CUSTOM_METRIC:
+            if metric_name is None:
+                raise ValueError(
+                    "For the custom metric type, a valid metric_name has to be provided."
+                )
+            metric_result_key = Guard.get_stage_str(stage) + "_" + metric_name
+        elif ootb_type == OOTBType.COST:
+            metric_result_key = COST_COLUMN_NAME
+        elif ootb_type == OOTBType.TASK_ADHERENCE:
+            metric_result_key = TASK_ADHERENCE_SCORE_COLUMN_NAME
+        else:
+            raise ValueError("The provided OOTB type is not implemented.")
+    elif guard_type == GuardType.NEMO_GUARDRAILS:
+        metric_result_key = Guard.get_stage_str(stage) + "_" + NEMO_GUARD_COLUMN_NAME
+    else:
+        raise ValueError("The provided guard type is not implemented.")
+    return metric_result_key
+
+
 class Guard(ABC):
     def __init__(self, config: dict, stage=None):
         self._name = config["name"]
@@ -244,6 +291,13 @@ class Guard(ABC):
         self._faas_url = config.get("faas_url")
         self._copy_citations = config["copy_citations"]
         self.is_agentic = config.get("is_agentic", False)
+        self.metric_column_name = get_metric_column_name(
+            config["type"],
+            config.get("ootb_type"),
+            self._stage,
+            config.get("model_info", {}).get("target_name"),
+            config["name"],
+        )
 
         if config.get("intervention"):
             self.intervention = GuardIntervention(config["intervention"])
@@ -448,11 +502,6 @@ class ModelGuard(Guard):
     def model_info(self):
         return self._model_info
 
-    def get_metric_column_name(self, stage):
-        if self.model_info is None:
-            raise NotImplementedError("Missing model_info for model guard")
-        return self.get_stage_str(stage) + "_" + self._model_info.target_name
-
     def get_span_column_name(self, _):
         if self.model_info is None:
             raise NotImplementedError("Missing model_info for model guard")
@@ -580,9 +629,6 @@ class NeMoGuard(Guard, GuardLLMMixin):
         """No average score metrics for NemoGuard's"""
         return False
 
-    def get_metric_column_name(self, stage):
-        return self.get_stage_str(stage) + "_" + NEMO_GUARD_COLUMN_NAME
-
     @property
     def nemo_llm_rails(self):
         return self._nemo_llm_rails
@@ -601,16 +647,6 @@ class OOTBGuard(Guard):
         """Latency is not tracked for token counts guards"""
         return self._ootb_type != OOTBType.TOKEN_COUNT
 
-    def get_metric_column_name(self, stage):
-        if self._ootb_type == OOTBType.TOKEN_COUNT:
-            return self.get_stage_str(stage) + "_" + TOKEN_COUNT_COLUMN_NAME
-        elif self._ootb_type == OOTBType.ROUGE_1:
-            return self.get_stage_str(stage) + "_" + ROUGE_1_COLUMN_NAME
-        elif self._ootb_type == OOTBType.CUSTOM_METRIC:
-            return self.get_stage_str(stage) + "_" + self.name
-        else:
-            raise NotImplementedError(f"No metric column name defined for {self._ootb_type} guard")
-
     def get_span_column_name(self, _):
         if self._ootb_type == OOTBType.TOKEN_COUNT:
             return TOKEN_COUNT_COLUMN_NAME
@@ -640,9 +676,6 @@ class OOTBCostMetric(OOTBGuard):
         self.output_unit = cost_config["output_unit"]
         self.output_multiplier = self.output_price / self.output_unit
 
-    def get_metric_column_name(self, _):
-        return COST_COLUMN_NAME
-
     def get_average_score_custom_metric_name(self, _):
         return f"Total cost in {self.currency}"
 
@@ -681,9 +714,6 @@ class FaithfulnessGuard(OOTBGuard, GuardLLMMixin):
         Settings.embed_model = None
         self._evaluator = FaithfulnessEvaluator()
 
-    def get_metric_column_name(self, stage):
-        return self.get_stage_str(stage) + "_" + FAITHFULLNESS_COLUMN_NAME
-
     @property
     def faithfulness_evaluator(self):
         return self._evaluator
@@ -705,12 +735,12 @@ class AgentGoalAccuracyGuard(OOTBGuard, GuardLLMMixin):
         # Default LLM Type for Agent Goal Accuracy is set to Azure OpenAI
         self._llm_type = config.get("llm_type", GuardLLMType.AZURE_OPENAI)
         llm = self.get_llm(config, self._llm_type)
-        evaluator_llm = LangchainLLMWrapper(llm)
+        if self._llm_type == GuardLLMType.AZURE_OPENAI:
+            evaluator_llm = LangchainLLMWrapper(llm)
+        else:
+            evaluator_llm = LlamaIndexLLMWrapper(llm)
         self.scorer = AgentGoalAccuracyWithoutReference(llm=evaluator_llm)
 
-    def get_metric_column_name(self, _):
-        return AGENT_GOAL_ACCURACY_COLUMN_NAME
-
     @property
     def accuracy_scorer(self):
         return self.scorer
@@ -735,9 +765,6 @@ class TaskAdherenceGuard(OOTBGuard, GuardLLMMixin):
         deepeval_llm = ModerationDeepEvalLLM(llm)
         self.scorer = TaskCompletionMetric(model=deepeval_llm, include_reason=True)
 
-    def get_metric_column_name(self, _):
-        return TASK_ADHERENCE_SCORE_COLUMN_NAME
-
     @property
     def task_adherence_scorer(self):
         return self.scorer

datarobot_dome/guard_executor.py

@@ -113,7 +113,7 @@ class AsyncGuardExecutor:
             if guard.has_latency_custom_metric():
                 self.pipeline.report_guard_latency(guard, latency)
             if guard.has_average_score_custom_metric():
-                metric_column_name = guard.get_metric_column_name(stage)
+                metric_column_name = guard.metric_column_name
                 if metric_column_name in df.columns:
                     span.set_attribute(
                         guard.get_span_attribute_name(stage),
@@ -222,7 +222,7 @@ class AsyncGuardExecutor:
             # and "Response_toxicity_toxic_PREDICTION", if toxicity is configured for both
             # prompts and responses
             copy_df.rename(
-                columns={metric_column: guard.get_metric_column_name(stage)},
+                columns={metric_column: guard.metric_column_name},
                 inplace=True,
             )
         except Exception as ex:
@@ -359,7 +359,7 @@ class AsyncGuardExecutor:
         else:
             prompt_column_name = self.pipeline.get_input_column(GuardStage.PROMPT)
             response_column_name = self.pipeline.get_input_column(GuardStage.RESPONSE)
-            metric_column_name = guard.get_metric_column_name(stage)
+            metric_column_name = guard.metric_column_name
 
             try:
                 copy_df[metric_column_name] = copy_df.apply(
@@ -407,7 +407,7 @@ class AsyncGuardExecutor:
         else:
             prompt_column_name = self.pipeline.get_input_column(GuardStage.PROMPT)
             response_column_name = self.pipeline.get_input_column(GuardStage.RESPONSE)
-            metric_column_name = guard.get_metric_column_name(stage)
+            metric_column_name = guard.metric_column_name
 
             try:
                 copy_df[metric_column_name] = copy_df.apply(
@@ -454,7 +454,7 @@ class AsyncGuardExecutor:
         else:
             prompt_column_name = self.pipeline.get_input_column(GuardStage.PROMPT)
             response_column_name = self.pipeline.get_input_column(GuardStage.RESPONSE)
-            metric_column_name = guard.get_metric_column_name(stage)
+            metric_column_name = guard.metric_column_name
 
             try:
                 copy_df[metric_column_name] = copy_df.apply(
@@ -500,7 +500,7 @@ class AsyncGuardExecutor:
             intervene = False
         else:
             input_column = self.pipeline.get_input_column(stage)
-            metric_column_name = guard.get_metric_column_name(stage)
+            metric_column_name = guard.metric_column_name
             copy_df[metric_column_name] = copy_df.apply(
                 lambda x: get_rouge_1_score(
                     scorer=self.pipeline.rouge_scorer,
@@ -517,7 +517,7 @@ class AsyncGuardExecutor:
 
         prompt_column_name = self.pipeline.get_input_column(GuardStage.PROMPT)
         response_column_name = self.pipeline.get_input_column(GuardStage.RESPONSE)
-        metric_column_name = guard.get_metric_column_name(stage)
+        metric_column_name = guard.metric_column_name
         if (
             PROMPT_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE not in copy_df.columns
             or RESPONSE_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE not in copy_df.columns
@@ -539,7 +539,7 @@ class AsyncGuardExecutor:
         if not isinstance(guard, OOTBGuard):
             raise ValueError(f"Guard object should be of type OOTBGuard, got: {type(guard)}")
         input_column = self.pipeline.get_input_column(stage)
-        metric_column_name = guard.get_metric_column_name(stage)
+        metric_column_name = guard.metric_column_name
         intervene = self._should_intervene(guard)
 
         if guard.ootb_type == OOTBType.TOKEN_COUNT:
@@ -594,7 +594,7 @@ class AsyncGuardExecutor:
             raise ValueError(f"Guard object should be of type NeMoGuard, got: {type(guard)}")
 
         input_column = self.pipeline.get_input_column(stage)
-        metric_column_name = guard.get_metric_column_name(stage)
+        metric_column_name = guard.metric_column_name
         intervene = self._should_intervene(guard)
 
         try:

datarobot_dome/guard_helpers.py

@@ -19,6 +19,7 @@ import tiktoken
 from deepeval.metrics import TaskCompletionMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.test_case import LLMTestCase
+from langchain_core.language_models import BaseLanguageModel
 from langchain_nvidia_ai_endpoints import ChatNVIDIA
 from langchain_nvidia_ai_endpoints import Model
 from langchain_nvidia_ai_endpoints import register_model
@@ -441,11 +442,21 @@ class ModerationDeepEvalLLM(DeepEvalBaseLLM):
         return self.llm
 
     def generate(self, prompt: str) -> str:
-        return self.llm.invoke(prompt).content
+        if isinstance(self.llm, BaseLanguageModel):
+            # Langchain LLM
+            return self.llm.invoke(prompt).content
+        else:
+            # LlamaIndex LLM
+            return self.llm.complete(prompt)
 
     async def a_generate(self, prompt: str) -> str:
-        res = await self.llm.ainvoke(prompt)
-        return res.content
+        if isinstance(self.llm, BaseLanguageModel):
+            # Langchain LLM
+            res = await self.llm.ainvoke(prompt)
+            return res.content
+        else:
+            res = await self.llm.acomplete(prompt)
+            return res.text
 
     def get_model_name(self):
         return "DeepEval LLM for Moderation"

datarobot_dome/guards/guard_llm_mixin.py

@@ -14,6 +14,7 @@ import os
 
 import datarobot as dr
 import trafaret as t
+from llama_index.llms.openai import OpenAI
 
 from datarobot_dome.constants import AWS_ACCOUNT_SECRET_DEFINITION_SUFFIX
 from datarobot_dome.constants import GOOGLE_SERVICE_ACCOUNT_SECRET_DEFINITION_SUFFIX
@@ -172,7 +173,7 @@ class GuardLLMMixin:
                         "api_key": openai_api_key,
                     }
                     os.environ["OPENAI_API_KEY"] = openai_api_key
-                    llm = "default"
+                    llm = OpenAI()
                 elif llm_type == GuardLLMType.AZURE_OPENAI:
                     if openai_api_base is None:
                         raise ValueError("OpenAI API base url is required for LLM Guard")
@@ -229,13 +230,6 @@ class GuardLLMMixin:
                     aws_region=config["aws_region"],
                 )
             elif llm_type == GuardLLMType.DATAROBOT:
-                if config["type"] == GuardType.OOTB and config["ootb_type"] in [
-                    OOTBType.AGENT_GOAL_ACCURACY,
-                    OOTBType.TASK_ADHERENCE,
-                ]:
-                    # DataRobot LLM does not implement generate / agenerate yet
-                    # so can't support it for Agent Goal Accuracy
-                    raise NotImplementedError
                 if config.get("deployment_id") is None:
                     raise ValueError("Deployment ID is required for LLM Guard")
                 deployment = dr.Deployment.get(config["deployment_id"])

datarobot_dome/pipeline/llm_pipeline.py

@@ -304,7 +304,7 @@ class LLMPipeline(Pipeline):
             metric_list = [
                 (
                     guard.get_average_score_custom_metric_name(guard.stage),
-                    guard.get_metric_column_name(guard.stage),
+                    guard.metric_column_name,
                 )
             ]
         if intervention_action:

datarobot_dome/streaming.py

@@ -323,7 +323,7 @@ class ModerationIterator:
                 self.aggregated_metrics_df[column_name] = metrics_df[column_name]
             elif guard.type == GuardType.OOTB:
                 if guard.ootb_type == OOTBType.TOKEN_COUNT:
-                    column_name = guard.get_metric_column_name(GuardStage.RESPONSE)
+                    column_name = guard.metric_column_name
                     self.aggregated_metrics_df[column_name] += metrics_df[column_name]
                 else:
                     # Faithfulness, ROUGE-1 can't run on chunks so no merging
@@ -368,7 +368,7 @@ class ModerationIterator:
         postscore_df = postscore_df_chunk.copy(deep=True)
         for guard in self.pipeline.get_postscore_guards():
             if not self._guard_can_work_on_chunk(guard):
-                metric_column_name = guard.get_metric_column_name(GuardStage.RESPONSE)
+                metric_column_name = guard.metric_column_name
                 if metric_column_name in postscore_df_assembled.columns:
                     postscore_df[metric_column_name] = postscore_df_assembled[metric_column_name]
                 if guard.has_latency_custom_metric():

{datarobot_moderations-11.2.1.dist-info → datarobot_moderations-11.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datarobot-moderations
-Version: 11.2.1
+Version: 11.2.3
 Summary: DataRobot Monitoring and Moderation framework
 License: DataRobot Tool and Utility Agreement
 Author: DataRobot
@@ -15,7 +15,7 @@ Requires-Dist: aiohttp (>=3.9.5)
 Requires-Dist: backoff (>=2.2.1)
 Requires-Dist: datarobot (>=3.6.0)
 Requires-Dist: datarobot-predict (>=1.9.6)
-Requires-Dist: deepeval (==2.7.9)
+Requires-Dist: deepeval (>=3.3.5)
 Requires-Dist: langchain (>=0.1.12)
 Requires-Dist: langchain-nvidia-ai-endpoints (>=0.3.9)
 Requires-Dist: langchain-openai (>=0.1.7)

{datarobot_moderations-11.2.1.dist-info → datarobot_moderations-11.2.3.dist-info}/RECORD

@@ -2,23 +2,23 @@ datarobot_dome/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,58
 datarobot_dome/async_http_client.py,sha256=wkB4irwvnchNGzO1bk2C_HWM-GOSB3AUn5TXKl-X0ZI,9649
 datarobot_dome/chat_helper.py,sha256=BzvtUyZSZxzOqq-5a2wQKhHhr2kMlcP1MFrHaDAeD_o,9671
 datarobot_dome/constants.py,sha256=vM2_JkXbn4dkWARCqxNfLriSo0E05LDXVrwNktptpuc,10416
-datarobot_dome/drum_integration.py,sha256=HresblJwlCk_sRnWReWQWeZMg5rYzKTA2hjmy1Rcn6U,40553
-datarobot_dome/guard.py,sha256=1INYx17n9ToiB5bzI-jIReUUuqkK_ucxpOx4jQLts6g,33264
-datarobot_dome/guard_executor.py,sha256=AOI8MZeZETHMoFgBePe0wa2vE9d2975MYQnEDHLZL7s,35462
-datarobot_dome/guard_helpers.py,sha256=ajxm-w7MS7eN5DMMO-jbbzjcOYMZ-LvhO53n2NI5_Fk,16773
+datarobot_dome/drum_integration.py,sha256=BnhAP-D4AaEeh4ferZ-qXnORuWQzYzw9qKAZUTZZnJU,40542
+datarobot_dome/guard.py,sha256=xJds9hcbUaS-KD5nC1mn0GiPdBrileFUu6BuTAjDNuY,34668
+datarobot_dome/guard_executor.py,sha256=ox5_jOHcqMaxaaagIYJJHhCwEI7Wg-rUEiu5rutsfVU,35363
+datarobot_dome/guard_helpers.py,sha256=jfu8JTWCcxu4WD1MKxeP1n53DeebY3SSuP-t5sWyV1U,17187
 datarobot_dome/guards/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
-datarobot_dome/guards/guard_llm_mixin.py,sha256=VovlpNZjWIGamF4SSvLF5lzOFyApH5IoOiB_qtCmRg0,12216
+datarobot_dome/guards/guard_llm_mixin.py,sha256=OIjOHeIAwJiM8BQOfqj1fY2jy-jJfc_CNToYrzG_6xk,11871
 datarobot_dome/llm.py,sha256=L02OvTrflmD34-FrfXebfF-zzKTeuin7fpne1Cl5psg,5719
 datarobot_dome/metrics/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
 datarobot_dome/metrics/citation_metrics.py,sha256=l2mnV1gz7nQeJ_yfaS4dcP3DFWf0p5QIBnKQ6shLnw4,4652
 datarobot_dome/metrics/factory.py,sha256=7caa8paI9LuFXDgguXdC4on28V7IwwIsKJT2Z-Aps8A,2187
 datarobot_dome/metrics/metric_scorer.py,sha256=uJ_IJRw7ZFHueg8xjsaXbt0ypO7JiydZ0WapCp96yng,2540
 datarobot_dome/pipeline/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
-datarobot_dome/pipeline/llm_pipeline.py,sha256=g7PAiLOMADr2DQFrtg2NrUj4u_tcvnoiJXrBR8xWsmY,18789
+datarobot_dome/pipeline/llm_pipeline.py,sha256=DMZ4gu88MiSSEQtshDyHOzT3R2Seuf8UqZ7A36QHj3M,18772
 datarobot_dome/pipeline/pipeline.py,sha256=7UmvrZtNxTGewpgM4cf2oThHPoJSarEU1Dyp7xEsASU,17401
 datarobot_dome/pipeline/vdb_pipeline.py,sha256=q3c_Z-hGUqhH6j6n8VpS3wZiBIkWgpRDsBnyJyZhiw4,9855
 datarobot_dome/runtime.py,sha256=FD8wXOweqoQVzbZMh-mucL66xT2kGxPsJUGAcJBgwxw,1468
-datarobot_dome/streaming.py,sha256=6nYvh6SoxPRLfO6GGdEoHsQuyLP9oX1lDMe8IeGo4lw,17801
-datarobot_moderations-11.2.1.dist-info/METADATA,sha256=fEyM5I3z0qS9dT2ofZ3J8UAN_ybm2SBF_46mifEHIpA,4742
-datarobot_moderations-11.2.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datarobot_moderations-11.2.1.dist-info/RECORD,,
+datarobot_dome/streaming.py,sha256=DkvKEH0yN0aPEWMTAjMFJB3Kx4iLGdjUMQU1pAplbeg,17751
+datarobot_moderations-11.2.3.dist-info/METADATA,sha256=dzpTYxhAXg-NEm8Rrko8U8qvbQncQoGw93a9ZhWV3jo,4742
+datarobot_moderations-11.2.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datarobot_moderations-11.2.3.dist-info/RECORD,,