datarobot-moderations 11.2.10__py3-none-any.whl → 11.2.11__py3-none-any.whl

datarobot_dome/__init__.py

@@ -1,5 +1,5 @@
 #  ---------------------------------------------------------------------------------
-#  Copyright (c) 2025 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
 #  Last updated 2025.
 #
 #  DataRobot, Inc. Confidential.

datarobot_dome/async_http_client.py

@@ -1,5 +1,5 @@
 #  ---------------------------------------------------------------------------------
-#  Copyright (c) 2025 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
 #  Last updated 2025.
 #
 #  DataRobot, Inc. Confidential.

datarobot_dome/chat_helper.py

@@ -1,5 +1,5 @@
 #  ---------------------------------------------------------------------------------
-#  Copyright (c) 2025 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
 #  Last updated 2025.
 #
 #  DataRobot, Inc. Confidential.

datarobot_dome/constants.py

@@ -1,5 +1,5 @@
 #  ---------------------------------------------------------------------------------
-#  Copyright (c) 2025 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
 #  Last updated 2025.
 #
 #  DataRobot, Inc. Confidential.
@@ -31,6 +31,7 @@ COST_COLUMN_NAME = "cost"
 FAITHFULLNESS_COLUMN_NAME = "faithfulness"
 AGENT_GOAL_ACCURACY_COLUMN_NAME = "agent_goal_accuracy"
 TASK_ADHERENCE_SCORE_COLUMN_NAME = "task_adherence_score"
+GUIDELINE_ADHERENCE_COLUMN_NAME = "agent_guideline_adherence_score"
 
 CUSTOM_METRIC_DESCRIPTION_SUFFIX = "Created by DataRobot Moderation System"
 
@@ -105,8 +106,9 @@ class GuardType:
     OOTB = "ootb"  # Out of the Box
     MODEL = "model"  # wraps a deployed model
     NEMO_GUARDRAILS = "nemo_guardrails"  # Nemo guardrails
+    NEMO_EVALUATOR = "nemo_evaluator"  # Nemo evaluator
 
-    ALL = [MODEL, NEMO_GUARDRAILS, OOTB]
+    ALL = [OOTB, MODEL, NEMO_GUARDRAILS, NEMO_EVALUATOR]
 
 
 class OOTBType:
@@ -117,6 +119,7 @@ class OOTBType:
     CUSTOM_METRIC = "custom_metric"
     COST = "cost"
     TASK_ADHERENCE = "task_adherence"
+    GUIDELINE_ADHERENCE = "agent_guideline_adherence"
 
     ALL = [
         TOKEN_COUNT,
@@ -126,6 +129,27 @@ class OOTBType:
         COST,
         AGENT_GOAL_ACCURACY,
         TASK_ADHERENCE,
+        GUIDELINE_ADHERENCE,
+    ]
+
+
+class NemoEvaluatorType:
+    LLM_JUDGE = "llm_judge"
+    CONTEXT_RELEVANCE = "context_relevance"
+    RESPONSE_GROUNDEDNESS = "response_groundedness"
+    TOPIC_ADHERENCE = "topic_adherence"
+    AGENT_GOAL_ACCURACY = "agent_goal_accuracy"
+    RESPONSE_RELEVANCY = "response_relevancy"
+    FAITHFULNESS = "faithfulness"
+
+    ALL = [
+        LLM_JUDGE,
+        CONTEXT_RELEVANCE,
+        RESPONSE_GROUNDEDNESS,
+        TOPIC_ADHERENCE,
+        AGENT_GOAL_ACCURACY,
+        RESPONSE_RELEVANCY,
+        FAITHFULNESS,
     ]
 
 

datarobot_dome/drum_integration.py

@@ -1,5 +1,5 @@
 #  ---------------------------------------------------------------------------------
-#  Copyright (c) 2025 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
 #  Last updated 2025.
 #
 #  DataRobot, Inc. Confidential.

datarobot_dome/guard_executor.py

@@ -1,5 +1,5 @@
 #  ---------------------------------------------------------------------------------
-#  Copyright (c) 2025 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
 #  Last updated 2025.
 #
 #  DataRobot, Inc. Confidential.
@@ -32,14 +32,8 @@ from datarobot_dome.constants import GuardTimeoutAction
 from datarobot_dome.constants import GuardType
 from datarobot_dome.constants import ModerationEventTypes
 from datarobot_dome.constants import OOTBType
-from datarobot_dome.guard import AgentGoalAccuracyGuard
-from datarobot_dome.guard import FaithfulnessGuard
-from datarobot_dome.guard import ModelGuard
-from datarobot_dome.guard import NeMoGuard
-from datarobot_dome.guard import OOTBCostMetric
-from datarobot_dome.guard import OOTBGuard
-from datarobot_dome.guard import TaskAdherenceGuard
 from datarobot_dome.guard_helpers import calculate_agent_goal_accuracy
+from datarobot_dome.guard_helpers import calculate_agent_guideline_adherence
 from datarobot_dome.guard_helpers import calculate_faithfulness
 from datarobot_dome.guard_helpers import calculate_task_adherence
 from datarobot_dome.guard_helpers import calculate_token_counts_for_cost_calculations
@@ -48,6 +42,14 @@ from datarobot_dome.guard_helpers import get_rouge_1_score
 from datarobot_dome.guard_helpers import get_token_count
 from datarobot_dome.guard_helpers import nemo_response_stage_input_formatter
 from datarobot_dome.guard_helpers import nemo_response_stage_output_formatter
+from datarobot_dome.guards import ModelGuard
+from datarobot_dome.guards import NeMoGuard
+from datarobot_dome.guards import OOTBAgentGoalAccuracyGuard
+from datarobot_dome.guards import OOTBCostMetric
+from datarobot_dome.guards import OOTBFaithfulnessGuard
+from datarobot_dome.guards import OOTBGuard
+from datarobot_dome.guards import OOTBTaskAdherenceGuard
+from datarobot_dome.guards.ootb_guard import OOTBAgentGuidelineAdherence
 
 tracer = trace.get_tracer(__name__)
 
@@ -173,7 +175,7 @@ class AsyncGuardExecutor:
         metric_column = guard.model_info.target_name
 
         llm_input_column = self.pipeline.get_input_column(stage)
-        guard_input_column = guard.get_input_column(stage)
+        guard_input_column = guard.get_input_column_name(stage)
 
         intervene = self._should_intervene(guard)
         try:
@@ -335,9 +337,9 @@ class AsyncGuardExecutor:
         return self.intervene(guard, copy_df, input_column, metric_column)
 
     async def _handle_faithfulness(self, guard, copy_df, stage, intervene):
-        if not isinstance(guard, FaithfulnessGuard):
+        if not isinstance(guard, OOTBFaithfulnessGuard):
             raise ValueError(
-                f"Guard object should be of type FaithfulnessGuard, got: {type(guard)}"
+                f"Guard object should be of type OOTBFaithfulnessGuard, got: {type(guard)}"
             )
         if stage == GuardStage.PROMPT:
             raise ValueError("Faithfulness only supports evaluating the response")
@@ -387,9 +389,9 @@ class AsyncGuardExecutor:
         return copy_df, intervene
 
     async def _handle_agent_goal_accuracy(self, guard, copy_df, stage):
-        if not isinstance(guard, AgentGoalAccuracyGuard):
+        if not isinstance(guard, OOTBAgentGoalAccuracyGuard):
             raise ValueError(
-                f"Guard object should be of type AgentGoalAccuracyGuard, got: {type(guard)}"
+                f"Guard object should be of type OOTBAgentGoalAccuracyGuard, got: {type(guard)}"
             )
         if stage == GuardStage.PROMPT:
             raise ValueError("Agent Goal Accuracy only supports evaluating the response")
@@ -434,9 +436,9 @@ class AsyncGuardExecutor:
         return copy_df
 
     async def _handle_task_adherence(self, guard, copy_df, stage):
-        if not isinstance(guard, TaskAdherenceGuard):
+        if not isinstance(guard, OOTBTaskAdherenceGuard):
             raise ValueError(
-                f"Guard object should be of type TaskAdherenceGuard, got: {type(guard)}"
+                f"Guard object should be of type OOTBTaskAdherenceGuard, got: {type(guard)}"
             )
         if stage == GuardStage.PROMPT:
             raise ValueError("Task Adherence only supports evaluating the response")
@@ -511,6 +513,46 @@ class AsyncGuardExecutor:
             )
         return copy_df, intervene
 
+    async def _handle_guideline_adherence(self, guard, copy_df, stage):
+        if not isinstance(guard, OOTBAgentGuidelineAdherence):
+            raise ValueError(
+                f"Guard object should be of type AgentGuidelineAdherence, got: {type(guard)}"
+            )
+
+        if stage != GuardStage.RESPONSE:
+            raise ValueError("Guideline Adherence only supports evaluating the response")
+
+        citation_columns = get_citation_columns(copy_df.columns)
+        prompt_column_name = self.pipeline.get_input_column(GuardStage.PROMPT)
+        response_column_name = self.pipeline.get_input_column(GuardStage.RESPONSE)
+        metric_column_name = guard.metric_column_name
+        try:
+            copy_df[metric_column_name] = copy_df.apply(
+                lambda x: calculate_agent_guideline_adherence(
+                    scorer=guard.guideline_adherence_scorer,
+                    prompt=x[prompt_column_name],
+                    response=x[response_column_name],
+                    citations=[x[col] for col in citation_columns]
+                    if len(citation_columns) > 0
+                    else None,
+                ),
+                axis=1,
+            )
+
+        except Exception as e:
+            title = "Guideline Adherence calculation failed"
+            message = f"Exception: {e}"
+            self._logger.error(title + " " + message)
+            self._logger.error(traceback.format_exc())
+            await self.pipeline.send_event_async(
+                title,
+                message,
+                ModerationEventTypes.MODERATION_MODEL_RUNTIME_ERROR,
+                guard_name=guard.name,
+            )
+
+        return copy_df
+
     async def _handle_cost(self, guard, copy_df, stage):
         if not isinstance(guard, OOTBCostMetric):
             raise ValueError(f"Guard object should be of type OOTBCostMetric, got: {type(guard)}")
@@ -560,6 +602,10 @@ class AsyncGuardExecutor:
             copy_df = await self._handle_task_adherence(guard, copy_df, stage)
             # No intervention for task adherence
             intervene = False
+        elif guard.ootb_type == OOTBType.GUIDELINE_ADHERENCE:
+            copy_df = await self._handle_guideline_adherence(guard, copy_df, stage)
+            # No intervention for guideline adherence
+            intervene = False
         elif guard.ootb_type == OOTBType.CUSTOM_METRIC:
             body = {
                 "df": copy_df.to_dict(),
@@ -694,13 +740,18 @@ class AsyncGuardExecutor:
 
     def _get_input_df_for_the_guard(self, _input_df, join_columns, guard, stage):
         if stage == GuardStage.RESPONSE and isinstance(guard, OOTBGuard):
-            if guard.ootb_type in [OOTBType.ROUGE_1, OOTBType.FAITHFULNESS] or guard.copy_citations:
+            if (
+                guard.ootb_type
+                in [OOTBType.ROUGE_1, OOTBType.FAITHFULNESS, OOTBType.GUIDELINE_ADHERENCE]
+                or guard.copy_citations
+            ):
                 join_columns = join_columns.union(set(get_citation_columns(_input_df.columns)))
             if guard.ootb_type in [
                 OOTBType.FAITHFULNESS,
                 OOTBType.COST,
                 OOTBType.AGENT_GOAL_ACCURACY,
                 OOTBType.TASK_ADHERENCE,
+                OOTBType.GUIDELINE_ADHERENCE,
             ]:
                 join_columns.add(self.pipeline.get_input_column(GuardStage.PROMPT))
             if guard.ootb_type in [OOTBType.AGENT_GOAL_ACCURACY, OOTBType.TASK_ADHERENCE]:

datarobot_dome/guard_factory.py

@@ -0,0 +1,126 @@
+#  ---------------------------------------------------------------------------------
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Last updated 2025.
+#
+#  DataRobot, Inc. Confidential.
+#  This is proprietary source code of DataRobot, Inc. and its affiliates.
+#
+#  This file and its contents are subject to DataRobot Tool and Utility Agreement.
+#  For details, see
+#  https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
+#  ---------------------------------------------------------------------------------
+import os
+
+from datarobot_dome.constants import GuardAction
+from datarobot_dome.constants import GuardOperatorType
+from datarobot_dome.constants import GuardType
+from datarobot_dome.constants import NemoEvaluatorType
+from datarobot_dome.constants import OOTBType
+from datarobot_dome.guards import ModelGuard
+from datarobot_dome.guards import NeMoAgentGoalAccuracyGuard
+from datarobot_dome.guards import NeMoContextRelevanceGuard
+from datarobot_dome.guards import NeMoFaithfulnessGuard
+from datarobot_dome.guards import NeMoGuard
+from datarobot_dome.guards import NeMoLLMJudgeGuard
+from datarobot_dome.guards import NeMoResponseGroundednessGuard
+from datarobot_dome.guards import NeMoResponseRelevancyGuard
+from datarobot_dome.guards import NeMoTopicAdherenceGuard
+from datarobot_dome.guards import OOTBAgentGoalAccuracyGuard
+from datarobot_dome.guards import OOTBCostMetric
+from datarobot_dome.guards import OOTBFaithfulnessGuard
+from datarobot_dome.guards import OOTBGuard
+from datarobot_dome.guards import OOTBTaskAdherenceGuard
+from datarobot_dome.guards.base import Guard
+from datarobot_dome.guards.ootb_guard import OOTBAgentGuidelineAdherence
+from datarobot_dome.guards.validation import guard_trafaret
+
+
+class GuardFactory:
+    @classmethod
+    def _perform_post_validation_checks(cls, guard_config):
+        if not guard_config.get("intervention"):
+            return
+
+        if guard_config["intervention"]["action"] == GuardAction.BLOCK and (
+            guard_config["intervention"]["message"] is None
+            or len(guard_config["intervention"]["message"]) == 0
+        ):
+            raise ValueError("Blocked action needs a blocking message")
+
+        if guard_config["intervention"]["action"] == GuardAction.REPLACE:
+            if "model_info" not in guard_config:
+                raise ValueError("'Replace' action needs model_info section")
+            if (
+                "replacement_text_column_name" not in guard_config["model_info"]
+                or guard_config["model_info"]["replacement_text_column_name"] is None
+                or len(guard_config["model_info"]["replacement_text_column_name"]) == 0
+            ):
+                raise ValueError(
+                    "'Replace' action needs valid 'replacement_text_column_name' "
+                    "in 'model_info' section of the guard"
+                )
+
+        if not guard_config["intervention"].get("conditions"):
+            return
+
+        if len(guard_config["intervention"]["conditions"]) == 0:
+            return
+
+        condition = guard_config["intervention"]["conditions"][0]
+        if condition["comparator"] in GuardOperatorType.REQUIRES_LIST_COMPARAND:
+            if not isinstance(condition["comparand"], list):
+                raise ValueError(
+                    f"Comparand needs to be a list with {condition['comparator']} comparator"
+                )
+        elif isinstance(condition["comparand"], list):
+            raise ValueError(
+                f"Comparand needs to be a scalar with {condition['comparator']} comparator"
+            )
+
+    @staticmethod
+    def create(input_config: dict, stage=None, model_dir: str = os.getcwd()) -> Guard:
+        config = guard_trafaret.check(input_config)
+
+        GuardFactory._perform_post_validation_checks(config)
+
+        if config["type"] == GuardType.MODEL:
+            guard = ModelGuard(config, stage)
+        elif config["type"] == GuardType.OOTB:
+            if config["ootb_type"] == OOTBType.FAITHFULNESS:
+                guard = OOTBFaithfulnessGuard(config, stage)
+            elif config["ootb_type"] == OOTBType.COST:
+                guard = OOTBCostMetric(config, stage)
+            elif config["ootb_type"] == OOTBType.AGENT_GOAL_ACCURACY:
+                guard = OOTBAgentGoalAccuracyGuard(config, stage)
+            elif config["ootb_type"] == OOTBType.TASK_ADHERENCE:
+                guard = OOTBTaskAdherenceGuard(config, stage)
+            elif config["ootb_type"] == OOTBType.GUIDELINE_ADHERENCE:
+                guard = OOTBAgentGuidelineAdherence(config, stage)
+            else:
+                guard = OOTBGuard(config, stage)
+        elif config["type"] == GuardType.NEMO_GUARDRAILS:
+            guard = NeMoGuard(config, stage, model_dir)
+        elif config["type"] == GuardType.NEMO_EVALUATOR:
+            match config["nemo_evaluator_type"]:
+                case NemoEvaluatorType.LLM_JUDGE:
+                    guard = NeMoLLMJudgeGuard(config, stage)
+                case NemoEvaluatorType.CONTEXT_RELEVANCE:
+                    guard = NeMoContextRelevanceGuard(config, stage)
+                case NemoEvaluatorType.RESPONSE_GROUNDEDNESS:
+                    guard = NeMoResponseGroundednessGuard(config, stage)
+                case NemoEvaluatorType.TOPIC_ADHERENCE:
+                    guard = NeMoTopicAdherenceGuard(config, stage)
+                case NemoEvaluatorType.AGENT_GOAL_ACCURACY:
+                    guard = NeMoAgentGoalAccuracyGuard(config, stage)
+                case NemoEvaluatorType.RESPONSE_RELEVANCY:
+                    guard = NeMoResponseRelevancyGuard(config, stage)
+                case NemoEvaluatorType.FAITHFULNESS:
+                    guard = NeMoFaithfulnessGuard(config, stage)
+                case _:
+                    raise ValueError(
+                        f"Invalid guard type: {config['type']} - {config['nemo_evaluator_type']}"
+                    )
+        else:
+            raise ValueError(f"Invalid guard type: {config['type']}")
+
+        return guard

datarobot_dome/guard_helpers.py

@@ -1,5 +1,5 @@
 #  ---------------------------------------------------------------------------------
-#  Copyright (c) 2025 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
 #  Last updated 2025.
 #
 #  DataRobot, Inc. Confidential.
@@ -27,6 +27,7 @@ from langchain_nvidia_ai_endpoints._statics import determine_model
 from langchain_openai import AzureChatOpenAI
 from langchain_openai import ChatOpenAI
 from llama_index.core.evaluation import FaithfulnessEvaluator
+from llama_index.core.evaluation import GuidelineEvaluator
 from ragas import MultiTurnSample
 from ragas.messages import AIMessage
 from ragas.messages import HumanMessage
@@ -434,6 +435,20 @@ def calculate_agent_goal_accuracy(
     return scorer.multi_turn_score(sample)
 
 
+def calculate_agent_guideline_adherence(
+    scorer: GuidelineEvaluator,
+    prompt: str,
+    response: str,
+    citations: list[str] | None = None,
+):
+    evaluation_results = scorer.evaluate(
+        query=prompt,
+        context=citations,
+        response=response,
+    )
+    return evaluation_results.passing
+
+
 class ModerationDeepEvalLLM(DeepEvalBaseLLM):
     def __init__(self, llm, *args, **kwargs):
         self.llm = llm

datarobot_dome/guards/__init__.py

@@ -1,5 +1,5 @@
 #  ---------------------------------------------------------------------------------
-#  Copyright (c) 2025 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
 #  Last updated 2025.
 #
 #  DataRobot, Inc. Confidential.
@@ -9,3 +9,18 @@
 #  For details, see
 #  https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
 #  ---------------------------------------------------------------------------------
+from .model_guard import ModelGuard
+from .nemo_evaluator import NeMoAgentGoalAccuracyGuard
+from .nemo_evaluator import NeMoContextRelevanceGuard
+from .nemo_evaluator import NeMoEvaluatorGuard
+from .nemo_evaluator import NeMoFaithfulnessGuard
+from .nemo_evaluator import NeMoLLMJudgeGuard
+from .nemo_evaluator import NeMoResponseGroundednessGuard
+from .nemo_evaluator import NeMoResponseRelevancyGuard
+from .nemo_evaluator import NeMoTopicAdherenceGuard
+from .nemo_guard import NeMoGuard
+from .ootb_guard import OOTBAgentGoalAccuracyGuard
+from .ootb_guard import OOTBCostMetric
+from .ootb_guard import OOTBFaithfulnessGuard
+from .ootb_guard import OOTBGuard
+from .ootb_guard import OOTBTaskAdherenceGuard