datarobot-moderations 11.2.10__py3-none-any.whl → 11.2.12__py3-none-any.whl

datarobot_dome/guards/base.py

@@ -0,0 +1,259 @@
+#  ---------------------------------------------------------------------------------
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Last updated 2026.
+#
+#  DataRobot, Inc. Confidential.
+#  This is proprietary source code of DataRobot, Inc. and its affiliates.
+#
+#  This file and its contents are subject to DataRobot Tool and Utility Agreement.
+#  For details, see
+#  https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
+#  ---------------------------------------------------------------------------------
+from abc import ABC
+
+from datarobot.enums import CustomMetricAggregationType
+from datarobot.enums import CustomMetricDirectionality
+
+from datarobot_dome.constants import AGENT_GOAL_ACCURACY_COLUMN_NAME
+from datarobot_dome.constants import COST_COLUMN_NAME
+from datarobot_dome.constants import CUSTOM_METRIC_DESCRIPTION_SUFFIX
+from datarobot_dome.constants import DEFAULT_PROMPT_COLUMN_NAME
+from datarobot_dome.constants import DEFAULT_RESPONSE_COLUMN_NAME
+from datarobot_dome.constants import FAITHFULLNESS_COLUMN_NAME
+from datarobot_dome.constants import GUIDELINE_ADHERENCE_COLUMN_NAME
+from datarobot_dome.constants import NEMO_GUARD_COLUMN_NAME
+from datarobot_dome.constants import ROUGE_1_COLUMN_NAME
+from datarobot_dome.constants import SPAN_PREFIX
+from datarobot_dome.constants import TASK_ADHERENCE_SCORE_COLUMN_NAME
+from datarobot_dome.constants import TOKEN_COUNT_COLUMN_NAME
+from datarobot_dome.constants import GuardAction
+from datarobot_dome.constants import GuardStage
+from datarobot_dome.constants import GuardType
+from datarobot_dome.constants import NemoEvaluatorType
+from datarobot_dome.constants import OOTBType
+
+
+def get_metric_column_name(
+    guard_type: GuardType,
+    ootb_type: OOTBType | None,
+    stage: GuardStage,
+    model_guard_target_name: str | None = None,
+    metric_name: str | None = None,
+    nemo_evaluator_type: str | None = None,
+) -> str:
+    """Gets the metric column name. Note that this function gets used in buzok code. If you update
+    it, please also update the moderation library in the buzok worker image.
+    """
+    if guard_type == GuardType.MODEL:
+        if model_guard_target_name is None:
+            raise ValueError(
+                "For the model guard type, a valid model_guard_target_name has to be provided."
+            )
+        metric_result_key = Guard.get_stage_str(stage) + "_" + model_guard_target_name
+    elif guard_type == GuardType.OOTB:
+        if ootb_type is None:
+            raise ValueError("For the OOTB type, a valid OOTB guard type has to be provided.")
+        elif ootb_type == OOTBType.TOKEN_COUNT:
+            metric_result_key = Guard.get_stage_str(stage) + "_" + TOKEN_COUNT_COLUMN_NAME
+        elif ootb_type == OOTBType.ROUGE_1:
+            metric_result_key = Guard.get_stage_str(stage) + "_" + ROUGE_1_COLUMN_NAME
+        elif ootb_type == OOTBType.FAITHFULNESS:
+            metric_result_key = Guard.get_stage_str(stage) + "_" + FAITHFULLNESS_COLUMN_NAME
+        elif ootb_type == OOTBType.AGENT_GOAL_ACCURACY:
+            metric_result_key = AGENT_GOAL_ACCURACY_COLUMN_NAME
+        elif ootb_type == OOTBType.CUSTOM_METRIC:
+            if metric_name is None:
+                raise ValueError(
+                    "For the custom metric type, a valid metric_name has to be provided."
+                )
+            metric_result_key = Guard.get_stage_str(stage) + "_" + metric_name
+        elif ootb_type == OOTBType.COST:
+            metric_result_key = COST_COLUMN_NAME
+        elif ootb_type == OOTBType.TASK_ADHERENCE:
+            metric_result_key = TASK_ADHERENCE_SCORE_COLUMN_NAME
+        elif ootb_type == OOTBType.GUIDELINE_ADHERENCE:
+            metric_result_key = GUIDELINE_ADHERENCE_COLUMN_NAME
+        else:
+            raise ValueError("The provided OOTB type is not implemented.")
+    elif guard_type == GuardType.NEMO_GUARDRAILS:
+        metric_result_key = Guard.get_stage_str(stage) + "_" + NEMO_GUARD_COLUMN_NAME
+    elif guard_type == GuardType.NEMO_EVALUATOR:
+        if nemo_evaluator_type == NemoEvaluatorType.LLM_JUDGE:
+            metric_result_key = f"{Guard.get_stage_str(stage)}_nemo_{nemo_evaluator_type}"
+        elif nemo_evaluator_type in NemoEvaluatorType.ALL:
+            metric_result_key = f"nemo_{nemo_evaluator_type}"
+        else:
+            raise ValueError("The provided NeMo Evaluator type is not implemented.")
+    else:
+        raise ValueError("The provided guard type is not implemented.")
+    return metric_result_key
+
+
+class GuardIntervention:
+    def __init__(self, intervention_config: dict) -> None:
+        self.action = intervention_config["action"]
+        self.message = intervention_config.get("message")
+        self.threshold = None
+        self.comparator = None
+        if (
+            "conditions" in intervention_config
+            and intervention_config["conditions"] is not None
+            and len(intervention_config["conditions"]) > 0
+        ):
+            self.threshold = intervention_config["conditions"][0].get("comparand")
+            self.comparator = intervention_config["conditions"][0].get("comparator")
+
+
+class Guard(ABC):
+    def __init__(self, config: dict, stage=None):
+        self._name = config["name"]
+        self._description = config.get("description")
+        self._type = config["type"]
+        self._stage = stage if stage else config["stage"]
+        self._pipeline = None
+        self.intervention = None
+        self._deployment_id = config.get("deployment_id")
+        self._dr_cm = None
+        self._faas_url = config.get("faas_url")
+        self._copy_citations = config["copy_citations"]
+        self.is_agentic = config.get("is_agentic", False)
+        self.metric_column_name = get_metric_column_name(
+            config["type"],
+            config.get("ootb_type"),
+            self._stage,
+            config.get("model_info", {}).get("target_name"),
+            config["name"],
+            config.get("nemo_evaluator_type"),
+        )
+        if config.get("intervention"):
+            self.intervention = GuardIntervention(config["intervention"])
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    @property
+    def type(self) -> GuardType:
+        return self._type
+
+    @property
+    def stage(self) -> GuardStage:
+        return self._stage
+
+    @property
+    def faas_url(self) -> str:
+        return self._faas_url
+
+    @property
+    def copy_citations(self) -> str:
+        return self._copy_citations
+
+    def set_pipeline(self, pipeline):
+        self._pipeline = pipeline
+
+    @property
+    def llm_type(self):
+        return self._llm_type
+
+    @staticmethod
+    def get_stage_str(stage):
+        return "Prompts" if stage == GuardStage.PROMPT else "Responses"
+
+    def get_input_column_name(self, stage) -> str:
+        match stage:
+            case GuardStage.PROMPT:
+                return DEFAULT_PROMPT_COLUMN_NAME
+            case GuardStage.RESPONSE:
+                return DEFAULT_RESPONSE_COLUMN_NAME
+            case _:
+                raise ValueError(f"Stage ({stage}) is not supported.")
+
+    def has_latency_custom_metric(self) -> bool:
+        """Determines if latency metric is tracked for this guard type. Default is True."""
+        return True
+
+    def get_latency_custom_metric_name(self):
+        return f"{self.name} Guard Latency"
+
+    def get_latency_custom_metric(self):
+        return {
+            "name": self.get_latency_custom_metric_name(),
+            "directionality": CustomMetricDirectionality.LOWER_IS_BETTER,
+            "units": "seconds",
+            "type": CustomMetricAggregationType.AVERAGE,
+            "baselineValue": 0,
+            "isModelSpecific": True,
+            "timeStep": "hour",
+            "description": (
+                f"{self.get_latency_custom_metric_name()}.  {CUSTOM_METRIC_DESCRIPTION_SUFFIX}"
+            ),
+        }
+
+    def has_average_score_custom_metric(self) -> bool:
+        """Determines if an average score metric is tracked for this guard type. Default is True."""
+        return True
+
+    def get_average_score_custom_metric_name(self, stage):
+        return f"{self.name} Guard Average Score for {self.get_stage_str(stage)}"
+
+    def get_average_score_metric(self, stage):
+        return {
+            "name": self.get_average_score_custom_metric_name(stage),
+            "directionality": CustomMetricDirectionality.LOWER_IS_BETTER,
+            "units": "probability",
+            "type": CustomMetricAggregationType.AVERAGE,
+            "baselineValue": 0,
+            "isModelSpecific": True,
+            "timeStep": "hour",
+            "description": (
+                f"{self.get_average_score_custom_metric_name(stage)}. "
+                f" {CUSTOM_METRIC_DESCRIPTION_SUFFIX}"
+            ),
+        }
+
+    def get_guard_enforced_custom_metric_name(self, stage, moderation_method):
+        if moderation_method == GuardAction.REPLACE:
+            return f"{self.name} Guard replaced {self.get_stage_str(stage)}"
+        return f"{self.name} Guard {moderation_method}ed {self.get_stage_str(stage)}"
+
+    def get_enforced_custom_metric(self, stage, moderation_method):
+        return {
+            "name": self.get_guard_enforced_custom_metric_name(stage, moderation_method),
+            "directionality": CustomMetricDirectionality.LOWER_IS_BETTER,
+            "units": "count",
+            "type": CustomMetricAggregationType.SUM,
+            "baselineValue": 0,
+            "isModelSpecific": True,
+            "timeStep": "hour",
+            "description": (
+                f"Number of {self.get_stage_str(stage)} {moderation_method}ed by the "
+                f"{self.name} guard.  {CUSTOM_METRIC_DESCRIPTION_SUFFIX}"
+            ),
+        }
+
+    def get_intervention_action(self):
+        if not self.intervention:
+            return GuardAction.NONE
+        return self.intervention.action
+
+    def get_comparand(self):
+        return self.intervention.threshold
+
+    def get_enforced_span_attribute_name(self, stage):
+        intervention_action = self.get_intervention_action()
+        if intervention_action in [GuardAction.BLOCK, GuardAction.REPORT]:
+            return f"{SPAN_PREFIX}.{stage.lower()}.{intervention_action}ed"
+        elif intervention_action == GuardAction.REPLACE:
+            return f"{SPAN_PREFIX}.{stage.lower()}.replaced"
+        else:
+            raise NotImplementedError
+
+    def get_span_column_name(self, _):
+        raise NotImplementedError
+
+    def get_span_attribute_name(self, _):
+        raise NotImplementedError

datarobot_dome/guards/guard_llm_mixin.py

@@ -1,6 +1,6 @@
 #  ---------------------------------------------------------------------------------
-#  Copyright (c) 2025 DataRobot, Inc. and its affiliates. All rights reserved.
-#  Last updated 2025.
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Last updated 2026.
 #
 #  DataRobot, Inc. Confidential.
 #  This is proprietary source code of DataRobot, Inc. and its affiliates.
@@ -94,6 +94,8 @@ class GuardLLMMixin:
                 return f"{secret_env_var_name_prefix}{OOTBType.AGENT_GOAL_ACCURACY}_{llm_type_str}"
             elif config["ootb_type"] == OOTBType.TASK_ADHERENCE:
                 return f"{secret_env_var_name_prefix}{OOTBType.TASK_ADHERENCE}_{llm_type_str}"
+            elif config["ootb_type"] == OOTBType.GUIDELINE_ADHERENCE:
+                return f"{secret_env_var_name_prefix}{OOTBType.GUIDELINE_ADHERENCE}_{llm_type_str}"
             else:
                 raise Exception("Invalid guard config for building env var name")
         else:

datarobot_dome/guards/model_guard.py

@@ -0,0 +1,84 @@
+#  ---------------------------------------------------------------------------------
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Last updated 2026.
+#
+#  DataRobot, Inc. Confidential.
+#  This is proprietary source code of DataRobot, Inc. and its affiliates.
+#
+#  This file and its contents are subject to DataRobot Tool and Utility Agreement.
+#  For details, see
+#  https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
+#  ---------------------------------------------------------------------------------
+import datarobot as dr
+
+from datarobot_dome.constants import SPAN_PREFIX
+
+from .base import Guard
+
+
+class GuardModelInfo:
+    def __init__(self, model_config: dict):
+        self._model_id = model_config.get("model_id")
+        self._target_name = model_config["target_name"]
+        self._target_type = model_config["target_type"]
+        self._class_names = model_config.get("class_names", [])
+        self._input_column_name = model_config["input_column_name"]
+        self._replacement_text_column_name = model_config.get("replacement_text_column_name")
+
+    @property
+    def model_id(self) -> str:
+        return self._model_id
+
+    @property
+    def target_name(self) -> str:
+        return self._target_name
+
+    @property
+    def target_type(self) -> str:
+        return self._target_type
+
+    @property
+    def class_names(self) -> list[str]:
+        return self._class_names
+
+    @property
+    def input_column_name(self) -> str:
+        return self._input_column_name
+
+    @property
+    def replacement_text_column_name(self) -> str:
+        return self._replacement_text_column_name
+
+
+class ModelGuard(Guard):
+    def __init__(self, config: dict, stage=None):
+        super().__init__(config, stage)
+        self._deployment_id = config["deployment_id"]
+        self._model_info = GuardModelInfo(config["model_info"])
+        # dr.Client is set in the Pipeline init, Lets query the deployment
+        # to get the prediction server information
+        self.deployment = dr.Deployment.get(self._deployment_id)
+
+    @property
+    def deployment_id(self) -> str:
+        return self._deployment_id
+
+    @property
+    def model_info(self):
+        return self._model_info
+
+    def get_input_column_name(self, stage) -> str:
+        return self._model_info.input_column_name
+
+    def get_span_column_name(self, _):
+        if self.model_info is None:
+            raise NotImplementedError("Missing model_info for model guard")
+        # Typically 0th index is the target name
+        return self._model_info.target_name.split("_")[0]
+
+    def get_span_attribute_name(self, stage):
+        return f"{SPAN_PREFIX}.{stage.lower()}.{self.get_span_column_name(stage)}"
+
+    def has_average_score_custom_metric(self) -> bool:
+        """A couple ModelGuard types do not have an average score metric"""
+        return self.model_info.target_type not in ["Multiclass", "TextGeneration"]

datarobot_dome/guards/nemo_evaluator.py

@@ -0,0 +1,329 @@
+#  ---------------------------------------------------------------------------------
+#  Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
+#  Last updated 2026.
+#
+#  DataRobot, Inc. Confidential.
+#  This is proprietary source code of DataRobot, Inc. and its affiliates.
+#
+#  This file and its contents are subject to DataRobot Tool and Utility Agreement.
+#  For details, see
+#  https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
+#  ---------------------------------------------------------------------------------
+from functools import cached_property
+from typing import Optional
+
+import requests
+from nemo_microservices import AsyncNeMoMicroservices
+from nemo_microservices.types import EvaluationConfigParam
+from nemo_microservices.types import EvaluationTargetParam
+from nemo_microservices.types import LiveEvaluation
+from nemo_microservices.types import MetricConfigParam
+from nemo_microservices.types import TaskConfigParam
+
+from datarobot_dome.constants import GuardLLMType
+from datarobot_dome.guard_helpers import get_datarobot_endpoint_and_token
+
+from .base import Guard
+
+
+class NeMoEvaluatorGuard(Guard):
+    def __init__(self, config: dict, stage=None):
+        super().__init__(config, stage)
+        self.nemo_evaluator_type = config["nemo_evaluator_type"]
+        self._llm_type = config["llm_type"]
+        self.llm_deployment_id = config.get("deployment_id")
+        self.llm_gateway_model_id = config.get("llm_gateway_model_id")
+
+    @cached_property
+    def _client(self) -> AsyncNeMoMicroservices:
+        """
+        Using localhost for development purpose only.
+        It will be replaced with url to a deployed NeMo evaluator instance later in the PBMP.
+        """
+        return AsyncNeMoMicroservices(base_url="http://localhost:8080")
+
+    @staticmethod
+    def _get_default_model_id(deployment_id: str) -> str:
+        """Get id of first model as default from a deployment that responds to /models endpoint."""
+        datarobot_endpoint, datarobot_api_token = get_datarobot_endpoint_and_token()
+        response = requests.get(
+            f"{datarobot_endpoint}/deployments/{deployment_id}/directAccess/models",
+            headers={"Authorization": f"Bearer {datarobot_api_token}"},
+        )
+        if response.status_code != 200:
+            raise ValueError(f"Unable to query for default model for deployment {deployment_id}")
+        try:
+            return response.json()["data"][0]["id"]
+        except (requests.JSONDecodeError, KeyError, IndexError, TypeError):
+            raise ValueError(f"Unable to select default model for deployment {deployment_id}")
+
+    @cached_property
+    def llm_judge_api_endpoint(self) -> dict:
+        """LLM Judge API endpoint, to be passed to NeMo evaluator."""
+        datarobot_endpoint, datarobot_api_token = get_datarobot_endpoint_and_token()
+        if self.llm_type == GuardLLMType.DATAROBOT:
+            url = f"{datarobot_endpoint}/deployments/{self.llm_deployment_id}/chat/completions"
+            model_id = self._get_default_model_id(self.llm_deployment_id)
+        elif self.llm_type == GuardLLMType.LLM_GATEWAY:
+            url = f"{datarobot_endpoint}/genai/llmgw/chat/completions"
+            model_id = self.llm_gateway_model_id
+        else:
+            raise ValueError(
+                f"LLM type {self.llm_type} is not supported by NeMo Evaluator based guards."
+            )
+        return {"url": url, "api_key": datarobot_api_token, "model_id": model_id}
+
+    def has_average_score_custom_metric(self) -> bool:
+        return False
+
+    async def evaluate(
+        self,
+        *,
+        prompt: Optional[str],
+        response: Optional[str],
+        retrieved_contexts: Optional[list[str]],
+    ) -> float:
+        raise NotImplementedError
+
+    def _extract_score(self, evaluation: LiveEvaluation) -> int | float:
+        task = evaluation.result.tasks[self.nemo_evaluator_type]
+        metric = task.metrics[self.nemo_evaluator_type]
+        score = metric.scores[self.nemo_evaluator_type]
+        return score.value
+
+
+class NeMoLLMJudgeGuard(NeMoEvaluatorGuard):
+    def __init__(self, config: dict, stage=None):
+        super().__init__(config, stage)
+        self.nemo_llm_judge_config = config.get("nemo_llm_judge_config", {})
+
+    async def evaluate(self, *, prompt: str, response: str, **kwargs) -> float:
+        system_prompt = self.nemo_llm_judge_config["system_prompt"]
+        user_prompt = self.nemo_llm_judge_config["user_prompt"]
+        score_parsing_regex = self.nemo_llm_judge_config["score_parsing_regex"]
+
+        config = EvaluationConfigParam(
+            type="custom",
+            tasks={
+                self.nemo_evaluator_type: TaskConfigParam(
+                    type="data",
+                    metrics={
+                        self.nemo_evaluator_type: MetricConfigParam(
+                            type="llm-judge",
+                            params={
+                                "model": {"api_endpoint": self.llm_judge_api_endpoint},
+                                "template": {
+                                    "messages": [
+                                        {"role": "system", "content": system_prompt},
+                                        {"role": "user", "content": user_prompt},
+                                    ]
+                                },
+                                "scores": {
+                                    self.nemo_evaluator_type: {
+                                        "type": "int",
+                                        "parser": {"type": "regex", "pattern": score_parsing_regex},
+                                    }
+                                },
+                            },
+                        )
+                    },
+                )
+            },
+        )
+        target = EvaluationTargetParam(
+            type="rows", rows=[{"promptText": prompt, "responseText": response}]
+        )
+        evaluation = await self._client.evaluation.live(config=config, target=target)
+        return self._extract_score(evaluation)
+
+
+class NeMoContextRelevanceGuard(NeMoEvaluatorGuard):
+    async def evaluate(self, *, prompt: str, retrieved_contexts: list[str], **kwargs) -> float:
+        config = EvaluationConfigParam(
+            type="custom",
+            tasks={
+                self.nemo_evaluator_type: TaskConfigParam(
+                    type="data",
+                    metrics={
+                        self.nemo_evaluator_type: MetricConfigParam(
+                            type=self.nemo_evaluator_type,
+                            params={
+                                "judge": {"model": {"api_endpoint": self.llm_judge_api_endpoint}},
+                            },
+                        )
+                    },
+                )
+            },
+        )
+        target = EvaluationTargetParam(
+            type="rows",
+            rows=[{"user_input": prompt, "retrieved_contexts": retrieved_contexts}],
+        )
+        evaluation = await self._client.evaluation.live(config=config, target=target)
+        return self._extract_score(evaluation)
+
+
+class NeMoResponseGroundednessGuard(NeMoEvaluatorGuard):
+    async def evaluate(self, *, response: str, retrieved_contexts: list[str], **kwargs) -> float:
+        config = EvaluationConfigParam(
+            type="custom",
+            tasks={
+                self.nemo_evaluator_type: TaskConfigParam(
+                    type="data",
+                    metrics={
+                        self.nemo_evaluator_type: MetricConfigParam(
+                            type=self.nemo_evaluator_type,
+                            params={
+                                "judge": {"model": {"api_endpoint": self.llm_judge_api_endpoint}},
+                            },
+                        )
+                    },
+                )
+            },
+        )
+        target = EvaluationTargetParam(
+            type="rows",
+            rows=[{"response": response, "retrieved_contexts": retrieved_contexts}],
+        )
+        evaluation = await self._client.evaluation.live(config=config, target=target)
+        return self._extract_score(evaluation)
+
+
+class NeMoTopicAdherenceGuard(NeMoEvaluatorGuard):
+    def __init__(self, config: dict, stage=None):
+        super().__init__(config, stage)
+        self.nemo_topic_adherence_config = config["nemo_topic_adherence_config"]
+
+    async def evaluate(self, *, prompt: str, response: str, **kwargs) -> float:
+        config = EvaluationConfigParam(
+            type="custom",
+            tasks={
+                self.nemo_evaluator_type: TaskConfigParam(
+                    type="data",
+                    metrics={
+                        self.nemo_evaluator_type: MetricConfigParam(
+                            type=self.nemo_evaluator_type,
+                            params={
+                                "judge": {"model": {"api_endpoint": self.llm_judge_api_endpoint}},
+                                "metric_mode": self.nemo_topic_adherence_config["metric_mode"],
+                            },
+                        )
+                    },
+                )
+            },
+        )
+        target = EvaluationTargetParam(
+            type="rows",
+            rows=[
+                {
+                    "user_input": [
+                        {"content": prompt, "type": "human"},
+                        {"content": response, "type": "ai"},
+                    ],
+                    "reference_topics": self.nemo_topic_adherence_config["reference_topics"],
+                }
+            ],
+        )
+        evaluation = await self._client.evaluation.live(config=config, target=target)
+        return self._extract_score(evaluation)
+
+
+class NeMoAgentGoalAccuracyGuard(NeMoEvaluatorGuard):
+    async def evaluate(self, *, prompt: str, response: str, **kwargs) -> float:
+        config = EvaluationConfigParam(
+            type="custom",
+            tasks={
+                self.nemo_evaluator_type: TaskConfigParam(
+                    type="data",
+                    metrics={
+                        self.nemo_evaluator_type: MetricConfigParam(
+                            type=self.nemo_evaluator_type,
+                            params={
+                                "judge": {"model": {"api_endpoint": self.llm_judge_api_endpoint}},
+                                "use_reference": False,
+                            },
+                        )
+                    },
+                )
+            },
+        )
+        target = EvaluationTargetParam(
+            type="rows",
+            rows=[
+                {
+                    "user_input": [
+                        {"content": prompt, "type": "human"},
+                        {"content": response, "type": "ai"},
+                    ],
+                }
+            ],
+        )
+        evaluation = await self._client.evaluation.live(config=config, target=target)
+        return self._extract_score(evaluation)
+
+
+class NeMoResponseRelevancyGuard(NeMoEvaluatorGuard):
+    def __init__(self, config: dict, stage=None):
+        super().__init__(config, stage)
+        self.response_relevancy_config = config["nemo_response_relevancy_config"]
+
+    @cached_property
+    def embedding_judge_api_endpoint(self) -> dict:
+        """Embedding judge API endpoint, to be passed to NeMo evaluator."""
+        datarobot_endpoint, datarobot_api_token = get_datarobot_endpoint_and_token()
+        deployment_id = self.response_relevancy_config["embedding_deployment_id"]
+        url = f"{datarobot_endpoint}/deployments/{deployment_id}/directAccess/nim/v1/"
+        return {"url": url, "api_key": datarobot_api_token, "model_id": ""}
+
+    async def evaluate(
+        self, *, prompt: str, response: str, retrieved_context: Optional[list[str]]
+    ) -> float:
+        config = EvaluationConfigParam(
+            type="custom",
+            tasks={
+                self.nemo_evaluator_type: TaskConfigParam(
+                    type="data",
+                    metrics={
+                        self.nemo_evaluator_type: MetricConfigParam(
+                            type=self.nemo_evaluator_type,
+                            params={
+                                "judge": {"model": {"api_endpoint": self.llm_judge_api_endpoint}},
+                                "judge_embeddings": {
+                                    "model": {"api_endpoint": self.embedding_judge_api_endpoint},
+                                },
+                            },
+                        )
+                    },
+                )
+            },
+        )
+        row = {"user_input": prompt, "response": response}
+        if retrieved_context:
+            row["retrieved_contexts"] = retrieved_context
+        target = EvaluationTargetParam(type="rows", rows=[row])
+        evaluation = await self._client.evaluation.live(config=config, target=target)
+        return self._extract_score(evaluation)
+
+
+class NeMoFaithfulnessGuard(NeMoEvaluatorGuard):
+    async def evaluate(self, *, prompt: str, response: str, retrieved_contexts: list[str]) -> float:
+        config = EvaluationConfigParam(
+            type="custom",
+            tasks={
+                self.nemo_evaluator_type: TaskConfigParam(
+                    type="data",
+                    metrics={
+                        self.nemo_evaluator_type: MetricConfigParam(
+                            type=self.nemo_evaluator_type,
+                            params={
+                                "judge": {"model": {"api_endpoint": self.llm_judge_api_endpoint}},
+                            },
+                        )
+                    },
+                )
+            },
+        )
+        row = {"user_input": prompt, "response": response, "retrieved_contexts": retrieved_contexts}
+        target = EvaluationTargetParam(type="rows", rows=[row])
+        evaluation = await self._client.evaluation.live(config=config, target=target)
+        return self._extract_score(evaluation)