datarobot-moderations 11.2.10__py3-none-any.whl → 11.2.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datarobot_dome/__init__.py +1 -1
- datarobot_dome/async_http_client.py +1 -1
- datarobot_dome/chat_helper.py +1 -1
- datarobot_dome/constants.py +26 -2
- datarobot_dome/drum_integration.py +1 -1
- datarobot_dome/guard_executor.py +67 -16
- datarobot_dome/guard_factory.py +126 -0
- datarobot_dome/guard_helpers.py +16 -1
- datarobot_dome/guards/__init__.py +16 -1
- datarobot_dome/guards/base.py +259 -0
- datarobot_dome/guards/guard_llm_mixin.py +3 -1
- datarobot_dome/guards/model_guard.py +84 -0
- datarobot_dome/guards/nemo_evaluator.py +73 -0
- datarobot_dome/guards/nemo_guard.py +146 -0
- datarobot_dome/guards/ootb_guard.py +209 -0
- datarobot_dome/guards/validation.py +201 -0
- datarobot_dome/llm.py +1 -1
- datarobot_dome/metrics/__init__.py +1 -1
- datarobot_dome/metrics/citation_metrics.py +1 -1
- datarobot_dome/metrics/factory.py +1 -1
- datarobot_dome/metrics/metric_scorer.py +1 -1
- datarobot_dome/pipeline/__init__.py +1 -1
- datarobot_dome/pipeline/llm_pipeline.py +3 -3
- datarobot_dome/pipeline/pipeline.py +1 -1
- datarobot_dome/pipeline/vdb_pipeline.py +1 -1
- datarobot_dome/runtime.py +1 -1
- datarobot_dome/streaming.py +2 -2
- {datarobot_moderations-11.2.10.dist-info → datarobot_moderations-11.2.11.dist-info}/METADATA +2 -1
- datarobot_moderations-11.2.11.dist-info/RECORD +30 -0
- {datarobot_moderations-11.2.10.dist-info → datarobot_moderations-11.2.11.dist-info}/WHEEL +1 -1
- datarobot_dome/guard.py +0 -845
- datarobot_moderations-11.2.10.dist-info/RECORD +0 -24
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
# ---------------------------------------------------------------------------------
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
|
+
# Last updated 2025.
|
|
4
|
+
#
|
|
5
|
+
# DataRobot, Inc. Confidential.
|
|
6
|
+
# This is proprietary source code of DataRobot, Inc. and its affiliates.
|
|
7
|
+
#
|
|
8
|
+
# This file and its contents are subject to DataRobot Tool and Utility Agreement.
|
|
9
|
+
# For details, see
|
|
10
|
+
# https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
|
|
11
|
+
# ---------------------------------------------------------------------------------
|
|
12
|
+
from datarobot.enums import CustomMetricAggregationType
|
|
13
|
+
from datarobot.enums import CustomMetricDirectionality
|
|
14
|
+
from deepeval.metrics import TaskCompletionMetric
|
|
15
|
+
from llama_index.core import Settings
|
|
16
|
+
from llama_index.core.evaluation import FaithfulnessEvaluator
|
|
17
|
+
from llama_index.core.evaluation import GuidelineEvaluator
|
|
18
|
+
from ragas.llms import LangchainLLMWrapper
|
|
19
|
+
from ragas.llms import LlamaIndexLLMWrapper
|
|
20
|
+
from ragas.metrics import AgentGoalAccuracyWithoutReference
|
|
21
|
+
|
|
22
|
+
from datarobot_dome.constants import AGENT_GOAL_ACCURACY_COLUMN_NAME
|
|
23
|
+
from datarobot_dome.constants import COST_COLUMN_NAME
|
|
24
|
+
from datarobot_dome.constants import CUSTOM_METRIC_DESCRIPTION_SUFFIX
|
|
25
|
+
from datarobot_dome.constants import FAITHFULLNESS_COLUMN_NAME
|
|
26
|
+
from datarobot_dome.constants import GUIDELINE_ADHERENCE_COLUMN_NAME
|
|
27
|
+
from datarobot_dome.constants import ROUGE_1_COLUMN_NAME
|
|
28
|
+
from datarobot_dome.constants import SPAN_PREFIX
|
|
29
|
+
from datarobot_dome.constants import TASK_ADHERENCE_SCORE_COLUMN_NAME
|
|
30
|
+
from datarobot_dome.constants import TOKEN_COUNT_COLUMN_NAME
|
|
31
|
+
from datarobot_dome.constants import GuardLLMType
|
|
32
|
+
from datarobot_dome.constants import GuardStage
|
|
33
|
+
from datarobot_dome.constants import OOTBType
|
|
34
|
+
from datarobot_dome.guard_helpers import ModerationDeepEvalLLM
|
|
35
|
+
|
|
36
|
+
from .base import Guard
|
|
37
|
+
from .guard_llm_mixin import GuardLLMMixin
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class OOTBGuard(Guard):
|
|
41
|
+
def __init__(self, config: dict, stage=None):
|
|
42
|
+
super().__init__(config, stage)
|
|
43
|
+
self._ootb_type = config["ootb_type"]
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def ootb_type(self):
|
|
47
|
+
return self._ootb_type
|
|
48
|
+
|
|
49
|
+
def has_latency_custom_metric(self):
|
|
50
|
+
"""Latency is not tracked for token counts guards"""
|
|
51
|
+
return self._ootb_type != OOTBType.TOKEN_COUNT
|
|
52
|
+
|
|
53
|
+
def get_span_column_name(self, _):
|
|
54
|
+
if self._ootb_type == OOTBType.TOKEN_COUNT:
|
|
55
|
+
return TOKEN_COUNT_COLUMN_NAME
|
|
56
|
+
elif self._ootb_type == OOTBType.ROUGE_1:
|
|
57
|
+
return ROUGE_1_COLUMN_NAME
|
|
58
|
+
elif self._ootb_type == OOTBType.CUSTOM_METRIC:
|
|
59
|
+
return self.name
|
|
60
|
+
else:
|
|
61
|
+
raise NotImplementedError(f"No span attribute name defined for {self._ootb_type} guard")
|
|
62
|
+
|
|
63
|
+
def get_span_attribute_name(self, stage):
|
|
64
|
+
return f"{SPAN_PREFIX}.{stage.lower()}.{self.get_span_column_name(stage)}"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class OOTBCostMetric(OOTBGuard):
|
|
68
|
+
def __init__(self, config, stage):
|
|
69
|
+
super().__init__(config, stage)
|
|
70
|
+
# The cost is calculated based on the usage metrics returned by the
|
|
71
|
+
# completion object, so it can be evaluated only at response stage
|
|
72
|
+
self._stage = GuardStage.RESPONSE
|
|
73
|
+
cost_config = config["additional_guard_config"]["cost"]
|
|
74
|
+
self.currency = cost_config["currency"]
|
|
75
|
+
self.input_price = cost_config["input_price"]
|
|
76
|
+
self.input_unit = cost_config["input_unit"]
|
|
77
|
+
self.input_multiplier = self.input_price / self.input_unit
|
|
78
|
+
self.output_price = cost_config["output_price"]
|
|
79
|
+
self.output_unit = cost_config["output_unit"]
|
|
80
|
+
self.output_multiplier = self.output_price / self.output_unit
|
|
81
|
+
|
|
82
|
+
def get_average_score_custom_metric_name(self, _):
|
|
83
|
+
return f"Total cost in {self.currency}"
|
|
84
|
+
|
|
85
|
+
def get_average_score_metric(self, _):
|
|
86
|
+
return {
|
|
87
|
+
"name": self.get_average_score_custom_metric_name(_),
|
|
88
|
+
"directionality": CustomMetricDirectionality.LOWER_IS_BETTER,
|
|
89
|
+
"units": "value",
|
|
90
|
+
"type": CustomMetricAggregationType.SUM,
|
|
91
|
+
"baselineValue": 0,
|
|
92
|
+
"isModelSpecific": True,
|
|
93
|
+
"timeStep": "hour",
|
|
94
|
+
"description": (
|
|
95
|
+
f"{self.get_average_score_custom_metric_name(_)}. "
|
|
96
|
+
f" {CUSTOM_METRIC_DESCRIPTION_SUFFIX}"
|
|
97
|
+
),
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
def get_span_column_name(self, _):
|
|
101
|
+
return f"{COST_COLUMN_NAME}.{self.currency.lower()}"
|
|
102
|
+
|
|
103
|
+
def get_span_attribute_name(self, _):
|
|
104
|
+
return f"{SPAN_PREFIX}.{self._stage.lower()}.{self.get_span_column_name(_)}"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class OOTBFaithfulnessGuard(OOTBGuard, GuardLLMMixin):
|
|
108
|
+
def __init__(self, config: dict, stage=None):
|
|
109
|
+
super().__init__(config, stage)
|
|
110
|
+
|
|
111
|
+
if self.stage == GuardStage.PROMPT:
|
|
112
|
+
raise Exception("Faithfulness cannot be configured for the Prompt stage")
|
|
113
|
+
|
|
114
|
+
# Default LLM Type for Faithfulness is set to Azure OpenAI
|
|
115
|
+
self._llm_type = config.get("llm_type", GuardLLMType.AZURE_OPENAI)
|
|
116
|
+
Settings.llm = self.get_llm(config, self._llm_type)
|
|
117
|
+
Settings.embed_model = None
|
|
118
|
+
self._evaluator = FaithfulnessEvaluator()
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def faithfulness_evaluator(self):
|
|
122
|
+
return self._evaluator
|
|
123
|
+
|
|
124
|
+
def get_span_column_name(self, _):
|
|
125
|
+
return FAITHFULLNESS_COLUMN_NAME
|
|
126
|
+
|
|
127
|
+
def get_span_attribute_name(self, _):
|
|
128
|
+
return f"{SPAN_PREFIX}.{self._stage.lower()}.{self.get_span_column_name(_)}"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class OOTBAgentGoalAccuracyGuard(OOTBGuard, GuardLLMMixin):
|
|
132
|
+
def __init__(self, config: dict, stage=None):
|
|
133
|
+
super().__init__(config, stage)
|
|
134
|
+
|
|
135
|
+
if self.stage == GuardStage.PROMPT:
|
|
136
|
+
raise Exception("Agent Goal Accuracy guard cannot be configured for the Prompt stage")
|
|
137
|
+
|
|
138
|
+
# Default LLM Type for Agent Goal Accuracy is set to Azure OpenAI
|
|
139
|
+
self._llm_type = config.get("llm_type", GuardLLMType.AZURE_OPENAI)
|
|
140
|
+
llm = self.get_llm(config, self._llm_type)
|
|
141
|
+
if self._llm_type == GuardLLMType.AZURE_OPENAI:
|
|
142
|
+
evaluator_llm = LangchainLLMWrapper(llm)
|
|
143
|
+
else:
|
|
144
|
+
evaluator_llm = LlamaIndexLLMWrapper(llm)
|
|
145
|
+
self.scorer = AgentGoalAccuracyWithoutReference(llm=evaluator_llm)
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def accuracy_scorer(self):
|
|
149
|
+
return self.scorer
|
|
150
|
+
|
|
151
|
+
def get_span_column_name(self, _):
|
|
152
|
+
return AGENT_GOAL_ACCURACY_COLUMN_NAME
|
|
153
|
+
|
|
154
|
+
def get_span_attribute_name(self, _):
|
|
155
|
+
return f"{SPAN_PREFIX}.{self._stage.lower()}.{self.get_span_column_name(_)}"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class OOTBTaskAdherenceGuard(OOTBGuard, GuardLLMMixin):
|
|
159
|
+
def __init__(self, config: dict, stage=None):
|
|
160
|
+
super().__init__(config, stage)
|
|
161
|
+
|
|
162
|
+
if self.stage == GuardStage.PROMPT:
|
|
163
|
+
raise Exception("Agent Goal Accuracy guard cannot be configured for the Prompt stage")
|
|
164
|
+
|
|
165
|
+
# Default LLM Type for Task Adherence is set to Azure OpenAI
|
|
166
|
+
self._llm_type = config.get("llm_type", GuardLLMType.AZURE_OPENAI)
|
|
167
|
+
llm = self.get_llm(config, self._llm_type)
|
|
168
|
+
deepeval_llm = ModerationDeepEvalLLM(llm)
|
|
169
|
+
self.scorer = TaskCompletionMetric(model=deepeval_llm, include_reason=True)
|
|
170
|
+
|
|
171
|
+
@property
|
|
172
|
+
def task_adherence_scorer(self):
|
|
173
|
+
return self.scorer
|
|
174
|
+
|
|
175
|
+
def get_span_column_name(self, _):
|
|
176
|
+
return TASK_ADHERENCE_SCORE_COLUMN_NAME
|
|
177
|
+
|
|
178
|
+
def get_span_attribute_name(self, _):
|
|
179
|
+
return f"{SPAN_PREFIX}.{self._stage.lower()}.{self.get_span_column_name(_)}"
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class OOTBAgentGuidelineAdherence(OOTBGuard, GuardLLMMixin):
|
|
183
|
+
def __init__(self, config: dict, stage=None):
|
|
184
|
+
super().__init__(config, stage)
|
|
185
|
+
|
|
186
|
+
if self.stage == GuardStage.PROMPT:
|
|
187
|
+
raise Exception(
|
|
188
|
+
"Agent Guideline Adherence guard cannot be configured for the Prompt stage"
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
self.guideline = config.get("additional_guard_config", {}).get("agent_guideline")
|
|
192
|
+
if self.guideline is None or len(self.guideline) == 0:
|
|
193
|
+
raise Exception("Agent Guideline Adherence requires at least one guideline.")
|
|
194
|
+
|
|
195
|
+
# Default LLM Type for Guideline Adherence is set to Azure OpenAI
|
|
196
|
+
self._llm_type = config.get("llm_type", GuardLLMType.AZURE_OPENAI)
|
|
197
|
+
llm = self.get_llm(config, self._llm_type)
|
|
198
|
+
|
|
199
|
+
self.scorer = GuidelineEvaluator(llm=llm, guidelines=self.guideline)
|
|
200
|
+
|
|
201
|
+
@property
|
|
202
|
+
def guideline_adherence_scorer(self):
|
|
203
|
+
return self.scorer
|
|
204
|
+
|
|
205
|
+
def get_span_column_name(self, _):
|
|
206
|
+
return GUIDELINE_ADHERENCE_COLUMN_NAME
|
|
207
|
+
|
|
208
|
+
def get_span_attribute_name(self, _):
|
|
209
|
+
return f"{SPAN_PREFIX}.{self._stage.lower()}.{self.get_span_column_name(_)}"
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
# ---------------------------------------------------------------------------------
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
|
+
# Last updated 2025.
|
|
4
|
+
#
|
|
5
|
+
# DataRobot, Inc. Confidential.
|
|
6
|
+
# This is proprietary source code of DataRobot, Inc. and its affiliates.
|
|
7
|
+
#
|
|
8
|
+
# This file and its contents are subject to DataRobot Tool and Utility Agreement.
|
|
9
|
+
# For details, see
|
|
10
|
+
# https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
|
|
11
|
+
# ---------------------------------------------------------------------------------
|
|
12
|
+
import trafaret as t
|
|
13
|
+
|
|
14
|
+
from datarobot_dome.constants import DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC
|
|
15
|
+
from datarobot_dome.constants import AwsModel
|
|
16
|
+
from datarobot_dome.constants import CostCurrency
|
|
17
|
+
from datarobot_dome.constants import GoogleModel
|
|
18
|
+
from datarobot_dome.constants import GuardAction
|
|
19
|
+
from datarobot_dome.constants import GuardLLMType
|
|
20
|
+
from datarobot_dome.constants import GuardModelTargetType
|
|
21
|
+
from datarobot_dome.constants import GuardOperatorType
|
|
22
|
+
from datarobot_dome.constants import GuardStage
|
|
23
|
+
from datarobot_dome.constants import GuardTimeoutAction
|
|
24
|
+
from datarobot_dome.constants import GuardType
|
|
25
|
+
from datarobot_dome.constants import NemoEvaluatorType
|
|
26
|
+
from datarobot_dome.constants import OOTBType
|
|
27
|
+
|
|
28
|
+
MAX_GUARD_NAME_LENGTH = 255
|
|
29
|
+
MAX_COLUMN_NAME_LENGTH = 255
|
|
30
|
+
MAX_GUARD_COLUMN_NAME_LENGTH = 255
|
|
31
|
+
MAX_GUARD_MESSAGE_LENGTH = 4096
|
|
32
|
+
MAX_GUARD_DESCRIPTION_LENGTH = 4096
|
|
33
|
+
OBJECT_ID_LENGTH = 24
|
|
34
|
+
MAX_REGEX_LENGTH = 255
|
|
35
|
+
MAX_URL_LENGTH = 255
|
|
36
|
+
MAX_TOKEN_LENGTH = 255
|
|
37
|
+
MAX_GUIDELINE_LENGTH = 4096
|
|
38
|
+
|
|
39
|
+
cost_metric_trafaret = t.Dict(
|
|
40
|
+
{
|
|
41
|
+
t.Key("currency", to_name="currency", optional=True, default=CostCurrency.USD): t.Enum(
|
|
42
|
+
*CostCurrency.ALL
|
|
43
|
+
),
|
|
44
|
+
t.Key("input_price", to_name="input_price", optional=False): t.Float(),
|
|
45
|
+
t.Key("input_unit", to_name="input_unit", optional=False): t.Int(),
|
|
46
|
+
t.Key("output_price", to_name="output_price", optional=False): t.Float(),
|
|
47
|
+
t.Key("output_unit", to_name="output_unit", optional=False): t.Int(),
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
model_info_trafaret = t.Dict(
|
|
53
|
+
{
|
|
54
|
+
t.Key("class_names", to_name="class_names", optional=True): t.List(
|
|
55
|
+
t.String(max_length=MAX_COLUMN_NAME_LENGTH)
|
|
56
|
+
),
|
|
57
|
+
t.Key("model_id", to_name="model_id", optional=True): t.String(max_length=OBJECT_ID_LENGTH),
|
|
58
|
+
t.Key("input_column_name", to_name="input_column_name", optional=False): t.String(
|
|
59
|
+
max_length=MAX_COLUMN_NAME_LENGTH
|
|
60
|
+
),
|
|
61
|
+
t.Key("target_name", to_name="target_name", optional=False): t.String(
|
|
62
|
+
max_length=MAX_COLUMN_NAME_LENGTH
|
|
63
|
+
),
|
|
64
|
+
t.Key(
|
|
65
|
+
"replacement_text_column_name", to_name="replacement_text_column_name", optional=True
|
|
66
|
+
): t.Or(t.String(allow_blank=True, max_length=MAX_COLUMN_NAME_LENGTH), t.Null),
|
|
67
|
+
t.Key("target_type", to_name="target_type", optional=False): t.Enum(
|
|
68
|
+
*GuardModelTargetType.ALL
|
|
69
|
+
),
|
|
70
|
+
},
|
|
71
|
+
allow_extra=["*"],
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
model_guard_intervention_trafaret = t.Dict(
|
|
76
|
+
{
|
|
77
|
+
t.Key("comparand", to_name="comparand", optional=False): t.Or(
|
|
78
|
+
t.String(max_length=MAX_GUARD_NAME_LENGTH),
|
|
79
|
+
t.Float(),
|
|
80
|
+
t.Bool(),
|
|
81
|
+
t.List(t.String(max_length=MAX_GUARD_NAME_LENGTH)),
|
|
82
|
+
t.List(t.Float()),
|
|
83
|
+
),
|
|
84
|
+
t.Key("comparator", to_name="comparator", optional=False): t.Enum(*GuardOperatorType.ALL),
|
|
85
|
+
},
|
|
86
|
+
allow_extra=["*"],
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
guard_intervention_trafaret = t.Dict(
|
|
91
|
+
{
|
|
92
|
+
t.Key("action", to_name="action", optional=False): t.Enum(*GuardAction.ALL),
|
|
93
|
+
t.Key("message", to_name="message", optional=True): t.String(
|
|
94
|
+
max_length=MAX_GUARD_MESSAGE_LENGTH, allow_blank=True
|
|
95
|
+
),
|
|
96
|
+
t.Key("conditions", to_name="conditions", optional=True): t.Or(
|
|
97
|
+
t.List(
|
|
98
|
+
model_guard_intervention_trafaret,
|
|
99
|
+
max_length=1,
|
|
100
|
+
min_length=0,
|
|
101
|
+
),
|
|
102
|
+
t.Null,
|
|
103
|
+
),
|
|
104
|
+
t.Key("send_notification", to_name="send_notification", optional=True): t.Bool(),
|
|
105
|
+
},
|
|
106
|
+
allow_extra=["*"],
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
additional_guard_config_trafaret = t.Dict(
|
|
110
|
+
{
|
|
111
|
+
t.Key("cost", to_name="cost", optional=True): t.Or(cost_metric_trafaret, t.Null),
|
|
112
|
+
t.Key("tool_call", to_name="tool_call", optional=True): t.Or(t.Any(), t.Null),
|
|
113
|
+
t.Key("agent_guideline", to_name="agent_guideline", optional=True): t.String(
|
|
114
|
+
max_length=MAX_GUIDELINE_LENGTH, allow_blank=True
|
|
115
|
+
),
|
|
116
|
+
}
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
guard_trafaret = t.Dict(
|
|
121
|
+
{
|
|
122
|
+
t.Key("name", to_name="name", optional=False): t.String(max_length=MAX_GUARD_NAME_LENGTH),
|
|
123
|
+
t.Key("description", to_name="description", optional=True): t.String(
|
|
124
|
+
max_length=MAX_GUARD_DESCRIPTION_LENGTH
|
|
125
|
+
),
|
|
126
|
+
t.Key("stage", to_name="stage", optional=False): t.Or(
|
|
127
|
+
t.List(t.Enum(*GuardStage.ALL)), t.Enum(*GuardStage.ALL)
|
|
128
|
+
),
|
|
129
|
+
t.Key("type", to_name="type", optional=False): t.Enum(*GuardType.ALL),
|
|
130
|
+
t.Key("ootb_type", to_name="ootb_type", optional=True): t.Enum(*OOTBType.ALL),
|
|
131
|
+
t.Key("nemo_evaluator_type", to_name="nemo_evaluator_type", optional=True): t.Enum(
|
|
132
|
+
*NemoEvaluatorType.ALL
|
|
133
|
+
),
|
|
134
|
+
t.Key("llm_type", to_name="llm_type", optional=True): t.Enum(*GuardLLMType.ALL),
|
|
135
|
+
t.Key("deployment_id", to_name="deployment_id", optional=True): t.Or(
|
|
136
|
+
t.String(max_length=OBJECT_ID_LENGTH), t.Null
|
|
137
|
+
),
|
|
138
|
+
t.Key("model_info", to_name="model_info", optional=True): model_info_trafaret,
|
|
139
|
+
t.Key("intervention", to_name="intervention", optional=True): t.Or(
|
|
140
|
+
guard_intervention_trafaret, t.Null
|
|
141
|
+
),
|
|
142
|
+
t.Key("openai_api_key", to_name="openai_api_key", optional=True): t.Or(
|
|
143
|
+
t.String(max_length=MAX_TOKEN_LENGTH), t.Null
|
|
144
|
+
),
|
|
145
|
+
t.Key("openai_deployment_id", to_name="openai_deployment_id", optional=True): t.Or(
|
|
146
|
+
t.String(max_length=OBJECT_ID_LENGTH), t.Null
|
|
147
|
+
),
|
|
148
|
+
t.Key("openai_api_base", to_name="openai_api_base", optional=True): t.Or(
|
|
149
|
+
t.String(max_length=MAX_URL_LENGTH), t.Null
|
|
150
|
+
),
|
|
151
|
+
t.Key("google_region", to_name="google_region", optional=True): t.Or(t.String, t.Null),
|
|
152
|
+
t.Key("google_model", to_name="google_model", optional=True): t.Or(
|
|
153
|
+
t.Enum(*GoogleModel.ALL), t.Null
|
|
154
|
+
),
|
|
155
|
+
t.Key("aws_region", to_name="aws_region", optional=True): t.Or(t.String, t.Null),
|
|
156
|
+
t.Key("aws_model", to_name="aws_model", optional=True): t.Or(t.Enum(*AwsModel.ALL), t.Null),
|
|
157
|
+
t.Key("faas_url", optional=True): t.Or(t.String(max_length=MAX_URL_LENGTH), t.Null),
|
|
158
|
+
t.Key("copy_citations", optional=True, default=False): t.Bool(),
|
|
159
|
+
t.Key("is_agentic", to_name="is_agentic", optional=True, default=False): t.Bool(),
|
|
160
|
+
t.Key(
|
|
161
|
+
"additional_guard_config",
|
|
162
|
+
to_name="additional_guard_config",
|
|
163
|
+
optional=True,
|
|
164
|
+
default=None,
|
|
165
|
+
): t.Or(additional_guard_config_trafaret, t.Null),
|
|
166
|
+
},
|
|
167
|
+
allow_extra=["*"],
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
moderation_config_trafaret = t.Dict(
|
|
172
|
+
{
|
|
173
|
+
t.Key(
|
|
174
|
+
"timeout_sec",
|
|
175
|
+
to_name="timeout_sec",
|
|
176
|
+
optional=True,
|
|
177
|
+
default=DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC,
|
|
178
|
+
): t.Int(gt=1),
|
|
179
|
+
t.Key(
|
|
180
|
+
"timeout_action",
|
|
181
|
+
to_name="timeout_action",
|
|
182
|
+
optional=True,
|
|
183
|
+
default=GuardTimeoutAction.SCORE,
|
|
184
|
+
): t.Enum(*GuardTimeoutAction.ALL),
|
|
185
|
+
# Why default is True?
|
|
186
|
+
# We manually tested it and sending extra output with OpenAI completion object under
|
|
187
|
+
# "datarobot_moderations" field seems to be working by default, "EVEN WITH" OpenAI client
|
|
188
|
+
# It will always work with the API response (because it will simply be treated as extra data
|
|
189
|
+
# in the json response). So, most of the times it is going to work. In future, if the
|
|
190
|
+
# OpenAI client couldn't recognize extra data - we can simply disable this flag, so that
|
|
191
|
+
# it won't break the client and user flow
|
|
192
|
+
t.Key(
|
|
193
|
+
"enable_extra_model_output_for_chat",
|
|
194
|
+
to_name="enable_extra_model_output_for_chat",
|
|
195
|
+
optional=True,
|
|
196
|
+
default=True,
|
|
197
|
+
): t.Bool(),
|
|
198
|
+
t.Key("guards", to_name="guards", optional=False): t.List(guard_trafaret),
|
|
199
|
+
},
|
|
200
|
+
allow_extra=["*"],
|
|
201
|
+
)
|
datarobot_dome/llm.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
|
@@ -24,9 +24,9 @@ from datarobot_dome.constants import LOGGER_NAME_PREFIX
|
|
|
24
24
|
from datarobot_dome.constants import GuardAction
|
|
25
25
|
from datarobot_dome.constants import GuardOperatorType
|
|
26
26
|
from datarobot_dome.constants import GuardStage
|
|
27
|
-
from datarobot_dome.
|
|
28
|
-
from datarobot_dome.guard import moderation_config_trafaret
|
|
27
|
+
from datarobot_dome.guard_factory import GuardFactory
|
|
29
28
|
from datarobot_dome.guard_helpers import get_rouge_1_scorer
|
|
29
|
+
from datarobot_dome.guards.validation import moderation_config_trafaret
|
|
30
30
|
from datarobot_dome.pipeline.pipeline import Pipeline
|
|
31
31
|
|
|
32
32
|
CUSTOM_METRICS_BULK_UPLOAD_API_PREFIX = "deployments"
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
datarobot_dome/runtime.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
datarobot_dome/streaming.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# ---------------------------------------------------------------------------------
|
|
2
|
-
# Copyright (c)
|
|
2
|
+
# Copyright (c) 2026 DataRobot, Inc. and its affiliates. All rights reserved.
|
|
3
3
|
# Last updated 2025.
|
|
4
4
|
#
|
|
5
5
|
# DataRobot, Inc. Confidential.
|
|
@@ -35,7 +35,7 @@ from datarobot_dome.constants import GuardAction
|
|
|
35
35
|
from datarobot_dome.constants import GuardStage
|
|
36
36
|
from datarobot_dome.constants import GuardType
|
|
37
37
|
from datarobot_dome.constants import OOTBType
|
|
38
|
-
from datarobot_dome.
|
|
38
|
+
from datarobot_dome.guard_factory import Guard
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
class StreamingContext:
|
{datarobot_moderations-11.2.10.dist-info → datarobot_moderations-11.2.11.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datarobot-moderations
|
|
3
|
-
Version: 11.2.
|
|
3
|
+
Version: 11.2.11
|
|
4
4
|
Summary: DataRobot Monitoring and Moderation framework
|
|
5
5
|
License: DataRobot Tool and Utility Agreement
|
|
6
6
|
Author: DataRobot
|
|
@@ -24,6 +24,7 @@ Requires-Dist: llama-index-embeddings-azure-openai (>=0.1.6)
|
|
|
24
24
|
Requires-Dist: llama-index-llms-bedrock-converse (>=0.1.6)
|
|
25
25
|
Requires-Dist: llama-index-llms-langchain (>=0.1.3)
|
|
26
26
|
Requires-Dist: llama-index-llms-vertex (>=0.1.5)
|
|
27
|
+
Requires-Dist: nemo-microservices (>=1.5.0,<2.0.0)
|
|
27
28
|
Requires-Dist: nemoguardrails (>=0.9.0)
|
|
28
29
|
Requires-Dist: nest-asyncio (>=1.6.0)
|
|
29
30
|
Requires-Dist: numpy (>=1.25.0)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
datarobot_dome/__init__.py,sha256=S5kGePu3juoQk4ztKosOjtJyOwRqzN5quavyuO79Vvg,583
|
|
2
|
+
datarobot_dome/async_http_client.py,sha256=g6smItjY7YX7v8WwjOGJrlEpYmv596W6QUXAQ5SlUBQ,9863
|
|
3
|
+
datarobot_dome/chat_helper.py,sha256=maSxpTESdKCeo8jtLID6zquICxymzEOnrpuvqsfx9Dk,9671
|
|
4
|
+
datarobot_dome/constants.py,sha256=Mt_IFjIyMRmp0huRb8AN9MYqGf9OSgWB8WWQ7iqcx9I,11404
|
|
5
|
+
datarobot_dome/drum_integration.py,sha256=tg_rpwlClc0Dt1_X-bHG3qEKAwbaPJu_qb4VyA1lUFg,46480
|
|
6
|
+
datarobot_dome/guard_executor.py,sha256=z62P4HS0PGiKX1AAKxxTBuWMiNjdFgsqJKlpnvWcPSI,37607
|
|
7
|
+
datarobot_dome/guard_factory.py,sha256=g92bQ1zqlDJ5lpjDc1s60Is5RlFPbs0HkExgPgoIbg8,6081
|
|
8
|
+
datarobot_dome/guard_helpers.py,sha256=B5v0nP5wZk9am_-rDRWuuk6jYuaYVh4t9ClHQM-O7c4,17581
|
|
9
|
+
datarobot_dome/guards/__init__.py,sha256=sL2DO-IvQkS3pgpurfmbQf_Ot-5yyyTFp0RiH0pNjOA,1287
|
|
10
|
+
datarobot_dome/guards/base.py,sha256=0eIr3uAeIIQ8-YnBVM1z452HOyE7wyeqIuNgeYVhVN0,10646
|
|
11
|
+
datarobot_dome/guards/guard_llm_mixin.py,sha256=QnrQjyBrgKlDZ4yPy10kdDzleuQ4h54aGld8EykCweQ,12041
|
|
12
|
+
datarobot_dome/guards/model_guard.py,sha256=2bTQYFNuLYT169ex17cclyznirHtbqtSpOSnkP_Sh5E,2970
|
|
13
|
+
datarobot_dome/guards/nemo_evaluator.py,sha256=85-_V4aTLA-Ry8_rWjKI9IMlC9Nparwm_KvUy08cwVw,2497
|
|
14
|
+
datarobot_dome/guards/nemo_guard.py,sha256=E50OS6tdw5PnUX_DY6U-zEkUuP6T-63YTVYOgdvvK2Y,7114
|
|
15
|
+
datarobot_dome/guards/ootb_guard.py,sha256=c7SKb7zy9cMBRiP9eSrZJsAgQYFyMygjEBhdn1YZfT8,8469
|
|
16
|
+
datarobot_dome/guards/validation.py,sha256=39W4wM7g6lAuVnM078gRvXFx_sYe9-YioN8T600EXv0,8367
|
|
17
|
+
datarobot_dome/llm.py,sha256=e6dCK-apbr3l4Njztgm8my8vorFvvw1L7MbaHUiCOZ0,5719
|
|
18
|
+
datarobot_dome/metrics/__init__.py,sha256=S5kGePu3juoQk4ztKosOjtJyOwRqzN5quavyuO79Vvg,583
|
|
19
|
+
datarobot_dome/metrics/citation_metrics.py,sha256=q-uSJKNQjLizdpiju7e1qgVPQVu3SJHDm4Abc8e4m9k,4652
|
|
20
|
+
datarobot_dome/metrics/factory.py,sha256=C7M_hhN6z-_Q8LWfuGhuU_gUc87TJHmvYoY6wFC6O3w,2153
|
|
21
|
+
datarobot_dome/metrics/metric_scorer.py,sha256=pFWkOv1-O6zdCQOdFJwIYTvJkJOzJ2LqKiizVI1cKHA,2540
|
|
22
|
+
datarobot_dome/pipeline/__init__.py,sha256=S5kGePu3juoQk4ztKosOjtJyOwRqzN5quavyuO79Vvg,583
|
|
23
|
+
datarobot_dome/pipeline/llm_pipeline.py,sha256=DzSNxNH8uu46IX-cCYMhVck6yWdCBqGbr9AJbI6tSUs,20981
|
|
24
|
+
datarobot_dome/pipeline/pipeline.py,sha256=TUzTux9BRQnUhiHr0yhB8nm4CsOoPNTDALZcAo8KmVE,19486
|
|
25
|
+
datarobot_dome/pipeline/vdb_pipeline.py,sha256=z72xMm5gVfAYJ3vqWEy190AORrXFsJNcbrZ3y_Biod4,9835
|
|
26
|
+
datarobot_dome/runtime.py,sha256=4G23IWzu6QSHEpy3INlb3s9mHNxsZlSalOHV4gN6L18,1468
|
|
27
|
+
datarobot_dome/streaming.py,sha256=NRykFbv7gJQ02brZ9u5rwTTKMq6REibcLtgTw-18294,18248
|
|
28
|
+
datarobot_moderations-11.2.11.dist-info/METADATA,sha256=Erh64WdPsSUZxRyqnnlEyKdviIAXfLBi2d_MZoYs9XE,4855
|
|
29
|
+
datarobot_moderations-11.2.11.dist-info/WHEEL,sha256=3ny-bZhpXrU6vSQ1UPG34FoxZBp3lVcvK0LkgUz6VLk,88
|
|
30
|
+
datarobot_moderations-11.2.11.dist-info/RECORD,,
|