datarobot-moderations 11.2.3__py3-none-any.whl → 11.2.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -28,6 +28,7 @@ from datarobot_dome.constants import DATAROBOT_CONFIGURED_ON_PREM_ST_SAAS_URL
28
28
  from datarobot_dome.constants import DATAROBOT_SERVERLESS_PLATFORM
29
29
  from datarobot_dome.constants import DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC
30
30
  from datarobot_dome.constants import LOGGER_NAME_PREFIX
31
+ from datarobot_dome.constants import MODERATIONS_USER_AGENT
31
32
  from datarobot_dome.constants import RETRY_COUNT
32
33
  from datarobot_dome.constants import ModerationEventTypes
33
34
 
@@ -81,11 +82,13 @@ class AsyncHTTPClient:
81
82
  "Content-Type": "text/csv",
82
83
  "Accept": "text/csv",
83
84
  "Authorization": f"Bearer {os.environ['DATAROBOT_API_TOKEN']}",
85
+ "User-Agent": MODERATIONS_USER_AGENT,
84
86
  }
85
87
  self.json_headers = {
86
88
  "Content-Type": "application/json",
87
89
  "Accept": "application/json",
88
90
  "Authorization": f"Bearer {os.environ['DATAROBOT_API_TOKEN']}",
91
+ "User-Agent": MODERATIONS_USER_AGENT,
89
92
  }
90
93
  self.session = None
91
94
  self.events_url = f"{os.environ['DATAROBOT_ENDPOINT']}/remoteEvents/"
@@ -100,9 +103,9 @@ class AsyncHTTPClient:
100
103
  asyncio.set_event_loop(self.loop)
101
104
  else:
102
105
  raise
106
+ nest_asyncio.apply(loop=self.loop)
103
107
  self.loop.run_until_complete(self.__create_client_session(timeout))
104
108
  self.loop.set_debug(True)
105
- nest_asyncio.apply(loop=self.loop)
106
109
 
107
110
  atexit.register(self.shutdown)
108
111
 
@@ -10,9 +10,12 @@
10
10
  # https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
11
11
  # ---------------------------------------------------------------------------------
12
12
  from enum import Enum
13
+ from importlib.metadata import version
13
14
 
14
15
  __GUARD_ASSOCIATION_IDS_COLUMN_NAME__ = "datarobot_guard_association_id"
15
16
 
17
+ MODERATIONS_USER_AGENT = f"datarobot-moderations:{version('datarobot-moderations')}"
18
+
16
19
  LOGGER_NAME_PREFIX = "moderations"
17
20
 
18
21
  DEFAULT_PROMPT_COLUMN_NAME = "promptText"
@@ -71,6 +74,8 @@ PROMPT_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE = "prompt_token_count_from_usage"
71
74
  RESPONSE_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE = "response_token_count_from_usage"
72
75
 
73
76
  SPAN_PREFIX = "datarobot.guard"
77
+ DATAROBOT_ASSOCIATION_ID_FIELD_NAME = "datarobot_association_id"
78
+ DATAROBOT_METRICS_DICT_FIELD_NAME = "datarobot_metrics"
74
79
 
75
80
 
76
81
  class TargetType(str, Enum):
@@ -24,6 +24,7 @@ import numpy as np
24
24
  import pandas as pd
25
25
  import yaml
26
26
  from openai.types.chat import ChatCompletionChunk
27
+ from openai.types.chat import CompletionCreateParams
27
28
  from openai.types.chat.chat_completion import ChatCompletion
28
29
  from openai.types.chat.chat_completion import Choice
29
30
  from openai.types.chat.chat_completion_message import ChatCompletionMessage
@@ -40,6 +41,8 @@ from datarobot_dome.chat_helper import run_postscore_guards
40
41
  from datarobot_dome.constants import AGENTIC_PIPELINE_INTERACTIONS_ATTR
41
42
  from datarobot_dome.constants import CHAT_COMPLETION_OBJECT
42
43
  from datarobot_dome.constants import CITATIONS_ATTR
44
+ from datarobot_dome.constants import DATAROBOT_ASSOCIATION_ID_FIELD_NAME
45
+ from datarobot_dome.constants import DATAROBOT_METRICS_DICT_FIELD_NAME
43
46
  from datarobot_dome.constants import DATAROBOT_MODERATIONS_ATTR
44
47
  from datarobot_dome.constants import DISABLE_MODERATION_RUNTIME_PARAM_NAME
45
48
  from datarobot_dome.constants import LLM_BLUEPRINT_ID_ATTR
@@ -590,6 +593,29 @@ def _set_moderation_attribute_to_completion(pipeline, chat_completion, df, assoc
590
593
 
591
594
 
592
595
  def get_chat_prompt(completion_create_params):
596
+ """
597
+ Validate and extract the user prompt from completion create parameters (CCP).
598
+ Include tool calls if they were provided.
599
+
600
+ CCP "messages" list must be non-empty and include content with "user" role.
601
+ Example: "messages": [{"role": "user", "content": "What is the meaning of life?"}]
602
+
603
+ :param completion_create_params: dict containing chat request
604
+ :return: constructed prompt based on CCP content.
605
+ :raise ValueError if completion create parameters is not valid.
606
+ """
607
+ # ensure message content exists
608
+ if (
609
+ "messages" not in completion_create_params
610
+ or completion_create_params["messages"] is None
611
+ or len(completion_create_params["messages"]) == 0
612
+ or not isinstance(completion_create_params["messages"][-1], dict)
613
+ or "content" not in completion_create_params["messages"][-1]
614
+ ):
615
+ raise ValueError(
616
+ f"Chat input for moderation does not contain a message: {completion_create_params}"
617
+ )
618
+
593
619
  # Get the prompt with role = User
594
620
  last_user_message = None
595
621
  tool_calls = []
@@ -599,7 +625,7 @@ def get_chat_prompt(completion_create_params):
599
625
  if message["role"] == "tool":
600
626
  tool_calls.append(f"{message.get('name', '')}_{message['content']}")
601
627
  if last_user_message is None:
602
- raise Exception("No message with 'user' role found in input")
628
+ raise ValueError("No message with 'user' role found in input")
603
629
 
604
630
  prompt_content = last_user_message["content"]
605
631
  tool_names = []
@@ -623,7 +649,7 @@ def get_chat_prompt(completion_create_params):
623
649
  concatenated_prompt.append(message)
624
650
  chat_prompt = "\n".join(concatenated_prompt)
625
651
  else:
626
- raise Exception(f"Unhandled prompt type: {type(prompt_content)}")
652
+ raise ValueError(f"Unhandled prompt type: {type(prompt_content)}")
627
653
 
628
654
  if len(tool_calls) > 0:
629
655
  # Lets not add tool names if tool calls are present. Tool calls are more
@@ -693,32 +719,91 @@ def report_otel_evaluation_set_metric(pipeline, result_df):
693
719
  current_span.set_attribute("datarobot.moderation.evaluation", json.dumps(final_value))
694
720
 
695
721
 
722
+ def filter_extra_body(
723
+ completion_create_params: CompletionCreateParams,
724
+ ) -> tuple[CompletionCreateParams, dict]:
725
+ """
726
+ completion_create_params is a typed dict of a few standard fields,
727
+ and arbitrary fields from extra_body.
728
+ If "datarobot_metrics" is in extra_body, process it here.
729
+ Save its value only if it is a dict as expected.
730
+ :param completion_create_params: the chat completion params from OpenAI client via DRUM
731
+ :return: filtered completion_create_params; dict of {name: value} for "datarobot_" fields
732
+ """
733
+ datarobot_extra_body_params = {}
734
+ name = DATAROBOT_METRICS_DICT_FIELD_NAME
735
+ if name in completion_create_params:
736
+ value = completion_create_params[name]
737
+ _logger.debug("found DataRobot metrics in extra_body: %s", f"{name}={value}")
738
+ if isinstance(value, dict):
739
+ datarobot_extra_body_params = copy.deepcopy(value)
740
+ else:
741
+ _logger.warning("DataRobot metrics in extra_body is not a dict: %s", f"{name}={value}")
742
+ completion_create_params.pop(name, None)
743
+ return completion_create_params, datarobot_extra_body_params
744
+
745
+
746
+ def filter_association_id(
747
+ completion_create_params: CompletionCreateParams,
748
+ ) -> tuple[CompletionCreateParams, str | None]:
749
+ """
750
+ completion_create_params (CCP) is a typed dict of a few standard fields,
751
+ and arbitrary fields from extra_body.
752
+ If a field for the association ID exists, extract that value and remove it from the CCP.
753
+ Do this before calling filter_extra_body(), which would otherwise capture the association ID.
754
+ :param completion_create_params: the chat completion params from OpenAI client via DRUM
755
+ :return: filtered completion_create_params, association ID value
756
+
757
+ If no association ID was found in extra body: return original CCP,None
758
+ """
759
+ name = DATAROBOT_ASSOCIATION_ID_FIELD_NAME
760
+ if name in completion_create_params:
761
+ value = completion_create_params[name]
762
+ _logger.debug("found association ID in extra_body: %s", f"{name}={value}")
763
+ completion_create_params.pop(name, None)
764
+ return completion_create_params, value
765
+ return completion_create_params, None
766
+
767
+
696
768
  def guard_chat_wrapper(
697
769
  completion_create_params, model, pipeline, drum_chat_fn, association_id=None, **kwargs
698
770
  ):
771
+ # if association ID was included in extra_body, extract field name and value
772
+ completion_create_params, eb_assoc_id_value = filter_association_id(completion_create_params)
773
+
774
+ # extract any fields mentioned in "datarobot_metrics" to send as custom metrics later
775
+ completion_create_params, chat_extra_body_params = filter_extra_body(completion_create_params)
776
+
777
+ # register them for ID lookup later
778
+ pipeline.add_extra_body_custom_metric_definition(chat_extra_body_params)
779
+
780
+ # define all pipeline-based and guard-based custom metrics (but not those from extra_body)
781
+ # sends single API request to look up IDs for all custom metrics (including extra_body)
699
782
  pipeline.get_new_metrics_payload()
700
783
 
784
+ # the chat request is not a dataframe, but we'll build a DF internally for moderation.
701
785
  prompt_column_name = pipeline.get_input_column(GuardStage.PROMPT)
702
- if (
703
- "messages" not in completion_create_params
704
- or completion_create_params["messages"] is None
705
- or len(completion_create_params["messages"]) == 0
706
- or not isinstance(completion_create_params["messages"][-1], dict)
707
- or "content" not in completion_create_params["messages"][-1]
708
- ):
709
- raise ValueError(f"Invalid chat input for moderation: {completion_create_params}")
710
-
711
786
  prompt = get_chat_prompt(completion_create_params)
712
787
  streaming_response_requested = completion_create_params.get("stream", False)
713
788
 
714
789
  data = pd.DataFrame({prompt_column_name: [prompt]})
790
+ # for association IDs (with or without extra_body): the column must be defined in the deployment
791
+ # (here, this means pipeline.get_association_id_column_name() ("standard name") is not empty.)
792
+ # there are 3 likely cases for association ID, and 1 corner case:
793
+ # 1. ID value not provided (drum or extra_body) => no association ID column
794
+ # 2. ID value provided by DRUM => new DF column with standard name and provided value
795
+ # 3. ID defined in extra_body => new DF column with standard name and extra_body value
796
+ # 4. ID in extra_body with empty value => no association ID column
797
+ # Moderation library no longer auto-generates an association ID for chat. However, DRUM does.
715
798
  association_id_column_name = pipeline.get_association_id_column_name()
799
+ association_id = eb_assoc_id_value or association_id
716
800
  if association_id_column_name:
717
801
  if association_id:
718
802
  data[association_id_column_name] = [association_id]
719
- elif pipeline.auto_generate_association_ids:
720
- data[association_id_column_name] = pipeline.generate_association_ids(1)
721
- association_id = data[association_id_column_name].tolist()[0]
803
+
804
+ # report any metrics from extra_body. They are not tied to a prompt or response phase.
805
+ _logger.debug("Report extra_body params as custom metrics")
806
+ pipeline.report_custom_metrics_from_extra_body(association_id, chat_extra_body_params)
722
807
 
723
808
  # ==================================================================
724
809
  # Step 1: Prescore Guards processing
@@ -890,7 +975,13 @@ def init(model_dir: str = os.getcwd()):
890
975
 
891
976
 
892
977
  class ModerationPipeline:
893
- """Base class to simplify interactions with DRUM."""
978
+ """
979
+ Base class to simplify interactions with DRUM.
980
+ This class is not used outside of testing;
981
+ moderation_pipeline_factory() will select the LLM or VDB subclass instead.
982
+ Also: Pipeline and ModerationPipeline are separate classes (not in samm hierarchy)
983
+ However, LlmModerationPipeline includes LLMPipeline by composition.
984
+ """
894
985
 
895
986
  def score(self, input_df: pd.DataFrame, model, drum_score_fn, **kwargs):
896
987
  """Default score function just runs the DRUM score function."""
@@ -898,7 +989,7 @@ class ModerationPipeline:
898
989
 
899
990
  def chat(
900
991
  self,
901
- completion_create_params: pd.DataFrame,
992
+ completion_create_params: CompletionCreateParams,
902
993
  model,
903
994
  drum_chat_fn,
904
995
  association_id: str = None,
@@ -920,13 +1011,16 @@ class LlmModerationPipeline(ModerationPipeline):
920
1011
 
921
1012
  def chat(
922
1013
  self,
923
- completion_create_params: pd.DataFrame,
1014
+ completion_create_params: CompletionCreateParams,
924
1015
  model,
925
1016
  drum_chat_fn,
926
1017
  association_id=None,
927
1018
  **kwargs,
928
1019
  ):
929
- """Calls the standard guard chat function."""
1020
+ """
1021
+ Calls the standard guard chat function.
1022
+ See PythonModelAdapter.chat() in DRUM, which calls chat() here.
1023
+ """
930
1024
  return guard_chat_wrapper(
931
1025
  completion_create_params,
932
1026
  model,
@@ -949,6 +1043,14 @@ class VdbModerationPipeline(ModerationPipeline):
949
1043
  def moderation_pipeline_factory(
950
1044
  target_type: str, model_dir: str = os.getcwd()
951
1045
  ) -> Optional[ModerationPipeline]:
1046
+ """
1047
+ Create and return a moderation pipeline based on model target type.
1048
+ This function is the main integration point with DRUM;
1049
+ called by DRUM's PythonModelAdapter._load_moderation_hooks.
1050
+ :param target_type: usually textgen, agentic, or vdb
1051
+ :param model_dir:
1052
+ :return:
1053
+ """
952
1054
  # Disable ragas and deepeval tracking while loading the module.
953
1055
  os.environ["RAGAS_DO_NOT_TRACK"] = "true"
954
1056
  os.environ["DEEPEVAL_TELEMETRY_OPT_OUT"] = "YES"
@@ -113,7 +113,7 @@ class LLMPipeline(Pipeline):
113
113
  self._custom_model_dir = os.path.dirname(guards_config_filename)
114
114
 
115
115
  self._modifier_guard_seen = {stage: None for stage in GuardStage.ALL}
116
- self.auto_generate_association_ids = False
116
+ self.auto_generate_association_ids = False # used for score, but not used for chat
117
117
 
118
118
  # Dictionary of async http clients per process - its important to maintain
119
119
  # this when moderation is running with CUSTOM_MODEL_WORKERS > 1
@@ -197,17 +197,17 @@ class LLMPipeline(Pipeline):
197
197
 
198
198
  if guard.has_average_score_custom_metric():
199
199
  metric_def = self._get_average_score_metric_definition(guard)
200
- self.add_custom_metric(metric_def, True)
200
+ self.add_custom_metric_definition(metric_def, True)
201
201
 
202
202
  if guard.has_latency_custom_metric():
203
203
  metric_def = guard.get_latency_custom_metric()
204
- self.add_custom_metric(metric_def, False)
204
+ self.add_custom_metric_definition(metric_def, False)
205
205
 
206
206
  if intervention_action:
207
207
  # Enforced metric for all kinds of guards, as long as they have intervention
208
208
  # action defined - even for token count
209
209
  metric_def = guard.get_enforced_custom_metric(guard_stage, intervention_action)
210
- self.add_custom_metric(metric_def, True)
210
+ self.add_custom_metric_definition(metric_def, True)
211
211
 
212
212
  def _add_default_custom_metrics(self):
213
213
  """Default custom metrics"""
@@ -219,14 +219,14 @@ class LLMPipeline(Pipeline):
219
219
  postscore_guard_latency_custom_metric,
220
220
  score_latency,
221
221
  ]:
222
- self.add_custom_metric(metric_def, False)
222
+ self.add_custom_metric_definition(metric_def, False)
223
223
 
224
224
  # These metrics report with an association-id
225
225
  for metric_def in [
226
226
  get_blocked_custom_metric(GuardStage.PROMPT),
227
227
  get_blocked_custom_metric(GuardStage.RESPONSE),
228
228
  ]:
229
- self.add_custom_metric(metric_def, True)
229
+ self.add_custom_metric_definition(metric_def, True)
230
230
 
231
231
  def _add_guard_to_pipeline(self, guard):
232
232
  if guard.stage == GuardStage.PROMPT:
@@ -380,6 +380,55 @@ class LLMPipeline(Pipeline):
380
380
  buckets = self._add_guard_specific_custom_metrics(row, self.get_postscore_guards())
381
381
  payload["buckets"].extend(buckets)
382
382
 
383
+ def add_extra_body_custom_metric_definition(self, chat_extra_body_params: dict) -> None:
384
+ """
385
+ For each name found in extra_body earlier, add to the internal map for reporting later.
386
+ Custom metric IDs are looked up in create_custom_metrics().
387
+ :param chat_extra_body_params: dict of name=value pairs
388
+ """
389
+ for name in chat_extra_body_params.keys():
390
+ self.extra_body_custom_metric_map[name] = {"id": None}
391
+
392
+ def report_custom_metrics_from_extra_body(
393
+ self, association_id: str, extra_params: dict
394
+ ) -> None:
395
+ """
396
+ Add any key-value pairs extracted from extra_body as custom metrics.
397
+ :param association_id: Association ID of the chat request
398
+ :param extra_params: a dict of {"name": value} for all extra_body parameters found
399
+ """
400
+ # If no association ID is defined for deployment, custom metrics will not be processed
401
+ if self._association_id_column_name is None:
402
+ return
403
+ if not extra_params:
404
+ return # nothing to send
405
+ payload = {"buckets": []}
406
+ for name, value in extra_params.items():
407
+ if name in self.custom_metric_map:
408
+ # In case of name collision:
409
+ # the extra_body metric will _not_ override the other moderation metric
410
+ self._logger.warning(
411
+ "extra_body custom metric name is already in use in moderation; "
412
+ f"will not be sent: {name}"
413
+ )
414
+ continue
415
+ if name not in self.extra_body_custom_metric_map:
416
+ self._logger.warning(f"extra_body custom metric ID not in map: {name}")
417
+ continue
418
+ metric_id = self.extra_body_custom_metric_map[name].get("id")
419
+ if not metric_id:
420
+ self._logger.warning(f"extra_body custom metric has missing ID: {name}")
421
+ continue
422
+ payload["buckets"].append(
423
+ {
424
+ "customMetricId": metric_id,
425
+ "value": value,
426
+ "associationId": association_id,
427
+ }
428
+ )
429
+ self._logger.debug(f"Sending custom metrics payload from extra_body: {payload}")
430
+ self.upload_custom_metrics(payload)
431
+
383
432
  def report_custom_metrics(self, result_df):
384
433
  if self.delayed_custom_metric_creation:
385
434
  # Flag is not set yet, so no point reporting custom metrics
@@ -29,6 +29,7 @@ from datarobot.models.deployment import CustomMetric
29
29
  from datarobot_dome.async_http_client import AsyncHTTPClient
30
30
  from datarobot_dome.constants import DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC
31
31
  from datarobot_dome.constants import LOGGER_NAME_PREFIX
32
+ from datarobot_dome.constants import MODERATIONS_USER_AGENT
32
33
  from datarobot_dome.constants import ModerationEventTypes
33
34
 
34
35
  CUSTOM_METRICS_BULK_UPLOAD_API_PREFIX = "deployments"
@@ -40,7 +41,6 @@ class Pipeline:
40
41
 
41
42
  def __init__(self, async_http_timeout_sec=DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC):
42
43
  self._logger = logging.getLogger(LOGGER_NAME_PREFIX + "." + self.__class__.__name__)
43
- self.custom_metric = {}
44
44
  self._deployment = None
45
45
  self._association_id_column_name = None
46
46
  self._datarobot_url = None
@@ -53,6 +53,8 @@ class Pipeline:
53
53
  self._custom_metrics_bulk_upload_url = None
54
54
  self.aggregate_custom_metric = None
55
55
  self.custom_metric_map = dict()
56
+ # even though only LLMPipeline really uses the next map, Pipeline still references it
57
+ self.extra_body_custom_metric_map = dict()
56
58
  self.delayed_custom_metric_creation = False
57
59
  self.upload_custom_metrics_tasks = set()
58
60
 
@@ -76,11 +78,19 @@ class Pipeline:
76
78
  self._logger.warning(f"Missing DataRobot API Token, {self.common_message}")
77
79
  return
78
80
 
81
+ self._deployment_id = os.environ.get("MLOPS_DEPLOYMENT_ID", None)
82
+ if self._deployment_id is None:
83
+ self._logger.warning(
84
+ f"DataRobot deployment id not exported (MLOPS_DEPLOYMENT_ID), {self.common_message}"
85
+ )
86
+ return
87
+
79
88
  # This is regular / default DataRobot Client
80
89
  self.dr_client = dr.Client(endpoint=self._datarobot_url, token=self._datarobot_api_token)
81
90
  self._headers = {
82
91
  "Content-Type": "application/json",
83
92
  "Authorization": f"Bearer {self._datarobot_api_token}",
93
+ "User-Agent": MODERATIONS_USER_AGENT,
84
94
  }
85
95
 
86
96
  def _query_self_deployment(self):
@@ -89,7 +99,6 @@ class Pipeline:
89
99
  moderations for
90
100
  :return:
91
101
  """
92
- self._deployment_id = os.environ.get("MLOPS_DEPLOYMENT_ID", None)
93
102
  if self._deployment_id is None:
94
103
  self._logger.warning(f'Custom Model workshop "test" mode?, {self.common_message}')
95
104
  return
@@ -170,13 +179,17 @@ class Pipeline:
170
179
  self.create_custom_metrics()
171
180
  self.delayed_custom_metric_creation = False
172
181
 
173
- def add_custom_metric(
182
+ def add_custom_metric_definition(
174
183
  self, metric_definition: dict[str, Any], requires_association_id: bool, **kwargs
175
184
  ) -> None:
176
185
  """
177
186
  Adds an entry to the `custom_metric_map`.
187
+ Only 2 functions should write to this map:
188
+ * this function -- links the custom metric definition to its name
189
+ * create_custom_metrics() -- queries DR for the object ID and links it to the name
178
190
 
179
191
  NOTE: the kwargs allow implementations to add their own specialized values.
192
+ Currently only VDBPipeline calls this with kwargs.
180
193
  """
181
194
  name = metric_definition["name"]
182
195
  self.custom_metric_map[name] = {
@@ -191,6 +204,8 @@ class Pipeline:
191
204
 
192
205
  Updates the `custom_metric_map` with id's to insure the appropriate data
193
206
  is put in place for reporting.
207
+
208
+ Every custom metric we want to use must already exist by name in the map.
194
209
  """
195
210
  cleanup_metrics_list = list()
196
211
  for index, (metric_name, custom_metric) in enumerate(self.custom_metric_map.items()):
@@ -254,6 +269,10 @@ class Pipeline:
254
269
  # required in case a metric is duplicated, in which case, we don't have its
255
270
  # id in the loop above
256
271
  #
272
+ # Note: user-defined custom metrics (such as for extra_body params) will also
273
+ # be included in the list. We will warn if an existing metric is not found
274
+ # in either list, which means this chat request does not have a value for it.
275
+ #
257
276
  # We have to go through pagination - dmm list_custom_metrics does not implement
258
277
  # pagination
259
278
  custom_metrics_list = []
@@ -267,12 +286,19 @@ class Pipeline:
267
286
  if response_list["next"] is None:
268
287
  break
269
288
 
289
+ # assign IDs to the "metric by name" maps so we can upload by ID later
270
290
  for metric in custom_metrics_list:
271
291
  metric_name = metric["name"]
272
- if metric_name not in self.custom_metric_map:
273
- self._logger.error(f"Metric '{metric_name}' exists at DR but not in moderation")
292
+ if metric_name in self.custom_metric_map:
293
+ self.custom_metric_map[metric_name]["id"] = metric["id"]
294
+ elif metric_name in self.extra_body_custom_metric_map:
295
+ self.extra_body_custom_metric_map[metric_name]["id"] = metric["id"]
296
+ else:
297
+ self._logger.warning(
298
+ f"Metric '{metric_name}' exists at DR but not in moderation; "
299
+ "no value will be reported"
300
+ )
274
301
  continue
275
- self.custom_metric_map[metric_name]["id"] = metric["id"]
276
302
 
277
303
  # These are the metrics we couldn't create - so, don't track them
278
304
  for metric_name in cleanup_metrics_list:
@@ -70,7 +70,7 @@ class VDBPipeline(Pipeline):
70
70
 
71
71
  # Metric list so far does not need association id for reporting
72
72
  for metric_def, per_row, score_type in metric_list:
73
- self.add_custom_metric(metric_def, per_row, scorer_type=score_type)
73
+ self.add_custom_metric_definition(metric_def, per_row, scorer_type=score_type)
74
74
 
75
75
  def create_scorers(self):
76
76
  """
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.3
1
+ Metadata-Version: 2.4
2
2
  Name: datarobot-moderations
3
- Version: 11.2.3
3
+ Version: 11.2.5
4
4
  Summary: DataRobot Monitoring and Moderation framework
5
5
  License: DataRobot Tool and Utility Agreement
6
6
  Author: DataRobot
@@ -19,7 +19,7 @@ Requires-Dist: deepeval (>=3.3.5)
19
19
  Requires-Dist: langchain (>=0.1.12)
20
20
  Requires-Dist: langchain-nvidia-ai-endpoints (>=0.3.9)
21
21
  Requires-Dist: langchain-openai (>=0.1.7)
22
- Requires-Dist: llama-index (>=0.12.49)
22
+ Requires-Dist: llama-index (>=0.13.0)
23
23
  Requires-Dist: llama-index-embeddings-azure-openai (>=0.1.6)
24
24
  Requires-Dist: llama-index-llms-bedrock-converse (>=0.1.6)
25
25
  Requires-Dist: llama-index-llms-langchain (>=0.1.3)
@@ -1,8 +1,8 @@
1
1
  datarobot_dome/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
2
- datarobot_dome/async_http_client.py,sha256=wkB4irwvnchNGzO1bk2C_HWM-GOSB3AUn5TXKl-X0ZI,9649
2
+ datarobot_dome/async_http_client.py,sha256=cQFoSI2ovt0Kyk4XWQPXod5PAfA-ZPkjLYVWQZhDGDE,9809
3
3
  datarobot_dome/chat_helper.py,sha256=BzvtUyZSZxzOqq-5a2wQKhHhr2kMlcP1MFrHaDAeD_o,9671
4
- datarobot_dome/constants.py,sha256=vM2_JkXbn4dkWARCqxNfLriSo0E05LDXVrwNktptpuc,10416
5
- datarobot_dome/drum_integration.py,sha256=BnhAP-D4AaEeh4ferZ-qXnORuWQzYzw9qKAZUTZZnJU,40542
4
+ datarobot_dome/constants.py,sha256=jvgpHa3Wh_nZVZmfU-6ab8FHnKNW3KxOPYIIEb_oS6U,10662
5
+ datarobot_dome/drum_integration.py,sha256=DIGeiw6fwKCbgi6mb-mJdjPEB6C1uMc-pZotpTnlGZQ,45770
6
6
  datarobot_dome/guard.py,sha256=xJds9hcbUaS-KD5nC1mn0GiPdBrileFUu6BuTAjDNuY,34668
7
7
  datarobot_dome/guard_executor.py,sha256=ox5_jOHcqMaxaaagIYJJHhCwEI7Wg-rUEiu5rutsfVU,35363
8
8
  datarobot_dome/guard_helpers.py,sha256=jfu8JTWCcxu4WD1MKxeP1n53DeebY3SSuP-t5sWyV1U,17187
@@ -14,11 +14,11 @@ datarobot_dome/metrics/citation_metrics.py,sha256=l2mnV1gz7nQeJ_yfaS4dcP3DFWf0p5
14
14
  datarobot_dome/metrics/factory.py,sha256=7caa8paI9LuFXDgguXdC4on28V7IwwIsKJT2Z-Aps8A,2187
15
15
  datarobot_dome/metrics/metric_scorer.py,sha256=uJ_IJRw7ZFHueg8xjsaXbt0ypO7JiydZ0WapCp96yng,2540
16
16
  datarobot_dome/pipeline/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
17
- datarobot_dome/pipeline/llm_pipeline.py,sha256=DMZ4gu88MiSSEQtshDyHOzT3R2Seuf8UqZ7A36QHj3M,18772
18
- datarobot_dome/pipeline/pipeline.py,sha256=7UmvrZtNxTGewpgM4cf2oThHPoJSarEU1Dyp7xEsASU,17401
19
- datarobot_dome/pipeline/vdb_pipeline.py,sha256=q3c_Z-hGUqhH6j6n8VpS3wZiBIkWgpRDsBnyJyZhiw4,9855
17
+ datarobot_dome/pipeline/llm_pipeline.py,sha256=g8XZQjJPPs43GPqaBG4TtsCbvu1o7TWraOmzhtUGB_o,21172
18
+ datarobot_dome/pipeline/pipeline.py,sha256=iDR6h25RDp4bnTW1eGWErG9W_ZXDdJ_sQGVD-tT_Ll0,18805
19
+ datarobot_dome/pipeline/vdb_pipeline.py,sha256=zt5d_41oJjdT8qOtvpgz-l5uvImwKE9f6pQsAU_TdR4,9866
20
20
  datarobot_dome/runtime.py,sha256=FD8wXOweqoQVzbZMh-mucL66xT2kGxPsJUGAcJBgwxw,1468
21
21
  datarobot_dome/streaming.py,sha256=DkvKEH0yN0aPEWMTAjMFJB3Kx4iLGdjUMQU1pAplbeg,17751
22
- datarobot_moderations-11.2.3.dist-info/METADATA,sha256=dzpTYxhAXg-NEm8Rrko8U8qvbQncQoGw93a9ZhWV3jo,4742
23
- datarobot_moderations-11.2.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
24
- datarobot_moderations-11.2.3.dist-info/RECORD,,
22
+ datarobot_moderations-11.2.5.dist-info/METADATA,sha256=9Aj3fZnLvcgKpiAQwnvkkAzA3SxaBq2j45ctWEMCEaI,4741
23
+ datarobot_moderations-11.2.5.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
24
+ datarobot_moderations-11.2.5.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: poetry-core 2.1.3
2
+ Generator: poetry-core 2.2.1
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any