datarobot-moderations 11.2.3__py3-none-any.whl → 11.2.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datarobot_dome/async_http_client.py +4 -1
- datarobot_dome/constants.py +5 -0
- datarobot_dome/drum_integration.py +120 -18
- datarobot_dome/pipeline/llm_pipeline.py +55 -6
- datarobot_dome/pipeline/pipeline.py +32 -6
- datarobot_dome/pipeline/vdb_pipeline.py +1 -1
- {datarobot_moderations-11.2.3.dist-info → datarobot_moderations-11.2.5.dist-info}/METADATA +3 -3
- {datarobot_moderations-11.2.3.dist-info → datarobot_moderations-11.2.5.dist-info}/RECORD +9 -9
- {datarobot_moderations-11.2.3.dist-info → datarobot_moderations-11.2.5.dist-info}/WHEEL +1 -1
|
@@ -28,6 +28,7 @@ from datarobot_dome.constants import DATAROBOT_CONFIGURED_ON_PREM_ST_SAAS_URL
|
|
|
28
28
|
from datarobot_dome.constants import DATAROBOT_SERVERLESS_PLATFORM
|
|
29
29
|
from datarobot_dome.constants import DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC
|
|
30
30
|
from datarobot_dome.constants import LOGGER_NAME_PREFIX
|
|
31
|
+
from datarobot_dome.constants import MODERATIONS_USER_AGENT
|
|
31
32
|
from datarobot_dome.constants import RETRY_COUNT
|
|
32
33
|
from datarobot_dome.constants import ModerationEventTypes
|
|
33
34
|
|
|
@@ -81,11 +82,13 @@ class AsyncHTTPClient:
|
|
|
81
82
|
"Content-Type": "text/csv",
|
|
82
83
|
"Accept": "text/csv",
|
|
83
84
|
"Authorization": f"Bearer {os.environ['DATAROBOT_API_TOKEN']}",
|
|
85
|
+
"User-Agent": MODERATIONS_USER_AGENT,
|
|
84
86
|
}
|
|
85
87
|
self.json_headers = {
|
|
86
88
|
"Content-Type": "application/json",
|
|
87
89
|
"Accept": "application/json",
|
|
88
90
|
"Authorization": f"Bearer {os.environ['DATAROBOT_API_TOKEN']}",
|
|
91
|
+
"User-Agent": MODERATIONS_USER_AGENT,
|
|
89
92
|
}
|
|
90
93
|
self.session = None
|
|
91
94
|
self.events_url = f"{os.environ['DATAROBOT_ENDPOINT']}/remoteEvents/"
|
|
@@ -100,9 +103,9 @@ class AsyncHTTPClient:
|
|
|
100
103
|
asyncio.set_event_loop(self.loop)
|
|
101
104
|
else:
|
|
102
105
|
raise
|
|
106
|
+
nest_asyncio.apply(loop=self.loop)
|
|
103
107
|
self.loop.run_until_complete(self.__create_client_session(timeout))
|
|
104
108
|
self.loop.set_debug(True)
|
|
105
|
-
nest_asyncio.apply(loop=self.loop)
|
|
106
109
|
|
|
107
110
|
atexit.register(self.shutdown)
|
|
108
111
|
|
datarobot_dome/constants.py
CHANGED
|
@@ -10,9 +10,12 @@
|
|
|
10
10
|
# https://www.datarobot.com/wp-content/uploads/2021/07/DataRobot-Tool-and-Utility-Agreement.pdf.
|
|
11
11
|
# ---------------------------------------------------------------------------------
|
|
12
12
|
from enum import Enum
|
|
13
|
+
from importlib.metadata import version
|
|
13
14
|
|
|
14
15
|
__GUARD_ASSOCIATION_IDS_COLUMN_NAME__ = "datarobot_guard_association_id"
|
|
15
16
|
|
|
17
|
+
MODERATIONS_USER_AGENT = f"datarobot-moderations:{version('datarobot-moderations')}"
|
|
18
|
+
|
|
16
19
|
LOGGER_NAME_PREFIX = "moderations"
|
|
17
20
|
|
|
18
21
|
DEFAULT_PROMPT_COLUMN_NAME = "promptText"
|
|
@@ -71,6 +74,8 @@ PROMPT_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE = "prompt_token_count_from_usage"
|
|
|
71
74
|
RESPONSE_TOKEN_COUNT_COLUMN_NAME_FROM_USAGE = "response_token_count_from_usage"
|
|
72
75
|
|
|
73
76
|
SPAN_PREFIX = "datarobot.guard"
|
|
77
|
+
DATAROBOT_ASSOCIATION_ID_FIELD_NAME = "datarobot_association_id"
|
|
78
|
+
DATAROBOT_METRICS_DICT_FIELD_NAME = "datarobot_metrics"
|
|
74
79
|
|
|
75
80
|
|
|
76
81
|
class TargetType(str, Enum):
|
|
@@ -24,6 +24,7 @@ import numpy as np
|
|
|
24
24
|
import pandas as pd
|
|
25
25
|
import yaml
|
|
26
26
|
from openai.types.chat import ChatCompletionChunk
|
|
27
|
+
from openai.types.chat import CompletionCreateParams
|
|
27
28
|
from openai.types.chat.chat_completion import ChatCompletion
|
|
28
29
|
from openai.types.chat.chat_completion import Choice
|
|
29
30
|
from openai.types.chat.chat_completion_message import ChatCompletionMessage
|
|
@@ -40,6 +41,8 @@ from datarobot_dome.chat_helper import run_postscore_guards
|
|
|
40
41
|
from datarobot_dome.constants import AGENTIC_PIPELINE_INTERACTIONS_ATTR
|
|
41
42
|
from datarobot_dome.constants import CHAT_COMPLETION_OBJECT
|
|
42
43
|
from datarobot_dome.constants import CITATIONS_ATTR
|
|
44
|
+
from datarobot_dome.constants import DATAROBOT_ASSOCIATION_ID_FIELD_NAME
|
|
45
|
+
from datarobot_dome.constants import DATAROBOT_METRICS_DICT_FIELD_NAME
|
|
43
46
|
from datarobot_dome.constants import DATAROBOT_MODERATIONS_ATTR
|
|
44
47
|
from datarobot_dome.constants import DISABLE_MODERATION_RUNTIME_PARAM_NAME
|
|
45
48
|
from datarobot_dome.constants import LLM_BLUEPRINT_ID_ATTR
|
|
@@ -590,6 +593,29 @@ def _set_moderation_attribute_to_completion(pipeline, chat_completion, df, assoc
|
|
|
590
593
|
|
|
591
594
|
|
|
592
595
|
def get_chat_prompt(completion_create_params):
|
|
596
|
+
"""
|
|
597
|
+
Validate and extract the user prompt from completion create parameters (CCP).
|
|
598
|
+
Include tool calls if they were provided.
|
|
599
|
+
|
|
600
|
+
CCP "messages" list must be non-empty and include content with "user" role.
|
|
601
|
+
Example: "messages": [{"role": "user", "content": "What is the meaning of life?"}]
|
|
602
|
+
|
|
603
|
+
:param completion_create_params: dict containing chat request
|
|
604
|
+
:return: constructed prompt based on CCP content.
|
|
605
|
+
:raise ValueError if completion create parameters is not valid.
|
|
606
|
+
"""
|
|
607
|
+
# ensure message content exists
|
|
608
|
+
if (
|
|
609
|
+
"messages" not in completion_create_params
|
|
610
|
+
or completion_create_params["messages"] is None
|
|
611
|
+
or len(completion_create_params["messages"]) == 0
|
|
612
|
+
or not isinstance(completion_create_params["messages"][-1], dict)
|
|
613
|
+
or "content" not in completion_create_params["messages"][-1]
|
|
614
|
+
):
|
|
615
|
+
raise ValueError(
|
|
616
|
+
f"Chat input for moderation does not contain a message: {completion_create_params}"
|
|
617
|
+
)
|
|
618
|
+
|
|
593
619
|
# Get the prompt with role = User
|
|
594
620
|
last_user_message = None
|
|
595
621
|
tool_calls = []
|
|
@@ -599,7 +625,7 @@ def get_chat_prompt(completion_create_params):
|
|
|
599
625
|
if message["role"] == "tool":
|
|
600
626
|
tool_calls.append(f"{message.get('name', '')}_{message['content']}")
|
|
601
627
|
if last_user_message is None:
|
|
602
|
-
raise
|
|
628
|
+
raise ValueError("No message with 'user' role found in input")
|
|
603
629
|
|
|
604
630
|
prompt_content = last_user_message["content"]
|
|
605
631
|
tool_names = []
|
|
@@ -623,7 +649,7 @@ def get_chat_prompt(completion_create_params):
|
|
|
623
649
|
concatenated_prompt.append(message)
|
|
624
650
|
chat_prompt = "\n".join(concatenated_prompt)
|
|
625
651
|
else:
|
|
626
|
-
raise
|
|
652
|
+
raise ValueError(f"Unhandled prompt type: {type(prompt_content)}")
|
|
627
653
|
|
|
628
654
|
if len(tool_calls) > 0:
|
|
629
655
|
# Lets not add tool names if tool calls are present. Tool calls are more
|
|
@@ -693,32 +719,91 @@ def report_otel_evaluation_set_metric(pipeline, result_df):
|
|
|
693
719
|
current_span.set_attribute("datarobot.moderation.evaluation", json.dumps(final_value))
|
|
694
720
|
|
|
695
721
|
|
|
722
|
+
def filter_extra_body(
|
|
723
|
+
completion_create_params: CompletionCreateParams,
|
|
724
|
+
) -> tuple[CompletionCreateParams, dict]:
|
|
725
|
+
"""
|
|
726
|
+
completion_create_params is a typed dict of a few standard fields,
|
|
727
|
+
and arbitrary fields from extra_body.
|
|
728
|
+
If "datarobot_metrics" is in extra_body, process it here.
|
|
729
|
+
Save its value only if it is a dict as expected.
|
|
730
|
+
:param completion_create_params: the chat completion params from OpenAI client via DRUM
|
|
731
|
+
:return: filtered completion_create_params; dict of {name: value} for "datarobot_" fields
|
|
732
|
+
"""
|
|
733
|
+
datarobot_extra_body_params = {}
|
|
734
|
+
name = DATAROBOT_METRICS_DICT_FIELD_NAME
|
|
735
|
+
if name in completion_create_params:
|
|
736
|
+
value = completion_create_params[name]
|
|
737
|
+
_logger.debug("found DataRobot metrics in extra_body: %s", f"{name}={value}")
|
|
738
|
+
if isinstance(value, dict):
|
|
739
|
+
datarobot_extra_body_params = copy.deepcopy(value)
|
|
740
|
+
else:
|
|
741
|
+
_logger.warning("DataRobot metrics in extra_body is not a dict: %s", f"{name}={value}")
|
|
742
|
+
completion_create_params.pop(name, None)
|
|
743
|
+
return completion_create_params, datarobot_extra_body_params
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def filter_association_id(
|
|
747
|
+
completion_create_params: CompletionCreateParams,
|
|
748
|
+
) -> tuple[CompletionCreateParams, str | None]:
|
|
749
|
+
"""
|
|
750
|
+
completion_create_params (CCP) is a typed dict of a few standard fields,
|
|
751
|
+
and arbitrary fields from extra_body.
|
|
752
|
+
If a field for the association ID exists, extract that value and remove it from the CCP.
|
|
753
|
+
Do this before calling filter_extra_body(), which would otherwise capture the association ID.
|
|
754
|
+
:param completion_create_params: the chat completion params from OpenAI client via DRUM
|
|
755
|
+
:return: filtered completion_create_params, association ID value
|
|
756
|
+
|
|
757
|
+
If no association ID was found in extra body: return original CCP,None
|
|
758
|
+
"""
|
|
759
|
+
name = DATAROBOT_ASSOCIATION_ID_FIELD_NAME
|
|
760
|
+
if name in completion_create_params:
|
|
761
|
+
value = completion_create_params[name]
|
|
762
|
+
_logger.debug("found association ID in extra_body: %s", f"{name}={value}")
|
|
763
|
+
completion_create_params.pop(name, None)
|
|
764
|
+
return completion_create_params, value
|
|
765
|
+
return completion_create_params, None
|
|
766
|
+
|
|
767
|
+
|
|
696
768
|
def guard_chat_wrapper(
|
|
697
769
|
completion_create_params, model, pipeline, drum_chat_fn, association_id=None, **kwargs
|
|
698
770
|
):
|
|
771
|
+
# if association ID was included in extra_body, extract field name and value
|
|
772
|
+
completion_create_params, eb_assoc_id_value = filter_association_id(completion_create_params)
|
|
773
|
+
|
|
774
|
+
# extract any fields mentioned in "datarobot_metrics" to send as custom metrics later
|
|
775
|
+
completion_create_params, chat_extra_body_params = filter_extra_body(completion_create_params)
|
|
776
|
+
|
|
777
|
+
# register them for ID lookup later
|
|
778
|
+
pipeline.add_extra_body_custom_metric_definition(chat_extra_body_params)
|
|
779
|
+
|
|
780
|
+
# define all pipeline-based and guard-based custom metrics (but not those from extra_body)
|
|
781
|
+
# sends single API request to look up IDs for all custom metrics (including extra_body)
|
|
699
782
|
pipeline.get_new_metrics_payload()
|
|
700
783
|
|
|
784
|
+
# the chat request is not a dataframe, but we'll build a DF internally for moderation.
|
|
701
785
|
prompt_column_name = pipeline.get_input_column(GuardStage.PROMPT)
|
|
702
|
-
if (
|
|
703
|
-
"messages" not in completion_create_params
|
|
704
|
-
or completion_create_params["messages"] is None
|
|
705
|
-
or len(completion_create_params["messages"]) == 0
|
|
706
|
-
or not isinstance(completion_create_params["messages"][-1], dict)
|
|
707
|
-
or "content" not in completion_create_params["messages"][-1]
|
|
708
|
-
):
|
|
709
|
-
raise ValueError(f"Invalid chat input for moderation: {completion_create_params}")
|
|
710
|
-
|
|
711
786
|
prompt = get_chat_prompt(completion_create_params)
|
|
712
787
|
streaming_response_requested = completion_create_params.get("stream", False)
|
|
713
788
|
|
|
714
789
|
data = pd.DataFrame({prompt_column_name: [prompt]})
|
|
790
|
+
# for association IDs (with or without extra_body): the column must be defined in the deployment
|
|
791
|
+
# (here, this means pipeline.get_association_id_column_name() ("standard name") is not empty.)
|
|
792
|
+
# there are 3 likely cases for association ID, and 1 corner case:
|
|
793
|
+
# 1. ID value not provided (drum or extra_body) => no association ID column
|
|
794
|
+
# 2. ID value provided by DRUM => new DF column with standard name and provided value
|
|
795
|
+
# 3. ID defined in extra_body => new DF column with standard name and extra_body value
|
|
796
|
+
# 4. ID in extra_body with empty value => no association ID column
|
|
797
|
+
# Moderation library no longer auto-generates an association ID for chat. However, DRUM does.
|
|
715
798
|
association_id_column_name = pipeline.get_association_id_column_name()
|
|
799
|
+
association_id = eb_assoc_id_value or association_id
|
|
716
800
|
if association_id_column_name:
|
|
717
801
|
if association_id:
|
|
718
802
|
data[association_id_column_name] = [association_id]
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
803
|
+
|
|
804
|
+
# report any metrics from extra_body. They are not tied to a prompt or response phase.
|
|
805
|
+
_logger.debug("Report extra_body params as custom metrics")
|
|
806
|
+
pipeline.report_custom_metrics_from_extra_body(association_id, chat_extra_body_params)
|
|
722
807
|
|
|
723
808
|
# ==================================================================
|
|
724
809
|
# Step 1: Prescore Guards processing
|
|
@@ -890,7 +975,13 @@ def init(model_dir: str = os.getcwd()):
|
|
|
890
975
|
|
|
891
976
|
|
|
892
977
|
class ModerationPipeline:
|
|
893
|
-
"""
|
|
978
|
+
"""
|
|
979
|
+
Base class to simplify interactions with DRUM.
|
|
980
|
+
This class is not used outside of testing;
|
|
981
|
+
moderation_pipeline_factory() will select the LLM or VDB subclass instead.
|
|
982
|
+
Also: Pipeline and ModerationPipeline are separate classes (not in samm hierarchy)
|
|
983
|
+
However, LlmModerationPipeline includes LLMPipeline by composition.
|
|
984
|
+
"""
|
|
894
985
|
|
|
895
986
|
def score(self, input_df: pd.DataFrame, model, drum_score_fn, **kwargs):
|
|
896
987
|
"""Default score function just runs the DRUM score function."""
|
|
@@ -898,7 +989,7 @@ class ModerationPipeline:
|
|
|
898
989
|
|
|
899
990
|
def chat(
|
|
900
991
|
self,
|
|
901
|
-
completion_create_params:
|
|
992
|
+
completion_create_params: CompletionCreateParams,
|
|
902
993
|
model,
|
|
903
994
|
drum_chat_fn,
|
|
904
995
|
association_id: str = None,
|
|
@@ -920,13 +1011,16 @@ class LlmModerationPipeline(ModerationPipeline):
|
|
|
920
1011
|
|
|
921
1012
|
def chat(
|
|
922
1013
|
self,
|
|
923
|
-
completion_create_params:
|
|
1014
|
+
completion_create_params: CompletionCreateParams,
|
|
924
1015
|
model,
|
|
925
1016
|
drum_chat_fn,
|
|
926
1017
|
association_id=None,
|
|
927
1018
|
**kwargs,
|
|
928
1019
|
):
|
|
929
|
-
"""
|
|
1020
|
+
"""
|
|
1021
|
+
Calls the standard guard chat function.
|
|
1022
|
+
See PythonModelAdapter.chat() in DRUM, which calls chat() here.
|
|
1023
|
+
"""
|
|
930
1024
|
return guard_chat_wrapper(
|
|
931
1025
|
completion_create_params,
|
|
932
1026
|
model,
|
|
@@ -949,6 +1043,14 @@ class VdbModerationPipeline(ModerationPipeline):
|
|
|
949
1043
|
def moderation_pipeline_factory(
|
|
950
1044
|
target_type: str, model_dir: str = os.getcwd()
|
|
951
1045
|
) -> Optional[ModerationPipeline]:
|
|
1046
|
+
"""
|
|
1047
|
+
Create and return a moderation pipeline based on model target type.
|
|
1048
|
+
This function is the main integration point with DRUM;
|
|
1049
|
+
called by DRUM's PythonModelAdapter._load_moderation_hooks.
|
|
1050
|
+
:param target_type: usually textgen, agentic, or vdb
|
|
1051
|
+
:param model_dir:
|
|
1052
|
+
:return:
|
|
1053
|
+
"""
|
|
952
1054
|
# Disable ragas and deepeval tracking while loading the module.
|
|
953
1055
|
os.environ["RAGAS_DO_NOT_TRACK"] = "true"
|
|
954
1056
|
os.environ["DEEPEVAL_TELEMETRY_OPT_OUT"] = "YES"
|
|
@@ -113,7 +113,7 @@ class LLMPipeline(Pipeline):
|
|
|
113
113
|
self._custom_model_dir = os.path.dirname(guards_config_filename)
|
|
114
114
|
|
|
115
115
|
self._modifier_guard_seen = {stage: None for stage in GuardStage.ALL}
|
|
116
|
-
self.auto_generate_association_ids = False
|
|
116
|
+
self.auto_generate_association_ids = False # used for score, but not used for chat
|
|
117
117
|
|
|
118
118
|
# Dictionary of async http clients per process - its important to maintain
|
|
119
119
|
# this when moderation is running with CUSTOM_MODEL_WORKERS > 1
|
|
@@ -197,17 +197,17 @@ class LLMPipeline(Pipeline):
|
|
|
197
197
|
|
|
198
198
|
if guard.has_average_score_custom_metric():
|
|
199
199
|
metric_def = self._get_average_score_metric_definition(guard)
|
|
200
|
-
self.
|
|
200
|
+
self.add_custom_metric_definition(metric_def, True)
|
|
201
201
|
|
|
202
202
|
if guard.has_latency_custom_metric():
|
|
203
203
|
metric_def = guard.get_latency_custom_metric()
|
|
204
|
-
self.
|
|
204
|
+
self.add_custom_metric_definition(metric_def, False)
|
|
205
205
|
|
|
206
206
|
if intervention_action:
|
|
207
207
|
# Enforced metric for all kinds of guards, as long as they have intervention
|
|
208
208
|
# action defined - even for token count
|
|
209
209
|
metric_def = guard.get_enforced_custom_metric(guard_stage, intervention_action)
|
|
210
|
-
self.
|
|
210
|
+
self.add_custom_metric_definition(metric_def, True)
|
|
211
211
|
|
|
212
212
|
def _add_default_custom_metrics(self):
|
|
213
213
|
"""Default custom metrics"""
|
|
@@ -219,14 +219,14 @@ class LLMPipeline(Pipeline):
|
|
|
219
219
|
postscore_guard_latency_custom_metric,
|
|
220
220
|
score_latency,
|
|
221
221
|
]:
|
|
222
|
-
self.
|
|
222
|
+
self.add_custom_metric_definition(metric_def, False)
|
|
223
223
|
|
|
224
224
|
# These metrics report with an association-id
|
|
225
225
|
for metric_def in [
|
|
226
226
|
get_blocked_custom_metric(GuardStage.PROMPT),
|
|
227
227
|
get_blocked_custom_metric(GuardStage.RESPONSE),
|
|
228
228
|
]:
|
|
229
|
-
self.
|
|
229
|
+
self.add_custom_metric_definition(metric_def, True)
|
|
230
230
|
|
|
231
231
|
def _add_guard_to_pipeline(self, guard):
|
|
232
232
|
if guard.stage == GuardStage.PROMPT:
|
|
@@ -380,6 +380,55 @@ class LLMPipeline(Pipeline):
|
|
|
380
380
|
buckets = self._add_guard_specific_custom_metrics(row, self.get_postscore_guards())
|
|
381
381
|
payload["buckets"].extend(buckets)
|
|
382
382
|
|
|
383
|
+
def add_extra_body_custom_metric_definition(self, chat_extra_body_params: dict) -> None:
|
|
384
|
+
"""
|
|
385
|
+
For each name found in extra_body earlier, add to the internal map for reporting later.
|
|
386
|
+
Custom metric IDs are looked up in create_custom_metrics().
|
|
387
|
+
:param chat_extra_body_params: dict of name=value pairs
|
|
388
|
+
"""
|
|
389
|
+
for name in chat_extra_body_params.keys():
|
|
390
|
+
self.extra_body_custom_metric_map[name] = {"id": None}
|
|
391
|
+
|
|
392
|
+
def report_custom_metrics_from_extra_body(
|
|
393
|
+
self, association_id: str, extra_params: dict
|
|
394
|
+
) -> None:
|
|
395
|
+
"""
|
|
396
|
+
Add any key-value pairs extracted from extra_body as custom metrics.
|
|
397
|
+
:param association_id: Association ID of the chat request
|
|
398
|
+
:param extra_params: a dict of {"name": value} for all extra_body parameters found
|
|
399
|
+
"""
|
|
400
|
+
# If no association ID is defined for deployment, custom metrics will not be processed
|
|
401
|
+
if self._association_id_column_name is None:
|
|
402
|
+
return
|
|
403
|
+
if not extra_params:
|
|
404
|
+
return # nothing to send
|
|
405
|
+
payload = {"buckets": []}
|
|
406
|
+
for name, value in extra_params.items():
|
|
407
|
+
if name in self.custom_metric_map:
|
|
408
|
+
# In case of name collision:
|
|
409
|
+
# the extra_body metric will _not_ override the other moderation metric
|
|
410
|
+
self._logger.warning(
|
|
411
|
+
"extra_body custom metric name is already in use in moderation; "
|
|
412
|
+
f"will not be sent: {name}"
|
|
413
|
+
)
|
|
414
|
+
continue
|
|
415
|
+
if name not in self.extra_body_custom_metric_map:
|
|
416
|
+
self._logger.warning(f"extra_body custom metric ID not in map: {name}")
|
|
417
|
+
continue
|
|
418
|
+
metric_id = self.extra_body_custom_metric_map[name].get("id")
|
|
419
|
+
if not metric_id:
|
|
420
|
+
self._logger.warning(f"extra_body custom metric has missing ID: {name}")
|
|
421
|
+
continue
|
|
422
|
+
payload["buckets"].append(
|
|
423
|
+
{
|
|
424
|
+
"customMetricId": metric_id,
|
|
425
|
+
"value": value,
|
|
426
|
+
"associationId": association_id,
|
|
427
|
+
}
|
|
428
|
+
)
|
|
429
|
+
self._logger.debug(f"Sending custom metrics payload from extra_body: {payload}")
|
|
430
|
+
self.upload_custom_metrics(payload)
|
|
431
|
+
|
|
383
432
|
def report_custom_metrics(self, result_df):
|
|
384
433
|
if self.delayed_custom_metric_creation:
|
|
385
434
|
# Flag is not set yet, so no point reporting custom metrics
|
|
@@ -29,6 +29,7 @@ from datarobot.models.deployment import CustomMetric
|
|
|
29
29
|
from datarobot_dome.async_http_client import AsyncHTTPClient
|
|
30
30
|
from datarobot_dome.constants import DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC
|
|
31
31
|
from datarobot_dome.constants import LOGGER_NAME_PREFIX
|
|
32
|
+
from datarobot_dome.constants import MODERATIONS_USER_AGENT
|
|
32
33
|
from datarobot_dome.constants import ModerationEventTypes
|
|
33
34
|
|
|
34
35
|
CUSTOM_METRICS_BULK_UPLOAD_API_PREFIX = "deployments"
|
|
@@ -40,7 +41,6 @@ class Pipeline:
|
|
|
40
41
|
|
|
41
42
|
def __init__(self, async_http_timeout_sec=DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC):
|
|
42
43
|
self._logger = logging.getLogger(LOGGER_NAME_PREFIX + "." + self.__class__.__name__)
|
|
43
|
-
self.custom_metric = {}
|
|
44
44
|
self._deployment = None
|
|
45
45
|
self._association_id_column_name = None
|
|
46
46
|
self._datarobot_url = None
|
|
@@ -53,6 +53,8 @@ class Pipeline:
|
|
|
53
53
|
self._custom_metrics_bulk_upload_url = None
|
|
54
54
|
self.aggregate_custom_metric = None
|
|
55
55
|
self.custom_metric_map = dict()
|
|
56
|
+
# even though only LLMPipeline really uses the next map, Pipeline still references it
|
|
57
|
+
self.extra_body_custom_metric_map = dict()
|
|
56
58
|
self.delayed_custom_metric_creation = False
|
|
57
59
|
self.upload_custom_metrics_tasks = set()
|
|
58
60
|
|
|
@@ -76,11 +78,19 @@ class Pipeline:
|
|
|
76
78
|
self._logger.warning(f"Missing DataRobot API Token, {self.common_message}")
|
|
77
79
|
return
|
|
78
80
|
|
|
81
|
+
self._deployment_id = os.environ.get("MLOPS_DEPLOYMENT_ID", None)
|
|
82
|
+
if self._deployment_id is None:
|
|
83
|
+
self._logger.warning(
|
|
84
|
+
f"DataRobot deployment id not exported (MLOPS_DEPLOYMENT_ID), {self.common_message}"
|
|
85
|
+
)
|
|
86
|
+
return
|
|
87
|
+
|
|
79
88
|
# This is regular / default DataRobot Client
|
|
80
89
|
self.dr_client = dr.Client(endpoint=self._datarobot_url, token=self._datarobot_api_token)
|
|
81
90
|
self._headers = {
|
|
82
91
|
"Content-Type": "application/json",
|
|
83
92
|
"Authorization": f"Bearer {self._datarobot_api_token}",
|
|
93
|
+
"User-Agent": MODERATIONS_USER_AGENT,
|
|
84
94
|
}
|
|
85
95
|
|
|
86
96
|
def _query_self_deployment(self):
|
|
@@ -89,7 +99,6 @@ class Pipeline:
|
|
|
89
99
|
moderations for
|
|
90
100
|
:return:
|
|
91
101
|
"""
|
|
92
|
-
self._deployment_id = os.environ.get("MLOPS_DEPLOYMENT_ID", None)
|
|
93
102
|
if self._deployment_id is None:
|
|
94
103
|
self._logger.warning(f'Custom Model workshop "test" mode?, {self.common_message}')
|
|
95
104
|
return
|
|
@@ -170,13 +179,17 @@ class Pipeline:
|
|
|
170
179
|
self.create_custom_metrics()
|
|
171
180
|
self.delayed_custom_metric_creation = False
|
|
172
181
|
|
|
173
|
-
def
|
|
182
|
+
def add_custom_metric_definition(
|
|
174
183
|
self, metric_definition: dict[str, Any], requires_association_id: bool, **kwargs
|
|
175
184
|
) -> None:
|
|
176
185
|
"""
|
|
177
186
|
Adds an entry to the `custom_metric_map`.
|
|
187
|
+
Only 2 functions should write to this map:
|
|
188
|
+
* this function -- links the custom metric definition to its name
|
|
189
|
+
* create_custom_metrics() -- queries DR for the object ID and links it to the name
|
|
178
190
|
|
|
179
191
|
NOTE: the kwargs allow implementations to add their own specialized values.
|
|
192
|
+
Currently only VDBPipeline calls this with kwargs.
|
|
180
193
|
"""
|
|
181
194
|
name = metric_definition["name"]
|
|
182
195
|
self.custom_metric_map[name] = {
|
|
@@ -191,6 +204,8 @@ class Pipeline:
|
|
|
191
204
|
|
|
192
205
|
Updates the `custom_metric_map` with id's to insure the appropriate data
|
|
193
206
|
is put in place for reporting.
|
|
207
|
+
|
|
208
|
+
Every custom metric we want to use must already exist by name in the map.
|
|
194
209
|
"""
|
|
195
210
|
cleanup_metrics_list = list()
|
|
196
211
|
for index, (metric_name, custom_metric) in enumerate(self.custom_metric_map.items()):
|
|
@@ -254,6 +269,10 @@ class Pipeline:
|
|
|
254
269
|
# required in case a metric is duplicated, in which case, we don't have its
|
|
255
270
|
# id in the loop above
|
|
256
271
|
#
|
|
272
|
+
# Note: user-defined custom metrics (such as for extra_body params) will also
|
|
273
|
+
# be included in the list. We will warn if an existing metric is not found
|
|
274
|
+
# in either list, which means this chat request does not have a value for it.
|
|
275
|
+
#
|
|
257
276
|
# We have to go through pagination - dmm list_custom_metrics does not implement
|
|
258
277
|
# pagination
|
|
259
278
|
custom_metrics_list = []
|
|
@@ -267,12 +286,19 @@ class Pipeline:
|
|
|
267
286
|
if response_list["next"] is None:
|
|
268
287
|
break
|
|
269
288
|
|
|
289
|
+
# assign IDs to the "metric by name" maps so we can upload by ID later
|
|
270
290
|
for metric in custom_metrics_list:
|
|
271
291
|
metric_name = metric["name"]
|
|
272
|
-
if metric_name
|
|
273
|
-
self.
|
|
292
|
+
if metric_name in self.custom_metric_map:
|
|
293
|
+
self.custom_metric_map[metric_name]["id"] = metric["id"]
|
|
294
|
+
elif metric_name in self.extra_body_custom_metric_map:
|
|
295
|
+
self.extra_body_custom_metric_map[metric_name]["id"] = metric["id"]
|
|
296
|
+
else:
|
|
297
|
+
self._logger.warning(
|
|
298
|
+
f"Metric '{metric_name}' exists at DR but not in moderation; "
|
|
299
|
+
"no value will be reported"
|
|
300
|
+
)
|
|
274
301
|
continue
|
|
275
|
-
self.custom_metric_map[metric_name]["id"] = metric["id"]
|
|
276
302
|
|
|
277
303
|
# These are the metrics we couldn't create - so, don't track them
|
|
278
304
|
for metric_name in cleanup_metrics_list:
|
|
@@ -70,7 +70,7 @@ class VDBPipeline(Pipeline):
|
|
|
70
70
|
|
|
71
71
|
# Metric list so far does not need association id for reporting
|
|
72
72
|
for metric_def, per_row, score_type in metric_list:
|
|
73
|
-
self.
|
|
73
|
+
self.add_custom_metric_definition(metric_def, per_row, scorer_type=score_type)
|
|
74
74
|
|
|
75
75
|
def create_scorers(self):
|
|
76
76
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: datarobot-moderations
|
|
3
|
-
Version: 11.2.
|
|
3
|
+
Version: 11.2.5
|
|
4
4
|
Summary: DataRobot Monitoring and Moderation framework
|
|
5
5
|
License: DataRobot Tool and Utility Agreement
|
|
6
6
|
Author: DataRobot
|
|
@@ -19,7 +19,7 @@ Requires-Dist: deepeval (>=3.3.5)
|
|
|
19
19
|
Requires-Dist: langchain (>=0.1.12)
|
|
20
20
|
Requires-Dist: langchain-nvidia-ai-endpoints (>=0.3.9)
|
|
21
21
|
Requires-Dist: langchain-openai (>=0.1.7)
|
|
22
|
-
Requires-Dist: llama-index (>=0.
|
|
22
|
+
Requires-Dist: llama-index (>=0.13.0)
|
|
23
23
|
Requires-Dist: llama-index-embeddings-azure-openai (>=0.1.6)
|
|
24
24
|
Requires-Dist: llama-index-llms-bedrock-converse (>=0.1.6)
|
|
25
25
|
Requires-Dist: llama-index-llms-langchain (>=0.1.3)
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
datarobot_dome/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
|
|
2
|
-
datarobot_dome/async_http_client.py,sha256=
|
|
2
|
+
datarobot_dome/async_http_client.py,sha256=cQFoSI2ovt0Kyk4XWQPXod5PAfA-ZPkjLYVWQZhDGDE,9809
|
|
3
3
|
datarobot_dome/chat_helper.py,sha256=BzvtUyZSZxzOqq-5a2wQKhHhr2kMlcP1MFrHaDAeD_o,9671
|
|
4
|
-
datarobot_dome/constants.py,sha256=
|
|
5
|
-
datarobot_dome/drum_integration.py,sha256=
|
|
4
|
+
datarobot_dome/constants.py,sha256=jvgpHa3Wh_nZVZmfU-6ab8FHnKNW3KxOPYIIEb_oS6U,10662
|
|
5
|
+
datarobot_dome/drum_integration.py,sha256=DIGeiw6fwKCbgi6mb-mJdjPEB6C1uMc-pZotpTnlGZQ,45770
|
|
6
6
|
datarobot_dome/guard.py,sha256=xJds9hcbUaS-KD5nC1mn0GiPdBrileFUu6BuTAjDNuY,34668
|
|
7
7
|
datarobot_dome/guard_executor.py,sha256=ox5_jOHcqMaxaaagIYJJHhCwEI7Wg-rUEiu5rutsfVU,35363
|
|
8
8
|
datarobot_dome/guard_helpers.py,sha256=jfu8JTWCcxu4WD1MKxeP1n53DeebY3SSuP-t5sWyV1U,17187
|
|
@@ -14,11 +14,11 @@ datarobot_dome/metrics/citation_metrics.py,sha256=l2mnV1gz7nQeJ_yfaS4dcP3DFWf0p5
|
|
|
14
14
|
datarobot_dome/metrics/factory.py,sha256=7caa8paI9LuFXDgguXdC4on28V7IwwIsKJT2Z-Aps8A,2187
|
|
15
15
|
datarobot_dome/metrics/metric_scorer.py,sha256=uJ_IJRw7ZFHueg8xjsaXbt0ypO7JiydZ0WapCp96yng,2540
|
|
16
16
|
datarobot_dome/pipeline/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
|
|
17
|
-
datarobot_dome/pipeline/llm_pipeline.py,sha256=
|
|
18
|
-
datarobot_dome/pipeline/pipeline.py,sha256=
|
|
19
|
-
datarobot_dome/pipeline/vdb_pipeline.py,sha256=
|
|
17
|
+
datarobot_dome/pipeline/llm_pipeline.py,sha256=g8XZQjJPPs43GPqaBG4TtsCbvu1o7TWraOmzhtUGB_o,21172
|
|
18
|
+
datarobot_dome/pipeline/pipeline.py,sha256=iDR6h25RDp4bnTW1eGWErG9W_ZXDdJ_sQGVD-tT_Ll0,18805
|
|
19
|
+
datarobot_dome/pipeline/vdb_pipeline.py,sha256=zt5d_41oJjdT8qOtvpgz-l5uvImwKE9f6pQsAU_TdR4,9866
|
|
20
20
|
datarobot_dome/runtime.py,sha256=FD8wXOweqoQVzbZMh-mucL66xT2kGxPsJUGAcJBgwxw,1468
|
|
21
21
|
datarobot_dome/streaming.py,sha256=DkvKEH0yN0aPEWMTAjMFJB3Kx4iLGdjUMQU1pAplbeg,17751
|
|
22
|
-
datarobot_moderations-11.2.
|
|
23
|
-
datarobot_moderations-11.2.
|
|
24
|
-
datarobot_moderations-11.2.
|
|
22
|
+
datarobot_moderations-11.2.5.dist-info/METADATA,sha256=9Aj3fZnLvcgKpiAQwnvkkAzA3SxaBq2j45ctWEMCEaI,4741
|
|
23
|
+
datarobot_moderations-11.2.5.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
24
|
+
datarobot_moderations-11.2.5.dist-info/RECORD,,
|