datarobot-moderations 11.2.8__py3-none-any.whl → 11.2.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datarobot_dome/drum_integration.py +1 -2
- datarobot_dome/guard_helpers.py +1 -1
- datarobot_dome/metrics/factory.py +2 -3
- datarobot_dome/pipeline/pipeline.py +19 -16
- datarobot_dome/pipeline/vdb_pipeline.py +1 -2
- datarobot_dome/streaming.py +10 -1
- {datarobot_moderations-11.2.8.dist-info → datarobot_moderations-11.2.10.dist-info}/METADATA +2 -1
- {datarobot_moderations-11.2.8.dist-info → datarobot_moderations-11.2.10.dist-info}/RECORD +9 -9
- {datarobot_moderations-11.2.8.dist-info → datarobot_moderations-11.2.10.dist-info}/WHEEL +0 -0
|
@@ -19,7 +19,6 @@ import traceback
|
|
|
19
19
|
import uuid
|
|
20
20
|
from collections.abc import Iterable
|
|
21
21
|
from inspect import signature
|
|
22
|
-
from typing import Optional
|
|
23
22
|
|
|
24
23
|
import numpy as np
|
|
25
24
|
import pandas as pd
|
|
@@ -1055,7 +1054,7 @@ class VdbModerationPipeline(ModerationPipeline):
|
|
|
1055
1054
|
|
|
1056
1055
|
def moderation_pipeline_factory(
|
|
1057
1056
|
target_type: str, model_dir: str = os.getcwd()
|
|
1058
|
-
) ->
|
|
1057
|
+
) -> ModerationPipeline | None:
|
|
1059
1058
|
"""
|
|
1060
1059
|
Create and return a moderation pipeline based on model target type.
|
|
1061
1060
|
This function is the main integration point with DRUM;
|
datarobot_dome/guard_helpers.py
CHANGED
|
@@ -369,7 +369,7 @@ def get_chat_nvidia_llm(api_key: str, base_url: str) -> ChatNVIDIA:
|
|
|
369
369
|
# This registration is for the sake of NeMo guardrails to find
|
|
370
370
|
# the datarobot LLM
|
|
371
371
|
register_model(nim_model)
|
|
372
|
-
return ChatNVIDIA(model=nim_model.id, api_key=api_key)
|
|
372
|
+
return ChatNVIDIA(model=nim_model.id, api_key=api_key, base_url=base_url)
|
|
373
373
|
|
|
374
374
|
|
|
375
375
|
def calculate_faithfulness(
|
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
# ---------------------------------------------------------------------------------
|
|
12
12
|
from typing import Any
|
|
13
13
|
from typing import ClassVar
|
|
14
|
-
from typing import Optional
|
|
15
14
|
|
|
16
15
|
from datarobot_dome.metrics.citation_metrics import CitationTokenAverageScorer
|
|
17
16
|
from datarobot_dome.metrics.citation_metrics import CitationTokenCountScorer
|
|
@@ -38,14 +37,14 @@ class MetricScorerFactory:
|
|
|
38
37
|
return clazz
|
|
39
38
|
|
|
40
39
|
@staticmethod
|
|
41
|
-
def create(metric_type: ScorerType, config:
|
|
40
|
+
def create(metric_type: ScorerType, config: dict[str, Any] | None = None) -> MetricScorer:
|
|
42
41
|
_config = config or {}
|
|
43
42
|
clazz = MetricScorerFactory.get_class(metric_type)
|
|
44
43
|
return clazz(_config)
|
|
45
44
|
|
|
46
45
|
@staticmethod
|
|
47
46
|
def custom_metric_config(
|
|
48
|
-
metric_type: ScorerType, config:
|
|
47
|
+
metric_type: ScorerType, config: dict[str, Any] | None = None
|
|
49
48
|
) -> dict[str, Any]:
|
|
50
49
|
_config = config or {}
|
|
51
50
|
clazz = MetricScorerFactory.get_class(metric_type)
|
|
@@ -18,13 +18,13 @@ import uuid
|
|
|
18
18
|
from datetime import datetime
|
|
19
19
|
from datetime import timezone
|
|
20
20
|
from typing import Any
|
|
21
|
-
from typing import Optional
|
|
22
21
|
|
|
23
22
|
import datarobot as dr
|
|
24
23
|
import numpy as np
|
|
25
24
|
from datarobot.errors import ClientError
|
|
26
25
|
from datarobot.mlops.events import MLOpsEvent
|
|
27
26
|
from datarobot.models.deployment import CustomMetric
|
|
27
|
+
from opentelemetry.instrumentation.utils import suppress_instrumentation
|
|
28
28
|
|
|
29
29
|
from datarobot_dome.async_http_client import AsyncHTTPClient
|
|
30
30
|
from datarobot_dome.constants import DEFAULT_GUARD_PREDICTION_TIMEOUT_IN_SEC
|
|
@@ -108,7 +108,8 @@ class Pipeline:
|
|
|
108
108
|
self._logger.info(f"Model ID from env variable {self._model_id}")
|
|
109
109
|
|
|
110
110
|
try:
|
|
111
|
-
|
|
111
|
+
with suppress_instrumentation():
|
|
112
|
+
self._deployment = dr.Deployment.get(deployment_id=self._deployment_id)
|
|
112
113
|
self._logger.info(f"Model ID set on the deployment {self._deployment.model['id']}")
|
|
113
114
|
except Exception as e:
|
|
114
115
|
self._logger.warning(
|
|
@@ -213,9 +214,10 @@ class Pipeline:
|
|
|
213
214
|
custom_metrics_list = []
|
|
214
215
|
offset, limit = 0, 50
|
|
215
216
|
while True:
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
217
|
+
with suppress_instrumentation():
|
|
218
|
+
response_list = self.dr_client.get(
|
|
219
|
+
f"deployments/{self._deployment_id}/customMetrics/?offset={offset}&limit={limit}"
|
|
220
|
+
).json()
|
|
219
221
|
custom_metrics_list.extend(response_list["data"])
|
|
220
222
|
offset += response_list["count"]
|
|
221
223
|
if response_list["next"] is None:
|
|
@@ -246,16 +248,17 @@ class Pipeline:
|
|
|
246
248
|
# us control over which are duplicates, if max limit reached etc and we can
|
|
247
249
|
# take appropriate actions accordingly. Performance wise it is same, because
|
|
248
250
|
# library also runs a loop to create custom metrics one by one
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
251
|
+
with suppress_instrumentation():
|
|
252
|
+
_metric_obj = CustomMetric.create(
|
|
253
|
+
deployment_id=self._deployment_id,
|
|
254
|
+
name=metric_name,
|
|
255
|
+
directionality=metric_definition["directionality"],
|
|
256
|
+
aggregation_type=metric_definition["type"],
|
|
257
|
+
time_step=metric_definition["timeStep"],
|
|
258
|
+
units=metric_definition["units"],
|
|
259
|
+
baseline_value=metric_definition["baselineValue"],
|
|
260
|
+
is_model_specific=metric_definition["isModelSpecific"],
|
|
261
|
+
)
|
|
259
262
|
custom_metric["id"] = _metric_obj.id
|
|
260
263
|
except ClientError as e:
|
|
261
264
|
if e.status_code == 409:
|
|
@@ -318,7 +321,7 @@ class Pipeline:
|
|
|
318
321
|
self._logger.error(f"Skipping metric creation: {metric_name}")
|
|
319
322
|
del self.custom_metric_map[metric_name]
|
|
320
323
|
|
|
321
|
-
def custom_metric_id_from_name(self, name: str) ->
|
|
324
|
+
def custom_metric_id_from_name(self, name: str) -> str | None:
|
|
322
325
|
"""Gets the custom-metric id from the name of a custom metric."""
|
|
323
326
|
identifier = self.custom_metric_map.get(name, {}).get("id")
|
|
324
327
|
return str(identifier) if identifier else None
|
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
import logging
|
|
13
13
|
import time
|
|
14
14
|
from typing import Any
|
|
15
|
-
from typing import Optional
|
|
16
15
|
|
|
17
16
|
import pandas as pd
|
|
18
17
|
from datarobot.enums import CustomMetricAggregationType
|
|
@@ -42,7 +41,7 @@ score_latency = {
|
|
|
42
41
|
|
|
43
42
|
|
|
44
43
|
class VDBPipeline(Pipeline):
|
|
45
|
-
def __init__(self, config:
|
|
44
|
+
def __init__(self, config: dict[str, Any] | None = None):
|
|
46
45
|
super().__init__()
|
|
47
46
|
metric_config = config.get("metrics", {}) if config else {}
|
|
48
47
|
self._score_configs: dict[ScorerType, dict[str, Any]] = {
|
datarobot_dome/streaming.py
CHANGED
|
@@ -250,8 +250,17 @@ class ModerationIterator:
|
|
|
250
250
|
self.aggregated_metrics_df = postscore_df_assembled
|
|
251
251
|
postscore_df = postscore_df_assembled
|
|
252
252
|
|
|
253
|
+
# It's possible (but not a normal case) to get an empty first chunk where content is None
|
|
254
|
+
# This leads to KeyError: 'promptText' when attempting to merge
|
|
255
|
+
# Log a warning, and use postscore DF for moderations (as if not first chunk)
|
|
253
256
|
if self.first_chunk:
|
|
254
|
-
|
|
257
|
+
try:
|
|
258
|
+
moderations_df = postscore_df.merge(
|
|
259
|
+
self.prescore_df, on=list(self.input_df.columns)
|
|
260
|
+
)
|
|
261
|
+
except KeyError as e:
|
|
262
|
+
self.logger.warning(f"received first chunk with possible empty content; {e}")
|
|
263
|
+
moderations_df = postscore_df
|
|
255
264
|
self.first_chunk = False
|
|
256
265
|
else:
|
|
257
266
|
moderations_df = postscore_df
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datarobot-moderations
|
|
3
|
-
Version: 11.2.
|
|
3
|
+
Version: 11.2.10
|
|
4
4
|
Summary: DataRobot Monitoring and Moderation framework
|
|
5
5
|
License: DataRobot Tool and Utility Agreement
|
|
6
6
|
Author: DataRobot
|
|
@@ -29,6 +29,7 @@ Requires-Dist: nest-asyncio (>=1.6.0)
|
|
|
29
29
|
Requires-Dist: numpy (>=1.25.0)
|
|
30
30
|
Requires-Dist: openai (>=1.14.3)
|
|
31
31
|
Requires-Dist: opentelemetry-api (>=1.16.0)
|
|
32
|
+
Requires-Dist: opentelemetry-instrumentation (>=0.60b1,<0.61)
|
|
32
33
|
Requires-Dist: opentelemetry-sdk (>=1.16.0)
|
|
33
34
|
Requires-Dist: pandas (>=2.0.3)
|
|
34
35
|
Requires-Dist: ragas (>=0.2.15)
|
|
@@ -2,23 +2,23 @@ datarobot_dome/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,58
|
|
|
2
2
|
datarobot_dome/async_http_client.py,sha256=YVkzSSP7ItOEwEskjH0MAs1kJW0jXfbvJWkF_52K7CE,9863
|
|
3
3
|
datarobot_dome/chat_helper.py,sha256=BzvtUyZSZxzOqq-5a2wQKhHhr2kMlcP1MFrHaDAeD_o,9671
|
|
4
4
|
datarobot_dome/constants.py,sha256=jvgpHa3Wh_nZVZmfU-6ab8FHnKNW3KxOPYIIEb_oS6U,10662
|
|
5
|
-
datarobot_dome/drum_integration.py,sha256=
|
|
5
|
+
datarobot_dome/drum_integration.py,sha256=s_8TbUzMj7P6TBV1UiSOaCqHDqY7EIIX0hDkXEW3Kes,46480
|
|
6
6
|
datarobot_dome/guard.py,sha256=xJds9hcbUaS-KD5nC1mn0GiPdBrileFUu6BuTAjDNuY,34668
|
|
7
7
|
datarobot_dome/guard_executor.py,sha256=ox5_jOHcqMaxaaagIYJJHhCwEI7Wg-rUEiu5rutsfVU,35363
|
|
8
|
-
datarobot_dome/guard_helpers.py,sha256=
|
|
8
|
+
datarobot_dome/guard_helpers.py,sha256=29bLktNjvtC7rU4StXADz_FQ1OwIF2xMEKf9GxUd77Q,17206
|
|
9
9
|
datarobot_dome/guards/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
|
|
10
10
|
datarobot_dome/guards/guard_llm_mixin.py,sha256=OIjOHeIAwJiM8BQOfqj1fY2jy-jJfc_CNToYrzG_6xk,11871
|
|
11
11
|
datarobot_dome/llm.py,sha256=L02OvTrflmD34-FrfXebfF-zzKTeuin7fpne1Cl5psg,5719
|
|
12
12
|
datarobot_dome/metrics/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
|
|
13
13
|
datarobot_dome/metrics/citation_metrics.py,sha256=l2mnV1gz7nQeJ_yfaS4dcP3DFWf0p5QIBnKQ6shLnw4,4652
|
|
14
|
-
datarobot_dome/metrics/factory.py,sha256=
|
|
14
|
+
datarobot_dome/metrics/factory.py,sha256=uHQyvkgPHRUQVtZfe5pIrjPrOwQY_Wqs-70AGpNtc_Q,2153
|
|
15
15
|
datarobot_dome/metrics/metric_scorer.py,sha256=uJ_IJRw7ZFHueg8xjsaXbt0ypO7JiydZ0WapCp96yng,2540
|
|
16
16
|
datarobot_dome/pipeline/__init__.py,sha256=B5Rx8_CNCNsOpxBbRj27XOXCfRZmvmrAR-NzlzIKnDw,583
|
|
17
17
|
datarobot_dome/pipeline/llm_pipeline.py,sha256=bindyorJq9VSPZt2X8w0-3REaXui7wIA-8c-zNOIdZU,20961
|
|
18
|
-
datarobot_dome/pipeline/pipeline.py,sha256=
|
|
19
|
-
datarobot_dome/pipeline/vdb_pipeline.py,sha256=
|
|
18
|
+
datarobot_dome/pipeline/pipeline.py,sha256=39ilS5KGwgvlj3Kuntiey8QPV0uq0jsXGnxqx5WpN-w,19486
|
|
19
|
+
datarobot_dome/pipeline/vdb_pipeline.py,sha256=divxwPh_PrEd-eX2ybjMlE7vq6VBQrh320zysLiU36E,9835
|
|
20
20
|
datarobot_dome/runtime.py,sha256=FD8wXOweqoQVzbZMh-mucL66xT2kGxPsJUGAcJBgwxw,1468
|
|
21
|
-
datarobot_dome/streaming.py,sha256=
|
|
22
|
-
datarobot_moderations-11.2.
|
|
23
|
-
datarobot_moderations-11.2.
|
|
24
|
-
datarobot_moderations-11.2.
|
|
21
|
+
datarobot_dome/streaming.py,sha256=TusPJ5nf0HWvdUdfg_DGHfNCqnkFoEb3bmpVDh5fCLc,18240
|
|
22
|
+
datarobot_moderations-11.2.10.dist-info/METADATA,sha256=AVCJ2aWvMfxRMSS9lxVPEw_7xUyeomtESC5tocgYeJU,4804
|
|
23
|
+
datarobot_moderations-11.2.10.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
24
|
+
datarobot_moderations-11.2.10.dist-info/RECORD,,
|
|
File without changes
|