beekeeper-monitors-watsonx 1.1.2__py3-none-any.whl → 1.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- beekeeper/monitors/watsonx/base.py +6 -3
- beekeeper/monitors/watsonx/custom_metric.py +59 -57
- beekeeper/monitors/watsonx/supporting_classes/enums.py +32 -0
- {beekeeper_monitors_watsonx-1.1.2.dist-info → beekeeper_monitors_watsonx-1.1.3.dist-info}/METADATA +1 -1
- {beekeeper_monitors_watsonx-1.1.2.dist-info → beekeeper_monitors_watsonx-1.1.3.dist-info}/RECORD +6 -6
- {beekeeper_monitors_watsonx-1.1.2.dist-info → beekeeper_monitors_watsonx-1.1.3.dist-info}/WHEEL +0 -0
|
@@ -627,11 +627,12 @@ class WatsonxExternalPromptMonitor(PromptMonitor):
|
|
|
627
627
|
rollback_step()
|
|
628
628
|
raise Exception(wos_status.get("failure"))
|
|
629
629
|
|
|
630
|
-
|
|
631
630
|
return {
|
|
632
631
|
"detached_prompt_template_asset_id": detached_pta_id,
|
|
633
632
|
"deployment_id": deployment_id,
|
|
634
|
-
"subscription_id": generative_ai_monitor_details.get(
|
|
633
|
+
"subscription_id": generative_ai_monitor_details.get(
|
|
634
|
+
"subscription_id", None
|
|
635
|
+
),
|
|
635
636
|
}
|
|
636
637
|
|
|
637
638
|
def store_payload_records(
|
|
@@ -1385,7 +1386,9 @@ class WatsonxPromptMonitor(PromptMonitor):
|
|
|
1385
1386
|
return {
|
|
1386
1387
|
"prompt_template_asset_id": pta_id,
|
|
1387
1388
|
"deployment_id": deployment_id,
|
|
1388
|
-
"subscription_id": generative_ai_monitor_details.get(
|
|
1389
|
+
"subscription_id": generative_ai_monitor_details.get(
|
|
1390
|
+
"subscription_id", None
|
|
1391
|
+
),
|
|
1389
1392
|
}
|
|
1390
1393
|
|
|
1391
1394
|
def store_payload_records(
|
|
@@ -7,7 +7,7 @@ from beekeeper.monitors.watsonx.supporting_classes.credentials import (
|
|
|
7
7
|
CloudPakforDataCredentials,
|
|
8
8
|
IntegratedSystemCredentials,
|
|
9
9
|
)
|
|
10
|
-
from beekeeper.monitors.watsonx.supporting_classes.enums import Region
|
|
10
|
+
from beekeeper.monitors.watsonx.supporting_classes.enums import DataSetType, Region
|
|
11
11
|
from beekeeper.monitors.watsonx.supporting_classes.metric import (
|
|
12
12
|
WatsonxLocalMetric,
|
|
13
13
|
WatsonxMetric,
|
|
@@ -531,6 +531,49 @@ class WatsonxCustomMetricsManager:
|
|
|
531
531
|
json_patch_operation=patch_payload,
|
|
532
532
|
).result
|
|
533
533
|
|
|
534
|
+
def store_record_metric_data(
|
|
535
|
+
self,
|
|
536
|
+
custom_data_set_id: str,
|
|
537
|
+
computed_on: Union[DataSetType, str],
|
|
538
|
+
run_id: str,
|
|
539
|
+
request_records: List[Dict],
|
|
540
|
+
):
|
|
541
|
+
"""
|
|
542
|
+
Stores computed metrics data to the specified transaction record.
|
|
543
|
+
|
|
544
|
+
Args:
|
|
545
|
+
custom_data_set_id (str): The ID of the custom metric data set.
|
|
546
|
+
computed_on (DataSetType): The dataset on which the metric was calculated (e.g., payload or feedback).
|
|
547
|
+
run_id (str): The ID of the monitor run that generated the metrics.
|
|
548
|
+
request_records (List[Dict]): A list of dictionaries containing the records to be stored.
|
|
549
|
+
|
|
550
|
+
Example:
|
|
551
|
+
```python
|
|
552
|
+
wxgov_client.store_record_metric_data(
|
|
553
|
+
reference_record_id="0196ad39-1b75-7e77-bddb-cc5393d575c2",
|
|
554
|
+
run_id="RUN_ID",
|
|
555
|
+
request_records=[
|
|
556
|
+
{
|
|
557
|
+
"reference_record_id": "304a9270-44a1-4c4d-bfd4-f756541011f8",
|
|
558
|
+
"record_timestamp": "2025-12-09T00:00:00Z",
|
|
559
|
+
"context_quality": 0.786,
|
|
560
|
+
"pii": 0.05,
|
|
561
|
+
}
|
|
562
|
+
],
|
|
563
|
+
)
|
|
564
|
+
```
|
|
565
|
+
"""
|
|
566
|
+
computed_on = DataSetType.from_value(computed_on).value
|
|
567
|
+
|
|
568
|
+
for record in request_records:
|
|
569
|
+
record["run_id"] = run_id
|
|
570
|
+
record["computed_on"] = computed_on
|
|
571
|
+
|
|
572
|
+
return self._wos_client.data_sets.store_records(
|
|
573
|
+
data_set_id=custom_data_set_id,
|
|
574
|
+
request_body=request_records,
|
|
575
|
+
).result
|
|
576
|
+
|
|
534
577
|
# ===== Local Custom Metrics =====
|
|
535
578
|
@deprecated(
|
|
536
579
|
reason="'add_local_metric_definition()' is deprecated and will be removed in a future version. Use 'create_local_metric_definition()' from 'beekeeper-monitors-watsonx' instead.",
|
|
@@ -549,33 +592,17 @@ class WatsonxCustomMetricsManager:
|
|
|
549
592
|
subscription_id=subscription_id,
|
|
550
593
|
)
|
|
551
594
|
|
|
595
|
+
@deprecated(
|
|
596
|
+
reason="'create_local_metric_definition()' is deprecated and will be removed in a future version. Use OOB record level metrics instead.",
|
|
597
|
+
version="1.1.3",
|
|
598
|
+
action="always",
|
|
599
|
+
)
|
|
552
600
|
def create_local_metric_definition(
|
|
553
601
|
self,
|
|
554
602
|
name: str,
|
|
555
603
|
metrics: List[WatsonxLocalMetric],
|
|
556
604
|
subscription_id: str,
|
|
557
605
|
) -> str:
|
|
558
|
-
"""
|
|
559
|
-
Creates a custom metric definition to compute metrics at the local (transaction) level for IBM watsonx.governance.
|
|
560
|
-
|
|
561
|
-
Args:
|
|
562
|
-
name (str): The name of the custom transaction metric group.
|
|
563
|
-
metrics (List[WatsonxLocalMetric]): A list of metrics to be monitored at the local (transaction) level.
|
|
564
|
-
subscription_id (str): The IBM watsonx.governance subscription ID associated with the metric definition.
|
|
565
|
-
|
|
566
|
-
Example:
|
|
567
|
-
```python
|
|
568
|
-
from beekeeper.monitors.watsonx import WatsonxLocalMetric
|
|
569
|
-
|
|
570
|
-
wxgov_client.create_local_metric_definition(
|
|
571
|
-
name="Custom LLM Local Metric",
|
|
572
|
-
subscription_id="019674ca-0c38-745f-8e9b-58546e95174e",
|
|
573
|
-
metrics=[
|
|
574
|
-
WatsonxLocalMetric(name="context_quality", data_type="double")
|
|
575
|
-
],
|
|
576
|
-
)
|
|
577
|
-
```
|
|
578
|
-
"""
|
|
579
606
|
from ibm_watson_openscale.base_classes.watson_open_scale_v2 import (
|
|
580
607
|
LocationTableName,
|
|
581
608
|
SparkStruct,
|
|
@@ -636,55 +663,30 @@ class WatsonxCustomMetricsManager:
|
|
|
636
663
|
request_records=request_records,
|
|
637
664
|
)
|
|
638
665
|
|
|
666
|
+
@deprecated(
|
|
667
|
+
reason="'store_local_metric_data()' is deprecated and will be removed in a future version. Use OOB record level metrics instead.",
|
|
668
|
+
version="1.1.3",
|
|
669
|
+
action="always",
|
|
670
|
+
)
|
|
639
671
|
def store_local_metric_data(
|
|
640
672
|
self,
|
|
641
673
|
metric_instance_id: str,
|
|
642
674
|
request_records: List[Dict],
|
|
643
675
|
):
|
|
644
|
-
"""
|
|
645
|
-
Stores computed metrics data to the specified transaction record.
|
|
646
|
-
|
|
647
|
-
Args:
|
|
648
|
-
metric_instance_id (str): The unique ID of the custom transaction metric.
|
|
649
|
-
request_records (List[Dict]): A list of dictionaries containing the records to be stored.
|
|
650
|
-
|
|
651
|
-
Example:
|
|
652
|
-
```python
|
|
653
|
-
wxgov_client.store_local_metric_data(
|
|
654
|
-
metric_instance_id="0196ad39-1b75-7e77-bddb-cc5393d575c2",
|
|
655
|
-
request_records=[
|
|
656
|
-
{
|
|
657
|
-
"scoring_id": "304a9270-44a1-4c4d-bfd4-f756541011f8",
|
|
658
|
-
"run_id": "RUN_ID",
|
|
659
|
-
"computed_on": "payload",
|
|
660
|
-
"context_quality": 0.786,
|
|
661
|
-
}
|
|
662
|
-
],
|
|
663
|
-
)
|
|
664
|
-
```
|
|
665
|
-
"""
|
|
666
676
|
return self._wos_client.data_sets.store_records(
|
|
667
677
|
data_set_id=metric_instance_id,
|
|
668
678
|
request_body=request_records,
|
|
669
679
|
).result
|
|
670
680
|
|
|
681
|
+
@deprecated(
|
|
682
|
+
reason="'list_local_metrics()' is deprecated and will be removed in a future version. Use OOB record level metrics instead.",
|
|
683
|
+
version="1.1.3",
|
|
684
|
+
action="always",
|
|
685
|
+
)
|
|
671
686
|
def list_local_metrics(
|
|
672
687
|
self,
|
|
673
688
|
metric_instance_id: str,
|
|
674
689
|
):
|
|
675
|
-
"""
|
|
676
|
-
Lists records from a custom local metric definition.
|
|
677
|
-
|
|
678
|
-
Args:
|
|
679
|
-
metric_instance_id (str): The unique ID of the custom transaction metric.
|
|
680
|
-
|
|
681
|
-
Example:
|
|
682
|
-
```python
|
|
683
|
-
wxgov_client.list_local_metrics(
|
|
684
|
-
metric_instance_id="0196ad47-c505-73c0-9d7b-91c082b697e3"
|
|
685
|
-
)
|
|
686
|
-
```
|
|
687
|
-
"""
|
|
688
690
|
return self._get_dataset_data(metric_instance_id)
|
|
689
691
|
|
|
690
692
|
|
|
@@ -111,3 +111,35 @@ class TaskType(Enum):
|
|
|
111
111
|
raise TypeError(
|
|
112
112
|
f"Invalid type for parameter 'task_id'. Expected str or TaskType, but received {type(value).__name__}."
|
|
113
113
|
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class DataSetType(Enum):
|
|
117
|
+
"""
|
|
118
|
+
Supported IBM watsonx.governance tasks.
|
|
119
|
+
|
|
120
|
+
Attributes:
|
|
121
|
+
PAYLOAD (str): "payload"
|
|
122
|
+
FEEDBACK (str): "feedback"
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
PAYLOAD = "payload"
|
|
126
|
+
FEEDBACK = "feedback"
|
|
127
|
+
|
|
128
|
+
@classmethod
|
|
129
|
+
def from_value(cls, value: str) -> "DataSetType":
|
|
130
|
+
if isinstance(value, cls):
|
|
131
|
+
return value
|
|
132
|
+
|
|
133
|
+
if isinstance(value, str):
|
|
134
|
+
try:
|
|
135
|
+
return cls(value.lower())
|
|
136
|
+
except ValueError:
|
|
137
|
+
raise ValueError(
|
|
138
|
+
"Invalid value. Received: '{}'. Valid values are: {}.".format(
|
|
139
|
+
value, [item.value for item in DataSetType]
|
|
140
|
+
)
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
raise TypeError(
|
|
144
|
+
f"Invalid type. Expected str or DataSetType, but received {type(value).__name__}."
|
|
145
|
+
)
|
{beekeeper_monitors_watsonx-1.1.2.dist-info → beekeeper_monitors_watsonx-1.1.3.dist-info}/RECORD
RENAMED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
beekeeper/monitors/watsonx/__init__.py,sha256=iJv6D68IT00ZC40TNSVYtqyFTen9sWoDqUtxvVVJjOE,789
|
|
2
|
-
beekeeper/monitors/watsonx/base.py,sha256=
|
|
3
|
-
beekeeper/monitors/watsonx/custom_metric.py,sha256=
|
|
2
|
+
beekeeper/monitors/watsonx/base.py,sha256=HbMPlPDHr6Z4gaolExleDHSR8YqisNzIC3aR2vC2aH8,62233
|
|
3
|
+
beekeeper/monitors/watsonx/custom_metric.py,sha256=7HSn_jMLRK0y4qFx84sra38cdJk9ps-LEGN7QJceG5k,24105
|
|
4
4
|
beekeeper/monitors/watsonx/supporting_classes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
beekeeper/monitors/watsonx/supporting_classes/credentials.py,sha256=x4rYoOFvx0pWDhFZfuy6fM0rj7JCivaSYn_jYFXlV8U,5190
|
|
6
|
-
beekeeper/monitors/watsonx/supporting_classes/enums.py,sha256=
|
|
6
|
+
beekeeper/monitors/watsonx/supporting_classes/enums.py,sha256=KrVgBjhnUIUglWmx0OgY3sWoeKMHE-4pZ_ozxzB8IQE,4111
|
|
7
7
|
beekeeper/monitors/watsonx/supporting_classes/metric.py,sha256=iERXRi0iBFHITFaLtonoV97nNUCPXbieD7Z1wX6bvX8,3370
|
|
8
8
|
beekeeper/monitors/watsonx/utils/data_utils.py,sha256=qBLYtHGY0MJ0JJ8BpFDT2YIjA3QOYJQNemLvpA3DMz0,1252
|
|
9
9
|
beekeeper/monitors/watsonx/utils/instrumentation.py,sha256=ztgR1kZ9h-JvASzRA47AYHdc-Isv33EQum9XBLg-dQk,525
|
|
10
|
-
beekeeper_monitors_watsonx-1.1.
|
|
11
|
-
beekeeper_monitors_watsonx-1.1.
|
|
12
|
-
beekeeper_monitors_watsonx-1.1.
|
|
10
|
+
beekeeper_monitors_watsonx-1.1.3.dist-info/METADATA,sha256=xo8UL2_D8Ecft2N0nC7uZwVs67shn6cC1JHpkHO2HVo,716
|
|
11
|
+
beekeeper_monitors_watsonx-1.1.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
12
|
+
beekeeper_monitors_watsonx-1.1.3.dist-info/RECORD,,
|
{beekeeper_monitors_watsonx-1.1.2.dist-info → beekeeper_monitors_watsonx-1.1.3.dist-info}/WHEEL
RENAMED
|
File without changes
|