fiddler-client 2.4.0.dev2__py3-none-any.whl → 2.5.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fiddler/__init__.py +2 -0
- fiddler/_version.py +1 -1
- fiddler/api/alert_mixin.py +51 -1
- fiddler/api/webhooks_mixin.py +6 -8
- fiddler/core_objects.py +76 -26
- fiddler/schema/alert.py +9 -2
- fiddler/schemas/custom_features.py +44 -1
- fiddler3/__init__.py +121 -38
- fiddler3/configs.py +3 -0
- fiddler3/connection.py +68 -1
- fiddler3/constants/alert_rule.py +30 -0
- fiddler3/constants/events.py +7 -0
- fiddler3/constants/model.py +9 -0
- fiddler3/entities/__init__.py +0 -7
- fiddler3/entities/alert_record.py +106 -0
- fiddler3/entities/alert_rule.py +323 -0
- fiddler3/entities/base.py +2 -16
- fiddler3/entities/baseline.py +21 -15
- fiddler3/entities/dataset.py +1 -1
- fiddler3/entities/events.py +136 -0
- fiddler3/entities/file.py +162 -0
- fiddler3/entities/model.py +141 -24
- fiddler3/entities/model_artifact.py +52 -224
- fiddler3/entities/model_deployment.py +12 -1
- fiddler3/entities/project.py +2 -2
- fiddler3/entities/{model_surrogate.py → surrogate.py} +27 -29
- fiddler3/entities/webhook.py +128 -0
- fiddler3/entities/xai.py +271 -5
- fiddler3/libs/http_client.py +1 -1
- fiddler3/libs/json_encoder.py +1 -1
- fiddler3/schemas/__init__.py +0 -9
- fiddler3/schemas/alert_record.py +27 -0
- fiddler3/schemas/alert_rule.py +39 -0
- fiddler3/schemas/custom_features.py +68 -12
- fiddler3/schemas/events.py +15 -0
- fiddler3/schemas/file.py +23 -0
- fiddler3/schemas/model.py +1 -6
- fiddler3/schemas/model_deployment.py +5 -4
- fiddler3/schemas/webhook.py +23 -0
- fiddler3/tests/apis/test_alert_record.py +99 -0
- fiddler3/tests/apis/test_alert_rule.py +393 -0
- fiddler3/tests/apis/test_events.py +169 -0
- fiddler3/tests/apis/test_files.py +164 -0
- fiddler3/tests/apis/test_model.py +60 -9
- fiddler3/tests/apis/test_model_artifact.py +21 -0
- fiddler3/tests/apis/test_project.py +3 -3
- fiddler3/tests/apis/test_webhook.py +257 -0
- fiddler3/tests/apis/test_xai.py +233 -1
- fiddler3/tests/conftest.py +2 -2
- fiddler3/tests/constants.py +5 -0
- fiddler3/tests/test_connection.py +24 -1
- fiddler3/tests/test_json_encoder.py +2 -2
- fiddler3/utils/helpers.py +20 -0
- {fiddler_client-2.4.0.dev2.dist-info → fiddler_client-2.5.0.dev2.dist-info}/METADATA +15 -5
- {fiddler_client-2.4.0.dev2.dist-info → fiddler_client-2.5.0.dev2.dist-info}/RECORD +59 -44
- tests/fiddler/test_alert.py +95 -5
- fiddler3/entities/helpers.py +0 -21
- fiddler3/schemas/model_artifact.py +0 -6
- {fiddler_client-2.4.0.dev2.dist-info → fiddler_client-2.5.0.dev2.dist-info}/LICENSE.txt +0 -0
- {fiddler_client-2.4.0.dev2.dist-info → fiddler_client-2.5.0.dev2.dist-info}/WHEEL +0 -0
- {fiddler_client-2.4.0.dev2.dist-info → fiddler_client-2.5.0.dev2.dist-info}/top_level.txt +0 -0
tests/fiddler/test_alert.py
CHANGED
|
@@ -10,6 +10,7 @@ from fiddler.exceptions import NotFound
|
|
|
10
10
|
from fiddler.schema.alert import (
|
|
11
11
|
AlertCondition,
|
|
12
12
|
AlertRule,
|
|
13
|
+
AlertRuleWithNotifications,
|
|
13
14
|
AlertType,
|
|
14
15
|
BinSize,
|
|
15
16
|
ComparePeriod,
|
|
@@ -1185,6 +1186,8 @@ class TestAlert(BaseTestCase):
|
|
|
1185
1186
|
'severity': None,
|
|
1186
1187
|
'failure_reason': 'NA',
|
|
1187
1188
|
'message': 'In project bank_churn and model bank_churn, during the time period of one hour starting |2022-07-13 11:00:00 (UTC time)|, the prediction drift score was 0.789, greater than 0.1.',
|
|
1189
|
+
'alert_record_main_version': 1,
|
|
1190
|
+
'alert_record_sub_version': 1,
|
|
1188
1191
|
},
|
|
1189
1192
|
{
|
|
1190
1193
|
'id': 2,
|
|
@@ -1199,6 +1202,8 @@ class TestAlert(BaseTestCase):
|
|
|
1199
1202
|
'severity': None,
|
|
1200
1203
|
'failure_reason': 'NA',
|
|
1201
1204
|
'message': 'In project bank_churn and model bank_churn, during the time period of one hour starting |2022-07-13 14:00:00 (UTC time)|, the prediction drift score was 0.789, greater than 0.1.',
|
|
1205
|
+
'alert_record_main_version': 1,
|
|
1206
|
+
'alert_record_sub_version': 1,
|
|
1202
1207
|
},
|
|
1203
1208
|
{
|
|
1204
1209
|
'id': 3,
|
|
@@ -1214,6 +1219,8 @@ class TestAlert(BaseTestCase):
|
|
|
1214
1219
|
'failure_reason': 'NA',
|
|
1215
1220
|
'message': 'In project bank_churn and model bank_churn, during the time period of one hour starting |2022-07-13 15:00:00 (UTC time)|, the prediction drift score was 0.789, greater than 0.1.',
|
|
1216
1221
|
'feature_name': 'feature_name',
|
|
1222
|
+
'alert_record_main_version': 1,
|
|
1223
|
+
'alert_record_sub_version': 1,
|
|
1217
1224
|
},
|
|
1218
1225
|
],
|
|
1219
1226
|
}
|
|
@@ -1397,7 +1404,6 @@ class TestAlert(BaseTestCase):
|
|
|
1397
1404
|
self.assertEqual(alert_rule_parsed_frequency.column, 'geography')
|
|
1398
1405
|
self.assertEqual(alert_rule_parsed_frequency.category, 'Hawaii')
|
|
1399
1406
|
|
|
1400
|
-
|
|
1401
1407
|
def test_triggered_alert_record_serializer(self) -> None:
|
|
1402
1408
|
tar_with_value_as_float = {
|
|
1403
1409
|
'id': 1,
|
|
@@ -1414,6 +1420,8 @@ class TestAlert(BaseTestCase):
|
|
|
1414
1420
|
'message': 'some message',
|
|
1415
1421
|
'multi_col_values': {'a': 1.0, 'b': 2.0},
|
|
1416
1422
|
'feature_name': 'age',
|
|
1423
|
+
'alert_record_main_version': 1,
|
|
1424
|
+
'alert_record_sub_version': 1,
|
|
1417
1425
|
}
|
|
1418
1426
|
tar: TriggeredAlerts = TriggeredAlerts.parse_obj(tar_with_value_as_float)
|
|
1419
1427
|
self.assertEqual(tar.alert_value, 1.0)
|
|
@@ -1432,17 +1440,99 @@ class TestAlert(BaseTestCase):
|
|
|
1432
1440
|
'severity': 'HIGH',
|
|
1433
1441
|
'failure_reason': 'NA',
|
|
1434
1442
|
'message': 'some message',
|
|
1443
|
+
'alert_record_main_version': 1,
|
|
1444
|
+
'alert_record_sub_version': 1,
|
|
1435
1445
|
}
|
|
1436
1446
|
tar_dict: TriggeredAlerts = TriggeredAlerts.parse_obj(tar_with_value_as_dict)
|
|
1437
1447
|
self.assertEqual(tar_dict.alert_value, {'a': 1.0, 'b': 2.0})
|
|
1438
1448
|
|
|
1439
1449
|
def test_enum_values(self) -> None:
|
|
1440
|
-
metric_values = [item.value for item in Metric]
|
|
1441
|
-
for metric in [
|
|
1442
|
-
'
|
|
1443
|
-
'
|
|
1450
|
+
metric_values = [item.value for item in Metric]
|
|
1451
|
+
for metric in [
|
|
1452
|
+
'null_violation_count',
|
|
1453
|
+
'range_violation_count',
|
|
1454
|
+
'type_violation_count',
|
|
1455
|
+
'expected_calibration_error',
|
|
1456
|
+
'calibrated_threshold',
|
|
1457
|
+
'geometric_mean',
|
|
1458
|
+
'ndcg_mean',
|
|
1459
|
+
'traffic',
|
|
1460
|
+
'frequency',
|
|
1461
|
+
]:
|
|
1444
1462
|
self.assertIn(metric, metric_values)
|
|
1445
1463
|
|
|
1464
|
+
def test_update_alert_notification_status(self) -> None:
|
|
1465
|
+
response = {
|
|
1466
|
+
'data': {
|
|
1467
|
+
'created_by': 'admin@fiddler.ai',
|
|
1468
|
+
'priority': 'LOW',
|
|
1469
|
+
'alert_type': 'drift',
|
|
1470
|
+
'compare_period': 86400000,
|
|
1471
|
+
'time_bucket': 3600000,
|
|
1472
|
+
'metric': 'jsd',
|
|
1473
|
+
'model_name': self._model_id,
|
|
1474
|
+
'organization_name': self._org,
|
|
1475
|
+
'created_at': '2022-09-15T18:10:38.286686+00:00',
|
|
1476
|
+
'critical_threshold': 0.2,
|
|
1477
|
+
'feature_names': ['age'],
|
|
1478
|
+
'is_active': True,
|
|
1479
|
+
'enable_notification': True,
|
|
1480
|
+
'uuid': '9f8180d3-3fa0-40c4-8656-b9b1d2de1b69',
|
|
1481
|
+
'compare_to': 'time_period',
|
|
1482
|
+
'warning_threshold': 0.7,
|
|
1483
|
+
'condition': 'lesser',
|
|
1484
|
+
'project_name': self._project_id,
|
|
1485
|
+
'id': 86,
|
|
1486
|
+
'name': 'new_name',
|
|
1487
|
+
'metric_display_name': 'Jenson-Shannon Distance',
|
|
1488
|
+
'alert_type_display_name': 'Data Drift',
|
|
1489
|
+
'notifications': {
|
|
1490
|
+
'emails': {'email': 'nikhil@fiddler.ai, admin@fiddler.ai'},
|
|
1491
|
+
'pagerduty': {
|
|
1492
|
+
'service': 'pd_test_1, pd_test_2',
|
|
1493
|
+
'severity': 'critical',
|
|
1494
|
+
},
|
|
1495
|
+
'webhooks': [{'uuid': '5e4eceae-0387-44fe-b726-7516c33e33fe'}],
|
|
1496
|
+
},
|
|
1497
|
+
},
|
|
1498
|
+
'api_version': '2.0',
|
|
1499
|
+
'kind': 'NORMAL',
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
url = self._get_url(response['data']['uuid'])
|
|
1503
|
+
|
|
1504
|
+
self.requests_mock.patch(
|
|
1505
|
+
url,
|
|
1506
|
+
json=response,
|
|
1507
|
+
)
|
|
1508
|
+
|
|
1509
|
+
self.requests_mock.get(
|
|
1510
|
+
url,
|
|
1511
|
+
json=response,
|
|
1512
|
+
)
|
|
1513
|
+
|
|
1514
|
+
# columns parameter is supported from 23.3.0
|
|
1515
|
+
server_info = ServerInfo(**{'feature_flags': {}, 'server_version': '23.3.0'})
|
|
1516
|
+
with mock.patch.object(
|
|
1517
|
+
self.client, 'server_info', server_info
|
|
1518
|
+
) as mock_server_info:
|
|
1519
|
+
alert_rules: list[
|
|
1520
|
+
AlertRuleWithNotifications
|
|
1521
|
+
] = self.client.update_alert_notification_status(
|
|
1522
|
+
alert_config_ids=[response['data']['uuid']], notification_status=True
|
|
1523
|
+
)
|
|
1524
|
+
alert_rule = alert_rules[0]
|
|
1525
|
+
self.assertEqual(
|
|
1526
|
+
alert_rule.alert_rule_uuid, '9f8180d3-3fa0-40c4-8656-b9b1d2de1b69'
|
|
1527
|
+
)
|
|
1528
|
+
self.assertEqual(alert_rule.critical_threshold, 0.2)
|
|
1529
|
+
self.assertEqual(alert_rule.name, 'new_name')
|
|
1530
|
+
self.assertEqual(alert_rule.columns, ['age'])
|
|
1531
|
+
self.assertEqual(alert_rule.compare_period, ComparePeriod.ONE_DAY)
|
|
1532
|
+
self.assertIsNone(alert_rule.baseline_name)
|
|
1533
|
+
|
|
1534
|
+
self.assertEqual(len(self.requests_mock.calls), 2)
|
|
1535
|
+
|
|
1446
1536
|
|
|
1447
1537
|
if __name__ == '__main__':
|
|
1448
1538
|
unittest.main()
|
fiddler3/entities/helpers.py
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
from http import HTTPStatus
|
|
2
|
-
|
|
3
|
-
from fiddler3.exceptions import NotFound
|
|
4
|
-
from fiddler3.schemas.response import ErrorData
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
def raise_not_found(message: str) -> None:
|
|
8
|
-
"""Raise NotFound if the resource is not found while fetching with names"""
|
|
9
|
-
raise NotFound(
|
|
10
|
-
error=ErrorData(
|
|
11
|
-
code=HTTPStatus.NOT_FOUND,
|
|
12
|
-
message=message,
|
|
13
|
-
errors=[
|
|
14
|
-
{
|
|
15
|
-
'reason': 'ObjectNotFound', # type: ignore
|
|
16
|
-
'message': message,
|
|
17
|
-
'help': '',
|
|
18
|
-
}
|
|
19
|
-
],
|
|
20
|
-
),
|
|
21
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|