fiddler-client 3.5.0.dev1__py3-none-any.whl → 3.6.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fiddler/VERSION +1 -1
- fiddler/__init__.py +0 -2
- fiddler/configs.py +6 -1
- fiddler/decorators.py +47 -15
- fiddler/entities/alert_rule.py +1 -1
- fiddler/entities/job.py +46 -11
- fiddler/entities/xai.py +19 -81
- fiddler/exceptions.py +14 -3
- fiddler/libs/http_client.py +245 -56
- fiddler/schemas/model.py +1 -1
- fiddler/schemas/xai.py +0 -6
- fiddler/tests/apis/test_alert_rule.py +3 -12
- fiddler/tests/apis/test_xai.py +0 -108
- {fiddler_client-3.5.0.dev1.dist-info → fiddler_client-3.6.0.dev1.dist-info}/METADATA +1 -1
- {fiddler_client-3.5.0.dev1.dist-info → fiddler_client-3.6.0.dev1.dist-info}/RECORD +18 -18
- {fiddler_client-3.5.0.dev1.dist-info → fiddler_client-3.6.0.dev1.dist-info}/WHEEL +1 -1
- {fiddler_client-3.5.0.dev1.dist-info → fiddler_client-3.6.0.dev1.dist-info}/LICENSE.txt +0 -0
- {fiddler_client-3.5.0.dev1.dist-info → fiddler_client-3.6.0.dev1.dist-info}/top_level.txt +0 -0
fiddler/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
3.
|
|
1
|
+
3.6.0.dev1
|
fiddler/__init__.py
CHANGED
|
@@ -57,7 +57,6 @@ from fiddler.schemas.xai import ( # noqa
|
|
|
57
57
|
DatasetDataSource,
|
|
58
58
|
EventIdDataSource,
|
|
59
59
|
RowDataSource,
|
|
60
|
-
SqlSliceQueryDataSource,
|
|
61
60
|
)
|
|
62
61
|
from fiddler.schemas.xai_params import XaiParams # noqa
|
|
63
62
|
from fiddler.utils.helpers import group_by # noqa
|
|
@@ -110,7 +109,6 @@ __all__ = [
|
|
|
110
109
|
'ModelTaskParams',
|
|
111
110
|
'Multivariate',
|
|
112
111
|
'RowDataSource',
|
|
113
|
-
'SqlSliceQueryDataSource',
|
|
114
112
|
'TextEmbedding',
|
|
115
113
|
'VectorFeature',
|
|
116
114
|
'XaiParams',
|
fiddler/configs.py
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
DEFAULT_CONNECTION_TIMEOUT = 60 # In seconds
|
|
2
2
|
MIN_SERVER_VERSION = '24.4.0'
|
|
3
3
|
DEFAULT_PAGE_SIZE = 50
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
# In seconds. For example, this generates a log message every N seconds.
|
|
6
|
+
# Was once set to 3, and I think that was too frequent. If this needs to
|
|
7
|
+
# be snappier, we can grow the interval.
|
|
8
|
+
JOB_POLL_INTERVAL = 6
|
|
5
9
|
JOB_WAIT_TIMEOUT = 1800
|
|
10
|
+
|
|
6
11
|
STREAM_EVENT_LIMIT = 1000
|
|
7
12
|
DEFAULT_NUM_CLUSTERS = 5
|
|
8
13
|
DEFAULT_NUM_TAGS = 5
|
fiddler/decorators.py
CHANGED
|
@@ -3,6 +3,7 @@ from http import HTTPStatus
|
|
|
3
3
|
from typing import Any, Callable, TypeVar, cast
|
|
4
4
|
|
|
5
5
|
import requests
|
|
6
|
+
import pydantic
|
|
6
7
|
|
|
7
8
|
from fiddler.exceptions import ApiError, Conflict, HttpError, NotFound, Unsupported
|
|
8
9
|
from fiddler.schemas.response import ErrorResponse
|
|
@@ -15,36 +16,67 @@ _WrappedFuncType = TypeVar( # pylint: disable=invalid-name
|
|
|
15
16
|
)
|
|
16
17
|
|
|
17
18
|
|
|
18
|
-
|
|
19
|
+
# ignore: fiddler/decorators.py:19:1: C901 'handle_api_error' is too complex (11)
|
|
20
|
+
def handle_api_error(func: _WrappedFuncType) -> _WrappedFuncType: # noqa: C901
|
|
19
21
|
@wraps(func)
|
|
20
22
|
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
21
23
|
try:
|
|
22
24
|
return func(*args, **kwargs)
|
|
23
25
|
except requests.JSONDecodeError as e:
|
|
26
|
+
# This often is premature use of `json()`. Calling code should
|
|
27
|
+
# first check the response for the expected status code, and only
|
|
28
|
+
# then attempt JSON-deserialization.
|
|
24
29
|
raise HttpError(message=f'Invalid JSON response - {e.doc}') # type: ignore
|
|
25
|
-
except requests.HTTPError as
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
30
|
+
except requests.HTTPError as http_exc:
|
|
31
|
+
# Note(JP): when we're here it means we got an HTTP response.
|
|
32
|
+
# That's how this exception is documented.
|
|
33
|
+
|
|
34
|
+
status = http_exc.response.status_code
|
|
35
|
+
# The logging levels 'critical' and 'error' are generally meant to
|
|
36
|
+
# be used for terminal / fatal errors. Here, we handle many
|
|
37
|
+
# transient (retryable) errors, and it's fair to log them not as
|
|
38
|
+
# serious as actual errors. Therefore level WARNING and not ERROR.
|
|
39
|
+
# Even INFO might make sense.
|
|
40
|
+
logger.warning(
|
|
41
|
+
'%s HTTP request to %s failed with %s - %s',
|
|
42
|
+
http_exc.request.method,
|
|
43
|
+
getattr(http_exc.request, 'url', 'unknown'),
|
|
44
|
+
status,
|
|
45
|
+
# Not entire response body in case we have a megabyte response.
|
|
46
|
+
getattr(http_exc.response, 'content', 'missing')[:200],
|
|
31
47
|
)
|
|
32
48
|
|
|
33
|
-
|
|
49
|
+
# Now, map the error at hand to an exception to be thrown towards
|
|
50
|
+
# the caller.
|
|
34
51
|
|
|
35
|
-
if
|
|
52
|
+
if status == HTTPStatus.METHOD_NOT_ALLOWED:
|
|
36
53
|
raise Unsupported()
|
|
37
54
|
|
|
55
|
+
# Did we get structured error information? This might even
|
|
56
|
+
# be wrapped in a 501 response for example.
|
|
38
57
|
try:
|
|
39
|
-
|
|
58
|
+
# https://github.com/fiddler-labs/fiddler/issues/9314
|
|
59
|
+
error_resp = ErrorResponse(**http_exc.response.json())
|
|
60
|
+
except pydantic.ValidationError:
|
|
61
|
+
# We couldn't turn the HTTP error response into an object
|
|
62
|
+
# satisfying model='ErrorResponse'. Now, instead of exposing
|
|
63
|
+
# this to users as a pydantic.ValidationError (which is kinda
|
|
64
|
+
# useless to them) -- just re-raise requests.HTTPError, with
|
|
65
|
+
# all its useful error detail -- it is likely to show the
|
|
66
|
+
# actual problem.
|
|
67
|
+
raise http_exc
|
|
40
68
|
except requests.JSONDecodeError:
|
|
41
|
-
raise
|
|
42
|
-
|
|
43
|
-
|
|
69
|
+
# Re-raise requests.HTTPError, with all its useful error
|
|
70
|
+
# detail.
|
|
71
|
+
raise http_exc
|
|
72
|
+
|
|
73
|
+
# This code path is only reachable if we could successfully
|
|
74
|
+
# deserialize an ErrorResponse object from the HTTP response body.
|
|
44
75
|
|
|
45
|
-
if
|
|
76
|
+
if status == HTTPStatus.CONFLICT:
|
|
46
77
|
raise Conflict(error_resp.error)
|
|
47
|
-
|
|
78
|
+
|
|
79
|
+
if status == HTTPStatus.NOT_FOUND:
|
|
48
80
|
raise NotFound(error_resp.error)
|
|
49
81
|
|
|
50
82
|
raise ApiError(error_resp.error)
|
fiddler/entities/alert_rule.py
CHANGED
|
@@ -247,7 +247,7 @@ class AlertRule(
|
|
|
247
247
|
url=self._get_url(id_=self.id), data={'enable_notification': True}
|
|
248
248
|
)
|
|
249
249
|
logger.info(
|
|
250
|
-
'
|
|
250
|
+
'notifications have been enabled for alert rule with id: %s', self.id
|
|
251
251
|
)
|
|
252
252
|
|
|
253
253
|
@handle_api_error
|
fiddler/entities/job.py
CHANGED
|
@@ -4,6 +4,8 @@ import time
|
|
|
4
4
|
from typing import Iterator
|
|
5
5
|
from uuid import UUID
|
|
6
6
|
|
|
7
|
+
import requests
|
|
8
|
+
|
|
7
9
|
from fiddler.configs import JOB_POLL_INTERVAL, JOB_WAIT_TIMEOUT
|
|
8
10
|
from fiddler.constants.job import JobStatus
|
|
9
11
|
from fiddler.decorators import handle_api_error
|
|
@@ -112,24 +114,57 @@ class Job(BaseEntity): # pylint: disable=too-many-instance-attributes
|
|
|
112
114
|
:return: Iterator of job objects
|
|
113
115
|
"""
|
|
114
116
|
assert self.id is not None
|
|
117
|
+
deadline = time.monotonic() + timeout
|
|
115
118
|
|
|
116
|
-
start_time = time.time()
|
|
117
119
|
while True:
|
|
120
|
+
|
|
121
|
+
if time.monotonic() > deadline:
|
|
122
|
+
raise TimeoutError(f'Deadline exceeded while watching job {self.id}')
|
|
123
|
+
|
|
118
124
|
try:
|
|
119
|
-
|
|
125
|
+
# This can raise requests.HTTPError to represent non-2xx
|
|
126
|
+
# responses.
|
|
127
|
+
response = self._client().get(
|
|
128
|
+
url=self._get_url(id_=self.id),
|
|
129
|
+
# Short-ish TCP connect timeout, to stay responsive in
|
|
130
|
+
# terms of logging. The HTTP response latency for GET
|
|
131
|
+
# /jobs/<id> is expected to be less than couple of seconds
|
|
132
|
+
# (i.e., 30 s includes lots of leeway).
|
|
133
|
+
timeout=(5, 30),
|
|
134
|
+
# Inject `retry="off"` to disable the centralized retry
|
|
135
|
+
# machinery: ask for being confronted with the details
|
|
136
|
+
# (throw exceptions my way, so I an do my own "responsive"
|
|
137
|
+
# type of retrying). Otherwise, we'd give up control.
|
|
138
|
+
retry="off",
|
|
139
|
+
)
|
|
120
140
|
self._refresh_from_response(response)
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
141
|
+
|
|
142
|
+
except requests.exceptions.HTTPError as exc:
|
|
143
|
+
# Note(JP): got a non-2xx HTTP response. The main purpose of
|
|
144
|
+
# this handler is to keep going after having received a 5xx
|
|
145
|
+
# response. In this case of GETting job status even some 404s
|
|
146
|
+
# might be worth retrying. That is, it's fine to give up only
|
|
147
|
+
# after reaching the deadline even if we collect some 4xx
|
|
148
|
+
# responses along the way. Noteworthy: Receiving a 5xx response
|
|
149
|
+
# here is not an error, it's an expected scenario to be
|
|
150
|
+
# accounted for.
|
|
151
|
+
logger.info(
|
|
152
|
+
'watch: ignore unexpected response %s (URL: %s, response body prefix: %s...)',
|
|
153
|
+
exc.response,
|
|
154
|
+
exc.request.url,
|
|
155
|
+
exc.response.text[:120],
|
|
125
156
|
)
|
|
126
157
|
continue
|
|
127
158
|
|
|
128
|
-
|
|
159
|
+
except requests.exceptions.RequestException:
|
|
160
|
+
# This error is in the hierarchy _above_ `HTTPError`, and in
|
|
161
|
+
# this setup catches all errors that are _not_ a bad response:
|
|
162
|
+
# DNS error, TCP connect timeout, err during sending request,
|
|
163
|
+
# err during receiving response. Rely on the error detail to
|
|
164
|
+
# have already been logged.
|
|
165
|
+
continue
|
|
129
166
|
|
|
130
|
-
|
|
131
|
-
if (current_time - start_time) > timeout:
|
|
132
|
-
raise TimeoutError(f'Timed out while watching job {self.id}')
|
|
167
|
+
yield self
|
|
133
168
|
|
|
134
169
|
if self.status in [
|
|
135
170
|
JobStatus.SUCCESS,
|
|
@@ -153,7 +188,7 @@ class Job(BaseEntity): # pylint: disable=too-many-instance-attributes
|
|
|
153
188
|
|
|
154
189
|
for job in self.watch(interval=interval, timeout=timeout):
|
|
155
190
|
logger.info(
|
|
156
|
-
'%s:
|
|
191
|
+
'%s: %s, progress: %.1f%%',
|
|
157
192
|
log_prefix,
|
|
158
193
|
job.status,
|
|
159
194
|
job.progress,
|
fiddler/entities/xai.py
CHANGED
|
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|
|
3
3
|
import os
|
|
4
4
|
from collections import namedtuple
|
|
5
5
|
from datetime import datetime, timezone
|
|
6
|
-
from io import BytesIO
|
|
7
6
|
from pathlib import Path
|
|
8
7
|
from typing import Any, Callable
|
|
9
8
|
from uuid import UUID
|
|
@@ -18,15 +17,10 @@ from fiddler.constants.xai import (
|
|
|
18
17
|
)
|
|
19
18
|
from fiddler.decorators import handle_api_error
|
|
20
19
|
from fiddler.entities.job import Job
|
|
20
|
+
from fiddler.exceptions import Unsupported
|
|
21
21
|
from fiddler.schemas.job import JobCompactResp
|
|
22
|
-
from fiddler.schemas.xai import
|
|
23
|
-
DatasetDataSource,
|
|
24
|
-
EventIdDataSource,
|
|
25
|
-
RowDataSource,
|
|
26
|
-
SqlSliceQueryDataSource,
|
|
27
|
-
)
|
|
22
|
+
from fiddler.schemas.xai import DatasetDataSource, EventIdDataSource, RowDataSource
|
|
28
23
|
from fiddler.utils.decorators import check_version
|
|
29
|
-
from fiddler.utils.helpers import try_series_retype
|
|
30
24
|
from fiddler.utils.logger import get_logger
|
|
31
25
|
|
|
32
26
|
logger = get_logger(__name__)
|
|
@@ -44,7 +38,7 @@ class XaiMixin:
|
|
|
44
38
|
def explain( # pylint: disable=too-many-arguments
|
|
45
39
|
self,
|
|
46
40
|
input_data_source: RowDataSource | EventIdDataSource,
|
|
47
|
-
ref_data_source: DatasetDataSource |
|
|
41
|
+
ref_data_source: DatasetDataSource | None = None,
|
|
48
42
|
method: ExplainMethod | str = ExplainMethod.FIDDLER_SHAP,
|
|
49
43
|
num_permutations: int | None = None,
|
|
50
44
|
ci_level: float | None = None,
|
|
@@ -55,8 +49,8 @@ class XaiMixin:
|
|
|
55
49
|
|
|
56
50
|
:param input_data_source: DataSource for the input data to compute explanation
|
|
57
51
|
on (RowDataSource, EventIdDataSource)
|
|
58
|
-
:param ref_data_source:
|
|
59
|
-
on
|
|
52
|
+
:param ref_data_source: Dataset data source for the reference data to compute explanation
|
|
53
|
+
on.
|
|
60
54
|
Only used for non-text models and the following methods:
|
|
61
55
|
'SHAP', 'FIDDLER_SHAP', 'PERMUTE', 'MEAN_RESET'
|
|
62
56
|
:param method: Explanation method name. Could be your custom
|
|
@@ -102,7 +96,7 @@ class XaiMixin:
|
|
|
102
96
|
sample: bool = False,
|
|
103
97
|
max_rows: int | None = None,
|
|
104
98
|
columns: list[str] | None = None,
|
|
105
|
-
) ->
|
|
99
|
+
) -> None:
|
|
106
100
|
"""
|
|
107
101
|
Fetch data with slice query.
|
|
108
102
|
|
|
@@ -111,35 +105,12 @@ class XaiMixin:
|
|
|
111
105
|
columns to select overriding columns selected in the query.
|
|
112
106
|
:param max_rows: Number of maximum rows to fetch
|
|
113
107
|
:param sample: Whether rows should be sample or not from the database
|
|
114
|
-
|
|
115
108
|
:return: Dataframe of the query output
|
|
116
109
|
"""
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
payload: dict[str, Any] = {
|
|
120
|
-
'model_id': self.id,
|
|
121
|
-
'query': query,
|
|
122
|
-
'sample': sample,
|
|
123
|
-
}
|
|
124
|
-
if max_rows:
|
|
125
|
-
payload['max_rows'] = max_rows
|
|
126
|
-
if columns:
|
|
127
|
-
payload['columns'] = columns
|
|
128
|
-
|
|
129
|
-
response = self._client().post(
|
|
130
|
-
url='/v3/slice-query/fetch',
|
|
131
|
-
data=payload,
|
|
110
|
+
raise Unsupported(
|
|
111
|
+
'This method is not supported since version 3.4. Please use `model.download_data` instead.'
|
|
132
112
|
)
|
|
133
113
|
|
|
134
|
-
response_dict = response.json()['data']
|
|
135
|
-
|
|
136
|
-
column_names = response_dict['metadata']['columns']
|
|
137
|
-
dtype_strings = response_dict['metadata']['dtypes']
|
|
138
|
-
df = pd.DataFrame(response_dict['rows'], columns=column_names)
|
|
139
|
-
for column_name, dtype in zip(column_names, dtype_strings):
|
|
140
|
-
df[column_name] = try_series_retype(df[column_name], dtype)
|
|
141
|
-
return df
|
|
142
|
-
|
|
143
114
|
@handle_api_error
|
|
144
115
|
def download_data( # pylint: disable=too-many-arguments
|
|
145
116
|
self,
|
|
@@ -250,25 +221,9 @@ class XaiMixin:
|
|
|
250
221
|
:param max_rows: Number of maximum rows to fetch
|
|
251
222
|
:param sample: Whether rows should be sample or not from the database
|
|
252
223
|
"""
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
os.makedirs(output_dir)
|
|
257
|
-
payload: dict[str, Any] = {
|
|
258
|
-
'model_id': self.id,
|
|
259
|
-
'query': query,
|
|
260
|
-
'sample': sample,
|
|
261
|
-
}
|
|
262
|
-
if max_rows:
|
|
263
|
-
payload['max_rows'] = max_rows
|
|
264
|
-
if columns:
|
|
265
|
-
payload['columns'] = columns
|
|
266
|
-
|
|
267
|
-
file_path = os.path.join(output_dir, 'output.parquet')
|
|
268
|
-
with self._client().post(url='/v3/slice-query/download', data=payload) as resp:
|
|
269
|
-
# Download parquet file
|
|
270
|
-
df = pd.read_parquet(BytesIO(resp.content))
|
|
271
|
-
df.to_parquet(file_path)
|
|
224
|
+
raise Unsupported(
|
|
225
|
+
'This method is not supported since version 3.4. Please use `model.download_data` instead.'
|
|
226
|
+
)
|
|
272
227
|
|
|
273
228
|
@handle_api_error
|
|
274
229
|
def get_mutual_info(
|
|
@@ -277,7 +232,7 @@ class XaiMixin:
|
|
|
277
232
|
column_name: str,
|
|
278
233
|
num_samples: int | None = None,
|
|
279
234
|
normalized: bool = False,
|
|
280
|
-
) ->
|
|
235
|
+
) -> None:
|
|
281
236
|
"""
|
|
282
237
|
Get mutual information.
|
|
283
238
|
|
|
@@ -290,27 +245,10 @@ class XaiMixin:
|
|
|
290
245
|
all the variables in the dataset.
|
|
291
246
|
:param num_samples: Number of samples to select for computation
|
|
292
247
|
:param normalized: If set to True, it will compute Normalized Mutual Information
|
|
293
|
-
|
|
294
248
|
:return: a dictionary of mutual information w.r.t the given feature
|
|
295
249
|
for each column given
|
|
296
250
|
"""
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
payload: dict[str, Any] = {
|
|
300
|
-
'model_id': self.id,
|
|
301
|
-
'query': query,
|
|
302
|
-
'column_name': column_name,
|
|
303
|
-
'normalized': normalized,
|
|
304
|
-
}
|
|
305
|
-
if num_samples:
|
|
306
|
-
payload['num_samples'] = num_samples
|
|
307
|
-
|
|
308
|
-
response = self._client().post(
|
|
309
|
-
url='/v3/analytics/mutual-info',
|
|
310
|
-
data=payload,
|
|
311
|
-
)
|
|
312
|
-
|
|
313
|
-
return response.json()['data']
|
|
251
|
+
raise Unsupported('This method is not supported since version 3.4.')
|
|
314
252
|
|
|
315
253
|
@handle_api_error
|
|
316
254
|
def predict(
|
|
@@ -344,7 +282,7 @@ class XaiMixin:
|
|
|
344
282
|
@handle_api_error
|
|
345
283
|
def get_feature_impact( # pylint: disable=too-many-arguments
|
|
346
284
|
self,
|
|
347
|
-
data_source: DatasetDataSource
|
|
285
|
+
data_source: DatasetDataSource,
|
|
348
286
|
num_iterations: int | None = None,
|
|
349
287
|
num_refs: int | None = None,
|
|
350
288
|
ci_level: float | None = None,
|
|
@@ -354,8 +292,8 @@ class XaiMixin:
|
|
|
354
292
|
"""
|
|
355
293
|
Get global feature impact for a model over a dataset or a slice.
|
|
356
294
|
|
|
357
|
-
:param data_source:
|
|
358
|
-
impact on
|
|
295
|
+
:param data_source: Dataset data Source for the input dataset to compute feature
|
|
296
|
+
impact on
|
|
359
297
|
:param num_iterations: The maximum number of ablated model inferences per feature
|
|
360
298
|
:param num_refs: The number of reference points used in the explanation
|
|
361
299
|
:param ci_level: The confidence level (between 0 and 1)
|
|
@@ -398,7 +336,7 @@ class XaiMixin:
|
|
|
398
336
|
@handle_api_error
|
|
399
337
|
def get_feature_importance( # pylint: disable=too-many-arguments
|
|
400
338
|
self,
|
|
401
|
-
data_source: DatasetDataSource
|
|
339
|
+
data_source: DatasetDataSource,
|
|
402
340
|
num_iterations: int | None = None,
|
|
403
341
|
num_refs: int | None = None,
|
|
404
342
|
ci_level: float | None = None,
|
|
@@ -406,8 +344,8 @@ class XaiMixin:
|
|
|
406
344
|
"""
|
|
407
345
|
Get global feature importance for a model over a dataset or a slice.
|
|
408
346
|
|
|
409
|
-
:param data_source:
|
|
410
|
-
importance on
|
|
347
|
+
:param data_source: Dataset data aSource for the input dataset to compute feature
|
|
348
|
+
importance on
|
|
411
349
|
:param num_iterations: The maximum number of ablated model inferences per feature
|
|
412
350
|
:param num_refs: The number of reference points used in the explanation
|
|
413
351
|
:param ci_level: The confidence level (between 0 and 1)
|
fiddler/exceptions.py
CHANGED
|
@@ -17,6 +17,8 @@ class BaseError(Exception):
|
|
|
17
17
|
super().__init__(self.message)
|
|
18
18
|
|
|
19
19
|
def __str__(self) -> str:
|
|
20
|
+
# It would be better to prepend context: like FiddlerBaseError. Note
|
|
21
|
+
# that this here is the string representation of an exception.
|
|
20
22
|
return str(self.message)
|
|
21
23
|
|
|
22
24
|
@property
|
|
@@ -53,17 +55,26 @@ class Unsupported(BaseError):
|
|
|
53
55
|
|
|
54
56
|
|
|
55
57
|
class HttpError(BaseError):
|
|
56
|
-
"""Base class for all HTTP errors
|
|
58
|
+
"""Base class for all HTTP errors
|
|
59
|
+
|
|
60
|
+
Deprecated. Not thrown anymore.
|
|
61
|
+
"""
|
|
57
62
|
|
|
58
63
|
|
|
59
64
|
class ConnTimeout(HttpError):
|
|
60
|
-
"""Connection timeout error
|
|
65
|
+
"""Connection timeout error
|
|
66
|
+
|
|
67
|
+
Deprecated. Not thrown anymore.
|
|
68
|
+
"""
|
|
61
69
|
|
|
62
70
|
message = 'Request timed out while trying to reach endpoint'
|
|
63
71
|
|
|
64
72
|
|
|
65
73
|
class ConnError(HttpError):
|
|
66
|
-
"""Connection error
|
|
74
|
+
"""Connection error
|
|
75
|
+
|
|
76
|
+
Deprecated. Not thrown anymore.
|
|
77
|
+
"""
|
|
67
78
|
|
|
68
79
|
message = 'Unable to reach the given endpoint'
|
|
69
80
|
|
fiddler/libs/http_client.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import time
|
|
3
4
|
from copy import deepcopy
|
|
4
5
|
from typing import Any
|
|
5
6
|
from urllib.parse import urljoin
|
|
@@ -10,16 +11,11 @@ from requests.adapters import HTTPAdapter
|
|
|
10
11
|
|
|
11
12
|
import fiddler.libs.aws
|
|
12
13
|
from fiddler.constants.common import JSON_CONTENT_TYPE
|
|
13
|
-
from fiddler.exceptions import
|
|
14
|
-
ConnError,
|
|
15
|
-
ConnTimeout,
|
|
16
|
-
HttpError,
|
|
17
|
-
)
|
|
14
|
+
from fiddler.exceptions import HttpError, BaseError # pylint: disable=redefined-builtin
|
|
18
15
|
from fiddler.libs.json_encoder import RequestClientJSONEncoder
|
|
19
16
|
from fiddler.utils.logger import get_logger
|
|
20
17
|
|
|
21
|
-
|
|
22
|
-
|
|
18
|
+
log = get_logger(__name__)
|
|
23
19
|
|
|
24
20
|
# Note(JP): this is doing conditional runtime modification for AWS Hadron /
|
|
25
21
|
# SageMaker, depending on
|
|
@@ -38,7 +34,21 @@ class RequestClient:
|
|
|
38
34
|
verify: bool = True,
|
|
39
35
|
proxies: dict | None = None,
|
|
40
36
|
) -> None:
|
|
41
|
-
"""
|
|
37
|
+
"""
|
|
38
|
+
HTTP client abstraction.
|
|
39
|
+
|
|
40
|
+
For centralized logging and retrying and error handling
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
# The default retry mechanism keep retrying HTTP requests up to a
|
|
44
|
+
# certain deadline N seconds in the future. The idea is to survive
|
|
45
|
+
# micro outages, i.e. this should be O(1 min). Idea: make configurable,
|
|
46
|
+
# maybe have a conservative default and then we can tune this in our CI
|
|
47
|
+
# to try longer.
|
|
48
|
+
self.default_retry_for_seconds = 60 * 10
|
|
49
|
+
self.timeout_long_running_requests = (5, 120)
|
|
50
|
+
self.timeout_short_running_requests = (5, 15)
|
|
51
|
+
|
|
42
52
|
self.base_url = base_url
|
|
43
53
|
self.proxies = proxies
|
|
44
54
|
self.headers = headers
|
|
@@ -55,6 +65,7 @@ class RequestClient:
|
|
|
55
65
|
|
|
56
66
|
self.session.verify = verify
|
|
57
67
|
adapter = HTTPAdapter(
|
|
68
|
+
# Note(JP): what's the relevance of setting these?
|
|
58
69
|
pool_connections=25,
|
|
59
70
|
pool_maxsize=25,
|
|
60
71
|
)
|
|
@@ -63,7 +74,7 @@ class RequestClient:
|
|
|
63
74
|
# change from above?
|
|
64
75
|
self.session.mount(self.base_url, adapter)
|
|
65
76
|
|
|
66
|
-
def
|
|
77
|
+
def _request_with_retry(
|
|
67
78
|
self,
|
|
68
79
|
*,
|
|
69
80
|
method: str,
|
|
@@ -71,11 +82,13 @@ class RequestClient:
|
|
|
71
82
|
params: dict | None = None,
|
|
72
83
|
headers: dict | None = None,
|
|
73
84
|
data: dict | bytes | None = None,
|
|
74
|
-
timeout:
|
|
85
|
+
timeout: float | tuple[float, float] | None = None,
|
|
75
86
|
**kwargs: Any,
|
|
76
87
|
) -> requests.Response:
|
|
77
88
|
"""
|
|
78
|
-
|
|
89
|
+
Emit HTTP request.
|
|
90
|
+
|
|
91
|
+
Always apply retry strategy, unless told not to.
|
|
79
92
|
|
|
80
93
|
:param method: HTTP method like
|
|
81
94
|
:param url: API endpoint
|
|
@@ -83,12 +96,11 @@ class RequestClient:
|
|
|
83
96
|
:param headers: Request headers
|
|
84
97
|
:param data: Dict/binary data
|
|
85
98
|
:param timeout: Request timeout in seconds
|
|
99
|
+
:param kwargs: passed on to requests.session.request()
|
|
86
100
|
"""
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
full_url = urljoin(self.base_url, url)
|
|
90
|
-
|
|
101
|
+
abs_url = urljoin(self.base_url, url)
|
|
91
102
|
request_headers = self.headers
|
|
103
|
+
|
|
92
104
|
# override/update headers coming from the calling method
|
|
93
105
|
if headers:
|
|
94
106
|
request_headers = deepcopy(self.headers)
|
|
@@ -96,38 +108,220 @@ class RequestClient:
|
|
|
96
108
|
|
|
97
109
|
content_type = request_headers.get('Content-Type')
|
|
98
110
|
if data is not None and content_type == JSON_CONTENT_TYPE:
|
|
99
|
-
|
|
111
|
+
# Why did we decide on simplejson here instead of stdlib? For
|
|
112
|
+
# behavior, for perf?
|
|
113
|
+
data = simplejson.dumps(
|
|
114
|
+
data, ignore_nan=True, cls=RequestClientJSONEncoder # type: ignore
|
|
115
|
+
)
|
|
100
116
|
|
|
117
|
+
# Use `setdefault`: define properties in kwargs when the caller doesn't
|
|
118
|
+
# specify. verify: default to session config, but allow for override.
|
|
101
119
|
kwargs.setdefault('allow_redirects', True)
|
|
102
|
-
# requests is not able to pass the value of self.session.verify to the
|
|
103
|
-
# verify param in kwargs when REQUESTS_CA_BUNDLE is set.
|
|
104
|
-
# So setting that as default here
|
|
105
120
|
kwargs.setdefault('verify', self.session.verify)
|
|
121
|
+
|
|
122
|
+
# Construct dictionary of parameters passed to session.request() with
|
|
123
|
+
# all parameters except for method and URL.
|
|
124
|
+
kwargs.update(
|
|
125
|
+
{
|
|
126
|
+
"params": params,
|
|
127
|
+
"data": data,
|
|
128
|
+
"headers": request_headers,
|
|
129
|
+
"timeout": timeout,
|
|
130
|
+
"proxies": self.proxies,
|
|
131
|
+
}
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
return self._make_request_retry_until_deadline(method, abs_url, **kwargs)
|
|
135
|
+
|
|
136
|
+
def _make_request_retry_until_deadline(
|
|
137
|
+
self, method: str, url: str, **kwargs: Any
|
|
138
|
+
) -> requests.Response:
|
|
139
|
+
"""
|
|
140
|
+
Return `requests.Response` object when the request was sent out and
|
|
141
|
+
responded to with an HTTP response with a 2xx status code.
|
|
142
|
+
|
|
143
|
+
Implement a retry loop with deadline control.
|
|
144
|
+
|
|
145
|
+
Raise an exception derived from `requests.exceptions.RequestException`
|
|
146
|
+
to indicate a non-retryable error, such as various 4xx responses.
|
|
147
|
+
|
|
148
|
+
Interrupt the retry loop and raise ryingHTTPClientNonRetryableResponse
|
|
149
|
+
|
|
150
|
+
Raise `fiddler.exceptions.BaseError` (from last exception) when
|
|
151
|
+
reaching the deadline (it might raise a minute early, or even be
|
|
152
|
+
briefly exceeded).
|
|
153
|
+
"""
|
|
154
|
+
logpfx = f"http: {method} {url} --"
|
|
155
|
+
|
|
156
|
+
t0 = time.monotonic()
|
|
157
|
+
deadline = t0 + self.default_retry_for_seconds
|
|
158
|
+
cycle: int = 0
|
|
159
|
+
|
|
160
|
+
log.debug("%s try", logpfx)
|
|
161
|
+
|
|
162
|
+
while time.monotonic() < deadline:
|
|
163
|
+
cycle += 1
|
|
164
|
+
|
|
165
|
+
result = self._make_request_retry_guts(method, url, **kwargs)
|
|
166
|
+
|
|
167
|
+
if isinstance(result, requests.Response):
|
|
168
|
+
if cycle > 1:
|
|
169
|
+
log.info("%s success after %.3f s", logpfx, time.monotonic() - t0)
|
|
170
|
+
return result
|
|
171
|
+
|
|
172
|
+
# Rename, for clarity. This exception was considered retryable, but
|
|
173
|
+
# we may have to throw it below upon approaching the deadline.
|
|
174
|
+
last_exception = result
|
|
175
|
+
|
|
176
|
+
# Desired behavior: rather fast first retry attempt; followed by
|
|
177
|
+
# slow exponential growth, and an upper bound: 0.66, 1.33, 2.66,
|
|
178
|
+
# 5.33, 10.66, 21.33, 42.66, 60, 60, .... (seconds)
|
|
179
|
+
wait_seconds = min((2**cycle) / 3.0, 60)
|
|
180
|
+
log.info(
|
|
181
|
+
"%s cycle %s failed, wait for %.1f s, deadline in %.1f min",
|
|
182
|
+
logpfx,
|
|
183
|
+
cycle,
|
|
184
|
+
wait_seconds,
|
|
185
|
+
(deadline - time.monotonic()) / 60.0,
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# Would the next wait exceed the deadline?
|
|
189
|
+
if (time.monotonic() + wait_seconds) > deadline:
|
|
190
|
+
break
|
|
191
|
+
|
|
192
|
+
time.sleep(wait_seconds)
|
|
193
|
+
|
|
194
|
+
# Give up after retrying. Structurally emit error detail of last seen
|
|
195
|
+
# (retryable) exception.
|
|
196
|
+
raise BaseError(
|
|
197
|
+
f"{method} request to {url}: giving up after {time.monotonic() - t0:.3f} s"
|
|
198
|
+
) from last_exception
|
|
199
|
+
|
|
200
|
+
def _make_request_retry_guts(
|
|
201
|
+
self, method: str, url: str, **kwargs: Any
|
|
202
|
+
) -> requests.Response | requests.RequestException:
|
|
203
|
+
"""
|
|
204
|
+
Return `requests.Response` object when the request was sent out and
|
|
205
|
+
responded to with an HTTP response that does not throw upon
|
|
206
|
+
`resp.raise_for_status()` (a 2xx status code).
|
|
207
|
+
|
|
208
|
+
Return an exception object to indicate a _retryable_ error to the
|
|
209
|
+
caller. The caller will then retry.
|
|
210
|
+
|
|
211
|
+
Raise an exception derived from `requests.exceptions.RequestException`
|
|
212
|
+
to indicate a non-retryable error, such as various 4xx responses.
|
|
213
|
+
"""
|
|
214
|
+
logpfx = f"http: {method} {url} --"
|
|
215
|
+
|
|
216
|
+
if "timeout" not in kwargs:
|
|
217
|
+
kwargs["timeout"] = self.timeout_long_running_requests
|
|
218
|
+
|
|
219
|
+
# Magic argument passed down to here. Can be set to "off".
|
|
220
|
+
retry_strategy = kwargs.pop('retry', 'default')
|
|
221
|
+
|
|
222
|
+
t0 = time.monotonic()
|
|
223
|
+
log.info("%s emit", logpfx)
|
|
224
|
+
|
|
225
|
+
try:
|
|
226
|
+
# Here, we want to have a tight TCP connect() timeout and a
|
|
227
|
+
# meaningful TCP recv timeout, also a meaningful global
|
|
228
|
+
# request-response cycle timeout (accounting for the ~expected HTTP
|
|
229
|
+
# request processing time in the API implementation, plus leeway).
|
|
230
|
+
resp = self.session.request(method=method, url=url, **kwargs)
|
|
231
|
+
except requests.exceptions.RequestException as exc:
|
|
232
|
+
|
|
233
|
+
if retry_strategy == "off":
|
|
234
|
+
log.info("%s error: %s", logpfx, exc)
|
|
235
|
+
# Do not retry, let the caller deal with this exception.
|
|
236
|
+
raise
|
|
237
|
+
|
|
238
|
+
# Note(JP): we did not get a response. We might not even have sent
|
|
239
|
+
# the request. An error happened before sending the request, while
|
|
240
|
+
# sending the request, while waiting for response, or while
|
|
241
|
+
# receiving the response. High probability for this being a
|
|
242
|
+
# transient problem. A few examples for errors handled here:
|
|
243
|
+
#
|
|
244
|
+
# - DNS resolution error
|
|
245
|
+
# - TCP connect() timeout
|
|
246
|
+
# - Connection refused during TCP connect()
|
|
247
|
+
# - Timeout while waiting for the other end to start sending the
|
|
248
|
+
# HTTP response (after having sent the request).
|
|
249
|
+
# - RECV timeout between trying to receive two response bytes.
|
|
250
|
+
#
|
|
251
|
+
# Note(JP): do this, regardless of the type of request that we
|
|
252
|
+
# sent. I know we've expressed concerns about idempotency here and
|
|
253
|
+
# there. But I believe that it will be a big step forward to have
|
|
254
|
+
# more or less *easy-to-reason-about* retrying in the client and to
|
|
255
|
+
# maybe risk a rare idempotency problem and debug it and fix it in
|
|
256
|
+
# the backend.
|
|
257
|
+
|
|
258
|
+
# Convention: returning the exception tells the caller to retry.
|
|
259
|
+
log.info("%s retry soon: %s", logpfx, exc)
|
|
260
|
+
return exc
|
|
261
|
+
|
|
262
|
+
# Got an HTTP response. In the scope below, `resp` reflects that.
|
|
263
|
+
log.info(
|
|
264
|
+
"%s request took %.3f s, response code: %s, resp body size: %s, req body size: %s",
|
|
265
|
+
logpfx,
|
|
266
|
+
time.monotonic() - t0,
|
|
267
|
+
resp.status_code,
|
|
268
|
+
len(resp.content),
|
|
269
|
+
0 if resp.request.body is None else len(resp.request.body),
|
|
270
|
+
)
|
|
271
|
+
|
|
106
272
|
try:
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
273
|
+
resp.raise_for_status()
|
|
274
|
+
|
|
275
|
+
# The criterion for a good-looking response. Ideally we check for
|
|
276
|
+
# the _precisely_ expected status code, but can do that later.
|
|
277
|
+
return resp
|
|
278
|
+
except requests.HTTPError as exc:
|
|
279
|
+
# Decide whether or not this is a retryable based on the response
|
|
280
|
+
# details. Put into a log message before leaving this function.
|
|
281
|
+
treat_retryable = self._is_retryable_resp(resp)
|
|
282
|
+
|
|
283
|
+
# Log body prefix: sometimes this is critical for debuggability.
|
|
284
|
+
log.warning(
|
|
285
|
+
"%s error response with code %s%s, body bytes: <%s ...>",
|
|
286
|
+
logpfx,
|
|
287
|
+
resp.status_code,
|
|
288
|
+
" (treat retryable)" if treat_retryable else "",
|
|
289
|
+
resp.text[:500],
|
|
116
290
|
)
|
|
117
291
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
#
|
|
126
|
-
|
|
127
|
-
raise
|
|
292
|
+
if retry_strategy == "off":
|
|
293
|
+
raise
|
|
294
|
+
|
|
295
|
+
if treat_retryable:
|
|
296
|
+
return exc
|
|
297
|
+
|
|
298
|
+
# Otherwise let this nice requests.HTTPError object bubble up to
|
|
299
|
+
# calling code. This is the expected code path for most 4xx
|
|
300
|
+
# error responses.
|
|
301
|
+
raise
|
|
302
|
+
|
|
303
|
+
def _is_retryable_resp(self, resp: requests.Response) -> bool:
|
|
304
|
+
"""
|
|
305
|
+
Do we (want to) consider this response as retryable, based on the
|
|
306
|
+
status code alone?
|
|
307
|
+
"""
|
|
308
|
+
if resp.status_code == 429:
|
|
309
|
+
# Canonical way to signal "back off, retry soon".
|
|
310
|
+
return True
|
|
311
|
+
|
|
312
|
+
if resp.status_code == 501:
|
|
313
|
+
# Means: "not implemented" or "not supported". Might be a little
|
|
314
|
+
# exotic to use but we seem to rely on it. Do not retry those
|
|
315
|
+
# for now, in general
|
|
316
|
+
return False
|
|
317
|
+
|
|
318
|
+
# Retry any 5xx response, later maybe fine-tune this by specific status
|
|
319
|
+
# code. Certain 500 Internal Server Error are probably permanent, but
|
|
320
|
+
# it's worth retrying them, too.
|
|
321
|
+
if str(resp.status_code).startswith("5"):
|
|
322
|
+
return True
|
|
128
323
|
|
|
129
|
-
|
|
130
|
-
return response
|
|
324
|
+
return False
|
|
131
325
|
|
|
132
326
|
def get(
|
|
133
327
|
self,
|
|
@@ -135,11 +329,10 @@ class RequestClient:
|
|
|
135
329
|
url: str,
|
|
136
330
|
params: dict | None = None,
|
|
137
331
|
headers: dict | None = None,
|
|
138
|
-
timeout:
|
|
139
|
-
**kwargs:
|
|
332
|
+
timeout: float | tuple[float, float] | None = None,
|
|
333
|
+
**kwargs: Any,
|
|
140
334
|
) -> requests.Response:
|
|
141
|
-
|
|
142
|
-
return self.call(
|
|
335
|
+
return self._request_with_retry(
|
|
143
336
|
method='GET',
|
|
144
337
|
url=url,
|
|
145
338
|
params=params,
|
|
@@ -155,10 +348,9 @@ class RequestClient:
|
|
|
155
348
|
params: dict | None = None,
|
|
156
349
|
headers: dict | None = None,
|
|
157
350
|
timeout: int | None = None,
|
|
158
|
-
**kwargs:
|
|
351
|
+
**kwargs: Any,
|
|
159
352
|
) -> requests.Response:
|
|
160
|
-
|
|
161
|
-
return self.call(
|
|
353
|
+
return self._request_with_retry(
|
|
162
354
|
method='DELETE',
|
|
163
355
|
url=url,
|
|
164
356
|
params=params,
|
|
@@ -175,10 +367,9 @@ class RequestClient:
|
|
|
175
367
|
headers: dict | None = None,
|
|
176
368
|
timeout: int | None = None,
|
|
177
369
|
data: dict | bytes | None = None,
|
|
178
|
-
**kwargs:
|
|
370
|
+
**kwargs: Any,
|
|
179
371
|
) -> requests.Response:
|
|
180
|
-
|
|
181
|
-
return self.call(
|
|
372
|
+
return self._request_with_retry(
|
|
182
373
|
method='POST',
|
|
183
374
|
url=url,
|
|
184
375
|
params=params,
|
|
@@ -196,10 +387,9 @@ class RequestClient:
|
|
|
196
387
|
headers: dict | None = None,
|
|
197
388
|
timeout: int | None = None,
|
|
198
389
|
data: dict | None = None,
|
|
199
|
-
**kwargs:
|
|
390
|
+
**kwargs: Any,
|
|
200
391
|
) -> requests.Response:
|
|
201
|
-
|
|
202
|
-
return self.call(
|
|
392
|
+
return self._request_with_retry(
|
|
203
393
|
method='PUT',
|
|
204
394
|
url=url,
|
|
205
395
|
params=params,
|
|
@@ -217,10 +407,9 @@ class RequestClient:
|
|
|
217
407
|
headers: dict | None = None,
|
|
218
408
|
timeout: int | None = None,
|
|
219
409
|
data: dict | None = None,
|
|
220
|
-
**kwargs:
|
|
410
|
+
**kwargs: Any,
|
|
221
411
|
) -> requests.Response:
|
|
222
|
-
|
|
223
|
-
return self.call(
|
|
412
|
+
return self._request_with_retry(
|
|
224
413
|
method='PATCH',
|
|
225
414
|
url=url,
|
|
226
415
|
params=params,
|
fiddler/schemas/model.py
CHANGED
fiddler/schemas/xai.py
CHANGED
|
@@ -24,9 +24,3 @@ class DatasetDataSource(BaseModel):
|
|
|
24
24
|
env_type: str
|
|
25
25
|
num_samples: Optional[int]
|
|
26
26
|
env_id: Optional[Union[str, UUID]] = Field(alias='dataset_id')
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
class SqlSliceQueryDataSource(BaseModel):
|
|
30
|
-
source_type = 'SQL_SLICE_QUERY'
|
|
31
|
-
query: str
|
|
32
|
-
num_samples: Optional[int]
|
|
@@ -288,7 +288,7 @@ def test_add_alert_rule_success() -> None:
|
|
|
288
288
|
|
|
289
289
|
|
|
290
290
|
@responses.activate
|
|
291
|
-
def test_enable_notifications(
|
|
291
|
+
def test_enable_notifications() -> None:
|
|
292
292
|
responses.get(
|
|
293
293
|
url=f'{URL}/v3/alert-rules/{ALERT_RULE_ID}',
|
|
294
294
|
json=API_RESPONSE_200,
|
|
@@ -304,14 +304,10 @@ def test_enable_notifications(caplog) -> None:
|
|
|
304
304
|
|
|
305
305
|
alert_rule.enable_notifications()
|
|
306
306
|
assert json.loads(resp.calls[0].request.body) == {'enable_notification': True}
|
|
307
|
-
assert (
|
|
308
|
-
f'Notifications have been enabled for alert rule with id: {ALERT_RULE_ID}'
|
|
309
|
-
== caplog.messages[0]
|
|
310
|
-
)
|
|
311
307
|
|
|
312
308
|
|
|
313
309
|
@responses.activate
|
|
314
|
-
def test_disable_notifications(
|
|
310
|
+
def test_disable_notifications() -> None:
|
|
315
311
|
responses.get(
|
|
316
312
|
url=f'{URL}/v3/alert-rules/{ALERT_RULE_ID}',
|
|
317
313
|
json=API_RESPONSE_200,
|
|
@@ -329,10 +325,6 @@ def test_disable_notifications(caplog) -> None:
|
|
|
329
325
|
set_logging(logging.INFO)
|
|
330
326
|
alert_rule.disable_notifications()
|
|
331
327
|
assert json.loads(resp.calls[0].request.body) == {'enable_notification': False}
|
|
332
|
-
assert (
|
|
333
|
-
f'Notifications have been disabled for alert rule with id: {ALERT_RULE_ID}'
|
|
334
|
-
== caplog.messages[0]
|
|
335
|
-
)
|
|
336
328
|
|
|
337
329
|
|
|
338
330
|
@responses.activate
|
|
@@ -459,7 +451,7 @@ def test_set_notifications_webhooks() -> None:
|
|
|
459
451
|
|
|
460
452
|
|
|
461
453
|
@responses.activate
|
|
462
|
-
def test_set_notifications_invalid_input(
|
|
454
|
+
def test_set_notifications_invalid_input() -> None:
|
|
463
455
|
responses.get(
|
|
464
456
|
url=f'{URL}/v3/alert-rules/{ALERT_RULE_ID}',
|
|
465
457
|
json=API_RESPONSE_200,
|
|
@@ -472,7 +464,6 @@ def test_set_notifications_invalid_input(caplog) -> None:
|
|
|
472
464
|
emails=TEST_EMAILS[0],
|
|
473
465
|
webhooks=TEST_WEBHOOKS,
|
|
474
466
|
)
|
|
475
|
-
assert 'Invalid input: The format of input is not correct' in caplog.messages[0]
|
|
476
467
|
|
|
477
468
|
|
|
478
469
|
@responses.activate
|
fiddler/tests/apis/test_xai.py
CHANGED
|
@@ -219,48 +219,6 @@ EXPLAIN_RESPONSE_200 = {
|
|
|
219
219
|
'kind': 'NORMAL',
|
|
220
220
|
}
|
|
221
221
|
|
|
222
|
-
FETCH_SLICE_RESPONSE_200 = {
|
|
223
|
-
'data': {
|
|
224
|
-
'metadata': {
|
|
225
|
-
'query': "SELECT * FROM test_bank_churn.bank_churn WHERE geography='France' order by balance desc LIMIT 2",
|
|
226
|
-
'is_slice': True,
|
|
227
|
-
'columns': ['Age'],
|
|
228
|
-
'dtypes': ['int'],
|
|
229
|
-
'model': {
|
|
230
|
-
'id': '36bc3613-49d5-46b5-a0e0-a45c3aa4a9d9',
|
|
231
|
-
'name': 'bank_churn',
|
|
232
|
-
},
|
|
233
|
-
'env': {
|
|
234
|
-
'id': '86e59334-2b86-4726-a970-6900db46e437',
|
|
235
|
-
'type': 'PRE_PRODUCTION',
|
|
236
|
-
'name': 'test_bank_churn',
|
|
237
|
-
},
|
|
238
|
-
},
|
|
239
|
-
'rows': [[57], [37]],
|
|
240
|
-
},
|
|
241
|
-
'api_version': '3.0',
|
|
242
|
-
'kind': 'NORMAL',
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
MUTUAL_INFO_RESPONSE_200 = {
|
|
246
|
-
'data': {
|
|
247
|
-
'CreditScore': 0.009774993816146854,
|
|
248
|
-
'Geography': 1.0387361759590092,
|
|
249
|
-
'Gender': 0.000345600896640319,
|
|
250
|
-
'Age': 0.006747713128151839,
|
|
251
|
-
'Tenure': 0.0017159873190972647,
|
|
252
|
-
'Balance': 0.14278016623398132,
|
|
253
|
-
'NumOfProducts': 0.002155101383831244,
|
|
254
|
-
'HasCrCard': 0.00011152799557884174,
|
|
255
|
-
'IsActiveMember': 0.00026523254634630566,
|
|
256
|
-
'EstimatedSalary': 0.010603971349228138,
|
|
257
|
-
'probability_churned': 0.06073798415644517,
|
|
258
|
-
'Decisions': 2.3682864881247045e-05,
|
|
259
|
-
'Churned': 0.014017045772466472,
|
|
260
|
-
},
|
|
261
|
-
'api_version': '3.0',
|
|
262
|
-
'kind': 'NORMAL',
|
|
263
|
-
}
|
|
264
222
|
|
|
265
223
|
PREDICT_RESPONSE_200 = {
|
|
266
224
|
'data': {'predictions': [{'predicted_quality': 5.759617514660622}]},
|
|
@@ -555,27 +513,6 @@ def test_explain() -> None:
|
|
|
555
513
|
)
|
|
556
514
|
|
|
557
515
|
|
|
558
|
-
@responses.activate
|
|
559
|
-
def test_get_slice() -> None:
|
|
560
|
-
responses.get(
|
|
561
|
-
url=f'{URL}/v3/models/{MODEL_ID}',
|
|
562
|
-
json=MODEL_API_RESPONSE_200,
|
|
563
|
-
)
|
|
564
|
-
model = Model.get(id_=MODEL_ID)
|
|
565
|
-
|
|
566
|
-
responses.post(
|
|
567
|
-
url=f'{URL}/v3/slice-query/fetch',
|
|
568
|
-
json=FETCH_SLICE_RESPONSE_200,
|
|
569
|
-
)
|
|
570
|
-
|
|
571
|
-
slice_df = model.get_slice(
|
|
572
|
-
query="SELECT * FROM test_bank_churn.bank_churn WHERE geography='France' order by balance desc LIMIT 3",
|
|
573
|
-
columns=['Age'],
|
|
574
|
-
max_rows=2,
|
|
575
|
-
)
|
|
576
|
-
expected_df = pd.DataFrame({'Age': [57, 37]})
|
|
577
|
-
pd.testing.assert_frame_equal(expected_df, slice_df)
|
|
578
|
-
|
|
579
516
|
|
|
580
517
|
@responses.activate
|
|
581
518
|
def test_download_data() -> None:
|
|
@@ -607,51 +544,6 @@ def test_download_data() -> None:
|
|
|
607
544
|
shutil.rmtree(str(parquet_output))
|
|
608
545
|
|
|
609
546
|
|
|
610
|
-
@responses.activate
|
|
611
|
-
def test_slice_download() -> None:
|
|
612
|
-
responses.get(
|
|
613
|
-
url=f'{URL}/v3/models/{MODEL_ID}',
|
|
614
|
-
json=MODEL_API_RESPONSE_200,
|
|
615
|
-
)
|
|
616
|
-
model = Model.get(id_=MODEL_ID)
|
|
617
|
-
|
|
618
|
-
parquet_path = os.path.join(OUTPUT_DIR, 'test_slice_download.parquet')
|
|
619
|
-
parquet_output = os.path.join(BASE_TEST_DIR, 'slice_test_dir')
|
|
620
|
-
with open(parquet_path, 'rb') as parquet_file:
|
|
621
|
-
data = io.BufferedReader(parquet_file)
|
|
622
|
-
responses.post(
|
|
623
|
-
url=f'{URL}/v3/slice-query/download',
|
|
624
|
-
body=data,
|
|
625
|
-
)
|
|
626
|
-
expected_df = pd.DataFrame({'Age': [38, 57, 42]})
|
|
627
|
-
model.download_slice(
|
|
628
|
-
output_dir=parquet_output,
|
|
629
|
-
query='SELECT * FROM test_bank_churn.bank_churn WHERE age>=20 order by balance desc',
|
|
630
|
-
columns=['Age'],
|
|
631
|
-
max_rows=3,
|
|
632
|
-
)
|
|
633
|
-
slice_df = pd.read_parquet(parquet_path)
|
|
634
|
-
pd.testing.assert_frame_equal(expected_df, slice_df)
|
|
635
|
-
shutil.rmtree(str(parquet_output))
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
@responses.activate
|
|
639
|
-
def test_mutual_info() -> None:
|
|
640
|
-
responses.get(
|
|
641
|
-
url=f'{URL}/v3/models/{MODEL_ID}',
|
|
642
|
-
json=MODEL_API_RESPONSE_200,
|
|
643
|
-
)
|
|
644
|
-
model = Model.get(id_=MODEL_ID)
|
|
645
|
-
|
|
646
|
-
responses.post(
|
|
647
|
-
url=f'{URL}/v3/analytics/mutual-info',
|
|
648
|
-
json=MUTUAL_INFO_RESPONSE_200,
|
|
649
|
-
)
|
|
650
|
-
mutual_info = model.get_mutual_info(
|
|
651
|
-
query=f'select * from {DATASET_NAME}.{MODEL_NAME}',
|
|
652
|
-
column_name='Geography',
|
|
653
|
-
)
|
|
654
|
-
assert mutual_info == MUTUAL_INFO_RESPONSE_200['data']
|
|
655
547
|
|
|
656
548
|
|
|
657
549
|
@responses.activate
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
fiddler/VERSION,sha256=
|
|
2
|
-
fiddler/__init__.py,sha256=
|
|
3
|
-
fiddler/configs.py,sha256=
|
|
1
|
+
fiddler/VERSION,sha256=GEhNOdsvFfk1-UqmBoYWxbLNcwiqgDmoJMtNkUTmTIs,11
|
|
2
|
+
fiddler/__init__.py,sha256=3gbyU75uS19XTOqscenO38oPfC7qf9d44Cgt9UYyTJI,3772
|
|
3
|
+
fiddler/configs.py,sha256=qwmxc6oYKUoNgzXyY7nSkPMeEk0KehYTl7qNi-uKul4,406
|
|
4
4
|
fiddler/connection.py,sha256=gm7dShXhNFRTyHKsAEtQ56_lDlArLdxjKJQaQMK5-fE,5775
|
|
5
|
-
fiddler/decorators.py,sha256=
|
|
6
|
-
fiddler/exceptions.py,sha256=
|
|
5
|
+
fiddler/decorators.py,sha256=yfA_7s41kzZOBqc0ZqluGezvkoTlHyL8q2dLF2amMyA,3624
|
|
6
|
+
fiddler/exceptions.py,sha256=LrGXSNP3N5aQKyiVX7H1mX2hs23YWZev1WxlRzxiWis,2534
|
|
7
7
|
fiddler/version.py,sha256=8fyg3UhFpqghMpxbtYIwNcB1lhCG6yTkPBbDS7IrwxY,140
|
|
8
8
|
fiddler/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
9
|
fiddler/constants/alert_rule.py,sha256=cmFq-VKAlQbRKEkTtMmVNdD5R52uMI2zJai6TESFwfs,512
|
|
@@ -17,14 +17,14 @@ fiddler/constants/model_deployment.py,sha256=SfagLKVD7tK1QmYQFJLDrA4-6FaSeNF6tik
|
|
|
17
17
|
fiddler/constants/xai.py,sha256=m6uO9DleXj7l9ZwqscmAkr3ZNzdcgLMtPPKwUNh8B3k,344
|
|
18
18
|
fiddler/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
19
|
fiddler/entities/alert_record.py,sha256=RcDACWDNpntkiId-JpkLQvVeIbuaZAzn6AWl2zYIh9A,3564
|
|
20
|
-
fiddler/entities/alert_rule.py,sha256=
|
|
20
|
+
fiddler/entities/alert_rule.py,sha256=MmYr0_RVWFoMvp7EPE5u23hvDO45rNNuuhuKOnLv9D4,10512
|
|
21
21
|
fiddler/entities/base.py,sha256=YCTLt1ta8UbD7u-5BW7v_ocA4S7LevEncI5XPS_SB60,2106
|
|
22
22
|
fiddler/entities/baseline.py,sha256=k3D-K8WC5MkHDILyejRaH1v9jdCIymT96McDBOVQ9A0,7907
|
|
23
23
|
fiddler/entities/custom_expression.py,sha256=SdS61JiI0gOxWwHAGxkHh1vqS5XsRtyeNSDs5E-Sg2U,6568
|
|
24
24
|
fiddler/entities/dataset.py,sha256=kwD1pZMG6DQGIjejU7A2KUUBRgOi0o1qbbUX-PSlnGg,4399
|
|
25
25
|
fiddler/entities/events.py,sha256=9pzg7J1v03o2DGgHX_uPsCSujwk5iyPEDJPrrBNWJbo,5662
|
|
26
26
|
fiddler/entities/file.py,sha256=dk8ISfjy0--F2eF23Ae8vGiEZmJW6e9LMqNmKfrY2mE,4710
|
|
27
|
-
fiddler/entities/job.py,sha256=
|
|
27
|
+
fiddler/entities/job.py,sha256=C_P2w_sMPFgBIEa2ILyMV0B3dc3dPzuo9Yik1B5CP0k,7098
|
|
28
28
|
fiddler/entities/model.py,sha256=giqLWvnqXsqrekJ3sIeHf3vUUYO43PxrGh6I_KsGfH4,19271
|
|
29
29
|
fiddler/entities/model_artifact.py,sha256=Y7o1mFs-nVDz7HI9mVBu9ICwkqGD_QbT3q6vf8KKesk,5520
|
|
30
30
|
fiddler/entities/model_deployment.py,sha256=pzS6xPg4zuhrIzMbF44ptK_ZwV8yN6URF0hDwyVhnwY,3966
|
|
@@ -33,10 +33,10 @@ fiddler/entities/project.py,sha256=G5omjjKHEjvhenMvW-R0bYwrkRVHixIOhTPZb1pS7FI,4
|
|
|
33
33
|
fiddler/entities/surrogate.py,sha256=94RkJmv8uwNhRL8PaliuiRv-R0EXDMKJTimE9boXKwQ,3077
|
|
34
34
|
fiddler/entities/user.py,sha256=ev_Xx9QHAdEHYzNDafzQihXOcD8ME5XkpKgc8MiqCnw,1243
|
|
35
35
|
fiddler/entities/webhook.py,sha256=BhwV-05EaFtOATbnlPWUaGeziRSewBPSNSGsYFbIRfc,4514
|
|
36
|
-
fiddler/entities/xai.py,sha256
|
|
36
|
+
fiddler/entities/xai.py,sha256=-jbQEBFO1NxsBGld_bWV-nmeciex-jK4ett0_AjaO8I,22041
|
|
37
37
|
fiddler/libs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
38
|
fiddler/libs/aws.py,sha256=QZAVyNhZ5hUpP1GV9lBzldpKDVZvCenTWSZ4yM6g79E,3480
|
|
39
|
-
fiddler/libs/http_client.py,sha256=
|
|
39
|
+
fiddler/libs/http_client.py,sha256=fgawVffkW25xsswlIGy7TctDKgRFiQTsA5GCcaHe_qc,14945
|
|
40
40
|
fiddler/libs/json_encoder.py,sha256=brsrWySvtZdaBArIkqKM5WwJUbvZf6ZbKozn1CewjOc,512
|
|
41
41
|
fiddler/libs/semver.py,sha256=khO8HpPceBXbimD4Q7hqQB9BYFHl9G7z7--ycXDQQwo,15979
|
|
42
42
|
fiddler/packtools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -56,7 +56,7 @@ fiddler/schemas/events.py,sha256=dH6SGPBO_rgGZetIhSo4Fq0L1pTU7TEKR7zW7gg3fnI,409
|
|
|
56
56
|
fiddler/schemas/file.py,sha256=UNQAHQW1SILLC4SFnPnbZQ0oSSYEw1qzkN6REdD_chY,465
|
|
57
57
|
fiddler/schemas/filter_query.py,sha256=XP8SoFGSJOo9yf3fA9dPqpKdCJuaUpbKbYixn46lBF0,1325
|
|
58
58
|
fiddler/schemas/job.py,sha256=6q3SiHp43ah0R0eIO77DNgJRug-dejI13NZAd_7c0sk,354
|
|
59
|
-
fiddler/schemas/model.py,sha256=
|
|
59
|
+
fiddler/schemas/model.py,sha256=8PsF69DxC5Y8k8klGbYSZAS-xegXEqIygaW9pZq3nAE,1401
|
|
60
60
|
fiddler/schemas/model_deployment.py,sha256=s1moOBT7JIMWgGZ4ALxgZJhB4ulVK6i2VkNp7_mOJKo,1128
|
|
61
61
|
fiddler/schemas/model_schema.py,sha256=Kx5rMyDYvgrBv52Oy2RQrW4pn6D_aqVzrvMICaYckOs,1917
|
|
62
62
|
fiddler/schemas/model_spec.py,sha256=K7GkhkbL8vvoFGLNq-xy1aeiRcPOx0QBx9EaMwZfV-I,969
|
|
@@ -67,7 +67,7 @@ fiddler/schemas/response.py,sha256=kcSt3zmyjLpbjt_t2KOOzinBqgmfJFJS9mvx998hhj8,1
|
|
|
67
67
|
fiddler/schemas/server_info.py,sha256=nrykHzrcIaYX1TcK71RoGSQ6kUIinCfvMU8wxaGtQZQ,500
|
|
68
68
|
fiddler/schemas/user.py,sha256=p_Z-Zjj85HvJw2QGWwP_qqx-lAcNTgzMrVzgkKjstEU,140
|
|
69
69
|
fiddler/schemas/webhook.py,sha256=Utg1wRylr5L0CYc47k7iLQdlVCL0L6S29jVntNwMzbA,464
|
|
70
|
-
fiddler/schemas/xai.py,sha256=
|
|
70
|
+
fiddler/schemas/xai.py,sha256=j16vu4iDhTuD42kKEjukbMFkRFgKfkHzMRVDWTsEFz8,616
|
|
71
71
|
fiddler/schemas/xai_params.py,sha256=VX9dXumBrnCAmxyvsQWagj4kYTaQr9Rv1hGsSimhFjU,316
|
|
72
72
|
fiddler/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
73
73
|
fiddler/tests/conftest.py,sha256=Dmz9FKAFHYdq1AWmcY7NES9pU4cbY9qKWQiojjaX_0E,840
|
|
@@ -80,7 +80,7 @@ fiddler/tests/test_utils.py,sha256=AF6y7UfboEkCa-nNw0aF30SdHC7yW2Hh78J4H_4s78k,3
|
|
|
80
80
|
fiddler/tests/utils.py,sha256=FEDbASW4pYZktdEK2FLnIDyZCwBzpb60m6EzkvOkwSc,335
|
|
81
81
|
fiddler/tests/apis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
82
82
|
fiddler/tests/apis/test_alert_record.py,sha256=_0EBRDn87TqTiO1KYEHKQQVtVZjj4lYjbt8Ix-RRTSg,3118
|
|
83
|
-
fiddler/tests/apis/test_alert_rule.py,sha256=
|
|
83
|
+
fiddler/tests/apis/test_alert_rule.py,sha256=KlS6FOl3MGcfslAPnqvy6EQmsKDulyqAXlRrCI_AebM,14273
|
|
84
84
|
fiddler/tests/apis/test_baseline.py,sha256=R_vvbZdSenpYMSydurENJrE_uNWE3sJR9sHJksfU_Jk,7494
|
|
85
85
|
fiddler/tests/apis/test_custom_metric.py,sha256=QlNa80M9PxqaqzW-3nCtH5kKLRHYu2iqFDautBusbos,8208
|
|
86
86
|
fiddler/tests/apis/test_dataset.py,sha256=1Xc9Ng91fLMc2C6dpKfse42fQ0anza3ZOKOD1TqCvTQ,4636
|
|
@@ -96,7 +96,7 @@ fiddler/tests/apis/test_model_surrogate.py,sha256=h3OPgD1-_n6gTd0YH_hDcBBRjBfXv1
|
|
|
96
96
|
fiddler/tests/apis/test_project.py,sha256=IEYT3imD4dZJg-OT888G4jQJ_AxxHN5_VkCHcmwUnGA,6015
|
|
97
97
|
fiddler/tests/apis/test_segment.py,sha256=hQiXpZEKLJrnlRBLLMsP3jph8H8K7AEUw17sMK6BuF8,7804
|
|
98
98
|
fiddler/tests/apis/test_webhook.py,sha256=RyBw7p2xNp-LFmq6n9mgG-GQqSu2Y3FzsccM8aSyLwc,8180
|
|
99
|
-
fiddler/tests/apis/test_xai.py,sha256=
|
|
99
|
+
fiddler/tests/apis/test_xai.py,sha256=OSExXaA0ajKR7-qvTsbP5uZ72pzbqTVxVn92SDAHLzs,24485
|
|
100
100
|
fiddler/utils/__init__.py,sha256=ooAJR6_tVaF6UKZriz5ynZPj8IeCzaDIPKV23F-e4Fk,53
|
|
101
101
|
fiddler/utils/decorators.py,sha256=bEMhbfxoeyfJRnPGTY3xfBJbGYe9j7W4jbkhke8yuls,1521
|
|
102
102
|
fiddler/utils/helpers.py,sha256=Xl_T_se9SOeMTmVJAOVzzmkekUsmpFMlAb2KvPCWfTw,2908
|
|
@@ -104,8 +104,8 @@ fiddler/utils/logger.py,sha256=-UTpjbhGZxBiZrK6UQ-ayZ37L9tv_ZbG1AnC-k2riiQ,1113
|
|
|
104
104
|
fiddler/utils/model_generator.py,sha256=TwQMQDpy-0gcq6HyL68s37N-sk_nGPq33mmppK6i7hI,2203
|
|
105
105
|
fiddler/utils/validations.py,sha256=i8NtgrpCsitq6BPa5lygpKDh07oaOXu7PAlCIcMvqzY,408
|
|
106
106
|
fiddler/utils/version.py,sha256=iC8Ry7UFl9EJW_xU1WbzO_l4yK6V7eQ6U5exB3xT864,531
|
|
107
|
-
fiddler_client-3.
|
|
108
|
-
fiddler_client-3.
|
|
109
|
-
fiddler_client-3.
|
|
110
|
-
fiddler_client-3.
|
|
111
|
-
fiddler_client-3.
|
|
107
|
+
fiddler_client-3.6.0.dev1.dist-info/LICENSE.txt,sha256=w8-LUAb_VLBWSsCNmih0pAqLIicJfGu8OJXpDjkIg_o,559
|
|
108
|
+
fiddler_client-3.6.0.dev1.dist-info/METADATA,sha256=r39vKnQ3TFVYlnBGy_z1o1dXEuWi0OA8OkzhEHp6Zao,1591
|
|
109
|
+
fiddler_client-3.6.0.dev1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
110
|
+
fiddler_client-3.6.0.dev1.dist-info/top_level.txt,sha256=ndC6LqvKmrTTs8czRlBOYdYgSqZbHuMf0zHt_HWpzzk,8
|
|
111
|
+
fiddler_client-3.6.0.dev1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|