databricks-sdk 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of databricks-sdk might be problematic. Click here for more details.
- databricks/sdk/__init__.py +79 -0
- databricks/sdk/core.py +1 -1
- databricks/sdk/errors/base.py +45 -1
- databricks/sdk/errors/mapper.py +9 -1
- databricks/sdk/errors/overrides.py +25 -0
- databricks/sdk/service/catalog.py +367 -244
- databricks/sdk/service/compute.py +78 -23
- databricks/sdk/service/iam.py +16 -24
- databricks/sdk/service/jobs.py +55 -65
- databricks/sdk/service/marketplace.py +3571 -0
- databricks/sdk/version.py +1 -1
- {databricks_sdk-0.24.0.dist-info → databricks_sdk-0.25.0.dist-info}/METADATA +19 -1
- {databricks_sdk-0.24.0.dist-info → databricks_sdk-0.25.0.dist-info}/RECORD +17 -15
- {databricks_sdk-0.24.0.dist-info → databricks_sdk-0.25.0.dist-info}/LICENSE +0 -0
- {databricks_sdk-0.24.0.dist-info → databricks_sdk-0.25.0.dist-info}/NOTICE +0 -0
- {databricks_sdk-0.24.0.dist-info → databricks_sdk-0.25.0.dist-info}/WHEEL +0 -0
- {databricks_sdk-0.24.0.dist-info → databricks_sdk-0.25.0.dist-info}/top_level.txt +0 -0
|
@@ -320,7 +320,34 @@ class AwsIamRoleResponse:
|
|
|
320
320
|
|
|
321
321
|
|
|
322
322
|
@dataclass
|
|
323
|
-
class
|
|
323
|
+
class AzureManagedIdentityRequest:
|
|
324
|
+
access_connector_id: str
|
|
325
|
+
"""The Azure resource ID of the Azure Databricks Access Connector. Use the format
|
|
326
|
+
/subscriptions/{guid}/resourceGroups/{rg-name}/providers/Microsoft.Databricks/accessConnectors/{connector-name}."""
|
|
327
|
+
|
|
328
|
+
managed_identity_id: Optional[str] = None
|
|
329
|
+
"""The Azure resource ID of the managed identity. Use the format
|
|
330
|
+
/subscriptions/{guid}/resourceGroups/{rg-name}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identity-name}.
|
|
331
|
+
This is only available for user-assgined identities. For system-assigned identities, the
|
|
332
|
+
access_connector_id is used to identify the identity. If this field is not provided, then we
|
|
333
|
+
assume the AzureManagedIdentity is for a system-assigned identity."""
|
|
334
|
+
|
|
335
|
+
def as_dict(self) -> dict:
|
|
336
|
+
"""Serializes the AzureManagedIdentityRequest into a dictionary suitable for use as a JSON request body."""
|
|
337
|
+
body = {}
|
|
338
|
+
if self.access_connector_id is not None: body['access_connector_id'] = self.access_connector_id
|
|
339
|
+
if self.managed_identity_id is not None: body['managed_identity_id'] = self.managed_identity_id
|
|
340
|
+
return body
|
|
341
|
+
|
|
342
|
+
@classmethod
|
|
343
|
+
def from_dict(cls, d: Dict[str, any]) -> AzureManagedIdentityRequest:
|
|
344
|
+
"""Deserializes the AzureManagedIdentityRequest from a dictionary."""
|
|
345
|
+
return cls(access_connector_id=d.get('access_connector_id', None),
|
|
346
|
+
managed_identity_id=d.get('managed_identity_id', None))
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
@dataclass
|
|
350
|
+
class AzureManagedIdentityResponse:
|
|
324
351
|
access_connector_id: str
|
|
325
352
|
"""The Azure resource ID of the Azure Databricks Access Connector. Use the format
|
|
326
353
|
/subscriptions/{guid}/resourceGroups/{rg-name}/providers/Microsoft.Databricks/accessConnectors/{connector-name}."""
|
|
@@ -336,7 +363,7 @@ class AzureManagedIdentity:
|
|
|
336
363
|
assume the AzureManagedIdentity is for a system-assigned identity."""
|
|
337
364
|
|
|
338
365
|
def as_dict(self) -> dict:
|
|
339
|
-
"""Serializes the
|
|
366
|
+
"""Serializes the AzureManagedIdentityResponse into a dictionary suitable for use as a JSON request body."""
|
|
340
367
|
body = {}
|
|
341
368
|
if self.access_connector_id is not None: body['access_connector_id'] = self.access_connector_id
|
|
342
369
|
if self.credential_id is not None: body['credential_id'] = self.credential_id
|
|
@@ -344,8 +371,8 @@ class AzureManagedIdentity:
|
|
|
344
371
|
return body
|
|
345
372
|
|
|
346
373
|
@classmethod
|
|
347
|
-
def from_dict(cls, d: Dict[str, any]) ->
|
|
348
|
-
"""Deserializes the
|
|
374
|
+
def from_dict(cls, d: Dict[str, any]) -> AzureManagedIdentityResponse:
|
|
375
|
+
"""Deserializes the AzureManagedIdentityResponse from a dictionary."""
|
|
349
376
|
return cls(access_connector_id=d.get('access_connector_id', None),
|
|
350
377
|
credential_id=d.get('credential_id', None),
|
|
351
378
|
managed_identity_id=d.get('managed_identity_id', None))
|
|
@@ -1256,7 +1283,7 @@ class CreateMonitor:
|
|
|
1256
1283
|
"""Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
1257
1284
|
table should also be present in the baseline table."""
|
|
1258
1285
|
|
|
1259
|
-
custom_metrics: Optional[List[
|
|
1286
|
+
custom_metrics: Optional[List[MonitorMetric]] = None
|
|
1260
1287
|
"""Custom metrics to compute on the monitored table. These can be aggregate metrics, derived
|
|
1261
1288
|
metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across
|
|
1262
1289
|
time windows)."""
|
|
@@ -1264,13 +1291,10 @@ class CreateMonitor:
|
|
|
1264
1291
|
data_classification_config: Optional[MonitorDataClassificationConfig] = None
|
|
1265
1292
|
"""The data classification config for the monitor."""
|
|
1266
1293
|
|
|
1267
|
-
|
|
1268
|
-
"""Full name of the table."""
|
|
1269
|
-
|
|
1270
|
-
inference_log: Optional[MonitorInferenceLogProfileType] = None
|
|
1294
|
+
inference_log: Optional[MonitorInferenceLog] = None
|
|
1271
1295
|
"""Configuration for monitoring inference logs."""
|
|
1272
1296
|
|
|
1273
|
-
notifications: Optional[
|
|
1297
|
+
notifications: Optional[MonitorNotifications] = None
|
|
1274
1298
|
"""The notification settings for the monitor."""
|
|
1275
1299
|
|
|
1276
1300
|
schedule: Optional[MonitorCronSchedule] = None
|
|
@@ -1284,10 +1308,13 @@ class CreateMonitor:
|
|
|
1284
1308
|
expression independently, resulting in a separate slice for each predicate and its complements.
|
|
1285
1309
|
For high-cardinality columns, only the top 100 unique values by frequency will generate slices."""
|
|
1286
1310
|
|
|
1287
|
-
snapshot: Optional[
|
|
1311
|
+
snapshot: Optional[MonitorSnapshot] = None
|
|
1288
1312
|
"""Configuration for monitoring snapshot tables."""
|
|
1289
1313
|
|
|
1290
|
-
|
|
1314
|
+
table_name: Optional[str] = None
|
|
1315
|
+
"""Full name of the table."""
|
|
1316
|
+
|
|
1317
|
+
time_series: Optional[MonitorTimeSeries] = None
|
|
1291
1318
|
"""Configuration for monitoring time series tables."""
|
|
1292
1319
|
|
|
1293
1320
|
warehouse_id: Optional[str] = None
|
|
@@ -1302,7 +1329,6 @@ class CreateMonitor:
|
|
|
1302
1329
|
if self.custom_metrics: body['custom_metrics'] = [v.as_dict() for v in self.custom_metrics]
|
|
1303
1330
|
if self.data_classification_config:
|
|
1304
1331
|
body['data_classification_config'] = self.data_classification_config.as_dict()
|
|
1305
|
-
if self.full_name is not None: body['full_name'] = self.full_name
|
|
1306
1332
|
if self.inference_log: body['inference_log'] = self.inference_log.as_dict()
|
|
1307
1333
|
if self.notifications: body['notifications'] = self.notifications.as_dict()
|
|
1308
1334
|
if self.output_schema_name is not None: body['output_schema_name'] = self.output_schema_name
|
|
@@ -1311,6 +1337,7 @@ class CreateMonitor:
|
|
|
1311
1337
|
body['skip_builtin_dashboard'] = self.skip_builtin_dashboard
|
|
1312
1338
|
if self.slicing_exprs: body['slicing_exprs'] = [v for v in self.slicing_exprs]
|
|
1313
1339
|
if self.snapshot: body['snapshot'] = self.snapshot.as_dict()
|
|
1340
|
+
if self.table_name is not None: body['table_name'] = self.table_name
|
|
1314
1341
|
if self.time_series: body['time_series'] = self.time_series.as_dict()
|
|
1315
1342
|
if self.warehouse_id is not None: body['warehouse_id'] = self.warehouse_id
|
|
1316
1343
|
return body
|
|
@@ -1320,18 +1347,18 @@ class CreateMonitor:
|
|
|
1320
1347
|
"""Deserializes the CreateMonitor from a dictionary."""
|
|
1321
1348
|
return cls(assets_dir=d.get('assets_dir', None),
|
|
1322
1349
|
baseline_table_name=d.get('baseline_table_name', None),
|
|
1323
|
-
custom_metrics=_repeated_dict(d, 'custom_metrics',
|
|
1350
|
+
custom_metrics=_repeated_dict(d, 'custom_metrics', MonitorMetric),
|
|
1324
1351
|
data_classification_config=_from_dict(d, 'data_classification_config',
|
|
1325
1352
|
MonitorDataClassificationConfig),
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
notifications=_from_dict(d, 'notifications', MonitorNotificationsConfig),
|
|
1353
|
+
inference_log=_from_dict(d, 'inference_log', MonitorInferenceLog),
|
|
1354
|
+
notifications=_from_dict(d, 'notifications', MonitorNotifications),
|
|
1329
1355
|
output_schema_name=d.get('output_schema_name', None),
|
|
1330
1356
|
schedule=_from_dict(d, 'schedule', MonitorCronSchedule),
|
|
1331
1357
|
skip_builtin_dashboard=d.get('skip_builtin_dashboard', None),
|
|
1332
1358
|
slicing_exprs=d.get('slicing_exprs', None),
|
|
1333
|
-
snapshot=_from_dict(d, 'snapshot',
|
|
1334
|
-
|
|
1359
|
+
snapshot=_from_dict(d, 'snapshot', MonitorSnapshot),
|
|
1360
|
+
table_name=d.get('table_name', None),
|
|
1361
|
+
time_series=_from_dict(d, 'time_series', MonitorTimeSeries),
|
|
1335
1362
|
warehouse_id=d.get('warehouse_id', None))
|
|
1336
1363
|
|
|
1337
1364
|
|
|
@@ -1454,7 +1481,7 @@ class CreateStorageCredential:
|
|
|
1454
1481
|
aws_iam_role: Optional[AwsIamRoleRequest] = None
|
|
1455
1482
|
"""The AWS IAM role configuration."""
|
|
1456
1483
|
|
|
1457
|
-
azure_managed_identity: Optional[
|
|
1484
|
+
azure_managed_identity: Optional[AzureManagedIdentityRequest] = None
|
|
1458
1485
|
"""The Azure managed identity configuration."""
|
|
1459
1486
|
|
|
1460
1487
|
azure_service_principal: Optional[AzureServicePrincipal] = None
|
|
@@ -1495,7 +1522,8 @@ class CreateStorageCredential:
|
|
|
1495
1522
|
def from_dict(cls, d: Dict[str, any]) -> CreateStorageCredential:
|
|
1496
1523
|
"""Deserializes the CreateStorageCredential from a dictionary."""
|
|
1497
1524
|
return cls(aws_iam_role=_from_dict(d, 'aws_iam_role', AwsIamRoleRequest),
|
|
1498
|
-
azure_managed_identity=_from_dict(d, 'azure_managed_identity',
|
|
1525
|
+
azure_managed_identity=_from_dict(d, 'azure_managed_identity',
|
|
1526
|
+
AzureManagedIdentityRequest),
|
|
1499
1527
|
azure_service_principal=_from_dict(d, 'azure_service_principal', AzureServicePrincipal),
|
|
1500
1528
|
cloudflare_api_token=_from_dict(d, 'cloudflare_api_token', CloudflareApiToken),
|
|
1501
1529
|
comment=d.get('comment', None),
|
|
@@ -3068,14 +3096,16 @@ class ModelVersionInfoStatus(Enum):
|
|
|
3068
3096
|
|
|
3069
3097
|
@dataclass
|
|
3070
3098
|
class MonitorCronSchedule:
|
|
3071
|
-
|
|
3072
|
-
"""
|
|
3099
|
+
quartz_cron_expression: str
|
|
3100
|
+
"""The expression that determines when to run the monitor. See [examples].
|
|
3101
|
+
|
|
3102
|
+
[examples]: https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html"""
|
|
3073
3103
|
|
|
3074
|
-
|
|
3075
|
-
"""
|
|
3104
|
+
timezone_id: str
|
|
3105
|
+
"""The timezone id (e.g., ``"PST"``) in which to evaluate the quartz expression."""
|
|
3076
3106
|
|
|
3077
|
-
|
|
3078
|
-
"""
|
|
3107
|
+
pause_status: Optional[MonitorCronSchedulePauseStatus] = None
|
|
3108
|
+
"""Read only field that indicates whether a schedule is paused or not."""
|
|
3079
3109
|
|
|
3080
3110
|
def as_dict(self) -> dict:
|
|
3081
3111
|
"""Serializes the MonitorCronSchedule into a dictionary suitable for use as a JSON request body."""
|
|
@@ -3095,62 +3125,12 @@ class MonitorCronSchedule:
|
|
|
3095
3125
|
|
|
3096
3126
|
|
|
3097
3127
|
class MonitorCronSchedulePauseStatus(Enum):
|
|
3098
|
-
"""
|
|
3128
|
+
"""Read only field that indicates whether a schedule is paused or not."""
|
|
3099
3129
|
|
|
3100
3130
|
PAUSED = 'PAUSED'
|
|
3101
3131
|
UNPAUSED = 'UNPAUSED'
|
|
3102
3132
|
|
|
3103
3133
|
|
|
3104
|
-
@dataclass
|
|
3105
|
-
class MonitorCustomMetric:
|
|
3106
|
-
definition: Optional[str] = None
|
|
3107
|
-
"""Jinja template for a SQL expression that specifies how to compute the metric. See [create metric
|
|
3108
|
-
definition].
|
|
3109
|
-
|
|
3110
|
-
[create metric definition]: https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition"""
|
|
3111
|
-
|
|
3112
|
-
input_columns: Optional[List[str]] = None
|
|
3113
|
-
"""Columns on the monitored table to apply the custom metrics to."""
|
|
3114
|
-
|
|
3115
|
-
name: Optional[str] = None
|
|
3116
|
-
"""Name of the custom metric."""
|
|
3117
|
-
|
|
3118
|
-
output_data_type: Optional[str] = None
|
|
3119
|
-
"""The output type of the custom metric."""
|
|
3120
|
-
|
|
3121
|
-
type: Optional[MonitorCustomMetricType] = None
|
|
3122
|
-
"""The type of the custom metric."""
|
|
3123
|
-
|
|
3124
|
-
def as_dict(self) -> dict:
|
|
3125
|
-
"""Serializes the MonitorCustomMetric into a dictionary suitable for use as a JSON request body."""
|
|
3126
|
-
body = {}
|
|
3127
|
-
if self.definition is not None: body['definition'] = self.definition
|
|
3128
|
-
if self.input_columns: body['input_columns'] = [v for v in self.input_columns]
|
|
3129
|
-
if self.name is not None: body['name'] = self.name
|
|
3130
|
-
if self.output_data_type is not None: body['output_data_type'] = self.output_data_type
|
|
3131
|
-
if self.type is not None: body['type'] = self.type.value
|
|
3132
|
-
return body
|
|
3133
|
-
|
|
3134
|
-
@classmethod
|
|
3135
|
-
def from_dict(cls, d: Dict[str, any]) -> MonitorCustomMetric:
|
|
3136
|
-
"""Deserializes the MonitorCustomMetric from a dictionary."""
|
|
3137
|
-
return cls(definition=d.get('definition', None),
|
|
3138
|
-
input_columns=d.get('input_columns', None),
|
|
3139
|
-
name=d.get('name', None),
|
|
3140
|
-
output_data_type=d.get('output_data_type', None),
|
|
3141
|
-
type=_enum(d, 'type', MonitorCustomMetricType))
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
class MonitorCustomMetricType(Enum):
|
|
3145
|
-
"""The type of the custom metric."""
|
|
3146
|
-
|
|
3147
|
-
CUSTOM_METRIC_TYPE_AGGREGATE = 'CUSTOM_METRIC_TYPE_AGGREGATE'
|
|
3148
|
-
CUSTOM_METRIC_TYPE_DERIVED = 'CUSTOM_METRIC_TYPE_DERIVED'
|
|
3149
|
-
CUSTOM_METRIC_TYPE_DRIFT = 'CUSTOM_METRIC_TYPE_DRIFT'
|
|
3150
|
-
MONITOR_STATUS_ERROR = 'MONITOR_STATUS_ERROR'
|
|
3151
|
-
MONITOR_STATUS_FAILED = 'MONITOR_STATUS_FAILED'
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
3134
|
@dataclass
|
|
3155
3135
|
class MonitorDataClassificationConfig:
|
|
3156
3136
|
enabled: Optional[bool] = None
|
|
@@ -3169,48 +3149,58 @@ class MonitorDataClassificationConfig:
|
|
|
3169
3149
|
|
|
3170
3150
|
|
|
3171
3151
|
@dataclass
|
|
3172
|
-
class
|
|
3152
|
+
class MonitorDestination:
|
|
3173
3153
|
email_addresses: Optional[List[str]] = None
|
|
3174
3154
|
"""The list of email addresses to send the notification to. A maximum of 5 email addresses is
|
|
3175
3155
|
supported."""
|
|
3176
3156
|
|
|
3177
3157
|
def as_dict(self) -> dict:
|
|
3178
|
-
"""Serializes the
|
|
3158
|
+
"""Serializes the MonitorDestination into a dictionary suitable for use as a JSON request body."""
|
|
3179
3159
|
body = {}
|
|
3180
3160
|
if self.email_addresses: body['email_addresses'] = [v for v in self.email_addresses]
|
|
3181
3161
|
return body
|
|
3182
3162
|
|
|
3183
3163
|
@classmethod
|
|
3184
|
-
def from_dict(cls, d: Dict[str, any]) ->
|
|
3185
|
-
"""Deserializes the
|
|
3164
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorDestination:
|
|
3165
|
+
"""Deserializes the MonitorDestination from a dictionary."""
|
|
3186
3166
|
return cls(email_addresses=d.get('email_addresses', None))
|
|
3187
3167
|
|
|
3188
3168
|
|
|
3189
3169
|
@dataclass
|
|
3190
|
-
class
|
|
3191
|
-
|
|
3192
|
-
"""
|
|
3170
|
+
class MonitorInferenceLog:
|
|
3171
|
+
timestamp_col: str
|
|
3172
|
+
"""Column that contains the timestamps of requests. The column must be one of the following: - A
|
|
3173
|
+
``TimestampType`` column - A column whose values can be converted to timestamps through the
|
|
3174
|
+
pyspark ``to_timestamp`` [function].
|
|
3175
|
+
|
|
3176
|
+
[function]: https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html"""
|
|
3193
3177
|
|
|
3194
|
-
|
|
3195
|
-
"""
|
|
3178
|
+
granularities: List[str]
|
|
3179
|
+
"""Granularities for aggregating data into time windows based on their timestamp. Currently the
|
|
3180
|
+
following static granularities are supported: {``"5 minutes"``, ``"30 minutes"``, ``"1 hour"``,
|
|
3181
|
+
``"1 day"``, ``"<n> week(s)"``, ``"1 month"``, ``"1 year"``}."""
|
|
3196
3182
|
|
|
3197
|
-
model_id_col:
|
|
3198
|
-
"""Column of the model
|
|
3183
|
+
model_id_col: str
|
|
3184
|
+
"""Column that contains the id of the model generating the predictions. Metrics will be computed
|
|
3185
|
+
per model id by default, and also across all model ids."""
|
|
3199
3186
|
|
|
3200
|
-
|
|
3201
|
-
"""
|
|
3187
|
+
problem_type: MonitorInferenceLogProblemType
|
|
3188
|
+
"""Problem type the model aims to solve. Determines the type of model-quality metrics that will be
|
|
3189
|
+
computed."""
|
|
3202
3190
|
|
|
3203
|
-
|
|
3204
|
-
"""Column
|
|
3191
|
+
prediction_col: str
|
|
3192
|
+
"""Column that contains the output/prediction from the model."""
|
|
3205
3193
|
|
|
3206
|
-
|
|
3207
|
-
"""
|
|
3194
|
+
label_col: Optional[str] = None
|
|
3195
|
+
"""Optional column that contains the ground truth for the prediction."""
|
|
3208
3196
|
|
|
3209
|
-
|
|
3210
|
-
"""
|
|
3197
|
+
prediction_proba_col: Optional[str] = None
|
|
3198
|
+
"""Optional column that contains the prediction probabilities for each class in a classification
|
|
3199
|
+
problem type. The values in this column should be a map, mapping each class label to the
|
|
3200
|
+
prediction probability for a given sample. The map should be of PySpark MapType()."""
|
|
3211
3201
|
|
|
3212
3202
|
def as_dict(self) -> dict:
|
|
3213
|
-
"""Serializes the
|
|
3203
|
+
"""Serializes the MonitorInferenceLog into a dictionary suitable for use as a JSON request body."""
|
|
3214
3204
|
body = {}
|
|
3215
3205
|
if self.granularities: body['granularities'] = [v for v in self.granularities]
|
|
3216
3206
|
if self.label_col is not None: body['label_col'] = self.label_col
|
|
@@ -3222,19 +3212,20 @@ class MonitorInferenceLogProfileType:
|
|
|
3222
3212
|
return body
|
|
3223
3213
|
|
|
3224
3214
|
@classmethod
|
|
3225
|
-
def from_dict(cls, d: Dict[str, any]) ->
|
|
3226
|
-
"""Deserializes the
|
|
3215
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorInferenceLog:
|
|
3216
|
+
"""Deserializes the MonitorInferenceLog from a dictionary."""
|
|
3227
3217
|
return cls(granularities=d.get('granularities', None),
|
|
3228
3218
|
label_col=d.get('label_col', None),
|
|
3229
3219
|
model_id_col=d.get('model_id_col', None),
|
|
3230
3220
|
prediction_col=d.get('prediction_col', None),
|
|
3231
3221
|
prediction_proba_col=d.get('prediction_proba_col', None),
|
|
3232
|
-
problem_type=_enum(d, 'problem_type',
|
|
3222
|
+
problem_type=_enum(d, 'problem_type', MonitorInferenceLogProblemType),
|
|
3233
3223
|
timestamp_col=d.get('timestamp_col', None))
|
|
3234
3224
|
|
|
3235
3225
|
|
|
3236
|
-
class
|
|
3237
|
-
"""Problem type the model aims to solve.
|
|
3226
|
+
class MonitorInferenceLogProblemType(Enum):
|
|
3227
|
+
"""Problem type the model aims to solve. Determines the type of model-quality metrics that will be
|
|
3228
|
+
computed."""
|
|
3238
3229
|
|
|
3239
3230
|
PROBLEM_TYPE_CLASSIFICATION = 'PROBLEM_TYPE_CLASSIFICATION'
|
|
3240
3231
|
PROBLEM_TYPE_REGRESSION = 'PROBLEM_TYPE_REGRESSION'
|
|
@@ -3242,6 +3233,23 @@ class MonitorInferenceLogProfileTypeProblemType(Enum):
|
|
|
3242
3233
|
|
|
3243
3234
|
@dataclass
|
|
3244
3235
|
class MonitorInfo:
|
|
3236
|
+
table_name: str
|
|
3237
|
+
"""The full name of the table to monitor. Format: __catalog_name__.__schema_name__.__table_name__."""
|
|
3238
|
+
|
|
3239
|
+
status: MonitorInfoStatus
|
|
3240
|
+
"""The status of the monitor."""
|
|
3241
|
+
|
|
3242
|
+
monitor_version: str
|
|
3243
|
+
"""The version of the monitor config (e.g. 1,2,3). If negative, the monitor may be corrupted."""
|
|
3244
|
+
|
|
3245
|
+
profile_metrics_table_name: str
|
|
3246
|
+
"""The full name of the profile metrics table. Format:
|
|
3247
|
+
__catalog_name__.__schema_name__.__table_name__."""
|
|
3248
|
+
|
|
3249
|
+
drift_metrics_table_name: str
|
|
3250
|
+
"""The full name of the drift metrics table. Format:
|
|
3251
|
+
__catalog_name__.__schema_name__.__table_name__."""
|
|
3252
|
+
|
|
3245
3253
|
assets_dir: Optional[str] = None
|
|
3246
3254
|
"""The directory to store monitoring assets (e.g. dashboard, metric tables)."""
|
|
3247
3255
|
|
|
@@ -3249,40 +3257,30 @@ class MonitorInfo:
|
|
|
3249
3257
|
"""Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
3250
3258
|
table should also be present in the baseline table."""
|
|
3251
3259
|
|
|
3252
|
-
custom_metrics: Optional[List[
|
|
3260
|
+
custom_metrics: Optional[List[MonitorMetric]] = None
|
|
3253
3261
|
"""Custom metrics to compute on the monitored table. These can be aggregate metrics, derived
|
|
3254
3262
|
metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across
|
|
3255
3263
|
time windows)."""
|
|
3256
3264
|
|
|
3257
3265
|
dashboard_id: Optional[str] = None
|
|
3258
|
-
"""
|
|
3266
|
+
"""Id of dashboard that visualizes the computed metrics. This can be empty if the monitor is in
|
|
3267
|
+
PENDING state."""
|
|
3259
3268
|
|
|
3260
3269
|
data_classification_config: Optional[MonitorDataClassificationConfig] = None
|
|
3261
3270
|
"""The data classification config for the monitor."""
|
|
3262
3271
|
|
|
3263
|
-
|
|
3264
|
-
"""The full name of the drift metrics table. Format:
|
|
3265
|
-
__catalog_name__.__schema_name__.__table_name__."""
|
|
3266
|
-
|
|
3267
|
-
inference_log: Optional[MonitorInferenceLogProfileType] = None
|
|
3272
|
+
inference_log: Optional[MonitorInferenceLog] = None
|
|
3268
3273
|
"""Configuration for monitoring inference logs."""
|
|
3269
3274
|
|
|
3270
3275
|
latest_monitor_failure_msg: Optional[str] = None
|
|
3271
3276
|
"""The latest failure message of the monitor (if any)."""
|
|
3272
3277
|
|
|
3273
|
-
|
|
3274
|
-
"""The version of the monitor config (e.g. 1,2,3). If negative, the monitor may be corrupted."""
|
|
3275
|
-
|
|
3276
|
-
notifications: Optional[MonitorNotificationsConfig] = None
|
|
3278
|
+
notifications: Optional[MonitorNotifications] = None
|
|
3277
3279
|
"""The notification settings for the monitor."""
|
|
3278
3280
|
|
|
3279
3281
|
output_schema_name: Optional[str] = None
|
|
3280
3282
|
"""Schema where output metric tables are created."""
|
|
3281
3283
|
|
|
3282
|
-
profile_metrics_table_name: Optional[str] = None
|
|
3283
|
-
"""The full name of the profile metrics table. Format:
|
|
3284
|
-
__catalog_name__.__schema_name__.__table_name__."""
|
|
3285
|
-
|
|
3286
3284
|
schedule: Optional[MonitorCronSchedule] = None
|
|
3287
3285
|
"""The schedule for automatically updating and refreshing metric tables."""
|
|
3288
3286
|
|
|
@@ -3291,16 +3289,10 @@ class MonitorInfo:
|
|
|
3291
3289
|
expression independently, resulting in a separate slice for each predicate and its complements.
|
|
3292
3290
|
For high-cardinality columns, only the top 100 unique values by frequency will generate slices."""
|
|
3293
3291
|
|
|
3294
|
-
snapshot: Optional[
|
|
3292
|
+
snapshot: Optional[MonitorSnapshot] = None
|
|
3295
3293
|
"""Configuration for monitoring snapshot tables."""
|
|
3296
3294
|
|
|
3297
|
-
|
|
3298
|
-
"""The status of the monitor."""
|
|
3299
|
-
|
|
3300
|
-
table_name: Optional[str] = None
|
|
3301
|
-
"""The full name of the table to monitor. Format: __catalog_name__.__schema_name__.__table_name__."""
|
|
3302
|
-
|
|
3303
|
-
time_series: Optional[MonitorTimeSeriesProfileType] = None
|
|
3295
|
+
time_series: Optional[MonitorTimeSeries] = None
|
|
3304
3296
|
"""Configuration for monitoring time series tables."""
|
|
3305
3297
|
|
|
3306
3298
|
def as_dict(self) -> dict:
|
|
@@ -3335,23 +3327,23 @@ class MonitorInfo:
|
|
|
3335
3327
|
"""Deserializes the MonitorInfo from a dictionary."""
|
|
3336
3328
|
return cls(assets_dir=d.get('assets_dir', None),
|
|
3337
3329
|
baseline_table_name=d.get('baseline_table_name', None),
|
|
3338
|
-
custom_metrics=_repeated_dict(d, 'custom_metrics',
|
|
3330
|
+
custom_metrics=_repeated_dict(d, 'custom_metrics', MonitorMetric),
|
|
3339
3331
|
dashboard_id=d.get('dashboard_id', None),
|
|
3340
3332
|
data_classification_config=_from_dict(d, 'data_classification_config',
|
|
3341
3333
|
MonitorDataClassificationConfig),
|
|
3342
3334
|
drift_metrics_table_name=d.get('drift_metrics_table_name', None),
|
|
3343
|
-
inference_log=_from_dict(d, 'inference_log',
|
|
3335
|
+
inference_log=_from_dict(d, 'inference_log', MonitorInferenceLog),
|
|
3344
3336
|
latest_monitor_failure_msg=d.get('latest_monitor_failure_msg', None),
|
|
3345
3337
|
monitor_version=d.get('monitor_version', None),
|
|
3346
|
-
notifications=_from_dict(d, 'notifications',
|
|
3338
|
+
notifications=_from_dict(d, 'notifications', MonitorNotifications),
|
|
3347
3339
|
output_schema_name=d.get('output_schema_name', None),
|
|
3348
3340
|
profile_metrics_table_name=d.get('profile_metrics_table_name', None),
|
|
3349
3341
|
schedule=_from_dict(d, 'schedule', MonitorCronSchedule),
|
|
3350
3342
|
slicing_exprs=d.get('slicing_exprs', None),
|
|
3351
|
-
snapshot=_from_dict(d, 'snapshot',
|
|
3343
|
+
snapshot=_from_dict(d, 'snapshot', MonitorSnapshot),
|
|
3352
3344
|
status=_enum(d, 'status', MonitorInfoStatus),
|
|
3353
3345
|
table_name=d.get('table_name', None),
|
|
3354
|
-
time_series=_from_dict(d, 'time_series',
|
|
3346
|
+
time_series=_from_dict(d, 'time_series', MonitorTimeSeries))
|
|
3355
3347
|
|
|
3356
3348
|
|
|
3357
3349
|
class MonitorInfoStatus(Enum):
|
|
@@ -3365,38 +3357,109 @@ class MonitorInfoStatus(Enum):
|
|
|
3365
3357
|
|
|
3366
3358
|
|
|
3367
3359
|
@dataclass
|
|
3368
|
-
class
|
|
3369
|
-
|
|
3360
|
+
class MonitorMetric:
|
|
3361
|
+
name: str
|
|
3362
|
+
"""Name of the metric in the output tables."""
|
|
3363
|
+
|
|
3364
|
+
definition: str
|
|
3365
|
+
"""Jinja template for a SQL expression that specifies how to compute the metric. See [create metric
|
|
3366
|
+
definition].
|
|
3367
|
+
|
|
3368
|
+
[create metric definition]: https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition"""
|
|
3369
|
+
|
|
3370
|
+
input_columns: List[str]
|
|
3371
|
+
"""A list of column names in the input table the metric should be computed for. Can use
|
|
3372
|
+
``":table"`` to indicate that the metric needs information from multiple columns."""
|
|
3373
|
+
|
|
3374
|
+
output_data_type: str
|
|
3375
|
+
"""The output type of the custom metric."""
|
|
3376
|
+
|
|
3377
|
+
type: MonitorMetricType
|
|
3378
|
+
"""Can only be one of ``"CUSTOM_METRIC_TYPE_AGGREGATE"``, ``"CUSTOM_METRIC_TYPE_DERIVED"``, or
|
|
3379
|
+
``"CUSTOM_METRIC_TYPE_DRIFT"``. The ``"CUSTOM_METRIC_TYPE_AGGREGATE"`` and
|
|
3380
|
+
``"CUSTOM_METRIC_TYPE_DERIVED"`` metrics are computed on a single table, whereas the
|
|
3381
|
+
``"CUSTOM_METRIC_TYPE_DRIFT"`` compare metrics across baseline and input table, or across the
|
|
3382
|
+
two consecutive time windows. - CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing
|
|
3383
|
+
columns in your table - CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate
|
|
3384
|
+
metrics - CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics"""
|
|
3385
|
+
|
|
3386
|
+
def as_dict(self) -> dict:
|
|
3387
|
+
"""Serializes the MonitorMetric into a dictionary suitable for use as a JSON request body."""
|
|
3388
|
+
body = {}
|
|
3389
|
+
if self.definition is not None: body['definition'] = self.definition
|
|
3390
|
+
if self.input_columns: body['input_columns'] = [v for v in self.input_columns]
|
|
3391
|
+
if self.name is not None: body['name'] = self.name
|
|
3392
|
+
if self.output_data_type is not None: body['output_data_type'] = self.output_data_type
|
|
3393
|
+
if self.type is not None: body['type'] = self.type.value
|
|
3394
|
+
return body
|
|
3395
|
+
|
|
3396
|
+
@classmethod
|
|
3397
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorMetric:
|
|
3398
|
+
"""Deserializes the MonitorMetric from a dictionary."""
|
|
3399
|
+
return cls(definition=d.get('definition', None),
|
|
3400
|
+
input_columns=d.get('input_columns', None),
|
|
3401
|
+
name=d.get('name', None),
|
|
3402
|
+
output_data_type=d.get('output_data_type', None),
|
|
3403
|
+
type=_enum(d, 'type', MonitorMetricType))
|
|
3404
|
+
|
|
3405
|
+
|
|
3406
|
+
class MonitorMetricType(Enum):
|
|
3407
|
+
"""Can only be one of ``"CUSTOM_METRIC_TYPE_AGGREGATE"``, ``"CUSTOM_METRIC_TYPE_DERIVED"``, or
|
|
3408
|
+
``"CUSTOM_METRIC_TYPE_DRIFT"``. The ``"CUSTOM_METRIC_TYPE_AGGREGATE"`` and
|
|
3409
|
+
``"CUSTOM_METRIC_TYPE_DERIVED"`` metrics are computed on a single table, whereas the
|
|
3410
|
+
``"CUSTOM_METRIC_TYPE_DRIFT"`` compare metrics across baseline and input table, or across the
|
|
3411
|
+
two consecutive time windows. - CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing
|
|
3412
|
+
columns in your table - CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate
|
|
3413
|
+
metrics - CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics"""
|
|
3414
|
+
|
|
3415
|
+
CUSTOM_METRIC_TYPE_AGGREGATE = 'CUSTOM_METRIC_TYPE_AGGREGATE'
|
|
3416
|
+
CUSTOM_METRIC_TYPE_DERIVED = 'CUSTOM_METRIC_TYPE_DERIVED'
|
|
3417
|
+
CUSTOM_METRIC_TYPE_DRIFT = 'CUSTOM_METRIC_TYPE_DRIFT'
|
|
3418
|
+
|
|
3419
|
+
|
|
3420
|
+
@dataclass
|
|
3421
|
+
class MonitorNotifications:
|
|
3422
|
+
on_failure: Optional[MonitorDestination] = None
|
|
3370
3423
|
"""Who to send notifications to on monitor failure."""
|
|
3371
3424
|
|
|
3425
|
+
on_new_classification_tag_detected: Optional[MonitorDestination] = None
|
|
3426
|
+
"""Who to send notifications to when new data classification tags are detected."""
|
|
3427
|
+
|
|
3372
3428
|
def as_dict(self) -> dict:
|
|
3373
|
-
"""Serializes the
|
|
3429
|
+
"""Serializes the MonitorNotifications into a dictionary suitable for use as a JSON request body."""
|
|
3374
3430
|
body = {}
|
|
3375
3431
|
if self.on_failure: body['on_failure'] = self.on_failure.as_dict()
|
|
3432
|
+
if self.on_new_classification_tag_detected:
|
|
3433
|
+
body['on_new_classification_tag_detected'] = self.on_new_classification_tag_detected.as_dict()
|
|
3376
3434
|
return body
|
|
3377
3435
|
|
|
3378
3436
|
@classmethod
|
|
3379
|
-
def from_dict(cls, d: Dict[str, any]) ->
|
|
3380
|
-
"""Deserializes the
|
|
3381
|
-
return cls(on_failure=_from_dict(d, 'on_failure',
|
|
3437
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorNotifications:
|
|
3438
|
+
"""Deserializes the MonitorNotifications from a dictionary."""
|
|
3439
|
+
return cls(on_failure=_from_dict(d, 'on_failure', MonitorDestination),
|
|
3440
|
+
on_new_classification_tag_detected=_from_dict(d, 'on_new_classification_tag_detected',
|
|
3441
|
+
MonitorDestination))
|
|
3382
3442
|
|
|
3383
3443
|
|
|
3384
3444
|
@dataclass
|
|
3385
3445
|
class MonitorRefreshInfo:
|
|
3446
|
+
refresh_id: int
|
|
3447
|
+
"""Unique id of the refresh operation."""
|
|
3448
|
+
|
|
3449
|
+
state: MonitorRefreshInfoState
|
|
3450
|
+
"""The current state of the refresh."""
|
|
3451
|
+
|
|
3452
|
+
start_time_ms: int
|
|
3453
|
+
"""Time at which refresh operation was initiated (milliseconds since 1/1/1970 UTC)."""
|
|
3454
|
+
|
|
3386
3455
|
end_time_ms: Optional[int] = None
|
|
3387
|
-
"""
|
|
3456
|
+
"""Time at which refresh operation completed (milliseconds since 1/1/1970 UTC)."""
|
|
3388
3457
|
|
|
3389
3458
|
message: Optional[str] = None
|
|
3390
3459
|
"""An optional message to give insight into the current state of the job (e.g. FAILURE messages)."""
|
|
3391
3460
|
|
|
3392
|
-
|
|
3393
|
-
"""The
|
|
3394
|
-
|
|
3395
|
-
start_time_ms: Optional[int] = None
|
|
3396
|
-
"""The time at which the refresh started, in epoch milliseconds."""
|
|
3397
|
-
|
|
3398
|
-
state: Optional[MonitorRefreshInfoState] = None
|
|
3399
|
-
"""The current state of the refresh."""
|
|
3461
|
+
trigger: Optional[MonitorRefreshInfoTrigger] = None
|
|
3462
|
+
"""The method by which the refresh was triggered."""
|
|
3400
3463
|
|
|
3401
3464
|
def as_dict(self) -> dict:
|
|
3402
3465
|
"""Serializes the MonitorRefreshInfo into a dictionary suitable for use as a JSON request body."""
|
|
@@ -3406,6 +3469,7 @@ class MonitorRefreshInfo:
|
|
|
3406
3469
|
if self.refresh_id is not None: body['refresh_id'] = self.refresh_id
|
|
3407
3470
|
if self.start_time_ms is not None: body['start_time_ms'] = self.start_time_ms
|
|
3408
3471
|
if self.state is not None: body['state'] = self.state.value
|
|
3472
|
+
if self.trigger is not None: body['trigger'] = self.trigger.value
|
|
3409
3473
|
return body
|
|
3410
3474
|
|
|
3411
3475
|
@classmethod
|
|
@@ -3415,7 +3479,8 @@ class MonitorRefreshInfo:
|
|
|
3415
3479
|
message=d.get('message', None),
|
|
3416
3480
|
refresh_id=d.get('refresh_id', None),
|
|
3417
3481
|
start_time_ms=d.get('start_time_ms', None),
|
|
3418
|
-
state=_enum(d, 'state', MonitorRefreshInfoState)
|
|
3482
|
+
state=_enum(d, 'state', MonitorRefreshInfoState),
|
|
3483
|
+
trigger=_enum(d, 'trigger', MonitorRefreshInfoTrigger))
|
|
3419
3484
|
|
|
3420
3485
|
|
|
3421
3486
|
class MonitorRefreshInfoState(Enum):
|
|
@@ -3428,39 +3493,51 @@ class MonitorRefreshInfoState(Enum):
|
|
|
3428
3493
|
SUCCESS = 'SUCCESS'
|
|
3429
3494
|
|
|
3430
3495
|
|
|
3496
|
+
class MonitorRefreshInfoTrigger(Enum):
|
|
3497
|
+
"""The method by which the refresh was triggered."""
|
|
3498
|
+
|
|
3499
|
+
MANUAL = 'MANUAL'
|
|
3500
|
+
SCHEDULE = 'SCHEDULE'
|
|
3501
|
+
|
|
3502
|
+
|
|
3431
3503
|
@dataclass
|
|
3432
|
-
class
|
|
3504
|
+
class MonitorSnapshot:
|
|
3433
3505
|
|
|
3434
3506
|
def as_dict(self) -> dict:
|
|
3435
|
-
"""Serializes the
|
|
3507
|
+
"""Serializes the MonitorSnapshot into a dictionary suitable for use as a JSON request body."""
|
|
3436
3508
|
body = {}
|
|
3437
3509
|
return body
|
|
3438
3510
|
|
|
3439
3511
|
@classmethod
|
|
3440
|
-
def from_dict(cls, d: Dict[str, any]) ->
|
|
3441
|
-
"""Deserializes the
|
|
3512
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorSnapshot:
|
|
3513
|
+
"""Deserializes the MonitorSnapshot from a dictionary."""
|
|
3442
3514
|
return cls()
|
|
3443
3515
|
|
|
3444
3516
|
|
|
3445
3517
|
@dataclass
|
|
3446
|
-
class
|
|
3447
|
-
|
|
3448
|
-
"""
|
|
3518
|
+
class MonitorTimeSeries:
|
|
3519
|
+
timestamp_col: str
|
|
3520
|
+
"""Column that contains the timestamps of requests. The column must be one of the following: - A
|
|
3521
|
+
``TimestampType`` column - A column whose values can be converted to timestamps through the
|
|
3522
|
+
pyspark ``to_timestamp`` [function].
|
|
3523
|
+
|
|
3524
|
+
[function]: https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html"""
|
|
3449
3525
|
|
|
3450
|
-
|
|
3451
|
-
"""
|
|
3452
|
-
|
|
3526
|
+
granularities: List[str]
|
|
3527
|
+
"""Granularities for aggregating data into time windows based on their timestamp. Currently the
|
|
3528
|
+
following static granularities are supported: {``"5 minutes"``, ``"30 minutes"``, ``"1 hour"``,
|
|
3529
|
+
``"1 day"``, ``"<n> week(s)"``, ``"1 month"``, ``"1 year"``}."""
|
|
3453
3530
|
|
|
3454
3531
|
def as_dict(self) -> dict:
|
|
3455
|
-
"""Serializes the
|
|
3532
|
+
"""Serializes the MonitorTimeSeries into a dictionary suitable for use as a JSON request body."""
|
|
3456
3533
|
body = {}
|
|
3457
3534
|
if self.granularities: body['granularities'] = [v for v in self.granularities]
|
|
3458
3535
|
if self.timestamp_col is not None: body['timestamp_col'] = self.timestamp_col
|
|
3459
3536
|
return body
|
|
3460
3537
|
|
|
3461
3538
|
@classmethod
|
|
3462
|
-
def from_dict(cls, d: Dict[str, any]) ->
|
|
3463
|
-
"""Deserializes the
|
|
3539
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorTimeSeries:
|
|
3540
|
+
"""Deserializes the MonitorTimeSeries from a dictionary."""
|
|
3464
3541
|
return cls(granularities=d.get('granularities', None), timestamp_col=d.get('timestamp_col', None))
|
|
3465
3542
|
|
|
3466
3543
|
|
|
@@ -4207,7 +4284,7 @@ class StorageCredentialInfo:
|
|
|
4207
4284
|
aws_iam_role: Optional[AwsIamRoleResponse] = None
|
|
4208
4285
|
"""The AWS IAM role configuration."""
|
|
4209
4286
|
|
|
4210
|
-
azure_managed_identity: Optional[
|
|
4287
|
+
azure_managed_identity: Optional[AzureManagedIdentityResponse] = None
|
|
4211
4288
|
"""The Azure managed identity configuration."""
|
|
4212
4289
|
|
|
4213
4290
|
azure_service_principal: Optional[AzureServicePrincipal] = None
|
|
@@ -4280,7 +4357,8 @@ class StorageCredentialInfo:
|
|
|
4280
4357
|
def from_dict(cls, d: Dict[str, any]) -> StorageCredentialInfo:
|
|
4281
4358
|
"""Deserializes the StorageCredentialInfo from a dictionary."""
|
|
4282
4359
|
return cls(aws_iam_role=_from_dict(d, 'aws_iam_role', AwsIamRoleResponse),
|
|
4283
|
-
azure_managed_identity=_from_dict(d, 'azure_managed_identity',
|
|
4360
|
+
azure_managed_identity=_from_dict(d, 'azure_managed_identity',
|
|
4361
|
+
AzureManagedIdentityResponse),
|
|
4284
4362
|
azure_service_principal=_from_dict(d, 'azure_service_principal', AzureServicePrincipal),
|
|
4285
4363
|
cloudflare_api_token=_from_dict(d, 'cloudflare_api_token', CloudflareApiToken),
|
|
4286
4364
|
comment=d.get('comment', None),
|
|
@@ -4583,7 +4661,7 @@ class TableInfo:
|
|
|
4583
4661
|
|
|
4584
4662
|
@dataclass
|
|
4585
4663
|
class TableRowFilter:
|
|
4586
|
-
|
|
4664
|
+
function_name: str
|
|
4587
4665
|
"""The full name of the row filter SQL UDF."""
|
|
4588
4666
|
|
|
4589
4667
|
input_column_names: List[str]
|
|
@@ -4593,14 +4671,15 @@ class TableRowFilter:
|
|
|
4593
4671
|
def as_dict(self) -> dict:
|
|
4594
4672
|
"""Serializes the TableRowFilter into a dictionary suitable for use as a JSON request body."""
|
|
4595
4673
|
body = {}
|
|
4674
|
+
if self.function_name is not None: body['function_name'] = self.function_name
|
|
4596
4675
|
if self.input_column_names: body['input_column_names'] = [v for v in self.input_column_names]
|
|
4597
|
-
if self.name is not None: body['name'] = self.name
|
|
4598
4676
|
return body
|
|
4599
4677
|
|
|
4600
4678
|
@classmethod
|
|
4601
4679
|
def from_dict(cls, d: Dict[str, any]) -> TableRowFilter:
|
|
4602
4680
|
"""Deserializes the TableRowFilter from a dictionary."""
|
|
4603
|
-
return cls(
|
|
4681
|
+
return cls(function_name=d.get('function_name', None),
|
|
4682
|
+
input_column_names=d.get('input_column_names', None))
|
|
4604
4683
|
|
|
4605
4684
|
|
|
4606
4685
|
@dataclass
|
|
@@ -4993,7 +5072,7 @@ class UpdateMonitor:
|
|
|
4993
5072
|
"""Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
4994
5073
|
table should also be present in the baseline table."""
|
|
4995
5074
|
|
|
4996
|
-
custom_metrics: Optional[List[
|
|
5075
|
+
custom_metrics: Optional[List[MonitorMetric]] = None
|
|
4997
5076
|
"""Custom metrics to compute on the monitored table. These can be aggregate metrics, derived
|
|
4998
5077
|
metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across
|
|
4999
5078
|
time windows)."""
|
|
@@ -5001,13 +5080,10 @@ class UpdateMonitor:
|
|
|
5001
5080
|
data_classification_config: Optional[MonitorDataClassificationConfig] = None
|
|
5002
5081
|
"""The data classification config for the monitor."""
|
|
5003
5082
|
|
|
5004
|
-
|
|
5005
|
-
"""Full name of the table."""
|
|
5006
|
-
|
|
5007
|
-
inference_log: Optional[MonitorInferenceLogProfileType] = None
|
|
5083
|
+
inference_log: Optional[MonitorInferenceLog] = None
|
|
5008
5084
|
"""Configuration for monitoring inference logs."""
|
|
5009
5085
|
|
|
5010
|
-
notifications: Optional[
|
|
5086
|
+
notifications: Optional[MonitorNotifications] = None
|
|
5011
5087
|
"""The notification settings for the monitor."""
|
|
5012
5088
|
|
|
5013
5089
|
schedule: Optional[MonitorCronSchedule] = None
|
|
@@ -5018,10 +5094,13 @@ class UpdateMonitor:
|
|
|
5018
5094
|
expression independently, resulting in a separate slice for each predicate and its complements.
|
|
5019
5095
|
For high-cardinality columns, only the top 100 unique values by frequency will generate slices."""
|
|
5020
5096
|
|
|
5021
|
-
snapshot: Optional[
|
|
5097
|
+
snapshot: Optional[MonitorSnapshot] = None
|
|
5022
5098
|
"""Configuration for monitoring snapshot tables."""
|
|
5023
5099
|
|
|
5024
|
-
|
|
5100
|
+
table_name: Optional[str] = None
|
|
5101
|
+
"""Full name of the table."""
|
|
5102
|
+
|
|
5103
|
+
time_series: Optional[MonitorTimeSeries] = None
|
|
5025
5104
|
"""Configuration for monitoring time series tables."""
|
|
5026
5105
|
|
|
5027
5106
|
def as_dict(self) -> dict:
|
|
@@ -5031,13 +5110,13 @@ class UpdateMonitor:
|
|
|
5031
5110
|
if self.custom_metrics: body['custom_metrics'] = [v.as_dict() for v in self.custom_metrics]
|
|
5032
5111
|
if self.data_classification_config:
|
|
5033
5112
|
body['data_classification_config'] = self.data_classification_config.as_dict()
|
|
5034
|
-
if self.full_name is not None: body['full_name'] = self.full_name
|
|
5035
5113
|
if self.inference_log: body['inference_log'] = self.inference_log.as_dict()
|
|
5036
5114
|
if self.notifications: body['notifications'] = self.notifications.as_dict()
|
|
5037
5115
|
if self.output_schema_name is not None: body['output_schema_name'] = self.output_schema_name
|
|
5038
5116
|
if self.schedule: body['schedule'] = self.schedule.as_dict()
|
|
5039
5117
|
if self.slicing_exprs: body['slicing_exprs'] = [v for v in self.slicing_exprs]
|
|
5040
5118
|
if self.snapshot: body['snapshot'] = self.snapshot.as_dict()
|
|
5119
|
+
if self.table_name is not None: body['table_name'] = self.table_name
|
|
5041
5120
|
if self.time_series: body['time_series'] = self.time_series.as_dict()
|
|
5042
5121
|
return body
|
|
5043
5122
|
|
|
@@ -5045,17 +5124,17 @@ class UpdateMonitor:
|
|
|
5045
5124
|
def from_dict(cls, d: Dict[str, any]) -> UpdateMonitor:
|
|
5046
5125
|
"""Deserializes the UpdateMonitor from a dictionary."""
|
|
5047
5126
|
return cls(baseline_table_name=d.get('baseline_table_name', None),
|
|
5048
|
-
custom_metrics=_repeated_dict(d, 'custom_metrics',
|
|
5127
|
+
custom_metrics=_repeated_dict(d, 'custom_metrics', MonitorMetric),
|
|
5049
5128
|
data_classification_config=_from_dict(d, 'data_classification_config',
|
|
5050
5129
|
MonitorDataClassificationConfig),
|
|
5051
|
-
|
|
5052
|
-
|
|
5053
|
-
notifications=_from_dict(d, 'notifications', MonitorNotificationsConfig),
|
|
5130
|
+
inference_log=_from_dict(d, 'inference_log', MonitorInferenceLog),
|
|
5131
|
+
notifications=_from_dict(d, 'notifications', MonitorNotifications),
|
|
5054
5132
|
output_schema_name=d.get('output_schema_name', None),
|
|
5055
5133
|
schedule=_from_dict(d, 'schedule', MonitorCronSchedule),
|
|
5056
5134
|
slicing_exprs=d.get('slicing_exprs', None),
|
|
5057
|
-
snapshot=_from_dict(d, 'snapshot',
|
|
5058
|
-
|
|
5135
|
+
snapshot=_from_dict(d, 'snapshot', MonitorSnapshot),
|
|
5136
|
+
table_name=d.get('table_name', None),
|
|
5137
|
+
time_series=_from_dict(d, 'time_series', MonitorTimeSeries))
|
|
5059
5138
|
|
|
5060
5139
|
|
|
5061
5140
|
@dataclass
|
|
@@ -5180,7 +5259,7 @@ class UpdateStorageCredential:
|
|
|
5180
5259
|
aws_iam_role: Optional[AwsIamRoleRequest] = None
|
|
5181
5260
|
"""The AWS IAM role configuration."""
|
|
5182
5261
|
|
|
5183
|
-
azure_managed_identity: Optional[
|
|
5262
|
+
azure_managed_identity: Optional[AzureManagedIdentityResponse] = None
|
|
5184
5263
|
"""The Azure managed identity configuration."""
|
|
5185
5264
|
|
|
5186
5265
|
azure_service_principal: Optional[AzureServicePrincipal] = None
|
|
@@ -5236,7 +5315,8 @@ class UpdateStorageCredential:
|
|
|
5236
5315
|
def from_dict(cls, d: Dict[str, any]) -> UpdateStorageCredential:
|
|
5237
5316
|
"""Deserializes the UpdateStorageCredential from a dictionary."""
|
|
5238
5317
|
return cls(aws_iam_role=_from_dict(d, 'aws_iam_role', AwsIamRoleRequest),
|
|
5239
|
-
azure_managed_identity=_from_dict(d, 'azure_managed_identity',
|
|
5318
|
+
azure_managed_identity=_from_dict(d, 'azure_managed_identity',
|
|
5319
|
+
AzureManagedIdentityResponse),
|
|
5240
5320
|
azure_service_principal=_from_dict(d, 'azure_service_principal', AzureServicePrincipal),
|
|
5241
5321
|
cloudflare_api_token=_from_dict(d, 'cloudflare_api_token', CloudflareApiToken),
|
|
5242
5322
|
comment=d.get('comment', None),
|
|
@@ -5346,7 +5426,7 @@ class ValidateStorageCredential:
|
|
|
5346
5426
|
aws_iam_role: Optional[AwsIamRoleRequest] = None
|
|
5347
5427
|
"""The AWS IAM role configuration."""
|
|
5348
5428
|
|
|
5349
|
-
azure_managed_identity: Optional[
|
|
5429
|
+
azure_managed_identity: Optional[AzureManagedIdentityRequest] = None
|
|
5350
5430
|
"""The Azure managed identity configuration."""
|
|
5351
5431
|
|
|
5352
5432
|
azure_service_principal: Optional[AzureServicePrincipal] = None
|
|
@@ -5392,7 +5472,8 @@ class ValidateStorageCredential:
|
|
|
5392
5472
|
def from_dict(cls, d: Dict[str, any]) -> ValidateStorageCredential:
|
|
5393
5473
|
"""Deserializes the ValidateStorageCredential from a dictionary."""
|
|
5394
5474
|
return cls(aws_iam_role=_from_dict(d, 'aws_iam_role', AwsIamRoleRequest),
|
|
5395
|
-
azure_managed_identity=_from_dict(d, 'azure_managed_identity',
|
|
5475
|
+
azure_managed_identity=_from_dict(d, 'azure_managed_identity',
|
|
5476
|
+
AzureManagedIdentityRequest),
|
|
5396
5477
|
azure_service_principal=_from_dict(d, 'azure_service_principal', AzureServicePrincipal),
|
|
5397
5478
|
cloudflare_api_token=_from_dict(d, 'cloudflare_api_token', CloudflareApiToken),
|
|
5398
5479
|
databricks_gcp_service_account=_from_dict(d, 'databricks_gcp_service_account',
|
|
@@ -5426,36 +5507,68 @@ class ValidateStorageCredentialResponse:
|
|
|
5426
5507
|
|
|
5427
5508
|
@dataclass
|
|
5428
5509
|
class ValidationResult:
|
|
5429
|
-
|
|
5430
|
-
"""
|
|
5510
|
+
aws_operation: Optional[ValidationResultAwsOperation] = None
|
|
5511
|
+
"""The operation tested."""
|
|
5512
|
+
|
|
5513
|
+
azure_operation: Optional[ValidationResultAzureOperation] = None
|
|
5514
|
+
"""The operation tested."""
|
|
5431
5515
|
|
|
5432
|
-
|
|
5516
|
+
gcp_operation: Optional[ValidationResultGcpOperation] = None
|
|
5433
5517
|
"""The operation tested."""
|
|
5434
5518
|
|
|
5519
|
+
message: Optional[str] = None
|
|
5520
|
+
"""Error message would exist when the result does not equal to **PASS**."""
|
|
5521
|
+
|
|
5435
5522
|
result: Optional[ValidationResultResult] = None
|
|
5436
5523
|
"""The results of the tested operation."""
|
|
5437
5524
|
|
|
5438
5525
|
def as_dict(self) -> dict:
|
|
5439
5526
|
"""Serializes the ValidationResult into a dictionary suitable for use as a JSON request body."""
|
|
5440
5527
|
body = {}
|
|
5528
|
+
if self.aws_operation is not None: body['aws_operation'] = self.aws_operation.value
|
|
5529
|
+
if self.azure_operation is not None: body['azure_operation'] = self.azure_operation.value
|
|
5530
|
+
if self.gcp_operation is not None: body['gcp_operation'] = self.gcp_operation.value
|
|
5441
5531
|
if self.message is not None: body['message'] = self.message
|
|
5442
|
-
if self.operation is not None: body['operation'] = self.operation.value
|
|
5443
5532
|
if self.result is not None: body['result'] = self.result.value
|
|
5444
5533
|
return body
|
|
5445
5534
|
|
|
5446
5535
|
@classmethod
|
|
5447
5536
|
def from_dict(cls, d: Dict[str, any]) -> ValidationResult:
|
|
5448
5537
|
"""Deserializes the ValidationResult from a dictionary."""
|
|
5449
|
-
return cls(
|
|
5450
|
-
|
|
5538
|
+
return cls(aws_operation=_enum(d, 'aws_operation', ValidationResultAwsOperation),
|
|
5539
|
+
azure_operation=_enum(d, 'azure_operation', ValidationResultAzureOperation),
|
|
5540
|
+
gcp_operation=_enum(d, 'gcp_operation', ValidationResultGcpOperation),
|
|
5541
|
+
message=d.get('message', None),
|
|
5451
5542
|
result=_enum(d, 'result', ValidationResultResult))
|
|
5452
5543
|
|
|
5453
5544
|
|
|
5454
|
-
class
|
|
5545
|
+
class ValidationResultAwsOperation(Enum):
|
|
5546
|
+
"""The operation tested."""
|
|
5547
|
+
|
|
5548
|
+
DELETE = 'DELETE'
|
|
5549
|
+
LIST = 'LIST'
|
|
5550
|
+
PATH_EXISTS = 'PATH_EXISTS'
|
|
5551
|
+
READ = 'READ'
|
|
5552
|
+
WRITE = 'WRITE'
|
|
5553
|
+
|
|
5554
|
+
|
|
5555
|
+
class ValidationResultAzureOperation(Enum):
|
|
5455
5556
|
"""The operation tested."""
|
|
5456
5557
|
|
|
5457
5558
|
DELETE = 'DELETE'
|
|
5559
|
+
HIERARCHICAL_NAMESPACE_ENABLED = 'HIERARCHICAL_NAMESPACE_ENABLED'
|
|
5458
5560
|
LIST = 'LIST'
|
|
5561
|
+
PATH_EXISTS = 'PATH_EXISTS'
|
|
5562
|
+
READ = 'READ'
|
|
5563
|
+
WRITE = 'WRITE'
|
|
5564
|
+
|
|
5565
|
+
|
|
5566
|
+
class ValidationResultGcpOperation(Enum):
|
|
5567
|
+
"""The operation tested."""
|
|
5568
|
+
|
|
5569
|
+
DELETE = 'DELETE'
|
|
5570
|
+
LIST = 'LIST'
|
|
5571
|
+
PATH_EXISTS = 'PATH_EXISTS'
|
|
5459
5572
|
READ = 'READ'
|
|
5460
5573
|
WRITE = 'WRITE'
|
|
5461
5574
|
|
|
@@ -6818,7 +6931,7 @@ class LakehouseMonitorsAPI:
|
|
|
6818
6931
|
def __init__(self, api_client):
|
|
6819
6932
|
self._api = api_client
|
|
6820
6933
|
|
|
6821
|
-
def cancel_refresh(self,
|
|
6934
|
+
def cancel_refresh(self, table_name: str, refresh_id: str):
|
|
6822
6935
|
"""Cancel refresh.
|
|
6823
6936
|
|
|
6824
6937
|
Cancel an active monitor refresh for the given refresh ID.
|
|
@@ -6830,7 +6943,7 @@ class LakehouseMonitorsAPI:
|
|
|
6830
6943
|
|
|
6831
6944
|
Additionally, the call must be made from the workspace where the monitor was created.
|
|
6832
6945
|
|
|
6833
|
-
:param
|
|
6946
|
+
:param table_name: str
|
|
6834
6947
|
Full name of the table.
|
|
6835
6948
|
:param refresh_id: str
|
|
6836
6949
|
ID of the refresh.
|
|
@@ -6841,24 +6954,24 @@ class LakehouseMonitorsAPI:
|
|
|
6841
6954
|
headers = {}
|
|
6842
6955
|
|
|
6843
6956
|
self._api.do('POST',
|
|
6844
|
-
f'/api/2.1/unity-catalog/tables/{
|
|
6957
|
+
f'/api/2.1/unity-catalog/tables/{table_name}/monitor/refreshes/{refresh_id}/cancel',
|
|
6845
6958
|
headers=headers)
|
|
6846
6959
|
|
|
6847
6960
|
def create(self,
|
|
6848
|
-
|
|
6961
|
+
table_name: str,
|
|
6849
6962
|
assets_dir: str,
|
|
6850
6963
|
output_schema_name: str,
|
|
6851
6964
|
*,
|
|
6852
6965
|
baseline_table_name: Optional[str] = None,
|
|
6853
|
-
custom_metrics: Optional[List[
|
|
6966
|
+
custom_metrics: Optional[List[MonitorMetric]] = None,
|
|
6854
6967
|
data_classification_config: Optional[MonitorDataClassificationConfig] = None,
|
|
6855
|
-
inference_log: Optional[
|
|
6856
|
-
notifications: Optional[
|
|
6968
|
+
inference_log: Optional[MonitorInferenceLog] = None,
|
|
6969
|
+
notifications: Optional[MonitorNotifications] = None,
|
|
6857
6970
|
schedule: Optional[MonitorCronSchedule] = None,
|
|
6858
6971
|
skip_builtin_dashboard: Optional[bool] = None,
|
|
6859
6972
|
slicing_exprs: Optional[List[str]] = None,
|
|
6860
|
-
snapshot: Optional[
|
|
6861
|
-
time_series: Optional[
|
|
6973
|
+
snapshot: Optional[MonitorSnapshot] = None,
|
|
6974
|
+
time_series: Optional[MonitorTimeSeries] = None,
|
|
6862
6975
|
warehouse_id: Optional[str] = None) -> MonitorInfo:
|
|
6863
6976
|
"""Create a table monitor.
|
|
6864
6977
|
|
|
@@ -6872,7 +6985,7 @@ class LakehouseMonitorsAPI:
|
|
|
6872
6985
|
|
|
6873
6986
|
Workspace assets, such as the dashboard, will be created in the workspace where this call was made.
|
|
6874
6987
|
|
|
6875
|
-
:param
|
|
6988
|
+
:param table_name: str
|
|
6876
6989
|
Full name of the table.
|
|
6877
6990
|
:param assets_dir: str
|
|
6878
6991
|
The directory to store monitoring assets (e.g. dashboard, metric tables).
|
|
@@ -6881,14 +6994,14 @@ class LakehouseMonitorsAPI:
|
|
|
6881
6994
|
:param baseline_table_name: str (optional)
|
|
6882
6995
|
Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
6883
6996
|
table should also be present in the baseline table.
|
|
6884
|
-
:param custom_metrics: List[:class:`
|
|
6997
|
+
:param custom_metrics: List[:class:`MonitorMetric`] (optional)
|
|
6885
6998
|
Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics
|
|
6886
6999
|
(from already computed aggregate metrics), or drift metrics (comparing metrics across time windows).
|
|
6887
7000
|
:param data_classification_config: :class:`MonitorDataClassificationConfig` (optional)
|
|
6888
7001
|
The data classification config for the monitor.
|
|
6889
|
-
:param inference_log: :class:`
|
|
7002
|
+
:param inference_log: :class:`MonitorInferenceLog` (optional)
|
|
6890
7003
|
Configuration for monitoring inference logs.
|
|
6891
|
-
:param notifications: :class:`
|
|
7004
|
+
:param notifications: :class:`MonitorNotifications` (optional)
|
|
6892
7005
|
The notification settings for the monitor.
|
|
6893
7006
|
:param schedule: :class:`MonitorCronSchedule` (optional)
|
|
6894
7007
|
The schedule for automatically updating and refreshing metric tables.
|
|
@@ -6898,9 +7011,9 @@ class LakehouseMonitorsAPI:
|
|
|
6898
7011
|
List of column expressions to slice data with for targeted analysis. The data is grouped by each
|
|
6899
7012
|
expression independently, resulting in a separate slice for each predicate and its complements. For
|
|
6900
7013
|
high-cardinality columns, only the top 100 unique values by frequency will generate slices.
|
|
6901
|
-
:param snapshot: :class:`
|
|
7014
|
+
:param snapshot: :class:`MonitorSnapshot` (optional)
|
|
6902
7015
|
Configuration for monitoring snapshot tables.
|
|
6903
|
-
:param time_series: :class:`
|
|
7016
|
+
:param time_series: :class:`MonitorTimeSeries` (optional)
|
|
6904
7017
|
Configuration for monitoring time series tables.
|
|
6905
7018
|
:param warehouse_id: str (optional)
|
|
6906
7019
|
Optional argument to specify the warehouse for dashboard creation. If not specified, the first
|
|
@@ -6926,12 +7039,12 @@ class LakehouseMonitorsAPI:
|
|
|
6926
7039
|
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
|
|
6927
7040
|
|
|
6928
7041
|
res = self._api.do('POST',
|
|
6929
|
-
f'/api/2.1/unity-catalog/tables/{
|
|
7042
|
+
f'/api/2.1/unity-catalog/tables/{table_name}/monitor',
|
|
6930
7043
|
body=body,
|
|
6931
7044
|
headers=headers)
|
|
6932
7045
|
return MonitorInfo.from_dict(res)
|
|
6933
7046
|
|
|
6934
|
-
def delete(self,
|
|
7047
|
+
def delete(self, table_name: str):
|
|
6935
7048
|
"""Delete a table monitor.
|
|
6936
7049
|
|
|
6937
7050
|
Deletes a monitor for the specified table.
|
|
@@ -6946,7 +7059,7 @@ class LakehouseMonitorsAPI:
|
|
|
6946
7059
|
Note that the metric tables and dashboard will not be deleted as part of this call; those assets must
|
|
6947
7060
|
be manually cleaned up (if desired).
|
|
6948
7061
|
|
|
6949
|
-
:param
|
|
7062
|
+
:param table_name: str
|
|
6950
7063
|
Full name of the table.
|
|
6951
7064
|
|
|
6952
7065
|
|
|
@@ -6954,9 +7067,9 @@ class LakehouseMonitorsAPI:
|
|
|
6954
7067
|
|
|
6955
7068
|
headers = {}
|
|
6956
7069
|
|
|
6957
|
-
self._api.do('DELETE', f'/api/2.1/unity-catalog/tables/{
|
|
7070
|
+
self._api.do('DELETE', f'/api/2.1/unity-catalog/tables/{table_name}/monitor', headers=headers)
|
|
6958
7071
|
|
|
6959
|
-
def get(self,
|
|
7072
|
+
def get(self, table_name: str) -> MonitorInfo:
|
|
6960
7073
|
"""Get a table monitor.
|
|
6961
7074
|
|
|
6962
7075
|
Gets a monitor for the specified table.
|
|
@@ -6970,7 +7083,7 @@ class LakehouseMonitorsAPI:
|
|
|
6970
7083
|
the monitor. Some information (e.g., dashboard) may be filtered out if the caller is in a different
|
|
6971
7084
|
workspace than where the monitor was created.
|
|
6972
7085
|
|
|
6973
|
-
:param
|
|
7086
|
+
:param table_name: str
|
|
6974
7087
|
Full name of the table.
|
|
6975
7088
|
|
|
6976
7089
|
:returns: :class:`MonitorInfo`
|
|
@@ -6978,10 +7091,10 @@ class LakehouseMonitorsAPI:
|
|
|
6978
7091
|
|
|
6979
7092
|
headers = {'Accept': 'application/json', }
|
|
6980
7093
|
|
|
6981
|
-
res = self._api.do('GET', f'/api/2.1/unity-catalog/tables/{
|
|
7094
|
+
res = self._api.do('GET', f'/api/2.1/unity-catalog/tables/{table_name}/monitor', headers=headers)
|
|
6982
7095
|
return MonitorInfo.from_dict(res)
|
|
6983
7096
|
|
|
6984
|
-
def get_refresh(self,
|
|
7097
|
+
def get_refresh(self, table_name: str, refresh_id: str) -> MonitorRefreshInfo:
|
|
6985
7098
|
"""Get refresh.
|
|
6986
7099
|
|
|
6987
7100
|
Gets info about a specific monitor refresh using the given refresh ID.
|
|
@@ -6993,7 +7106,7 @@ class LakehouseMonitorsAPI:
|
|
|
6993
7106
|
|
|
6994
7107
|
Additionally, the call must be made from the workspace where the monitor was created.
|
|
6995
7108
|
|
|
6996
|
-
:param
|
|
7109
|
+
:param table_name: str
|
|
6997
7110
|
Full name of the table.
|
|
6998
7111
|
:param refresh_id: str
|
|
6999
7112
|
ID of the refresh.
|
|
@@ -7004,11 +7117,11 @@ class LakehouseMonitorsAPI:
|
|
|
7004
7117
|
headers = {'Accept': 'application/json', }
|
|
7005
7118
|
|
|
7006
7119
|
res = self._api.do('GET',
|
|
7007
|
-
f'/api/2.1/unity-catalog/tables/{
|
|
7120
|
+
f'/api/2.1/unity-catalog/tables/{table_name}/monitor/refreshes/{refresh_id}',
|
|
7008
7121
|
headers=headers)
|
|
7009
7122
|
return MonitorRefreshInfo.from_dict(res)
|
|
7010
7123
|
|
|
7011
|
-
def list_refreshes(self,
|
|
7124
|
+
def list_refreshes(self, table_name: str) -> Iterator[MonitorRefreshInfo]:
|
|
7012
7125
|
"""List refreshes.
|
|
7013
7126
|
|
|
7014
7127
|
Gets an array containing the history of the most recent refreshes (up to 25) for this table.
|
|
@@ -7020,7 +7133,7 @@ class LakehouseMonitorsAPI:
|
|
|
7020
7133
|
|
|
7021
7134
|
Additionally, the call must be made from the workspace where the monitor was created.
|
|
7022
7135
|
|
|
7023
|
-
:param
|
|
7136
|
+
:param table_name: str
|
|
7024
7137
|
Full name of the table.
|
|
7025
7138
|
|
|
7026
7139
|
:returns: Iterator over :class:`MonitorRefreshInfo`
|
|
@@ -7029,11 +7142,11 @@ class LakehouseMonitorsAPI:
|
|
|
7029
7142
|
headers = {'Accept': 'application/json', }
|
|
7030
7143
|
|
|
7031
7144
|
res = self._api.do('GET',
|
|
7032
|
-
f'/api/2.1/unity-catalog/tables/{
|
|
7145
|
+
f'/api/2.1/unity-catalog/tables/{table_name}/monitor/refreshes',
|
|
7033
7146
|
headers=headers)
|
|
7034
7147
|
return [MonitorRefreshInfo.from_dict(v) for v in res]
|
|
7035
7148
|
|
|
7036
|
-
def run_refresh(self,
|
|
7149
|
+
def run_refresh(self, table_name: str) -> MonitorRefreshInfo:
|
|
7037
7150
|
"""Queue a metric refresh for a monitor.
|
|
7038
7151
|
|
|
7039
7152
|
Queues a metric refresh on the monitor for the specified table. The refresh will execute in the
|
|
@@ -7046,7 +7159,7 @@ class LakehouseMonitorsAPI:
|
|
|
7046
7159
|
|
|
7047
7160
|
Additionally, the call must be made from the workspace where the monitor was created.
|
|
7048
7161
|
|
|
7049
|
-
:param
|
|
7162
|
+
:param table_name: str
|
|
7050
7163
|
Full name of the table.
|
|
7051
7164
|
|
|
7052
7165
|
:returns: :class:`MonitorRefreshInfo`
|
|
@@ -7055,23 +7168,23 @@ class LakehouseMonitorsAPI:
|
|
|
7055
7168
|
headers = {'Accept': 'application/json', }
|
|
7056
7169
|
|
|
7057
7170
|
res = self._api.do('POST',
|
|
7058
|
-
f'/api/2.1/unity-catalog/tables/{
|
|
7171
|
+
f'/api/2.1/unity-catalog/tables/{table_name}/monitor/refreshes',
|
|
7059
7172
|
headers=headers)
|
|
7060
7173
|
return MonitorRefreshInfo.from_dict(res)
|
|
7061
7174
|
|
|
7062
7175
|
def update(self,
|
|
7063
|
-
|
|
7176
|
+
table_name: str,
|
|
7064
7177
|
output_schema_name: str,
|
|
7065
7178
|
*,
|
|
7066
7179
|
baseline_table_name: Optional[str] = None,
|
|
7067
|
-
custom_metrics: Optional[List[
|
|
7180
|
+
custom_metrics: Optional[List[MonitorMetric]] = None,
|
|
7068
7181
|
data_classification_config: Optional[MonitorDataClassificationConfig] = None,
|
|
7069
|
-
inference_log: Optional[
|
|
7070
|
-
notifications: Optional[
|
|
7182
|
+
inference_log: Optional[MonitorInferenceLog] = None,
|
|
7183
|
+
notifications: Optional[MonitorNotifications] = None,
|
|
7071
7184
|
schedule: Optional[MonitorCronSchedule] = None,
|
|
7072
7185
|
slicing_exprs: Optional[List[str]] = None,
|
|
7073
|
-
snapshot: Optional[
|
|
7074
|
-
time_series: Optional[
|
|
7186
|
+
snapshot: Optional[MonitorSnapshot] = None,
|
|
7187
|
+
time_series: Optional[MonitorTimeSeries] = None) -> MonitorInfo:
|
|
7075
7188
|
"""Update a table monitor.
|
|
7076
7189
|
|
|
7077
7190
|
Updates a monitor for the specified table.
|
|
@@ -7086,21 +7199,21 @@ class LakehouseMonitorsAPI:
|
|
|
7086
7199
|
|
|
7087
7200
|
Certain configuration fields, such as output asset identifiers, cannot be updated.
|
|
7088
7201
|
|
|
7089
|
-
:param
|
|
7202
|
+
:param table_name: str
|
|
7090
7203
|
Full name of the table.
|
|
7091
7204
|
:param output_schema_name: str
|
|
7092
7205
|
Schema where output metric tables are created.
|
|
7093
7206
|
:param baseline_table_name: str (optional)
|
|
7094
7207
|
Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
7095
7208
|
table should also be present in the baseline table.
|
|
7096
|
-
:param custom_metrics: List[:class:`
|
|
7209
|
+
:param custom_metrics: List[:class:`MonitorMetric`] (optional)
|
|
7097
7210
|
Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics
|
|
7098
7211
|
(from already computed aggregate metrics), or drift metrics (comparing metrics across time windows).
|
|
7099
7212
|
:param data_classification_config: :class:`MonitorDataClassificationConfig` (optional)
|
|
7100
7213
|
The data classification config for the monitor.
|
|
7101
|
-
:param inference_log: :class:`
|
|
7214
|
+
:param inference_log: :class:`MonitorInferenceLog` (optional)
|
|
7102
7215
|
Configuration for monitoring inference logs.
|
|
7103
|
-
:param notifications: :class:`
|
|
7216
|
+
:param notifications: :class:`MonitorNotifications` (optional)
|
|
7104
7217
|
The notification settings for the monitor.
|
|
7105
7218
|
:param schedule: :class:`MonitorCronSchedule` (optional)
|
|
7106
7219
|
The schedule for automatically updating and refreshing metric tables.
|
|
@@ -7108,9 +7221,9 @@ class LakehouseMonitorsAPI:
|
|
|
7108
7221
|
List of column expressions to slice data with for targeted analysis. The data is grouped by each
|
|
7109
7222
|
expression independently, resulting in a separate slice for each predicate and its complements. For
|
|
7110
7223
|
high-cardinality columns, only the top 100 unique values by frequency will generate slices.
|
|
7111
|
-
:param snapshot: :class:`
|
|
7224
|
+
:param snapshot: :class:`MonitorSnapshot` (optional)
|
|
7112
7225
|
Configuration for monitoring snapshot tables.
|
|
7113
|
-
:param time_series: :class:`
|
|
7226
|
+
:param time_series: :class:`MonitorTimeSeries` (optional)
|
|
7114
7227
|
Configuration for monitoring time series tables.
|
|
7115
7228
|
|
|
7116
7229
|
:returns: :class:`MonitorInfo`
|
|
@@ -7130,7 +7243,7 @@ class LakehouseMonitorsAPI:
|
|
|
7130
7243
|
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
|
|
7131
7244
|
|
|
7132
7245
|
res = self._api.do('PUT',
|
|
7133
|
-
f'/api/2.1/unity-catalog/tables/{
|
|
7246
|
+
f'/api/2.1/unity-catalog/tables/{table_name}/monitor',
|
|
7134
7247
|
body=body,
|
|
7135
7248
|
headers=headers)
|
|
7136
7249
|
return MonitorInfo.from_dict(res)
|
|
@@ -7795,9 +7908,19 @@ class RegisteredModelsAPI:
|
|
|
7795
7908
|
Whether to include registered models in the response for which the principal can only access
|
|
7796
7909
|
selective metadata for
|
|
7797
7910
|
:param max_results: int (optional)
|
|
7798
|
-
Max number of registered models to return.
|
|
7799
|
-
|
|
7800
|
-
|
|
7911
|
+
Max number of registered models to return.
|
|
7912
|
+
|
|
7913
|
+
If both catalog and schema are specified: - when max_results is not specified, the page length is
|
|
7914
|
+
set to a server configured value (10000, as of 4/2/2024). - when set to a value greater than 0, the
|
|
7915
|
+
page length is the minimum of this value and a server configured value (10000, as of 4/2/2024); -
|
|
7916
|
+
when set to 0, the page length is set to a server configured value (10000, as of 4/2/2024); - when
|
|
7917
|
+
set to a value less than 0, an invalid parameter error is returned;
|
|
7918
|
+
|
|
7919
|
+
If neither schema nor catalog is specified: - when max_results is not specified, the page length is
|
|
7920
|
+
set to a server configured value (100, as of 4/2/2024). - when set to a value greater than 0, the
|
|
7921
|
+
page length is the minimum of this value and a server configured value (1000, as of 4/2/2024); -
|
|
7922
|
+
when set to 0, the page length is set to a server configured value (100, as of 4/2/2024); - when set
|
|
7923
|
+
to a value less than 0, an invalid parameter error is returned;
|
|
7801
7924
|
:param page_token: str (optional)
|
|
7802
7925
|
Opaque token to send for the next page of results (pagination).
|
|
7803
7926
|
:param schema_name: str (optional)
|
|
@@ -8079,7 +8202,7 @@ class StorageCredentialsAPI:
|
|
|
8079
8202
|
name: str,
|
|
8080
8203
|
*,
|
|
8081
8204
|
aws_iam_role: Optional[AwsIamRoleRequest] = None,
|
|
8082
|
-
azure_managed_identity: Optional[
|
|
8205
|
+
azure_managed_identity: Optional[AzureManagedIdentityRequest] = None,
|
|
8083
8206
|
azure_service_principal: Optional[AzureServicePrincipal] = None,
|
|
8084
8207
|
cloudflare_api_token: Optional[CloudflareApiToken] = None,
|
|
8085
8208
|
comment: Optional[str] = None,
|
|
@@ -8094,7 +8217,7 @@ class StorageCredentialsAPI:
|
|
|
8094
8217
|
The credential name. The name must be unique within the metastore.
|
|
8095
8218
|
:param aws_iam_role: :class:`AwsIamRoleRequest` (optional)
|
|
8096
8219
|
The AWS IAM role configuration.
|
|
8097
|
-
:param azure_managed_identity: :class:`
|
|
8220
|
+
:param azure_managed_identity: :class:`AzureManagedIdentityRequest` (optional)
|
|
8098
8221
|
The Azure managed identity configuration.
|
|
8099
8222
|
:param azure_service_principal: :class:`AzureServicePrincipal` (optional)
|
|
8100
8223
|
The Azure service principal configuration.
|
|
@@ -8213,7 +8336,7 @@ class StorageCredentialsAPI:
|
|
|
8213
8336
|
name: str,
|
|
8214
8337
|
*,
|
|
8215
8338
|
aws_iam_role: Optional[AwsIamRoleRequest] = None,
|
|
8216
|
-
azure_managed_identity: Optional[
|
|
8339
|
+
azure_managed_identity: Optional[AzureManagedIdentityResponse] = None,
|
|
8217
8340
|
azure_service_principal: Optional[AzureServicePrincipal] = None,
|
|
8218
8341
|
cloudflare_api_token: Optional[CloudflareApiToken] = None,
|
|
8219
8342
|
comment: Optional[str] = None,
|
|
@@ -8231,7 +8354,7 @@ class StorageCredentialsAPI:
|
|
|
8231
8354
|
Name of the storage credential.
|
|
8232
8355
|
:param aws_iam_role: :class:`AwsIamRoleRequest` (optional)
|
|
8233
8356
|
The AWS IAM role configuration.
|
|
8234
|
-
:param azure_managed_identity: :class:`
|
|
8357
|
+
:param azure_managed_identity: :class:`AzureManagedIdentityResponse` (optional)
|
|
8235
8358
|
The Azure managed identity configuration.
|
|
8236
8359
|
:param azure_service_principal: :class:`AzureServicePrincipal` (optional)
|
|
8237
8360
|
The Azure service principal configuration.
|
|
@@ -8280,7 +8403,7 @@ class StorageCredentialsAPI:
|
|
|
8280
8403
|
def validate(self,
|
|
8281
8404
|
*,
|
|
8282
8405
|
aws_iam_role: Optional[AwsIamRoleRequest] = None,
|
|
8283
|
-
azure_managed_identity: Optional[
|
|
8406
|
+
azure_managed_identity: Optional[AzureManagedIdentityRequest] = None,
|
|
8284
8407
|
azure_service_principal: Optional[AzureServicePrincipal] = None,
|
|
8285
8408
|
cloudflare_api_token: Optional[CloudflareApiToken] = None,
|
|
8286
8409
|
databricks_gcp_service_account: Optional[DatabricksGcpServiceAccountRequest] = None,
|
|
@@ -8302,7 +8425,7 @@ class StorageCredentialsAPI:
|
|
|
8302
8425
|
|
|
8303
8426
|
:param aws_iam_role: :class:`AwsIamRoleRequest` (optional)
|
|
8304
8427
|
The AWS IAM role configuration.
|
|
8305
|
-
:param azure_managed_identity: :class:`
|
|
8428
|
+
:param azure_managed_identity: :class:`AzureManagedIdentityRequest` (optional)
|
|
8306
8429
|
The Azure managed identity configuration.
|
|
8307
8430
|
:param azure_service_principal: :class:`AzureServicePrincipal` (optional)
|
|
8308
8431
|
The Azure service principal configuration.
|