databricks-sdk 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of databricks-sdk might be problematic. Click here for more details.
- databricks/sdk/__init__.py +41 -5
- databricks/sdk/azure.py +17 -7
- databricks/sdk/clock.py +49 -0
- databricks/sdk/config.py +459 -0
- databricks/sdk/core.py +7 -1026
- databricks/sdk/credentials_provider.py +628 -0
- databricks/sdk/environments.py +72 -0
- databricks/sdk/errors/__init__.py +1 -1
- databricks/sdk/errors/mapper.py +5 -5
- databricks/sdk/mixins/workspace.py +3 -3
- databricks/sdk/oauth.py +2 -1
- databricks/sdk/retries.py +9 -5
- databricks/sdk/service/_internal.py +1 -1
- databricks/sdk/service/catalog.py +946 -82
- databricks/sdk/service/compute.py +106 -41
- databricks/sdk/service/files.py +145 -31
- databricks/sdk/service/iam.py +44 -40
- databricks/sdk/service/jobs.py +199 -20
- databricks/sdk/service/ml.py +33 -42
- databricks/sdk/service/oauth2.py +3 -4
- databricks/sdk/service/pipelines.py +51 -31
- databricks/sdk/service/serving.py +1 -2
- databricks/sdk/service/settings.py +377 -72
- databricks/sdk/service/sharing.py +3 -4
- databricks/sdk/service/sql.py +27 -19
- databricks/sdk/service/vectorsearch.py +13 -17
- databricks/sdk/service/workspace.py +20 -11
- databricks/sdk/version.py +1 -1
- {databricks_sdk-0.17.0.dist-info → databricks_sdk-0.19.0.dist-info}/METADATA +4 -4
- databricks_sdk-0.19.0.dist-info/RECORD +53 -0
- databricks_sdk-0.17.0.dist-info/RECORD +0 -49
- /databricks/sdk/errors/{mapping.py → platform.py} +0 -0
- {databricks_sdk-0.17.0.dist-info → databricks_sdk-0.19.0.dist-info}/LICENSE +0 -0
- {databricks_sdk-0.17.0.dist-info → databricks_sdk-0.19.0.dist-info}/NOTICE +0 -0
- {databricks_sdk-0.17.0.dist-info → databricks_sdk-0.19.0.dist-info}/WHEEL +0 -0
- {databricks_sdk-0.17.0.dist-info → databricks_sdk-0.19.0.dist-info}/top_level.txt +0 -0
|
@@ -784,6 +784,7 @@ class ConnectionInfoSecurableKind(Enum):
|
|
|
784
784
|
class ConnectionType(Enum):
|
|
785
785
|
"""The type of connection."""
|
|
786
786
|
|
|
787
|
+
BIGQUERY = 'BIGQUERY'
|
|
787
788
|
DATABRICKS = 'DATABRICKS'
|
|
788
789
|
MYSQL = 'MYSQL'
|
|
789
790
|
POSTGRESQL = 'POSTGRESQL'
|
|
@@ -1162,6 +1163,97 @@ class CreateMetastoreAssignment:
|
|
|
1162
1163
|
workspace_id=d.get('workspace_id', None))
|
|
1163
1164
|
|
|
1164
1165
|
|
|
1166
|
+
@dataclass
|
|
1167
|
+
class CreateMonitor:
|
|
1168
|
+
assets_dir: str
|
|
1169
|
+
"""The directory to store monitoring assets (e.g. dashboard, metric tables)."""
|
|
1170
|
+
|
|
1171
|
+
output_schema_name: str
|
|
1172
|
+
"""Schema where output metric tables are created."""
|
|
1173
|
+
|
|
1174
|
+
baseline_table_name: Optional[str] = None
|
|
1175
|
+
"""Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
1176
|
+
table should also be present in the baseline table."""
|
|
1177
|
+
|
|
1178
|
+
custom_metrics: Optional[List[MonitorCustomMetric]] = None
|
|
1179
|
+
"""Custom metrics to compute on the monitored table. These can be aggregate metrics, derived
|
|
1180
|
+
metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across
|
|
1181
|
+
time windows)."""
|
|
1182
|
+
|
|
1183
|
+
data_classification_config: Optional[MonitorDataClassificationConfig] = None
|
|
1184
|
+
"""The data classification config for the monitor."""
|
|
1185
|
+
|
|
1186
|
+
full_name: Optional[str] = None
|
|
1187
|
+
"""Full name of the table."""
|
|
1188
|
+
|
|
1189
|
+
inference_log: Optional[MonitorInferenceLogProfileType] = None
|
|
1190
|
+
"""Configuration for monitoring inference logs."""
|
|
1191
|
+
|
|
1192
|
+
notifications: Optional[List[MonitorNotificationsConfig]] = None
|
|
1193
|
+
"""The notification settings for the monitor."""
|
|
1194
|
+
|
|
1195
|
+
schedule: Optional[MonitorCronSchedule] = None
|
|
1196
|
+
"""The schedule for automatically updating and refreshing metric tables."""
|
|
1197
|
+
|
|
1198
|
+
skip_builtin_dashboard: Optional[bool] = None
|
|
1199
|
+
"""Whether to skip creating a default dashboard summarizing data quality metrics."""
|
|
1200
|
+
|
|
1201
|
+
slicing_exprs: Optional[List[str]] = None
|
|
1202
|
+
"""List of column expressions to slice data with for targeted analysis. The data is grouped by each
|
|
1203
|
+
expression independently, resulting in a separate slice for each predicate and its complements.
|
|
1204
|
+
For high-cardinality columns, only the top 100 unique values by frequency will generate slices."""
|
|
1205
|
+
|
|
1206
|
+
snapshot: Optional[Any] = None
|
|
1207
|
+
"""Configuration for monitoring snapshot tables."""
|
|
1208
|
+
|
|
1209
|
+
time_series: Optional[MonitorTimeSeriesProfileType] = None
|
|
1210
|
+
"""Configuration for monitoring time series tables."""
|
|
1211
|
+
|
|
1212
|
+
warehouse_id: Optional[str] = None
|
|
1213
|
+
"""Optional argument to specify the warehouse for dashboard creation. If not specified, the first
|
|
1214
|
+
running warehouse will be used."""
|
|
1215
|
+
|
|
1216
|
+
def as_dict(self) -> dict:
|
|
1217
|
+
"""Serializes the CreateMonitor into a dictionary suitable for use as a JSON request body."""
|
|
1218
|
+
body = {}
|
|
1219
|
+
if self.assets_dir is not None: body['assets_dir'] = self.assets_dir
|
|
1220
|
+
if self.baseline_table_name is not None: body['baseline_table_name'] = self.baseline_table_name
|
|
1221
|
+
if self.custom_metrics: body['custom_metrics'] = [v.as_dict() for v in self.custom_metrics]
|
|
1222
|
+
if self.data_classification_config:
|
|
1223
|
+
body['data_classification_config'] = self.data_classification_config.as_dict()
|
|
1224
|
+
if self.full_name is not None: body['full_name'] = self.full_name
|
|
1225
|
+
if self.inference_log: body['inference_log'] = self.inference_log.as_dict()
|
|
1226
|
+
if self.notifications: body['notifications'] = [v.as_dict() for v in self.notifications]
|
|
1227
|
+
if self.output_schema_name is not None: body['output_schema_name'] = self.output_schema_name
|
|
1228
|
+
if self.schedule: body['schedule'] = self.schedule.as_dict()
|
|
1229
|
+
if self.skip_builtin_dashboard is not None:
|
|
1230
|
+
body['skip_builtin_dashboard'] = self.skip_builtin_dashboard
|
|
1231
|
+
if self.slicing_exprs: body['slicing_exprs'] = [v for v in self.slicing_exprs]
|
|
1232
|
+
if self.snapshot: body['snapshot'] = self.snapshot
|
|
1233
|
+
if self.time_series: body['time_series'] = self.time_series.as_dict()
|
|
1234
|
+
if self.warehouse_id is not None: body['warehouse_id'] = self.warehouse_id
|
|
1235
|
+
return body
|
|
1236
|
+
|
|
1237
|
+
@classmethod
|
|
1238
|
+
def from_dict(cls, d: Dict[str, any]) -> CreateMonitor:
|
|
1239
|
+
"""Deserializes the CreateMonitor from a dictionary."""
|
|
1240
|
+
return cls(assets_dir=d.get('assets_dir', None),
|
|
1241
|
+
baseline_table_name=d.get('baseline_table_name', None),
|
|
1242
|
+
custom_metrics=_repeated_dict(d, 'custom_metrics', MonitorCustomMetric),
|
|
1243
|
+
data_classification_config=_from_dict(d, 'data_classification_config',
|
|
1244
|
+
MonitorDataClassificationConfig),
|
|
1245
|
+
full_name=d.get('full_name', None),
|
|
1246
|
+
inference_log=_from_dict(d, 'inference_log', MonitorInferenceLogProfileType),
|
|
1247
|
+
notifications=_repeated_dict(d, 'notifications', MonitorNotificationsConfig),
|
|
1248
|
+
output_schema_name=d.get('output_schema_name', None),
|
|
1249
|
+
schedule=_from_dict(d, 'schedule', MonitorCronSchedule),
|
|
1250
|
+
skip_builtin_dashboard=d.get('skip_builtin_dashboard', None),
|
|
1251
|
+
slicing_exprs=d.get('slicing_exprs', None),
|
|
1252
|
+
snapshot=d.get('snapshot', None),
|
|
1253
|
+
time_series=_from_dict(d, 'time_series', MonitorTimeSeriesProfileType),
|
|
1254
|
+
warehouse_id=d.get('warehouse_id', None))
|
|
1255
|
+
|
|
1256
|
+
|
|
1165
1257
|
@dataclass
|
|
1166
1258
|
class CreateRegisteredModelRequest:
|
|
1167
1259
|
catalog_name: str
|
|
@@ -2731,6 +2823,389 @@ class ModelVersionInfoStatus(Enum):
|
|
|
2731
2823
|
READY = 'READY'
|
|
2732
2824
|
|
|
2733
2825
|
|
|
2826
|
+
@dataclass
|
|
2827
|
+
class MonitorCronSchedule:
|
|
2828
|
+
pause_status: Optional[MonitorCronSchedulePauseStatus] = None
|
|
2829
|
+
"""Whether the schedule is paused or not"""
|
|
2830
|
+
|
|
2831
|
+
quartz_cron_expression: Optional[str] = None
|
|
2832
|
+
"""A cron expression using quartz syntax that describes the schedule for a job."""
|
|
2833
|
+
|
|
2834
|
+
timezone_id: Optional[str] = None
|
|
2835
|
+
"""A Java timezone id. The schedule for a job will be resolved with respect to this timezone."""
|
|
2836
|
+
|
|
2837
|
+
def as_dict(self) -> dict:
|
|
2838
|
+
"""Serializes the MonitorCronSchedule into a dictionary suitable for use as a JSON request body."""
|
|
2839
|
+
body = {}
|
|
2840
|
+
if self.pause_status is not None: body['pause_status'] = self.pause_status.value
|
|
2841
|
+
if self.quartz_cron_expression is not None:
|
|
2842
|
+
body['quartz_cron_expression'] = self.quartz_cron_expression
|
|
2843
|
+
if self.timezone_id is not None: body['timezone_id'] = self.timezone_id
|
|
2844
|
+
return body
|
|
2845
|
+
|
|
2846
|
+
@classmethod
|
|
2847
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorCronSchedule:
|
|
2848
|
+
"""Deserializes the MonitorCronSchedule from a dictionary."""
|
|
2849
|
+
return cls(pause_status=_enum(d, 'pause_status', MonitorCronSchedulePauseStatus),
|
|
2850
|
+
quartz_cron_expression=d.get('quartz_cron_expression', None),
|
|
2851
|
+
timezone_id=d.get('timezone_id', None))
|
|
2852
|
+
|
|
2853
|
+
|
|
2854
|
+
class MonitorCronSchedulePauseStatus(Enum):
|
|
2855
|
+
"""Whether the schedule is paused or not"""
|
|
2856
|
+
|
|
2857
|
+
PAUSED = 'PAUSED'
|
|
2858
|
+
UNPAUSED = 'UNPAUSED'
|
|
2859
|
+
|
|
2860
|
+
|
|
2861
|
+
@dataclass
|
|
2862
|
+
class MonitorCustomMetric:
|
|
2863
|
+
definition: Optional[str] = None
|
|
2864
|
+
"""Jinja template for a SQL expression that specifies how to compute the metric. See [create metric
|
|
2865
|
+
definition].
|
|
2866
|
+
|
|
2867
|
+
[create metric definition]: https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition"""
|
|
2868
|
+
|
|
2869
|
+
input_columns: Optional[List[str]] = None
|
|
2870
|
+
"""Columns on the monitored table to apply the custom metrics to."""
|
|
2871
|
+
|
|
2872
|
+
name: Optional[str] = None
|
|
2873
|
+
"""Name of the custom metric."""
|
|
2874
|
+
|
|
2875
|
+
output_data_type: Optional[str] = None
|
|
2876
|
+
"""The output type of the custom metric."""
|
|
2877
|
+
|
|
2878
|
+
type: Optional[MonitorCustomMetricType] = None
|
|
2879
|
+
"""The type of the custom metric."""
|
|
2880
|
+
|
|
2881
|
+
def as_dict(self) -> dict:
|
|
2882
|
+
"""Serializes the MonitorCustomMetric into a dictionary suitable for use as a JSON request body."""
|
|
2883
|
+
body = {}
|
|
2884
|
+
if self.definition is not None: body['definition'] = self.definition
|
|
2885
|
+
if self.input_columns: body['input_columns'] = [v for v in self.input_columns]
|
|
2886
|
+
if self.name is not None: body['name'] = self.name
|
|
2887
|
+
if self.output_data_type is not None: body['output_data_type'] = self.output_data_type
|
|
2888
|
+
if self.type is not None: body['type'] = self.type.value
|
|
2889
|
+
return body
|
|
2890
|
+
|
|
2891
|
+
@classmethod
|
|
2892
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorCustomMetric:
|
|
2893
|
+
"""Deserializes the MonitorCustomMetric from a dictionary."""
|
|
2894
|
+
return cls(definition=d.get('definition', None),
|
|
2895
|
+
input_columns=d.get('input_columns', None),
|
|
2896
|
+
name=d.get('name', None),
|
|
2897
|
+
output_data_type=d.get('output_data_type', None),
|
|
2898
|
+
type=_enum(d, 'type', MonitorCustomMetricType))
|
|
2899
|
+
|
|
2900
|
+
|
|
2901
|
+
class MonitorCustomMetricType(Enum):
|
|
2902
|
+
"""The type of the custom metric."""
|
|
2903
|
+
|
|
2904
|
+
CUSTOM_METRIC_TYPE_AGGREGATE = 'CUSTOM_METRIC_TYPE_AGGREGATE'
|
|
2905
|
+
CUSTOM_METRIC_TYPE_DERIVED = 'CUSTOM_METRIC_TYPE_DERIVED'
|
|
2906
|
+
CUSTOM_METRIC_TYPE_DRIFT = 'CUSTOM_METRIC_TYPE_DRIFT'
|
|
2907
|
+
MONITOR_STATUS_ERROR = 'MONITOR_STATUS_ERROR'
|
|
2908
|
+
MONITOR_STATUS_FAILED = 'MONITOR_STATUS_FAILED'
|
|
2909
|
+
|
|
2910
|
+
|
|
2911
|
+
@dataclass
|
|
2912
|
+
class MonitorDataClassificationConfig:
|
|
2913
|
+
enabled: Optional[bool] = None
|
|
2914
|
+
"""Whether data classification is enabled."""
|
|
2915
|
+
|
|
2916
|
+
def as_dict(self) -> dict:
|
|
2917
|
+
"""Serializes the MonitorDataClassificationConfig into a dictionary suitable for use as a JSON request body."""
|
|
2918
|
+
body = {}
|
|
2919
|
+
if self.enabled is not None: body['enabled'] = self.enabled
|
|
2920
|
+
return body
|
|
2921
|
+
|
|
2922
|
+
@classmethod
|
|
2923
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorDataClassificationConfig:
|
|
2924
|
+
"""Deserializes the MonitorDataClassificationConfig from a dictionary."""
|
|
2925
|
+
return cls(enabled=d.get('enabled', None))
|
|
2926
|
+
|
|
2927
|
+
|
|
2928
|
+
@dataclass
|
|
2929
|
+
class MonitorDestinations:
|
|
2930
|
+
email_addresses: Optional[List[str]] = None
|
|
2931
|
+
"""The list of email addresses to send the notification to."""
|
|
2932
|
+
|
|
2933
|
+
def as_dict(self) -> dict:
|
|
2934
|
+
"""Serializes the MonitorDestinations into a dictionary suitable for use as a JSON request body."""
|
|
2935
|
+
body = {}
|
|
2936
|
+
if self.email_addresses: body['email_addresses'] = [v for v in self.email_addresses]
|
|
2937
|
+
return body
|
|
2938
|
+
|
|
2939
|
+
@classmethod
|
|
2940
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorDestinations:
|
|
2941
|
+
"""Deserializes the MonitorDestinations from a dictionary."""
|
|
2942
|
+
return cls(email_addresses=d.get('email_addresses', None))
|
|
2943
|
+
|
|
2944
|
+
|
|
2945
|
+
@dataclass
|
|
2946
|
+
class MonitorInferenceLogProfileType:
|
|
2947
|
+
granularities: Optional[List[str]] = None
|
|
2948
|
+
"""List of granularities to use when aggregating data into time windows based on their timestamp."""
|
|
2949
|
+
|
|
2950
|
+
label_col: Optional[str] = None
|
|
2951
|
+
"""Column of the model label."""
|
|
2952
|
+
|
|
2953
|
+
model_id_col: Optional[str] = None
|
|
2954
|
+
"""Column of the model id or version."""
|
|
2955
|
+
|
|
2956
|
+
prediction_col: Optional[str] = None
|
|
2957
|
+
"""Column of the model prediction."""
|
|
2958
|
+
|
|
2959
|
+
prediction_proba_col: Optional[str] = None
|
|
2960
|
+
"""Column of the model prediction probabilities."""
|
|
2961
|
+
|
|
2962
|
+
problem_type: Optional[MonitorInferenceLogProfileTypeProblemType] = None
|
|
2963
|
+
"""Problem type the model aims to solve."""
|
|
2964
|
+
|
|
2965
|
+
timestamp_col: Optional[str] = None
|
|
2966
|
+
"""Column of the timestamp of predictions."""
|
|
2967
|
+
|
|
2968
|
+
def as_dict(self) -> dict:
|
|
2969
|
+
"""Serializes the MonitorInferenceLogProfileType into a dictionary suitable for use as a JSON request body."""
|
|
2970
|
+
body = {}
|
|
2971
|
+
if self.granularities: body['granularities'] = [v for v in self.granularities]
|
|
2972
|
+
if self.label_col is not None: body['label_col'] = self.label_col
|
|
2973
|
+
if self.model_id_col is not None: body['model_id_col'] = self.model_id_col
|
|
2974
|
+
if self.prediction_col is not None: body['prediction_col'] = self.prediction_col
|
|
2975
|
+
if self.prediction_proba_col is not None: body['prediction_proba_col'] = self.prediction_proba_col
|
|
2976
|
+
if self.problem_type is not None: body['problem_type'] = self.problem_type.value
|
|
2977
|
+
if self.timestamp_col is not None: body['timestamp_col'] = self.timestamp_col
|
|
2978
|
+
return body
|
|
2979
|
+
|
|
2980
|
+
@classmethod
|
|
2981
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorInferenceLogProfileType:
|
|
2982
|
+
"""Deserializes the MonitorInferenceLogProfileType from a dictionary."""
|
|
2983
|
+
return cls(granularities=d.get('granularities', None),
|
|
2984
|
+
label_col=d.get('label_col', None),
|
|
2985
|
+
model_id_col=d.get('model_id_col', None),
|
|
2986
|
+
prediction_col=d.get('prediction_col', None),
|
|
2987
|
+
prediction_proba_col=d.get('prediction_proba_col', None),
|
|
2988
|
+
problem_type=_enum(d, 'problem_type', MonitorInferenceLogProfileTypeProblemType),
|
|
2989
|
+
timestamp_col=d.get('timestamp_col', None))
|
|
2990
|
+
|
|
2991
|
+
|
|
2992
|
+
class MonitorInferenceLogProfileTypeProblemType(Enum):
|
|
2993
|
+
"""Problem type the model aims to solve."""
|
|
2994
|
+
|
|
2995
|
+
PROBLEM_TYPE_CLASSIFICATION = 'PROBLEM_TYPE_CLASSIFICATION'
|
|
2996
|
+
PROBLEM_TYPE_REGRESSION = 'PROBLEM_TYPE_REGRESSION'
|
|
2997
|
+
|
|
2998
|
+
|
|
2999
|
+
@dataclass
|
|
3000
|
+
class MonitorInfo:
|
|
3001
|
+
assets_dir: Optional[str] = None
|
|
3002
|
+
"""The directory to store monitoring assets (e.g. dashboard, metric tables)."""
|
|
3003
|
+
|
|
3004
|
+
baseline_table_name: Optional[str] = None
|
|
3005
|
+
"""Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
3006
|
+
table should also be present in the baseline table."""
|
|
3007
|
+
|
|
3008
|
+
custom_metrics: Optional[List[MonitorCustomMetric]] = None
|
|
3009
|
+
"""Custom metrics to compute on the monitored table. These can be aggregate metrics, derived
|
|
3010
|
+
metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across
|
|
3011
|
+
time windows)."""
|
|
3012
|
+
|
|
3013
|
+
dashboard_id: Optional[str] = None
|
|
3014
|
+
"""The ID of the generated dashboard."""
|
|
3015
|
+
|
|
3016
|
+
data_classification_config: Optional[MonitorDataClassificationConfig] = None
|
|
3017
|
+
"""The data classification config for the monitor."""
|
|
3018
|
+
|
|
3019
|
+
drift_metrics_table_name: Optional[str] = None
|
|
3020
|
+
"""The full name of the drift metrics table. Format:
|
|
3021
|
+
__catalog_name__.__schema_name__.__table_name__."""
|
|
3022
|
+
|
|
3023
|
+
inference_log: Optional[MonitorInferenceLogProfileType] = None
|
|
3024
|
+
"""Configuration for monitoring inference logs."""
|
|
3025
|
+
|
|
3026
|
+
latest_monitor_failure_msg: Optional[str] = None
|
|
3027
|
+
"""The latest failure message of the monitor (if any)."""
|
|
3028
|
+
|
|
3029
|
+
monitor_version: Optional[str] = None
|
|
3030
|
+
"""The version of the monitor config (e.g. 1,2,3). If negative, the monitor may be corrupted."""
|
|
3031
|
+
|
|
3032
|
+
notifications: Optional[List[MonitorNotificationsConfig]] = None
|
|
3033
|
+
"""The notification settings for the monitor."""
|
|
3034
|
+
|
|
3035
|
+
output_schema_name: Optional[str] = None
|
|
3036
|
+
"""Schema where output metric tables are created."""
|
|
3037
|
+
|
|
3038
|
+
profile_metrics_table_name: Optional[str] = None
|
|
3039
|
+
"""The full name of the profile metrics table. Format:
|
|
3040
|
+
__catalog_name__.__schema_name__.__table_name__."""
|
|
3041
|
+
|
|
3042
|
+
schedule: Optional[MonitorCronSchedule] = None
|
|
3043
|
+
"""The schedule for automatically updating and refreshing metric tables."""
|
|
3044
|
+
|
|
3045
|
+
slicing_exprs: Optional[List[str]] = None
|
|
3046
|
+
"""List of column expressions to slice data with for targeted analysis. The data is grouped by each
|
|
3047
|
+
expression independently, resulting in a separate slice for each predicate and its complements.
|
|
3048
|
+
For high-cardinality columns, only the top 100 unique values by frequency will generate slices."""
|
|
3049
|
+
|
|
3050
|
+
snapshot: Optional[Any] = None
|
|
3051
|
+
"""Configuration for monitoring snapshot tables."""
|
|
3052
|
+
|
|
3053
|
+
status: Optional[MonitorInfoStatus] = None
|
|
3054
|
+
"""The status of the monitor."""
|
|
3055
|
+
|
|
3056
|
+
table_name: Optional[str] = None
|
|
3057
|
+
"""The full name of the table to monitor. Format: __catalog_name__.__schema_name__.__table_name__."""
|
|
3058
|
+
|
|
3059
|
+
time_series: Optional[MonitorTimeSeriesProfileType] = None
|
|
3060
|
+
"""Configuration for monitoring time series tables."""
|
|
3061
|
+
|
|
3062
|
+
def as_dict(self) -> dict:
|
|
3063
|
+
"""Serializes the MonitorInfo into a dictionary suitable for use as a JSON request body."""
|
|
3064
|
+
body = {}
|
|
3065
|
+
if self.assets_dir is not None: body['assets_dir'] = self.assets_dir
|
|
3066
|
+
if self.baseline_table_name is not None: body['baseline_table_name'] = self.baseline_table_name
|
|
3067
|
+
if self.custom_metrics: body['custom_metrics'] = [v.as_dict() for v in self.custom_metrics]
|
|
3068
|
+
if self.dashboard_id is not None: body['dashboard_id'] = self.dashboard_id
|
|
3069
|
+
if self.data_classification_config:
|
|
3070
|
+
body['data_classification_config'] = self.data_classification_config.as_dict()
|
|
3071
|
+
if self.drift_metrics_table_name is not None:
|
|
3072
|
+
body['drift_metrics_table_name'] = self.drift_metrics_table_name
|
|
3073
|
+
if self.inference_log: body['inference_log'] = self.inference_log.as_dict()
|
|
3074
|
+
if self.latest_monitor_failure_msg is not None:
|
|
3075
|
+
body['latest_monitor_failure_msg'] = self.latest_monitor_failure_msg
|
|
3076
|
+
if self.monitor_version is not None: body['monitor_version'] = self.monitor_version
|
|
3077
|
+
if self.notifications: body['notifications'] = [v.as_dict() for v in self.notifications]
|
|
3078
|
+
if self.output_schema_name is not None: body['output_schema_name'] = self.output_schema_name
|
|
3079
|
+
if self.profile_metrics_table_name is not None:
|
|
3080
|
+
body['profile_metrics_table_name'] = self.profile_metrics_table_name
|
|
3081
|
+
if self.schedule: body['schedule'] = self.schedule.as_dict()
|
|
3082
|
+
if self.slicing_exprs: body['slicing_exprs'] = [v for v in self.slicing_exprs]
|
|
3083
|
+
if self.snapshot: body['snapshot'] = self.snapshot
|
|
3084
|
+
if self.status is not None: body['status'] = self.status.value
|
|
3085
|
+
if self.table_name is not None: body['table_name'] = self.table_name
|
|
3086
|
+
if self.time_series: body['time_series'] = self.time_series.as_dict()
|
|
3087
|
+
return body
|
|
3088
|
+
|
|
3089
|
+
@classmethod
|
|
3090
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorInfo:
|
|
3091
|
+
"""Deserializes the MonitorInfo from a dictionary."""
|
|
3092
|
+
return cls(assets_dir=d.get('assets_dir', None),
|
|
3093
|
+
baseline_table_name=d.get('baseline_table_name', None),
|
|
3094
|
+
custom_metrics=_repeated_dict(d, 'custom_metrics', MonitorCustomMetric),
|
|
3095
|
+
dashboard_id=d.get('dashboard_id', None),
|
|
3096
|
+
data_classification_config=_from_dict(d, 'data_classification_config',
|
|
3097
|
+
MonitorDataClassificationConfig),
|
|
3098
|
+
drift_metrics_table_name=d.get('drift_metrics_table_name', None),
|
|
3099
|
+
inference_log=_from_dict(d, 'inference_log', MonitorInferenceLogProfileType),
|
|
3100
|
+
latest_monitor_failure_msg=d.get('latest_monitor_failure_msg', None),
|
|
3101
|
+
monitor_version=d.get('monitor_version', None),
|
|
3102
|
+
notifications=_repeated_dict(d, 'notifications', MonitorNotificationsConfig),
|
|
3103
|
+
output_schema_name=d.get('output_schema_name', None),
|
|
3104
|
+
profile_metrics_table_name=d.get('profile_metrics_table_name', None),
|
|
3105
|
+
schedule=_from_dict(d, 'schedule', MonitorCronSchedule),
|
|
3106
|
+
slicing_exprs=d.get('slicing_exprs', None),
|
|
3107
|
+
snapshot=d.get('snapshot', None),
|
|
3108
|
+
status=_enum(d, 'status', MonitorInfoStatus),
|
|
3109
|
+
table_name=d.get('table_name', None),
|
|
3110
|
+
time_series=_from_dict(d, 'time_series', MonitorTimeSeriesProfileType))
|
|
3111
|
+
|
|
3112
|
+
|
|
3113
|
+
class MonitorInfoStatus(Enum):
|
|
3114
|
+
"""The status of the monitor."""
|
|
3115
|
+
|
|
3116
|
+
MONITOR_STATUS_ACTIVE = 'MONITOR_STATUS_ACTIVE'
|
|
3117
|
+
MONITOR_STATUS_DELETE_PENDING = 'MONITOR_STATUS_DELETE_PENDING'
|
|
3118
|
+
MONITOR_STATUS_ERROR = 'MONITOR_STATUS_ERROR'
|
|
3119
|
+
MONITOR_STATUS_FAILED = 'MONITOR_STATUS_FAILED'
|
|
3120
|
+
MONITOR_STATUS_PENDING = 'MONITOR_STATUS_PENDING'
|
|
3121
|
+
|
|
3122
|
+
|
|
3123
|
+
@dataclass
|
|
3124
|
+
class MonitorNotificationsConfig:
|
|
3125
|
+
on_failure: Optional[MonitorDestinations] = None
|
|
3126
|
+
"""Who to send notifications to on monitor failure."""
|
|
3127
|
+
|
|
3128
|
+
def as_dict(self) -> dict:
|
|
3129
|
+
"""Serializes the MonitorNotificationsConfig into a dictionary suitable for use as a JSON request body."""
|
|
3130
|
+
body = {}
|
|
3131
|
+
if self.on_failure: body['on_failure'] = self.on_failure.as_dict()
|
|
3132
|
+
return body
|
|
3133
|
+
|
|
3134
|
+
@classmethod
|
|
3135
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorNotificationsConfig:
|
|
3136
|
+
"""Deserializes the MonitorNotificationsConfig from a dictionary."""
|
|
3137
|
+
return cls(on_failure=_from_dict(d, 'on_failure', MonitorDestinations))
|
|
3138
|
+
|
|
3139
|
+
|
|
3140
|
+
@dataclass
|
|
3141
|
+
class MonitorRefreshInfo:
|
|
3142
|
+
end_time_ms: Optional[int] = None
|
|
3143
|
+
"""The time at which the refresh ended, in epoch milliseconds."""
|
|
3144
|
+
|
|
3145
|
+
message: Optional[str] = None
|
|
3146
|
+
"""An optional message to give insight into the current state of the job (e.g. FAILURE messages)."""
|
|
3147
|
+
|
|
3148
|
+
refresh_id: Optional[int] = None
|
|
3149
|
+
"""The ID of the refresh."""
|
|
3150
|
+
|
|
3151
|
+
start_time_ms: Optional[int] = None
|
|
3152
|
+
"""The time at which the refresh started, in epoch milliseconds."""
|
|
3153
|
+
|
|
3154
|
+
state: Optional[MonitorRefreshInfoState] = None
|
|
3155
|
+
"""The current state of the refresh."""
|
|
3156
|
+
|
|
3157
|
+
def as_dict(self) -> dict:
|
|
3158
|
+
"""Serializes the MonitorRefreshInfo into a dictionary suitable for use as a JSON request body."""
|
|
3159
|
+
body = {}
|
|
3160
|
+
if self.end_time_ms is not None: body['end_time_ms'] = self.end_time_ms
|
|
3161
|
+
if self.message is not None: body['message'] = self.message
|
|
3162
|
+
if self.refresh_id is not None: body['refresh_id'] = self.refresh_id
|
|
3163
|
+
if self.start_time_ms is not None: body['start_time_ms'] = self.start_time_ms
|
|
3164
|
+
if self.state is not None: body['state'] = self.state.value
|
|
3165
|
+
return body
|
|
3166
|
+
|
|
3167
|
+
@classmethod
|
|
3168
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorRefreshInfo:
|
|
3169
|
+
"""Deserializes the MonitorRefreshInfo from a dictionary."""
|
|
3170
|
+
return cls(end_time_ms=d.get('end_time_ms', None),
|
|
3171
|
+
message=d.get('message', None),
|
|
3172
|
+
refresh_id=d.get('refresh_id', None),
|
|
3173
|
+
start_time_ms=d.get('start_time_ms', None),
|
|
3174
|
+
state=_enum(d, 'state', MonitorRefreshInfoState))
|
|
3175
|
+
|
|
3176
|
+
|
|
3177
|
+
class MonitorRefreshInfoState(Enum):
|
|
3178
|
+
"""The current state of the refresh."""
|
|
3179
|
+
|
|
3180
|
+
CANCELED = 'CANCELED'
|
|
3181
|
+
FAILED = 'FAILED'
|
|
3182
|
+
PENDING = 'PENDING'
|
|
3183
|
+
RUNNING = 'RUNNING'
|
|
3184
|
+
SUCCESS = 'SUCCESS'
|
|
3185
|
+
|
|
3186
|
+
|
|
3187
|
+
@dataclass
|
|
3188
|
+
class MonitorTimeSeriesProfileType:
|
|
3189
|
+
granularities: Optional[List[str]] = None
|
|
3190
|
+
"""List of granularities to use when aggregating data into time windows based on their timestamp."""
|
|
3191
|
+
|
|
3192
|
+
timestamp_col: Optional[str] = None
|
|
3193
|
+
"""The timestamp column. This must be timestamp types or convertible to timestamp types using the
|
|
3194
|
+
pyspark to_timestamp function."""
|
|
3195
|
+
|
|
3196
|
+
def as_dict(self) -> dict:
|
|
3197
|
+
"""Serializes the MonitorTimeSeriesProfileType into a dictionary suitable for use as a JSON request body."""
|
|
3198
|
+
body = {}
|
|
3199
|
+
if self.granularities: body['granularities'] = [v for v in self.granularities]
|
|
3200
|
+
if self.timestamp_col is not None: body['timestamp_col'] = self.timestamp_col
|
|
3201
|
+
return body
|
|
3202
|
+
|
|
3203
|
+
@classmethod
|
|
3204
|
+
def from_dict(cls, d: Dict[str, any]) -> MonitorTimeSeriesProfileType:
|
|
3205
|
+
"""Deserializes the MonitorTimeSeriesProfileType from a dictionary."""
|
|
3206
|
+
return cls(granularities=d.get('granularities', None), timestamp_col=d.get('timestamp_col', None))
|
|
3207
|
+
|
|
3208
|
+
|
|
2734
3209
|
@dataclass
|
|
2735
3210
|
class NamedTableConstraint:
|
|
2736
3211
|
name: str
|
|
@@ -3386,6 +3861,23 @@ class TableDependency:
|
|
|
3386
3861
|
return cls(table_full_name=d.get('table_full_name', None))
|
|
3387
3862
|
|
|
3388
3863
|
|
|
3864
|
+
@dataclass
|
|
3865
|
+
class TableExistsResponse:
|
|
3866
|
+
table_exists: Optional[bool] = None
|
|
3867
|
+
"""Whether the table exists or not."""
|
|
3868
|
+
|
|
3869
|
+
def as_dict(self) -> dict:
|
|
3870
|
+
"""Serializes the TableExistsResponse into a dictionary suitable for use as a JSON request body."""
|
|
3871
|
+
body = {}
|
|
3872
|
+
if self.table_exists is not None: body['table_exists'] = self.table_exists
|
|
3873
|
+
return body
|
|
3874
|
+
|
|
3875
|
+
@classmethod
|
|
3876
|
+
def from_dict(cls, d: Dict[str, any]) -> TableExistsResponse:
|
|
3877
|
+
"""Deserializes the TableExistsResponse from a dictionary."""
|
|
3878
|
+
return cls(table_exists=d.get('table_exists', None))
|
|
3879
|
+
|
|
3880
|
+
|
|
3389
3881
|
@dataclass
|
|
3390
3882
|
class TableInfo:
|
|
3391
3883
|
access_point: Optional[str] = None
|
|
@@ -3671,9 +4163,6 @@ class UpdateConnection:
|
|
|
3671
4163
|
options: Dict[str, str]
|
|
3672
4164
|
"""A map of key-value properties attached to the securable."""
|
|
3673
4165
|
|
|
3674
|
-
name: Optional[str] = None
|
|
3675
|
-
"""Name of the connection."""
|
|
3676
|
-
|
|
3677
4166
|
name_arg: Optional[str] = None
|
|
3678
4167
|
"""Name of the connection."""
|
|
3679
4168
|
|
|
@@ -3686,7 +4175,6 @@ class UpdateConnection:
|
|
|
3686
4175
|
def as_dict(self) -> dict:
|
|
3687
4176
|
"""Serializes the UpdateConnection into a dictionary suitable for use as a JSON request body."""
|
|
3688
4177
|
body = {}
|
|
3689
|
-
if self.name is not None: body['name'] = self.name
|
|
3690
4178
|
if self.name_arg is not None: body['name_arg'] = self.name_arg
|
|
3691
4179
|
if self.new_name is not None: body['new_name'] = self.new_name
|
|
3692
4180
|
if self.options: body['options'] = self.options
|
|
@@ -3696,8 +4184,7 @@ class UpdateConnection:
|
|
|
3696
4184
|
@classmethod
|
|
3697
4185
|
def from_dict(cls, d: Dict[str, any]) -> UpdateConnection:
|
|
3698
4186
|
"""Deserializes the UpdateConnection from a dictionary."""
|
|
3699
|
-
return cls(
|
|
3700
|
-
name_arg=d.get('name_arg', None),
|
|
4187
|
+
return cls(name_arg=d.get('name_arg', None),
|
|
3701
4188
|
new_name=d.get('new_name', None),
|
|
3702
4189
|
options=d.get('options', None),
|
|
3703
4190
|
owner=d.get('owner', None))
|
|
@@ -3807,9 +4294,6 @@ class UpdateMetastore:
|
|
|
3807
4294
|
id: Optional[str] = None
|
|
3808
4295
|
"""Unique ID of the metastore."""
|
|
3809
4296
|
|
|
3810
|
-
name: Optional[str] = None
|
|
3811
|
-
"""The user-specified name of the metastore."""
|
|
3812
|
-
|
|
3813
4297
|
new_name: Optional[str] = None
|
|
3814
4298
|
"""New name for the metastore."""
|
|
3815
4299
|
|
|
@@ -3832,7 +4316,6 @@ class UpdateMetastore:
|
|
|
3832
4316
|
'delta_sharing_recipient_token_lifetime_in_seconds'] = self.delta_sharing_recipient_token_lifetime_in_seconds
|
|
3833
4317
|
if self.delta_sharing_scope is not None: body['delta_sharing_scope'] = self.delta_sharing_scope.value
|
|
3834
4318
|
if self.id is not None: body['id'] = self.id
|
|
3835
|
-
if self.name is not None: body['name'] = self.name
|
|
3836
4319
|
if self.new_name is not None: body['new_name'] = self.new_name
|
|
3837
4320
|
if self.owner is not None: body['owner'] = self.owner
|
|
3838
4321
|
if self.privilege_model_version is not None:
|
|
@@ -3849,7 +4332,6 @@ class UpdateMetastore:
|
|
|
3849
4332
|
'delta_sharing_recipient_token_lifetime_in_seconds', None),
|
|
3850
4333
|
delta_sharing_scope=_enum(d, 'delta_sharing_scope', UpdateMetastoreDeltaSharingScope),
|
|
3851
4334
|
id=d.get('id', None),
|
|
3852
|
-
name=d.get('name', None),
|
|
3853
4335
|
new_name=d.get('new_name', None),
|
|
3854
4336
|
owner=d.get('owner', None),
|
|
3855
4337
|
privilege_model_version=d.get('privilege_model_version', None),
|
|
@@ -3917,6 +4399,85 @@ class UpdateModelVersionRequest:
|
|
|
3917
4399
|
version=d.get('version', None))
|
|
3918
4400
|
|
|
3919
4401
|
|
|
4402
|
+
@dataclass
|
|
4403
|
+
class UpdateMonitor:
|
|
4404
|
+
assets_dir: str
|
|
4405
|
+
"""The directory to store monitoring assets (e.g. dashboard, metric tables)."""
|
|
4406
|
+
|
|
4407
|
+
output_schema_name: str
|
|
4408
|
+
"""Schema where output metric tables are created."""
|
|
4409
|
+
|
|
4410
|
+
baseline_table_name: Optional[str] = None
|
|
4411
|
+
"""Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
4412
|
+
table should also be present in the baseline table."""
|
|
4413
|
+
|
|
4414
|
+
custom_metrics: Optional[List[MonitorCustomMetric]] = None
|
|
4415
|
+
"""Custom metrics to compute on the monitored table. These can be aggregate metrics, derived
|
|
4416
|
+
metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across
|
|
4417
|
+
time windows)."""
|
|
4418
|
+
|
|
4419
|
+
data_classification_config: Optional[MonitorDataClassificationConfig] = None
|
|
4420
|
+
"""The data classification config for the monitor."""
|
|
4421
|
+
|
|
4422
|
+
full_name: Optional[str] = None
|
|
4423
|
+
"""Full name of the table."""
|
|
4424
|
+
|
|
4425
|
+
inference_log: Optional[MonitorInferenceLogProfileType] = None
|
|
4426
|
+
"""Configuration for monitoring inference logs."""
|
|
4427
|
+
|
|
4428
|
+
notifications: Optional[List[MonitorNotificationsConfig]] = None
|
|
4429
|
+
"""The notification settings for the monitor."""
|
|
4430
|
+
|
|
4431
|
+
schedule: Optional[MonitorCronSchedule] = None
|
|
4432
|
+
"""The schedule for automatically updating and refreshing metric tables."""
|
|
4433
|
+
|
|
4434
|
+
slicing_exprs: Optional[List[str]] = None
|
|
4435
|
+
"""List of column expressions to slice data with for targeted analysis. The data is grouped by each
|
|
4436
|
+
expression independently, resulting in a separate slice for each predicate and its complements.
|
|
4437
|
+
For high-cardinality columns, only the top 100 unique values by frequency will generate slices."""
|
|
4438
|
+
|
|
4439
|
+
snapshot: Optional[Any] = None
|
|
4440
|
+
"""Configuration for monitoring snapshot tables."""
|
|
4441
|
+
|
|
4442
|
+
time_series: Optional[MonitorTimeSeriesProfileType] = None
|
|
4443
|
+
"""Configuration for monitoring time series tables."""
|
|
4444
|
+
|
|
4445
|
+
def as_dict(self) -> dict:
|
|
4446
|
+
"""Serializes the UpdateMonitor into a dictionary suitable for use as a JSON request body."""
|
|
4447
|
+
body = {}
|
|
4448
|
+
if self.assets_dir is not None: body['assets_dir'] = self.assets_dir
|
|
4449
|
+
if self.baseline_table_name is not None: body['baseline_table_name'] = self.baseline_table_name
|
|
4450
|
+
if self.custom_metrics: body['custom_metrics'] = [v.as_dict() for v in self.custom_metrics]
|
|
4451
|
+
if self.data_classification_config:
|
|
4452
|
+
body['data_classification_config'] = self.data_classification_config.as_dict()
|
|
4453
|
+
if self.full_name is not None: body['full_name'] = self.full_name
|
|
4454
|
+
if self.inference_log: body['inference_log'] = self.inference_log.as_dict()
|
|
4455
|
+
if self.notifications: body['notifications'] = [v.as_dict() for v in self.notifications]
|
|
4456
|
+
if self.output_schema_name is not None: body['output_schema_name'] = self.output_schema_name
|
|
4457
|
+
if self.schedule: body['schedule'] = self.schedule.as_dict()
|
|
4458
|
+
if self.slicing_exprs: body['slicing_exprs'] = [v for v in self.slicing_exprs]
|
|
4459
|
+
if self.snapshot: body['snapshot'] = self.snapshot
|
|
4460
|
+
if self.time_series: body['time_series'] = self.time_series.as_dict()
|
|
4461
|
+
return body
|
|
4462
|
+
|
|
4463
|
+
@classmethod
|
|
4464
|
+
def from_dict(cls, d: Dict[str, any]) -> UpdateMonitor:
|
|
4465
|
+
"""Deserializes the UpdateMonitor from a dictionary."""
|
|
4466
|
+
return cls(assets_dir=d.get('assets_dir', None),
|
|
4467
|
+
baseline_table_name=d.get('baseline_table_name', None),
|
|
4468
|
+
custom_metrics=_repeated_dict(d, 'custom_metrics', MonitorCustomMetric),
|
|
4469
|
+
data_classification_config=_from_dict(d, 'data_classification_config',
|
|
4470
|
+
MonitorDataClassificationConfig),
|
|
4471
|
+
full_name=d.get('full_name', None),
|
|
4472
|
+
inference_log=_from_dict(d, 'inference_log', MonitorInferenceLogProfileType),
|
|
4473
|
+
notifications=_repeated_dict(d, 'notifications', MonitorNotificationsConfig),
|
|
4474
|
+
output_schema_name=d.get('output_schema_name', None),
|
|
4475
|
+
schedule=_from_dict(d, 'schedule', MonitorCronSchedule),
|
|
4476
|
+
slicing_exprs=d.get('slicing_exprs', None),
|
|
4477
|
+
snapshot=d.get('snapshot', None),
|
|
4478
|
+
time_series=_from_dict(d, 'time_series', MonitorTimeSeriesProfileType))
|
|
4479
|
+
|
|
4480
|
+
|
|
3920
4481
|
@dataclass
|
|
3921
4482
|
class UpdatePermissions:
|
|
3922
4483
|
changes: Optional[List[PermissionsChange]] = None
|
|
@@ -3952,9 +4513,6 @@ class UpdateRegisteredModelRequest:
|
|
|
3952
4513
|
full_name: Optional[str] = None
|
|
3953
4514
|
"""The three-level (fully qualified) name of the registered model"""
|
|
3954
4515
|
|
|
3955
|
-
name: Optional[str] = None
|
|
3956
|
-
"""The name of the registered model"""
|
|
3957
|
-
|
|
3958
4516
|
new_name: Optional[str] = None
|
|
3959
4517
|
"""New name for the registered model."""
|
|
3960
4518
|
|
|
@@ -3966,7 +4524,6 @@ class UpdateRegisteredModelRequest:
|
|
|
3966
4524
|
body = {}
|
|
3967
4525
|
if self.comment is not None: body['comment'] = self.comment
|
|
3968
4526
|
if self.full_name is not None: body['full_name'] = self.full_name
|
|
3969
|
-
if self.name is not None: body['name'] = self.name
|
|
3970
4527
|
if self.new_name is not None: body['new_name'] = self.new_name
|
|
3971
4528
|
if self.owner is not None: body['owner'] = self.owner
|
|
3972
4529
|
return body
|
|
@@ -3976,7 +4533,6 @@ class UpdateRegisteredModelRequest:
|
|
|
3976
4533
|
"""Deserializes the UpdateRegisteredModelRequest from a dictionary."""
|
|
3977
4534
|
return cls(comment=d.get('comment', None),
|
|
3978
4535
|
full_name=d.get('full_name', None),
|
|
3979
|
-
name=d.get('name', None),
|
|
3980
4536
|
new_name=d.get('new_name', None),
|
|
3981
4537
|
owner=d.get('owner', None))
|
|
3982
4538
|
|
|
@@ -3992,9 +4548,6 @@ class UpdateSchema:
|
|
|
3992
4548
|
full_name: Optional[str] = None
|
|
3993
4549
|
"""Full name of the schema."""
|
|
3994
4550
|
|
|
3995
|
-
name: Optional[str] = None
|
|
3996
|
-
"""Name of schema, relative to parent catalog."""
|
|
3997
|
-
|
|
3998
4551
|
new_name: Optional[str] = None
|
|
3999
4552
|
"""New name for the schema."""
|
|
4000
4553
|
|
|
@@ -4011,7 +4564,6 @@ class UpdateSchema:
|
|
|
4011
4564
|
if self.enable_predictive_optimization is not None:
|
|
4012
4565
|
body['enable_predictive_optimization'] = self.enable_predictive_optimization.value
|
|
4013
4566
|
if self.full_name is not None: body['full_name'] = self.full_name
|
|
4014
|
-
if self.name is not None: body['name'] = self.name
|
|
4015
4567
|
if self.new_name is not None: body['new_name'] = self.new_name
|
|
4016
4568
|
if self.owner is not None: body['owner'] = self.owner
|
|
4017
4569
|
if self.properties: body['properties'] = self.properties
|
|
@@ -4024,7 +4576,6 @@ class UpdateSchema:
|
|
|
4024
4576
|
enable_predictive_optimization=_enum(d, 'enable_predictive_optimization',
|
|
4025
4577
|
EnablePredictiveOptimization),
|
|
4026
4578
|
full_name=d.get('full_name', None),
|
|
4027
|
-
name=d.get('name', None),
|
|
4028
4579
|
new_name=d.get('new_name', None),
|
|
4029
4580
|
owner=d.get('owner', None),
|
|
4030
4581
|
properties=d.get('properties', None))
|
|
@@ -4112,9 +4663,6 @@ class UpdateVolumeRequestContent:
|
|
|
4112
4663
|
full_name_arg: Optional[str] = None
|
|
4113
4664
|
"""The three-level (fully qualified) name of the volume"""
|
|
4114
4665
|
|
|
4115
|
-
name: Optional[str] = None
|
|
4116
|
-
"""The name of the volume"""
|
|
4117
|
-
|
|
4118
4666
|
new_name: Optional[str] = None
|
|
4119
4667
|
"""New name for the volume."""
|
|
4120
4668
|
|
|
@@ -4126,7 +4674,6 @@ class UpdateVolumeRequestContent:
|
|
|
4126
4674
|
body = {}
|
|
4127
4675
|
if self.comment is not None: body['comment'] = self.comment
|
|
4128
4676
|
if self.full_name_arg is not None: body['full_name_arg'] = self.full_name_arg
|
|
4129
|
-
if self.name is not None: body['name'] = self.name
|
|
4130
4677
|
if self.new_name is not None: body['new_name'] = self.new_name
|
|
4131
4678
|
if self.owner is not None: body['owner'] = self.owner
|
|
4132
4679
|
return body
|
|
@@ -4136,7 +4683,6 @@ class UpdateVolumeRequestContent:
|
|
|
4136
4683
|
"""Deserializes the UpdateVolumeRequestContent from a dictionary."""
|
|
4137
4684
|
return cls(comment=d.get('comment', None),
|
|
4138
4685
|
full_name_arg=d.get('full_name_arg', None),
|
|
4139
|
-
name=d.get('name', None),
|
|
4140
4686
|
new_name=d.get('new_name', None),
|
|
4141
4687
|
owner=d.get('owner', None))
|
|
4142
4688
|
|
|
@@ -5115,7 +5661,6 @@ class ConnectionsAPI:
|
|
|
5115
5661
|
name_arg: str,
|
|
5116
5662
|
options: Dict[str, str],
|
|
5117
5663
|
*,
|
|
5118
|
-
name: Optional[str] = None,
|
|
5119
5664
|
new_name: Optional[str] = None,
|
|
5120
5665
|
owner: Optional[str] = None) -> ConnectionInfo:
|
|
5121
5666
|
"""Update a connection.
|
|
@@ -5126,8 +5671,6 @@ class ConnectionsAPI:
|
|
|
5126
5671
|
Name of the connection.
|
|
5127
5672
|
:param options: Dict[str,str]
|
|
5128
5673
|
A map of key-value properties attached to the securable.
|
|
5129
|
-
:param name: str (optional)
|
|
5130
|
-
Name of the connection.
|
|
5131
5674
|
:param new_name: str (optional)
|
|
5132
5675
|
New name for the connection.
|
|
5133
5676
|
:param owner: str (optional)
|
|
@@ -5136,7 +5679,6 @@ class ConnectionsAPI:
|
|
|
5136
5679
|
:returns: :class:`ConnectionInfo`
|
|
5137
5680
|
"""
|
|
5138
5681
|
body = {}
|
|
5139
|
-
if name is not None: body['name'] = name
|
|
5140
5682
|
if new_name is not None: body['new_name'] = new_name
|
|
5141
5683
|
if options is not None: body['options'] = options
|
|
5142
5684
|
if owner is not None: body['owner'] = owner
|
|
@@ -5281,10 +5823,9 @@ class ExternalLocationsAPI:
|
|
|
5281
5823
|
'/api/2.1/unity-catalog/external-locations',
|
|
5282
5824
|
query=query,
|
|
5283
5825
|
headers=headers)
|
|
5284
|
-
if 'external_locations'
|
|
5285
|
-
|
|
5286
|
-
|
|
5287
|
-
yield ExternalLocationInfo.from_dict(v)
|
|
5826
|
+
if 'external_locations' in json:
|
|
5827
|
+
for v in json['external_locations']:
|
|
5828
|
+
yield ExternalLocationInfo.from_dict(v)
|
|
5288
5829
|
if 'next_page_token' not in json or not json['next_page_token']:
|
|
5289
5830
|
return
|
|
5290
5831
|
query['page_token'] = json['next_page_token']
|
|
@@ -5465,10 +6006,9 @@ class FunctionsAPI:
|
|
|
5465
6006
|
|
|
5466
6007
|
while True:
|
|
5467
6008
|
json = self._api.do('GET', '/api/2.1/unity-catalog/functions', query=query, headers=headers)
|
|
5468
|
-
if 'functions'
|
|
5469
|
-
|
|
5470
|
-
|
|
5471
|
-
yield FunctionInfo.from_dict(v)
|
|
6009
|
+
if 'functions' in json:
|
|
6010
|
+
for v in json['functions']:
|
|
6011
|
+
yield FunctionInfo.from_dict(v)
|
|
5472
6012
|
if 'next_page_token' not in json or not json['next_page_token']:
|
|
5473
6013
|
return
|
|
5474
6014
|
query['page_token'] = json['next_page_token']
|
|
@@ -5597,6 +6137,331 @@ class GrantsAPI:
|
|
|
5597
6137
|
return PermissionsList.from_dict(res)
|
|
5598
6138
|
|
|
5599
6139
|
|
|
6140
|
+
class LakehouseMonitorsAPI:
|
|
6141
|
+
"""A monitor computes and monitors data or model quality metrics for a table over time. It generates metrics
|
|
6142
|
+
tables and a dashboard that you can use to monitor table health and set alerts.
|
|
6143
|
+
|
|
6144
|
+
Most write operations require the user to be the owner of the table (or its parent schema or parent
|
|
6145
|
+
catalog). Viewing the dashboard, computed metrics, or monitor configuration only requires the user to have
|
|
6146
|
+
**SELECT** privileges on the table (along with **USE_SCHEMA** and **USE_CATALOG**)."""
|
|
6147
|
+
|
|
6148
|
+
def __init__(self, api_client):
|
|
6149
|
+
self._api = api_client
|
|
6150
|
+
|
|
6151
|
+
def cancel_refresh(self, full_name: str, refresh_id: str):
|
|
6152
|
+
"""Cancel refresh.
|
|
6153
|
+
|
|
6154
|
+
Cancel an active monitor refresh for the given refresh ID.
|
|
6155
|
+
|
|
6156
|
+
The caller must either: 1. be an owner of the table's parent catalog 2. have **USE_CATALOG** on the
|
|
6157
|
+
table's parent catalog and be an owner of the table's parent schema 3. have the following permissions:
|
|
6158
|
+
- **USE_CATALOG** on the table's parent catalog - **USE_SCHEMA** on the table's parent schema - be an
|
|
6159
|
+
owner of the table
|
|
6160
|
+
|
|
6161
|
+
Additionally, the call must be made from the workspace where the monitor was created.
|
|
6162
|
+
|
|
6163
|
+
:param full_name: str
|
|
6164
|
+
Full name of the table.
|
|
6165
|
+
:param refresh_id: str
|
|
6166
|
+
ID of the refresh.
|
|
6167
|
+
|
|
6168
|
+
|
|
6169
|
+
"""
|
|
6170
|
+
|
|
6171
|
+
headers = {}
|
|
6172
|
+
self._api.do('POST',
|
|
6173
|
+
f'/api/2.1/unity-catalog/tables/{full_name}/monitor/refreshes/{refresh_id}/cancel',
|
|
6174
|
+
headers=headers)
|
|
6175
|
+
|
|
6176
|
+
def create(self,
|
|
6177
|
+
full_name: str,
|
|
6178
|
+
assets_dir: str,
|
|
6179
|
+
output_schema_name: str,
|
|
6180
|
+
*,
|
|
6181
|
+
baseline_table_name: Optional[str] = None,
|
|
6182
|
+
custom_metrics: Optional[List[MonitorCustomMetric]] = None,
|
|
6183
|
+
data_classification_config: Optional[MonitorDataClassificationConfig] = None,
|
|
6184
|
+
inference_log: Optional[MonitorInferenceLogProfileType] = None,
|
|
6185
|
+
notifications: Optional[List[MonitorNotificationsConfig]] = None,
|
|
6186
|
+
schedule: Optional[MonitorCronSchedule] = None,
|
|
6187
|
+
skip_builtin_dashboard: Optional[bool] = None,
|
|
6188
|
+
slicing_exprs: Optional[List[str]] = None,
|
|
6189
|
+
snapshot: Optional[Any] = None,
|
|
6190
|
+
time_series: Optional[MonitorTimeSeriesProfileType] = None,
|
|
6191
|
+
warehouse_id: Optional[str] = None) -> MonitorInfo:
|
|
6192
|
+
"""Create a table monitor.
|
|
6193
|
+
|
|
6194
|
+
Creates a new monitor for the specified table.
|
|
6195
|
+
|
|
6196
|
+
The caller must either: 1. be an owner of the table's parent catalog, have **USE_SCHEMA** on the
|
|
6197
|
+
table's parent schema, and have **SELECT** access on the table 2. have **USE_CATALOG** on the table's
|
|
6198
|
+
parent catalog, be an owner of the table's parent schema, and have **SELECT** access on the table. 3.
|
|
6199
|
+
have the following permissions: - **USE_CATALOG** on the table's parent catalog - **USE_SCHEMA** on
|
|
6200
|
+
the table's parent schema - be an owner of the table.
|
|
6201
|
+
|
|
6202
|
+
Workspace assets, such as the dashboard, will be created in the workspace where this call was made.
|
|
6203
|
+
|
|
6204
|
+
:param full_name: str
|
|
6205
|
+
Full name of the table.
|
|
6206
|
+
:param assets_dir: str
|
|
6207
|
+
The directory to store monitoring assets (e.g. dashboard, metric tables).
|
|
6208
|
+
:param output_schema_name: str
|
|
6209
|
+
Schema where output metric tables are created.
|
|
6210
|
+
:param baseline_table_name: str (optional)
|
|
6211
|
+
Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
6212
|
+
table should also be present in the baseline table.
|
|
6213
|
+
:param custom_metrics: List[:class:`MonitorCustomMetric`] (optional)
|
|
6214
|
+
Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics
|
|
6215
|
+
(from already computed aggregate metrics), or drift metrics (comparing metrics across time windows).
|
|
6216
|
+
:param data_classification_config: :class:`MonitorDataClassificationConfig` (optional)
|
|
6217
|
+
The data classification config for the monitor.
|
|
6218
|
+
:param inference_log: :class:`MonitorInferenceLogProfileType` (optional)
|
|
6219
|
+
Configuration for monitoring inference logs.
|
|
6220
|
+
:param notifications: List[:class:`MonitorNotificationsConfig`] (optional)
|
|
6221
|
+
The notification settings for the monitor.
|
|
6222
|
+
:param schedule: :class:`MonitorCronSchedule` (optional)
|
|
6223
|
+
The schedule for automatically updating and refreshing metric tables.
|
|
6224
|
+
:param skip_builtin_dashboard: bool (optional)
|
|
6225
|
+
Whether to skip creating a default dashboard summarizing data quality metrics.
|
|
6226
|
+
:param slicing_exprs: List[str] (optional)
|
|
6227
|
+
List of column expressions to slice data with for targeted analysis. The data is grouped by each
|
|
6228
|
+
expression independently, resulting in a separate slice for each predicate and its complements. For
|
|
6229
|
+
high-cardinality columns, only the top 100 unique values by frequency will generate slices.
|
|
6230
|
+
:param snapshot: Any (optional)
|
|
6231
|
+
Configuration for monitoring snapshot tables.
|
|
6232
|
+
:param time_series: :class:`MonitorTimeSeriesProfileType` (optional)
|
|
6233
|
+
Configuration for monitoring time series tables.
|
|
6234
|
+
:param warehouse_id: str (optional)
|
|
6235
|
+
Optional argument to specify the warehouse for dashboard creation. If not specified, the first
|
|
6236
|
+
running warehouse will be used.
|
|
6237
|
+
|
|
6238
|
+
:returns: :class:`MonitorInfo`
|
|
6239
|
+
"""
|
|
6240
|
+
body = {}
|
|
6241
|
+
if assets_dir is not None: body['assets_dir'] = assets_dir
|
|
6242
|
+
if baseline_table_name is not None: body['baseline_table_name'] = baseline_table_name
|
|
6243
|
+
if custom_metrics is not None: body['custom_metrics'] = [v.as_dict() for v in custom_metrics]
|
|
6244
|
+
if data_classification_config is not None:
|
|
6245
|
+
body['data_classification_config'] = data_classification_config.as_dict()
|
|
6246
|
+
if inference_log is not None: body['inference_log'] = inference_log.as_dict()
|
|
6247
|
+
if notifications is not None: body['notifications'] = [v.as_dict() for v in notifications]
|
|
6248
|
+
if output_schema_name is not None: body['output_schema_name'] = output_schema_name
|
|
6249
|
+
if schedule is not None: body['schedule'] = schedule.as_dict()
|
|
6250
|
+
if skip_builtin_dashboard is not None: body['skip_builtin_dashboard'] = skip_builtin_dashboard
|
|
6251
|
+
if slicing_exprs is not None: body['slicing_exprs'] = [v for v in slicing_exprs]
|
|
6252
|
+
if snapshot is not None: body['snapshot'] = snapshot
|
|
6253
|
+
if time_series is not None: body['time_series'] = time_series.as_dict()
|
|
6254
|
+
if warehouse_id is not None: body['warehouse_id'] = warehouse_id
|
|
6255
|
+
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
|
|
6256
|
+
res = self._api.do('POST',
|
|
6257
|
+
f'/api/2.1/unity-catalog/tables/{full_name}/monitor',
|
|
6258
|
+
body=body,
|
|
6259
|
+
headers=headers)
|
|
6260
|
+
return MonitorInfo.from_dict(res)
|
|
6261
|
+
|
|
6262
|
+
def delete(self, full_name: str):
|
|
6263
|
+
"""Delete a table monitor.
|
|
6264
|
+
|
|
6265
|
+
Deletes a monitor for the specified table.
|
|
6266
|
+
|
|
6267
|
+
The caller must either: 1. be an owner of the table's parent catalog 2. have **USE_CATALOG** on the
|
|
6268
|
+
table's parent catalog and be an owner of the table's parent schema 3. have the following permissions:
|
|
6269
|
+
- **USE_CATALOG** on the table's parent catalog - **USE_SCHEMA** on the table's parent schema - be an
|
|
6270
|
+
owner of the table.
|
|
6271
|
+
|
|
6272
|
+
Additionally, the call must be made from the workspace where the monitor was created.
|
|
6273
|
+
|
|
6274
|
+
Note that the metric tables and dashboard will not be deleted as part of this call; those assets must
|
|
6275
|
+
be manually cleaned up (if desired).
|
|
6276
|
+
|
|
6277
|
+
:param full_name: str
|
|
6278
|
+
Full name of the table.
|
|
6279
|
+
|
|
6280
|
+
|
|
6281
|
+
"""
|
|
6282
|
+
|
|
6283
|
+
headers = {}
|
|
6284
|
+
self._api.do('DELETE', f'/api/2.1/unity-catalog/tables/{full_name}/monitor', headers=headers)
|
|
6285
|
+
|
|
6286
|
+
def get(self, full_name: str) -> MonitorInfo:
|
|
6287
|
+
"""Get a table monitor.
|
|
6288
|
+
|
|
6289
|
+
Gets a monitor for the specified table.
|
|
6290
|
+
|
|
6291
|
+
The caller must either: 1. be an owner of the table's parent catalog 2. have **USE_CATALOG** on the
|
|
6292
|
+
table's parent catalog and be an owner of the table's parent schema. 3. have the following
|
|
6293
|
+
permissions: - **USE_CATALOG** on the table's parent catalog - **USE_SCHEMA** on the table's parent
|
|
6294
|
+
schema - **SELECT** privilege on the table.
|
|
6295
|
+
|
|
6296
|
+
The returned information includes configuration values, as well as information on assets created by
|
|
6297
|
+
the monitor. Some information (e.g., dashboard) may be filtered out if the caller is in a different
|
|
6298
|
+
workspace than where the monitor was created.
|
|
6299
|
+
|
|
6300
|
+
:param full_name: str
|
|
6301
|
+
Full name of the table.
|
|
6302
|
+
|
|
6303
|
+
:returns: :class:`MonitorInfo`
|
|
6304
|
+
"""
|
|
6305
|
+
|
|
6306
|
+
headers = {'Accept': 'application/json', }
|
|
6307
|
+
res = self._api.do('GET', f'/api/2.1/unity-catalog/tables/{full_name}/monitor', headers=headers)
|
|
6308
|
+
return MonitorInfo.from_dict(res)
|
|
6309
|
+
|
|
6310
|
+
def get_refresh(self, full_name: str, refresh_id: str) -> MonitorRefreshInfo:
|
|
6311
|
+
"""Get refresh.
|
|
6312
|
+
|
|
6313
|
+
Gets info about a specific monitor refresh using the given refresh ID.
|
|
6314
|
+
|
|
6315
|
+
The caller must either: 1. be an owner of the table's parent catalog 2. have **USE_CATALOG** on the
|
|
6316
|
+
table's parent catalog and be an owner of the table's parent schema 3. have the following permissions:
|
|
6317
|
+
- **USE_CATALOG** on the table's parent catalog - **USE_SCHEMA** on the table's parent schema -
|
|
6318
|
+
**SELECT** privilege on the table.
|
|
6319
|
+
|
|
6320
|
+
Additionally, the call must be made from the workspace where the monitor was created.
|
|
6321
|
+
|
|
6322
|
+
:param full_name: str
|
|
6323
|
+
Full name of the table.
|
|
6324
|
+
:param refresh_id: str
|
|
6325
|
+
ID of the refresh.
|
|
6326
|
+
|
|
6327
|
+
:returns: :class:`MonitorRefreshInfo`
|
|
6328
|
+
"""
|
|
6329
|
+
|
|
6330
|
+
headers = {'Accept': 'application/json', }
|
|
6331
|
+
res = self._api.do('GET',
|
|
6332
|
+
f'/api/2.1/unity-catalog/tables/{full_name}/monitor/refreshes/{refresh_id}',
|
|
6333
|
+
headers=headers)
|
|
6334
|
+
return MonitorRefreshInfo.from_dict(res)
|
|
6335
|
+
|
|
6336
|
+
def list_refreshes(self, full_name: str) -> Iterator[MonitorRefreshInfo]:
|
|
6337
|
+
"""List refreshes.
|
|
6338
|
+
|
|
6339
|
+
Gets an array containing the history of the most recent refreshes (up to 25) for this table.
|
|
6340
|
+
|
|
6341
|
+
The caller must either: 1. be an owner of the table's parent catalog 2. have **USE_CATALOG** on the
|
|
6342
|
+
table's parent catalog and be an owner of the table's parent schema 3. have the following permissions:
|
|
6343
|
+
- **USE_CATALOG** on the table's parent catalog - **USE_SCHEMA** on the table's parent schema -
|
|
6344
|
+
**SELECT** privilege on the table.
|
|
6345
|
+
|
|
6346
|
+
Additionally, the call must be made from the workspace where the monitor was created.
|
|
6347
|
+
|
|
6348
|
+
:param full_name: str
|
|
6349
|
+
Full name of the table.
|
|
6350
|
+
|
|
6351
|
+
:returns: Iterator over :class:`MonitorRefreshInfo`
|
|
6352
|
+
"""
|
|
6353
|
+
|
|
6354
|
+
headers = {'Accept': 'application/json', }
|
|
6355
|
+
res = self._api.do('GET',
|
|
6356
|
+
f'/api/2.1/unity-catalog/tables/{full_name}/monitor/refreshes',
|
|
6357
|
+
headers=headers)
|
|
6358
|
+
return [MonitorRefreshInfo.from_dict(v) for v in res]
|
|
6359
|
+
|
|
6360
|
+
def run_refresh(self, full_name: str) -> MonitorRefreshInfo:
|
|
6361
|
+
"""Queue a metric refresh for a monitor.
|
|
6362
|
+
|
|
6363
|
+
Queues a metric refresh on the monitor for the specified table. The refresh will execute in the
|
|
6364
|
+
background.
|
|
6365
|
+
|
|
6366
|
+
The caller must either: 1. be an owner of the table's parent catalog 2. have **USE_CATALOG** on the
|
|
6367
|
+
table's parent catalog and be an owner of the table's parent schema 3. have the following permissions:
|
|
6368
|
+
- **USE_CATALOG** on the table's parent catalog - **USE_SCHEMA** on the table's parent schema - be an
|
|
6369
|
+
owner of the table
|
|
6370
|
+
|
|
6371
|
+
Additionally, the call must be made from the workspace where the monitor was created.
|
|
6372
|
+
|
|
6373
|
+
:param full_name: str
|
|
6374
|
+
Full name of the table.
|
|
6375
|
+
|
|
6376
|
+
:returns: :class:`MonitorRefreshInfo`
|
|
6377
|
+
"""
|
|
6378
|
+
|
|
6379
|
+
headers = {'Accept': 'application/json', }
|
|
6380
|
+
res = self._api.do('POST',
|
|
6381
|
+
f'/api/2.1/unity-catalog/tables/{full_name}/monitor/refreshes',
|
|
6382
|
+
headers=headers)
|
|
6383
|
+
return MonitorRefreshInfo.from_dict(res)
|
|
6384
|
+
|
|
6385
|
+
def update(self,
|
|
6386
|
+
full_name: str,
|
|
6387
|
+
assets_dir: str,
|
|
6388
|
+
output_schema_name: str,
|
|
6389
|
+
*,
|
|
6390
|
+
baseline_table_name: Optional[str] = None,
|
|
6391
|
+
custom_metrics: Optional[List[MonitorCustomMetric]] = None,
|
|
6392
|
+
data_classification_config: Optional[MonitorDataClassificationConfig] = None,
|
|
6393
|
+
inference_log: Optional[MonitorInferenceLogProfileType] = None,
|
|
6394
|
+
notifications: Optional[List[MonitorNotificationsConfig]] = None,
|
|
6395
|
+
schedule: Optional[MonitorCronSchedule] = None,
|
|
6396
|
+
slicing_exprs: Optional[List[str]] = None,
|
|
6397
|
+
snapshot: Optional[Any] = None,
|
|
6398
|
+
time_series: Optional[MonitorTimeSeriesProfileType] = None) -> MonitorInfo:
|
|
6399
|
+
"""Update a table monitor.
|
|
6400
|
+
|
|
6401
|
+
Updates a monitor for the specified table.
|
|
6402
|
+
|
|
6403
|
+
The caller must either: 1. be an owner of the table's parent catalog 2. have **USE_CATALOG** on the
|
|
6404
|
+
table's parent catalog and be an owner of the table's parent schema 3. have the following permissions:
|
|
6405
|
+
- **USE_CATALOG** on the table's parent catalog - **USE_SCHEMA** on the table's parent schema - be an
|
|
6406
|
+
owner of the table.
|
|
6407
|
+
|
|
6408
|
+
Additionally, the call must be made from the workspace where the monitor was created, and the caller
|
|
6409
|
+
must be the original creator of the monitor.
|
|
6410
|
+
|
|
6411
|
+
Certain configuration fields, such as output asset identifiers, cannot be updated.
|
|
6412
|
+
|
|
6413
|
+
:param full_name: str
|
|
6414
|
+
Full name of the table.
|
|
6415
|
+
:param assets_dir: str
|
|
6416
|
+
The directory to store monitoring assets (e.g. dashboard, metric tables).
|
|
6417
|
+
:param output_schema_name: str
|
|
6418
|
+
Schema where output metric tables are created.
|
|
6419
|
+
:param baseline_table_name: str (optional)
|
|
6420
|
+
Name of the baseline table from which drift metrics are computed from. Columns in the monitored
|
|
6421
|
+
table should also be present in the baseline table.
|
|
6422
|
+
:param custom_metrics: List[:class:`MonitorCustomMetric`] (optional)
|
|
6423
|
+
Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics
|
|
6424
|
+
(from already computed aggregate metrics), or drift metrics (comparing metrics across time windows).
|
|
6425
|
+
:param data_classification_config: :class:`MonitorDataClassificationConfig` (optional)
|
|
6426
|
+
The data classification config for the monitor.
|
|
6427
|
+
:param inference_log: :class:`MonitorInferenceLogProfileType` (optional)
|
|
6428
|
+
Configuration for monitoring inference logs.
|
|
6429
|
+
:param notifications: List[:class:`MonitorNotificationsConfig`] (optional)
|
|
6430
|
+
The notification settings for the monitor.
|
|
6431
|
+
:param schedule: :class:`MonitorCronSchedule` (optional)
|
|
6432
|
+
The schedule for automatically updating and refreshing metric tables.
|
|
6433
|
+
:param slicing_exprs: List[str] (optional)
|
|
6434
|
+
List of column expressions to slice data with for targeted analysis. The data is grouped by each
|
|
6435
|
+
expression independently, resulting in a separate slice for each predicate and its complements. For
|
|
6436
|
+
high-cardinality columns, only the top 100 unique values by frequency will generate slices.
|
|
6437
|
+
:param snapshot: Any (optional)
|
|
6438
|
+
Configuration for monitoring snapshot tables.
|
|
6439
|
+
:param time_series: :class:`MonitorTimeSeriesProfileType` (optional)
|
|
6440
|
+
Configuration for monitoring time series tables.
|
|
6441
|
+
|
|
6442
|
+
:returns: :class:`MonitorInfo`
|
|
6443
|
+
"""
|
|
6444
|
+
body = {}
|
|
6445
|
+
if assets_dir is not None: body['assets_dir'] = assets_dir
|
|
6446
|
+
if baseline_table_name is not None: body['baseline_table_name'] = baseline_table_name
|
|
6447
|
+
if custom_metrics is not None: body['custom_metrics'] = [v.as_dict() for v in custom_metrics]
|
|
6448
|
+
if data_classification_config is not None:
|
|
6449
|
+
body['data_classification_config'] = data_classification_config.as_dict()
|
|
6450
|
+
if inference_log is not None: body['inference_log'] = inference_log.as_dict()
|
|
6451
|
+
if notifications is not None: body['notifications'] = [v.as_dict() for v in notifications]
|
|
6452
|
+
if output_schema_name is not None: body['output_schema_name'] = output_schema_name
|
|
6453
|
+
if schedule is not None: body['schedule'] = schedule.as_dict()
|
|
6454
|
+
if slicing_exprs is not None: body['slicing_exprs'] = [v for v in slicing_exprs]
|
|
6455
|
+
if snapshot is not None: body['snapshot'] = snapshot
|
|
6456
|
+
if time_series is not None: body['time_series'] = time_series.as_dict()
|
|
6457
|
+
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
|
|
6458
|
+
res = self._api.do('PUT',
|
|
6459
|
+
f'/api/2.1/unity-catalog/tables/{full_name}/monitor',
|
|
6460
|
+
body=body,
|
|
6461
|
+
headers=headers)
|
|
6462
|
+
return MonitorInfo.from_dict(res)
|
|
6463
|
+
|
|
6464
|
+
|
|
5600
6465
|
class MetastoresAPI:
|
|
5601
6466
|
"""A metastore is the top-level container of objects in Unity Catalog. It stores data assets (tables and
|
|
5602
6467
|
views) and the permissions that govern access to them. Databricks account admins can create metastores and
|
|
@@ -5768,7 +6633,6 @@ class MetastoresAPI:
|
|
|
5768
6633
|
delta_sharing_organization_name: Optional[str] = None,
|
|
5769
6634
|
delta_sharing_recipient_token_lifetime_in_seconds: Optional[int] = None,
|
|
5770
6635
|
delta_sharing_scope: Optional[UpdateMetastoreDeltaSharingScope] = None,
|
|
5771
|
-
name: Optional[str] = None,
|
|
5772
6636
|
new_name: Optional[str] = None,
|
|
5773
6637
|
owner: Optional[str] = None,
|
|
5774
6638
|
privilege_model_version: Optional[str] = None,
|
|
@@ -5787,8 +6651,6 @@ class MetastoresAPI:
|
|
|
5787
6651
|
The lifetime of delta sharing recipient token in seconds.
|
|
5788
6652
|
:param delta_sharing_scope: :class:`UpdateMetastoreDeltaSharingScope` (optional)
|
|
5789
6653
|
The scope of Delta Sharing enabled for the metastore.
|
|
5790
|
-
:param name: str (optional)
|
|
5791
|
-
The user-specified name of the metastore.
|
|
5792
6654
|
:param new_name: str (optional)
|
|
5793
6655
|
New name for the metastore.
|
|
5794
6656
|
:param owner: str (optional)
|
|
@@ -5807,7 +6669,6 @@ class MetastoresAPI:
|
|
|
5807
6669
|
body[
|
|
5808
6670
|
'delta_sharing_recipient_token_lifetime_in_seconds'] = delta_sharing_recipient_token_lifetime_in_seconds
|
|
5809
6671
|
if delta_sharing_scope is not None: body['delta_sharing_scope'] = delta_sharing_scope.value
|
|
5810
|
-
if name is not None: body['name'] = name
|
|
5811
6672
|
if new_name is not None: body['new_name'] = new_name
|
|
5812
6673
|
if owner is not None: body['owner'] = owner
|
|
5813
6674
|
if privilege_model_version is not None: body['privilege_model_version'] = privilege_model_version
|
|
@@ -5970,10 +6831,9 @@ class ModelVersionsAPI:
|
|
|
5970
6831
|
f'/api/2.1/unity-catalog/models/{full_name}/versions',
|
|
5971
6832
|
query=query,
|
|
5972
6833
|
headers=headers)
|
|
5973
|
-
if 'model_versions'
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
yield ModelVersionInfo.from_dict(v)
|
|
6834
|
+
if 'model_versions' in json:
|
|
6835
|
+
for v in json['model_versions']:
|
|
6836
|
+
yield ModelVersionInfo.from_dict(v)
|
|
5977
6837
|
if 'next_page_token' not in json or not json['next_page_token']:
|
|
5978
6838
|
return
|
|
5979
6839
|
query['page_token'] = json['next_page_token']
|
|
@@ -6180,10 +7040,9 @@ class RegisteredModelsAPI:
|
|
|
6180
7040
|
|
|
6181
7041
|
while True:
|
|
6182
7042
|
json = self._api.do('GET', '/api/2.1/unity-catalog/models', query=query, headers=headers)
|
|
6183
|
-
if 'registered_models'
|
|
6184
|
-
|
|
6185
|
-
|
|
6186
|
-
yield RegisteredModelInfo.from_dict(v)
|
|
7043
|
+
if 'registered_models' in json:
|
|
7044
|
+
for v in json['registered_models']:
|
|
7045
|
+
yield RegisteredModelInfo.from_dict(v)
|
|
6187
7046
|
if 'next_page_token' not in json or not json['next_page_token']:
|
|
6188
7047
|
return
|
|
6189
7048
|
query['page_token'] = json['next_page_token']
|
|
@@ -6219,7 +7078,6 @@ class RegisteredModelsAPI:
|
|
|
6219
7078
|
full_name: str,
|
|
6220
7079
|
*,
|
|
6221
7080
|
comment: Optional[str] = None,
|
|
6222
|
-
name: Optional[str] = None,
|
|
6223
7081
|
new_name: Optional[str] = None,
|
|
6224
7082
|
owner: Optional[str] = None) -> RegisteredModelInfo:
|
|
6225
7083
|
"""Update a Registered Model.
|
|
@@ -6236,8 +7094,6 @@ class RegisteredModelsAPI:
|
|
|
6236
7094
|
The three-level (fully qualified) name of the registered model
|
|
6237
7095
|
:param comment: str (optional)
|
|
6238
7096
|
The comment attached to the registered model
|
|
6239
|
-
:param name: str (optional)
|
|
6240
|
-
The name of the registered model
|
|
6241
7097
|
:param new_name: str (optional)
|
|
6242
7098
|
New name for the registered model.
|
|
6243
7099
|
:param owner: str (optional)
|
|
@@ -6247,7 +7103,6 @@ class RegisteredModelsAPI:
|
|
|
6247
7103
|
"""
|
|
6248
7104
|
body = {}
|
|
6249
7105
|
if comment is not None: body['comment'] = comment
|
|
6250
|
-
if name is not None: body['name'] = name
|
|
6251
7106
|
if new_name is not None: body['new_name'] = new_name
|
|
6252
7107
|
if owner is not None: body['owner'] = owner
|
|
6253
7108
|
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
|
|
@@ -6364,10 +7219,9 @@ class SchemasAPI:
|
|
|
6364
7219
|
|
|
6365
7220
|
while True:
|
|
6366
7221
|
json = self._api.do('GET', '/api/2.1/unity-catalog/schemas', query=query, headers=headers)
|
|
6367
|
-
if 'schemas'
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
yield SchemaInfo.from_dict(v)
|
|
7222
|
+
if 'schemas' in json:
|
|
7223
|
+
for v in json['schemas']:
|
|
7224
|
+
yield SchemaInfo.from_dict(v)
|
|
6371
7225
|
if 'next_page_token' not in json or not json['next_page_token']:
|
|
6372
7226
|
return
|
|
6373
7227
|
query['page_token'] = json['next_page_token']
|
|
@@ -6377,7 +7231,6 @@ class SchemasAPI:
|
|
|
6377
7231
|
*,
|
|
6378
7232
|
comment: Optional[str] = None,
|
|
6379
7233
|
enable_predictive_optimization: Optional[EnablePredictiveOptimization] = None,
|
|
6380
|
-
name: Optional[str] = None,
|
|
6381
7234
|
new_name: Optional[str] = None,
|
|
6382
7235
|
owner: Optional[str] = None,
|
|
6383
7236
|
properties: Optional[Dict[str, str]] = None) -> SchemaInfo:
|
|
@@ -6394,8 +7247,6 @@ class SchemasAPI:
|
|
|
6394
7247
|
User-provided free-form text description.
|
|
6395
7248
|
:param enable_predictive_optimization: :class:`EnablePredictiveOptimization` (optional)
|
|
6396
7249
|
Whether predictive optimization should be enabled for this object and objects under it.
|
|
6397
|
-
:param name: str (optional)
|
|
6398
|
-
Name of schema, relative to parent catalog.
|
|
6399
7250
|
:param new_name: str (optional)
|
|
6400
7251
|
New name for the schema.
|
|
6401
7252
|
:param owner: str (optional)
|
|
@@ -6409,7 +7260,6 @@ class SchemasAPI:
|
|
|
6409
7260
|
if comment is not None: body['comment'] = comment
|
|
6410
7261
|
if enable_predictive_optimization is not None:
|
|
6411
7262
|
body['enable_predictive_optimization'] = enable_predictive_optimization.value
|
|
6412
|
-
if name is not None: body['name'] = name
|
|
6413
7263
|
if new_name is not None: body['new_name'] = new_name
|
|
6414
7264
|
if owner is not None: body['owner'] = owner
|
|
6415
7265
|
if properties is not None: body['properties'] = properties
|
|
@@ -6558,10 +7408,9 @@ class StorageCredentialsAPI:
|
|
|
6558
7408
|
'/api/2.1/unity-catalog/storage-credentials',
|
|
6559
7409
|
query=query,
|
|
6560
7410
|
headers=headers)
|
|
6561
|
-
if 'storage_credentials'
|
|
6562
|
-
|
|
6563
|
-
|
|
6564
|
-
yield StorageCredentialInfo.from_dict(v)
|
|
7411
|
+
if 'storage_credentials' in json:
|
|
7412
|
+
for v in json['storage_credentials']:
|
|
7413
|
+
yield StorageCredentialInfo.from_dict(v)
|
|
6565
7414
|
if 'next_page_token' not in json or not json['next_page_token']:
|
|
6566
7415
|
return
|
|
6567
7416
|
query['page_token'] = json['next_page_token']
|
|
@@ -6868,13 +7717,34 @@ class TablesAPI:
|
|
|
6868
7717
|
headers = {'Accept': 'application/json', }
|
|
6869
7718
|
self._api.do('DELETE', f'/api/2.1/unity-catalog/tables/{full_name}', headers=headers)
|
|
6870
7719
|
|
|
7720
|
+
def exists(self, full_name: str) -> TableExistsResponse:
|
|
7721
|
+
"""Get boolean reflecting if table exists.
|
|
7722
|
+
|
|
7723
|
+
Gets if a table exists in the metastore for a specific catalog and schema. The caller must satisfy one
|
|
7724
|
+
of the following requirements: * Be a metastore admin * Be the owner of the parent catalog * Be the
|
|
7725
|
+
owner of the parent schema and have the USE_CATALOG privilege on the parent catalog * Have the
|
|
7726
|
+
**USE_CATALOG** privilege on the parent catalog and the **USE_SCHEMA** privilege on the parent schema,
|
|
7727
|
+
and either be the table owner or have the SELECT privilege on the table. * Have BROWSE privilege on
|
|
7728
|
+
the parent catalog * Have BROWSE privilege on the parent schema.
|
|
7729
|
+
|
|
7730
|
+
:param full_name: str
|
|
7731
|
+
Full name of the table.
|
|
7732
|
+
|
|
7733
|
+
:returns: :class:`TableExistsResponse`
|
|
7734
|
+
"""
|
|
7735
|
+
|
|
7736
|
+
headers = {'Accept': 'application/json', }
|
|
7737
|
+
res = self._api.do('GET', f'/api/2.1/unity-catalog/tables/{full_name}/exists', headers=headers)
|
|
7738
|
+
return TableExistsResponse.from_dict(res)
|
|
7739
|
+
|
|
6871
7740
|
def get(self, full_name: str, *, include_delta_metadata: Optional[bool] = None) -> TableInfo:
|
|
6872
7741
|
"""Get a table.
|
|
6873
7742
|
|
|
6874
|
-
Gets a table from the metastore for a specific catalog and schema. The caller must
|
|
6875
|
-
|
|
6876
|
-
|
|
6877
|
-
privilege on
|
|
7743
|
+
Gets a table from the metastore for a specific catalog and schema. The caller must satisfy one of the
|
|
7744
|
+
following requirements: * Be a metastore admin * Be the owner of the parent catalog * Be the owner of
|
|
7745
|
+
the parent schema and have the USE_CATALOG privilege on the parent catalog * Have the **USE_CATALOG**
|
|
7746
|
+
privilege on the parent catalog and the **USE_SCHEMA** privilege on the parent schema, and either be
|
|
7747
|
+
the table owner or have the SELECT privilege on the table.
|
|
6878
7748
|
|
|
6879
7749
|
:param full_name: str
|
|
6880
7750
|
Full name of the table.
|
|
@@ -6940,10 +7810,9 @@ class TablesAPI:
|
|
|
6940
7810
|
|
|
6941
7811
|
while True:
|
|
6942
7812
|
json = self._api.do('GET', '/api/2.1/unity-catalog/tables', query=query, headers=headers)
|
|
6943
|
-
if 'tables'
|
|
6944
|
-
|
|
6945
|
-
|
|
6946
|
-
yield TableInfo.from_dict(v)
|
|
7813
|
+
if 'tables' in json:
|
|
7814
|
+
for v in json['tables']:
|
|
7815
|
+
yield TableInfo.from_dict(v)
|
|
6947
7816
|
if 'next_page_token' not in json or not json['next_page_token']:
|
|
6948
7817
|
return
|
|
6949
7818
|
query['page_token'] = json['next_page_token']
|
|
@@ -6996,10 +7865,9 @@ class TablesAPI:
|
|
|
6996
7865
|
|
|
6997
7866
|
while True:
|
|
6998
7867
|
json = self._api.do('GET', '/api/2.1/unity-catalog/table-summaries', query=query, headers=headers)
|
|
6999
|
-
if 'tables'
|
|
7000
|
-
|
|
7001
|
-
|
|
7002
|
-
yield TableSummary.from_dict(v)
|
|
7868
|
+
if 'tables' in json:
|
|
7869
|
+
for v in json['tables']:
|
|
7870
|
+
yield TableSummary.from_dict(v)
|
|
7003
7871
|
if 'next_page_token' not in json or not json['next_page_token']:
|
|
7004
7872
|
return
|
|
7005
7873
|
query['page_token'] = json['next_page_token']
|
|
@@ -7156,7 +8024,6 @@ class VolumesAPI:
|
|
|
7156
8024
|
full_name_arg: str,
|
|
7157
8025
|
*,
|
|
7158
8026
|
comment: Optional[str] = None,
|
|
7159
|
-
name: Optional[str] = None,
|
|
7160
8027
|
new_name: Optional[str] = None,
|
|
7161
8028
|
owner: Optional[str] = None) -> VolumeInfo:
|
|
7162
8029
|
"""Update a Volume.
|
|
@@ -7173,8 +8040,6 @@ class VolumesAPI:
|
|
|
7173
8040
|
The three-level (fully qualified) name of the volume
|
|
7174
8041
|
:param comment: str (optional)
|
|
7175
8042
|
The comment attached to the volume
|
|
7176
|
-
:param name: str (optional)
|
|
7177
|
-
The name of the volume
|
|
7178
8043
|
:param new_name: str (optional)
|
|
7179
8044
|
New name for the volume.
|
|
7180
8045
|
:param owner: str (optional)
|
|
@@ -7184,7 +8049,6 @@ class VolumesAPI:
|
|
|
7184
8049
|
"""
|
|
7185
8050
|
body = {}
|
|
7186
8051
|
if comment is not None: body['comment'] = comment
|
|
7187
|
-
if name is not None: body['name'] = name
|
|
7188
8052
|
if new_name is not None: body['new_name'] = new_name
|
|
7189
8053
|
if owner is not None: body['owner'] = owner
|
|
7190
8054
|
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
|