oracle-ads 2.12.11__py3-none-any.whl → 2.13.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/__init__.py +7 -1
- ads/aqua/app.py +41 -27
- ads/aqua/client/client.py +48 -11
- ads/aqua/common/entities.py +28 -1
- ads/aqua/common/enums.py +32 -21
- ads/aqua/common/errors.py +3 -4
- ads/aqua/common/utils.py +10 -15
- ads/aqua/config/container_config.py +203 -0
- ads/aqua/config/evaluation/evaluation_service_config.py +5 -181
- ads/aqua/constants.py +1 -1
- ads/aqua/evaluation/constants.py +7 -7
- ads/aqua/evaluation/errors.py +3 -4
- ads/aqua/evaluation/evaluation.py +4 -4
- ads/aqua/extension/base_handler.py +4 -0
- ads/aqua/extension/model_handler.py +41 -27
- ads/aqua/extension/models/ws_models.py +5 -6
- ads/aqua/finetuning/constants.py +3 -3
- ads/aqua/finetuning/finetuning.py +2 -3
- ads/aqua/model/constants.py +7 -7
- ads/aqua/model/entities.py +2 -3
- ads/aqua/model/enums.py +4 -5
- ads/aqua/model/model.py +46 -29
- ads/aqua/modeldeployment/deployment.py +6 -14
- ads/aqua/modeldeployment/entities.py +5 -3
- ads/aqua/server/__init__.py +4 -0
- ads/aqua/server/__main__.py +24 -0
- ads/aqua/server/app.py +47 -0
- ads/aqua/server/aqua_spec.yml +1291 -0
- ads/aqua/ui.py +5 -199
- ads/common/auth.py +50 -28
- ads/common/extended_enum.py +52 -44
- ads/common/utils.py +91 -11
- ads/config.py +3 -0
- ads/llm/__init__.py +12 -8
- ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +32 -23
- ads/model/artifact_downloader.py +6 -4
- ads/model/common/utils.py +15 -3
- ads/model/datascience_model.py +422 -71
- ads/model/generic_model.py +3 -3
- ads/model/model_metadata.py +70 -24
- ads/model/model_version_set.py +5 -3
- ads/model/service/oci_datascience_model.py +487 -17
- ads/opctl/anomaly_detection.py +11 -0
- ads/opctl/backend/marketplace/helm_helper.py +13 -14
- ads/opctl/cli.py +4 -5
- ads/opctl/cmds.py +28 -32
- ads/opctl/config/merger.py +8 -11
- ads/opctl/config/resolver.py +25 -30
- ads/opctl/forecast.py +11 -0
- ads/opctl/operator/cli.py +9 -9
- ads/opctl/operator/common/backend_factory.py +56 -60
- ads/opctl/operator/common/const.py +5 -5
- ads/opctl/operator/common/utils.py +16 -0
- ads/opctl/operator/lowcode/anomaly/const.py +8 -9
- ads/opctl/operator/lowcode/common/data.py +5 -2
- ads/opctl/operator/lowcode/common/transformations.py +2 -12
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +43 -48
- ads/opctl/operator/lowcode/forecast/__main__.py +5 -5
- ads/opctl/operator/lowcode/forecast/const.py +6 -6
- ads/opctl/operator/lowcode/forecast/model/arima.py +6 -3
- ads/opctl/operator/lowcode/forecast/model/automlx.py +61 -31
- ads/opctl/operator/lowcode/forecast/model/base_model.py +66 -40
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +79 -13
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +5 -2
- ads/opctl/operator/lowcode/forecast/model/prophet.py +28 -15
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +13 -15
- ads/opctl/operator/lowcode/forecast/schema.yaml +1 -1
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +7 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +19 -11
- ads/opctl/operator/lowcode/pii/constant.py +6 -7
- ads/opctl/operator/lowcode/recommender/constant.py +12 -7
- ads/opctl/operator/runtime/marketplace_runtime.py +4 -10
- ads/opctl/operator/runtime/runtime.py +4 -6
- ads/pipeline/ads_pipeline_run.py +13 -25
- ads/pipeline/visualizer/graph_renderer.py +3 -4
- {oracle_ads-2.12.11.dist-info → oracle_ads-2.13.1.dist-info}/METADATA +18 -15
- {oracle_ads-2.12.11.dist-info → oracle_ads-2.13.1.dist-info}/RECORD +82 -74
- {oracle_ads-2.12.11.dist-info → oracle_ads-2.13.1.dist-info}/WHEEL +1 -1
- ads/aqua/config/evaluation/evaluation_service_model_config.py +0 -8
- {oracle_ads-2.12.11.dist-info → oracle_ads-2.13.1.dist-info}/entry_points.txt +0 -0
- {oracle_ads-2.12.11.dist-info → oracle_ads-2.13.1.dist-info/licenses}/LICENSE.txt +0 -0
ads/model/datascience_model.py
CHANGED
@@ -1,10 +1,8 @@
|
|
1
1
|
#!/usr/bin/env python
|
2
|
-
# -*- coding: utf-8; -*-
|
3
2
|
|
4
|
-
# Copyright (c) 2022,
|
3
|
+
# Copyright (c) 2022, 2025 Oracle and/or its affiliates.
|
5
4
|
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
5
|
|
7
|
-
import cgi
|
8
6
|
import json
|
9
7
|
import logging
|
10
8
|
import os
|
@@ -19,12 +17,16 @@ from jsonschema import ValidationError, validate
|
|
19
17
|
|
20
18
|
from ads.common import oci_client as oc
|
21
19
|
from ads.common import utils
|
22
|
-
from ads.common.extended_enum import
|
20
|
+
from ads.common.extended_enum import ExtendedEnum
|
23
21
|
from ads.common.object_storage_details import ObjectStorageDetails
|
22
|
+
from ads.common.utils import is_path_exists
|
23
|
+
from ads.config import (
|
24
|
+
AQUA_SERVICE_MODELS_BUCKET as SERVICE_MODELS_BUCKET,
|
25
|
+
)
|
24
26
|
from ads.config import (
|
25
27
|
COMPARTMENT_OCID,
|
26
28
|
PROJECT_OCID,
|
27
|
-
|
29
|
+
USER,
|
28
30
|
)
|
29
31
|
from ads.feature_engineering.schema import Schema
|
30
32
|
from ads.jobs.builders.base import Builder
|
@@ -33,6 +35,7 @@ from ads.model.artifact_downloader import (
|
|
33
35
|
SmallArtifactDownloader,
|
34
36
|
)
|
35
37
|
from ads.model.artifact_uploader import LargeArtifactUploader, SmallArtifactUploader
|
38
|
+
from ads.model.common.utils import MetadataArtifactPathType
|
36
39
|
from ads.model.model_metadata import (
|
37
40
|
MetadataCustomCategory,
|
38
41
|
ModelCustomMetadata,
|
@@ -41,6 +44,7 @@ from ads.model.model_metadata import (
|
|
41
44
|
ModelTaxonomyMetadata,
|
42
45
|
)
|
43
46
|
from ads.model.service.oci_datascience_model import (
|
47
|
+
ModelMetadataArtifactDetails,
|
44
48
|
ModelProvenanceNotFoundError,
|
45
49
|
OCIDataScienceModel,
|
46
50
|
)
|
@@ -71,6 +75,11 @@ class BucketNotVersionedError(Exception): # pragma: no cover
|
|
71
75
|
super().__init__(msg)
|
72
76
|
|
73
77
|
|
78
|
+
class PathNotFoundError(Exception):
|
79
|
+
def __init__(self, msg="The given path doesn't exist."):
|
80
|
+
super().__init__(msg)
|
81
|
+
|
82
|
+
|
74
83
|
class ModelFileDescriptionError(Exception): # pragma: no cover
|
75
84
|
def __init__(self, msg="Model File Description file is not set up."):
|
76
85
|
super().__init__(msg)
|
@@ -80,14 +89,14 @@ class InvalidArtifactType(Exception): # pragma: no cover
|
|
80
89
|
pass
|
81
90
|
|
82
91
|
|
83
|
-
class CustomerNotificationType(
|
92
|
+
class CustomerNotificationType(ExtendedEnum):
|
84
93
|
NONE = "NONE"
|
85
94
|
ALL = "ALL"
|
86
95
|
ON_FAILURE = "ON_FAILURE"
|
87
96
|
ON_SUCCESS = "ON_SUCCESS"
|
88
97
|
|
89
98
|
|
90
|
-
class SettingStatus(
|
99
|
+
class SettingStatus(ExtendedEnum):
|
91
100
|
"""Enum to represent the status of retention settings."""
|
92
101
|
|
93
102
|
PENDING = "PENDING"
|
@@ -116,17 +125,17 @@ class ModelBackupSetting:
|
|
116
125
|
"""
|
117
126
|
|
118
127
|
def __init__(
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
128
|
+
self,
|
129
|
+
is_backup_enabled: Optional[bool] = None,
|
130
|
+
backup_region: Optional[str] = None,
|
131
|
+
customer_notification_type: Optional[CustomerNotificationType] = None,
|
123
132
|
):
|
124
133
|
self.is_backup_enabled = (
|
125
134
|
is_backup_enabled if is_backup_enabled is not None else False
|
126
135
|
)
|
127
136
|
self.backup_region = backup_region
|
128
137
|
self.customer_notification_type = (
|
129
|
-
|
138
|
+
customer_notification_type or CustomerNotificationType.NONE
|
130
139
|
)
|
131
140
|
|
132
141
|
def to_dict(self) -> Dict:
|
@@ -143,10 +152,7 @@ class ModelBackupSetting:
|
|
143
152
|
return cls(
|
144
153
|
is_backup_enabled=data.get("is_backup_enabled"),
|
145
154
|
backup_region=data.get("backup_region"),
|
146
|
-
customer_notification_type=
|
147
|
-
data.get("customer_notification_type")
|
148
|
-
)
|
149
|
-
or None,
|
155
|
+
customer_notification_type=data.get("customer_notification_type") or None,
|
150
156
|
)
|
151
157
|
|
152
158
|
def to_json(self) -> str:
|
@@ -166,12 +172,15 @@ class ModelBackupSetting:
|
|
166
172
|
|
167
173
|
def validate(self) -> bool:
|
168
174
|
"""Validates the backup settings details. Returns True if valid, False otherwise."""
|
169
|
-
return all(
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
+
return all(
|
176
|
+
[
|
177
|
+
isinstance(self.is_backup_enabled, bool),
|
178
|
+
not self.backup_region or isinstance(self.backup_region, str),
|
179
|
+
isinstance(self.customer_notification_type, str)
|
180
|
+
and self.customer_notification_type
|
181
|
+
in CustomerNotificationType.values(),
|
182
|
+
]
|
183
|
+
)
|
175
184
|
|
176
185
|
def __repr__(self):
|
177
186
|
return self.to_yaml()
|
@@ -198,15 +207,15 @@ class ModelRetentionSetting:
|
|
198
207
|
"""
|
199
208
|
|
200
209
|
def __init__(
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
210
|
+
self,
|
211
|
+
archive_after_days: Optional[int] = None,
|
212
|
+
delete_after_days: Optional[int] = None,
|
213
|
+
customer_notification_type: Optional[CustomerNotificationType] = None,
|
205
214
|
):
|
206
215
|
self.archive_after_days = archive_after_days
|
207
216
|
self.delete_after_days = delete_after_days
|
208
217
|
self.customer_notification_type = (
|
209
|
-
|
218
|
+
customer_notification_type or CustomerNotificationType.NONE
|
210
219
|
)
|
211
220
|
|
212
221
|
def to_dict(self) -> Dict:
|
@@ -223,10 +232,7 @@ class ModelRetentionSetting:
|
|
223
232
|
return cls(
|
224
233
|
archive_after_days=data.get("archive_after_days"),
|
225
234
|
delete_after_days=data.get("delete_after_days"),
|
226
|
-
customer_notification_type=
|
227
|
-
data.get("customer_notification_type")
|
228
|
-
)
|
229
|
-
or None,
|
235
|
+
customer_notification_type=data.get("customer_notification_type") or None,
|
230
236
|
)
|
231
237
|
|
232
238
|
def to_json(self) -> str:
|
@@ -245,13 +251,23 @@ class ModelRetentionSetting:
|
|
245
251
|
|
246
252
|
def validate(self) -> bool:
|
247
253
|
"""Validates the retention settings details. Returns True if valid, False otherwise."""
|
248
|
-
return all(
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
254
|
+
return all(
|
255
|
+
[
|
256
|
+
self.archive_after_days is None
|
257
|
+
or (
|
258
|
+
isinstance(self.archive_after_days, int)
|
259
|
+
and self.archive_after_days >= 0
|
260
|
+
),
|
261
|
+
self.delete_after_days is None
|
262
|
+
or (
|
263
|
+
isinstance(self.delete_after_days, int)
|
264
|
+
and self.delete_after_days >= 0
|
265
|
+
),
|
266
|
+
isinstance(self.customer_notification_type, str)
|
267
|
+
and self.customer_notification_type
|
268
|
+
in CustomerNotificationType.values(),
|
269
|
+
]
|
270
|
+
)
|
255
271
|
|
256
272
|
def __repr__(self):
|
257
273
|
return self.to_yaml()
|
@@ -278,13 +294,13 @@ class ModelRetentionOperationDetails:
|
|
278
294
|
"""
|
279
295
|
|
280
296
|
def __init__(
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
297
|
+
self,
|
298
|
+
archive_state: Optional[SettingStatus] = None,
|
299
|
+
archive_state_details: Optional[str] = None,
|
300
|
+
delete_state: Optional[SettingStatus] = None,
|
301
|
+
delete_state_details: Optional[str] = None,
|
302
|
+
time_archival_scheduled: Optional[int] = None,
|
303
|
+
time_deletion_scheduled: Optional[int] = None,
|
288
304
|
):
|
289
305
|
self.archive_state = archive_state
|
290
306
|
self.archive_state_details = archive_state_details
|
@@ -308,9 +324,9 @@ class ModelRetentionOperationDetails:
|
|
308
324
|
def from_dict(cls, data: Dict) -> "ModelRetentionOperationDetails":
|
309
325
|
"""Constructs retention operation details from a dictionary."""
|
310
326
|
return cls(
|
311
|
-
archive_state=
|
327
|
+
archive_state=data.get("archive_state") or None,
|
312
328
|
archive_state_details=data.get("archive_state_details"),
|
313
|
-
delete_state=
|
329
|
+
delete_state=data.get("delete_state") or None,
|
314
330
|
delete_state_details=data.get("delete_state_details"),
|
315
331
|
time_archival_scheduled=data.get("time_archival_scheduled"),
|
316
332
|
time_deletion_scheduled=data.get("time_deletion_scheduled"),
|
@@ -334,8 +350,10 @@ class ModelRetentionOperationDetails:
|
|
334
350
|
"""Validates the retention operation details."""
|
335
351
|
return all(
|
336
352
|
[
|
337
|
-
self.archive_state is None
|
338
|
-
|
353
|
+
self.archive_state is None
|
354
|
+
or self.archive_state in SettingStatus.values(),
|
355
|
+
self.delete_state is None
|
356
|
+
or self.delete_state in SettingStatus.values(),
|
339
357
|
self.time_archival_scheduled is None
|
340
358
|
or isinstance(self.time_archival_scheduled, int),
|
341
359
|
self.time_deletion_scheduled is None
|
@@ -368,10 +386,10 @@ class ModelBackupOperationDetails:
|
|
368
386
|
"""
|
369
387
|
|
370
388
|
def __init__(
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
389
|
+
self,
|
390
|
+
backup_state: Optional[SettingStatus] = None,
|
391
|
+
backup_state_details: Optional[str] = None,
|
392
|
+
time_last_backup: Optional[int] = None,
|
375
393
|
):
|
376
394
|
self.backup_state = backup_state
|
377
395
|
self.backup_state_details = backup_state_details
|
@@ -389,7 +407,7 @@ class ModelBackupOperationDetails:
|
|
389
407
|
def from_dict(cls, data: Dict) -> "ModelBackupOperationDetails":
|
390
408
|
"""Constructs backup operation details from a dictionary."""
|
391
409
|
return cls(
|
392
|
-
backup_state=
|
410
|
+
backup_state=data.get("backup_state") or None,
|
393
411
|
backup_state_details=data.get("backup_state_details"),
|
394
412
|
time_last_backup=data.get("time_last_backup"),
|
395
413
|
)
|
@@ -411,8 +429,14 @@ class ModelBackupOperationDetails:
|
|
411
429
|
def validate(self) -> bool:
|
412
430
|
"""Validates the backup operation details."""
|
413
431
|
return not (
|
414
|
-
|
415
|
-
|
432
|
+
(
|
433
|
+
self.backup_state is not None
|
434
|
+
and self.backup_state not in SettingStatus.values()
|
435
|
+
)
|
436
|
+
or (
|
437
|
+
self.time_last_backup is not None
|
438
|
+
and not isinstance(self.time_last_backup, int)
|
439
|
+
)
|
416
440
|
)
|
417
441
|
|
418
442
|
def __repr__(self):
|
@@ -1042,7 +1066,7 @@ class DataScienceModel(Builder):
|
|
1042
1066
|
elif json_string:
|
1043
1067
|
json_data = json.loads(json_string)
|
1044
1068
|
elif json_uri:
|
1045
|
-
with open(json_uri
|
1069
|
+
with open(json_uri) as json_file:
|
1046
1070
|
json_data = json.load(json_file)
|
1047
1071
|
else:
|
1048
1072
|
raise ValueError("Must provide either a valid json string or URI location.")
|
@@ -1077,7 +1101,7 @@ class DataScienceModel(Builder):
|
|
1077
1101
|
return self.get_spec(self.CONST_RETENTION_SETTING)
|
1078
1102
|
|
1079
1103
|
def with_retention_setting(
|
1080
|
-
|
1104
|
+
self, retention_setting: Union[Dict, ModelRetentionSetting]
|
1081
1105
|
) -> "DataScienceModel":
|
1082
1106
|
"""
|
1083
1107
|
Sets the retention setting details for the model.
|
@@ -1106,7 +1130,7 @@ class DataScienceModel(Builder):
|
|
1106
1130
|
return self.get_spec(self.CONST_BACKUP_SETTING)
|
1107
1131
|
|
1108
1132
|
def with_backup_setting(
|
1109
|
-
|
1133
|
+
self, backup_setting: Union[Dict, ModelBackupSetting]
|
1110
1134
|
) -> "DataScienceModel":
|
1111
1135
|
"""
|
1112
1136
|
Sets the model's backup setting details.
|
@@ -1368,8 +1392,8 @@ class DataScienceModel(Builder):
|
|
1368
1392
|
shutil.rmtree(self.local_copy_dir, ignore_errors=True)
|
1369
1393
|
|
1370
1394
|
def restore_model(
|
1371
|
-
|
1372
|
-
|
1395
|
+
self,
|
1396
|
+
restore_model_for_hours_specified: Optional[int] = None,
|
1373
1397
|
) -> None:
|
1374
1398
|
"""
|
1375
1399
|
Restore archived model artifact.
|
@@ -1398,8 +1422,12 @@ class DataScienceModel(Builder):
|
|
1398
1422
|
|
1399
1423
|
# Optional: Validate restore_model_for_hours_specified
|
1400
1424
|
if restore_model_for_hours_specified is not None and (
|
1401
|
-
|
1402
|
-
|
1425
|
+
not isinstance(restore_model_for_hours_specified, int)
|
1426
|
+
or restore_model_for_hours_specified <= 0
|
1427
|
+
):
|
1428
|
+
raise ValueError(
|
1429
|
+
"restore_model_for_hours_specified must be a positive integer."
|
1430
|
+
)
|
1403
1431
|
|
1404
1432
|
self.dsc_model.restore_archived_model_artifact(
|
1405
1433
|
restore_model_for_hours_specified=restore_model_for_hours_specified,
|
@@ -1571,7 +1599,11 @@ class DataScienceModel(Builder):
|
|
1571
1599
|
|
1572
1600
|
@classmethod
|
1573
1601
|
def list(
|
1574
|
-
cls,
|
1602
|
+
cls,
|
1603
|
+
compartment_id: str = None,
|
1604
|
+
project_id: str = None,
|
1605
|
+
category: str = USER,
|
1606
|
+
**kwargs,
|
1575
1607
|
) -> List["DataScienceModel"]:
|
1576
1608
|
"""Lists datascience models in a given compartment.
|
1577
1609
|
|
@@ -1581,6 +1613,8 @@ class DataScienceModel(Builder):
|
|
1581
1613
|
The compartment OCID.
|
1582
1614
|
project_id: (str, optional). Defaults to `None`.
|
1583
1615
|
The project OCID.
|
1616
|
+
category: (str, optional). Defaults to `USER`.
|
1617
|
+
The category of Model. Allowed values are: "USER", "SERVICE"
|
1584
1618
|
kwargs
|
1585
1619
|
Additional keyword arguments for filtering models.
|
1586
1620
|
|
@@ -1592,13 +1626,17 @@ class DataScienceModel(Builder):
|
|
1592
1626
|
return [
|
1593
1627
|
cls()._update_from_oci_dsc_model(model)
|
1594
1628
|
for model in OCIDataScienceModel.list_resource(
|
1595
|
-
compartment_id, project_id=project_id, **kwargs
|
1629
|
+
compartment_id, project_id=project_id, category=category, **kwargs
|
1596
1630
|
)
|
1597
1631
|
]
|
1598
1632
|
|
1599
1633
|
@classmethod
|
1600
1634
|
def list_df(
|
1601
|
-
cls,
|
1635
|
+
cls,
|
1636
|
+
compartment_id: str = None,
|
1637
|
+
project_id: str = None,
|
1638
|
+
category: str = USER,
|
1639
|
+
**kwargs,
|
1602
1640
|
) -> "pandas.DataFrame":
|
1603
1641
|
"""Lists datascience models in a given compartment.
|
1604
1642
|
|
@@ -1608,6 +1646,8 @@ class DataScienceModel(Builder):
|
|
1608
1646
|
The compartment OCID.
|
1609
1647
|
project_id: (str, optional). Defaults to `None`.
|
1610
1648
|
The project OCID.
|
1649
|
+
category: (str, optional). Defaults to `None`.
|
1650
|
+
The category of Model.
|
1611
1651
|
kwargs
|
1612
1652
|
Additional keyword arguments for filtering models.
|
1613
1653
|
|
@@ -1618,7 +1658,7 @@ class DataScienceModel(Builder):
|
|
1618
1658
|
"""
|
1619
1659
|
records = []
|
1620
1660
|
for model in OCIDataScienceModel.list_resource(
|
1621
|
-
compartment_id, project_id=project_id, **kwargs
|
1661
|
+
compartment_id, project_id=project_id, category=category, **kwargs
|
1622
1662
|
):
|
1623
1663
|
records.append(
|
1624
1664
|
{
|
@@ -1721,7 +1761,7 @@ class DataScienceModel(Builder):
|
|
1721
1761
|
self.CONST_BACKUP_SETTING: ModelBackupSetting.to_dict,
|
1722
1762
|
self.CONST_RETENTION_SETTING: ModelRetentionSetting.to_dict,
|
1723
1763
|
self.CONST_BACKUP_OPERATION_DETAILS: ModelBackupOperationDetails.to_dict,
|
1724
|
-
self.CONST_RETENTION_OPERATION_DETAILS: ModelRetentionOperationDetails.to_dict
|
1764
|
+
self.CONST_RETENTION_OPERATION_DETAILS: ModelRetentionOperationDetails.to_dict,
|
1725
1765
|
}
|
1726
1766
|
|
1727
1767
|
# Update the main properties
|
@@ -1756,9 +1796,11 @@ class DataScienceModel(Builder):
|
|
1756
1796
|
# Update artifact info
|
1757
1797
|
try:
|
1758
1798
|
artifact_info = self.dsc_model.get_artifact_info()
|
1759
|
-
_, file_name_info =
|
1799
|
+
_, file_name_info = utils.parse_content_disposition(
|
1800
|
+
artifact_info["Content-Disposition"]
|
1801
|
+
)
|
1760
1802
|
|
1761
|
-
if self.dsc_model.
|
1803
|
+
if self.dsc_model.is_model_created_by_reference():
|
1762
1804
|
_, file_extension = os.path.splitext(file_name_info["filename"])
|
1763
1805
|
if file_extension.lower() == ".json":
|
1764
1806
|
bucket_uri, _ = self._download_file_description_artifact()
|
@@ -1767,7 +1809,6 @@ class DataScienceModel(Builder):
|
|
1767
1809
|
self.set_spec(self.CONST_ARTIFACT, file_name_info["filename"])
|
1768
1810
|
except:
|
1769
1811
|
pass
|
1770
|
-
|
1771
1812
|
return self
|
1772
1813
|
|
1773
1814
|
def to_dict(self) -> Dict:
|
@@ -2193,3 +2234,313 @@ class DataScienceModel(Builder):
|
|
2193
2234
|
else:
|
2194
2235
|
# model found case
|
2195
2236
|
self.model_file_description["models"].pop(modelSearchIdx)
|
2237
|
+
|
2238
|
+
def create_custom_metadata_artifact(
|
2239
|
+
self,
|
2240
|
+
metadata_key_name: str,
|
2241
|
+
artifact_path_or_content: str,
|
2242
|
+
path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
|
2243
|
+
) -> ModelMetadataArtifactDetails:
|
2244
|
+
"""Creates model custom metadata artifact for specified model.
|
2245
|
+
|
2246
|
+
Parameters
|
2247
|
+
----------
|
2248
|
+
metadata_key_name: str
|
2249
|
+
The name of the model custom metadata key
|
2250
|
+
|
2251
|
+
artifact_path_or_content: str
|
2252
|
+
The model custom metadata artifact path to be upload. It can also be the actual content of the custom metadata
|
2253
|
+
|
2254
|
+
path_type: MetadataArtifactPathType
|
2255
|
+
Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
|
2256
|
+
Specifies what type of path is to be provided for metadata artifact.
|
2257
|
+
Can be either local , oss or the actual content itself
|
2258
|
+
|
2259
|
+
Returns
|
2260
|
+
-------
|
2261
|
+
ModelMetadataArtifactDetails
|
2262
|
+
The model custom metadata artifact creation info.
|
2263
|
+
Example:
|
2264
|
+
{
|
2265
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2266
|
+
'opc-request-id': 'E4F7',
|
2267
|
+
'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
|
2268
|
+
'X-Content-Type-Options': 'nosniff',
|
2269
|
+
'Content-Length': '4029958',
|
2270
|
+
'Vary': 'Origin',
|
2271
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2272
|
+
'status': 204
|
2273
|
+
}
|
2274
|
+
|
2275
|
+
"""
|
2276
|
+
return self.dsc_model.create_custom_metadata_artifact(
|
2277
|
+
metadata_key_name=metadata_key_name,
|
2278
|
+
artifact_path=artifact_path_or_content,
|
2279
|
+
path_type=path_type,
|
2280
|
+
)
|
2281
|
+
|
2282
|
+
def create_defined_metadata_artifact(
|
2283
|
+
self,
|
2284
|
+
metadata_key_name: str,
|
2285
|
+
artifact_path_or_content: str,
|
2286
|
+
path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
|
2287
|
+
) -> ModelMetadataArtifactDetails:
|
2288
|
+
"""Creates model defined metadata artifact for specified model.
|
2289
|
+
|
2290
|
+
Parameters
|
2291
|
+
----------
|
2292
|
+
metadata_key_name: str
|
2293
|
+
The name of the model defined metadata key
|
2294
|
+
|
2295
|
+
artifact_path_or_content: str
|
2296
|
+
The model defined metadata artifact path to be upload. It can also be the actual content of the defined metadata
|
2297
|
+
|
2298
|
+
path_type: MetadataArtifactPathType
|
2299
|
+
Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
|
2300
|
+
Specifies what type of path is to be provided for metadata artifact.
|
2301
|
+
Can be either local , oss or the actual content itself
|
2302
|
+
|
2303
|
+
Returns
|
2304
|
+
-------
|
2305
|
+
ModelMetadataArtifactDetails
|
2306
|
+
The model defined metadata artifact creation info.
|
2307
|
+
Example:
|
2308
|
+
{
|
2309
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2310
|
+
'opc-request-id': 'E4F7',
|
2311
|
+
'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
|
2312
|
+
'X-Content-Type-Options': 'nosniff',
|
2313
|
+
'Content-Length': '4029958',
|
2314
|
+
'Vary': 'Origin',
|
2315
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2316
|
+
'status': 204
|
2317
|
+
}
|
2318
|
+
|
2319
|
+
"""
|
2320
|
+
return self.dsc_model.create_defined_metadata_artifact(
|
2321
|
+
metadata_key_name=metadata_key_name,
|
2322
|
+
artifact_path=artifact_path_or_content,
|
2323
|
+
path_type=path_type,
|
2324
|
+
)
|
2325
|
+
|
2326
|
+
def update_custom_metadata_artifact(
|
2327
|
+
self,
|
2328
|
+
metadata_key_name: str,
|
2329
|
+
artifact_path_or_content: str,
|
2330
|
+
path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
|
2331
|
+
) -> ModelMetadataArtifactDetails:
|
2332
|
+
"""Update model custom metadata artifact for specified model.
|
2333
|
+
|
2334
|
+
Parameters
|
2335
|
+
----------
|
2336
|
+
metadata_key_name: str
|
2337
|
+
The name of the model custom metadata key
|
2338
|
+
|
2339
|
+
artifact_path_or_content: str
|
2340
|
+
The model custom metadata artifact path. It can also be the actual content of the custom metadata
|
2341
|
+
|
2342
|
+
path_type: MetadataArtifactPathType
|
2343
|
+
Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
|
2344
|
+
Specifies what type of path is to be provided for metadata artifact.
|
2345
|
+
Can be either local , oss or the actual content itself
|
2346
|
+
|
2347
|
+
Returns
|
2348
|
+
-------
|
2349
|
+
ModelMetadataArtifactDetails
|
2350
|
+
The model custom metadata artifact update info.
|
2351
|
+
Example:
|
2352
|
+
{
|
2353
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2354
|
+
'opc-request-id': 'E4F7',
|
2355
|
+
'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
|
2356
|
+
'X-Content-Type-Options': 'nosniff',
|
2357
|
+
'Content-Length': '4029958',
|
2358
|
+
'Vary': 'Origin',
|
2359
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2360
|
+
'status': 204
|
2361
|
+
}
|
2362
|
+
|
2363
|
+
"""
|
2364
|
+
return self.dsc_model.update_custom_metadata_artifact(
|
2365
|
+
metadata_key_name=metadata_key_name,
|
2366
|
+
artifact_path=artifact_path_or_content,
|
2367
|
+
path_type=path_type,
|
2368
|
+
)
|
2369
|
+
|
2370
|
+
def update_defined_metadata_artifact(
|
2371
|
+
self,
|
2372
|
+
metadata_key_name: str,
|
2373
|
+
artifact_path_or_content: str,
|
2374
|
+
path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
|
2375
|
+
) -> ModelMetadataArtifactDetails:
|
2376
|
+
"""Update model defined metadata artifact for specified model.
|
2377
|
+
|
2378
|
+
Parameters
|
2379
|
+
----------
|
2380
|
+
metadata_key_name: str
|
2381
|
+
The name of the model defined metadata key
|
2382
|
+
|
2383
|
+
artifact_path_or_content: str
|
2384
|
+
The model defined metadata artifact path. It can also be the actual content of the defined metadata
|
2385
|
+
|
2386
|
+
path_type: MetadataArtifactPathType
|
2387
|
+
Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
|
2388
|
+
Specifies what type of path is to be provided for metadata artifact.
|
2389
|
+
Can be either local , oss or the actual content itself
|
2390
|
+
|
2391
|
+
Returns
|
2392
|
+
-------
|
2393
|
+
ModelMetadataArtifactDetails
|
2394
|
+
The model defined metadata artifact update info.
|
2395
|
+
Example:
|
2396
|
+
{
|
2397
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2398
|
+
'opc-request-id': 'E4F7',
|
2399
|
+
'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
|
2400
|
+
'X-Content-Type-Options': 'nosniff',
|
2401
|
+
'Content-Length': '4029958',
|
2402
|
+
'Vary': 'Origin',
|
2403
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2404
|
+
'status': 204
|
2405
|
+
}
|
2406
|
+
|
2407
|
+
"""
|
2408
|
+
return self.dsc_model.update_defined_metadata_artifact(
|
2409
|
+
metadata_key_name=metadata_key_name,
|
2410
|
+
artifact_path=artifact_path_or_content,
|
2411
|
+
path_type=path_type,
|
2412
|
+
)
|
2413
|
+
|
2414
|
+
def get_custom_metadata_artifact(
|
2415
|
+
self, metadata_key_name: str, target_dir: str, override: bool = False
|
2416
|
+
) -> bytes:
|
2417
|
+
"""Downloads model custom metadata artifact content for specified model metadata key.
|
2418
|
+
|
2419
|
+
Parameters
|
2420
|
+
----------
|
2421
|
+
metadata_key_name: str
|
2422
|
+
The name of the custom metadata key of the model
|
2423
|
+
|
2424
|
+
target_dir: str
|
2425
|
+
The local file path where downloaded model custom metadata artifact will be saved.
|
2426
|
+
|
2427
|
+
override: bool
|
2428
|
+
A boolean flag that controls downloaded metadata artifact file overwriting
|
2429
|
+
- If True, overwrites the file if it already exists.
|
2430
|
+
- If False (default), raises a `FileExistsError` if the file exists.
|
2431
|
+
Returns
|
2432
|
+
-------
|
2433
|
+
bytes
|
2434
|
+
File content of the custom metadata artifact
|
2435
|
+
|
2436
|
+
"""
|
2437
|
+
if not is_path_exists(target_dir):
|
2438
|
+
raise PathNotFoundError(f"Path : {target_dir} does not exist")
|
2439
|
+
|
2440
|
+
file_content = self.dsc_model.get_custom_metadata_artifact(
|
2441
|
+
metadata_key_name=metadata_key_name
|
2442
|
+
)
|
2443
|
+
artifact_file_path = os.path.join(target_dir, f"{metadata_key_name}")
|
2444
|
+
|
2445
|
+
if not override and os.path.exists(artifact_file_path):
|
2446
|
+
raise FileExistsError(f"File already exists: {artifact_file_path}")
|
2447
|
+
|
2448
|
+
with open(artifact_file_path, "wb") as _file:
|
2449
|
+
_file.write(file_content)
|
2450
|
+
logger.debug(f"Artifact downloaded to location - {artifact_file_path}")
|
2451
|
+
return file_content
|
2452
|
+
|
2453
|
+
def get_defined_metadata_artifact(
|
2454
|
+
self, metadata_key_name: str, target_dir: str, override: bool = False
|
2455
|
+
) -> bytes:
|
2456
|
+
"""Downloads model defined metadata artifact content for specified model metadata key.
|
2457
|
+
|
2458
|
+
Parameters
|
2459
|
+
----------
|
2460
|
+
metadata_key_name: str
|
2461
|
+
The name of the model metadatum in the metadata.
|
2462
|
+
|
2463
|
+
target_dir: str
|
2464
|
+
The local file path where downloaded model defined metadata artifact will be saved.
|
2465
|
+
|
2466
|
+
override: bool
|
2467
|
+
A boolean flag that controls downloaded metadata artifact file overwriting
|
2468
|
+
- If True, overwrites the file if it already exists.
|
2469
|
+
- If False (default), raises a `FileExistsError` if the file exists.
|
2470
|
+
Returns
|
2471
|
+
-------
|
2472
|
+
bytes
|
2473
|
+
File content of the custom metadata artifact
|
2474
|
+
|
2475
|
+
"""
|
2476
|
+
if not is_path_exists(target_dir):
|
2477
|
+
raise PathNotFoundError(f"Path : {target_dir} does not exist")
|
2478
|
+
|
2479
|
+
file_content = self.dsc_model.get_defined_metadata_artifact(
|
2480
|
+
metadata_key_name=metadata_key_name
|
2481
|
+
)
|
2482
|
+
artifact_file_path = os.path.join(target_dir, f"{metadata_key_name}")
|
2483
|
+
|
2484
|
+
if not override and os.path.exists(artifact_file_path):
|
2485
|
+
raise FileExistsError(f"File already exists: {artifact_file_path}")
|
2486
|
+
|
2487
|
+
with open(artifact_file_path, "wb") as _file:
|
2488
|
+
_file.write(file_content)
|
2489
|
+
logger.debug(f"Artifact downloaded to location - {artifact_file_path}")
|
2490
|
+
return file_content
|
2491
|
+
|
2492
|
+
def delete_custom_metadata_artifact(
|
2493
|
+
self, metadata_key_name: str
|
2494
|
+
) -> ModelMetadataArtifactDetails:
|
2495
|
+
"""Deletes model custom metadata artifact for specified model metadata key.
|
2496
|
+
|
2497
|
+
Parameters
|
2498
|
+
----------
|
2499
|
+
metadata_key_name: str
|
2500
|
+
The name of the model metadatum in the metadata.
|
2501
|
+
Returns
|
2502
|
+
-------
|
2503
|
+
ModelMetadataArtifactDetails
|
2504
|
+
The model custom metadata artifact delete call info.
|
2505
|
+
Example:
|
2506
|
+
{
|
2507
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2508
|
+
'opc-request-id': 'E4F7',
|
2509
|
+
'X-Content-Type-Options': 'nosniff',
|
2510
|
+
'Vary': 'Origin',
|
2511
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2512
|
+
'status': 204
|
2513
|
+
}
|
2514
|
+
|
2515
|
+
"""
|
2516
|
+
return self.dsc_model.delete_custom_metadata_artifact(
|
2517
|
+
metadata_key_name=metadata_key_name
|
2518
|
+
)
|
2519
|
+
|
2520
|
+
def delete_defined_metadata_artifact(
|
2521
|
+
self, metadata_key_name: str
|
2522
|
+
) -> ModelMetadataArtifactDetails:
|
2523
|
+
"""Deletes model defined metadata artifact for specified model metadata key.
|
2524
|
+
|
2525
|
+
Parameters
|
2526
|
+
----------
|
2527
|
+
metadata_key_name: str
|
2528
|
+
The name of the model metadatum in the metadata.
|
2529
|
+
Returns
|
2530
|
+
-------
|
2531
|
+
ModelMetadataArtifactDetails
|
2532
|
+
The model defined metadata artifact delete call info.
|
2533
|
+
Example:
|
2534
|
+
{
|
2535
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2536
|
+
'opc-request-id': 'E4F7',
|
2537
|
+
'X-Content-Type-Options': 'nosniff',
|
2538
|
+
'Vary': 'Origin',
|
2539
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2540
|
+
'status': 204
|
2541
|
+
}
|
2542
|
+
|
2543
|
+
"""
|
2544
|
+
return self.dsc_model.delete_defined_metadata_artifact(
|
2545
|
+
metadata_key_name=metadata_key_name
|
2546
|
+
)
|