oracle-ads 2.10.1__py3-none-any.whl → 2.11.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/__init__.py +12 -0
- ads/aqua/base.py +324 -0
- ads/aqua/cli.py +19 -0
- ads/aqua/config/deployment_config_defaults.json +9 -0
- ads/aqua/config/resource_limit_names.json +7 -0
- ads/aqua/constants.py +45 -0
- ads/aqua/data.py +40 -0
- ads/aqua/decorator.py +101 -0
- ads/aqua/deployment.py +643 -0
- ads/aqua/dummy_data/icon.txt +1 -0
- ads/aqua/dummy_data/oci_model_deployments.json +56 -0
- ads/aqua/dummy_data/oci_models.json +1 -0
- ads/aqua/dummy_data/readme.md +26 -0
- ads/aqua/evaluation.py +1751 -0
- ads/aqua/exception.py +82 -0
- ads/aqua/extension/__init__.py +40 -0
- ads/aqua/extension/base_handler.py +138 -0
- ads/aqua/extension/common_handler.py +21 -0
- ads/aqua/extension/deployment_handler.py +202 -0
- ads/aqua/extension/evaluation_handler.py +135 -0
- ads/aqua/extension/finetune_handler.py +66 -0
- ads/aqua/extension/model_handler.py +59 -0
- ads/aqua/extension/ui_handler.py +201 -0
- ads/aqua/extension/utils.py +23 -0
- ads/aqua/finetune.py +579 -0
- ads/aqua/job.py +29 -0
- ads/aqua/model.py +819 -0
- ads/aqua/training/__init__.py +4 -0
- ads/aqua/training/exceptions.py +459 -0
- ads/aqua/ui.py +453 -0
- ads/aqua/utils.py +715 -0
- ads/cli.py +37 -6
- ads/common/decorator/__init__.py +7 -3
- ads/common/decorator/require_nonempty_arg.py +65 -0
- ads/common/object_storage_details.py +166 -7
- ads/common/oci_client.py +18 -1
- ads/common/oci_logging.py +2 -2
- ads/common/oci_mixin.py +4 -5
- ads/common/serializer.py +34 -5
- ads/common/utils.py +75 -10
- ads/config.py +40 -1
- ads/jobs/ads_job.py +43 -25
- ads/jobs/builders/infrastructure/base.py +4 -2
- ads/jobs/builders/infrastructure/dsc_job.py +49 -39
- ads/jobs/builders/runtimes/base.py +71 -1
- ads/jobs/builders/runtimes/container_runtime.py +4 -4
- ads/jobs/builders/runtimes/pytorch_runtime.py +10 -63
- ads/jobs/templates/driver_pytorch.py +27 -10
- ads/model/artifact_downloader.py +84 -14
- ads/model/artifact_uploader.py +25 -23
- ads/model/datascience_model.py +388 -38
- ads/model/deployment/model_deployment.py +10 -2
- ads/model/generic_model.py +8 -0
- ads/model/model_file_description_schema.json +68 -0
- ads/model/model_metadata.py +1 -1
- ads/model/service/oci_datascience_model.py +34 -5
- ads/opctl/operator/lowcode/anomaly/README.md +2 -1
- ads/opctl/operator/lowcode/anomaly/__main__.py +10 -4
- ads/opctl/operator/lowcode/anomaly/environment.yaml +2 -1
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +12 -6
- ads/opctl/operator/lowcode/forecast/README.md +3 -2
- ads/opctl/operator/lowcode/forecast/environment.yaml +3 -2
- ads/opctl/operator/lowcode/forecast/model/automlx.py +12 -23
- ads/telemetry/base.py +62 -0
- ads/telemetry/client.py +105 -0
- ads/telemetry/telemetry.py +6 -3
- {oracle_ads-2.10.1.dist-info → oracle_ads-2.11.1.dist-info}/METADATA +37 -7
- {oracle_ads-2.10.1.dist-info → oracle_ads-2.11.1.dist-info}/RECORD +71 -36
- {oracle_ads-2.10.1.dist-info → oracle_ads-2.11.1.dist-info}/LICENSE.txt +0 -0
- {oracle_ads-2.10.1.dist-info → oracle_ads-2.11.1.dist-info}/WHEEL +0 -0
- {oracle_ads-2.10.1.dist-info → oracle_ads-2.11.1.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,59 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# Copyright (c) 2024 Oracle and/or its affiliates.
|
4
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
5
|
+
|
6
|
+
from urllib.parse import urlparse
|
7
|
+
|
8
|
+
from tornado.web import HTTPError
|
9
|
+
|
10
|
+
from ads.aqua.decorator import handle_exceptions
|
11
|
+
from ads.aqua.extension.base_handler import AquaAPIhandler
|
12
|
+
from ads.aqua.model import AquaModelApp
|
13
|
+
|
14
|
+
|
15
|
+
class AquaModelHandler(AquaAPIhandler):
|
16
|
+
"""Handler for Aqua Model REST APIs."""
|
17
|
+
|
18
|
+
@handle_exceptions
|
19
|
+
def get(self, model_id=""):
|
20
|
+
"""Handle GET request."""
|
21
|
+
if not model_id:
|
22
|
+
return self.list()
|
23
|
+
return self.read(model_id)
|
24
|
+
|
25
|
+
def read(self, model_id):
|
26
|
+
"""Read the information of an Aqua model."""
|
27
|
+
return self.finish(AquaModelApp().get(model_id))
|
28
|
+
|
29
|
+
@handle_exceptions
|
30
|
+
def delete(self, id=""):
|
31
|
+
"""Handles DELETE request for clearing cache"""
|
32
|
+
url_parse = urlparse(self.request.path)
|
33
|
+
paths = url_parse.path.strip("/")
|
34
|
+
if paths.startswith("aqua/model/cache"):
|
35
|
+
return self.finish(AquaModelApp().clear_model_list_cache())
|
36
|
+
else:
|
37
|
+
raise HTTPError(400, f"The request {self.request.path} is invalid.")
|
38
|
+
|
39
|
+
def list(self):
|
40
|
+
"""List Aqua models."""
|
41
|
+
compartment_id = self.get_argument("compartment_id", default=None)
|
42
|
+
# project_id is no needed.
|
43
|
+
project_id = self.get_argument("project_id", default=None)
|
44
|
+
return self.finish(AquaModelApp().list(compartment_id, project_id))
|
45
|
+
|
46
|
+
class AquaModelLicenseHandler(AquaAPIhandler):
|
47
|
+
"""Handler for Aqua Model license REST APIs."""
|
48
|
+
|
49
|
+
@handle_exceptions
|
50
|
+
def get(self, model_id):
|
51
|
+
"""Handle GET request."""
|
52
|
+
|
53
|
+
model_id = model_id.split("/")[0]
|
54
|
+
return self.finish(AquaModelApp().load_license(model_id))
|
55
|
+
|
56
|
+
__handlers__ = [
|
57
|
+
("model/?([^/]*)", AquaModelHandler),
|
58
|
+
("model/?([^/]*)/license", AquaModelLicenseHandler),
|
59
|
+
]
|
@@ -0,0 +1,201 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# Copyright (c) 2024 Oracle and/or its affiliates.
|
4
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
5
|
+
|
6
|
+
from urllib.parse import urlparse
|
7
|
+
|
8
|
+
from tornado.web import HTTPError
|
9
|
+
|
10
|
+
from ads.aqua.data import Tags
|
11
|
+
from ads.aqua.decorator import handle_exceptions
|
12
|
+
from ads.aqua.extension.base_handler import AquaAPIhandler
|
13
|
+
from ads.aqua.ui import AquaUIApp
|
14
|
+
from ads.config import COMPARTMENT_OCID
|
15
|
+
|
16
|
+
|
17
|
+
class AquaUIHandler(AquaAPIhandler):
|
18
|
+
"""
|
19
|
+
Handler for Aqua UI REST APIs.
|
20
|
+
|
21
|
+
Methods
|
22
|
+
-------
|
23
|
+
get(self, id="")
|
24
|
+
Routes the request to fetch log groups, log ids details or compartments
|
25
|
+
list_log_groups(self, id: str)
|
26
|
+
Reads the AQUA deployment information.
|
27
|
+
list_logs(self, log_group_id: str, **kwargs)
|
28
|
+
Lists the specified log group's log objects.
|
29
|
+
list_compartments(self, **kwargs)
|
30
|
+
Lists the compartments in a compartment specified by ODSC_MODEL_COMPARTMENT_OCID env variable.
|
31
|
+
|
32
|
+
Raises
|
33
|
+
------
|
34
|
+
HTTPError: For various failure scenarios such as invalid input format, missing data, etc.
|
35
|
+
"""
|
36
|
+
|
37
|
+
def get(self, id=""):
|
38
|
+
"""Handle GET request."""
|
39
|
+
url_parse = urlparse(self.request.path)
|
40
|
+
paths = url_parse.path.strip("/")
|
41
|
+
if paths.startswith("aqua/logging"):
|
42
|
+
if not id:
|
43
|
+
return self.list_log_groups()
|
44
|
+
return self.list_logs(id)
|
45
|
+
elif paths.startswith("aqua/compartments/default"):
|
46
|
+
return self.get_default_compartment()
|
47
|
+
elif paths.startswith("aqua/compartments"):
|
48
|
+
return self.list_compartments()
|
49
|
+
elif paths.startswith("aqua/experiment"):
|
50
|
+
return self.list_experiments()
|
51
|
+
elif paths.startswith("aqua/versionsets"):
|
52
|
+
return self.list_model_version_sets()
|
53
|
+
elif paths.startswith("aqua/buckets"):
|
54
|
+
return self.list_buckets()
|
55
|
+
elif paths.startswith("aqua/job/shapes"):
|
56
|
+
return self.list_job_shapes()
|
57
|
+
elif paths.startswith("aqua/vcn"):
|
58
|
+
return self.list_vcn()
|
59
|
+
elif paths.startswith("aqua/subnets"):
|
60
|
+
return self.list_subnets()
|
61
|
+
elif paths.startswith("aqua/shapes/limit"):
|
62
|
+
return self.get_shape_availability()
|
63
|
+
elif paths.startswith("aqua/bucket/versioning"):
|
64
|
+
return self.is_bucket_versioned()
|
65
|
+
else:
|
66
|
+
raise HTTPError(400, f"The request {self.request.path} is invalid.")
|
67
|
+
|
68
|
+
@handle_exceptions
|
69
|
+
def delete(self, id=""):
|
70
|
+
"""Handles DELETE request for clearing cache"""
|
71
|
+
# todo: added for dev work, to be deleted if there's no feature to refresh cache in Aqua
|
72
|
+
url_parse = urlparse(self.request.path)
|
73
|
+
paths = url_parse.path.strip("/")
|
74
|
+
if paths.startswith("aqua/compartments/cache"):
|
75
|
+
return self.finish(AquaUIApp().clear_compartments_list_cache())
|
76
|
+
else:
|
77
|
+
raise HTTPError(400, f"The request {self.request.path} is invalid.")
|
78
|
+
|
79
|
+
@handle_exceptions
|
80
|
+
def list_log_groups(self, **kwargs):
|
81
|
+
"""Lists all log groups for the specified compartment or tenancy."""
|
82
|
+
compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
|
83
|
+
return self.finish(
|
84
|
+
AquaUIApp().list_log_groups(compartment_id=compartment_id, **kwargs)
|
85
|
+
)
|
86
|
+
|
87
|
+
@handle_exceptions
|
88
|
+
def list_logs(self, log_group_id: str, **kwargs):
|
89
|
+
"""Lists the specified log group's log objects."""
|
90
|
+
return self.finish(AquaUIApp().list_logs(log_group_id=log_group_id, **kwargs))
|
91
|
+
|
92
|
+
@handle_exceptions
|
93
|
+
def list_compartments(self):
|
94
|
+
"""Lists the compartments in a compartment specified by ODSC_MODEL_COMPARTMENT_OCID env variable."""
|
95
|
+
return self.finish(AquaUIApp().list_compartments())
|
96
|
+
|
97
|
+
@handle_exceptions
|
98
|
+
def get_default_compartment(self):
|
99
|
+
"""Returns user compartment ocid."""
|
100
|
+
return self.finish(AquaUIApp().get_default_compartment())
|
101
|
+
|
102
|
+
@handle_exceptions
|
103
|
+
def list_model_version_sets(self, **kwargs):
|
104
|
+
"""Lists all model version sets for the specified compartment or tenancy."""
|
105
|
+
|
106
|
+
compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
|
107
|
+
return self.finish(
|
108
|
+
AquaUIApp().list_model_version_sets(
|
109
|
+
compartment_id=compartment_id,
|
110
|
+
target_tag=Tags.AQUA_FINE_TUNING.value,
|
111
|
+
**kwargs,
|
112
|
+
)
|
113
|
+
)
|
114
|
+
|
115
|
+
@handle_exceptions
|
116
|
+
def list_experiments(self, **kwargs):
|
117
|
+
"""Lists all experiments for the specified compartment or tenancy."""
|
118
|
+
|
119
|
+
compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
|
120
|
+
return self.finish(
|
121
|
+
AquaUIApp().list_model_version_sets(
|
122
|
+
compartment_id=compartment_id,
|
123
|
+
target_tag=Tags.AQUA_EVALUATION.value,
|
124
|
+
**kwargs,
|
125
|
+
)
|
126
|
+
)
|
127
|
+
|
128
|
+
@handle_exceptions
|
129
|
+
def list_buckets(self, **kwargs):
|
130
|
+
"""Lists all model version sets for the specified compartment or tenancy."""
|
131
|
+
compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
|
132
|
+
versioned = self.get_argument("versioned", default=None)
|
133
|
+
versioned = True if versioned and versioned.lower() == "true" else False
|
134
|
+
|
135
|
+
return self.finish(
|
136
|
+
AquaUIApp().list_buckets(
|
137
|
+
compartment_id=compartment_id, versioned=versioned, **kwargs
|
138
|
+
)
|
139
|
+
)
|
140
|
+
|
141
|
+
@handle_exceptions
|
142
|
+
def list_job_shapes(self, **kwargs):
|
143
|
+
"""Lists job shapes available in the specified compartment."""
|
144
|
+
compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
|
145
|
+
return self.finish(
|
146
|
+
AquaUIApp().list_job_shapes(compartment_id=compartment_id, **kwargs)
|
147
|
+
)
|
148
|
+
|
149
|
+
@handle_exceptions
|
150
|
+
def list_vcn(self, **kwargs):
|
151
|
+
"""Lists the virtual cloud networks (VCNs) in the specified compartment."""
|
152
|
+
compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
|
153
|
+
return self.finish(
|
154
|
+
AquaUIApp().list_vcn(compartment_id=compartment_id, **kwargs)
|
155
|
+
)
|
156
|
+
|
157
|
+
@handle_exceptions
|
158
|
+
def list_subnets(self, **kwargs):
|
159
|
+
"""Lists the subnets in the specified VCN and the specified compartment."""
|
160
|
+
compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
|
161
|
+
vcn_id = self.get_argument("vcn_id")
|
162
|
+
return self.finish(
|
163
|
+
AquaUIApp().list_subnets(
|
164
|
+
compartment_id=compartment_id, vcn_id=vcn_id, **kwargs
|
165
|
+
)
|
166
|
+
)
|
167
|
+
|
168
|
+
@handle_exceptions
|
169
|
+
def get_shape_availability(self, **kwargs):
|
170
|
+
"""For a given compartmentId, resource limit name, and scope, returns the number of available resources associated
|
171
|
+
with the given limit."""
|
172
|
+
compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
|
173
|
+
instance_shape = self.get_argument("instance_shape")
|
174
|
+
|
175
|
+
return self.finish(
|
176
|
+
AquaUIApp().get_shape_availability(
|
177
|
+
compartment_id=compartment_id, instance_shape=instance_shape, **kwargs
|
178
|
+
)
|
179
|
+
)
|
180
|
+
|
181
|
+
@handle_exceptions
|
182
|
+
def is_bucket_versioned(self):
|
183
|
+
"""For a given compartmentId, resource limit name, and scope, returns the number of available resources associated
|
184
|
+
with the given limit."""
|
185
|
+
bucket_uri = self.get_argument("bucket_uri")
|
186
|
+
return self.finish(AquaUIApp().is_bucket_versioned(bucket_uri=bucket_uri))
|
187
|
+
|
188
|
+
|
189
|
+
__handlers__ = [
|
190
|
+
("logging/?([^/]*)", AquaUIHandler),
|
191
|
+
("compartments/?([^/]*)", AquaUIHandler),
|
192
|
+
# TODO: change url to evaluation/experiements/?([^/]*)
|
193
|
+
("experiment/?([^/]*)", AquaUIHandler),
|
194
|
+
("versionsets/?([^/]*)", AquaUIHandler),
|
195
|
+
("buckets/?([^/]*)", AquaUIHandler),
|
196
|
+
("job/shapes/?([^/]*)", AquaUIHandler),
|
197
|
+
("vcn/?([^/]*)", AquaUIHandler),
|
198
|
+
("subnets/?([^/]*)", AquaUIHandler),
|
199
|
+
("shapes/limit/?([^/]*)", AquaUIHandler),
|
200
|
+
("bucket/versioning/?([^/]*)", AquaUIHandler),
|
201
|
+
]
|
@@ -0,0 +1,23 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# Copyright (c) 2024 Oracle and/or its affiliates.
|
4
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
5
|
+
from dataclasses import fields
|
6
|
+
from typing import Dict, Optional
|
7
|
+
from requests import HTTPError
|
8
|
+
|
9
|
+
from ads.aqua.extension.base_handler import Errors
|
10
|
+
|
11
|
+
|
12
|
+
def validate_function_parameters(data_class, input_data: Dict):
|
13
|
+
"""Validates if the required parameters are provided in input data."""
|
14
|
+
required_parameters = [
|
15
|
+
field.name for field in fields(data_class)
|
16
|
+
if field.type != Optional[field.type]
|
17
|
+
]
|
18
|
+
|
19
|
+
for required_parameter in required_parameters:
|
20
|
+
if not input_data.get(required_parameter):
|
21
|
+
raise HTTPError(
|
22
|
+
400, Errors.MISSING_REQUIRED_PARAMETER.format(required_parameter)
|
23
|
+
)
|