oracle-ads 2.13.12__py3-none-any.whl → 2.13.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/app.py +60 -8
- ads/aqua/cli.py +2 -0
- ads/aqua/client/client.py +38 -21
- ads/aqua/client/openai_client.py +20 -10
- ads/aqua/common/entities.py +58 -18
- ads/aqua/constants.py +2 -0
- ads/aqua/extension/common_handler.py +47 -2
- ads/aqua/extension/model_handler.py +50 -8
- ads/aqua/model/constants.py +1 -0
- ads/aqua/model/model.py +115 -56
- ads/aqua/modeldeployment/deployment.py +93 -20
- ads/aqua/modeldeployment/entities.py +146 -0
- ads/aqua/modeldeployment/model_group_config.py +7 -14
- ads/aqua/verify_policies/__init__.py +8 -0
- ads/aqua/verify_policies/constants.py +13 -0
- ads/aqua/verify_policies/entities.py +29 -0
- ads/aqua/verify_policies/messages.py +101 -0
- ads/aqua/verify_policies/utils.py +432 -0
- ads/aqua/verify_policies/verify.py +345 -0
- ads/aqua/version.json +3 -0
- ads/common/oci_logging.py +4 -7
- ads/model/deployment/model_deployment.py +51 -38
- {oracle_ads-2.13.12.dist-info → oracle_ads-2.13.13.dist-info}/METADATA +2 -1
- {oracle_ads-2.13.12.dist-info → oracle_ads-2.13.13.dist-info}/RECORD +27 -20
- {oracle_ads-2.13.12.dist-info → oracle_ads-2.13.13.dist-info}/WHEEL +0 -0
- {oracle_ads-2.13.12.dist-info → oracle_ads-2.13.13.dist-info}/entry_points.txt +0 -0
- {oracle_ads-2.13.12.dist-info → oracle_ads-2.13.13.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,13 @@
|
|
1
|
+
OBS_MANAGE_TEST_FILE = "AQUA Policy Verification - OBJECT STORAGE"
|
2
|
+
TEST_MODEL_NAME="AQUA Policy Verification - Model"
|
3
|
+
TEST_MD_NAME="AQUA Policy Verification - Model Deployment"
|
4
|
+
TEST_JOB_NAME="AQUA Policy Verification - Job"
|
5
|
+
TEST_JOB_RUN_NAME="AQUA Policy Verification - Job Run"
|
6
|
+
TEST_MVS_NAME="AQUA Policy Verification - Model Version Set"
|
7
|
+
TEST_VM_SHAPE="VM.Standard.E4.Flex"
|
8
|
+
TEST_DEFAULT_JOB_SHAPE = "VM.Standard.E3.Flex"
|
9
|
+
TEST_LIMIT_NAME = "ds-gpu-a10-count"
|
10
|
+
DUMMY_TEST_BYTE = b"7IV6cktUGcHIhur4bXTv"
|
11
|
+
POLICY_HELP_LINK = "https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/ai-quick-actions/policies/README.md"
|
12
|
+
|
13
|
+
|
@@ -0,0 +1,29 @@
|
|
1
|
+
from dataclasses import dataclass
|
2
|
+
from ads.common.extended_enum import ExtendedEnum
|
3
|
+
from ads.common.serializer import DataClassSerializable
|
4
|
+
|
5
|
+
|
6
|
+
class PolicyStatus(ExtendedEnum):
|
7
|
+
SUCCESS = "SUCCESS"
|
8
|
+
FAILURE = "FAILURE"
|
9
|
+
UNVERIFIED = "UNVERIFIED"
|
10
|
+
|
11
|
+
|
12
|
+
@dataclass(repr=False)
|
13
|
+
class OperationResultSuccess(DataClassSerializable):
|
14
|
+
operation: str
|
15
|
+
status: PolicyStatus = PolicyStatus.SUCCESS
|
16
|
+
|
17
|
+
|
18
|
+
@dataclass(repr=False)
|
19
|
+
class OperationResultFailure(DataClassSerializable):
|
20
|
+
operation: str
|
21
|
+
error: str
|
22
|
+
policy_hint: str
|
23
|
+
status: PolicyStatus = PolicyStatus.FAILURE
|
24
|
+
|
25
|
+
|
26
|
+
@dataclass(repr=False)
|
27
|
+
class CommonSettings(DataClassSerializable):
|
28
|
+
compartment_id: str
|
29
|
+
project_id: str
|
@@ -0,0 +1,101 @@
|
|
1
|
+
from ads.aqua.verify_policies.utils import VerifyPoliciesUtils
|
2
|
+
|
3
|
+
utils = VerifyPoliciesUtils()
|
4
|
+
operation_messages = {
|
5
|
+
utils.list_compartments.__name__: {
|
6
|
+
"name": "List Compartments",
|
7
|
+
"error": "Unable to retrieve the list of compartments. Please verify that you have the required permissions to list compartments in your tenancy. ",
|
8
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to inspect compartments in tenancy"
|
9
|
+
},
|
10
|
+
utils.list_models.__name__: {
|
11
|
+
"name": "List Models",
|
12
|
+
"error": "Failed to fetch available models. Ensure that the policies allow you to list models from the Model Catalog in the selected compartment.",
|
13
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-models in compartment <your-compartment-name>"
|
14
|
+
},
|
15
|
+
utils.list_project.__name__: {
|
16
|
+
"name": "List Projects",
|
17
|
+
"error": "Failed to list Data Science projects. Verify that you have the appropriate permission to access projects in the selected compartment.",
|
18
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-projects in compartment <your-compartment-name>"
|
19
|
+
},
|
20
|
+
utils.list_model_version_sets.__name__: {
|
21
|
+
"name": "List Model Version Sets",
|
22
|
+
"error": "Unable to load model version sets. Check your access rights to list model version sets in the selected compartment.",
|
23
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-modelversionsets in compartment <your-compartment-name>"
|
24
|
+
},
|
25
|
+
utils.list_jobs.__name__: {
|
26
|
+
"name": "List Jobs",
|
27
|
+
"error": "Job list could not be retrieved. Please confirm that you have the necessary permissions to view jobs in the compartment.",
|
28
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-jobs in compartment <your-compartment-name>"
|
29
|
+
},
|
30
|
+
utils.list_job_runs.__name__: {
|
31
|
+
"name": "List Job Runs",
|
32
|
+
"error": "Job Runs list could not be retrieved. Please confirm that you have the necessary permissions to view job runs in the compartme",
|
33
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-job-runs in compartment <your-compartment-name>"
|
34
|
+
},
|
35
|
+
utils.list_buckets.__name__: {
|
36
|
+
"name": "List Object Storage Buckets",
|
37
|
+
"error": "Cannot fetch Object Storage buckets. Verify that you have access to list buckets within the specified compartment.",
|
38
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to read objectstorage-namespaces in compartment <your-compartment-name>\nAllow dynamic-group aqua-dynamic-group to read buckets in compartment <your-compartment-name>"
|
39
|
+
},
|
40
|
+
utils.manage_bucket.__name__: {
|
41
|
+
"name": "Object Storage Access",
|
42
|
+
"error": "Failed to access the Object Storage bucket. Verify that the bucket exists and you have write permissions.",
|
43
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage object-family in compartment <your-compartment-name> where any {target.bucket.name='<your-bucket-name>'}"
|
44
|
+
},
|
45
|
+
utils.list_log_groups.__name__: {
|
46
|
+
"name": "List Log Groups",
|
47
|
+
"error": "Log groups or logs could not be retrieved. Please confirm you have logging read access for the selected compartment.",
|
48
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to use logging-family in compartment <your-compartment-name>"
|
49
|
+
},
|
50
|
+
utils.get_resource_availability.__name__: {
|
51
|
+
"name": "Verify Shape Limits",
|
52
|
+
"error": "Failed to retrieve shape limits for your compartment. Make sure the required policies are in place to read shape and quota data.",
|
53
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to read resource-availability in compartment <your-compartment-name>"
|
54
|
+
},
|
55
|
+
utils.register_model.__name__: {
|
56
|
+
"name": "Register Model",
|
57
|
+
"error": "Model registration failed. Ensure you have the correct permissions to write to the Model Catalog and access Object Storage.",
|
58
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-models in compartment <your-compartment-name>"
|
59
|
+
},
|
60
|
+
utils.aqua_model.delete_model.__name__: {
|
61
|
+
"name": "Delete Model",
|
62
|
+
"error": "Could not delete model. Please confirm you have delete permissions for Model Catalog resources in the compartment.",
|
63
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-models in compartment <your-compartment-name>"
|
64
|
+
},
|
65
|
+
utils.create_job.__name__: {
|
66
|
+
"name": "Create Job",
|
67
|
+
"error": "Job could not be created. Please check if you have permissions to create Data Science jobs.",
|
68
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-jobs in compartment <your-compartment-name>"
|
69
|
+
},
|
70
|
+
utils.create_job_run.__name__: {
|
71
|
+
"name": "Create Job Run",
|
72
|
+
"error": "Job Run could not be created. Confirm that you are allowed to run jobs in the selected compartment.",
|
73
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-job-runs in compartment <your-compartment-name>"
|
74
|
+
},
|
75
|
+
"delete_job": {
|
76
|
+
"name": "Delete Job",
|
77
|
+
"error": "Job could not be deleted. Please check if you have permissions to delete Data Science jobs.",
|
78
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-jobs in compartment <your-compartment-name>"
|
79
|
+
},
|
80
|
+
utils.aqua_model.create_model_version_set.__name__: {
|
81
|
+
"name": "Create Model Version Set",
|
82
|
+
"error": "Unable to create a model version set for storing evaluation results. Ensure that required Model Catalog permissions are set.",
|
83
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-modelversionsets in compartment <your-compartment-name>"
|
84
|
+
},
|
85
|
+
utils.aqua_model.ds_client.delete_model_version_set.__name__: {
|
86
|
+
"name": "Delete Model Version Set",
|
87
|
+
"error": "Unable to delete a model version. Ensure that required Model Catalog permissions are set.",
|
88
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-modelversionsets in compartment <your-compartment-name>"
|
89
|
+
},
|
90
|
+
utils.create_model_deployment.__name__: {
|
91
|
+
"name": "Create Model Deployment",
|
92
|
+
"error": "Model deployment could not be created. Confirm you have correct permissions to deploy models to the Model Deployment service.",
|
93
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-model-deployments in compartment <your-compartment-name>"
|
94
|
+
},
|
95
|
+
utils.aqua_model.ds_client.delete_model_deployment.__name__: {
|
96
|
+
"name": "Delete Model Deployment",
|
97
|
+
"error": "Unable to delete the model deployment. Please check if you have appropriate permissions to manage deployments.",
|
98
|
+
"policy_hint": "Allow dynamic-group aqua-dynamic-group to manage data-science-model-deployments in compartment <your-compartment-name>"
|
99
|
+
}
|
100
|
+
|
101
|
+
}
|
@@ -0,0 +1,432 @@
|
|
1
|
+
import logging
|
2
|
+
from ads.aqua.model.model import AquaModelApp
|
3
|
+
from ads.aqua.verify_policies.constants import DUMMY_TEST_BYTE, OBS_MANAGE_TEST_FILE, TEST_DEFAULT_JOB_SHAPE, TEST_MD_NAME, \
|
4
|
+
TEST_MODEL_NAME
|
5
|
+
from ads.aqua.verify_policies.entities import PolicyStatus
|
6
|
+
from ads.common.auth import default_signer
|
7
|
+
from ads.common.oci_mixin import LIFECYCLE_STOP_STATE
|
8
|
+
from ads.config import COMPARTMENT_OCID, DATA_SCIENCE_SERVICE_NAME, TENANCY_OCID, PROJECT_OCID
|
9
|
+
from ads.common import oci_client
|
10
|
+
from rich.console import Console
|
11
|
+
from rich.logging import RichHandler
|
12
|
+
|
13
|
+
logger = logging.getLogger("aqua.policies")
|
14
|
+
|
15
|
+
import oci
|
16
|
+
|
17
|
+
class PolicyValidationError(Exception):
|
18
|
+
def __init__(self, error: str):
|
19
|
+
super().__init__(error)
|
20
|
+
|
21
|
+
class VerifyPoliciesUtils:
|
22
|
+
"""
|
23
|
+
Utility class for verifying OCI IAM policies through operations on Data Science resources.
|
24
|
+
Provides methods to interact with models, model deployments, jobs, object storage, and limits APIs
|
25
|
+
using Oracle Accelerated Data Science (ADS) SDK.
|
26
|
+
"""
|
27
|
+
def __init__(self):
|
28
|
+
self.aqua_model = AquaModelApp()
|
29
|
+
self.obs_client = oci_client.OCIClientFactory(**default_signer()).object_storage
|
30
|
+
self.model_id = None
|
31
|
+
self.job_id = None
|
32
|
+
self.limit = 3
|
33
|
+
|
34
|
+
def list_compartments(self, **kwargs):
|
35
|
+
"""
|
36
|
+
List compartments in a given tenancy.
|
37
|
+
|
38
|
+
Parameters:
|
39
|
+
compartment_id (str, optional): OCID of the parent compartment. Defaults to TENANCY_OCID.
|
40
|
+
limit (int, optional): Maximum number of compartments to return. Defaults to 3.
|
41
|
+
|
42
|
+
Returns:
|
43
|
+
List[oci.identity.models.Compartment]: List of compartment data objects.
|
44
|
+
"""
|
45
|
+
compartment_id = kwargs.pop("compartment_id", TENANCY_OCID)
|
46
|
+
limit = kwargs.pop("limit", self.limit)
|
47
|
+
return self.aqua_model.identity_client.list_compartments(compartment_id=compartment_id, limit=limit,
|
48
|
+
**kwargs).data
|
49
|
+
|
50
|
+
def list_models(self, **kwargs):
|
51
|
+
"""
|
52
|
+
List models registered in Data Science.
|
53
|
+
|
54
|
+
Parameters:
|
55
|
+
**kwParameters: Filters such as display_name, lifecycle_state, etc.
|
56
|
+
|
57
|
+
Returns:
|
58
|
+
List[oci.data_science.models.Model]: List of model metadata.
|
59
|
+
"""
|
60
|
+
return self.aqua_model.list(**kwargs)
|
61
|
+
|
62
|
+
def list_log_groups(self, **kwargs):
|
63
|
+
"""
|
64
|
+
List log groups in the compartment.
|
65
|
+
|
66
|
+
Parameters:
|
67
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
68
|
+
limit (int, optional): Maximum results. Defaults to 3.
|
69
|
+
|
70
|
+
Returns:
|
71
|
+
List[oci.logging.models.LogGroupSummary]: List of log groups.
|
72
|
+
"""
|
73
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
74
|
+
limit = kwargs.pop("limit", self.limit)
|
75
|
+
return self.aqua_model.logging_client.list_log_groups(compartment_id=compartment_id, limit=limit, **kwargs).data
|
76
|
+
|
77
|
+
def list_log(self, **kwargs):
|
78
|
+
"""
|
79
|
+
List logs under a specific log group.
|
80
|
+
|
81
|
+
Parameters:
|
82
|
+
log_group_id (str): OCID of the log group.
|
83
|
+
limit (int, optional): Maximum number of logs to return. Defaults to 3.
|
84
|
+
|
85
|
+
Returns:
|
86
|
+
List[oci.logging.models.LogSummary]: List of log metadata.
|
87
|
+
"""
|
88
|
+
log_group_id = kwargs.pop("log_group_id")
|
89
|
+
limit = kwargs.pop("limit", self.limit)
|
90
|
+
return self.aqua_model.logging_client.list_logs(log_group_id=log_group_id, limit=limit, **kwargs).data
|
91
|
+
|
92
|
+
def list_project(self, **kwargs):
|
93
|
+
"""
|
94
|
+
List Data Science projects in a compartment.
|
95
|
+
|
96
|
+
Parameters:
|
97
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
98
|
+
limit (int, optional): Maximum number of projects to return. Defaults to 3.
|
99
|
+
|
100
|
+
Returns:
|
101
|
+
List[oci.data_science.models.ProjectSummary]: List of project summaries.
|
102
|
+
"""
|
103
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
104
|
+
limit = kwargs.pop("limit", self.limit)
|
105
|
+
return self.aqua_model.ds_client.list_projects(compartment_id=compartment_id, limit=limit, **kwargs).data
|
106
|
+
|
107
|
+
def list_model_version_sets(self, **kwargs):
|
108
|
+
"""
|
109
|
+
List model version sets in a compartment.
|
110
|
+
|
111
|
+
Parameters:
|
112
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
113
|
+
limit (int, optional): Max results. Defaults to 3.
|
114
|
+
|
115
|
+
Returns:
|
116
|
+
List[oci.data_science.models.ModelVersionSetSummary]: List of version sets.
|
117
|
+
"""
|
118
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
119
|
+
limit = kwargs.pop("limit", self.limit)
|
120
|
+
return self.aqua_model.ds_client.list_model_version_sets(compartment_id=compartment_id, limit=limit,
|
121
|
+
**kwargs).data
|
122
|
+
|
123
|
+
def list_jobs(self, **kwargs):
|
124
|
+
"""
|
125
|
+
List Data Science jobs in a compartment.
|
126
|
+
|
127
|
+
Parameters:
|
128
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
129
|
+
limit (int, optional): Max results. Defaults to 3.
|
130
|
+
|
131
|
+
Returns:
|
132
|
+
List[oci.data_science.models.JobSummary]: List of job summaries.
|
133
|
+
"""
|
134
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
135
|
+
limit = kwargs.pop("limit", self.limit)
|
136
|
+
return self.aqua_model.ds_client.list_jobs(compartment_id=compartment_id, limit=limit, **kwargs).data
|
137
|
+
|
138
|
+
def list_job_runs(self, **kwargs):
|
139
|
+
"""
|
140
|
+
List job runs in a compartment.
|
141
|
+
|
142
|
+
Parameters:
|
143
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
144
|
+
limit (int, optional): Max results. Defaults to 3.
|
145
|
+
|
146
|
+
Returns:
|
147
|
+
List[oci.data_science.models.JobRunSummary]: List of job run records.
|
148
|
+
"""
|
149
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
150
|
+
limit = kwargs.pop("limit", self.limit)
|
151
|
+
return self.aqua_model.ds_client.list_job_runs(compartment_id=compartment_id, limit=limit, **kwargs).data
|
152
|
+
|
153
|
+
def list_buckets(self, **kwargs):
|
154
|
+
"""
|
155
|
+
List Object Storage buckets in a compartment.
|
156
|
+
|
157
|
+
Parameters:
|
158
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
159
|
+
limit (int, optional): Max results. Defaults to self.limit.
|
160
|
+
|
161
|
+
Returns:
|
162
|
+
List[oci.object_storage.models.BucketSummary]: List of buckets.
|
163
|
+
"""
|
164
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
165
|
+
limit = kwargs.pop("limit", self.limit)
|
166
|
+
namespace_name = self.obs_client.get_namespace(compartment_id=compartment_id).data
|
167
|
+
return self.obs_client.list_buckets(namespace_name=namespace_name, compartment_id=compartment_id, limit=limit,
|
168
|
+
**kwargs).data
|
169
|
+
|
170
|
+
def manage_bucket(self, **kwargs):
|
171
|
+
"""
|
172
|
+
Verify Object Storage access by creating and deleting a test file in a bucket.
|
173
|
+
|
174
|
+
Parameters:
|
175
|
+
bucket (str): Name of the bucket to test access in.
|
176
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
177
|
+
|
178
|
+
Returns:
|
179
|
+
bool: True if test object operations succeeded.
|
180
|
+
"""
|
181
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
182
|
+
namespace_name = self.obs_client.get_namespace(compartment_id=compartment_id).data
|
183
|
+
bucket = kwargs.pop("bucket")
|
184
|
+
logger.info(f"Creating file in object storage with name {OBS_MANAGE_TEST_FILE} in bucket {bucket}")
|
185
|
+
self.obs_client.put_object(namespace_name, bucket, object_name=OBS_MANAGE_TEST_FILE, put_object_body="TEST")
|
186
|
+
logger.info(f"Deleting file {OBS_MANAGE_TEST_FILE} from object storage")
|
187
|
+
self.obs_client.delete_object(namespace_name, bucket, object_name=OBS_MANAGE_TEST_FILE)
|
188
|
+
return True
|
189
|
+
|
190
|
+
def list_model_deployment_shapes(self, **kwargs):
|
191
|
+
"""
|
192
|
+
List available model deployment compute shapes.
|
193
|
+
|
194
|
+
Parameters:
|
195
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
196
|
+
limit (int, optional): Max results. Defaults to 3.
|
197
|
+
|
198
|
+
Returns:
|
199
|
+
List[oci.data_science.models.ModelDeploymentShapeSummary]: List of shapes.
|
200
|
+
"""
|
201
|
+
limit = kwargs.pop("limit", self.limit)
|
202
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
203
|
+
return self.aqua_model.ds_client.list_model_deployment_shapes(compartment_id=compartment_id, limit=limit,
|
204
|
+
**kwargs).data
|
205
|
+
|
206
|
+
def get_resource_availability(self, **kwargs):
|
207
|
+
"""
|
208
|
+
Get quota availability for a specific resource.
|
209
|
+
|
210
|
+
Parameters:
|
211
|
+
limit_name (str): Name of the limit (e.g., 'ds-gpu-a10-count').
|
212
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
213
|
+
|
214
|
+
Returns:
|
215
|
+
oci.limits.models.ResourceAvailability: Quota availability information.
|
216
|
+
"""
|
217
|
+
limits_client = oci_client.OCIClientFactory(**default_signer()).limits
|
218
|
+
limit_name = kwargs.pop("limit_name")
|
219
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
220
|
+
return limits_client.get_resource_availability(compartment_id=compartment_id,
|
221
|
+
service_name=DATA_SCIENCE_SERVICE_NAME,
|
222
|
+
limit_name=limit_name).data
|
223
|
+
|
224
|
+
def register_model(self, **kwargs):
|
225
|
+
"""
|
226
|
+
Register a new model with test metadata and a dummy artifact.
|
227
|
+
|
228
|
+
Parameters:
|
229
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
230
|
+
project_id (str, optional): Project OCID. Defaults to PROJECT_OCID.
|
231
|
+
|
232
|
+
Returns:
|
233
|
+
str: OCID of the registered model.
|
234
|
+
"""
|
235
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
236
|
+
project_id = kwargs.pop("project_id", PROJECT_OCID)
|
237
|
+
|
238
|
+
create_model_details = oci.data_science.models.CreateModelDetails(
|
239
|
+
compartment_id=compartment_id,
|
240
|
+
project_id=project_id,
|
241
|
+
display_name=TEST_MODEL_NAME
|
242
|
+
)
|
243
|
+
logger.info(f"Registering test model `{TEST_MODEL_NAME}`")
|
244
|
+
model_id = self.aqua_model.ds_client.create_model(create_model_details=create_model_details).data.id
|
245
|
+
self.aqua_model.ds_client.create_model_artifact(
|
246
|
+
model_id=model_id,
|
247
|
+
model_artifact=DUMMY_TEST_BYTE
|
248
|
+
).data
|
249
|
+
self.model_id = model_id
|
250
|
+
return model_id
|
251
|
+
|
252
|
+
def create_model_deployment(self, **kwargs):
|
253
|
+
"""
|
254
|
+
Create and deploy a model using a predefined container image and configuration.
|
255
|
+
|
256
|
+
Parameters:
|
257
|
+
model_id (str): OCID of the model to deploy.
|
258
|
+
instance_shape (str): Compute shape to use (e.g., 'VM.Standard2.1').
|
259
|
+
|
260
|
+
Returns:
|
261
|
+
str: OCID of the created model deployment.
|
262
|
+
"""
|
263
|
+
model_id = kwargs.pop("model_id")
|
264
|
+
instance_shape = kwargs.pop("instance_shape")
|
265
|
+
model_deployment_instance_shape_config_details = oci.data_science.models.ModelDeploymentInstanceShapeConfigDetails(
|
266
|
+
ocpus=1,
|
267
|
+
memory_in_gbs=6)
|
268
|
+
instance_configuration = oci.data_science.models.InstanceConfiguration(
|
269
|
+
instance_shape_name=instance_shape,
|
270
|
+
model_deployment_instance_shape_config_details=model_deployment_instance_shape_config_details
|
271
|
+
)
|
272
|
+
model_configuration_details = oci.data_science.models.ModelConfigurationDetails(
|
273
|
+
model_id=model_id,
|
274
|
+
instance_configuration=instance_configuration
|
275
|
+
)
|
276
|
+
|
277
|
+
model_deployment_configuration_details = oci.data_science.models.SingleModelDeploymentConfigurationDetails(
|
278
|
+
model_configuration_details=model_configuration_details
|
279
|
+
)
|
280
|
+
create_model_deployment_details = oci.data_science.models.CreateModelDeploymentDetails(
|
281
|
+
compartment_id=COMPARTMENT_OCID,
|
282
|
+
project_id=PROJECT_OCID,
|
283
|
+
display_name=TEST_MD_NAME,
|
284
|
+
model_deployment_configuration_details=model_deployment_configuration_details,
|
285
|
+
)
|
286
|
+
md_ocid = self.aqua_model.ds_client.create_model_deployment(
|
287
|
+
create_model_deployment_details=create_model_deployment_details).data.id
|
288
|
+
waiter_result = oci.wait_until(
|
289
|
+
self.aqua_model.ds_client,
|
290
|
+
self.aqua_model.ds_client.get_model_deployment(md_ocid),
|
291
|
+
evaluate_response=lambda r: self._evaluate_response(wait_message="Waiting for Model Deployment to finish",
|
292
|
+
response=r),
|
293
|
+
max_interval_seconds=30,
|
294
|
+
)
|
295
|
+
logger.info("Model Deployment may result in FAILED state.")
|
296
|
+
return md_ocid
|
297
|
+
|
298
|
+
def _evaluate_response(self, wait_message, response):
|
299
|
+
logger.info(f"{wait_message}, Current state: {response.data.lifecycle_state}")
|
300
|
+
return getattr(response.data, 'lifecycle_state').upper() in LIFECYCLE_STOP_STATE
|
301
|
+
|
302
|
+
def create_job(self, **kwargs):
|
303
|
+
"""
|
304
|
+
Create a standalone Data Science job with default config and dummy artifact.
|
305
|
+
|
306
|
+
Parameters:
|
307
|
+
display_name (str): Display name of the job.
|
308
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
309
|
+
project_id (str, optional): Project OCID. Defaults to PROJECT_OCID.
|
310
|
+
shape_name (str, optional): Compute shape name. Defaults to TEST_DEFAULT_JOB_SHAPE.
|
311
|
+
subnet_id (str, optional): Optional subnet ID.
|
312
|
+
|
313
|
+
Returns:
|
314
|
+
str: OCID of the created job.
|
315
|
+
"""
|
316
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
317
|
+
project_id = kwargs.pop("project_id", PROJECT_OCID)
|
318
|
+
shape_name = kwargs.pop("shape_name", TEST_DEFAULT_JOB_SHAPE)
|
319
|
+
display_name = kwargs.pop("display_name")
|
320
|
+
subnet_id = kwargs.pop("subnet_id", None)
|
321
|
+
job_infrastructure_type = "STANDALONE" if subnet_id is not None else "ME_STANDALONE"
|
322
|
+
|
323
|
+
response = self.aqua_model.ds_client.create_job(
|
324
|
+
create_job_details=oci.data_science.models.CreateJobDetails(
|
325
|
+
display_name=display_name,
|
326
|
+
project_id=project_id,
|
327
|
+
compartment_id=compartment_id,
|
328
|
+
job_configuration_details=oci.data_science.models.DefaultJobConfigurationDetails(
|
329
|
+
job_type="DEFAULT",
|
330
|
+
environment_variables={}),
|
331
|
+
job_infrastructure_configuration_details=oci.data_science.models.StandaloneJobInfrastructureConfigurationDetails(
|
332
|
+
job_infrastructure_type=job_infrastructure_type,
|
333
|
+
shape_name=shape_name,
|
334
|
+
subnet_id=subnet_id,
|
335
|
+
job_shape_config_details=oci.data_science.models.JobShapeConfigDetails(
|
336
|
+
ocpus=1,
|
337
|
+
memory_in_gbs=16),
|
338
|
+
block_storage_size_in_gbs=50
|
339
|
+
)
|
340
|
+
)
|
341
|
+
)
|
342
|
+
|
343
|
+
job_id = response.data.id
|
344
|
+
self.aqua_model.ds_client.create_job_artifact(job_id=job_id, job_artifact=b"echo OK\n",
|
345
|
+
content_disposition="attachment; filename=entry.sh")
|
346
|
+
self.job_id = job_id
|
347
|
+
return job_id
|
348
|
+
|
349
|
+
def create_job_run(self, **kwargs):
|
350
|
+
"""
|
351
|
+
Start a job run from an existing job and wait for its completion.
|
352
|
+
|
353
|
+
Parameters:
|
354
|
+
job_id (str): OCID of the job to run.
|
355
|
+
display_name (str): Display name of the job run.
|
356
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
357
|
+
project_id (str, optional): Project OCID. Defaults to PROJECT_OCID.
|
358
|
+
|
359
|
+
Returns:
|
360
|
+
oci.data_science.models.JobRun: Job run response after completion.
|
361
|
+
"""
|
362
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
363
|
+
project_id = kwargs.pop("project_id", PROJECT_OCID)
|
364
|
+
job_id = kwargs.pop("job_id")
|
365
|
+
display_name = kwargs.pop("display_name")
|
366
|
+
response = self.aqua_model.ds_client.create_job_run(
|
367
|
+
create_job_run_details=oci.data_science.models.CreateJobRunDetails(
|
368
|
+
project_id=project_id,
|
369
|
+
compartment_id=compartment_id,
|
370
|
+
job_id=job_id,
|
371
|
+
display_name=display_name
|
372
|
+
)
|
373
|
+
)
|
374
|
+
job_run_id = response.data.id
|
375
|
+
|
376
|
+
waiter_result = oci.wait_until(
|
377
|
+
self.aqua_model.ds_client,
|
378
|
+
self.aqua_model.ds_client.get_job_run(job_run_id),
|
379
|
+
evaluate_response=lambda r: self._evaluate_response(wait_message="Waiting for job run to finish",
|
380
|
+
response=r),
|
381
|
+
max_interval_seconds=30,
|
382
|
+
max_wait_seconds=600
|
383
|
+
)
|
384
|
+
|
385
|
+
job_run_status = waiter_result.data
|
386
|
+
if job_run_status.lifecycle_state == "FAILED":
|
387
|
+
logger.warning(f"Job run failed: {job_run_status.lifecycle_details}")
|
388
|
+
raise PolicyValidationError("Job Run Failed")
|
389
|
+
return job_run_status
|
390
|
+
|
391
|
+
def create_model_version_set(self, **kwargs):
|
392
|
+
"""
|
393
|
+
Create a new model version set with the specified name.
|
394
|
+
|
395
|
+
Parameters:
|
396
|
+
name (str): Name of the model version set.
|
397
|
+
compartment_id (str, optional): Compartment OCID. Defaults to COMPARTMENT_OCID.
|
398
|
+
project_id (str, optional): Project OCID. Defaults to PROJECT_OCID.
|
399
|
+
|
400
|
+
Returns:
|
401
|
+
oci.data_science.models.ModelVersionSet: Model version set creation response.
|
402
|
+
"""
|
403
|
+
name = kwargs.pop("name")
|
404
|
+
compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
|
405
|
+
project_id = kwargs.pop("project_id", PROJECT_OCID)
|
406
|
+
return self.aqua_model.create_model_version_set(model_version_set_name=name, compartment_id=compartment_id,
|
407
|
+
project_id=project_id)
|
408
|
+
|
409
|
+
|
410
|
+
class RichStatusLog:
|
411
|
+
def __init__(self):
|
412
|
+
self.console = Console()
|
413
|
+
# logger = logging.("aqua.policies")
|
414
|
+
handler = RichHandler(console=self.console,
|
415
|
+
markup=True,
|
416
|
+
rich_tracebacks=False,
|
417
|
+
show_time=False,
|
418
|
+
show_path=False)
|
419
|
+
logger.addHandler(handler)
|
420
|
+
logger.propagate = False
|
421
|
+
self.logger = logger
|
422
|
+
|
423
|
+
def get_logger(self):
|
424
|
+
return self.logger
|
425
|
+
|
426
|
+
def get_status_emoji(self, status: PolicyStatus):
|
427
|
+
if status == PolicyStatus.SUCCESS:
|
428
|
+
return ":white_check_mark:[green]"
|
429
|
+
if status == PolicyStatus.FAILURE:
|
430
|
+
return ":cross_mark:[red]"
|
431
|
+
if status == PolicyStatus.UNVERIFIED:
|
432
|
+
return ":exclamation_question_mark:[yellow]"
|