oracle-ads 2.11.14__py3-none-any.whl → 2.11.16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/common/entities.py +17 -0
- ads/aqua/common/enums.py +5 -1
- ads/aqua/common/utils.py +109 -22
- ads/aqua/config/config.py +1 -1
- ads/aqua/config/deployment_config_defaults.json +29 -1
- ads/aqua/config/resource_limit_names.json +1 -0
- ads/aqua/constants.py +35 -18
- ads/aqua/evaluation/entities.py +0 -1
- ads/aqua/evaluation/evaluation.py +165 -121
- ads/aqua/extension/common_ws_msg_handler.py +57 -0
- ads/aqua/extension/deployment_handler.py +14 -13
- ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
- ads/aqua/extension/errors.py +1 -1
- ads/aqua/extension/evaluation_handler.py +4 -7
- ads/aqua/extension/evaluation_ws_msg_handler.py +28 -10
- ads/aqua/extension/model_handler.py +31 -6
- ads/aqua/extension/models/ws_models.py +78 -3
- ads/aqua/extension/models_ws_msg_handler.py +49 -0
- ads/aqua/extension/ui_websocket_handler.py +7 -1
- ads/aqua/model/entities.py +17 -9
- ads/aqua/model/model.py +260 -90
- ads/aqua/modeldeployment/constants.py +0 -16
- ads/aqua/modeldeployment/deployment.py +97 -74
- ads/aqua/modeldeployment/entities.py +9 -20
- ads/aqua/ui.py +152 -28
- ads/common/object_storage_details.py +2 -5
- ads/common/serializer.py +2 -3
- ads/jobs/builders/infrastructure/dsc_job.py +29 -3
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +74 -27
- ads/jobs/builders/runtimes/container_runtime.py +83 -4
- ads/opctl/operator/common/operator_config.py +1 -0
- ads/opctl/operator/lowcode/anomaly/README.md +3 -3
- ads/opctl/operator/lowcode/anomaly/__main__.py +5 -6
- ads/opctl/operator/lowcode/anomaly/const.py +9 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +6 -2
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +51 -26
- ads/opctl/operator/lowcode/anomaly/model/factory.py +41 -13
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +79 -0
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +79 -0
- ads/opctl/operator/lowcode/anomaly/operator_config.py +1 -0
- ads/opctl/operator/lowcode/anomaly/schema.yaml +16 -2
- ads/opctl/operator/lowcode/anomaly/utils.py +16 -13
- ads/opctl/operator/lowcode/common/data.py +2 -1
- ads/opctl/operator/lowcode/common/errors.py +6 -0
- ads/opctl/operator/lowcode/common/transformations.py +37 -9
- ads/opctl/operator/lowcode/common/utils.py +32 -10
- ads/opctl/operator/lowcode/forecast/model/base_model.py +21 -13
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +14 -18
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +15 -4
- ads/opctl/operator/lowcode/forecast/schema.yaml +9 -0
- ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
- ads/opctl/operator/lowcode/recommender/README.md +206 -0
- ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
- ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
- ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
- ads/opctl/operator/lowcode/recommender/constant.py +25 -0
- ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
- ads/opctl/operator/lowcode/recommender/model/base_model.py +198 -0
- ads/opctl/operator/lowcode/recommender/model/factory.py +58 -0
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
- ads/opctl/operator/lowcode/recommender/model/svd.py +88 -0
- ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
- ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
- ads/opctl/operator/lowcode/recommender/utils.py +13 -0
- ads/pipeline/ads_pipeline_run.py +13 -2
- {oracle_ads-2.11.14.dist-info → oracle_ads-2.11.16.dist-info}/METADATA +6 -1
- {oracle_ads-2.11.14.dist-info → oracle_ads-2.11.16.dist-info}/RECORD +70 -50
- {oracle_ads-2.11.14.dist-info → oracle_ads-2.11.16.dist-info}/LICENSE.txt +0 -0
- {oracle_ads-2.11.14.dist-info → oracle_ads-2.11.16.dist-info}/WHEEL +0 -0
- {oracle_ads-2.11.14.dist-info → oracle_ads-2.11.16.dist-info}/entry_points.txt +0 -0
@@ -1,5 +1,4 @@
|
|
1
1
|
#!/usr/bin/env python
|
2
|
-
# -*- coding: utf-8 -*-
|
3
2
|
# Copyright (c) 2024 Oracle and/or its affiliates.
|
4
3
|
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
5
4
|
import base64
|
@@ -8,7 +7,7 @@ import os
|
|
8
7
|
import re
|
9
8
|
import tempfile
|
10
9
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
11
|
-
from dataclasses import asdict
|
10
|
+
from dataclasses import asdict, fields
|
12
11
|
from datetime import datetime, timedelta
|
13
12
|
from pathlib import Path
|
14
13
|
from threading import Lock
|
@@ -47,17 +46,40 @@ from ads.aqua.common.utils import (
|
|
47
46
|
upload_local_to_os,
|
48
47
|
)
|
49
48
|
from ads.aqua.constants import (
|
49
|
+
CONSOLE_LINK_RESOURCE_TYPE_MAPPING,
|
50
|
+
EVALUATION_REPORT,
|
51
|
+
EVALUATION_REPORT_JSON,
|
52
|
+
EVALUATION_REPORT_MD,
|
50
53
|
JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING,
|
54
|
+
LIFECYCLE_DETAILS_MISSING_JOBRUN,
|
51
55
|
NB_SESSION_IDENTIFIER,
|
52
56
|
UNKNOWN,
|
53
|
-
CONSOLE_LINK_RESOURCE_TYPE_MAPPING,
|
54
57
|
)
|
55
|
-
from ads.aqua.evaluation.constants import
|
56
|
-
|
57
|
-
|
58
|
+
from ads.aqua.evaluation.constants import (
|
59
|
+
EVAL_TERMINATION_STATE,
|
60
|
+
EvaluationConfig,
|
61
|
+
EvaluationCustomMetadata,
|
62
|
+
EvaluationMetricResult,
|
63
|
+
EvaluationReportJson,
|
64
|
+
)
|
65
|
+
from ads.aqua.evaluation.entities import (
|
66
|
+
AquaEvalMetric,
|
67
|
+
AquaEvalMetrics,
|
68
|
+
AquaEvalMetricSummary,
|
69
|
+
AquaEvalParams,
|
70
|
+
AquaEvalReport,
|
71
|
+
AquaEvaluationCommands,
|
72
|
+
AquaEvaluationDetail,
|
73
|
+
AquaEvaluationSummary,
|
74
|
+
AquaResourceIdentifier,
|
75
|
+
CreateAquaEvaluationDetails,
|
76
|
+
ModelParams,
|
77
|
+
)
|
78
|
+
from ads.aqua.evaluation.errors import EVALUATION_JOB_EXIT_CODE_MESSAGE
|
79
|
+
from ads.aqua.ui import AquaContainerConfig
|
58
80
|
from ads.common.auth import default_signer
|
59
81
|
from ads.common.object_storage_details import ObjectStorageDetails
|
60
|
-
from ads.common.utils import get_console_link, get_files, get_log_links
|
82
|
+
from ads.common.utils import get_console_link, get_files, get_log_links
|
61
83
|
from ads.config import (
|
62
84
|
AQUA_JOB_SUBNET_ID,
|
63
85
|
COMPARTMENT_OCID,
|
@@ -69,7 +91,9 @@ from ads.jobs.builders.infrastructure.dsc_job import DataScienceJob
|
|
69
91
|
from ads.jobs.builders.runtimes.base import Runtime
|
70
92
|
from ads.jobs.builders.runtimes.container_runtime import ContainerRuntime
|
71
93
|
from ads.model.datascience_model import DataScienceModel
|
94
|
+
from ads.model.deployment import ModelDeploymentContainerRuntime
|
72
95
|
from ads.model.deployment.model_deployment import ModelDeployment
|
96
|
+
from ads.model.generic_model import ModelDeploymentRuntimeType
|
73
97
|
from ads.model.model_metadata import (
|
74
98
|
MetadataTaxonomyKeys,
|
75
99
|
ModelCustomMetadata,
|
@@ -134,19 +158,20 @@ class AquaEvaluationApp(AquaApp):
|
|
134
158
|
if not create_aqua_evaluation_details:
|
135
159
|
try:
|
136
160
|
create_aqua_evaluation_details = CreateAquaEvaluationDetails(**kwargs)
|
137
|
-
except:
|
161
|
+
except Exception as ex:
|
138
162
|
raise AquaValueError(
|
139
|
-
"Invalid create evaluation parameters.
|
140
|
-
|
141
|
-
|
163
|
+
"Invalid create evaluation parameters. "
|
164
|
+
"Allowable parameters are: "
|
165
|
+
f"{', '.join([field.name for field in fields(CreateAquaEvaluationDetails)])}."
|
166
|
+
) from ex
|
142
167
|
|
143
168
|
if not is_valid_ocid(create_aqua_evaluation_details.evaluation_source_id):
|
144
169
|
raise AquaValueError(
|
145
170
|
f"Invalid evaluation source {create_aqua_evaluation_details.evaluation_source_id}. "
|
146
171
|
"Specify either a model or model deployment id."
|
147
172
|
)
|
148
|
-
|
149
173
|
evaluation_source = None
|
174
|
+
eval_inference_configuration = None
|
150
175
|
if (
|
151
176
|
DataScienceResource.MODEL_DEPLOYMENT
|
152
177
|
in create_aqua_evaluation_details.evaluation_source_id
|
@@ -154,6 +179,28 @@ class AquaEvaluationApp(AquaApp):
|
|
154
179
|
evaluation_source = ModelDeployment.from_id(
|
155
180
|
create_aqua_evaluation_details.evaluation_source_id
|
156
181
|
)
|
182
|
+
try:
|
183
|
+
if (
|
184
|
+
evaluation_source.runtime.type
|
185
|
+
== ModelDeploymentRuntimeType.CONTAINER
|
186
|
+
):
|
187
|
+
runtime = ModelDeploymentContainerRuntime.from_dict(
|
188
|
+
evaluation_source.runtime.to_dict()
|
189
|
+
)
|
190
|
+
inference_config = AquaContainerConfig.from_container_index_json(
|
191
|
+
enable_spec=True
|
192
|
+
).inference
|
193
|
+
for container in inference_config.values():
|
194
|
+
if container.name == runtime.image.split(":")[0]:
|
195
|
+
eval_inference_configuration = (
|
196
|
+
container.spec.evaluation_configuration
|
197
|
+
)
|
198
|
+
except Exception:
|
199
|
+
logger.debug(
|
200
|
+
f"Could not load inference config details for the evaluation id: "
|
201
|
+
f"{create_aqua_evaluation_details.evaluation_source_id}. Please check if the container"
|
202
|
+
f" runtime has the correct SMC image information."
|
203
|
+
)
|
157
204
|
elif (
|
158
205
|
DataScienceResource.MODEL
|
159
206
|
in create_aqua_evaluation_details.evaluation_source_id
|
@@ -186,11 +233,11 @@ class AquaEvaluationApp(AquaApp):
|
|
186
233
|
auth=default_signer(),
|
187
234
|
force_overwrite=create_aqua_evaluation_details.force_overwrite,
|
188
235
|
)
|
189
|
-
except FileExistsError:
|
236
|
+
except FileExistsError as err:
|
190
237
|
raise AquaFileExistsError(
|
191
238
|
f"Dataset {dataset_file} already exists in {create_aqua_evaluation_details.report_path}. "
|
192
239
|
"Please use a new dataset file name, report path or set `force_overwrite` as True."
|
193
|
-
)
|
240
|
+
) from err
|
194
241
|
logger.debug(
|
195
242
|
f"Uploaded local file {evaluation_dataset_path} to object storage {dst_uri}."
|
196
243
|
)
|
@@ -210,11 +257,11 @@ class AquaEvaluationApp(AquaApp):
|
|
210
257
|
report_path=create_aqua_evaluation_details.report_path,
|
211
258
|
**create_aqua_evaluation_details.model_parameters,
|
212
259
|
)
|
213
|
-
except:
|
260
|
+
except Exception as ex:
|
214
261
|
raise AquaValueError(
|
215
262
|
"Invalid model parameters. Model parameters should "
|
216
263
|
f"be a dictionary with keys: {', '.join(list(ModelParams.__annotations__.keys()))}."
|
217
|
-
)
|
264
|
+
) from ex
|
218
265
|
|
219
266
|
target_compartment = (
|
220
267
|
create_aqua_evaluation_details.compartment_id or COMPARTMENT_OCID
|
@@ -244,7 +291,7 @@ class AquaEvaluationApp(AquaApp):
|
|
244
291
|
raise AquaValueError(
|
245
292
|
f"Invalid experiment name. Please provide an experiment with `{Tags.AQUA_EVALUATION}` in tags."
|
246
293
|
)
|
247
|
-
except:
|
294
|
+
except Exception:
|
248
295
|
logger.debug(
|
249
296
|
f"Model version set {experiment_model_version_set_name} doesn't exist. "
|
250
297
|
"Creating new model version set."
|
@@ -295,11 +342,7 @@ class AquaEvaluationApp(AquaApp):
|
|
295
342
|
evaluation_model_taxonomy_metadata = ModelTaxonomyMetadata()
|
296
343
|
evaluation_model_taxonomy_metadata[
|
297
344
|
MetadataTaxonomyKeys.HYPERPARAMETERS
|
298
|
-
].value = {
|
299
|
-
"model_params": {
|
300
|
-
key: value for key, value in asdict(evaluation_model_parameters).items()
|
301
|
-
}
|
302
|
-
}
|
345
|
+
].value = {"model_params": dict(asdict(evaluation_model_parameters))}
|
303
346
|
|
304
347
|
evaluation_model = (
|
305
348
|
DataScienceModel()
|
@@ -350,14 +393,13 @@ class AquaEvaluationApp(AquaApp):
|
|
350
393
|
)
|
351
394
|
if AQUA_JOB_SUBNET_ID:
|
352
395
|
evaluation_job.infrastructure.with_subnet_id(AQUA_JOB_SUBNET_ID)
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
)
|
396
|
+
elif NB_SESSION_IDENTIFIER in os.environ:
|
397
|
+
# apply default subnet id for job by setting ME_STANDALONE
|
398
|
+
# so as to avoid using the notebook session's networking when running on it
|
399
|
+
# https://accelerated-data-science.readthedocs.io/en/latest/user_guide/jobs/infra_and_runtime.html#networking
|
400
|
+
evaluation_job.infrastructure.with_job_infrastructure_type(
|
401
|
+
JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING
|
402
|
+
)
|
361
403
|
|
362
404
|
container_image = self._get_evaluation_container(
|
363
405
|
create_aqua_evaluation_details.evaluation_source_id
|
@@ -374,10 +416,11 @@ class AquaEvaluationApp(AquaApp):
|
|
374
416
|
report_path=create_aqua_evaluation_details.report_path,
|
375
417
|
model_parameters=create_aqua_evaluation_details.model_parameters,
|
376
418
|
metrics=create_aqua_evaluation_details.metrics,
|
419
|
+
inference_configuration=eval_inference_configuration.to_filtered_dict()
|
420
|
+
if eval_inference_configuration
|
421
|
+
else {},
|
377
422
|
)
|
378
|
-
).create(
|
379
|
-
**kwargs
|
380
|
-
) ## TODO: decide what parameters will be needed
|
423
|
+
).create(**kwargs) ## TODO: decide what parameters will be needed
|
381
424
|
logger.debug(
|
382
425
|
f"Successfully created evaluation job {evaluation_job.id} for {create_aqua_evaluation_details.evaluation_source_id}."
|
383
426
|
)
|
@@ -479,12 +522,12 @@ class AquaEvaluationApp(AquaApp):
|
|
479
522
|
region=self.region,
|
480
523
|
),
|
481
524
|
),
|
482
|
-
tags=
|
483
|
-
aqua_evaluation
|
484
|
-
evaluation_job_id
|
485
|
-
evaluation_source
|
486
|
-
evaluation_experiment_id
|
487
|
-
|
525
|
+
tags={
|
526
|
+
"aqua_evaluation": Tags.AQUA_EVALUATION,
|
527
|
+
"evaluation_job_id": evaluation_job.id,
|
528
|
+
"evaluation_source": create_aqua_evaluation_details.evaluation_source_id,
|
529
|
+
"evaluation_experiment_id": experiment_model_version_set_id,
|
530
|
+
},
|
488
531
|
parameters=AquaEvalParams(),
|
489
532
|
)
|
490
533
|
|
@@ -497,6 +540,7 @@ class AquaEvaluationApp(AquaApp):
|
|
497
540
|
report_path: str,
|
498
541
|
model_parameters: dict,
|
499
542
|
metrics: List = None,
|
543
|
+
inference_configuration: dict = None,
|
500
544
|
) -> Runtime:
|
501
545
|
"""Builds evaluation runtime for Job."""
|
502
546
|
# TODO the image name needs to be extracted from the mapping index.json file.
|
@@ -506,16 +550,19 @@ class AquaEvaluationApp(AquaApp):
|
|
506
550
|
.with_environment_variable(
|
507
551
|
**{
|
508
552
|
"AIP_SMC_EVALUATION_ARGUMENTS": json.dumps(
|
509
|
-
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
553
|
+
{
|
554
|
+
**asdict(
|
555
|
+
self._build_launch_cmd(
|
556
|
+
evaluation_id=evaluation_id,
|
557
|
+
evaluation_source_id=evaluation_source_id,
|
558
|
+
dataset_path=dataset_path,
|
559
|
+
report_path=report_path,
|
560
|
+
model_parameters=model_parameters,
|
561
|
+
metrics=metrics,
|
562
|
+
),
|
563
|
+
),
|
564
|
+
**(inference_configuration or {}),
|
565
|
+
},
|
519
566
|
),
|
520
567
|
"CONDA_BUCKET_NS": CONDA_BUCKET_NS,
|
521
568
|
},
|
@@ -526,7 +573,7 @@ class AquaEvaluationApp(AquaApp):
|
|
526
573
|
|
527
574
|
@staticmethod
|
528
575
|
def _get_service_model_name(
|
529
|
-
source: Union[ModelDeployment, DataScienceModel]
|
576
|
+
source: Union[ModelDeployment, DataScienceModel],
|
530
577
|
) -> str:
|
531
578
|
"""Gets the service model name from source. If it's ModelDeployment, needs to check
|
532
579
|
if its model has been fine tuned or not.
|
@@ -652,21 +699,21 @@ class AquaEvaluationApp(AquaApp):
|
|
652
699
|
try:
|
653
700
|
log = utils.query_resource(log_id, return_all=False)
|
654
701
|
log_name = log.display_name if log else ""
|
655
|
-
except:
|
702
|
+
except Exception:
|
656
703
|
pass
|
657
704
|
|
658
705
|
if loggroup_id:
|
659
706
|
try:
|
660
707
|
loggroup = utils.query_resource(loggroup_id, return_all=False)
|
661
708
|
loggroup_name = loggroup.display_name if loggroup else ""
|
662
|
-
except:
|
709
|
+
except Exception:
|
663
710
|
pass
|
664
711
|
|
665
712
|
try:
|
666
713
|
introspection = json.loads(
|
667
714
|
self._get_attribute_from_model_metadata(resource, "ArtifactTestResults")
|
668
715
|
)
|
669
|
-
except:
|
716
|
+
except Exception:
|
670
717
|
introspection = {}
|
671
718
|
|
672
719
|
summary = AquaEvaluationDetail(
|
@@ -685,19 +732,13 @@ class AquaEvaluationApp(AquaApp):
|
|
685
732
|
return summary
|
686
733
|
|
687
734
|
@telemetry(entry_point="plugin=evaluation&action=list", name="aqua")
|
688
|
-
def list(
|
689
|
-
self, compartment_id: str = None, project_id: str = None, **kwargs
|
690
|
-
) -> List[AquaEvaluationSummary]:
|
735
|
+
def list(self, compartment_id: str = None) -> List[AquaEvaluationSummary]:
|
691
736
|
"""List Aqua evaluations in a given compartment and under certain project.
|
692
737
|
|
693
738
|
Parameters
|
694
739
|
----------
|
695
740
|
compartment_id: (str, optional). Defaults to `None`.
|
696
741
|
The compartment OCID.
|
697
|
-
project_id: (str, optional). Defaults to `None`.
|
698
|
-
The project OCID.
|
699
|
-
kwargs
|
700
|
-
Additional keyword arguments.
|
701
742
|
|
702
743
|
Returns
|
703
744
|
-------
|
@@ -718,7 +759,7 @@ class AquaEvaluationApp(AquaApp):
|
|
718
759
|
evaluations = []
|
719
760
|
async_tasks = []
|
720
761
|
for model in models:
|
721
|
-
if model.identifier in self._eval_cache
|
762
|
+
if model.identifier in self._eval_cache:
|
722
763
|
logger.debug(f"Retrieving evaluation {model.identifier} from cache.")
|
723
764
|
evaluations.append(self._eval_cache.get(model.identifier))
|
724
765
|
|
@@ -790,7 +831,7 @@ class AquaEvaluationApp(AquaApp):
|
|
790
831
|
"""Checks if the evaluation artifact exists."""
|
791
832
|
try:
|
792
833
|
response = self.ds_client.head_model_artifact(model_id=model.identifier)
|
793
|
-
return
|
834
|
+
return response.status == 200
|
794
835
|
except oci.exceptions.ServiceError as ex:
|
795
836
|
if ex.status == 404:
|
796
837
|
logger.debug(f"Evaluation artifact not found for {model.identifier}.")
|
@@ -846,18 +887,17 @@ class AquaEvaluationApp(AquaApp):
|
|
846
887
|
if job_run_details
|
847
888
|
else ""
|
848
889
|
)
|
849
|
-
|
850
|
-
|
851
|
-
id=eval_id,
|
890
|
+
return {
|
891
|
+
"id": eval_id,
|
852
892
|
**self._get_status(
|
853
893
|
model=eval,
|
854
894
|
jobrun=job_run_details,
|
855
895
|
),
|
856
|
-
log_id
|
857
|
-
log_url
|
858
|
-
loggroup_id
|
859
|
-
loggroup_url
|
860
|
-
|
896
|
+
"log_id": log_id,
|
897
|
+
"log_url": log_url,
|
898
|
+
"loggroup_id": loggroup_id,
|
899
|
+
"loggroup_url": loggroup_url,
|
900
|
+
}
|
861
901
|
|
862
902
|
def get_supported_metrics(self) -> dict:
|
863
903
|
"""Gets a list of supported metrics for evaluation."""
|
@@ -919,8 +959,8 @@ class AquaEvaluationApp(AquaApp):
|
|
919
959
|
AquaEvalMetrics:
|
920
960
|
An instance of AquaEvalMetrics.
|
921
961
|
"""
|
922
|
-
if eval_id in self._metrics_cache
|
923
|
-
logger.info(
|
962
|
+
if eval_id in self._metrics_cache:
|
963
|
+
logger.info("Returning metrics from cache.")
|
924
964
|
eval_metrics = self._metrics_cache.get(eval_id)
|
925
965
|
if len(eval_metrics.report) > 0:
|
926
966
|
return eval_metrics
|
@@ -934,14 +974,14 @@ class AquaEvaluationApp(AquaApp):
|
|
934
974
|
|
935
975
|
files_in_artifact = get_files(temp_dir)
|
936
976
|
md_report_content = self._read_from_artifact(
|
937
|
-
temp_dir, files_in_artifact,
|
977
|
+
temp_dir, files_in_artifact, EVALUATION_REPORT_MD
|
938
978
|
)
|
939
979
|
|
940
980
|
# json report not availiable for failed evaluation
|
941
981
|
try:
|
942
982
|
json_report = json.loads(
|
943
983
|
self._read_from_artifact(
|
944
|
-
temp_dir, files_in_artifact,
|
984
|
+
temp_dir, files_in_artifact, EVALUATION_REPORT_JSON
|
945
985
|
)
|
946
986
|
)
|
947
987
|
except Exception as e:
|
@@ -1028,8 +1068,8 @@ class AquaEvaluationApp(AquaApp):
|
|
1028
1068
|
AquaFileNotFoundError:
|
1029
1069
|
When missing `report.html` in evaluation artifact.
|
1030
1070
|
"""
|
1031
|
-
if eval_id in self._report_cache
|
1032
|
-
logger.info(
|
1071
|
+
if eval_id in self._report_cache:
|
1072
|
+
logger.info("Returning report from cache.")
|
1033
1073
|
report = self._report_cache.get(eval_id)
|
1034
1074
|
if report.content:
|
1035
1075
|
return report
|
@@ -1040,7 +1080,7 @@ class AquaEvaluationApp(AquaApp):
|
|
1040
1080
|
auth=self._auth,
|
1041
1081
|
)
|
1042
1082
|
content = self._read_from_artifact(
|
1043
|
-
temp_dir, get_files(temp_dir),
|
1083
|
+
temp_dir, get_files(temp_dir), EVALUATION_REPORT
|
1044
1084
|
)
|
1045
1085
|
|
1046
1086
|
report = AquaEvalReport(
|
@@ -1084,7 +1124,7 @@ class AquaEvaluationApp(AquaApp):
|
|
1084
1124
|
"Model provenance is missing job run training_id key"
|
1085
1125
|
)
|
1086
1126
|
|
1087
|
-
status =
|
1127
|
+
status = {"id": eval_id, "lifecycle_state": UNKNOWN, "time_accepted": UNKNOWN}
|
1088
1128
|
run = DataScienceJobRun.from_ocid(job_run_id)
|
1089
1129
|
if run.lifecycle_state in [
|
1090
1130
|
DataScienceJobRun.LIFECYCLE_STATE_ACCEPTED,
|
@@ -1092,11 +1132,11 @@ class AquaEvaluationApp(AquaApp):
|
|
1092
1132
|
DataScienceJobRun.LIFECYCLE_STATE_NEEDS_ATTENTION,
|
1093
1133
|
]:
|
1094
1134
|
self._cancel_job_run(run, model)
|
1095
|
-
status =
|
1096
|
-
id
|
1097
|
-
lifecycle_state
|
1098
|
-
time_accepted
|
1099
|
-
|
1135
|
+
status = {
|
1136
|
+
"id": eval_id,
|
1137
|
+
"lifecycle_state": "CANCELING",
|
1138
|
+
"time_accepted": datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f%z"),
|
1139
|
+
}
|
1100
1140
|
return status
|
1101
1141
|
|
1102
1142
|
@staticmethod
|
@@ -1142,10 +1182,10 @@ class AquaEvaluationApp(AquaApp):
|
|
1142
1182
|
job_id = model.custom_metadata_list.get(
|
1143
1183
|
EvaluationCustomMetadata.EVALUATION_JOB_ID
|
1144
1184
|
).value
|
1145
|
-
except Exception:
|
1185
|
+
except Exception as ex:
|
1146
1186
|
raise AquaMissingKeyError(
|
1147
1187
|
f"Custom metadata is missing {EvaluationCustomMetadata.EVALUATION_JOB_ID} key"
|
1148
|
-
)
|
1188
|
+
) from ex
|
1149
1189
|
|
1150
1190
|
job = DataScienceJob.from_id(job_id)
|
1151
1191
|
|
@@ -1163,11 +1203,11 @@ class AquaEvaluationApp(AquaApp):
|
|
1163
1203
|
self._eval_cache.pop(key=eval_id, default=None)
|
1164
1204
|
self._deletion_cache.__setitem__(key=eval_id, value="")
|
1165
1205
|
|
1166
|
-
status =
|
1167
|
-
id
|
1168
|
-
lifecycle_state
|
1169
|
-
time_accepted
|
1170
|
-
|
1206
|
+
status = {
|
1207
|
+
"id": eval_id,
|
1208
|
+
"lifecycle_state": jobrun.lifecycle_state if jobrun else "DELETING",
|
1209
|
+
"time_accepted": datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f%z"),
|
1210
|
+
}
|
1171
1211
|
return status
|
1172
1212
|
|
1173
1213
|
@staticmethod
|
@@ -1236,7 +1276,7 @@ class AquaEvaluationApp(AquaApp):
|
|
1236
1276
|
model.additional_details.get(RqsAdditionalDetails.METADATA),
|
1237
1277
|
target_attribute,
|
1238
1278
|
)
|
1239
|
-
except:
|
1279
|
+
except Exception:
|
1240
1280
|
logger.debug(
|
1241
1281
|
f"Missing `{target_attribute}` in custom metadata of the evaluation."
|
1242
1282
|
f"Evaluation id: {model.identifier} "
|
@@ -1254,7 +1294,7 @@ class AquaEvaluationApp(AquaApp):
|
|
1254
1294
|
def _get_source(
|
1255
1295
|
self,
|
1256
1296
|
evaluation: oci.resource_search.models.ResourceSummary,
|
1257
|
-
resources_mapping: dict =
|
1297
|
+
resources_mapping: dict = None,
|
1258
1298
|
) -> tuple:
|
1259
1299
|
"""Returns ocid and name of the model has been evaluated."""
|
1260
1300
|
source_id = self._get_attribute_from_model_metadata(
|
@@ -1263,14 +1303,16 @@ class AquaEvaluationApp(AquaApp):
|
|
1263
1303
|
)
|
1264
1304
|
|
1265
1305
|
try:
|
1266
|
-
|
1267
|
-
|
1268
|
-
source.
|
1269
|
-
|
1270
|
-
|
1271
|
-
|
1306
|
+
source_name = None
|
1307
|
+
if resources_mapping:
|
1308
|
+
source = resources_mapping.get(source_id)
|
1309
|
+
source_name = (
|
1310
|
+
source.display_name
|
1311
|
+
if source
|
1312
|
+
else self._get_attribute_from_model_metadata(
|
1313
|
+
evaluation, EvaluationCustomMetadata.EVALUATION_SOURCE_NAME
|
1314
|
+
)
|
1272
1315
|
)
|
1273
|
-
)
|
1274
1316
|
|
1275
1317
|
# try to resolve source_name from source id
|
1276
1318
|
if source_id and not source_name:
|
@@ -1286,13 +1328,13 @@ class AquaEvaluationApp(AquaApp):
|
|
1286
1328
|
raise AquaRuntimeError(
|
1287
1329
|
f"Not supported source type: {resource_type}"
|
1288
1330
|
)
|
1289
|
-
except Exception
|
1331
|
+
except Exception:
|
1290
1332
|
logger.debug(
|
1291
1333
|
f"Failed to retrieve source information for evaluation {evaluation.identifier}."
|
1292
1334
|
)
|
1293
1335
|
source_name = ""
|
1294
1336
|
|
1295
|
-
return
|
1337
|
+
return source_id, source_name
|
1296
1338
|
|
1297
1339
|
def _get_experiment_info(
|
1298
1340
|
self, model: oci.resource_search.models.ResourceSummary
|
@@ -1306,7 +1348,7 @@ class AquaEvaluationApp(AquaApp):
|
|
1306
1348
|
def _process(
|
1307
1349
|
self,
|
1308
1350
|
model: oci.resource_search.models.ResourceSummary,
|
1309
|
-
resources_mapping: dict =
|
1351
|
+
resources_mapping: dict = None,
|
1310
1352
|
) -> dict:
|
1311
1353
|
"""Constructs AquaEvaluationSummary from `oci.resource_search.models.ResourceSummary`."""
|
1312
1354
|
|
@@ -1320,25 +1362,27 @@ class AquaEvaluationApp(AquaApp):
|
|
1320
1362
|
ocid=model_id,
|
1321
1363
|
region=self.region,
|
1322
1364
|
)
|
1323
|
-
source_model_id, source_model_name = self._get_source(
|
1365
|
+
source_model_id, source_model_name = self._get_source(
|
1366
|
+
model, resources_mapping if resources_mapping else {}
|
1367
|
+
)
|
1324
1368
|
experiment_id, experiment_name = self._get_experiment_info(model)
|
1325
1369
|
parameters = self._fetch_runtime_params(model)
|
1326
1370
|
|
1327
|
-
return
|
1328
|
-
id
|
1329
|
-
name
|
1330
|
-
console_url
|
1331
|
-
time_created
|
1332
|
-
tags
|
1333
|
-
experiment
|
1371
|
+
return {
|
1372
|
+
"id": model_id,
|
1373
|
+
"name": model.display_name,
|
1374
|
+
"console_url": console_url,
|
1375
|
+
"time_created": model.time_created,
|
1376
|
+
"tags": tags,
|
1377
|
+
"experiment": self._build_resource_identifier(
|
1334
1378
|
id=experiment_id,
|
1335
1379
|
name=experiment_name,
|
1336
1380
|
),
|
1337
|
-
source
|
1381
|
+
"source": self._build_resource_identifier(
|
1338
1382
|
id=source_model_id, name=source_model_name
|
1339
1383
|
),
|
1340
|
-
parameters
|
1341
|
-
|
1384
|
+
"parameters": parameters,
|
1385
|
+
}
|
1342
1386
|
|
1343
1387
|
def _build_resource_identifier(
|
1344
1388
|
self, id: str = None, name: str = None
|
@@ -1465,7 +1509,7 @@ class AquaEvaluationApp(AquaApp):
|
|
1465
1509
|
job_run_status = jobrun.lifecycle_state
|
1466
1510
|
|
1467
1511
|
if jobrun is None:
|
1468
|
-
if model.identifier in self._deletion_cache
|
1512
|
+
if model.identifier in self._deletion_cache:
|
1469
1513
|
job_run_status = JobRun.LIFECYCLE_STATE_DELETED
|
1470
1514
|
|
1471
1515
|
elif self._get_attribute_from_model_metadata(
|
@@ -1484,20 +1528,20 @@ class AquaEvaluationApp(AquaApp):
|
|
1484
1528
|
|
1485
1529
|
try:
|
1486
1530
|
lifecycle_details = (
|
1487
|
-
|
1531
|
+
LIFECYCLE_DETAILS_MISSING_JOBRUN
|
1488
1532
|
if not jobrun
|
1489
1533
|
else self._extract_job_lifecycle_details(jobrun.lifecycle_details)
|
1490
1534
|
)
|
1491
|
-
except:
|
1535
|
+
except Exception:
|
1492
1536
|
# ResourceSummary does not have lifecycle_details attr
|
1493
1537
|
lifecycle_details = ""
|
1494
1538
|
|
1495
|
-
return
|
1496
|
-
lifecycle_state
|
1539
|
+
return {
|
1540
|
+
"lifecycle_state": (
|
1497
1541
|
lifecycle_state if isinstance(lifecycle_state, str) else lifecycle_state
|
1498
1542
|
),
|
1499
|
-
lifecycle_details
|
1500
|
-
|
1543
|
+
"lifecycle_details": lifecycle_details,
|
1544
|
+
}
|
1501
1545
|
|
1502
1546
|
def _prefetch_resources(self, compartment_id) -> dict:
|
1503
1547
|
"""Fetches all AQUA resources."""
|
@@ -1554,7 +1598,7 @@ class AquaEvaluationApp(AquaApp):
|
|
1554
1598
|
exit_code_message = EVALUATION_JOB_EXIT_CODE_MESSAGE.get(exit_code)
|
1555
1599
|
if exit_code_message:
|
1556
1600
|
message = f"{exit_code_message} Exit code: {exit_code}."
|
1557
|
-
except:
|
1601
|
+
except Exception:
|
1558
1602
|
pass
|
1559
1603
|
|
1560
1604
|
return message
|
@@ -0,0 +1,57 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
|
3
|
+
# Copyright (c) 2024 Oracle and/or its affiliates.
|
4
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
5
|
+
|
6
|
+
import json
|
7
|
+
from importlib import metadata
|
8
|
+
from typing import List, Union
|
9
|
+
|
10
|
+
from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID, fetch_service_compartment
|
11
|
+
from ads.aqua.common.decorator import handle_exceptions
|
12
|
+
from ads.aqua.common.errors import AquaResourceAccessError
|
13
|
+
from ads.aqua.common.utils import known_realm
|
14
|
+
from ads.aqua.extension.aqua_ws_msg_handler import AquaWSMsgHandler
|
15
|
+
from ads.aqua.extension.models.ws_models import (
|
16
|
+
AdsVersionResponse,
|
17
|
+
CompatibilityCheckResponse,
|
18
|
+
RequestResponseType,
|
19
|
+
)
|
20
|
+
|
21
|
+
|
22
|
+
class AquaCommonWsMsgHandler(AquaWSMsgHandler):
|
23
|
+
@staticmethod
|
24
|
+
def get_message_types() -> List[RequestResponseType]:
|
25
|
+
return [RequestResponseType.AdsVersion, RequestResponseType.CompatibilityCheck]
|
26
|
+
|
27
|
+
def __init__(self, message: Union[str, bytes]):
|
28
|
+
super().__init__(message)
|
29
|
+
|
30
|
+
@handle_exceptions
|
31
|
+
def process(self) -> Union[AdsVersionResponse, CompatibilityCheckResponse]:
|
32
|
+
request = json.loads(self.message)
|
33
|
+
if request.get("kind") == "AdsVersion":
|
34
|
+
version = metadata.version("oracle_ads")
|
35
|
+
response = AdsVersionResponse(
|
36
|
+
message_id=request.get("message_id"),
|
37
|
+
kind=RequestResponseType.AdsVersion,
|
38
|
+
data=version,
|
39
|
+
)
|
40
|
+
return response
|
41
|
+
if request.get("kind") == "CompatibilityCheck":
|
42
|
+
if ODSC_MODEL_COMPARTMENT_OCID or fetch_service_compartment():
|
43
|
+
return CompatibilityCheckResponse(
|
44
|
+
message_id=request.get("message_id"),
|
45
|
+
kind=RequestResponseType.CompatibilityCheck,
|
46
|
+
data={"status": "ok"},
|
47
|
+
)
|
48
|
+
elif known_realm():
|
49
|
+
return CompatibilityCheckResponse(
|
50
|
+
message_id=request.get("message_id"),
|
51
|
+
kind=RequestResponseType.CompatibilityCheck,
|
52
|
+
data={"status": "compatible"},
|
53
|
+
)
|
54
|
+
else:
|
55
|
+
raise AquaResourceAccessError(
|
56
|
+
"The AI Quick actions extension is not compatible in the given region."
|
57
|
+
)
|