oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/__init__.py +40 -0
- ads/aqua/app.py +507 -0
- ads/aqua/cli.py +96 -0
- ads/aqua/client/__init__.py +3 -0
- ads/aqua/client/client.py +836 -0
- ads/aqua/client/openai_client.py +305 -0
- ads/aqua/common/__init__.py +5 -0
- ads/aqua/common/decorator.py +125 -0
- ads/aqua/common/entities.py +274 -0
- ads/aqua/common/enums.py +134 -0
- ads/aqua/common/errors.py +109 -0
- ads/aqua/common/utils.py +1295 -0
- ads/aqua/config/__init__.py +4 -0
- ads/aqua/config/container_config.py +247 -0
- ads/aqua/config/evaluation/__init__.py +4 -0
- ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
- ads/aqua/config/utils/__init__.py +4 -0
- ads/aqua/config/utils/serializer.py +339 -0
- ads/aqua/constants.py +116 -0
- ads/aqua/data.py +14 -0
- ads/aqua/dummy_data/icon.txt +1 -0
- ads/aqua/dummy_data/oci_model_deployments.json +56 -0
- ads/aqua/dummy_data/oci_models.json +1 -0
- ads/aqua/dummy_data/readme.md +26 -0
- ads/aqua/evaluation/__init__.py +8 -0
- ads/aqua/evaluation/constants.py +53 -0
- ads/aqua/evaluation/entities.py +186 -0
- ads/aqua/evaluation/errors.py +70 -0
- ads/aqua/evaluation/evaluation.py +1814 -0
- ads/aqua/extension/__init__.py +42 -0
- ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
- ads/aqua/extension/base_handler.py +90 -0
- ads/aqua/extension/common_handler.py +121 -0
- ads/aqua/extension/common_ws_msg_handler.py +36 -0
- ads/aqua/extension/deployment_handler.py +381 -0
- ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
- ads/aqua/extension/errors.py +30 -0
- ads/aqua/extension/evaluation_handler.py +129 -0
- ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
- ads/aqua/extension/finetune_handler.py +96 -0
- ads/aqua/extension/model_handler.py +390 -0
- ads/aqua/extension/models/__init__.py +0 -0
- ads/aqua/extension/models/ws_models.py +145 -0
- ads/aqua/extension/models_ws_msg_handler.py +50 -0
- ads/aqua/extension/ui_handler.py +300 -0
- ads/aqua/extension/ui_websocket_handler.py +130 -0
- ads/aqua/extension/utils.py +133 -0
- ads/aqua/finetuning/__init__.py +7 -0
- ads/aqua/finetuning/constants.py +23 -0
- ads/aqua/finetuning/entities.py +181 -0
- ads/aqua/finetuning/finetuning.py +749 -0
- ads/aqua/model/__init__.py +8 -0
- ads/aqua/model/constants.py +60 -0
- ads/aqua/model/entities.py +385 -0
- ads/aqua/model/enums.py +32 -0
- ads/aqua/model/model.py +2134 -0
- ads/aqua/model/utils.py +52 -0
- ads/aqua/modeldeployment/__init__.py +6 -0
- ads/aqua/modeldeployment/constants.py +10 -0
- ads/aqua/modeldeployment/deployment.py +1315 -0
- ads/aqua/modeldeployment/entities.py +653 -0
- ads/aqua/modeldeployment/utils.py +543 -0
- ads/aqua/resources/gpu_shapes_index.json +94 -0
- ads/aqua/server/__init__.py +4 -0
- ads/aqua/server/__main__.py +24 -0
- ads/aqua/server/app.py +47 -0
- ads/aqua/server/aqua_spec.yml +1291 -0
- ads/aqua/training/__init__.py +4 -0
- ads/aqua/training/exceptions.py +476 -0
- ads/aqua/ui.py +519 -0
- ads/automl/__init__.py +9 -0
- ads/automl/driver.py +330 -0
- ads/automl/provider.py +975 -0
- ads/bds/__init__.py +5 -0
- ads/bds/auth.py +127 -0
- ads/bds/big_data_service.py +255 -0
- ads/catalog/__init__.py +19 -0
- ads/catalog/model.py +1576 -0
- ads/catalog/notebook.py +461 -0
- ads/catalog/project.py +468 -0
- ads/catalog/summary.py +178 -0
- ads/common/__init__.py +11 -0
- ads/common/analyzer.py +65 -0
- ads/common/artifact/.model-ignore +63 -0
- ads/common/artifact/__init__.py +10 -0
- ads/common/auth.py +1122 -0
- ads/common/card_identifier.py +83 -0
- ads/common/config.py +647 -0
- ads/common/data.py +165 -0
- ads/common/decorator/__init__.py +9 -0
- ads/common/decorator/argument_to_case.py +88 -0
- ads/common/decorator/deprecate.py +69 -0
- ads/common/decorator/require_nonempty_arg.py +65 -0
- ads/common/decorator/runtime_dependency.py +178 -0
- ads/common/decorator/threaded.py +97 -0
- ads/common/decorator/utils.py +35 -0
- ads/common/dsc_file_system.py +303 -0
- ads/common/error.py +14 -0
- ads/common/extended_enum.py +81 -0
- ads/common/function/__init__.py +5 -0
- ads/common/function/fn_util.py +142 -0
- ads/common/function/func_conf.yaml +25 -0
- ads/common/ipython.py +76 -0
- ads/common/model.py +679 -0
- ads/common/model_artifact.py +1759 -0
- ads/common/model_artifact_schema.json +107 -0
- ads/common/model_export_util.py +664 -0
- ads/common/model_metadata.py +24 -0
- ads/common/object_storage_details.py +296 -0
- ads/common/oci_client.py +179 -0
- ads/common/oci_datascience.py +46 -0
- ads/common/oci_logging.py +1144 -0
- ads/common/oci_mixin.py +957 -0
- ads/common/oci_resource.py +136 -0
- ads/common/serializer.py +559 -0
- ads/common/utils.py +1852 -0
- ads/common/word_lists.py +1491 -0
- ads/common/work_request.py +189 -0
- ads/config.py +1 -0
- ads/data_labeling/__init__.py +13 -0
- ads/data_labeling/boundingbox.py +253 -0
- ads/data_labeling/constants.py +47 -0
- ads/data_labeling/data_labeling_service.py +244 -0
- ads/data_labeling/interface/__init__.py +5 -0
- ads/data_labeling/interface/loader.py +16 -0
- ads/data_labeling/interface/parser.py +16 -0
- ads/data_labeling/interface/reader.py +23 -0
- ads/data_labeling/loader/__init__.py +5 -0
- ads/data_labeling/loader/file_loader.py +241 -0
- ads/data_labeling/metadata.py +110 -0
- ads/data_labeling/mixin/__init__.py +5 -0
- ads/data_labeling/mixin/data_labeling.py +232 -0
- ads/data_labeling/ner.py +129 -0
- ads/data_labeling/parser/__init__.py +5 -0
- ads/data_labeling/parser/dls_record_parser.py +388 -0
- ads/data_labeling/parser/export_metadata_parser.py +94 -0
- ads/data_labeling/parser/export_record_parser.py +473 -0
- ads/data_labeling/reader/__init__.py +5 -0
- ads/data_labeling/reader/dataset_reader.py +574 -0
- ads/data_labeling/reader/dls_record_reader.py +121 -0
- ads/data_labeling/reader/export_record_reader.py +62 -0
- ads/data_labeling/reader/jsonl_reader.py +75 -0
- ads/data_labeling/reader/metadata_reader.py +203 -0
- ads/data_labeling/reader/record_reader.py +263 -0
- ads/data_labeling/record.py +52 -0
- ads/data_labeling/visualizer/__init__.py +5 -0
- ads/data_labeling/visualizer/image_visualizer.py +525 -0
- ads/data_labeling/visualizer/text_visualizer.py +357 -0
- ads/database/__init__.py +5 -0
- ads/database/connection.py +338 -0
- ads/dataset/__init__.py +10 -0
- ads/dataset/capabilities.md +51 -0
- ads/dataset/classification_dataset.py +339 -0
- ads/dataset/correlation.py +226 -0
- ads/dataset/correlation_plot.py +563 -0
- ads/dataset/dask_series.py +173 -0
- ads/dataset/dataframe_transformer.py +110 -0
- ads/dataset/dataset.py +1979 -0
- ads/dataset/dataset_browser.py +360 -0
- ads/dataset/dataset_with_target.py +995 -0
- ads/dataset/exception.py +25 -0
- ads/dataset/factory.py +987 -0
- ads/dataset/feature_engineering_transformer.py +35 -0
- ads/dataset/feature_selection.py +107 -0
- ads/dataset/forecasting_dataset.py +26 -0
- ads/dataset/helper.py +1450 -0
- ads/dataset/label_encoder.py +99 -0
- ads/dataset/mixin/__init__.py +5 -0
- ads/dataset/mixin/dataset_accessor.py +134 -0
- ads/dataset/pipeline.py +58 -0
- ads/dataset/plot.py +710 -0
- ads/dataset/progress.py +86 -0
- ads/dataset/recommendation.py +297 -0
- ads/dataset/recommendation_transformer.py +502 -0
- ads/dataset/regression_dataset.py +14 -0
- ads/dataset/sampled_dataset.py +1050 -0
- ads/dataset/target.py +98 -0
- ads/dataset/timeseries.py +18 -0
- ads/dbmixin/__init__.py +5 -0
- ads/dbmixin/db_pandas_accessor.py +153 -0
- ads/environment/__init__.py +9 -0
- ads/environment/ml_runtime.py +66 -0
- ads/evaluations/README.md +14 -0
- ads/evaluations/__init__.py +109 -0
- ads/evaluations/evaluation_plot.py +983 -0
- ads/evaluations/evaluator.py +1334 -0
- ads/evaluations/statistical_metrics.py +543 -0
- ads/experiments/__init__.py +9 -0
- ads/experiments/capabilities.md +0 -0
- ads/explanations/__init__.py +21 -0
- ads/explanations/base_explainer.py +142 -0
- ads/explanations/capabilities.md +83 -0
- ads/explanations/explainer.py +190 -0
- ads/explanations/mlx_global_explainer.py +1050 -0
- ads/explanations/mlx_interface.py +386 -0
- ads/explanations/mlx_local_explainer.py +287 -0
- ads/explanations/mlx_whatif_explainer.py +201 -0
- ads/feature_engineering/__init__.py +20 -0
- ads/feature_engineering/accessor/__init__.py +5 -0
- ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
- ads/feature_engineering/accessor/mixin/__init__.py +5 -0
- ads/feature_engineering/accessor/mixin/correlation.py +166 -0
- ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
- ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
- ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
- ads/feature_engineering/accessor/mixin/utils.py +65 -0
- ads/feature_engineering/accessor/series_accessor.py +431 -0
- ads/feature_engineering/adsimage/__init__.py +5 -0
- ads/feature_engineering/adsimage/image.py +192 -0
- ads/feature_engineering/adsimage/image_reader.py +170 -0
- ads/feature_engineering/adsimage/interface/__init__.py +5 -0
- ads/feature_engineering/adsimage/interface/reader.py +19 -0
- ads/feature_engineering/adsstring/__init__.py +7 -0
- ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
- ads/feature_engineering/adsstring/string/__init__.py +8 -0
- ads/feature_engineering/data_schema.json +57 -0
- ads/feature_engineering/dataset/__init__.py +5 -0
- ads/feature_engineering/dataset/zip_code_data.py +42062 -0
- ads/feature_engineering/exceptions.py +40 -0
- ads/feature_engineering/feature_type/__init__.py +133 -0
- ads/feature_engineering/feature_type/address.py +184 -0
- ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
- ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
- ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
- ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
- ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
- ads/feature_engineering/feature_type/adsstring/string.py +258 -0
- ads/feature_engineering/feature_type/base.py +58 -0
- ads/feature_engineering/feature_type/boolean.py +183 -0
- ads/feature_engineering/feature_type/category.py +146 -0
- ads/feature_engineering/feature_type/constant.py +137 -0
- ads/feature_engineering/feature_type/continuous.py +151 -0
- ads/feature_engineering/feature_type/creditcard.py +314 -0
- ads/feature_engineering/feature_type/datetime.py +190 -0
- ads/feature_engineering/feature_type/discrete.py +134 -0
- ads/feature_engineering/feature_type/document.py +43 -0
- ads/feature_engineering/feature_type/gis.py +251 -0
- ads/feature_engineering/feature_type/handler/__init__.py +5 -0
- ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
- ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
- ads/feature_engineering/feature_type/handler/warnings.py +128 -0
- ads/feature_engineering/feature_type/integer.py +142 -0
- ads/feature_engineering/feature_type/ip_address.py +144 -0
- ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
- ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
- ads/feature_engineering/feature_type/lat_long.py +256 -0
- ads/feature_engineering/feature_type/object.py +43 -0
- ads/feature_engineering/feature_type/ordinal.py +132 -0
- ads/feature_engineering/feature_type/phone_number.py +135 -0
- ads/feature_engineering/feature_type/string.py +171 -0
- ads/feature_engineering/feature_type/text.py +93 -0
- ads/feature_engineering/feature_type/unknown.py +43 -0
- ads/feature_engineering/feature_type/zip_code.py +164 -0
- ads/feature_engineering/feature_type_manager.py +406 -0
- ads/feature_engineering/schema.py +795 -0
- ads/feature_engineering/utils.py +245 -0
- ads/feature_store/.readthedocs.yaml +19 -0
- ads/feature_store/README.md +65 -0
- ads/feature_store/__init__.py +9 -0
- ads/feature_store/common/__init__.py +0 -0
- ads/feature_store/common/enums.py +339 -0
- ads/feature_store/common/exceptions.py +18 -0
- ads/feature_store/common/spark_session_singleton.py +125 -0
- ads/feature_store/common/utils/__init__.py +0 -0
- ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
- ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
- ads/feature_store/common/utils/transformation_utils.py +82 -0
- ads/feature_store/common/utils/utility.py +403 -0
- ads/feature_store/data_validation/__init__.py +0 -0
- ads/feature_store/data_validation/great_expectation.py +129 -0
- ads/feature_store/dataset.py +1230 -0
- ads/feature_store/dataset_job.py +530 -0
- ads/feature_store/docs/Dockerfile +7 -0
- ads/feature_store/docs/Makefile +44 -0
- ads/feature_store/docs/conf.py +28 -0
- ads/feature_store/docs/requirements.txt +14 -0
- ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
- ads/feature_store/docs/source/cicd.rst +137 -0
- ads/feature_store/docs/source/conf.py +86 -0
- ads/feature_store/docs/source/data_versioning.rst +33 -0
- ads/feature_store/docs/source/dataset.rst +388 -0
- ads/feature_store/docs/source/dataset_job.rst +27 -0
- ads/feature_store/docs/source/demo.rst +70 -0
- ads/feature_store/docs/source/entity.rst +78 -0
- ads/feature_store/docs/source/feature_group.rst +624 -0
- ads/feature_store/docs/source/feature_group_job.rst +29 -0
- ads/feature_store/docs/source/feature_store.rst +122 -0
- ads/feature_store/docs/source/feature_store_class.rst +123 -0
- ads/feature_store/docs/source/feature_validation.rst +66 -0
- ads/feature_store/docs/source/figures/cicd.png +0 -0
- ads/feature_store/docs/source/figures/data_validation.png +0 -0
- ads/feature_store/docs/source/figures/data_versioning.png +0 -0
- ads/feature_store/docs/source/figures/dataset.gif +0 -0
- ads/feature_store/docs/source/figures/dataset.png +0 -0
- ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
- ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
- ads/feature_store/docs/source/figures/entity.png +0 -0
- ads/feature_store/docs/source/figures/feature_group.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
- ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
- ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
- ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
- ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
- ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
- ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
- ads/feature_store/docs/source/figures/overview.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
- ads/feature_store/docs/source/figures/stats_1.png +0 -0
- ads/feature_store/docs/source/figures/stats_2.png +0 -0
- ads/feature_store/docs/source/figures/stats_d.png +0 -0
- ads/feature_store/docs/source/figures/stats_fg.png +0 -0
- ads/feature_store/docs/source/figures/transformation.png +0 -0
- ads/feature_store/docs/source/figures/transformations.gif +0 -0
- ads/feature_store/docs/source/figures/validation.png +0 -0
- ads/feature_store/docs/source/figures/validation_fg.png +0 -0
- ads/feature_store/docs/source/figures/validation_results.png +0 -0
- ads/feature_store/docs/source/figures/validation_summary.png +0 -0
- ads/feature_store/docs/source/index.rst +81 -0
- ads/feature_store/docs/source/module.rst +8 -0
- ads/feature_store/docs/source/notebook.rst +94 -0
- ads/feature_store/docs/source/overview.rst +47 -0
- ads/feature_store/docs/source/quickstart.rst +176 -0
- ads/feature_store/docs/source/release_notes.rst +194 -0
- ads/feature_store/docs/source/setup_feature_store.rst +81 -0
- ads/feature_store/docs/source/statistics.rst +58 -0
- ads/feature_store/docs/source/transformation.rst +199 -0
- ads/feature_store/docs/source/ui.rst +65 -0
- ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
- ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
- ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
- ads/feature_store/entity.py +718 -0
- ads/feature_store/execution_strategy/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
- ads/feature_store/execution_strategy/engine/__init__.py +0 -0
- ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
- ads/feature_store/execution_strategy/execution_strategy.py +113 -0
- ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
- ads/feature_store/execution_strategy/spark/__init__.py +0 -0
- ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
- ads/feature_store/feature.py +192 -0
- ads/feature_store/feature_group.py +1494 -0
- ads/feature_store/feature_group_expectation.py +346 -0
- ads/feature_store/feature_group_job.py +602 -0
- ads/feature_store/feature_lineage/__init__.py +0 -0
- ads/feature_store/feature_lineage/graphviz_service.py +180 -0
- ads/feature_store/feature_option_details.py +50 -0
- ads/feature_store/feature_statistics/__init__.py +0 -0
- ads/feature_store/feature_statistics/statistics_service.py +99 -0
- ads/feature_store/feature_store.py +699 -0
- ads/feature_store/feature_store_registrar.py +518 -0
- ads/feature_store/input_feature_detail.py +149 -0
- ads/feature_store/mixin/__init__.py +4 -0
- ads/feature_store/mixin/oci_feature_store.py +145 -0
- ads/feature_store/model_details.py +73 -0
- ads/feature_store/query/__init__.py +0 -0
- ads/feature_store/query/filter.py +266 -0
- ads/feature_store/query/generator/__init__.py +0 -0
- ads/feature_store/query/generator/query_generator.py +298 -0
- ads/feature_store/query/join.py +161 -0
- ads/feature_store/query/query.py +403 -0
- ads/feature_store/query/validator/__init__.py +0 -0
- ads/feature_store/query/validator/query_validator.py +57 -0
- ads/feature_store/response/__init__.py +0 -0
- ads/feature_store/response/response_builder.py +68 -0
- ads/feature_store/service/__init__.py +0 -0
- ads/feature_store/service/oci_dataset.py +139 -0
- ads/feature_store/service/oci_dataset_job.py +199 -0
- ads/feature_store/service/oci_entity.py +125 -0
- ads/feature_store/service/oci_feature_group.py +164 -0
- ads/feature_store/service/oci_feature_group_job.py +214 -0
- ads/feature_store/service/oci_feature_store.py +182 -0
- ads/feature_store/service/oci_lineage.py +87 -0
- ads/feature_store/service/oci_transformation.py +104 -0
- ads/feature_store/statistics/__init__.py +0 -0
- ads/feature_store/statistics/abs_feature_value.py +49 -0
- ads/feature_store/statistics/charts/__init__.py +0 -0
- ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
- ads/feature_store/statistics/charts/box_plot.py +148 -0
- ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
- ads/feature_store/statistics/charts/probability_distribution.py +68 -0
- ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
- ads/feature_store/statistics/feature_stat.py +126 -0
- ads/feature_store/statistics/generic_feature_value.py +33 -0
- ads/feature_store/statistics/statistics.py +41 -0
- ads/feature_store/statistics_config.py +101 -0
- ads/feature_store/templates/feature_store_template.yaml +45 -0
- ads/feature_store/transformation.py +499 -0
- ads/feature_store/validation_output.py +57 -0
- ads/hpo/__init__.py +9 -0
- ads/hpo/_imports.py +91 -0
- ads/hpo/ads_search_space.py +439 -0
- ads/hpo/distributions.py +325 -0
- ads/hpo/objective.py +280 -0
- ads/hpo/search_cv.py +1657 -0
- ads/hpo/stopping_criterion.py +75 -0
- ads/hpo/tuner_artifact.py +413 -0
- ads/hpo/utils.py +91 -0
- ads/hpo/validation.py +140 -0
- ads/hpo/visualization/__init__.py +5 -0
- ads/hpo/visualization/_contour.py +23 -0
- ads/hpo/visualization/_edf.py +20 -0
- ads/hpo/visualization/_intermediate_values.py +21 -0
- ads/hpo/visualization/_optimization_history.py +25 -0
- ads/hpo/visualization/_parallel_coordinate.py +169 -0
- ads/hpo/visualization/_param_importances.py +26 -0
- ads/jobs/__init__.py +53 -0
- ads/jobs/ads_job.py +663 -0
- ads/jobs/builders/__init__.py +5 -0
- ads/jobs/builders/base.py +156 -0
- ads/jobs/builders/infrastructure/__init__.py +6 -0
- ads/jobs/builders/infrastructure/base.py +165 -0
- ads/jobs/builders/infrastructure/dataflow.py +1252 -0
- ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
- ads/jobs/builders/infrastructure/utils.py +65 -0
- ads/jobs/builders/runtimes/__init__.py +5 -0
- ads/jobs/builders/runtimes/artifact.py +338 -0
- ads/jobs/builders/runtimes/base.py +325 -0
- ads/jobs/builders/runtimes/container_runtime.py +242 -0
- ads/jobs/builders/runtimes/python_runtime.py +1016 -0
- ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
- ads/jobs/cli.py +104 -0
- ads/jobs/env_var_parser.py +131 -0
- ads/jobs/extension.py +160 -0
- ads/jobs/schema/__init__.py +5 -0
- ads/jobs/schema/infrastructure_schema.json +116 -0
- ads/jobs/schema/job_schema.json +42 -0
- ads/jobs/schema/runtime_schema.json +183 -0
- ads/jobs/schema/validator.py +141 -0
- ads/jobs/serializer.py +296 -0
- ads/jobs/templates/__init__.py +5 -0
- ads/jobs/templates/container.py +6 -0
- ads/jobs/templates/driver_notebook.py +177 -0
- ads/jobs/templates/driver_oci.py +500 -0
- ads/jobs/templates/driver_python.py +48 -0
- ads/jobs/templates/driver_pytorch.py +852 -0
- ads/jobs/templates/driver_utils.py +615 -0
- ads/jobs/templates/hostname_from_env.c +55 -0
- ads/jobs/templates/oci_metrics.py +181 -0
- ads/jobs/utils.py +104 -0
- ads/llm/__init__.py +28 -0
- ads/llm/autogen/__init__.py +2 -0
- ads/llm/autogen/constants.py +15 -0
- ads/llm/autogen/reports/__init__.py +2 -0
- ads/llm/autogen/reports/base.py +67 -0
- ads/llm/autogen/reports/data.py +103 -0
- ads/llm/autogen/reports/session.py +526 -0
- ads/llm/autogen/reports/templates/chat_box.html +13 -0
- ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
- ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
- ads/llm/autogen/reports/utils.py +56 -0
- ads/llm/autogen/v02/__init__.py +4 -0
- ads/llm/autogen/v02/client.py +295 -0
- ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
- ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
- ads/llm/autogen/v02/loggers/__init__.py +6 -0
- ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
- ads/llm/autogen/v02/loggers/session_logger.py +580 -0
- ads/llm/autogen/v02/loggers/utils.py +86 -0
- ads/llm/autogen/v02/runtime_logging.py +163 -0
- ads/llm/chain.py +268 -0
- ads/llm/chat_template.py +31 -0
- ads/llm/deploy.py +63 -0
- ads/llm/guardrails/__init__.py +5 -0
- ads/llm/guardrails/base.py +442 -0
- ads/llm/guardrails/huggingface.py +44 -0
- ads/llm/langchain/__init__.py +5 -0
- ads/llm/langchain/plugins/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
- ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
- ads/llm/langchain/plugins/llms/__init__.py +5 -0
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
- ads/llm/requirements.txt +3 -0
- ads/llm/serialize.py +219 -0
- ads/llm/serializers/__init__.py +0 -0
- ads/llm/serializers/retrieval_qa.py +153 -0
- ads/llm/serializers/runnable_parallel.py +27 -0
- ads/llm/templates/score_chain.jinja2 +155 -0
- ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
- ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
- ads/model/__init__.py +52 -0
- ads/model/artifact.py +573 -0
- ads/model/artifact_downloader.py +254 -0
- ads/model/artifact_uploader.py +267 -0
- ads/model/base_properties.py +238 -0
- ads/model/common/.model-ignore +66 -0
- ads/model/common/__init__.py +5 -0
- ads/model/common/utils.py +142 -0
- ads/model/datascience_model.py +2635 -0
- ads/model/deployment/__init__.py +20 -0
- ads/model/deployment/common/__init__.py +5 -0
- ads/model/deployment/common/utils.py +308 -0
- ads/model/deployment/model_deployer.py +466 -0
- ads/model/deployment/model_deployment.py +1846 -0
- ads/model/deployment/model_deployment_infrastructure.py +671 -0
- ads/model/deployment/model_deployment_properties.py +493 -0
- ads/model/deployment/model_deployment_runtime.py +838 -0
- ads/model/extractor/__init__.py +5 -0
- ads/model/extractor/automl_extractor.py +74 -0
- ads/model/extractor/embedding_onnx_extractor.py +80 -0
- ads/model/extractor/huggingface_extractor.py +88 -0
- ads/model/extractor/keras_extractor.py +84 -0
- ads/model/extractor/lightgbm_extractor.py +93 -0
- ads/model/extractor/model_info_extractor.py +114 -0
- ads/model/extractor/model_info_extractor_factory.py +105 -0
- ads/model/extractor/pytorch_extractor.py +87 -0
- ads/model/extractor/sklearn_extractor.py +112 -0
- ads/model/extractor/spark_extractor.py +89 -0
- ads/model/extractor/tensorflow_extractor.py +85 -0
- ads/model/extractor/xgboost_extractor.py +94 -0
- ads/model/framework/__init__.py +5 -0
- ads/model/framework/automl_model.py +178 -0
- ads/model/framework/embedding_onnx_model.py +438 -0
- ads/model/framework/huggingface_model.py +399 -0
- ads/model/framework/lightgbm_model.py +266 -0
- ads/model/framework/pytorch_model.py +266 -0
- ads/model/framework/sklearn_model.py +250 -0
- ads/model/framework/spark_model.py +326 -0
- ads/model/framework/tensorflow_model.py +254 -0
- ads/model/framework/xgboost_model.py +258 -0
- ads/model/generic_model.py +3518 -0
- ads/model/model_artifact_boilerplate/README.md +381 -0
- ads/model/model_artifact_boilerplate/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
- ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
- ads/model/model_artifact_boilerplate/score.py +61 -0
- ads/model/model_file_description_schema.json +68 -0
- ads/model/model_introspect.py +331 -0
- ads/model/model_metadata.py +1810 -0
- ads/model/model_metadata_mixin.py +460 -0
- ads/model/model_properties.py +63 -0
- ads/model/model_version_set.py +739 -0
- ads/model/runtime/__init__.py +5 -0
- ads/model/runtime/env_info.py +306 -0
- ads/model/runtime/model_deployment_details.py +37 -0
- ads/model/runtime/model_provenance_details.py +58 -0
- ads/model/runtime/runtime_info.py +81 -0
- ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
- ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
- ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
- ads/model/runtime/utils.py +201 -0
- ads/model/serde/__init__.py +5 -0
- ads/model/serde/common.py +40 -0
- ads/model/serde/model_input.py +547 -0
- ads/model/serde/model_serializer.py +1184 -0
- ads/model/service/__init__.py +5 -0
- ads/model/service/oci_datascience_model.py +1076 -0
- ads/model/service/oci_datascience_model_deployment.py +500 -0
- ads/model/service/oci_datascience_model_version_set.py +176 -0
- ads/model/transformer/__init__.py +5 -0
- ads/model/transformer/onnx_transformer.py +324 -0
- ads/mysqldb/__init__.py +5 -0
- ads/mysqldb/mysql_db.py +227 -0
- ads/opctl/__init__.py +18 -0
- ads/opctl/anomaly_detection.py +11 -0
- ads/opctl/backend/__init__.py +5 -0
- ads/opctl/backend/ads_dataflow.py +353 -0
- ads/opctl/backend/ads_ml_job.py +710 -0
- ads/opctl/backend/ads_ml_pipeline.py +164 -0
- ads/opctl/backend/ads_model_deployment.py +209 -0
- ads/opctl/backend/base.py +146 -0
- ads/opctl/backend/local.py +1053 -0
- ads/opctl/backend/marketplace/__init__.py +9 -0
- ads/opctl/backend/marketplace/helm_helper.py +173 -0
- ads/opctl/backend/marketplace/local_marketplace.py +271 -0
- ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
- ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
- ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
- ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
- ads/opctl/backend/marketplace/models/__init__.py +5 -0
- ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
- ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
- ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
- ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
- ads/opctl/cli.py +707 -0
- ads/opctl/cmds.py +869 -0
- ads/opctl/conda/__init__.py +5 -0
- ads/opctl/conda/cli.py +193 -0
- ads/opctl/conda/cmds.py +749 -0
- ads/opctl/conda/config.yaml +34 -0
- ads/opctl/conda/manifest_template.yaml +13 -0
- ads/opctl/conda/multipart_uploader.py +188 -0
- ads/opctl/conda/pack.py +89 -0
- ads/opctl/config/__init__.py +5 -0
- ads/opctl/config/base.py +57 -0
- ads/opctl/config/diagnostics/__init__.py +5 -0
- ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
- ads/opctl/config/merger.py +255 -0
- ads/opctl/config/resolver.py +297 -0
- ads/opctl/config/utils.py +79 -0
- ads/opctl/config/validator.py +17 -0
- ads/opctl/config/versioner.py +68 -0
- ads/opctl/config/yaml_parsers/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/base.py +58 -0
- ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
- ads/opctl/constants.py +66 -0
- ads/opctl/decorator/__init__.py +5 -0
- ads/opctl/decorator/common.py +129 -0
- ads/opctl/diagnostics/__init__.py +5 -0
- ads/opctl/diagnostics/__main__.py +25 -0
- ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
- ads/opctl/diagnostics/check_requirements.py +144 -0
- ads/opctl/diagnostics/requirement_exception.py +9 -0
- ads/opctl/distributed/README.md +109 -0
- ads/opctl/distributed/__init__.py +5 -0
- ads/opctl/distributed/certificates.py +32 -0
- ads/opctl/distributed/cli.py +207 -0
- ads/opctl/distributed/cmds.py +731 -0
- ads/opctl/distributed/common/__init__.py +5 -0
- ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
- ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
- ads/opctl/distributed/common/cluster_config_helper.py +103 -0
- ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
- ads/opctl/distributed/common/cluster_runner.py +54 -0
- ads/opctl/distributed/common/framework_factory.py +29 -0
- ads/opctl/docker/Dockerfile.job +103 -0
- ads/opctl/docker/Dockerfile.job.arm +107 -0
- ads/opctl/docker/Dockerfile.job.gpu +175 -0
- ads/opctl/docker/base-env.yaml +13 -0
- ads/opctl/docker/cuda.repo +6 -0
- ads/opctl/docker/operator/.dockerignore +0 -0
- ads/opctl/docker/operator/Dockerfile +41 -0
- ads/opctl/docker/operator/Dockerfile.gpu +85 -0
- ads/opctl/docker/operator/cuda.repo +6 -0
- ads/opctl/docker/operator/environment.yaml +8 -0
- ads/opctl/forecast.py +11 -0
- ads/opctl/index.yaml +3 -0
- ads/opctl/model/__init__.py +5 -0
- ads/opctl/model/cli.py +65 -0
- ads/opctl/model/cmds.py +73 -0
- ads/opctl/operator/README.md +4 -0
- ads/opctl/operator/__init__.py +31 -0
- ads/opctl/operator/cli.py +344 -0
- ads/opctl/operator/cmd.py +596 -0
- ads/opctl/operator/common/__init__.py +5 -0
- ads/opctl/operator/common/backend_factory.py +460 -0
- ads/opctl/operator/common/const.py +27 -0
- ads/opctl/operator/common/data/synthetic.csv +16001 -0
- ads/opctl/operator/common/dictionary_merger.py +148 -0
- ads/opctl/operator/common/errors.py +42 -0
- ads/opctl/operator/common/operator_config.py +99 -0
- ads/opctl/operator/common/operator_loader.py +811 -0
- ads/opctl/operator/common/operator_schema.yaml +130 -0
- ads/opctl/operator/common/operator_yaml_generator.py +152 -0
- ads/opctl/operator/common/utils.py +208 -0
- ads/opctl/operator/lowcode/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
- ads/opctl/operator/lowcode/anomaly/README.md +207 -0
- ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
- ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
- ads/opctl/operator/lowcode/anomaly/const.py +167 -0
- ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
- ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
- ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
- ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
- ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
- ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
- ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
- ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
- ads/opctl/operator/lowcode/common/__init__.py +5 -0
- ads/opctl/operator/lowcode/common/const.py +10 -0
- ads/opctl/operator/lowcode/common/data.py +116 -0
- ads/opctl/operator/lowcode/common/errors.py +47 -0
- ads/opctl/operator/lowcode/common/transformations.py +296 -0
- ads/opctl/operator/lowcode/common/utils.py +384 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
- ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
- ads/opctl/operator/lowcode/forecast/README.md +209 -0
- ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
- ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
- ads/opctl/operator/lowcode/forecast/const.py +92 -0
- ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
- ads/opctl/operator/lowcode/forecast/errors.py +26 -0
- ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
- ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
- ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
- ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
- ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
- ads/opctl/operator/lowcode/forecast/model/prophet.py +450 -0
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
- ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
- ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
- ads/opctl/operator/lowcode/forecast/utils.py +397 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
- ads/opctl/operator/lowcode/pii/MLoperator +17 -0
- ads/opctl/operator/lowcode/pii/README.md +208 -0
- ads/opctl/operator/lowcode/pii/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/__main__.py +78 -0
- ads/opctl/operator/lowcode/pii/cmd.py +39 -0
- ads/opctl/operator/lowcode/pii/constant.py +84 -0
- ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
- ads/opctl/operator/lowcode/pii/errors.py +27 -0
- ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
- ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
- ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
- ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
- ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
- ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
- ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
- ads/opctl/operator/lowcode/pii/model/report.py +487 -0
- ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
- ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
- ads/opctl/operator/lowcode/pii/utils.py +43 -0
- ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
- ads/opctl/operator/lowcode/recommender/README.md +206 -0
- ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
- ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
- ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
- ads/opctl/operator/lowcode/recommender/constant.py +30 -0
- ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
- ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
- ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
- ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
- ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
- ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
- ads/opctl/operator/lowcode/recommender/utils.py +13 -0
- ads/opctl/operator/runtime/__init__.py +5 -0
- ads/opctl/operator/runtime/const.py +17 -0
- ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
- ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
- ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/runtime.py +115 -0
- ads/opctl/schema.yaml.yml +36 -0
- ads/opctl/script.py +40 -0
- ads/opctl/spark/__init__.py +5 -0
- ads/opctl/spark/cli.py +43 -0
- ads/opctl/spark/cmds.py +147 -0
- ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
- ads/opctl/utils.py +344 -0
- ads/oracledb/__init__.py +5 -0
- ads/oracledb/oracle_db.py +346 -0
- ads/pipeline/__init__.py +39 -0
- ads/pipeline/ads_pipeline.py +2279 -0
- ads/pipeline/ads_pipeline_run.py +772 -0
- ads/pipeline/ads_pipeline_step.py +605 -0
- ads/pipeline/builders/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/custom_script.py +32 -0
- ads/pipeline/cli.py +119 -0
- ads/pipeline/extension.py +291 -0
- ads/pipeline/schema/__init__.py +5 -0
- ads/pipeline/schema/cs_step_schema.json +35 -0
- ads/pipeline/schema/ml_step_schema.json +31 -0
- ads/pipeline/schema/pipeline_schema.json +71 -0
- ads/pipeline/visualizer/__init__.py +5 -0
- ads/pipeline/visualizer/base.py +570 -0
- ads/pipeline/visualizer/graph_renderer.py +272 -0
- ads/pipeline/visualizer/text_renderer.py +84 -0
- ads/secrets/__init__.py +11 -0
- ads/secrets/adb.py +386 -0
- ads/secrets/auth_token.py +86 -0
- ads/secrets/big_data_service.py +365 -0
- ads/secrets/mysqldb.py +149 -0
- ads/secrets/oracledb.py +160 -0
- ads/secrets/secrets.py +407 -0
- ads/telemetry/__init__.py +7 -0
- ads/telemetry/base.py +69 -0
- ads/telemetry/client.py +122 -0
- ads/telemetry/telemetry.py +257 -0
- ads/templates/dataflow_pyspark.jinja2 +13 -0
- ads/templates/dataflow_sparksql.jinja2 +22 -0
- ads/templates/func.jinja2 +20 -0
- ads/templates/schemas/openapi.json +1740 -0
- ads/templates/score-pkl.jinja2 +173 -0
- ads/templates/score.jinja2 +322 -0
- ads/templates/score_embedding_onnx.jinja2 +202 -0
- ads/templates/score_generic.jinja2 +165 -0
- ads/templates/score_huggingface_pipeline.jinja2 +217 -0
- ads/templates/score_lightgbm.jinja2 +185 -0
- ads/templates/score_onnx.jinja2 +407 -0
- ads/templates/score_onnx_new.jinja2 +473 -0
- ads/templates/score_oracle_automl.jinja2 +185 -0
- ads/templates/score_pyspark.jinja2 +154 -0
- ads/templates/score_pytorch.jinja2 +219 -0
- ads/templates/score_scikit-learn.jinja2 +184 -0
- ads/templates/score_tensorflow.jinja2 +184 -0
- ads/templates/score_xgboost.jinja2 +178 -0
- ads/text_dataset/__init__.py +5 -0
- ads/text_dataset/backends.py +211 -0
- ads/text_dataset/dataset.py +445 -0
- ads/text_dataset/extractor.py +207 -0
- ads/text_dataset/options.py +53 -0
- ads/text_dataset/udfs.py +22 -0
- ads/text_dataset/utils.py +49 -0
- ads/type_discovery/__init__.py +9 -0
- ads/type_discovery/abstract_detector.py +21 -0
- ads/type_discovery/constant_detector.py +41 -0
- ads/type_discovery/continuous_detector.py +54 -0
- ads/type_discovery/credit_card_detector.py +99 -0
- ads/type_discovery/datetime_detector.py +92 -0
- ads/type_discovery/discrete_detector.py +118 -0
- ads/type_discovery/document_detector.py +146 -0
- ads/type_discovery/ip_detector.py +68 -0
- ads/type_discovery/latlon_detector.py +90 -0
- ads/type_discovery/phone_number_detector.py +63 -0
- ads/type_discovery/type_discovery_driver.py +87 -0
- ads/type_discovery/typed_feature.py +594 -0
- ads/type_discovery/unknown_detector.py +41 -0
- ads/type_discovery/zipcode_detector.py +48 -0
- ads/vault/__init__.py +7 -0
- ads/vault/vault.py +237 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/METADATA +150 -149
- oracle_ads-2.13.10rc0.dist-info/RECORD +858 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/WHEEL +1 -2
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/entry_points.txt +2 -1
- oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
- oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,2635 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
|
3
|
+
# Copyright (c) 2022, 2025 Oracle and/or its affiliates.
|
4
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
5
|
+
|
6
|
+
import json
|
7
|
+
import logging
|
8
|
+
import os
|
9
|
+
import shutil
|
10
|
+
import tempfile
|
11
|
+
from copy import deepcopy
|
12
|
+
from typing import Dict, List, Optional, Tuple, Union
|
13
|
+
|
14
|
+
import pandas
|
15
|
+
import yaml
|
16
|
+
from jsonschema import ValidationError, validate
|
17
|
+
|
18
|
+
from ads.common import oci_client as oc
|
19
|
+
from ads.common import utils
|
20
|
+
from ads.common.extended_enum import ExtendedEnum
|
21
|
+
from ads.common.object_storage_details import ObjectStorageDetails
|
22
|
+
from ads.common.utils import is_path_exists
|
23
|
+
from ads.config import (
|
24
|
+
AQUA_SERVICE_MODELS_BUCKET as SERVICE_MODELS_BUCKET,
|
25
|
+
)
|
26
|
+
from ads.config import (
|
27
|
+
COMPARTMENT_OCID,
|
28
|
+
PROJECT_OCID,
|
29
|
+
USER,
|
30
|
+
)
|
31
|
+
from ads.feature_engineering.schema import Schema
|
32
|
+
from ads.jobs.builders.base import Builder
|
33
|
+
from ads.model.artifact_downloader import (
|
34
|
+
LargeArtifactDownloader,
|
35
|
+
SmallArtifactDownloader,
|
36
|
+
)
|
37
|
+
from ads.model.artifact_uploader import LargeArtifactUploader, SmallArtifactUploader
|
38
|
+
from ads.model.common.utils import MetadataArtifactPathType
|
39
|
+
from ads.model.model_metadata import (
|
40
|
+
MetadataCustomCategory,
|
41
|
+
ModelCustomMetadata,
|
42
|
+
ModelCustomMetadataItem,
|
43
|
+
ModelProvenanceMetadata,
|
44
|
+
ModelTaxonomyMetadata,
|
45
|
+
)
|
46
|
+
from ads.model.service.oci_datascience_model import (
|
47
|
+
ModelMetadataArtifactDetails,
|
48
|
+
ModelProvenanceNotFoundError,
|
49
|
+
OCIDataScienceModel,
|
50
|
+
)
|
51
|
+
|
52
|
+
logger = logging.getLogger(__name__)
|
53
|
+
|
54
|
+
|
55
|
+
_MAX_ARTIFACT_SIZE_IN_BYTES = 2147483648 # 2GB
|
56
|
+
MODEL_BY_REFERENCE_VERSION = "1.0"
|
57
|
+
MODEL_BY_REFERENCE_JSON_FILE_NAME = "model_description.json"
|
58
|
+
|
59
|
+
|
60
|
+
class ModelArtifactSizeError(Exception): # pragma: no cover
|
61
|
+
def __init__(self, max_artifact_size: str):
|
62
|
+
super().__init__(
|
63
|
+
f"The model artifacts size is greater than `{max_artifact_size}`. "
|
64
|
+
"The `bucket_uri` needs to be specified to "
|
65
|
+
"copy artifacts to the object storage bucket. "
|
66
|
+
"Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`"
|
67
|
+
)
|
68
|
+
|
69
|
+
|
70
|
+
class BucketNotVersionedError(Exception): # pragma: no cover
|
71
|
+
def __init__(
|
72
|
+
self,
|
73
|
+
msg="Model artifact bucket is not versioned. Enable versioning on the bucket to proceed with model creation by reference.",
|
74
|
+
):
|
75
|
+
super().__init__(msg)
|
76
|
+
|
77
|
+
|
78
|
+
class PathNotFoundError(Exception):
|
79
|
+
def __init__(self, msg="The given path doesn't exist."):
|
80
|
+
super().__init__(msg)
|
81
|
+
|
82
|
+
|
83
|
+
class ModelFileDescriptionError(Exception): # pragma: no cover
|
84
|
+
def __init__(self, msg="Model File Description file is not set up."):
|
85
|
+
super().__init__(msg)
|
86
|
+
|
87
|
+
|
88
|
+
class InvalidArtifactType(Exception): # pragma: no cover
|
89
|
+
pass
|
90
|
+
|
91
|
+
|
92
|
+
class InvalidArtifactPathTypeOrContentError(Exception): # pragma: no cover
|
93
|
+
def __init__(self, msg="Invalid type of Metdata artifact content"):
|
94
|
+
super().__init__(msg)
|
95
|
+
|
96
|
+
|
97
|
+
class CustomerNotificationType(ExtendedEnum):
|
98
|
+
NONE = "NONE"
|
99
|
+
ALL = "ALL"
|
100
|
+
ON_FAILURE = "ON_FAILURE"
|
101
|
+
ON_SUCCESS = "ON_SUCCESS"
|
102
|
+
|
103
|
+
|
104
|
+
class SettingStatus(ExtendedEnum):
|
105
|
+
"""Enum to represent the status of retention settings."""
|
106
|
+
|
107
|
+
PENDING = "PENDING"
|
108
|
+
SUCCEEDED = "SUCCEEDED"
|
109
|
+
FAILED = "FAILED"
|
110
|
+
|
111
|
+
|
112
|
+
class ModelBackupSetting:
|
113
|
+
"""
|
114
|
+
Class that represents Model Backup Setting Details Metadata.
|
115
|
+
|
116
|
+
Methods
|
117
|
+
-------
|
118
|
+
to_dict(self) -> Dict:
|
119
|
+
Serializes the backup settings into a dictionary.
|
120
|
+
from_dict(cls, data: Dict) -> 'ModelBackupSetting':
|
121
|
+
Constructs backup settings from a dictionary.
|
122
|
+
to_json(self) -> str:
|
123
|
+
Serializes the backup settings into a JSON string.
|
124
|
+
from_json(cls, json_str: str) -> 'ModelBackupSetting':
|
125
|
+
Constructs backup settings from a JSON string.
|
126
|
+
to_yaml(self) -> str:
|
127
|
+
Serializes the backup settings into a YAML string.
|
128
|
+
validate(self) -> bool:
|
129
|
+
Validates the backup settings details.
|
130
|
+
"""
|
131
|
+
|
132
|
+
def __init__(
|
133
|
+
self,
|
134
|
+
is_backup_enabled: Optional[bool] = None,
|
135
|
+
backup_region: Optional[str] = None,
|
136
|
+
customer_notification_type: Optional[CustomerNotificationType] = None,
|
137
|
+
):
|
138
|
+
self.is_backup_enabled = (
|
139
|
+
is_backup_enabled if is_backup_enabled is not None else False
|
140
|
+
)
|
141
|
+
self.backup_region = backup_region
|
142
|
+
self.customer_notification_type = (
|
143
|
+
customer_notification_type or CustomerNotificationType.NONE
|
144
|
+
)
|
145
|
+
|
146
|
+
def to_dict(self) -> Dict:
|
147
|
+
"""Serializes the backup settings into a dictionary."""
|
148
|
+
return {
|
149
|
+
"is_backup_enabled": self.is_backup_enabled,
|
150
|
+
"backup_region": self.backup_region,
|
151
|
+
"customer_notification_type": self.customer_notification_type,
|
152
|
+
}
|
153
|
+
|
154
|
+
@classmethod
|
155
|
+
def from_dict(cls, data: Dict) -> "ModelBackupSetting":
|
156
|
+
"""Constructs backup settings from a dictionary."""
|
157
|
+
return cls(
|
158
|
+
is_backup_enabled=data.get("is_backup_enabled"),
|
159
|
+
backup_region=data.get("backup_region"),
|
160
|
+
customer_notification_type=data.get("customer_notification_type") or None,
|
161
|
+
)
|
162
|
+
|
163
|
+
def to_json(self) -> str:
|
164
|
+
"""Serializes the backup settings into a JSON string."""
|
165
|
+
return json.dumps(self.to_dict())
|
166
|
+
|
167
|
+
@classmethod
|
168
|
+
def from_json(cls, json_str) -> "ModelBackupSetting":
|
169
|
+
"""Constructs backup settings from a JSON string or dictionary."""
|
170
|
+
data = json.loads(json_str) if isinstance(json_str, str) else json_str
|
171
|
+
|
172
|
+
return cls.from_dict(data)
|
173
|
+
|
174
|
+
def to_yaml(self) -> str:
|
175
|
+
"""Serializes the backup settings into a YAML string."""
|
176
|
+
return yaml.dump(self.to_dict())
|
177
|
+
|
178
|
+
def validate(self) -> bool:
|
179
|
+
"""Validates the backup settings details. Returns True if valid, False otherwise."""
|
180
|
+
return all(
|
181
|
+
[
|
182
|
+
isinstance(self.is_backup_enabled, bool),
|
183
|
+
not self.backup_region or isinstance(self.backup_region, str),
|
184
|
+
isinstance(self.customer_notification_type, str)
|
185
|
+
and self.customer_notification_type
|
186
|
+
in CustomerNotificationType.values(),
|
187
|
+
]
|
188
|
+
)
|
189
|
+
|
190
|
+
def __repr__(self):
|
191
|
+
return self.to_yaml()
|
192
|
+
|
193
|
+
|
194
|
+
class ModelRetentionSetting:
|
195
|
+
"""
|
196
|
+
Class that represents Model Retention Setting Details Metadata.
|
197
|
+
|
198
|
+
Methods
|
199
|
+
-------
|
200
|
+
to_dict(self) -> Dict:
|
201
|
+
Serializes the retention settings into a dictionary.
|
202
|
+
from_dict(cls, data: Dict) -> 'ModelRetentionSetting':
|
203
|
+
Constructs retention settings from a dictionary.
|
204
|
+
to_json(self) -> str:
|
205
|
+
Serializes the retention settings into a JSON string.
|
206
|
+
from_json(cls, json_str: str) -> 'ModelRetentionSetting':
|
207
|
+
Constructs retention settings from a JSON string.
|
208
|
+
to_yaml(self) -> str:
|
209
|
+
Serializes the retention settings into a YAML string.
|
210
|
+
validate(self) -> bool:
|
211
|
+
Validates the retention settings details.
|
212
|
+
"""
|
213
|
+
|
214
|
+
def __init__(
|
215
|
+
self,
|
216
|
+
archive_after_days: Optional[int] = None,
|
217
|
+
delete_after_days: Optional[int] = None,
|
218
|
+
customer_notification_type: Optional[CustomerNotificationType] = None,
|
219
|
+
):
|
220
|
+
self.archive_after_days = archive_after_days
|
221
|
+
self.delete_after_days = delete_after_days
|
222
|
+
self.customer_notification_type = (
|
223
|
+
customer_notification_type or CustomerNotificationType.NONE
|
224
|
+
)
|
225
|
+
|
226
|
+
def to_dict(self) -> Dict:
|
227
|
+
"""Serializes the retention settings into a dictionary."""
|
228
|
+
return {
|
229
|
+
"archive_after_days": self.archive_after_days,
|
230
|
+
"delete_after_days": self.delete_after_days,
|
231
|
+
"customer_notification_type": self.customer_notification_type,
|
232
|
+
}
|
233
|
+
|
234
|
+
@classmethod
|
235
|
+
def from_dict(cls, data: Dict) -> "ModelRetentionSetting":
|
236
|
+
"""Constructs retention settings from a dictionary."""
|
237
|
+
return cls(
|
238
|
+
archive_after_days=data.get("archive_after_days"),
|
239
|
+
delete_after_days=data.get("delete_after_days"),
|
240
|
+
customer_notification_type=data.get("customer_notification_type") or None,
|
241
|
+
)
|
242
|
+
|
243
|
+
def to_json(self) -> str:
|
244
|
+
"""Serializes the retention settings into a JSON string."""
|
245
|
+
return json.dumps(self.to_dict())
|
246
|
+
|
247
|
+
@classmethod
|
248
|
+
def from_json(cls, json_str) -> "ModelRetentionSetting":
|
249
|
+
"""Constructs retention settings from a JSON string."""
|
250
|
+
data = json.loads(json_str) if isinstance(json_str, str) else json_str
|
251
|
+
return cls.from_dict(data)
|
252
|
+
|
253
|
+
def to_yaml(self) -> str:
|
254
|
+
"""Serializes the retention settings into a YAML string."""
|
255
|
+
return yaml.dump(self.to_dict())
|
256
|
+
|
257
|
+
def validate(self) -> bool:
|
258
|
+
"""Validates the retention settings details. Returns True if valid, False otherwise."""
|
259
|
+
return all(
|
260
|
+
[
|
261
|
+
self.archive_after_days is None
|
262
|
+
or (
|
263
|
+
isinstance(self.archive_after_days, int)
|
264
|
+
and self.archive_after_days >= 0
|
265
|
+
),
|
266
|
+
self.delete_after_days is None
|
267
|
+
or (
|
268
|
+
isinstance(self.delete_after_days, int)
|
269
|
+
and self.delete_after_days >= 0
|
270
|
+
),
|
271
|
+
isinstance(self.customer_notification_type, str)
|
272
|
+
and self.customer_notification_type
|
273
|
+
in CustomerNotificationType.values(),
|
274
|
+
]
|
275
|
+
)
|
276
|
+
|
277
|
+
def __repr__(self):
|
278
|
+
return self.to_yaml()
|
279
|
+
|
280
|
+
|
281
|
+
class ModelRetentionOperationDetails:
|
282
|
+
"""
|
283
|
+
Class that represents Model Retention Operation Details Metadata.
|
284
|
+
|
285
|
+
Methods
|
286
|
+
-------
|
287
|
+
to_dict(self) -> Dict:
|
288
|
+
Serializes the retention operation details into a dictionary.
|
289
|
+
from_dict(cls, data: Dict) -> 'ModelRetentionOperationDetails':
|
290
|
+
Constructs retention operation details from a dictionary.
|
291
|
+
to_json(self) -> str:
|
292
|
+
Serializes the retention operation details into a JSON string.
|
293
|
+
from_json(cls, json_str: str) -> 'ModelRetentionOperationDetails':
|
294
|
+
Constructs retention operation details from a JSON string.
|
295
|
+
to_yaml(self) -> str:
|
296
|
+
Serializes the retention operation details into a YAML string.
|
297
|
+
validate(self) -> bool:
|
298
|
+
Validates the retention operation details.
|
299
|
+
"""
|
300
|
+
|
301
|
+
def __init__(
|
302
|
+
self,
|
303
|
+
archive_state: Optional[SettingStatus] = None,
|
304
|
+
archive_state_details: Optional[str] = None,
|
305
|
+
delete_state: Optional[SettingStatus] = None,
|
306
|
+
delete_state_details: Optional[str] = None,
|
307
|
+
time_archival_scheduled: Optional[int] = None,
|
308
|
+
time_deletion_scheduled: Optional[int] = None,
|
309
|
+
):
|
310
|
+
self.archive_state = archive_state
|
311
|
+
self.archive_state_details = archive_state_details
|
312
|
+
self.delete_state = delete_state
|
313
|
+
self.delete_state_details = delete_state_details
|
314
|
+
self.time_archival_scheduled = time_archival_scheduled
|
315
|
+
self.time_deletion_scheduled = time_deletion_scheduled
|
316
|
+
|
317
|
+
def to_dict(self) -> Dict:
|
318
|
+
"""Serializes the retention operation details into a dictionary."""
|
319
|
+
return {
|
320
|
+
"archive_state": self.archive_state or None,
|
321
|
+
"archive_state_details": self.archive_state_details,
|
322
|
+
"delete_state": self.delete_state or None,
|
323
|
+
"delete_state_details": self.delete_state_details,
|
324
|
+
"time_archival_scheduled": self.time_archival_scheduled,
|
325
|
+
"time_deletion_scheduled": self.time_deletion_scheduled,
|
326
|
+
}
|
327
|
+
|
328
|
+
@classmethod
|
329
|
+
def from_dict(cls, data: Dict) -> "ModelRetentionOperationDetails":
|
330
|
+
"""Constructs retention operation details from a dictionary."""
|
331
|
+
return cls(
|
332
|
+
archive_state=data.get("archive_state") or None,
|
333
|
+
archive_state_details=data.get("archive_state_details"),
|
334
|
+
delete_state=data.get("delete_state") or None,
|
335
|
+
delete_state_details=data.get("delete_state_details"),
|
336
|
+
time_archival_scheduled=data.get("time_archival_scheduled"),
|
337
|
+
time_deletion_scheduled=data.get("time_deletion_scheduled"),
|
338
|
+
)
|
339
|
+
|
340
|
+
def to_json(self) -> str:
|
341
|
+
"""Serializes the retention operation details into a JSON string."""
|
342
|
+
return json.dumps(self.to_dict())
|
343
|
+
|
344
|
+
@classmethod
|
345
|
+
def from_json(cls, json_str: str) -> "ModelRetentionOperationDetails":
|
346
|
+
"""Constructs retention operation details from a JSON string."""
|
347
|
+
data = json.loads(json_str)
|
348
|
+
return cls.from_dict(data)
|
349
|
+
|
350
|
+
def to_yaml(self) -> str:
|
351
|
+
"""Serializes the retention operation details into a YAML string."""
|
352
|
+
return yaml.dump(self.to_dict())
|
353
|
+
|
354
|
+
def validate(self) -> bool:
|
355
|
+
"""Validates the retention operation details."""
|
356
|
+
return all(
|
357
|
+
[
|
358
|
+
self.archive_state is None
|
359
|
+
or self.archive_state in SettingStatus.values(),
|
360
|
+
self.delete_state is None
|
361
|
+
or self.delete_state in SettingStatus.values(),
|
362
|
+
self.time_archival_scheduled is None
|
363
|
+
or isinstance(self.time_archival_scheduled, int),
|
364
|
+
self.time_deletion_scheduled is None
|
365
|
+
or isinstance(self.time_deletion_scheduled, int),
|
366
|
+
]
|
367
|
+
)
|
368
|
+
|
369
|
+
def __repr__(self):
|
370
|
+
return self.to_yaml()
|
371
|
+
|
372
|
+
|
373
|
+
class ModelBackupOperationDetails:
|
374
|
+
"""
|
375
|
+
Class that represents Model Backup Operation Details Metadata.
|
376
|
+
|
377
|
+
Methods
|
378
|
+
-------
|
379
|
+
to_dict(self) -> Dict:
|
380
|
+
Serializes the backup operation details into a dictionary.
|
381
|
+
from_dict(cls, data: Dict) -> 'ModelBackupOperationDetails':
|
382
|
+
Constructs backup operation details from a dictionary.
|
383
|
+
to_json(self) -> str:
|
384
|
+
Serializes the backup operation details into a JSON string.
|
385
|
+
from_json(cls, json_str: str) -> 'ModelBackupOperationDetails':
|
386
|
+
Constructs backup operation details from a JSON string.
|
387
|
+
to_yaml(self) -> str:
|
388
|
+
Serializes the backup operation details into a YAML string.
|
389
|
+
validate(self) -> bool:
|
390
|
+
Validates the backup operation details.
|
391
|
+
"""
|
392
|
+
|
393
|
+
def __init__(
|
394
|
+
self,
|
395
|
+
backup_state: Optional[SettingStatus] = None,
|
396
|
+
backup_state_details: Optional[str] = None,
|
397
|
+
time_last_backup: Optional[int] = None,
|
398
|
+
):
|
399
|
+
self.backup_state = backup_state
|
400
|
+
self.backup_state_details = backup_state_details
|
401
|
+
self.time_last_backup = time_last_backup
|
402
|
+
|
403
|
+
def to_dict(self) -> Dict:
|
404
|
+
"""Serializes the backup operation details into a dictionary."""
|
405
|
+
return {
|
406
|
+
"backup_state": self.backup_state or None,
|
407
|
+
"backup_state_details": self.backup_state_details,
|
408
|
+
"time_last_backup": self.time_last_backup,
|
409
|
+
}
|
410
|
+
|
411
|
+
@classmethod
|
412
|
+
def from_dict(cls, data: Dict) -> "ModelBackupOperationDetails":
|
413
|
+
"""Constructs backup operation details from a dictionary."""
|
414
|
+
return cls(
|
415
|
+
backup_state=data.get("backup_state") or None,
|
416
|
+
backup_state_details=data.get("backup_state_details"),
|
417
|
+
time_last_backup=data.get("time_last_backup"),
|
418
|
+
)
|
419
|
+
|
420
|
+
def to_json(self) -> str:
|
421
|
+
"""Serializes the backup operation details into a JSON string."""
|
422
|
+
return json.dumps(self.to_dict())
|
423
|
+
|
424
|
+
@classmethod
|
425
|
+
def from_json(cls, json_str: str) -> "ModelBackupOperationDetails":
|
426
|
+
"""Constructs backup operation details from a JSON string."""
|
427
|
+
data = json.loads(json_str)
|
428
|
+
return cls.from_dict(data)
|
429
|
+
|
430
|
+
def to_yaml(self) -> str:
|
431
|
+
"""Serializes the backup operation details into a YAML string."""
|
432
|
+
return yaml.dump(self.to_dict())
|
433
|
+
|
434
|
+
def validate(self) -> bool:
|
435
|
+
"""Validates the backup operation details."""
|
436
|
+
return not (
|
437
|
+
(
|
438
|
+
self.backup_state is not None
|
439
|
+
and self.backup_state not in SettingStatus.values()
|
440
|
+
)
|
441
|
+
or (
|
442
|
+
self.time_last_backup is not None
|
443
|
+
and not isinstance(self.time_last_backup, int)
|
444
|
+
)
|
445
|
+
)
|
446
|
+
|
447
|
+
def __repr__(self):
|
448
|
+
return self.to_yaml()
|
449
|
+
|
450
|
+
|
451
|
+
class DataScienceModel(Builder):
|
452
|
+
"""Represents a Data Science Model.
|
453
|
+
|
454
|
+
Attributes
|
455
|
+
----------
|
456
|
+
id: str
|
457
|
+
Model ID.
|
458
|
+
project_id: str
|
459
|
+
Project OCID.
|
460
|
+
compartment_id: str
|
461
|
+
Compartment OCID.
|
462
|
+
name: str
|
463
|
+
Model name.
|
464
|
+
description: str
|
465
|
+
Model description.
|
466
|
+
freeform_tags: Dict[str, str]
|
467
|
+
Model freeform tags.
|
468
|
+
defined_tags: Dict[str, Dict[str, object]]
|
469
|
+
Model defined tags.
|
470
|
+
input_schema: ads.feature_engineering.Schema
|
471
|
+
Model input schema.
|
472
|
+
output_schema: ads.feature_engineering.Schema, Dict
|
473
|
+
Model output schema.
|
474
|
+
defined_metadata_list: ModelTaxonomyMetadata
|
475
|
+
Model defined metadata.
|
476
|
+
custom_metadata_list: ModelCustomMetadata
|
477
|
+
Model custom metadata.
|
478
|
+
provenance_metadata: ModelProvenanceMetadata
|
479
|
+
Model provenance metadata.
|
480
|
+
artifact: str
|
481
|
+
The artifact location. Can be either path to folder with artifacts or
|
482
|
+
path to zip archive.
|
483
|
+
status: Union[str, None]
|
484
|
+
Model status.
|
485
|
+
model_version_set_id: str
|
486
|
+
Model version set ID
|
487
|
+
version_label: str
|
488
|
+
Model version label
|
489
|
+
version_id: str
|
490
|
+
Model version id
|
491
|
+
model_file_description: dict
|
492
|
+
Contains object path details for models created by reference.
|
493
|
+
backup_setting: ModelBackupSetting
|
494
|
+
The value to assign to the backup_setting property of this CreateModelDetails.
|
495
|
+
retention_setting: ModelRetentionSetting
|
496
|
+
The value to assign to the retention_setting property of this CreateModelDetails.
|
497
|
+
retention_operation_details: ModelRetentionOperationDetails
|
498
|
+
The value to assign to the retention_operation_details property for the Model.
|
499
|
+
backup_operation_details: ModelBackupOperationDetails
|
500
|
+
The value to assign to the backup_operation_details property for the Model.
|
501
|
+
|
502
|
+
Methods
|
503
|
+
-------
|
504
|
+
create(self, **kwargs) -> "DataScienceModel"
|
505
|
+
Creates model.
|
506
|
+
delete(self, delete_associated_model_deployment: Optional[bool] = False) -> "DataScienceModel":
|
507
|
+
Removes model.
|
508
|
+
to_dict(self) -> dict
|
509
|
+
Serializes model to a dictionary.
|
510
|
+
from_id(cls, id: str) -> "DataScienceModel"
|
511
|
+
Gets an existing model by OCID.
|
512
|
+
from_dict(cls, config: dict) -> "DataScienceModel"
|
513
|
+
Loads model instance from a dictionary of configurations.
|
514
|
+
upload_artifact(self, ...) -> None
|
515
|
+
Uploads model artifacts to the model catalog.
|
516
|
+
download_artifact(self, ...) -> None
|
517
|
+
Downloads model artifacts from the model catalog.
|
518
|
+
update(self, **kwargs) -> "DataScienceModel"
|
519
|
+
Updates datascience model in model catalog.
|
520
|
+
list(cls, compartment_id: str = None, **kwargs) -> List["DataScienceModel"]
|
521
|
+
Lists datascience models in a given compartment.
|
522
|
+
sync(self):
|
523
|
+
Sync up a datascience model with OCI datascience model.
|
524
|
+
with_project_id(self, project_id: str) -> "DataScienceModel"
|
525
|
+
Sets the project ID.
|
526
|
+
with_description(self, description: str) -> "DataScienceModel"
|
527
|
+
Sets the description.
|
528
|
+
with_compartment_id(self, compartment_id: str) -> "DataScienceModel"
|
529
|
+
Sets the compartment ID.
|
530
|
+
with_display_name(self, name: str) -> "DataScienceModel"
|
531
|
+
Sets the name.
|
532
|
+
with_freeform_tags(self, **kwargs: Dict[str, str]) -> "DataScienceModel"
|
533
|
+
Sets freeform tags.
|
534
|
+
with_defined_tags(self, **kwargs: Dict[str, Dict[str, object]]) -> "DataScienceModel"
|
535
|
+
Sets defined tags.
|
536
|
+
with_input_schema(self, schema: Union[Schema, Dict]) -> "DataScienceModel"
|
537
|
+
Sets the model input schema.
|
538
|
+
with_output_schema(self, schema: Union[Schema, Dict]) -> "DataScienceModel"
|
539
|
+
Sets the model output schema.
|
540
|
+
with_defined_metadata_list(self, metadata: Union[ModelTaxonomyMetadata, Dict]) -> "DataScienceModel"
|
541
|
+
Sets model taxonomy (defined) metadata.
|
542
|
+
with_custom_metadata_list(self, metadata: Union[ModelCustomMetadata, Dict]) -> "DataScienceModel"
|
543
|
+
Sets model custom metadata.
|
544
|
+
with_provenance_metadata(self, metadata: Union[ModelProvenanceMetadata, Dict]) -> "DataScienceModel"
|
545
|
+
Sets model provenance metadata.
|
546
|
+
with_artifact(self, *uri: str)
|
547
|
+
Sets the artifact location. Can be a local. For models created by reference, uri can take in single arg or multiple args in case
|
548
|
+
of a fine-tuned or multimodel setting.
|
549
|
+
with_model_version_set_id(self, model_version_set_id: str):
|
550
|
+
Sets the model version set ID.
|
551
|
+
with_version_label(self, version_label: str):
|
552
|
+
Sets the model version label.
|
553
|
+
with_version_id(self, version_id: str):
|
554
|
+
Sets the model version id.
|
555
|
+
with_model_file_description: dict
|
556
|
+
Sets path details for models created by reference. Input can be either a dict, string or json file and
|
557
|
+
the schema is dictated by model_file_description_schema.json
|
558
|
+
|
559
|
+
|
560
|
+
Examples
|
561
|
+
--------
|
562
|
+
>>> ds_model = (DataScienceModel()
|
563
|
+
... .with_compartment_id(os.environ["NB_SESSION_COMPARTMENT_OCID"])
|
564
|
+
... .with_project_id(os.environ["PROJECT_OCID"])
|
565
|
+
... .with_display_name("TestModel")
|
566
|
+
... .with_description("Testing the test model")
|
567
|
+
... .with_freeform_tags(tag1="val1", tag2="val2")
|
568
|
+
... .with_artifact("/path/to/the/model/artifacts/"))
|
569
|
+
>>> ds_model.create()
|
570
|
+
>>> ds_model.status()
|
571
|
+
>>> ds_model.with_description("new description").update()
|
572
|
+
>>> ds_model.download_artifact("/path/to/dst/folder/")
|
573
|
+
>>> ds_model.delete()
|
574
|
+
>>> DataScienceModel.list()
|
575
|
+
"""
|
576
|
+
|
577
|
+
_PREFIX = "datascience_model"
|
578
|
+
|
579
|
+
CONST_ID = "id"
|
580
|
+
CONST_PROJECT_ID = "projectId"
|
581
|
+
CONST_COMPARTMENT_ID = "compartmentId"
|
582
|
+
CONST_DISPLAY_NAME = "displayName"
|
583
|
+
CONST_DESCRIPTION = "description"
|
584
|
+
CONST_FREEFORM_TAG = "freeformTags"
|
585
|
+
CONST_DEFINED_TAG = "definedTags"
|
586
|
+
CONST_INPUT_SCHEMA = "inputSchema"
|
587
|
+
CONST_OUTPUT_SCHEMA = "outputSchema"
|
588
|
+
CONST_CUSTOM_METADATA = "customMetadataList"
|
589
|
+
CONST_DEFINED_METADATA = "definedMetadataList"
|
590
|
+
CONST_PROVENANCE_METADATA = "provenanceMetadata"
|
591
|
+
CONST_ARTIFACT = "artifact"
|
592
|
+
CONST_MODEL_VERSION_SET_ID = "modelVersionSetId"
|
593
|
+
CONST_MODEL_VERSION_SET_NAME = "modelVersionSetName"
|
594
|
+
CONST_MODEL_VERSION_LABEL = "versionLabel"
|
595
|
+
CONST_MODEL_VERSION_ID = "versionId"
|
596
|
+
CONST_TIME_CREATED = "timeCreated"
|
597
|
+
CONST_LIFECYCLE_STATE = "lifecycleState"
|
598
|
+
CONST_LIFECYCLE_DETAILS = "lifecycleDetails"
|
599
|
+
CONST_MODEL_FILE_DESCRIPTION = "modelDescription"
|
600
|
+
CONST_BACKUP_SETTING = "backupSetting"
|
601
|
+
CONST_RETENTION_SETTING = "retentionSetting"
|
602
|
+
CONST_BACKUP_OPERATION_DETAILS = "backupOperationDetails"
|
603
|
+
CONST_RETENTION_OPERATION_DETAILS = "retentionOperationDetails"
|
604
|
+
|
605
|
+
attribute_map = {
|
606
|
+
CONST_ID: "id",
|
607
|
+
CONST_PROJECT_ID: "project_id",
|
608
|
+
CONST_COMPARTMENT_ID: "compartment_id",
|
609
|
+
CONST_DISPLAY_NAME: "display_name",
|
610
|
+
CONST_DESCRIPTION: "description",
|
611
|
+
CONST_FREEFORM_TAG: "freeform_tags",
|
612
|
+
CONST_DEFINED_TAG: "defined_tags",
|
613
|
+
CONST_INPUT_SCHEMA: "input_schema",
|
614
|
+
CONST_OUTPUT_SCHEMA: "output_schema",
|
615
|
+
CONST_CUSTOM_METADATA: "custom_metadata_list",
|
616
|
+
CONST_DEFINED_METADATA: "defined_metadata_list",
|
617
|
+
CONST_PROVENANCE_METADATA: "provenance_metadata",
|
618
|
+
CONST_ARTIFACT: "artifact",
|
619
|
+
CONST_MODEL_VERSION_SET_ID: "model_version_set_id",
|
620
|
+
CONST_MODEL_VERSION_SET_NAME: "model_version_set_name",
|
621
|
+
CONST_MODEL_VERSION_LABEL: "version_label",
|
622
|
+
CONST_MODEL_VERSION_ID: "version_id",
|
623
|
+
CONST_TIME_CREATED: "time_created",
|
624
|
+
CONST_LIFECYCLE_STATE: "lifecycle_state",
|
625
|
+
CONST_LIFECYCLE_DETAILS: "lifecycle_details",
|
626
|
+
CONST_MODEL_FILE_DESCRIPTION: "model_description",
|
627
|
+
CONST_BACKUP_SETTING: "backup_setting",
|
628
|
+
CONST_RETENTION_SETTING: "retention_setting",
|
629
|
+
CONST_BACKUP_OPERATION_DETAILS: "backup_operation_details",
|
630
|
+
CONST_RETENTION_OPERATION_DETAILS: "retention_operation_details",
|
631
|
+
}
|
632
|
+
|
633
|
+
def __init__(self, spec: Dict = None, **kwargs) -> None:
|
634
|
+
"""Initializes datascience model.
|
635
|
+
|
636
|
+
Parameters
|
637
|
+
----------
|
638
|
+
spec: (Dict, optional). Defaults to None.
|
639
|
+
Object specification.
|
640
|
+
|
641
|
+
kwargs: Dict
|
642
|
+
Specification as keyword arguments.
|
643
|
+
If 'spec' contains the same key as the one in kwargs,
|
644
|
+
the value from kwargs will be used.
|
645
|
+
|
646
|
+
- project_id: str
|
647
|
+
- compartment_id: str
|
648
|
+
- name: str
|
649
|
+
- description: str
|
650
|
+
- defined_tags: Dict[str, Dict[str, object]]
|
651
|
+
- freeform_tags: Dict[str, str]
|
652
|
+
- input_schema: Union[ads.feature_engineering.Schema, Dict]
|
653
|
+
- output_schema: Union[ads.feature_engineering.Schema, Dict]
|
654
|
+
- defined_metadata_list: Union[ModelTaxonomyMetadata, Dict]
|
655
|
+
- custom_metadata_list: Union[ModelCustomMetadata, Dict]
|
656
|
+
- provenance_metadata: Union[ModelProvenanceMetadata, Dict]
|
657
|
+
- artifact: str
|
658
|
+
"""
|
659
|
+
super().__init__(spec=spec, **deepcopy(kwargs))
|
660
|
+
# Reinitiate complex attributes
|
661
|
+
self._init_complex_attributes()
|
662
|
+
# Specify oci datascience model instance
|
663
|
+
self.dsc_model = self._to_oci_dsc_model()
|
664
|
+
self.local_copy_dir = None
|
665
|
+
|
666
|
+
@property
|
667
|
+
def id(self) -> Optional[str]:
|
668
|
+
"""The model OCID."""
|
669
|
+
if self.dsc_model:
|
670
|
+
return self.dsc_model.id
|
671
|
+
return None
|
672
|
+
|
673
|
+
@property
|
674
|
+
def status(self) -> Union[str, None]:
|
675
|
+
"""Status of the model.
|
676
|
+
|
677
|
+
Returns
|
678
|
+
-------
|
679
|
+
str
|
680
|
+
Status of the model.
|
681
|
+
"""
|
682
|
+
if self.dsc_model:
|
683
|
+
return self.dsc_model.status
|
684
|
+
return None
|
685
|
+
|
686
|
+
@property
|
687
|
+
def lifecycle_state(self) -> Union[str, None]:
|
688
|
+
"""Status of the model.
|
689
|
+
|
690
|
+
Returns
|
691
|
+
-------
|
692
|
+
str
|
693
|
+
Status of the model.
|
694
|
+
"""
|
695
|
+
if self.dsc_model:
|
696
|
+
return self.dsc_model.status
|
697
|
+
return None
|
698
|
+
|
699
|
+
@property
|
700
|
+
def lifecycle_details(self) -> str:
|
701
|
+
"""
|
702
|
+
Gets the lifecycle_details of this DataScienceModel.
|
703
|
+
Details about the lifecycle state of the model.
|
704
|
+
|
705
|
+
:return: The lifecycle_details of this DataScienceModel.
|
706
|
+
:rtype: str
|
707
|
+
"""
|
708
|
+
return self.get_spec(self.CONST_LIFECYCLE_DETAILS)
|
709
|
+
|
710
|
+
@lifecycle_details.setter
|
711
|
+
def lifecycle_details(self, lifecycle_details: str) -> "DataScienceModel":
|
712
|
+
"""
|
713
|
+
Sets the lifecycle_details of this DataScienceModel.
|
714
|
+
Details about the lifecycle state of the model.
|
715
|
+
|
716
|
+
:param lifecycle_details: The lifecycle_details of this DataScienceModel.
|
717
|
+
:type: str
|
718
|
+
"""
|
719
|
+
return self.set_spec(self.CONST_LIFECYCLE_DETAILS, lifecycle_details)
|
720
|
+
|
721
|
+
@property
|
722
|
+
def kind(self) -> str:
|
723
|
+
"""The kind of the object as showing in a YAML."""
|
724
|
+
return "datascienceModel"
|
725
|
+
|
726
|
+
@property
|
727
|
+
def project_id(self) -> str:
|
728
|
+
return self.get_spec(self.CONST_PROJECT_ID)
|
729
|
+
|
730
|
+
def with_project_id(self, project_id: str) -> "DataScienceModel":
|
731
|
+
"""Sets the project ID.
|
732
|
+
|
733
|
+
Parameters
|
734
|
+
----------
|
735
|
+
project_id: str
|
736
|
+
The project ID.
|
737
|
+
|
738
|
+
Returns
|
739
|
+
-------
|
740
|
+
DataScienceModel
|
741
|
+
The DataScienceModel instance (self)
|
742
|
+
"""
|
743
|
+
return self.set_spec(self.CONST_PROJECT_ID, project_id)
|
744
|
+
|
745
|
+
@property
|
746
|
+
def time_created(self) -> str:
|
747
|
+
return self.get_spec(self.CONST_TIME_CREATED)
|
748
|
+
|
749
|
+
@property
|
750
|
+
def description(self) -> str:
|
751
|
+
return self.get_spec(self.CONST_DESCRIPTION)
|
752
|
+
|
753
|
+
def with_description(self, description: str) -> "DataScienceModel":
|
754
|
+
"""Sets the description.
|
755
|
+
|
756
|
+
Parameters
|
757
|
+
----------
|
758
|
+
description: str
|
759
|
+
The description of the model.
|
760
|
+
|
761
|
+
Returns
|
762
|
+
-------
|
763
|
+
DataScienceModel
|
764
|
+
The DataScienceModel instance (self)
|
765
|
+
"""
|
766
|
+
return self.set_spec(self.CONST_DESCRIPTION, description)
|
767
|
+
|
768
|
+
@property
|
769
|
+
def compartment_id(self) -> str:
|
770
|
+
return self.get_spec(self.CONST_COMPARTMENT_ID)
|
771
|
+
|
772
|
+
def with_compartment_id(self, compartment_id: str) -> "DataScienceModel":
|
773
|
+
"""Sets the compartment ID.
|
774
|
+
|
775
|
+
Parameters
|
776
|
+
----------
|
777
|
+
compartment_id: str
|
778
|
+
The compartment ID.
|
779
|
+
|
780
|
+
Returns
|
781
|
+
-------
|
782
|
+
DataScienceModel
|
783
|
+
The DataScienceModel instance (self)
|
784
|
+
"""
|
785
|
+
return self.set_spec(self.CONST_COMPARTMENT_ID, compartment_id)
|
786
|
+
|
787
|
+
@property
|
788
|
+
def display_name(self) -> str:
|
789
|
+
return self.get_spec(self.CONST_DISPLAY_NAME)
|
790
|
+
|
791
|
+
@display_name.setter
|
792
|
+
def display_name(self, name: str) -> "DataScienceModel":
|
793
|
+
return self.set_spec(self.CONST_DISPLAY_NAME, name)
|
794
|
+
|
795
|
+
def with_display_name(self, name: str) -> "DataScienceModel":
|
796
|
+
"""Sets the name.
|
797
|
+
|
798
|
+
Parameters
|
799
|
+
----------
|
800
|
+
name: str
|
801
|
+
The name.
|
802
|
+
|
803
|
+
Returns
|
804
|
+
-------
|
805
|
+
DataScienceModel
|
806
|
+
The DataScienceModel instance (self)
|
807
|
+
"""
|
808
|
+
return self.set_spec(self.CONST_DISPLAY_NAME, name)
|
809
|
+
|
810
|
+
@property
|
811
|
+
def freeform_tags(self) -> Dict[str, str]:
|
812
|
+
return self.get_spec(self.CONST_FREEFORM_TAG)
|
813
|
+
|
814
|
+
def with_freeform_tags(self, **kwargs: Dict[str, str]) -> "DataScienceModel":
|
815
|
+
"""Sets freeform tags.
|
816
|
+
|
817
|
+
Returns
|
818
|
+
-------
|
819
|
+
DataScienceModel
|
820
|
+
The DataScienceModel instance (self)
|
821
|
+
"""
|
822
|
+
return self.set_spec(self.CONST_FREEFORM_TAG, kwargs)
|
823
|
+
|
824
|
+
@property
|
825
|
+
def defined_tags(self) -> Dict[str, Dict[str, object]]:
|
826
|
+
return self.get_spec(self.CONST_DEFINED_TAG)
|
827
|
+
|
828
|
+
def with_defined_tags(
|
829
|
+
self, **kwargs: Dict[str, Dict[str, object]]
|
830
|
+
) -> "DataScienceModel":
|
831
|
+
"""Sets defined tags.
|
832
|
+
|
833
|
+
Returns
|
834
|
+
-------
|
835
|
+
DataScienceModel
|
836
|
+
The DataScienceModel instance (self)
|
837
|
+
"""
|
838
|
+
return self.set_spec(self.CONST_DEFINED_TAG, kwargs)
|
839
|
+
|
840
|
+
@property
|
841
|
+
def input_schema(self) -> Union[Schema, Dict]:
|
842
|
+
"""Returns model input schema.
|
843
|
+
|
844
|
+
Returns
|
845
|
+
-------
|
846
|
+
ads.feature_engineering.Schema
|
847
|
+
Model input schema.
|
848
|
+
"""
|
849
|
+
return self.get_spec(self.CONST_INPUT_SCHEMA)
|
850
|
+
|
851
|
+
def with_input_schema(self, schema: Union[Schema, Dict]) -> "DataScienceModel":
|
852
|
+
"""Sets the model input schema.
|
853
|
+
|
854
|
+
Parameters
|
855
|
+
----------
|
856
|
+
schema: Union[ads.feature_engineering.Schema, Dict]
|
857
|
+
The model input schema.
|
858
|
+
|
859
|
+
Returns
|
860
|
+
-------
|
861
|
+
DataScienceModel
|
862
|
+
The DataScienceModel instance (self)
|
863
|
+
"""
|
864
|
+
if schema and isinstance(schema, Dict):
|
865
|
+
try:
|
866
|
+
schema = Schema.from_dict(schema)
|
867
|
+
except Exception as err:
|
868
|
+
logger.warn(err)
|
869
|
+
|
870
|
+
return self.set_spec(self.CONST_INPUT_SCHEMA, schema)
|
871
|
+
|
872
|
+
@property
|
873
|
+
def output_schema(self) -> Union[Schema, Dict]:
|
874
|
+
"""Returns model output schema.
|
875
|
+
|
876
|
+
Returns
|
877
|
+
-------
|
878
|
+
ads.feature_engineering.Schema
|
879
|
+
Model output schema.
|
880
|
+
"""
|
881
|
+
return self.get_spec(self.CONST_OUTPUT_SCHEMA)
|
882
|
+
|
883
|
+
def with_output_schema(self, schema: Union[Schema, Dict]) -> "DataScienceModel":
|
884
|
+
"""Sets the model output schema.
|
885
|
+
|
886
|
+
Parameters
|
887
|
+
----------
|
888
|
+
schema: Union[ads.feature_engineering.Schema, Dict]
|
889
|
+
The model output schema.
|
890
|
+
|
891
|
+
Returns
|
892
|
+
-------
|
893
|
+
DataScienceModel
|
894
|
+
The DataScienceModel instance (self)
|
895
|
+
"""
|
896
|
+
if schema and isinstance(schema, Dict):
|
897
|
+
try:
|
898
|
+
schema = Schema.from_dict(schema)
|
899
|
+
except Exception as err:
|
900
|
+
logger.warn(err)
|
901
|
+
|
902
|
+
return self.set_spec(self.CONST_OUTPUT_SCHEMA, schema)
|
903
|
+
|
904
|
+
@property
|
905
|
+
def defined_metadata_list(self) -> ModelTaxonomyMetadata:
|
906
|
+
"""Returns model taxonomy (defined) metadatda."""
|
907
|
+
return self.get_spec(self.CONST_DEFINED_METADATA)
|
908
|
+
|
909
|
+
def with_defined_metadata_list(
|
910
|
+
self, metadata: Union[ModelTaxonomyMetadata, Dict]
|
911
|
+
) -> "DataScienceModel":
|
912
|
+
"""Sets model taxonomy (defined) metadata.
|
913
|
+
|
914
|
+
Parameters
|
915
|
+
----------
|
916
|
+
metadata: Union[ModelTaxonomyMetadata, Dict]
|
917
|
+
The defined metadata.
|
918
|
+
|
919
|
+
Returns
|
920
|
+
-------
|
921
|
+
DataScienceModel
|
922
|
+
The DataScienceModel instance (self)
|
923
|
+
"""
|
924
|
+
if metadata and isinstance(metadata, Dict):
|
925
|
+
metadata = ModelTaxonomyMetadata.from_dict(metadata)
|
926
|
+
return self.set_spec(self.CONST_DEFINED_METADATA, metadata)
|
927
|
+
|
928
|
+
@property
|
929
|
+
def custom_metadata_list(self) -> ModelCustomMetadata:
|
930
|
+
"""Returns model custom metadatda."""
|
931
|
+
return self.get_spec(self.CONST_CUSTOM_METADATA)
|
932
|
+
|
933
|
+
def with_custom_metadata_list(
|
934
|
+
self, metadata: Union[ModelCustomMetadata, Dict]
|
935
|
+
) -> "DataScienceModel":
|
936
|
+
"""Sets model custom metadata.
|
937
|
+
|
938
|
+
Parameters
|
939
|
+
----------
|
940
|
+
metadata: Union[ModelCustomMetadata, Dict]
|
941
|
+
The custom metadata.
|
942
|
+
|
943
|
+
Returns
|
944
|
+
-------
|
945
|
+
DataScienceModel
|
946
|
+
The DataScienceModel instance (self)
|
947
|
+
"""
|
948
|
+
if metadata and isinstance(metadata, Dict):
|
949
|
+
metadata = ModelCustomMetadata.from_dict(metadata)
|
950
|
+
return self.set_spec(self.CONST_CUSTOM_METADATA, metadata)
|
951
|
+
|
952
|
+
@property
|
953
|
+
def provenance_metadata(self) -> ModelProvenanceMetadata:
|
954
|
+
"""Returns model provenance metadatda."""
|
955
|
+
return self.get_spec(self.CONST_PROVENANCE_METADATA)
|
956
|
+
|
957
|
+
def with_provenance_metadata(
|
958
|
+
self, metadata: Union[ModelProvenanceMetadata, Dict]
|
959
|
+
) -> "DataScienceModel":
|
960
|
+
"""Sets model provenance metadata.
|
961
|
+
|
962
|
+
Parameters
|
963
|
+
----------
|
964
|
+
provenance_metadata: Union[ModelProvenanceMetadata, Dict]
|
965
|
+
The provenance metadata.
|
966
|
+
|
967
|
+
Returns
|
968
|
+
-------
|
969
|
+
DataScienceModel
|
970
|
+
The DataScienceModel instance (self)
|
971
|
+
"""
|
972
|
+
if metadata and isinstance(metadata, Dict):
|
973
|
+
metadata = ModelProvenanceMetadata.from_dict(metadata)
|
974
|
+
return self.set_spec(self.CONST_PROVENANCE_METADATA, metadata)
|
975
|
+
|
976
|
+
@property
|
977
|
+
def artifact(self) -> Union[str, list]:
|
978
|
+
return self.get_spec(self.CONST_ARTIFACT)
|
979
|
+
|
980
|
+
def with_artifact(self, uri: str, *args):
|
981
|
+
"""Sets the artifact location. Can be a local.
|
982
|
+
|
983
|
+
Parameters
|
984
|
+
----------
|
985
|
+
uri: str
|
986
|
+
Path to artifact directory or to the ZIP archive.
|
987
|
+
It could contain a serialized model(required) as well as any files needed for deployment.
|
988
|
+
The content of the source folder will be zipped and uploaded to the model catalog.
|
989
|
+
For models created by reference, uri can take in single arg or multiple args in case of a fine-tuned or
|
990
|
+
multimodel setting.
|
991
|
+
Examples
|
992
|
+
--------
|
993
|
+
>>> .with_artifact(uri="./model1/")
|
994
|
+
>>> .with_artifact(uri="./model1.zip")
|
995
|
+
>>> .with_artifact("./model1", "./model2")
|
996
|
+
"""
|
997
|
+
|
998
|
+
return self.set_spec(self.CONST_ARTIFACT, [uri] + list(args) if args else uri)
|
999
|
+
|
1000
|
+
@property
|
1001
|
+
def model_version_set_id(self) -> str:
|
1002
|
+
return self.get_spec(self.CONST_MODEL_VERSION_SET_ID)
|
1003
|
+
|
1004
|
+
def with_model_version_set_id(self, model_version_set_id: str):
|
1005
|
+
"""Sets the model version set ID.
|
1006
|
+
|
1007
|
+
Parameters
|
1008
|
+
----------
|
1009
|
+
urmodel_version_set_idi: str
|
1010
|
+
The Model version set OCID.
|
1011
|
+
"""
|
1012
|
+
return self.set_spec(self.CONST_MODEL_VERSION_SET_ID, model_version_set_id)
|
1013
|
+
|
1014
|
+
@property
|
1015
|
+
def model_version_set_name(self) -> str:
|
1016
|
+
return self.get_spec(self.CONST_MODEL_VERSION_SET_NAME)
|
1017
|
+
|
1018
|
+
@property
|
1019
|
+
def version_label(self) -> str:
|
1020
|
+
return self.get_spec(self.CONST_MODEL_VERSION_LABEL)
|
1021
|
+
|
1022
|
+
def with_version_label(self, version_label: str):
|
1023
|
+
"""Sets the model version label.
|
1024
|
+
|
1025
|
+
Parameters
|
1026
|
+
----------
|
1027
|
+
version_label: str
|
1028
|
+
The model version label.
|
1029
|
+
"""
|
1030
|
+
return self.set_spec(self.CONST_MODEL_VERSION_LABEL, version_label)
|
1031
|
+
|
1032
|
+
@property
|
1033
|
+
def version_id(self) -> str:
|
1034
|
+
return self.get_spec(self.CONST_MODEL_VERSION_ID)
|
1035
|
+
|
1036
|
+
def with_version_id(self, version_id: str):
|
1037
|
+
"""Sets the model version id.
|
1038
|
+
|
1039
|
+
Parameters
|
1040
|
+
----------
|
1041
|
+
version_id: str
|
1042
|
+
The model version id.
|
1043
|
+
"""
|
1044
|
+
return self.set_spec(self.CONST_MODEL_VERSION_ID, version_id)
|
1045
|
+
|
1046
|
+
@property
|
1047
|
+
def model_file_description(self) -> dict:
|
1048
|
+
return self.get_spec(self.CONST_MODEL_FILE_DESCRIPTION)
|
1049
|
+
|
1050
|
+
def with_model_file_description(
|
1051
|
+
self, json_dict: dict = None, json_string: str = None, json_uri: str = None
|
1052
|
+
):
|
1053
|
+
"""Sets the json file description for model passed by reference
|
1054
|
+
Parameters
|
1055
|
+
----------
|
1056
|
+
json_dict : dict, optional
|
1057
|
+
json dict, by default None
|
1058
|
+
json_string : str, optional
|
1059
|
+
json string, by default None
|
1060
|
+
json_uri : str, optional
|
1061
|
+
URI location of file containing json, by default None
|
1062
|
+
|
1063
|
+
Examples
|
1064
|
+
--------
|
1065
|
+
>>> DataScienceModel().with_model_file_description(json_string="<json_string>")
|
1066
|
+
>>> DataScienceModel().with_model_file_description(json_dict=dict())
|
1067
|
+
>>> DataScienceModel().with_model_file_description(json_uri="./model_description.json")
|
1068
|
+
"""
|
1069
|
+
if json_dict:
|
1070
|
+
json_data = json_dict
|
1071
|
+
elif json_string:
|
1072
|
+
json_data = json.loads(json_string)
|
1073
|
+
elif json_uri:
|
1074
|
+
with open(json_uri) as json_file:
|
1075
|
+
json_data = json.load(json_file)
|
1076
|
+
else:
|
1077
|
+
raise ValueError("Must provide either a valid json string or URI location.")
|
1078
|
+
|
1079
|
+
schema_file_path = os.path.join(
|
1080
|
+
os.path.dirname(os.path.abspath(__file__)),
|
1081
|
+
"model_file_description_schema.json",
|
1082
|
+
)
|
1083
|
+
with open(schema_file_path, encoding="utf-8") as schema_file:
|
1084
|
+
schema = json.load(schema_file)
|
1085
|
+
|
1086
|
+
try:
|
1087
|
+
validate(json_data, schema)
|
1088
|
+
except ValidationError as ve:
|
1089
|
+
message = (
|
1090
|
+
f"model_file_description_schema.json validation failed. "
|
1091
|
+
f"See Exception: {ve}"
|
1092
|
+
)
|
1093
|
+
logging.error(message)
|
1094
|
+
raise ModelFileDescriptionError(message)
|
1095
|
+
|
1096
|
+
return self.set_spec(self.CONST_MODEL_FILE_DESCRIPTION, json_data)
|
1097
|
+
|
1098
|
+
@property
|
1099
|
+
def retention_setting(self) -> ModelRetentionSetting:
|
1100
|
+
"""
|
1101
|
+
Gets the retention_setting of this model.
|
1102
|
+
|
1103
|
+
:return: The retention_setting of this model.
|
1104
|
+
:rtype: RetentionSetting
|
1105
|
+
"""
|
1106
|
+
return self.get_spec(self.CONST_RETENTION_SETTING)
|
1107
|
+
|
1108
|
+
def with_retention_setting(
|
1109
|
+
self, retention_setting: Union[Dict, ModelRetentionSetting]
|
1110
|
+
) -> "DataScienceModel":
|
1111
|
+
"""
|
1112
|
+
Sets the retention setting details for the model.
|
1113
|
+
|
1114
|
+
Parameters
|
1115
|
+
----------
|
1116
|
+
retention_setting : Union[Dict, RetentionSetting]
|
1117
|
+
The retention setting details for the model. Can be provided as either a dictionary or
|
1118
|
+
an instance of the `RetentionSetting` class.
|
1119
|
+
|
1120
|
+
Returns
|
1121
|
+
-------
|
1122
|
+
DataScienceModel
|
1123
|
+
The `DataScienceModel` instance (self) for method chaining.
|
1124
|
+
"""
|
1125
|
+
return self.set_spec(self.CONST_RETENTION_SETTING, retention_setting)
|
1126
|
+
|
1127
|
+
@property
|
1128
|
+
def backup_setting(self) -> ModelBackupSetting:
|
1129
|
+
"""
|
1130
|
+
Gets the backup_setting of this model.
|
1131
|
+
|
1132
|
+
:return: The backup_setting of this model.
|
1133
|
+
:rtype: BackupSetting
|
1134
|
+
"""
|
1135
|
+
return self.get_spec(self.CONST_BACKUP_SETTING)
|
1136
|
+
|
1137
|
+
def with_backup_setting(
|
1138
|
+
self, backup_setting: Union[Dict, ModelBackupSetting]
|
1139
|
+
) -> "DataScienceModel":
|
1140
|
+
"""
|
1141
|
+
Sets the model's backup setting details.
|
1142
|
+
|
1143
|
+
Parameters
|
1144
|
+
----------
|
1145
|
+
backup_setting : Union[Dict, BackupSetting]
|
1146
|
+
The backup setting details for the model. This can be passed as either a dictionary or
|
1147
|
+
an instance of the `BackupSetting` class.
|
1148
|
+
|
1149
|
+
Returns
|
1150
|
+
-------
|
1151
|
+
DataScienceModel
|
1152
|
+
The `DataScienceModel` instance (self) for method chaining.
|
1153
|
+
"""
|
1154
|
+
|
1155
|
+
return self.set_spec(self.CONST_BACKUP_SETTING, backup_setting)
|
1156
|
+
|
1157
|
+
@property
|
1158
|
+
def retention_operation_details(self) -> ModelRetentionOperationDetails:
|
1159
|
+
"""
|
1160
|
+
Gets the retention_operation_details of this Model using the spec constant.
|
1161
|
+
|
1162
|
+
:return: The retention_operation_details of this Model.
|
1163
|
+
:rtype: ModelRetentionOperationDetails
|
1164
|
+
"""
|
1165
|
+
return self.get_spec(self.CONST_RETENTION_OPERATION_DETAILS)
|
1166
|
+
|
1167
|
+
@property
|
1168
|
+
def backup_operation_details(self) -> "ModelBackupOperationDetails":
|
1169
|
+
"""
|
1170
|
+
Gets the backup_operation_details of this Model using the spec constant.
|
1171
|
+
|
1172
|
+
:return: The backup_operation_details of this Model.
|
1173
|
+
:rtype: ModelBackupOperationDetails
|
1174
|
+
"""
|
1175
|
+
return self.get_spec(self.CONST_BACKUP_OPERATION_DETAILS)
|
1176
|
+
|
1177
|
+
def create(self, **kwargs) -> "DataScienceModel":
|
1178
|
+
"""Creates datascience model.
|
1179
|
+
|
1180
|
+
Parameters
|
1181
|
+
----------
|
1182
|
+
kwargs
|
1183
|
+
Additional kwargs arguments.
|
1184
|
+
Can be any attribute that `oci.data_science.models.Model` accepts.
|
1185
|
+
|
1186
|
+
In addition can be also provided the attributes listed below.
|
1187
|
+
|
1188
|
+
bucket_uri: (str, optional). Defaults to None.
|
1189
|
+
The OCI Object Storage URI where model artifacts will be copied to.
|
1190
|
+
The `bucket_uri` is only necessary for uploading large artifacts which
|
1191
|
+
size is greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
|
1192
|
+
|
1193
|
+
.. versionadded:: 2.8.10
|
1194
|
+
|
1195
|
+
If `artifact` is provided as an object storage path to a zip archive, `bucket_uri` will be ignored.
|
1196
|
+
|
1197
|
+
overwrite_existing_artifact: (bool, optional). Defaults to `True`.
|
1198
|
+
Overwrite target bucket artifact if exists.
|
1199
|
+
remove_existing_artifact: (bool, optional). Defaults to `True`.
|
1200
|
+
Wether artifacts uploaded to object storage bucket need to be removed or not.
|
1201
|
+
region: (str, optional). Defaults to `None`.
|
1202
|
+
The destination Object Storage bucket region.
|
1203
|
+
By default the value will be extracted from the `OCI_REGION_METADATA` environment variable.
|
1204
|
+
auth: (Dict, optional). Defaults to `None`.
|
1205
|
+
The default authentication is set using `ads.set_auth` API.
|
1206
|
+
If you need to override the default, use the `ads.common.auth.api_keys` or
|
1207
|
+
`ads.common.auth.resource_principal` to create appropriate authentication signer
|
1208
|
+
and kwargs required to instantiate IdentityClient object.
|
1209
|
+
timeout: (int, optional). Defaults to 10 seconds.
|
1210
|
+
The connection timeout in seconds for the client.
|
1211
|
+
parallel_process_count: (int, optional).
|
1212
|
+
The number of worker processes to use in parallel for uploading individual parts of a multipart upload.
|
1213
|
+
model_by_reference: (bool, optional)
|
1214
|
+
Whether model artifact is made available to Model Store by reference. Requires artifact location to be
|
1215
|
+
provided using with_artifact method.
|
1216
|
+
|
1217
|
+
Returns
|
1218
|
+
-------
|
1219
|
+
DataScienceModel
|
1220
|
+
The DataScienceModel instance (self)
|
1221
|
+
|
1222
|
+
Raises
|
1223
|
+
------
|
1224
|
+
ValueError
|
1225
|
+
If compartment id not provided.
|
1226
|
+
If project id not provided.
|
1227
|
+
"""
|
1228
|
+
if not self.compartment_id:
|
1229
|
+
raise ValueError("Compartment id must be provided.")
|
1230
|
+
|
1231
|
+
if not self.project_id:
|
1232
|
+
raise ValueError("Project id must be provided.")
|
1233
|
+
|
1234
|
+
if not self.display_name:
|
1235
|
+
self.display_name = self._random_display_name()
|
1236
|
+
|
1237
|
+
model_by_reference = kwargs.pop("model_by_reference", False)
|
1238
|
+
if model_by_reference:
|
1239
|
+
# Update custom metadata
|
1240
|
+
logger.info("Update custom metadata field with model by reference flag.")
|
1241
|
+
metadata_item = ModelCustomMetadataItem(
|
1242
|
+
key=self.CONST_MODEL_FILE_DESCRIPTION,
|
1243
|
+
value="true",
|
1244
|
+
description="model by reference flag",
|
1245
|
+
category=MetadataCustomCategory.OTHER,
|
1246
|
+
)
|
1247
|
+
if self.custom_metadata_list:
|
1248
|
+
self.custom_metadata_list._add(metadata_item, replace=True)
|
1249
|
+
else:
|
1250
|
+
custom_metadata = ModelCustomMetadata()
|
1251
|
+
custom_metadata._add(metadata_item)
|
1252
|
+
self.with_custom_metadata_list(custom_metadata)
|
1253
|
+
|
1254
|
+
payload = deepcopy(self._spec)
|
1255
|
+
payload.pop("id", None)
|
1256
|
+
logger.debug(f"Creating a model with payload {payload}")
|
1257
|
+
|
1258
|
+
# Create model in the model catalog
|
1259
|
+
logger.info("Saving model to the Model Catalog.")
|
1260
|
+
self.dsc_model = self._to_oci_dsc_model(**kwargs).create()
|
1261
|
+
|
1262
|
+
# Create model provenance
|
1263
|
+
if self.provenance_metadata:
|
1264
|
+
logger.info("Saving model provenance metadata.")
|
1265
|
+
self.dsc_model.create_model_provenance(
|
1266
|
+
self.provenance_metadata._to_oci_metadata()
|
1267
|
+
)
|
1268
|
+
|
1269
|
+
# Upload artifacts
|
1270
|
+
logger.info("Uploading model artifacts.")
|
1271
|
+
self.upload_artifact(
|
1272
|
+
bucket_uri=kwargs.pop("bucket_uri", None),
|
1273
|
+
overwrite_existing_artifact=kwargs.pop("overwrite_existing_artifact", True),
|
1274
|
+
remove_existing_artifact=kwargs.pop("remove_existing_artifact", True),
|
1275
|
+
region=kwargs.pop("region", None),
|
1276
|
+
auth=kwargs.pop("auth", None),
|
1277
|
+
timeout=kwargs.pop("timeout", None),
|
1278
|
+
parallel_process_count=kwargs.pop("parallel_process_count", None),
|
1279
|
+
model_by_reference=model_by_reference,
|
1280
|
+
)
|
1281
|
+
|
1282
|
+
# Sync up model
|
1283
|
+
self.sync()
|
1284
|
+
logger.info(f"Model {self.id} has been successfully saved.")
|
1285
|
+
|
1286
|
+
return self
|
1287
|
+
|
1288
|
+
def upload_artifact(
|
1289
|
+
self,
|
1290
|
+
bucket_uri: Optional[str] = None,
|
1291
|
+
auth: Optional[Dict] = None,
|
1292
|
+
region: Optional[str] = None,
|
1293
|
+
overwrite_existing_artifact: Optional[bool] = True,
|
1294
|
+
remove_existing_artifact: Optional[bool] = True,
|
1295
|
+
timeout: Optional[int] = None,
|
1296
|
+
parallel_process_count: int = utils.DEFAULT_PARALLEL_PROCESS_COUNT,
|
1297
|
+
model_by_reference: Optional[bool] = False,
|
1298
|
+
) -> None:
|
1299
|
+
"""Uploads model artifacts to the model catalog.
|
1300
|
+
|
1301
|
+
Parameters
|
1302
|
+
----------
|
1303
|
+
bucket_uri: (str, optional). Defaults to None.
|
1304
|
+
The OCI Object Storage URI where model artifacts will be copied to.
|
1305
|
+
The `bucket_uri` is only necessary for uploading large artifacts which
|
1306
|
+
size is greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
|
1307
|
+
|
1308
|
+
.. versionadded:: 2.8.10
|
1309
|
+
|
1310
|
+
If `artifact` is provided as an object storage path to a zip archive, `bucket_uri` will be ignored.
|
1311
|
+
|
1312
|
+
auth: (Dict, optional). Defaults to `None`.
|
1313
|
+
The default authentication is set using `ads.set_auth` API.
|
1314
|
+
If you need to override the default, use the `ads.common.auth.api_keys` or
|
1315
|
+
`ads.common.auth.resource_principal` to create appropriate authentication signer
|
1316
|
+
and kwargs required to instantiate IdentityClient object.
|
1317
|
+
region: (str, optional). Defaults to `None`.
|
1318
|
+
The destination Object Storage bucket region.
|
1319
|
+
By default the value will be extracted from the `OCI_REGION_METADATA` environment variables.
|
1320
|
+
overwrite_existing_artifact: (bool, optional). Defaults to `True`.
|
1321
|
+
Overwrite target bucket artifact if exists.
|
1322
|
+
remove_existing_artifact: (bool, optional). Defaults to `True`.
|
1323
|
+
Wether artifacts uploaded to object storage bucket need to be removed or not.
|
1324
|
+
timeout: (int, optional). Defaults to 10 seconds.
|
1325
|
+
The connection timeout in seconds for the client.
|
1326
|
+
parallel_process_count: (int, optional)
|
1327
|
+
The number of worker processes to use in parallel for uploading individual parts of a multipart upload.
|
1328
|
+
model_by_reference: (bool, optional)
|
1329
|
+
Whether model artifact is made available to Model Store by reference.
|
1330
|
+
"""
|
1331
|
+
# Upload artifact to the model catalog
|
1332
|
+
if model_by_reference and self.model_file_description:
|
1333
|
+
logger.info(
|
1334
|
+
"Model artifact will be uploaded using model_file_description contents, "
|
1335
|
+
"artifact location will not be used."
|
1336
|
+
)
|
1337
|
+
elif not self.artifact:
|
1338
|
+
logger.warn(
|
1339
|
+
"Model artifact location not provided. "
|
1340
|
+
"Provide the artifact location to upload artifacts to the model catalog."
|
1341
|
+
)
|
1342
|
+
return
|
1343
|
+
|
1344
|
+
if timeout:
|
1345
|
+
self.dsc_model._client = None
|
1346
|
+
self.dsc_model.__class__.kwargs = {
|
1347
|
+
**(self.dsc_model.__class__.kwargs or {}),
|
1348
|
+
"timeout": timeout,
|
1349
|
+
}
|
1350
|
+
|
1351
|
+
if model_by_reference:
|
1352
|
+
self._validate_prepare_file_description_artifact()
|
1353
|
+
else:
|
1354
|
+
if isinstance(self.artifact, list):
|
1355
|
+
raise InvalidArtifactType(
|
1356
|
+
"Multiple artifacts are only allowed for models created by reference."
|
1357
|
+
)
|
1358
|
+
|
1359
|
+
if ObjectStorageDetails.is_oci_path(self.artifact):
|
1360
|
+
if bucket_uri and bucket_uri != self.artifact:
|
1361
|
+
logger.warn(
|
1362
|
+
"The `bucket_uri` will be ignored and the value of `self.artifact` will be used instead."
|
1363
|
+
)
|
1364
|
+
bucket_uri = self.artifact
|
1365
|
+
|
1366
|
+
if not model_by_reference and (
|
1367
|
+
bucket_uri or utils.folder_size(self.artifact) > _MAX_ARTIFACT_SIZE_IN_BYTES
|
1368
|
+
):
|
1369
|
+
if not bucket_uri:
|
1370
|
+
raise ModelArtifactSizeError(
|
1371
|
+
max_artifact_size=utils.human_size(_MAX_ARTIFACT_SIZE_IN_BYTES)
|
1372
|
+
)
|
1373
|
+
|
1374
|
+
artifact_uploader = LargeArtifactUploader(
|
1375
|
+
dsc_model=self.dsc_model,
|
1376
|
+
artifact_path=self.artifact,
|
1377
|
+
auth=auth,
|
1378
|
+
region=region,
|
1379
|
+
bucket_uri=bucket_uri,
|
1380
|
+
overwrite_existing_artifact=overwrite_existing_artifact,
|
1381
|
+
remove_existing_artifact=remove_existing_artifact,
|
1382
|
+
parallel_process_count=parallel_process_count,
|
1383
|
+
)
|
1384
|
+
else:
|
1385
|
+
artifact_uploader = SmallArtifactUploader(
|
1386
|
+
dsc_model=self.dsc_model,
|
1387
|
+
artifact_path=self.artifact,
|
1388
|
+
)
|
1389
|
+
artifact_uploader.upload()
|
1390
|
+
|
1391
|
+
self._remove_file_description_artifact()
|
1392
|
+
|
1393
|
+
def _remove_file_description_artifact(self):
|
1394
|
+
"""Removes temporary model file description artifact for model by reference."""
|
1395
|
+
# delete if local copy directory was created
|
1396
|
+
if self.local_copy_dir:
|
1397
|
+
shutil.rmtree(self.local_copy_dir, ignore_errors=True)
|
1398
|
+
|
1399
|
+
def restore_model(
|
1400
|
+
self,
|
1401
|
+
restore_model_for_hours_specified: Optional[int] = None,
|
1402
|
+
) -> None:
|
1403
|
+
"""
|
1404
|
+
Restore archived model artifact.
|
1405
|
+
|
1406
|
+
Parameters
|
1407
|
+
----------
|
1408
|
+
|
1409
|
+
restore_model_for_hours_specified : Optional[int]
|
1410
|
+
Duration in hours for which the archived model is available for access.
|
1411
|
+
|
1412
|
+
Returns
|
1413
|
+
-------
|
1414
|
+
None
|
1415
|
+
|
1416
|
+
Raises
|
1417
|
+
------
|
1418
|
+
ValueError
|
1419
|
+
If the model ID is invalid or if any parameters are incorrect.
|
1420
|
+
"""
|
1421
|
+
# Validate model_id
|
1422
|
+
if not self.id:
|
1423
|
+
logger.warn(
|
1424
|
+
"Model needs to be saved to the model catalog before it can be restored."
|
1425
|
+
)
|
1426
|
+
return
|
1427
|
+
|
1428
|
+
# Optional: Validate restore_model_for_hours_specified
|
1429
|
+
if restore_model_for_hours_specified is not None and (
|
1430
|
+
not isinstance(restore_model_for_hours_specified, int)
|
1431
|
+
or restore_model_for_hours_specified <= 0
|
1432
|
+
):
|
1433
|
+
raise ValueError(
|
1434
|
+
"restore_model_for_hours_specified must be a positive integer."
|
1435
|
+
)
|
1436
|
+
|
1437
|
+
self.dsc_model.restore_archived_model_artifact(
|
1438
|
+
restore_model_for_hours_specified=restore_model_for_hours_specified,
|
1439
|
+
)
|
1440
|
+
|
1441
|
+
def download_artifact(
|
1442
|
+
self,
|
1443
|
+
target_dir: str,
|
1444
|
+
auth: Optional[Dict] = None,
|
1445
|
+
force_overwrite: Optional[bool] = False,
|
1446
|
+
bucket_uri: Optional[str] = None,
|
1447
|
+
region: Optional[str] = None,
|
1448
|
+
overwrite_existing_artifact: Optional[bool] = True,
|
1449
|
+
remove_existing_artifact: Optional[bool] = True,
|
1450
|
+
timeout: Optional[int] = None,
|
1451
|
+
):
|
1452
|
+
"""Downloads model artifacts from the model catalog.
|
1453
|
+
|
1454
|
+
Parameters
|
1455
|
+
----------
|
1456
|
+
target_dir: str
|
1457
|
+
The target location of model artifacts.
|
1458
|
+
auth: (Dict, optional). Defaults to `None`.
|
1459
|
+
The default authentication is set using `ads.set_auth` API.
|
1460
|
+
If you need to override the default, use the `ads.common.auth.api_keys` or
|
1461
|
+
`ads.common.auth.resource_principal` to create appropriate authentication signer
|
1462
|
+
and kwargs required to instantiate IdentityClient object.
|
1463
|
+
force_overwrite: (bool, optional). Defaults to `False`.
|
1464
|
+
Overwrite target directory if exists.
|
1465
|
+
bucket_uri: (str, optional). Defaults to None.
|
1466
|
+
The OCI Object Storage URI where model artifacts will be copied to.
|
1467
|
+
The `bucket_uri` is only necessary for uploading large artifacts which
|
1468
|
+
size is greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
|
1469
|
+
region: (str, optional). Defaults to `None`.
|
1470
|
+
The destination Object Storage bucket region.
|
1471
|
+
By default the value will be extracted from the `OCI_REGION_METADATA` environment variables.
|
1472
|
+
overwrite_existing_artifact: (bool, optional). Defaults to `True`.
|
1473
|
+
Overwrite target bucket artifact if exists.
|
1474
|
+
remove_existing_artifact: (bool, optional). Defaults to `True`.
|
1475
|
+
Wether artifacts uploaded to object storage bucket need to be removed or not.
|
1476
|
+
timeout: (int, optional). Defaults to 10 seconds.
|
1477
|
+
The connection timeout in seconds for the client.
|
1478
|
+
|
1479
|
+
Raises
|
1480
|
+
------
|
1481
|
+
ModelArtifactSizeError
|
1482
|
+
If model artifacts size greater than 2GB and temporary OS bucket uri not provided.
|
1483
|
+
"""
|
1484
|
+
# Upload artifact to the model catalog
|
1485
|
+
if not self.artifact:
|
1486
|
+
logger.warn(
|
1487
|
+
"Model doesn't contain an artifact. "
|
1488
|
+
"The artifact needs to be uploaded to the model catalog at first. "
|
1489
|
+
)
|
1490
|
+
return
|
1491
|
+
|
1492
|
+
if timeout:
|
1493
|
+
self.dsc_model._client = None
|
1494
|
+
self.dsc_model.__class__.kwargs = {
|
1495
|
+
**(self.dsc_model.__class__.kwargs or {}),
|
1496
|
+
"timeout": timeout,
|
1497
|
+
}
|
1498
|
+
try:
|
1499
|
+
model_by_reference = self.custom_metadata_list.get(
|
1500
|
+
self.CONST_MODEL_FILE_DESCRIPTION
|
1501
|
+
).value
|
1502
|
+
logging.info(
|
1503
|
+
f"modelDescription tag found in custom metadata list with value {model_by_reference}"
|
1504
|
+
)
|
1505
|
+
except ValueError:
|
1506
|
+
model_by_reference = False
|
1507
|
+
|
1508
|
+
if model_by_reference:
|
1509
|
+
_, artifact_size = self._download_file_description_artifact()
|
1510
|
+
logging.warning(
|
1511
|
+
f"Model {self.dsc_model.id} was created by reference, artifacts will be downloaded from the bucket {bucket_uri}"
|
1512
|
+
)
|
1513
|
+
# artifacts will be downloaded from model_file_description
|
1514
|
+
bucket_uri = None
|
1515
|
+
else:
|
1516
|
+
artifact_info = self.dsc_model.get_artifact_info()
|
1517
|
+
artifact_size = int(artifact_info.get("content-length"))
|
1518
|
+
|
1519
|
+
if not bucket_uri and artifact_size > _MAX_ARTIFACT_SIZE_IN_BYTES:
|
1520
|
+
raise ModelArtifactSizeError(
|
1521
|
+
utils.human_size(_MAX_ARTIFACT_SIZE_IN_BYTES)
|
1522
|
+
)
|
1523
|
+
|
1524
|
+
if (
|
1525
|
+
artifact_size > _MAX_ARTIFACT_SIZE_IN_BYTES
|
1526
|
+
or bucket_uri
|
1527
|
+
or model_by_reference
|
1528
|
+
):
|
1529
|
+
artifact_downloader = LargeArtifactDownloader(
|
1530
|
+
dsc_model=self.dsc_model,
|
1531
|
+
target_dir=target_dir,
|
1532
|
+
auth=auth,
|
1533
|
+
force_overwrite=force_overwrite,
|
1534
|
+
region=region,
|
1535
|
+
bucket_uri=bucket_uri,
|
1536
|
+
overwrite_existing_artifact=overwrite_existing_artifact,
|
1537
|
+
remove_existing_artifact=remove_existing_artifact,
|
1538
|
+
model_file_description=self.model_file_description,
|
1539
|
+
)
|
1540
|
+
else:
|
1541
|
+
artifact_downloader = SmallArtifactDownloader(
|
1542
|
+
dsc_model=self.dsc_model,
|
1543
|
+
target_dir=target_dir,
|
1544
|
+
force_overwrite=force_overwrite,
|
1545
|
+
)
|
1546
|
+
artifact_downloader.download()
|
1547
|
+
|
1548
|
+
def update(self, **kwargs) -> "DataScienceModel":
|
1549
|
+
"""Updates datascience model in model catalog.
|
1550
|
+
|
1551
|
+
Parameters
|
1552
|
+
----------
|
1553
|
+
kwargs
|
1554
|
+
Additional kwargs arguments.
|
1555
|
+
Can be any attribute that `oci.data_science.models.Model` accepts.
|
1556
|
+
|
1557
|
+
Returns
|
1558
|
+
-------
|
1559
|
+
DataScienceModel
|
1560
|
+
The DataScienceModel instance (self).
|
1561
|
+
"""
|
1562
|
+
if not self.id:
|
1563
|
+
logger.warn(
|
1564
|
+
"Model needs to be saved to the model catalog before it can be updated."
|
1565
|
+
)
|
1566
|
+
return
|
1567
|
+
|
1568
|
+
logger.debug(f"Updating a model with payload {self._spec}")
|
1569
|
+
logger.info(f"Updating model {self.id} in the Model Catalog.")
|
1570
|
+
self.dsc_model = self._to_oci_dsc_model(**kwargs).update()
|
1571
|
+
|
1572
|
+
logger.debug(f"Updating a model provenance metadata {self.provenance_metadata}")
|
1573
|
+
if self.provenance_metadata:
|
1574
|
+
try:
|
1575
|
+
self.dsc_model.get_model_provenance()
|
1576
|
+
self.dsc_model.update_model_provenance(
|
1577
|
+
self.provenance_metadata._to_oci_metadata()
|
1578
|
+
)
|
1579
|
+
except ModelProvenanceNotFoundError:
|
1580
|
+
self.dsc_model.create_model_provenance(
|
1581
|
+
self.provenance_metadata._to_oci_metadata()
|
1582
|
+
)
|
1583
|
+
|
1584
|
+
return self.sync()
|
1585
|
+
|
1586
|
+
def delete(
|
1587
|
+
self,
|
1588
|
+
delete_associated_model_deployment: Optional[bool] = False,
|
1589
|
+
) -> "DataScienceModel":
|
1590
|
+
"""Removes model from the model catalog.
|
1591
|
+
|
1592
|
+
Parameters
|
1593
|
+
----------
|
1594
|
+
delete_associated_model_deployment: (bool, optional). Defaults to `False`.
|
1595
|
+
Whether associated model deployments need to be deleted or not.
|
1596
|
+
|
1597
|
+
Returns
|
1598
|
+
-------
|
1599
|
+
DataScienceModel
|
1600
|
+
The DataScienceModel instance (self).
|
1601
|
+
"""
|
1602
|
+
self.dsc_model.delete(delete_associated_model_deployment)
|
1603
|
+
return self.sync()
|
1604
|
+
|
1605
|
+
@classmethod
|
1606
|
+
def list(
|
1607
|
+
cls,
|
1608
|
+
compartment_id: str = None,
|
1609
|
+
project_id: str = None,
|
1610
|
+
category: str = USER,
|
1611
|
+
**kwargs,
|
1612
|
+
) -> List["DataScienceModel"]:
|
1613
|
+
"""Lists datascience models in a given compartment.
|
1614
|
+
|
1615
|
+
Parameters
|
1616
|
+
----------
|
1617
|
+
compartment_id: (str, optional). Defaults to `None`.
|
1618
|
+
The compartment OCID.
|
1619
|
+
project_id: (str, optional). Defaults to `None`.
|
1620
|
+
The project OCID.
|
1621
|
+
category: (str, optional). Defaults to `USER`.
|
1622
|
+
The category of Model. Allowed values are: "USER", "SERVICE"
|
1623
|
+
kwargs
|
1624
|
+
Additional keyword arguments for filtering models.
|
1625
|
+
|
1626
|
+
Returns
|
1627
|
+
-------
|
1628
|
+
List[DataScienceModel]
|
1629
|
+
The list of the datascience models.
|
1630
|
+
"""
|
1631
|
+
return [
|
1632
|
+
cls()._update_from_oci_dsc_model(model)
|
1633
|
+
for model in OCIDataScienceModel.list_resource(
|
1634
|
+
compartment_id, project_id=project_id, category=category, **kwargs
|
1635
|
+
)
|
1636
|
+
]
|
1637
|
+
|
1638
|
+
@classmethod
|
1639
|
+
def list_df(
|
1640
|
+
cls,
|
1641
|
+
compartment_id: str = None,
|
1642
|
+
project_id: str = None,
|
1643
|
+
category: str = USER,
|
1644
|
+
**kwargs,
|
1645
|
+
) -> "pandas.DataFrame":
|
1646
|
+
"""Lists datascience models in a given compartment.
|
1647
|
+
|
1648
|
+
Parameters
|
1649
|
+
----------
|
1650
|
+
compartment_id: (str, optional). Defaults to `None`.
|
1651
|
+
The compartment OCID.
|
1652
|
+
project_id: (str, optional). Defaults to `None`.
|
1653
|
+
The project OCID.
|
1654
|
+
category: (str, optional). Defaults to `None`.
|
1655
|
+
The category of Model.
|
1656
|
+
kwargs
|
1657
|
+
Additional keyword arguments for filtering models.
|
1658
|
+
|
1659
|
+
Returns
|
1660
|
+
-------
|
1661
|
+
pandas.DataFrame
|
1662
|
+
The list of the datascience models in a pandas dataframe format.
|
1663
|
+
"""
|
1664
|
+
records = []
|
1665
|
+
for model in OCIDataScienceModel.list_resource(
|
1666
|
+
compartment_id, project_id=project_id, category=category, **kwargs
|
1667
|
+
):
|
1668
|
+
records.append(
|
1669
|
+
{
|
1670
|
+
"id": f"...{model.id[-6:]}",
|
1671
|
+
"display_name": model.display_name,
|
1672
|
+
"description": model.description,
|
1673
|
+
"time_created": model.time_created.strftime(utils.date_format),
|
1674
|
+
"lifecycle_state": model.lifecycle_state,
|
1675
|
+
"created_by": f"...{model.created_by[-6:]}",
|
1676
|
+
"compartment_id": f"...{model.compartment_id[-6:]}",
|
1677
|
+
"project_id": f"...{model.project_id[-6:]}",
|
1678
|
+
}
|
1679
|
+
)
|
1680
|
+
return pandas.DataFrame.from_records(records)
|
1681
|
+
|
1682
|
+
@classmethod
|
1683
|
+
def from_id(cls, id: str) -> "DataScienceModel":
|
1684
|
+
"""Gets an existing model by OCID.
|
1685
|
+
|
1686
|
+
Parameters
|
1687
|
+
----------
|
1688
|
+
id: str
|
1689
|
+
The model OCID.
|
1690
|
+
|
1691
|
+
Returns
|
1692
|
+
-------
|
1693
|
+
DataScienceModel
|
1694
|
+
An instance of DataScienceModel.
|
1695
|
+
"""
|
1696
|
+
return cls()._update_from_oci_dsc_model(OCIDataScienceModel.from_id(id))
|
1697
|
+
|
1698
|
+
def sync(self):
|
1699
|
+
"""Sync up a datascience model with OCI datascience model."""
|
1700
|
+
return self._update_from_oci_dsc_model(OCIDataScienceModel.from_id(self.id))
|
1701
|
+
|
1702
|
+
def _init_complex_attributes(self):
|
1703
|
+
"""Initiates complex attributes."""
|
1704
|
+
self.with_custom_metadata_list(self.custom_metadata_list)
|
1705
|
+
self.with_defined_metadata_list(self.defined_metadata_list)
|
1706
|
+
self.with_provenance_metadata(self.provenance_metadata)
|
1707
|
+
self.with_input_schema(self.input_schema)
|
1708
|
+
self.with_output_schema(self.output_schema)
|
1709
|
+
|
1710
|
+
def _to_oci_dsc_model(self, **kwargs):
|
1711
|
+
"""Creates an `OCIDataScienceModel` instance from the `DataScienceModel`.
|
1712
|
+
|
1713
|
+
kwargs
|
1714
|
+
Additional kwargs arguments.
|
1715
|
+
Can be any attribute that `oci.data_science.models.Model` accepts.
|
1716
|
+
|
1717
|
+
Returns
|
1718
|
+
-------
|
1719
|
+
OCIDataScienceModel
|
1720
|
+
The instance of the OCIDataScienceModel.
|
1721
|
+
"""
|
1722
|
+
COMPLEX_ATTRIBUTES_CONVERTER = {
|
1723
|
+
self.CONST_INPUT_SCHEMA: "to_json",
|
1724
|
+
self.CONST_OUTPUT_SCHEMA: "to_json",
|
1725
|
+
self.CONST_CUSTOM_METADATA: "_to_oci_metadata",
|
1726
|
+
self.CONST_DEFINED_METADATA: "_to_oci_metadata",
|
1727
|
+
self.CONST_PROVENANCE_METADATA: "_to_oci_metadata",
|
1728
|
+
}
|
1729
|
+
dsc_spec = {}
|
1730
|
+
for infra_attr, dsc_attr in self.attribute_map.items():
|
1731
|
+
value = self.get_spec(infra_attr)
|
1732
|
+
if infra_attr in COMPLEX_ATTRIBUTES_CONVERTER and value:
|
1733
|
+
if isinstance(value, dict):
|
1734
|
+
dsc_spec[dsc_attr] = json.dumps(value)
|
1735
|
+
else:
|
1736
|
+
dsc_spec[dsc_attr] = getattr(
|
1737
|
+
self.get_spec(infra_attr),
|
1738
|
+
COMPLEX_ATTRIBUTES_CONVERTER[infra_attr],
|
1739
|
+
)()
|
1740
|
+
else:
|
1741
|
+
dsc_spec[dsc_attr] = value
|
1742
|
+
|
1743
|
+
dsc_spec.update(**kwargs)
|
1744
|
+
return OCIDataScienceModel(**dsc_spec)
|
1745
|
+
|
1746
|
+
def _update_from_oci_dsc_model(
|
1747
|
+
self, dsc_model: OCIDataScienceModel
|
1748
|
+
) -> "DataScienceModel":
|
1749
|
+
"""Update the properties from an OCIDataScienceModel object.
|
1750
|
+
|
1751
|
+
Parameters
|
1752
|
+
----------
|
1753
|
+
dsc_model: OCIDataScienceModel
|
1754
|
+
An instance of OCIDataScienceModel.
|
1755
|
+
|
1756
|
+
Returns
|
1757
|
+
-------
|
1758
|
+
DataScienceModel
|
1759
|
+
The DataScienceModel instance (self).
|
1760
|
+
"""
|
1761
|
+
COMPLEX_ATTRIBUTES_CONVERTER = {
|
1762
|
+
self.CONST_INPUT_SCHEMA: [Schema.from_json, json.loads],
|
1763
|
+
self.CONST_OUTPUT_SCHEMA: [Schema.from_json, json.loads],
|
1764
|
+
self.CONST_CUSTOM_METADATA: ModelCustomMetadata._from_oci_metadata,
|
1765
|
+
self.CONST_DEFINED_METADATA: ModelTaxonomyMetadata._from_oci_metadata,
|
1766
|
+
self.CONST_BACKUP_SETTING: ModelBackupSetting.to_dict,
|
1767
|
+
self.CONST_RETENTION_SETTING: ModelRetentionSetting.to_dict,
|
1768
|
+
self.CONST_BACKUP_OPERATION_DETAILS: ModelBackupOperationDetails.to_dict,
|
1769
|
+
self.CONST_RETENTION_OPERATION_DETAILS: ModelRetentionOperationDetails.to_dict,
|
1770
|
+
}
|
1771
|
+
|
1772
|
+
# Update the main properties
|
1773
|
+
self.dsc_model = dsc_model
|
1774
|
+
for infra_attr, dsc_attr in self.attribute_map.items():
|
1775
|
+
value = utils.get_value(dsc_model, dsc_attr)
|
1776
|
+
if value:
|
1777
|
+
if infra_attr in COMPLEX_ATTRIBUTES_CONVERTER:
|
1778
|
+
converter = COMPLEX_ATTRIBUTES_CONVERTER[infra_attr]
|
1779
|
+
if isinstance(converter, List):
|
1780
|
+
for converter_item in converter:
|
1781
|
+
try:
|
1782
|
+
value = converter_item(value)
|
1783
|
+
except Exception as err:
|
1784
|
+
logger.warn(err)
|
1785
|
+
pass
|
1786
|
+
else:
|
1787
|
+
value = converter(value)
|
1788
|
+
self.set_spec(infra_attr, value)
|
1789
|
+
|
1790
|
+
# Update provenance metadata
|
1791
|
+
try:
|
1792
|
+
self.set_spec(
|
1793
|
+
self.CONST_PROVENANCE_METADATA,
|
1794
|
+
ModelProvenanceMetadata._from_oci_metadata(
|
1795
|
+
self.dsc_model.get_model_provenance()
|
1796
|
+
),
|
1797
|
+
)
|
1798
|
+
except ModelProvenanceNotFoundError:
|
1799
|
+
pass
|
1800
|
+
|
1801
|
+
# Update artifact info
|
1802
|
+
try:
|
1803
|
+
artifact_info = self.dsc_model.get_artifact_info()
|
1804
|
+
_, file_name_info = utils.parse_content_disposition(
|
1805
|
+
artifact_info["Content-Disposition"]
|
1806
|
+
)
|
1807
|
+
|
1808
|
+
if self.dsc_model.is_model_created_by_reference():
|
1809
|
+
_, file_extension = os.path.splitext(file_name_info["filename"])
|
1810
|
+
if file_extension.lower() == ".json":
|
1811
|
+
bucket_uri, _ = self._download_file_description_artifact()
|
1812
|
+
self.set_spec(self.CONST_ARTIFACT, bucket_uri)
|
1813
|
+
else:
|
1814
|
+
self.set_spec(self.CONST_ARTIFACT, file_name_info["filename"])
|
1815
|
+
except:
|
1816
|
+
pass
|
1817
|
+
return self
|
1818
|
+
|
1819
|
+
def to_dict(self) -> Dict:
|
1820
|
+
"""Serializes model to a dictionary.
|
1821
|
+
|
1822
|
+
Returns
|
1823
|
+
-------
|
1824
|
+
dict
|
1825
|
+
The model serialized as a dictionary.
|
1826
|
+
"""
|
1827
|
+
spec = deepcopy(self._spec)
|
1828
|
+
for key, value in spec.items():
|
1829
|
+
if hasattr(value, "to_dict"):
|
1830
|
+
value = value.to_dict()
|
1831
|
+
spec[key] = value
|
1832
|
+
|
1833
|
+
return {
|
1834
|
+
"kind": self.kind,
|
1835
|
+
"type": self.type,
|
1836
|
+
"spec": utils.batch_convert_case(spec, "camel"),
|
1837
|
+
}
|
1838
|
+
|
1839
|
+
@classmethod
|
1840
|
+
def from_dict(cls, config: Dict) -> "DataScienceModel":
|
1841
|
+
"""Loads model instance from a dictionary of configurations.
|
1842
|
+
|
1843
|
+
Parameters
|
1844
|
+
----------
|
1845
|
+
config: Dict
|
1846
|
+
A dictionary of configurations.
|
1847
|
+
|
1848
|
+
Returns
|
1849
|
+
-------
|
1850
|
+
DataScienceModel
|
1851
|
+
The model instance.
|
1852
|
+
"""
|
1853
|
+
return cls(spec=utils.batch_convert_case(deepcopy(config["spec"]), "snake"))
|
1854
|
+
|
1855
|
+
def _random_display_name(self):
|
1856
|
+
"""Generates a random display name."""
|
1857
|
+
return f"{self._PREFIX}-{utils.get_random_name_for_resource()}"
|
1858
|
+
|
1859
|
+
def _load_default_properties(self) -> Dict:
|
1860
|
+
"""Load default properties from environment variables, notebook session, etc.
|
1861
|
+
|
1862
|
+
Returns
|
1863
|
+
-------
|
1864
|
+
Dict
|
1865
|
+
A dictionary of default properties.
|
1866
|
+
"""
|
1867
|
+
defaults = super()._load_default_properties()
|
1868
|
+
compartment_ocid = COMPARTMENT_OCID
|
1869
|
+
if compartment_ocid:
|
1870
|
+
defaults[self.CONST_COMPARTMENT_ID] = compartment_ocid
|
1871
|
+
if PROJECT_OCID:
|
1872
|
+
defaults[self.CONST_PROJECT_ID] = PROJECT_OCID
|
1873
|
+
defaults[self.CONST_DISPLAY_NAME] = self._random_display_name()
|
1874
|
+
|
1875
|
+
return defaults
|
1876
|
+
|
1877
|
+
def __getattr__(self, item):
|
1878
|
+
if f"with_{item}" in self.__dir__():
|
1879
|
+
return self.get_spec(item)
|
1880
|
+
raise AttributeError(f"Attribute {item} not found.")
|
1881
|
+
|
1882
|
+
def _validate_prepare_file_description_artifact(self):
|
1883
|
+
"""This helper method validates the path to check if the buckets are versioned and if the OSS location and
|
1884
|
+
the files exist. Next, it creates a json dict with the path information and sets it as the artifact to be
|
1885
|
+
uploaded."""
|
1886
|
+
|
1887
|
+
if not self.model_file_description:
|
1888
|
+
bucket_uri = self.artifact
|
1889
|
+
if isinstance(bucket_uri, str):
|
1890
|
+
bucket_uri = [bucket_uri]
|
1891
|
+
|
1892
|
+
for uri in bucket_uri:
|
1893
|
+
os_path = ObjectStorageDetails.from_path(uri)
|
1894
|
+
# for aqua use case, user may not have access to the service bucket.
|
1895
|
+
if os_path.bucket == SERVICE_MODELS_BUCKET:
|
1896
|
+
continue
|
1897
|
+
if not os_path.is_bucket_versioned():
|
1898
|
+
message = f"Model artifact bucket {uri} is not versioned. Enable versioning on the bucket to proceed with model creation by reference."
|
1899
|
+
logger.error(message)
|
1900
|
+
raise BucketNotVersionedError(message)
|
1901
|
+
|
1902
|
+
json_data = self._prepare_file_description_artifact(bucket_uri)
|
1903
|
+
self.with_model_file_description(json_dict=json_data)
|
1904
|
+
|
1905
|
+
self.local_copy_dir = tempfile.mkdtemp()
|
1906
|
+
# create temp directory for model description file
|
1907
|
+
json_file_path = os.path.join(
|
1908
|
+
self.local_copy_dir, MODEL_BY_REFERENCE_JSON_FILE_NAME
|
1909
|
+
)
|
1910
|
+
with open(json_file_path, "w") as outfile:
|
1911
|
+
json.dump(self.model_file_description, outfile, indent=2)
|
1912
|
+
|
1913
|
+
self.with_artifact(json_file_path)
|
1914
|
+
|
1915
|
+
@staticmethod
|
1916
|
+
def _prepare_file_description_artifact(bucket_uri: list) -> dict:
|
1917
|
+
"""Prepares yaml file config if model is passed by reference and uploaded to catalog.
|
1918
|
+
|
1919
|
+
Returns
|
1920
|
+
-------
|
1921
|
+
dict
|
1922
|
+
json dict with the model by reference artifact details
|
1923
|
+
"""
|
1924
|
+
|
1925
|
+
# create json content
|
1926
|
+
content = dict()
|
1927
|
+
content["version"] = MODEL_BY_REFERENCE_VERSION
|
1928
|
+
content["type"] = "modelOSSReferenceDescription"
|
1929
|
+
content["models"] = []
|
1930
|
+
|
1931
|
+
for uri in bucket_uri:
|
1932
|
+
if not ObjectStorageDetails.is_oci_path(uri) or uri.endswith(".zip"):
|
1933
|
+
msg = "Artifact path cannot be a zip file or local directory for model creation by reference."
|
1934
|
+
logging.error(msg)
|
1935
|
+
raise InvalidArtifactType(msg)
|
1936
|
+
|
1937
|
+
# read list from objects from artifact location
|
1938
|
+
oss_details = ObjectStorageDetails.from_path(uri)
|
1939
|
+
|
1940
|
+
# first retrieve the etag and version id
|
1941
|
+
object_versions = oss_details.list_object_versions(fields="etag")
|
1942
|
+
version_dict = {
|
1943
|
+
obj.etag: obj.version_id
|
1944
|
+
for obj in object_versions
|
1945
|
+
if obj.etag is not None
|
1946
|
+
}
|
1947
|
+
|
1948
|
+
# add version id based on etag for each object
|
1949
|
+
objects = oss_details.list_objects(fields="name,etag,size").objects
|
1950
|
+
|
1951
|
+
if len(objects) == 0:
|
1952
|
+
raise ModelFileDescriptionError(
|
1953
|
+
f"The path {oss_details.path} does not exist or no objects were found in the path. "
|
1954
|
+
)
|
1955
|
+
|
1956
|
+
object_list = []
|
1957
|
+
for obj in objects:
|
1958
|
+
object_list.append(
|
1959
|
+
{
|
1960
|
+
"name": obj.name,
|
1961
|
+
"version": version_dict[obj.etag],
|
1962
|
+
"sizeInBytes": obj.size,
|
1963
|
+
}
|
1964
|
+
)
|
1965
|
+
content["models"].extend(
|
1966
|
+
[
|
1967
|
+
{
|
1968
|
+
"namespace": oss_details.namespace,
|
1969
|
+
"bucketName": oss_details.bucket,
|
1970
|
+
"prefix": oss_details.filepath,
|
1971
|
+
"objects": object_list,
|
1972
|
+
}
|
1973
|
+
]
|
1974
|
+
)
|
1975
|
+
|
1976
|
+
return content
|
1977
|
+
|
1978
|
+
def _download_file_description_artifact(self) -> Tuple[Union[str, List[str]], int]:
|
1979
|
+
"""Loads the json file from model artifact, updates the
|
1980
|
+
model file description property, and returns the bucket uri and artifact size details.
|
1981
|
+
|
1982
|
+
Returns
|
1983
|
+
-------
|
1984
|
+
bucket_uri: Union[str, List[str]]
|
1985
|
+
Location(s) of bucket where model artifacts are present
|
1986
|
+
artifact_size: int
|
1987
|
+
estimated size of the model files in bytes
|
1988
|
+
|
1989
|
+
"""
|
1990
|
+
if not self.model_file_description:
|
1991
|
+
# get model file description from model artifact json
|
1992
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
1993
|
+
artifact_downloader = SmallArtifactDownloader(
|
1994
|
+
dsc_model=self.dsc_model,
|
1995
|
+
target_dir=temp_dir,
|
1996
|
+
)
|
1997
|
+
artifact_downloader.download()
|
1998
|
+
# create temp directory for model description file
|
1999
|
+
json_file_path = os.path.join(
|
2000
|
+
temp_dir, MODEL_BY_REFERENCE_JSON_FILE_NAME
|
2001
|
+
)
|
2002
|
+
self.with_model_file_description(json_uri=json_file_path)
|
2003
|
+
|
2004
|
+
model_file_desc_dict = self.model_file_description
|
2005
|
+
models = model_file_desc_dict["models"]
|
2006
|
+
|
2007
|
+
bucket_uri = list()
|
2008
|
+
artifact_size = 0
|
2009
|
+
for model in models:
|
2010
|
+
namespace = model["namespace"]
|
2011
|
+
bucket_name = model["bucketName"]
|
2012
|
+
prefix = model["prefix"]
|
2013
|
+
objects = model["objects"]
|
2014
|
+
uri = f"oci://{bucket_name}@{namespace}/{prefix}"
|
2015
|
+
artifact_size += sum([obj["sizeInBytes"] for obj in objects])
|
2016
|
+
bucket_uri.append(uri)
|
2017
|
+
|
2018
|
+
return bucket_uri[0] if len(bucket_uri) == 1 else bucket_uri, artifact_size
|
2019
|
+
|
2020
|
+
def add_artifact(
|
2021
|
+
self,
|
2022
|
+
uri: Optional[str] = None,
|
2023
|
+
namespace: Optional[str] = None,
|
2024
|
+
bucket: Optional[str] = None,
|
2025
|
+
prefix: Optional[str] = None,
|
2026
|
+
files: Optional[List[str]] = None,
|
2027
|
+
):
|
2028
|
+
"""
|
2029
|
+
Adds information about objects in a specified bucket to the model description JSON.
|
2030
|
+
|
2031
|
+
Parameters
|
2032
|
+
----------
|
2033
|
+
uri : str, optional
|
2034
|
+
The URI representing the location of the artifact in OCI object storage.
|
2035
|
+
namespace : str, optional
|
2036
|
+
The namespace of the bucket containing the objects. Required if `uri` is not provided.
|
2037
|
+
bucket : str, optional
|
2038
|
+
The name of the bucket containing the objects. Required if `uri` is not provided.
|
2039
|
+
prefix : str, optional
|
2040
|
+
The prefix of the objects to add. Defaults to None. Cannot be provided if `files` is provided.
|
2041
|
+
files : list of str, optional
|
2042
|
+
A list of file names to include in the model description. If provided, only objects with matching file names will be included. Cannot be provided if `prefix` is provided.
|
2043
|
+
|
2044
|
+
Returns
|
2045
|
+
-------
|
2046
|
+
None
|
2047
|
+
|
2048
|
+
Raises
|
2049
|
+
------
|
2050
|
+
ValueError
|
2051
|
+
- If both `uri` and (`namespace` and `bucket`) are provided.
|
2052
|
+
- If neither `uri` nor both `namespace` and `bucket` are provided.
|
2053
|
+
- If both `prefix` and `files` are provided.
|
2054
|
+
- If no files are found to add to the model description.
|
2055
|
+
|
2056
|
+
Note
|
2057
|
+
----
|
2058
|
+
- If `files` is not provided, it retrieves information about all objects in the bucket.
|
2059
|
+
- If `files` is provided, it only retrieves information about objects with matching file names.
|
2060
|
+
- If no objects are found to add to the model description, a ValueError is raised.
|
2061
|
+
"""
|
2062
|
+
|
2063
|
+
if uri and (namespace or bucket):
|
2064
|
+
raise ValueError(
|
2065
|
+
"Either 'uri' must be provided or both 'namespace' and 'bucket' must be provided."
|
2066
|
+
)
|
2067
|
+
if uri:
|
2068
|
+
object_storage_details = ObjectStorageDetails.from_path(uri)
|
2069
|
+
bucket = object_storage_details.bucket
|
2070
|
+
namespace = object_storage_details.namespace
|
2071
|
+
prefix = (
|
2072
|
+
None
|
2073
|
+
if object_storage_details.filepath == ""
|
2074
|
+
else object_storage_details.filepath
|
2075
|
+
)
|
2076
|
+
if (not namespace) or (not bucket):
|
2077
|
+
raise ValueError("Both 'namespace' and 'bucket' must be provided.")
|
2078
|
+
|
2079
|
+
# Check if both prefix and files are provided
|
2080
|
+
if prefix is not None and files is not None:
|
2081
|
+
raise ValueError(
|
2082
|
+
"Both 'prefix' and 'files' cannot be provided. Please provide only one."
|
2083
|
+
)
|
2084
|
+
|
2085
|
+
if self.model_file_description is None:
|
2086
|
+
self.empty_json = {
|
2087
|
+
"version": "1.0",
|
2088
|
+
"type": "modelOSSReferenceDescription",
|
2089
|
+
"models": [],
|
2090
|
+
}
|
2091
|
+
self.set_spec(self.CONST_MODEL_FILE_DESCRIPTION, self.empty_json)
|
2092
|
+
|
2093
|
+
# Get object storage client
|
2094
|
+
self.object_storage_client = oc.OCIClientFactory(
|
2095
|
+
**(self.dsc_model.auth)
|
2096
|
+
).object_storage
|
2097
|
+
|
2098
|
+
# Remove if the model already exists
|
2099
|
+
self.remove_artifact(namespace=namespace, bucket=bucket, prefix=prefix)
|
2100
|
+
|
2101
|
+
def check_if_file_exists(fileName):
|
2102
|
+
isExists = False
|
2103
|
+
try:
|
2104
|
+
headResponse = self.object_storage_client.head_object(
|
2105
|
+
namespace, bucket, object_name=fileName
|
2106
|
+
)
|
2107
|
+
if headResponse.status == 200:
|
2108
|
+
isExists = True
|
2109
|
+
except Exception as e:
|
2110
|
+
if hasattr(e, "status") and e.status == 404:
|
2111
|
+
logger.error(f"File not found in bucket: {fileName}")
|
2112
|
+
else:
|
2113
|
+
logger.error(f"An error occured: {e}")
|
2114
|
+
return isExists
|
2115
|
+
|
2116
|
+
# Function to un-paginate the api call with while loop
|
2117
|
+
def list_obj_versions_unpaginated():
|
2118
|
+
objectStorageList = []
|
2119
|
+
has_next_page, opc_next_page = True, None
|
2120
|
+
while has_next_page:
|
2121
|
+
response = self.object_storage_client.list_object_versions(
|
2122
|
+
namespace_name=namespace,
|
2123
|
+
bucket_name=bucket,
|
2124
|
+
prefix=prefix,
|
2125
|
+
fields="name,size",
|
2126
|
+
page=opc_next_page,
|
2127
|
+
)
|
2128
|
+
objectStorageList.extend(response.data.items)
|
2129
|
+
has_next_page = response.has_next_page
|
2130
|
+
opc_next_page = response.next_page
|
2131
|
+
return objectStorageList
|
2132
|
+
|
2133
|
+
# Fetch object details and put it into the objects variable
|
2134
|
+
objectStorageList = []
|
2135
|
+
if files is None:
|
2136
|
+
objectStorageList = list_obj_versions_unpaginated()
|
2137
|
+
else:
|
2138
|
+
for fileName in files:
|
2139
|
+
if check_if_file_exists(fileName=fileName):
|
2140
|
+
objectStorageList.append(
|
2141
|
+
self.object_storage_client.list_object_versions(
|
2142
|
+
namespace_name=namespace,
|
2143
|
+
bucket_name=bucket,
|
2144
|
+
prefix=fileName,
|
2145
|
+
fields="name,size",
|
2146
|
+
).data.items[0]
|
2147
|
+
)
|
2148
|
+
|
2149
|
+
objects = [
|
2150
|
+
{"name": obj.name, "version": obj.version_id, "sizeInBytes": obj.size}
|
2151
|
+
for obj in objectStorageList
|
2152
|
+
if obj.size > 0
|
2153
|
+
]
|
2154
|
+
|
2155
|
+
if len(objects) == 0:
|
2156
|
+
error_message = (
|
2157
|
+
f"No files to add in the bucket: {bucket} with namespace: {namespace} "
|
2158
|
+
f"and prefix: {prefix}. File names: {files}"
|
2159
|
+
)
|
2160
|
+
logger.error(error_message)
|
2161
|
+
raise ValueError(error_message)
|
2162
|
+
|
2163
|
+
tmp_model_file_description = self.model_file_description
|
2164
|
+
tmp_model_file_description["models"].append(
|
2165
|
+
{
|
2166
|
+
"namespace": namespace,
|
2167
|
+
"bucketName": bucket,
|
2168
|
+
"prefix": "" if not prefix else prefix,
|
2169
|
+
"objects": objects,
|
2170
|
+
}
|
2171
|
+
)
|
2172
|
+
self.set_spec(self.CONST_MODEL_FILE_DESCRIPTION, tmp_model_file_description)
|
2173
|
+
|
2174
|
+
def remove_artifact(
|
2175
|
+
self,
|
2176
|
+
uri: Optional[str] = None,
|
2177
|
+
namespace: Optional[str] = None,
|
2178
|
+
bucket: Optional[str] = None,
|
2179
|
+
prefix: Optional[str] = None,
|
2180
|
+
):
|
2181
|
+
"""
|
2182
|
+
Removes information about objects in a specified bucket or using a specified URI from the model description JSON.
|
2183
|
+
|
2184
|
+
Parameters
|
2185
|
+
----------
|
2186
|
+
uri : str, optional
|
2187
|
+
The URI representing the location of the artifact in OCI object storage.
|
2188
|
+
namespace : str, optional
|
2189
|
+
The namespace of the bucket containing the objects. Required if `uri` is not provided.
|
2190
|
+
bucket : str, optional
|
2191
|
+
The name of the bucket containing the objects. Required if `uri` is not provided.
|
2192
|
+
prefix : str, optional
|
2193
|
+
The prefix of the objects to remove. Defaults to None.
|
2194
|
+
|
2195
|
+
Returns
|
2196
|
+
-------
|
2197
|
+
None
|
2198
|
+
|
2199
|
+
Raises
|
2200
|
+
------
|
2201
|
+
ValueError
|
2202
|
+
- If both 'uri' and ('namespace' and 'bucket') are provided.
|
2203
|
+
- If neither 'uri' nor both 'namespace' and 'bucket' are provided.
|
2204
|
+
- If the model description JSON is None.
|
2205
|
+
"""
|
2206
|
+
|
2207
|
+
if uri and (namespace or bucket):
|
2208
|
+
raise ValueError(
|
2209
|
+
"Either 'uri' must be provided or both 'namespace' and 'bucket' must be provided."
|
2210
|
+
)
|
2211
|
+
if uri:
|
2212
|
+
object_storage_details = ObjectStorageDetails.from_path(uri)
|
2213
|
+
bucket = object_storage_details.bucket
|
2214
|
+
namespace = object_storage_details.namespace
|
2215
|
+
prefix = (
|
2216
|
+
None
|
2217
|
+
if object_storage_details.filepath == ""
|
2218
|
+
else object_storage_details.filepath
|
2219
|
+
)
|
2220
|
+
if (not namespace) or (not bucket):
|
2221
|
+
raise ValueError("Both 'namespace' and 'bucket' must be provided.")
|
2222
|
+
|
2223
|
+
def find_model_idx():
|
2224
|
+
for idx, model in enumerate(self.model_file_description["models"]):
|
2225
|
+
if (
|
2226
|
+
model["namespace"],
|
2227
|
+
model["bucketName"],
|
2228
|
+
(model["prefix"] if ("prefix" in model) else None),
|
2229
|
+
) == (namespace, bucket, "" if not prefix else prefix):
|
2230
|
+
return idx
|
2231
|
+
return -1
|
2232
|
+
|
2233
|
+
if self.model_file_description is None:
|
2234
|
+
return
|
2235
|
+
|
2236
|
+
modelSearchIdx = find_model_idx()
|
2237
|
+
if modelSearchIdx == -1:
|
2238
|
+
return
|
2239
|
+
else:
|
2240
|
+
# model found case
|
2241
|
+
self.model_file_description["models"].pop(modelSearchIdx)
|
2242
|
+
|
2243
|
+
def if_model_custom_metadata_artifact_exist(
|
2244
|
+
self, metadata_key_name: str, **kwargs
|
2245
|
+
) -> bool:
|
2246
|
+
"""Checks if the custom metadata artifact exists for the model.
|
2247
|
+
|
2248
|
+
Parameters
|
2249
|
+
----------
|
2250
|
+
metadata_key_name: str
|
2251
|
+
Custom metadata key name
|
2252
|
+
**kwargs :
|
2253
|
+
Additional keyword arguments passed in head_model_artifact.
|
2254
|
+
|
2255
|
+
Returns
|
2256
|
+
-------
|
2257
|
+
bool
|
2258
|
+
Whether the artifact exists.
|
2259
|
+
"""
|
2260
|
+
|
2261
|
+
try:
|
2262
|
+
response = self.dsc_model.head_custom_metadata_artifact(
|
2263
|
+
metadata_key_name=metadata_key_name, **kwargs
|
2264
|
+
)
|
2265
|
+
return int(response.status) == 200
|
2266
|
+
except Exception as ex:
|
2267
|
+
logger.info(
|
2268
|
+
f"Error fetching custom metadata: {metadata_key_name} for model {self.id}. {ex}"
|
2269
|
+
)
|
2270
|
+
return False
|
2271
|
+
|
2272
|
+
def create_custom_metadata_artifact(
|
2273
|
+
self,
|
2274
|
+
metadata_key_name: str,
|
2275
|
+
artifact_path_or_content: Union[str, bytes],
|
2276
|
+
path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
|
2277
|
+
) -> ModelMetadataArtifactDetails:
|
2278
|
+
"""Creates model custom metadata artifact for specified model.
|
2279
|
+
|
2280
|
+
Parameters
|
2281
|
+
----------
|
2282
|
+
metadata_key_name: str
|
2283
|
+
The name of the model custom metadata key
|
2284
|
+
|
2285
|
+
artifact_path_or_content: Union[str,bytes]
|
2286
|
+
The model custom metadata artifact path to be uploaded. It can also be the actual content of the custom metadata artifact
|
2287
|
+
The type is string when it represents local path or oss path.
|
2288
|
+
The type is bytes when it represents content itself
|
2289
|
+
|
2290
|
+
path_type: MetadataArtifactPathType
|
2291
|
+
Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
|
2292
|
+
Specifies what type of path is to be provided for metadata artifact.
|
2293
|
+
|
2294
|
+
Example:
|
2295
|
+
>>> ds_model=DataScienceModel.from_id("ocid1.datasciencemodel.iad.xxyxz...")
|
2296
|
+
>>> ds_model.create_custom_metadata_artifact(
|
2297
|
+
... "README",
|
2298
|
+
... artifact_path_or_content="/Users/<username>/Downloads/README.md",
|
2299
|
+
... path_type=MetadataArtifactPathType.LOCAL
|
2300
|
+
... )
|
2301
|
+
|
2302
|
+
Returns
|
2303
|
+
-------
|
2304
|
+
ModelMetadataArtifactDetails
|
2305
|
+
The model custom metadata artifact creation info.
|
2306
|
+
Example:
|
2307
|
+
{
|
2308
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2309
|
+
'opc-request-id': 'E4F7',
|
2310
|
+
'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
|
2311
|
+
'X-Content-Type-Options': 'nosniff',
|
2312
|
+
'Content-Length': '4029958',
|
2313
|
+
'Vary': 'Origin',
|
2314
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2315
|
+
'status': 204
|
2316
|
+
}
|
2317
|
+
|
2318
|
+
"""
|
2319
|
+
if path_type == MetadataArtifactPathType.CONTENT and not isinstance(
|
2320
|
+
artifact_path_or_content, bytes
|
2321
|
+
):
|
2322
|
+
raise InvalidArtifactPathTypeOrContentError(
|
2323
|
+
f"Invalid type of artifact content: {type(artifact_path_or_content)}. It should be bytes."
|
2324
|
+
)
|
2325
|
+
|
2326
|
+
return self.dsc_model.create_custom_metadata_artifact(
|
2327
|
+
metadata_key_name=metadata_key_name,
|
2328
|
+
artifact_path_or_content=artifact_path_or_content,
|
2329
|
+
path_type=path_type,
|
2330
|
+
)
|
2331
|
+
|
2332
|
+
def create_defined_metadata_artifact(
|
2333
|
+
self,
|
2334
|
+
metadata_key_name: str,
|
2335
|
+
artifact_path_or_content: Union[str, bytes],
|
2336
|
+
path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
|
2337
|
+
) -> ModelMetadataArtifactDetails:
|
2338
|
+
"""Creates model defined metadata artifact for specified model.
|
2339
|
+
|
2340
|
+
Parameters
|
2341
|
+
----------
|
2342
|
+
metadata_key_name: str
|
2343
|
+
The name of the model defined metadata key
|
2344
|
+
|
2345
|
+
artifact_path_or_content: Union[str,bytes]
|
2346
|
+
The model defined metadata artifact path to be uploaded. It can also be the actual content of the defined metadata
|
2347
|
+
The type is string when it represents local path or oss path.
|
2348
|
+
The type is bytes when it represents content itself
|
2349
|
+
|
2350
|
+
path_type: MetadataArtifactPathType
|
2351
|
+
Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
|
2352
|
+
Specifies what type of path is to be provided for metadata artifact.
|
2353
|
+
Can be either local , oss or the actual content itself
|
2354
|
+
|
2355
|
+
Example:
|
2356
|
+
>>> ds_model=DataScienceModel.from_id("ocid1.datasciencemodel.iad.xxyxz...")
|
2357
|
+
>>> ds_model.create_defined_metadata_artifact(
|
2358
|
+
... "README",
|
2359
|
+
... artifact_path_or_content="oci://path/to/bucket/README.md",
|
2360
|
+
... path_type=MetadataArtifactPathType.OSS
|
2361
|
+
... )
|
2362
|
+
|
2363
|
+
Returns
|
2364
|
+
-------
|
2365
|
+
ModelMetadataArtifactDetails
|
2366
|
+
The model defined metadata artifact creation info.
|
2367
|
+
Example:
|
2368
|
+
{
|
2369
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2370
|
+
'opc-request-id': 'E4F7',
|
2371
|
+
'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
|
2372
|
+
'X-Content-Type-Options': 'nosniff',
|
2373
|
+
'Content-Length': '4029958',
|
2374
|
+
'Vary': 'Origin',
|
2375
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2376
|
+
'status': 204
|
2377
|
+
}
|
2378
|
+
|
2379
|
+
"""
|
2380
|
+
if path_type == MetadataArtifactPathType.CONTENT and not isinstance(
|
2381
|
+
artifact_path_or_content, bytes
|
2382
|
+
):
|
2383
|
+
raise InvalidArtifactPathTypeOrContentError(
|
2384
|
+
f"Invalid type of artifact content: {type(artifact_path_or_content)}. It should be bytes."
|
2385
|
+
)
|
2386
|
+
|
2387
|
+
return self.dsc_model.create_defined_metadata_artifact(
|
2388
|
+
metadata_key_name=metadata_key_name,
|
2389
|
+
artifact_path_or_content=artifact_path_or_content,
|
2390
|
+
path_type=path_type,
|
2391
|
+
)
|
2392
|
+
|
2393
|
+
def update_custom_metadata_artifact(
|
2394
|
+
self,
|
2395
|
+
metadata_key_name: str,
|
2396
|
+
artifact_path_or_content: Union[str, bytes],
|
2397
|
+
path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
|
2398
|
+
) -> ModelMetadataArtifactDetails:
|
2399
|
+
"""Update model custom metadata artifact for specified model.
|
2400
|
+
|
2401
|
+
Parameters
|
2402
|
+
----------
|
2403
|
+
metadata_key_name: str
|
2404
|
+
The name of the model custom metadata key
|
2405
|
+
|
2406
|
+
artifact_path_or_content: Union[str,bytes]
|
2407
|
+
The model custom metadata artifact path to be uploaded. It can also be the actual content of the custom metadata
|
2408
|
+
The type is string when it represents local path or oss path.
|
2409
|
+
The type is bytes when it represents content itself
|
2410
|
+
|
2411
|
+
path_type: MetadataArtifactPathType
|
2412
|
+
Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
|
2413
|
+
Specifies what type of path is to be provided for metadata artifact.
|
2414
|
+
Can be either local , oss or the actual content itself
|
2415
|
+
|
2416
|
+
Returns
|
2417
|
+
-------
|
2418
|
+
ModelMetadataArtifactDetails
|
2419
|
+
The model custom metadata artifact update info.
|
2420
|
+
Example:
|
2421
|
+
{
|
2422
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2423
|
+
'opc-request-id': 'E4F7',
|
2424
|
+
'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
|
2425
|
+
'X-Content-Type-Options': 'nosniff',
|
2426
|
+
'Content-Length': '4029958',
|
2427
|
+
'Vary': 'Origin',
|
2428
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2429
|
+
'status': 204
|
2430
|
+
}
|
2431
|
+
|
2432
|
+
"""
|
2433
|
+
if path_type == MetadataArtifactPathType.CONTENT and not isinstance(
|
2434
|
+
artifact_path_or_content, bytes
|
2435
|
+
):
|
2436
|
+
raise InvalidArtifactPathTypeOrContentError(
|
2437
|
+
f"Invalid type of artifact content: {type(artifact_path_or_content)}. It should be bytes."
|
2438
|
+
)
|
2439
|
+
|
2440
|
+
return self.dsc_model.update_custom_metadata_artifact(
|
2441
|
+
metadata_key_name=metadata_key_name,
|
2442
|
+
artifact_path_or_content=artifact_path_or_content,
|
2443
|
+
path_type=path_type,
|
2444
|
+
)
|
2445
|
+
|
2446
|
+
def update_defined_metadata_artifact(
|
2447
|
+
self,
|
2448
|
+
metadata_key_name: str,
|
2449
|
+
artifact_path_or_content: Union[str, bytes],
|
2450
|
+
path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
|
2451
|
+
) -> ModelMetadataArtifactDetails:
|
2452
|
+
"""Update model defined metadata artifact for specified model.
|
2453
|
+
|
2454
|
+
Parameters
|
2455
|
+
----------
|
2456
|
+
metadata_key_name: str
|
2457
|
+
The name of the model defined metadata key
|
2458
|
+
|
2459
|
+
artifact_path_or_content: Union[str,bytes]
|
2460
|
+
The model defined metadata artifact path to be uploaded. It can also be the actual content of the defined metadata
|
2461
|
+
The type is string when it represents local path or oss path.
|
2462
|
+
The type is bytes when it represents content itself
|
2463
|
+
|
2464
|
+
path_type: MetadataArtifactPathType
|
2465
|
+
Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
|
2466
|
+
Specifies what type of path is to be provided for metadata artifact.
|
2467
|
+
Can be either local , oss or the actual content itself
|
2468
|
+
|
2469
|
+
Returns
|
2470
|
+
-------
|
2471
|
+
ModelMetadataArtifactDetails
|
2472
|
+
The model defined metadata artifact update info.
|
2473
|
+
Example:
|
2474
|
+
{
|
2475
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2476
|
+
'opc-request-id': 'E4F7',
|
2477
|
+
'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
|
2478
|
+
'X-Content-Type-Options': 'nosniff',
|
2479
|
+
'Content-Length': '4029958',
|
2480
|
+
'Vary': 'Origin',
|
2481
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2482
|
+
'status': 204
|
2483
|
+
}
|
2484
|
+
|
2485
|
+
"""
|
2486
|
+
if path_type == MetadataArtifactPathType.CONTENT and not isinstance(
|
2487
|
+
artifact_path_or_content, bytes
|
2488
|
+
):
|
2489
|
+
raise InvalidArtifactPathTypeOrContentError(
|
2490
|
+
f"Invalid type of artifact content: {type(artifact_path_or_content)}. It should be bytes."
|
2491
|
+
)
|
2492
|
+
|
2493
|
+
return self.dsc_model.update_defined_metadata_artifact(
|
2494
|
+
metadata_key_name=metadata_key_name,
|
2495
|
+
artifact_path_or_content=artifact_path_or_content,
|
2496
|
+
path_type=path_type,
|
2497
|
+
)
|
2498
|
+
|
2499
|
+
def get_custom_metadata_artifact(
|
2500
|
+
self, metadata_key_name: str, target_dir: str, override: bool = False
|
2501
|
+
) -> bytes:
|
2502
|
+
"""Downloads model custom metadata artifact content for specified model metadata key.
|
2503
|
+
|
2504
|
+
Parameters
|
2505
|
+
----------
|
2506
|
+
metadata_key_name: str
|
2507
|
+
The name of the custom metadata key of the model
|
2508
|
+
|
2509
|
+
target_dir: str
|
2510
|
+
The local file path where downloaded model custom metadata artifact will be saved.
|
2511
|
+
|
2512
|
+
override: bool
|
2513
|
+
A boolean flag that controls downloaded metadata artifact file overwriting
|
2514
|
+
- If True, overwrites the file if it already exists.
|
2515
|
+
- If False (default), raises a `FileExistsError` if the file exists.
|
2516
|
+
Returns
|
2517
|
+
-------
|
2518
|
+
bytes
|
2519
|
+
File content of the custom metadata artifact
|
2520
|
+
|
2521
|
+
"""
|
2522
|
+
if not is_path_exists(target_dir):
|
2523
|
+
raise PathNotFoundError(f"Path : {target_dir} does not exist")
|
2524
|
+
|
2525
|
+
file_content = self.dsc_model.get_custom_metadata_artifact(
|
2526
|
+
metadata_key_name=metadata_key_name
|
2527
|
+
)
|
2528
|
+
artifact_file_path = os.path.join(target_dir, f"{metadata_key_name}")
|
2529
|
+
|
2530
|
+
if not override and is_path_exists(artifact_file_path):
|
2531
|
+
raise FileExistsError(
|
2532
|
+
f"File already exists: {artifact_file_path}. Please use boolean override parameter to override the file content."
|
2533
|
+
)
|
2534
|
+
|
2535
|
+
with open(artifact_file_path, "wb") as _file:
|
2536
|
+
_file.write(file_content)
|
2537
|
+
logger.debug(f"Artifact downloaded to location - {artifact_file_path}")
|
2538
|
+
return file_content
|
2539
|
+
|
2540
|
+
def get_defined_metadata_artifact(
|
2541
|
+
self, metadata_key_name: str, target_dir: str, override: bool = False
|
2542
|
+
) -> bytes:
|
2543
|
+
"""Downloads model defined metadata artifact content for specified model metadata key.
|
2544
|
+
|
2545
|
+
Parameters
|
2546
|
+
----------
|
2547
|
+
metadata_key_name: str
|
2548
|
+
The name of the model metadatum in the metadata.
|
2549
|
+
|
2550
|
+
target_dir: str
|
2551
|
+
The local file path where downloaded model defined metadata artifact will be saved.
|
2552
|
+
|
2553
|
+
override: bool
|
2554
|
+
A boolean flag that controls downloaded metadata artifact file overwriting
|
2555
|
+
- If True, overwrites the file if it already exists.
|
2556
|
+
- If False (default), raises a `FileExistsError` if the file exists.
|
2557
|
+
Returns
|
2558
|
+
-------
|
2559
|
+
bytes
|
2560
|
+
File content of the custom metadata artifact
|
2561
|
+
|
2562
|
+
"""
|
2563
|
+
if not is_path_exists(target_dir):
|
2564
|
+
raise PathNotFoundError(f"Path : {target_dir} does not exist")
|
2565
|
+
|
2566
|
+
file_content = self.dsc_model.get_defined_metadata_artifact(
|
2567
|
+
metadata_key_name=metadata_key_name
|
2568
|
+
)
|
2569
|
+
artifact_file_path = os.path.join(target_dir, f"{metadata_key_name}")
|
2570
|
+
|
2571
|
+
if not override and is_path_exists(artifact_file_path):
|
2572
|
+
raise FileExistsError(
|
2573
|
+
f"File already exists: {artifact_file_path}. Please use boolean override parameter to override the file content."
|
2574
|
+
)
|
2575
|
+
|
2576
|
+
with open(artifact_file_path, "wb") as _file:
|
2577
|
+
_file.write(file_content)
|
2578
|
+
logger.debug(f"Artifact downloaded to location - {artifact_file_path}")
|
2579
|
+
return file_content
|
2580
|
+
|
2581
|
+
def delete_custom_metadata_artifact(
|
2582
|
+
self, metadata_key_name: str
|
2583
|
+
) -> ModelMetadataArtifactDetails:
|
2584
|
+
"""Deletes model custom metadata artifact for specified model metadata key.
|
2585
|
+
|
2586
|
+
Parameters
|
2587
|
+
----------
|
2588
|
+
metadata_key_name: str
|
2589
|
+
The name of the model metadatum in the metadata.
|
2590
|
+
Returns
|
2591
|
+
-------
|
2592
|
+
ModelMetadataArtifactDetails
|
2593
|
+
The model custom metadata artifact delete call info.
|
2594
|
+
Example:
|
2595
|
+
{
|
2596
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2597
|
+
'opc-request-id': 'E4F7',
|
2598
|
+
'X-Content-Type-Options': 'nosniff',
|
2599
|
+
'Vary': 'Origin',
|
2600
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2601
|
+
'status': 204
|
2602
|
+
}
|
2603
|
+
|
2604
|
+
"""
|
2605
|
+
return self.dsc_model.delete_custom_metadata_artifact(
|
2606
|
+
metadata_key_name=metadata_key_name
|
2607
|
+
)
|
2608
|
+
|
2609
|
+
def delete_defined_metadata_artifact(
|
2610
|
+
self, metadata_key_name: str
|
2611
|
+
) -> ModelMetadataArtifactDetails:
|
2612
|
+
"""Deletes model defined metadata artifact for specified model metadata key.
|
2613
|
+
|
2614
|
+
Parameters
|
2615
|
+
----------
|
2616
|
+
metadata_key_name: str
|
2617
|
+
The name of the model metadatum in the metadata.
|
2618
|
+
Returns
|
2619
|
+
-------
|
2620
|
+
ModelMetadataArtifactDetails
|
2621
|
+
The model defined metadata artifact delete call info.
|
2622
|
+
Example:
|
2623
|
+
{
|
2624
|
+
'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
|
2625
|
+
'opc-request-id': 'E4F7',
|
2626
|
+
'X-Content-Type-Options': 'nosniff',
|
2627
|
+
'Vary': 'Origin',
|
2628
|
+
'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
|
2629
|
+
'status': 204
|
2630
|
+
}
|
2631
|
+
|
2632
|
+
"""
|
2633
|
+
return self.dsc_model.delete_defined_metadata_artifact(
|
2634
|
+
metadata_key_name=metadata_key_name
|
2635
|
+
)
|