oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/METADATA +151 -151
- oracle_ads-2.13.9rc0.dist-info/RECORD +9 -0
- {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/WHEEL +2 -1
- {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/entry_points.txt +1 -2
- oracle_ads-2.13.9rc0.dist-info/top_level.txt +1 -0
- ads/aqua/__init__.py +0 -40
- ads/aqua/app.py +0 -506
- ads/aqua/cli.py +0 -96
- ads/aqua/client/__init__.py +0 -3
- ads/aqua/client/client.py +0 -836
- ads/aqua/client/openai_client.py +0 -305
- ads/aqua/common/__init__.py +0 -5
- ads/aqua/common/decorator.py +0 -125
- ads/aqua/common/entities.py +0 -266
- ads/aqua/common/enums.py +0 -122
- ads/aqua/common/errors.py +0 -109
- ads/aqua/common/utils.py +0 -1285
- ads/aqua/config/__init__.py +0 -4
- ads/aqua/config/container_config.py +0 -248
- ads/aqua/config/evaluation/__init__.py +0 -4
- ads/aqua/config/evaluation/evaluation_service_config.py +0 -147
- ads/aqua/config/utils/__init__.py +0 -4
- ads/aqua/config/utils/serializer.py +0 -339
- ads/aqua/constants.py +0 -114
- ads/aqua/data.py +0 -14
- ads/aqua/dummy_data/icon.txt +0 -1
- ads/aqua/dummy_data/oci_model_deployments.json +0 -56
- ads/aqua/dummy_data/oci_models.json +0 -1
- ads/aqua/dummy_data/readme.md +0 -26
- ads/aqua/evaluation/__init__.py +0 -8
- ads/aqua/evaluation/constants.py +0 -53
- ads/aqua/evaluation/entities.py +0 -186
- ads/aqua/evaluation/errors.py +0 -70
- ads/aqua/evaluation/evaluation.py +0 -1814
- ads/aqua/extension/__init__.py +0 -42
- ads/aqua/extension/aqua_ws_msg_handler.py +0 -76
- ads/aqua/extension/base_handler.py +0 -90
- ads/aqua/extension/common_handler.py +0 -121
- ads/aqua/extension/common_ws_msg_handler.py +0 -36
- ads/aqua/extension/deployment_handler.py +0 -298
- ads/aqua/extension/deployment_ws_msg_handler.py +0 -54
- ads/aqua/extension/errors.py +0 -30
- ads/aqua/extension/evaluation_handler.py +0 -129
- ads/aqua/extension/evaluation_ws_msg_handler.py +0 -61
- ads/aqua/extension/finetune_handler.py +0 -96
- ads/aqua/extension/model_handler.py +0 -390
- ads/aqua/extension/models/__init__.py +0 -0
- ads/aqua/extension/models/ws_models.py +0 -145
- ads/aqua/extension/models_ws_msg_handler.py +0 -50
- ads/aqua/extension/ui_handler.py +0 -282
- ads/aqua/extension/ui_websocket_handler.py +0 -130
- ads/aqua/extension/utils.py +0 -133
- ads/aqua/finetuning/__init__.py +0 -7
- ads/aqua/finetuning/constants.py +0 -23
- ads/aqua/finetuning/entities.py +0 -181
- ads/aqua/finetuning/finetuning.py +0 -731
- ads/aqua/model/__init__.py +0 -8
- ads/aqua/model/constants.py +0 -60
- ads/aqua/model/entities.py +0 -306
- ads/aqua/model/enums.py +0 -30
- ads/aqua/model/model.py +0 -2079
- ads/aqua/modeldeployment/__init__.py +0 -8
- ads/aqua/modeldeployment/constants.py +0 -10
- ads/aqua/modeldeployment/deployment.py +0 -1324
- ads/aqua/modeldeployment/entities.py +0 -653
- ads/aqua/modeldeployment/inference.py +0 -74
- ads/aqua/modeldeployment/utils.py +0 -543
- ads/aqua/resources/gpu_shapes_index.json +0 -94
- ads/aqua/server/__init__.py +0 -4
- ads/aqua/server/__main__.py +0 -24
- ads/aqua/server/app.py +0 -47
- ads/aqua/server/aqua_spec.yml +0 -1291
- ads/aqua/training/__init__.py +0 -4
- ads/aqua/training/exceptions.py +0 -476
- ads/aqua/ui.py +0 -499
- ads/automl/__init__.py +0 -9
- ads/automl/driver.py +0 -330
- ads/automl/provider.py +0 -975
- ads/bds/__init__.py +0 -5
- ads/bds/auth.py +0 -127
- ads/bds/big_data_service.py +0 -255
- ads/catalog/__init__.py +0 -19
- ads/catalog/model.py +0 -1576
- ads/catalog/notebook.py +0 -461
- ads/catalog/project.py +0 -468
- ads/catalog/summary.py +0 -178
- ads/common/__init__.py +0 -11
- ads/common/analyzer.py +0 -65
- ads/common/artifact/.model-ignore +0 -63
- ads/common/artifact/__init__.py +0 -10
- ads/common/auth.py +0 -1122
- ads/common/card_identifier.py +0 -83
- ads/common/config.py +0 -647
- ads/common/data.py +0 -165
- ads/common/decorator/__init__.py +0 -9
- ads/common/decorator/argument_to_case.py +0 -88
- ads/common/decorator/deprecate.py +0 -69
- ads/common/decorator/require_nonempty_arg.py +0 -65
- ads/common/decorator/runtime_dependency.py +0 -178
- ads/common/decorator/threaded.py +0 -97
- ads/common/decorator/utils.py +0 -35
- ads/common/dsc_file_system.py +0 -303
- ads/common/error.py +0 -14
- ads/common/extended_enum.py +0 -81
- ads/common/function/__init__.py +0 -5
- ads/common/function/fn_util.py +0 -142
- ads/common/function/func_conf.yaml +0 -25
- ads/common/ipython.py +0 -76
- ads/common/model.py +0 -679
- ads/common/model_artifact.py +0 -1759
- ads/common/model_artifact_schema.json +0 -107
- ads/common/model_export_util.py +0 -664
- ads/common/model_metadata.py +0 -24
- ads/common/object_storage_details.py +0 -296
- ads/common/oci_client.py +0 -175
- ads/common/oci_datascience.py +0 -46
- ads/common/oci_logging.py +0 -1144
- ads/common/oci_mixin.py +0 -957
- ads/common/oci_resource.py +0 -136
- ads/common/serializer.py +0 -559
- ads/common/utils.py +0 -1852
- ads/common/word_lists.py +0 -1491
- ads/common/work_request.py +0 -189
- ads/data_labeling/__init__.py +0 -13
- ads/data_labeling/boundingbox.py +0 -253
- ads/data_labeling/constants.py +0 -47
- ads/data_labeling/data_labeling_service.py +0 -244
- ads/data_labeling/interface/__init__.py +0 -5
- ads/data_labeling/interface/loader.py +0 -16
- ads/data_labeling/interface/parser.py +0 -16
- ads/data_labeling/interface/reader.py +0 -23
- ads/data_labeling/loader/__init__.py +0 -5
- ads/data_labeling/loader/file_loader.py +0 -241
- ads/data_labeling/metadata.py +0 -110
- ads/data_labeling/mixin/__init__.py +0 -5
- ads/data_labeling/mixin/data_labeling.py +0 -232
- ads/data_labeling/ner.py +0 -129
- ads/data_labeling/parser/__init__.py +0 -5
- ads/data_labeling/parser/dls_record_parser.py +0 -388
- ads/data_labeling/parser/export_metadata_parser.py +0 -94
- ads/data_labeling/parser/export_record_parser.py +0 -473
- ads/data_labeling/reader/__init__.py +0 -5
- ads/data_labeling/reader/dataset_reader.py +0 -574
- ads/data_labeling/reader/dls_record_reader.py +0 -121
- ads/data_labeling/reader/export_record_reader.py +0 -62
- ads/data_labeling/reader/jsonl_reader.py +0 -75
- ads/data_labeling/reader/metadata_reader.py +0 -203
- ads/data_labeling/reader/record_reader.py +0 -263
- ads/data_labeling/record.py +0 -52
- ads/data_labeling/visualizer/__init__.py +0 -5
- ads/data_labeling/visualizer/image_visualizer.py +0 -525
- ads/data_labeling/visualizer/text_visualizer.py +0 -357
- ads/database/__init__.py +0 -5
- ads/database/connection.py +0 -338
- ads/dataset/__init__.py +0 -10
- ads/dataset/capabilities.md +0 -51
- ads/dataset/classification_dataset.py +0 -339
- ads/dataset/correlation.py +0 -226
- ads/dataset/correlation_plot.py +0 -563
- ads/dataset/dask_series.py +0 -173
- ads/dataset/dataframe_transformer.py +0 -110
- ads/dataset/dataset.py +0 -1979
- ads/dataset/dataset_browser.py +0 -360
- ads/dataset/dataset_with_target.py +0 -995
- ads/dataset/exception.py +0 -25
- ads/dataset/factory.py +0 -987
- ads/dataset/feature_engineering_transformer.py +0 -35
- ads/dataset/feature_selection.py +0 -107
- ads/dataset/forecasting_dataset.py +0 -26
- ads/dataset/helper.py +0 -1450
- ads/dataset/label_encoder.py +0 -99
- ads/dataset/mixin/__init__.py +0 -5
- ads/dataset/mixin/dataset_accessor.py +0 -134
- ads/dataset/pipeline.py +0 -58
- ads/dataset/plot.py +0 -710
- ads/dataset/progress.py +0 -86
- ads/dataset/recommendation.py +0 -297
- ads/dataset/recommendation_transformer.py +0 -502
- ads/dataset/regression_dataset.py +0 -14
- ads/dataset/sampled_dataset.py +0 -1050
- ads/dataset/target.py +0 -98
- ads/dataset/timeseries.py +0 -18
- ads/dbmixin/__init__.py +0 -5
- ads/dbmixin/db_pandas_accessor.py +0 -153
- ads/environment/__init__.py +0 -9
- ads/environment/ml_runtime.py +0 -66
- ads/evaluations/README.md +0 -14
- ads/evaluations/__init__.py +0 -109
- ads/evaluations/evaluation_plot.py +0 -983
- ads/evaluations/evaluator.py +0 -1334
- ads/evaluations/statistical_metrics.py +0 -543
- ads/experiments/__init__.py +0 -9
- ads/experiments/capabilities.md +0 -0
- ads/explanations/__init__.py +0 -21
- ads/explanations/base_explainer.py +0 -142
- ads/explanations/capabilities.md +0 -83
- ads/explanations/explainer.py +0 -190
- ads/explanations/mlx_global_explainer.py +0 -1050
- ads/explanations/mlx_interface.py +0 -386
- ads/explanations/mlx_local_explainer.py +0 -287
- ads/explanations/mlx_whatif_explainer.py +0 -201
- ads/feature_engineering/__init__.py +0 -20
- ads/feature_engineering/accessor/__init__.py +0 -5
- ads/feature_engineering/accessor/dataframe_accessor.py +0 -535
- ads/feature_engineering/accessor/mixin/__init__.py +0 -5
- ads/feature_engineering/accessor/mixin/correlation.py +0 -166
- ads/feature_engineering/accessor/mixin/eda_mixin.py +0 -266
- ads/feature_engineering/accessor/mixin/eda_mixin_series.py +0 -85
- ads/feature_engineering/accessor/mixin/feature_types_mixin.py +0 -211
- ads/feature_engineering/accessor/mixin/utils.py +0 -65
- ads/feature_engineering/accessor/series_accessor.py +0 -431
- ads/feature_engineering/adsimage/__init__.py +0 -5
- ads/feature_engineering/adsimage/image.py +0 -192
- ads/feature_engineering/adsimage/image_reader.py +0 -170
- ads/feature_engineering/adsimage/interface/__init__.py +0 -5
- ads/feature_engineering/adsimage/interface/reader.py +0 -19
- ads/feature_engineering/adsstring/__init__.py +0 -7
- ads/feature_engineering/adsstring/oci_language/__init__.py +0 -8
- ads/feature_engineering/adsstring/string/__init__.py +0 -8
- ads/feature_engineering/data_schema.json +0 -57
- ads/feature_engineering/dataset/__init__.py +0 -5
- ads/feature_engineering/dataset/zip_code_data.py +0 -42062
- ads/feature_engineering/exceptions.py +0 -40
- ads/feature_engineering/feature_type/__init__.py +0 -133
- ads/feature_engineering/feature_type/address.py +0 -184
- ads/feature_engineering/feature_type/adsstring/__init__.py +0 -5
- ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +0 -164
- ads/feature_engineering/feature_type/adsstring/oci_language.py +0 -93
- ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +0 -5
- ads/feature_engineering/feature_type/adsstring/parsers/base.py +0 -47
- ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +0 -96
- ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +0 -221
- ads/feature_engineering/feature_type/adsstring/string.py +0 -258
- ads/feature_engineering/feature_type/base.py +0 -58
- ads/feature_engineering/feature_type/boolean.py +0 -183
- ads/feature_engineering/feature_type/category.py +0 -146
- ads/feature_engineering/feature_type/constant.py +0 -137
- ads/feature_engineering/feature_type/continuous.py +0 -151
- ads/feature_engineering/feature_type/creditcard.py +0 -314
- ads/feature_engineering/feature_type/datetime.py +0 -190
- ads/feature_engineering/feature_type/discrete.py +0 -134
- ads/feature_engineering/feature_type/document.py +0 -43
- ads/feature_engineering/feature_type/gis.py +0 -251
- ads/feature_engineering/feature_type/handler/__init__.py +0 -5
- ads/feature_engineering/feature_type/handler/feature_validator.py +0 -524
- ads/feature_engineering/feature_type/handler/feature_warning.py +0 -319
- ads/feature_engineering/feature_type/handler/warnings.py +0 -128
- ads/feature_engineering/feature_type/integer.py +0 -142
- ads/feature_engineering/feature_type/ip_address.py +0 -144
- ads/feature_engineering/feature_type/ip_address_v4.py +0 -138
- ads/feature_engineering/feature_type/ip_address_v6.py +0 -138
- ads/feature_engineering/feature_type/lat_long.py +0 -256
- ads/feature_engineering/feature_type/object.py +0 -43
- ads/feature_engineering/feature_type/ordinal.py +0 -132
- ads/feature_engineering/feature_type/phone_number.py +0 -135
- ads/feature_engineering/feature_type/string.py +0 -171
- ads/feature_engineering/feature_type/text.py +0 -93
- ads/feature_engineering/feature_type/unknown.py +0 -43
- ads/feature_engineering/feature_type/zip_code.py +0 -164
- ads/feature_engineering/feature_type_manager.py +0 -406
- ads/feature_engineering/schema.py +0 -795
- ads/feature_engineering/utils.py +0 -245
- ads/feature_store/.readthedocs.yaml +0 -19
- ads/feature_store/README.md +0 -65
- ads/feature_store/__init__.py +0 -9
- ads/feature_store/common/__init__.py +0 -0
- ads/feature_store/common/enums.py +0 -339
- ads/feature_store/common/exceptions.py +0 -18
- ads/feature_store/common/spark_session_singleton.py +0 -125
- ads/feature_store/common/utils/__init__.py +0 -0
- ads/feature_store/common/utils/base64_encoder_decoder.py +0 -72
- ads/feature_store/common/utils/feature_schema_mapper.py +0 -283
- ads/feature_store/common/utils/transformation_utils.py +0 -82
- ads/feature_store/common/utils/utility.py +0 -403
- ads/feature_store/data_validation/__init__.py +0 -0
- ads/feature_store/data_validation/great_expectation.py +0 -129
- ads/feature_store/dataset.py +0 -1230
- ads/feature_store/dataset_job.py +0 -530
- ads/feature_store/docs/Dockerfile +0 -7
- ads/feature_store/docs/Makefile +0 -44
- ads/feature_store/docs/conf.py +0 -28
- ads/feature_store/docs/requirements.txt +0 -14
- ads/feature_store/docs/source/ads.feature_store.query.rst +0 -20
- ads/feature_store/docs/source/cicd.rst +0 -137
- ads/feature_store/docs/source/conf.py +0 -86
- ads/feature_store/docs/source/data_versioning.rst +0 -33
- ads/feature_store/docs/source/dataset.rst +0 -388
- ads/feature_store/docs/source/dataset_job.rst +0 -27
- ads/feature_store/docs/source/demo.rst +0 -70
- ads/feature_store/docs/source/entity.rst +0 -78
- ads/feature_store/docs/source/feature_group.rst +0 -624
- ads/feature_store/docs/source/feature_group_job.rst +0 -29
- ads/feature_store/docs/source/feature_store.rst +0 -122
- ads/feature_store/docs/source/feature_store_class.rst +0 -123
- ads/feature_store/docs/source/feature_validation.rst +0 -66
- ads/feature_store/docs/source/figures/cicd.png +0 -0
- ads/feature_store/docs/source/figures/data_validation.png +0 -0
- ads/feature_store/docs/source/figures/data_versioning.png +0 -0
- ads/feature_store/docs/source/figures/dataset.gif +0 -0
- ads/feature_store/docs/source/figures/dataset.png +0 -0
- ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
- ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
- ads/feature_store/docs/source/figures/entity.png +0 -0
- ads/feature_store/docs/source/figures/feature_group.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
- ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
- ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
- ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
- ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
- ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
- ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
- ads/feature_store/docs/source/figures/overview.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
- ads/feature_store/docs/source/figures/stats_1.png +0 -0
- ads/feature_store/docs/source/figures/stats_2.png +0 -0
- ads/feature_store/docs/source/figures/stats_d.png +0 -0
- ads/feature_store/docs/source/figures/stats_fg.png +0 -0
- ads/feature_store/docs/source/figures/transformation.png +0 -0
- ads/feature_store/docs/source/figures/transformations.gif +0 -0
- ads/feature_store/docs/source/figures/validation.png +0 -0
- ads/feature_store/docs/source/figures/validation_fg.png +0 -0
- ads/feature_store/docs/source/figures/validation_results.png +0 -0
- ads/feature_store/docs/source/figures/validation_summary.png +0 -0
- ads/feature_store/docs/source/index.rst +0 -81
- ads/feature_store/docs/source/module.rst +0 -8
- ads/feature_store/docs/source/notebook.rst +0 -94
- ads/feature_store/docs/source/overview.rst +0 -47
- ads/feature_store/docs/source/quickstart.rst +0 -176
- ads/feature_store/docs/source/release_notes.rst +0 -194
- ads/feature_store/docs/source/setup_feature_store.rst +0 -81
- ads/feature_store/docs/source/statistics.rst +0 -58
- ads/feature_store/docs/source/transformation.rst +0 -199
- ads/feature_store/docs/source/ui.rst +0 -65
- ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +0 -66
- ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +0 -192
- ads/feature_store/docs/source/user_guides.setup.terraform.rst +0 -338
- ads/feature_store/entity.py +0 -718
- ads/feature_store/execution_strategy/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +0 -375
- ads/feature_store/execution_strategy/engine/__init__.py +0 -0
- ads/feature_store/execution_strategy/engine/spark_engine.py +0 -316
- ads/feature_store/execution_strategy/execution_strategy.py +0 -113
- ads/feature_store/execution_strategy/execution_strategy_provider.py +0 -47
- ads/feature_store/execution_strategy/spark/__init__.py +0 -0
- ads/feature_store/execution_strategy/spark/spark_execution.py +0 -618
- ads/feature_store/feature.py +0 -192
- ads/feature_store/feature_group.py +0 -1494
- ads/feature_store/feature_group_expectation.py +0 -346
- ads/feature_store/feature_group_job.py +0 -602
- ads/feature_store/feature_lineage/__init__.py +0 -0
- ads/feature_store/feature_lineage/graphviz_service.py +0 -180
- ads/feature_store/feature_option_details.py +0 -50
- ads/feature_store/feature_statistics/__init__.py +0 -0
- ads/feature_store/feature_statistics/statistics_service.py +0 -99
- ads/feature_store/feature_store.py +0 -699
- ads/feature_store/feature_store_registrar.py +0 -518
- ads/feature_store/input_feature_detail.py +0 -149
- ads/feature_store/mixin/__init__.py +0 -4
- ads/feature_store/mixin/oci_feature_store.py +0 -145
- ads/feature_store/model_details.py +0 -73
- ads/feature_store/query/__init__.py +0 -0
- ads/feature_store/query/filter.py +0 -266
- ads/feature_store/query/generator/__init__.py +0 -0
- ads/feature_store/query/generator/query_generator.py +0 -298
- ads/feature_store/query/join.py +0 -161
- ads/feature_store/query/query.py +0 -403
- ads/feature_store/query/validator/__init__.py +0 -0
- ads/feature_store/query/validator/query_validator.py +0 -57
- ads/feature_store/response/__init__.py +0 -0
- ads/feature_store/response/response_builder.py +0 -68
- ads/feature_store/service/__init__.py +0 -0
- ads/feature_store/service/oci_dataset.py +0 -139
- ads/feature_store/service/oci_dataset_job.py +0 -199
- ads/feature_store/service/oci_entity.py +0 -125
- ads/feature_store/service/oci_feature_group.py +0 -164
- ads/feature_store/service/oci_feature_group_job.py +0 -214
- ads/feature_store/service/oci_feature_store.py +0 -182
- ads/feature_store/service/oci_lineage.py +0 -87
- ads/feature_store/service/oci_transformation.py +0 -104
- ads/feature_store/statistics/__init__.py +0 -0
- ads/feature_store/statistics/abs_feature_value.py +0 -49
- ads/feature_store/statistics/charts/__init__.py +0 -0
- ads/feature_store/statistics/charts/abstract_feature_plot.py +0 -37
- ads/feature_store/statistics/charts/box_plot.py +0 -148
- ads/feature_store/statistics/charts/frequency_distribution.py +0 -65
- ads/feature_store/statistics/charts/probability_distribution.py +0 -68
- ads/feature_store/statistics/charts/top_k_frequent_elements.py +0 -98
- ads/feature_store/statistics/feature_stat.py +0 -126
- ads/feature_store/statistics/generic_feature_value.py +0 -33
- ads/feature_store/statistics/statistics.py +0 -41
- ads/feature_store/statistics_config.py +0 -101
- ads/feature_store/templates/feature_store_template.yaml +0 -45
- ads/feature_store/transformation.py +0 -499
- ads/feature_store/validation_output.py +0 -57
- ads/hpo/__init__.py +0 -9
- ads/hpo/_imports.py +0 -91
- ads/hpo/ads_search_space.py +0 -439
- ads/hpo/distributions.py +0 -325
- ads/hpo/objective.py +0 -280
- ads/hpo/search_cv.py +0 -1657
- ads/hpo/stopping_criterion.py +0 -75
- ads/hpo/tuner_artifact.py +0 -413
- ads/hpo/utils.py +0 -91
- ads/hpo/validation.py +0 -140
- ads/hpo/visualization/__init__.py +0 -5
- ads/hpo/visualization/_contour.py +0 -23
- ads/hpo/visualization/_edf.py +0 -20
- ads/hpo/visualization/_intermediate_values.py +0 -21
- ads/hpo/visualization/_optimization_history.py +0 -25
- ads/hpo/visualization/_parallel_coordinate.py +0 -169
- ads/hpo/visualization/_param_importances.py +0 -26
- ads/jobs/__init__.py +0 -53
- ads/jobs/ads_job.py +0 -663
- ads/jobs/builders/__init__.py +0 -5
- ads/jobs/builders/base.py +0 -156
- ads/jobs/builders/infrastructure/__init__.py +0 -6
- ads/jobs/builders/infrastructure/base.py +0 -165
- ads/jobs/builders/infrastructure/dataflow.py +0 -1252
- ads/jobs/builders/infrastructure/dsc_job.py +0 -1894
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +0 -1233
- ads/jobs/builders/infrastructure/utils.py +0 -65
- ads/jobs/builders/runtimes/__init__.py +0 -5
- ads/jobs/builders/runtimes/artifact.py +0 -338
- ads/jobs/builders/runtimes/base.py +0 -325
- ads/jobs/builders/runtimes/container_runtime.py +0 -242
- ads/jobs/builders/runtimes/python_runtime.py +0 -1016
- ads/jobs/builders/runtimes/pytorch_runtime.py +0 -204
- ads/jobs/cli.py +0 -104
- ads/jobs/env_var_parser.py +0 -131
- ads/jobs/extension.py +0 -160
- ads/jobs/schema/__init__.py +0 -5
- ads/jobs/schema/infrastructure_schema.json +0 -116
- ads/jobs/schema/job_schema.json +0 -42
- ads/jobs/schema/runtime_schema.json +0 -183
- ads/jobs/schema/validator.py +0 -141
- ads/jobs/serializer.py +0 -296
- ads/jobs/templates/__init__.py +0 -5
- ads/jobs/templates/container.py +0 -6
- ads/jobs/templates/driver_notebook.py +0 -177
- ads/jobs/templates/driver_oci.py +0 -500
- ads/jobs/templates/driver_python.py +0 -48
- ads/jobs/templates/driver_pytorch.py +0 -852
- ads/jobs/templates/driver_utils.py +0 -615
- ads/jobs/templates/hostname_from_env.c +0 -55
- ads/jobs/templates/oci_metrics.py +0 -181
- ads/jobs/utils.py +0 -104
- ads/llm/__init__.py +0 -28
- ads/llm/autogen/__init__.py +0 -2
- ads/llm/autogen/constants.py +0 -15
- ads/llm/autogen/reports/__init__.py +0 -2
- ads/llm/autogen/reports/base.py +0 -67
- ads/llm/autogen/reports/data.py +0 -103
- ads/llm/autogen/reports/session.py +0 -526
- ads/llm/autogen/reports/templates/chat_box.html +0 -13
- ads/llm/autogen/reports/templates/chat_box_lt.html +0 -5
- ads/llm/autogen/reports/templates/chat_box_rt.html +0 -6
- ads/llm/autogen/reports/utils.py +0 -56
- ads/llm/autogen/v02/__init__.py +0 -4
- ads/llm/autogen/v02/client.py +0 -295
- ads/llm/autogen/v02/log_handlers/__init__.py +0 -2
- ads/llm/autogen/v02/log_handlers/oci_file_handler.py +0 -83
- ads/llm/autogen/v02/loggers/__init__.py +0 -6
- ads/llm/autogen/v02/loggers/metric_logger.py +0 -320
- ads/llm/autogen/v02/loggers/session_logger.py +0 -580
- ads/llm/autogen/v02/loggers/utils.py +0 -86
- ads/llm/autogen/v02/runtime_logging.py +0 -163
- ads/llm/chain.py +0 -268
- ads/llm/chat_template.py +0 -31
- ads/llm/deploy.py +0 -63
- ads/llm/guardrails/__init__.py +0 -5
- ads/llm/guardrails/base.py +0 -442
- ads/llm/guardrails/huggingface.py +0 -44
- ads/llm/langchain/__init__.py +0 -5
- ads/llm/langchain/plugins/__init__.py +0 -5
- ads/llm/langchain/plugins/chat_models/__init__.py +0 -5
- ads/llm/langchain/plugins/chat_models/oci_data_science.py +0 -1027
- ads/llm/langchain/plugins/embeddings/__init__.py +0 -4
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +0 -184
- ads/llm/langchain/plugins/llms/__init__.py +0 -5
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +0 -979
- ads/llm/requirements.txt +0 -3
- ads/llm/serialize.py +0 -219
- ads/llm/serializers/__init__.py +0 -0
- ads/llm/serializers/retrieval_qa.py +0 -153
- ads/llm/serializers/runnable_parallel.py +0 -27
- ads/llm/templates/score_chain.jinja2 +0 -155
- ads/llm/templates/tool_chat_template_hermes.jinja +0 -130
- ads/llm/templates/tool_chat_template_mistral_parallel.jinja +0 -94
- ads/model/__init__.py +0 -52
- ads/model/artifact.py +0 -573
- ads/model/artifact_downloader.py +0 -254
- ads/model/artifact_uploader.py +0 -267
- ads/model/base_properties.py +0 -238
- ads/model/common/.model-ignore +0 -66
- ads/model/common/__init__.py +0 -5
- ads/model/common/utils.py +0 -142
- ads/model/datascience_model.py +0 -2635
- ads/model/deployment/__init__.py +0 -20
- ads/model/deployment/common/__init__.py +0 -5
- ads/model/deployment/common/utils.py +0 -308
- ads/model/deployment/model_deployer.py +0 -466
- ads/model/deployment/model_deployment.py +0 -1846
- ads/model/deployment/model_deployment_infrastructure.py +0 -671
- ads/model/deployment/model_deployment_properties.py +0 -493
- ads/model/deployment/model_deployment_runtime.py +0 -838
- ads/model/extractor/__init__.py +0 -5
- ads/model/extractor/automl_extractor.py +0 -74
- ads/model/extractor/embedding_onnx_extractor.py +0 -80
- ads/model/extractor/huggingface_extractor.py +0 -88
- ads/model/extractor/keras_extractor.py +0 -84
- ads/model/extractor/lightgbm_extractor.py +0 -93
- ads/model/extractor/model_info_extractor.py +0 -114
- ads/model/extractor/model_info_extractor_factory.py +0 -105
- ads/model/extractor/pytorch_extractor.py +0 -87
- ads/model/extractor/sklearn_extractor.py +0 -112
- ads/model/extractor/spark_extractor.py +0 -89
- ads/model/extractor/tensorflow_extractor.py +0 -85
- ads/model/extractor/xgboost_extractor.py +0 -94
- ads/model/framework/__init__.py +0 -5
- ads/model/framework/automl_model.py +0 -178
- ads/model/framework/embedding_onnx_model.py +0 -438
- ads/model/framework/huggingface_model.py +0 -399
- ads/model/framework/lightgbm_model.py +0 -266
- ads/model/framework/pytorch_model.py +0 -266
- ads/model/framework/sklearn_model.py +0 -250
- ads/model/framework/spark_model.py +0 -326
- ads/model/framework/tensorflow_model.py +0 -254
- ads/model/framework/xgboost_model.py +0 -258
- ads/model/generic_model.py +0 -3518
- ads/model/model_artifact_boilerplate/README.md +0 -381
- ads/model/model_artifact_boilerplate/__init__.py +0 -5
- ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +0 -5
- ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +0 -427
- ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +0 -2
- ads/model/model_artifact_boilerplate/runtime.yaml +0 -7
- ads/model/model_artifact_boilerplate/score.py +0 -61
- ads/model/model_file_description_schema.json +0 -68
- ads/model/model_introspect.py +0 -331
- ads/model/model_metadata.py +0 -1810
- ads/model/model_metadata_mixin.py +0 -460
- ads/model/model_properties.py +0 -63
- ads/model/model_version_set.py +0 -739
- ads/model/runtime/__init__.py +0 -5
- ads/model/runtime/env_info.py +0 -306
- ads/model/runtime/model_deployment_details.py +0 -37
- ads/model/runtime/model_provenance_details.py +0 -58
- ads/model/runtime/runtime_info.py +0 -81
- ads/model/runtime/schemas/inference_env_info_schema.yaml +0 -16
- ads/model/runtime/schemas/model_provenance_schema.yaml +0 -36
- ads/model/runtime/schemas/training_env_info_schema.yaml +0 -16
- ads/model/runtime/utils.py +0 -201
- ads/model/serde/__init__.py +0 -5
- ads/model/serde/common.py +0 -40
- ads/model/serde/model_input.py +0 -547
- ads/model/serde/model_serializer.py +0 -1184
- ads/model/service/__init__.py +0 -5
- ads/model/service/oci_datascience_model.py +0 -1076
- ads/model/service/oci_datascience_model_deployment.py +0 -500
- ads/model/service/oci_datascience_model_version_set.py +0 -176
- ads/model/transformer/__init__.py +0 -5
- ads/model/transformer/onnx_transformer.py +0 -324
- ads/mysqldb/__init__.py +0 -5
- ads/mysqldb/mysql_db.py +0 -227
- ads/opctl/__init__.py +0 -18
- ads/opctl/anomaly_detection.py +0 -11
- ads/opctl/backend/__init__.py +0 -5
- ads/opctl/backend/ads_dataflow.py +0 -353
- ads/opctl/backend/ads_ml_job.py +0 -710
- ads/opctl/backend/ads_ml_pipeline.py +0 -164
- ads/opctl/backend/ads_model_deployment.py +0 -209
- ads/opctl/backend/base.py +0 -146
- ads/opctl/backend/local.py +0 -1053
- ads/opctl/backend/marketplace/__init__.py +0 -9
- ads/opctl/backend/marketplace/helm_helper.py +0 -173
- ads/opctl/backend/marketplace/local_marketplace.py +0 -271
- ads/opctl/backend/marketplace/marketplace_backend_runner.py +0 -71
- ads/opctl/backend/marketplace/marketplace_operator_interface.py +0 -44
- ads/opctl/backend/marketplace/marketplace_operator_runner.py +0 -24
- ads/opctl/backend/marketplace/marketplace_utils.py +0 -212
- ads/opctl/backend/marketplace/models/__init__.py +0 -5
- ads/opctl/backend/marketplace/models/bearer_token.py +0 -94
- ads/opctl/backend/marketplace/models/marketplace_type.py +0 -70
- ads/opctl/backend/marketplace/models/ocir_details.py +0 -56
- ads/opctl/backend/marketplace/prerequisite_checker.py +0 -238
- ads/opctl/cli.py +0 -707
- ads/opctl/cmds.py +0 -869
- ads/opctl/conda/__init__.py +0 -5
- ads/opctl/conda/cli.py +0 -193
- ads/opctl/conda/cmds.py +0 -749
- ads/opctl/conda/config.yaml +0 -34
- ads/opctl/conda/manifest_template.yaml +0 -13
- ads/opctl/conda/multipart_uploader.py +0 -188
- ads/opctl/conda/pack.py +0 -89
- ads/opctl/config/__init__.py +0 -5
- ads/opctl/config/base.py +0 -57
- ads/opctl/config/diagnostics/__init__.py +0 -5
- ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +0 -62
- ads/opctl/config/merger.py +0 -255
- ads/opctl/config/resolver.py +0 -297
- ads/opctl/config/utils.py +0 -79
- ads/opctl/config/validator.py +0 -17
- ads/opctl/config/versioner.py +0 -68
- ads/opctl/config/yaml_parsers/__init__.py +0 -7
- ads/opctl/config/yaml_parsers/base.py +0 -58
- ads/opctl/config/yaml_parsers/distributed/__init__.py +0 -7
- ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +0 -201
- ads/opctl/constants.py +0 -66
- ads/opctl/decorator/__init__.py +0 -5
- ads/opctl/decorator/common.py +0 -129
- ads/opctl/diagnostics/__init__.py +0 -5
- ads/opctl/diagnostics/__main__.py +0 -25
- ads/opctl/diagnostics/check_distributed_job_requirements.py +0 -212
- ads/opctl/diagnostics/check_requirements.py +0 -144
- ads/opctl/diagnostics/requirement_exception.py +0 -9
- ads/opctl/distributed/README.md +0 -109
- ads/opctl/distributed/__init__.py +0 -5
- ads/opctl/distributed/certificates.py +0 -32
- ads/opctl/distributed/cli.py +0 -207
- ads/opctl/distributed/cmds.py +0 -731
- ads/opctl/distributed/common/__init__.py +0 -5
- ads/opctl/distributed/common/abstract_cluster_provider.py +0 -449
- ads/opctl/distributed/common/abstract_framework_spec_builder.py +0 -88
- ads/opctl/distributed/common/cluster_config_helper.py +0 -103
- ads/opctl/distributed/common/cluster_provider_factory.py +0 -21
- ads/opctl/distributed/common/cluster_runner.py +0 -54
- ads/opctl/distributed/common/framework_factory.py +0 -29
- ads/opctl/docker/Dockerfile.job +0 -103
- ads/opctl/docker/Dockerfile.job.arm +0 -107
- ads/opctl/docker/Dockerfile.job.gpu +0 -175
- ads/opctl/docker/base-env.yaml +0 -13
- ads/opctl/docker/cuda.repo +0 -6
- ads/opctl/docker/operator/.dockerignore +0 -0
- ads/opctl/docker/operator/Dockerfile +0 -41
- ads/opctl/docker/operator/Dockerfile.gpu +0 -85
- ads/opctl/docker/operator/cuda.repo +0 -6
- ads/opctl/docker/operator/environment.yaml +0 -8
- ads/opctl/forecast.py +0 -11
- ads/opctl/index.yaml +0 -3
- ads/opctl/model/__init__.py +0 -5
- ads/opctl/model/cli.py +0 -65
- ads/opctl/model/cmds.py +0 -73
- ads/opctl/operator/README.md +0 -4
- ads/opctl/operator/__init__.py +0 -31
- ads/opctl/operator/cli.py +0 -344
- ads/opctl/operator/cmd.py +0 -596
- ads/opctl/operator/common/__init__.py +0 -5
- ads/opctl/operator/common/backend_factory.py +0 -460
- ads/opctl/operator/common/const.py +0 -27
- ads/opctl/operator/common/data/synthetic.csv +0 -16001
- ads/opctl/operator/common/dictionary_merger.py +0 -148
- ads/opctl/operator/common/errors.py +0 -42
- ads/opctl/operator/common/operator_config.py +0 -99
- ads/opctl/operator/common/operator_loader.py +0 -811
- ads/opctl/operator/common/operator_schema.yaml +0 -130
- ads/opctl/operator/common/operator_yaml_generator.py +0 -152
- ads/opctl/operator/common/utils.py +0 -208
- ads/opctl/operator/lowcode/__init__.py +0 -5
- ads/opctl/operator/lowcode/anomaly/MLoperator +0 -16
- ads/opctl/operator/lowcode/anomaly/README.md +0 -207
- ads/opctl/operator/lowcode/anomaly/__init__.py +0 -5
- ads/opctl/operator/lowcode/anomaly/__main__.py +0 -103
- ads/opctl/operator/lowcode/anomaly/cmd.py +0 -35
- ads/opctl/operator/lowcode/anomaly/const.py +0 -167
- ads/opctl/operator/lowcode/anomaly/environment.yaml +0 -10
- ads/opctl/operator/lowcode/anomaly/model/__init__.py +0 -5
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +0 -146
- ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +0 -162
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +0 -99
- ads/opctl/operator/lowcode/anomaly/model/autots.py +0 -115
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +0 -404
- ads/opctl/operator/lowcode/anomaly/model/factory.py +0 -110
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +0 -78
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +0 -78
- ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +0 -120
- ads/opctl/operator/lowcode/anomaly/model/tods.py +0 -119
- ads/opctl/operator/lowcode/anomaly/operator_config.py +0 -127
- ads/opctl/operator/lowcode/anomaly/schema.yaml +0 -401
- ads/opctl/operator/lowcode/anomaly/utils.py +0 -88
- ads/opctl/operator/lowcode/common/__init__.py +0 -5
- ads/opctl/operator/lowcode/common/const.py +0 -10
- ads/opctl/operator/lowcode/common/data.py +0 -116
- ads/opctl/operator/lowcode/common/errors.py +0 -47
- ads/opctl/operator/lowcode/common/transformations.py +0 -296
- ads/opctl/operator/lowcode/common/utils.py +0 -293
- ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +0 -13
- ads/opctl/operator/lowcode/feature_store_marketplace/README.md +0 -30
- ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +0 -5
- ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +0 -116
- ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +0 -85
- ads/opctl/operator/lowcode/feature_store_marketplace/const.py +0 -15
- ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +0 -4
- ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +0 -32
- ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +0 -43
- ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +0 -120
- ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +0 -34
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +0 -386
- ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +0 -160
- ads/opctl/operator/lowcode/forecast/MLoperator +0 -25
- ads/opctl/operator/lowcode/forecast/README.md +0 -209
- ads/opctl/operator/lowcode/forecast/__init__.py +0 -5
- ads/opctl/operator/lowcode/forecast/__main__.py +0 -89
- ads/opctl/operator/lowcode/forecast/cmd.py +0 -40
- ads/opctl/operator/lowcode/forecast/const.py +0 -92
- ads/opctl/operator/lowcode/forecast/environment.yaml +0 -20
- ads/opctl/operator/lowcode/forecast/errors.py +0 -26
- ads/opctl/operator/lowcode/forecast/model/__init__.py +0 -5
- ads/opctl/operator/lowcode/forecast/model/arima.py +0 -279
- ads/opctl/operator/lowcode/forecast/model/automlx.py +0 -542
- ads/opctl/operator/lowcode/forecast/model/autots.py +0 -312
- ads/opctl/operator/lowcode/forecast/model/base_model.py +0 -863
- ads/opctl/operator/lowcode/forecast/model/factory.py +0 -106
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +0 -492
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +0 -243
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +0 -486
- ads/opctl/operator/lowcode/forecast/model/prophet.py +0 -445
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +0 -244
- ads/opctl/operator/lowcode/forecast/operator_config.py +0 -234
- ads/opctl/operator/lowcode/forecast/schema.yaml +0 -506
- ads/opctl/operator/lowcode/forecast/utils.py +0 -413
- ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +0 -7
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +0 -285
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +0 -246
- ads/opctl/operator/lowcode/pii/MLoperator +0 -17
- ads/opctl/operator/lowcode/pii/README.md +0 -208
- ads/opctl/operator/lowcode/pii/__init__.py +0 -5
- ads/opctl/operator/lowcode/pii/__main__.py +0 -78
- ads/opctl/operator/lowcode/pii/cmd.py +0 -39
- ads/opctl/operator/lowcode/pii/constant.py +0 -84
- ads/opctl/operator/lowcode/pii/environment.yaml +0 -17
- ads/opctl/operator/lowcode/pii/errors.py +0 -27
- ads/opctl/operator/lowcode/pii/model/__init__.py +0 -5
- ads/opctl/operator/lowcode/pii/model/factory.py +0 -82
- ads/opctl/operator/lowcode/pii/model/guardrails.py +0 -167
- ads/opctl/operator/lowcode/pii/model/pii.py +0 -145
- ads/opctl/operator/lowcode/pii/model/processor/__init__.py +0 -34
- ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +0 -34
- ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +0 -35
- ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +0 -225
- ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +0 -73
- ads/opctl/operator/lowcode/pii/model/processor/remover.py +0 -26
- ads/opctl/operator/lowcode/pii/model/report.py +0 -487
- ads/opctl/operator/lowcode/pii/operator_config.py +0 -95
- ads/opctl/operator/lowcode/pii/schema.yaml +0 -108
- ads/opctl/operator/lowcode/pii/utils.py +0 -43
- ads/opctl/operator/lowcode/recommender/MLoperator +0 -16
- ads/opctl/operator/lowcode/recommender/README.md +0 -206
- ads/opctl/operator/lowcode/recommender/__init__.py +0 -5
- ads/opctl/operator/lowcode/recommender/__main__.py +0 -82
- ads/opctl/operator/lowcode/recommender/cmd.py +0 -33
- ads/opctl/operator/lowcode/recommender/constant.py +0 -30
- ads/opctl/operator/lowcode/recommender/environment.yaml +0 -11
- ads/opctl/operator/lowcode/recommender/model/base_model.py +0 -212
- ads/opctl/operator/lowcode/recommender/model/factory.py +0 -56
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +0 -25
- ads/opctl/operator/lowcode/recommender/model/svd.py +0 -106
- ads/opctl/operator/lowcode/recommender/operator_config.py +0 -81
- ads/opctl/operator/lowcode/recommender/schema.yaml +0 -265
- ads/opctl/operator/lowcode/recommender/utils.py +0 -13
- ads/opctl/operator/runtime/__init__.py +0 -5
- ads/opctl/operator/runtime/const.py +0 -17
- ads/opctl/operator/runtime/container_runtime_schema.yaml +0 -50
- ads/opctl/operator/runtime/marketplace_runtime.py +0 -50
- ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +0 -21
- ads/opctl/operator/runtime/python_runtime_schema.yaml +0 -21
- ads/opctl/operator/runtime/runtime.py +0 -115
- ads/opctl/schema.yaml.yml +0 -36
- ads/opctl/script.py +0 -40
- ads/opctl/spark/__init__.py +0 -5
- ads/opctl/spark/cli.py +0 -43
- ads/opctl/spark/cmds.py +0 -147
- ads/opctl/templates/diagnostic_report_template.jinja2 +0 -102
- ads/opctl/utils.py +0 -344
- ads/oracledb/__init__.py +0 -5
- ads/oracledb/oracle_db.py +0 -346
- ads/pipeline/__init__.py +0 -39
- ads/pipeline/ads_pipeline.py +0 -2279
- ads/pipeline/ads_pipeline_run.py +0 -772
- ads/pipeline/ads_pipeline_step.py +0 -605
- ads/pipeline/builders/__init__.py +0 -5
- ads/pipeline/builders/infrastructure/__init__.py +0 -5
- ads/pipeline/builders/infrastructure/custom_script.py +0 -32
- ads/pipeline/cli.py +0 -119
- ads/pipeline/extension.py +0 -291
- ads/pipeline/schema/__init__.py +0 -5
- ads/pipeline/schema/cs_step_schema.json +0 -35
- ads/pipeline/schema/ml_step_schema.json +0 -31
- ads/pipeline/schema/pipeline_schema.json +0 -71
- ads/pipeline/visualizer/__init__.py +0 -5
- ads/pipeline/visualizer/base.py +0 -570
- ads/pipeline/visualizer/graph_renderer.py +0 -272
- ads/pipeline/visualizer/text_renderer.py +0 -84
- ads/secrets/__init__.py +0 -11
- ads/secrets/adb.py +0 -386
- ads/secrets/auth_token.py +0 -86
- ads/secrets/big_data_service.py +0 -365
- ads/secrets/mysqldb.py +0 -149
- ads/secrets/oracledb.py +0 -160
- ads/secrets/secrets.py +0 -407
- ads/telemetry/__init__.py +0 -7
- ads/telemetry/base.py +0 -69
- ads/telemetry/client.py +0 -125
- ads/telemetry/telemetry.py +0 -257
- ads/templates/dataflow_pyspark.jinja2 +0 -13
- ads/templates/dataflow_sparksql.jinja2 +0 -22
- ads/templates/func.jinja2 +0 -20
- ads/templates/schemas/openapi.json +0 -1740
- ads/templates/score-pkl.jinja2 +0 -173
- ads/templates/score.jinja2 +0 -322
- ads/templates/score_embedding_onnx.jinja2 +0 -202
- ads/templates/score_generic.jinja2 +0 -165
- ads/templates/score_huggingface_pipeline.jinja2 +0 -217
- ads/templates/score_lightgbm.jinja2 +0 -185
- ads/templates/score_onnx.jinja2 +0 -407
- ads/templates/score_onnx_new.jinja2 +0 -473
- ads/templates/score_oracle_automl.jinja2 +0 -185
- ads/templates/score_pyspark.jinja2 +0 -154
- ads/templates/score_pytorch.jinja2 +0 -219
- ads/templates/score_scikit-learn.jinja2 +0 -184
- ads/templates/score_tensorflow.jinja2 +0 -184
- ads/templates/score_xgboost.jinja2 +0 -178
- ads/text_dataset/__init__.py +0 -5
- ads/text_dataset/backends.py +0 -211
- ads/text_dataset/dataset.py +0 -445
- ads/text_dataset/extractor.py +0 -207
- ads/text_dataset/options.py +0 -53
- ads/text_dataset/udfs.py +0 -22
- ads/text_dataset/utils.py +0 -49
- ads/type_discovery/__init__.py +0 -9
- ads/type_discovery/abstract_detector.py +0 -21
- ads/type_discovery/constant_detector.py +0 -41
- ads/type_discovery/continuous_detector.py +0 -54
- ads/type_discovery/credit_card_detector.py +0 -99
- ads/type_discovery/datetime_detector.py +0 -92
- ads/type_discovery/discrete_detector.py +0 -118
- ads/type_discovery/document_detector.py +0 -146
- ads/type_discovery/ip_detector.py +0 -68
- ads/type_discovery/latlon_detector.py +0 -90
- ads/type_discovery/phone_number_detector.py +0 -63
- ads/type_discovery/type_discovery_driver.py +0 -87
- ads/type_discovery/typed_feature.py +0 -594
- ads/type_discovery/unknown_detector.py +0 -41
- ads/type_discovery/zipcode_detector.py +0 -48
- ads/vault/__init__.py +0 -7
- ads/vault/vault.py +0 -237
- oracle_ads-2.13.7.dist-info/RECORD +0 -858
- {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/licenses/LICENSE.txt +0 -0
ads/catalog/model.py
DELETED
@@ -1,1576 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python
|
2
|
-
# -*- coding: utf-8; -*-
|
3
|
-
|
4
|
-
# Copyright (c) 2020, 2024 Oracle and/or its affiliates.
|
5
|
-
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
|
-
|
7
|
-
import warnings
|
8
|
-
|
9
|
-
warnings.warn(
|
10
|
-
(
|
11
|
-
"The `ads.catalog.model` is deprecated in `oracle-ads 2.6.9` and will be removed in `oracle-ads 3.0`. "
|
12
|
-
"Use framework specific Model utility class for saving and deploying model. "
|
13
|
-
"Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html"
|
14
|
-
),
|
15
|
-
DeprecationWarning,
|
16
|
-
stacklevel=2,
|
17
|
-
)
|
18
|
-
|
19
|
-
import json
|
20
|
-
import os
|
21
|
-
import shutil
|
22
|
-
import tempfile
|
23
|
-
import time
|
24
|
-
import uuid
|
25
|
-
from typing import Dict, Optional, Union
|
26
|
-
from zipfile import ZipFile
|
27
|
-
|
28
|
-
import pandas as pd
|
29
|
-
import yaml
|
30
|
-
from ads.catalog.summary import SummaryList
|
31
|
-
from ads.common import auth, logger, oci_client, utils
|
32
|
-
from ads.common.decorator.deprecate import deprecated
|
33
|
-
from ads.common.decorator.runtime_dependency import (
|
34
|
-
runtime_dependency,
|
35
|
-
OptionalDependency,
|
36
|
-
)
|
37
|
-
from ads.common.model_artifact import ConflictStrategy, ModelArtifact
|
38
|
-
from ads.model.model_metadata import (
|
39
|
-
METADATA_SIZE_LIMIT,
|
40
|
-
MetadataSizeTooLarge,
|
41
|
-
ModelCustomMetadata,
|
42
|
-
ModelTaxonomyMetadata,
|
43
|
-
)
|
44
|
-
from ads.common.object_storage_details import ObjectStorageDetails
|
45
|
-
from ads.common.oci_resource import SEARCH_TYPE, OCIResource
|
46
|
-
from ads.config import (
|
47
|
-
NB_SESSION_COMPARTMENT_OCID,
|
48
|
-
OCI_ODSC_SERVICE_ENDPOINT,
|
49
|
-
OCI_REGION_METADATA,
|
50
|
-
PROJECT_OCID,
|
51
|
-
)
|
52
|
-
from ads.dataset.progress import TqdmProgressBar
|
53
|
-
from ads.feature_engineering.schema import Schema
|
54
|
-
from ads.model.model_version_set import ModelVersionSet, _extract_model_version_set_id
|
55
|
-
from ads.model.deployment.model_deployer import ModelDeployer
|
56
|
-
from oci.data_science.data_science_client import DataScienceClient
|
57
|
-
from oci.data_science.models import (
|
58
|
-
ArtifactExportDetailsObjectStorage,
|
59
|
-
ArtifactImportDetailsObjectStorage,
|
60
|
-
CreateModelDetails,
|
61
|
-
ExportModelArtifactDetails,
|
62
|
-
ImportModelArtifactDetails,
|
63
|
-
)
|
64
|
-
from oci.data_science.models import Model as OCIModel
|
65
|
-
from oci.data_science.models import ModelSummary, WorkRequest
|
66
|
-
from oci.data_science.models.model_provenance import ModelProvenance
|
67
|
-
from oci.data_science.models.update_model_details import UpdateModelDetails
|
68
|
-
from oci.exceptions import ServiceError
|
69
|
-
from oci.identity import IdentityClient
|
70
|
-
|
71
|
-
_UPDATE_MODEL_DETAILS_ATTRIBUTES = [
|
72
|
-
"display_name",
|
73
|
-
"description",
|
74
|
-
"freeform_tags",
|
75
|
-
"defined_tags",
|
76
|
-
"model_version_set_id",
|
77
|
-
"version_label",
|
78
|
-
]
|
79
|
-
_MODEL_PROVENANCE_ATTRIBUTES = ModelProvenance().swagger_types.keys()
|
80
|
-
_ETAG_KEY = "ETag"
|
81
|
-
_MAX_ARTIFACT_SIZE_IN_BYTES = 2147483648 # 2GB
|
82
|
-
_WORK_REQUEST_INTERVAL_IN_SEC = 3
|
83
|
-
|
84
|
-
|
85
|
-
class ModelWithActiveDeploymentError(Exception): # pragma: no cover
|
86
|
-
pass
|
87
|
-
|
88
|
-
|
89
|
-
class ModelArtifactSizeError(Exception): # pragma: no cover
|
90
|
-
def __init__(self, max_artifact_size: str):
|
91
|
-
super().__init__(
|
92
|
-
f"The model artifacts size is greater than `{max_artifact_size}`. "
|
93
|
-
"The `bucket_uri` needs to be specified to "
|
94
|
-
"copy artifacts to the object storage bucket. "
|
95
|
-
"Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`"
|
96
|
-
)
|
97
|
-
|
98
|
-
|
99
|
-
def _get_etag(response) -> str:
|
100
|
-
"""Gets etag from the response."""
|
101
|
-
if _ETAG_KEY in response.headers:
|
102
|
-
return response.headers[_ETAG_KEY].split("--")[0]
|
103
|
-
return None
|
104
|
-
|
105
|
-
|
106
|
-
class ModelSummaryList(SummaryList):
|
107
|
-
"""Model Summary List which represents a list of Model Object.
|
108
|
-
|
109
|
-
Methods
|
110
|
-
-------
|
111
|
-
sort_by(self, columns, reverse=False)
|
112
|
-
Performs a multi-key sort on a particular set of columns and returns the sorted ModelSummaryList.
|
113
|
-
Results are listed in a descending order by default.
|
114
|
-
filter(self, selection, instance=None)
|
115
|
-
Filters the model list according to a lambda filter function, or list comprehension.
|
116
|
-
"""
|
117
|
-
|
118
|
-
@deprecated(
|
119
|
-
"2.6.6",
|
120
|
-
details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
|
121
|
-
)
|
122
|
-
def __init__(
|
123
|
-
self,
|
124
|
-
model_catalog,
|
125
|
-
model_list,
|
126
|
-
response=None,
|
127
|
-
datetime_format=utils.date_format,
|
128
|
-
):
|
129
|
-
super(ModelSummaryList, self).__init__(
|
130
|
-
model_list, datetime_format=datetime_format
|
131
|
-
)
|
132
|
-
self.mc = model_catalog
|
133
|
-
self.response = response
|
134
|
-
|
135
|
-
def __add__(self, rhs):
|
136
|
-
return ModelSummaryList(
|
137
|
-
self.mc, list.__add__(self, rhs), datetime_format=self.datetime_format
|
138
|
-
)
|
139
|
-
|
140
|
-
def __getitem__(self, item):
|
141
|
-
return self.mc.get_model(super(ModelSummaryList, self).__getitem__(item).id)
|
142
|
-
|
143
|
-
def sort_by(self, columns, reverse=False):
|
144
|
-
"""
|
145
|
-
Performs a multi-key sort on a particular set of columns and returns the sorted ModelSummaryList.
|
146
|
-
Results are listed in a descending order by default.
|
147
|
-
|
148
|
-
Parameters
|
149
|
-
----------
|
150
|
-
columns: List of string
|
151
|
-
A list of columns which are provided to sort on
|
152
|
-
reverse: Boolean (defaults to false)
|
153
|
-
If you'd like to reverse the results (for example, to get ascending instead of descending results)
|
154
|
-
|
155
|
-
Returns
|
156
|
-
-------
|
157
|
-
ModelSummaryList: A sorted ModelSummaryList
|
158
|
-
"""
|
159
|
-
return ModelSummaryList(
|
160
|
-
self.mc,
|
161
|
-
self._sort_by(columns, reverse=reverse),
|
162
|
-
datetime_format=self.datetime_format,
|
163
|
-
)
|
164
|
-
|
165
|
-
def filter(self, selection, instance=None):
|
166
|
-
"""
|
167
|
-
Filters the model list according to a lambda filter function, or list comprehension.
|
168
|
-
|
169
|
-
Parameters
|
170
|
-
----------
|
171
|
-
selection: lambda function filtering model instances, or a list-comprehension
|
172
|
-
function of list filtering projects
|
173
|
-
instance: list, optional
|
174
|
-
list to filter, optional, defaults to self
|
175
|
-
|
176
|
-
Returns
|
177
|
-
-------
|
178
|
-
ModelSummaryList: A filtered ModelSummaryList
|
179
|
-
"""
|
180
|
-
instance = instance if instance is not None else self
|
181
|
-
|
182
|
-
if callable(selection):
|
183
|
-
res = list(filter(selection, instance))
|
184
|
-
# lambda filtering
|
185
|
-
if len(res) == 0:
|
186
|
-
print("No models found")
|
187
|
-
return
|
188
|
-
return ModelSummaryList(self.mc, res, datetime_format=self.datetime_format)
|
189
|
-
elif isinstance(selection, list):
|
190
|
-
# list comprehension
|
191
|
-
if len(selection) == 0:
|
192
|
-
print("No models found")
|
193
|
-
return
|
194
|
-
return ModelSummaryList(
|
195
|
-
self.mc, selection, datetime_format=self.datetime_format
|
196
|
-
)
|
197
|
-
else:
|
198
|
-
raise ValueError(
|
199
|
-
"Filter selection must be a function or a ProjectSummaryList"
|
200
|
-
)
|
201
|
-
|
202
|
-
|
203
|
-
class Model:
|
204
|
-
"""Class that represents the ADS implementation of model catalog item.
|
205
|
-
Converts the metadata and schema from OCI implememtation to ADS implementation.
|
206
|
-
|
207
|
-
Methods
|
208
|
-
-------
|
209
|
-
to_dataframe
|
210
|
-
Converts model to dataframe format.
|
211
|
-
show_in_notebook
|
212
|
-
Shows model in the notebook in dataframe or YAML representation.
|
213
|
-
activate
|
214
|
-
Activates model.
|
215
|
-
deactivate
|
216
|
-
Deactivates model.
|
217
|
-
commit
|
218
|
-
Commits the changes made to the model.
|
219
|
-
rollback
|
220
|
-
Rollbacks the changes made to the model.
|
221
|
-
load_model
|
222
|
-
Loads the model from the model catalog based on model ID.
|
223
|
-
"""
|
224
|
-
|
225
|
-
_FIELDS_TO_DECORATE = [
|
226
|
-
"schema_input",
|
227
|
-
"schema_output",
|
228
|
-
"metadata_custom",
|
229
|
-
"metadata_taxonomy",
|
230
|
-
]
|
231
|
-
|
232
|
-
_NEW_ATTRIBUTES = [
|
233
|
-
"input_schema",
|
234
|
-
"output_schema",
|
235
|
-
"custom_metadata_list",
|
236
|
-
"defined_metadata_list",
|
237
|
-
]
|
238
|
-
|
239
|
-
@deprecated(
|
240
|
-
"2.6.6",
|
241
|
-
details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
|
242
|
-
)
|
243
|
-
def __init__(
|
244
|
-
self,
|
245
|
-
model: OCIModel,
|
246
|
-
model_etag: str,
|
247
|
-
provenance_metadata: ModelProvenance,
|
248
|
-
provenance_etag: str,
|
249
|
-
ds_client: DataScienceClient,
|
250
|
-
identity_client: IdentityClient,
|
251
|
-
) -> None:
|
252
|
-
"""Initializes the Model.
|
253
|
-
|
254
|
-
Parameters
|
255
|
-
----------
|
256
|
-
model: OCIModel
|
257
|
-
The OCI model object.
|
258
|
-
model_etag: str
|
259
|
-
The model ETag.
|
260
|
-
provenance_metadata: ModelProvenance
|
261
|
-
The model provenance metadata.
|
262
|
-
provenance_etag: str
|
263
|
-
The model provenance metadata ETag.
|
264
|
-
ds_client: DataScienceClient
|
265
|
-
The Oracle DataScience client.
|
266
|
-
identity_client: IdentityClient
|
267
|
-
The Orcale Identity Service Client.
|
268
|
-
"""
|
269
|
-
self.ds_client = ds_client
|
270
|
-
self.identity_client = identity_client
|
271
|
-
self.user_name = ""
|
272
|
-
self._etag = model_etag
|
273
|
-
self._provenance_metadata_etag = provenance_etag
|
274
|
-
self.provenance_metadata = provenance_metadata
|
275
|
-
self._extract_oci_model(model)
|
276
|
-
self._extract_user_name(model)
|
277
|
-
|
278
|
-
def _extract_oci_model(self, model: OCIModel) -> None:
|
279
|
-
"""Extracts the model information from OCI model."""
|
280
|
-
for key in model.swagger_types.keys():
|
281
|
-
if key not in self._NEW_ATTRIBUTES:
|
282
|
-
val = getattr(model, key)
|
283
|
-
setattr(self, key, val)
|
284
|
-
self.schema_input = self._extract_schema("input_schema", model)
|
285
|
-
self.schema_output = self._extract_schema("output_schema", model)
|
286
|
-
self.metadata_custom = self._extract_metadata_custom(model)
|
287
|
-
self.metadata_taxonomy = self._extract_metadata_taxonomy(model)
|
288
|
-
self.swagger_types = model.swagger_types
|
289
|
-
self.lifecycle_state = model.lifecycle_state
|
290
|
-
|
291
|
-
def _validate_metadata(self):
|
292
|
-
self.metadata_custom.validate()
|
293
|
-
self.metadata_taxonomy.validate()
|
294
|
-
total_size = self.metadata_custom.size() + self.metadata_taxonomy.size()
|
295
|
-
if total_size > METADATA_SIZE_LIMIT:
|
296
|
-
raise MetadataSizeTooLarge(total_size)
|
297
|
-
return True
|
298
|
-
|
299
|
-
def _extract_user_name(self, model: OCIModel) -> None:
|
300
|
-
try:
|
301
|
-
user = self.identity_client.get_user(model.created_by)
|
302
|
-
self.user_name = user.data.name
|
303
|
-
except:
|
304
|
-
pass
|
305
|
-
|
306
|
-
@staticmethod
|
307
|
-
def _extract_schema(key, model):
|
308
|
-
"""Extracts the input and output schema."""
|
309
|
-
schema = Schema()
|
310
|
-
if hasattr(model, key):
|
311
|
-
try:
|
312
|
-
schema = (
|
313
|
-
Schema.from_dict(json.loads(getattr(model, key)))
|
314
|
-
if getattr(model, key)
|
315
|
-
else Schema()
|
316
|
-
)
|
317
|
-
except Exception as e:
|
318
|
-
logger.warning(str(e))
|
319
|
-
return schema
|
320
|
-
|
321
|
-
@staticmethod
|
322
|
-
def _extract_metadata_taxonomy(model):
|
323
|
-
"""Extracts the taxonomy metadata."""
|
324
|
-
metadata_taxonomy = ModelTaxonomyMetadata()
|
325
|
-
if hasattr(model, "defined_metadata_list"):
|
326
|
-
try:
|
327
|
-
metadata_taxonomy = ModelTaxonomyMetadata._from_oci_metadata(
|
328
|
-
model.defined_metadata_list
|
329
|
-
)
|
330
|
-
except Exception as e:
|
331
|
-
logger.warning(str(e))
|
332
|
-
return metadata_taxonomy
|
333
|
-
|
334
|
-
@staticmethod
|
335
|
-
def _extract_metadata_custom(model):
|
336
|
-
"""Extracts the custom metadata."""
|
337
|
-
metadata_custom = ModelCustomMetadata()
|
338
|
-
if hasattr(model, "custom_metadata_list"):
|
339
|
-
try:
|
340
|
-
metadata_custom = ModelCustomMetadata._from_oci_metadata(
|
341
|
-
model.custom_metadata_list
|
342
|
-
)
|
343
|
-
except Exception as e:
|
344
|
-
logger.warning(str(e))
|
345
|
-
return metadata_custom
|
346
|
-
|
347
|
-
def _to_dict(self):
|
348
|
-
"""Converts the model attributes to dictionary format."""
|
349
|
-
attributes = {}
|
350
|
-
for key in _UPDATE_MODEL_DETAILS_ATTRIBUTES:
|
351
|
-
if hasattr(self, key):
|
352
|
-
attributes[key] = getattr(self, key)
|
353
|
-
|
354
|
-
if self.provenance_metadata is not None:
|
355
|
-
attributes.update(
|
356
|
-
{
|
357
|
-
key: getattr(self.provenance_metadata, key)
|
358
|
-
for key in _MODEL_PROVENANCE_ATTRIBUTES
|
359
|
-
}
|
360
|
-
)
|
361
|
-
for field in self._FIELDS_TO_DECORATE:
|
362
|
-
attributes[field] = getattr(self, field).to_dict()
|
363
|
-
return attributes
|
364
|
-
|
365
|
-
def _to_yaml(self):
|
366
|
-
"""Converts the model attributes to yaml format."""
|
367
|
-
attributes = self._to_dict()
|
368
|
-
return yaml.safe_dump(attributes)
|
369
|
-
|
370
|
-
def to_dataframe(self) -> pd.DataFrame:
|
371
|
-
"""
|
372
|
-
Converts the model to dataframe format.
|
373
|
-
|
374
|
-
Returns
|
375
|
-
-------
|
376
|
-
panadas.DataFrame
|
377
|
-
Pandas dataframe.
|
378
|
-
"""
|
379
|
-
attributes = self._to_dict()
|
380
|
-
df = pd.DataFrame.from_dict(attributes, orient="index", columns=[""]).dropna()
|
381
|
-
return df
|
382
|
-
|
383
|
-
@runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
|
384
|
-
def show_in_notebook(self, display_format: str = "dataframe") -> None:
|
385
|
-
"""Shows model in dataframe or yaml format.
|
386
|
-
Supported formats: `dataframe` and `yaml`. Defaults to dataframe format.
|
387
|
-
|
388
|
-
Returns
|
389
|
-
-------
|
390
|
-
None
|
391
|
-
Nothing.
|
392
|
-
"""
|
393
|
-
if display_format == "dataframe":
|
394
|
-
from IPython.core.display import display
|
395
|
-
|
396
|
-
display(self.to_dataframe())
|
397
|
-
elif display_format == "yaml":
|
398
|
-
print(self._to_yaml())
|
399
|
-
else:
|
400
|
-
NotImplementedError(
|
401
|
-
"`display_format` is not supported. Choose 'dataframe' or 'yaml'"
|
402
|
-
)
|
403
|
-
|
404
|
-
def _repr_html_(self):
|
405
|
-
"""Shows model in dataframe format."""
|
406
|
-
return (
|
407
|
-
self.to_dataframe().style.set_properties(**{"margin-left": "0px"}).to_html()
|
408
|
-
)
|
409
|
-
|
410
|
-
def __repr__(self):
|
411
|
-
"""Shows model in dataframe format."""
|
412
|
-
return (
|
413
|
-
self.to_dataframe().style.set_properties(**{"margin-left": "0px"}).to_html()
|
414
|
-
)
|
415
|
-
|
416
|
-
def activate(self) -> None:
|
417
|
-
"""Activates model.
|
418
|
-
|
419
|
-
Returns
|
420
|
-
-------
|
421
|
-
None
|
422
|
-
Nothing.
|
423
|
-
"""
|
424
|
-
self.lifecycle_state = OCIModel.LIFECYCLE_STATE_ACTIVE
|
425
|
-
|
426
|
-
def deactivate(self) -> None:
|
427
|
-
"""Deactivates model.
|
428
|
-
|
429
|
-
Returns
|
430
|
-
-------
|
431
|
-
None
|
432
|
-
Nothing.
|
433
|
-
"""
|
434
|
-
self.lifecycle_state = OCIModel.LIFECYCLE_STATE_INACTIVE
|
435
|
-
|
436
|
-
def commit(self, force: bool = True) -> None:
|
437
|
-
"""Commits model changes.
|
438
|
-
|
439
|
-
Parameters
|
440
|
-
----------
|
441
|
-
force: bool
|
442
|
-
If True, any remote changes on this model would be lost.
|
443
|
-
|
444
|
-
Returns
|
445
|
-
-------
|
446
|
-
None
|
447
|
-
Nothing.
|
448
|
-
"""
|
449
|
-
self._validate_metadata()
|
450
|
-
attributes = {
|
451
|
-
key: getattr(self, key) for key in _UPDATE_MODEL_DETAILS_ATTRIBUTES
|
452
|
-
}
|
453
|
-
|
454
|
-
if hasattr(self, "metadata_custom"):
|
455
|
-
attributes["custom_metadata_list"] = self.metadata_custom._to_oci_metadata()
|
456
|
-
if hasattr(self, "metadata_taxonomy"):
|
457
|
-
attributes[
|
458
|
-
"defined_metadata_list"
|
459
|
-
] = self.metadata_taxonomy._to_oci_metadata()
|
460
|
-
|
461
|
-
update_model_details = UpdateModelDetails(**attributes)
|
462
|
-
# freeform_tags=self._model.freeform_tags, defined_tags=self._model.defined_tags)
|
463
|
-
|
464
|
-
# update model
|
465
|
-
# https://docs.oracle.com/en-us/iaas/Content/API/Concepts/usingapi.htm#eleven
|
466
|
-
# The API supports etags for the purposes of optimistic concurrency control.
|
467
|
-
# The GET and POST calls return an etag response header with a value you should store.
|
468
|
-
# When you later want to update or delete the resource, set the if-match header to the ETag
|
469
|
-
# you received for the resource. The resource will then be updated or deleted
|
470
|
-
# only if the ETag you provide matches the current value of that resource's ETag.
|
471
|
-
kwargs = {}
|
472
|
-
if not force:
|
473
|
-
kwargs["if_match"] = self._etag
|
474
|
-
|
475
|
-
self.ds_client.update_model(
|
476
|
-
self.id, update_model_details=update_model_details, **kwargs
|
477
|
-
)
|
478
|
-
# store the lifecycle status, as updating the model will delete info not included in "update_model_details"
|
479
|
-
lifecycle_status = self.lifecycle_state
|
480
|
-
self.__dict__.update(self._load_model().__dict__)
|
481
|
-
self.lifecycle_state = lifecycle_status
|
482
|
-
|
483
|
-
# update model state
|
484
|
-
if not force:
|
485
|
-
kwargs["if_match"] = self._etag
|
486
|
-
if self.lifecycle_state == OCIModel.LIFECYCLE_STATE_ACTIVE:
|
487
|
-
self.ds_client.activate_model(self.id, **kwargs)
|
488
|
-
elif self.lifecycle_state == OCIModel.LIFECYCLE_STATE_INACTIVE:
|
489
|
-
self.ds_client.deactivate_model(self.id, **kwargs)
|
490
|
-
self.__dict__.update(self._load_model().__dict__)
|
491
|
-
|
492
|
-
# update model provenance
|
493
|
-
if self.provenance_metadata != ModelProvenance():
|
494
|
-
if not force:
|
495
|
-
kwargs["if_match"] = self._provenance_metadata_etag
|
496
|
-
response = self.ds_client.update_model_provenance(
|
497
|
-
self.id, self.provenance_metadata, **kwargs
|
498
|
-
)
|
499
|
-
# get model etag again, as updating model provenance changes it
|
500
|
-
self.__dict__.update(self._load_model().__dict__)
|
501
|
-
|
502
|
-
@staticmethod
|
503
|
-
def _get_provenance_metadata(ds_client: DataScienceClient, model_id: str):
|
504
|
-
"""Gets provenance information for specified model."""
|
505
|
-
try:
|
506
|
-
provenance_response = ds_client.get_model_provenance(model_id)
|
507
|
-
except ServiceError as e:
|
508
|
-
if e.status == 404:
|
509
|
-
try:
|
510
|
-
provenance_response = ds_client.create_model_provenance(
|
511
|
-
model_id, ModelProvenance()
|
512
|
-
)
|
513
|
-
except ServiceError as e2:
|
514
|
-
raise e2
|
515
|
-
elif e.status == 409:
|
516
|
-
print("The model has been deleted.")
|
517
|
-
raise e
|
518
|
-
else:
|
519
|
-
raise e
|
520
|
-
return provenance_response
|
521
|
-
|
522
|
-
@classmethod
|
523
|
-
def load_model(
|
524
|
-
cls,
|
525
|
-
ds_client: DataScienceClient,
|
526
|
-
identity_client: IdentityClient,
|
527
|
-
model_id: str,
|
528
|
-
) -> "Model":
|
529
|
-
"""Loads the model from the model catalog based on model ID.
|
530
|
-
|
531
|
-
Parameters
|
532
|
-
----------
|
533
|
-
ds_client: DataScienceClient
|
534
|
-
The Oracle DataScience client.
|
535
|
-
identity_client: IdentityClient
|
536
|
-
The Orcale Identity Service Client.
|
537
|
-
model_id: str
|
538
|
-
The model ID.
|
539
|
-
|
540
|
-
Returns
|
541
|
-
-------
|
542
|
-
Model
|
543
|
-
The ADS model catalog item.
|
544
|
-
|
545
|
-
Raises
|
546
|
-
------
|
547
|
-
ServiceError: If error occures while getting model from server.
|
548
|
-
KeyError: If model not found.
|
549
|
-
ValueError: If error occures while getting model provenance mettadata from server.
|
550
|
-
"""
|
551
|
-
|
552
|
-
try:
|
553
|
-
model_response = ds_client.get_model(model_id)
|
554
|
-
except ServiceError as e:
|
555
|
-
if e.status == 404:
|
556
|
-
raise KeyError(e.message) from e
|
557
|
-
raise e
|
558
|
-
|
559
|
-
try:
|
560
|
-
provenance_response = cls._get_provenance_metadata(ds_client, model_id)
|
561
|
-
except Exception as e:
|
562
|
-
raise ValueError(
|
563
|
-
f"Unable to fetch model provenance metadata for model {model_id}"
|
564
|
-
)
|
565
|
-
|
566
|
-
return cls(
|
567
|
-
model_response.data,
|
568
|
-
_get_etag(model_response),
|
569
|
-
provenance_response.data,
|
570
|
-
_get_etag(provenance_response),
|
571
|
-
ds_client,
|
572
|
-
identity_client,
|
573
|
-
)
|
574
|
-
|
575
|
-
def _load_model(self):
|
576
|
-
"""Loads the model from model catalog."""
|
577
|
-
return self.load_model(self.ds_client, self.identity_client, self.id)
|
578
|
-
|
579
|
-
def rollback(self) -> None:
|
580
|
-
"""Rollbacks the changes made to the model.
|
581
|
-
|
582
|
-
Returns
|
583
|
-
-------
|
584
|
-
None
|
585
|
-
Nothing.
|
586
|
-
"""
|
587
|
-
self.__dict__.update(self._load_model().__dict__)
|
588
|
-
|
589
|
-
|
590
|
-
class ModelCatalog:
|
591
|
-
"""
|
592
|
-
Allows to list, load, update, download, upload and delete models from model catalog.
|
593
|
-
|
594
|
-
Methods
|
595
|
-
-------
|
596
|
-
get_model(self, model_id)
|
597
|
-
Loads the model from the model catalog based on model_id.
|
598
|
-
list_models(self, project_id=None, include_deleted=False, datetime_format=utils.date_format, **kwargs)
|
599
|
-
Lists all models in a given compartment, or in the current project if project_id is specified.
|
600
|
-
list_model_deployment(self, model_id, config=None, tenant_id=None, limit=500, page=None, **kwargs)
|
601
|
-
Gets the list of model deployments by model Id across the compartments.
|
602
|
-
update_model(self, model_id, update_model_details=None, **kwargs)
|
603
|
-
Updates a model with given model_id, using the provided update data.
|
604
|
-
delete_model(self, model, **kwargs)
|
605
|
-
Deletes the model based on model_id.
|
606
|
-
download_model(self, model_id, target_dir, force_overwrite=False, install_libs=False, conflict_strategy=ConflictStrategy.IGNORE)
|
607
|
-
Downloads the model from model_dir to target_dir based on model_id.
|
608
|
-
upload_model(self, model_artifact, provenance_metadata=None, project_id=None, display_name=None, description=None)
|
609
|
-
Uploads the model artifact to cloud storage.
|
610
|
-
"""
|
611
|
-
|
612
|
-
@deprecated(
|
613
|
-
"2.6.6",
|
614
|
-
details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
|
615
|
-
)
|
616
|
-
def __init__(
|
617
|
-
self,
|
618
|
-
compartment_id: Optional[str] = None,
|
619
|
-
ds_client_auth: Optional[dict] = None,
|
620
|
-
identity_client_auth: Optional[dict] = None,
|
621
|
-
timeout: Optional[int] = None,
|
622
|
-
ds_client: Optional[DataScienceClient] = None,
|
623
|
-
identity_client: Optional[IdentityClient] = None,
|
624
|
-
):
|
625
|
-
"""Initializes model catalog instance.
|
626
|
-
|
627
|
-
Parameters
|
628
|
-
----------
|
629
|
-
compartment_id : (str, optional). Defaults to None.
|
630
|
-
Model compartment OCID. If `None`, the `config.NB_SESSION_COMPARTMENT_OCID` would be used.
|
631
|
-
ds_client_auth : (dict, optional). Defaults to None.
|
632
|
-
The default authetication is set using `ads.set_auth` API. If you need to override the
|
633
|
-
default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
|
634
|
-
authentication signer and kwargs required to instantiate DataScienceClient object.
|
635
|
-
identity_client_auth : (dict, optional). Defaults to None.
|
636
|
-
The default authetication is set using `ads.set_auth` API. If you need to override the
|
637
|
-
default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
|
638
|
-
authentication signer and kwargs required to instantiate IdentityClient object.
|
639
|
-
timeout: (int, optional). Defaults to 10 seconds.
|
640
|
-
The connection timeout in seconds for the client.
|
641
|
-
ds_client: DataScienceClient
|
642
|
-
The Oracle DataScience client.
|
643
|
-
identity_client: IdentityClient
|
644
|
-
The Orcale Identity Service Client.
|
645
|
-
|
646
|
-
Raises
|
647
|
-
------
|
648
|
-
ValueError
|
649
|
-
If compartment ID not specified.
|
650
|
-
TypeError
|
651
|
-
If timeout not an integer.
|
652
|
-
"""
|
653
|
-
self.compartment_id = (
|
654
|
-
NB_SESSION_COMPARTMENT_OCID if compartment_id is None else compartment_id
|
655
|
-
)
|
656
|
-
if self.compartment_id is None:
|
657
|
-
raise ValueError("compartment_id needs to be specified.")
|
658
|
-
|
659
|
-
if timeout and not isinstance(timeout, int):
|
660
|
-
raise TypeError("Timeout must be an integer.")
|
661
|
-
|
662
|
-
self.ds_client_auth = ds_client_auth
|
663
|
-
self.identity_client_auth = identity_client_auth
|
664
|
-
self.ds_client = ds_client
|
665
|
-
self.identity_client = identity_client
|
666
|
-
|
667
|
-
if not self.ds_client:
|
668
|
-
self.ds_client_auth = (
|
669
|
-
ds_client_auth
|
670
|
-
if ds_client_auth
|
671
|
-
else auth.default_signer(
|
672
|
-
{"service_endpoint": OCI_ODSC_SERVICE_ENDPOINT}
|
673
|
-
)
|
674
|
-
)
|
675
|
-
if timeout:
|
676
|
-
if not self.ds_client_auth.get("client_kwargs"):
|
677
|
-
self.ds_client_auth["client_kwargs"] = {}
|
678
|
-
self.ds_client_auth["client_kwargs"]["timeout"] = timeout
|
679
|
-
self.ds_client = oci_client.OCIClientFactory(
|
680
|
-
**self.ds_client_auth
|
681
|
-
).data_science
|
682
|
-
|
683
|
-
if not self.identity_client:
|
684
|
-
self.identity_client_auth = (
|
685
|
-
identity_client_auth
|
686
|
-
if identity_client_auth
|
687
|
-
else auth.default_signer(
|
688
|
-
{"service_endpoint": OCI_ODSC_SERVICE_ENDPOINT}
|
689
|
-
)
|
690
|
-
)
|
691
|
-
if timeout:
|
692
|
-
if not self.identity_client_auth.get("client_kwargs"):
|
693
|
-
self.identity_client_auth["client_kwargs"] = {}
|
694
|
-
self.identity_client_auth["client_kwargs"]["timeout"] = timeout
|
695
|
-
self.identity_client = oci_client.OCIClientFactory(
|
696
|
-
**self.identity_client_auth
|
697
|
-
).identity
|
698
|
-
|
699
|
-
self.short_id_index = {}
|
700
|
-
|
701
|
-
def __getitem__(self, model_id): # pragma: no cover
|
702
|
-
return self.get_model(model_id)
|
703
|
-
|
704
|
-
def __contains__(self, model_id): # pragma: no cover
|
705
|
-
try:
|
706
|
-
return self.get_model(model_id) is not None
|
707
|
-
except KeyError:
|
708
|
-
return False
|
709
|
-
except Exception:
|
710
|
-
raise
|
711
|
-
|
712
|
-
def __iter__(self): # pragma: no cover
|
713
|
-
return self.list_models().__iter__()
|
714
|
-
|
715
|
-
def __len__(self): # pragma: no cover
|
716
|
-
return len(self.list_models())
|
717
|
-
|
718
|
-
def get_model(self, model_id):
|
719
|
-
"""
|
720
|
-
Loads the model from the model catalog based on model_id.
|
721
|
-
|
722
|
-
Parameters
|
723
|
-
----------
|
724
|
-
model_id: str, required
|
725
|
-
The model ID.
|
726
|
-
|
727
|
-
Returns
|
728
|
-
-------
|
729
|
-
ads.catalog.Model
|
730
|
-
The ads.catalog.Model with the matching ID.
|
731
|
-
"""
|
732
|
-
if not model_id.startswith("ocid"):
|
733
|
-
model_id = self.short_id_index[model_id]
|
734
|
-
self.id = model_id
|
735
|
-
return Model.load_model(self.ds_client, self.identity_client, model_id)
|
736
|
-
|
737
|
-
def list_models(
|
738
|
-
self,
|
739
|
-
project_id: str = None,
|
740
|
-
include_deleted: bool = False,
|
741
|
-
datetime_format: str = utils.date_format,
|
742
|
-
**kwargs,
|
743
|
-
):
|
744
|
-
"""
|
745
|
-
Lists all models in a given compartment, or in the current project if project_id is specified.
|
746
|
-
|
747
|
-
Parameters
|
748
|
-
----------
|
749
|
-
project_id: str
|
750
|
-
The project_id of model.
|
751
|
-
include_deleted: bool, optional, default=False
|
752
|
-
Whether to include deleted models in the returned list.
|
753
|
-
datetime_format: str, optional, default: '%Y-%m-%d %H:%M:%S'
|
754
|
-
Change format for date time fields.
|
755
|
-
|
756
|
-
Returns
|
757
|
-
-------
|
758
|
-
ModelSummaryList
|
759
|
-
A list of models.
|
760
|
-
"""
|
761
|
-
try:
|
762
|
-
list_models_response = self.ds_client.list_models(
|
763
|
-
self.compartment_id, project_id=project_id, **kwargs
|
764
|
-
)
|
765
|
-
if list_models_response.data is None or len(list_models_response.data) == 0:
|
766
|
-
print("No model found.")
|
767
|
-
return
|
768
|
-
except ServiceError as se:
|
769
|
-
if se.status == 404:
|
770
|
-
raise KeyError(se.message) from se
|
771
|
-
else:
|
772
|
-
raise
|
773
|
-
|
774
|
-
model_list_filtered = [
|
775
|
-
Model(
|
776
|
-
model=model,
|
777
|
-
model_etag=None,
|
778
|
-
provenance_metadata=None,
|
779
|
-
provenance_etag=None,
|
780
|
-
ds_client=self.ds_client,
|
781
|
-
identity_client=self.identity_client,
|
782
|
-
)
|
783
|
-
for model in list_models_response.data
|
784
|
-
if include_deleted
|
785
|
-
or model.lifecycle_state != ModelSummary.LIFECYCLE_STATE_DELETED
|
786
|
-
]
|
787
|
-
# handle empty list
|
788
|
-
if model_list_filtered is None or len(model_list_filtered) == 0:
|
789
|
-
print("No model found.")
|
790
|
-
return []
|
791
|
-
|
792
|
-
msl = ModelSummaryList(
|
793
|
-
self,
|
794
|
-
model_list_filtered,
|
795
|
-
list_models_response,
|
796
|
-
datetime_format=datetime_format,
|
797
|
-
)
|
798
|
-
self.short_id_index.update(msl.short_id_index)
|
799
|
-
return msl
|
800
|
-
|
801
|
-
def list_model_deployment(
|
802
|
-
self,
|
803
|
-
model_id: str,
|
804
|
-
config: dict = None,
|
805
|
-
tenant_id: str = None,
|
806
|
-
limit: int = 500,
|
807
|
-
page: str = None,
|
808
|
-
**kwargs,
|
809
|
-
):
|
810
|
-
"""
|
811
|
-
Gets the list of model deployments by model Id across the compartments.
|
812
|
-
|
813
|
-
Parameters
|
814
|
-
----------
|
815
|
-
model_id: str
|
816
|
-
The model ID.
|
817
|
-
config: dict (optional)
|
818
|
-
Configuration keys and values as per SDK and Tool Configuration.
|
819
|
-
The from_file() method can be used to load configuration from a file.
|
820
|
-
Alternatively, a dict can be passed. You can validate_config the dict
|
821
|
-
using validate_config(). Defaults to None.
|
822
|
-
tenant_id: str (optional)
|
823
|
-
The tenancy ID, which can be used to specify a different tenancy
|
824
|
-
(for cross-tenancy authorization) when searching for resources in
|
825
|
-
a different tenancy. Defaults to None.
|
826
|
-
limit: int (optional)
|
827
|
-
The maximum number of items to return. The value must be between
|
828
|
-
1 and 1000. Defaults to 500.
|
829
|
-
page: str (optional)
|
830
|
-
The page at which to start retrieving results.
|
831
|
-
|
832
|
-
Returns
|
833
|
-
-------
|
834
|
-
The list of model deployments.
|
835
|
-
"""
|
836
|
-
query = f"query datasciencemodeldeployment resources where ModelId='{model_id}'"
|
837
|
-
return OCIResource.search(
|
838
|
-
query,
|
839
|
-
type=SEARCH_TYPE.STRUCTURED,
|
840
|
-
config=config,
|
841
|
-
tenant_id=tenant_id,
|
842
|
-
limit=limit,
|
843
|
-
page=page,
|
844
|
-
**kwargs,
|
845
|
-
)
|
846
|
-
|
847
|
-
def update_model(self, model_id, update_model_details=None, **kwargs) -> Model:
|
848
|
-
"""
|
849
|
-
Updates a model with given model_id, using the provided update data.
|
850
|
-
|
851
|
-
Parameters
|
852
|
-
----------
|
853
|
-
model_id: str
|
854
|
-
The model ID.
|
855
|
-
update_model_details: UpdateModelDetails
|
856
|
-
Contains the update model details data to apply.
|
857
|
-
Mandatory unless kwargs are supplied.
|
858
|
-
kwargs: dict, optional
|
859
|
-
Update model details can be supplied instead as kwargs.
|
860
|
-
|
861
|
-
Returns
|
862
|
-
-------
|
863
|
-
Model
|
864
|
-
The ads.catalog.Model with the matching ID.
|
865
|
-
"""
|
866
|
-
if not model_id.startswith("ocid"):
|
867
|
-
model_id = self.short_id_index[model_id]
|
868
|
-
if update_model_details is None:
|
869
|
-
update_model_details = UpdateModelDetails(
|
870
|
-
**{
|
871
|
-
k: v
|
872
|
-
for k, v in kwargs.items()
|
873
|
-
if k in _UPDATE_MODEL_DETAILS_ATTRIBUTES
|
874
|
-
}
|
875
|
-
)
|
876
|
-
update_model_details.compartment_id = self.compartment_id
|
877
|
-
# filter kwargs removing used keys
|
878
|
-
kwargs = {
|
879
|
-
k: v
|
880
|
-
for k, v in kwargs.items()
|
881
|
-
if k not in _UPDATE_MODEL_DETAILS_ATTRIBUTES
|
882
|
-
}
|
883
|
-
update_model_response = self.ds_client.update_model(
|
884
|
-
model_id, update_model_details, **kwargs
|
885
|
-
)
|
886
|
-
provenance_response = Model._get_provenance_metadata(self.ds_client, model_id)
|
887
|
-
return Model(
|
888
|
-
model=update_model_response.data,
|
889
|
-
model_etag=_get_etag(update_model_response),
|
890
|
-
provenance_metadata=provenance_response.data,
|
891
|
-
provenance_etag=_get_etag(provenance_response),
|
892
|
-
ds_client=self.ds_client,
|
893
|
-
identity_client=self.identity_client,
|
894
|
-
)
|
895
|
-
|
896
|
-
def delete_model(self, model: Union[str, "ads.catalog.Model"], **kwargs) -> bool:
|
897
|
-
"""
|
898
|
-
Deletes the model from Model Catalog.
|
899
|
-
|
900
|
-
Parameters
|
901
|
-
----------
|
902
|
-
model: Union[str, "ads.catalog.Model"]
|
903
|
-
The OCID of the model to delete as a string, or a `ads.catalog.Model` instance.
|
904
|
-
|
905
|
-
kwargs:
|
906
|
-
delete_associated_model_deployment: (bool, optional). Defaults to `False`.
|
907
|
-
Whether associated model deployments need to be deletet or not.
|
908
|
-
|
909
|
-
Returns
|
910
|
-
-------
|
911
|
-
bool
|
912
|
-
`True` if the model was successfully deleted.
|
913
|
-
|
914
|
-
Raises
|
915
|
-
------
|
916
|
-
ModelWithActiveDeploymentError
|
917
|
-
If model has active model deployments ant inout attribute
|
918
|
-
`delete_associated_model_deployment` set to `False`.
|
919
|
-
"""
|
920
|
-
model_id = (
|
921
|
-
model.id
|
922
|
-
if isinstance(model, Model)
|
923
|
-
else self.short_id_index[model]
|
924
|
-
if not model.startswith("ocid")
|
925
|
-
else model
|
926
|
-
)
|
927
|
-
delete_associated_model_deployment = kwargs.pop(
|
928
|
-
"delete_associated_model_deployment", None
|
929
|
-
)
|
930
|
-
active_deployments = tuple(
|
931
|
-
item
|
932
|
-
for item in self.list_model_deployment(model_id)
|
933
|
-
if item.lifecycle_state == "ACTIVE"
|
934
|
-
)
|
935
|
-
|
936
|
-
if len(active_deployments) > 0:
|
937
|
-
if not delete_associated_model_deployment:
|
938
|
-
raise ModelWithActiveDeploymentError(
|
939
|
-
f"The model `{model_id}` has active model deployments: "
|
940
|
-
f"{[item.identifier for item in active_deployments]}. "
|
941
|
-
"Delete associated model deployments before deleting the model or "
|
942
|
-
"set the `delete_associated_model_deployment` attribute to `True`."
|
943
|
-
)
|
944
|
-
|
945
|
-
logger.info(
|
946
|
-
f"Deleting model deployments associated with the model `{model_id}`."
|
947
|
-
)
|
948
|
-
for oci_model_deployment in active_deployments:
|
949
|
-
(
|
950
|
-
ModelDeployer(config=self.ds_client_auth)
|
951
|
-
.get_model_deployment(oci_model_deployment.identifier)
|
952
|
-
.delete(wait_for_completion=True)
|
953
|
-
)
|
954
|
-
|
955
|
-
logger.info(f"Deleting model `{model_id}`.")
|
956
|
-
self.ds_client.delete_model(model_id, **kwargs)
|
957
|
-
return True
|
958
|
-
|
959
|
-
def _download_artifact(
|
960
|
-
self,
|
961
|
-
model_id: str,
|
962
|
-
target_dir: str,
|
963
|
-
force_overwrite: Optional[bool] = False,
|
964
|
-
bucket_uri: Optional[str] = None,
|
965
|
-
remove_existing_artifact: Optional[bool] = True,
|
966
|
-
) -> None:
|
967
|
-
"""
|
968
|
-
Downloads the model artifacts from model catalog to target_dir based on `model_id`.
|
969
|
-
|
970
|
-
Parameters
|
971
|
-
----------
|
972
|
-
model_id: str
|
973
|
-
The OCID of the model to download.
|
974
|
-
target_dir: str
|
975
|
-
The target location of model artifacts.
|
976
|
-
force_overwrite: (bool, optional). Defaults to `False`.
|
977
|
-
Overwrite target directory if exists.
|
978
|
-
bucket_uri: (str, optional). Defaults to None.
|
979
|
-
The OCI Object Storage URI where model artifacts will be copied to.
|
980
|
-
The `bucket_uri` is only necessary for downloading large artifacts with
|
981
|
-
size is greater than 2GB. Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`.
|
982
|
-
remove_existing_artifact: (bool, optional). Defaults to `True`.
|
983
|
-
Whether artifacts uploaded to object storage bucket need to be removed or not.
|
984
|
-
|
985
|
-
Raises
|
986
|
-
------
|
987
|
-
ValueError
|
988
|
-
If targeted directory does not exist.
|
989
|
-
KeyError
|
990
|
-
If model id not found.
|
991
|
-
|
992
|
-
Returns
|
993
|
-
-------
|
994
|
-
None
|
995
|
-
Nothing
|
996
|
-
"""
|
997
|
-
if os.path.exists(target_dir) and os.listdir(target_dir):
|
998
|
-
if not force_overwrite:
|
999
|
-
raise ValueError(
|
1000
|
-
f"The `{target_dir}` directory already exists. "
|
1001
|
-
"Set `force_overwrite` to `True` if you wish to overwrite."
|
1002
|
-
)
|
1003
|
-
shutil.rmtree(target_dir)
|
1004
|
-
|
1005
|
-
with utils.get_progress_bar(6) as progress:
|
1006
|
-
progress.update("Getting information about model artifacts")
|
1007
|
-
|
1008
|
-
# If the size of artifacts greater than 2GB, then artifacts
|
1009
|
-
# need to be imported to OS bucket at first.
|
1010
|
-
try:
|
1011
|
-
model_artifact_info = self.ds_client.head_model_artifact(
|
1012
|
-
model_id=model_id
|
1013
|
-
).headers
|
1014
|
-
except ServiceError as ex:
|
1015
|
-
if ex.status == 404:
|
1016
|
-
raise KeyError(f"The model `{model_id}` not found.") from ex
|
1017
|
-
raise
|
1018
|
-
|
1019
|
-
artifact_size = int(model_artifact_info.get("content-length"))
|
1020
|
-
|
1021
|
-
if bucket_uri or artifact_size > _MAX_ARTIFACT_SIZE_IN_BYTES:
|
1022
|
-
if not bucket_uri:
|
1023
|
-
raise ModelArtifactSizeError(
|
1024
|
-
utils.human_size(_MAX_ARTIFACT_SIZE_IN_BYTES)
|
1025
|
-
)
|
1026
|
-
|
1027
|
-
self._download_large_artifact(
|
1028
|
-
model_id=model_id,
|
1029
|
-
target_dir=target_dir,
|
1030
|
-
bucket_uri=os.path.join(bucket_uri, f"{model_id}.zip"),
|
1031
|
-
progress=progress,
|
1032
|
-
remove_existing_artifact=remove_existing_artifact,
|
1033
|
-
)
|
1034
|
-
else:
|
1035
|
-
self._download_small_artifact(
|
1036
|
-
model_id=model_id, target_dir=target_dir, progress=progress
|
1037
|
-
)
|
1038
|
-
progress.update()
|
1039
|
-
|
1040
|
-
progress.update("Done")
|
1041
|
-
|
1042
|
-
def _download_large_artifact(
|
1043
|
-
self,
|
1044
|
-
model_id: str,
|
1045
|
-
target_dir: str,
|
1046
|
-
bucket_uri: str,
|
1047
|
-
progress: TqdmProgressBar,
|
1048
|
-
remove_existing_artifact: Optional[bool] = True,
|
1049
|
-
) -> None:
|
1050
|
-
"""
|
1051
|
-
Downloads the model artifacts from model catalog to target_dir based on `model_id`.
|
1052
|
-
This method is used for artifacts with size greater than 2GB.
|
1053
|
-
|
1054
|
-
Parameters
|
1055
|
-
----------
|
1056
|
-
model_id: str
|
1057
|
-
The OCID of the model to download.
|
1058
|
-
target_dir: str
|
1059
|
-
The target location of model artifacts.
|
1060
|
-
bucket_uri: str
|
1061
|
-
The OCI Object Storage URI where model artifacts will be copied to.
|
1062
|
-
The `bucket_uri` is only necessary for downloading large artifacts with
|
1063
|
-
size is greater than 2GB. Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`.
|
1064
|
-
progress: TqdmProgressBar
|
1065
|
-
The progress bar.
|
1066
|
-
remove_existing_artifact: (bool, optional). Defaults to `True`.
|
1067
|
-
Whether artifacts uploaded to object storage bucket need to be removed or not.
|
1068
|
-
|
1069
|
-
Returns
|
1070
|
-
-------
|
1071
|
-
None
|
1072
|
-
Nothing.
|
1073
|
-
"""
|
1074
|
-
progress.update(f"Importing model artifacts from model catalog")
|
1075
|
-
self._import_model_artifact(model_id=model_id, bucket_uri=bucket_uri)
|
1076
|
-
|
1077
|
-
progress.update("Copying model artifacts to the artifact directory")
|
1078
|
-
with tempfile.TemporaryDirectory() as temp_dir:
|
1079
|
-
zip_file_path = os.path.join(temp_dir, f"{str(uuid.uuid4())}.zip")
|
1080
|
-
zip_file_path = utils.copy_file(
|
1081
|
-
uri_src=bucket_uri,
|
1082
|
-
uri_dst=zip_file_path,
|
1083
|
-
progressbar_description="Copying model artifacts to the artifact directory",
|
1084
|
-
)
|
1085
|
-
|
1086
|
-
progress.update("Extracting model artifacts")
|
1087
|
-
with ZipFile(zip_file_path) as zip_file:
|
1088
|
-
zip_file.extractall(target_dir)
|
1089
|
-
|
1090
|
-
if remove_existing_artifact:
|
1091
|
-
progress.update("Removing temporary artifacts")
|
1092
|
-
utils.remove_file(bucket_uri, self.ds_client_auth)
|
1093
|
-
else:
|
1094
|
-
progress.update()
|
1095
|
-
|
1096
|
-
def _import_model_artifact(
|
1097
|
-
self,
|
1098
|
-
model_id: str,
|
1099
|
-
bucket_uri: str,
|
1100
|
-
):
|
1101
|
-
"""Imports model artifact from the model catalog to the object storage bucket.
|
1102
|
-
This method is used for the case when the artifact size is greater than 2GB.
|
1103
|
-
"""
|
1104
|
-
bucket_details = ObjectStorageDetails.from_path(bucket_uri)
|
1105
|
-
response = self.ds_client.import_model_artifact(
|
1106
|
-
model_id=model_id,
|
1107
|
-
import_model_artifact_details=ImportModelArtifactDetails(
|
1108
|
-
artifact_import_details=ArtifactImportDetailsObjectStorage(
|
1109
|
-
namespace=bucket_details.namespace,
|
1110
|
-
destination_bucket=bucket_details.bucket,
|
1111
|
-
destination_object_name=bucket_details.filepath,
|
1112
|
-
destination_region=self._region,
|
1113
|
-
)
|
1114
|
-
),
|
1115
|
-
)
|
1116
|
-
|
1117
|
-
self._wait_for_work_request(
|
1118
|
-
response=response,
|
1119
|
-
first_step_description="Preparing to import model artifacts from the model catalog",
|
1120
|
-
num_steps=3,
|
1121
|
-
)
|
1122
|
-
|
1123
|
-
def _download_small_artifact(
|
1124
|
-
self,
|
1125
|
-
model_id: str,
|
1126
|
-
target_dir: str,
|
1127
|
-
progress: TqdmProgressBar,
|
1128
|
-
) -> None:
|
1129
|
-
"""
|
1130
|
-
Downloads the model artifacts from model catalog to target_dir based on `model_id`.
|
1131
|
-
This method is used for the artifacts with size less than 2GB.
|
1132
|
-
|
1133
|
-
Parameters
|
1134
|
-
----------
|
1135
|
-
model_id: str
|
1136
|
-
The OCID of the model to download.
|
1137
|
-
target_dir: str
|
1138
|
-
The target location of model artifacts.
|
1139
|
-
progress: TqdmProgressBar
|
1140
|
-
The progress bar.
|
1141
|
-
|
1142
|
-
Returns
|
1143
|
-
-------
|
1144
|
-
None
|
1145
|
-
Nothing
|
1146
|
-
"""
|
1147
|
-
progress.update("Importing model artifacts from catalog")
|
1148
|
-
try:
|
1149
|
-
zip_contents = self.ds_client.get_model_artifact_content(
|
1150
|
-
model_id
|
1151
|
-
).data.content
|
1152
|
-
except ServiceError as ex:
|
1153
|
-
if ex.status == 404:
|
1154
|
-
raise KeyError(ex.message) from ex
|
1155
|
-
raise
|
1156
|
-
|
1157
|
-
with tempfile.TemporaryDirectory() as temp_dir:
|
1158
|
-
progress.update("Copying model artifacts to the artifact directory")
|
1159
|
-
zip_file_path = os.path.join(temp_dir, f"{str(uuid.uuid4())}.zip")
|
1160
|
-
with open(zip_file_path, "wb") as zip_file:
|
1161
|
-
zip_file.write(zip_contents)
|
1162
|
-
progress.update("Extracting model artifacts")
|
1163
|
-
with ZipFile(zip_file_path) as zip_file:
|
1164
|
-
zip_file.extractall(target_dir)
|
1165
|
-
|
1166
|
-
@deprecated(
|
1167
|
-
"2.5.9",
|
1168
|
-
details="Instead use `ads.common.model_artifact.ModelArtifact.from_model_catalog()`.",
|
1169
|
-
)
|
1170
|
-
def download_model(
|
1171
|
-
self,
|
1172
|
-
model_id: str,
|
1173
|
-
target_dir: str,
|
1174
|
-
force_overwrite: bool = False,
|
1175
|
-
install_libs: bool = False,
|
1176
|
-
conflict_strategy=ConflictStrategy.IGNORE,
|
1177
|
-
bucket_uri: Optional[str] = None,
|
1178
|
-
remove_existing_artifact: Optional[bool] = True,
|
1179
|
-
):
|
1180
|
-
"""
|
1181
|
-
Downloads the model from model_dir to target_dir based on model_id.
|
1182
|
-
|
1183
|
-
Parameters
|
1184
|
-
----------
|
1185
|
-
model_id: str
|
1186
|
-
The OCID of the model to download.
|
1187
|
-
target_dir: str
|
1188
|
-
The target location of model after download.
|
1189
|
-
force_overwrite: bool
|
1190
|
-
Overwrite target_dir if exists.
|
1191
|
-
install_libs: bool, default: False
|
1192
|
-
Install the libraries specified in ds-requirements.txt which are missing in the current environment.
|
1193
|
-
conflict_strategy: ConflictStrategy, default: IGNORE
|
1194
|
-
Determines how to handle version conflicts between the current environment and requirements of
|
1195
|
-
model artifact.
|
1196
|
-
Valid values: "IGNORE", "UPDATE" or ConflictStrategy.
|
1197
|
-
IGNORE: Use the installed version in case of conflict
|
1198
|
-
UPDATE: Force update dependency to the version required by model artifact in case of conflict
|
1199
|
-
bucket_uri: (str, optional). Defaults to None.
|
1200
|
-
The OCI Object Storage URI where model artifacts will be copied to.
|
1201
|
-
The `bucket_uri` is only necessary for downloading large artifacts with
|
1202
|
-
size is greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
|
1203
|
-
remove_existing_artifact: (bool, optional). Defaults to `True`.
|
1204
|
-
Whether artifacts uploaded to object storage bucket need to be removed or not.
|
1205
|
-
|
1206
|
-
Returns
|
1207
|
-
-------
|
1208
|
-
ModelArtifact
|
1209
|
-
A ModelArtifact instance.
|
1210
|
-
"""
|
1211
|
-
self._download_artifact(
|
1212
|
-
model_id,
|
1213
|
-
target_dir,
|
1214
|
-
force_overwrite,
|
1215
|
-
bucket_uri=bucket_uri,
|
1216
|
-
remove_existing_artifact=remove_existing_artifact,
|
1217
|
-
)
|
1218
|
-
|
1219
|
-
result = ModelArtifact(
|
1220
|
-
target_dir,
|
1221
|
-
conflict_strategy=conflict_strategy,
|
1222
|
-
install_libs=install_libs,
|
1223
|
-
reload=False,
|
1224
|
-
)
|
1225
|
-
|
1226
|
-
try:
|
1227
|
-
model_response = self.ds_client.get_model(model_id)
|
1228
|
-
except ServiceError as e:
|
1229
|
-
if e.status == 404:
|
1230
|
-
raise KeyError(e.message) from e
|
1231
|
-
raise e
|
1232
|
-
|
1233
|
-
if hasattr(model_response.data, "custom_metadata_list"):
|
1234
|
-
try:
|
1235
|
-
result.metadata_custom = ModelCustomMetadata._from_oci_metadata(
|
1236
|
-
model_response.data.custom_metadata_list
|
1237
|
-
)
|
1238
|
-
except:
|
1239
|
-
result.metadata_custom = ModelCustomMetadata()
|
1240
|
-
if hasattr(model_response.data, "defined_metadata_list"):
|
1241
|
-
try:
|
1242
|
-
result.metadata_taxonomy = ModelTaxonomyMetadata._from_oci_metadata(
|
1243
|
-
model_response.data.defined_metadata_list
|
1244
|
-
)
|
1245
|
-
except:
|
1246
|
-
result.metadata_taxonomy = ModelTaxonomyMetadata()
|
1247
|
-
if hasattr(model_response.data, "input_schema"):
|
1248
|
-
try:
|
1249
|
-
result.schema_input = Schema.from_dict(
|
1250
|
-
json.loads(model_response.data.input_schema)
|
1251
|
-
if model_response.data.input_schema != ""
|
1252
|
-
else Schema()
|
1253
|
-
)
|
1254
|
-
except:
|
1255
|
-
result.schema_input = Schema()
|
1256
|
-
if hasattr(model_response.data, "output_schema"):
|
1257
|
-
try:
|
1258
|
-
result.schema_output = Schema.from_dict(
|
1259
|
-
json.loads(model_response.data.output_schema)
|
1260
|
-
if model_response.data.output_schema != ""
|
1261
|
-
else Schema()
|
1262
|
-
)
|
1263
|
-
except:
|
1264
|
-
result.schema_output = Schema()
|
1265
|
-
|
1266
|
-
if not install_libs:
|
1267
|
-
logger.warning(
|
1268
|
-
"Libraries in `ds-requirements.txt` were not installed. "
|
1269
|
-
"Use `install_requirements()` to install the required dependencies."
|
1270
|
-
)
|
1271
|
-
return result
|
1272
|
-
|
1273
|
-
@deprecated(
|
1274
|
-
"2.6.6",
|
1275
|
-
details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
|
1276
|
-
)
|
1277
|
-
def upload_model(
|
1278
|
-
self,
|
1279
|
-
model_artifact: ModelArtifact,
|
1280
|
-
provenance_metadata: Optional[ModelProvenance] = None,
|
1281
|
-
project_id: Optional[str] = None,
|
1282
|
-
display_name: Optional[str] = None,
|
1283
|
-
description: Optional[str] = None,
|
1284
|
-
freeform_tags: Optional[Dict[str, Dict[str, object]]] = None,
|
1285
|
-
defined_tags: Optional[Dict[str, Dict[str, object]]] = None,
|
1286
|
-
bucket_uri: Optional[str] = None,
|
1287
|
-
remove_existing_artifact: Optional[bool] = True,
|
1288
|
-
overwrite_existing_artifact: Optional[bool] = True,
|
1289
|
-
model_version_set: Optional[Union[str, ModelVersionSet]] = None,
|
1290
|
-
version_label: Optional[str] = None,
|
1291
|
-
):
|
1292
|
-
"""
|
1293
|
-
Uploads the model artifact to cloud storage.
|
1294
|
-
|
1295
|
-
Parameters
|
1296
|
-
----------
|
1297
|
-
model_artifact: Union[ModelArtifact, GenericModel]
|
1298
|
-
The model artifacts or generic model instance.
|
1299
|
-
provenance_metadata: (ModelProvenance, optional). Defaults to None.
|
1300
|
-
Model provenance gives data scientists information about the origin of their model.
|
1301
|
-
This information allows data scientists to reproduce
|
1302
|
-
the development environment in which the model was trained.
|
1303
|
-
project_id: (str, optional). Defaults to None.
|
1304
|
-
The project_id of model.
|
1305
|
-
display_name: (str, optional). Defaults to None.
|
1306
|
-
The name of model. If a display_name is not provided, a randomly generated easy to remember name
|
1307
|
-
with timestamp will be generated, like 'strange-spider-2022-08-17-23:55.02'.
|
1308
|
-
description: (str, optional). Defaults to None.
|
1309
|
-
The description of model.
|
1310
|
-
freeform_tags: (Dict[str, str], optional). Defaults to None.
|
1311
|
-
Freeform tags for the model, by default None
|
1312
|
-
defined_tags: (Dict[str, dict[str, object]], optional). Defaults to None.
|
1313
|
-
Defined tags for the model, by default None.
|
1314
|
-
bucket_uri: (str, optional). Defaults to None.
|
1315
|
-
The OCI Object Storage URI where model artifacts will be copied to.
|
1316
|
-
The `bucket_uri` is only necessary for uploading large artifacts which
|
1317
|
-
size greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
|
1318
|
-
remove_existing_artifact: (bool, optional). Defaults to `True`.
|
1319
|
-
Whether artifacts uploaded to object storage bucket need to be removed or not.
|
1320
|
-
overwrite_existing_artifact: (bool, optional). Defaults to `True`.
|
1321
|
-
Overwrite target bucket artifact if exists.
|
1322
|
-
model_version_set: (Union[str, ModelVersionSet], optional). Defaults to None.
|
1323
|
-
The Model version set OCID, or name, or `ModelVersionSet` instance.
|
1324
|
-
version_label: (str, optional). Defaults to None.
|
1325
|
-
The model version label.
|
1326
|
-
|
1327
|
-
Returns
|
1328
|
-
-------
|
1329
|
-
ads.catalog.Model
|
1330
|
-
The ads.catalog.Model with the matching ID.
|
1331
|
-
"""
|
1332
|
-
project_id = project_id or PROJECT_OCID
|
1333
|
-
if not project_id:
|
1334
|
-
raise ValueError("`project_id` needs to be specified.")
|
1335
|
-
|
1336
|
-
copy_artifact_to_os = False
|
1337
|
-
if (
|
1338
|
-
bucket_uri
|
1339
|
-
or utils.folder_size(model_artifact.artifact_dir)
|
1340
|
-
> _MAX_ARTIFACT_SIZE_IN_BYTES
|
1341
|
-
):
|
1342
|
-
if not bucket_uri:
|
1343
|
-
raise ValueError(
|
1344
|
-
f"The model artifacts size is greater than `{utils.human_size(_MAX_ARTIFACT_SIZE_IN_BYTES)}`. "
|
1345
|
-
"The `bucket_uri` needs to be specified to "
|
1346
|
-
"copy artifacts to the object storage bucket. "
|
1347
|
-
"Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`"
|
1348
|
-
)
|
1349
|
-
copy_artifact_to_os = True
|
1350
|
-
|
1351
|
-
# extract model_version_set_id from model_version_set attribute or environment
|
1352
|
-
# variables in case of saving model in context of model version set.
|
1353
|
-
model_version_set_id = _extract_model_version_set_id(model_version_set)
|
1354
|
-
# Set default display_name if not specified - randomly generated easy to remember name generated
|
1355
|
-
display_name = display_name or utils.get_random_name_for_resource()
|
1356
|
-
|
1357
|
-
with utils.get_progress_bar(5) as progress:
|
1358
|
-
project_id = PROJECT_OCID if project_id is None else project_id
|
1359
|
-
if project_id is None:
|
1360
|
-
raise ValueError("project_id needs to be specified.")
|
1361
|
-
schema_file = os.path.join(model_artifact.artifact_dir, "schema.json")
|
1362
|
-
if os.path.exists(schema_file):
|
1363
|
-
with open(schema_file, "r") as schema:
|
1364
|
-
metadata = json.load(schema)
|
1365
|
-
freeform_tags = {"problem_type": metadata["problem_type"]}
|
1366
|
-
|
1367
|
-
progress.update("Saving model in the model catalog")
|
1368
|
-
create_model_details = CreateModelDetails(
|
1369
|
-
display_name=display_name,
|
1370
|
-
description=description,
|
1371
|
-
project_id=project_id,
|
1372
|
-
compartment_id=self.compartment_id,
|
1373
|
-
custom_metadata_list=model_artifact.metadata_custom._to_oci_metadata()
|
1374
|
-
if model_artifact.metadata_custom is not None
|
1375
|
-
else [],
|
1376
|
-
defined_metadata_list=model_artifact.metadata_taxonomy._to_oci_metadata()
|
1377
|
-
if model_artifact.metadata_taxonomy is not None
|
1378
|
-
else [],
|
1379
|
-
input_schema=model_artifact.schema_input.to_json()
|
1380
|
-
if model_artifact.schema_input is not None
|
1381
|
-
else '{"schema": []}',
|
1382
|
-
output_schema=model_artifact.schema_output.to_json()
|
1383
|
-
if model_artifact.schema_output is not None
|
1384
|
-
else '{"schema": []}',
|
1385
|
-
freeform_tags=freeform_tags,
|
1386
|
-
defined_tags=defined_tags,
|
1387
|
-
model_version_set_id=model_version_set_id,
|
1388
|
-
version_label=version_label,
|
1389
|
-
)
|
1390
|
-
model = self.ds_client.create_model(create_model_details)
|
1391
|
-
|
1392
|
-
if provenance_metadata is not None:
|
1393
|
-
progress.update("Saving provenance metadata")
|
1394
|
-
self.ds_client.create_model_provenance(
|
1395
|
-
model.data.id, provenance_metadata
|
1396
|
-
)
|
1397
|
-
else:
|
1398
|
-
progress.update()
|
1399
|
-
|
1400
|
-
# if the model artifact size greater than 2GB then export function
|
1401
|
-
# needs to be used instead of upload. The export function will copy
|
1402
|
-
# model artifacts to the OS bucket at first and then will upload
|
1403
|
-
# the artifacts to the model catalog.
|
1404
|
-
if copy_artifact_to_os:
|
1405
|
-
self._export_model_artifact(
|
1406
|
-
model_id=model.data.id,
|
1407
|
-
model_artifact=model_artifact,
|
1408
|
-
progress=progress,
|
1409
|
-
bucket_uri=bucket_uri,
|
1410
|
-
remove_existing_artifact=remove_existing_artifact,
|
1411
|
-
overwrite_existing_artifact=overwrite_existing_artifact,
|
1412
|
-
)
|
1413
|
-
else:
|
1414
|
-
self._upload_model_artifact(
|
1415
|
-
model_id=model.data.id,
|
1416
|
-
model_artifact=model_artifact,
|
1417
|
-
progress=progress,
|
1418
|
-
)
|
1419
|
-
progress.update()
|
1420
|
-
|
1421
|
-
progress.update("Done")
|
1422
|
-
return self.get_model(model.data.id)
|
1423
|
-
|
1424
|
-
def _prepare_model_artifact(self, model_artifact, progress: TqdmProgressBar) -> str:
|
1425
|
-
"""Prepares model artifacts to save in the Model Catalog.
|
1426
|
-
|
1427
|
-
Returns
|
1428
|
-
-------
|
1429
|
-
str
|
1430
|
-
The path to the model artifact zip archive.
|
1431
|
-
"""
|
1432
|
-
progress.update("Preparing model artifacts zip")
|
1433
|
-
files_to_upload = model_artifact._get_files()
|
1434
|
-
artifact_path = "/tmp/saved_model_" + str(uuid.uuid4()) + ".zip"
|
1435
|
-
zf = ZipFile(artifact_path, "w")
|
1436
|
-
for matched_file in files_to_upload:
|
1437
|
-
zf.write(
|
1438
|
-
os.path.join(model_artifact.artifact_dir, matched_file),
|
1439
|
-
arcname=matched_file,
|
1440
|
-
)
|
1441
|
-
zf.close()
|
1442
|
-
return artifact_path
|
1443
|
-
|
1444
|
-
def _upload_model_artifact(self, model_id, model_artifact, progress):
|
1445
|
-
"""Uploads model artifact to the model catalog.
|
1446
|
-
This method can be used only if the size of model artifact is less than 2GB.
|
1447
|
-
For the artifacts with size greater than 2 GB the `_export_model_artifact`
|
1448
|
-
method should be used instead.
|
1449
|
-
"""
|
1450
|
-
artifact_zip_path = self._prepare_model_artifact(model_artifact, progress)
|
1451
|
-
progress.update("Uploading model artifacts to the catalog")
|
1452
|
-
with open(artifact_zip_path, "rb") as file_data:
|
1453
|
-
bytes_content = file_data.read()
|
1454
|
-
self.ds_client.create_model_artifact(
|
1455
|
-
model_id,
|
1456
|
-
bytes_content,
|
1457
|
-
content_disposition=f'attachment; filename="{model_id}.zip"',
|
1458
|
-
)
|
1459
|
-
os.remove(artifact_zip_path)
|
1460
|
-
progress.update()
|
1461
|
-
|
1462
|
-
def _export_model_artifact(
|
1463
|
-
self,
|
1464
|
-
model_id: str,
|
1465
|
-
model_artifact: ModelArtifact,
|
1466
|
-
bucket_uri: str,
|
1467
|
-
progress,
|
1468
|
-
remove_existing_artifact: Optional[bool] = True,
|
1469
|
-
overwrite_existing_artifact: Optional[bool] = True,
|
1470
|
-
):
|
1471
|
-
"""Exports model artifact to the model catalog.
|
1472
|
-
This method is used for the case when the artifact size is greater than 2GB.
|
1473
|
-
1. Archive model artifact.
|
1474
|
-
2. Copies the artifact to the object storage bucket.
|
1475
|
-
3. Exports artifact from the user's object storage bucket to the system one.
|
1476
|
-
"""
|
1477
|
-
artifact_zip_path = self._prepare_model_artifact(model_artifact, progress)
|
1478
|
-
progress.update(f"Copying model artifact to the Object Storage bucket")
|
1479
|
-
|
1480
|
-
try:
|
1481
|
-
bucket_uri_file_name = os.path.basename(bucket_uri)
|
1482
|
-
if not bucket_uri_file_name:
|
1483
|
-
bucket_uri = os.path.join(bucket_uri, f"{model_id}.zip")
|
1484
|
-
elif not bucket_uri.lower().endswith(".zip"):
|
1485
|
-
bucket_uri = f"{bucket_uri}.zip"
|
1486
|
-
|
1487
|
-
bucket_file_name = utils.copy_file(
|
1488
|
-
artifact_zip_path,
|
1489
|
-
bucket_uri,
|
1490
|
-
force_overwrite=overwrite_existing_artifact,
|
1491
|
-
auth=self.ds_client_auth,
|
1492
|
-
progressbar_description="Copying model artifact to the Object Storage bucket",
|
1493
|
-
)
|
1494
|
-
except FileExistsError:
|
1495
|
-
raise FileExistsError(
|
1496
|
-
f"The `{bucket_uri}` exists. Please use a new file name or "
|
1497
|
-
"set `overwrite_existing_artifact` to `True` if you wish to overwrite."
|
1498
|
-
)
|
1499
|
-
|
1500
|
-
os.remove(artifact_zip_path)
|
1501
|
-
|
1502
|
-
progress.update("Exporting model artifact to the model catalog")
|
1503
|
-
bucket_details = ObjectStorageDetails.from_path(bucket_file_name)
|
1504
|
-
response = self.ds_client.export_model_artifact(
|
1505
|
-
model_id=model_id,
|
1506
|
-
export_model_artifact_details=ExportModelArtifactDetails(
|
1507
|
-
artifact_export_details=ArtifactExportDetailsObjectStorage(
|
1508
|
-
namespace=bucket_details.namespace,
|
1509
|
-
source_bucket=bucket_details.bucket,
|
1510
|
-
source_object_name=bucket_details.filepath,
|
1511
|
-
source_region=self._region,
|
1512
|
-
)
|
1513
|
-
),
|
1514
|
-
)
|
1515
|
-
|
1516
|
-
self._wait_for_work_request(
|
1517
|
-
response=response,
|
1518
|
-
first_step_description="Preparing to export model artifact to the model catalog",
|
1519
|
-
num_steps=4,
|
1520
|
-
)
|
1521
|
-
|
1522
|
-
if remove_existing_artifact:
|
1523
|
-
progress.update(
|
1524
|
-
"Removing temporary model artifact from the Object Storage bucket"
|
1525
|
-
)
|
1526
|
-
utils.remove_file(bucket_file_name, self.ds_client_auth)
|
1527
|
-
else:
|
1528
|
-
progress.update()
|
1529
|
-
|
1530
|
-
@property
|
1531
|
-
def _region(self):
|
1532
|
-
"""Gets current region."""
|
1533
|
-
if "region" in self.ds_client_auth.get("config", {}):
|
1534
|
-
return self.ds_client_auth["config"]["region"]
|
1535
|
-
return json.loads(OCI_REGION_METADATA)["regionIdentifier"]
|
1536
|
-
|
1537
|
-
def _wait_for_work_request(
|
1538
|
-
self, response, first_step_description: str = "", num_steps=4
|
1539
|
-
):
|
1540
|
-
"""Waits for the work request to be completed."""
|
1541
|
-
STOP_STATE = (
|
1542
|
-
WorkRequest.STATUS_SUCCEEDED,
|
1543
|
-
WorkRequest.STATUS_CANCELED,
|
1544
|
-
WorkRequest.STATUS_CANCELING,
|
1545
|
-
WorkRequest.STATUS_FAILED,
|
1546
|
-
)
|
1547
|
-
work_request_id = response.headers["opc-work-request-id"]
|
1548
|
-
work_request_logs = None
|
1549
|
-
|
1550
|
-
i = 0
|
1551
|
-
with utils.get_progress_bar(num_steps) as progress:
|
1552
|
-
progress.update(first_step_description)
|
1553
|
-
while not work_request_logs or len(work_request_logs) < num_steps:
|
1554
|
-
time.sleep(_WORK_REQUEST_INTERVAL_IN_SEC)
|
1555
|
-
work_request = self.ds_client.get_work_request(work_request_id)
|
1556
|
-
work_request_logs = self.ds_client.list_work_request_logs(
|
1557
|
-
work_request_id
|
1558
|
-
).data
|
1559
|
-
if work_request_logs:
|
1560
|
-
new_work_request_logs = work_request_logs[i:]
|
1561
|
-
|
1562
|
-
for wr_item in new_work_request_logs:
|
1563
|
-
progress.update(wr_item.message)
|
1564
|
-
i += 1
|
1565
|
-
|
1566
|
-
if work_request.data.status in STOP_STATE:
|
1567
|
-
if work_request.data.status != WorkRequest.STATUS_SUCCEEDED:
|
1568
|
-
if work_request_logs:
|
1569
|
-
raise Exception(work_request_logs[-1].message)
|
1570
|
-
else:
|
1571
|
-
raise Exception(
|
1572
|
-
"An error occurred in attempt to perform the operation. Check the service logs to get more details."
|
1573
|
-
)
|
1574
|
-
else:
|
1575
|
-
break
|
1576
|
-
return work_request
|