oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/METADATA +151 -151
- oracle_ads-2.13.9rc0.dist-info/RECORD +9 -0
- {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/WHEEL +2 -1
- {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/entry_points.txt +1 -2
- oracle_ads-2.13.9rc0.dist-info/top_level.txt +1 -0
- ads/aqua/__init__.py +0 -40
- ads/aqua/app.py +0 -506
- ads/aqua/cli.py +0 -96
- ads/aqua/client/__init__.py +0 -3
- ads/aqua/client/client.py +0 -836
- ads/aqua/client/openai_client.py +0 -305
- ads/aqua/common/__init__.py +0 -5
- ads/aqua/common/decorator.py +0 -125
- ads/aqua/common/entities.py +0 -266
- ads/aqua/common/enums.py +0 -122
- ads/aqua/common/errors.py +0 -109
- ads/aqua/common/utils.py +0 -1285
- ads/aqua/config/__init__.py +0 -4
- ads/aqua/config/container_config.py +0 -248
- ads/aqua/config/evaluation/__init__.py +0 -4
- ads/aqua/config/evaluation/evaluation_service_config.py +0 -147
- ads/aqua/config/utils/__init__.py +0 -4
- ads/aqua/config/utils/serializer.py +0 -339
- ads/aqua/constants.py +0 -114
- ads/aqua/data.py +0 -14
- ads/aqua/dummy_data/icon.txt +0 -1
- ads/aqua/dummy_data/oci_model_deployments.json +0 -56
- ads/aqua/dummy_data/oci_models.json +0 -1
- ads/aqua/dummy_data/readme.md +0 -26
- ads/aqua/evaluation/__init__.py +0 -8
- ads/aqua/evaluation/constants.py +0 -53
- ads/aqua/evaluation/entities.py +0 -186
- ads/aqua/evaluation/errors.py +0 -70
- ads/aqua/evaluation/evaluation.py +0 -1814
- ads/aqua/extension/__init__.py +0 -42
- ads/aqua/extension/aqua_ws_msg_handler.py +0 -76
- ads/aqua/extension/base_handler.py +0 -90
- ads/aqua/extension/common_handler.py +0 -121
- ads/aqua/extension/common_ws_msg_handler.py +0 -36
- ads/aqua/extension/deployment_handler.py +0 -298
- ads/aqua/extension/deployment_ws_msg_handler.py +0 -54
- ads/aqua/extension/errors.py +0 -30
- ads/aqua/extension/evaluation_handler.py +0 -129
- ads/aqua/extension/evaluation_ws_msg_handler.py +0 -61
- ads/aqua/extension/finetune_handler.py +0 -96
- ads/aqua/extension/model_handler.py +0 -390
- ads/aqua/extension/models/__init__.py +0 -0
- ads/aqua/extension/models/ws_models.py +0 -145
- ads/aqua/extension/models_ws_msg_handler.py +0 -50
- ads/aqua/extension/ui_handler.py +0 -282
- ads/aqua/extension/ui_websocket_handler.py +0 -130
- ads/aqua/extension/utils.py +0 -133
- ads/aqua/finetuning/__init__.py +0 -7
- ads/aqua/finetuning/constants.py +0 -23
- ads/aqua/finetuning/entities.py +0 -181
- ads/aqua/finetuning/finetuning.py +0 -731
- ads/aqua/model/__init__.py +0 -8
- ads/aqua/model/constants.py +0 -60
- ads/aqua/model/entities.py +0 -306
- ads/aqua/model/enums.py +0 -30
- ads/aqua/model/model.py +0 -2079
- ads/aqua/modeldeployment/__init__.py +0 -8
- ads/aqua/modeldeployment/constants.py +0 -10
- ads/aqua/modeldeployment/deployment.py +0 -1324
- ads/aqua/modeldeployment/entities.py +0 -653
- ads/aqua/modeldeployment/inference.py +0 -74
- ads/aqua/modeldeployment/utils.py +0 -543
- ads/aqua/resources/gpu_shapes_index.json +0 -94
- ads/aqua/server/__init__.py +0 -4
- ads/aqua/server/__main__.py +0 -24
- ads/aqua/server/app.py +0 -47
- ads/aqua/server/aqua_spec.yml +0 -1291
- ads/aqua/training/__init__.py +0 -4
- ads/aqua/training/exceptions.py +0 -476
- ads/aqua/ui.py +0 -499
- ads/automl/__init__.py +0 -9
- ads/automl/driver.py +0 -330
- ads/automl/provider.py +0 -975
- ads/bds/__init__.py +0 -5
- ads/bds/auth.py +0 -127
- ads/bds/big_data_service.py +0 -255
- ads/catalog/__init__.py +0 -19
- ads/catalog/model.py +0 -1576
- ads/catalog/notebook.py +0 -461
- ads/catalog/project.py +0 -468
- ads/catalog/summary.py +0 -178
- ads/common/__init__.py +0 -11
- ads/common/analyzer.py +0 -65
- ads/common/artifact/.model-ignore +0 -63
- ads/common/artifact/__init__.py +0 -10
- ads/common/auth.py +0 -1122
- ads/common/card_identifier.py +0 -83
- ads/common/config.py +0 -647
- ads/common/data.py +0 -165
- ads/common/decorator/__init__.py +0 -9
- ads/common/decorator/argument_to_case.py +0 -88
- ads/common/decorator/deprecate.py +0 -69
- ads/common/decorator/require_nonempty_arg.py +0 -65
- ads/common/decorator/runtime_dependency.py +0 -178
- ads/common/decorator/threaded.py +0 -97
- ads/common/decorator/utils.py +0 -35
- ads/common/dsc_file_system.py +0 -303
- ads/common/error.py +0 -14
- ads/common/extended_enum.py +0 -81
- ads/common/function/__init__.py +0 -5
- ads/common/function/fn_util.py +0 -142
- ads/common/function/func_conf.yaml +0 -25
- ads/common/ipython.py +0 -76
- ads/common/model.py +0 -679
- ads/common/model_artifact.py +0 -1759
- ads/common/model_artifact_schema.json +0 -107
- ads/common/model_export_util.py +0 -664
- ads/common/model_metadata.py +0 -24
- ads/common/object_storage_details.py +0 -296
- ads/common/oci_client.py +0 -175
- ads/common/oci_datascience.py +0 -46
- ads/common/oci_logging.py +0 -1144
- ads/common/oci_mixin.py +0 -957
- ads/common/oci_resource.py +0 -136
- ads/common/serializer.py +0 -559
- ads/common/utils.py +0 -1852
- ads/common/word_lists.py +0 -1491
- ads/common/work_request.py +0 -189
- ads/data_labeling/__init__.py +0 -13
- ads/data_labeling/boundingbox.py +0 -253
- ads/data_labeling/constants.py +0 -47
- ads/data_labeling/data_labeling_service.py +0 -244
- ads/data_labeling/interface/__init__.py +0 -5
- ads/data_labeling/interface/loader.py +0 -16
- ads/data_labeling/interface/parser.py +0 -16
- ads/data_labeling/interface/reader.py +0 -23
- ads/data_labeling/loader/__init__.py +0 -5
- ads/data_labeling/loader/file_loader.py +0 -241
- ads/data_labeling/metadata.py +0 -110
- ads/data_labeling/mixin/__init__.py +0 -5
- ads/data_labeling/mixin/data_labeling.py +0 -232
- ads/data_labeling/ner.py +0 -129
- ads/data_labeling/parser/__init__.py +0 -5
- ads/data_labeling/parser/dls_record_parser.py +0 -388
- ads/data_labeling/parser/export_metadata_parser.py +0 -94
- ads/data_labeling/parser/export_record_parser.py +0 -473
- ads/data_labeling/reader/__init__.py +0 -5
- ads/data_labeling/reader/dataset_reader.py +0 -574
- ads/data_labeling/reader/dls_record_reader.py +0 -121
- ads/data_labeling/reader/export_record_reader.py +0 -62
- ads/data_labeling/reader/jsonl_reader.py +0 -75
- ads/data_labeling/reader/metadata_reader.py +0 -203
- ads/data_labeling/reader/record_reader.py +0 -263
- ads/data_labeling/record.py +0 -52
- ads/data_labeling/visualizer/__init__.py +0 -5
- ads/data_labeling/visualizer/image_visualizer.py +0 -525
- ads/data_labeling/visualizer/text_visualizer.py +0 -357
- ads/database/__init__.py +0 -5
- ads/database/connection.py +0 -338
- ads/dataset/__init__.py +0 -10
- ads/dataset/capabilities.md +0 -51
- ads/dataset/classification_dataset.py +0 -339
- ads/dataset/correlation.py +0 -226
- ads/dataset/correlation_plot.py +0 -563
- ads/dataset/dask_series.py +0 -173
- ads/dataset/dataframe_transformer.py +0 -110
- ads/dataset/dataset.py +0 -1979
- ads/dataset/dataset_browser.py +0 -360
- ads/dataset/dataset_with_target.py +0 -995
- ads/dataset/exception.py +0 -25
- ads/dataset/factory.py +0 -987
- ads/dataset/feature_engineering_transformer.py +0 -35
- ads/dataset/feature_selection.py +0 -107
- ads/dataset/forecasting_dataset.py +0 -26
- ads/dataset/helper.py +0 -1450
- ads/dataset/label_encoder.py +0 -99
- ads/dataset/mixin/__init__.py +0 -5
- ads/dataset/mixin/dataset_accessor.py +0 -134
- ads/dataset/pipeline.py +0 -58
- ads/dataset/plot.py +0 -710
- ads/dataset/progress.py +0 -86
- ads/dataset/recommendation.py +0 -297
- ads/dataset/recommendation_transformer.py +0 -502
- ads/dataset/regression_dataset.py +0 -14
- ads/dataset/sampled_dataset.py +0 -1050
- ads/dataset/target.py +0 -98
- ads/dataset/timeseries.py +0 -18
- ads/dbmixin/__init__.py +0 -5
- ads/dbmixin/db_pandas_accessor.py +0 -153
- ads/environment/__init__.py +0 -9
- ads/environment/ml_runtime.py +0 -66
- ads/evaluations/README.md +0 -14
- ads/evaluations/__init__.py +0 -109
- ads/evaluations/evaluation_plot.py +0 -983
- ads/evaluations/evaluator.py +0 -1334
- ads/evaluations/statistical_metrics.py +0 -543
- ads/experiments/__init__.py +0 -9
- ads/experiments/capabilities.md +0 -0
- ads/explanations/__init__.py +0 -21
- ads/explanations/base_explainer.py +0 -142
- ads/explanations/capabilities.md +0 -83
- ads/explanations/explainer.py +0 -190
- ads/explanations/mlx_global_explainer.py +0 -1050
- ads/explanations/mlx_interface.py +0 -386
- ads/explanations/mlx_local_explainer.py +0 -287
- ads/explanations/mlx_whatif_explainer.py +0 -201
- ads/feature_engineering/__init__.py +0 -20
- ads/feature_engineering/accessor/__init__.py +0 -5
- ads/feature_engineering/accessor/dataframe_accessor.py +0 -535
- ads/feature_engineering/accessor/mixin/__init__.py +0 -5
- ads/feature_engineering/accessor/mixin/correlation.py +0 -166
- ads/feature_engineering/accessor/mixin/eda_mixin.py +0 -266
- ads/feature_engineering/accessor/mixin/eda_mixin_series.py +0 -85
- ads/feature_engineering/accessor/mixin/feature_types_mixin.py +0 -211
- ads/feature_engineering/accessor/mixin/utils.py +0 -65
- ads/feature_engineering/accessor/series_accessor.py +0 -431
- ads/feature_engineering/adsimage/__init__.py +0 -5
- ads/feature_engineering/adsimage/image.py +0 -192
- ads/feature_engineering/adsimage/image_reader.py +0 -170
- ads/feature_engineering/adsimage/interface/__init__.py +0 -5
- ads/feature_engineering/adsimage/interface/reader.py +0 -19
- ads/feature_engineering/adsstring/__init__.py +0 -7
- ads/feature_engineering/adsstring/oci_language/__init__.py +0 -8
- ads/feature_engineering/adsstring/string/__init__.py +0 -8
- ads/feature_engineering/data_schema.json +0 -57
- ads/feature_engineering/dataset/__init__.py +0 -5
- ads/feature_engineering/dataset/zip_code_data.py +0 -42062
- ads/feature_engineering/exceptions.py +0 -40
- ads/feature_engineering/feature_type/__init__.py +0 -133
- ads/feature_engineering/feature_type/address.py +0 -184
- ads/feature_engineering/feature_type/adsstring/__init__.py +0 -5
- ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +0 -164
- ads/feature_engineering/feature_type/adsstring/oci_language.py +0 -93
- ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +0 -5
- ads/feature_engineering/feature_type/adsstring/parsers/base.py +0 -47
- ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +0 -96
- ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +0 -221
- ads/feature_engineering/feature_type/adsstring/string.py +0 -258
- ads/feature_engineering/feature_type/base.py +0 -58
- ads/feature_engineering/feature_type/boolean.py +0 -183
- ads/feature_engineering/feature_type/category.py +0 -146
- ads/feature_engineering/feature_type/constant.py +0 -137
- ads/feature_engineering/feature_type/continuous.py +0 -151
- ads/feature_engineering/feature_type/creditcard.py +0 -314
- ads/feature_engineering/feature_type/datetime.py +0 -190
- ads/feature_engineering/feature_type/discrete.py +0 -134
- ads/feature_engineering/feature_type/document.py +0 -43
- ads/feature_engineering/feature_type/gis.py +0 -251
- ads/feature_engineering/feature_type/handler/__init__.py +0 -5
- ads/feature_engineering/feature_type/handler/feature_validator.py +0 -524
- ads/feature_engineering/feature_type/handler/feature_warning.py +0 -319
- ads/feature_engineering/feature_type/handler/warnings.py +0 -128
- ads/feature_engineering/feature_type/integer.py +0 -142
- ads/feature_engineering/feature_type/ip_address.py +0 -144
- ads/feature_engineering/feature_type/ip_address_v4.py +0 -138
- ads/feature_engineering/feature_type/ip_address_v6.py +0 -138
- ads/feature_engineering/feature_type/lat_long.py +0 -256
- ads/feature_engineering/feature_type/object.py +0 -43
- ads/feature_engineering/feature_type/ordinal.py +0 -132
- ads/feature_engineering/feature_type/phone_number.py +0 -135
- ads/feature_engineering/feature_type/string.py +0 -171
- ads/feature_engineering/feature_type/text.py +0 -93
- ads/feature_engineering/feature_type/unknown.py +0 -43
- ads/feature_engineering/feature_type/zip_code.py +0 -164
- ads/feature_engineering/feature_type_manager.py +0 -406
- ads/feature_engineering/schema.py +0 -795
- ads/feature_engineering/utils.py +0 -245
- ads/feature_store/.readthedocs.yaml +0 -19
- ads/feature_store/README.md +0 -65
- ads/feature_store/__init__.py +0 -9
- ads/feature_store/common/__init__.py +0 -0
- ads/feature_store/common/enums.py +0 -339
- ads/feature_store/common/exceptions.py +0 -18
- ads/feature_store/common/spark_session_singleton.py +0 -125
- ads/feature_store/common/utils/__init__.py +0 -0
- ads/feature_store/common/utils/base64_encoder_decoder.py +0 -72
- ads/feature_store/common/utils/feature_schema_mapper.py +0 -283
- ads/feature_store/common/utils/transformation_utils.py +0 -82
- ads/feature_store/common/utils/utility.py +0 -403
- ads/feature_store/data_validation/__init__.py +0 -0
- ads/feature_store/data_validation/great_expectation.py +0 -129
- ads/feature_store/dataset.py +0 -1230
- ads/feature_store/dataset_job.py +0 -530
- ads/feature_store/docs/Dockerfile +0 -7
- ads/feature_store/docs/Makefile +0 -44
- ads/feature_store/docs/conf.py +0 -28
- ads/feature_store/docs/requirements.txt +0 -14
- ads/feature_store/docs/source/ads.feature_store.query.rst +0 -20
- ads/feature_store/docs/source/cicd.rst +0 -137
- ads/feature_store/docs/source/conf.py +0 -86
- ads/feature_store/docs/source/data_versioning.rst +0 -33
- ads/feature_store/docs/source/dataset.rst +0 -388
- ads/feature_store/docs/source/dataset_job.rst +0 -27
- ads/feature_store/docs/source/demo.rst +0 -70
- ads/feature_store/docs/source/entity.rst +0 -78
- ads/feature_store/docs/source/feature_group.rst +0 -624
- ads/feature_store/docs/source/feature_group_job.rst +0 -29
- ads/feature_store/docs/source/feature_store.rst +0 -122
- ads/feature_store/docs/source/feature_store_class.rst +0 -123
- ads/feature_store/docs/source/feature_validation.rst +0 -66
- ads/feature_store/docs/source/figures/cicd.png +0 -0
- ads/feature_store/docs/source/figures/data_validation.png +0 -0
- ads/feature_store/docs/source/figures/data_versioning.png +0 -0
- ads/feature_store/docs/source/figures/dataset.gif +0 -0
- ads/feature_store/docs/source/figures/dataset.png +0 -0
- ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
- ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
- ads/feature_store/docs/source/figures/entity.png +0 -0
- ads/feature_store/docs/source/figures/feature_group.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
- ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
- ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
- ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
- ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
- ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
- ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
- ads/feature_store/docs/source/figures/overview.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
- ads/feature_store/docs/source/figures/stats_1.png +0 -0
- ads/feature_store/docs/source/figures/stats_2.png +0 -0
- ads/feature_store/docs/source/figures/stats_d.png +0 -0
- ads/feature_store/docs/source/figures/stats_fg.png +0 -0
- ads/feature_store/docs/source/figures/transformation.png +0 -0
- ads/feature_store/docs/source/figures/transformations.gif +0 -0
- ads/feature_store/docs/source/figures/validation.png +0 -0
- ads/feature_store/docs/source/figures/validation_fg.png +0 -0
- ads/feature_store/docs/source/figures/validation_results.png +0 -0
- ads/feature_store/docs/source/figures/validation_summary.png +0 -0
- ads/feature_store/docs/source/index.rst +0 -81
- ads/feature_store/docs/source/module.rst +0 -8
- ads/feature_store/docs/source/notebook.rst +0 -94
- ads/feature_store/docs/source/overview.rst +0 -47
- ads/feature_store/docs/source/quickstart.rst +0 -176
- ads/feature_store/docs/source/release_notes.rst +0 -194
- ads/feature_store/docs/source/setup_feature_store.rst +0 -81
- ads/feature_store/docs/source/statistics.rst +0 -58
- ads/feature_store/docs/source/transformation.rst +0 -199
- ads/feature_store/docs/source/ui.rst +0 -65
- ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +0 -66
- ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +0 -192
- ads/feature_store/docs/source/user_guides.setup.terraform.rst +0 -338
- ads/feature_store/entity.py +0 -718
- ads/feature_store/execution_strategy/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +0 -375
- ads/feature_store/execution_strategy/engine/__init__.py +0 -0
- ads/feature_store/execution_strategy/engine/spark_engine.py +0 -316
- ads/feature_store/execution_strategy/execution_strategy.py +0 -113
- ads/feature_store/execution_strategy/execution_strategy_provider.py +0 -47
- ads/feature_store/execution_strategy/spark/__init__.py +0 -0
- ads/feature_store/execution_strategy/spark/spark_execution.py +0 -618
- ads/feature_store/feature.py +0 -192
- ads/feature_store/feature_group.py +0 -1494
- ads/feature_store/feature_group_expectation.py +0 -346
- ads/feature_store/feature_group_job.py +0 -602
- ads/feature_store/feature_lineage/__init__.py +0 -0
- ads/feature_store/feature_lineage/graphviz_service.py +0 -180
- ads/feature_store/feature_option_details.py +0 -50
- ads/feature_store/feature_statistics/__init__.py +0 -0
- ads/feature_store/feature_statistics/statistics_service.py +0 -99
- ads/feature_store/feature_store.py +0 -699
- ads/feature_store/feature_store_registrar.py +0 -518
- ads/feature_store/input_feature_detail.py +0 -149
- ads/feature_store/mixin/__init__.py +0 -4
- ads/feature_store/mixin/oci_feature_store.py +0 -145
- ads/feature_store/model_details.py +0 -73
- ads/feature_store/query/__init__.py +0 -0
- ads/feature_store/query/filter.py +0 -266
- ads/feature_store/query/generator/__init__.py +0 -0
- ads/feature_store/query/generator/query_generator.py +0 -298
- ads/feature_store/query/join.py +0 -161
- ads/feature_store/query/query.py +0 -403
- ads/feature_store/query/validator/__init__.py +0 -0
- ads/feature_store/query/validator/query_validator.py +0 -57
- ads/feature_store/response/__init__.py +0 -0
- ads/feature_store/response/response_builder.py +0 -68
- ads/feature_store/service/__init__.py +0 -0
- ads/feature_store/service/oci_dataset.py +0 -139
- ads/feature_store/service/oci_dataset_job.py +0 -199
- ads/feature_store/service/oci_entity.py +0 -125
- ads/feature_store/service/oci_feature_group.py +0 -164
- ads/feature_store/service/oci_feature_group_job.py +0 -214
- ads/feature_store/service/oci_feature_store.py +0 -182
- ads/feature_store/service/oci_lineage.py +0 -87
- ads/feature_store/service/oci_transformation.py +0 -104
- ads/feature_store/statistics/__init__.py +0 -0
- ads/feature_store/statistics/abs_feature_value.py +0 -49
- ads/feature_store/statistics/charts/__init__.py +0 -0
- ads/feature_store/statistics/charts/abstract_feature_plot.py +0 -37
- ads/feature_store/statistics/charts/box_plot.py +0 -148
- ads/feature_store/statistics/charts/frequency_distribution.py +0 -65
- ads/feature_store/statistics/charts/probability_distribution.py +0 -68
- ads/feature_store/statistics/charts/top_k_frequent_elements.py +0 -98
- ads/feature_store/statistics/feature_stat.py +0 -126
- ads/feature_store/statistics/generic_feature_value.py +0 -33
- ads/feature_store/statistics/statistics.py +0 -41
- ads/feature_store/statistics_config.py +0 -101
- ads/feature_store/templates/feature_store_template.yaml +0 -45
- ads/feature_store/transformation.py +0 -499
- ads/feature_store/validation_output.py +0 -57
- ads/hpo/__init__.py +0 -9
- ads/hpo/_imports.py +0 -91
- ads/hpo/ads_search_space.py +0 -439
- ads/hpo/distributions.py +0 -325
- ads/hpo/objective.py +0 -280
- ads/hpo/search_cv.py +0 -1657
- ads/hpo/stopping_criterion.py +0 -75
- ads/hpo/tuner_artifact.py +0 -413
- ads/hpo/utils.py +0 -91
- ads/hpo/validation.py +0 -140
- ads/hpo/visualization/__init__.py +0 -5
- ads/hpo/visualization/_contour.py +0 -23
- ads/hpo/visualization/_edf.py +0 -20
- ads/hpo/visualization/_intermediate_values.py +0 -21
- ads/hpo/visualization/_optimization_history.py +0 -25
- ads/hpo/visualization/_parallel_coordinate.py +0 -169
- ads/hpo/visualization/_param_importances.py +0 -26
- ads/jobs/__init__.py +0 -53
- ads/jobs/ads_job.py +0 -663
- ads/jobs/builders/__init__.py +0 -5
- ads/jobs/builders/base.py +0 -156
- ads/jobs/builders/infrastructure/__init__.py +0 -6
- ads/jobs/builders/infrastructure/base.py +0 -165
- ads/jobs/builders/infrastructure/dataflow.py +0 -1252
- ads/jobs/builders/infrastructure/dsc_job.py +0 -1894
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +0 -1233
- ads/jobs/builders/infrastructure/utils.py +0 -65
- ads/jobs/builders/runtimes/__init__.py +0 -5
- ads/jobs/builders/runtimes/artifact.py +0 -338
- ads/jobs/builders/runtimes/base.py +0 -325
- ads/jobs/builders/runtimes/container_runtime.py +0 -242
- ads/jobs/builders/runtimes/python_runtime.py +0 -1016
- ads/jobs/builders/runtimes/pytorch_runtime.py +0 -204
- ads/jobs/cli.py +0 -104
- ads/jobs/env_var_parser.py +0 -131
- ads/jobs/extension.py +0 -160
- ads/jobs/schema/__init__.py +0 -5
- ads/jobs/schema/infrastructure_schema.json +0 -116
- ads/jobs/schema/job_schema.json +0 -42
- ads/jobs/schema/runtime_schema.json +0 -183
- ads/jobs/schema/validator.py +0 -141
- ads/jobs/serializer.py +0 -296
- ads/jobs/templates/__init__.py +0 -5
- ads/jobs/templates/container.py +0 -6
- ads/jobs/templates/driver_notebook.py +0 -177
- ads/jobs/templates/driver_oci.py +0 -500
- ads/jobs/templates/driver_python.py +0 -48
- ads/jobs/templates/driver_pytorch.py +0 -852
- ads/jobs/templates/driver_utils.py +0 -615
- ads/jobs/templates/hostname_from_env.c +0 -55
- ads/jobs/templates/oci_metrics.py +0 -181
- ads/jobs/utils.py +0 -104
- ads/llm/__init__.py +0 -28
- ads/llm/autogen/__init__.py +0 -2
- ads/llm/autogen/constants.py +0 -15
- ads/llm/autogen/reports/__init__.py +0 -2
- ads/llm/autogen/reports/base.py +0 -67
- ads/llm/autogen/reports/data.py +0 -103
- ads/llm/autogen/reports/session.py +0 -526
- ads/llm/autogen/reports/templates/chat_box.html +0 -13
- ads/llm/autogen/reports/templates/chat_box_lt.html +0 -5
- ads/llm/autogen/reports/templates/chat_box_rt.html +0 -6
- ads/llm/autogen/reports/utils.py +0 -56
- ads/llm/autogen/v02/__init__.py +0 -4
- ads/llm/autogen/v02/client.py +0 -295
- ads/llm/autogen/v02/log_handlers/__init__.py +0 -2
- ads/llm/autogen/v02/log_handlers/oci_file_handler.py +0 -83
- ads/llm/autogen/v02/loggers/__init__.py +0 -6
- ads/llm/autogen/v02/loggers/metric_logger.py +0 -320
- ads/llm/autogen/v02/loggers/session_logger.py +0 -580
- ads/llm/autogen/v02/loggers/utils.py +0 -86
- ads/llm/autogen/v02/runtime_logging.py +0 -163
- ads/llm/chain.py +0 -268
- ads/llm/chat_template.py +0 -31
- ads/llm/deploy.py +0 -63
- ads/llm/guardrails/__init__.py +0 -5
- ads/llm/guardrails/base.py +0 -442
- ads/llm/guardrails/huggingface.py +0 -44
- ads/llm/langchain/__init__.py +0 -5
- ads/llm/langchain/plugins/__init__.py +0 -5
- ads/llm/langchain/plugins/chat_models/__init__.py +0 -5
- ads/llm/langchain/plugins/chat_models/oci_data_science.py +0 -1027
- ads/llm/langchain/plugins/embeddings/__init__.py +0 -4
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +0 -184
- ads/llm/langchain/plugins/llms/__init__.py +0 -5
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +0 -979
- ads/llm/requirements.txt +0 -3
- ads/llm/serialize.py +0 -219
- ads/llm/serializers/__init__.py +0 -0
- ads/llm/serializers/retrieval_qa.py +0 -153
- ads/llm/serializers/runnable_parallel.py +0 -27
- ads/llm/templates/score_chain.jinja2 +0 -155
- ads/llm/templates/tool_chat_template_hermes.jinja +0 -130
- ads/llm/templates/tool_chat_template_mistral_parallel.jinja +0 -94
- ads/model/__init__.py +0 -52
- ads/model/artifact.py +0 -573
- ads/model/artifact_downloader.py +0 -254
- ads/model/artifact_uploader.py +0 -267
- ads/model/base_properties.py +0 -238
- ads/model/common/.model-ignore +0 -66
- ads/model/common/__init__.py +0 -5
- ads/model/common/utils.py +0 -142
- ads/model/datascience_model.py +0 -2635
- ads/model/deployment/__init__.py +0 -20
- ads/model/deployment/common/__init__.py +0 -5
- ads/model/deployment/common/utils.py +0 -308
- ads/model/deployment/model_deployer.py +0 -466
- ads/model/deployment/model_deployment.py +0 -1846
- ads/model/deployment/model_deployment_infrastructure.py +0 -671
- ads/model/deployment/model_deployment_properties.py +0 -493
- ads/model/deployment/model_deployment_runtime.py +0 -838
- ads/model/extractor/__init__.py +0 -5
- ads/model/extractor/automl_extractor.py +0 -74
- ads/model/extractor/embedding_onnx_extractor.py +0 -80
- ads/model/extractor/huggingface_extractor.py +0 -88
- ads/model/extractor/keras_extractor.py +0 -84
- ads/model/extractor/lightgbm_extractor.py +0 -93
- ads/model/extractor/model_info_extractor.py +0 -114
- ads/model/extractor/model_info_extractor_factory.py +0 -105
- ads/model/extractor/pytorch_extractor.py +0 -87
- ads/model/extractor/sklearn_extractor.py +0 -112
- ads/model/extractor/spark_extractor.py +0 -89
- ads/model/extractor/tensorflow_extractor.py +0 -85
- ads/model/extractor/xgboost_extractor.py +0 -94
- ads/model/framework/__init__.py +0 -5
- ads/model/framework/automl_model.py +0 -178
- ads/model/framework/embedding_onnx_model.py +0 -438
- ads/model/framework/huggingface_model.py +0 -399
- ads/model/framework/lightgbm_model.py +0 -266
- ads/model/framework/pytorch_model.py +0 -266
- ads/model/framework/sklearn_model.py +0 -250
- ads/model/framework/spark_model.py +0 -326
- ads/model/framework/tensorflow_model.py +0 -254
- ads/model/framework/xgboost_model.py +0 -258
- ads/model/generic_model.py +0 -3518
- ads/model/model_artifact_boilerplate/README.md +0 -381
- ads/model/model_artifact_boilerplate/__init__.py +0 -5
- ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +0 -5
- ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +0 -427
- ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +0 -2
- ads/model/model_artifact_boilerplate/runtime.yaml +0 -7
- ads/model/model_artifact_boilerplate/score.py +0 -61
- ads/model/model_file_description_schema.json +0 -68
- ads/model/model_introspect.py +0 -331
- ads/model/model_metadata.py +0 -1810
- ads/model/model_metadata_mixin.py +0 -460
- ads/model/model_properties.py +0 -63
- ads/model/model_version_set.py +0 -739
- ads/model/runtime/__init__.py +0 -5
- ads/model/runtime/env_info.py +0 -306
- ads/model/runtime/model_deployment_details.py +0 -37
- ads/model/runtime/model_provenance_details.py +0 -58
- ads/model/runtime/runtime_info.py +0 -81
- ads/model/runtime/schemas/inference_env_info_schema.yaml +0 -16
- ads/model/runtime/schemas/model_provenance_schema.yaml +0 -36
- ads/model/runtime/schemas/training_env_info_schema.yaml +0 -16
- ads/model/runtime/utils.py +0 -201
- ads/model/serde/__init__.py +0 -5
- ads/model/serde/common.py +0 -40
- ads/model/serde/model_input.py +0 -547
- ads/model/serde/model_serializer.py +0 -1184
- ads/model/service/__init__.py +0 -5
- ads/model/service/oci_datascience_model.py +0 -1076
- ads/model/service/oci_datascience_model_deployment.py +0 -500
- ads/model/service/oci_datascience_model_version_set.py +0 -176
- ads/model/transformer/__init__.py +0 -5
- ads/model/transformer/onnx_transformer.py +0 -324
- ads/mysqldb/__init__.py +0 -5
- ads/mysqldb/mysql_db.py +0 -227
- ads/opctl/__init__.py +0 -18
- ads/opctl/anomaly_detection.py +0 -11
- ads/opctl/backend/__init__.py +0 -5
- ads/opctl/backend/ads_dataflow.py +0 -353
- ads/opctl/backend/ads_ml_job.py +0 -710
- ads/opctl/backend/ads_ml_pipeline.py +0 -164
- ads/opctl/backend/ads_model_deployment.py +0 -209
- ads/opctl/backend/base.py +0 -146
- ads/opctl/backend/local.py +0 -1053
- ads/opctl/backend/marketplace/__init__.py +0 -9
- ads/opctl/backend/marketplace/helm_helper.py +0 -173
- ads/opctl/backend/marketplace/local_marketplace.py +0 -271
- ads/opctl/backend/marketplace/marketplace_backend_runner.py +0 -71
- ads/opctl/backend/marketplace/marketplace_operator_interface.py +0 -44
- ads/opctl/backend/marketplace/marketplace_operator_runner.py +0 -24
- ads/opctl/backend/marketplace/marketplace_utils.py +0 -212
- ads/opctl/backend/marketplace/models/__init__.py +0 -5
- ads/opctl/backend/marketplace/models/bearer_token.py +0 -94
- ads/opctl/backend/marketplace/models/marketplace_type.py +0 -70
- ads/opctl/backend/marketplace/models/ocir_details.py +0 -56
- ads/opctl/backend/marketplace/prerequisite_checker.py +0 -238
- ads/opctl/cli.py +0 -707
- ads/opctl/cmds.py +0 -869
- ads/opctl/conda/__init__.py +0 -5
- ads/opctl/conda/cli.py +0 -193
- ads/opctl/conda/cmds.py +0 -749
- ads/opctl/conda/config.yaml +0 -34
- ads/opctl/conda/manifest_template.yaml +0 -13
- ads/opctl/conda/multipart_uploader.py +0 -188
- ads/opctl/conda/pack.py +0 -89
- ads/opctl/config/__init__.py +0 -5
- ads/opctl/config/base.py +0 -57
- ads/opctl/config/diagnostics/__init__.py +0 -5
- ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +0 -62
- ads/opctl/config/merger.py +0 -255
- ads/opctl/config/resolver.py +0 -297
- ads/opctl/config/utils.py +0 -79
- ads/opctl/config/validator.py +0 -17
- ads/opctl/config/versioner.py +0 -68
- ads/opctl/config/yaml_parsers/__init__.py +0 -7
- ads/opctl/config/yaml_parsers/base.py +0 -58
- ads/opctl/config/yaml_parsers/distributed/__init__.py +0 -7
- ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +0 -201
- ads/opctl/constants.py +0 -66
- ads/opctl/decorator/__init__.py +0 -5
- ads/opctl/decorator/common.py +0 -129
- ads/opctl/diagnostics/__init__.py +0 -5
- ads/opctl/diagnostics/__main__.py +0 -25
- ads/opctl/diagnostics/check_distributed_job_requirements.py +0 -212
- ads/opctl/diagnostics/check_requirements.py +0 -144
- ads/opctl/diagnostics/requirement_exception.py +0 -9
- ads/opctl/distributed/README.md +0 -109
- ads/opctl/distributed/__init__.py +0 -5
- ads/opctl/distributed/certificates.py +0 -32
- ads/opctl/distributed/cli.py +0 -207
- ads/opctl/distributed/cmds.py +0 -731
- ads/opctl/distributed/common/__init__.py +0 -5
- ads/opctl/distributed/common/abstract_cluster_provider.py +0 -449
- ads/opctl/distributed/common/abstract_framework_spec_builder.py +0 -88
- ads/opctl/distributed/common/cluster_config_helper.py +0 -103
- ads/opctl/distributed/common/cluster_provider_factory.py +0 -21
- ads/opctl/distributed/common/cluster_runner.py +0 -54
- ads/opctl/distributed/common/framework_factory.py +0 -29
- ads/opctl/docker/Dockerfile.job +0 -103
- ads/opctl/docker/Dockerfile.job.arm +0 -107
- ads/opctl/docker/Dockerfile.job.gpu +0 -175
- ads/opctl/docker/base-env.yaml +0 -13
- ads/opctl/docker/cuda.repo +0 -6
- ads/opctl/docker/operator/.dockerignore +0 -0
- ads/opctl/docker/operator/Dockerfile +0 -41
- ads/opctl/docker/operator/Dockerfile.gpu +0 -85
- ads/opctl/docker/operator/cuda.repo +0 -6
- ads/opctl/docker/operator/environment.yaml +0 -8
- ads/opctl/forecast.py +0 -11
- ads/opctl/index.yaml +0 -3
- ads/opctl/model/__init__.py +0 -5
- ads/opctl/model/cli.py +0 -65
- ads/opctl/model/cmds.py +0 -73
- ads/opctl/operator/README.md +0 -4
- ads/opctl/operator/__init__.py +0 -31
- ads/opctl/operator/cli.py +0 -344
- ads/opctl/operator/cmd.py +0 -596
- ads/opctl/operator/common/__init__.py +0 -5
- ads/opctl/operator/common/backend_factory.py +0 -460
- ads/opctl/operator/common/const.py +0 -27
- ads/opctl/operator/common/data/synthetic.csv +0 -16001
- ads/opctl/operator/common/dictionary_merger.py +0 -148
- ads/opctl/operator/common/errors.py +0 -42
- ads/opctl/operator/common/operator_config.py +0 -99
- ads/opctl/operator/common/operator_loader.py +0 -811
- ads/opctl/operator/common/operator_schema.yaml +0 -130
- ads/opctl/operator/common/operator_yaml_generator.py +0 -152
- ads/opctl/operator/common/utils.py +0 -208
- ads/opctl/operator/lowcode/__init__.py +0 -5
- ads/opctl/operator/lowcode/anomaly/MLoperator +0 -16
- ads/opctl/operator/lowcode/anomaly/README.md +0 -207
- ads/opctl/operator/lowcode/anomaly/__init__.py +0 -5
- ads/opctl/operator/lowcode/anomaly/__main__.py +0 -103
- ads/opctl/operator/lowcode/anomaly/cmd.py +0 -35
- ads/opctl/operator/lowcode/anomaly/const.py +0 -167
- ads/opctl/operator/lowcode/anomaly/environment.yaml +0 -10
- ads/opctl/operator/lowcode/anomaly/model/__init__.py +0 -5
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +0 -146
- ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +0 -162
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +0 -99
- ads/opctl/operator/lowcode/anomaly/model/autots.py +0 -115
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +0 -404
- ads/opctl/operator/lowcode/anomaly/model/factory.py +0 -110
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +0 -78
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +0 -78
- ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +0 -120
- ads/opctl/operator/lowcode/anomaly/model/tods.py +0 -119
- ads/opctl/operator/lowcode/anomaly/operator_config.py +0 -127
- ads/opctl/operator/lowcode/anomaly/schema.yaml +0 -401
- ads/opctl/operator/lowcode/anomaly/utils.py +0 -88
- ads/opctl/operator/lowcode/common/__init__.py +0 -5
- ads/opctl/operator/lowcode/common/const.py +0 -10
- ads/opctl/operator/lowcode/common/data.py +0 -116
- ads/opctl/operator/lowcode/common/errors.py +0 -47
- ads/opctl/operator/lowcode/common/transformations.py +0 -296
- ads/opctl/operator/lowcode/common/utils.py +0 -293
- ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +0 -13
- ads/opctl/operator/lowcode/feature_store_marketplace/README.md +0 -30
- ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +0 -5
- ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +0 -116
- ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +0 -85
- ads/opctl/operator/lowcode/feature_store_marketplace/const.py +0 -15
- ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +0 -4
- ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +0 -32
- ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +0 -43
- ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +0 -120
- ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +0 -34
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +0 -386
- ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +0 -160
- ads/opctl/operator/lowcode/forecast/MLoperator +0 -25
- ads/opctl/operator/lowcode/forecast/README.md +0 -209
- ads/opctl/operator/lowcode/forecast/__init__.py +0 -5
- ads/opctl/operator/lowcode/forecast/__main__.py +0 -89
- ads/opctl/operator/lowcode/forecast/cmd.py +0 -40
- ads/opctl/operator/lowcode/forecast/const.py +0 -92
- ads/opctl/operator/lowcode/forecast/environment.yaml +0 -20
- ads/opctl/operator/lowcode/forecast/errors.py +0 -26
- ads/opctl/operator/lowcode/forecast/model/__init__.py +0 -5
- ads/opctl/operator/lowcode/forecast/model/arima.py +0 -279
- ads/opctl/operator/lowcode/forecast/model/automlx.py +0 -542
- ads/opctl/operator/lowcode/forecast/model/autots.py +0 -312
- ads/opctl/operator/lowcode/forecast/model/base_model.py +0 -863
- ads/opctl/operator/lowcode/forecast/model/factory.py +0 -106
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +0 -492
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +0 -243
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +0 -486
- ads/opctl/operator/lowcode/forecast/model/prophet.py +0 -445
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +0 -244
- ads/opctl/operator/lowcode/forecast/operator_config.py +0 -234
- ads/opctl/operator/lowcode/forecast/schema.yaml +0 -506
- ads/opctl/operator/lowcode/forecast/utils.py +0 -413
- ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +0 -7
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +0 -285
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +0 -246
- ads/opctl/operator/lowcode/pii/MLoperator +0 -17
- ads/opctl/operator/lowcode/pii/README.md +0 -208
- ads/opctl/operator/lowcode/pii/__init__.py +0 -5
- ads/opctl/operator/lowcode/pii/__main__.py +0 -78
- ads/opctl/operator/lowcode/pii/cmd.py +0 -39
- ads/opctl/operator/lowcode/pii/constant.py +0 -84
- ads/opctl/operator/lowcode/pii/environment.yaml +0 -17
- ads/opctl/operator/lowcode/pii/errors.py +0 -27
- ads/opctl/operator/lowcode/pii/model/__init__.py +0 -5
- ads/opctl/operator/lowcode/pii/model/factory.py +0 -82
- ads/opctl/operator/lowcode/pii/model/guardrails.py +0 -167
- ads/opctl/operator/lowcode/pii/model/pii.py +0 -145
- ads/opctl/operator/lowcode/pii/model/processor/__init__.py +0 -34
- ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +0 -34
- ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +0 -35
- ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +0 -225
- ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +0 -73
- ads/opctl/operator/lowcode/pii/model/processor/remover.py +0 -26
- ads/opctl/operator/lowcode/pii/model/report.py +0 -487
- ads/opctl/operator/lowcode/pii/operator_config.py +0 -95
- ads/opctl/operator/lowcode/pii/schema.yaml +0 -108
- ads/opctl/operator/lowcode/pii/utils.py +0 -43
- ads/opctl/operator/lowcode/recommender/MLoperator +0 -16
- ads/opctl/operator/lowcode/recommender/README.md +0 -206
- ads/opctl/operator/lowcode/recommender/__init__.py +0 -5
- ads/opctl/operator/lowcode/recommender/__main__.py +0 -82
- ads/opctl/operator/lowcode/recommender/cmd.py +0 -33
- ads/opctl/operator/lowcode/recommender/constant.py +0 -30
- ads/opctl/operator/lowcode/recommender/environment.yaml +0 -11
- ads/opctl/operator/lowcode/recommender/model/base_model.py +0 -212
- ads/opctl/operator/lowcode/recommender/model/factory.py +0 -56
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +0 -25
- ads/opctl/operator/lowcode/recommender/model/svd.py +0 -106
- ads/opctl/operator/lowcode/recommender/operator_config.py +0 -81
- ads/opctl/operator/lowcode/recommender/schema.yaml +0 -265
- ads/opctl/operator/lowcode/recommender/utils.py +0 -13
- ads/opctl/operator/runtime/__init__.py +0 -5
- ads/opctl/operator/runtime/const.py +0 -17
- ads/opctl/operator/runtime/container_runtime_schema.yaml +0 -50
- ads/opctl/operator/runtime/marketplace_runtime.py +0 -50
- ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +0 -21
- ads/opctl/operator/runtime/python_runtime_schema.yaml +0 -21
- ads/opctl/operator/runtime/runtime.py +0 -115
- ads/opctl/schema.yaml.yml +0 -36
- ads/opctl/script.py +0 -40
- ads/opctl/spark/__init__.py +0 -5
- ads/opctl/spark/cli.py +0 -43
- ads/opctl/spark/cmds.py +0 -147
- ads/opctl/templates/diagnostic_report_template.jinja2 +0 -102
- ads/opctl/utils.py +0 -344
- ads/oracledb/__init__.py +0 -5
- ads/oracledb/oracle_db.py +0 -346
- ads/pipeline/__init__.py +0 -39
- ads/pipeline/ads_pipeline.py +0 -2279
- ads/pipeline/ads_pipeline_run.py +0 -772
- ads/pipeline/ads_pipeline_step.py +0 -605
- ads/pipeline/builders/__init__.py +0 -5
- ads/pipeline/builders/infrastructure/__init__.py +0 -5
- ads/pipeline/builders/infrastructure/custom_script.py +0 -32
- ads/pipeline/cli.py +0 -119
- ads/pipeline/extension.py +0 -291
- ads/pipeline/schema/__init__.py +0 -5
- ads/pipeline/schema/cs_step_schema.json +0 -35
- ads/pipeline/schema/ml_step_schema.json +0 -31
- ads/pipeline/schema/pipeline_schema.json +0 -71
- ads/pipeline/visualizer/__init__.py +0 -5
- ads/pipeline/visualizer/base.py +0 -570
- ads/pipeline/visualizer/graph_renderer.py +0 -272
- ads/pipeline/visualizer/text_renderer.py +0 -84
- ads/secrets/__init__.py +0 -11
- ads/secrets/adb.py +0 -386
- ads/secrets/auth_token.py +0 -86
- ads/secrets/big_data_service.py +0 -365
- ads/secrets/mysqldb.py +0 -149
- ads/secrets/oracledb.py +0 -160
- ads/secrets/secrets.py +0 -407
- ads/telemetry/__init__.py +0 -7
- ads/telemetry/base.py +0 -69
- ads/telemetry/client.py +0 -125
- ads/telemetry/telemetry.py +0 -257
- ads/templates/dataflow_pyspark.jinja2 +0 -13
- ads/templates/dataflow_sparksql.jinja2 +0 -22
- ads/templates/func.jinja2 +0 -20
- ads/templates/schemas/openapi.json +0 -1740
- ads/templates/score-pkl.jinja2 +0 -173
- ads/templates/score.jinja2 +0 -322
- ads/templates/score_embedding_onnx.jinja2 +0 -202
- ads/templates/score_generic.jinja2 +0 -165
- ads/templates/score_huggingface_pipeline.jinja2 +0 -217
- ads/templates/score_lightgbm.jinja2 +0 -185
- ads/templates/score_onnx.jinja2 +0 -407
- ads/templates/score_onnx_new.jinja2 +0 -473
- ads/templates/score_oracle_automl.jinja2 +0 -185
- ads/templates/score_pyspark.jinja2 +0 -154
- ads/templates/score_pytorch.jinja2 +0 -219
- ads/templates/score_scikit-learn.jinja2 +0 -184
- ads/templates/score_tensorflow.jinja2 +0 -184
- ads/templates/score_xgboost.jinja2 +0 -178
- ads/text_dataset/__init__.py +0 -5
- ads/text_dataset/backends.py +0 -211
- ads/text_dataset/dataset.py +0 -445
- ads/text_dataset/extractor.py +0 -207
- ads/text_dataset/options.py +0 -53
- ads/text_dataset/udfs.py +0 -22
- ads/text_dataset/utils.py +0 -49
- ads/type_discovery/__init__.py +0 -9
- ads/type_discovery/abstract_detector.py +0 -21
- ads/type_discovery/constant_detector.py +0 -41
- ads/type_discovery/continuous_detector.py +0 -54
- ads/type_discovery/credit_card_detector.py +0 -99
- ads/type_discovery/datetime_detector.py +0 -92
- ads/type_discovery/discrete_detector.py +0 -118
- ads/type_discovery/document_detector.py +0 -146
- ads/type_discovery/ip_detector.py +0 -68
- ads/type_discovery/latlon_detector.py +0 -90
- ads/type_discovery/phone_number_detector.py +0 -63
- ads/type_discovery/type_discovery_driver.py +0 -87
- ads/type_discovery/typed_feature.py +0 -594
- ads/type_discovery/unknown_detector.py +0 -41
- ads/type_discovery/zipcode_detector.py +0 -48
- ads/vault/__init__.py +0 -7
- ads/vault/vault.py +0 -237
- oracle_ads-2.13.7.dist-info/RECORD +0 -858
- {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/licenses/LICENSE.txt +0 -0
ads/opctl/conda/cmds.py
DELETED
@@ -1,749 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python
|
2
|
-
# -*- coding: utf-8; -*-
|
3
|
-
|
4
|
-
# Copyright (c) 2022 Oracle and/or its affiliates.
|
5
|
-
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
|
-
|
7
|
-
import datetime
|
8
|
-
import os
|
9
|
-
import shutil
|
10
|
-
import subprocess
|
11
|
-
import shlex
|
12
|
-
import json
|
13
|
-
import glob
|
14
|
-
from typing import Dict
|
15
|
-
|
16
|
-
import click
|
17
|
-
import yaml
|
18
|
-
|
19
|
-
from datetime import datetime
|
20
|
-
import ocifs
|
21
|
-
|
22
|
-
from ads.common.oci_client import OCIClientFactory
|
23
|
-
from ads.common.auth import create_signer
|
24
|
-
from ads.common.decorator.runtime_dependency import (
|
25
|
-
runtime_dependency,
|
26
|
-
OptionalDependency,
|
27
|
-
)
|
28
|
-
from ads.config import NO_CONTAINER
|
29
|
-
|
30
|
-
from ads.opctl.constants import (
|
31
|
-
ML_JOB_GPU_IMAGE,
|
32
|
-
ML_JOB_IMAGE,
|
33
|
-
DEFAULT_IMAGE_HOME_DIR,
|
34
|
-
DEFAULT_IMAGE_CONDA_DIR,
|
35
|
-
DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR,
|
36
|
-
)
|
37
|
-
from ads.opctl import logger
|
38
|
-
from ads.opctl.config.utils import read_from_ini
|
39
|
-
from ads.opctl.utils import (
|
40
|
-
parse_conda_uri,
|
41
|
-
run_container,
|
42
|
-
get_docker_client,
|
43
|
-
is_in_notebook_session,
|
44
|
-
run_command,
|
45
|
-
)
|
46
|
-
from ads.opctl.config.base import ConfigProcessor
|
47
|
-
from ads.opctl.config.merger import ConfigMerger
|
48
|
-
from ads.opctl.conda.multipart_uploader import MultiPartUploader
|
49
|
-
import tempfile
|
50
|
-
|
51
|
-
|
52
|
-
def _fetch_manifest_template() -> Dict:
|
53
|
-
manifest_template_file = os.path.join(
|
54
|
-
os.path.dirname(os.path.abspath(__file__)), "manifest_template.yaml"
|
55
|
-
)
|
56
|
-
with open(manifest_template_file) as manifest_file:
|
57
|
-
manifest_template = yaml.safe_load(manifest_file)
|
58
|
-
return manifest_template
|
59
|
-
|
60
|
-
|
61
|
-
@runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
|
62
|
-
def _check_job_image_exists(gpu: bool) -> None:
|
63
|
-
if gpu:
|
64
|
-
image = ML_JOB_GPU_IMAGE
|
65
|
-
else:
|
66
|
-
image = ML_JOB_IMAGE
|
67
|
-
try:
|
68
|
-
client = get_docker_client()
|
69
|
-
client.api.inspect_image(image)
|
70
|
-
except docker.errors.ImageNotFound:
|
71
|
-
if gpu:
|
72
|
-
cmd = "`ads opctl build-image -g job-local`"
|
73
|
-
else:
|
74
|
-
cmd = "`ads opctl build-image job-local`"
|
75
|
-
raise RuntimeError(
|
76
|
-
f"Please run {cmd} to build a local image for Jobs development first."
|
77
|
-
)
|
78
|
-
|
79
|
-
|
80
|
-
def _get_name(name: str, env_file: str) -> str:
|
81
|
-
if not name and env_file:
|
82
|
-
with open(env_file) as f:
|
83
|
-
name = yaml.safe_load(f.read()).get("manifest").get("name", None)
|
84
|
-
if not name:
|
85
|
-
raise ValueError(
|
86
|
-
"Either specify environment name in environment yaml or with `--name`."
|
87
|
-
)
|
88
|
-
return name
|
89
|
-
|
90
|
-
|
91
|
-
def create(**kwargs) -> str:
|
92
|
-
p = ConfigProcessor().step(ConfigMerger, **kwargs)
|
93
|
-
exec_config = p.config["execution"]
|
94
|
-
name = _get_name(exec_config.get("name"), exec_config.get("environment_file"))
|
95
|
-
_create(
|
96
|
-
name=name,
|
97
|
-
version=exec_config["version"],
|
98
|
-
env_file=exec_config["environment_file"],
|
99
|
-
conda_pack_folder=exec_config["conda_pack_folder"],
|
100
|
-
gpu=exec_config.get("gpu", False),
|
101
|
-
overwrite=exec_config.get("overwrite", False),
|
102
|
-
)
|
103
|
-
|
104
|
-
|
105
|
-
def _create(
|
106
|
-
name: str,
|
107
|
-
version: str,
|
108
|
-
env_file: str,
|
109
|
-
conda_pack_folder: str,
|
110
|
-
gpu: bool,
|
111
|
-
overwrite: bool,
|
112
|
-
prepare_publish: bool = False,
|
113
|
-
) -> str:
|
114
|
-
"""Create a conda pack given an environment yaml file under conda pack folder specified.
|
115
|
-
|
116
|
-
Parameters
|
117
|
-
----------
|
118
|
-
slug : str
|
119
|
-
slug of the conda pack
|
120
|
-
env_file : str
|
121
|
-
path to conda environment yaml
|
122
|
-
conda_pack_folder : str
|
123
|
-
path to conda pack folder
|
124
|
-
gpu : bool
|
125
|
-
whether to build against GPU image
|
126
|
-
overwrite : bool
|
127
|
-
whether to overwrite existing pack of the same slug
|
128
|
-
prepare_pubish : bool
|
129
|
-
whether to create conda pack archive after conda pack is created
|
130
|
-
|
131
|
-
Raises
|
132
|
-
------
|
133
|
-
FileNotFoundError
|
134
|
-
Environment file not found
|
135
|
-
RuntimeError
|
136
|
-
Environment file may be inproperly removed
|
137
|
-
RuntimeError
|
138
|
-
Creating pack failed
|
139
|
-
|
140
|
-
Return
|
141
|
-
------
|
142
|
-
str:
|
143
|
-
slug of the environment
|
144
|
-
"""
|
145
|
-
|
146
|
-
if not os.path.exists(env_file):
|
147
|
-
raise FileNotFoundError(f"Environment file {env_file} is not found.")
|
148
|
-
|
149
|
-
conda_dep = None
|
150
|
-
with open(env_file) as mfile:
|
151
|
-
conda_dep = yaml.safe_load(mfile.read())
|
152
|
-
# If manifest exists in the environment.yaml file, use that
|
153
|
-
manifest = conda_dep.get("manifest", {})
|
154
|
-
slug = manifest.get(
|
155
|
-
"slug", f"{name}_v{version}".replace(" ", "").replace(".", "_").lower()
|
156
|
-
)
|
157
|
-
pack_folder_path = os.path.join(
|
158
|
-
os.path.abspath(os.path.expanduser(conda_pack_folder)), slug
|
159
|
-
)
|
160
|
-
|
161
|
-
if os.path.exists(pack_folder_path):
|
162
|
-
overwrite = overwrite or click.confirm(
|
163
|
-
f"Conda pack with slug {slug} already exists. Do you wish to overwrite?"
|
164
|
-
)
|
165
|
-
if overwrite:
|
166
|
-
if (
|
167
|
-
os.path.commonpath(
|
168
|
-
[pack_folder_path, os.path.abspath(os.path.expanduser(env_file))]
|
169
|
-
)
|
170
|
-
== pack_folder_path
|
171
|
-
):
|
172
|
-
raise RuntimeError(
|
173
|
-
f"Environment file {os.path.abspath(os.path.expanduser(env_file))} is in {pack_folder_path}. Overwriting this folder will remove the file."
|
174
|
-
)
|
175
|
-
shutil.rmtree(pack_folder_path)
|
176
|
-
else:
|
177
|
-
return
|
178
|
-
|
179
|
-
os.makedirs(pack_folder_path, exist_ok=True)
|
180
|
-
|
181
|
-
logger.info(
|
182
|
-
f"Preparing manifest. Manifest in the environment: {conda_dep.get('manifest')}"
|
183
|
-
)
|
184
|
-
manifest_template = _fetch_manifest_template()
|
185
|
-
if "name" not in manifest:
|
186
|
-
manifest_template["manifest"]["name"] = name
|
187
|
-
manifest_template["manifest"]["slug"] = slug
|
188
|
-
if "type" not in manifest:
|
189
|
-
logger.info("Setting manifest to published")
|
190
|
-
manifest_template["manifest"]["type"] = "published"
|
191
|
-
if "version" not in manifest:
|
192
|
-
manifest_template["manifest"]["version"] = version
|
193
|
-
manifest_template["manifest"]["arch_type"] = "GPU" if gpu else "CPU"
|
194
|
-
|
195
|
-
manifest_template["manifest"]["create_date"] = datetime.utcnow().strftime(
|
196
|
-
"%a, %b %d, %Y, %H:%M:%S %Z UTC"
|
197
|
-
)
|
198
|
-
|
199
|
-
if not "manifest_version" in manifest:
|
200
|
-
manifest_template["manifest"]["manifest_version"] = "1.0"
|
201
|
-
|
202
|
-
logger.info(f"Creating conda environment {slug}")
|
203
|
-
manifest_dict = {
|
204
|
-
k: manifest_template["manifest"][k]
|
205
|
-
for k in manifest_template["manifest"]
|
206
|
-
if manifest_template["manifest"][k]
|
207
|
-
}
|
208
|
-
if "manifest" in conda_dep:
|
209
|
-
conda_dep["manifest"].update(manifest_dict)
|
210
|
-
else:
|
211
|
-
conda_dep["manifest"] = manifest_dict
|
212
|
-
logger.info(f"Updated conda environment manifest: {conda_dep.get('manifest')}")
|
213
|
-
|
214
|
-
if is_in_notebook_session() or NO_CONTAINER:
|
215
|
-
command = f"conda env create --prefix {pack_folder_path} --file {os.path.abspath(os.path.expanduser(env_file))}"
|
216
|
-
proc = run_command(command, shell=True)
|
217
|
-
if proc.returncode != 0:
|
218
|
-
raise RuntimeError(
|
219
|
-
f"Failed to create conda environment. (exit code {proc.returncode})"
|
220
|
-
)
|
221
|
-
else:
|
222
|
-
_check_job_image_exists(gpu)
|
223
|
-
docker_pack_folder_path = os.path.join(DEFAULT_IMAGE_HOME_DIR, slug)
|
224
|
-
docker_env_file_path = os.path.join(
|
225
|
-
DEFAULT_IMAGE_HOME_DIR, os.path.basename(env_file)
|
226
|
-
)
|
227
|
-
|
228
|
-
create_command = f"conda env create --prefix {docker_pack_folder_path} --file {docker_env_file_path}"
|
229
|
-
|
230
|
-
volumes = {
|
231
|
-
pack_folder_path: {"bind": docker_pack_folder_path},
|
232
|
-
os.path.abspath(os.path.expanduser(env_file)): {
|
233
|
-
"bind": docker_env_file_path
|
234
|
-
},
|
235
|
-
}
|
236
|
-
|
237
|
-
if gpu:
|
238
|
-
image = ML_JOB_GPU_IMAGE
|
239
|
-
else:
|
240
|
-
image = ML_JOB_IMAGE
|
241
|
-
try:
|
242
|
-
if prepare_publish:
|
243
|
-
tmp_file = tempfile.NamedTemporaryFile(suffix=".yaml")
|
244
|
-
# Save the manifest in the temp file that can be mounted inside the container so that archiving will work
|
245
|
-
with open(tmp_file.name, "w") as f:
|
246
|
-
yaml.safe_dump(conda_dep, f)
|
247
|
-
|
248
|
-
pack_script = os.path.join(
|
249
|
-
os.path.dirname(os.path.abspath(__file__)), "pack.py"
|
250
|
-
)
|
251
|
-
pack_command = f"python {os.path.join(DEFAULT_IMAGE_HOME_DIR, 'pack.py')} --conda-path {docker_pack_folder_path} --manifest-location {os.path.join(DEFAULT_IMAGE_HOME_DIR, 'manifest.yaml')}"
|
252
|
-
|
253
|
-
# add pack script and manifest file to the mount so that archive can be created in the same container run
|
254
|
-
condapack_script = {
|
255
|
-
pack_script: {
|
256
|
-
"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, "pack.py")
|
257
|
-
},
|
258
|
-
tmp_file.name: {
|
259
|
-
"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, "manifest.yaml")
|
260
|
-
},
|
261
|
-
}
|
262
|
-
volumes = {
|
263
|
-
**volumes,
|
264
|
-
**condapack_script,
|
265
|
-
} # | not supported in python 3.8
|
266
|
-
|
267
|
-
run_container(
|
268
|
-
image=image,
|
269
|
-
bind_volumes=volumes,
|
270
|
-
entrypoint="/bin/bash -c ",
|
271
|
-
env_vars={},
|
272
|
-
command=f" '{create_command} && {pack_command}'",
|
273
|
-
)
|
274
|
-
else:
|
275
|
-
run_container(
|
276
|
-
image=image,
|
277
|
-
bind_volumes=volumes,
|
278
|
-
env_vars={},
|
279
|
-
command=create_command,
|
280
|
-
)
|
281
|
-
except Exception:
|
282
|
-
if os.path.exists(pack_folder_path):
|
283
|
-
shutil.rmtree(pack_folder_path)
|
284
|
-
raise RuntimeError(f"Could not create environment {slug}.")
|
285
|
-
|
286
|
-
# Save the manifest file inside the host machine, where the conda environment is saved.
|
287
|
-
manifest_location = f"{os.path.join(pack_folder_path, slug)}_manifest.yaml"
|
288
|
-
with open(manifest_location, "w") as mfile:
|
289
|
-
yaml.safe_dump(conda_dep, mfile)
|
290
|
-
|
291
|
-
logger.info(f"Environment `{slug}` setup complete.")
|
292
|
-
print(f"Pack {slug} created under {pack_folder_path}.")
|
293
|
-
|
294
|
-
return slug
|
295
|
-
|
296
|
-
|
297
|
-
def _fetch_pack_path(
|
298
|
-
slug: str,
|
299
|
-
conda_pack_os_prefix: str,
|
300
|
-
oci_config: str,
|
301
|
-
oci_profile: str,
|
302
|
-
auth_type: str,
|
303
|
-
) -> str:
|
304
|
-
oci_auth = create_signer(auth_type, oci_config, oci_profile)
|
305
|
-
fs = ocifs.OCIFileSystem(**oci_auth)
|
306
|
-
fnames = fs.ls(conda_pack_os_prefix, detail=True, refresh=True)
|
307
|
-
for f in fnames:
|
308
|
-
if os.path.basename(f["name"]) == slug:
|
309
|
-
return f"oci://{f['name']}"
|
310
|
-
elif f["type"] == "directory":
|
311
|
-
path = _fetch_pack_path(
|
312
|
-
slug, f"oci://{f['name']}", oci_config, oci_profile, auth_type
|
313
|
-
)
|
314
|
-
if path:
|
315
|
-
return path
|
316
|
-
return None
|
317
|
-
|
318
|
-
|
319
|
-
def install(**kwargs) -> None:
|
320
|
-
p = ConfigProcessor().step(ConfigMerger, **kwargs)
|
321
|
-
exec_config = p.config["execution"]
|
322
|
-
if exec_config.get("slug") and exec_config.get("conda_pack_os_prefix"):
|
323
|
-
conda_uri = _fetch_pack_path(
|
324
|
-
exec_config["slug"],
|
325
|
-
exec_config["conda_pack_os_prefix"],
|
326
|
-
exec_config.get("oci_config"),
|
327
|
-
exec_config.get("oci_profile"),
|
328
|
-
exec_config.get("auth"),
|
329
|
-
)
|
330
|
-
if not conda_uri:
|
331
|
-
raise FileNotFoundError(
|
332
|
-
f"{exec_config['slug']} not found under {exec_config['conda_pack_os_prefix']}"
|
333
|
-
)
|
334
|
-
elif exec_config.get("conda_uri"):
|
335
|
-
conda_uri = exec_config["conda_uri"]
|
336
|
-
else:
|
337
|
-
raise ValueError(
|
338
|
-
"Either `--conda-uri`, or `--slug` and `--conda-pack-os-prefix` has to be specified."
|
339
|
-
)
|
340
|
-
_install(
|
341
|
-
conda_uri,
|
342
|
-
os.path.abspath(os.path.expanduser(exec_config["conda_pack_folder"])),
|
343
|
-
exec_config.get("oci_config"),
|
344
|
-
exec_config.get("oci_profile"),
|
345
|
-
overwrite=exec_config.get("overwrite", False),
|
346
|
-
debug=kwargs.get("debug", False),
|
347
|
-
auth_type=exec_config.get("auth"),
|
348
|
-
)
|
349
|
-
|
350
|
-
|
351
|
-
def _install(
|
352
|
-
conda_uri: str,
|
353
|
-
conda_pack_folder: str,
|
354
|
-
oci_config: str = None,
|
355
|
-
oci_profile: str = None,
|
356
|
-
overwrite: bool = False,
|
357
|
-
debug: bool = False,
|
358
|
-
auth_type: str = None,
|
359
|
-
) -> None:
|
360
|
-
"""
|
361
|
-
Install conda pack.
|
362
|
-
|
363
|
-
Parameters
|
364
|
-
----------
|
365
|
-
conda_uri: str
|
366
|
-
OCI object storage uri to conda pack
|
367
|
-
conda_pack_folder: str
|
368
|
-
local folder to save conda packs
|
369
|
-
oci_config: str
|
370
|
-
path to OCI config file
|
371
|
-
oci_profile: str
|
372
|
-
OCI config profile
|
373
|
-
overwrite: bool
|
374
|
-
whether to overwrite existing pack
|
375
|
-
debug: bool
|
376
|
-
whether to turn on debug mode
|
377
|
-
auth_type : str
|
378
|
-
authentication method
|
379
|
-
Returns
|
380
|
-
-------
|
381
|
-
None
|
382
|
-
"""
|
383
|
-
ns, bucket, path, slug = parse_conda_uri(conda_uri)
|
384
|
-
if bucket == "service-conda-packs":
|
385
|
-
raise ValueError(
|
386
|
-
"Download service conda pack is not allowed. Only custom conda pack can be downloaded to local machine. You need to publish it to your own bucket first."
|
387
|
-
)
|
388
|
-
os.makedirs(conda_pack_folder, exist_ok=True)
|
389
|
-
pack_path = os.path.join(os.path.expanduser(conda_pack_folder), slug + ".tar.gz")
|
390
|
-
pack_folder_path = os.path.join(os.path.expanduser(conda_pack_folder), slug)
|
391
|
-
|
392
|
-
if not (is_in_notebook_session() or NO_CONTAINER):
|
393
|
-
_check_job_image_exists(gpu=False)
|
394
|
-
|
395
|
-
while os.path.exists(pack_folder_path):
|
396
|
-
if overwrite:
|
397
|
-
break
|
398
|
-
else:
|
399
|
-
ans = click.prompt(
|
400
|
-
f"conda pack with slug {slug} already exists. Enter a new name or 'o' for overwrite:",
|
401
|
-
default="o",
|
402
|
-
)
|
403
|
-
if ans == "o":
|
404
|
-
overwrite = True
|
405
|
-
break
|
406
|
-
else:
|
407
|
-
slug = ans
|
408
|
-
pack_path = os.path.join(
|
409
|
-
os.path.expanduser(conda_pack_folder), slug + ".tar.gz"
|
410
|
-
)
|
411
|
-
pack_folder_path = os.path.join(
|
412
|
-
os.path.expanduser(conda_pack_folder), slug
|
413
|
-
)
|
414
|
-
|
415
|
-
if oci_config is None or oci_profile is None:
|
416
|
-
download_command = [
|
417
|
-
"oci",
|
418
|
-
"os",
|
419
|
-
"object",
|
420
|
-
"get",
|
421
|
-
"--name",
|
422
|
-
f"{path}",
|
423
|
-
"-bn",
|
424
|
-
bucket,
|
425
|
-
"-ns",
|
426
|
-
ns,
|
427
|
-
"--file",
|
428
|
-
pack_path,
|
429
|
-
"--auth",
|
430
|
-
auth_type,
|
431
|
-
]
|
432
|
-
else:
|
433
|
-
oci_config = os.path.abspath(os.path.expanduser(oci_config))
|
434
|
-
parser = read_from_ini(os.path.expanduser(oci_config))
|
435
|
-
if oci_profile not in parser:
|
436
|
-
raise ValueError(f"PROFILE {oci_profile} not found in {oci_config}.")
|
437
|
-
download_command = [
|
438
|
-
"oci",
|
439
|
-
"os",
|
440
|
-
"object",
|
441
|
-
"get",
|
442
|
-
"--name",
|
443
|
-
f"{path}",
|
444
|
-
"-bn",
|
445
|
-
bucket,
|
446
|
-
"-ns",
|
447
|
-
ns,
|
448
|
-
"--file",
|
449
|
-
pack_path,
|
450
|
-
"--profile",
|
451
|
-
oci_profile,
|
452
|
-
"--config-file",
|
453
|
-
oci_config,
|
454
|
-
]
|
455
|
-
if debug:
|
456
|
-
download_command.append("-d")
|
457
|
-
print(" ".join([shlex.quote(c) for c in download_command]))
|
458
|
-
process = subprocess.Popen(download_command)
|
459
|
-
process.communicate()
|
460
|
-
if process.returncode != 0:
|
461
|
-
if os.path.exists(pack_path):
|
462
|
-
os.remove(pack_path)
|
463
|
-
raise RuntimeError(
|
464
|
-
f"Downloading pack failed with return code {process.returncode}. Please double check the path and make sure you have access."
|
465
|
-
)
|
466
|
-
else:
|
467
|
-
print(f"Download {conda_uri} completed")
|
468
|
-
|
469
|
-
print(f"Start unpacking {pack_path}")
|
470
|
-
os.makedirs(pack_folder_path, exist_ok=True)
|
471
|
-
|
472
|
-
process = subprocess.Popen(["tar", "-xf", pack_path, "-C", pack_folder_path])
|
473
|
-
process.communicate()
|
474
|
-
if process.returncode != 0:
|
475
|
-
shutil.rmtree(pack_folder_path)
|
476
|
-
raise Exception(
|
477
|
-
f"Error extracting {pack_path} to {pack_folder_path}. Please try again."
|
478
|
-
)
|
479
|
-
|
480
|
-
# Run the conda-unpack script to clean up prefixes
|
481
|
-
# This will fix problems related to repeated "placehold" paths.
|
482
|
-
# See https://conda.github.io/conda-pack/#commandline-usage
|
483
|
-
unpack_script = os.path.join(pack_folder_path, "bin", "conda-unpack")
|
484
|
-
if os.path.exists(unpack_script):
|
485
|
-
if is_in_notebook_session() or NO_CONTAINER:
|
486
|
-
run_command(unpack_script)
|
487
|
-
else:
|
488
|
-
volumes = {
|
489
|
-
pack_folder_path: {"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, slug)}
|
490
|
-
}
|
491
|
-
try:
|
492
|
-
run_container(
|
493
|
-
image=ML_JOB_IMAGE,
|
494
|
-
bind_volumes=volumes,
|
495
|
-
env_vars={},
|
496
|
-
command=os.path.join(
|
497
|
-
DEFAULT_IMAGE_HOME_DIR, slug, "bin/conda-unpack"
|
498
|
-
),
|
499
|
-
)
|
500
|
-
except Exception:
|
501
|
-
raise RuntimeError(f"Error unpacking environment {slug}.")
|
502
|
-
if os.path.exists(os.path.join(pack_folder_path, "spark-defaults.conf")):
|
503
|
-
if is_in_notebook_session() or NO_CONTAINER:
|
504
|
-
if not os.path.exists(DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR):
|
505
|
-
os.makedirs(DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR)
|
506
|
-
shutil.copy(
|
507
|
-
os.path.join(pack_folder_path, "spark-defaults.conf"),
|
508
|
-
DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR,
|
509
|
-
)
|
510
|
-
for file in os.listdir(pack_folder_path):
|
511
|
-
if os.path.splitext(file)[1] == ".jar":
|
512
|
-
shutil.copy(
|
513
|
-
os.path.join(pack_folder_path, file),
|
514
|
-
DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR,
|
515
|
-
)
|
516
|
-
else:
|
517
|
-
with open(os.path.join(pack_folder_path, "spark-defaults.conf")) as f:
|
518
|
-
confs = f.read()
|
519
|
-
confs = confs.replace(
|
520
|
-
DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR,
|
521
|
-
os.path.join(DEFAULT_IMAGE_CONDA_DIR, slug),
|
522
|
-
)
|
523
|
-
with open(
|
524
|
-
os.path.join(pack_folder_path, "spark-defaults.conf"), "w"
|
525
|
-
) as f:
|
526
|
-
f.write(confs)
|
527
|
-
|
528
|
-
os.remove(pack_path)
|
529
|
-
manifest_path = glob.glob(os.path.join(pack_folder_path, "*_manifest.yaml"))[0]
|
530
|
-
with open(manifest_path) as f:
|
531
|
-
env = yaml.safe_load(f.read())
|
532
|
-
env["manifest"]["pack_uri"] = conda_uri
|
533
|
-
with open(manifest_path, "w") as f:
|
534
|
-
yaml.safe_dump(env, f)
|
535
|
-
print(f"{slug} set up at {pack_folder_path}.")
|
536
|
-
|
537
|
-
|
538
|
-
def publish(**kwargs) -> None:
|
539
|
-
p = ConfigProcessor().step(ConfigMerger, **kwargs)
|
540
|
-
exec_config = p.config["execution"]
|
541
|
-
skip_archive = False
|
542
|
-
if exec_config.get("environment_file", None):
|
543
|
-
name = _get_name(exec_config.get("name"), exec_config.get("environment_file"))
|
544
|
-
slug = _create(
|
545
|
-
name=name,
|
546
|
-
version=exec_config["version"],
|
547
|
-
env_file=exec_config["environment_file"],
|
548
|
-
conda_pack_folder=exec_config["conda_pack_folder"],
|
549
|
-
gpu=exec_config.get("gpu", False),
|
550
|
-
overwrite=exec_config["overwrite"],
|
551
|
-
prepare_publish=True,
|
552
|
-
)
|
553
|
-
skip_archive = (
|
554
|
-
True # The conda pack archive is already created during create process.
|
555
|
-
)
|
556
|
-
else:
|
557
|
-
slug = exec_config.get("slug")
|
558
|
-
if not slug:
|
559
|
-
raise ValueError("Please specify slug of the conda pack via `--slug`.")
|
560
|
-
if not exec_config.get("conda_pack_os_prefix"):
|
561
|
-
raise ValueError(
|
562
|
-
"Please specify object storage path to save conda pack either via `ads opctl configure` or `--conda-pack-os-prefix`."
|
563
|
-
)
|
564
|
-
_publish(
|
565
|
-
conda_slug=slug,
|
566
|
-
conda_uri_prefix=exec_config["conda_pack_os_prefix"],
|
567
|
-
conda_pack_folder=exec_config["conda_pack_folder"],
|
568
|
-
oci_config=exec_config.get("oci_config"),
|
569
|
-
oci_profile=exec_config.get("oci_profile"),
|
570
|
-
overwrite=exec_config["overwrite"],
|
571
|
-
auth_type=exec_config["auth"],
|
572
|
-
skip_archive=skip_archive,
|
573
|
-
)
|
574
|
-
|
575
|
-
|
576
|
-
def _publish(
|
577
|
-
conda_slug: str,
|
578
|
-
conda_uri_prefix: str,
|
579
|
-
conda_pack_folder: str,
|
580
|
-
oci_config: str,
|
581
|
-
oci_profile: str,
|
582
|
-
overwrite: bool,
|
583
|
-
auth_type: str,
|
584
|
-
skip_archive: bool = False,
|
585
|
-
) -> None:
|
586
|
-
"""Publish a local conda pack to object storage location
|
587
|
-
|
588
|
-
Parameters
|
589
|
-
----------
|
590
|
-
conda_slug : str
|
591
|
-
slug of conda pack
|
592
|
-
conda_uri_prefix : str
|
593
|
-
object storage prefix to save conda pack
|
594
|
-
conda_pack_folder : str
|
595
|
-
path to local conda folder
|
596
|
-
oci_config : str
|
597
|
-
oci config file path
|
598
|
-
oci_profile : str
|
599
|
-
oci config profile
|
600
|
-
overwrite : bool
|
601
|
-
whether to overwrite existing pack of the same slug
|
602
|
-
auth_type : str
|
603
|
-
authentication method
|
604
|
-
|
605
|
-
Raises
|
606
|
-
------
|
607
|
-
FileNotFoundError
|
608
|
-
local conda pack folder not found
|
609
|
-
FileNotFoundError
|
610
|
-
manifest file not found inside conda pack
|
611
|
-
RuntimeError
|
612
|
-
IP packages found inside conda pack
|
613
|
-
RuntimeError
|
614
|
-
Packing conda failed
|
615
|
-
RuntimeError
|
616
|
-
Pack file not found
|
617
|
-
"""
|
618
|
-
ns, bucket, prefix, _ = parse_conda_uri(conda_uri_prefix)
|
619
|
-
|
620
|
-
# ======= check if conda pack and manifest exists ==============
|
621
|
-
pack_folder_path = os.path.abspath(
|
622
|
-
os.path.expanduser(os.path.join(conda_pack_folder, conda_slug))
|
623
|
-
)
|
624
|
-
if not os.path.exists(pack_folder_path):
|
625
|
-
raise FileNotFoundError(
|
626
|
-
f"Could not find environment {conda_slug} in {conda_pack_folder}."
|
627
|
-
)
|
628
|
-
paths = glob.glob(os.path.join(pack_folder_path, "*_manifest.yaml"))
|
629
|
-
if len(paths) != 1:
|
630
|
-
raise FileNotFoundError(
|
631
|
-
"Could not locate manifest file in the provided environment."
|
632
|
-
)
|
633
|
-
else:
|
634
|
-
manifest_location = paths[0]
|
635
|
-
|
636
|
-
# ===== check if pack contains IP packages =========
|
637
|
-
print(f"Loading environment information from {manifest_location}.")
|
638
|
-
with open(manifest_location) as mlf:
|
639
|
-
env = yaml.safe_load(mlf.read())
|
640
|
-
if "IP" in env["manifest"] and env["manifest"]["IP"].lower() == "y":
|
641
|
-
raise RuntimeError("This environment has IP restricted packages.")
|
642
|
-
|
643
|
-
oci_auth = create_signer(auth_type, oci_config, oci_profile)
|
644
|
-
client = OCIClientFactory(**oci_auth).object_storage
|
645
|
-
publish_slug = conda_slug
|
646
|
-
if not overwrite:
|
647
|
-
existing_packs = client.list_objects(ns, bucket, prefix=prefix).data.objects
|
648
|
-
pack_names = [os.path.basename(pack.name) for pack in existing_packs]
|
649
|
-
while publish_slug in pack_names:
|
650
|
-
ans = click.prompt(
|
651
|
-
f"{conda_slug} exists in {conda_uri_prefix}. Enter a new name or overwrite (o)",
|
652
|
-
default="o",
|
653
|
-
)
|
654
|
-
if ans == "o":
|
655
|
-
break
|
656
|
-
else:
|
657
|
-
publish_slug = "_".join(ans.lower().split(" "))
|
658
|
-
|
659
|
-
pack_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pack.py")
|
660
|
-
if not skip_archive:
|
661
|
-
if is_in_notebook_session() or NO_CONTAINER:
|
662
|
-
# Set the CONDA_PUBLISH_TYPE environment variable so that the `type` attribute inside the manifest is not changed
|
663
|
-
publish_type = os.environ.get("CONDA_PUBLISH_TYPE")
|
664
|
-
command = f"python {pack_script} --conda-path {pack_folder_path}"
|
665
|
-
if publish_type:
|
666
|
-
command = f"CONDA_PUBLISH_TYPE={publish_type} {command}"
|
667
|
-
proc = run_command(command, shell=True)
|
668
|
-
if proc.returncode != 0:
|
669
|
-
raise RuntimeError(
|
670
|
-
f"Failed to archive the conda environment. (exit code {proc.returncode})"
|
671
|
-
)
|
672
|
-
else:
|
673
|
-
volumes = {
|
674
|
-
pack_folder_path: {
|
675
|
-
"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, conda_slug)
|
676
|
-
},
|
677
|
-
pack_script: {"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, "pack.py")},
|
678
|
-
}
|
679
|
-
command = f"python {os.path.join(DEFAULT_IMAGE_HOME_DIR, 'pack.py')} --conda-path {os.path.join(DEFAULT_IMAGE_HOME_DIR, conda_slug)}"
|
680
|
-
gpu = env["manifest"]["arch_type"] == "GPU"
|
681
|
-
_check_job_image_exists(gpu)
|
682
|
-
if gpu:
|
683
|
-
image = ML_JOB_GPU_IMAGE
|
684
|
-
else:
|
685
|
-
image = ML_JOB_IMAGE
|
686
|
-
try:
|
687
|
-
run_container(
|
688
|
-
image=image, bind_volumes=volumes, env_vars={}, command=command
|
689
|
-
)
|
690
|
-
except Exception:
|
691
|
-
raise RuntimeError(f"Could not pack environment {conda_slug}.")
|
692
|
-
NOT_ALLOWED_CHARS = "@#$%^&*/"
|
693
|
-
|
694
|
-
if any(chr in conda_slug for chr in NOT_ALLOWED_CHARS):
|
695
|
-
raise ValueError(
|
696
|
-
f"Invalid conda_slug. Found {NOT_ALLOWED_CHARS} in slug name. Please use a different slug name."
|
697
|
-
)
|
698
|
-
pack_file = os.path.join(pack_folder_path, f"{conda_slug}.tar.gz")
|
699
|
-
if not os.path.exists(pack_file):
|
700
|
-
raise RuntimeError(f"Pack {pack_file} was not created.")
|
701
|
-
pack_size = round(os.path.getsize(pack_file) / 2**20, 2)
|
702
|
-
|
703
|
-
with open(manifest_location) as mlf:
|
704
|
-
env = yaml.safe_load(mlf.read())
|
705
|
-
manifest = env["manifest"]
|
706
|
-
manifest["slug"] = conda_slug
|
707
|
-
manifest["create_date"] = datetime.utcnow().strftime(
|
708
|
-
"%a, %b %d, %Y, %H:%M:%S %Z UTC"
|
709
|
-
)
|
710
|
-
manifest["size_mb"] = pack_size
|
711
|
-
ns, bucket, prefix, _ = parse_conda_uri(conda_uri_prefix)
|
712
|
-
|
713
|
-
pack_uri = os.path.join(
|
714
|
-
conda_uri_prefix,
|
715
|
-
manifest.get("arch_type", "CPU").lower(),
|
716
|
-
manifest["name"],
|
717
|
-
str(manifest["version"]),
|
718
|
-
publish_slug,
|
719
|
-
)
|
720
|
-
if os.environ.get("CONDA_PUBLISH_TYPE") != "service":
|
721
|
-
# Set these values only for published conda pack
|
722
|
-
manifest["pack_path"] = os.path.join(
|
723
|
-
prefix,
|
724
|
-
manifest.get("arch_type", "CPU").lower(),
|
725
|
-
manifest["name"],
|
726
|
-
str(manifest["version"]),
|
727
|
-
publish_slug,
|
728
|
-
)
|
729
|
-
manifest["pack_uri"] = pack_uri
|
730
|
-
else:
|
731
|
-
manifest["type"] = "published"
|
732
|
-
|
733
|
-
with open(manifest_location, "w") as f:
|
734
|
-
yaml.safe_dump(env, f)
|
735
|
-
if pack_size > 100:
|
736
|
-
MultiPartUploader(
|
737
|
-
pack_file, pack_uri, 10, oci_config, oci_profile, auth_type
|
738
|
-
).upload(manifest)
|
739
|
-
else:
|
740
|
-
with open(pack_file, "rb") as pkf:
|
741
|
-
client.put_object(
|
742
|
-
ns,
|
743
|
-
bucket,
|
744
|
-
manifest["pack_path"],
|
745
|
-
pkf,
|
746
|
-
opc_meta={"manifest": json.dumps(manifest)},
|
747
|
-
)
|
748
|
-
print(f"Conda pack {pack_uri} published.")
|
749
|
-
os.remove(pack_file)
|