oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/__init__.py +40 -0
- ads/aqua/app.py +506 -0
- ads/aqua/cli.py +96 -0
- ads/aqua/client/__init__.py +3 -0
- ads/aqua/client/client.py +836 -0
- ads/aqua/client/openai_client.py +305 -0
- ads/aqua/common/__init__.py +5 -0
- ads/aqua/common/decorator.py +125 -0
- ads/aqua/common/entities.py +269 -0
- ads/aqua/common/enums.py +122 -0
- ads/aqua/common/errors.py +109 -0
- ads/aqua/common/utils.py +1285 -0
- ads/aqua/config/__init__.py +4 -0
- ads/aqua/config/container_config.py +248 -0
- ads/aqua/config/evaluation/__init__.py +4 -0
- ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
- ads/aqua/config/utils/__init__.py +4 -0
- ads/aqua/config/utils/serializer.py +339 -0
- ads/aqua/constants.py +116 -0
- ads/aqua/data.py +14 -0
- ads/aqua/dummy_data/icon.txt +1 -0
- ads/aqua/dummy_data/oci_model_deployments.json +56 -0
- ads/aqua/dummy_data/oci_models.json +1 -0
- ads/aqua/dummy_data/readme.md +26 -0
- ads/aqua/evaluation/__init__.py +8 -0
- ads/aqua/evaluation/constants.py +53 -0
- ads/aqua/evaluation/entities.py +186 -0
- ads/aqua/evaluation/errors.py +70 -0
- ads/aqua/evaluation/evaluation.py +1814 -0
- ads/aqua/extension/__init__.py +42 -0
- ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
- ads/aqua/extension/base_handler.py +90 -0
- ads/aqua/extension/common_handler.py +121 -0
- ads/aqua/extension/common_ws_msg_handler.py +36 -0
- ads/aqua/extension/deployment_handler.py +298 -0
- ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
- ads/aqua/extension/errors.py +30 -0
- ads/aqua/extension/evaluation_handler.py +129 -0
- ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
- ads/aqua/extension/finetune_handler.py +96 -0
- ads/aqua/extension/model_handler.py +390 -0
- ads/aqua/extension/models/__init__.py +0 -0
- ads/aqua/extension/models/ws_models.py +145 -0
- ads/aqua/extension/models_ws_msg_handler.py +50 -0
- ads/aqua/extension/ui_handler.py +282 -0
- ads/aqua/extension/ui_websocket_handler.py +130 -0
- ads/aqua/extension/utils.py +133 -0
- ads/aqua/finetuning/__init__.py +7 -0
- ads/aqua/finetuning/constants.py +23 -0
- ads/aqua/finetuning/entities.py +181 -0
- ads/aqua/finetuning/finetuning.py +749 -0
- ads/aqua/model/__init__.py +8 -0
- ads/aqua/model/constants.py +60 -0
- ads/aqua/model/entities.py +385 -0
- ads/aqua/model/enums.py +32 -0
- ads/aqua/model/model.py +2114 -0
- ads/aqua/modeldeployment/__init__.py +8 -0
- ads/aqua/modeldeployment/constants.py +10 -0
- ads/aqua/modeldeployment/deployment.py +1326 -0
- ads/aqua/modeldeployment/entities.py +653 -0
- ads/aqua/modeldeployment/inference.py +74 -0
- ads/aqua/modeldeployment/utils.py +543 -0
- ads/aqua/resources/gpu_shapes_index.json +94 -0
- ads/aqua/server/__init__.py +4 -0
- ads/aqua/server/__main__.py +24 -0
- ads/aqua/server/app.py +47 -0
- ads/aqua/server/aqua_spec.yml +1291 -0
- ads/aqua/training/__init__.py +4 -0
- ads/aqua/training/exceptions.py +476 -0
- ads/aqua/ui.py +499 -0
- ads/automl/__init__.py +9 -0
- ads/automl/driver.py +330 -0
- ads/automl/provider.py +975 -0
- ads/bds/__init__.py +5 -0
- ads/bds/auth.py +127 -0
- ads/bds/big_data_service.py +255 -0
- ads/catalog/__init__.py +19 -0
- ads/catalog/model.py +1576 -0
- ads/catalog/notebook.py +461 -0
- ads/catalog/project.py +468 -0
- ads/catalog/summary.py +178 -0
- ads/common/__init__.py +11 -0
- ads/common/analyzer.py +65 -0
- ads/common/artifact/.model-ignore +63 -0
- ads/common/artifact/__init__.py +10 -0
- ads/common/auth.py +1122 -0
- ads/common/card_identifier.py +83 -0
- ads/common/config.py +647 -0
- ads/common/data.py +165 -0
- ads/common/decorator/__init__.py +9 -0
- ads/common/decorator/argument_to_case.py +88 -0
- ads/common/decorator/deprecate.py +69 -0
- ads/common/decorator/require_nonempty_arg.py +65 -0
- ads/common/decorator/runtime_dependency.py +178 -0
- ads/common/decorator/threaded.py +97 -0
- ads/common/decorator/utils.py +35 -0
- ads/common/dsc_file_system.py +303 -0
- ads/common/error.py +14 -0
- ads/common/extended_enum.py +81 -0
- ads/common/function/__init__.py +5 -0
- ads/common/function/fn_util.py +142 -0
- ads/common/function/func_conf.yaml +25 -0
- ads/common/ipython.py +76 -0
- ads/common/model.py +679 -0
- ads/common/model_artifact.py +1759 -0
- ads/common/model_artifact_schema.json +107 -0
- ads/common/model_export_util.py +664 -0
- ads/common/model_metadata.py +24 -0
- ads/common/object_storage_details.py +296 -0
- ads/common/oci_client.py +175 -0
- ads/common/oci_datascience.py +46 -0
- ads/common/oci_logging.py +1144 -0
- ads/common/oci_mixin.py +957 -0
- ads/common/oci_resource.py +136 -0
- ads/common/serializer.py +559 -0
- ads/common/utils.py +1852 -0
- ads/common/word_lists.py +1491 -0
- ads/common/work_request.py +189 -0
- ads/data_labeling/__init__.py +13 -0
- ads/data_labeling/boundingbox.py +253 -0
- ads/data_labeling/constants.py +47 -0
- ads/data_labeling/data_labeling_service.py +244 -0
- ads/data_labeling/interface/__init__.py +5 -0
- ads/data_labeling/interface/loader.py +16 -0
- ads/data_labeling/interface/parser.py +16 -0
- ads/data_labeling/interface/reader.py +23 -0
- ads/data_labeling/loader/__init__.py +5 -0
- ads/data_labeling/loader/file_loader.py +241 -0
- ads/data_labeling/metadata.py +110 -0
- ads/data_labeling/mixin/__init__.py +5 -0
- ads/data_labeling/mixin/data_labeling.py +232 -0
- ads/data_labeling/ner.py +129 -0
- ads/data_labeling/parser/__init__.py +5 -0
- ads/data_labeling/parser/dls_record_parser.py +388 -0
- ads/data_labeling/parser/export_metadata_parser.py +94 -0
- ads/data_labeling/parser/export_record_parser.py +473 -0
- ads/data_labeling/reader/__init__.py +5 -0
- ads/data_labeling/reader/dataset_reader.py +574 -0
- ads/data_labeling/reader/dls_record_reader.py +121 -0
- ads/data_labeling/reader/export_record_reader.py +62 -0
- ads/data_labeling/reader/jsonl_reader.py +75 -0
- ads/data_labeling/reader/metadata_reader.py +203 -0
- ads/data_labeling/reader/record_reader.py +263 -0
- ads/data_labeling/record.py +52 -0
- ads/data_labeling/visualizer/__init__.py +5 -0
- ads/data_labeling/visualizer/image_visualizer.py +525 -0
- ads/data_labeling/visualizer/text_visualizer.py +357 -0
- ads/database/__init__.py +5 -0
- ads/database/connection.py +338 -0
- ads/dataset/__init__.py +10 -0
- ads/dataset/capabilities.md +51 -0
- ads/dataset/classification_dataset.py +339 -0
- ads/dataset/correlation.py +226 -0
- ads/dataset/correlation_plot.py +563 -0
- ads/dataset/dask_series.py +173 -0
- ads/dataset/dataframe_transformer.py +110 -0
- ads/dataset/dataset.py +1979 -0
- ads/dataset/dataset_browser.py +360 -0
- ads/dataset/dataset_with_target.py +995 -0
- ads/dataset/exception.py +25 -0
- ads/dataset/factory.py +987 -0
- ads/dataset/feature_engineering_transformer.py +35 -0
- ads/dataset/feature_selection.py +107 -0
- ads/dataset/forecasting_dataset.py +26 -0
- ads/dataset/helper.py +1450 -0
- ads/dataset/label_encoder.py +99 -0
- ads/dataset/mixin/__init__.py +5 -0
- ads/dataset/mixin/dataset_accessor.py +134 -0
- ads/dataset/pipeline.py +58 -0
- ads/dataset/plot.py +710 -0
- ads/dataset/progress.py +86 -0
- ads/dataset/recommendation.py +297 -0
- ads/dataset/recommendation_transformer.py +502 -0
- ads/dataset/regression_dataset.py +14 -0
- ads/dataset/sampled_dataset.py +1050 -0
- ads/dataset/target.py +98 -0
- ads/dataset/timeseries.py +18 -0
- ads/dbmixin/__init__.py +5 -0
- ads/dbmixin/db_pandas_accessor.py +153 -0
- ads/environment/__init__.py +9 -0
- ads/environment/ml_runtime.py +66 -0
- ads/evaluations/README.md +14 -0
- ads/evaluations/__init__.py +109 -0
- ads/evaluations/evaluation_plot.py +983 -0
- ads/evaluations/evaluator.py +1334 -0
- ads/evaluations/statistical_metrics.py +543 -0
- ads/experiments/__init__.py +9 -0
- ads/experiments/capabilities.md +0 -0
- ads/explanations/__init__.py +21 -0
- ads/explanations/base_explainer.py +142 -0
- ads/explanations/capabilities.md +83 -0
- ads/explanations/explainer.py +190 -0
- ads/explanations/mlx_global_explainer.py +1050 -0
- ads/explanations/mlx_interface.py +386 -0
- ads/explanations/mlx_local_explainer.py +287 -0
- ads/explanations/mlx_whatif_explainer.py +201 -0
- ads/feature_engineering/__init__.py +20 -0
- ads/feature_engineering/accessor/__init__.py +5 -0
- ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
- ads/feature_engineering/accessor/mixin/__init__.py +5 -0
- ads/feature_engineering/accessor/mixin/correlation.py +166 -0
- ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
- ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
- ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
- ads/feature_engineering/accessor/mixin/utils.py +65 -0
- ads/feature_engineering/accessor/series_accessor.py +431 -0
- ads/feature_engineering/adsimage/__init__.py +5 -0
- ads/feature_engineering/adsimage/image.py +192 -0
- ads/feature_engineering/adsimage/image_reader.py +170 -0
- ads/feature_engineering/adsimage/interface/__init__.py +5 -0
- ads/feature_engineering/adsimage/interface/reader.py +19 -0
- ads/feature_engineering/adsstring/__init__.py +7 -0
- ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
- ads/feature_engineering/adsstring/string/__init__.py +8 -0
- ads/feature_engineering/data_schema.json +57 -0
- ads/feature_engineering/dataset/__init__.py +5 -0
- ads/feature_engineering/dataset/zip_code_data.py +42062 -0
- ads/feature_engineering/exceptions.py +40 -0
- ads/feature_engineering/feature_type/__init__.py +133 -0
- ads/feature_engineering/feature_type/address.py +184 -0
- ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
- ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
- ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
- ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
- ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
- ads/feature_engineering/feature_type/adsstring/string.py +258 -0
- ads/feature_engineering/feature_type/base.py +58 -0
- ads/feature_engineering/feature_type/boolean.py +183 -0
- ads/feature_engineering/feature_type/category.py +146 -0
- ads/feature_engineering/feature_type/constant.py +137 -0
- ads/feature_engineering/feature_type/continuous.py +151 -0
- ads/feature_engineering/feature_type/creditcard.py +314 -0
- ads/feature_engineering/feature_type/datetime.py +190 -0
- ads/feature_engineering/feature_type/discrete.py +134 -0
- ads/feature_engineering/feature_type/document.py +43 -0
- ads/feature_engineering/feature_type/gis.py +251 -0
- ads/feature_engineering/feature_type/handler/__init__.py +5 -0
- ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
- ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
- ads/feature_engineering/feature_type/handler/warnings.py +128 -0
- ads/feature_engineering/feature_type/integer.py +142 -0
- ads/feature_engineering/feature_type/ip_address.py +144 -0
- ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
- ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
- ads/feature_engineering/feature_type/lat_long.py +256 -0
- ads/feature_engineering/feature_type/object.py +43 -0
- ads/feature_engineering/feature_type/ordinal.py +132 -0
- ads/feature_engineering/feature_type/phone_number.py +135 -0
- ads/feature_engineering/feature_type/string.py +171 -0
- ads/feature_engineering/feature_type/text.py +93 -0
- ads/feature_engineering/feature_type/unknown.py +43 -0
- ads/feature_engineering/feature_type/zip_code.py +164 -0
- ads/feature_engineering/feature_type_manager.py +406 -0
- ads/feature_engineering/schema.py +795 -0
- ads/feature_engineering/utils.py +245 -0
- ads/feature_store/.readthedocs.yaml +19 -0
- ads/feature_store/README.md +65 -0
- ads/feature_store/__init__.py +9 -0
- ads/feature_store/common/__init__.py +0 -0
- ads/feature_store/common/enums.py +339 -0
- ads/feature_store/common/exceptions.py +18 -0
- ads/feature_store/common/spark_session_singleton.py +125 -0
- ads/feature_store/common/utils/__init__.py +0 -0
- ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
- ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
- ads/feature_store/common/utils/transformation_utils.py +82 -0
- ads/feature_store/common/utils/utility.py +403 -0
- ads/feature_store/data_validation/__init__.py +0 -0
- ads/feature_store/data_validation/great_expectation.py +129 -0
- ads/feature_store/dataset.py +1230 -0
- ads/feature_store/dataset_job.py +530 -0
- ads/feature_store/docs/Dockerfile +7 -0
- ads/feature_store/docs/Makefile +44 -0
- ads/feature_store/docs/conf.py +28 -0
- ads/feature_store/docs/requirements.txt +14 -0
- ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
- ads/feature_store/docs/source/cicd.rst +137 -0
- ads/feature_store/docs/source/conf.py +86 -0
- ads/feature_store/docs/source/data_versioning.rst +33 -0
- ads/feature_store/docs/source/dataset.rst +388 -0
- ads/feature_store/docs/source/dataset_job.rst +27 -0
- ads/feature_store/docs/source/demo.rst +70 -0
- ads/feature_store/docs/source/entity.rst +78 -0
- ads/feature_store/docs/source/feature_group.rst +624 -0
- ads/feature_store/docs/source/feature_group_job.rst +29 -0
- ads/feature_store/docs/source/feature_store.rst +122 -0
- ads/feature_store/docs/source/feature_store_class.rst +123 -0
- ads/feature_store/docs/source/feature_validation.rst +66 -0
- ads/feature_store/docs/source/figures/cicd.png +0 -0
- ads/feature_store/docs/source/figures/data_validation.png +0 -0
- ads/feature_store/docs/source/figures/data_versioning.png +0 -0
- ads/feature_store/docs/source/figures/dataset.gif +0 -0
- ads/feature_store/docs/source/figures/dataset.png +0 -0
- ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
- ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
- ads/feature_store/docs/source/figures/entity.png +0 -0
- ads/feature_store/docs/source/figures/feature_group.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
- ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
- ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
- ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
- ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
- ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
- ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
- ads/feature_store/docs/source/figures/overview.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
- ads/feature_store/docs/source/figures/stats_1.png +0 -0
- ads/feature_store/docs/source/figures/stats_2.png +0 -0
- ads/feature_store/docs/source/figures/stats_d.png +0 -0
- ads/feature_store/docs/source/figures/stats_fg.png +0 -0
- ads/feature_store/docs/source/figures/transformation.png +0 -0
- ads/feature_store/docs/source/figures/transformations.gif +0 -0
- ads/feature_store/docs/source/figures/validation.png +0 -0
- ads/feature_store/docs/source/figures/validation_fg.png +0 -0
- ads/feature_store/docs/source/figures/validation_results.png +0 -0
- ads/feature_store/docs/source/figures/validation_summary.png +0 -0
- ads/feature_store/docs/source/index.rst +81 -0
- ads/feature_store/docs/source/module.rst +8 -0
- ads/feature_store/docs/source/notebook.rst +94 -0
- ads/feature_store/docs/source/overview.rst +47 -0
- ads/feature_store/docs/source/quickstart.rst +176 -0
- ads/feature_store/docs/source/release_notes.rst +194 -0
- ads/feature_store/docs/source/setup_feature_store.rst +81 -0
- ads/feature_store/docs/source/statistics.rst +58 -0
- ads/feature_store/docs/source/transformation.rst +199 -0
- ads/feature_store/docs/source/ui.rst +65 -0
- ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
- ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
- ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
- ads/feature_store/entity.py +718 -0
- ads/feature_store/execution_strategy/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
- ads/feature_store/execution_strategy/engine/__init__.py +0 -0
- ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
- ads/feature_store/execution_strategy/execution_strategy.py +113 -0
- ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
- ads/feature_store/execution_strategy/spark/__init__.py +0 -0
- ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
- ads/feature_store/feature.py +192 -0
- ads/feature_store/feature_group.py +1494 -0
- ads/feature_store/feature_group_expectation.py +346 -0
- ads/feature_store/feature_group_job.py +602 -0
- ads/feature_store/feature_lineage/__init__.py +0 -0
- ads/feature_store/feature_lineage/graphviz_service.py +180 -0
- ads/feature_store/feature_option_details.py +50 -0
- ads/feature_store/feature_statistics/__init__.py +0 -0
- ads/feature_store/feature_statistics/statistics_service.py +99 -0
- ads/feature_store/feature_store.py +699 -0
- ads/feature_store/feature_store_registrar.py +518 -0
- ads/feature_store/input_feature_detail.py +149 -0
- ads/feature_store/mixin/__init__.py +4 -0
- ads/feature_store/mixin/oci_feature_store.py +145 -0
- ads/feature_store/model_details.py +73 -0
- ads/feature_store/query/__init__.py +0 -0
- ads/feature_store/query/filter.py +266 -0
- ads/feature_store/query/generator/__init__.py +0 -0
- ads/feature_store/query/generator/query_generator.py +298 -0
- ads/feature_store/query/join.py +161 -0
- ads/feature_store/query/query.py +403 -0
- ads/feature_store/query/validator/__init__.py +0 -0
- ads/feature_store/query/validator/query_validator.py +57 -0
- ads/feature_store/response/__init__.py +0 -0
- ads/feature_store/response/response_builder.py +68 -0
- ads/feature_store/service/__init__.py +0 -0
- ads/feature_store/service/oci_dataset.py +139 -0
- ads/feature_store/service/oci_dataset_job.py +199 -0
- ads/feature_store/service/oci_entity.py +125 -0
- ads/feature_store/service/oci_feature_group.py +164 -0
- ads/feature_store/service/oci_feature_group_job.py +214 -0
- ads/feature_store/service/oci_feature_store.py +182 -0
- ads/feature_store/service/oci_lineage.py +87 -0
- ads/feature_store/service/oci_transformation.py +104 -0
- ads/feature_store/statistics/__init__.py +0 -0
- ads/feature_store/statistics/abs_feature_value.py +49 -0
- ads/feature_store/statistics/charts/__init__.py +0 -0
- ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
- ads/feature_store/statistics/charts/box_plot.py +148 -0
- ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
- ads/feature_store/statistics/charts/probability_distribution.py +68 -0
- ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
- ads/feature_store/statistics/feature_stat.py +126 -0
- ads/feature_store/statistics/generic_feature_value.py +33 -0
- ads/feature_store/statistics/statistics.py +41 -0
- ads/feature_store/statistics_config.py +101 -0
- ads/feature_store/templates/feature_store_template.yaml +45 -0
- ads/feature_store/transformation.py +499 -0
- ads/feature_store/validation_output.py +57 -0
- ads/hpo/__init__.py +9 -0
- ads/hpo/_imports.py +91 -0
- ads/hpo/ads_search_space.py +439 -0
- ads/hpo/distributions.py +325 -0
- ads/hpo/objective.py +280 -0
- ads/hpo/search_cv.py +1657 -0
- ads/hpo/stopping_criterion.py +75 -0
- ads/hpo/tuner_artifact.py +413 -0
- ads/hpo/utils.py +91 -0
- ads/hpo/validation.py +140 -0
- ads/hpo/visualization/__init__.py +5 -0
- ads/hpo/visualization/_contour.py +23 -0
- ads/hpo/visualization/_edf.py +20 -0
- ads/hpo/visualization/_intermediate_values.py +21 -0
- ads/hpo/visualization/_optimization_history.py +25 -0
- ads/hpo/visualization/_parallel_coordinate.py +169 -0
- ads/hpo/visualization/_param_importances.py +26 -0
- ads/jobs/__init__.py +53 -0
- ads/jobs/ads_job.py +663 -0
- ads/jobs/builders/__init__.py +5 -0
- ads/jobs/builders/base.py +156 -0
- ads/jobs/builders/infrastructure/__init__.py +6 -0
- ads/jobs/builders/infrastructure/base.py +165 -0
- ads/jobs/builders/infrastructure/dataflow.py +1252 -0
- ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
- ads/jobs/builders/infrastructure/utils.py +65 -0
- ads/jobs/builders/runtimes/__init__.py +5 -0
- ads/jobs/builders/runtimes/artifact.py +338 -0
- ads/jobs/builders/runtimes/base.py +325 -0
- ads/jobs/builders/runtimes/container_runtime.py +242 -0
- ads/jobs/builders/runtimes/python_runtime.py +1016 -0
- ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
- ads/jobs/cli.py +104 -0
- ads/jobs/env_var_parser.py +131 -0
- ads/jobs/extension.py +160 -0
- ads/jobs/schema/__init__.py +5 -0
- ads/jobs/schema/infrastructure_schema.json +116 -0
- ads/jobs/schema/job_schema.json +42 -0
- ads/jobs/schema/runtime_schema.json +183 -0
- ads/jobs/schema/validator.py +141 -0
- ads/jobs/serializer.py +296 -0
- ads/jobs/templates/__init__.py +5 -0
- ads/jobs/templates/container.py +6 -0
- ads/jobs/templates/driver_notebook.py +177 -0
- ads/jobs/templates/driver_oci.py +500 -0
- ads/jobs/templates/driver_python.py +48 -0
- ads/jobs/templates/driver_pytorch.py +852 -0
- ads/jobs/templates/driver_utils.py +615 -0
- ads/jobs/templates/hostname_from_env.c +55 -0
- ads/jobs/templates/oci_metrics.py +181 -0
- ads/jobs/utils.py +104 -0
- ads/llm/__init__.py +28 -0
- ads/llm/autogen/__init__.py +2 -0
- ads/llm/autogen/constants.py +15 -0
- ads/llm/autogen/reports/__init__.py +2 -0
- ads/llm/autogen/reports/base.py +67 -0
- ads/llm/autogen/reports/data.py +103 -0
- ads/llm/autogen/reports/session.py +526 -0
- ads/llm/autogen/reports/templates/chat_box.html +13 -0
- ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
- ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
- ads/llm/autogen/reports/utils.py +56 -0
- ads/llm/autogen/v02/__init__.py +4 -0
- ads/llm/autogen/v02/client.py +295 -0
- ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
- ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
- ads/llm/autogen/v02/loggers/__init__.py +6 -0
- ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
- ads/llm/autogen/v02/loggers/session_logger.py +580 -0
- ads/llm/autogen/v02/loggers/utils.py +86 -0
- ads/llm/autogen/v02/runtime_logging.py +163 -0
- ads/llm/chain.py +268 -0
- ads/llm/chat_template.py +31 -0
- ads/llm/deploy.py +63 -0
- ads/llm/guardrails/__init__.py +5 -0
- ads/llm/guardrails/base.py +442 -0
- ads/llm/guardrails/huggingface.py +44 -0
- ads/llm/langchain/__init__.py +5 -0
- ads/llm/langchain/plugins/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
- ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
- ads/llm/langchain/plugins/llms/__init__.py +5 -0
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
- ads/llm/requirements.txt +3 -0
- ads/llm/serialize.py +219 -0
- ads/llm/serializers/__init__.py +0 -0
- ads/llm/serializers/retrieval_qa.py +153 -0
- ads/llm/serializers/runnable_parallel.py +27 -0
- ads/llm/templates/score_chain.jinja2 +155 -0
- ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
- ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
- ads/model/__init__.py +52 -0
- ads/model/artifact.py +573 -0
- ads/model/artifact_downloader.py +254 -0
- ads/model/artifact_uploader.py +267 -0
- ads/model/base_properties.py +238 -0
- ads/model/common/.model-ignore +66 -0
- ads/model/common/__init__.py +5 -0
- ads/model/common/utils.py +142 -0
- ads/model/datascience_model.py +2635 -0
- ads/model/deployment/__init__.py +20 -0
- ads/model/deployment/common/__init__.py +5 -0
- ads/model/deployment/common/utils.py +308 -0
- ads/model/deployment/model_deployer.py +466 -0
- ads/model/deployment/model_deployment.py +1846 -0
- ads/model/deployment/model_deployment_infrastructure.py +671 -0
- ads/model/deployment/model_deployment_properties.py +493 -0
- ads/model/deployment/model_deployment_runtime.py +838 -0
- ads/model/extractor/__init__.py +5 -0
- ads/model/extractor/automl_extractor.py +74 -0
- ads/model/extractor/embedding_onnx_extractor.py +80 -0
- ads/model/extractor/huggingface_extractor.py +88 -0
- ads/model/extractor/keras_extractor.py +84 -0
- ads/model/extractor/lightgbm_extractor.py +93 -0
- ads/model/extractor/model_info_extractor.py +114 -0
- ads/model/extractor/model_info_extractor_factory.py +105 -0
- ads/model/extractor/pytorch_extractor.py +87 -0
- ads/model/extractor/sklearn_extractor.py +112 -0
- ads/model/extractor/spark_extractor.py +89 -0
- ads/model/extractor/tensorflow_extractor.py +85 -0
- ads/model/extractor/xgboost_extractor.py +94 -0
- ads/model/framework/__init__.py +5 -0
- ads/model/framework/automl_model.py +178 -0
- ads/model/framework/embedding_onnx_model.py +438 -0
- ads/model/framework/huggingface_model.py +399 -0
- ads/model/framework/lightgbm_model.py +266 -0
- ads/model/framework/pytorch_model.py +266 -0
- ads/model/framework/sklearn_model.py +250 -0
- ads/model/framework/spark_model.py +326 -0
- ads/model/framework/tensorflow_model.py +254 -0
- ads/model/framework/xgboost_model.py +258 -0
- ads/model/generic_model.py +3518 -0
- ads/model/model_artifact_boilerplate/README.md +381 -0
- ads/model/model_artifact_boilerplate/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
- ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
- ads/model/model_artifact_boilerplate/score.py +61 -0
- ads/model/model_file_description_schema.json +68 -0
- ads/model/model_introspect.py +331 -0
- ads/model/model_metadata.py +1810 -0
- ads/model/model_metadata_mixin.py +460 -0
- ads/model/model_properties.py +63 -0
- ads/model/model_version_set.py +739 -0
- ads/model/runtime/__init__.py +5 -0
- ads/model/runtime/env_info.py +306 -0
- ads/model/runtime/model_deployment_details.py +37 -0
- ads/model/runtime/model_provenance_details.py +58 -0
- ads/model/runtime/runtime_info.py +81 -0
- ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
- ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
- ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
- ads/model/runtime/utils.py +201 -0
- ads/model/serde/__init__.py +5 -0
- ads/model/serde/common.py +40 -0
- ads/model/serde/model_input.py +547 -0
- ads/model/serde/model_serializer.py +1184 -0
- ads/model/service/__init__.py +5 -0
- ads/model/service/oci_datascience_model.py +1076 -0
- ads/model/service/oci_datascience_model_deployment.py +500 -0
- ads/model/service/oci_datascience_model_version_set.py +176 -0
- ads/model/transformer/__init__.py +5 -0
- ads/model/transformer/onnx_transformer.py +324 -0
- ads/mysqldb/__init__.py +5 -0
- ads/mysqldb/mysql_db.py +227 -0
- ads/opctl/__init__.py +18 -0
- ads/opctl/anomaly_detection.py +11 -0
- ads/opctl/backend/__init__.py +5 -0
- ads/opctl/backend/ads_dataflow.py +353 -0
- ads/opctl/backend/ads_ml_job.py +710 -0
- ads/opctl/backend/ads_ml_pipeline.py +164 -0
- ads/opctl/backend/ads_model_deployment.py +209 -0
- ads/opctl/backend/base.py +146 -0
- ads/opctl/backend/local.py +1053 -0
- ads/opctl/backend/marketplace/__init__.py +9 -0
- ads/opctl/backend/marketplace/helm_helper.py +173 -0
- ads/opctl/backend/marketplace/local_marketplace.py +271 -0
- ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
- ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
- ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
- ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
- ads/opctl/backend/marketplace/models/__init__.py +5 -0
- ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
- ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
- ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
- ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
- ads/opctl/cli.py +707 -0
- ads/opctl/cmds.py +869 -0
- ads/opctl/conda/__init__.py +5 -0
- ads/opctl/conda/cli.py +193 -0
- ads/opctl/conda/cmds.py +749 -0
- ads/opctl/conda/config.yaml +34 -0
- ads/opctl/conda/manifest_template.yaml +13 -0
- ads/opctl/conda/multipart_uploader.py +188 -0
- ads/opctl/conda/pack.py +89 -0
- ads/opctl/config/__init__.py +5 -0
- ads/opctl/config/base.py +57 -0
- ads/opctl/config/diagnostics/__init__.py +5 -0
- ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
- ads/opctl/config/merger.py +255 -0
- ads/opctl/config/resolver.py +297 -0
- ads/opctl/config/utils.py +79 -0
- ads/opctl/config/validator.py +17 -0
- ads/opctl/config/versioner.py +68 -0
- ads/opctl/config/yaml_parsers/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/base.py +58 -0
- ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
- ads/opctl/constants.py +66 -0
- ads/opctl/decorator/__init__.py +5 -0
- ads/opctl/decorator/common.py +129 -0
- ads/opctl/diagnostics/__init__.py +5 -0
- ads/opctl/diagnostics/__main__.py +25 -0
- ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
- ads/opctl/diagnostics/check_requirements.py +144 -0
- ads/opctl/diagnostics/requirement_exception.py +9 -0
- ads/opctl/distributed/README.md +109 -0
- ads/opctl/distributed/__init__.py +5 -0
- ads/opctl/distributed/certificates.py +32 -0
- ads/opctl/distributed/cli.py +207 -0
- ads/opctl/distributed/cmds.py +731 -0
- ads/opctl/distributed/common/__init__.py +5 -0
- ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
- ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
- ads/opctl/distributed/common/cluster_config_helper.py +103 -0
- ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
- ads/opctl/distributed/common/cluster_runner.py +54 -0
- ads/opctl/distributed/common/framework_factory.py +29 -0
- ads/opctl/docker/Dockerfile.job +103 -0
- ads/opctl/docker/Dockerfile.job.arm +107 -0
- ads/opctl/docker/Dockerfile.job.gpu +175 -0
- ads/opctl/docker/base-env.yaml +13 -0
- ads/opctl/docker/cuda.repo +6 -0
- ads/opctl/docker/operator/.dockerignore +0 -0
- ads/opctl/docker/operator/Dockerfile +41 -0
- ads/opctl/docker/operator/Dockerfile.gpu +85 -0
- ads/opctl/docker/operator/cuda.repo +6 -0
- ads/opctl/docker/operator/environment.yaml +8 -0
- ads/opctl/forecast.py +11 -0
- ads/opctl/index.yaml +3 -0
- ads/opctl/model/__init__.py +5 -0
- ads/opctl/model/cli.py +65 -0
- ads/opctl/model/cmds.py +73 -0
- ads/opctl/operator/README.md +4 -0
- ads/opctl/operator/__init__.py +31 -0
- ads/opctl/operator/cli.py +344 -0
- ads/opctl/operator/cmd.py +596 -0
- ads/opctl/operator/common/__init__.py +5 -0
- ads/opctl/operator/common/backend_factory.py +460 -0
- ads/opctl/operator/common/const.py +27 -0
- ads/opctl/operator/common/data/synthetic.csv +16001 -0
- ads/opctl/operator/common/dictionary_merger.py +148 -0
- ads/opctl/operator/common/errors.py +42 -0
- ads/opctl/operator/common/operator_config.py +99 -0
- ads/opctl/operator/common/operator_loader.py +811 -0
- ads/opctl/operator/common/operator_schema.yaml +130 -0
- ads/opctl/operator/common/operator_yaml_generator.py +152 -0
- ads/opctl/operator/common/utils.py +208 -0
- ads/opctl/operator/lowcode/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
- ads/opctl/operator/lowcode/anomaly/README.md +207 -0
- ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
- ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
- ads/opctl/operator/lowcode/anomaly/const.py +167 -0
- ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
- ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
- ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
- ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
- ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
- ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
- ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
- ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
- ads/opctl/operator/lowcode/common/__init__.py +5 -0
- ads/opctl/operator/lowcode/common/const.py +10 -0
- ads/opctl/operator/lowcode/common/data.py +116 -0
- ads/opctl/operator/lowcode/common/errors.py +47 -0
- ads/opctl/operator/lowcode/common/transformations.py +296 -0
- ads/opctl/operator/lowcode/common/utils.py +384 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
- ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
- ads/opctl/operator/lowcode/forecast/README.md +209 -0
- ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
- ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
- ads/opctl/operator/lowcode/forecast/const.py +92 -0
- ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
- ads/opctl/operator/lowcode/forecast/errors.py +26 -0
- ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
- ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
- ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
- ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
- ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
- ads/opctl/operator/lowcode/forecast/model/prophet.py +445 -0
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
- ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
- ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
- ads/opctl/operator/lowcode/forecast/utils.py +397 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
- ads/opctl/operator/lowcode/pii/MLoperator +17 -0
- ads/opctl/operator/lowcode/pii/README.md +208 -0
- ads/opctl/operator/lowcode/pii/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/__main__.py +78 -0
- ads/opctl/operator/lowcode/pii/cmd.py +39 -0
- ads/opctl/operator/lowcode/pii/constant.py +84 -0
- ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
- ads/opctl/operator/lowcode/pii/errors.py +27 -0
- ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
- ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
- ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
- ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
- ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
- ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
- ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
- ads/opctl/operator/lowcode/pii/model/report.py +487 -0
- ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
- ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
- ads/opctl/operator/lowcode/pii/utils.py +43 -0
- ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
- ads/opctl/operator/lowcode/recommender/README.md +206 -0
- ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
- ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
- ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
- ads/opctl/operator/lowcode/recommender/constant.py +30 -0
- ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
- ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
- ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
- ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
- ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
- ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
- ads/opctl/operator/lowcode/recommender/utils.py +13 -0
- ads/opctl/operator/runtime/__init__.py +5 -0
- ads/opctl/operator/runtime/const.py +17 -0
- ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
- ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
- ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/runtime.py +115 -0
- ads/opctl/schema.yaml.yml +36 -0
- ads/opctl/script.py +40 -0
- ads/opctl/spark/__init__.py +5 -0
- ads/opctl/spark/cli.py +43 -0
- ads/opctl/spark/cmds.py +147 -0
- ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
- ads/opctl/utils.py +344 -0
- ads/oracledb/__init__.py +5 -0
- ads/oracledb/oracle_db.py +346 -0
- ads/pipeline/__init__.py +39 -0
- ads/pipeline/ads_pipeline.py +2279 -0
- ads/pipeline/ads_pipeline_run.py +772 -0
- ads/pipeline/ads_pipeline_step.py +605 -0
- ads/pipeline/builders/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/custom_script.py +32 -0
- ads/pipeline/cli.py +119 -0
- ads/pipeline/extension.py +291 -0
- ads/pipeline/schema/__init__.py +5 -0
- ads/pipeline/schema/cs_step_schema.json +35 -0
- ads/pipeline/schema/ml_step_schema.json +31 -0
- ads/pipeline/schema/pipeline_schema.json +71 -0
- ads/pipeline/visualizer/__init__.py +5 -0
- ads/pipeline/visualizer/base.py +570 -0
- ads/pipeline/visualizer/graph_renderer.py +272 -0
- ads/pipeline/visualizer/text_renderer.py +84 -0
- ads/secrets/__init__.py +11 -0
- ads/secrets/adb.py +386 -0
- ads/secrets/auth_token.py +86 -0
- ads/secrets/big_data_service.py +365 -0
- ads/secrets/mysqldb.py +149 -0
- ads/secrets/oracledb.py +160 -0
- ads/secrets/secrets.py +407 -0
- ads/telemetry/__init__.py +7 -0
- ads/telemetry/base.py +69 -0
- ads/telemetry/client.py +125 -0
- ads/telemetry/telemetry.py +257 -0
- ads/templates/dataflow_pyspark.jinja2 +13 -0
- ads/templates/dataflow_sparksql.jinja2 +22 -0
- ads/templates/func.jinja2 +20 -0
- ads/templates/schemas/openapi.json +1740 -0
- ads/templates/score-pkl.jinja2 +173 -0
- ads/templates/score.jinja2 +322 -0
- ads/templates/score_embedding_onnx.jinja2 +202 -0
- ads/templates/score_generic.jinja2 +165 -0
- ads/templates/score_huggingface_pipeline.jinja2 +217 -0
- ads/templates/score_lightgbm.jinja2 +185 -0
- ads/templates/score_onnx.jinja2 +407 -0
- ads/templates/score_onnx_new.jinja2 +473 -0
- ads/templates/score_oracle_automl.jinja2 +185 -0
- ads/templates/score_pyspark.jinja2 +154 -0
- ads/templates/score_pytorch.jinja2 +219 -0
- ads/templates/score_scikit-learn.jinja2 +184 -0
- ads/templates/score_tensorflow.jinja2 +184 -0
- ads/templates/score_xgboost.jinja2 +178 -0
- ads/text_dataset/__init__.py +5 -0
- ads/text_dataset/backends.py +211 -0
- ads/text_dataset/dataset.py +445 -0
- ads/text_dataset/extractor.py +207 -0
- ads/text_dataset/options.py +53 -0
- ads/text_dataset/udfs.py +22 -0
- ads/text_dataset/utils.py +49 -0
- ads/type_discovery/__init__.py +9 -0
- ads/type_discovery/abstract_detector.py +21 -0
- ads/type_discovery/constant_detector.py +41 -0
- ads/type_discovery/continuous_detector.py +54 -0
- ads/type_discovery/credit_card_detector.py +99 -0
- ads/type_discovery/datetime_detector.py +92 -0
- ads/type_discovery/discrete_detector.py +118 -0
- ads/type_discovery/document_detector.py +146 -0
- ads/type_discovery/ip_detector.py +68 -0
- ads/type_discovery/latlon_detector.py +90 -0
- ads/type_discovery/phone_number_detector.py +63 -0
- ads/type_discovery/type_discovery_driver.py +87 -0
- ads/type_discovery/typed_feature.py +594 -0
- ads/type_discovery/unknown_detector.py +41 -0
- ads/type_discovery/zipcode_detector.py +48 -0
- ads/vault/__init__.py +7 -0
- ads/vault/vault.py +237 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.9rc1.dist-info}/METADATA +150 -150
- oracle_ads-2.13.9rc1.dist-info/RECORD +858 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.9rc1.dist-info}/WHEEL +1 -2
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.9rc1.dist-info}/entry_points.txt +2 -1
- oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
- oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.9rc1.dist-info}/licenses/LICENSE.txt +0 -0
ads/common/oci_mixin.py
ADDED
@@ -0,0 +1,957 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8; -*-
|
3
|
+
|
4
|
+
# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
|
5
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
|
+
|
7
|
+
"""Contains Mixins for integrating OCI data models
|
8
|
+
"""
|
9
|
+
import inspect
|
10
|
+
import json
|
11
|
+
import logging
|
12
|
+
import os
|
13
|
+
import re
|
14
|
+
import traceback
|
15
|
+
from datetime import date, datetime
|
16
|
+
from enum import Enum
|
17
|
+
from typing import Callable, Optional, Union
|
18
|
+
|
19
|
+
import oci
|
20
|
+
import yaml
|
21
|
+
from dateutil import tz
|
22
|
+
from dateutil.parser import parse
|
23
|
+
from oci._vendor import six
|
24
|
+
|
25
|
+
from ads.common import auth
|
26
|
+
from ads.common.decorator.utils import class_or_instance_method
|
27
|
+
from ads.common.utils import camel_to_snake
|
28
|
+
from ads.config import COMPARTMENT_OCID
|
29
|
+
|
30
|
+
logger = logging.getLogger(__name__)
|
31
|
+
|
32
|
+
LIFECYCLE_STOP_STATE = ("SUCCEEDED", "FAILED", "CANCELED", "DELETED")
|
33
|
+
WORK_REQUEST_STOP_STATE = ("SUCCEEDED", "FAILED", "CANCELED")
|
34
|
+
DEFAULT_WAIT_TIME = 1200
|
35
|
+
DEFAULT_POLL_INTERVAL = 10
|
36
|
+
WORK_REQUEST_PERCENTAGE = 100
|
37
|
+
|
38
|
+
|
39
|
+
class MergeStrategy(Enum):
|
40
|
+
OVERRIDE = "override"
|
41
|
+
MERGE = "merge"
|
42
|
+
|
43
|
+
|
44
|
+
class OCIModelNotExists(Exception): # pragma: no cover
|
45
|
+
pass
|
46
|
+
|
47
|
+
|
48
|
+
class OCIClientMixin:
|
49
|
+
"""Mixin class for representing OCI resource/service with OCI client.
|
50
|
+
|
51
|
+
Most OCI requests requires a client for making the connection.
|
52
|
+
Usually the same client will be used for the requests related the same resource/service type.
|
53
|
+
This Mixin adds a "client" property to simplify accessing the client.
|
54
|
+
The actual client will be initialize lazily so that it is not required for a sub-class
|
55
|
+
To use the client, sub-class should override the init_client() method pass in the "client" keyword argument.
|
56
|
+
For example:
|
57
|
+
|
58
|
+
@class_or_instance_method
|
59
|
+
def init_client(cls, **kwargs) -> oci.logging.LoggingManagementClient:
|
60
|
+
return cls._init_client(client=oci.logging.LoggingManagementClient, **kwargs)
|
61
|
+
|
62
|
+
Instance methods in the sub-class can use self.client to access the client.
|
63
|
+
The init_client() method is a class method used to create the client.
|
64
|
+
Any class method using the client should use init_client() to create the client.
|
65
|
+
The call to this method may come from an instance or a class.
|
66
|
+
When the method is called from a class,
|
67
|
+
the default authentication configured at ADS level with ads.set_auth() will be used.
|
68
|
+
When the method is called from an instance,
|
69
|
+
the config, signer and kwargs specified when initializing the instance will be used.
|
70
|
+
The sub-class's __init__ method should take config, signer and client_kwargs as argument,
|
71
|
+
then pass them to the __init__ method of this class.
|
72
|
+
This allows users to override the authentication and client initialization parameters.
|
73
|
+
For example, log_group = OCILogGroup(config=config, signer=signer, client_kwargs=kwargs)
|
74
|
+
"""
|
75
|
+
|
76
|
+
config = None
|
77
|
+
signer = None
|
78
|
+
kwargs = None
|
79
|
+
|
80
|
+
@class_or_instance_method
|
81
|
+
def _get_auth(cls):
|
82
|
+
client_kwargs = dict(retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY)
|
83
|
+
if cls.kwargs:
|
84
|
+
client_kwargs.update(cls.kwargs)
|
85
|
+
|
86
|
+
if cls.config is None and cls.signer is None:
|
87
|
+
oci_auth = auth.default_signer(client_kwargs)
|
88
|
+
elif not cls.signer and cls.config:
|
89
|
+
oci_auth = {"config": cls.config, "client_kwargs": client_kwargs}
|
90
|
+
else:
|
91
|
+
oci_auth = {
|
92
|
+
"config": cls.config,
|
93
|
+
"signer": cls.signer,
|
94
|
+
"client_kwargs": client_kwargs,
|
95
|
+
}
|
96
|
+
return oci_auth
|
97
|
+
|
98
|
+
@class_or_instance_method
|
99
|
+
def init_client(cls, **kwargs):
|
100
|
+
"""Initializes the OCI client specified in the "client" keyword argument
|
101
|
+
Sub-class should override this method and call cls._init_client(client=OCI_CLIENT)
|
102
|
+
|
103
|
+
Parameters
|
104
|
+
----------
|
105
|
+
**kwargs :
|
106
|
+
Additional keyword arguments for initializing the OCI client.
|
107
|
+
|
108
|
+
Returns
|
109
|
+
-------
|
110
|
+
An instance of OCI client.
|
111
|
+
|
112
|
+
"""
|
113
|
+
return cls._init_client(**kwargs)
|
114
|
+
|
115
|
+
@class_or_instance_method
|
116
|
+
def _init_client(cls, client, **kwargs):
|
117
|
+
"""Initializes the OCI client specified in the "client" argument
|
118
|
+
|
119
|
+
Parameters
|
120
|
+
----------
|
121
|
+
client :
|
122
|
+
The OCI client class to be initialized, e.g., oci.data_science.DataScienceClient
|
123
|
+
**kwargs :
|
124
|
+
Additional keyword arguments for initializing the OCI client.
|
125
|
+
|
126
|
+
Returns
|
127
|
+
-------
|
128
|
+
An instance of OCI client.
|
129
|
+
|
130
|
+
"""
|
131
|
+
auth = cls._get_auth()
|
132
|
+
auth_kwargs = auth.pop("client_kwargs", {})
|
133
|
+
if kwargs:
|
134
|
+
auth_kwargs.update(kwargs)
|
135
|
+
auth.update(auth_kwargs)
|
136
|
+
return client(**auth)
|
137
|
+
|
138
|
+
@class_or_instance_method
|
139
|
+
def create_instance(obj, *args, **kwargs):
|
140
|
+
"""Creates an instance using the same authentication as the class or an existing instance.
|
141
|
+
If this method is called by a class, the default ADS authentication method will be used.
|
142
|
+
If this method is called by an instance, the authentication method set in the instance will be used.
|
143
|
+
"""
|
144
|
+
# Here cls could be a class or an instance.
|
145
|
+
# If it is a class, it can be used to create the new instance directly.
|
146
|
+
# If it is an instance, the new instance should be created using cls.__class__.
|
147
|
+
# Calling the classmethod _create_instance() will make sure that class will be used.
|
148
|
+
return obj._create_instance(
|
149
|
+
config=obj.config,
|
150
|
+
signer=obj.signer,
|
151
|
+
client_kwargs=obj.kwargs,
|
152
|
+
*args,
|
153
|
+
**kwargs,
|
154
|
+
)
|
155
|
+
|
156
|
+
@classmethod
|
157
|
+
def _create_instance(cls, *args, **kwargs):
|
158
|
+
"""Initialize an instance."""
|
159
|
+
return cls(*args, **kwargs)
|
160
|
+
|
161
|
+
def __init__(self, config=None, signer=None, client_kwargs=None) -> None:
|
162
|
+
"""Initializes a service/resource with OCI client as a property.
|
163
|
+
If config or signer is specified, it will be used to initialize the OCI client.
|
164
|
+
If neither of them is specified, the client will be initialized with ads.common.auth.default_signer.
|
165
|
+
If both of them are specified, both of them will be passed into the OCI client,
|
166
|
+
and the authentication will be determined by OCI Python SDK.
|
167
|
+
|
168
|
+
Parameters
|
169
|
+
----------
|
170
|
+
config : dict, optional
|
171
|
+
OCI API key config dictionary, by default None.
|
172
|
+
signer : oci.signer.Signer, optional
|
173
|
+
OCI authentication signer, by default None.
|
174
|
+
client_kwargs : dict, optional
|
175
|
+
Additional keyword arguments for initializing the OCI client.
|
176
|
+
"""
|
177
|
+
super().__init__()
|
178
|
+
self.config = config
|
179
|
+
self.signer = signer
|
180
|
+
self.kwargs = client_kwargs
|
181
|
+
self._client = None
|
182
|
+
|
183
|
+
@property
|
184
|
+
def auth(self) -> dict:
|
185
|
+
"""The ADS authentication config used to initialize the client.
|
186
|
+
This auth has the same format as those obtained by calling functions in ads.common.auth.
|
187
|
+
The config is a dict containing the following key-value pairs:
|
188
|
+
config: The config contains the config loaded from the configuration loaded from `oci_config`.
|
189
|
+
signer: The signer contains the signer object created from the api keys.
|
190
|
+
client_kwargs: client_kwargs contains the `client_kwargs` that was passed in as input parameter.
|
191
|
+
"""
|
192
|
+
return self._get_auth()
|
193
|
+
|
194
|
+
@property
|
195
|
+
def client(self):
|
196
|
+
"""OCI client"""
|
197
|
+
if self._client is None:
|
198
|
+
self._client = self.init_client()
|
199
|
+
return self._client
|
200
|
+
|
201
|
+
|
202
|
+
class OCISerializableMixin(OCIClientMixin):
|
203
|
+
"""Mixin class containing OCI serialization/de-serialization methods.
|
204
|
+
These methods are copied and modified from the OCI BaseClient.
|
205
|
+
|
206
|
+
"""
|
207
|
+
|
208
|
+
type_mappings = None
|
209
|
+
|
210
|
+
def serialize(self):
|
211
|
+
"""Serialize the model to a dictionary that is ready to be send to OCI API.
|
212
|
+
|
213
|
+
Returns
|
214
|
+
-------
|
215
|
+
dict
|
216
|
+
A dictionary that is ready to be send to OCI API.
|
217
|
+
|
218
|
+
"""
|
219
|
+
return self.client.base_client.sanitize_for_serialization(self)
|
220
|
+
|
221
|
+
@staticmethod
|
222
|
+
def _parse_kwargs(attribute_map: dict, **kwargs):
|
223
|
+
"""Parse kwargs to make all key in camel format."""
|
224
|
+
parsed_kwargs = {}
|
225
|
+
for key, val in kwargs.items():
|
226
|
+
if key in attribute_map:
|
227
|
+
parsed_kwargs[attribute_map[key]] = val
|
228
|
+
else:
|
229
|
+
parsed_kwargs[key] = val
|
230
|
+
|
231
|
+
return parsed_kwargs
|
232
|
+
|
233
|
+
@class_or_instance_method
|
234
|
+
def deserialize(cls, data, to_cls):
|
235
|
+
"""De-serialize data from dictionary to an OCI model"""
|
236
|
+
if cls.type_mappings is None:
|
237
|
+
cls.type_mappings = cls.init_client().base_client.type_mappings
|
238
|
+
|
239
|
+
if data is None:
|
240
|
+
return None
|
241
|
+
|
242
|
+
# This is a work around for enums not being present
|
243
|
+
# in the type mappings.
|
244
|
+
# See OraclePythonSdkCodegen removeEnumsFromModelGeneration().
|
245
|
+
if to_cls in cls.type_mappings:
|
246
|
+
to_cls = cls.type_mappings[to_cls]
|
247
|
+
|
248
|
+
if isinstance(data, dict):
|
249
|
+
data = cls._parse_kwargs(to_cls().attribute_map, **data)
|
250
|
+
else:
|
251
|
+
return cls.__deserialize_primitive(data, to_cls)
|
252
|
+
|
253
|
+
if hasattr(to_cls, "get_subtype"):
|
254
|
+
# Return the object as is if it is already deserialized.
|
255
|
+
if isinstance(data, to_cls) or issubclass(data.__class__, to_cls):
|
256
|
+
return data
|
257
|
+
|
258
|
+
# Use the discriminator value to get the correct subtype.
|
259
|
+
to_cls = to_cls.get_subtype(data) # get_subtype returns a str
|
260
|
+
to_cls = cls.type_mappings[to_cls]
|
261
|
+
|
262
|
+
# kwargs needs to be parsed again as there are more attributes in the sub types.
|
263
|
+
if isinstance(data, dict):
|
264
|
+
data = cls._parse_kwargs(to_cls().attribute_map, **data)
|
265
|
+
|
266
|
+
if to_cls in [int, float, six.u, bool]:
|
267
|
+
return cls.__deserialize_primitive(data, to_cls)
|
268
|
+
elif to_cls == object:
|
269
|
+
return data
|
270
|
+
elif to_cls == date:
|
271
|
+
return cls.__deserialize_date(data)
|
272
|
+
elif to_cls == datetime:
|
273
|
+
return cls.__deserialize_datetime(data)
|
274
|
+
else:
|
275
|
+
return cls.__deserialize_model(data, to_cls)
|
276
|
+
|
277
|
+
@class_or_instance_method
|
278
|
+
def __deserialize_model(cls, data, to_cls):
|
279
|
+
"""De-serializes list or dict to model."""
|
280
|
+
if isinstance(data, to_cls):
|
281
|
+
return data
|
282
|
+
|
283
|
+
instance = to_cls()
|
284
|
+
|
285
|
+
for attr, attr_type in instance.swagger_types.items():
|
286
|
+
property = instance.attribute_map[attr]
|
287
|
+
if property in data:
|
288
|
+
value = data[property]
|
289
|
+
setattr(instance, attr, cls.deserialize(value, attr_type))
|
290
|
+
|
291
|
+
return instance
|
292
|
+
|
293
|
+
@staticmethod
|
294
|
+
def __deserialize_primitive(data, cls):
|
295
|
+
"""De-serializes string to primitive type."""
|
296
|
+
try:
|
297
|
+
value = cls(data)
|
298
|
+
except UnicodeEncodeError:
|
299
|
+
value = data
|
300
|
+
except TypeError:
|
301
|
+
value = data
|
302
|
+
return value
|
303
|
+
|
304
|
+
@staticmethod
|
305
|
+
def __deserialize_date(string):
|
306
|
+
"""De-serializes string to date."""
|
307
|
+
try:
|
308
|
+
return parse(string).date()
|
309
|
+
except ImportError:
|
310
|
+
return string
|
311
|
+
except ValueError:
|
312
|
+
raise Exception("Failed to parse `{0}` into a date object".format(string))
|
313
|
+
|
314
|
+
@staticmethod
|
315
|
+
def __deserialize_datetime(string):
|
316
|
+
"""De-serializes string to datetime.
|
317
|
+
|
318
|
+
The string should be in iso8601 datetime format.
|
319
|
+
"""
|
320
|
+
if isinstance(string, datetime):
|
321
|
+
return string
|
322
|
+
try:
|
323
|
+
# If this parser creates a date without raising an exception
|
324
|
+
# then the time zone is utc and needs to be set.
|
325
|
+
naivedatetime = datetime.strptime(string, "%Y-%m-%dT%H:%M:%S.%fZ")
|
326
|
+
awaredatetime = naivedatetime.replace(tzinfo=tz.tzutc())
|
327
|
+
return awaredatetime
|
328
|
+
|
329
|
+
except ValueError:
|
330
|
+
try:
|
331
|
+
return parse(string)
|
332
|
+
except ImportError:
|
333
|
+
return string
|
334
|
+
except ValueError:
|
335
|
+
raise Exception(
|
336
|
+
"Failed to parse `{0}` into a datetime object".format(string)
|
337
|
+
)
|
338
|
+
except ImportError:
|
339
|
+
return string
|
340
|
+
|
341
|
+
|
342
|
+
class OCIModelMixin(OCISerializableMixin):
|
343
|
+
"""Mixin class to operate OCI model.
|
344
|
+
OCI resources are represented by models in the OCI Python SDK.
|
345
|
+
|
346
|
+
Unifying OCI models for the same resource
|
347
|
+
-----------------------------------------
|
348
|
+
OCI SDK uses different models to represent the same resource for different operations.
|
349
|
+
For example, CreateLogDetails is used when creating a log resource,
|
350
|
+
while LogSummary is returned when listing the log resources.
|
351
|
+
However, both CreateLogDetails and LogSummary have the same commonly used attribute like
|
352
|
+
compartment_id, display_name, log_type, etc.
|
353
|
+
In general, there is a class with a super set of all properties.
|
354
|
+
For example, the Log class contains all properties of CreateLogDetails and LogSummary,
|
355
|
+
as well as other classes representing an OCI log resource.
|
356
|
+
A subclass can be implemented with this Mixin to unify the OCI models,
|
357
|
+
so that all properties are available to the user.
|
358
|
+
For example, if we define the Mixin model as ``class OCILog(OCIModelMixin, oci.logging.models.Log)``,
|
359
|
+
users will be able to access properties like ``OCILog().display_name``
|
360
|
+
Since this sub-class contains all the properties, it can be converted to any related OCI model in an operation.
|
361
|
+
For example, we can create ``CreateLogDetails`` from ``OCILog`` by extracting a subset of the properties.
|
362
|
+
When listing the resources, properties from ``LogSummary`` can be used to update
|
363
|
+
the corresponding properties of ``OCILog``.
|
364
|
+
Such convertion can be done be the generic methods provided by this Mixin.
|
365
|
+
Although OCI SDK accepts dictionary (JSON) data instead of objects like CreateLogDetails when creating or
|
366
|
+
updating the resource, the structure and keys of the dictionary is not easy for a user to remember.
|
367
|
+
It is also unnecessary for the users to construct the entire dictionary if they only want to update a single value.
|
368
|
+
|
369
|
+
This Mixin class should be the first parent as the class from OCI SDK does not call ``super().__init__()``
|
370
|
+
in its ``__init__()`` constructor.
|
371
|
+
Mixin properties may not be intialized correctly if ``super().__init__()`` is not called.
|
372
|
+
|
373
|
+
|
374
|
+
Provide generic methods for CRUDL operations
|
375
|
+
--------------------------------------------
|
376
|
+
Since OCI SDK uses different models in CRUDL operations,
|
377
|
+
this Mixin provides the following method to convert between them.
|
378
|
+
An OCI model instance can be any OCI model of the resource containing some properties, e.g. LogSummary
|
379
|
+
``from_oci_model()`` static method can be used to initialize a new instance from an OCI model instance.
|
380
|
+
``update_from_oci_model()`` can be used to update the existing properties from an OCI model instance.
|
381
|
+
``to_oci_model()`` can be used to extract properties from the Mixin model to OCI model.
|
382
|
+
|
383
|
+
"""
|
384
|
+
|
385
|
+
# Regex pattern matching the module name of an OCI model.
|
386
|
+
OCI_MODEL_PATTERN = r"(oci|feature_store_client).[^.]+\.models[\..*]?"
|
387
|
+
# Constants
|
388
|
+
CONS_COMPARTMENT_ID = "compartment_id"
|
389
|
+
|
390
|
+
@staticmethod
|
391
|
+
def check_compartment_id(compartment_id: Optional[str]) -> str:
|
392
|
+
"""Checks if a compartment ID has value and
|
393
|
+
return the value from NB_SESSION_COMPARTMENT_OCID environment variable if it is not specified.
|
394
|
+
|
395
|
+
Parameters
|
396
|
+
----------
|
397
|
+
compartment_id : str
|
398
|
+
Compartment OCID or None
|
399
|
+
|
400
|
+
Returns
|
401
|
+
-------
|
402
|
+
type
|
403
|
+
str: Compartment OCID
|
404
|
+
|
405
|
+
Raises
|
406
|
+
------
|
407
|
+
ValueError
|
408
|
+
compartment_id is not specified and NB_SESSION_COMPARTMENT_OCID environment variable is not set
|
409
|
+
|
410
|
+
"""
|
411
|
+
compartment_id = compartment_id or COMPARTMENT_OCID
|
412
|
+
if not compartment_id:
|
413
|
+
raise ValueError("Specify compartment OCID.")
|
414
|
+
return compartment_id
|
415
|
+
|
416
|
+
@class_or_instance_method
|
417
|
+
def list_resource(
|
418
|
+
cls, compartment_id: str = None, limit: int = 0, **kwargs
|
419
|
+
) -> list:
|
420
|
+
"""Generic method to list OCI resources
|
421
|
+
|
422
|
+
Parameters
|
423
|
+
----------
|
424
|
+
compartment_id : str
|
425
|
+
Compartment ID of the OCI resources. Defaults to None.
|
426
|
+
If compartment_id is not specified,
|
427
|
+
the value of NB_SESSION_COMPARTMENT_OCID in environment variable will be used.
|
428
|
+
limit : int
|
429
|
+
The maximum number of items to return. Defaults to 0, All items will be returned
|
430
|
+
**kwargs :
|
431
|
+
Additional keyword arguments to filter the resource.
|
432
|
+
The kwargs are passed into OCI API.
|
433
|
+
|
434
|
+
Returns
|
435
|
+
-------
|
436
|
+
list
|
437
|
+
A list of OCI resources
|
438
|
+
|
439
|
+
Raises
|
440
|
+
------
|
441
|
+
NotImplementedError
|
442
|
+
List method is not supported or implemented.
|
443
|
+
|
444
|
+
"""
|
445
|
+
if limit:
|
446
|
+
items = cls._find_oci_method("list")(
|
447
|
+
cls.check_compartment_id(compartment_id), limit=limit, **kwargs
|
448
|
+
).data
|
449
|
+
else:
|
450
|
+
items = oci.pagination.list_call_get_all_results(
|
451
|
+
cls._find_oci_method("list"),
|
452
|
+
cls.check_compartment_id(compartment_id),
|
453
|
+
**kwargs,
|
454
|
+
).data
|
455
|
+
return [cls.from_oci_model(item) for item in items]
|
456
|
+
|
457
|
+
@classmethod
|
458
|
+
def _find_oci_parent(cls):
|
459
|
+
"""Finds the parent OCI model class
|
460
|
+
|
461
|
+
Parameters
|
462
|
+
----------
|
463
|
+
|
464
|
+
Returns
|
465
|
+
-------
|
466
|
+
class
|
467
|
+
An OCI model class
|
468
|
+
|
469
|
+
Raises
|
470
|
+
------
|
471
|
+
AttributeError
|
472
|
+
If the class if not inherit from an OCI model.
|
473
|
+
|
474
|
+
"""
|
475
|
+
oci_model = None
|
476
|
+
for parent in inspect.getmro(cls):
|
477
|
+
if re.match(OCIModelMixin.OCI_MODEL_PATTERN, parent.__module__):
|
478
|
+
oci_model = parent
|
479
|
+
break
|
480
|
+
if not oci_model:
|
481
|
+
raise AttributeError(f"{cls.__name__} is not inherited from an OCI model.")
|
482
|
+
return oci_model
|
483
|
+
|
484
|
+
@class_or_instance_method
|
485
|
+
def _find_oci_method(cls, method: str) -> Callable:
|
486
|
+
"""Finds the OCI method for operations like get/list
|
487
|
+
|
488
|
+
Parameters
|
489
|
+
----------
|
490
|
+
method : str
|
491
|
+
The operation to be performed, e.g. get, list, delete
|
492
|
+
|
493
|
+
|
494
|
+
Returns
|
495
|
+
-------
|
496
|
+
callable
|
497
|
+
The method from OCI client to perform the operation.
|
498
|
+
|
499
|
+
"""
|
500
|
+
client = cls.init_client()
|
501
|
+
|
502
|
+
user_define_method = f"oci_{method}_method".upper()
|
503
|
+
if hasattr(cls, user_define_method) and getattr(cls, user_define_method):
|
504
|
+
return getattr(cls, user_define_method).__get__(client, client.__class__)
|
505
|
+
|
506
|
+
oci_class = cls._find_oci_parent()
|
507
|
+
if method == "list":
|
508
|
+
method_name = f"{method}_{camel_to_snake(oci_class.__name__)}s"
|
509
|
+
else:
|
510
|
+
method_name = f"{method}_{camel_to_snake(oci_class.__name__)}"
|
511
|
+
|
512
|
+
if hasattr(client, method_name):
|
513
|
+
return getattr(client, method_name)
|
514
|
+
|
515
|
+
raise NotImplementedError(
|
516
|
+
f"{str(oci_class)} does not have {method_name} method. "
|
517
|
+
f"Define {user_define_method} in {str(cls)}"
|
518
|
+
)
|
519
|
+
|
520
|
+
@class_or_instance_method
|
521
|
+
def from_oci_model(cls, oci_instance):
|
522
|
+
"""Initialize an instance from an instance of OCI model.
|
523
|
+
|
524
|
+
Parameters
|
525
|
+
----------
|
526
|
+
oci_instance :
|
527
|
+
An instance of an OCI model.
|
528
|
+
|
529
|
+
"""
|
530
|
+
oci_model = cls._find_oci_parent()
|
531
|
+
kwargs = {}
|
532
|
+
for attr in oci_model().swagger_types.keys():
|
533
|
+
if hasattr(oci_instance, attr):
|
534
|
+
kwargs[attr] = getattr(oci_instance, attr)
|
535
|
+
instance = cls.create_instance(**kwargs)
|
536
|
+
if hasattr(oci_instance, "attribute_map"):
|
537
|
+
instance._oci_attributes = oci_instance.attribute_map
|
538
|
+
return instance
|
539
|
+
|
540
|
+
@class_or_instance_method
|
541
|
+
def from_dict(cls, data):
|
542
|
+
"""Initialize an instance from a dictionary.
|
543
|
+
|
544
|
+
Parameters
|
545
|
+
----------
|
546
|
+
data : dict
|
547
|
+
A dictionary containing the properties to initialize the class.
|
548
|
+
|
549
|
+
"""
|
550
|
+
return cls.create_instance(**data)
|
551
|
+
|
552
|
+
@class_or_instance_method
|
553
|
+
def deserialize(cls, data: dict, to_cls: str = None):
|
554
|
+
"""Deserialize data
|
555
|
+
|
556
|
+
Parameters
|
557
|
+
----------
|
558
|
+
data : dict
|
559
|
+
A dictionary containing the data to be deserialized.
|
560
|
+
|
561
|
+
to_cls : str
|
562
|
+
The name of the OCI model class to be initialized using the data.
|
563
|
+
The OCI model class must be from the same OCI service of the OCI client (self.client).
|
564
|
+
Defaults to None, the parent OCI model class name will be used
|
565
|
+
if current class is inherited from an OCI model.
|
566
|
+
If parent OCI model class is not found or not from the same OCI service,
|
567
|
+
the data will be returned as is.
|
568
|
+
|
569
|
+
"""
|
570
|
+
if to_cls is None:
|
571
|
+
oci_model = cls._find_oci_parent()
|
572
|
+
to_cls = oci_model.__name__
|
573
|
+
return super().deserialize(data, to_cls)
|
574
|
+
|
575
|
+
@class_or_instance_method
|
576
|
+
def _get(cls, ocid):
|
577
|
+
get_method = cls._find_oci_method("get")
|
578
|
+
return get_method(ocid).data
|
579
|
+
|
580
|
+
@class_or_instance_method
|
581
|
+
def from_ocid(cls, ocid: str):
|
582
|
+
"""Initializes an object from OCID
|
583
|
+
|
584
|
+
Parameters
|
585
|
+
----------
|
586
|
+
ocid : str
|
587
|
+
The OCID of the object
|
588
|
+
|
589
|
+
"""
|
590
|
+
oci_instance = cls._get(ocid)
|
591
|
+
return cls.from_oci_model(oci_instance)
|
592
|
+
|
593
|
+
def __init__(
|
594
|
+
self,
|
595
|
+
config: dict = None,
|
596
|
+
signer: oci.signer.Signer = None,
|
597
|
+
client_kwargs: dict = None,
|
598
|
+
**kwargs,
|
599
|
+
) -> None:
|
600
|
+
# Initialize an empty object
|
601
|
+
super().__init__(config=config, signer=signer, client_kwargs=client_kwargs)
|
602
|
+
|
603
|
+
if kwargs:
|
604
|
+
self.update_from_oci_model(
|
605
|
+
self.deserialize(kwargs), merge_strategy=MergeStrategy.MERGE
|
606
|
+
)
|
607
|
+
# When from_oci_model is called to initialize the object from an OCI model,
|
608
|
+
# _oci_attributes stores the attribute names from the OCI model
|
609
|
+
# This is used to determine if an additional get request should be called to get additional attributes.
|
610
|
+
# For example, some attributes are not included in the results from OCI list request.
|
611
|
+
# When user access such attributes, an additional get request should be made to get the data from OCI.
|
612
|
+
# See __getattribute__() for the implementation.
|
613
|
+
self._oci_attributes = {}
|
614
|
+
|
615
|
+
@property
|
616
|
+
def name(self) -> str:
|
617
|
+
"""Gets the name of the object."""
|
618
|
+
if hasattr(self, "display_name"):
|
619
|
+
return getattr(self, "display_name")
|
620
|
+
return ""
|
621
|
+
|
622
|
+
@name.setter
|
623
|
+
def name(self, value: str):
|
624
|
+
"""Sets the name of the object.
|
625
|
+
|
626
|
+
Parameters
|
627
|
+
----------
|
628
|
+
value : str
|
629
|
+
The name of the object.
|
630
|
+
"""
|
631
|
+
setattr(self, "display_name", value)
|
632
|
+
|
633
|
+
def load_properties_from_env(self):
|
634
|
+
"""Loads properties from the environment"""
|
635
|
+
env_var_mapping = {
|
636
|
+
"project_id": ["PROJECT_OCID"],
|
637
|
+
OCIModelMixin.CONS_COMPARTMENT_ID: [
|
638
|
+
"JOB_RUN_COMPARTMENT_OCID",
|
639
|
+
"NB_SESSION_COMPARTMENT_OCID",
|
640
|
+
],
|
641
|
+
}
|
642
|
+
for attr, env_names in env_var_mapping.items():
|
643
|
+
try:
|
644
|
+
env_value = next(
|
645
|
+
os.environ.get(env_name)
|
646
|
+
for env_name in env_names
|
647
|
+
if os.environ.get(env_name, None) is not None
|
648
|
+
)
|
649
|
+
if getattr(self, attr, "") is None:
|
650
|
+
setattr(self, attr, env_value)
|
651
|
+
except:
|
652
|
+
pass
|
653
|
+
|
654
|
+
def to_oci_model(self, oci_model):
|
655
|
+
"""Converts the object into an instance of OCI data model.
|
656
|
+
|
657
|
+
Parameters
|
658
|
+
----------
|
659
|
+
oci_model : class or str
|
660
|
+
The OCI model to be converted to. This can be a string of the model name.
|
661
|
+
type_mapping : dict
|
662
|
+
A dictionary mapping the models.
|
663
|
+
Returns: An instance of the oci_model
|
664
|
+
|
665
|
+
Returns
|
666
|
+
-------
|
667
|
+
|
668
|
+
"""
|
669
|
+
if isinstance(oci_model, str):
|
670
|
+
oci_model_name = oci_model
|
671
|
+
else:
|
672
|
+
oci_model_name = oci_model.__name__
|
673
|
+
data = json.dumps(self.to_dict()).encode("utf8")
|
674
|
+
return self.client.base_client.deserialize_response_data(data, oci_model_name)
|
675
|
+
|
676
|
+
@staticmethod
|
677
|
+
def flatten(data: dict) -> dict:
|
678
|
+
"""Flattens a nested dictionary.
|
679
|
+
|
680
|
+
Parameters
|
681
|
+
----------
|
682
|
+
data : A nested dictionary
|
683
|
+
|
684
|
+
|
685
|
+
Returns
|
686
|
+
-------
|
687
|
+
dict
|
688
|
+
The flattened dictionary.
|
689
|
+
|
690
|
+
"""
|
691
|
+
flatten_dict = {}
|
692
|
+
for key, value in data.items():
|
693
|
+
if isinstance(value, dict):
|
694
|
+
flatten_dict.update(OCIModelMixin.flatten(value))
|
695
|
+
else:
|
696
|
+
flatten_dict[key] = value
|
697
|
+
return flatten_dict
|
698
|
+
|
699
|
+
def to_dict(self, flatten: bool = False) -> dict:
|
700
|
+
"""Converts the properties to a dictionary
|
701
|
+
|
702
|
+
Parameters
|
703
|
+
----------
|
704
|
+
flatten :
|
705
|
+
(Default value = False)
|
706
|
+
|
707
|
+
Returns
|
708
|
+
-------
|
709
|
+
|
710
|
+
"""
|
711
|
+
data = self.serialize()
|
712
|
+
if flatten:
|
713
|
+
data = OCIModelMixin.flatten(data)
|
714
|
+
return data
|
715
|
+
|
716
|
+
def update_from_oci_model(
|
717
|
+
self, oci_model_instance, merge_strategy: MergeStrategy = MergeStrategy.OVERRIDE
|
718
|
+
):
|
719
|
+
"""Updates the properties from OCI model with the same properties.
|
720
|
+
|
721
|
+
Parameters
|
722
|
+
----------
|
723
|
+
oci_model_instance :
|
724
|
+
An instance of OCI model, which should have the same properties of this class.
|
725
|
+
"""
|
726
|
+
for attr in self.swagger_types.keys():
|
727
|
+
if (
|
728
|
+
hasattr(oci_model_instance, attr)
|
729
|
+
and getattr(oci_model_instance, attr)
|
730
|
+
and (
|
731
|
+
not hasattr(self, attr)
|
732
|
+
or not getattr(self, attr)
|
733
|
+
or merge_strategy == MergeStrategy.OVERRIDE
|
734
|
+
)
|
735
|
+
):
|
736
|
+
setattr(self, attr, getattr(oci_model_instance, attr))
|
737
|
+
|
738
|
+
if hasattr(oci_model_instance, "attribute_map"):
|
739
|
+
self._oci_attributes = oci_model_instance.attribute_map
|
740
|
+
return self
|
741
|
+
|
742
|
+
def sync(self, merge_strategy: MergeStrategy = MergeStrategy.OVERRIDE):
|
743
|
+
"""Refreshes the properties of the object from OCI"""
|
744
|
+
return self.update_from_oci_model(
|
745
|
+
self.from_ocid(self.id), merge_strategy=merge_strategy
|
746
|
+
)
|
747
|
+
|
748
|
+
def delete(self):
|
749
|
+
"""Deletes the resource"""
|
750
|
+
delete_method = self._find_oci_method("delete")
|
751
|
+
delete_method(self.id)
|
752
|
+
return self
|
753
|
+
|
754
|
+
def __getattribute__(self, name: str):
|
755
|
+
"""Returns an attribute value of the resource.
|
756
|
+
|
757
|
+
This method will try to sync the values from OCI service when it is not already available locally.
|
758
|
+
Some attribute value may not be available locally if the previous OCI API call returns an OCI model
|
759
|
+
that contains only a subset of the attributes.
|
760
|
+
For example, JobSummary model contains only a subset of the attributes from the Job model.
|
761
|
+
|
762
|
+
Parameters
|
763
|
+
----------
|
764
|
+
name : str
|
765
|
+
Attribute name.
|
766
|
+
"""
|
767
|
+
skip_lookup = ["id", "attribute_map", "_oci_attributes"]
|
768
|
+
|
769
|
+
if name in skip_lookup or name.startswith("_"):
|
770
|
+
return super().__getattribute__(name)
|
771
|
+
|
772
|
+
# Ignore if _oci_attributes is not initialized
|
773
|
+
if not hasattr(self, "_oci_attributes"):
|
774
|
+
return super().__getattribute__(name)
|
775
|
+
|
776
|
+
if (
|
777
|
+
hasattr(self, "attribute_map")
|
778
|
+
and name in self.attribute_map
|
779
|
+
and name not in self._oci_attributes
|
780
|
+
and hasattr(self, "id")
|
781
|
+
and self.id
|
782
|
+
):
|
783
|
+
# Do not sync if it is in the sync process
|
784
|
+
stack = traceback.extract_stack()
|
785
|
+
for frame in reversed(stack):
|
786
|
+
if frame.name == "sync":
|
787
|
+
return super().__getattribute__(name)
|
788
|
+
# Sync only if there is no value
|
789
|
+
if not super().__getattribute__(name):
|
790
|
+
try:
|
791
|
+
self.sync(merge_strategy=MergeStrategy.MERGE)
|
792
|
+
except oci.exceptions.ServiceError as ex:
|
793
|
+
# 400 errors are usually cause by the user
|
794
|
+
if ex.status == 400:
|
795
|
+
logger.error("%s - %s: %s", self.__class__, ex.code, ex.message)
|
796
|
+
self._oci_attributes = self.attribute_map
|
797
|
+
else:
|
798
|
+
logger.error(
|
799
|
+
"Failed to synchronize the properties of %s due to service error:\n%s",
|
800
|
+
self.__class__,
|
801
|
+
traceback.format_exc(),
|
802
|
+
)
|
803
|
+
except Exception:
|
804
|
+
logger.error(
|
805
|
+
"Failed to synchronize the properties of %s.\n%s",
|
806
|
+
self.__class__,
|
807
|
+
traceback.format_exc(),
|
808
|
+
)
|
809
|
+
return super().__getattribute__(name)
|
810
|
+
|
811
|
+
@property
|
812
|
+
def status(self) -> Optional[str]:
|
813
|
+
"""Status of the object.
|
814
|
+
|
815
|
+
Returns
|
816
|
+
-------
|
817
|
+
str
|
818
|
+
Status of the object.
|
819
|
+
"""
|
820
|
+
if not self.lifecycle_state or not self.lifecycle_state in LIFECYCLE_STOP_STATE:
|
821
|
+
self.sync()
|
822
|
+
return self.lifecycle_state
|
823
|
+
|
824
|
+
def __repr__(self) -> str:
|
825
|
+
"""Displays the object as YAML."""
|
826
|
+
return self.to_yaml()
|
827
|
+
|
828
|
+
def to_yaml(self) -> str:
|
829
|
+
"""Serializes the object into YAML string.
|
830
|
+
|
831
|
+
Returns
|
832
|
+
-------
|
833
|
+
str
|
834
|
+
YAML stored in a string.
|
835
|
+
"""
|
836
|
+
return yaml.safe_dump(self.to_dict())
|
837
|
+
|
838
|
+
|
839
|
+
class OCIWorkRequestMixin:
|
840
|
+
"""Mixin class containing methods related to OCI work request"""
|
841
|
+
|
842
|
+
def wait_for_work_request(
|
843
|
+
self,
|
844
|
+
work_request_id: str,
|
845
|
+
wait_for_state: Union[str, tuple],
|
846
|
+
max_wait_seconds: int = None,
|
847
|
+
wait_interval_seconds: int = None,
|
848
|
+
):
|
849
|
+
"""Wait for a work request to be completed.
|
850
|
+
|
851
|
+
Parameters
|
852
|
+
----------
|
853
|
+
work_request_id : str
|
854
|
+
OCI work request ID
|
855
|
+
wait_for_state : str or tuple
|
856
|
+
The state to wait for. Must be a tuple for multiple states.
|
857
|
+
max_wait_seconds : int
|
858
|
+
Max wait seconds for the work request. Defaults to None (Default value from OCI SDK will be used).
|
859
|
+
wait_interval_seconds : int
|
860
|
+
Interval in seconds between each status check. Defaults to None (Default value from OCI SDK will be used).
|
861
|
+
|
862
|
+
Returns
|
863
|
+
-------
|
864
|
+
Response
|
865
|
+
OCI API Response
|
866
|
+
|
867
|
+
"""
|
868
|
+
result = None
|
869
|
+
if hasattr(self.client, "get_work_request") and callable(
|
870
|
+
getattr(self.client, "get_work_request")
|
871
|
+
):
|
872
|
+
try:
|
873
|
+
wait_period_kwargs = {}
|
874
|
+
if max_wait_seconds is not None:
|
875
|
+
wait_period_kwargs["max_wait_seconds"] = max_wait_seconds
|
876
|
+
if wait_interval_seconds is not None:
|
877
|
+
wait_period_kwargs["max_interval_seconds"] = wait_interval_seconds
|
878
|
+
|
879
|
+
logger.debug(
|
880
|
+
"Action completed. Waiting until the work request has entered state: %s",
|
881
|
+
wait_for_state,
|
882
|
+
)
|
883
|
+
result = oci.wait_until(
|
884
|
+
self.client,
|
885
|
+
self.client.get_work_request(work_request_id),
|
886
|
+
"status",
|
887
|
+
wait_for_state,
|
888
|
+
**wait_period_kwargs,
|
889
|
+
)
|
890
|
+
except oci.exceptions.MaximumWaitTimeExceeded:
|
891
|
+
# If we fail, we should show an error, but we should still provide the information to the customer
|
892
|
+
logger.error(
|
893
|
+
"Failed to wait until the work request %s entered the specified state. Maximum wait time exceeded",
|
894
|
+
work_request_id,
|
895
|
+
)
|
896
|
+
except Exception:
|
897
|
+
logger.error(
|
898
|
+
"Encountered error while waiting for work request to enter the specified state."
|
899
|
+
)
|
900
|
+
raise
|
901
|
+
else:
|
902
|
+
logger.error(
|
903
|
+
"%s does not support wait for the work request to enter the specified state",
|
904
|
+
str(self.client.__class__),
|
905
|
+
)
|
906
|
+
return result
|
907
|
+
|
908
|
+
def get_work_request_response(
|
909
|
+
self,
|
910
|
+
response: str,
|
911
|
+
wait_for_state: Union[str, tuple],
|
912
|
+
success_state: str,
|
913
|
+
max_wait_seconds: int = None,
|
914
|
+
wait_interval_seconds: int = None,
|
915
|
+
error_msg: str = "",
|
916
|
+
):
|
917
|
+
if "opc-work-request-id" in response.headers:
|
918
|
+
opc_work_request_id = response.headers.get("opc-work-request-id")
|
919
|
+
work_request_response = self.wait_for_work_request(
|
920
|
+
opc_work_request_id,
|
921
|
+
wait_for_state=wait_for_state,
|
922
|
+
max_wait_seconds=max_wait_seconds,
|
923
|
+
wait_interval_seconds=wait_interval_seconds,
|
924
|
+
)
|
925
|
+
# Raise an error if the failed to create the resource.
|
926
|
+
if work_request_response.data.status != success_state:
|
927
|
+
raise oci.exceptions.RequestException(
|
928
|
+
f"{error_msg}\n" + str(work_request_response.data)
|
929
|
+
)
|
930
|
+
|
931
|
+
else:
|
932
|
+
# This will likely never happen as OCI SDK will raise an error if the HTTP request is not successful.
|
933
|
+
raise oci.exceptions.RequestException(
|
934
|
+
f"opc-work-request-id not found in response headers: {response.headers}"
|
935
|
+
)
|
936
|
+
return work_request_response
|
937
|
+
|
938
|
+
|
939
|
+
class OCIModelWithNameMixin:
|
940
|
+
"""Mixin class to operate OCI model which contains name property."""
|
941
|
+
|
942
|
+
@classmethod
|
943
|
+
def from_name(cls, name: str, compartment_id: Optional[str] = None):
|
944
|
+
"""Initializes an object from name.
|
945
|
+
|
946
|
+
Parameters
|
947
|
+
----------
|
948
|
+
name: str
|
949
|
+
The name of the object.
|
950
|
+
compartment_id: (str, optional). Defaults to None.
|
951
|
+
Compartment OCID of the OCI resources. If `compartment_id` is not specified,
|
952
|
+
the value will be taken from environment variables.
|
953
|
+
"""
|
954
|
+
res = cls.list_resource(compartment_id=compartment_id, limit=1, name=name)
|
955
|
+
if not res:
|
956
|
+
raise OCIModelNotExists()
|
957
|
+
return cls.from_oci_model(res[0])
|