oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/__init__.py +40 -0
- ads/aqua/app.py +507 -0
- ads/aqua/cli.py +96 -0
- ads/aqua/client/__init__.py +3 -0
- ads/aqua/client/client.py +836 -0
- ads/aqua/client/openai_client.py +305 -0
- ads/aqua/common/__init__.py +5 -0
- ads/aqua/common/decorator.py +125 -0
- ads/aqua/common/entities.py +274 -0
- ads/aqua/common/enums.py +134 -0
- ads/aqua/common/errors.py +109 -0
- ads/aqua/common/utils.py +1295 -0
- ads/aqua/config/__init__.py +4 -0
- ads/aqua/config/container_config.py +246 -0
- ads/aqua/config/evaluation/__init__.py +4 -0
- ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
- ads/aqua/config/utils/__init__.py +4 -0
- ads/aqua/config/utils/serializer.py +339 -0
- ads/aqua/constants.py +116 -0
- ads/aqua/data.py +14 -0
- ads/aqua/dummy_data/icon.txt +1 -0
- ads/aqua/dummy_data/oci_model_deployments.json +56 -0
- ads/aqua/dummy_data/oci_models.json +1 -0
- ads/aqua/dummy_data/readme.md +26 -0
- ads/aqua/evaluation/__init__.py +8 -0
- ads/aqua/evaluation/constants.py +53 -0
- ads/aqua/evaluation/entities.py +186 -0
- ads/aqua/evaluation/errors.py +70 -0
- ads/aqua/evaluation/evaluation.py +1814 -0
- ads/aqua/extension/__init__.py +42 -0
- ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
- ads/aqua/extension/base_handler.py +90 -0
- ads/aqua/extension/common_handler.py +121 -0
- ads/aqua/extension/common_ws_msg_handler.py +36 -0
- ads/aqua/extension/deployment_handler.py +381 -0
- ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
- ads/aqua/extension/errors.py +30 -0
- ads/aqua/extension/evaluation_handler.py +129 -0
- ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
- ads/aqua/extension/finetune_handler.py +96 -0
- ads/aqua/extension/model_handler.py +390 -0
- ads/aqua/extension/models/__init__.py +0 -0
- ads/aqua/extension/models/ws_models.py +145 -0
- ads/aqua/extension/models_ws_msg_handler.py +50 -0
- ads/aqua/extension/ui_handler.py +300 -0
- ads/aqua/extension/ui_websocket_handler.py +130 -0
- ads/aqua/extension/utils.py +133 -0
- ads/aqua/finetuning/__init__.py +7 -0
- ads/aqua/finetuning/constants.py +23 -0
- ads/aqua/finetuning/entities.py +181 -0
- ads/aqua/finetuning/finetuning.py +749 -0
- ads/aqua/model/__init__.py +8 -0
- ads/aqua/model/constants.py +60 -0
- ads/aqua/model/entities.py +385 -0
- ads/aqua/model/enums.py +32 -0
- ads/aqua/model/model.py +2134 -0
- ads/aqua/model/utils.py +52 -0
- ads/aqua/modeldeployment/__init__.py +6 -0
- ads/aqua/modeldeployment/constants.py +10 -0
- ads/aqua/modeldeployment/deployment.py +1315 -0
- ads/aqua/modeldeployment/entities.py +653 -0
- ads/aqua/modeldeployment/utils.py +543 -0
- ads/aqua/resources/gpu_shapes_index.json +94 -0
- ads/aqua/server/__init__.py +4 -0
- ads/aqua/server/__main__.py +24 -0
- ads/aqua/server/app.py +47 -0
- ads/aqua/server/aqua_spec.yml +1291 -0
- ads/aqua/training/__init__.py +4 -0
- ads/aqua/training/exceptions.py +476 -0
- ads/aqua/ui.py +519 -0
- ads/automl/__init__.py +9 -0
- ads/automl/driver.py +330 -0
- ads/automl/provider.py +975 -0
- ads/bds/__init__.py +5 -0
- ads/bds/auth.py +127 -0
- ads/bds/big_data_service.py +255 -0
- ads/catalog/__init__.py +19 -0
- ads/catalog/model.py +1576 -0
- ads/catalog/notebook.py +461 -0
- ads/catalog/project.py +468 -0
- ads/catalog/summary.py +178 -0
- ads/common/__init__.py +11 -0
- ads/common/analyzer.py +65 -0
- ads/common/artifact/.model-ignore +63 -0
- ads/common/artifact/__init__.py +10 -0
- ads/common/auth.py +1122 -0
- ads/common/card_identifier.py +83 -0
- ads/common/config.py +647 -0
- ads/common/data.py +165 -0
- ads/common/decorator/__init__.py +9 -0
- ads/common/decorator/argument_to_case.py +88 -0
- ads/common/decorator/deprecate.py +69 -0
- ads/common/decorator/require_nonempty_arg.py +65 -0
- ads/common/decorator/runtime_dependency.py +178 -0
- ads/common/decorator/threaded.py +97 -0
- ads/common/decorator/utils.py +35 -0
- ads/common/dsc_file_system.py +303 -0
- ads/common/error.py +14 -0
- ads/common/extended_enum.py +81 -0
- ads/common/function/__init__.py +5 -0
- ads/common/function/fn_util.py +142 -0
- ads/common/function/func_conf.yaml +25 -0
- ads/common/ipython.py +76 -0
- ads/common/model.py +679 -0
- ads/common/model_artifact.py +1759 -0
- ads/common/model_artifact_schema.json +107 -0
- ads/common/model_export_util.py +664 -0
- ads/common/model_metadata.py +24 -0
- ads/common/object_storage_details.py +296 -0
- ads/common/oci_client.py +179 -0
- ads/common/oci_datascience.py +46 -0
- ads/common/oci_logging.py +1144 -0
- ads/common/oci_mixin.py +957 -0
- ads/common/oci_resource.py +136 -0
- ads/common/serializer.py +559 -0
- ads/common/utils.py +1852 -0
- ads/common/word_lists.py +1491 -0
- ads/common/work_request.py +189 -0
- ads/config.py +1 -0
- ads/data_labeling/__init__.py +13 -0
- ads/data_labeling/boundingbox.py +253 -0
- ads/data_labeling/constants.py +47 -0
- ads/data_labeling/data_labeling_service.py +244 -0
- ads/data_labeling/interface/__init__.py +5 -0
- ads/data_labeling/interface/loader.py +16 -0
- ads/data_labeling/interface/parser.py +16 -0
- ads/data_labeling/interface/reader.py +23 -0
- ads/data_labeling/loader/__init__.py +5 -0
- ads/data_labeling/loader/file_loader.py +241 -0
- ads/data_labeling/metadata.py +110 -0
- ads/data_labeling/mixin/__init__.py +5 -0
- ads/data_labeling/mixin/data_labeling.py +232 -0
- ads/data_labeling/ner.py +129 -0
- ads/data_labeling/parser/__init__.py +5 -0
- ads/data_labeling/parser/dls_record_parser.py +388 -0
- ads/data_labeling/parser/export_metadata_parser.py +94 -0
- ads/data_labeling/parser/export_record_parser.py +473 -0
- ads/data_labeling/reader/__init__.py +5 -0
- ads/data_labeling/reader/dataset_reader.py +574 -0
- ads/data_labeling/reader/dls_record_reader.py +121 -0
- ads/data_labeling/reader/export_record_reader.py +62 -0
- ads/data_labeling/reader/jsonl_reader.py +75 -0
- ads/data_labeling/reader/metadata_reader.py +203 -0
- ads/data_labeling/reader/record_reader.py +263 -0
- ads/data_labeling/record.py +52 -0
- ads/data_labeling/visualizer/__init__.py +5 -0
- ads/data_labeling/visualizer/image_visualizer.py +525 -0
- ads/data_labeling/visualizer/text_visualizer.py +357 -0
- ads/database/__init__.py +5 -0
- ads/database/connection.py +338 -0
- ads/dataset/__init__.py +10 -0
- ads/dataset/capabilities.md +51 -0
- ads/dataset/classification_dataset.py +339 -0
- ads/dataset/correlation.py +226 -0
- ads/dataset/correlation_plot.py +563 -0
- ads/dataset/dask_series.py +173 -0
- ads/dataset/dataframe_transformer.py +110 -0
- ads/dataset/dataset.py +1979 -0
- ads/dataset/dataset_browser.py +360 -0
- ads/dataset/dataset_with_target.py +995 -0
- ads/dataset/exception.py +25 -0
- ads/dataset/factory.py +987 -0
- ads/dataset/feature_engineering_transformer.py +35 -0
- ads/dataset/feature_selection.py +107 -0
- ads/dataset/forecasting_dataset.py +26 -0
- ads/dataset/helper.py +1450 -0
- ads/dataset/label_encoder.py +99 -0
- ads/dataset/mixin/__init__.py +5 -0
- ads/dataset/mixin/dataset_accessor.py +134 -0
- ads/dataset/pipeline.py +58 -0
- ads/dataset/plot.py +710 -0
- ads/dataset/progress.py +86 -0
- ads/dataset/recommendation.py +297 -0
- ads/dataset/recommendation_transformer.py +502 -0
- ads/dataset/regression_dataset.py +14 -0
- ads/dataset/sampled_dataset.py +1050 -0
- ads/dataset/target.py +98 -0
- ads/dataset/timeseries.py +18 -0
- ads/dbmixin/__init__.py +5 -0
- ads/dbmixin/db_pandas_accessor.py +153 -0
- ads/environment/__init__.py +9 -0
- ads/environment/ml_runtime.py +66 -0
- ads/evaluations/README.md +14 -0
- ads/evaluations/__init__.py +109 -0
- ads/evaluations/evaluation_plot.py +983 -0
- ads/evaluations/evaluator.py +1334 -0
- ads/evaluations/statistical_metrics.py +543 -0
- ads/experiments/__init__.py +9 -0
- ads/experiments/capabilities.md +0 -0
- ads/explanations/__init__.py +21 -0
- ads/explanations/base_explainer.py +142 -0
- ads/explanations/capabilities.md +83 -0
- ads/explanations/explainer.py +190 -0
- ads/explanations/mlx_global_explainer.py +1050 -0
- ads/explanations/mlx_interface.py +386 -0
- ads/explanations/mlx_local_explainer.py +287 -0
- ads/explanations/mlx_whatif_explainer.py +201 -0
- ads/feature_engineering/__init__.py +20 -0
- ads/feature_engineering/accessor/__init__.py +5 -0
- ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
- ads/feature_engineering/accessor/mixin/__init__.py +5 -0
- ads/feature_engineering/accessor/mixin/correlation.py +166 -0
- ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
- ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
- ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
- ads/feature_engineering/accessor/mixin/utils.py +65 -0
- ads/feature_engineering/accessor/series_accessor.py +431 -0
- ads/feature_engineering/adsimage/__init__.py +5 -0
- ads/feature_engineering/adsimage/image.py +192 -0
- ads/feature_engineering/adsimage/image_reader.py +170 -0
- ads/feature_engineering/adsimage/interface/__init__.py +5 -0
- ads/feature_engineering/adsimage/interface/reader.py +19 -0
- ads/feature_engineering/adsstring/__init__.py +7 -0
- ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
- ads/feature_engineering/adsstring/string/__init__.py +8 -0
- ads/feature_engineering/data_schema.json +57 -0
- ads/feature_engineering/dataset/__init__.py +5 -0
- ads/feature_engineering/dataset/zip_code_data.py +42062 -0
- ads/feature_engineering/exceptions.py +40 -0
- ads/feature_engineering/feature_type/__init__.py +133 -0
- ads/feature_engineering/feature_type/address.py +184 -0
- ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
- ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
- ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
- ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
- ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
- ads/feature_engineering/feature_type/adsstring/string.py +258 -0
- ads/feature_engineering/feature_type/base.py +58 -0
- ads/feature_engineering/feature_type/boolean.py +183 -0
- ads/feature_engineering/feature_type/category.py +146 -0
- ads/feature_engineering/feature_type/constant.py +137 -0
- ads/feature_engineering/feature_type/continuous.py +151 -0
- ads/feature_engineering/feature_type/creditcard.py +314 -0
- ads/feature_engineering/feature_type/datetime.py +190 -0
- ads/feature_engineering/feature_type/discrete.py +134 -0
- ads/feature_engineering/feature_type/document.py +43 -0
- ads/feature_engineering/feature_type/gis.py +251 -0
- ads/feature_engineering/feature_type/handler/__init__.py +5 -0
- ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
- ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
- ads/feature_engineering/feature_type/handler/warnings.py +128 -0
- ads/feature_engineering/feature_type/integer.py +142 -0
- ads/feature_engineering/feature_type/ip_address.py +144 -0
- ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
- ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
- ads/feature_engineering/feature_type/lat_long.py +256 -0
- ads/feature_engineering/feature_type/object.py +43 -0
- ads/feature_engineering/feature_type/ordinal.py +132 -0
- ads/feature_engineering/feature_type/phone_number.py +135 -0
- ads/feature_engineering/feature_type/string.py +171 -0
- ads/feature_engineering/feature_type/text.py +93 -0
- ads/feature_engineering/feature_type/unknown.py +43 -0
- ads/feature_engineering/feature_type/zip_code.py +164 -0
- ads/feature_engineering/feature_type_manager.py +406 -0
- ads/feature_engineering/schema.py +795 -0
- ads/feature_engineering/utils.py +245 -0
- ads/feature_store/.readthedocs.yaml +19 -0
- ads/feature_store/README.md +65 -0
- ads/feature_store/__init__.py +9 -0
- ads/feature_store/common/__init__.py +0 -0
- ads/feature_store/common/enums.py +339 -0
- ads/feature_store/common/exceptions.py +18 -0
- ads/feature_store/common/spark_session_singleton.py +125 -0
- ads/feature_store/common/utils/__init__.py +0 -0
- ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
- ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
- ads/feature_store/common/utils/transformation_utils.py +82 -0
- ads/feature_store/common/utils/utility.py +403 -0
- ads/feature_store/data_validation/__init__.py +0 -0
- ads/feature_store/data_validation/great_expectation.py +129 -0
- ads/feature_store/dataset.py +1230 -0
- ads/feature_store/dataset_job.py +530 -0
- ads/feature_store/docs/Dockerfile +7 -0
- ads/feature_store/docs/Makefile +44 -0
- ads/feature_store/docs/conf.py +28 -0
- ads/feature_store/docs/requirements.txt +14 -0
- ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
- ads/feature_store/docs/source/cicd.rst +137 -0
- ads/feature_store/docs/source/conf.py +86 -0
- ads/feature_store/docs/source/data_versioning.rst +33 -0
- ads/feature_store/docs/source/dataset.rst +388 -0
- ads/feature_store/docs/source/dataset_job.rst +27 -0
- ads/feature_store/docs/source/demo.rst +70 -0
- ads/feature_store/docs/source/entity.rst +78 -0
- ads/feature_store/docs/source/feature_group.rst +624 -0
- ads/feature_store/docs/source/feature_group_job.rst +29 -0
- ads/feature_store/docs/source/feature_store.rst +122 -0
- ads/feature_store/docs/source/feature_store_class.rst +123 -0
- ads/feature_store/docs/source/feature_validation.rst +66 -0
- ads/feature_store/docs/source/figures/cicd.png +0 -0
- ads/feature_store/docs/source/figures/data_validation.png +0 -0
- ads/feature_store/docs/source/figures/data_versioning.png +0 -0
- ads/feature_store/docs/source/figures/dataset.gif +0 -0
- ads/feature_store/docs/source/figures/dataset.png +0 -0
- ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
- ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
- ads/feature_store/docs/source/figures/entity.png +0 -0
- ads/feature_store/docs/source/figures/feature_group.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
- ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
- ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
- ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
- ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
- ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
- ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
- ads/feature_store/docs/source/figures/overview.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
- ads/feature_store/docs/source/figures/stats_1.png +0 -0
- ads/feature_store/docs/source/figures/stats_2.png +0 -0
- ads/feature_store/docs/source/figures/stats_d.png +0 -0
- ads/feature_store/docs/source/figures/stats_fg.png +0 -0
- ads/feature_store/docs/source/figures/transformation.png +0 -0
- ads/feature_store/docs/source/figures/transformations.gif +0 -0
- ads/feature_store/docs/source/figures/validation.png +0 -0
- ads/feature_store/docs/source/figures/validation_fg.png +0 -0
- ads/feature_store/docs/source/figures/validation_results.png +0 -0
- ads/feature_store/docs/source/figures/validation_summary.png +0 -0
- ads/feature_store/docs/source/index.rst +81 -0
- ads/feature_store/docs/source/module.rst +8 -0
- ads/feature_store/docs/source/notebook.rst +94 -0
- ads/feature_store/docs/source/overview.rst +47 -0
- ads/feature_store/docs/source/quickstart.rst +176 -0
- ads/feature_store/docs/source/release_notes.rst +194 -0
- ads/feature_store/docs/source/setup_feature_store.rst +81 -0
- ads/feature_store/docs/source/statistics.rst +58 -0
- ads/feature_store/docs/source/transformation.rst +199 -0
- ads/feature_store/docs/source/ui.rst +65 -0
- ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
- ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
- ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
- ads/feature_store/entity.py +718 -0
- ads/feature_store/execution_strategy/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
- ads/feature_store/execution_strategy/engine/__init__.py +0 -0
- ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
- ads/feature_store/execution_strategy/execution_strategy.py +113 -0
- ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
- ads/feature_store/execution_strategy/spark/__init__.py +0 -0
- ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
- ads/feature_store/feature.py +192 -0
- ads/feature_store/feature_group.py +1494 -0
- ads/feature_store/feature_group_expectation.py +346 -0
- ads/feature_store/feature_group_job.py +602 -0
- ads/feature_store/feature_lineage/__init__.py +0 -0
- ads/feature_store/feature_lineage/graphviz_service.py +180 -0
- ads/feature_store/feature_option_details.py +50 -0
- ads/feature_store/feature_statistics/__init__.py +0 -0
- ads/feature_store/feature_statistics/statistics_service.py +99 -0
- ads/feature_store/feature_store.py +699 -0
- ads/feature_store/feature_store_registrar.py +518 -0
- ads/feature_store/input_feature_detail.py +149 -0
- ads/feature_store/mixin/__init__.py +4 -0
- ads/feature_store/mixin/oci_feature_store.py +145 -0
- ads/feature_store/model_details.py +73 -0
- ads/feature_store/query/__init__.py +0 -0
- ads/feature_store/query/filter.py +266 -0
- ads/feature_store/query/generator/__init__.py +0 -0
- ads/feature_store/query/generator/query_generator.py +298 -0
- ads/feature_store/query/join.py +161 -0
- ads/feature_store/query/query.py +403 -0
- ads/feature_store/query/validator/__init__.py +0 -0
- ads/feature_store/query/validator/query_validator.py +57 -0
- ads/feature_store/response/__init__.py +0 -0
- ads/feature_store/response/response_builder.py +68 -0
- ads/feature_store/service/__init__.py +0 -0
- ads/feature_store/service/oci_dataset.py +139 -0
- ads/feature_store/service/oci_dataset_job.py +199 -0
- ads/feature_store/service/oci_entity.py +125 -0
- ads/feature_store/service/oci_feature_group.py +164 -0
- ads/feature_store/service/oci_feature_group_job.py +214 -0
- ads/feature_store/service/oci_feature_store.py +182 -0
- ads/feature_store/service/oci_lineage.py +87 -0
- ads/feature_store/service/oci_transformation.py +104 -0
- ads/feature_store/statistics/__init__.py +0 -0
- ads/feature_store/statistics/abs_feature_value.py +49 -0
- ads/feature_store/statistics/charts/__init__.py +0 -0
- ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
- ads/feature_store/statistics/charts/box_plot.py +148 -0
- ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
- ads/feature_store/statistics/charts/probability_distribution.py +68 -0
- ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
- ads/feature_store/statistics/feature_stat.py +126 -0
- ads/feature_store/statistics/generic_feature_value.py +33 -0
- ads/feature_store/statistics/statistics.py +41 -0
- ads/feature_store/statistics_config.py +101 -0
- ads/feature_store/templates/feature_store_template.yaml +45 -0
- ads/feature_store/transformation.py +499 -0
- ads/feature_store/validation_output.py +57 -0
- ads/hpo/__init__.py +9 -0
- ads/hpo/_imports.py +91 -0
- ads/hpo/ads_search_space.py +439 -0
- ads/hpo/distributions.py +325 -0
- ads/hpo/objective.py +280 -0
- ads/hpo/search_cv.py +1657 -0
- ads/hpo/stopping_criterion.py +75 -0
- ads/hpo/tuner_artifact.py +413 -0
- ads/hpo/utils.py +91 -0
- ads/hpo/validation.py +140 -0
- ads/hpo/visualization/__init__.py +5 -0
- ads/hpo/visualization/_contour.py +23 -0
- ads/hpo/visualization/_edf.py +20 -0
- ads/hpo/visualization/_intermediate_values.py +21 -0
- ads/hpo/visualization/_optimization_history.py +25 -0
- ads/hpo/visualization/_parallel_coordinate.py +169 -0
- ads/hpo/visualization/_param_importances.py +26 -0
- ads/jobs/__init__.py +53 -0
- ads/jobs/ads_job.py +663 -0
- ads/jobs/builders/__init__.py +5 -0
- ads/jobs/builders/base.py +156 -0
- ads/jobs/builders/infrastructure/__init__.py +6 -0
- ads/jobs/builders/infrastructure/base.py +165 -0
- ads/jobs/builders/infrastructure/dataflow.py +1252 -0
- ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
- ads/jobs/builders/infrastructure/utils.py +65 -0
- ads/jobs/builders/runtimes/__init__.py +5 -0
- ads/jobs/builders/runtimes/artifact.py +338 -0
- ads/jobs/builders/runtimes/base.py +325 -0
- ads/jobs/builders/runtimes/container_runtime.py +242 -0
- ads/jobs/builders/runtimes/python_runtime.py +1016 -0
- ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
- ads/jobs/cli.py +104 -0
- ads/jobs/env_var_parser.py +131 -0
- ads/jobs/extension.py +160 -0
- ads/jobs/schema/__init__.py +5 -0
- ads/jobs/schema/infrastructure_schema.json +116 -0
- ads/jobs/schema/job_schema.json +42 -0
- ads/jobs/schema/runtime_schema.json +183 -0
- ads/jobs/schema/validator.py +141 -0
- ads/jobs/serializer.py +296 -0
- ads/jobs/templates/__init__.py +5 -0
- ads/jobs/templates/container.py +6 -0
- ads/jobs/templates/driver_notebook.py +177 -0
- ads/jobs/templates/driver_oci.py +500 -0
- ads/jobs/templates/driver_python.py +48 -0
- ads/jobs/templates/driver_pytorch.py +852 -0
- ads/jobs/templates/driver_utils.py +615 -0
- ads/jobs/templates/hostname_from_env.c +55 -0
- ads/jobs/templates/oci_metrics.py +181 -0
- ads/jobs/utils.py +104 -0
- ads/llm/__init__.py +28 -0
- ads/llm/autogen/__init__.py +2 -0
- ads/llm/autogen/constants.py +15 -0
- ads/llm/autogen/reports/__init__.py +2 -0
- ads/llm/autogen/reports/base.py +67 -0
- ads/llm/autogen/reports/data.py +103 -0
- ads/llm/autogen/reports/session.py +526 -0
- ads/llm/autogen/reports/templates/chat_box.html +13 -0
- ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
- ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
- ads/llm/autogen/reports/utils.py +56 -0
- ads/llm/autogen/v02/__init__.py +4 -0
- ads/llm/autogen/v02/client.py +295 -0
- ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
- ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
- ads/llm/autogen/v02/loggers/__init__.py +6 -0
- ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
- ads/llm/autogen/v02/loggers/session_logger.py +580 -0
- ads/llm/autogen/v02/loggers/utils.py +86 -0
- ads/llm/autogen/v02/runtime_logging.py +163 -0
- ads/llm/chain.py +268 -0
- ads/llm/chat_template.py +31 -0
- ads/llm/deploy.py +63 -0
- ads/llm/guardrails/__init__.py +5 -0
- ads/llm/guardrails/base.py +442 -0
- ads/llm/guardrails/huggingface.py +44 -0
- ads/llm/langchain/__init__.py +5 -0
- ads/llm/langchain/plugins/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
- ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
- ads/llm/langchain/plugins/llms/__init__.py +5 -0
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
- ads/llm/requirements.txt +3 -0
- ads/llm/serialize.py +219 -0
- ads/llm/serializers/__init__.py +0 -0
- ads/llm/serializers/retrieval_qa.py +153 -0
- ads/llm/serializers/runnable_parallel.py +27 -0
- ads/llm/templates/score_chain.jinja2 +155 -0
- ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
- ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
- ads/model/__init__.py +52 -0
- ads/model/artifact.py +573 -0
- ads/model/artifact_downloader.py +254 -0
- ads/model/artifact_uploader.py +267 -0
- ads/model/base_properties.py +238 -0
- ads/model/common/.model-ignore +66 -0
- ads/model/common/__init__.py +5 -0
- ads/model/common/utils.py +142 -0
- ads/model/datascience_model.py +2635 -0
- ads/model/deployment/__init__.py +20 -0
- ads/model/deployment/common/__init__.py +5 -0
- ads/model/deployment/common/utils.py +308 -0
- ads/model/deployment/model_deployer.py +466 -0
- ads/model/deployment/model_deployment.py +1846 -0
- ads/model/deployment/model_deployment_infrastructure.py +671 -0
- ads/model/deployment/model_deployment_properties.py +493 -0
- ads/model/deployment/model_deployment_runtime.py +838 -0
- ads/model/extractor/__init__.py +5 -0
- ads/model/extractor/automl_extractor.py +74 -0
- ads/model/extractor/embedding_onnx_extractor.py +80 -0
- ads/model/extractor/huggingface_extractor.py +88 -0
- ads/model/extractor/keras_extractor.py +84 -0
- ads/model/extractor/lightgbm_extractor.py +93 -0
- ads/model/extractor/model_info_extractor.py +114 -0
- ads/model/extractor/model_info_extractor_factory.py +105 -0
- ads/model/extractor/pytorch_extractor.py +87 -0
- ads/model/extractor/sklearn_extractor.py +112 -0
- ads/model/extractor/spark_extractor.py +89 -0
- ads/model/extractor/tensorflow_extractor.py +85 -0
- ads/model/extractor/xgboost_extractor.py +94 -0
- ads/model/framework/__init__.py +5 -0
- ads/model/framework/automl_model.py +178 -0
- ads/model/framework/embedding_onnx_model.py +438 -0
- ads/model/framework/huggingface_model.py +399 -0
- ads/model/framework/lightgbm_model.py +266 -0
- ads/model/framework/pytorch_model.py +266 -0
- ads/model/framework/sklearn_model.py +250 -0
- ads/model/framework/spark_model.py +326 -0
- ads/model/framework/tensorflow_model.py +254 -0
- ads/model/framework/xgboost_model.py +258 -0
- ads/model/generic_model.py +3518 -0
- ads/model/model_artifact_boilerplate/README.md +381 -0
- ads/model/model_artifact_boilerplate/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
- ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
- ads/model/model_artifact_boilerplate/score.py +61 -0
- ads/model/model_file_description_schema.json +68 -0
- ads/model/model_introspect.py +331 -0
- ads/model/model_metadata.py +1810 -0
- ads/model/model_metadata_mixin.py +460 -0
- ads/model/model_properties.py +63 -0
- ads/model/model_version_set.py +739 -0
- ads/model/runtime/__init__.py +5 -0
- ads/model/runtime/env_info.py +306 -0
- ads/model/runtime/model_deployment_details.py +37 -0
- ads/model/runtime/model_provenance_details.py +58 -0
- ads/model/runtime/runtime_info.py +81 -0
- ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
- ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
- ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
- ads/model/runtime/utils.py +201 -0
- ads/model/serde/__init__.py +5 -0
- ads/model/serde/common.py +40 -0
- ads/model/serde/model_input.py +547 -0
- ads/model/serde/model_serializer.py +1184 -0
- ads/model/service/__init__.py +5 -0
- ads/model/service/oci_datascience_model.py +1076 -0
- ads/model/service/oci_datascience_model_deployment.py +500 -0
- ads/model/service/oci_datascience_model_version_set.py +176 -0
- ads/model/transformer/__init__.py +5 -0
- ads/model/transformer/onnx_transformer.py +324 -0
- ads/mysqldb/__init__.py +5 -0
- ads/mysqldb/mysql_db.py +227 -0
- ads/opctl/__init__.py +18 -0
- ads/opctl/anomaly_detection.py +11 -0
- ads/opctl/backend/__init__.py +5 -0
- ads/opctl/backend/ads_dataflow.py +353 -0
- ads/opctl/backend/ads_ml_job.py +710 -0
- ads/opctl/backend/ads_ml_pipeline.py +164 -0
- ads/opctl/backend/ads_model_deployment.py +209 -0
- ads/opctl/backend/base.py +146 -0
- ads/opctl/backend/local.py +1053 -0
- ads/opctl/backend/marketplace/__init__.py +9 -0
- ads/opctl/backend/marketplace/helm_helper.py +173 -0
- ads/opctl/backend/marketplace/local_marketplace.py +271 -0
- ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
- ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
- ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
- ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
- ads/opctl/backend/marketplace/models/__init__.py +5 -0
- ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
- ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
- ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
- ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
- ads/opctl/cli.py +707 -0
- ads/opctl/cmds.py +869 -0
- ads/opctl/conda/__init__.py +5 -0
- ads/opctl/conda/cli.py +193 -0
- ads/opctl/conda/cmds.py +749 -0
- ads/opctl/conda/config.yaml +34 -0
- ads/opctl/conda/manifest_template.yaml +13 -0
- ads/opctl/conda/multipart_uploader.py +188 -0
- ads/opctl/conda/pack.py +89 -0
- ads/opctl/config/__init__.py +5 -0
- ads/opctl/config/base.py +57 -0
- ads/opctl/config/diagnostics/__init__.py +5 -0
- ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
- ads/opctl/config/merger.py +255 -0
- ads/opctl/config/resolver.py +297 -0
- ads/opctl/config/utils.py +79 -0
- ads/opctl/config/validator.py +17 -0
- ads/opctl/config/versioner.py +68 -0
- ads/opctl/config/yaml_parsers/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/base.py +58 -0
- ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
- ads/opctl/constants.py +66 -0
- ads/opctl/decorator/__init__.py +5 -0
- ads/opctl/decorator/common.py +129 -0
- ads/opctl/diagnostics/__init__.py +5 -0
- ads/opctl/diagnostics/__main__.py +25 -0
- ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
- ads/opctl/diagnostics/check_requirements.py +144 -0
- ads/opctl/diagnostics/requirement_exception.py +9 -0
- ads/opctl/distributed/README.md +109 -0
- ads/opctl/distributed/__init__.py +5 -0
- ads/opctl/distributed/certificates.py +32 -0
- ads/opctl/distributed/cli.py +207 -0
- ads/opctl/distributed/cmds.py +731 -0
- ads/opctl/distributed/common/__init__.py +5 -0
- ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
- ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
- ads/opctl/distributed/common/cluster_config_helper.py +103 -0
- ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
- ads/opctl/distributed/common/cluster_runner.py +54 -0
- ads/opctl/distributed/common/framework_factory.py +29 -0
- ads/opctl/docker/Dockerfile.job +103 -0
- ads/opctl/docker/Dockerfile.job.arm +107 -0
- ads/opctl/docker/Dockerfile.job.gpu +175 -0
- ads/opctl/docker/base-env.yaml +13 -0
- ads/opctl/docker/cuda.repo +6 -0
- ads/opctl/docker/operator/.dockerignore +0 -0
- ads/opctl/docker/operator/Dockerfile +41 -0
- ads/opctl/docker/operator/Dockerfile.gpu +85 -0
- ads/opctl/docker/operator/cuda.repo +6 -0
- ads/opctl/docker/operator/environment.yaml +8 -0
- ads/opctl/forecast.py +11 -0
- ads/opctl/index.yaml +3 -0
- ads/opctl/model/__init__.py +5 -0
- ads/opctl/model/cli.py +65 -0
- ads/opctl/model/cmds.py +73 -0
- ads/opctl/operator/README.md +4 -0
- ads/opctl/operator/__init__.py +31 -0
- ads/opctl/operator/cli.py +344 -0
- ads/opctl/operator/cmd.py +596 -0
- ads/opctl/operator/common/__init__.py +5 -0
- ads/opctl/operator/common/backend_factory.py +460 -0
- ads/opctl/operator/common/const.py +27 -0
- ads/opctl/operator/common/data/synthetic.csv +16001 -0
- ads/opctl/operator/common/dictionary_merger.py +148 -0
- ads/opctl/operator/common/errors.py +42 -0
- ads/opctl/operator/common/operator_config.py +99 -0
- ads/opctl/operator/common/operator_loader.py +811 -0
- ads/opctl/operator/common/operator_schema.yaml +130 -0
- ads/opctl/operator/common/operator_yaml_generator.py +152 -0
- ads/opctl/operator/common/utils.py +208 -0
- ads/opctl/operator/lowcode/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
- ads/opctl/operator/lowcode/anomaly/README.md +207 -0
- ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
- ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
- ads/opctl/operator/lowcode/anomaly/const.py +167 -0
- ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
- ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
- ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
- ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
- ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
- ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
- ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
- ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
- ads/opctl/operator/lowcode/common/__init__.py +5 -0
- ads/opctl/operator/lowcode/common/const.py +10 -0
- ads/opctl/operator/lowcode/common/data.py +116 -0
- ads/opctl/operator/lowcode/common/errors.py +47 -0
- ads/opctl/operator/lowcode/common/transformations.py +296 -0
- ads/opctl/operator/lowcode/common/utils.py +384 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
- ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
- ads/opctl/operator/lowcode/forecast/README.md +209 -0
- ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
- ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
- ads/opctl/operator/lowcode/forecast/const.py +92 -0
- ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
- ads/opctl/operator/lowcode/forecast/errors.py +26 -0
- ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
- ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
- ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
- ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
- ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
- ads/opctl/operator/lowcode/forecast/model/prophet.py +450 -0
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
- ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
- ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
- ads/opctl/operator/lowcode/forecast/utils.py +397 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
- ads/opctl/operator/lowcode/pii/MLoperator +17 -0
- ads/opctl/operator/lowcode/pii/README.md +208 -0
- ads/opctl/operator/lowcode/pii/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/__main__.py +78 -0
- ads/opctl/operator/lowcode/pii/cmd.py +39 -0
- ads/opctl/operator/lowcode/pii/constant.py +84 -0
- ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
- ads/opctl/operator/lowcode/pii/errors.py +27 -0
- ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
- ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
- ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
- ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
- ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
- ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
- ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
- ads/opctl/operator/lowcode/pii/model/report.py +487 -0
- ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
- ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
- ads/opctl/operator/lowcode/pii/utils.py +43 -0
- ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
- ads/opctl/operator/lowcode/recommender/README.md +206 -0
- ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
- ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
- ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
- ads/opctl/operator/lowcode/recommender/constant.py +30 -0
- ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
- ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
- ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
- ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
- ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
- ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
- ads/opctl/operator/lowcode/recommender/utils.py +13 -0
- ads/opctl/operator/runtime/__init__.py +5 -0
- ads/opctl/operator/runtime/const.py +17 -0
- ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
- ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
- ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/runtime.py +115 -0
- ads/opctl/schema.yaml.yml +36 -0
- ads/opctl/script.py +40 -0
- ads/opctl/spark/__init__.py +5 -0
- ads/opctl/spark/cli.py +43 -0
- ads/opctl/spark/cmds.py +147 -0
- ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
- ads/opctl/utils.py +344 -0
- ads/oracledb/__init__.py +5 -0
- ads/oracledb/oracle_db.py +346 -0
- ads/pipeline/__init__.py +39 -0
- ads/pipeline/ads_pipeline.py +2279 -0
- ads/pipeline/ads_pipeline_run.py +772 -0
- ads/pipeline/ads_pipeline_step.py +605 -0
- ads/pipeline/builders/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/custom_script.py +32 -0
- ads/pipeline/cli.py +119 -0
- ads/pipeline/extension.py +291 -0
- ads/pipeline/schema/__init__.py +5 -0
- ads/pipeline/schema/cs_step_schema.json +35 -0
- ads/pipeline/schema/ml_step_schema.json +31 -0
- ads/pipeline/schema/pipeline_schema.json +71 -0
- ads/pipeline/visualizer/__init__.py +5 -0
- ads/pipeline/visualizer/base.py +570 -0
- ads/pipeline/visualizer/graph_renderer.py +272 -0
- ads/pipeline/visualizer/text_renderer.py +84 -0
- ads/secrets/__init__.py +11 -0
- ads/secrets/adb.py +386 -0
- ads/secrets/auth_token.py +86 -0
- ads/secrets/big_data_service.py +365 -0
- ads/secrets/mysqldb.py +149 -0
- ads/secrets/oracledb.py +160 -0
- ads/secrets/secrets.py +407 -0
- ads/telemetry/__init__.py +7 -0
- ads/telemetry/base.py +69 -0
- ads/telemetry/client.py +122 -0
- ads/telemetry/telemetry.py +257 -0
- ads/templates/dataflow_pyspark.jinja2 +13 -0
- ads/templates/dataflow_sparksql.jinja2 +22 -0
- ads/templates/func.jinja2 +20 -0
- ads/templates/schemas/openapi.json +1740 -0
- ads/templates/score-pkl.jinja2 +173 -0
- ads/templates/score.jinja2 +322 -0
- ads/templates/score_embedding_onnx.jinja2 +202 -0
- ads/templates/score_generic.jinja2 +165 -0
- ads/templates/score_huggingface_pipeline.jinja2 +217 -0
- ads/templates/score_lightgbm.jinja2 +185 -0
- ads/templates/score_onnx.jinja2 +407 -0
- ads/templates/score_onnx_new.jinja2 +473 -0
- ads/templates/score_oracle_automl.jinja2 +185 -0
- ads/templates/score_pyspark.jinja2 +154 -0
- ads/templates/score_pytorch.jinja2 +219 -0
- ads/templates/score_scikit-learn.jinja2 +184 -0
- ads/templates/score_tensorflow.jinja2 +184 -0
- ads/templates/score_xgboost.jinja2 +178 -0
- ads/text_dataset/__init__.py +5 -0
- ads/text_dataset/backends.py +211 -0
- ads/text_dataset/dataset.py +445 -0
- ads/text_dataset/extractor.py +207 -0
- ads/text_dataset/options.py +53 -0
- ads/text_dataset/udfs.py +22 -0
- ads/text_dataset/utils.py +49 -0
- ads/type_discovery/__init__.py +9 -0
- ads/type_discovery/abstract_detector.py +21 -0
- ads/type_discovery/constant_detector.py +41 -0
- ads/type_discovery/continuous_detector.py +54 -0
- ads/type_discovery/credit_card_detector.py +99 -0
- ads/type_discovery/datetime_detector.py +92 -0
- ads/type_discovery/discrete_detector.py +118 -0
- ads/type_discovery/document_detector.py +146 -0
- ads/type_discovery/ip_detector.py +68 -0
- ads/type_discovery/latlon_detector.py +90 -0
- ads/type_discovery/phone_number_detector.py +63 -0
- ads/type_discovery/type_discovery_driver.py +87 -0
- ads/type_discovery/typed_feature.py +594 -0
- ads/type_discovery/unknown_detector.py +41 -0
- ads/type_discovery/zipcode_detector.py +48 -0
- ads/vault/__init__.py +7 -0
- ads/vault/vault.py +237 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/METADATA +150 -149
- oracle_ads-2.13.10.dist-info/RECORD +858 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/WHEEL +1 -2
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/entry_points.txt +2 -1
- oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
- oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,772 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
|
3
|
+
# Copyright (c) 2022, 2025 Oracle and/or its affiliates.
|
4
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
5
|
+
import copy
|
6
|
+
import logging
|
7
|
+
import time
|
8
|
+
from typing import List, Optional
|
9
|
+
|
10
|
+
import oci
|
11
|
+
import yaml
|
12
|
+
|
13
|
+
from ads.common.decorator.runtime_dependency import (
|
14
|
+
OptionalDependency,
|
15
|
+
runtime_dependency,
|
16
|
+
)
|
17
|
+
from ads.common.extended_enum import ExtendedEnum
|
18
|
+
from ads.common.oci_datascience import OCIDataScienceMixin
|
19
|
+
from ads.common.oci_logging import ConsolidatedLog, OCILog
|
20
|
+
from ads.jobs.builders.infrastructure.base import RunInstance
|
21
|
+
from ads.pipeline.ads_pipeline_step import PipelineStep
|
22
|
+
from ads.pipeline.visualizer.base import (
|
23
|
+
GraphOrientation,
|
24
|
+
PipelineVisualizer,
|
25
|
+
StepStatus,
|
26
|
+
)
|
27
|
+
from ads.pipeline.visualizer.graph_renderer import PipelineGraphRenderer
|
28
|
+
from ads.pipeline.visualizer.text_renderer import PipelineTextRenderer
|
29
|
+
|
30
|
+
PIPELINE_RUN_TERMINAL_STATE = {
|
31
|
+
StepStatus.FAILED,
|
32
|
+
StepStatus.SUCCEEDED,
|
33
|
+
StepStatus.CANCELED,
|
34
|
+
StepStatus.DELETED,
|
35
|
+
StepStatus.SKIPPED,
|
36
|
+
}
|
37
|
+
|
38
|
+
LOG_INTERVAL = 3
|
39
|
+
SLEEP_INTERVAL = 3
|
40
|
+
MAXIMUM_TIMEOUT_SECONDS = 1800
|
41
|
+
LOG_RECORDS_LIMIT = 100
|
42
|
+
ALLOWED_OPERATION_KWARGS = [
|
43
|
+
"allow_control_chars",
|
44
|
+
"retry_strategy",
|
45
|
+
"delete_related_job_runs",
|
46
|
+
"if_match",
|
47
|
+
"opc_request_id",
|
48
|
+
]
|
49
|
+
ALLOWED_WAITER_KWARGS = [
|
50
|
+
"max_interval_seconds",
|
51
|
+
"max_wait_seconds",
|
52
|
+
"wait_callback",
|
53
|
+
"fetch_func",
|
54
|
+
]
|
55
|
+
|
56
|
+
logger = logging.getLogger(__name__)
|
57
|
+
|
58
|
+
|
59
|
+
class LogType(ExtendedEnum):
|
60
|
+
CUSTOM_LOG = "custom_log"
|
61
|
+
SERVICE_LOG = "service_log"
|
62
|
+
|
63
|
+
|
64
|
+
class ShowMode(ExtendedEnum):
|
65
|
+
GRAPH = "graph"
|
66
|
+
TEXT = "text"
|
67
|
+
|
68
|
+
|
69
|
+
class StepType(ExtendedEnum):
|
70
|
+
ML_JOB = "ML_JOB"
|
71
|
+
CUSTOM_SCRIPT = "CUSTOM_SCRIPT"
|
72
|
+
|
73
|
+
|
74
|
+
class LogNotConfiguredError(Exception): # pragma: no cover
|
75
|
+
pass
|
76
|
+
|
77
|
+
|
78
|
+
class PipelineRun(
|
79
|
+
OCIDataScienceMixin, oci.data_science.models.PipelineRun, RunInstance
|
80
|
+
):
|
81
|
+
"""
|
82
|
+
Attributes
|
83
|
+
----------
|
84
|
+
pipeline: Pipeline
|
85
|
+
Returns the ADS pipeline object for run instance.
|
86
|
+
status: str
|
87
|
+
Returns Lifecycle status.
|
88
|
+
custom_logging: OCILog
|
89
|
+
Returns the OCILog object containing the custom logs from the pipeline.
|
90
|
+
|
91
|
+
Methods
|
92
|
+
-------
|
93
|
+
create(self) -> PipelineRun
|
94
|
+
Creates an OCI pipeline run.
|
95
|
+
delete(self, delete_related_job_runs: Optional[bool] = True, max_wait_seconds: Optional[int] = MAXIMUM_TIMEOUT, **kwargs) -> PipelineRun
|
96
|
+
Deletes an OCI pipeline run.
|
97
|
+
cancel(self, maximum_timeout: int = MAXIMUM_TIMEOUT) -> PipelineRun
|
98
|
+
Cancels an OCI pipeline run.
|
99
|
+
watch(self, steps: List[str] = None, interval: float = LOG_INTERVAL, log_type: str = LogType.CUSTOM_LOG, *args) -> PipelineRun
|
100
|
+
Watches the pipeline run until it finishes.
|
101
|
+
list(cls, pipeline_id: str, compartment_id: Optional[str] = None, **kwargs) -> List[PipelineRun]:
|
102
|
+
Lists pipeline runs for a given pipeline.
|
103
|
+
to_yaml(self) -> str
|
104
|
+
Serializes the object into YAML string.
|
105
|
+
show(self, mode: str = ShowMode.GRAPH, wait: bool = False, rankdir: str = GraphOrientation.TOP_BOTTOM) -> None
|
106
|
+
Renders pipeline run. Can be `text` or `graph` representation.
|
107
|
+
to_svg(self, uri: str = None, rankdir: str = GraphOrientation.TOP_BOTTOM, **kwargs)
|
108
|
+
Renders pipeline run graph to SVG.
|
109
|
+
sync(self) -> None
|
110
|
+
Syncs status of Pipeline run.
|
111
|
+
"""
|
112
|
+
|
113
|
+
_DETAILS_LINK = (
|
114
|
+
"https://console.{region}.oraclecloud.com/data-science/pipeline-runs/{id}"
|
115
|
+
)
|
116
|
+
|
117
|
+
def __init__(
|
118
|
+
self,
|
119
|
+
config: dict = None,
|
120
|
+
signer: oci.signer.Signer = None,
|
121
|
+
client_kwargs: dict = None,
|
122
|
+
**kwargs,
|
123
|
+
) -> None:
|
124
|
+
super().__init__(config, signer, client_kwargs, **kwargs)
|
125
|
+
self._service_logging = None
|
126
|
+
self._custom_logging = None
|
127
|
+
self._pipeline = None
|
128
|
+
|
129
|
+
self._graphViz = PipelineVisualizer().with_renderer(
|
130
|
+
PipelineGraphRenderer(show_status=True)
|
131
|
+
)
|
132
|
+
self._textViz = PipelineVisualizer().with_renderer(PipelineTextRenderer())
|
133
|
+
|
134
|
+
def sync(self, **kwargs) -> None:
|
135
|
+
"""Syncs status of the Pipeline Run.
|
136
|
+
|
137
|
+
Returns
|
138
|
+
-------
|
139
|
+
None
|
140
|
+
"""
|
141
|
+
super().sync(**kwargs)
|
142
|
+
self._graphViz.with_pipeline(self.pipeline).with_pipeline_run(self)
|
143
|
+
self._textViz.with_pipeline(self.pipeline).with_pipeline_run(self)
|
144
|
+
return self
|
145
|
+
|
146
|
+
def show(
|
147
|
+
self,
|
148
|
+
mode: str = ShowMode.GRAPH,
|
149
|
+
wait: bool = False,
|
150
|
+
rankdir: str = GraphOrientation.TOP_BOTTOM,
|
151
|
+
) -> None:
|
152
|
+
"""
|
153
|
+
Renders pipeline run. Can be `text` or `graph` representation.
|
154
|
+
|
155
|
+
Parameters
|
156
|
+
----------
|
157
|
+
mode: (str, optional). Defaults to `graph`.
|
158
|
+
Pipeline run display mode. Allowed values: `graph` or `text`.
|
159
|
+
wait: (bool, optional). Default to `False`.
|
160
|
+
Whether to wait until the completion of the pipeline run.
|
161
|
+
rankdir: (str, optional). Default to `TB`.
|
162
|
+
Direction of the rendered graph. Allowed Values: `TB` or `LR`.
|
163
|
+
Applicable only for graph mode.
|
164
|
+
|
165
|
+
Returns
|
166
|
+
-------
|
167
|
+
None
|
168
|
+
"""
|
169
|
+
self.sync()
|
170
|
+
renderer = self._graphViz if mode.lower() == ShowMode.GRAPH else self._textViz
|
171
|
+
if not wait:
|
172
|
+
renderer.render(rankdir=rankdir)
|
173
|
+
return
|
174
|
+
self._show(renderer, rankdir=rankdir)
|
175
|
+
|
176
|
+
def to_svg(
|
177
|
+
self, uri: str = None, rankdir: str = GraphOrientation.TOP_BOTTOM, **kwargs
|
178
|
+
) -> str:
|
179
|
+
"""
|
180
|
+
Renders pipeline run graph to SVG.
|
181
|
+
|
182
|
+
Parameters
|
183
|
+
----------
|
184
|
+
uri: (string, optional). Defaults to `None`.
|
185
|
+
URI location to save the SVG string.
|
186
|
+
rankdir: (str, optional). Default to `TB`.
|
187
|
+
Direction of the rendered graph. Allowed Values: `TB` or `LR`.
|
188
|
+
Applicable only for graph mode.
|
189
|
+
|
190
|
+
Returns
|
191
|
+
-------
|
192
|
+
str
|
193
|
+
Pipeline run graph in svg format.
|
194
|
+
"""
|
195
|
+
self.sync()
|
196
|
+
return self._graphViz.to_svg(uri=uri, rankdir=rankdir, **kwargs)
|
197
|
+
|
198
|
+
@runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
|
199
|
+
def _show(
|
200
|
+
self,
|
201
|
+
viz,
|
202
|
+
rankdir: str = GraphOrientation.TOP_BOTTOM,
|
203
|
+
refresh_interval=SLEEP_INTERVAL,
|
204
|
+
):
|
205
|
+
"""
|
206
|
+
Renders pipeline run in text or graph until the completion of the pipeline.
|
207
|
+
|
208
|
+
Parameters
|
209
|
+
----------
|
210
|
+
viz: PipelineRenderer
|
211
|
+
The `PipelineTextRenderer` or `PipelineGraphRenderer` object.
|
212
|
+
rankdir: (str, optional). Default to `TB`.
|
213
|
+
Direction of the rendered graph. Allowed Values: `TB` or `LR`.
|
214
|
+
Applicable only for graph mode.
|
215
|
+
refresh_interval: (int, optional). Defaults to 5.
|
216
|
+
Time interval in seconds to refresh pipeline run status.
|
217
|
+
|
218
|
+
Returns
|
219
|
+
-------
|
220
|
+
None
|
221
|
+
"""
|
222
|
+
from IPython.display import clear_output
|
223
|
+
|
224
|
+
try:
|
225
|
+
while self.status not in PIPELINE_RUN_TERMINAL_STATE:
|
226
|
+
clear_output(wait=True)
|
227
|
+
viz.render(rankdir=rankdir)
|
228
|
+
time.sleep(refresh_interval)
|
229
|
+
|
230
|
+
clear_output(wait=True)
|
231
|
+
viz.render(rankdir=rankdir)
|
232
|
+
except KeyboardInterrupt:
|
233
|
+
pass
|
234
|
+
|
235
|
+
def logs(self, log_type: str = None) -> ConsolidatedLog:
|
236
|
+
"""Builds the consolidated log for pipeline run.
|
237
|
+
|
238
|
+
Parameters
|
239
|
+
----------
|
240
|
+
log_type: str
|
241
|
+
The log type of the pipeline run. Defaults to None.
|
242
|
+
Can be custom_log, service_log or None.
|
243
|
+
|
244
|
+
Returns
|
245
|
+
-------
|
246
|
+
ConsolidatedLog
|
247
|
+
The ConsolidatedLog instance.
|
248
|
+
"""
|
249
|
+
logging_list = []
|
250
|
+
if not log_type:
|
251
|
+
try:
|
252
|
+
logging_list.append(self.custom_logging)
|
253
|
+
except LogNotConfiguredError:
|
254
|
+
pass
|
255
|
+
|
256
|
+
try:
|
257
|
+
logging_list.append(self.service_logging)
|
258
|
+
except LogNotConfiguredError:
|
259
|
+
pass
|
260
|
+
|
261
|
+
if not logging_list:
|
262
|
+
raise LogNotConfiguredError(
|
263
|
+
"Neither `custom` nor `service` log was configured for the pipeline run."
|
264
|
+
)
|
265
|
+
elif log_type == LogType.SERVICE_LOG:
|
266
|
+
logging_list = [self.service_logging]
|
267
|
+
elif log_type == LogType.CUSTOM_LOG:
|
268
|
+
logging_list = [self.custom_logging]
|
269
|
+
else:
|
270
|
+
raise ValueError(
|
271
|
+
"Parameter log_type should be either custom_log, service_log or None."
|
272
|
+
)
|
273
|
+
|
274
|
+
return ConsolidatedLog(*logging_list)
|
275
|
+
|
276
|
+
@property
|
277
|
+
def pipeline(self):
|
278
|
+
"""Returns the ADS Pipeline instance.
|
279
|
+
Step details will be synched with the Pipeline Run.
|
280
|
+
|
281
|
+
Parameters
|
282
|
+
----------
|
283
|
+
None
|
284
|
+
|
285
|
+
Returns
|
286
|
+
-------
|
287
|
+
Pipeline
|
288
|
+
The ADS Pipeline instance, where Step details will be synched with the Pipeline Run.
|
289
|
+
"""
|
290
|
+
from ads.pipeline.ads_pipeline import Pipeline
|
291
|
+
|
292
|
+
if not self._pipeline:
|
293
|
+
self._pipeline = Pipeline.from_ocid(self.pipeline_id)
|
294
|
+
self._sync_step_details()
|
295
|
+
return self._pipeline
|
296
|
+
|
297
|
+
@property
|
298
|
+
def status(self) -> str:
|
299
|
+
"""Lifecycle status.
|
300
|
+
|
301
|
+
Returns
|
302
|
+
-------
|
303
|
+
str
|
304
|
+
Status in a string.
|
305
|
+
"""
|
306
|
+
self.sync()
|
307
|
+
return self.lifecycle_state
|
308
|
+
|
309
|
+
@property
|
310
|
+
def custom_logging(self) -> OCILog:
|
311
|
+
"""The OCILog object containing the custom logs from the pipeline run."""
|
312
|
+
if not self._custom_logging:
|
313
|
+
self._check_log_details()
|
314
|
+
|
315
|
+
while not self._stop_condition():
|
316
|
+
# Break if pipeline run has log ID.
|
317
|
+
if self.log_details.log_id:
|
318
|
+
break
|
319
|
+
time.sleep(LOG_INTERVAL)
|
320
|
+
|
321
|
+
self._custom_logging = OCILog(
|
322
|
+
id=self.log_details.log_id,
|
323
|
+
log_group_id=self.log_details.log_group_id,
|
324
|
+
compartment_id=self.compartment_id,
|
325
|
+
annotation="custom",
|
326
|
+
)
|
327
|
+
return self._custom_logging
|
328
|
+
|
329
|
+
@property
|
330
|
+
def service_logging(self) -> OCILog:
|
331
|
+
"""The OCILog object containing the service logs from the pipeline run."""
|
332
|
+
if not self._service_logging:
|
333
|
+
self._check_log_details()
|
334
|
+
self._service_logging = self._get_service_logging()
|
335
|
+
return self._service_logging
|
336
|
+
|
337
|
+
def _check_log_details(self):
|
338
|
+
if not self.log_details:
|
339
|
+
raise LogNotConfiguredError(
|
340
|
+
"Pipeline log is not configured. Make sure log group id is added."
|
341
|
+
)
|
342
|
+
if not self.log_details.log_group_id:
|
343
|
+
raise LogNotConfiguredError(
|
344
|
+
"Log group OCID is not specified for this pipeline. Call with_log_group_id to add it."
|
345
|
+
)
|
346
|
+
|
347
|
+
def _get_service_logging(self) -> OCILog:
|
348
|
+
"""Builds the OCI service log instance for pipeline run.
|
349
|
+
|
350
|
+
Returns
|
351
|
+
-------
|
352
|
+
OCILog
|
353
|
+
The OCILog instance.
|
354
|
+
"""
|
355
|
+
log_summary = self._search_service_logs()
|
356
|
+
|
357
|
+
if not log_summary:
|
358
|
+
raise LogNotConfiguredError("Service log is not configured for pipeline.")
|
359
|
+
|
360
|
+
# each pipeline can only have one service log
|
361
|
+
service_log_id = log_summary[0].id
|
362
|
+
return OCILog(
|
363
|
+
id=service_log_id,
|
364
|
+
log_group_id=self.log_details.log_group_id,
|
365
|
+
compartment_id=self.compartment_id,
|
366
|
+
annotation="service",
|
367
|
+
)
|
368
|
+
|
369
|
+
def _search_service_logs(self) -> List[oci.logging.models.log_summary.LogSummary]:
|
370
|
+
"""Search the service log of pipeline run based on
|
371
|
+
log_group_id, source_service, source_resource and log_type.
|
372
|
+
|
373
|
+
Returns
|
374
|
+
-------
|
375
|
+
list
|
376
|
+
A list of oci.logging.models.log_summary.LogSummary.
|
377
|
+
"""
|
378
|
+
return (
|
379
|
+
OCILog(compartment_id=self.compartment_id)
|
380
|
+
.client.list_logs(
|
381
|
+
log_group_id=self.log_details.log_group_id,
|
382
|
+
source_service=self.pipeline.CONST_SERVICE,
|
383
|
+
source_resource=self.pipeline.id,
|
384
|
+
log_type="SERVICE",
|
385
|
+
)
|
386
|
+
.data
|
387
|
+
)
|
388
|
+
|
389
|
+
def _sync_step_details(self) -> None:
|
390
|
+
"""Combines pipeline step details with override step details.
|
391
|
+
|
392
|
+
Returns
|
393
|
+
-------
|
394
|
+
None
|
395
|
+
"""
|
396
|
+
if not self._pipeline or not self._pipeline.step_details:
|
397
|
+
return None
|
398
|
+
|
399
|
+
updated_step_details = []
|
400
|
+
for step in self._pipeline.step_details:
|
401
|
+
updated_step_detail = copy.deepcopy(step.to_dict())
|
402
|
+
# restore dependencies information
|
403
|
+
updated_step_detail["spec"]["dependsOn"] = step.depends_on
|
404
|
+
# override step details if necessary
|
405
|
+
if self.step_override_details:
|
406
|
+
for override_step in self.step_override_details:
|
407
|
+
if type(override_step) == dict:
|
408
|
+
break
|
409
|
+
if step.name == override_step.step_name:
|
410
|
+
if override_step.step_configuration_details:
|
411
|
+
if (
|
412
|
+
"stepConfigurationDetails"
|
413
|
+
not in updated_step_detail["spec"]
|
414
|
+
):
|
415
|
+
updated_step_detail["spec"][
|
416
|
+
"stepConfigurationDetails"
|
417
|
+
] = {}
|
418
|
+
|
419
|
+
if override_step.step_configuration_details.maximum_runtime_in_minutes:
|
420
|
+
updated_step_detail["spec"]["stepConfigurationDetails"][
|
421
|
+
"maximumRuntimeInMinutes"
|
422
|
+
] = override_step.step_configuration_details.maximum_runtime_in_minutes
|
423
|
+
if override_step.step_configuration_details.environment_variables:
|
424
|
+
updated_step_detail["spec"]["stepConfigurationDetails"][
|
425
|
+
"environmentVariables"
|
426
|
+
] = override_step.step_configuration_details.environment_variables
|
427
|
+
if override_step.step_configuration_details.command_line_arguments:
|
428
|
+
updated_step_detail["spec"]["stepConfigurationDetails"][
|
429
|
+
"commandLineArguments"
|
430
|
+
] = override_step.step_configuration_details.command_line_arguments
|
431
|
+
|
432
|
+
updated_step_details.append(PipelineStep.from_dict(updated_step_detail))
|
433
|
+
self._pipeline.with_step_details(updated_step_details)
|
434
|
+
|
435
|
+
def _set_service_logging_resource(self, service_logging: OCILog):
|
436
|
+
"""Sets the service logging resource for pipeline run.
|
437
|
+
|
438
|
+
Parameters
|
439
|
+
----------
|
440
|
+
service_logging: OCILog instance.
|
441
|
+
The OCILog instance.
|
442
|
+
"""
|
443
|
+
self._service_logging = service_logging
|
444
|
+
|
445
|
+
def create(self) -> "PipelineRun":
|
446
|
+
"""Creates an OCI pipeline run.
|
447
|
+
|
448
|
+
Returns
|
449
|
+
-------
|
450
|
+
PipelineRun:
|
451
|
+
Pipeline run instance (self).
|
452
|
+
"""
|
453
|
+
self.load_properties_from_env()
|
454
|
+
response = self.client.create_pipeline_run(
|
455
|
+
self.to_oci_model(oci.data_science.models.CreatePipelineRunDetails)
|
456
|
+
)
|
457
|
+
self.update_from_oci_model(response.data)
|
458
|
+
return self
|
459
|
+
|
460
|
+
def delete(
|
461
|
+
self,
|
462
|
+
delete_related_job_runs: Optional[bool] = True,
|
463
|
+
max_wait_seconds: Optional[int] = MAXIMUM_TIMEOUT_SECONDS,
|
464
|
+
**kwargs,
|
465
|
+
) -> "PipelineRun":
|
466
|
+
"""Deletes an OCI pipeline run.
|
467
|
+
|
468
|
+
Parameters
|
469
|
+
----------
|
470
|
+
delete_related_job_runs: bool, optional
|
471
|
+
Specify whether to delete related JobRuns or not. Defaults to True.
|
472
|
+
max_wait_seconds: int, optional
|
473
|
+
The maximum time to wait, in seconds. Defaults to 1800.
|
474
|
+
|
475
|
+
kwargs: optional
|
476
|
+
The kwargs to be executed when deleting the pipeline.
|
477
|
+
The allowed keys are:
|
478
|
+
* "allow_control_chars": bool, to indicate whether or not this request should
|
479
|
+
allow control characters in the response object. By default, the response will
|
480
|
+
not allow control characters in strings.
|
481
|
+
* "retry_strategy": obj, to apply to this specific operation/call. This will
|
482
|
+
override any retry strategy set at the client-level. This should be one of the
|
483
|
+
strategies available in the :py:mod:`~oci.retry` module. This operation will not
|
484
|
+
retry by default, users can also use the convenient :py:data:`~oci.retry.DEFAULT_RETRY_STRATEGY`
|
485
|
+
provided by the SDK to enable retries for it. The specifics of the default retry
|
486
|
+
strategy are described `here <https://docs.oracle.com/en-us/iaas/tools/python/latest/sdk_behaviors/retries.html>`__.
|
487
|
+
To have this operation explicitly not perform any retries, pass an instance of :py:class:`~oci.retry.NoneRetryStrategy`.
|
488
|
+
* "if_match": str, for optimistic concurrency control. In the PUT or DELETE call
|
489
|
+
for a resource, set the `if-match` parameter to the value of the etag from a
|
490
|
+
previous GET or POST response for that resource. The resource is updated or
|
491
|
+
deleted only if the `etag` you provide matches the resource's current `etag` value.
|
492
|
+
* "opc_request_id": str, unique Oracle assigned identifier for the request.
|
493
|
+
If you need to contact Oracle about a particular request, then provide the request ID.
|
494
|
+
* "max_interval_seconds": int, the maximum interval between queries, in seconds.
|
495
|
+
* "wait_callback": A function which will be called each time that we have to do an initial
|
496
|
+
wait (i.e. because the property of the resource was not in the correct state,
|
497
|
+
or the ``evaluate_response`` function returned False). This function should take two
|
498
|
+
arguments - the first argument is the number of times we have checked the resource,
|
499
|
+
and the second argument is the result of the most recent check.
|
500
|
+
* "fetch_func": A function to be called to fetch the updated state from the server.
|
501
|
+
This can be used if the call to check for state needs to be more complex than a single
|
502
|
+
GET request. For example, if the goal is to wait until an item appears in a list,
|
503
|
+
fetch_func can be a function that paginates through a full list on the server.
|
504
|
+
|
505
|
+
Returns
|
506
|
+
-------
|
507
|
+
PipelineRun:
|
508
|
+
Pipeline run instance (self).
|
509
|
+
"""
|
510
|
+
operation_kwargs = {"delete_related_job_runs": delete_related_job_runs}
|
511
|
+
waiter_kwargs = {"max_wait_seconds": max_wait_seconds}
|
512
|
+
for key, value in kwargs.items():
|
513
|
+
if key in ALLOWED_OPERATION_KWARGS:
|
514
|
+
operation_kwargs[key] = value
|
515
|
+
elif key in ALLOWED_WAITER_KWARGS:
|
516
|
+
waiter_kwargs[key] = value
|
517
|
+
|
518
|
+
self.client_composite.delete_pipeline_run_and_wait_for_state(
|
519
|
+
pipeline_run_id=self.id,
|
520
|
+
wait_for_states=[PipelineRun.LIFECYCLE_STATE_DELETED],
|
521
|
+
operation_kwargs=operation_kwargs,
|
522
|
+
waiter_kwargs=waiter_kwargs,
|
523
|
+
)
|
524
|
+
return self.sync()
|
525
|
+
|
526
|
+
def cancel(self, maximum_timeout: int = MAXIMUM_TIMEOUT_SECONDS) -> "PipelineRun":
|
527
|
+
"""Cancels an OCI pipeline run.
|
528
|
+
|
529
|
+
Parameters
|
530
|
+
----------
|
531
|
+
maximum_timeout: int, optional
|
532
|
+
The maximum timeout to cancel the pipeline run. Defaults to 1800 seconds.
|
533
|
+
|
534
|
+
Returns
|
535
|
+
-------
|
536
|
+
PipelineRun:
|
537
|
+
Pipeline run instance (self).
|
538
|
+
"""
|
539
|
+
self.client.cancel_pipeline_run(self.id)
|
540
|
+
|
541
|
+
time_counter = 0
|
542
|
+
while self.sync().lifecycle_state not in [
|
543
|
+
PipelineRun.LIFECYCLE_STATE_CANCELED,
|
544
|
+
PipelineRun.LIFECYCLE_STATE_FAILED,
|
545
|
+
]:
|
546
|
+
time.sleep(SLEEP_INTERVAL)
|
547
|
+
if time_counter > maximum_timeout:
|
548
|
+
print(
|
549
|
+
"Pipeline run stopping after ",
|
550
|
+
maximum_timeout,
|
551
|
+
" seconds of not reaching CANCELLED state.",
|
552
|
+
)
|
553
|
+
break
|
554
|
+
time_counter += SLEEP_INTERVAL
|
555
|
+
|
556
|
+
if self.sync().lifecycle_state != PipelineRun.LIFECYCLE_STATE_CANCELED:
|
557
|
+
raise Exception("Error occurred in attempt to cancel the pipeline run.")
|
558
|
+
return self
|
559
|
+
|
560
|
+
def watch(
|
561
|
+
self,
|
562
|
+
steps: List[str] = None,
|
563
|
+
interval: float = LOG_INTERVAL,
|
564
|
+
log_type: str = None,
|
565
|
+
*args,
|
566
|
+
) -> "PipelineRun":
|
567
|
+
"""Watches the pipeline run until it finishes.
|
568
|
+
This method will keep streamming the service log of the pipeline run until it's succeeded, failed or cancelled.
|
569
|
+
|
570
|
+
Parameters
|
571
|
+
----------
|
572
|
+
steps: (List[str], optional). Defaults to None.
|
573
|
+
Pipeline steps passed in to filter the logs.
|
574
|
+
interval: (float, optional). Defaults to 3 seconds.
|
575
|
+
Time interval in seconds between each request to update the logs.
|
576
|
+
log_type: (str, optional). Defaults to None.
|
577
|
+
The log type. Can be `custom_log`, `service_log` or None.
|
578
|
+
*args:
|
579
|
+
Pipeline steps passed in to filter the logs.
|
580
|
+
Example: `.watch("step1", "step2")`
|
581
|
+
|
582
|
+
Examples
|
583
|
+
--------
|
584
|
+
>>> .watch()
|
585
|
+
>>> .watch(log_type="service_log")
|
586
|
+
>>> .watch("step1", "step2", log_type="custom_log", interval=3)
|
587
|
+
>>> .watch(steps=["step1", "step2"], log_type="custom_log", interval=3)
|
588
|
+
|
589
|
+
Returns
|
590
|
+
-------
|
591
|
+
PipelineRun:
|
592
|
+
Pipeline run instance (self).
|
593
|
+
"""
|
594
|
+
logging = self.logs(log_type=log_type)
|
595
|
+
|
596
|
+
steps_to_monitor = list(set(steps or ()) | set(args))
|
597
|
+
|
598
|
+
try:
|
599
|
+
return self.__stream_log(
|
600
|
+
logging,
|
601
|
+
steps_to_monitor,
|
602
|
+
interval,
|
603
|
+
log_type,
|
604
|
+
)
|
605
|
+
except KeyboardInterrupt:
|
606
|
+
print("Stop watching logs.")
|
607
|
+
pass
|
608
|
+
|
609
|
+
def __stream_log(
|
610
|
+
self,
|
611
|
+
logging: ConsolidatedLog,
|
612
|
+
pipeline_steps: List = None,
|
613
|
+
interval: float = LOG_INTERVAL,
|
614
|
+
log_type: str = None,
|
615
|
+
) -> "PipelineRun":
|
616
|
+
"""Stream logs from OCI pipeline backends.
|
617
|
+
|
618
|
+
Parameters
|
619
|
+
----------
|
620
|
+
logging : ConsolidatedLog.
|
621
|
+
The ConsolidatedLog instance.
|
622
|
+
pipeline_steps: list
|
623
|
+
A list of pipeline step name.
|
624
|
+
interval : float
|
625
|
+
Time interval in seconds between each request to update the logs.
|
626
|
+
log_type : str
|
627
|
+
The log type.
|
628
|
+
|
629
|
+
Returns
|
630
|
+
-------
|
631
|
+
PipelineRun:
|
632
|
+
Pipeline run instance (self).
|
633
|
+
"""
|
634
|
+
print(f"Pipeline OCID: {self.pipeline_id}")
|
635
|
+
print(f"Pipeline Run OCID: {self.id}")
|
636
|
+
|
637
|
+
if self.time_accepted:
|
638
|
+
count = logging.stream(
|
639
|
+
interval=interval,
|
640
|
+
stop_condition=self._stop_condition,
|
641
|
+
time_start=self.time_accepted,
|
642
|
+
log_filter=self._build_filter_expression(pipeline_steps, log_type),
|
643
|
+
)
|
644
|
+
if not count:
|
645
|
+
print(
|
646
|
+
"No logs in the last 14 days. Please set time_start to see older logs."
|
647
|
+
)
|
648
|
+
|
649
|
+
return self.sync()
|
650
|
+
|
651
|
+
def _build_filter_expression(self, steps: List = [], log_type: str = None) -> str:
|
652
|
+
"""Builds query expression for logs that are generated by pipeline run and job run.
|
653
|
+
The query expression consists of two parts:
|
654
|
+
1. Logs that are generated by pipeline run:
|
655
|
+
- service and custom logs for CUSTOM_SCRIPT step
|
656
|
+
- service log for ML_JOB step
|
657
|
+
Format: (source = *<pipeline_run_id> AND ( subject = <pipeline_step_name> OR subject = <pipeline_step_name> OR ...))
|
658
|
+
2. Logs that are generated by job run:
|
659
|
+
- custom log for ML_JOB step
|
660
|
+
Format: source = *<job_run_id> OR source = *<job_run_id> OR source = *<job_run_id> OR ...
|
661
|
+
|
662
|
+
TODO:
|
663
|
+
This is a temporary solution, and the real fix will be done after the jobs service add pipleine run details in the log data panel.
|
664
|
+
|
665
|
+
Parameters
|
666
|
+
----------
|
667
|
+
steps: list
|
668
|
+
A list of pipeline step name.
|
669
|
+
log_type : str
|
670
|
+
The log type.
|
671
|
+
|
672
|
+
Returns
|
673
|
+
-------
|
674
|
+
str:
|
675
|
+
Query string to search the logs of pipeline.
|
676
|
+
"""
|
677
|
+
sources = []
|
678
|
+
subjects = []
|
679
|
+
skipped_step_list = []
|
680
|
+
|
681
|
+
is_service_logging_enabled = False
|
682
|
+
try:
|
683
|
+
if self.service_logging:
|
684
|
+
is_service_logging_enabled = True
|
685
|
+
except LogNotConfiguredError:
|
686
|
+
logger.warning(
|
687
|
+
"Service log is not configured for pipeline. Streaming custom log."
|
688
|
+
)
|
689
|
+
|
690
|
+
for step_run in self.step_runs:
|
691
|
+
if not steps or (step_run.step_name in steps):
|
692
|
+
step_name = step_run.step_name
|
693
|
+
if step_run.step_type == StepType.ML_JOB:
|
694
|
+
if not step_run.job_run_id:
|
695
|
+
skipped_step_list.append(step_run.step_name)
|
696
|
+
continue
|
697
|
+
job_run_id = step_run.job_run_id
|
698
|
+
if log_type == LogType.CUSTOM_LOG:
|
699
|
+
sources.append(f"source = '*{job_run_id}'")
|
700
|
+
elif log_type == LogType.SERVICE_LOG:
|
701
|
+
subjects.append(f"subject = '{step_name}'")
|
702
|
+
else:
|
703
|
+
sources.append(f"source = '*{job_run_id}'")
|
704
|
+
if is_service_logging_enabled:
|
705
|
+
subjects.append(f"subject = '{step_name}'")
|
706
|
+
else:
|
707
|
+
subjects.append(f"subject = '{step_name}'")
|
708
|
+
|
709
|
+
if skipped_step_list:
|
710
|
+
logger.warning(
|
711
|
+
f"ML Jobs: {', '.join(skipped_step_list)} log can't be printed since their job run ids are not known at this time. Please stop and rerun the watch() command again to retrieve the job run ids and print the logs."
|
712
|
+
)
|
713
|
+
|
714
|
+
filter_list = []
|
715
|
+
|
716
|
+
# add query for logs that are generated by pipeline run
|
717
|
+
if subjects:
|
718
|
+
pipeline_log_filters = [f"source = '*{self.id}'"]
|
719
|
+
pipeline_log_filters.append("(" + " OR ".join(subjects) + ")")
|
720
|
+
filter_list = ["(" + " AND ".join(pipeline_log_filters) + ")"]
|
721
|
+
|
722
|
+
# add query for logs that are generated by job run
|
723
|
+
if sources:
|
724
|
+
filter_list.extend(sources)
|
725
|
+
|
726
|
+
return " OR ".join(filter_list)
|
727
|
+
|
728
|
+
def _stop_condition(self):
|
729
|
+
"""Stops the sync once the job is in a terminal state."""
|
730
|
+
self.sync()
|
731
|
+
return self.lifecycle_state in PIPELINE_RUN_TERMINAL_STATE
|
732
|
+
|
733
|
+
@classmethod
|
734
|
+
def list(
|
735
|
+
cls, pipeline_id: str, compartment_id: Optional[str] = None, **kwargs
|
736
|
+
) -> List["PipelineRun"]:
|
737
|
+
"""
|
738
|
+
List pipeline runs for a given pipeline.
|
739
|
+
|
740
|
+
Parameters
|
741
|
+
----------
|
742
|
+
pipeline_id: str.
|
743
|
+
The OCID of pipeline.
|
744
|
+
compartment_id: (str, optional). Defaults to None.
|
745
|
+
The OCID of compartment.
|
746
|
+
If `None`, the value will be taken from the environment variables.
|
747
|
+
kwargs
|
748
|
+
Additional keyword arguments for filtering pipelines.
|
749
|
+
- lifecycle_state: str. Allowed values: "CREATING", "ACTIVE", "DELETING", "FAILED", "DELETED"
|
750
|
+
- created_by: str
|
751
|
+
- limit: int
|
752
|
+
|
753
|
+
Returns
|
754
|
+
-------
|
755
|
+
List[PipelineRun]
|
756
|
+
The list of pipeline runs.
|
757
|
+
"""
|
758
|
+
PipelineRun.list_resource(compartment_id, pipeline_id=pipeline_id, **kwargs)
|
759
|
+
|
760
|
+
def __repr__(self) -> str:
|
761
|
+
"""Displays the object as YAML."""
|
762
|
+
return self.to_yaml()
|
763
|
+
|
764
|
+
def to_yaml(self) -> str:
|
765
|
+
"""Serializes the object into YAML string.
|
766
|
+
|
767
|
+
Returns
|
768
|
+
-------
|
769
|
+
str
|
770
|
+
YAML stored in a string.
|
771
|
+
"""
|
772
|
+
return yaml.safe_dump(self.to_dict())
|