oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/__init__.py +40 -0
- ads/aqua/app.py +507 -0
- ads/aqua/cli.py +96 -0
- ads/aqua/client/__init__.py +3 -0
- ads/aqua/client/client.py +836 -0
- ads/aqua/client/openai_client.py +305 -0
- ads/aqua/common/__init__.py +5 -0
- ads/aqua/common/decorator.py +125 -0
- ads/aqua/common/entities.py +274 -0
- ads/aqua/common/enums.py +134 -0
- ads/aqua/common/errors.py +109 -0
- ads/aqua/common/utils.py +1295 -0
- ads/aqua/config/__init__.py +4 -0
- ads/aqua/config/container_config.py +246 -0
- ads/aqua/config/evaluation/__init__.py +4 -0
- ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
- ads/aqua/config/utils/__init__.py +4 -0
- ads/aqua/config/utils/serializer.py +339 -0
- ads/aqua/constants.py +116 -0
- ads/aqua/data.py +14 -0
- ads/aqua/dummy_data/icon.txt +1 -0
- ads/aqua/dummy_data/oci_model_deployments.json +56 -0
- ads/aqua/dummy_data/oci_models.json +1 -0
- ads/aqua/dummy_data/readme.md +26 -0
- ads/aqua/evaluation/__init__.py +8 -0
- ads/aqua/evaluation/constants.py +53 -0
- ads/aqua/evaluation/entities.py +186 -0
- ads/aqua/evaluation/errors.py +70 -0
- ads/aqua/evaluation/evaluation.py +1814 -0
- ads/aqua/extension/__init__.py +42 -0
- ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
- ads/aqua/extension/base_handler.py +90 -0
- ads/aqua/extension/common_handler.py +121 -0
- ads/aqua/extension/common_ws_msg_handler.py +36 -0
- ads/aqua/extension/deployment_handler.py +381 -0
- ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
- ads/aqua/extension/errors.py +30 -0
- ads/aqua/extension/evaluation_handler.py +129 -0
- ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
- ads/aqua/extension/finetune_handler.py +96 -0
- ads/aqua/extension/model_handler.py +390 -0
- ads/aqua/extension/models/__init__.py +0 -0
- ads/aqua/extension/models/ws_models.py +145 -0
- ads/aqua/extension/models_ws_msg_handler.py +50 -0
- ads/aqua/extension/ui_handler.py +300 -0
- ads/aqua/extension/ui_websocket_handler.py +130 -0
- ads/aqua/extension/utils.py +133 -0
- ads/aqua/finetuning/__init__.py +7 -0
- ads/aqua/finetuning/constants.py +23 -0
- ads/aqua/finetuning/entities.py +181 -0
- ads/aqua/finetuning/finetuning.py +749 -0
- ads/aqua/model/__init__.py +8 -0
- ads/aqua/model/constants.py +60 -0
- ads/aqua/model/entities.py +385 -0
- ads/aqua/model/enums.py +32 -0
- ads/aqua/model/model.py +2134 -0
- ads/aqua/model/utils.py +52 -0
- ads/aqua/modeldeployment/__init__.py +6 -0
- ads/aqua/modeldeployment/constants.py +10 -0
- ads/aqua/modeldeployment/deployment.py +1315 -0
- ads/aqua/modeldeployment/entities.py +653 -0
- ads/aqua/modeldeployment/utils.py +543 -0
- ads/aqua/resources/gpu_shapes_index.json +94 -0
- ads/aqua/server/__init__.py +4 -0
- ads/aqua/server/__main__.py +24 -0
- ads/aqua/server/app.py +47 -0
- ads/aqua/server/aqua_spec.yml +1291 -0
- ads/aqua/training/__init__.py +4 -0
- ads/aqua/training/exceptions.py +476 -0
- ads/aqua/ui.py +519 -0
- ads/automl/__init__.py +9 -0
- ads/automl/driver.py +330 -0
- ads/automl/provider.py +975 -0
- ads/bds/__init__.py +5 -0
- ads/bds/auth.py +127 -0
- ads/bds/big_data_service.py +255 -0
- ads/catalog/__init__.py +19 -0
- ads/catalog/model.py +1576 -0
- ads/catalog/notebook.py +461 -0
- ads/catalog/project.py +468 -0
- ads/catalog/summary.py +178 -0
- ads/common/__init__.py +11 -0
- ads/common/analyzer.py +65 -0
- ads/common/artifact/.model-ignore +63 -0
- ads/common/artifact/__init__.py +10 -0
- ads/common/auth.py +1122 -0
- ads/common/card_identifier.py +83 -0
- ads/common/config.py +647 -0
- ads/common/data.py +165 -0
- ads/common/decorator/__init__.py +9 -0
- ads/common/decorator/argument_to_case.py +88 -0
- ads/common/decorator/deprecate.py +69 -0
- ads/common/decorator/require_nonempty_arg.py +65 -0
- ads/common/decorator/runtime_dependency.py +178 -0
- ads/common/decorator/threaded.py +97 -0
- ads/common/decorator/utils.py +35 -0
- ads/common/dsc_file_system.py +303 -0
- ads/common/error.py +14 -0
- ads/common/extended_enum.py +81 -0
- ads/common/function/__init__.py +5 -0
- ads/common/function/fn_util.py +142 -0
- ads/common/function/func_conf.yaml +25 -0
- ads/common/ipython.py +76 -0
- ads/common/model.py +679 -0
- ads/common/model_artifact.py +1759 -0
- ads/common/model_artifact_schema.json +107 -0
- ads/common/model_export_util.py +664 -0
- ads/common/model_metadata.py +24 -0
- ads/common/object_storage_details.py +296 -0
- ads/common/oci_client.py +179 -0
- ads/common/oci_datascience.py +46 -0
- ads/common/oci_logging.py +1144 -0
- ads/common/oci_mixin.py +957 -0
- ads/common/oci_resource.py +136 -0
- ads/common/serializer.py +559 -0
- ads/common/utils.py +1852 -0
- ads/common/word_lists.py +1491 -0
- ads/common/work_request.py +189 -0
- ads/config.py +1 -0
- ads/data_labeling/__init__.py +13 -0
- ads/data_labeling/boundingbox.py +253 -0
- ads/data_labeling/constants.py +47 -0
- ads/data_labeling/data_labeling_service.py +244 -0
- ads/data_labeling/interface/__init__.py +5 -0
- ads/data_labeling/interface/loader.py +16 -0
- ads/data_labeling/interface/parser.py +16 -0
- ads/data_labeling/interface/reader.py +23 -0
- ads/data_labeling/loader/__init__.py +5 -0
- ads/data_labeling/loader/file_loader.py +241 -0
- ads/data_labeling/metadata.py +110 -0
- ads/data_labeling/mixin/__init__.py +5 -0
- ads/data_labeling/mixin/data_labeling.py +232 -0
- ads/data_labeling/ner.py +129 -0
- ads/data_labeling/parser/__init__.py +5 -0
- ads/data_labeling/parser/dls_record_parser.py +388 -0
- ads/data_labeling/parser/export_metadata_parser.py +94 -0
- ads/data_labeling/parser/export_record_parser.py +473 -0
- ads/data_labeling/reader/__init__.py +5 -0
- ads/data_labeling/reader/dataset_reader.py +574 -0
- ads/data_labeling/reader/dls_record_reader.py +121 -0
- ads/data_labeling/reader/export_record_reader.py +62 -0
- ads/data_labeling/reader/jsonl_reader.py +75 -0
- ads/data_labeling/reader/metadata_reader.py +203 -0
- ads/data_labeling/reader/record_reader.py +263 -0
- ads/data_labeling/record.py +52 -0
- ads/data_labeling/visualizer/__init__.py +5 -0
- ads/data_labeling/visualizer/image_visualizer.py +525 -0
- ads/data_labeling/visualizer/text_visualizer.py +357 -0
- ads/database/__init__.py +5 -0
- ads/database/connection.py +338 -0
- ads/dataset/__init__.py +10 -0
- ads/dataset/capabilities.md +51 -0
- ads/dataset/classification_dataset.py +339 -0
- ads/dataset/correlation.py +226 -0
- ads/dataset/correlation_plot.py +563 -0
- ads/dataset/dask_series.py +173 -0
- ads/dataset/dataframe_transformer.py +110 -0
- ads/dataset/dataset.py +1979 -0
- ads/dataset/dataset_browser.py +360 -0
- ads/dataset/dataset_with_target.py +995 -0
- ads/dataset/exception.py +25 -0
- ads/dataset/factory.py +987 -0
- ads/dataset/feature_engineering_transformer.py +35 -0
- ads/dataset/feature_selection.py +107 -0
- ads/dataset/forecasting_dataset.py +26 -0
- ads/dataset/helper.py +1450 -0
- ads/dataset/label_encoder.py +99 -0
- ads/dataset/mixin/__init__.py +5 -0
- ads/dataset/mixin/dataset_accessor.py +134 -0
- ads/dataset/pipeline.py +58 -0
- ads/dataset/plot.py +710 -0
- ads/dataset/progress.py +86 -0
- ads/dataset/recommendation.py +297 -0
- ads/dataset/recommendation_transformer.py +502 -0
- ads/dataset/regression_dataset.py +14 -0
- ads/dataset/sampled_dataset.py +1050 -0
- ads/dataset/target.py +98 -0
- ads/dataset/timeseries.py +18 -0
- ads/dbmixin/__init__.py +5 -0
- ads/dbmixin/db_pandas_accessor.py +153 -0
- ads/environment/__init__.py +9 -0
- ads/environment/ml_runtime.py +66 -0
- ads/evaluations/README.md +14 -0
- ads/evaluations/__init__.py +109 -0
- ads/evaluations/evaluation_plot.py +983 -0
- ads/evaluations/evaluator.py +1334 -0
- ads/evaluations/statistical_metrics.py +543 -0
- ads/experiments/__init__.py +9 -0
- ads/experiments/capabilities.md +0 -0
- ads/explanations/__init__.py +21 -0
- ads/explanations/base_explainer.py +142 -0
- ads/explanations/capabilities.md +83 -0
- ads/explanations/explainer.py +190 -0
- ads/explanations/mlx_global_explainer.py +1050 -0
- ads/explanations/mlx_interface.py +386 -0
- ads/explanations/mlx_local_explainer.py +287 -0
- ads/explanations/mlx_whatif_explainer.py +201 -0
- ads/feature_engineering/__init__.py +20 -0
- ads/feature_engineering/accessor/__init__.py +5 -0
- ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
- ads/feature_engineering/accessor/mixin/__init__.py +5 -0
- ads/feature_engineering/accessor/mixin/correlation.py +166 -0
- ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
- ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
- ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
- ads/feature_engineering/accessor/mixin/utils.py +65 -0
- ads/feature_engineering/accessor/series_accessor.py +431 -0
- ads/feature_engineering/adsimage/__init__.py +5 -0
- ads/feature_engineering/adsimage/image.py +192 -0
- ads/feature_engineering/adsimage/image_reader.py +170 -0
- ads/feature_engineering/adsimage/interface/__init__.py +5 -0
- ads/feature_engineering/adsimage/interface/reader.py +19 -0
- ads/feature_engineering/adsstring/__init__.py +7 -0
- ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
- ads/feature_engineering/adsstring/string/__init__.py +8 -0
- ads/feature_engineering/data_schema.json +57 -0
- ads/feature_engineering/dataset/__init__.py +5 -0
- ads/feature_engineering/dataset/zip_code_data.py +42062 -0
- ads/feature_engineering/exceptions.py +40 -0
- ads/feature_engineering/feature_type/__init__.py +133 -0
- ads/feature_engineering/feature_type/address.py +184 -0
- ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
- ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
- ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
- ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
- ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
- ads/feature_engineering/feature_type/adsstring/string.py +258 -0
- ads/feature_engineering/feature_type/base.py +58 -0
- ads/feature_engineering/feature_type/boolean.py +183 -0
- ads/feature_engineering/feature_type/category.py +146 -0
- ads/feature_engineering/feature_type/constant.py +137 -0
- ads/feature_engineering/feature_type/continuous.py +151 -0
- ads/feature_engineering/feature_type/creditcard.py +314 -0
- ads/feature_engineering/feature_type/datetime.py +190 -0
- ads/feature_engineering/feature_type/discrete.py +134 -0
- ads/feature_engineering/feature_type/document.py +43 -0
- ads/feature_engineering/feature_type/gis.py +251 -0
- ads/feature_engineering/feature_type/handler/__init__.py +5 -0
- ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
- ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
- ads/feature_engineering/feature_type/handler/warnings.py +128 -0
- ads/feature_engineering/feature_type/integer.py +142 -0
- ads/feature_engineering/feature_type/ip_address.py +144 -0
- ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
- ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
- ads/feature_engineering/feature_type/lat_long.py +256 -0
- ads/feature_engineering/feature_type/object.py +43 -0
- ads/feature_engineering/feature_type/ordinal.py +132 -0
- ads/feature_engineering/feature_type/phone_number.py +135 -0
- ads/feature_engineering/feature_type/string.py +171 -0
- ads/feature_engineering/feature_type/text.py +93 -0
- ads/feature_engineering/feature_type/unknown.py +43 -0
- ads/feature_engineering/feature_type/zip_code.py +164 -0
- ads/feature_engineering/feature_type_manager.py +406 -0
- ads/feature_engineering/schema.py +795 -0
- ads/feature_engineering/utils.py +245 -0
- ads/feature_store/.readthedocs.yaml +19 -0
- ads/feature_store/README.md +65 -0
- ads/feature_store/__init__.py +9 -0
- ads/feature_store/common/__init__.py +0 -0
- ads/feature_store/common/enums.py +339 -0
- ads/feature_store/common/exceptions.py +18 -0
- ads/feature_store/common/spark_session_singleton.py +125 -0
- ads/feature_store/common/utils/__init__.py +0 -0
- ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
- ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
- ads/feature_store/common/utils/transformation_utils.py +82 -0
- ads/feature_store/common/utils/utility.py +403 -0
- ads/feature_store/data_validation/__init__.py +0 -0
- ads/feature_store/data_validation/great_expectation.py +129 -0
- ads/feature_store/dataset.py +1230 -0
- ads/feature_store/dataset_job.py +530 -0
- ads/feature_store/docs/Dockerfile +7 -0
- ads/feature_store/docs/Makefile +44 -0
- ads/feature_store/docs/conf.py +28 -0
- ads/feature_store/docs/requirements.txt +14 -0
- ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
- ads/feature_store/docs/source/cicd.rst +137 -0
- ads/feature_store/docs/source/conf.py +86 -0
- ads/feature_store/docs/source/data_versioning.rst +33 -0
- ads/feature_store/docs/source/dataset.rst +388 -0
- ads/feature_store/docs/source/dataset_job.rst +27 -0
- ads/feature_store/docs/source/demo.rst +70 -0
- ads/feature_store/docs/source/entity.rst +78 -0
- ads/feature_store/docs/source/feature_group.rst +624 -0
- ads/feature_store/docs/source/feature_group_job.rst +29 -0
- ads/feature_store/docs/source/feature_store.rst +122 -0
- ads/feature_store/docs/source/feature_store_class.rst +123 -0
- ads/feature_store/docs/source/feature_validation.rst +66 -0
- ads/feature_store/docs/source/figures/cicd.png +0 -0
- ads/feature_store/docs/source/figures/data_validation.png +0 -0
- ads/feature_store/docs/source/figures/data_versioning.png +0 -0
- ads/feature_store/docs/source/figures/dataset.gif +0 -0
- ads/feature_store/docs/source/figures/dataset.png +0 -0
- ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
- ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
- ads/feature_store/docs/source/figures/entity.png +0 -0
- ads/feature_store/docs/source/figures/feature_group.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
- ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
- ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
- ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
- ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
- ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
- ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
- ads/feature_store/docs/source/figures/overview.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
- ads/feature_store/docs/source/figures/stats_1.png +0 -0
- ads/feature_store/docs/source/figures/stats_2.png +0 -0
- ads/feature_store/docs/source/figures/stats_d.png +0 -0
- ads/feature_store/docs/source/figures/stats_fg.png +0 -0
- ads/feature_store/docs/source/figures/transformation.png +0 -0
- ads/feature_store/docs/source/figures/transformations.gif +0 -0
- ads/feature_store/docs/source/figures/validation.png +0 -0
- ads/feature_store/docs/source/figures/validation_fg.png +0 -0
- ads/feature_store/docs/source/figures/validation_results.png +0 -0
- ads/feature_store/docs/source/figures/validation_summary.png +0 -0
- ads/feature_store/docs/source/index.rst +81 -0
- ads/feature_store/docs/source/module.rst +8 -0
- ads/feature_store/docs/source/notebook.rst +94 -0
- ads/feature_store/docs/source/overview.rst +47 -0
- ads/feature_store/docs/source/quickstart.rst +176 -0
- ads/feature_store/docs/source/release_notes.rst +194 -0
- ads/feature_store/docs/source/setup_feature_store.rst +81 -0
- ads/feature_store/docs/source/statistics.rst +58 -0
- ads/feature_store/docs/source/transformation.rst +199 -0
- ads/feature_store/docs/source/ui.rst +65 -0
- ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
- ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
- ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
- ads/feature_store/entity.py +718 -0
- ads/feature_store/execution_strategy/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
- ads/feature_store/execution_strategy/engine/__init__.py +0 -0
- ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
- ads/feature_store/execution_strategy/execution_strategy.py +113 -0
- ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
- ads/feature_store/execution_strategy/spark/__init__.py +0 -0
- ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
- ads/feature_store/feature.py +192 -0
- ads/feature_store/feature_group.py +1494 -0
- ads/feature_store/feature_group_expectation.py +346 -0
- ads/feature_store/feature_group_job.py +602 -0
- ads/feature_store/feature_lineage/__init__.py +0 -0
- ads/feature_store/feature_lineage/graphviz_service.py +180 -0
- ads/feature_store/feature_option_details.py +50 -0
- ads/feature_store/feature_statistics/__init__.py +0 -0
- ads/feature_store/feature_statistics/statistics_service.py +99 -0
- ads/feature_store/feature_store.py +699 -0
- ads/feature_store/feature_store_registrar.py +518 -0
- ads/feature_store/input_feature_detail.py +149 -0
- ads/feature_store/mixin/__init__.py +4 -0
- ads/feature_store/mixin/oci_feature_store.py +145 -0
- ads/feature_store/model_details.py +73 -0
- ads/feature_store/query/__init__.py +0 -0
- ads/feature_store/query/filter.py +266 -0
- ads/feature_store/query/generator/__init__.py +0 -0
- ads/feature_store/query/generator/query_generator.py +298 -0
- ads/feature_store/query/join.py +161 -0
- ads/feature_store/query/query.py +403 -0
- ads/feature_store/query/validator/__init__.py +0 -0
- ads/feature_store/query/validator/query_validator.py +57 -0
- ads/feature_store/response/__init__.py +0 -0
- ads/feature_store/response/response_builder.py +68 -0
- ads/feature_store/service/__init__.py +0 -0
- ads/feature_store/service/oci_dataset.py +139 -0
- ads/feature_store/service/oci_dataset_job.py +199 -0
- ads/feature_store/service/oci_entity.py +125 -0
- ads/feature_store/service/oci_feature_group.py +164 -0
- ads/feature_store/service/oci_feature_group_job.py +214 -0
- ads/feature_store/service/oci_feature_store.py +182 -0
- ads/feature_store/service/oci_lineage.py +87 -0
- ads/feature_store/service/oci_transformation.py +104 -0
- ads/feature_store/statistics/__init__.py +0 -0
- ads/feature_store/statistics/abs_feature_value.py +49 -0
- ads/feature_store/statistics/charts/__init__.py +0 -0
- ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
- ads/feature_store/statistics/charts/box_plot.py +148 -0
- ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
- ads/feature_store/statistics/charts/probability_distribution.py +68 -0
- ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
- ads/feature_store/statistics/feature_stat.py +126 -0
- ads/feature_store/statistics/generic_feature_value.py +33 -0
- ads/feature_store/statistics/statistics.py +41 -0
- ads/feature_store/statistics_config.py +101 -0
- ads/feature_store/templates/feature_store_template.yaml +45 -0
- ads/feature_store/transformation.py +499 -0
- ads/feature_store/validation_output.py +57 -0
- ads/hpo/__init__.py +9 -0
- ads/hpo/_imports.py +91 -0
- ads/hpo/ads_search_space.py +439 -0
- ads/hpo/distributions.py +325 -0
- ads/hpo/objective.py +280 -0
- ads/hpo/search_cv.py +1657 -0
- ads/hpo/stopping_criterion.py +75 -0
- ads/hpo/tuner_artifact.py +413 -0
- ads/hpo/utils.py +91 -0
- ads/hpo/validation.py +140 -0
- ads/hpo/visualization/__init__.py +5 -0
- ads/hpo/visualization/_contour.py +23 -0
- ads/hpo/visualization/_edf.py +20 -0
- ads/hpo/visualization/_intermediate_values.py +21 -0
- ads/hpo/visualization/_optimization_history.py +25 -0
- ads/hpo/visualization/_parallel_coordinate.py +169 -0
- ads/hpo/visualization/_param_importances.py +26 -0
- ads/jobs/__init__.py +53 -0
- ads/jobs/ads_job.py +663 -0
- ads/jobs/builders/__init__.py +5 -0
- ads/jobs/builders/base.py +156 -0
- ads/jobs/builders/infrastructure/__init__.py +6 -0
- ads/jobs/builders/infrastructure/base.py +165 -0
- ads/jobs/builders/infrastructure/dataflow.py +1252 -0
- ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
- ads/jobs/builders/infrastructure/utils.py +65 -0
- ads/jobs/builders/runtimes/__init__.py +5 -0
- ads/jobs/builders/runtimes/artifact.py +338 -0
- ads/jobs/builders/runtimes/base.py +325 -0
- ads/jobs/builders/runtimes/container_runtime.py +242 -0
- ads/jobs/builders/runtimes/python_runtime.py +1016 -0
- ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
- ads/jobs/cli.py +104 -0
- ads/jobs/env_var_parser.py +131 -0
- ads/jobs/extension.py +160 -0
- ads/jobs/schema/__init__.py +5 -0
- ads/jobs/schema/infrastructure_schema.json +116 -0
- ads/jobs/schema/job_schema.json +42 -0
- ads/jobs/schema/runtime_schema.json +183 -0
- ads/jobs/schema/validator.py +141 -0
- ads/jobs/serializer.py +296 -0
- ads/jobs/templates/__init__.py +5 -0
- ads/jobs/templates/container.py +6 -0
- ads/jobs/templates/driver_notebook.py +177 -0
- ads/jobs/templates/driver_oci.py +500 -0
- ads/jobs/templates/driver_python.py +48 -0
- ads/jobs/templates/driver_pytorch.py +852 -0
- ads/jobs/templates/driver_utils.py +615 -0
- ads/jobs/templates/hostname_from_env.c +55 -0
- ads/jobs/templates/oci_metrics.py +181 -0
- ads/jobs/utils.py +104 -0
- ads/llm/__init__.py +28 -0
- ads/llm/autogen/__init__.py +2 -0
- ads/llm/autogen/constants.py +15 -0
- ads/llm/autogen/reports/__init__.py +2 -0
- ads/llm/autogen/reports/base.py +67 -0
- ads/llm/autogen/reports/data.py +103 -0
- ads/llm/autogen/reports/session.py +526 -0
- ads/llm/autogen/reports/templates/chat_box.html +13 -0
- ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
- ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
- ads/llm/autogen/reports/utils.py +56 -0
- ads/llm/autogen/v02/__init__.py +4 -0
- ads/llm/autogen/v02/client.py +295 -0
- ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
- ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
- ads/llm/autogen/v02/loggers/__init__.py +6 -0
- ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
- ads/llm/autogen/v02/loggers/session_logger.py +580 -0
- ads/llm/autogen/v02/loggers/utils.py +86 -0
- ads/llm/autogen/v02/runtime_logging.py +163 -0
- ads/llm/chain.py +268 -0
- ads/llm/chat_template.py +31 -0
- ads/llm/deploy.py +63 -0
- ads/llm/guardrails/__init__.py +5 -0
- ads/llm/guardrails/base.py +442 -0
- ads/llm/guardrails/huggingface.py +44 -0
- ads/llm/langchain/__init__.py +5 -0
- ads/llm/langchain/plugins/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
- ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
- ads/llm/langchain/plugins/llms/__init__.py +5 -0
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
- ads/llm/requirements.txt +3 -0
- ads/llm/serialize.py +219 -0
- ads/llm/serializers/__init__.py +0 -0
- ads/llm/serializers/retrieval_qa.py +153 -0
- ads/llm/serializers/runnable_parallel.py +27 -0
- ads/llm/templates/score_chain.jinja2 +155 -0
- ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
- ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
- ads/model/__init__.py +52 -0
- ads/model/artifact.py +573 -0
- ads/model/artifact_downloader.py +254 -0
- ads/model/artifact_uploader.py +267 -0
- ads/model/base_properties.py +238 -0
- ads/model/common/.model-ignore +66 -0
- ads/model/common/__init__.py +5 -0
- ads/model/common/utils.py +142 -0
- ads/model/datascience_model.py +2635 -0
- ads/model/deployment/__init__.py +20 -0
- ads/model/deployment/common/__init__.py +5 -0
- ads/model/deployment/common/utils.py +308 -0
- ads/model/deployment/model_deployer.py +466 -0
- ads/model/deployment/model_deployment.py +1846 -0
- ads/model/deployment/model_deployment_infrastructure.py +671 -0
- ads/model/deployment/model_deployment_properties.py +493 -0
- ads/model/deployment/model_deployment_runtime.py +838 -0
- ads/model/extractor/__init__.py +5 -0
- ads/model/extractor/automl_extractor.py +74 -0
- ads/model/extractor/embedding_onnx_extractor.py +80 -0
- ads/model/extractor/huggingface_extractor.py +88 -0
- ads/model/extractor/keras_extractor.py +84 -0
- ads/model/extractor/lightgbm_extractor.py +93 -0
- ads/model/extractor/model_info_extractor.py +114 -0
- ads/model/extractor/model_info_extractor_factory.py +105 -0
- ads/model/extractor/pytorch_extractor.py +87 -0
- ads/model/extractor/sklearn_extractor.py +112 -0
- ads/model/extractor/spark_extractor.py +89 -0
- ads/model/extractor/tensorflow_extractor.py +85 -0
- ads/model/extractor/xgboost_extractor.py +94 -0
- ads/model/framework/__init__.py +5 -0
- ads/model/framework/automl_model.py +178 -0
- ads/model/framework/embedding_onnx_model.py +438 -0
- ads/model/framework/huggingface_model.py +399 -0
- ads/model/framework/lightgbm_model.py +266 -0
- ads/model/framework/pytorch_model.py +266 -0
- ads/model/framework/sklearn_model.py +250 -0
- ads/model/framework/spark_model.py +326 -0
- ads/model/framework/tensorflow_model.py +254 -0
- ads/model/framework/xgboost_model.py +258 -0
- ads/model/generic_model.py +3518 -0
- ads/model/model_artifact_boilerplate/README.md +381 -0
- ads/model/model_artifact_boilerplate/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
- ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
- ads/model/model_artifact_boilerplate/score.py +61 -0
- ads/model/model_file_description_schema.json +68 -0
- ads/model/model_introspect.py +331 -0
- ads/model/model_metadata.py +1810 -0
- ads/model/model_metadata_mixin.py +460 -0
- ads/model/model_properties.py +63 -0
- ads/model/model_version_set.py +739 -0
- ads/model/runtime/__init__.py +5 -0
- ads/model/runtime/env_info.py +306 -0
- ads/model/runtime/model_deployment_details.py +37 -0
- ads/model/runtime/model_provenance_details.py +58 -0
- ads/model/runtime/runtime_info.py +81 -0
- ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
- ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
- ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
- ads/model/runtime/utils.py +201 -0
- ads/model/serde/__init__.py +5 -0
- ads/model/serde/common.py +40 -0
- ads/model/serde/model_input.py +547 -0
- ads/model/serde/model_serializer.py +1184 -0
- ads/model/service/__init__.py +5 -0
- ads/model/service/oci_datascience_model.py +1076 -0
- ads/model/service/oci_datascience_model_deployment.py +500 -0
- ads/model/service/oci_datascience_model_version_set.py +176 -0
- ads/model/transformer/__init__.py +5 -0
- ads/model/transformer/onnx_transformer.py +324 -0
- ads/mysqldb/__init__.py +5 -0
- ads/mysqldb/mysql_db.py +227 -0
- ads/opctl/__init__.py +18 -0
- ads/opctl/anomaly_detection.py +11 -0
- ads/opctl/backend/__init__.py +5 -0
- ads/opctl/backend/ads_dataflow.py +353 -0
- ads/opctl/backend/ads_ml_job.py +710 -0
- ads/opctl/backend/ads_ml_pipeline.py +164 -0
- ads/opctl/backend/ads_model_deployment.py +209 -0
- ads/opctl/backend/base.py +146 -0
- ads/opctl/backend/local.py +1053 -0
- ads/opctl/backend/marketplace/__init__.py +9 -0
- ads/opctl/backend/marketplace/helm_helper.py +173 -0
- ads/opctl/backend/marketplace/local_marketplace.py +271 -0
- ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
- ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
- ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
- ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
- ads/opctl/backend/marketplace/models/__init__.py +5 -0
- ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
- ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
- ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
- ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
- ads/opctl/cli.py +707 -0
- ads/opctl/cmds.py +869 -0
- ads/opctl/conda/__init__.py +5 -0
- ads/opctl/conda/cli.py +193 -0
- ads/opctl/conda/cmds.py +749 -0
- ads/opctl/conda/config.yaml +34 -0
- ads/opctl/conda/manifest_template.yaml +13 -0
- ads/opctl/conda/multipart_uploader.py +188 -0
- ads/opctl/conda/pack.py +89 -0
- ads/opctl/config/__init__.py +5 -0
- ads/opctl/config/base.py +57 -0
- ads/opctl/config/diagnostics/__init__.py +5 -0
- ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
- ads/opctl/config/merger.py +255 -0
- ads/opctl/config/resolver.py +297 -0
- ads/opctl/config/utils.py +79 -0
- ads/opctl/config/validator.py +17 -0
- ads/opctl/config/versioner.py +68 -0
- ads/opctl/config/yaml_parsers/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/base.py +58 -0
- ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
- ads/opctl/constants.py +66 -0
- ads/opctl/decorator/__init__.py +5 -0
- ads/opctl/decorator/common.py +129 -0
- ads/opctl/diagnostics/__init__.py +5 -0
- ads/opctl/diagnostics/__main__.py +25 -0
- ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
- ads/opctl/diagnostics/check_requirements.py +144 -0
- ads/opctl/diagnostics/requirement_exception.py +9 -0
- ads/opctl/distributed/README.md +109 -0
- ads/opctl/distributed/__init__.py +5 -0
- ads/opctl/distributed/certificates.py +32 -0
- ads/opctl/distributed/cli.py +207 -0
- ads/opctl/distributed/cmds.py +731 -0
- ads/opctl/distributed/common/__init__.py +5 -0
- ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
- ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
- ads/opctl/distributed/common/cluster_config_helper.py +103 -0
- ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
- ads/opctl/distributed/common/cluster_runner.py +54 -0
- ads/opctl/distributed/common/framework_factory.py +29 -0
- ads/opctl/docker/Dockerfile.job +103 -0
- ads/opctl/docker/Dockerfile.job.arm +107 -0
- ads/opctl/docker/Dockerfile.job.gpu +175 -0
- ads/opctl/docker/base-env.yaml +13 -0
- ads/opctl/docker/cuda.repo +6 -0
- ads/opctl/docker/operator/.dockerignore +0 -0
- ads/opctl/docker/operator/Dockerfile +41 -0
- ads/opctl/docker/operator/Dockerfile.gpu +85 -0
- ads/opctl/docker/operator/cuda.repo +6 -0
- ads/opctl/docker/operator/environment.yaml +8 -0
- ads/opctl/forecast.py +11 -0
- ads/opctl/index.yaml +3 -0
- ads/opctl/model/__init__.py +5 -0
- ads/opctl/model/cli.py +65 -0
- ads/opctl/model/cmds.py +73 -0
- ads/opctl/operator/README.md +4 -0
- ads/opctl/operator/__init__.py +31 -0
- ads/opctl/operator/cli.py +344 -0
- ads/opctl/operator/cmd.py +596 -0
- ads/opctl/operator/common/__init__.py +5 -0
- ads/opctl/operator/common/backend_factory.py +460 -0
- ads/opctl/operator/common/const.py +27 -0
- ads/opctl/operator/common/data/synthetic.csv +16001 -0
- ads/opctl/operator/common/dictionary_merger.py +148 -0
- ads/opctl/operator/common/errors.py +42 -0
- ads/opctl/operator/common/operator_config.py +99 -0
- ads/opctl/operator/common/operator_loader.py +811 -0
- ads/opctl/operator/common/operator_schema.yaml +130 -0
- ads/opctl/operator/common/operator_yaml_generator.py +152 -0
- ads/opctl/operator/common/utils.py +208 -0
- ads/opctl/operator/lowcode/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
- ads/opctl/operator/lowcode/anomaly/README.md +207 -0
- ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
- ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
- ads/opctl/operator/lowcode/anomaly/const.py +167 -0
- ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
- ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
- ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
- ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
- ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
- ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
- ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
- ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
- ads/opctl/operator/lowcode/common/__init__.py +5 -0
- ads/opctl/operator/lowcode/common/const.py +10 -0
- ads/opctl/operator/lowcode/common/data.py +116 -0
- ads/opctl/operator/lowcode/common/errors.py +47 -0
- ads/opctl/operator/lowcode/common/transformations.py +296 -0
- ads/opctl/operator/lowcode/common/utils.py +384 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
- ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
- ads/opctl/operator/lowcode/forecast/README.md +209 -0
- ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
- ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
- ads/opctl/operator/lowcode/forecast/const.py +92 -0
- ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
- ads/opctl/operator/lowcode/forecast/errors.py +26 -0
- ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
- ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
- ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
- ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
- ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
- ads/opctl/operator/lowcode/forecast/model/prophet.py +450 -0
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
- ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
- ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
- ads/opctl/operator/lowcode/forecast/utils.py +397 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
- ads/opctl/operator/lowcode/pii/MLoperator +17 -0
- ads/opctl/operator/lowcode/pii/README.md +208 -0
- ads/opctl/operator/lowcode/pii/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/__main__.py +78 -0
- ads/opctl/operator/lowcode/pii/cmd.py +39 -0
- ads/opctl/operator/lowcode/pii/constant.py +84 -0
- ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
- ads/opctl/operator/lowcode/pii/errors.py +27 -0
- ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
- ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
- ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
- ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
- ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
- ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
- ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
- ads/opctl/operator/lowcode/pii/model/report.py +487 -0
- ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
- ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
- ads/opctl/operator/lowcode/pii/utils.py +43 -0
- ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
- ads/opctl/operator/lowcode/recommender/README.md +206 -0
- ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
- ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
- ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
- ads/opctl/operator/lowcode/recommender/constant.py +30 -0
- ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
- ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
- ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
- ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
- ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
- ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
- ads/opctl/operator/lowcode/recommender/utils.py +13 -0
- ads/opctl/operator/runtime/__init__.py +5 -0
- ads/opctl/operator/runtime/const.py +17 -0
- ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
- ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
- ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/runtime.py +115 -0
- ads/opctl/schema.yaml.yml +36 -0
- ads/opctl/script.py +40 -0
- ads/opctl/spark/__init__.py +5 -0
- ads/opctl/spark/cli.py +43 -0
- ads/opctl/spark/cmds.py +147 -0
- ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
- ads/opctl/utils.py +344 -0
- ads/oracledb/__init__.py +5 -0
- ads/oracledb/oracle_db.py +346 -0
- ads/pipeline/__init__.py +39 -0
- ads/pipeline/ads_pipeline.py +2279 -0
- ads/pipeline/ads_pipeline_run.py +772 -0
- ads/pipeline/ads_pipeline_step.py +605 -0
- ads/pipeline/builders/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/custom_script.py +32 -0
- ads/pipeline/cli.py +119 -0
- ads/pipeline/extension.py +291 -0
- ads/pipeline/schema/__init__.py +5 -0
- ads/pipeline/schema/cs_step_schema.json +35 -0
- ads/pipeline/schema/ml_step_schema.json +31 -0
- ads/pipeline/schema/pipeline_schema.json +71 -0
- ads/pipeline/visualizer/__init__.py +5 -0
- ads/pipeline/visualizer/base.py +570 -0
- ads/pipeline/visualizer/graph_renderer.py +272 -0
- ads/pipeline/visualizer/text_renderer.py +84 -0
- ads/secrets/__init__.py +11 -0
- ads/secrets/adb.py +386 -0
- ads/secrets/auth_token.py +86 -0
- ads/secrets/big_data_service.py +365 -0
- ads/secrets/mysqldb.py +149 -0
- ads/secrets/oracledb.py +160 -0
- ads/secrets/secrets.py +407 -0
- ads/telemetry/__init__.py +7 -0
- ads/telemetry/base.py +69 -0
- ads/telemetry/client.py +122 -0
- ads/telemetry/telemetry.py +257 -0
- ads/templates/dataflow_pyspark.jinja2 +13 -0
- ads/templates/dataflow_sparksql.jinja2 +22 -0
- ads/templates/func.jinja2 +20 -0
- ads/templates/schemas/openapi.json +1740 -0
- ads/templates/score-pkl.jinja2 +173 -0
- ads/templates/score.jinja2 +322 -0
- ads/templates/score_embedding_onnx.jinja2 +202 -0
- ads/templates/score_generic.jinja2 +165 -0
- ads/templates/score_huggingface_pipeline.jinja2 +217 -0
- ads/templates/score_lightgbm.jinja2 +185 -0
- ads/templates/score_onnx.jinja2 +407 -0
- ads/templates/score_onnx_new.jinja2 +473 -0
- ads/templates/score_oracle_automl.jinja2 +185 -0
- ads/templates/score_pyspark.jinja2 +154 -0
- ads/templates/score_pytorch.jinja2 +219 -0
- ads/templates/score_scikit-learn.jinja2 +184 -0
- ads/templates/score_tensorflow.jinja2 +184 -0
- ads/templates/score_xgboost.jinja2 +178 -0
- ads/text_dataset/__init__.py +5 -0
- ads/text_dataset/backends.py +211 -0
- ads/text_dataset/dataset.py +445 -0
- ads/text_dataset/extractor.py +207 -0
- ads/text_dataset/options.py +53 -0
- ads/text_dataset/udfs.py +22 -0
- ads/text_dataset/utils.py +49 -0
- ads/type_discovery/__init__.py +9 -0
- ads/type_discovery/abstract_detector.py +21 -0
- ads/type_discovery/constant_detector.py +41 -0
- ads/type_discovery/continuous_detector.py +54 -0
- ads/type_discovery/credit_card_detector.py +99 -0
- ads/type_discovery/datetime_detector.py +92 -0
- ads/type_discovery/discrete_detector.py +118 -0
- ads/type_discovery/document_detector.py +146 -0
- ads/type_discovery/ip_detector.py +68 -0
- ads/type_discovery/latlon_detector.py +90 -0
- ads/type_discovery/phone_number_detector.py +63 -0
- ads/type_discovery/type_discovery_driver.py +87 -0
- ads/type_discovery/typed_feature.py +594 -0
- ads/type_discovery/unknown_detector.py +41 -0
- ads/type_discovery/zipcode_detector.py +48 -0
- ads/vault/__init__.py +7 -0
- ads/vault/vault.py +237 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/METADATA +150 -149
- oracle_ads-2.13.10.dist-info/RECORD +858 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/WHEEL +1 -2
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/entry_points.txt +2 -1
- oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
- oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,1810 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
|
3
|
+
# Copyright (c) 2021, 2025 Oracle and/or its affiliates.
|
4
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
5
|
+
|
6
|
+
import json
|
7
|
+
import logging
|
8
|
+
import os
|
9
|
+
import sys
|
10
|
+
from abc import ABC, abstractmethod
|
11
|
+
from dataclasses import dataclass, field, fields
|
12
|
+
from pathlib import Path
|
13
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
14
|
+
|
15
|
+
import fsspec
|
16
|
+
import git
|
17
|
+
import oci.data_science.models
|
18
|
+
import pandas as pd
|
19
|
+
import yaml
|
20
|
+
from oci.util import to_dict
|
21
|
+
|
22
|
+
from ads.common import logger
|
23
|
+
from ads.common.error import ChangesNotCommitted
|
24
|
+
from ads.common.extended_enum import ExtendedEnum
|
25
|
+
from ads.common.object_storage_details import ObjectStorageDetails
|
26
|
+
from ads.common.serializer import DataClassSerializable
|
27
|
+
from ads.common.utils import parse_bool
|
28
|
+
from ads.dataset import factory
|
29
|
+
|
30
|
+
try:
|
31
|
+
from yaml import CDumper as dumper
|
32
|
+
except:
|
33
|
+
from yaml import Dumper as dumper
|
34
|
+
|
35
|
+
|
36
|
+
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
|
37
|
+
logger = logging.getLogger("ADS")
|
38
|
+
|
39
|
+
METADATA_SIZE_LIMIT = 32000
|
40
|
+
METADATA_VALUE_LENGTH_LIMIT = 255
|
41
|
+
METADATA_DESCRIPTION_LENGTH_LIMIT = 255
|
42
|
+
_METADATA_EMPTY_VALUE = "NA"
|
43
|
+
CURRENT_WORKING_DIR = "."
|
44
|
+
|
45
|
+
_sentinel = object()
|
46
|
+
|
47
|
+
|
48
|
+
class MetadataSizeTooLarge(ValueError):
|
49
|
+
"""Maximum allowed size for model metadata has been exceeded.
|
50
|
+
See https://docs.oracle.com/en-us/iaas/data-science/using/models_saving_catalog.htm for more details.
|
51
|
+
"""
|
52
|
+
|
53
|
+
def __init__(self, size: int):
|
54
|
+
super().__init__(
|
55
|
+
f"The metadata is `{size}` bytes and exceeds "
|
56
|
+
f"the size limit of `{METADATA_SIZE_LIMIT}` bytes. "
|
57
|
+
"Reduce the metadata size."
|
58
|
+
)
|
59
|
+
|
60
|
+
|
61
|
+
class MetadataValueTooLong(ValueError):
|
62
|
+
"""Maximum allowed length of metadata value has been exceeded.
|
63
|
+
See https://docs.oracle.com/en-us/iaas/data-science/using/models_saving_catalog.htm for more details.
|
64
|
+
"""
|
65
|
+
|
66
|
+
def __init__(self, key: str, length: int):
|
67
|
+
super().__init__(
|
68
|
+
f"The custom metadata value of `{key}` is `{length}` characters and exceeds "
|
69
|
+
f"the length limit of `{METADATA_VALUE_LENGTH_LIMIT}` characters."
|
70
|
+
)
|
71
|
+
|
72
|
+
|
73
|
+
class MetadataDescriptionTooLong(ValueError):
|
74
|
+
"""Maximum allowed length of metadata description has been exceeded.
|
75
|
+
See https://docs.oracle.com/en-us/iaas/data-science/using/models_saving_catalog.htm for more details.
|
76
|
+
"""
|
77
|
+
|
78
|
+
def __init__(self, key: str, length: int):
|
79
|
+
super().__init__(
|
80
|
+
f"The custom metadata description of `{key}` is `{length}` characters and exceeds "
|
81
|
+
f"the length limit of `{METADATA_DESCRIPTION_LENGTH_LIMIT}` characters."
|
82
|
+
)
|
83
|
+
|
84
|
+
|
85
|
+
class MetadataCustomPrintColumns(ExtendedEnum):
|
86
|
+
KEY = "Key"
|
87
|
+
VALUE = "Value"
|
88
|
+
DESCRIPTION = "Description"
|
89
|
+
CATEGORY = "Category"
|
90
|
+
HAS_ARTIFACT = "HasArtifact"
|
91
|
+
|
92
|
+
|
93
|
+
class MetadataTaxonomyPrintColumns(ExtendedEnum):
|
94
|
+
KEY = "Key"
|
95
|
+
VALUE = "Value"
|
96
|
+
HAS_ARTIFACT = "HasArtifact"
|
97
|
+
|
98
|
+
|
99
|
+
class MetadataTaxonomyKeys(ExtendedEnum):
|
100
|
+
USE_CASE_TYPE = "UseCaseType"
|
101
|
+
FRAMEWORK = "Framework"
|
102
|
+
FRAMEWORK_VERSION = "FrameworkVersion"
|
103
|
+
ALGORITHM = "Algorithm"
|
104
|
+
HYPERPARAMETERS = "Hyperparameters"
|
105
|
+
ARTIFACT_TEST_RESULT = "ArtifactTestResults"
|
106
|
+
# README = "Readme"
|
107
|
+
# LICENSE = "License"
|
108
|
+
# DEPLOYMENT_CONFIGURATION = "DeploymentConfiguration"
|
109
|
+
# FINETUNE_CONFIGURATION = "FineTuneConfiguration"
|
110
|
+
|
111
|
+
|
112
|
+
class MetadataCustomKeys(ExtendedEnum):
|
113
|
+
SLUG_NAME = "SlugName"
|
114
|
+
CONDA_ENVIRONMENT = "CondaEnvironment"
|
115
|
+
CONDA_ENVIRONMENT_PATH = "CondaEnvironmentPath"
|
116
|
+
ENVIRONMENT_TYPE = "EnvironmentType"
|
117
|
+
MODEL_ARTIFACTS = "ModelArtifacts"
|
118
|
+
TRAINING_DATASET = "TrainingDataset"
|
119
|
+
VALIDATION_DATASET = "ValidationDataset"
|
120
|
+
MODEL_SERIALIZATION_FORMAT = "ModelSerializationFormat"
|
121
|
+
TRAINING_DATASET_SIZE = "TrainingDatasetSize"
|
122
|
+
VALIDATION_DATASET_SIZE = "ValidationDatasetSize"
|
123
|
+
TRAINING_DATASET_NUMBER_OF_ROWS = "TrainingDatasetNumberOfRows"
|
124
|
+
TRAINING_DATASET_NUMBER_OF_COLS = "TrainingDatasetNumberOfCols"
|
125
|
+
VALIDATION_DATASET_NUMBER_OF_ROWS = "ValidationDatasetNumberOfRows"
|
126
|
+
VALIDATION_DATASET_NUMBER_OF_COLS = "ValidationDataSetNumberOfCols"
|
127
|
+
CLIENT_LIBRARY = "ClientLibrary"
|
128
|
+
MODEL_FILE_NAME = "ModelFileName"
|
129
|
+
|
130
|
+
|
131
|
+
class MetadataCustomCategory(ExtendedEnum):
|
132
|
+
PERFORMANCE = "Performance"
|
133
|
+
TRAINING_PROFILE = "Training Profile"
|
134
|
+
TRAINING_AND_VALIDATION_DATASETS = "Training and Validation Datasets"
|
135
|
+
TRAINING_ENV = "Training Environment"
|
136
|
+
OTHER = "Other"
|
137
|
+
|
138
|
+
|
139
|
+
class UseCaseType(ExtendedEnum):
|
140
|
+
BINARY_CLASSIFICATION = "binary_classification"
|
141
|
+
REGRESSION = "regression"
|
142
|
+
MULTINOMIAL_CLASSIFICATION = "multinomial_classification"
|
143
|
+
CLUSTERING = "clustering"
|
144
|
+
RECOMMENDER = "recommender"
|
145
|
+
DIMENSIONALITY_REDUCTION = "dimensionality_reduction/representation"
|
146
|
+
TIME_SERIES_FORECASTING = "time_series_forecasting"
|
147
|
+
ANOMALY_DETECTION = "anomaly_detection"
|
148
|
+
TOPIC_MODELING = "topic_modeling"
|
149
|
+
NER = "ner"
|
150
|
+
SENTIMENT_ANALYSIS = "sentiment_analysis"
|
151
|
+
IMAGE_CLASSIFICATION = "image_classification"
|
152
|
+
OBJECT_LOCALIZATION = "object_localization"
|
153
|
+
OTHER = "other"
|
154
|
+
|
155
|
+
|
156
|
+
class Framework(ExtendedEnum):
|
157
|
+
SCIKIT_LEARN = "scikit-learn"
|
158
|
+
XGBOOST = "xgboost"
|
159
|
+
TENSORFLOW = "tensorflow"
|
160
|
+
PYTORCH = "pytorch"
|
161
|
+
MXNET = "mxnet"
|
162
|
+
KERAS = "keras"
|
163
|
+
LIGHT_GBM = "lightgbm"
|
164
|
+
PYMC3 = "pymc3"
|
165
|
+
PYOD = "pyod"
|
166
|
+
SPACY = "spacy"
|
167
|
+
PROPHET = "prophet"
|
168
|
+
SKTIME = "sktime"
|
169
|
+
STATSMODELS = "statsmodels"
|
170
|
+
CUML = "cuml"
|
171
|
+
ORACLE_AUTOML = "oracle_automl"
|
172
|
+
H20 = "h2o"
|
173
|
+
TRANSFORMERS = "transformers"
|
174
|
+
NLTK = "nltk"
|
175
|
+
EMCEE = "emcee"
|
176
|
+
PYSTAN = "pystan"
|
177
|
+
BERT = "bert"
|
178
|
+
GENSIM = "gensim"
|
179
|
+
FLAIR = "flair"
|
180
|
+
WORD2VEC = "word2vec"
|
181
|
+
ENSEMBLE = "ensemble"
|
182
|
+
SPARK = "pyspark"
|
183
|
+
EMBEDDING_ONNX = "embedding_onnx"
|
184
|
+
OTHER = "other"
|
185
|
+
|
186
|
+
|
187
|
+
class ModelMetadataItem(ABC):
|
188
|
+
"""The base abstract class representing model metadata item.
|
189
|
+
|
190
|
+
Methods
|
191
|
+
-------
|
192
|
+
to_dict(self) -> Dict
|
193
|
+
Serializes model metadata item to dictionary.
|
194
|
+
from_dict(cls, data: Dict) -> ModelMetadataItem
|
195
|
+
Constructs an instance of ModelMetadataItem from a dictionary.
|
196
|
+
to_yaml(self)
|
197
|
+
Serializes model metadata item to YAML.
|
198
|
+
size(self) -> int
|
199
|
+
Returns the size of the metadata in bytes.
|
200
|
+
to_json(self) -> JSON
|
201
|
+
Serializes metadata item to JSON.
|
202
|
+
to_json_file(self, file_path: str, storage_options: dict = None) -> None
|
203
|
+
Saves the metadata item value to a local file or object storage.
|
204
|
+
validate(self) -> bool
|
205
|
+
Validates metadata item.
|
206
|
+
"""
|
207
|
+
|
208
|
+
_FIELDS = []
|
209
|
+
|
210
|
+
@classmethod
|
211
|
+
def from_dict(cls, data: Dict) -> "ModelMetadataItem":
|
212
|
+
"""Constructs an instance of `ModelMetadataItem` from a dictionary.
|
213
|
+
|
214
|
+
Parameters
|
215
|
+
----------
|
216
|
+
data : Dict
|
217
|
+
Metadata item in a dictionary format.
|
218
|
+
|
219
|
+
Returns
|
220
|
+
-------
|
221
|
+
ModelMetadataItem
|
222
|
+
An instance of model metadata item.
|
223
|
+
"""
|
224
|
+
return cls(**data or {})
|
225
|
+
|
226
|
+
def to_dict(self) -> dict:
|
227
|
+
"""Serializes model metadata item to dictionary.
|
228
|
+
|
229
|
+
Returns
|
230
|
+
-------
|
231
|
+
dict
|
232
|
+
The dictionary representation of model metadata item.
|
233
|
+
"""
|
234
|
+
return {field: getattr(self, field) for field in self._FIELDS}
|
235
|
+
|
236
|
+
def to_yaml(self):
|
237
|
+
"""Serializes model metadata item to YAML.
|
238
|
+
|
239
|
+
Returns
|
240
|
+
-------
|
241
|
+
Yaml
|
242
|
+
The model metadata item in a YAML representation.
|
243
|
+
"""
|
244
|
+
return yaml.dump(self.to_dict(), Dumper=dumper)
|
245
|
+
|
246
|
+
def size(self) -> int:
|
247
|
+
"""Returns the size of the model metadata in bytes.
|
248
|
+
|
249
|
+
Returns
|
250
|
+
-------
|
251
|
+
int
|
252
|
+
The size of model metadata in bytes.
|
253
|
+
"""
|
254
|
+
return len(json.dumps(self.to_dict()).encode("utf-16"))
|
255
|
+
|
256
|
+
def to_json(self):
|
257
|
+
"""Serializes metadata item into a JSON.
|
258
|
+
|
259
|
+
Returns
|
260
|
+
-------
|
261
|
+
JSON
|
262
|
+
The metadata item in a JSON representation.
|
263
|
+
"""
|
264
|
+
return json.dumps(self.to_dict())
|
265
|
+
|
266
|
+
def to_json_file(
|
267
|
+
self,
|
268
|
+
file_path: str,
|
269
|
+
storage_options: dict = None,
|
270
|
+
) -> None:
|
271
|
+
"""Saves the metadata item value to a local file or object storage.
|
272
|
+
|
273
|
+
Parameters
|
274
|
+
----------
|
275
|
+
file_path : str
|
276
|
+
The file path to store the data.
|
277
|
+
"oci://bucket_name@namespace/folder_name/"
|
278
|
+
"oci://bucket_name@namespace/folder_name/result.json"
|
279
|
+
"path/to/local/folder"
|
280
|
+
"path/to/local/folder/result.json"
|
281
|
+
storage_options : dict. Default None
|
282
|
+
Parameters passed on to the backend filesystem class.
|
283
|
+
Defaults to `options` set using `DatasetFactory.set_default_storage()`.
|
284
|
+
|
285
|
+
Returns
|
286
|
+
-------
|
287
|
+
None
|
288
|
+
Nothing.
|
289
|
+
|
290
|
+
Raises
|
291
|
+
------
|
292
|
+
ValueError: When file path is empty.
|
293
|
+
TypeError: When file path not a string.
|
294
|
+
|
295
|
+
Examples
|
296
|
+
--------
|
297
|
+
>>> metadata_item = ModelCustomMetadataItem(key="key1", value="value1")
|
298
|
+
>>> storage_options = {"config": oci.config.from_file(os.path.join("~/.oci", "config"))}
|
299
|
+
>>> storage_options
|
300
|
+
{'log_requests': False,
|
301
|
+
'additional_user_agent': '',
|
302
|
+
'pass_phrase': None,
|
303
|
+
'user': '<user-id>',
|
304
|
+
'fingerprint': '05:15:2b:b1:46:8a:32:ec:e2:69:5b:32:01:**:**:**)',
|
305
|
+
'tenancy': '<tenency-id>',
|
306
|
+
'region': 'us-ashburn-1',
|
307
|
+
'key_file': '/home/datascience/.oci/oci_api_key.pem'}
|
308
|
+
>>> metadata_item.to_json_file(file_path = 'oci://bucket_name@namespace/folder_name/file.json', storage_options=storage_options)
|
309
|
+
>>> metadata_item.to_json_file("path/to/local/folder/file.json")
|
310
|
+
"""
|
311
|
+
if not file_path:
|
312
|
+
raise ValueError("File path must be specified.")
|
313
|
+
|
314
|
+
if not isinstance(file_path, str):
|
315
|
+
raise TypeError("File path must be a string.")
|
316
|
+
|
317
|
+
if not Path(os.path.basename(file_path)).suffix:
|
318
|
+
file_path = os.path.join(file_path, f"{self.key}.json")
|
319
|
+
|
320
|
+
if not storage_options:
|
321
|
+
storage_options = factory.default_storage_options or {"config": {}}
|
322
|
+
|
323
|
+
with fsspec.open(
|
324
|
+
file_path,
|
325
|
+
mode="w",
|
326
|
+
**(storage_options),
|
327
|
+
) as f:
|
328
|
+
f.write(json.dumps(self.value))
|
329
|
+
|
330
|
+
def _to_oci_metadata(self):
|
331
|
+
"""Converts metadata item to OCI metadata item."""
|
332
|
+
dict = self.to_dict()
|
333
|
+
if not dict["value"]:
|
334
|
+
return oci.data_science.models.Metadata(**dict)
|
335
|
+
if isinstance(dict["value"], (str, int, float)):
|
336
|
+
dict["value"] = str(dict["value"]).replace("NaN", "null")
|
337
|
+
else:
|
338
|
+
dict["value"] = json.dumps(dict["value"]).replace("NaN", "null")
|
339
|
+
return oci.data_science.models.Metadata(**dict)
|
340
|
+
|
341
|
+
@classmethod
|
342
|
+
def _from_oci_metadata(cls, oci_metadata_item) -> "ModelMetadataItem":
|
343
|
+
"""Creates a new metadata item from the OCI metadata item."""
|
344
|
+
oci_metadata_item = to_dict(oci_metadata_item)
|
345
|
+
key_value_map = {field: oci_metadata_item.get(field) for field in cls._FIELDS}
|
346
|
+
if isinstance(key_value_map["value"], str):
|
347
|
+
try:
|
348
|
+
key_value_map["value"] = json.loads(oci_metadata_item.get("value"))
|
349
|
+
key_value_map["has_artifact"] = parse_bool(
|
350
|
+
oci_metadata_item.get("has_artifact")
|
351
|
+
)
|
352
|
+
except Exception:
|
353
|
+
pass
|
354
|
+
return cls(**key_value_map)
|
355
|
+
|
356
|
+
def __hash__(self):
|
357
|
+
return hash(self.key.lower())
|
358
|
+
|
359
|
+
def __eq__(self, other):
|
360
|
+
return hash(self) == hash(other)
|
361
|
+
|
362
|
+
def __repr__(self):
|
363
|
+
return self.to_yaml()
|
364
|
+
|
365
|
+
@abstractmethod
|
366
|
+
def validate(self) -> bool:
|
367
|
+
"""Validates metadata item.
|
368
|
+
|
369
|
+
Returns
|
370
|
+
-------
|
371
|
+
bool
|
372
|
+
True if validation passed.
|
373
|
+
"""
|
374
|
+
pass
|
375
|
+
|
376
|
+
|
377
|
+
class ModelTaxonomyMetadataItem(ModelMetadataItem):
|
378
|
+
"""Class that represents model taxonomy metadata item.
|
379
|
+
|
380
|
+
Attributes
|
381
|
+
----------
|
382
|
+
key: str
|
383
|
+
The model metadata item key.
|
384
|
+
value: str
|
385
|
+
The model metadata item value.
|
386
|
+
|
387
|
+
Methods
|
388
|
+
-------
|
389
|
+
reset(self) -> None
|
390
|
+
Resets model metadata item.
|
391
|
+
to_dict(self) -> Dict
|
392
|
+
Serializes model metadata item to dictionary.
|
393
|
+
from_dict(cls) -> ModelTaxonomyMetadataItem
|
394
|
+
Constructs model metadata item from dictionary.
|
395
|
+
to_yaml(self)
|
396
|
+
Serializes model metadata item to YAML.
|
397
|
+
size(self) -> int
|
398
|
+
Returns the size of the metadata in bytes.
|
399
|
+
update(self, value: str = "") -> None
|
400
|
+
Updates metadata item information.
|
401
|
+
to_json(self) -> JSON
|
402
|
+
Serializes metadata item into a JSON.
|
403
|
+
to_json_file(self, file_path: str, storage_options: dict = None) -> None
|
404
|
+
Saves the metadata item value to a local file or object storage.
|
405
|
+
validate(self) -> bool
|
406
|
+
Validates metadata item.
|
407
|
+
"""
|
408
|
+
|
409
|
+
_FIELDS = ["key", "value", "has_artifact"]
|
410
|
+
|
411
|
+
def __init__(self, key: str, value: str = None, has_artifact: bool = False):
|
412
|
+
self.key = key
|
413
|
+
self.value = value
|
414
|
+
self.has_artifact = has_artifact
|
415
|
+
|
416
|
+
@property
|
417
|
+
def key(self) -> str:
|
418
|
+
return self._key
|
419
|
+
|
420
|
+
@key.setter
|
421
|
+
def key(self, key: str):
|
422
|
+
"""The model metadata key setter.
|
423
|
+
|
424
|
+
Raises
|
425
|
+
------
|
426
|
+
TypeError
|
427
|
+
If provided key is not a string.
|
428
|
+
ValueError
|
429
|
+
If provided key is already setup.
|
430
|
+
If provided key is empty.
|
431
|
+
"""
|
432
|
+
if hasattr(self, "_key"):
|
433
|
+
raise ValueError("The key field is immutable and cannot be changed.")
|
434
|
+
if not isinstance(key, str):
|
435
|
+
raise TypeError("The key must be a string.")
|
436
|
+
if key is None or key == "":
|
437
|
+
raise ValueError("The key cannot be empty.")
|
438
|
+
self._key = key
|
439
|
+
|
440
|
+
@property
|
441
|
+
def has_artifact(self) -> bool:
|
442
|
+
return self._has_artifact
|
443
|
+
|
444
|
+
@has_artifact.setter
|
445
|
+
def has_artifact(self, has_artifact: bool):
|
446
|
+
self._has_artifact = has_artifact is True
|
447
|
+
|
448
|
+
@property
|
449
|
+
def value(self) -> str:
|
450
|
+
return self._value
|
451
|
+
|
452
|
+
@value.setter
|
453
|
+
def value(self, value: str):
|
454
|
+
"""The model metadata value setter. Accepts any JSON serializable value.
|
455
|
+
|
456
|
+
Raises
|
457
|
+
------
|
458
|
+
ValueError
|
459
|
+
If provided value cannot be serialized to JSON.
|
460
|
+
"""
|
461
|
+
if value is None or value == "":
|
462
|
+
self._value = value
|
463
|
+
return
|
464
|
+
|
465
|
+
try:
|
466
|
+
json.dumps(value)
|
467
|
+
except TypeError:
|
468
|
+
raise ValueError(
|
469
|
+
f"An error occurred in attempt to serialize the value of {self.key} to JSON. "
|
470
|
+
"The value must be JSON serializable."
|
471
|
+
)
|
472
|
+
|
473
|
+
self._value = value
|
474
|
+
|
475
|
+
def reset(self) -> None:
|
476
|
+
"""Resets model metadata item.
|
477
|
+
|
478
|
+
Resets value to None.
|
479
|
+
|
480
|
+
Returns
|
481
|
+
-------
|
482
|
+
None
|
483
|
+
Nothing.
|
484
|
+
"""
|
485
|
+
self.update(value=None)
|
486
|
+
|
487
|
+
def update(self, value: str, has_artifact: bool = False) -> None:
|
488
|
+
"""Updates metadata item value.
|
489
|
+
|
490
|
+
Parameters
|
491
|
+
----------
|
492
|
+
value: str
|
493
|
+
The value of model metadata item.
|
494
|
+
|
495
|
+
Returns
|
496
|
+
-------
|
497
|
+
None
|
498
|
+
Nothing.
|
499
|
+
"""
|
500
|
+
self.value = value
|
501
|
+
self.has_artifact = has_artifact
|
502
|
+
|
503
|
+
def validate(self) -> bool:
|
504
|
+
"""Validates metadata item.
|
505
|
+
|
506
|
+
Returns
|
507
|
+
-------
|
508
|
+
bool
|
509
|
+
True if validation passed.
|
510
|
+
|
511
|
+
Raises
|
512
|
+
------
|
513
|
+
ValueError
|
514
|
+
If invalid UseCaseType provided.
|
515
|
+
If invalid Framework provided.
|
516
|
+
"""
|
517
|
+
if (
|
518
|
+
self.key.lower() == MetadataTaxonomyKeys.USE_CASE_TYPE.lower()
|
519
|
+
and self.value
|
520
|
+
and (not isinstance(self.value, str) or self.value not in UseCaseType)
|
521
|
+
):
|
522
|
+
raise ValueError(
|
523
|
+
f"Invalid value of `UseCaseType`. Choose from {UseCaseType.values()}."
|
524
|
+
)
|
525
|
+
if (
|
526
|
+
self.key.lower() == MetadataTaxonomyKeys.FRAMEWORK.lower()
|
527
|
+
and self.value
|
528
|
+
and (not isinstance(self.value, str) or self.value not in Framework)
|
529
|
+
):
|
530
|
+
raise ValueError(
|
531
|
+
f"Invalid value of `Framework`. Choose from {Framework.values()}."
|
532
|
+
)
|
533
|
+
return True
|
534
|
+
|
535
|
+
|
536
|
+
class ModelCustomMetadataItem(ModelTaxonomyMetadataItem):
|
537
|
+
"""Class that represents model custom metadata item.
|
538
|
+
|
539
|
+
Attributes
|
540
|
+
----------
|
541
|
+
key: str
|
542
|
+
The model metadata item key.
|
543
|
+
value: str
|
544
|
+
The model metadata item value.
|
545
|
+
description: str
|
546
|
+
The model metadata item description.
|
547
|
+
category: str
|
548
|
+
The model metadata item category.
|
549
|
+
|
550
|
+
Methods
|
551
|
+
-------
|
552
|
+
reset(self) -> None
|
553
|
+
Resets model metadata item.
|
554
|
+
to_dict(self)->dict
|
555
|
+
Serializes model metadata item to dictionary.
|
556
|
+
from_dict(cls) -> ModelCustomMetadataItem
|
557
|
+
Constructs model metadata item from dictionary.
|
558
|
+
to_yaml(self)
|
559
|
+
Serializes model metadata item to YAML.
|
560
|
+
size(self) -> int
|
561
|
+
Returns the size of the metadata in bytes.
|
562
|
+
update(self, value: str = "", description: str = "", category: str = "") -> None
|
563
|
+
Updates metadata item information.
|
564
|
+
to_json(self) -> JSON
|
565
|
+
Serializes metadata item into a JSON.
|
566
|
+
to_json_file(self, file_path: str, storage_options: dict = None) -> None
|
567
|
+
Saves the metadata item value to a local file or object storage.
|
568
|
+
validate(self) -> bool
|
569
|
+
Validates metadata item.
|
570
|
+
"""
|
571
|
+
|
572
|
+
_FIELDS = ["key", "value", "description", "category", "has_artifact"]
|
573
|
+
|
574
|
+
def __init__(
|
575
|
+
self,
|
576
|
+
key: str,
|
577
|
+
value: str = None,
|
578
|
+
description: str = None,
|
579
|
+
category: str = None,
|
580
|
+
has_artifact: bool = False,
|
581
|
+
):
|
582
|
+
super().__init__(key=key, value=value)
|
583
|
+
self.description = description
|
584
|
+
self.category = category
|
585
|
+
self.has_artifact = has_artifact
|
586
|
+
|
587
|
+
@property
|
588
|
+
def description(self) -> str:
|
589
|
+
return self._description
|
590
|
+
|
591
|
+
@description.setter
|
592
|
+
def description(self, description: str):
|
593
|
+
"""The model metadata description setter.
|
594
|
+
|
595
|
+
Raises
|
596
|
+
------
|
597
|
+
TypeError
|
598
|
+
If provided key is not a string.
|
599
|
+
"""
|
600
|
+
if description != None and not isinstance(description, str):
|
601
|
+
raise TypeError("The description must be a string.")
|
602
|
+
|
603
|
+
self._description = description
|
604
|
+
|
605
|
+
@property
|
606
|
+
def has_artifact(self) -> bool:
|
607
|
+
return self._has_artifact
|
608
|
+
|
609
|
+
@has_artifact.setter
|
610
|
+
def has_artifact(self, has_artifact: bool):
|
611
|
+
if not has_artifact:
|
612
|
+
self._has_artifact = False
|
613
|
+
else:
|
614
|
+
self._has_artifact = has_artifact
|
615
|
+
|
616
|
+
@property
|
617
|
+
def category(self) -> str:
|
618
|
+
return self._category
|
619
|
+
|
620
|
+
@category.setter
|
621
|
+
def category(self, category: str):
|
622
|
+
"""The model metadata category setter.
|
623
|
+
|
624
|
+
Raises
|
625
|
+
------
|
626
|
+
TypeError
|
627
|
+
If provided category is not a string.
|
628
|
+
ValueError
|
629
|
+
If provided category not supported.
|
630
|
+
"""
|
631
|
+
if not category:
|
632
|
+
self._category = None
|
633
|
+
return
|
634
|
+
|
635
|
+
if not isinstance(category, str):
|
636
|
+
raise TypeError(
|
637
|
+
f"Invalid category type for the {self.key}."
|
638
|
+
"The category must be a string."
|
639
|
+
)
|
640
|
+
|
641
|
+
if category not in MetadataCustomCategory:
|
642
|
+
raise ValueError(
|
643
|
+
f"Invalid category value for the {self.key}. "
|
644
|
+
f"Choose from {MetadataCustomCategory.values()}."
|
645
|
+
)
|
646
|
+
|
647
|
+
self._category = category
|
648
|
+
|
649
|
+
def reset(self) -> None:
|
650
|
+
"""Resets model metadata item.
|
651
|
+
|
652
|
+
Resets value, description and category to None.
|
653
|
+
|
654
|
+
Returns
|
655
|
+
-------
|
656
|
+
None
|
657
|
+
Nothing.
|
658
|
+
"""
|
659
|
+
self.update(value=None, description=None, category=None)
|
660
|
+
|
661
|
+
def update(
|
662
|
+
self, value: str, description: str, category: str, has_artifact: bool = False
|
663
|
+
) -> None:
|
664
|
+
"""Updates metadata item.
|
665
|
+
|
666
|
+
Parameters
|
667
|
+
----------
|
668
|
+
value: str
|
669
|
+
The value of model metadata item.
|
670
|
+
description: str
|
671
|
+
The description of model metadata item.
|
672
|
+
category: str
|
673
|
+
The category of model metadata item.
|
674
|
+
|
675
|
+
Returns
|
676
|
+
-------
|
677
|
+
None
|
678
|
+
Nothing.
|
679
|
+
"""
|
680
|
+
self.value = value
|
681
|
+
self.description = description
|
682
|
+
self.category = category
|
683
|
+
self.has_artifact = has_artifact
|
684
|
+
|
685
|
+
def _to_oci_metadata(self):
|
686
|
+
"""Converts metadata item to OCI metadata item."""
|
687
|
+
oci_metadata_item = super()._to_oci_metadata()
|
688
|
+
if not oci_metadata_item.value:
|
689
|
+
oci_metadata_item.value = _METADATA_EMPTY_VALUE
|
690
|
+
if not oci_metadata_item.category:
|
691
|
+
oci_metadata_item.category = MetadataCustomCategory.OTHER
|
692
|
+
if not oci_metadata_item.has_artifact:
|
693
|
+
oci_metadata_item.has_artifact = False
|
694
|
+
return oci_metadata_item
|
695
|
+
|
696
|
+
def validate(self) -> bool:
|
697
|
+
"""Validates metadata item.
|
698
|
+
|
699
|
+
Returns
|
700
|
+
-------
|
701
|
+
bool
|
702
|
+
True if validation passed.
|
703
|
+
|
704
|
+
Raises
|
705
|
+
------
|
706
|
+
ValueError
|
707
|
+
If invalid category provided.
|
708
|
+
MetadataValueTooLong
|
709
|
+
If value exceeds the length limit.
|
710
|
+
"""
|
711
|
+
if self.category and self.category not in MetadataCustomCategory:
|
712
|
+
raise ValueError(
|
713
|
+
f"Invalid category value for the {self.key}. "
|
714
|
+
f"Choose from {MetadataCustomCategory.values()}."
|
715
|
+
)
|
716
|
+
|
717
|
+
if self.value:
|
718
|
+
value = (
|
719
|
+
self.value if isinstance(self.value, str) else json.dumps(self.value)
|
720
|
+
)
|
721
|
+
if len(value) > METADATA_VALUE_LENGTH_LIMIT:
|
722
|
+
raise MetadataValueTooLong(self.key, len(value))
|
723
|
+
|
724
|
+
if (
|
725
|
+
self.description
|
726
|
+
and len(self.description) > METADATA_DESCRIPTION_LENGTH_LIMIT
|
727
|
+
):
|
728
|
+
raise MetadataDescriptionTooLong(self.key, len(self.description))
|
729
|
+
|
730
|
+
return True
|
731
|
+
|
732
|
+
|
733
|
+
class ModelMetadata(ABC):
|
734
|
+
"""The base abstract class representing model metadata.
|
735
|
+
|
736
|
+
Methods
|
737
|
+
-------
|
738
|
+
get(self, key: str) -> ModelMetadataItem
|
739
|
+
Returns the model metadata item by provided key.
|
740
|
+
reset(self) -> None
|
741
|
+
Resets all model metadata items to empty values.
|
742
|
+
to_dataframe(self) -> pd.DataFrame
|
743
|
+
Returns the model metadata list in a data frame format.
|
744
|
+
size(self) -> int
|
745
|
+
Returns the size of the model metadata in bytes.
|
746
|
+
validate(self) -> bool
|
747
|
+
Validates metadata.
|
748
|
+
to_dict(self)
|
749
|
+
Serializes model metadata into a dictionary.
|
750
|
+
from_dict(cls) -> ModelMetadata
|
751
|
+
Constructs model metadata from dictionary.
|
752
|
+
to_yaml(self)
|
753
|
+
Serializes model metadata into a YAML.
|
754
|
+
to_json(self)
|
755
|
+
Serializes model metadata into a JSON.
|
756
|
+
to_json_file(self, file_path: str, storage_options: dict = None) -> None
|
757
|
+
Saves the metadata to a local file or object storage.
|
758
|
+
"""
|
759
|
+
|
760
|
+
@abstractmethod
|
761
|
+
def __init__(self):
|
762
|
+
"""Initializes Model Metadata."""
|
763
|
+
self._items = set()
|
764
|
+
|
765
|
+
def get(
|
766
|
+
self, key: str, value: Optional[Any] = _sentinel
|
767
|
+
) -> Union[ModelMetadataItem, Any]:
|
768
|
+
"""Returns the model metadata item by provided key.
|
769
|
+
|
770
|
+
Parameters
|
771
|
+
----------
|
772
|
+
key: str
|
773
|
+
The key of model metadata item.
|
774
|
+
value: (str, optional)
|
775
|
+
A value to return if the specified key does not exist. Defaults to `object()`.
|
776
|
+
If default value not specified, the ValueError will be returned.
|
777
|
+
|
778
|
+
Returns
|
779
|
+
-------
|
780
|
+
ModelMetadataItem
|
781
|
+
The model metadata item.
|
782
|
+
|
783
|
+
Raises
|
784
|
+
------
|
785
|
+
ValueError
|
786
|
+
If provided key is empty or metadata item not found.
|
787
|
+
"""
|
788
|
+
if key is None or not isinstance(key, str) or key == "":
|
789
|
+
raise ValueError("The key must not be an empty string.")
|
790
|
+
for item in self._items:
|
791
|
+
if item.key.lower() == key.lower():
|
792
|
+
return item
|
793
|
+
|
794
|
+
if value is _sentinel:
|
795
|
+
raise ValueError(f"The metadata with {key} not found.")
|
796
|
+
|
797
|
+
return value
|
798
|
+
|
799
|
+
def reset(self) -> None:
|
800
|
+
"""Resets all model metadata items to empty values.
|
801
|
+
|
802
|
+
Resets value, description and category to None for every metadata item.
|
803
|
+
"""
|
804
|
+
for item in self._items:
|
805
|
+
item.reset()
|
806
|
+
|
807
|
+
def size(self) -> int:
|
808
|
+
"""Returns the size of the model metadata in bytes.
|
809
|
+
|
810
|
+
Returns
|
811
|
+
-------
|
812
|
+
int
|
813
|
+
The size of model metadata in bytes.
|
814
|
+
"""
|
815
|
+
return sum(item.size() for item in self._items)
|
816
|
+
|
817
|
+
def validate_size(self) -> bool:
|
818
|
+
"""Validates model metadata size.
|
819
|
+
|
820
|
+
Validates the size of metadata. Throws an error if the size of the metadata
|
821
|
+
exceeds expected value.
|
822
|
+
|
823
|
+
Returns
|
824
|
+
-------
|
825
|
+
bool
|
826
|
+
True if metadata size is valid.
|
827
|
+
|
828
|
+
Raises
|
829
|
+
------
|
830
|
+
MetadataSizeTooLarge
|
831
|
+
If the size of the metadata exceeds expected value.
|
832
|
+
"""
|
833
|
+
if self.size() > METADATA_SIZE_LIMIT:
|
834
|
+
raise MetadataSizeTooLarge(self.size())
|
835
|
+
return True
|
836
|
+
|
837
|
+
def validate(self) -> bool:
|
838
|
+
"""Validates model metadata.
|
839
|
+
|
840
|
+
Returns
|
841
|
+
-------
|
842
|
+
bool
|
843
|
+
True if metadata is valid.
|
844
|
+
"""
|
845
|
+
for item in self._items:
|
846
|
+
item.validate()
|
847
|
+
return True
|
848
|
+
|
849
|
+
def to_dict(self):
|
850
|
+
"""Serializes model metadata into a dictionary.
|
851
|
+
|
852
|
+
Returns
|
853
|
+
-------
|
854
|
+
Dict
|
855
|
+
The model metadata in a dictionary representation.
|
856
|
+
"""
|
857
|
+
return {"data": [item.to_dict() for item in self._items]}
|
858
|
+
|
859
|
+
def to_yaml(self):
|
860
|
+
"""Serializes model metadata into a YAML.
|
861
|
+
|
862
|
+
Returns
|
863
|
+
-------
|
864
|
+
Yaml
|
865
|
+
The model metadata in a YAML representation.
|
866
|
+
"""
|
867
|
+
return yaml.dump(self.to_dict(), Dumper=dumper)
|
868
|
+
|
869
|
+
def to_json(self):
|
870
|
+
"""Serializes model metadata into a JSON.
|
871
|
+
|
872
|
+
Returns
|
873
|
+
-------
|
874
|
+
JSON
|
875
|
+
The model metadata in a JSON representation.
|
876
|
+
"""
|
877
|
+
return json.dumps(self.to_dict())
|
878
|
+
|
879
|
+
@property
|
880
|
+
def keys(self) -> Tuple[str]:
|
881
|
+
"""Returns all registered metadata keys.
|
882
|
+
|
883
|
+
Returns
|
884
|
+
-------
|
885
|
+
Tuple[str]
|
886
|
+
The list of metadata keys.
|
887
|
+
"""
|
888
|
+
return tuple(item.key for item in self._items)
|
889
|
+
|
890
|
+
def _to_oci_metadata(self):
|
891
|
+
"""Convert to a list of `oci.data_science.models.Metadata` objects.
|
892
|
+
|
893
|
+
Returns
|
894
|
+
-------
|
895
|
+
list[oci.data_science.models.Metadata]
|
896
|
+
A list of oci data science model metadata.
|
897
|
+
|
898
|
+
Examples
|
899
|
+
--------
|
900
|
+
>>> metadata_taxonomy = ModelTaxonomyMetadata()
|
901
|
+
>>> metadata_taxonomy.get(key="FrameworkVersion").update(value="2.3.1")
|
902
|
+
>>> metadata_taxonomy._to_oci_metadata()
|
903
|
+
[{
|
904
|
+
"key": "FrameworkVersion",
|
905
|
+
"value": "2.3.1"
|
906
|
+
},
|
907
|
+
{
|
908
|
+
"key": "UseCaseType",
|
909
|
+
"value": null
|
910
|
+
},
|
911
|
+
{
|
912
|
+
"key": "Algorithm",
|
913
|
+
"value": null
|
914
|
+
},
|
915
|
+
{
|
916
|
+
"key": "Framework",
|
917
|
+
"value": null
|
918
|
+
},
|
919
|
+
{
|
920
|
+
"key": "Hyperparameters",
|
921
|
+
"value": null
|
922
|
+
}]
|
923
|
+
"""
|
924
|
+
return [item._to_oci_metadata() for item in self._items]
|
925
|
+
|
926
|
+
def to_json_file(
|
927
|
+
self,
|
928
|
+
file_path: str,
|
929
|
+
storage_options: dict = None,
|
930
|
+
) -> None:
|
931
|
+
"""Saves the metadata to a local file or object storage.
|
932
|
+
|
933
|
+
Parameters
|
934
|
+
----------
|
935
|
+
file_path : str
|
936
|
+
The file path to store the data.
|
937
|
+
"oci://bucket_name@namespace/folder_name/"
|
938
|
+
"oci://bucket_name@namespace/folder_name/metadata.json"
|
939
|
+
"path/to/local/folder"
|
940
|
+
"path/to/local/folder/metadata.json"
|
941
|
+
storage_options : dict. Default None
|
942
|
+
Parameters passed on to the backend filesystem class.
|
943
|
+
Defaults to `options` set using `DatasetFactory.set_default_storage()`.
|
944
|
+
|
945
|
+
Returns
|
946
|
+
-------
|
947
|
+
None
|
948
|
+
Nothing.
|
949
|
+
|
950
|
+
Raises
|
951
|
+
------
|
952
|
+
ValueError: When file path is empty.
|
953
|
+
TypeError: When file path not a string.
|
954
|
+
|
955
|
+
Examples
|
956
|
+
--------
|
957
|
+
>>> metadata = ModelTaxonomyMetadataItem()
|
958
|
+
>>> storage_options = {"config": oci.config.from_file(os.path.join("~/.oci", "config"))}
|
959
|
+
>>> storage_options
|
960
|
+
{'log_requests': False,
|
961
|
+
'additional_user_agent': '',
|
962
|
+
'pass_phrase': None,
|
963
|
+
'user': '<user-id>',
|
964
|
+
'fingerprint': '05:15:2b:b1:46:8a:32:ec:e2:69:5b:32:01:**:**:**)',
|
965
|
+
'tenancy': '<tenancy-id>',
|
966
|
+
'region': 'us-ashburn-1',
|
967
|
+
'key_file': '/home/datascience/.oci/oci_api_key.pem'}
|
968
|
+
>>> metadata.to_json_file(file_path = 'oci://bucket_name@namespace/folder_name/metadata_taxonomy.json', storage_options=storage_options)
|
969
|
+
>>> metadata_item.to_json_file("path/to/local/folder/metadata_taxonomy.json")
|
970
|
+
"""
|
971
|
+
if not file_path:
|
972
|
+
raise ValueError("File path must be specified.")
|
973
|
+
|
974
|
+
if not isinstance(file_path, str):
|
975
|
+
raise TypeError("File path must be a string.")
|
976
|
+
|
977
|
+
if not Path(os.path.basename(file_path)).suffix:
|
978
|
+
file_path = os.path.join(file_path, f"{self.__class__.__name__}.json")
|
979
|
+
|
980
|
+
if not storage_options:
|
981
|
+
storage_options = factory.default_storage_options or {"config": {}}
|
982
|
+
|
983
|
+
with fsspec.open(
|
984
|
+
file_path,
|
985
|
+
mode="w",
|
986
|
+
**(storage_options),
|
987
|
+
) as f:
|
988
|
+
f.write(self.to_json())
|
989
|
+
|
990
|
+
def __getitem__(self, key: str) -> ModelMetadataItem:
|
991
|
+
return self.get(key)
|
992
|
+
|
993
|
+
def __repr__(self):
|
994
|
+
return self.to_yaml()
|
995
|
+
|
996
|
+
def __len__(self):
|
997
|
+
return len(self._items)
|
998
|
+
|
999
|
+
@abstractmethod
|
1000
|
+
def _from_oci_metadata(cls, metadata_list):
|
1001
|
+
pass
|
1002
|
+
|
1003
|
+
@abstractmethod
|
1004
|
+
def to_dataframe(self) -> pd.DataFrame:
|
1005
|
+
"""Returns the model metadata list in a data frame format.
|
1006
|
+
|
1007
|
+
Returns
|
1008
|
+
-------
|
1009
|
+
`pandas.DataFrame`
|
1010
|
+
The model metadata in a dataframe format.
|
1011
|
+
"""
|
1012
|
+
pass
|
1013
|
+
|
1014
|
+
@abstractmethod
|
1015
|
+
def from_dict(cls, data: Dict) -> "ModelMetadata":
|
1016
|
+
"""Constructs an instance of `ModelMetadata` from a dictionary.
|
1017
|
+
|
1018
|
+
Parameters
|
1019
|
+
----------
|
1020
|
+
data : Dict
|
1021
|
+
Model metadata in a dictionary format.
|
1022
|
+
|
1023
|
+
Returns
|
1024
|
+
-------
|
1025
|
+
ModelMetadata
|
1026
|
+
An instance of model metadata.
|
1027
|
+
"""
|
1028
|
+
pass
|
1029
|
+
|
1030
|
+
|
1031
|
+
class ModelCustomMetadata(ModelMetadata):
|
1032
|
+
"""Class that represents Model Custom Metadata.
|
1033
|
+
|
1034
|
+
Methods
|
1035
|
+
-------
|
1036
|
+
get(self, key: str) -> ModelCustomMetadataItem
|
1037
|
+
Returns the model metadata item by provided key.
|
1038
|
+
reset(self) -> None
|
1039
|
+
Resets all model metadata items to empty values.
|
1040
|
+
to_dataframe(self) -> pd.DataFrame
|
1041
|
+
Returns the model metadata list in a data frame format.
|
1042
|
+
size(self) -> int
|
1043
|
+
Returns the size of the model metadata in bytes.
|
1044
|
+
validate(self) -> bool
|
1045
|
+
Validates metadata.
|
1046
|
+
to_dict(self)
|
1047
|
+
Serializes model metadata into a dictionary.
|
1048
|
+
from_dict(cls) -> ModelCustomMetadata
|
1049
|
+
Constructs model metadata from dictionary.
|
1050
|
+
to_yaml(self)
|
1051
|
+
Serializes model metadata into a YAML.
|
1052
|
+
add(self, key: str, value: str, description: str = "", category: str = MetadataCustomCategory.OTHER, replace: bool = False) -> None:
|
1053
|
+
Adds a new model metadata item. Replaces existing one if replace flag is True.
|
1054
|
+
remove(self, key: str) -> None
|
1055
|
+
Removes a model metadata item by key.
|
1056
|
+
clear(self) -> None
|
1057
|
+
Removes all metadata items.
|
1058
|
+
isempty(self) -> bool
|
1059
|
+
Checks if metadata is empty.
|
1060
|
+
to_json(self)
|
1061
|
+
Serializes model metadata into a JSON.
|
1062
|
+
to_json_file(self, file_path: str, storage_options: dict = None) -> None
|
1063
|
+
Saves the metadata to a local file or object storage.
|
1064
|
+
|
1065
|
+
Examples
|
1066
|
+
--------
|
1067
|
+
>>> metadata_custom = ModelCustomMetadata()
|
1068
|
+
>>> metadata_custom.add(key="format", value="pickle")
|
1069
|
+
>>> metadata_custom.add(key="note", value="important note", description="some description")
|
1070
|
+
>>> metadata_custom["format"].description = "some description"
|
1071
|
+
>>> metadata_custom.to_dataframe()
|
1072
|
+
Key Value Description Category
|
1073
|
+
----------------------------------------------------------------------------
|
1074
|
+
0 format pickle some description user defined
|
1075
|
+
1 note important note some description user defined
|
1076
|
+
>>> metadata_custom
|
1077
|
+
metadata:
|
1078
|
+
- category: user defined
|
1079
|
+
description: some description
|
1080
|
+
key: format
|
1081
|
+
value: pickle
|
1082
|
+
- category: user defined
|
1083
|
+
description: some description
|
1084
|
+
key: note
|
1085
|
+
value: important note
|
1086
|
+
>>> metadata_custom.remove("format")
|
1087
|
+
>>> metadata_custom
|
1088
|
+
metadata:
|
1089
|
+
- category: user defined
|
1090
|
+
description: some description
|
1091
|
+
key: note
|
1092
|
+
value: important note
|
1093
|
+
>>> metadata_custom.to_dict()
|
1094
|
+
{'metadata': [{
|
1095
|
+
'key': 'note',
|
1096
|
+
'value': 'important note',
|
1097
|
+
'category': 'user defined',
|
1098
|
+
'description': 'some description'
|
1099
|
+
}]}
|
1100
|
+
>>> metadata_custom.reset()
|
1101
|
+
>>> metadata_custom
|
1102
|
+
metadata:
|
1103
|
+
- category: None
|
1104
|
+
description: None
|
1105
|
+
key: note
|
1106
|
+
value: None
|
1107
|
+
>>> metadata_custom.clear()
|
1108
|
+
>>> metadata_custom.to_dataframe()
|
1109
|
+
Key Value Description Category
|
1110
|
+
----------------------------------------------------------------------------
|
1111
|
+
"""
|
1112
|
+
|
1113
|
+
def __init__(self):
|
1114
|
+
"""Initializes custom model metadata."""
|
1115
|
+
self._items = set()
|
1116
|
+
|
1117
|
+
def add(
|
1118
|
+
self,
|
1119
|
+
key: str,
|
1120
|
+
value: str,
|
1121
|
+
description: str = "",
|
1122
|
+
category: str = MetadataCustomCategory.OTHER,
|
1123
|
+
replace: bool = False,
|
1124
|
+
) -> None:
|
1125
|
+
"""Adds a new model metadata item. Overrides the existing one if replace flag is True.
|
1126
|
+
|
1127
|
+
Parameters
|
1128
|
+
----------
|
1129
|
+
key: str
|
1130
|
+
The metadata item key.
|
1131
|
+
value: str
|
1132
|
+
The metadata item value.
|
1133
|
+
description: str
|
1134
|
+
The metadata item description.
|
1135
|
+
category: str
|
1136
|
+
The metadata item category.
|
1137
|
+
replace: bool
|
1138
|
+
Overrides the existing metadata item if replace flag is True.
|
1139
|
+
|
1140
|
+
Returns
|
1141
|
+
-------
|
1142
|
+
None
|
1143
|
+
Nothing.
|
1144
|
+
|
1145
|
+
Raises
|
1146
|
+
------
|
1147
|
+
TypeError
|
1148
|
+
If provided key is not a string.
|
1149
|
+
If provided description not a string.
|
1150
|
+
ValueError
|
1151
|
+
If provided key is empty.
|
1152
|
+
If provided value is empty.
|
1153
|
+
If provided value cannot be serialized to JSON.
|
1154
|
+
If item with provided key is already registered and replace flag is False.
|
1155
|
+
If provided category is not supported.
|
1156
|
+
MetadataValueTooLong
|
1157
|
+
If the length of provided value exceeds 255 charracters.
|
1158
|
+
MetadataDescriptionTooLong
|
1159
|
+
If the length of provided description exceeds 255 charracters.
|
1160
|
+
"""
|
1161
|
+
|
1162
|
+
if not category:
|
1163
|
+
category = MetadataCustomCategory.OTHER
|
1164
|
+
|
1165
|
+
if key is None or key == "":
|
1166
|
+
raise ValueError("The key cannot be empty.")
|
1167
|
+
|
1168
|
+
if value is None or value == "":
|
1169
|
+
raise ValueError("The value cannot be empty.")
|
1170
|
+
|
1171
|
+
if not isinstance(key, str):
|
1172
|
+
raise TypeError("The key must be a string.")
|
1173
|
+
|
1174
|
+
if not isinstance(category, str):
|
1175
|
+
raise TypeError("The category must be a string.")
|
1176
|
+
|
1177
|
+
if category not in MetadataCustomCategory:
|
1178
|
+
raise ValueError(
|
1179
|
+
f"Invalid category value. "
|
1180
|
+
f"Choose from {MetadataCustomCategory.values()}."
|
1181
|
+
)
|
1182
|
+
|
1183
|
+
if description and not isinstance(description, str):
|
1184
|
+
raise TypeError("The description must be a string.")
|
1185
|
+
|
1186
|
+
try:
|
1187
|
+
tmp_value = json.dumps(value)
|
1188
|
+
except TypeError:
|
1189
|
+
raise ValueError(
|
1190
|
+
f"An error occurred in attempt to serialize the value of `{key}` to JSON. "
|
1191
|
+
"The value must be JSON serializable."
|
1192
|
+
)
|
1193
|
+
|
1194
|
+
if len(tmp_value) > METADATA_VALUE_LENGTH_LIMIT:
|
1195
|
+
raise MetadataValueTooLong(key, len(tmp_value))
|
1196
|
+
|
1197
|
+
if description and len(description) > METADATA_DESCRIPTION_LENGTH_LIMIT:
|
1198
|
+
raise MetadataDescriptionTooLong(key, len(description))
|
1199
|
+
|
1200
|
+
if not replace and key in self.keys:
|
1201
|
+
raise ValueError(
|
1202
|
+
f"The metadata item with key {key} is already registered. "
|
1203
|
+
"Use replace=True to overwrite."
|
1204
|
+
)
|
1205
|
+
|
1206
|
+
self._add(
|
1207
|
+
ModelCustomMetadataItem(
|
1208
|
+
key=key,
|
1209
|
+
value=value,
|
1210
|
+
description=description,
|
1211
|
+
category=category,
|
1212
|
+
),
|
1213
|
+
replace=replace,
|
1214
|
+
)
|
1215
|
+
|
1216
|
+
def _add(self, item: ModelCustomMetadataItem, replace=False) -> None:
|
1217
|
+
"""Adds a new model metadata item.
|
1218
|
+
|
1219
|
+
Overrides the existing one if replace flag is True.
|
1220
|
+
|
1221
|
+
Parameters
|
1222
|
+
----------
|
1223
|
+
item: ModelCustomMetadataItem
|
1224
|
+
The model metadata item.
|
1225
|
+
replace: bool
|
1226
|
+
Overrides the existing metadata item if replace flag is True.
|
1227
|
+
|
1228
|
+
Returns
|
1229
|
+
-------
|
1230
|
+
None
|
1231
|
+
Nothing.
|
1232
|
+
|
1233
|
+
Raises
|
1234
|
+
------
|
1235
|
+
ValueError
|
1236
|
+
If item is already registered and replace flag is False.
|
1237
|
+
TypeError
|
1238
|
+
If input data has a wrong format.
|
1239
|
+
"""
|
1240
|
+
if not isinstance(item, ModelCustomMetadataItem):
|
1241
|
+
raise TypeError(
|
1242
|
+
"Argument must be an instance of the class ModelCustomMetadataItem."
|
1243
|
+
)
|
1244
|
+
if not replace and item in self._items:
|
1245
|
+
raise ValueError(
|
1246
|
+
f"The metadata item with key {item.key} is already registered. "
|
1247
|
+
"Use replace=True to overwrite."
|
1248
|
+
)
|
1249
|
+
self._items.discard(item)
|
1250
|
+
self._items.add(item)
|
1251
|
+
|
1252
|
+
def _add_many(self, items: List[ModelCustomMetadataItem], replace=False) -> None:
|
1253
|
+
"""Adds model metadata items into model metadata.
|
1254
|
+
|
1255
|
+
Overrides the existing ones if replace flag is True.
|
1256
|
+
|
1257
|
+
Parameters
|
1258
|
+
----------
|
1259
|
+
items: List[ModelCustomMetadataItem]
|
1260
|
+
The list of model metadata items.
|
1261
|
+
replace: bool
|
1262
|
+
Overrides the existing metadata items if replace flag is True.
|
1263
|
+
|
1264
|
+
Returns
|
1265
|
+
-------
|
1266
|
+
None
|
1267
|
+
Nothing.
|
1268
|
+
|
1269
|
+
Raises
|
1270
|
+
------
|
1271
|
+
TypeError
|
1272
|
+
If input data has wrong format.
|
1273
|
+
"""
|
1274
|
+
if not isinstance(items, list):
|
1275
|
+
raise TypeError("Argument must be a list of model metadata items.")
|
1276
|
+
for item in items:
|
1277
|
+
self._add(item, replace)
|
1278
|
+
|
1279
|
+
def set_training_data(self, path: str, data_size: str = None):
|
1280
|
+
"""Adds training_data path and data size information into model custom metadata.
|
1281
|
+
|
1282
|
+
Parameters
|
1283
|
+
----------
|
1284
|
+
path: str
|
1285
|
+
The path where the training_data is stored.
|
1286
|
+
data_size: str
|
1287
|
+
The size of the training_data.
|
1288
|
+
|
1289
|
+
Returns
|
1290
|
+
-------
|
1291
|
+
None
|
1292
|
+
Nothing.
|
1293
|
+
"""
|
1294
|
+
self.add(
|
1295
|
+
key=MetadataCustomKeys.TRAINING_DATASET,
|
1296
|
+
value=path,
|
1297
|
+
category=MetadataCustomCategory.TRAINING_AND_VALIDATION_DATASETS,
|
1298
|
+
description="The path where training dataset path are stored.",
|
1299
|
+
replace=True,
|
1300
|
+
)
|
1301
|
+
if data_size is not None:
|
1302
|
+
self.add(
|
1303
|
+
key=MetadataCustomKeys.TRAINING_DATASET_SIZE,
|
1304
|
+
value=data_size,
|
1305
|
+
category=MetadataCustomCategory.TRAINING_AND_VALIDATION_DATASETS,
|
1306
|
+
description="The size of the training data.",
|
1307
|
+
replace=True,
|
1308
|
+
)
|
1309
|
+
|
1310
|
+
def set_validation_data(self, path: str, data_size: str = None):
|
1311
|
+
"""Adds validation_data path and data size information into model custom metadata.
|
1312
|
+
|
1313
|
+
Parameters
|
1314
|
+
----------
|
1315
|
+
path: str
|
1316
|
+
The path where the validation_data is stored.
|
1317
|
+
data_size: str
|
1318
|
+
The size of the validation_data.
|
1319
|
+
|
1320
|
+
Returns
|
1321
|
+
-------
|
1322
|
+
None
|
1323
|
+
Nothing.
|
1324
|
+
"""
|
1325
|
+
self.add(
|
1326
|
+
key=MetadataCustomKeys.VALIDATION_DATASET,
|
1327
|
+
value=path,
|
1328
|
+
category=MetadataCustomCategory.TRAINING_AND_VALIDATION_DATASETS,
|
1329
|
+
description="The path where validation dataset path are stored.",
|
1330
|
+
replace=True,
|
1331
|
+
)
|
1332
|
+
if data_size is not None:
|
1333
|
+
self.add(
|
1334
|
+
key=MetadataCustomKeys.VALIDATION_DATASET_SIZE,
|
1335
|
+
value=data_size,
|
1336
|
+
category=MetadataCustomCategory.TRAINING_AND_VALIDATION_DATASETS,
|
1337
|
+
description="The size of the validation data.",
|
1338
|
+
replace=True,
|
1339
|
+
)
|
1340
|
+
|
1341
|
+
def remove(self, key: str) -> None:
|
1342
|
+
"""Removes a model metadata item.
|
1343
|
+
|
1344
|
+
Parameters
|
1345
|
+
----------
|
1346
|
+
key: str
|
1347
|
+
The key of the metadata item that should be removed.
|
1348
|
+
|
1349
|
+
Returns
|
1350
|
+
-------
|
1351
|
+
None
|
1352
|
+
Nothing.
|
1353
|
+
"""
|
1354
|
+
self._items.discard(self.get(key))
|
1355
|
+
|
1356
|
+
def clear(self) -> None:
|
1357
|
+
"""Removes all metadata items.
|
1358
|
+
|
1359
|
+
Returns
|
1360
|
+
-------
|
1361
|
+
None
|
1362
|
+
Nothing.
|
1363
|
+
"""
|
1364
|
+
self._items.clear()
|
1365
|
+
|
1366
|
+
def isempty(self) -> bool:
|
1367
|
+
"""Checks if metadata is empty.
|
1368
|
+
|
1369
|
+
Returns
|
1370
|
+
-------
|
1371
|
+
bool
|
1372
|
+
True if metadata is empty, False otherwise.
|
1373
|
+
"""
|
1374
|
+
return len(self._items) == 0
|
1375
|
+
|
1376
|
+
@classmethod
|
1377
|
+
def _from_oci_metadata(cls, metadata_list):
|
1378
|
+
"""Convert from list of OCI metadata list to an ModelCustomMetadata object.
|
1379
|
+
|
1380
|
+
Returns
|
1381
|
+
-------
|
1382
|
+
ModelCustomMetadata
|
1383
|
+
A `ModelCustomMetadata` instance.
|
1384
|
+
"""
|
1385
|
+
metadata = cls()
|
1386
|
+
for item in metadata_list:
|
1387
|
+
metadata._add(
|
1388
|
+
ModelCustomMetadataItem._from_oci_metadata(item), replace=True
|
1389
|
+
)
|
1390
|
+
return metadata
|
1391
|
+
|
1392
|
+
def to_dataframe(self) -> pd.DataFrame:
|
1393
|
+
"""Returns the model metadata list in a data frame format.
|
1394
|
+
|
1395
|
+
Returns
|
1396
|
+
-------
|
1397
|
+
`pandas.DataFrame`
|
1398
|
+
The model metadata in a dataframe format.
|
1399
|
+
"""
|
1400
|
+
return (
|
1401
|
+
pd.DataFrame(
|
1402
|
+
(
|
1403
|
+
(
|
1404
|
+
item.key,
|
1405
|
+
item.value,
|
1406
|
+
item.description,
|
1407
|
+
item.category,
|
1408
|
+
item.has_artifact,
|
1409
|
+
)
|
1410
|
+
for item in self._items
|
1411
|
+
),
|
1412
|
+
columns=[value for value in MetadataCustomPrintColumns.values()],
|
1413
|
+
)
|
1414
|
+
.sort_values(by=[MetadataCustomPrintColumns.KEY])
|
1415
|
+
.reset_index(drop=True)
|
1416
|
+
)
|
1417
|
+
|
1418
|
+
@classmethod
|
1419
|
+
def from_dict(cls, data: Dict) -> "ModelCustomMetadata":
|
1420
|
+
"""Constructs an instance of `ModelCustomMetadata` from a dictionary.
|
1421
|
+
|
1422
|
+
Parameters
|
1423
|
+
----------
|
1424
|
+
data : Dict
|
1425
|
+
Model metadata in a dictionary format.
|
1426
|
+
|
1427
|
+
Returns
|
1428
|
+
-------
|
1429
|
+
ModelCustomMetadata
|
1430
|
+
An instance of model custom metadata.
|
1431
|
+
|
1432
|
+
Raises
|
1433
|
+
------
|
1434
|
+
ValueError
|
1435
|
+
In case of the wrong input data format.
|
1436
|
+
"""
|
1437
|
+
if (
|
1438
|
+
not data
|
1439
|
+
or not isinstance(data, Dict)
|
1440
|
+
or "data" not in data
|
1441
|
+
or not isinstance(data["data"], List)
|
1442
|
+
):
|
1443
|
+
raise ValueError(
|
1444
|
+
"An error occurred when attempting to deserialize the model custom metadata from a dictionary. "
|
1445
|
+
"The input data must be a dictionary with `data` key. Example: `{'data': []}`"
|
1446
|
+
)
|
1447
|
+
|
1448
|
+
metadata = cls()
|
1449
|
+
for item in data["data"]:
|
1450
|
+
metadata._add(ModelCustomMetadataItem.from_dict(item), replace=True)
|
1451
|
+
return metadata
|
1452
|
+
|
1453
|
+
|
1454
|
+
class ModelTaxonomyMetadata(ModelMetadata):
|
1455
|
+
"""Class that represents Model Taxonomy Metadata.
|
1456
|
+
|
1457
|
+
Methods
|
1458
|
+
-------
|
1459
|
+
get(self, key: str) -> ModelTaxonomyMetadataItem
|
1460
|
+
Returns the model metadata item by provided key.
|
1461
|
+
reset(self) -> None
|
1462
|
+
Resets all model metadata items to empty values.
|
1463
|
+
to_dataframe(self) -> pd.DataFrame
|
1464
|
+
Returns the model metadata list in a data frame format.
|
1465
|
+
size(self) -> int
|
1466
|
+
Returns the size of the model metadata in bytes.
|
1467
|
+
validate(self) -> bool
|
1468
|
+
Validates metadata.
|
1469
|
+
to_dict(self)
|
1470
|
+
Serializes model metadata into a dictionary.
|
1471
|
+
from_dict(cls) -> ModelTaxonomyMetadata
|
1472
|
+
Constructs model metadata from dictionary.
|
1473
|
+
to_yaml(self)
|
1474
|
+
Serializes model metadata into a YAML.
|
1475
|
+
to_json(self)
|
1476
|
+
Serializes model metadata into a JSON.
|
1477
|
+
to_json_file(self, file_path: str, storage_options: dict = None) -> None
|
1478
|
+
Saves the metadata to a local file or object storage.
|
1479
|
+
|
1480
|
+
Examples
|
1481
|
+
--------
|
1482
|
+
>>> metadata_taxonomy = ModelTaxonomyMetadata()
|
1483
|
+
>>> metadata_taxonomy.to_dataframe()
|
1484
|
+
Key Value
|
1485
|
+
--------------------------------------------
|
1486
|
+
0 UseCaseType binary_classification
|
1487
|
+
1 Framework sklearn
|
1488
|
+
2 FrameworkVersion 0.2.2
|
1489
|
+
3 Algorithm algorithm
|
1490
|
+
4 Hyperparameters {}
|
1491
|
+
|
1492
|
+
>>> metadata_taxonomy.reset()
|
1493
|
+
>>> metadata_taxonomy.to_dataframe()
|
1494
|
+
Key Value
|
1495
|
+
--------------------------------------------
|
1496
|
+
0 UseCaseType None
|
1497
|
+
1 Framework None
|
1498
|
+
2 FrameworkVersion None
|
1499
|
+
3 Algorithm None
|
1500
|
+
4 Hyperparameters None
|
1501
|
+
|
1502
|
+
>>> metadata_taxonomy
|
1503
|
+
metadata:
|
1504
|
+
- key: UseCaseType
|
1505
|
+
category: None
|
1506
|
+
description: None
|
1507
|
+
value: None
|
1508
|
+
"""
|
1509
|
+
|
1510
|
+
def __init__(self):
|
1511
|
+
super().__init__()
|
1512
|
+
for key in MetadataTaxonomyKeys.values():
|
1513
|
+
self._items.add(ModelTaxonomyMetadataItem(key=key))
|
1514
|
+
|
1515
|
+
def _populate_from_map(self, map: Dict[str, str]) -> None:
|
1516
|
+
"""Populates metadata information from map.
|
1517
|
+
|
1518
|
+
Parameters
|
1519
|
+
----------
|
1520
|
+
map: Dict[str, str]
|
1521
|
+
The key/value map with model metadata information.
|
1522
|
+
|
1523
|
+
Returns
|
1524
|
+
-------
|
1525
|
+
None
|
1526
|
+
Nothing.
|
1527
|
+
"""
|
1528
|
+
for value in MetadataTaxonomyKeys.values():
|
1529
|
+
if value in map:
|
1530
|
+
self[value].update(value=map[value])
|
1531
|
+
|
1532
|
+
@classmethod
|
1533
|
+
def _from_oci_metadata(cls, metadata_list):
|
1534
|
+
"""
|
1535
|
+
Convert from list of oci metadata to a ModelTaxonomyMetadata object.
|
1536
|
+
|
1537
|
+
Parameters
|
1538
|
+
----------
|
1539
|
+
metadata_list: List
|
1540
|
+
List of oci metadata.
|
1541
|
+
|
1542
|
+
Returns
|
1543
|
+
-------
|
1544
|
+
ModelTaxonomyMetadata
|
1545
|
+
A `ModelTaxonomyMetadata` instance.
|
1546
|
+
"""
|
1547
|
+
metadata = cls()
|
1548
|
+
for oci_item in metadata_list:
|
1549
|
+
item = ModelTaxonomyMetadataItem._from_oci_metadata(oci_item)
|
1550
|
+
if item.key in metadata.keys:
|
1551
|
+
metadata[item.key].update(
|
1552
|
+
value=item.value, has_artifact=item.has_artifact
|
1553
|
+
)
|
1554
|
+
else:
|
1555
|
+
metadata._items.add(item)
|
1556
|
+
return metadata
|
1557
|
+
|
1558
|
+
def to_dataframe(self) -> pd.DataFrame:
|
1559
|
+
"""Returns the model metadata list in a data frame format.
|
1560
|
+
|
1561
|
+
Returns
|
1562
|
+
-------
|
1563
|
+
`pandas.DataFrame`
|
1564
|
+
The model metadata in a dataframe format.
|
1565
|
+
"""
|
1566
|
+
return (
|
1567
|
+
pd.DataFrame(
|
1568
|
+
((item.key, item.value, item.has_artifact) for item in self._items),
|
1569
|
+
columns=[value for value in MetadataTaxonomyPrintColumns.values()],
|
1570
|
+
)
|
1571
|
+
.sort_values(by=[MetadataTaxonomyPrintColumns.KEY])
|
1572
|
+
.reset_index(drop=True)
|
1573
|
+
)
|
1574
|
+
|
1575
|
+
@classmethod
|
1576
|
+
def from_dict(cls, data: Dict) -> "ModelTaxonomyMetadata":
|
1577
|
+
"""Constructs an instance of `ModelTaxonomyMetadata` from a dictionary.
|
1578
|
+
|
1579
|
+
Parameters
|
1580
|
+
----------
|
1581
|
+
data : Dict
|
1582
|
+
Model metadata in a dictionary format.
|
1583
|
+
|
1584
|
+
Returns
|
1585
|
+
-------
|
1586
|
+
ModelTaxonomyMetadata
|
1587
|
+
An instance of model taxonomy metadata.
|
1588
|
+
|
1589
|
+
Raises
|
1590
|
+
------
|
1591
|
+
ValueError
|
1592
|
+
In case of the wrong input data format.
|
1593
|
+
"""
|
1594
|
+
if (
|
1595
|
+
not data
|
1596
|
+
or not isinstance(data, Dict)
|
1597
|
+
or "data" not in data
|
1598
|
+
or not isinstance(data["data"], List)
|
1599
|
+
):
|
1600
|
+
raise ValueError(
|
1601
|
+
"An error occurred when attempting to deserialize the model taxonomy metadata from a dictionary. "
|
1602
|
+
"The input data must be a dictionary with `data` key. Example: `{'data': []}`"
|
1603
|
+
)
|
1604
|
+
|
1605
|
+
metadata = cls()
|
1606
|
+
for item in data["data"]:
|
1607
|
+
item = ModelTaxonomyMetadataItem.from_dict(item)
|
1608
|
+
if item.key in metadata.keys:
|
1609
|
+
metadata[item.key].update(value=item.value)
|
1610
|
+
else:
|
1611
|
+
metadata._items.add(item)
|
1612
|
+
return metadata
|
1613
|
+
|
1614
|
+
|
1615
|
+
@dataclass(repr=True)
|
1616
|
+
class ModelProvenanceMetadata(DataClassSerializable):
|
1617
|
+
"""ModelProvenanceMetadata class.
|
1618
|
+
|
1619
|
+
Examples
|
1620
|
+
--------
|
1621
|
+
>>> provenance_metadata = ModelProvenanceMetadata.fetch_training_code_details()
|
1622
|
+
ModelProvenanceMetadata(repo=<git.repo.base.Repo '/home/datascience/.git'>, git_branch='master', git_commit='99ad04c31803f1d4ffcc3bf4afbd6bcf69a06af2', repository_url='file:///home/datascience', "", "")
|
1623
|
+
>>> provenance_metadata.assert_path_not_dirty("your_path", ignore=False)
|
1624
|
+
"""
|
1625
|
+
|
1626
|
+
repo: str = field(default=None, metadata={"serializable": False})
|
1627
|
+
git_branch: str = None
|
1628
|
+
git_commit: str = None
|
1629
|
+
repository_url: str = None
|
1630
|
+
training_script_path: str = None
|
1631
|
+
training_id: str = None
|
1632
|
+
artifact_dir: str = None
|
1633
|
+
|
1634
|
+
@classmethod
|
1635
|
+
def fetch_training_code_details(
|
1636
|
+
cls,
|
1637
|
+
training_script_path: str = None,
|
1638
|
+
training_id: str = None,
|
1639
|
+
artifact_dir: str = None,
|
1640
|
+
):
|
1641
|
+
"""Fetches the training code details: repo, git_branch, git_commit, repository_url, training_script_path and training_id.
|
1642
|
+
|
1643
|
+
Parameters
|
1644
|
+
----------
|
1645
|
+
training_script_path: (str, optional). Defaults to None.
|
1646
|
+
Training script path.
|
1647
|
+
training_id: (str, optional). Defaults to None.
|
1648
|
+
The training OCID for model.
|
1649
|
+
artifact_dir: str
|
1650
|
+
artifact directory to store the files needed for deployment.
|
1651
|
+
|
1652
|
+
Returns
|
1653
|
+
-------
|
1654
|
+
ModelProvenanceMetadata
|
1655
|
+
A ModelProvenanceMetadata instance.
|
1656
|
+
"""
|
1657
|
+
git_dir = CURRENT_WORKING_DIR
|
1658
|
+
if training_script_path:
|
1659
|
+
if not os.path.exists(training_script_path):
|
1660
|
+
logger.warning(
|
1661
|
+
f"Training script {os.path.abspath(training_script_path)} does not exist."
|
1662
|
+
)
|
1663
|
+
else:
|
1664
|
+
training_script_path = os.path.abspath(training_script_path)
|
1665
|
+
git_dir = os.path.dirname(training_script_path)
|
1666
|
+
repo = git.Repo(git_dir, search_parent_directories=True)
|
1667
|
+
# get repository url
|
1668
|
+
if len(repo.remotes) > 0:
|
1669
|
+
repository_url = (
|
1670
|
+
repo.remotes.origin.url
|
1671
|
+
if repo.remotes.origin in repo.remotes
|
1672
|
+
else list(repo.remotes.values())[0].url
|
1673
|
+
)
|
1674
|
+
else:
|
1675
|
+
repository_url = "file://" + repo.working_dir # no remote repo
|
1676
|
+
|
1677
|
+
# get git commit
|
1678
|
+
git_branch = None
|
1679
|
+
git_commit = None
|
1680
|
+
|
1681
|
+
try:
|
1682
|
+
# get git branch
|
1683
|
+
git_branch = format(repo.active_branch)
|
1684
|
+
git_commit = format(str(repo.head.commit.hexsha)) or None
|
1685
|
+
except Exception:
|
1686
|
+
logger.warning("No commit found.")
|
1687
|
+
return cls(
|
1688
|
+
repo=repo,
|
1689
|
+
git_branch=git_branch,
|
1690
|
+
git_commit=git_commit,
|
1691
|
+
repository_url=repository_url,
|
1692
|
+
training_script_path=training_script_path,
|
1693
|
+
training_id=training_id,
|
1694
|
+
artifact_dir=artifact_dir,
|
1695
|
+
)
|
1696
|
+
|
1697
|
+
def assert_path_not_dirty(self, path: str, ignore: bool):
|
1698
|
+
"""Checks if all the changes in this path has been commited.
|
1699
|
+
|
1700
|
+
Parameters
|
1701
|
+
----------
|
1702
|
+
path: (str)
|
1703
|
+
path.
|
1704
|
+
ignore (bool)
|
1705
|
+
whether to ignore the changes or not.
|
1706
|
+
|
1707
|
+
Raises
|
1708
|
+
------
|
1709
|
+
ChangesNotCommitted: if there are changes not being commited.
|
1710
|
+
|
1711
|
+
Returns
|
1712
|
+
-------
|
1713
|
+
None
|
1714
|
+
Nothing.
|
1715
|
+
"""
|
1716
|
+
if ObjectStorageDetails.is_oci_path(path):
|
1717
|
+
return
|
1718
|
+
|
1719
|
+
if self.repo is not None and not ignore:
|
1720
|
+
path_abs = os.path.abspath(path)
|
1721
|
+
if (
|
1722
|
+
os.path.commonpath([path_abs, self.repo.working_dir])
|
1723
|
+
== self.repo.working_dir
|
1724
|
+
):
|
1725
|
+
path_relpath = os.path.relpath(path_abs, self.repo.working_dir)
|
1726
|
+
if self.repo.is_dirty(path=path_relpath) or any(
|
1727
|
+
[
|
1728
|
+
os.path.commonpath([path_relpath, untracked]) == path_relpath
|
1729
|
+
for untracked in self.repo.untracked_files
|
1730
|
+
]
|
1731
|
+
):
|
1732
|
+
raise ChangesNotCommitted(path_abs)
|
1733
|
+
|
1734
|
+
def _to_oci_metadata(self) -> oci.data_science.models.ModelProvenance:
|
1735
|
+
"""Convert to `oci.data_science.models.ModelProvenance` object.
|
1736
|
+
|
1737
|
+
Returns
|
1738
|
+
-------
|
1739
|
+
oci.data_science.models.ModelProvenance
|
1740
|
+
OCI model provenance object.
|
1741
|
+
"""
|
1742
|
+
return oci.data_science.models.ModelProvenance(
|
1743
|
+
repository_url=self.repository_url,
|
1744
|
+
git_branch=self.git_branch,
|
1745
|
+
git_commit=self.git_commit,
|
1746
|
+
script_dir=self.artifact_dir,
|
1747
|
+
training_script=self.training_script_path,
|
1748
|
+
training_id=self.training_id,
|
1749
|
+
)
|
1750
|
+
|
1751
|
+
@classmethod
|
1752
|
+
def _from_oci_metadata(
|
1753
|
+
cls, model_provenance: oci.data_science.models.ModelProvenance
|
1754
|
+
) -> "ModelProvenanceMetadata":
|
1755
|
+
"""Creates a new model provenance metadata item from the `oci.data_science.models.ModelProvenance` object.
|
1756
|
+
|
1757
|
+
Returns
|
1758
|
+
-------
|
1759
|
+
ModelProvenanceMetadata
|
1760
|
+
Model provenance metadata object.
|
1761
|
+
"""
|
1762
|
+
return ModelProvenanceMetadata(
|
1763
|
+
repo=model_provenance.repository_url,
|
1764
|
+
git_branch=model_provenance.git_branch,
|
1765
|
+
git_commit=model_provenance.git_commit,
|
1766
|
+
repository_url=model_provenance.repository_url,
|
1767
|
+
training_script_path=model_provenance.training_script,
|
1768
|
+
training_id=model_provenance.training_id,
|
1769
|
+
artifact_dir=model_provenance.script_dir,
|
1770
|
+
)
|
1771
|
+
|
1772
|
+
@classmethod
|
1773
|
+
def from_dict(cls, data: Dict[str, str]) -> "ModelProvenanceMetadata":
|
1774
|
+
"""Constructs an instance of ModelProvenanceMetadata from a dictionary.
|
1775
|
+
|
1776
|
+
Parameters
|
1777
|
+
----------
|
1778
|
+
data : Dict[str,str]
|
1779
|
+
Model provenance metadata in dictionary format.
|
1780
|
+
|
1781
|
+
Returns
|
1782
|
+
-------
|
1783
|
+
ModelProvenanceMetadata
|
1784
|
+
An instance of ModelProvenanceMetadata.
|
1785
|
+
"""
|
1786
|
+
return cls(**data or {})
|
1787
|
+
|
1788
|
+
def to_dict(self) -> dict:
|
1789
|
+
"""Serializes model provenance metadata into a dictionary.
|
1790
|
+
|
1791
|
+
Returns
|
1792
|
+
-------
|
1793
|
+
Dict
|
1794
|
+
The dictionary representation of the model provenance metadata.
|
1795
|
+
"""
|
1796
|
+
return {
|
1797
|
+
f.name: getattr(self, f.name)
|
1798
|
+
for f in fields(self)
|
1799
|
+
if ("serializable" not in f.metadata or f.metadata["serializable"])
|
1800
|
+
}
|
1801
|
+
|
1802
|
+
def __repr__(self):
|
1803
|
+
"""Returns printable version of object.
|
1804
|
+
|
1805
|
+
Parameters
|
1806
|
+
----------
|
1807
|
+
string
|
1808
|
+
Serialized version of object as a YAML string
|
1809
|
+
"""
|
1810
|
+
return self.to_yaml()
|