oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/__init__.py +40 -0
- ads/aqua/app.py +507 -0
- ads/aqua/cli.py +96 -0
- ads/aqua/client/__init__.py +3 -0
- ads/aqua/client/client.py +836 -0
- ads/aqua/client/openai_client.py +305 -0
- ads/aqua/common/__init__.py +5 -0
- ads/aqua/common/decorator.py +125 -0
- ads/aqua/common/entities.py +274 -0
- ads/aqua/common/enums.py +134 -0
- ads/aqua/common/errors.py +109 -0
- ads/aqua/common/utils.py +1295 -0
- ads/aqua/config/__init__.py +4 -0
- ads/aqua/config/container_config.py +247 -0
- ads/aqua/config/evaluation/__init__.py +4 -0
- ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
- ads/aqua/config/utils/__init__.py +4 -0
- ads/aqua/config/utils/serializer.py +339 -0
- ads/aqua/constants.py +116 -0
- ads/aqua/data.py +14 -0
- ads/aqua/dummy_data/icon.txt +1 -0
- ads/aqua/dummy_data/oci_model_deployments.json +56 -0
- ads/aqua/dummy_data/oci_models.json +1 -0
- ads/aqua/dummy_data/readme.md +26 -0
- ads/aqua/evaluation/__init__.py +8 -0
- ads/aqua/evaluation/constants.py +53 -0
- ads/aqua/evaluation/entities.py +186 -0
- ads/aqua/evaluation/errors.py +70 -0
- ads/aqua/evaluation/evaluation.py +1814 -0
- ads/aqua/extension/__init__.py +42 -0
- ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
- ads/aqua/extension/base_handler.py +90 -0
- ads/aqua/extension/common_handler.py +121 -0
- ads/aqua/extension/common_ws_msg_handler.py +36 -0
- ads/aqua/extension/deployment_handler.py +381 -0
- ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
- ads/aqua/extension/errors.py +30 -0
- ads/aqua/extension/evaluation_handler.py +129 -0
- ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
- ads/aqua/extension/finetune_handler.py +96 -0
- ads/aqua/extension/model_handler.py +390 -0
- ads/aqua/extension/models/__init__.py +0 -0
- ads/aqua/extension/models/ws_models.py +145 -0
- ads/aqua/extension/models_ws_msg_handler.py +50 -0
- ads/aqua/extension/ui_handler.py +300 -0
- ads/aqua/extension/ui_websocket_handler.py +130 -0
- ads/aqua/extension/utils.py +133 -0
- ads/aqua/finetuning/__init__.py +7 -0
- ads/aqua/finetuning/constants.py +23 -0
- ads/aqua/finetuning/entities.py +181 -0
- ads/aqua/finetuning/finetuning.py +749 -0
- ads/aqua/model/__init__.py +8 -0
- ads/aqua/model/constants.py +60 -0
- ads/aqua/model/entities.py +385 -0
- ads/aqua/model/enums.py +32 -0
- ads/aqua/model/model.py +2134 -0
- ads/aqua/model/utils.py +52 -0
- ads/aqua/modeldeployment/__init__.py +6 -0
- ads/aqua/modeldeployment/constants.py +10 -0
- ads/aqua/modeldeployment/deployment.py +1315 -0
- ads/aqua/modeldeployment/entities.py +653 -0
- ads/aqua/modeldeployment/utils.py +543 -0
- ads/aqua/resources/gpu_shapes_index.json +94 -0
- ads/aqua/server/__init__.py +4 -0
- ads/aqua/server/__main__.py +24 -0
- ads/aqua/server/app.py +47 -0
- ads/aqua/server/aqua_spec.yml +1291 -0
- ads/aqua/training/__init__.py +4 -0
- ads/aqua/training/exceptions.py +476 -0
- ads/aqua/ui.py +519 -0
- ads/automl/__init__.py +9 -0
- ads/automl/driver.py +330 -0
- ads/automl/provider.py +975 -0
- ads/bds/__init__.py +5 -0
- ads/bds/auth.py +127 -0
- ads/bds/big_data_service.py +255 -0
- ads/catalog/__init__.py +19 -0
- ads/catalog/model.py +1576 -0
- ads/catalog/notebook.py +461 -0
- ads/catalog/project.py +468 -0
- ads/catalog/summary.py +178 -0
- ads/common/__init__.py +11 -0
- ads/common/analyzer.py +65 -0
- ads/common/artifact/.model-ignore +63 -0
- ads/common/artifact/__init__.py +10 -0
- ads/common/auth.py +1122 -0
- ads/common/card_identifier.py +83 -0
- ads/common/config.py +647 -0
- ads/common/data.py +165 -0
- ads/common/decorator/__init__.py +9 -0
- ads/common/decorator/argument_to_case.py +88 -0
- ads/common/decorator/deprecate.py +69 -0
- ads/common/decorator/require_nonempty_arg.py +65 -0
- ads/common/decorator/runtime_dependency.py +178 -0
- ads/common/decorator/threaded.py +97 -0
- ads/common/decorator/utils.py +35 -0
- ads/common/dsc_file_system.py +303 -0
- ads/common/error.py +14 -0
- ads/common/extended_enum.py +81 -0
- ads/common/function/__init__.py +5 -0
- ads/common/function/fn_util.py +142 -0
- ads/common/function/func_conf.yaml +25 -0
- ads/common/ipython.py +76 -0
- ads/common/model.py +679 -0
- ads/common/model_artifact.py +1759 -0
- ads/common/model_artifact_schema.json +107 -0
- ads/common/model_export_util.py +664 -0
- ads/common/model_metadata.py +24 -0
- ads/common/object_storage_details.py +296 -0
- ads/common/oci_client.py +179 -0
- ads/common/oci_datascience.py +46 -0
- ads/common/oci_logging.py +1144 -0
- ads/common/oci_mixin.py +957 -0
- ads/common/oci_resource.py +136 -0
- ads/common/serializer.py +559 -0
- ads/common/utils.py +1852 -0
- ads/common/word_lists.py +1491 -0
- ads/common/work_request.py +189 -0
- ads/config.py +1 -0
- ads/data_labeling/__init__.py +13 -0
- ads/data_labeling/boundingbox.py +253 -0
- ads/data_labeling/constants.py +47 -0
- ads/data_labeling/data_labeling_service.py +244 -0
- ads/data_labeling/interface/__init__.py +5 -0
- ads/data_labeling/interface/loader.py +16 -0
- ads/data_labeling/interface/parser.py +16 -0
- ads/data_labeling/interface/reader.py +23 -0
- ads/data_labeling/loader/__init__.py +5 -0
- ads/data_labeling/loader/file_loader.py +241 -0
- ads/data_labeling/metadata.py +110 -0
- ads/data_labeling/mixin/__init__.py +5 -0
- ads/data_labeling/mixin/data_labeling.py +232 -0
- ads/data_labeling/ner.py +129 -0
- ads/data_labeling/parser/__init__.py +5 -0
- ads/data_labeling/parser/dls_record_parser.py +388 -0
- ads/data_labeling/parser/export_metadata_parser.py +94 -0
- ads/data_labeling/parser/export_record_parser.py +473 -0
- ads/data_labeling/reader/__init__.py +5 -0
- ads/data_labeling/reader/dataset_reader.py +574 -0
- ads/data_labeling/reader/dls_record_reader.py +121 -0
- ads/data_labeling/reader/export_record_reader.py +62 -0
- ads/data_labeling/reader/jsonl_reader.py +75 -0
- ads/data_labeling/reader/metadata_reader.py +203 -0
- ads/data_labeling/reader/record_reader.py +263 -0
- ads/data_labeling/record.py +52 -0
- ads/data_labeling/visualizer/__init__.py +5 -0
- ads/data_labeling/visualizer/image_visualizer.py +525 -0
- ads/data_labeling/visualizer/text_visualizer.py +357 -0
- ads/database/__init__.py +5 -0
- ads/database/connection.py +338 -0
- ads/dataset/__init__.py +10 -0
- ads/dataset/capabilities.md +51 -0
- ads/dataset/classification_dataset.py +339 -0
- ads/dataset/correlation.py +226 -0
- ads/dataset/correlation_plot.py +563 -0
- ads/dataset/dask_series.py +173 -0
- ads/dataset/dataframe_transformer.py +110 -0
- ads/dataset/dataset.py +1979 -0
- ads/dataset/dataset_browser.py +360 -0
- ads/dataset/dataset_with_target.py +995 -0
- ads/dataset/exception.py +25 -0
- ads/dataset/factory.py +987 -0
- ads/dataset/feature_engineering_transformer.py +35 -0
- ads/dataset/feature_selection.py +107 -0
- ads/dataset/forecasting_dataset.py +26 -0
- ads/dataset/helper.py +1450 -0
- ads/dataset/label_encoder.py +99 -0
- ads/dataset/mixin/__init__.py +5 -0
- ads/dataset/mixin/dataset_accessor.py +134 -0
- ads/dataset/pipeline.py +58 -0
- ads/dataset/plot.py +710 -0
- ads/dataset/progress.py +86 -0
- ads/dataset/recommendation.py +297 -0
- ads/dataset/recommendation_transformer.py +502 -0
- ads/dataset/regression_dataset.py +14 -0
- ads/dataset/sampled_dataset.py +1050 -0
- ads/dataset/target.py +98 -0
- ads/dataset/timeseries.py +18 -0
- ads/dbmixin/__init__.py +5 -0
- ads/dbmixin/db_pandas_accessor.py +153 -0
- ads/environment/__init__.py +9 -0
- ads/environment/ml_runtime.py +66 -0
- ads/evaluations/README.md +14 -0
- ads/evaluations/__init__.py +109 -0
- ads/evaluations/evaluation_plot.py +983 -0
- ads/evaluations/evaluator.py +1334 -0
- ads/evaluations/statistical_metrics.py +543 -0
- ads/experiments/__init__.py +9 -0
- ads/experiments/capabilities.md +0 -0
- ads/explanations/__init__.py +21 -0
- ads/explanations/base_explainer.py +142 -0
- ads/explanations/capabilities.md +83 -0
- ads/explanations/explainer.py +190 -0
- ads/explanations/mlx_global_explainer.py +1050 -0
- ads/explanations/mlx_interface.py +386 -0
- ads/explanations/mlx_local_explainer.py +287 -0
- ads/explanations/mlx_whatif_explainer.py +201 -0
- ads/feature_engineering/__init__.py +20 -0
- ads/feature_engineering/accessor/__init__.py +5 -0
- ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
- ads/feature_engineering/accessor/mixin/__init__.py +5 -0
- ads/feature_engineering/accessor/mixin/correlation.py +166 -0
- ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
- ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
- ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
- ads/feature_engineering/accessor/mixin/utils.py +65 -0
- ads/feature_engineering/accessor/series_accessor.py +431 -0
- ads/feature_engineering/adsimage/__init__.py +5 -0
- ads/feature_engineering/adsimage/image.py +192 -0
- ads/feature_engineering/adsimage/image_reader.py +170 -0
- ads/feature_engineering/adsimage/interface/__init__.py +5 -0
- ads/feature_engineering/adsimage/interface/reader.py +19 -0
- ads/feature_engineering/adsstring/__init__.py +7 -0
- ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
- ads/feature_engineering/adsstring/string/__init__.py +8 -0
- ads/feature_engineering/data_schema.json +57 -0
- ads/feature_engineering/dataset/__init__.py +5 -0
- ads/feature_engineering/dataset/zip_code_data.py +42062 -0
- ads/feature_engineering/exceptions.py +40 -0
- ads/feature_engineering/feature_type/__init__.py +133 -0
- ads/feature_engineering/feature_type/address.py +184 -0
- ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
- ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
- ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
- ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
- ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
- ads/feature_engineering/feature_type/adsstring/string.py +258 -0
- ads/feature_engineering/feature_type/base.py +58 -0
- ads/feature_engineering/feature_type/boolean.py +183 -0
- ads/feature_engineering/feature_type/category.py +146 -0
- ads/feature_engineering/feature_type/constant.py +137 -0
- ads/feature_engineering/feature_type/continuous.py +151 -0
- ads/feature_engineering/feature_type/creditcard.py +314 -0
- ads/feature_engineering/feature_type/datetime.py +190 -0
- ads/feature_engineering/feature_type/discrete.py +134 -0
- ads/feature_engineering/feature_type/document.py +43 -0
- ads/feature_engineering/feature_type/gis.py +251 -0
- ads/feature_engineering/feature_type/handler/__init__.py +5 -0
- ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
- ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
- ads/feature_engineering/feature_type/handler/warnings.py +128 -0
- ads/feature_engineering/feature_type/integer.py +142 -0
- ads/feature_engineering/feature_type/ip_address.py +144 -0
- ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
- ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
- ads/feature_engineering/feature_type/lat_long.py +256 -0
- ads/feature_engineering/feature_type/object.py +43 -0
- ads/feature_engineering/feature_type/ordinal.py +132 -0
- ads/feature_engineering/feature_type/phone_number.py +135 -0
- ads/feature_engineering/feature_type/string.py +171 -0
- ads/feature_engineering/feature_type/text.py +93 -0
- ads/feature_engineering/feature_type/unknown.py +43 -0
- ads/feature_engineering/feature_type/zip_code.py +164 -0
- ads/feature_engineering/feature_type_manager.py +406 -0
- ads/feature_engineering/schema.py +795 -0
- ads/feature_engineering/utils.py +245 -0
- ads/feature_store/.readthedocs.yaml +19 -0
- ads/feature_store/README.md +65 -0
- ads/feature_store/__init__.py +9 -0
- ads/feature_store/common/__init__.py +0 -0
- ads/feature_store/common/enums.py +339 -0
- ads/feature_store/common/exceptions.py +18 -0
- ads/feature_store/common/spark_session_singleton.py +125 -0
- ads/feature_store/common/utils/__init__.py +0 -0
- ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
- ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
- ads/feature_store/common/utils/transformation_utils.py +82 -0
- ads/feature_store/common/utils/utility.py +403 -0
- ads/feature_store/data_validation/__init__.py +0 -0
- ads/feature_store/data_validation/great_expectation.py +129 -0
- ads/feature_store/dataset.py +1230 -0
- ads/feature_store/dataset_job.py +530 -0
- ads/feature_store/docs/Dockerfile +7 -0
- ads/feature_store/docs/Makefile +44 -0
- ads/feature_store/docs/conf.py +28 -0
- ads/feature_store/docs/requirements.txt +14 -0
- ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
- ads/feature_store/docs/source/cicd.rst +137 -0
- ads/feature_store/docs/source/conf.py +86 -0
- ads/feature_store/docs/source/data_versioning.rst +33 -0
- ads/feature_store/docs/source/dataset.rst +388 -0
- ads/feature_store/docs/source/dataset_job.rst +27 -0
- ads/feature_store/docs/source/demo.rst +70 -0
- ads/feature_store/docs/source/entity.rst +78 -0
- ads/feature_store/docs/source/feature_group.rst +624 -0
- ads/feature_store/docs/source/feature_group_job.rst +29 -0
- ads/feature_store/docs/source/feature_store.rst +122 -0
- ads/feature_store/docs/source/feature_store_class.rst +123 -0
- ads/feature_store/docs/source/feature_validation.rst +66 -0
- ads/feature_store/docs/source/figures/cicd.png +0 -0
- ads/feature_store/docs/source/figures/data_validation.png +0 -0
- ads/feature_store/docs/source/figures/data_versioning.png +0 -0
- ads/feature_store/docs/source/figures/dataset.gif +0 -0
- ads/feature_store/docs/source/figures/dataset.png +0 -0
- ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
- ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
- ads/feature_store/docs/source/figures/entity.png +0 -0
- ads/feature_store/docs/source/figures/feature_group.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
- ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
- ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
- ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
- ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
- ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
- ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
- ads/feature_store/docs/source/figures/overview.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
- ads/feature_store/docs/source/figures/stats_1.png +0 -0
- ads/feature_store/docs/source/figures/stats_2.png +0 -0
- ads/feature_store/docs/source/figures/stats_d.png +0 -0
- ads/feature_store/docs/source/figures/stats_fg.png +0 -0
- ads/feature_store/docs/source/figures/transformation.png +0 -0
- ads/feature_store/docs/source/figures/transformations.gif +0 -0
- ads/feature_store/docs/source/figures/validation.png +0 -0
- ads/feature_store/docs/source/figures/validation_fg.png +0 -0
- ads/feature_store/docs/source/figures/validation_results.png +0 -0
- ads/feature_store/docs/source/figures/validation_summary.png +0 -0
- ads/feature_store/docs/source/index.rst +81 -0
- ads/feature_store/docs/source/module.rst +8 -0
- ads/feature_store/docs/source/notebook.rst +94 -0
- ads/feature_store/docs/source/overview.rst +47 -0
- ads/feature_store/docs/source/quickstart.rst +176 -0
- ads/feature_store/docs/source/release_notes.rst +194 -0
- ads/feature_store/docs/source/setup_feature_store.rst +81 -0
- ads/feature_store/docs/source/statistics.rst +58 -0
- ads/feature_store/docs/source/transformation.rst +199 -0
- ads/feature_store/docs/source/ui.rst +65 -0
- ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
- ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
- ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
- ads/feature_store/entity.py +718 -0
- ads/feature_store/execution_strategy/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
- ads/feature_store/execution_strategy/engine/__init__.py +0 -0
- ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
- ads/feature_store/execution_strategy/execution_strategy.py +113 -0
- ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
- ads/feature_store/execution_strategy/spark/__init__.py +0 -0
- ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
- ads/feature_store/feature.py +192 -0
- ads/feature_store/feature_group.py +1494 -0
- ads/feature_store/feature_group_expectation.py +346 -0
- ads/feature_store/feature_group_job.py +602 -0
- ads/feature_store/feature_lineage/__init__.py +0 -0
- ads/feature_store/feature_lineage/graphviz_service.py +180 -0
- ads/feature_store/feature_option_details.py +50 -0
- ads/feature_store/feature_statistics/__init__.py +0 -0
- ads/feature_store/feature_statistics/statistics_service.py +99 -0
- ads/feature_store/feature_store.py +699 -0
- ads/feature_store/feature_store_registrar.py +518 -0
- ads/feature_store/input_feature_detail.py +149 -0
- ads/feature_store/mixin/__init__.py +4 -0
- ads/feature_store/mixin/oci_feature_store.py +145 -0
- ads/feature_store/model_details.py +73 -0
- ads/feature_store/query/__init__.py +0 -0
- ads/feature_store/query/filter.py +266 -0
- ads/feature_store/query/generator/__init__.py +0 -0
- ads/feature_store/query/generator/query_generator.py +298 -0
- ads/feature_store/query/join.py +161 -0
- ads/feature_store/query/query.py +403 -0
- ads/feature_store/query/validator/__init__.py +0 -0
- ads/feature_store/query/validator/query_validator.py +57 -0
- ads/feature_store/response/__init__.py +0 -0
- ads/feature_store/response/response_builder.py +68 -0
- ads/feature_store/service/__init__.py +0 -0
- ads/feature_store/service/oci_dataset.py +139 -0
- ads/feature_store/service/oci_dataset_job.py +199 -0
- ads/feature_store/service/oci_entity.py +125 -0
- ads/feature_store/service/oci_feature_group.py +164 -0
- ads/feature_store/service/oci_feature_group_job.py +214 -0
- ads/feature_store/service/oci_feature_store.py +182 -0
- ads/feature_store/service/oci_lineage.py +87 -0
- ads/feature_store/service/oci_transformation.py +104 -0
- ads/feature_store/statistics/__init__.py +0 -0
- ads/feature_store/statistics/abs_feature_value.py +49 -0
- ads/feature_store/statistics/charts/__init__.py +0 -0
- ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
- ads/feature_store/statistics/charts/box_plot.py +148 -0
- ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
- ads/feature_store/statistics/charts/probability_distribution.py +68 -0
- ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
- ads/feature_store/statistics/feature_stat.py +126 -0
- ads/feature_store/statistics/generic_feature_value.py +33 -0
- ads/feature_store/statistics/statistics.py +41 -0
- ads/feature_store/statistics_config.py +101 -0
- ads/feature_store/templates/feature_store_template.yaml +45 -0
- ads/feature_store/transformation.py +499 -0
- ads/feature_store/validation_output.py +57 -0
- ads/hpo/__init__.py +9 -0
- ads/hpo/_imports.py +91 -0
- ads/hpo/ads_search_space.py +439 -0
- ads/hpo/distributions.py +325 -0
- ads/hpo/objective.py +280 -0
- ads/hpo/search_cv.py +1657 -0
- ads/hpo/stopping_criterion.py +75 -0
- ads/hpo/tuner_artifact.py +413 -0
- ads/hpo/utils.py +91 -0
- ads/hpo/validation.py +140 -0
- ads/hpo/visualization/__init__.py +5 -0
- ads/hpo/visualization/_contour.py +23 -0
- ads/hpo/visualization/_edf.py +20 -0
- ads/hpo/visualization/_intermediate_values.py +21 -0
- ads/hpo/visualization/_optimization_history.py +25 -0
- ads/hpo/visualization/_parallel_coordinate.py +169 -0
- ads/hpo/visualization/_param_importances.py +26 -0
- ads/jobs/__init__.py +53 -0
- ads/jobs/ads_job.py +663 -0
- ads/jobs/builders/__init__.py +5 -0
- ads/jobs/builders/base.py +156 -0
- ads/jobs/builders/infrastructure/__init__.py +6 -0
- ads/jobs/builders/infrastructure/base.py +165 -0
- ads/jobs/builders/infrastructure/dataflow.py +1252 -0
- ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
- ads/jobs/builders/infrastructure/utils.py +65 -0
- ads/jobs/builders/runtimes/__init__.py +5 -0
- ads/jobs/builders/runtimes/artifact.py +338 -0
- ads/jobs/builders/runtimes/base.py +325 -0
- ads/jobs/builders/runtimes/container_runtime.py +242 -0
- ads/jobs/builders/runtimes/python_runtime.py +1016 -0
- ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
- ads/jobs/cli.py +104 -0
- ads/jobs/env_var_parser.py +131 -0
- ads/jobs/extension.py +160 -0
- ads/jobs/schema/__init__.py +5 -0
- ads/jobs/schema/infrastructure_schema.json +116 -0
- ads/jobs/schema/job_schema.json +42 -0
- ads/jobs/schema/runtime_schema.json +183 -0
- ads/jobs/schema/validator.py +141 -0
- ads/jobs/serializer.py +296 -0
- ads/jobs/templates/__init__.py +5 -0
- ads/jobs/templates/container.py +6 -0
- ads/jobs/templates/driver_notebook.py +177 -0
- ads/jobs/templates/driver_oci.py +500 -0
- ads/jobs/templates/driver_python.py +48 -0
- ads/jobs/templates/driver_pytorch.py +852 -0
- ads/jobs/templates/driver_utils.py +615 -0
- ads/jobs/templates/hostname_from_env.c +55 -0
- ads/jobs/templates/oci_metrics.py +181 -0
- ads/jobs/utils.py +104 -0
- ads/llm/__init__.py +28 -0
- ads/llm/autogen/__init__.py +2 -0
- ads/llm/autogen/constants.py +15 -0
- ads/llm/autogen/reports/__init__.py +2 -0
- ads/llm/autogen/reports/base.py +67 -0
- ads/llm/autogen/reports/data.py +103 -0
- ads/llm/autogen/reports/session.py +526 -0
- ads/llm/autogen/reports/templates/chat_box.html +13 -0
- ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
- ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
- ads/llm/autogen/reports/utils.py +56 -0
- ads/llm/autogen/v02/__init__.py +4 -0
- ads/llm/autogen/v02/client.py +295 -0
- ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
- ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
- ads/llm/autogen/v02/loggers/__init__.py +6 -0
- ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
- ads/llm/autogen/v02/loggers/session_logger.py +580 -0
- ads/llm/autogen/v02/loggers/utils.py +86 -0
- ads/llm/autogen/v02/runtime_logging.py +163 -0
- ads/llm/chain.py +268 -0
- ads/llm/chat_template.py +31 -0
- ads/llm/deploy.py +63 -0
- ads/llm/guardrails/__init__.py +5 -0
- ads/llm/guardrails/base.py +442 -0
- ads/llm/guardrails/huggingface.py +44 -0
- ads/llm/langchain/__init__.py +5 -0
- ads/llm/langchain/plugins/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
- ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
- ads/llm/langchain/plugins/llms/__init__.py +5 -0
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
- ads/llm/requirements.txt +3 -0
- ads/llm/serialize.py +219 -0
- ads/llm/serializers/__init__.py +0 -0
- ads/llm/serializers/retrieval_qa.py +153 -0
- ads/llm/serializers/runnable_parallel.py +27 -0
- ads/llm/templates/score_chain.jinja2 +155 -0
- ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
- ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
- ads/model/__init__.py +52 -0
- ads/model/artifact.py +573 -0
- ads/model/artifact_downloader.py +254 -0
- ads/model/artifact_uploader.py +267 -0
- ads/model/base_properties.py +238 -0
- ads/model/common/.model-ignore +66 -0
- ads/model/common/__init__.py +5 -0
- ads/model/common/utils.py +142 -0
- ads/model/datascience_model.py +2635 -0
- ads/model/deployment/__init__.py +20 -0
- ads/model/deployment/common/__init__.py +5 -0
- ads/model/deployment/common/utils.py +308 -0
- ads/model/deployment/model_deployer.py +466 -0
- ads/model/deployment/model_deployment.py +1846 -0
- ads/model/deployment/model_deployment_infrastructure.py +671 -0
- ads/model/deployment/model_deployment_properties.py +493 -0
- ads/model/deployment/model_deployment_runtime.py +838 -0
- ads/model/extractor/__init__.py +5 -0
- ads/model/extractor/automl_extractor.py +74 -0
- ads/model/extractor/embedding_onnx_extractor.py +80 -0
- ads/model/extractor/huggingface_extractor.py +88 -0
- ads/model/extractor/keras_extractor.py +84 -0
- ads/model/extractor/lightgbm_extractor.py +93 -0
- ads/model/extractor/model_info_extractor.py +114 -0
- ads/model/extractor/model_info_extractor_factory.py +105 -0
- ads/model/extractor/pytorch_extractor.py +87 -0
- ads/model/extractor/sklearn_extractor.py +112 -0
- ads/model/extractor/spark_extractor.py +89 -0
- ads/model/extractor/tensorflow_extractor.py +85 -0
- ads/model/extractor/xgboost_extractor.py +94 -0
- ads/model/framework/__init__.py +5 -0
- ads/model/framework/automl_model.py +178 -0
- ads/model/framework/embedding_onnx_model.py +438 -0
- ads/model/framework/huggingface_model.py +399 -0
- ads/model/framework/lightgbm_model.py +266 -0
- ads/model/framework/pytorch_model.py +266 -0
- ads/model/framework/sklearn_model.py +250 -0
- ads/model/framework/spark_model.py +326 -0
- ads/model/framework/tensorflow_model.py +254 -0
- ads/model/framework/xgboost_model.py +258 -0
- ads/model/generic_model.py +3518 -0
- ads/model/model_artifact_boilerplate/README.md +381 -0
- ads/model/model_artifact_boilerplate/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
- ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
- ads/model/model_artifact_boilerplate/score.py +61 -0
- ads/model/model_file_description_schema.json +68 -0
- ads/model/model_introspect.py +331 -0
- ads/model/model_metadata.py +1810 -0
- ads/model/model_metadata_mixin.py +460 -0
- ads/model/model_properties.py +63 -0
- ads/model/model_version_set.py +739 -0
- ads/model/runtime/__init__.py +5 -0
- ads/model/runtime/env_info.py +306 -0
- ads/model/runtime/model_deployment_details.py +37 -0
- ads/model/runtime/model_provenance_details.py +58 -0
- ads/model/runtime/runtime_info.py +81 -0
- ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
- ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
- ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
- ads/model/runtime/utils.py +201 -0
- ads/model/serde/__init__.py +5 -0
- ads/model/serde/common.py +40 -0
- ads/model/serde/model_input.py +547 -0
- ads/model/serde/model_serializer.py +1184 -0
- ads/model/service/__init__.py +5 -0
- ads/model/service/oci_datascience_model.py +1076 -0
- ads/model/service/oci_datascience_model_deployment.py +500 -0
- ads/model/service/oci_datascience_model_version_set.py +176 -0
- ads/model/transformer/__init__.py +5 -0
- ads/model/transformer/onnx_transformer.py +324 -0
- ads/mysqldb/__init__.py +5 -0
- ads/mysqldb/mysql_db.py +227 -0
- ads/opctl/__init__.py +18 -0
- ads/opctl/anomaly_detection.py +11 -0
- ads/opctl/backend/__init__.py +5 -0
- ads/opctl/backend/ads_dataflow.py +353 -0
- ads/opctl/backend/ads_ml_job.py +710 -0
- ads/opctl/backend/ads_ml_pipeline.py +164 -0
- ads/opctl/backend/ads_model_deployment.py +209 -0
- ads/opctl/backend/base.py +146 -0
- ads/opctl/backend/local.py +1053 -0
- ads/opctl/backend/marketplace/__init__.py +9 -0
- ads/opctl/backend/marketplace/helm_helper.py +173 -0
- ads/opctl/backend/marketplace/local_marketplace.py +271 -0
- ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
- ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
- ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
- ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
- ads/opctl/backend/marketplace/models/__init__.py +5 -0
- ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
- ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
- ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
- ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
- ads/opctl/cli.py +707 -0
- ads/opctl/cmds.py +869 -0
- ads/opctl/conda/__init__.py +5 -0
- ads/opctl/conda/cli.py +193 -0
- ads/opctl/conda/cmds.py +749 -0
- ads/opctl/conda/config.yaml +34 -0
- ads/opctl/conda/manifest_template.yaml +13 -0
- ads/opctl/conda/multipart_uploader.py +188 -0
- ads/opctl/conda/pack.py +89 -0
- ads/opctl/config/__init__.py +5 -0
- ads/opctl/config/base.py +57 -0
- ads/opctl/config/diagnostics/__init__.py +5 -0
- ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
- ads/opctl/config/merger.py +255 -0
- ads/opctl/config/resolver.py +297 -0
- ads/opctl/config/utils.py +79 -0
- ads/opctl/config/validator.py +17 -0
- ads/opctl/config/versioner.py +68 -0
- ads/opctl/config/yaml_parsers/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/base.py +58 -0
- ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
- ads/opctl/constants.py +66 -0
- ads/opctl/decorator/__init__.py +5 -0
- ads/opctl/decorator/common.py +129 -0
- ads/opctl/diagnostics/__init__.py +5 -0
- ads/opctl/diagnostics/__main__.py +25 -0
- ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
- ads/opctl/diagnostics/check_requirements.py +144 -0
- ads/opctl/diagnostics/requirement_exception.py +9 -0
- ads/opctl/distributed/README.md +109 -0
- ads/opctl/distributed/__init__.py +5 -0
- ads/opctl/distributed/certificates.py +32 -0
- ads/opctl/distributed/cli.py +207 -0
- ads/opctl/distributed/cmds.py +731 -0
- ads/opctl/distributed/common/__init__.py +5 -0
- ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
- ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
- ads/opctl/distributed/common/cluster_config_helper.py +103 -0
- ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
- ads/opctl/distributed/common/cluster_runner.py +54 -0
- ads/opctl/distributed/common/framework_factory.py +29 -0
- ads/opctl/docker/Dockerfile.job +103 -0
- ads/opctl/docker/Dockerfile.job.arm +107 -0
- ads/opctl/docker/Dockerfile.job.gpu +175 -0
- ads/opctl/docker/base-env.yaml +13 -0
- ads/opctl/docker/cuda.repo +6 -0
- ads/opctl/docker/operator/.dockerignore +0 -0
- ads/opctl/docker/operator/Dockerfile +41 -0
- ads/opctl/docker/operator/Dockerfile.gpu +85 -0
- ads/opctl/docker/operator/cuda.repo +6 -0
- ads/opctl/docker/operator/environment.yaml +8 -0
- ads/opctl/forecast.py +11 -0
- ads/opctl/index.yaml +3 -0
- ads/opctl/model/__init__.py +5 -0
- ads/opctl/model/cli.py +65 -0
- ads/opctl/model/cmds.py +73 -0
- ads/opctl/operator/README.md +4 -0
- ads/opctl/operator/__init__.py +31 -0
- ads/opctl/operator/cli.py +344 -0
- ads/opctl/operator/cmd.py +596 -0
- ads/opctl/operator/common/__init__.py +5 -0
- ads/opctl/operator/common/backend_factory.py +460 -0
- ads/opctl/operator/common/const.py +27 -0
- ads/opctl/operator/common/data/synthetic.csv +16001 -0
- ads/opctl/operator/common/dictionary_merger.py +148 -0
- ads/opctl/operator/common/errors.py +42 -0
- ads/opctl/operator/common/operator_config.py +99 -0
- ads/opctl/operator/common/operator_loader.py +811 -0
- ads/opctl/operator/common/operator_schema.yaml +130 -0
- ads/opctl/operator/common/operator_yaml_generator.py +152 -0
- ads/opctl/operator/common/utils.py +208 -0
- ads/opctl/operator/lowcode/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
- ads/opctl/operator/lowcode/anomaly/README.md +207 -0
- ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
- ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
- ads/opctl/operator/lowcode/anomaly/const.py +167 -0
- ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
- ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
- ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
- ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
- ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
- ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
- ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
- ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
- ads/opctl/operator/lowcode/common/__init__.py +5 -0
- ads/opctl/operator/lowcode/common/const.py +10 -0
- ads/opctl/operator/lowcode/common/data.py +116 -0
- ads/opctl/operator/lowcode/common/errors.py +47 -0
- ads/opctl/operator/lowcode/common/transformations.py +296 -0
- ads/opctl/operator/lowcode/common/utils.py +384 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
- ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
- ads/opctl/operator/lowcode/forecast/README.md +209 -0
- ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
- ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
- ads/opctl/operator/lowcode/forecast/const.py +92 -0
- ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
- ads/opctl/operator/lowcode/forecast/errors.py +26 -0
- ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
- ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
- ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
- ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
- ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
- ads/opctl/operator/lowcode/forecast/model/prophet.py +450 -0
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
- ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
- ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
- ads/opctl/operator/lowcode/forecast/utils.py +397 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
- ads/opctl/operator/lowcode/pii/MLoperator +17 -0
- ads/opctl/operator/lowcode/pii/README.md +208 -0
- ads/opctl/operator/lowcode/pii/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/__main__.py +78 -0
- ads/opctl/operator/lowcode/pii/cmd.py +39 -0
- ads/opctl/operator/lowcode/pii/constant.py +84 -0
- ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
- ads/opctl/operator/lowcode/pii/errors.py +27 -0
- ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
- ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
- ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
- ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
- ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
- ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
- ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
- ads/opctl/operator/lowcode/pii/model/report.py +487 -0
- ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
- ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
- ads/opctl/operator/lowcode/pii/utils.py +43 -0
- ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
- ads/opctl/operator/lowcode/recommender/README.md +206 -0
- ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
- ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
- ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
- ads/opctl/operator/lowcode/recommender/constant.py +30 -0
- ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
- ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
- ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
- ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
- ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
- ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
- ads/opctl/operator/lowcode/recommender/utils.py +13 -0
- ads/opctl/operator/runtime/__init__.py +5 -0
- ads/opctl/operator/runtime/const.py +17 -0
- ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
- ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
- ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/runtime.py +115 -0
- ads/opctl/schema.yaml.yml +36 -0
- ads/opctl/script.py +40 -0
- ads/opctl/spark/__init__.py +5 -0
- ads/opctl/spark/cli.py +43 -0
- ads/opctl/spark/cmds.py +147 -0
- ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
- ads/opctl/utils.py +344 -0
- ads/oracledb/__init__.py +5 -0
- ads/oracledb/oracle_db.py +346 -0
- ads/pipeline/__init__.py +39 -0
- ads/pipeline/ads_pipeline.py +2279 -0
- ads/pipeline/ads_pipeline_run.py +772 -0
- ads/pipeline/ads_pipeline_step.py +605 -0
- ads/pipeline/builders/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/custom_script.py +32 -0
- ads/pipeline/cli.py +119 -0
- ads/pipeline/extension.py +291 -0
- ads/pipeline/schema/__init__.py +5 -0
- ads/pipeline/schema/cs_step_schema.json +35 -0
- ads/pipeline/schema/ml_step_schema.json +31 -0
- ads/pipeline/schema/pipeline_schema.json +71 -0
- ads/pipeline/visualizer/__init__.py +5 -0
- ads/pipeline/visualizer/base.py +570 -0
- ads/pipeline/visualizer/graph_renderer.py +272 -0
- ads/pipeline/visualizer/text_renderer.py +84 -0
- ads/secrets/__init__.py +11 -0
- ads/secrets/adb.py +386 -0
- ads/secrets/auth_token.py +86 -0
- ads/secrets/big_data_service.py +365 -0
- ads/secrets/mysqldb.py +149 -0
- ads/secrets/oracledb.py +160 -0
- ads/secrets/secrets.py +407 -0
- ads/telemetry/__init__.py +7 -0
- ads/telemetry/base.py +69 -0
- ads/telemetry/client.py +122 -0
- ads/telemetry/telemetry.py +257 -0
- ads/templates/dataflow_pyspark.jinja2 +13 -0
- ads/templates/dataflow_sparksql.jinja2 +22 -0
- ads/templates/func.jinja2 +20 -0
- ads/templates/schemas/openapi.json +1740 -0
- ads/templates/score-pkl.jinja2 +173 -0
- ads/templates/score.jinja2 +322 -0
- ads/templates/score_embedding_onnx.jinja2 +202 -0
- ads/templates/score_generic.jinja2 +165 -0
- ads/templates/score_huggingface_pipeline.jinja2 +217 -0
- ads/templates/score_lightgbm.jinja2 +185 -0
- ads/templates/score_onnx.jinja2 +407 -0
- ads/templates/score_onnx_new.jinja2 +473 -0
- ads/templates/score_oracle_automl.jinja2 +185 -0
- ads/templates/score_pyspark.jinja2 +154 -0
- ads/templates/score_pytorch.jinja2 +219 -0
- ads/templates/score_scikit-learn.jinja2 +184 -0
- ads/templates/score_tensorflow.jinja2 +184 -0
- ads/templates/score_xgboost.jinja2 +178 -0
- ads/text_dataset/__init__.py +5 -0
- ads/text_dataset/backends.py +211 -0
- ads/text_dataset/dataset.py +445 -0
- ads/text_dataset/extractor.py +207 -0
- ads/text_dataset/options.py +53 -0
- ads/text_dataset/udfs.py +22 -0
- ads/text_dataset/utils.py +49 -0
- ads/type_discovery/__init__.py +9 -0
- ads/type_discovery/abstract_detector.py +21 -0
- ads/type_discovery/constant_detector.py +41 -0
- ads/type_discovery/continuous_detector.py +54 -0
- ads/type_discovery/credit_card_detector.py +99 -0
- ads/type_discovery/datetime_detector.py +92 -0
- ads/type_discovery/discrete_detector.py +118 -0
- ads/type_discovery/document_detector.py +146 -0
- ads/type_discovery/ip_detector.py +68 -0
- ads/type_discovery/latlon_detector.py +90 -0
- ads/type_discovery/phone_number_detector.py +63 -0
- ads/type_discovery/type_discovery_driver.py +87 -0
- ads/type_discovery/typed_feature.py +594 -0
- ads/type_discovery/unknown_detector.py +41 -0
- ads/type_discovery/zipcode_detector.py +48 -0
- ads/vault/__init__.py +7 -0
- ads/vault/vault.py +237 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/METADATA +150 -149
- oracle_ads-2.13.10rc0.dist-info/RECORD +858 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/WHEEL +1 -2
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/entry_points.txt +2 -1
- oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
- oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/licenses/LICENSE.txt +0 -0
ads/dataset/plot.py
ADDED
@@ -0,0 +1,710 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8; -*-
|
3
|
+
|
4
|
+
# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
|
5
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
|
+
|
7
|
+
from __future__ import print_function, absolute_import
|
8
|
+
|
9
|
+
import random
|
10
|
+
from collections import defaultdict
|
11
|
+
from math import pi
|
12
|
+
import pandas as pd
|
13
|
+
import numpy as np
|
14
|
+
|
15
|
+
import matplotlib
|
16
|
+
import matplotlib.pyplot as plt
|
17
|
+
import numpy as np
|
18
|
+
from matplotlib import colors as mcolors
|
19
|
+
from ads.dataset.helper import _log_yscale_not_set
|
20
|
+
|
21
|
+
from ads.common.decorator.runtime_dependency import (
|
22
|
+
runtime_dependency,
|
23
|
+
OptionalDependency,
|
24
|
+
)
|
25
|
+
from ads.common.utils import _log_plot_high_cardinality_warning, MAX_DISPLAY_VALUES
|
26
|
+
from ads.type_discovery.latlon_detector import LatLonDetector
|
27
|
+
from ads.type_discovery.typed_feature import (
|
28
|
+
ContinuousTypedFeature,
|
29
|
+
DateTimeTypedFeature,
|
30
|
+
ConstantTypedFeature,
|
31
|
+
DiscreteTypedFeature,
|
32
|
+
CreditCardTypedFeature,
|
33
|
+
ZipcodeTypedFeature,
|
34
|
+
OrdinalTypedFeature,
|
35
|
+
CategoricalTypedFeature,
|
36
|
+
GISTypedFeature,
|
37
|
+
)
|
38
|
+
|
39
|
+
from ads.dataset import logger
|
40
|
+
|
41
|
+
|
42
|
+
class Plotting:
|
43
|
+
def __init__(self, df, feature_types, x, y=None, plot_type="infer", yscale=None):
|
44
|
+
self.df = df
|
45
|
+
self.feature_types = feature_types
|
46
|
+
self.x = x
|
47
|
+
self.y = y
|
48
|
+
self.x_type = self.feature_types[self.x]
|
49
|
+
self.y_type = self.feature_types[self.y] if self.y is not None else None
|
50
|
+
self.plot_type = plot_type
|
51
|
+
self.yscale = yscale
|
52
|
+
|
53
|
+
def __repr__(self):
|
54
|
+
choices = self._get_plot_method() # add (plot_type='{0}') in plot method"
|
55
|
+
if len(choices) > 1:
|
56
|
+
logger.info(f"Recommended plot type is {choices[0][1].__name__}.")
|
57
|
+
|
58
|
+
logger.info(
|
59
|
+
"Available plot types are ",
|
60
|
+
", ".join([x[1].__name__ for x in choices]),
|
61
|
+
".",
|
62
|
+
)
|
63
|
+
self.show_in_notebook()
|
64
|
+
return ""
|
65
|
+
|
66
|
+
def select_best_plot(self):
|
67
|
+
"""
|
68
|
+
Returns the best plot for a given dataset
|
69
|
+
"""
|
70
|
+
|
71
|
+
#
|
72
|
+
# auto logic
|
73
|
+
#
|
74
|
+
|
75
|
+
choices = self._get_plot_method()
|
76
|
+
|
77
|
+
if len(choices) > 1:
|
78
|
+
logger.info(
|
79
|
+
"select_best_plot (%s, %s) called, possible plot types are %s"
|
80
|
+
% (
|
81
|
+
self.x_type.meta_data["type"],
|
82
|
+
self.y_type.meta_data["type"] if self.y_type is not None else "",
|
83
|
+
", ".join([x[0].__name__ for x in choices]),
|
84
|
+
)
|
85
|
+
)
|
86
|
+
|
87
|
+
if self.plot_type != "infer":
|
88
|
+
for choice in choices:
|
89
|
+
if choice[1].__name__.lower().startswith(self.plot_type.lower()):
|
90
|
+
return choice
|
91
|
+
logger.info("invalid plot_type: {}".format(self.plot_type))
|
92
|
+
raise ValueError(
|
93
|
+
"plot_type: '%s' invalid, use one of: %s"
|
94
|
+
% (self.plot_type, ", ".join([x[0].__name__ for x in choices]))
|
95
|
+
)
|
96
|
+
|
97
|
+
return choices[0]
|
98
|
+
|
99
|
+
def show_in_notebook(self, **kwargs):
|
100
|
+
|
101
|
+
"""
|
102
|
+
Visualizes the dataset by plotting the distribution of a feature or relationship between two features.
|
103
|
+
|
104
|
+
Parameters
|
105
|
+
----------
|
106
|
+
figsize: tuple
|
107
|
+
defines the size of the fig
|
108
|
+
-------
|
109
|
+
"""
|
110
|
+
plotlib_type, plot_method, plot_kwargs = self.select_best_plot()
|
111
|
+
plotlib_type(plot_method, **plot_kwargs, **kwargs)
|
112
|
+
|
113
|
+
@staticmethod
|
114
|
+
def _add_identity(ax, *line_args, **line_kwargs):
|
115
|
+
ax.plot(ax.get_xlim(), ax.get_ylim(), ls="--", c="-.3", transform=ax.transAxes)
|
116
|
+
|
117
|
+
@staticmethod
|
118
|
+
def _build_plot_key(x_type, y_type=None):
|
119
|
+
if y_type is None:
|
120
|
+
return x_type.__name__
|
121
|
+
return x_type.__name__ + "," + y_type.__name__
|
122
|
+
|
123
|
+
@staticmethod
|
124
|
+
@runtime_dependency(module="scipy", install_from=OptionalDependency.VIZ)
|
125
|
+
def _gaussian_heatmap(x, y, data, s=10, edgecolor="white", cmap=plt.cm.jet):
|
126
|
+
"""
|
127
|
+
Generate a scatter plot and assign a color to each data point based on the local density (gaussian kernel) of
|
128
|
+
points.
|
129
|
+
|
130
|
+
Parameters
|
131
|
+
----------
|
132
|
+
x: str
|
133
|
+
name of the feature
|
134
|
+
y: str
|
135
|
+
name of the feature
|
136
|
+
data: object
|
137
|
+
The dataframe that contains x and y
|
138
|
+
s: int
|
139
|
+
area of each marker
|
140
|
+
edgecolor: str
|
141
|
+
edge color of each point. string value, e.g. 'blue'
|
142
|
+
cmap: object
|
143
|
+
color map for the heatmap
|
144
|
+
|
145
|
+
Returns
|
146
|
+
-------
|
147
|
+
ax.scatter() object: a scatter plot with colored density
|
148
|
+
|
149
|
+
Raises
|
150
|
+
------
|
151
|
+
ValueError
|
152
|
+
When the columns are identical or the columns are highly correlated.
|
153
|
+
"""
|
154
|
+
try:
|
155
|
+
_x = np.array(data[x])
|
156
|
+
_y = np.array(data[y])
|
157
|
+
xy = np.vstack([_x, _y])
|
158
|
+
z = scipy.stats.gaussian_kde(xy)(xy)
|
159
|
+
sc = plt.scatter(_x, _y, c=z, s=s, edgecolor=edgecolor, cmap=cmap)
|
160
|
+
plt.xlabel(x)
|
161
|
+
plt.ylabel(y)
|
162
|
+
return plt.colorbar(sc)
|
163
|
+
except:
|
164
|
+
return plt.scatter(_x, _y)
|
165
|
+
|
166
|
+
@staticmethod
|
167
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
168
|
+
def _categorical_vs_continuous_violin_plot(x, y, data):
|
169
|
+
# when x is categorical, we set the order based on counts of each category
|
170
|
+
vc = data[x].value_counts()
|
171
|
+
if len(vc.keys()) > 10:
|
172
|
+
_log_plot_high_cardinality_warning(x, len(vc.keys()))
|
173
|
+
idxes = vc[:10].index
|
174
|
+
else:
|
175
|
+
idxes = vc.index
|
176
|
+
seaborn.violinplot(x=x, y=y, data=data[data[x].isin(idxes)], order=idxes)
|
177
|
+
|
178
|
+
@staticmethod
|
179
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
180
|
+
def _ordinal_vs_continuous_violin_plot(x, y, data):
|
181
|
+
# when x is ordinal, we want to get a natural order to x values
|
182
|
+
vals = np.array(data[x].values)
|
183
|
+
sorted_x = list(np.sort(vals))
|
184
|
+
# get the frequency of each distinct element in the list using a dictionary.
|
185
|
+
freq = {}
|
186
|
+
for items in sorted_x:
|
187
|
+
freq[items] = sorted_x.count(items)
|
188
|
+
if len(freq) > MAX_DISPLAY_VALUES:
|
189
|
+
_log_plot_high_cardinality_warning(x, len(freq))
|
190
|
+
idxes = list(freq.keys())[:10]
|
191
|
+
else:
|
192
|
+
idxes = freq.keys()
|
193
|
+
seaborn.violinplot(x=x, y=y, data=data[data[x].isin(idxes)], order=idxes)
|
194
|
+
|
195
|
+
@staticmethod
|
196
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
197
|
+
def _categorical_vs_continuous_horizontal_violin_plot(x, y, data):
|
198
|
+
vc = data[x].value_counts()
|
199
|
+
if len(vc.keys()) > 10:
|
200
|
+
_log_plot_high_cardinality_warning(x, len(vc.keys()))
|
201
|
+
idxes = vc[:10].index
|
202
|
+
else:
|
203
|
+
idxes = vc.index
|
204
|
+
seaborn.violinplot(
|
205
|
+
x=y, y=x, data=data[data[x].isin(idxes)], order=idxes, orient="h"
|
206
|
+
)
|
207
|
+
|
208
|
+
@staticmethod
|
209
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
210
|
+
def _ordinal_vs_continuous_horizontal_violin_plot(x, y, data):
|
211
|
+
# when x is ordinal, we want to get a natural order to x values
|
212
|
+
vals = np.array(data[x].values)
|
213
|
+
sorted_x = list(np.sort(vals))
|
214
|
+
# get the frequency of each distinct element in the list using a dictionary.
|
215
|
+
freq = {}
|
216
|
+
for items in sorted_x:
|
217
|
+
freq[items] = sorted_x.count(items)
|
218
|
+
if len(freq) > MAX_DISPLAY_VALUES:
|
219
|
+
_log_plot_high_cardinality_warning(x, len(freq))
|
220
|
+
idxes = list(freq.keys())[:10]
|
221
|
+
else:
|
222
|
+
idxes = freq.keys()
|
223
|
+
seaborn.violinplot(
|
224
|
+
x=y, y=x, data=data[data[x].isin(idxes)], order=idxes, orient="h"
|
225
|
+
)
|
226
|
+
|
227
|
+
@staticmethod
|
228
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
229
|
+
def _categorical_vs_continuous_box_plot(x, y, data):
|
230
|
+
vc = data[x].value_counts()
|
231
|
+
if len(vc.keys()) > 10:
|
232
|
+
_log_plot_high_cardinality_warning(x, len(vc.keys()))
|
233
|
+
idxes = vc[:10].index
|
234
|
+
else:
|
235
|
+
idxes = vc.index
|
236
|
+
seaborn.boxplot(x=x, y=y, data=data[data[x].isin(idxes)], order=idxes)
|
237
|
+
|
238
|
+
@staticmethod
|
239
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
240
|
+
def _ordinal_vs_continuous_box_plot(x, y, data):
|
241
|
+
# when x is ordinal, we want to get a natural order to x values
|
242
|
+
vals = np.array(data[x].values)
|
243
|
+
sorted_x = list(np.sort(vals))
|
244
|
+
# get the frequency of each distinct element in the list using a dictionary.
|
245
|
+
freq = {}
|
246
|
+
for items in sorted_x:
|
247
|
+
freq[items] = sorted_x.count(items)
|
248
|
+
if len(freq) > MAX_DISPLAY_VALUES:
|
249
|
+
_log_plot_high_cardinality_warning(x, len(freq))
|
250
|
+
idxes = list(freq.keys())[:10]
|
251
|
+
else:
|
252
|
+
idxes = freq.keys()
|
253
|
+
seaborn.boxplot(x=x, y=y, data=data[data[x].isin(idxes)], order=idxes)
|
254
|
+
|
255
|
+
@staticmethod
|
256
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
257
|
+
def _ordinal_vs_continuous_horizontal_box_plot(x, y, data):
|
258
|
+
# when x is ordinal, we want to get a natural order to x values
|
259
|
+
vals = np.array(data[x].values)
|
260
|
+
sorted_x = list(np.sort(vals))
|
261
|
+
# get the frequency of each distinct element in the list using a dictionary.
|
262
|
+
freq = {}
|
263
|
+
for items in sorted_x:
|
264
|
+
freq[items] = sorted_x.count(items)
|
265
|
+
if len(freq) > MAX_DISPLAY_VALUES:
|
266
|
+
_log_plot_high_cardinality_warning(x, len(freq))
|
267
|
+
idxes = list(freq.keys())[:10]
|
268
|
+
else:
|
269
|
+
idxes = freq.keys()
|
270
|
+
seaborn.boxplot(
|
271
|
+
x=y, y=x, data=data[data[x].isin(idxes)], order=idxes, orient="h"
|
272
|
+
)
|
273
|
+
|
274
|
+
@staticmethod
|
275
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
276
|
+
def _count_plot(x, hue, data, yscale=None):
|
277
|
+
if not yscale:
|
278
|
+
_log_yscale_not_set()
|
279
|
+
# get the copy of data and convert categorical data type to object
|
280
|
+
data = data.copy()
|
281
|
+
data[hue] = data[hue].astype("object")
|
282
|
+
# get cardinality of categorical values
|
283
|
+
cat_cardi = data[hue].value_counts()
|
284
|
+
if len(cat_cardi) > 5:
|
285
|
+
top_categoricals = cat_cardi[:5].index
|
286
|
+
# modify the data to replace non top 5 categorical values to be the same value
|
287
|
+
data[hue] = np.where(
|
288
|
+
data[hue].isin(top_categoricals), data[hue], "all_other_categories"
|
289
|
+
)
|
290
|
+
cat_index = data[hue].value_counts().index
|
291
|
+
else:
|
292
|
+
cat_index = cat_cardi.index
|
293
|
+
# get cardinality of ordinal values
|
294
|
+
ordi_cardi = data[x].value_counts()
|
295
|
+
if len(ordi_cardi) > 10:
|
296
|
+
# bin the values and sort from small to large
|
297
|
+
data[x] = pd.cut(data[x], 10, precision=0)
|
298
|
+
data[x] = data[x].apply(lambda k: pd.Interval(int(k.left), int(k.right)))
|
299
|
+
g = seaborn.countplot(x=x, hue=hue, data=data, hue_order=cat_index)
|
300
|
+
if yscale:
|
301
|
+
g.set_yscale(yscale)
|
302
|
+
|
303
|
+
@staticmethod
|
304
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
305
|
+
def _ordinal_count_plot(x, data, yscale=None):
|
306
|
+
if not yscale:
|
307
|
+
_log_yscale_not_set()
|
308
|
+
data = data.copy()
|
309
|
+
ordi_cardi = data[x].value_counts()
|
310
|
+
if len(ordi_cardi) > 20:
|
311
|
+
intervals = pd.cut(data[x], 20, precision=0)
|
312
|
+
intervals = intervals.apply(
|
313
|
+
lambda k: pd.Interval(int(k.left), int(k.right))
|
314
|
+
)
|
315
|
+
else:
|
316
|
+
intervals = data[x]
|
317
|
+
g = seaborn.countplot(x=intervals, color="#1f77b4")
|
318
|
+
if yscale:
|
319
|
+
g.set_yscale(yscale)
|
320
|
+
|
321
|
+
@staticmethod
|
322
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
323
|
+
def _ordinal_vs_constant_count_plot(x, hue, data, yscale=None):
|
324
|
+
if not yscale:
|
325
|
+
_log_yscale_not_set()
|
326
|
+
data = data.copy()
|
327
|
+
intervals = pd.cut(data[x], 20, precision=0)
|
328
|
+
intervals = intervals.apply(lambda k: pd.Interval(int(k.left), int(k.right)))
|
329
|
+
g = seaborn.countplot(x=intervals, hue=hue, data=data)
|
330
|
+
if yscale:
|
331
|
+
g.set_yscale(yscale)
|
332
|
+
|
333
|
+
@staticmethod
|
334
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
335
|
+
def _single_column_count_plot(x, data, yscale=None):
|
336
|
+
if not yscale:
|
337
|
+
_log_yscale_not_set()
|
338
|
+
order = data[x].value_counts().iloc[:24].index
|
339
|
+
g = seaborn.countplot(x=x, data=data, order=order)
|
340
|
+
if yscale:
|
341
|
+
g.set_yscale(yscale)
|
342
|
+
|
343
|
+
@staticmethod
|
344
|
+
@runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
|
345
|
+
@runtime_dependency(module="folium", install_from=OptionalDependency.VIZ)
|
346
|
+
def _folium_map(x, data):
|
347
|
+
import folium.plugins
|
348
|
+
df = LatLonDetector.extract_x_y(data[x])
|
349
|
+
lat_min, lat_max, long_min, long_max = (
|
350
|
+
min(df.Y),
|
351
|
+
max(df.Y),
|
352
|
+
min(df.X),
|
353
|
+
max(df.X),
|
354
|
+
)
|
355
|
+
m = folium.Map(tiles="Stamen Terrain", zoom_control=False)
|
356
|
+
|
357
|
+
folium.plugins.HeatMap(df[["Y", "X"]]).add_to(m)
|
358
|
+
m.fit_bounds([[lat_min, long_min], [lat_max, long_max]])
|
359
|
+
|
360
|
+
from IPython.core.display import display
|
361
|
+
|
362
|
+
display(m)
|
363
|
+
|
364
|
+
@staticmethod
|
365
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
366
|
+
def _single_pdf(x, y, data):
|
367
|
+
seaborn.kdeplot(data[x], shade=True, shade_lowest=False)
|
368
|
+
|
369
|
+
@staticmethod
|
370
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
371
|
+
def _multiple_pdf(x, y, data):
|
372
|
+
colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS)
|
373
|
+
hues = [
|
374
|
+
colors[x]
|
375
|
+
for x in colors.keys()
|
376
|
+
if isinstance(colors[x], str) and colors[x].startswith("#")
|
377
|
+
]
|
378
|
+
|
379
|
+
for i, cat in enumerate(list(data[y].unique())):
|
380
|
+
s = data.loc[data[y] == cat][x]
|
381
|
+
color = random.choice(hues)
|
382
|
+
seaborn.kdeplot(s, color=color, shade=True, shade_lowest=False, label=cat)
|
383
|
+
plt.xlabel(x)
|
384
|
+
plt.ylabel(y)
|
385
|
+
|
386
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
387
|
+
def _matplot(self, plot_method, figsize=(4, 3), **kwargs):
|
388
|
+
|
389
|
+
plt.style.use("seaborn-white")
|
390
|
+
|
391
|
+
plt.rc("xtick", labelsize="x-small")
|
392
|
+
plt.rc("ytick", labelsize="x-small")
|
393
|
+
plt.rc("font", size=8)
|
394
|
+
plt.rc("figure", dpi=144)
|
395
|
+
|
396
|
+
fig = plt.figure(figsize=figsize)
|
397
|
+
|
398
|
+
#
|
399
|
+
# generate a title for the plot
|
400
|
+
#
|
401
|
+
text = '{}, "{}" ({})'.format(
|
402
|
+
plot_method.__name__.upper(), self.x, self.feature_types[self.x].type
|
403
|
+
)
|
404
|
+
if self.y:
|
405
|
+
text = '{} vs "{}" ({})'.format(
|
406
|
+
text, self.y, self.feature_types[self.y].type
|
407
|
+
)
|
408
|
+
|
409
|
+
plt.title(text, y=1.08)
|
410
|
+
plt.grid(linestyle="dotted")
|
411
|
+
|
412
|
+
# draw a 45 degree dotted or dashed line indicating equality when plot method is scatter plots and the span of x equals the span of y
|
413
|
+
if (
|
414
|
+
plot_method is plt.scatter or plot_method is Plotting._gaussian_heatmap
|
415
|
+
) and (self.df[kwargs["x"]].values.ptp() == self.df[kwargs["y"]].values.ptp()):
|
416
|
+
Plotting._add_identity(fig.axes[0], color="grey", ls="--")
|
417
|
+
|
418
|
+
plot_method(**kwargs, data=self.df)
|
419
|
+
|
420
|
+
# set x and y axis label
|
421
|
+
if self.y:
|
422
|
+
plt.ylabel(self.y)
|
423
|
+
if self.x:
|
424
|
+
plt.xlabel(self.x)
|
425
|
+
|
426
|
+
# rename the y-axis label and x-axis label when "count" is the y-axis label
|
427
|
+
if self.y == "count":
|
428
|
+
plt.xlabel("Column: {} values ".format(self.x))
|
429
|
+
plt.ylabel("instance count")
|
430
|
+
|
431
|
+
# add y-axis label as "count" when plot type is hist
|
432
|
+
if plot_method is plt.hist:
|
433
|
+
plt.ylabel("instance count")
|
434
|
+
# add tickmark on x-axis to see labeled values on x-axis in historgram. It has 30 intervals because that's the most tickmarks on graph can fit.
|
435
|
+
plt.xticks(
|
436
|
+
np.arange(
|
437
|
+
min(self.df[kwargs["x"]].values),
|
438
|
+
max(self.df[kwargs["x"]].values) + 1,
|
439
|
+
(
|
440
|
+
max(self.df[kwargs["x"]].values)
|
441
|
+
- min(self.df[kwargs["x"]].values)
|
442
|
+
)
|
443
|
+
/ 30,
|
444
|
+
)
|
445
|
+
)
|
446
|
+
# override y-axis label as "count" when plot type is _count_plot or countplot
|
447
|
+
if plot_method is Plotting._count_plot or plot_method is seaborn.countplot:
|
448
|
+
plt.ylabel("count")
|
449
|
+
|
450
|
+
plt.xticks(rotation=90)
|
451
|
+
|
452
|
+
def _generic_plot(self, plot_method, **kwargs):
|
453
|
+
plot_method(**kwargs, data=self.df)
|
454
|
+
|
455
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
456
|
+
def _get_plot_method(self):
|
457
|
+
|
458
|
+
#
|
459
|
+
# combos contains a dictionary with the key being a composite of the x and y types, the value will
|
460
|
+
# always be a list, possibly and empty list, indicating no match for combination
|
461
|
+
#
|
462
|
+
#
|
463
|
+
|
464
|
+
combos = defaultdict(list)
|
465
|
+
|
466
|
+
combos[
|
467
|
+
Plotting._build_plot_key(CategoricalTypedFeature, ContinuousTypedFeature)
|
468
|
+
] = [
|
469
|
+
(
|
470
|
+
self._matplot,
|
471
|
+
Plotting._categorical_vs_continuous_violin_plot,
|
472
|
+
{"x": self.x, "y": self.y},
|
473
|
+
),
|
474
|
+
(
|
475
|
+
self._matplot,
|
476
|
+
Plotting._categorical_vs_continuous_box_plot,
|
477
|
+
{"x": self.x, "y": self.y},
|
478
|
+
),
|
479
|
+
]
|
480
|
+
combos[
|
481
|
+
Plotting._build_plot_key(OrdinalTypedFeature, ContinuousTypedFeature)
|
482
|
+
] = [
|
483
|
+
(
|
484
|
+
self._matplot,
|
485
|
+
Plotting._ordinal_vs_continuous_violin_plot,
|
486
|
+
{"x": self.x, "y": self.y},
|
487
|
+
),
|
488
|
+
(
|
489
|
+
self._matplot,
|
490
|
+
Plotting._ordinal_vs_continuous_box_plot,
|
491
|
+
{"x": self.x, "y": self.y},
|
492
|
+
),
|
493
|
+
]
|
494
|
+
combos[
|
495
|
+
Plotting._build_plot_key(ContinuousTypedFeature, OrdinalTypedFeature)
|
496
|
+
] = [
|
497
|
+
(
|
498
|
+
self._matplot,
|
499
|
+
Plotting._ordinal_vs_continuous_horizontal_violin_plot,
|
500
|
+
{"x": self.y, "y": self.x},
|
501
|
+
),
|
502
|
+
(
|
503
|
+
self._matplot,
|
504
|
+
Plotting._ordinal_vs_continuous_horizontal_box_plot,
|
505
|
+
{"x": self.y, "y": self.x},
|
506
|
+
),
|
507
|
+
]
|
508
|
+
combos[
|
509
|
+
Plotting._build_plot_key(ContinuousTypedFeature, CategoricalTypedFeature)
|
510
|
+
] = [(self._matplot, Plotting._multiple_pdf, {"x": self.x, "y": self.y})]
|
511
|
+
combos[
|
512
|
+
Plotting._build_plot_key(ConstantTypedFeature, ContinuousTypedFeature)
|
513
|
+
] = [(self._matplot, seaborn.barplot, {"x": self.x, "y": self.y})]
|
514
|
+
combos[
|
515
|
+
Plotting._build_plot_key(ContinuousTypedFeature, ConstantTypedFeature)
|
516
|
+
] = [(self._matplot, Plotting._single_pdf, {"x": self.x, "y": self.y})]
|
517
|
+
combos[Plotting._build_plot_key(ConstantTypedFeature, DiscreteTypedFeature)] = [
|
518
|
+
(self._matplot, seaborn.barplot, {"x": self.x, "y": self.y})
|
519
|
+
]
|
520
|
+
combos[Plotting._build_plot_key(DiscreteTypedFeature, ConstantTypedFeature)] = [
|
521
|
+
(
|
522
|
+
self._matplot,
|
523
|
+
Plotting._ordinal_vs_constant_count_plot,
|
524
|
+
{"x": self.x, "hue": self.y, "yscale": self.yscale},
|
525
|
+
)
|
526
|
+
]
|
527
|
+
combos[
|
528
|
+
Plotting._build_plot_key(DateTimeTypedFeature, ContinuousTypedFeature)
|
529
|
+
] = [
|
530
|
+
(
|
531
|
+
self._matplot,
|
532
|
+
plt.scatter,
|
533
|
+
{
|
534
|
+
"x": self.x,
|
535
|
+
"y": self.y,
|
536
|
+
"s": pi / 10 * (matplotlib.rcParams["lines.markersize"] ** 2),
|
537
|
+
"edgecolor": "white",
|
538
|
+
"linewidths": "0.1",
|
539
|
+
},
|
540
|
+
)
|
541
|
+
]
|
542
|
+
combos[Plotting._build_plot_key(DateTimeTypedFeature, OrdinalTypedFeature)] = [
|
543
|
+
(
|
544
|
+
self._matplot,
|
545
|
+
plt.scatter,
|
546
|
+
{"x": self.x, "y": self.y, "edgecolor": "white", "linewidths": "0.1"},
|
547
|
+
)
|
548
|
+
]
|
549
|
+
combos[
|
550
|
+
Plotting._build_plot_key(ContinuousTypedFeature, ContinuousTypedFeature)
|
551
|
+
] = [
|
552
|
+
(
|
553
|
+
self._matplot,
|
554
|
+
Plotting._gaussian_heatmap,
|
555
|
+
{
|
556
|
+
"x": self.x,
|
557
|
+
"y": self.y,
|
558
|
+
"s": pi / 10 * (matplotlib.rcParams["lines.markersize"] ** 2),
|
559
|
+
},
|
560
|
+
),
|
561
|
+
(
|
562
|
+
self._matplot,
|
563
|
+
plt.scatter,
|
564
|
+
{"x": self.x, "y": self.y, "edgecolor": "white", "linewidths": "0.1"},
|
565
|
+
),
|
566
|
+
]
|
567
|
+
combos[Plotting._build_plot_key(OrdinalTypedFeature, OrdinalTypedFeature)] = [
|
568
|
+
(
|
569
|
+
self._matplot,
|
570
|
+
seaborn.scatterplot,
|
571
|
+
{
|
572
|
+
"x": self.x,
|
573
|
+
"y": self.y,
|
574
|
+
"s": pi / 10 * (matplotlib.rcParams["lines.markersize"] ** 2),
|
575
|
+
"edgecolor": "white",
|
576
|
+
"linewidths": "0.1",
|
577
|
+
},
|
578
|
+
)
|
579
|
+
]
|
580
|
+
combos[Plotting._build_plot_key(OrdinalTypedFeature, DiscreteTypedFeature)] = [
|
581
|
+
(self._matplot, seaborn.countplot, {"x": self.x, "hue": self.y})
|
582
|
+
]
|
583
|
+
combos[
|
584
|
+
Plotting._build_plot_key(OrdinalTypedFeature, CategoricalTypedFeature)
|
585
|
+
] = [
|
586
|
+
(
|
587
|
+
self._matplot,
|
588
|
+
Plotting._count_plot,
|
589
|
+
{"x": self.x, "hue": self.y, "yscale": self.yscale},
|
590
|
+
)
|
591
|
+
]
|
592
|
+
combos[
|
593
|
+
Plotting._build_plot_key(CategoricalTypedFeature, OrdinalTypedFeature)
|
594
|
+
] = [
|
595
|
+
(
|
596
|
+
self._matplot,
|
597
|
+
Plotting._count_plot,
|
598
|
+
{"x": self.y, "hue": self.x, "yscale": self.yscale},
|
599
|
+
)
|
600
|
+
]
|
601
|
+
combos[Plotting._build_plot_key(DiscreteTypedFeature, OrdinalTypedFeature)] = [
|
602
|
+
(
|
603
|
+
self._matplot,
|
604
|
+
seaborn.countplot,
|
605
|
+
{
|
606
|
+
"x": self.x,
|
607
|
+
"hue": self.y,
|
608
|
+
"order": self.df[self.x]
|
609
|
+
.value_counts(ascending=True)
|
610
|
+
.iloc[:10]
|
611
|
+
.index,
|
612
|
+
},
|
613
|
+
)
|
614
|
+
]
|
615
|
+
combos[
|
616
|
+
Plotting._build_plot_key(DiscreteTypedFeature, CategoricalTypedFeature)
|
617
|
+
] = [
|
618
|
+
(
|
619
|
+
self._matplot,
|
620
|
+
seaborn.countplot,
|
621
|
+
{
|
622
|
+
"x": self.x,
|
623
|
+
"hue": self.y,
|
624
|
+
"order": self.df[self.x].value_counts().iloc[:10].index,
|
625
|
+
},
|
626
|
+
)
|
627
|
+
]
|
628
|
+
combos[Plotting._build_plot_key(DateTimeTypedFeature, DateTimeTypedFeature)] = [
|
629
|
+
(
|
630
|
+
self._matplot,
|
631
|
+
plt.scatter,
|
632
|
+
{
|
633
|
+
"x": self.x,
|
634
|
+
"y": self.y,
|
635
|
+
"s": pi / 10 * (matplotlib.rcParams["lines.markersize"] ** 2),
|
636
|
+
"edgecolor": "white",
|
637
|
+
"linewidths": "0.1",
|
638
|
+
},
|
639
|
+
)
|
640
|
+
]
|
641
|
+
|
642
|
+
combos[Plotting._build_plot_key(ContinuousTypedFeature, None)] = [
|
643
|
+
(self._matplot, plt.hist, {"x": self.x})
|
644
|
+
]
|
645
|
+
combos[Plotting._build_plot_key(CategoricalTypedFeature, None)] = [
|
646
|
+
(
|
647
|
+
self._matplot,
|
648
|
+
Plotting._single_column_count_plot,
|
649
|
+
{"x": self.x, "yscale": self.yscale},
|
650
|
+
)
|
651
|
+
]
|
652
|
+
combos[Plotting._build_plot_key(OrdinalTypedFeature, None)] = [
|
653
|
+
(
|
654
|
+
self._matplot,
|
655
|
+
Plotting._ordinal_count_plot,
|
656
|
+
{"x": self.x, "yscale": self.yscale},
|
657
|
+
)
|
658
|
+
]
|
659
|
+
combos[Plotting._build_plot_key(ConstantTypedFeature, None)] = [
|
660
|
+
(self._matplot, seaborn.countplot, {"x": self.x})
|
661
|
+
]
|
662
|
+
combos[Plotting._build_plot_key(DateTimeTypedFeature, None)] = [
|
663
|
+
(self._matplot, plt.hist, {"x": self.x, "bins": 10, "color": "#1f77b4"})
|
664
|
+
]
|
665
|
+
combos[Plotting._build_plot_key(GISTypedFeature, None)] = [
|
666
|
+
(self._generic_plot, Plotting._folium_map, {"x": self.x})
|
667
|
+
]
|
668
|
+
|
669
|
+
y_type_name = None if self.y_type is None else self.y_type.__class__
|
670
|
+
keys_to_check = list(
|
671
|
+
[Plotting._build_plot_key(self.x_type.__class__, y_type_name)]
|
672
|
+
)
|
673
|
+
|
674
|
+
new_x_type = Plotting._change_type(self.x_type)
|
675
|
+
new_y_type = Plotting._change_type(self.y_type)
|
676
|
+
keys_to_check.append(Plotting._build_plot_key(new_x_type, y_type_name))
|
677
|
+
keys_to_check.append(
|
678
|
+
Plotting._build_plot_key(self.x_type.__class__, new_y_type)
|
679
|
+
)
|
680
|
+
keys_to_check.append(Plotting._build_plot_key(new_x_type, new_y_type))
|
681
|
+
for key in keys_to_check:
|
682
|
+
if key in combos and combos[key]:
|
683
|
+
assert isinstance(combos[key][0], tuple)
|
684
|
+
return combos[key]
|
685
|
+
|
686
|
+
if y_type_name is not None:
|
687
|
+
raise NotImplementedError(
|
688
|
+
"Plotting for the feature combination ({0} vs {1}) is not yet supported.".format(
|
689
|
+
self.x_type.meta_data["type"], self.y_type.meta_data["type"]
|
690
|
+
)
|
691
|
+
)
|
692
|
+
else:
|
693
|
+
raise NotImplementedError(
|
694
|
+
"Plotting for feature type {0} is not supported".format(
|
695
|
+
self.x_type.meta_data["type"]
|
696
|
+
)
|
697
|
+
)
|
698
|
+
|
699
|
+
def _change_type(feature_type):
|
700
|
+
if feature_type is None:
|
701
|
+
return None
|
702
|
+
|
703
|
+
return (
|
704
|
+
DiscreteTypedFeature
|
705
|
+
if isinstance(feature_type, DiscreteTypedFeature)
|
706
|
+
or isinstance(feature_type, CreditCardTypedFeature)
|
707
|
+
or isinstance(feature_type, ZipcodeTypedFeature)
|
708
|
+
or isinstance(feature_type, OrdinalTypedFeature)
|
709
|
+
else feature_type.__class__
|
710
|
+
)
|