oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/__init__.py +40 -0
- ads/aqua/app.py +507 -0
- ads/aqua/cli.py +96 -0
- ads/aqua/client/__init__.py +3 -0
- ads/aqua/client/client.py +836 -0
- ads/aqua/client/openai_client.py +305 -0
- ads/aqua/common/__init__.py +5 -0
- ads/aqua/common/decorator.py +125 -0
- ads/aqua/common/entities.py +274 -0
- ads/aqua/common/enums.py +134 -0
- ads/aqua/common/errors.py +109 -0
- ads/aqua/common/utils.py +1295 -0
- ads/aqua/config/__init__.py +4 -0
- ads/aqua/config/container_config.py +246 -0
- ads/aqua/config/evaluation/__init__.py +4 -0
- ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
- ads/aqua/config/utils/__init__.py +4 -0
- ads/aqua/config/utils/serializer.py +339 -0
- ads/aqua/constants.py +116 -0
- ads/aqua/data.py +14 -0
- ads/aqua/dummy_data/icon.txt +1 -0
- ads/aqua/dummy_data/oci_model_deployments.json +56 -0
- ads/aqua/dummy_data/oci_models.json +1 -0
- ads/aqua/dummy_data/readme.md +26 -0
- ads/aqua/evaluation/__init__.py +8 -0
- ads/aqua/evaluation/constants.py +53 -0
- ads/aqua/evaluation/entities.py +186 -0
- ads/aqua/evaluation/errors.py +70 -0
- ads/aqua/evaluation/evaluation.py +1814 -0
- ads/aqua/extension/__init__.py +42 -0
- ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
- ads/aqua/extension/base_handler.py +90 -0
- ads/aqua/extension/common_handler.py +121 -0
- ads/aqua/extension/common_ws_msg_handler.py +36 -0
- ads/aqua/extension/deployment_handler.py +381 -0
- ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
- ads/aqua/extension/errors.py +30 -0
- ads/aqua/extension/evaluation_handler.py +129 -0
- ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
- ads/aqua/extension/finetune_handler.py +96 -0
- ads/aqua/extension/model_handler.py +390 -0
- ads/aqua/extension/models/__init__.py +0 -0
- ads/aqua/extension/models/ws_models.py +145 -0
- ads/aqua/extension/models_ws_msg_handler.py +50 -0
- ads/aqua/extension/ui_handler.py +300 -0
- ads/aqua/extension/ui_websocket_handler.py +130 -0
- ads/aqua/extension/utils.py +133 -0
- ads/aqua/finetuning/__init__.py +7 -0
- ads/aqua/finetuning/constants.py +23 -0
- ads/aqua/finetuning/entities.py +181 -0
- ads/aqua/finetuning/finetuning.py +749 -0
- ads/aqua/model/__init__.py +8 -0
- ads/aqua/model/constants.py +60 -0
- ads/aqua/model/entities.py +385 -0
- ads/aqua/model/enums.py +32 -0
- ads/aqua/model/model.py +2134 -0
- ads/aqua/model/utils.py +52 -0
- ads/aqua/modeldeployment/__init__.py +6 -0
- ads/aqua/modeldeployment/constants.py +10 -0
- ads/aqua/modeldeployment/deployment.py +1315 -0
- ads/aqua/modeldeployment/entities.py +653 -0
- ads/aqua/modeldeployment/utils.py +543 -0
- ads/aqua/resources/gpu_shapes_index.json +94 -0
- ads/aqua/server/__init__.py +4 -0
- ads/aqua/server/__main__.py +24 -0
- ads/aqua/server/app.py +47 -0
- ads/aqua/server/aqua_spec.yml +1291 -0
- ads/aqua/training/__init__.py +4 -0
- ads/aqua/training/exceptions.py +476 -0
- ads/aqua/ui.py +519 -0
- ads/automl/__init__.py +9 -0
- ads/automl/driver.py +330 -0
- ads/automl/provider.py +975 -0
- ads/bds/__init__.py +5 -0
- ads/bds/auth.py +127 -0
- ads/bds/big_data_service.py +255 -0
- ads/catalog/__init__.py +19 -0
- ads/catalog/model.py +1576 -0
- ads/catalog/notebook.py +461 -0
- ads/catalog/project.py +468 -0
- ads/catalog/summary.py +178 -0
- ads/common/__init__.py +11 -0
- ads/common/analyzer.py +65 -0
- ads/common/artifact/.model-ignore +63 -0
- ads/common/artifact/__init__.py +10 -0
- ads/common/auth.py +1122 -0
- ads/common/card_identifier.py +83 -0
- ads/common/config.py +647 -0
- ads/common/data.py +165 -0
- ads/common/decorator/__init__.py +9 -0
- ads/common/decorator/argument_to_case.py +88 -0
- ads/common/decorator/deprecate.py +69 -0
- ads/common/decorator/require_nonempty_arg.py +65 -0
- ads/common/decorator/runtime_dependency.py +178 -0
- ads/common/decorator/threaded.py +97 -0
- ads/common/decorator/utils.py +35 -0
- ads/common/dsc_file_system.py +303 -0
- ads/common/error.py +14 -0
- ads/common/extended_enum.py +81 -0
- ads/common/function/__init__.py +5 -0
- ads/common/function/fn_util.py +142 -0
- ads/common/function/func_conf.yaml +25 -0
- ads/common/ipython.py +76 -0
- ads/common/model.py +679 -0
- ads/common/model_artifact.py +1759 -0
- ads/common/model_artifact_schema.json +107 -0
- ads/common/model_export_util.py +664 -0
- ads/common/model_metadata.py +24 -0
- ads/common/object_storage_details.py +296 -0
- ads/common/oci_client.py +179 -0
- ads/common/oci_datascience.py +46 -0
- ads/common/oci_logging.py +1144 -0
- ads/common/oci_mixin.py +957 -0
- ads/common/oci_resource.py +136 -0
- ads/common/serializer.py +559 -0
- ads/common/utils.py +1852 -0
- ads/common/word_lists.py +1491 -0
- ads/common/work_request.py +189 -0
- ads/config.py +1 -0
- ads/data_labeling/__init__.py +13 -0
- ads/data_labeling/boundingbox.py +253 -0
- ads/data_labeling/constants.py +47 -0
- ads/data_labeling/data_labeling_service.py +244 -0
- ads/data_labeling/interface/__init__.py +5 -0
- ads/data_labeling/interface/loader.py +16 -0
- ads/data_labeling/interface/parser.py +16 -0
- ads/data_labeling/interface/reader.py +23 -0
- ads/data_labeling/loader/__init__.py +5 -0
- ads/data_labeling/loader/file_loader.py +241 -0
- ads/data_labeling/metadata.py +110 -0
- ads/data_labeling/mixin/__init__.py +5 -0
- ads/data_labeling/mixin/data_labeling.py +232 -0
- ads/data_labeling/ner.py +129 -0
- ads/data_labeling/parser/__init__.py +5 -0
- ads/data_labeling/parser/dls_record_parser.py +388 -0
- ads/data_labeling/parser/export_metadata_parser.py +94 -0
- ads/data_labeling/parser/export_record_parser.py +473 -0
- ads/data_labeling/reader/__init__.py +5 -0
- ads/data_labeling/reader/dataset_reader.py +574 -0
- ads/data_labeling/reader/dls_record_reader.py +121 -0
- ads/data_labeling/reader/export_record_reader.py +62 -0
- ads/data_labeling/reader/jsonl_reader.py +75 -0
- ads/data_labeling/reader/metadata_reader.py +203 -0
- ads/data_labeling/reader/record_reader.py +263 -0
- ads/data_labeling/record.py +52 -0
- ads/data_labeling/visualizer/__init__.py +5 -0
- ads/data_labeling/visualizer/image_visualizer.py +525 -0
- ads/data_labeling/visualizer/text_visualizer.py +357 -0
- ads/database/__init__.py +5 -0
- ads/database/connection.py +338 -0
- ads/dataset/__init__.py +10 -0
- ads/dataset/capabilities.md +51 -0
- ads/dataset/classification_dataset.py +339 -0
- ads/dataset/correlation.py +226 -0
- ads/dataset/correlation_plot.py +563 -0
- ads/dataset/dask_series.py +173 -0
- ads/dataset/dataframe_transformer.py +110 -0
- ads/dataset/dataset.py +1979 -0
- ads/dataset/dataset_browser.py +360 -0
- ads/dataset/dataset_with_target.py +995 -0
- ads/dataset/exception.py +25 -0
- ads/dataset/factory.py +987 -0
- ads/dataset/feature_engineering_transformer.py +35 -0
- ads/dataset/feature_selection.py +107 -0
- ads/dataset/forecasting_dataset.py +26 -0
- ads/dataset/helper.py +1450 -0
- ads/dataset/label_encoder.py +99 -0
- ads/dataset/mixin/__init__.py +5 -0
- ads/dataset/mixin/dataset_accessor.py +134 -0
- ads/dataset/pipeline.py +58 -0
- ads/dataset/plot.py +710 -0
- ads/dataset/progress.py +86 -0
- ads/dataset/recommendation.py +297 -0
- ads/dataset/recommendation_transformer.py +502 -0
- ads/dataset/regression_dataset.py +14 -0
- ads/dataset/sampled_dataset.py +1050 -0
- ads/dataset/target.py +98 -0
- ads/dataset/timeseries.py +18 -0
- ads/dbmixin/__init__.py +5 -0
- ads/dbmixin/db_pandas_accessor.py +153 -0
- ads/environment/__init__.py +9 -0
- ads/environment/ml_runtime.py +66 -0
- ads/evaluations/README.md +14 -0
- ads/evaluations/__init__.py +109 -0
- ads/evaluations/evaluation_plot.py +983 -0
- ads/evaluations/evaluator.py +1334 -0
- ads/evaluations/statistical_metrics.py +543 -0
- ads/experiments/__init__.py +9 -0
- ads/experiments/capabilities.md +0 -0
- ads/explanations/__init__.py +21 -0
- ads/explanations/base_explainer.py +142 -0
- ads/explanations/capabilities.md +83 -0
- ads/explanations/explainer.py +190 -0
- ads/explanations/mlx_global_explainer.py +1050 -0
- ads/explanations/mlx_interface.py +386 -0
- ads/explanations/mlx_local_explainer.py +287 -0
- ads/explanations/mlx_whatif_explainer.py +201 -0
- ads/feature_engineering/__init__.py +20 -0
- ads/feature_engineering/accessor/__init__.py +5 -0
- ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
- ads/feature_engineering/accessor/mixin/__init__.py +5 -0
- ads/feature_engineering/accessor/mixin/correlation.py +166 -0
- ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
- ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
- ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
- ads/feature_engineering/accessor/mixin/utils.py +65 -0
- ads/feature_engineering/accessor/series_accessor.py +431 -0
- ads/feature_engineering/adsimage/__init__.py +5 -0
- ads/feature_engineering/adsimage/image.py +192 -0
- ads/feature_engineering/adsimage/image_reader.py +170 -0
- ads/feature_engineering/adsimage/interface/__init__.py +5 -0
- ads/feature_engineering/adsimage/interface/reader.py +19 -0
- ads/feature_engineering/adsstring/__init__.py +7 -0
- ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
- ads/feature_engineering/adsstring/string/__init__.py +8 -0
- ads/feature_engineering/data_schema.json +57 -0
- ads/feature_engineering/dataset/__init__.py +5 -0
- ads/feature_engineering/dataset/zip_code_data.py +42062 -0
- ads/feature_engineering/exceptions.py +40 -0
- ads/feature_engineering/feature_type/__init__.py +133 -0
- ads/feature_engineering/feature_type/address.py +184 -0
- ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
- ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
- ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
- ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
- ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
- ads/feature_engineering/feature_type/adsstring/string.py +258 -0
- ads/feature_engineering/feature_type/base.py +58 -0
- ads/feature_engineering/feature_type/boolean.py +183 -0
- ads/feature_engineering/feature_type/category.py +146 -0
- ads/feature_engineering/feature_type/constant.py +137 -0
- ads/feature_engineering/feature_type/continuous.py +151 -0
- ads/feature_engineering/feature_type/creditcard.py +314 -0
- ads/feature_engineering/feature_type/datetime.py +190 -0
- ads/feature_engineering/feature_type/discrete.py +134 -0
- ads/feature_engineering/feature_type/document.py +43 -0
- ads/feature_engineering/feature_type/gis.py +251 -0
- ads/feature_engineering/feature_type/handler/__init__.py +5 -0
- ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
- ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
- ads/feature_engineering/feature_type/handler/warnings.py +128 -0
- ads/feature_engineering/feature_type/integer.py +142 -0
- ads/feature_engineering/feature_type/ip_address.py +144 -0
- ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
- ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
- ads/feature_engineering/feature_type/lat_long.py +256 -0
- ads/feature_engineering/feature_type/object.py +43 -0
- ads/feature_engineering/feature_type/ordinal.py +132 -0
- ads/feature_engineering/feature_type/phone_number.py +135 -0
- ads/feature_engineering/feature_type/string.py +171 -0
- ads/feature_engineering/feature_type/text.py +93 -0
- ads/feature_engineering/feature_type/unknown.py +43 -0
- ads/feature_engineering/feature_type/zip_code.py +164 -0
- ads/feature_engineering/feature_type_manager.py +406 -0
- ads/feature_engineering/schema.py +795 -0
- ads/feature_engineering/utils.py +245 -0
- ads/feature_store/.readthedocs.yaml +19 -0
- ads/feature_store/README.md +65 -0
- ads/feature_store/__init__.py +9 -0
- ads/feature_store/common/__init__.py +0 -0
- ads/feature_store/common/enums.py +339 -0
- ads/feature_store/common/exceptions.py +18 -0
- ads/feature_store/common/spark_session_singleton.py +125 -0
- ads/feature_store/common/utils/__init__.py +0 -0
- ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
- ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
- ads/feature_store/common/utils/transformation_utils.py +82 -0
- ads/feature_store/common/utils/utility.py +403 -0
- ads/feature_store/data_validation/__init__.py +0 -0
- ads/feature_store/data_validation/great_expectation.py +129 -0
- ads/feature_store/dataset.py +1230 -0
- ads/feature_store/dataset_job.py +530 -0
- ads/feature_store/docs/Dockerfile +7 -0
- ads/feature_store/docs/Makefile +44 -0
- ads/feature_store/docs/conf.py +28 -0
- ads/feature_store/docs/requirements.txt +14 -0
- ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
- ads/feature_store/docs/source/cicd.rst +137 -0
- ads/feature_store/docs/source/conf.py +86 -0
- ads/feature_store/docs/source/data_versioning.rst +33 -0
- ads/feature_store/docs/source/dataset.rst +388 -0
- ads/feature_store/docs/source/dataset_job.rst +27 -0
- ads/feature_store/docs/source/demo.rst +70 -0
- ads/feature_store/docs/source/entity.rst +78 -0
- ads/feature_store/docs/source/feature_group.rst +624 -0
- ads/feature_store/docs/source/feature_group_job.rst +29 -0
- ads/feature_store/docs/source/feature_store.rst +122 -0
- ads/feature_store/docs/source/feature_store_class.rst +123 -0
- ads/feature_store/docs/source/feature_validation.rst +66 -0
- ads/feature_store/docs/source/figures/cicd.png +0 -0
- ads/feature_store/docs/source/figures/data_validation.png +0 -0
- ads/feature_store/docs/source/figures/data_versioning.png +0 -0
- ads/feature_store/docs/source/figures/dataset.gif +0 -0
- ads/feature_store/docs/source/figures/dataset.png +0 -0
- ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
- ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
- ads/feature_store/docs/source/figures/entity.png +0 -0
- ads/feature_store/docs/source/figures/feature_group.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
- ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
- ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
- ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
- ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
- ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
- ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
- ads/feature_store/docs/source/figures/overview.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
- ads/feature_store/docs/source/figures/stats_1.png +0 -0
- ads/feature_store/docs/source/figures/stats_2.png +0 -0
- ads/feature_store/docs/source/figures/stats_d.png +0 -0
- ads/feature_store/docs/source/figures/stats_fg.png +0 -0
- ads/feature_store/docs/source/figures/transformation.png +0 -0
- ads/feature_store/docs/source/figures/transformations.gif +0 -0
- ads/feature_store/docs/source/figures/validation.png +0 -0
- ads/feature_store/docs/source/figures/validation_fg.png +0 -0
- ads/feature_store/docs/source/figures/validation_results.png +0 -0
- ads/feature_store/docs/source/figures/validation_summary.png +0 -0
- ads/feature_store/docs/source/index.rst +81 -0
- ads/feature_store/docs/source/module.rst +8 -0
- ads/feature_store/docs/source/notebook.rst +94 -0
- ads/feature_store/docs/source/overview.rst +47 -0
- ads/feature_store/docs/source/quickstart.rst +176 -0
- ads/feature_store/docs/source/release_notes.rst +194 -0
- ads/feature_store/docs/source/setup_feature_store.rst +81 -0
- ads/feature_store/docs/source/statistics.rst +58 -0
- ads/feature_store/docs/source/transformation.rst +199 -0
- ads/feature_store/docs/source/ui.rst +65 -0
- ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
- ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
- ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
- ads/feature_store/entity.py +718 -0
- ads/feature_store/execution_strategy/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
- ads/feature_store/execution_strategy/engine/__init__.py +0 -0
- ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
- ads/feature_store/execution_strategy/execution_strategy.py +113 -0
- ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
- ads/feature_store/execution_strategy/spark/__init__.py +0 -0
- ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
- ads/feature_store/feature.py +192 -0
- ads/feature_store/feature_group.py +1494 -0
- ads/feature_store/feature_group_expectation.py +346 -0
- ads/feature_store/feature_group_job.py +602 -0
- ads/feature_store/feature_lineage/__init__.py +0 -0
- ads/feature_store/feature_lineage/graphviz_service.py +180 -0
- ads/feature_store/feature_option_details.py +50 -0
- ads/feature_store/feature_statistics/__init__.py +0 -0
- ads/feature_store/feature_statistics/statistics_service.py +99 -0
- ads/feature_store/feature_store.py +699 -0
- ads/feature_store/feature_store_registrar.py +518 -0
- ads/feature_store/input_feature_detail.py +149 -0
- ads/feature_store/mixin/__init__.py +4 -0
- ads/feature_store/mixin/oci_feature_store.py +145 -0
- ads/feature_store/model_details.py +73 -0
- ads/feature_store/query/__init__.py +0 -0
- ads/feature_store/query/filter.py +266 -0
- ads/feature_store/query/generator/__init__.py +0 -0
- ads/feature_store/query/generator/query_generator.py +298 -0
- ads/feature_store/query/join.py +161 -0
- ads/feature_store/query/query.py +403 -0
- ads/feature_store/query/validator/__init__.py +0 -0
- ads/feature_store/query/validator/query_validator.py +57 -0
- ads/feature_store/response/__init__.py +0 -0
- ads/feature_store/response/response_builder.py +68 -0
- ads/feature_store/service/__init__.py +0 -0
- ads/feature_store/service/oci_dataset.py +139 -0
- ads/feature_store/service/oci_dataset_job.py +199 -0
- ads/feature_store/service/oci_entity.py +125 -0
- ads/feature_store/service/oci_feature_group.py +164 -0
- ads/feature_store/service/oci_feature_group_job.py +214 -0
- ads/feature_store/service/oci_feature_store.py +182 -0
- ads/feature_store/service/oci_lineage.py +87 -0
- ads/feature_store/service/oci_transformation.py +104 -0
- ads/feature_store/statistics/__init__.py +0 -0
- ads/feature_store/statistics/abs_feature_value.py +49 -0
- ads/feature_store/statistics/charts/__init__.py +0 -0
- ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
- ads/feature_store/statistics/charts/box_plot.py +148 -0
- ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
- ads/feature_store/statistics/charts/probability_distribution.py +68 -0
- ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
- ads/feature_store/statistics/feature_stat.py +126 -0
- ads/feature_store/statistics/generic_feature_value.py +33 -0
- ads/feature_store/statistics/statistics.py +41 -0
- ads/feature_store/statistics_config.py +101 -0
- ads/feature_store/templates/feature_store_template.yaml +45 -0
- ads/feature_store/transformation.py +499 -0
- ads/feature_store/validation_output.py +57 -0
- ads/hpo/__init__.py +9 -0
- ads/hpo/_imports.py +91 -0
- ads/hpo/ads_search_space.py +439 -0
- ads/hpo/distributions.py +325 -0
- ads/hpo/objective.py +280 -0
- ads/hpo/search_cv.py +1657 -0
- ads/hpo/stopping_criterion.py +75 -0
- ads/hpo/tuner_artifact.py +413 -0
- ads/hpo/utils.py +91 -0
- ads/hpo/validation.py +140 -0
- ads/hpo/visualization/__init__.py +5 -0
- ads/hpo/visualization/_contour.py +23 -0
- ads/hpo/visualization/_edf.py +20 -0
- ads/hpo/visualization/_intermediate_values.py +21 -0
- ads/hpo/visualization/_optimization_history.py +25 -0
- ads/hpo/visualization/_parallel_coordinate.py +169 -0
- ads/hpo/visualization/_param_importances.py +26 -0
- ads/jobs/__init__.py +53 -0
- ads/jobs/ads_job.py +663 -0
- ads/jobs/builders/__init__.py +5 -0
- ads/jobs/builders/base.py +156 -0
- ads/jobs/builders/infrastructure/__init__.py +6 -0
- ads/jobs/builders/infrastructure/base.py +165 -0
- ads/jobs/builders/infrastructure/dataflow.py +1252 -0
- ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
- ads/jobs/builders/infrastructure/utils.py +65 -0
- ads/jobs/builders/runtimes/__init__.py +5 -0
- ads/jobs/builders/runtimes/artifact.py +338 -0
- ads/jobs/builders/runtimes/base.py +325 -0
- ads/jobs/builders/runtimes/container_runtime.py +242 -0
- ads/jobs/builders/runtimes/python_runtime.py +1016 -0
- ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
- ads/jobs/cli.py +104 -0
- ads/jobs/env_var_parser.py +131 -0
- ads/jobs/extension.py +160 -0
- ads/jobs/schema/__init__.py +5 -0
- ads/jobs/schema/infrastructure_schema.json +116 -0
- ads/jobs/schema/job_schema.json +42 -0
- ads/jobs/schema/runtime_schema.json +183 -0
- ads/jobs/schema/validator.py +141 -0
- ads/jobs/serializer.py +296 -0
- ads/jobs/templates/__init__.py +5 -0
- ads/jobs/templates/container.py +6 -0
- ads/jobs/templates/driver_notebook.py +177 -0
- ads/jobs/templates/driver_oci.py +500 -0
- ads/jobs/templates/driver_python.py +48 -0
- ads/jobs/templates/driver_pytorch.py +852 -0
- ads/jobs/templates/driver_utils.py +615 -0
- ads/jobs/templates/hostname_from_env.c +55 -0
- ads/jobs/templates/oci_metrics.py +181 -0
- ads/jobs/utils.py +104 -0
- ads/llm/__init__.py +28 -0
- ads/llm/autogen/__init__.py +2 -0
- ads/llm/autogen/constants.py +15 -0
- ads/llm/autogen/reports/__init__.py +2 -0
- ads/llm/autogen/reports/base.py +67 -0
- ads/llm/autogen/reports/data.py +103 -0
- ads/llm/autogen/reports/session.py +526 -0
- ads/llm/autogen/reports/templates/chat_box.html +13 -0
- ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
- ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
- ads/llm/autogen/reports/utils.py +56 -0
- ads/llm/autogen/v02/__init__.py +4 -0
- ads/llm/autogen/v02/client.py +295 -0
- ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
- ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
- ads/llm/autogen/v02/loggers/__init__.py +6 -0
- ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
- ads/llm/autogen/v02/loggers/session_logger.py +580 -0
- ads/llm/autogen/v02/loggers/utils.py +86 -0
- ads/llm/autogen/v02/runtime_logging.py +163 -0
- ads/llm/chain.py +268 -0
- ads/llm/chat_template.py +31 -0
- ads/llm/deploy.py +63 -0
- ads/llm/guardrails/__init__.py +5 -0
- ads/llm/guardrails/base.py +442 -0
- ads/llm/guardrails/huggingface.py +44 -0
- ads/llm/langchain/__init__.py +5 -0
- ads/llm/langchain/plugins/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
- ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
- ads/llm/langchain/plugins/llms/__init__.py +5 -0
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
- ads/llm/requirements.txt +3 -0
- ads/llm/serialize.py +219 -0
- ads/llm/serializers/__init__.py +0 -0
- ads/llm/serializers/retrieval_qa.py +153 -0
- ads/llm/serializers/runnable_parallel.py +27 -0
- ads/llm/templates/score_chain.jinja2 +155 -0
- ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
- ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
- ads/model/__init__.py +52 -0
- ads/model/artifact.py +573 -0
- ads/model/artifact_downloader.py +254 -0
- ads/model/artifact_uploader.py +267 -0
- ads/model/base_properties.py +238 -0
- ads/model/common/.model-ignore +66 -0
- ads/model/common/__init__.py +5 -0
- ads/model/common/utils.py +142 -0
- ads/model/datascience_model.py +2635 -0
- ads/model/deployment/__init__.py +20 -0
- ads/model/deployment/common/__init__.py +5 -0
- ads/model/deployment/common/utils.py +308 -0
- ads/model/deployment/model_deployer.py +466 -0
- ads/model/deployment/model_deployment.py +1846 -0
- ads/model/deployment/model_deployment_infrastructure.py +671 -0
- ads/model/deployment/model_deployment_properties.py +493 -0
- ads/model/deployment/model_deployment_runtime.py +838 -0
- ads/model/extractor/__init__.py +5 -0
- ads/model/extractor/automl_extractor.py +74 -0
- ads/model/extractor/embedding_onnx_extractor.py +80 -0
- ads/model/extractor/huggingface_extractor.py +88 -0
- ads/model/extractor/keras_extractor.py +84 -0
- ads/model/extractor/lightgbm_extractor.py +93 -0
- ads/model/extractor/model_info_extractor.py +114 -0
- ads/model/extractor/model_info_extractor_factory.py +105 -0
- ads/model/extractor/pytorch_extractor.py +87 -0
- ads/model/extractor/sklearn_extractor.py +112 -0
- ads/model/extractor/spark_extractor.py +89 -0
- ads/model/extractor/tensorflow_extractor.py +85 -0
- ads/model/extractor/xgboost_extractor.py +94 -0
- ads/model/framework/__init__.py +5 -0
- ads/model/framework/automl_model.py +178 -0
- ads/model/framework/embedding_onnx_model.py +438 -0
- ads/model/framework/huggingface_model.py +399 -0
- ads/model/framework/lightgbm_model.py +266 -0
- ads/model/framework/pytorch_model.py +266 -0
- ads/model/framework/sklearn_model.py +250 -0
- ads/model/framework/spark_model.py +326 -0
- ads/model/framework/tensorflow_model.py +254 -0
- ads/model/framework/xgboost_model.py +258 -0
- ads/model/generic_model.py +3518 -0
- ads/model/model_artifact_boilerplate/README.md +381 -0
- ads/model/model_artifact_boilerplate/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
- ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
- ads/model/model_artifact_boilerplate/score.py +61 -0
- ads/model/model_file_description_schema.json +68 -0
- ads/model/model_introspect.py +331 -0
- ads/model/model_metadata.py +1810 -0
- ads/model/model_metadata_mixin.py +460 -0
- ads/model/model_properties.py +63 -0
- ads/model/model_version_set.py +739 -0
- ads/model/runtime/__init__.py +5 -0
- ads/model/runtime/env_info.py +306 -0
- ads/model/runtime/model_deployment_details.py +37 -0
- ads/model/runtime/model_provenance_details.py +58 -0
- ads/model/runtime/runtime_info.py +81 -0
- ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
- ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
- ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
- ads/model/runtime/utils.py +201 -0
- ads/model/serde/__init__.py +5 -0
- ads/model/serde/common.py +40 -0
- ads/model/serde/model_input.py +547 -0
- ads/model/serde/model_serializer.py +1184 -0
- ads/model/service/__init__.py +5 -0
- ads/model/service/oci_datascience_model.py +1076 -0
- ads/model/service/oci_datascience_model_deployment.py +500 -0
- ads/model/service/oci_datascience_model_version_set.py +176 -0
- ads/model/transformer/__init__.py +5 -0
- ads/model/transformer/onnx_transformer.py +324 -0
- ads/mysqldb/__init__.py +5 -0
- ads/mysqldb/mysql_db.py +227 -0
- ads/opctl/__init__.py +18 -0
- ads/opctl/anomaly_detection.py +11 -0
- ads/opctl/backend/__init__.py +5 -0
- ads/opctl/backend/ads_dataflow.py +353 -0
- ads/opctl/backend/ads_ml_job.py +710 -0
- ads/opctl/backend/ads_ml_pipeline.py +164 -0
- ads/opctl/backend/ads_model_deployment.py +209 -0
- ads/opctl/backend/base.py +146 -0
- ads/opctl/backend/local.py +1053 -0
- ads/opctl/backend/marketplace/__init__.py +9 -0
- ads/opctl/backend/marketplace/helm_helper.py +173 -0
- ads/opctl/backend/marketplace/local_marketplace.py +271 -0
- ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
- ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
- ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
- ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
- ads/opctl/backend/marketplace/models/__init__.py +5 -0
- ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
- ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
- ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
- ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
- ads/opctl/cli.py +707 -0
- ads/opctl/cmds.py +869 -0
- ads/opctl/conda/__init__.py +5 -0
- ads/opctl/conda/cli.py +193 -0
- ads/opctl/conda/cmds.py +749 -0
- ads/opctl/conda/config.yaml +34 -0
- ads/opctl/conda/manifest_template.yaml +13 -0
- ads/opctl/conda/multipart_uploader.py +188 -0
- ads/opctl/conda/pack.py +89 -0
- ads/opctl/config/__init__.py +5 -0
- ads/opctl/config/base.py +57 -0
- ads/opctl/config/diagnostics/__init__.py +5 -0
- ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
- ads/opctl/config/merger.py +255 -0
- ads/opctl/config/resolver.py +297 -0
- ads/opctl/config/utils.py +79 -0
- ads/opctl/config/validator.py +17 -0
- ads/opctl/config/versioner.py +68 -0
- ads/opctl/config/yaml_parsers/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/base.py +58 -0
- ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
- ads/opctl/constants.py +66 -0
- ads/opctl/decorator/__init__.py +5 -0
- ads/opctl/decorator/common.py +129 -0
- ads/opctl/diagnostics/__init__.py +5 -0
- ads/opctl/diagnostics/__main__.py +25 -0
- ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
- ads/opctl/diagnostics/check_requirements.py +144 -0
- ads/opctl/diagnostics/requirement_exception.py +9 -0
- ads/opctl/distributed/README.md +109 -0
- ads/opctl/distributed/__init__.py +5 -0
- ads/opctl/distributed/certificates.py +32 -0
- ads/opctl/distributed/cli.py +207 -0
- ads/opctl/distributed/cmds.py +731 -0
- ads/opctl/distributed/common/__init__.py +5 -0
- ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
- ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
- ads/opctl/distributed/common/cluster_config_helper.py +103 -0
- ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
- ads/opctl/distributed/common/cluster_runner.py +54 -0
- ads/opctl/distributed/common/framework_factory.py +29 -0
- ads/opctl/docker/Dockerfile.job +103 -0
- ads/opctl/docker/Dockerfile.job.arm +107 -0
- ads/opctl/docker/Dockerfile.job.gpu +175 -0
- ads/opctl/docker/base-env.yaml +13 -0
- ads/opctl/docker/cuda.repo +6 -0
- ads/opctl/docker/operator/.dockerignore +0 -0
- ads/opctl/docker/operator/Dockerfile +41 -0
- ads/opctl/docker/operator/Dockerfile.gpu +85 -0
- ads/opctl/docker/operator/cuda.repo +6 -0
- ads/opctl/docker/operator/environment.yaml +8 -0
- ads/opctl/forecast.py +11 -0
- ads/opctl/index.yaml +3 -0
- ads/opctl/model/__init__.py +5 -0
- ads/opctl/model/cli.py +65 -0
- ads/opctl/model/cmds.py +73 -0
- ads/opctl/operator/README.md +4 -0
- ads/opctl/operator/__init__.py +31 -0
- ads/opctl/operator/cli.py +344 -0
- ads/opctl/operator/cmd.py +596 -0
- ads/opctl/operator/common/__init__.py +5 -0
- ads/opctl/operator/common/backend_factory.py +460 -0
- ads/opctl/operator/common/const.py +27 -0
- ads/opctl/operator/common/data/synthetic.csv +16001 -0
- ads/opctl/operator/common/dictionary_merger.py +148 -0
- ads/opctl/operator/common/errors.py +42 -0
- ads/opctl/operator/common/operator_config.py +99 -0
- ads/opctl/operator/common/operator_loader.py +811 -0
- ads/opctl/operator/common/operator_schema.yaml +130 -0
- ads/opctl/operator/common/operator_yaml_generator.py +152 -0
- ads/opctl/operator/common/utils.py +208 -0
- ads/opctl/operator/lowcode/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
- ads/opctl/operator/lowcode/anomaly/README.md +207 -0
- ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
- ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
- ads/opctl/operator/lowcode/anomaly/const.py +167 -0
- ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
- ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
- ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
- ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
- ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
- ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
- ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
- ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
- ads/opctl/operator/lowcode/common/__init__.py +5 -0
- ads/opctl/operator/lowcode/common/const.py +10 -0
- ads/opctl/operator/lowcode/common/data.py +116 -0
- ads/opctl/operator/lowcode/common/errors.py +47 -0
- ads/opctl/operator/lowcode/common/transformations.py +296 -0
- ads/opctl/operator/lowcode/common/utils.py +384 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
- ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
- ads/opctl/operator/lowcode/forecast/README.md +209 -0
- ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
- ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
- ads/opctl/operator/lowcode/forecast/const.py +92 -0
- ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
- ads/opctl/operator/lowcode/forecast/errors.py +26 -0
- ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
- ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
- ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
- ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
- ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
- ads/opctl/operator/lowcode/forecast/model/prophet.py +450 -0
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
- ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
- ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
- ads/opctl/operator/lowcode/forecast/utils.py +397 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
- ads/opctl/operator/lowcode/pii/MLoperator +17 -0
- ads/opctl/operator/lowcode/pii/README.md +208 -0
- ads/opctl/operator/lowcode/pii/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/__main__.py +78 -0
- ads/opctl/operator/lowcode/pii/cmd.py +39 -0
- ads/opctl/operator/lowcode/pii/constant.py +84 -0
- ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
- ads/opctl/operator/lowcode/pii/errors.py +27 -0
- ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
- ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
- ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
- ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
- ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
- ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
- ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
- ads/opctl/operator/lowcode/pii/model/report.py +487 -0
- ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
- ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
- ads/opctl/operator/lowcode/pii/utils.py +43 -0
- ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
- ads/opctl/operator/lowcode/recommender/README.md +206 -0
- ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
- ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
- ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
- ads/opctl/operator/lowcode/recommender/constant.py +30 -0
- ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
- ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
- ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
- ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
- ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
- ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
- ads/opctl/operator/lowcode/recommender/utils.py +13 -0
- ads/opctl/operator/runtime/__init__.py +5 -0
- ads/opctl/operator/runtime/const.py +17 -0
- ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
- ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
- ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/runtime.py +115 -0
- ads/opctl/schema.yaml.yml +36 -0
- ads/opctl/script.py +40 -0
- ads/opctl/spark/__init__.py +5 -0
- ads/opctl/spark/cli.py +43 -0
- ads/opctl/spark/cmds.py +147 -0
- ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
- ads/opctl/utils.py +344 -0
- ads/oracledb/__init__.py +5 -0
- ads/oracledb/oracle_db.py +346 -0
- ads/pipeline/__init__.py +39 -0
- ads/pipeline/ads_pipeline.py +2279 -0
- ads/pipeline/ads_pipeline_run.py +772 -0
- ads/pipeline/ads_pipeline_step.py +605 -0
- ads/pipeline/builders/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/custom_script.py +32 -0
- ads/pipeline/cli.py +119 -0
- ads/pipeline/extension.py +291 -0
- ads/pipeline/schema/__init__.py +5 -0
- ads/pipeline/schema/cs_step_schema.json +35 -0
- ads/pipeline/schema/ml_step_schema.json +31 -0
- ads/pipeline/schema/pipeline_schema.json +71 -0
- ads/pipeline/visualizer/__init__.py +5 -0
- ads/pipeline/visualizer/base.py +570 -0
- ads/pipeline/visualizer/graph_renderer.py +272 -0
- ads/pipeline/visualizer/text_renderer.py +84 -0
- ads/secrets/__init__.py +11 -0
- ads/secrets/adb.py +386 -0
- ads/secrets/auth_token.py +86 -0
- ads/secrets/big_data_service.py +365 -0
- ads/secrets/mysqldb.py +149 -0
- ads/secrets/oracledb.py +160 -0
- ads/secrets/secrets.py +407 -0
- ads/telemetry/__init__.py +7 -0
- ads/telemetry/base.py +69 -0
- ads/telemetry/client.py +122 -0
- ads/telemetry/telemetry.py +257 -0
- ads/templates/dataflow_pyspark.jinja2 +13 -0
- ads/templates/dataflow_sparksql.jinja2 +22 -0
- ads/templates/func.jinja2 +20 -0
- ads/templates/schemas/openapi.json +1740 -0
- ads/templates/score-pkl.jinja2 +173 -0
- ads/templates/score.jinja2 +322 -0
- ads/templates/score_embedding_onnx.jinja2 +202 -0
- ads/templates/score_generic.jinja2 +165 -0
- ads/templates/score_huggingface_pipeline.jinja2 +217 -0
- ads/templates/score_lightgbm.jinja2 +185 -0
- ads/templates/score_onnx.jinja2 +407 -0
- ads/templates/score_onnx_new.jinja2 +473 -0
- ads/templates/score_oracle_automl.jinja2 +185 -0
- ads/templates/score_pyspark.jinja2 +154 -0
- ads/templates/score_pytorch.jinja2 +219 -0
- ads/templates/score_scikit-learn.jinja2 +184 -0
- ads/templates/score_tensorflow.jinja2 +184 -0
- ads/templates/score_xgboost.jinja2 +178 -0
- ads/text_dataset/__init__.py +5 -0
- ads/text_dataset/backends.py +211 -0
- ads/text_dataset/dataset.py +445 -0
- ads/text_dataset/extractor.py +207 -0
- ads/text_dataset/options.py +53 -0
- ads/text_dataset/udfs.py +22 -0
- ads/text_dataset/utils.py +49 -0
- ads/type_discovery/__init__.py +9 -0
- ads/type_discovery/abstract_detector.py +21 -0
- ads/type_discovery/constant_detector.py +41 -0
- ads/type_discovery/continuous_detector.py +54 -0
- ads/type_discovery/credit_card_detector.py +99 -0
- ads/type_discovery/datetime_detector.py +92 -0
- ads/type_discovery/discrete_detector.py +118 -0
- ads/type_discovery/document_detector.py +146 -0
- ads/type_discovery/ip_detector.py +68 -0
- ads/type_discovery/latlon_detector.py +90 -0
- ads/type_discovery/phone_number_detector.py +63 -0
- ads/type_discovery/type_discovery_driver.py +87 -0
- ads/type_discovery/typed_feature.py +594 -0
- ads/type_discovery/unknown_detector.py +41 -0
- ads/type_discovery/zipcode_detector.py +48 -0
- ads/vault/__init__.py +7 -0
- ads/vault/vault.py +237 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/METADATA +150 -149
- oracle_ads-2.13.10.dist-info/RECORD +858 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/WHEEL +1 -2
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/entry_points.txt +2 -1
- oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
- oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,245 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*--
|
3
|
+
|
4
|
+
# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
|
5
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
|
+
|
7
|
+
"""
|
8
|
+
The module that represents utility functions.
|
9
|
+
|
10
|
+
Functions:
|
11
|
+
is_boolean(value: Any) -> bool
|
12
|
+
Checks if value type is boolean.
|
13
|
+
"""
|
14
|
+
import numpy as np
|
15
|
+
import matplotlib.pyplot as plt
|
16
|
+
import pandas as pd
|
17
|
+
import re
|
18
|
+
from ads.common.card_identifier import card_identify
|
19
|
+
from ads.common.decorator.runtime_dependency import (
|
20
|
+
runtime_dependency,
|
21
|
+
OptionalDependency,
|
22
|
+
)
|
23
|
+
from ads.feature_engineering.dataset.zip_code_data import zip_code_dict
|
24
|
+
from functools import lru_cache
|
25
|
+
from typing import Any
|
26
|
+
|
27
|
+
|
28
|
+
class SchemeNeutral(str):
|
29
|
+
BACKGROUND_LIGHT = "#F5F4F2"
|
30
|
+
BACKGROUND_DARK = "#E4E1DD"
|
31
|
+
AREA_LIGHT = "#BCB6B1"
|
32
|
+
AREA_DARK = "#9E9892"
|
33
|
+
LINE_LIGHT = "#665F5B"
|
34
|
+
LINE_DARK = "#47423E"
|
35
|
+
|
36
|
+
|
37
|
+
class SchemeTeal(str):
|
38
|
+
BACKGROUND_LIGHT = "#F0f6f5"
|
39
|
+
BACKGROUND_DARK = "#D6E5E5"
|
40
|
+
AREA_LIGHT = "#9ABFBF"
|
41
|
+
AREA_DARK = "#76A2A0"
|
42
|
+
LINE_LIGHT = "#3E686C"
|
43
|
+
LINE_DARK = "#2B484B"
|
44
|
+
|
45
|
+
|
46
|
+
def _tag_to_snake(name: str) -> str:
|
47
|
+
"""Conversts string to snake representation.
|
48
|
+
1. Converts the string to the lower case.
|
49
|
+
2. Converts all spaces to underscore.
|
50
|
+
|
51
|
+
Parameters
|
52
|
+
----------
|
53
|
+
name: string
|
54
|
+
The name to convert.
|
55
|
+
|
56
|
+
Returns
|
57
|
+
-------
|
58
|
+
str: The name converted to the snake representation.
|
59
|
+
"""
|
60
|
+
_name = name.strip().lower()
|
61
|
+
if len(_name.split()) > 1:
|
62
|
+
_name = "_".join(_name.split())
|
63
|
+
return _name
|
64
|
+
|
65
|
+
|
66
|
+
def _add_missing(x, df):
|
67
|
+
"""
|
68
|
+
Adds count of missing values.
|
69
|
+
"""
|
70
|
+
n_missing = pd.isnull(x.replace(r"", np.NaN)).sum()
|
71
|
+
if n_missing > 0:
|
72
|
+
df.loc["missing"] = n_missing
|
73
|
+
return df
|
74
|
+
|
75
|
+
|
76
|
+
def _count_unique_missing(x):
|
77
|
+
"""
|
78
|
+
Returns the total count, unique count and count of missing values of a series.
|
79
|
+
"""
|
80
|
+
df_stat = pd.Series(
|
81
|
+
{"count": len(x), "unique": len(x.replace(r"", np.NaN).dropna().unique())},
|
82
|
+
name=x.name,
|
83
|
+
).to_frame()
|
84
|
+
return _add_missing(x, df_stat)
|
85
|
+
|
86
|
+
|
87
|
+
def is_boolean(value: Any) -> bool:
|
88
|
+
"""Checks if value type is boolean.
|
89
|
+
|
90
|
+
Parameters
|
91
|
+
----------
|
92
|
+
value: Any
|
93
|
+
The value to check.
|
94
|
+
|
95
|
+
Returns
|
96
|
+
-------
|
97
|
+
bool: True if value is boolean, False otherwise.
|
98
|
+
"""
|
99
|
+
bool_values = ("yes", "y", "true", "t", "1", "no", "n", "false", "f", "0", "")
|
100
|
+
return isinstance(value, bool) or str(value).lower() in bool_values
|
101
|
+
|
102
|
+
|
103
|
+
def assign_issuer(cardnumber):
|
104
|
+
if pd.isnull(cardnumber):
|
105
|
+
return "missing"
|
106
|
+
else:
|
107
|
+
return card_identify().identify_issue_network(cardnumber)
|
108
|
+
|
109
|
+
|
110
|
+
def random_color_func(
|
111
|
+
z,
|
112
|
+
word=None,
|
113
|
+
font_size=None,
|
114
|
+
position=None,
|
115
|
+
orientation=None,
|
116
|
+
font_path=None,
|
117
|
+
random_state=None,
|
118
|
+
):
|
119
|
+
"""
|
120
|
+
Returns random color function use for color_func in creating WordCloud
|
121
|
+
"""
|
122
|
+
h = 179
|
123
|
+
s = 23
|
124
|
+
l = int(100.0 * float(random_state.randint(60, 120)) / 255.0)
|
125
|
+
return "hsl({}, {}%, {}%)".format(h, s, l)
|
126
|
+
|
127
|
+
|
128
|
+
def _is_float(s: str):
|
129
|
+
"""
|
130
|
+
Checks if given string can convert to float
|
131
|
+
"""
|
132
|
+
return re.match(r"^-?\d+(?:\.\d+)$", s) != None
|
133
|
+
|
134
|
+
|
135
|
+
def _str_lat_long_to_point(s):
|
136
|
+
"""
|
137
|
+
Converts input data into formated geometry point
|
138
|
+
Return formated geometry point string or np.NaN if input string is not valid
|
139
|
+
"""
|
140
|
+
if isinstance(s, str):
|
141
|
+
coords = s.split(",")
|
142
|
+
if len(coords) == 2:
|
143
|
+
lat, long = coords[0].lstrip(), coords[1].lstrip()
|
144
|
+
if lat.startswith("("):
|
145
|
+
lat = lat[1:]
|
146
|
+
if long.endswith(")"):
|
147
|
+
long = long[:-1]
|
148
|
+
if _is_float(lat) and _is_float(long):
|
149
|
+
return "POINT(" + long + " " + lat + ")"
|
150
|
+
return np.NaN
|
151
|
+
|
152
|
+
|
153
|
+
@runtime_dependency(module="geopandas", install_from=OptionalDependency.GEO)
|
154
|
+
def _plot_gis_scatter(df: pd.DataFrame, lon: str, lat: str):
|
155
|
+
"""
|
156
|
+
Parameters
|
157
|
+
----------
|
158
|
+
df : pd.DataFrame
|
159
|
+
dataframe contains gis data
|
160
|
+
lon: str
|
161
|
+
column name mapping to longitude
|
162
|
+
lat: str
|
163
|
+
column name mapping to latitude
|
164
|
+
Returns
|
165
|
+
-------
|
166
|
+
matplotlib.axes._subplots.AxesSubplot present location of given data on world map
|
167
|
+
"""
|
168
|
+
if len(df.index):
|
169
|
+
if lon in df.columns and lat in df.columns:
|
170
|
+
fig, ax = plt.subplots(facecolor=SchemeNeutral.BACKGROUND_LIGHT)
|
171
|
+
gdf = geopandas.GeoDataFrame(
|
172
|
+
df, geometry=geopandas.points_from_xy(df[lon], df[lat])
|
173
|
+
)
|
174
|
+
world = geopandas.read_file(
|
175
|
+
geopandas.datasets.get_path("naturalearth_lowres")
|
176
|
+
)
|
177
|
+
world = world[world["name"] == "United States of America"]
|
178
|
+
ax1 = world.plot(
|
179
|
+
ax=ax, color=SchemeNeutral.AREA_LIGHT, linewidth=0.5, edgecolor="white"
|
180
|
+
)
|
181
|
+
gdf.plot(ax=ax1, color=SchemeTeal.LINE_LIGHT, markersize=10)
|
182
|
+
plt.axis("off")
|
183
|
+
return ax1
|
184
|
+
|
185
|
+
|
186
|
+
def _to_lat_long(x: pd.Series, zipcode: pd.DataFrame):
|
187
|
+
"""
|
188
|
+
Parameters
|
189
|
+
----------
|
190
|
+
x: pd.Series
|
191
|
+
pandas series contains zip code data
|
192
|
+
zipcode : pd.DataFrame
|
193
|
+
dataframe contains gis data of zip code
|
194
|
+
Returns
|
195
|
+
-------
|
196
|
+
pd.DataFrame
|
197
|
+
dataframe with 2 columes including latitude and longitude data
|
198
|
+
"""
|
199
|
+
lats = []
|
200
|
+
longs = []
|
201
|
+
for s in x:
|
202
|
+
if s in zipcode.index:
|
203
|
+
lats.append(np.float64(zipcode.loc[s]["latitude"]))
|
204
|
+
longs.append(np.float64(zipcode.loc[s]["longitude"]))
|
205
|
+
df = pd.DataFrame(list(zip(lats, longs)), columns=["latitude", "longitude"])
|
206
|
+
return df
|
207
|
+
|
208
|
+
|
209
|
+
@runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
|
210
|
+
def _set_seaborn_theme():
|
211
|
+
"""
|
212
|
+
Sets seaborn figure & axes facecolor
|
213
|
+
"""
|
214
|
+
plt.figure(facecolor=SchemeNeutral.BACKGROUND_LIGHT)
|
215
|
+
seaborn.set(
|
216
|
+
rc={
|
217
|
+
"axes.facecolor": SchemeNeutral.BACKGROUND_LIGHT,
|
218
|
+
"figure.facecolor": SchemeNeutral.BACKGROUND_LIGHT,
|
219
|
+
}
|
220
|
+
)
|
221
|
+
|
222
|
+
|
223
|
+
def _format_stat(stat: pd.Series):
|
224
|
+
"""
|
225
|
+
Formats statistics row index.
|
226
|
+
"""
|
227
|
+
stat.rename(
|
228
|
+
{
|
229
|
+
"std": "standard deviation",
|
230
|
+
"min": "sample minimum",
|
231
|
+
"25%": "lower quartile",
|
232
|
+
"50%": "median",
|
233
|
+
"75%": "upper quartile",
|
234
|
+
"max": "sample maximum",
|
235
|
+
},
|
236
|
+
inplace=True,
|
237
|
+
)
|
238
|
+
|
239
|
+
|
240
|
+
@lru_cache(maxsize=1)
|
241
|
+
def _zip_code():
|
242
|
+
"""Returns dataframe contain zip code, latitude and longitude data."""
|
243
|
+
return pd.DataFrame.from_dict(
|
244
|
+
zip_code_dict, orient="index", columns=["longitude", "latitude"]
|
245
|
+
)
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# Read the Docs configuration file
|
2
|
+
|
3
|
+
# Required
|
4
|
+
version: 2
|
5
|
+
|
6
|
+
# Set the version of Python and other tools you might need
|
7
|
+
build:
|
8
|
+
os: ubuntu-22.04
|
9
|
+
tools:
|
10
|
+
python: "3.9"
|
11
|
+
|
12
|
+
# Build documentation in the docs/ directory with Sphinx
|
13
|
+
sphinx:
|
14
|
+
configuration: ads/feature_store/docs/source/conf.py
|
15
|
+
|
16
|
+
# Optionally declare the Python requirements required to build your docs
|
17
|
+
python:
|
18
|
+
install:
|
19
|
+
- requirements: ads/feature_store/docs/requirements.txt
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# Oracle Feature Store (ADS)
|
2
|
+
|
3
|
+
[](https://pypi.org/project/oracle-ads/) [](https://docs.oracle.com/en-us/iaas/data-science/using/conda-pyspark-fam.htm) [](https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples) [](https://delta.io/) [](https://spark.apache.org/docs/3.2.1/api/python/index.html) [](https://greatexpectations.io/) [](https://pandas.pydata.org/) [](https://arrow.apache.org/docs/python/index.html)
|
4
|
+
|
5
|
+
Managing many datasets, data sources, and transformations for machine learning is complex and costly. Poorly cleaned data, data issues, bugs in transformations, data drift, and training serving skew all lead to increased model development time and poor model performance. Feature store solves many of the problems because it is a centralized way to transform and access data for training and serving time, Feature stores help define a standardised pipeline for ingestion of data and querying of data.
|
6
|
+
|
7
|
+
ADS feature store is a stack-based solution that is deployed in your tenancy using OCI Resource Manager.
|
8
|
+
|
9
|
+
Following are brief descriptions of key concepts and the main components of ADS feature store.
|
10
|
+
|
11
|
+
- ``Feature Vector``: Set of feature values for any one primary and identifier key. For example, all and a subset of features of customer ID 2536 can be called as one feature vector .
|
12
|
+
- ``Feature``: A feature is an individual measurable property or characteristic of an event being observed.
|
13
|
+
- ``Entity``: An entity is a group of semantically related features. The first step a consumer of features would typically do when accessing the feature store service is to list the entities and the entities associated with features. Another way to look at it is that an entity is an object or concept that's described by its features. Examples of entities are customer, product, transaction, review, image, document, and so on.
|
14
|
+
- ``Feature Group``: A feature group in a feature store is a collection of related features that are often used together in ML models. It serves as an organizational unit within the feature store for users to manage, version, and share features across different ML projects. By organizing features into groups, data scientists and ML engineers can efficiently discover, reuse, and collaborate on features reducing the redundant work and ensuring consistency in feature engineering.
|
15
|
+
- ``Feature Group Job``: Feature group jobs are the processing instance of a feature group. Each feature group job includes validation results and statistics results.
|
16
|
+
- ``Dataset``: A dataset is a collection of features that are used together to either train a model or perform model inference.
|
17
|
+
- ``Dataset Job``: A dataset job is the processing instance of a dataset. Each dataset job includes validation results and statistics results.
|
18
|
+
|
19
|
+
## Documentation
|
20
|
+
|
21
|
+
- [Oracle Feature Store SDK (ADS) Documentation](https://feature-store-accelerated-data-science.readthedocs.io/en/latest/)
|
22
|
+
- [OCI Data Science and AI services Examples](https://github.com/oracle/oci-data-science-ai-samples)
|
23
|
+
- [Oracle AI & Data Science Blog](https://blogs.oracle.com/ai-and-datascience/)
|
24
|
+
- [OCI Documentation](https://docs.oracle.com/en-us/iaas/data-science/using/data-science.htm)
|
25
|
+
|
26
|
+
## Examples
|
27
|
+
|
28
|
+
### Quick start examples
|
29
|
+
|
30
|
+
| Jupyter Notebook | Description |
|
31
|
+
|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
32
|
+
| [Feature store querying](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_querying.ipynb) | - Ingestion, querying and exploration of data. |
|
33
|
+
| [Feature store quickstart](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_quickstart.ipynb) | - Ingestion, querying and exploration of data. |
|
34
|
+
| [Schema enforcement and schema evolution](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_schema_evolution.ipynb) | - `Schema evolution` allows you to easily change a table's current schema to accommodate data that is changing over time. `Schema enforcement`, also known as schema validation, is a safeguard in Delta Lake that ensures data quality by rejecting writes to a table that don't match the table's schema. |
|
35
|
+
| [Storage of medical records in feature store](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_ehr_data.ipynb) | Example to demonstrate storage of medical records in feature store |
|
36
|
+
|
37
|
+
### Big data operations using OCI DataFlow
|
38
|
+
|
39
|
+
| Jupyter Notebook | Description |
|
40
|
+
|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------|
|
41
|
+
| [Big data operations with feature store](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_spark_magic.ipynb) | - Ingestion of data using Spark Magic, querying and exploration of data using Spark Magic. |
|
42
|
+
|
43
|
+
### LLM Use cases
|
44
|
+
|
45
|
+
| Jupyter Notebook | Description |
|
46
|
+
|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
47
|
+
| [Embeddings in Feature Store](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_embeddings.ipynb) | - `Embedding feature stores` are optimized for fast and efficient retrieval of embeddings. This is important because embeddings can be high-dimensional and computationally expensive to calculate. By storing them in a dedicated store, you can avoid the need to recalculate embeddings for the same data repeatedly. |
|
48
|
+
| [Synthetic data generation in feature store using OpenAI and FewShotPromptTemplate](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_medical_synthetic_data_openai.ipynb) | - `Synthetic data` is artificially generated data, rather than data collected from real-world events. It's used to simulate real data without compromising privacy or encountering real-world limitations. |
|
49
|
+
| [PII Data redaction, Summarise Content and Translate content using doctran and open AI](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_pii_redaction_and_transformation.ipynb) | - One way to think of Doctran is a LLM-powered black box where messy strings go in and nice, clean, labelled strings come out. Another way to think about it is a modular, declarative wrapper over OpenAI's functional calling feature that significantly improves the developer experience. |
|
50
|
+
| [OpenAI embeddings in feature store](https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/notebook_examples/feature_store_embeddings_openai.ipynb) | - `Embedding feature stores` are optimized for fast and efficient retrieval of embeddings. This is important because embeddings can be high-dimensional and computationally expensive to calculate. By storing them in a dedicated store, you can avoid the need to recalculate embeddings for the same data repeatedly. |
|
51
|
+
|
52
|
+
|
53
|
+
## Contributing
|
54
|
+
|
55
|
+
This project welcomes contributions from the community. Before submitting a pull request, please [review our contribution guide](./../../CONTRIBUTING.md)
|
56
|
+
|
57
|
+
Find Getting Started instructions for developers in [README-development.md](https://github.com/oracle/accelerated-data-science/blob/main/README-development.md)
|
58
|
+
|
59
|
+
## Security
|
60
|
+
|
61
|
+
Consult the security guide [SECURITY.md](https://github.com/oracle/accelerated-data-science/blob/main/SECURITY.md) for our responsible security vulnerability disclosure process.
|
62
|
+
|
63
|
+
## License
|
64
|
+
|
65
|
+
Copyright (c) 2020, 2022 Oracle and/or its affiliates. Licensed under the [Universal Permissive License v1.0](https://oss.oracle.com/licenses/upl/)
|
@@ -0,0 +1,9 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*--
|
3
|
+
|
4
|
+
# Copyright (c) 2023 Oracle and/or its affiliates.
|
5
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
|
+
|
7
|
+
import logging
|
8
|
+
|
9
|
+
logger = logging.getLogger(__name__)
|
File without changes
|
@@ -0,0 +1,339 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8; -*-
|
3
|
+
|
4
|
+
# Copyright (c) 2023 Oracle and/or its affiliates.
|
5
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
|
+
|
7
|
+
from enum import Enum
|
8
|
+
|
9
|
+
|
10
|
+
class JobStatus(Enum):
|
11
|
+
"""
|
12
|
+
An enumeration that represents the supported Job status.
|
13
|
+
|
14
|
+
Attributes:
|
15
|
+
SUCCEEDED (str): A string representation of the state of Succeeded job.
|
16
|
+
FAILED (str): A string representation of the state of Failed job.
|
17
|
+
CODE_EXECUTION (str): A string representation of the state of CodeExecution job.
|
18
|
+
|
19
|
+
Methods:
|
20
|
+
None
|
21
|
+
"""
|
22
|
+
|
23
|
+
SUCCEEDED = "SUCCEEDED"
|
24
|
+
FAILED = "FAILED"
|
25
|
+
CODE_EXECUTION = "CODE_EXECUTION"
|
26
|
+
|
27
|
+
|
28
|
+
class LevelType(Enum):
|
29
|
+
"""
|
30
|
+
An enumeration defining the different types of logging levels.
|
31
|
+
|
32
|
+
Attributes:
|
33
|
+
ERROR (str): A string representing the highest logging level, indicating an error in the program.
|
34
|
+
WARNING (str): A string representing a lower logging level, indicating a potential issue or warning in the program.
|
35
|
+
"""
|
36
|
+
|
37
|
+
ERROR = "ERROR"
|
38
|
+
WARNING = "WARNING"
|
39
|
+
|
40
|
+
|
41
|
+
class DatasetIngestionMode(Enum):
|
42
|
+
"""
|
43
|
+
An enumeration defining the possible modes for ingesting datasets.
|
44
|
+
|
45
|
+
Attributes:
|
46
|
+
SQL (str): A string representing the SQL mode, which is used to ingest datasets using SQL.
|
47
|
+
"""
|
48
|
+
|
49
|
+
SQL = "SQL"
|
50
|
+
|
51
|
+
|
52
|
+
class IngestionType(Enum):
|
53
|
+
"""
|
54
|
+
The type of ingestion that can be performed.
|
55
|
+
|
56
|
+
Possible values:
|
57
|
+
* STREAMING: The data is ingested in real time.
|
58
|
+
* BATCH: The data is ingested in batches.
|
59
|
+
"""
|
60
|
+
|
61
|
+
STREAMING = "STREAMING"
|
62
|
+
BATCH = "BATCH"
|
63
|
+
|
64
|
+
|
65
|
+
class BatchIngestionMode(Enum):
|
66
|
+
"""
|
67
|
+
An enumeration that represents the supported Ingestion Mode in feature store.
|
68
|
+
|
69
|
+
Attributes:
|
70
|
+
OVERWRITE (str): Ingestion mode to overwrite the data in the system.
|
71
|
+
APPEND (str): Ingestion mode to append the data in the system.
|
72
|
+
UPSERT (str): Ingestion mode to insert and update the data in the system.
|
73
|
+
|
74
|
+
Methods:
|
75
|
+
None
|
76
|
+
"""
|
77
|
+
|
78
|
+
OVERWRITE = "OVERWRITE"
|
79
|
+
APPEND = "APPEND"
|
80
|
+
DEFAULT = "DEFAULT"
|
81
|
+
UPSERT = "UPSERT"
|
82
|
+
|
83
|
+
|
84
|
+
class StreamingIngestionMode(Enum):
|
85
|
+
"""
|
86
|
+
Enumeration for stream ingestion modes.
|
87
|
+
|
88
|
+
- `COMPLETE`: Represents complete stream ingestion where the entire dataset is replaced.
|
89
|
+
- `APPEND`: Represents appending new data to the existing dataset.
|
90
|
+
- `UPDATE`: Represents updating existing data in the dataset.
|
91
|
+
"""
|
92
|
+
|
93
|
+
COMPLETE = "COMPLETE"
|
94
|
+
APPEND = "APPEND"
|
95
|
+
UPDATE = "UPDATE"
|
96
|
+
|
97
|
+
|
98
|
+
class JoinType(Enum):
|
99
|
+
"""Enumeration of supported SQL join types.
|
100
|
+
|
101
|
+
Attributes:
|
102
|
+
INNER: Inner join.
|
103
|
+
LEFT: Left join.
|
104
|
+
RIGHT: Right join.
|
105
|
+
FULL: Full outer join.
|
106
|
+
CROSS: Cross join.
|
107
|
+
LEFT_SEMI_JOIN: Left semi join.
|
108
|
+
"""
|
109
|
+
|
110
|
+
INNER = "INNER"
|
111
|
+
LEFT = "LEFT"
|
112
|
+
RIGHT = "RIGHT"
|
113
|
+
FULL = "FULL"
|
114
|
+
CROSS = "CROSS"
|
115
|
+
LEFT_SEMI_JOIN = "LEFT_SEMI_JOIN"
|
116
|
+
|
117
|
+
|
118
|
+
class ExecutionEngine(Enum):
|
119
|
+
"""
|
120
|
+
An enumeration that represents the supported execution engines.
|
121
|
+
|
122
|
+
Attributes:
|
123
|
+
SPARK (str): A string representation of the Apache Spark execution engine.
|
124
|
+
PANDAS (str): A string representation of the Pandas execution engine.
|
125
|
+
|
126
|
+
Methods:
|
127
|
+
None
|
128
|
+
"""
|
129
|
+
|
130
|
+
SPARK = "SPARK"
|
131
|
+
PANDAS = "PANDAS"
|
132
|
+
|
133
|
+
|
134
|
+
class DataFrameType(Enum):
|
135
|
+
"""
|
136
|
+
An enumeration that represents the supported DataFrame types.
|
137
|
+
|
138
|
+
Attributes:
|
139
|
+
SPARK (str): A string representation for spark Data frame type.
|
140
|
+
PANDAS (str): A string representation for pandas Data frame type.
|
141
|
+
|
142
|
+
Methods:
|
143
|
+
None
|
144
|
+
"""
|
145
|
+
|
146
|
+
SPARK = "SPARK"
|
147
|
+
PANDAS = "PANDAS"
|
148
|
+
|
149
|
+
|
150
|
+
class ValidationEngineType(Enum):
|
151
|
+
"""
|
152
|
+
An enumeration that represents the supported validation engines.
|
153
|
+
|
154
|
+
Attributes:
|
155
|
+
GREAT_EXPECTATIONS (str): A string representation of the great expectation execution engine.
|
156
|
+
|
157
|
+
Methods:
|
158
|
+
None
|
159
|
+
"""
|
160
|
+
|
161
|
+
GREAT_EXPECTATIONS = "GREAT_EXPECTATIONS"
|
162
|
+
|
163
|
+
|
164
|
+
class FeatureStoreJobType(Enum):
|
165
|
+
"""
|
166
|
+
An enumeration that represents the Job type.
|
167
|
+
|
168
|
+
Attributes:
|
169
|
+
FEATURE_GROUP_INGESTION (str): A string representing that job is feature group ingestion.
|
170
|
+
DATASET_INGESTION (str): A string representing that job is dataset ingestion.
|
171
|
+
FEATURE_GROUP_DELETION (str): A string representing that job is feature group deletion.
|
172
|
+
DATASET_DELETION (str): A string representing that job is dataset deletion.
|
173
|
+
|
174
|
+
Methods:
|
175
|
+
None
|
176
|
+
"""
|
177
|
+
|
178
|
+
FEATURE_GROUP_INGESTION = "FEATURE_GROUP_INGESTION"
|
179
|
+
DATASET_INGESTION = "DATASET_INGESTION"
|
180
|
+
FEATURE_GROUP_DELETION = "FEATURE_GROUP_DELETION"
|
181
|
+
DATASET_DELETION = "DATASET_DELETION"
|
182
|
+
|
183
|
+
|
184
|
+
class LifecycleState(Enum):
|
185
|
+
"""
|
186
|
+
An enumeration that represents the lifecycle state of feature store resources.
|
187
|
+
|
188
|
+
Attributes:
|
189
|
+
ACTIVE (str): A string representing Active resource.
|
190
|
+
FAILED (str): A string representing Failed resource.
|
191
|
+
NEEDS_ATTENTION (str): A string representing needs_attention resource.
|
192
|
+
|
193
|
+
Methods:
|
194
|
+
None
|
195
|
+
"""
|
196
|
+
|
197
|
+
ACTIVE = "ACTIVE"
|
198
|
+
FAILED = "FAILED"
|
199
|
+
NEEDS_ATTENTION = "NEEDS_ATTENTION"
|
200
|
+
|
201
|
+
|
202
|
+
class JobConfigurationType(Enum):
|
203
|
+
"""
|
204
|
+
An enumeration defining the different types of job configuration modes for Spark.
|
205
|
+
|
206
|
+
Attributes:
|
207
|
+
SPARK_BATCH_AUTOMATIC (str): A string representing automatic job configuration mode for Spark Batch jobs.
|
208
|
+
SPARK_BATCH_MANUAL (str): A string representing manual job configuration mode for Spark Batch jobs.
|
209
|
+
"""
|
210
|
+
|
211
|
+
SPARK_BATCH_AUTOMATIC = "SPARK_BATCH_AUTOMATIC"
|
212
|
+
SPARK_BATCH_MANUAL = "SPARK_BATCH_MANUAL"
|
213
|
+
|
214
|
+
|
215
|
+
class ExpectationType(Enum):
|
216
|
+
"""
|
217
|
+
An enumeration of the available expectation types for a feature store.
|
218
|
+
|
219
|
+
Attributes:
|
220
|
+
STRICT (str): A strict expectation type.
|
221
|
+
LENIENT (str): A lenient expectation type.
|
222
|
+
NO_EXPECTATION (str): A no expectation type.
|
223
|
+
|
224
|
+
Methods:
|
225
|
+
None
|
226
|
+
"""
|
227
|
+
|
228
|
+
STRICT = "STRICT"
|
229
|
+
LENIENT = "LENIENT"
|
230
|
+
NO_EXPECTATION = "NO_EXPECTATION"
|
231
|
+
|
232
|
+
|
233
|
+
class TransformationMode(Enum):
|
234
|
+
"""
|
235
|
+
An enumeration defining the different modes for data transformation.
|
236
|
+
|
237
|
+
Attributes:
|
238
|
+
SQL (str): A string representing the SQL mode, which is used to transform data using SQL queries.
|
239
|
+
PANDAS (str): A string representing the Pandas mode, which is used to transform data using the Pandas library.
|
240
|
+
"""
|
241
|
+
|
242
|
+
SQL = "sql"
|
243
|
+
PANDAS = "pandas"
|
244
|
+
SPARK = "spark"
|
245
|
+
|
246
|
+
|
247
|
+
class FilterOperators(Enum):
|
248
|
+
"""
|
249
|
+
An enumeration defining the different comparison operators for data filtering.
|
250
|
+
|
251
|
+
Attributes:
|
252
|
+
GE (str): A string representing the greater than or equal to operator.
|
253
|
+
GT (str): A string representing the greater than operator.
|
254
|
+
NE (str): A string representing the not equals operator.
|
255
|
+
EQ (str): A string representing the equals operator.
|
256
|
+
LE (str): A string representing the less than or equal to operator.
|
257
|
+
LT (str): A string representing the less than operator.
|
258
|
+
IN (str): A string representing the in operator.
|
259
|
+
LK (str): A string representing the like operator.
|
260
|
+
"""
|
261
|
+
|
262
|
+
GE = "GREATER_THAN_OR_EQUAL"
|
263
|
+
GT = "GREATER_THAN"
|
264
|
+
NE = "NOT_EQUALS"
|
265
|
+
EQ = "EQUALS"
|
266
|
+
LE = "LESS_THAN_OR_EQUAL"
|
267
|
+
LT = "LESS_THAN"
|
268
|
+
IN = "IN"
|
269
|
+
LK = "LIKE"
|
270
|
+
|
271
|
+
|
272
|
+
class FeatureType(Enum):
|
273
|
+
"""
|
274
|
+
An enumeration of the available feature types for a feature store.
|
275
|
+
|
276
|
+
Attributes:
|
277
|
+
STRING (str): A string feature type.
|
278
|
+
INTEGER (str): An integer feature type.
|
279
|
+
FLOAT (str): A float feature type.
|
280
|
+
DOUBLE (str): A double feature type.
|
281
|
+
BOOLEAN (str): A boolean feature type.
|
282
|
+
DATE (str): A date feature type.
|
283
|
+
TIMESTAMP (str): A timestamp feature type.
|
284
|
+
DECIMAL (str): A decimal feature type.
|
285
|
+
BINARY (str): A binary feature type.
|
286
|
+
ARRAY (str): An array feature type.
|
287
|
+
MAP (str): A map feature type.
|
288
|
+
STRUCT (str): A struct feature type.
|
289
|
+
"""
|
290
|
+
|
291
|
+
STRING = "STRING"
|
292
|
+
SHORT = "SHORT"
|
293
|
+
INTEGER = "INTEGER"
|
294
|
+
LONG = "LONG"
|
295
|
+
FLOAT = "FLOAT"
|
296
|
+
DOUBLE = "DOUBLE"
|
297
|
+
BOOLEAN = "BOOLEAN"
|
298
|
+
DATE = "DATE"
|
299
|
+
TIMESTAMP = "TIMESTAMP"
|
300
|
+
DECIMAL = "DECIMAL"
|
301
|
+
BINARY = "BINARY"
|
302
|
+
BYTE = "BYTE"
|
303
|
+
STRING_ARRAY = "STRING_ARRAY"
|
304
|
+
INTEGER_ARRAY = "INTEGER_ARRAY"
|
305
|
+
SHORT_ARRAY = "SHORT_ARRAY"
|
306
|
+
LONG_ARRAY = "LONG_ARRAY"
|
307
|
+
FLOAT_ARRAY = "FLOAT_ARRAY"
|
308
|
+
DOUBLE_ARRAY = "DOUBLE_ARRAY"
|
309
|
+
BINARY_ARRAY = "BINARY_ARRAY"
|
310
|
+
DATE_ARRAY = "DATE_ARRAY"
|
311
|
+
TIMESTAMP_ARRAY = "TIMESTAMP_ARRAY"
|
312
|
+
BYTE_ARRAY = "BYTE_ARRAY"
|
313
|
+
BOOLEAN_ARRAY = "BOOLEAN_ARRAY"
|
314
|
+
STRING_STRING_MAP = "STRING_STRING_MAP"
|
315
|
+
STRING_INTEGER_MAP = "STRING_INTEGER_MAP"
|
316
|
+
STRING_SHORT_MAP = "STRING_SHORT_MAP"
|
317
|
+
STRING_LONG_MAP = "STRING_LONG_MAP"
|
318
|
+
STRING_FLOAT_MAP = "STRING_FLOAT_MAP"
|
319
|
+
STRING_DOUBLE_MAP = "STRING_DOUBLE_MAP"
|
320
|
+
STRING_TIMESTAMP_MAP = "STRING_TIMESTAMP_MAP"
|
321
|
+
STRING_DATE_MAP = "STRING_DATE_MAP"
|
322
|
+
STRING_BYTE_MAP = "STRING_BYTE_MAP"
|
323
|
+
STRING_BINARY_MAP = "STRING_BINARY_MAP"
|
324
|
+
STRING_BOOLEAN_MAP = "STRING_BOOLEAN_MAP"
|
325
|
+
UNKNOWN = "UNKNOWN"
|
326
|
+
COMPLEX = "COMPLEX"
|
327
|
+
|
328
|
+
|
329
|
+
class EntityType(Enum):
|
330
|
+
"""
|
331
|
+
An enumeration of the supported entity types.
|
332
|
+
|
333
|
+
Attributes:
|
334
|
+
FEATURE_GROUP (str): A string representing the feature group.
|
335
|
+
DATASET (str): An string representing the dataset.
|
336
|
+
"""
|
337
|
+
|
338
|
+
FEATURE_GROUP = "FEATURE_GROUP"
|
339
|
+
DATASET = "DATASET"
|