oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ads/aqua/__init__.py +40 -0
- ads/aqua/app.py +507 -0
- ads/aqua/cli.py +96 -0
- ads/aqua/client/__init__.py +3 -0
- ads/aqua/client/client.py +836 -0
- ads/aqua/client/openai_client.py +305 -0
- ads/aqua/common/__init__.py +5 -0
- ads/aqua/common/decorator.py +125 -0
- ads/aqua/common/entities.py +274 -0
- ads/aqua/common/enums.py +134 -0
- ads/aqua/common/errors.py +109 -0
- ads/aqua/common/utils.py +1295 -0
- ads/aqua/config/__init__.py +4 -0
- ads/aqua/config/container_config.py +247 -0
- ads/aqua/config/evaluation/__init__.py +4 -0
- ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
- ads/aqua/config/utils/__init__.py +4 -0
- ads/aqua/config/utils/serializer.py +339 -0
- ads/aqua/constants.py +116 -0
- ads/aqua/data.py +14 -0
- ads/aqua/dummy_data/icon.txt +1 -0
- ads/aqua/dummy_data/oci_model_deployments.json +56 -0
- ads/aqua/dummy_data/oci_models.json +1 -0
- ads/aqua/dummy_data/readme.md +26 -0
- ads/aqua/evaluation/__init__.py +8 -0
- ads/aqua/evaluation/constants.py +53 -0
- ads/aqua/evaluation/entities.py +186 -0
- ads/aqua/evaluation/errors.py +70 -0
- ads/aqua/evaluation/evaluation.py +1814 -0
- ads/aqua/extension/__init__.py +42 -0
- ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
- ads/aqua/extension/base_handler.py +90 -0
- ads/aqua/extension/common_handler.py +121 -0
- ads/aqua/extension/common_ws_msg_handler.py +36 -0
- ads/aqua/extension/deployment_handler.py +381 -0
- ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
- ads/aqua/extension/errors.py +30 -0
- ads/aqua/extension/evaluation_handler.py +129 -0
- ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
- ads/aqua/extension/finetune_handler.py +96 -0
- ads/aqua/extension/model_handler.py +390 -0
- ads/aqua/extension/models/__init__.py +0 -0
- ads/aqua/extension/models/ws_models.py +145 -0
- ads/aqua/extension/models_ws_msg_handler.py +50 -0
- ads/aqua/extension/ui_handler.py +300 -0
- ads/aqua/extension/ui_websocket_handler.py +130 -0
- ads/aqua/extension/utils.py +133 -0
- ads/aqua/finetuning/__init__.py +7 -0
- ads/aqua/finetuning/constants.py +23 -0
- ads/aqua/finetuning/entities.py +181 -0
- ads/aqua/finetuning/finetuning.py +749 -0
- ads/aqua/model/__init__.py +8 -0
- ads/aqua/model/constants.py +60 -0
- ads/aqua/model/entities.py +385 -0
- ads/aqua/model/enums.py +32 -0
- ads/aqua/model/model.py +2134 -0
- ads/aqua/model/utils.py +52 -0
- ads/aqua/modeldeployment/__init__.py +6 -0
- ads/aqua/modeldeployment/constants.py +10 -0
- ads/aqua/modeldeployment/deployment.py +1315 -0
- ads/aqua/modeldeployment/entities.py +653 -0
- ads/aqua/modeldeployment/utils.py +543 -0
- ads/aqua/resources/gpu_shapes_index.json +94 -0
- ads/aqua/server/__init__.py +4 -0
- ads/aqua/server/__main__.py +24 -0
- ads/aqua/server/app.py +47 -0
- ads/aqua/server/aqua_spec.yml +1291 -0
- ads/aqua/training/__init__.py +4 -0
- ads/aqua/training/exceptions.py +476 -0
- ads/aqua/ui.py +519 -0
- ads/automl/__init__.py +9 -0
- ads/automl/driver.py +330 -0
- ads/automl/provider.py +975 -0
- ads/bds/__init__.py +5 -0
- ads/bds/auth.py +127 -0
- ads/bds/big_data_service.py +255 -0
- ads/catalog/__init__.py +19 -0
- ads/catalog/model.py +1576 -0
- ads/catalog/notebook.py +461 -0
- ads/catalog/project.py +468 -0
- ads/catalog/summary.py +178 -0
- ads/common/__init__.py +11 -0
- ads/common/analyzer.py +65 -0
- ads/common/artifact/.model-ignore +63 -0
- ads/common/artifact/__init__.py +10 -0
- ads/common/auth.py +1122 -0
- ads/common/card_identifier.py +83 -0
- ads/common/config.py +647 -0
- ads/common/data.py +165 -0
- ads/common/decorator/__init__.py +9 -0
- ads/common/decorator/argument_to_case.py +88 -0
- ads/common/decorator/deprecate.py +69 -0
- ads/common/decorator/require_nonempty_arg.py +65 -0
- ads/common/decorator/runtime_dependency.py +178 -0
- ads/common/decorator/threaded.py +97 -0
- ads/common/decorator/utils.py +35 -0
- ads/common/dsc_file_system.py +303 -0
- ads/common/error.py +14 -0
- ads/common/extended_enum.py +81 -0
- ads/common/function/__init__.py +5 -0
- ads/common/function/fn_util.py +142 -0
- ads/common/function/func_conf.yaml +25 -0
- ads/common/ipython.py +76 -0
- ads/common/model.py +679 -0
- ads/common/model_artifact.py +1759 -0
- ads/common/model_artifact_schema.json +107 -0
- ads/common/model_export_util.py +664 -0
- ads/common/model_metadata.py +24 -0
- ads/common/object_storage_details.py +296 -0
- ads/common/oci_client.py +179 -0
- ads/common/oci_datascience.py +46 -0
- ads/common/oci_logging.py +1144 -0
- ads/common/oci_mixin.py +957 -0
- ads/common/oci_resource.py +136 -0
- ads/common/serializer.py +559 -0
- ads/common/utils.py +1852 -0
- ads/common/word_lists.py +1491 -0
- ads/common/work_request.py +189 -0
- ads/config.py +1 -0
- ads/data_labeling/__init__.py +13 -0
- ads/data_labeling/boundingbox.py +253 -0
- ads/data_labeling/constants.py +47 -0
- ads/data_labeling/data_labeling_service.py +244 -0
- ads/data_labeling/interface/__init__.py +5 -0
- ads/data_labeling/interface/loader.py +16 -0
- ads/data_labeling/interface/parser.py +16 -0
- ads/data_labeling/interface/reader.py +23 -0
- ads/data_labeling/loader/__init__.py +5 -0
- ads/data_labeling/loader/file_loader.py +241 -0
- ads/data_labeling/metadata.py +110 -0
- ads/data_labeling/mixin/__init__.py +5 -0
- ads/data_labeling/mixin/data_labeling.py +232 -0
- ads/data_labeling/ner.py +129 -0
- ads/data_labeling/parser/__init__.py +5 -0
- ads/data_labeling/parser/dls_record_parser.py +388 -0
- ads/data_labeling/parser/export_metadata_parser.py +94 -0
- ads/data_labeling/parser/export_record_parser.py +473 -0
- ads/data_labeling/reader/__init__.py +5 -0
- ads/data_labeling/reader/dataset_reader.py +574 -0
- ads/data_labeling/reader/dls_record_reader.py +121 -0
- ads/data_labeling/reader/export_record_reader.py +62 -0
- ads/data_labeling/reader/jsonl_reader.py +75 -0
- ads/data_labeling/reader/metadata_reader.py +203 -0
- ads/data_labeling/reader/record_reader.py +263 -0
- ads/data_labeling/record.py +52 -0
- ads/data_labeling/visualizer/__init__.py +5 -0
- ads/data_labeling/visualizer/image_visualizer.py +525 -0
- ads/data_labeling/visualizer/text_visualizer.py +357 -0
- ads/database/__init__.py +5 -0
- ads/database/connection.py +338 -0
- ads/dataset/__init__.py +10 -0
- ads/dataset/capabilities.md +51 -0
- ads/dataset/classification_dataset.py +339 -0
- ads/dataset/correlation.py +226 -0
- ads/dataset/correlation_plot.py +563 -0
- ads/dataset/dask_series.py +173 -0
- ads/dataset/dataframe_transformer.py +110 -0
- ads/dataset/dataset.py +1979 -0
- ads/dataset/dataset_browser.py +360 -0
- ads/dataset/dataset_with_target.py +995 -0
- ads/dataset/exception.py +25 -0
- ads/dataset/factory.py +987 -0
- ads/dataset/feature_engineering_transformer.py +35 -0
- ads/dataset/feature_selection.py +107 -0
- ads/dataset/forecasting_dataset.py +26 -0
- ads/dataset/helper.py +1450 -0
- ads/dataset/label_encoder.py +99 -0
- ads/dataset/mixin/__init__.py +5 -0
- ads/dataset/mixin/dataset_accessor.py +134 -0
- ads/dataset/pipeline.py +58 -0
- ads/dataset/plot.py +710 -0
- ads/dataset/progress.py +86 -0
- ads/dataset/recommendation.py +297 -0
- ads/dataset/recommendation_transformer.py +502 -0
- ads/dataset/regression_dataset.py +14 -0
- ads/dataset/sampled_dataset.py +1050 -0
- ads/dataset/target.py +98 -0
- ads/dataset/timeseries.py +18 -0
- ads/dbmixin/__init__.py +5 -0
- ads/dbmixin/db_pandas_accessor.py +153 -0
- ads/environment/__init__.py +9 -0
- ads/environment/ml_runtime.py +66 -0
- ads/evaluations/README.md +14 -0
- ads/evaluations/__init__.py +109 -0
- ads/evaluations/evaluation_plot.py +983 -0
- ads/evaluations/evaluator.py +1334 -0
- ads/evaluations/statistical_metrics.py +543 -0
- ads/experiments/__init__.py +9 -0
- ads/experiments/capabilities.md +0 -0
- ads/explanations/__init__.py +21 -0
- ads/explanations/base_explainer.py +142 -0
- ads/explanations/capabilities.md +83 -0
- ads/explanations/explainer.py +190 -0
- ads/explanations/mlx_global_explainer.py +1050 -0
- ads/explanations/mlx_interface.py +386 -0
- ads/explanations/mlx_local_explainer.py +287 -0
- ads/explanations/mlx_whatif_explainer.py +201 -0
- ads/feature_engineering/__init__.py +20 -0
- ads/feature_engineering/accessor/__init__.py +5 -0
- ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
- ads/feature_engineering/accessor/mixin/__init__.py +5 -0
- ads/feature_engineering/accessor/mixin/correlation.py +166 -0
- ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
- ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
- ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
- ads/feature_engineering/accessor/mixin/utils.py +65 -0
- ads/feature_engineering/accessor/series_accessor.py +431 -0
- ads/feature_engineering/adsimage/__init__.py +5 -0
- ads/feature_engineering/adsimage/image.py +192 -0
- ads/feature_engineering/adsimage/image_reader.py +170 -0
- ads/feature_engineering/adsimage/interface/__init__.py +5 -0
- ads/feature_engineering/adsimage/interface/reader.py +19 -0
- ads/feature_engineering/adsstring/__init__.py +7 -0
- ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
- ads/feature_engineering/adsstring/string/__init__.py +8 -0
- ads/feature_engineering/data_schema.json +57 -0
- ads/feature_engineering/dataset/__init__.py +5 -0
- ads/feature_engineering/dataset/zip_code_data.py +42062 -0
- ads/feature_engineering/exceptions.py +40 -0
- ads/feature_engineering/feature_type/__init__.py +133 -0
- ads/feature_engineering/feature_type/address.py +184 -0
- ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
- ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
- ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
- ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
- ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
- ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
- ads/feature_engineering/feature_type/adsstring/string.py +258 -0
- ads/feature_engineering/feature_type/base.py +58 -0
- ads/feature_engineering/feature_type/boolean.py +183 -0
- ads/feature_engineering/feature_type/category.py +146 -0
- ads/feature_engineering/feature_type/constant.py +137 -0
- ads/feature_engineering/feature_type/continuous.py +151 -0
- ads/feature_engineering/feature_type/creditcard.py +314 -0
- ads/feature_engineering/feature_type/datetime.py +190 -0
- ads/feature_engineering/feature_type/discrete.py +134 -0
- ads/feature_engineering/feature_type/document.py +43 -0
- ads/feature_engineering/feature_type/gis.py +251 -0
- ads/feature_engineering/feature_type/handler/__init__.py +5 -0
- ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
- ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
- ads/feature_engineering/feature_type/handler/warnings.py +128 -0
- ads/feature_engineering/feature_type/integer.py +142 -0
- ads/feature_engineering/feature_type/ip_address.py +144 -0
- ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
- ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
- ads/feature_engineering/feature_type/lat_long.py +256 -0
- ads/feature_engineering/feature_type/object.py +43 -0
- ads/feature_engineering/feature_type/ordinal.py +132 -0
- ads/feature_engineering/feature_type/phone_number.py +135 -0
- ads/feature_engineering/feature_type/string.py +171 -0
- ads/feature_engineering/feature_type/text.py +93 -0
- ads/feature_engineering/feature_type/unknown.py +43 -0
- ads/feature_engineering/feature_type/zip_code.py +164 -0
- ads/feature_engineering/feature_type_manager.py +406 -0
- ads/feature_engineering/schema.py +795 -0
- ads/feature_engineering/utils.py +245 -0
- ads/feature_store/.readthedocs.yaml +19 -0
- ads/feature_store/README.md +65 -0
- ads/feature_store/__init__.py +9 -0
- ads/feature_store/common/__init__.py +0 -0
- ads/feature_store/common/enums.py +339 -0
- ads/feature_store/common/exceptions.py +18 -0
- ads/feature_store/common/spark_session_singleton.py +125 -0
- ads/feature_store/common/utils/__init__.py +0 -0
- ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
- ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
- ads/feature_store/common/utils/transformation_utils.py +82 -0
- ads/feature_store/common/utils/utility.py +403 -0
- ads/feature_store/data_validation/__init__.py +0 -0
- ads/feature_store/data_validation/great_expectation.py +129 -0
- ads/feature_store/dataset.py +1230 -0
- ads/feature_store/dataset_job.py +530 -0
- ads/feature_store/docs/Dockerfile +7 -0
- ads/feature_store/docs/Makefile +44 -0
- ads/feature_store/docs/conf.py +28 -0
- ads/feature_store/docs/requirements.txt +14 -0
- ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
- ads/feature_store/docs/source/cicd.rst +137 -0
- ads/feature_store/docs/source/conf.py +86 -0
- ads/feature_store/docs/source/data_versioning.rst +33 -0
- ads/feature_store/docs/source/dataset.rst +388 -0
- ads/feature_store/docs/source/dataset_job.rst +27 -0
- ads/feature_store/docs/source/demo.rst +70 -0
- ads/feature_store/docs/source/entity.rst +78 -0
- ads/feature_store/docs/source/feature_group.rst +624 -0
- ads/feature_store/docs/source/feature_group_job.rst +29 -0
- ads/feature_store/docs/source/feature_store.rst +122 -0
- ads/feature_store/docs/source/feature_store_class.rst +123 -0
- ads/feature_store/docs/source/feature_validation.rst +66 -0
- ads/feature_store/docs/source/figures/cicd.png +0 -0
- ads/feature_store/docs/source/figures/data_validation.png +0 -0
- ads/feature_store/docs/source/figures/data_versioning.png +0 -0
- ads/feature_store/docs/source/figures/dataset.gif +0 -0
- ads/feature_store/docs/source/figures/dataset.png +0 -0
- ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
- ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
- ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
- ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
- ads/feature_store/docs/source/figures/entity.png +0 -0
- ads/feature_store/docs/source/figures/feature_group.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
- ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
- ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
- ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
- ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
- ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
- ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
- ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
- ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
- ads/feature_store/docs/source/figures/overview.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
- ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
- ads/feature_store/docs/source/figures/stats_1.png +0 -0
- ads/feature_store/docs/source/figures/stats_2.png +0 -0
- ads/feature_store/docs/source/figures/stats_d.png +0 -0
- ads/feature_store/docs/source/figures/stats_fg.png +0 -0
- ads/feature_store/docs/source/figures/transformation.png +0 -0
- ads/feature_store/docs/source/figures/transformations.gif +0 -0
- ads/feature_store/docs/source/figures/validation.png +0 -0
- ads/feature_store/docs/source/figures/validation_fg.png +0 -0
- ads/feature_store/docs/source/figures/validation_results.png +0 -0
- ads/feature_store/docs/source/figures/validation_summary.png +0 -0
- ads/feature_store/docs/source/index.rst +81 -0
- ads/feature_store/docs/source/module.rst +8 -0
- ads/feature_store/docs/source/notebook.rst +94 -0
- ads/feature_store/docs/source/overview.rst +47 -0
- ads/feature_store/docs/source/quickstart.rst +176 -0
- ads/feature_store/docs/source/release_notes.rst +194 -0
- ads/feature_store/docs/source/setup_feature_store.rst +81 -0
- ads/feature_store/docs/source/statistics.rst +58 -0
- ads/feature_store/docs/source/transformation.rst +199 -0
- ads/feature_store/docs/source/ui.rst +65 -0
- ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
- ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
- ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
- ads/feature_store/entity.py +718 -0
- ads/feature_store/execution_strategy/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
- ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
- ads/feature_store/execution_strategy/engine/__init__.py +0 -0
- ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
- ads/feature_store/execution_strategy/execution_strategy.py +113 -0
- ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
- ads/feature_store/execution_strategy/spark/__init__.py +0 -0
- ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
- ads/feature_store/feature.py +192 -0
- ads/feature_store/feature_group.py +1494 -0
- ads/feature_store/feature_group_expectation.py +346 -0
- ads/feature_store/feature_group_job.py +602 -0
- ads/feature_store/feature_lineage/__init__.py +0 -0
- ads/feature_store/feature_lineage/graphviz_service.py +180 -0
- ads/feature_store/feature_option_details.py +50 -0
- ads/feature_store/feature_statistics/__init__.py +0 -0
- ads/feature_store/feature_statistics/statistics_service.py +99 -0
- ads/feature_store/feature_store.py +699 -0
- ads/feature_store/feature_store_registrar.py +518 -0
- ads/feature_store/input_feature_detail.py +149 -0
- ads/feature_store/mixin/__init__.py +4 -0
- ads/feature_store/mixin/oci_feature_store.py +145 -0
- ads/feature_store/model_details.py +73 -0
- ads/feature_store/query/__init__.py +0 -0
- ads/feature_store/query/filter.py +266 -0
- ads/feature_store/query/generator/__init__.py +0 -0
- ads/feature_store/query/generator/query_generator.py +298 -0
- ads/feature_store/query/join.py +161 -0
- ads/feature_store/query/query.py +403 -0
- ads/feature_store/query/validator/__init__.py +0 -0
- ads/feature_store/query/validator/query_validator.py +57 -0
- ads/feature_store/response/__init__.py +0 -0
- ads/feature_store/response/response_builder.py +68 -0
- ads/feature_store/service/__init__.py +0 -0
- ads/feature_store/service/oci_dataset.py +139 -0
- ads/feature_store/service/oci_dataset_job.py +199 -0
- ads/feature_store/service/oci_entity.py +125 -0
- ads/feature_store/service/oci_feature_group.py +164 -0
- ads/feature_store/service/oci_feature_group_job.py +214 -0
- ads/feature_store/service/oci_feature_store.py +182 -0
- ads/feature_store/service/oci_lineage.py +87 -0
- ads/feature_store/service/oci_transformation.py +104 -0
- ads/feature_store/statistics/__init__.py +0 -0
- ads/feature_store/statistics/abs_feature_value.py +49 -0
- ads/feature_store/statistics/charts/__init__.py +0 -0
- ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
- ads/feature_store/statistics/charts/box_plot.py +148 -0
- ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
- ads/feature_store/statistics/charts/probability_distribution.py +68 -0
- ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
- ads/feature_store/statistics/feature_stat.py +126 -0
- ads/feature_store/statistics/generic_feature_value.py +33 -0
- ads/feature_store/statistics/statistics.py +41 -0
- ads/feature_store/statistics_config.py +101 -0
- ads/feature_store/templates/feature_store_template.yaml +45 -0
- ads/feature_store/transformation.py +499 -0
- ads/feature_store/validation_output.py +57 -0
- ads/hpo/__init__.py +9 -0
- ads/hpo/_imports.py +91 -0
- ads/hpo/ads_search_space.py +439 -0
- ads/hpo/distributions.py +325 -0
- ads/hpo/objective.py +280 -0
- ads/hpo/search_cv.py +1657 -0
- ads/hpo/stopping_criterion.py +75 -0
- ads/hpo/tuner_artifact.py +413 -0
- ads/hpo/utils.py +91 -0
- ads/hpo/validation.py +140 -0
- ads/hpo/visualization/__init__.py +5 -0
- ads/hpo/visualization/_contour.py +23 -0
- ads/hpo/visualization/_edf.py +20 -0
- ads/hpo/visualization/_intermediate_values.py +21 -0
- ads/hpo/visualization/_optimization_history.py +25 -0
- ads/hpo/visualization/_parallel_coordinate.py +169 -0
- ads/hpo/visualization/_param_importances.py +26 -0
- ads/jobs/__init__.py +53 -0
- ads/jobs/ads_job.py +663 -0
- ads/jobs/builders/__init__.py +5 -0
- ads/jobs/builders/base.py +156 -0
- ads/jobs/builders/infrastructure/__init__.py +6 -0
- ads/jobs/builders/infrastructure/base.py +165 -0
- ads/jobs/builders/infrastructure/dataflow.py +1252 -0
- ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
- ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
- ads/jobs/builders/infrastructure/utils.py +65 -0
- ads/jobs/builders/runtimes/__init__.py +5 -0
- ads/jobs/builders/runtimes/artifact.py +338 -0
- ads/jobs/builders/runtimes/base.py +325 -0
- ads/jobs/builders/runtimes/container_runtime.py +242 -0
- ads/jobs/builders/runtimes/python_runtime.py +1016 -0
- ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
- ads/jobs/cli.py +104 -0
- ads/jobs/env_var_parser.py +131 -0
- ads/jobs/extension.py +160 -0
- ads/jobs/schema/__init__.py +5 -0
- ads/jobs/schema/infrastructure_schema.json +116 -0
- ads/jobs/schema/job_schema.json +42 -0
- ads/jobs/schema/runtime_schema.json +183 -0
- ads/jobs/schema/validator.py +141 -0
- ads/jobs/serializer.py +296 -0
- ads/jobs/templates/__init__.py +5 -0
- ads/jobs/templates/container.py +6 -0
- ads/jobs/templates/driver_notebook.py +177 -0
- ads/jobs/templates/driver_oci.py +500 -0
- ads/jobs/templates/driver_python.py +48 -0
- ads/jobs/templates/driver_pytorch.py +852 -0
- ads/jobs/templates/driver_utils.py +615 -0
- ads/jobs/templates/hostname_from_env.c +55 -0
- ads/jobs/templates/oci_metrics.py +181 -0
- ads/jobs/utils.py +104 -0
- ads/llm/__init__.py +28 -0
- ads/llm/autogen/__init__.py +2 -0
- ads/llm/autogen/constants.py +15 -0
- ads/llm/autogen/reports/__init__.py +2 -0
- ads/llm/autogen/reports/base.py +67 -0
- ads/llm/autogen/reports/data.py +103 -0
- ads/llm/autogen/reports/session.py +526 -0
- ads/llm/autogen/reports/templates/chat_box.html +13 -0
- ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
- ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
- ads/llm/autogen/reports/utils.py +56 -0
- ads/llm/autogen/v02/__init__.py +4 -0
- ads/llm/autogen/v02/client.py +295 -0
- ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
- ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
- ads/llm/autogen/v02/loggers/__init__.py +6 -0
- ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
- ads/llm/autogen/v02/loggers/session_logger.py +580 -0
- ads/llm/autogen/v02/loggers/utils.py +86 -0
- ads/llm/autogen/v02/runtime_logging.py +163 -0
- ads/llm/chain.py +268 -0
- ads/llm/chat_template.py +31 -0
- ads/llm/deploy.py +63 -0
- ads/llm/guardrails/__init__.py +5 -0
- ads/llm/guardrails/base.py +442 -0
- ads/llm/guardrails/huggingface.py +44 -0
- ads/llm/langchain/__init__.py +5 -0
- ads/llm/langchain/plugins/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
- ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
- ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
- ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
- ads/llm/langchain/plugins/llms/__init__.py +5 -0
- ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
- ads/llm/requirements.txt +3 -0
- ads/llm/serialize.py +219 -0
- ads/llm/serializers/__init__.py +0 -0
- ads/llm/serializers/retrieval_qa.py +153 -0
- ads/llm/serializers/runnable_parallel.py +27 -0
- ads/llm/templates/score_chain.jinja2 +155 -0
- ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
- ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
- ads/model/__init__.py +52 -0
- ads/model/artifact.py +573 -0
- ads/model/artifact_downloader.py +254 -0
- ads/model/artifact_uploader.py +267 -0
- ads/model/base_properties.py +238 -0
- ads/model/common/.model-ignore +66 -0
- ads/model/common/__init__.py +5 -0
- ads/model/common/utils.py +142 -0
- ads/model/datascience_model.py +2635 -0
- ads/model/deployment/__init__.py +20 -0
- ads/model/deployment/common/__init__.py +5 -0
- ads/model/deployment/common/utils.py +308 -0
- ads/model/deployment/model_deployer.py +466 -0
- ads/model/deployment/model_deployment.py +1846 -0
- ads/model/deployment/model_deployment_infrastructure.py +671 -0
- ads/model/deployment/model_deployment_properties.py +493 -0
- ads/model/deployment/model_deployment_runtime.py +838 -0
- ads/model/extractor/__init__.py +5 -0
- ads/model/extractor/automl_extractor.py +74 -0
- ads/model/extractor/embedding_onnx_extractor.py +80 -0
- ads/model/extractor/huggingface_extractor.py +88 -0
- ads/model/extractor/keras_extractor.py +84 -0
- ads/model/extractor/lightgbm_extractor.py +93 -0
- ads/model/extractor/model_info_extractor.py +114 -0
- ads/model/extractor/model_info_extractor_factory.py +105 -0
- ads/model/extractor/pytorch_extractor.py +87 -0
- ads/model/extractor/sklearn_extractor.py +112 -0
- ads/model/extractor/spark_extractor.py +89 -0
- ads/model/extractor/tensorflow_extractor.py +85 -0
- ads/model/extractor/xgboost_extractor.py +94 -0
- ads/model/framework/__init__.py +5 -0
- ads/model/framework/automl_model.py +178 -0
- ads/model/framework/embedding_onnx_model.py +438 -0
- ads/model/framework/huggingface_model.py +399 -0
- ads/model/framework/lightgbm_model.py +266 -0
- ads/model/framework/pytorch_model.py +266 -0
- ads/model/framework/sklearn_model.py +250 -0
- ads/model/framework/spark_model.py +326 -0
- ads/model/framework/tensorflow_model.py +254 -0
- ads/model/framework/xgboost_model.py +258 -0
- ads/model/generic_model.py +3518 -0
- ads/model/model_artifact_boilerplate/README.md +381 -0
- ads/model/model_artifact_boilerplate/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
- ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
- ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
- ads/model/model_artifact_boilerplate/score.py +61 -0
- ads/model/model_file_description_schema.json +68 -0
- ads/model/model_introspect.py +331 -0
- ads/model/model_metadata.py +1810 -0
- ads/model/model_metadata_mixin.py +460 -0
- ads/model/model_properties.py +63 -0
- ads/model/model_version_set.py +739 -0
- ads/model/runtime/__init__.py +5 -0
- ads/model/runtime/env_info.py +306 -0
- ads/model/runtime/model_deployment_details.py +37 -0
- ads/model/runtime/model_provenance_details.py +58 -0
- ads/model/runtime/runtime_info.py +81 -0
- ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
- ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
- ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
- ads/model/runtime/utils.py +201 -0
- ads/model/serde/__init__.py +5 -0
- ads/model/serde/common.py +40 -0
- ads/model/serde/model_input.py +547 -0
- ads/model/serde/model_serializer.py +1184 -0
- ads/model/service/__init__.py +5 -0
- ads/model/service/oci_datascience_model.py +1076 -0
- ads/model/service/oci_datascience_model_deployment.py +500 -0
- ads/model/service/oci_datascience_model_version_set.py +176 -0
- ads/model/transformer/__init__.py +5 -0
- ads/model/transformer/onnx_transformer.py +324 -0
- ads/mysqldb/__init__.py +5 -0
- ads/mysqldb/mysql_db.py +227 -0
- ads/opctl/__init__.py +18 -0
- ads/opctl/anomaly_detection.py +11 -0
- ads/opctl/backend/__init__.py +5 -0
- ads/opctl/backend/ads_dataflow.py +353 -0
- ads/opctl/backend/ads_ml_job.py +710 -0
- ads/opctl/backend/ads_ml_pipeline.py +164 -0
- ads/opctl/backend/ads_model_deployment.py +209 -0
- ads/opctl/backend/base.py +146 -0
- ads/opctl/backend/local.py +1053 -0
- ads/opctl/backend/marketplace/__init__.py +9 -0
- ads/opctl/backend/marketplace/helm_helper.py +173 -0
- ads/opctl/backend/marketplace/local_marketplace.py +271 -0
- ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
- ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
- ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
- ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
- ads/opctl/backend/marketplace/models/__init__.py +5 -0
- ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
- ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
- ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
- ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
- ads/opctl/cli.py +707 -0
- ads/opctl/cmds.py +869 -0
- ads/opctl/conda/__init__.py +5 -0
- ads/opctl/conda/cli.py +193 -0
- ads/opctl/conda/cmds.py +749 -0
- ads/opctl/conda/config.yaml +34 -0
- ads/opctl/conda/manifest_template.yaml +13 -0
- ads/opctl/conda/multipart_uploader.py +188 -0
- ads/opctl/conda/pack.py +89 -0
- ads/opctl/config/__init__.py +5 -0
- ads/opctl/config/base.py +57 -0
- ads/opctl/config/diagnostics/__init__.py +5 -0
- ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
- ads/opctl/config/merger.py +255 -0
- ads/opctl/config/resolver.py +297 -0
- ads/opctl/config/utils.py +79 -0
- ads/opctl/config/validator.py +17 -0
- ads/opctl/config/versioner.py +68 -0
- ads/opctl/config/yaml_parsers/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/base.py +58 -0
- ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
- ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
- ads/opctl/constants.py +66 -0
- ads/opctl/decorator/__init__.py +5 -0
- ads/opctl/decorator/common.py +129 -0
- ads/opctl/diagnostics/__init__.py +5 -0
- ads/opctl/diagnostics/__main__.py +25 -0
- ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
- ads/opctl/diagnostics/check_requirements.py +144 -0
- ads/opctl/diagnostics/requirement_exception.py +9 -0
- ads/opctl/distributed/README.md +109 -0
- ads/opctl/distributed/__init__.py +5 -0
- ads/opctl/distributed/certificates.py +32 -0
- ads/opctl/distributed/cli.py +207 -0
- ads/opctl/distributed/cmds.py +731 -0
- ads/opctl/distributed/common/__init__.py +5 -0
- ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
- ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
- ads/opctl/distributed/common/cluster_config_helper.py +103 -0
- ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
- ads/opctl/distributed/common/cluster_runner.py +54 -0
- ads/opctl/distributed/common/framework_factory.py +29 -0
- ads/opctl/docker/Dockerfile.job +103 -0
- ads/opctl/docker/Dockerfile.job.arm +107 -0
- ads/opctl/docker/Dockerfile.job.gpu +175 -0
- ads/opctl/docker/base-env.yaml +13 -0
- ads/opctl/docker/cuda.repo +6 -0
- ads/opctl/docker/operator/.dockerignore +0 -0
- ads/opctl/docker/operator/Dockerfile +41 -0
- ads/opctl/docker/operator/Dockerfile.gpu +85 -0
- ads/opctl/docker/operator/cuda.repo +6 -0
- ads/opctl/docker/operator/environment.yaml +8 -0
- ads/opctl/forecast.py +11 -0
- ads/opctl/index.yaml +3 -0
- ads/opctl/model/__init__.py +5 -0
- ads/opctl/model/cli.py +65 -0
- ads/opctl/model/cmds.py +73 -0
- ads/opctl/operator/README.md +4 -0
- ads/opctl/operator/__init__.py +31 -0
- ads/opctl/operator/cli.py +344 -0
- ads/opctl/operator/cmd.py +596 -0
- ads/opctl/operator/common/__init__.py +5 -0
- ads/opctl/operator/common/backend_factory.py +460 -0
- ads/opctl/operator/common/const.py +27 -0
- ads/opctl/operator/common/data/synthetic.csv +16001 -0
- ads/opctl/operator/common/dictionary_merger.py +148 -0
- ads/opctl/operator/common/errors.py +42 -0
- ads/opctl/operator/common/operator_config.py +99 -0
- ads/opctl/operator/common/operator_loader.py +811 -0
- ads/opctl/operator/common/operator_schema.yaml +130 -0
- ads/opctl/operator/common/operator_yaml_generator.py +152 -0
- ads/opctl/operator/common/utils.py +208 -0
- ads/opctl/operator/lowcode/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
- ads/opctl/operator/lowcode/anomaly/README.md +207 -0
- ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
- ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
- ads/opctl/operator/lowcode/anomaly/const.py +167 -0
- ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
- ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
- ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
- ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
- ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
- ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
- ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
- ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
- ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
- ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
- ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
- ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
- ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
- ads/opctl/operator/lowcode/common/__init__.py +5 -0
- ads/opctl/operator/lowcode/common/const.py +10 -0
- ads/opctl/operator/lowcode/common/data.py +116 -0
- ads/opctl/operator/lowcode/common/errors.py +47 -0
- ads/opctl/operator/lowcode/common/transformations.py +296 -0
- ads/opctl/operator/lowcode/common/utils.py +384 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
- ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
- ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
- ads/opctl/operator/lowcode/forecast/README.md +209 -0
- ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
- ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
- ads/opctl/operator/lowcode/forecast/const.py +92 -0
- ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
- ads/opctl/operator/lowcode/forecast/errors.py +26 -0
- ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
- ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
- ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
- ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
- ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
- ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
- ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
- ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
- ads/opctl/operator/lowcode/forecast/model/prophet.py +450 -0
- ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
- ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
- ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
- ads/opctl/operator/lowcode/forecast/utils.py +397 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
- ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
- ads/opctl/operator/lowcode/pii/MLoperator +17 -0
- ads/opctl/operator/lowcode/pii/README.md +208 -0
- ads/opctl/operator/lowcode/pii/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/__main__.py +78 -0
- ads/opctl/operator/lowcode/pii/cmd.py +39 -0
- ads/opctl/operator/lowcode/pii/constant.py +84 -0
- ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
- ads/opctl/operator/lowcode/pii/errors.py +27 -0
- ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
- ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
- ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
- ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
- ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
- ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
- ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
- ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
- ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
- ads/opctl/operator/lowcode/pii/model/report.py +487 -0
- ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
- ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
- ads/opctl/operator/lowcode/pii/utils.py +43 -0
- ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
- ads/opctl/operator/lowcode/recommender/README.md +206 -0
- ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
- ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
- ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
- ads/opctl/operator/lowcode/recommender/constant.py +30 -0
- ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
- ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
- ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
- ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
- ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
- ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
- ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
- ads/opctl/operator/lowcode/recommender/utils.py +13 -0
- ads/opctl/operator/runtime/__init__.py +5 -0
- ads/opctl/operator/runtime/const.py +17 -0
- ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
- ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
- ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
- ads/opctl/operator/runtime/runtime.py +115 -0
- ads/opctl/schema.yaml.yml +36 -0
- ads/opctl/script.py +40 -0
- ads/opctl/spark/__init__.py +5 -0
- ads/opctl/spark/cli.py +43 -0
- ads/opctl/spark/cmds.py +147 -0
- ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
- ads/opctl/utils.py +344 -0
- ads/oracledb/__init__.py +5 -0
- ads/oracledb/oracle_db.py +346 -0
- ads/pipeline/__init__.py +39 -0
- ads/pipeline/ads_pipeline.py +2279 -0
- ads/pipeline/ads_pipeline_run.py +772 -0
- ads/pipeline/ads_pipeline_step.py +605 -0
- ads/pipeline/builders/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/__init__.py +5 -0
- ads/pipeline/builders/infrastructure/custom_script.py +32 -0
- ads/pipeline/cli.py +119 -0
- ads/pipeline/extension.py +291 -0
- ads/pipeline/schema/__init__.py +5 -0
- ads/pipeline/schema/cs_step_schema.json +35 -0
- ads/pipeline/schema/ml_step_schema.json +31 -0
- ads/pipeline/schema/pipeline_schema.json +71 -0
- ads/pipeline/visualizer/__init__.py +5 -0
- ads/pipeline/visualizer/base.py +570 -0
- ads/pipeline/visualizer/graph_renderer.py +272 -0
- ads/pipeline/visualizer/text_renderer.py +84 -0
- ads/secrets/__init__.py +11 -0
- ads/secrets/adb.py +386 -0
- ads/secrets/auth_token.py +86 -0
- ads/secrets/big_data_service.py +365 -0
- ads/secrets/mysqldb.py +149 -0
- ads/secrets/oracledb.py +160 -0
- ads/secrets/secrets.py +407 -0
- ads/telemetry/__init__.py +7 -0
- ads/telemetry/base.py +69 -0
- ads/telemetry/client.py +122 -0
- ads/telemetry/telemetry.py +257 -0
- ads/templates/dataflow_pyspark.jinja2 +13 -0
- ads/templates/dataflow_sparksql.jinja2 +22 -0
- ads/templates/func.jinja2 +20 -0
- ads/templates/schemas/openapi.json +1740 -0
- ads/templates/score-pkl.jinja2 +173 -0
- ads/templates/score.jinja2 +322 -0
- ads/templates/score_embedding_onnx.jinja2 +202 -0
- ads/templates/score_generic.jinja2 +165 -0
- ads/templates/score_huggingface_pipeline.jinja2 +217 -0
- ads/templates/score_lightgbm.jinja2 +185 -0
- ads/templates/score_onnx.jinja2 +407 -0
- ads/templates/score_onnx_new.jinja2 +473 -0
- ads/templates/score_oracle_automl.jinja2 +185 -0
- ads/templates/score_pyspark.jinja2 +154 -0
- ads/templates/score_pytorch.jinja2 +219 -0
- ads/templates/score_scikit-learn.jinja2 +184 -0
- ads/templates/score_tensorflow.jinja2 +184 -0
- ads/templates/score_xgboost.jinja2 +178 -0
- ads/text_dataset/__init__.py +5 -0
- ads/text_dataset/backends.py +211 -0
- ads/text_dataset/dataset.py +445 -0
- ads/text_dataset/extractor.py +207 -0
- ads/text_dataset/options.py +53 -0
- ads/text_dataset/udfs.py +22 -0
- ads/text_dataset/utils.py +49 -0
- ads/type_discovery/__init__.py +9 -0
- ads/type_discovery/abstract_detector.py +21 -0
- ads/type_discovery/constant_detector.py +41 -0
- ads/type_discovery/continuous_detector.py +54 -0
- ads/type_discovery/credit_card_detector.py +99 -0
- ads/type_discovery/datetime_detector.py +92 -0
- ads/type_discovery/discrete_detector.py +118 -0
- ads/type_discovery/document_detector.py +146 -0
- ads/type_discovery/ip_detector.py +68 -0
- ads/type_discovery/latlon_detector.py +90 -0
- ads/type_discovery/phone_number_detector.py +63 -0
- ads/type_discovery/type_discovery_driver.py +87 -0
- ads/type_discovery/typed_feature.py +594 -0
- ads/type_discovery/unknown_detector.py +41 -0
- ads/type_discovery/zipcode_detector.py +48 -0
- ads/vault/__init__.py +7 -0
- ads/vault/vault.py +237 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/METADATA +150 -149
- oracle_ads-2.13.10rc0.dist-info/RECORD +858 -0
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/WHEEL +1 -2
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/entry_points.txt +2 -1
- oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
- oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
- {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,399 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*--
|
3
|
+
|
4
|
+
# Copyright (c) 2023 Oracle and/or its affiliates.
|
5
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
|
+
|
7
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
8
|
+
|
9
|
+
import PIL
|
10
|
+
|
11
|
+
from ads.model.extractor.huggingface_extractor import HuggingFaceExtractor
|
12
|
+
from ads.model.generic_model import FrameworkSpecificModel
|
13
|
+
from ads.model.model_properties import ModelProperties
|
14
|
+
from ads.model.serde.model_serializer import HuggingFaceSerializerType
|
15
|
+
from ads.common.decorator.runtime_dependency import (
|
16
|
+
runtime_dependency,
|
17
|
+
OptionalDependency,
|
18
|
+
)
|
19
|
+
from ads.model.serde.model_serializer import ModelSerializerType
|
20
|
+
from ads.model.serde.common import SERDE
|
21
|
+
|
22
|
+
|
23
|
+
class HuggingFacePipelineModel(FrameworkSpecificModel):
|
24
|
+
"""HuggingFacePipelineModel class for estimators from HuggingFace framework.
|
25
|
+
|
26
|
+
Attributes
|
27
|
+
----------
|
28
|
+
algorithm: str
|
29
|
+
The algorithm of the model.
|
30
|
+
artifact_dir: str
|
31
|
+
Artifact directory to store the files needed for deployment.
|
32
|
+
auth: Dict
|
33
|
+
Default authentication is set using the `ads.set_auth` API. To override the
|
34
|
+
default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create
|
35
|
+
an authentication signer to instantiate an IdentityClient object.
|
36
|
+
estimator: Callable
|
37
|
+
A trained HuggingFace Pipeline using transformers.
|
38
|
+
framework: str
|
39
|
+
"transformers", the framework name of the model.
|
40
|
+
hyperparameter: dict
|
41
|
+
The hyperparameters of the estimator.
|
42
|
+
metadata_custom: ModelCustomMetadata
|
43
|
+
The model custom metadata.
|
44
|
+
metadata_provenance: ModelProvenanceMetadata
|
45
|
+
The model provenance metadata.
|
46
|
+
metadata_taxonomy: ModelTaxonomyMetadata
|
47
|
+
The model taxonomy metadata.
|
48
|
+
model_artifact: ModelArtifact
|
49
|
+
This is built by calling prepare.
|
50
|
+
model_deployment: ModelDeployment
|
51
|
+
A ModelDeployment instance.
|
52
|
+
model_file_name: str
|
53
|
+
Name of the serialized model.
|
54
|
+
model_id: str
|
55
|
+
The model ID.
|
56
|
+
properties: ModelProperties
|
57
|
+
ModelProperties object required to save and deploy model.
|
58
|
+
runtime_info: RuntimeInfo
|
59
|
+
A RuntimeInfo instance.
|
60
|
+
schema_input: Schema
|
61
|
+
Schema describes the structure of the input data.
|
62
|
+
schema_output: Schema
|
63
|
+
Schema describes the structure of the output data.
|
64
|
+
serialize: bool
|
65
|
+
Whether to serialize the model to pkl file by default. If False, you need to serialize the model manually,
|
66
|
+
save it under artifact_dir and update the score.py manually.
|
67
|
+
version: str
|
68
|
+
The framework version of the model.
|
69
|
+
|
70
|
+
Methods
|
71
|
+
-------
|
72
|
+
delete_deployment(...)
|
73
|
+
Deletes the current model deployment.
|
74
|
+
deploy(..., **kwargs)
|
75
|
+
Deploys a model.
|
76
|
+
from_model_artifact(uri, model_file_name, artifact_dir, ..., **kwargs)
|
77
|
+
Loads model from the specified folder, or zip/tar archive.
|
78
|
+
from_model_catalog(model_id, model_file_name, artifact_dir, ..., **kwargs)
|
79
|
+
Loads model from model catalog.
|
80
|
+
introspect(...)
|
81
|
+
Runs model introspection.
|
82
|
+
predict(data, ...)
|
83
|
+
Returns prediction of input data run against the model deployment endpoint.
|
84
|
+
prepare(..., **kwargs)
|
85
|
+
Prepare and save the score.py, serialized model and runtime.yaml file.
|
86
|
+
reload(...)
|
87
|
+
Reloads the model artifact files: `score.py` and the `runtime.yaml`.
|
88
|
+
save(..., **kwargs)
|
89
|
+
Saves model artifacts to the model catalog.
|
90
|
+
summary_status(...)
|
91
|
+
Gets a summary table of the current status.
|
92
|
+
verify(data, ...)
|
93
|
+
Tests if deployment works in local environment.
|
94
|
+
|
95
|
+
Examples
|
96
|
+
--------
|
97
|
+
>>> # Image Classification
|
98
|
+
>>> from transformers import pipeline
|
99
|
+
>>> import tempfile
|
100
|
+
>>> import PIL.Image
|
101
|
+
>>> import ads
|
102
|
+
>>> import requests
|
103
|
+
>>> import cloudpickle
|
104
|
+
>>> ## Download image data
|
105
|
+
>>> image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
|
106
|
+
>>> image = PIL.Image.open(requests.get(image_link, stream=True).raw)
|
107
|
+
>>> image_bytes = cloudpickle.dumps(image) # convert image to bytes
|
108
|
+
>>> ## Download a pretrained model
|
109
|
+
>>> vision_classifier = pipeline(model="google/vit-base-patch16-224")
|
110
|
+
>>> preds = vision_classifier(images=image)
|
111
|
+
>>> ## Initiate a HuggingFacePipelineModel instance
|
112
|
+
>>> vision_model = HuggingFacePipelineModel(vision_classifier, artifact_dir=tempfile.mkdtemp())
|
113
|
+
>>> ## Prepare
|
114
|
+
>>> vision_model.prepare(inference_conda_env="pytorch110_p38_cpu_v1", force_overwrite=True)
|
115
|
+
>>> ## Verify
|
116
|
+
>>> vision_model.verify(image)
|
117
|
+
>>> vision_model.verify(image_bytes)
|
118
|
+
>>> ## Save
|
119
|
+
>>> vision_model.save()
|
120
|
+
>>> ## Deploy
|
121
|
+
>>> log_group_id = "<log_group_id>"
|
122
|
+
>>> log_id = "<log_id>"
|
123
|
+
>>> vision_model.deploy(deployment_bandwidth_mbps=1000,
|
124
|
+
... wait_for_completion=False,
|
125
|
+
... deployment_log_group_id = log_group_id,
|
126
|
+
... deployment_access_log_id = log_id,
|
127
|
+
... deployment_predict_log_id = log_id)
|
128
|
+
>>> ## Predict from endpoint
|
129
|
+
>>> vision_model.predict(image)
|
130
|
+
>>> vision_model.predict(image_bytes)
|
131
|
+
>>> ### Invoke the model
|
132
|
+
>>> auth = ads.common.auth.default_signer()['signer']
|
133
|
+
>>> endpoint = vision_model.model_deployment.url + "/predict"
|
134
|
+
>>> headers = {"Content-Type": "application/octet-stream"}
|
135
|
+
>>> requests.post(endpoint, data=image_bytes, auth=auth, headers=headers).json()
|
136
|
+
|
137
|
+
Examples
|
138
|
+
--------
|
139
|
+
>>> # Image Segmentation
|
140
|
+
>>> from transformers import pipeline
|
141
|
+
>>> import tempfile
|
142
|
+
>>> import PIL.Image
|
143
|
+
>>> import ads
|
144
|
+
>>> import requests
|
145
|
+
>>> import cloudpickle
|
146
|
+
>>> ## Download image data
|
147
|
+
>>> image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
|
148
|
+
>>> image = PIL.Image.open(requests.get(image_link, stream=True).raw)
|
149
|
+
>>> image_bytes = cloudpickle.dumps(image) # convert image to bytes
|
150
|
+
>>> ## Download pretrained model
|
151
|
+
>>> segmenter = pipeline(task="image-segmentation")
|
152
|
+
>>> preds = segmenter(image)
|
153
|
+
>>> ## Initiate a HuggingFacePipelineModel instance
|
154
|
+
>>> segmentation_model = HuggingFacePipelineModel(segmenter, artifact_dir=empfile.mkdtemp())
|
155
|
+
>>> ## Prepare
|
156
|
+
>>> conda = "oci://bucket@namespace/path/to/conda/pack"
|
157
|
+
>>> python_version = "3.8"
|
158
|
+
>>> segmentation_model.prepare(inference_conda_env=conda, inference_python_version = python_version, force_overwrite=True)
|
159
|
+
>>> ## Verify
|
160
|
+
>>> segmentation_model.verify(data=image)
|
161
|
+
>>> segmentation_model.verify(data=image_bytes)
|
162
|
+
>>> ## Save
|
163
|
+
>>> segmentation_model.save()
|
164
|
+
>>> log_group_id = "<log_group_id>"
|
165
|
+
>>> log_id = "<log_id>"
|
166
|
+
>>> ## Deploy
|
167
|
+
>>> segmentation_model.deploy(deployment_bandwidth_mbps=1000,
|
168
|
+
wait_for_completion=False,
|
169
|
+
deployment_log_group_id = log_group_id,
|
170
|
+
deployment_access_log_id = log_id,
|
171
|
+
deployment_predict_log_id = log_id)
|
172
|
+
>>> ## Predict from endpoint
|
173
|
+
>>> segmentation_model.predict(image)
|
174
|
+
>>> segmentation_model.predict(image_bytes)
|
175
|
+
>>> ## Invoke the model
|
176
|
+
>>> auth = ads.common.auth.default_signer()['signer']
|
177
|
+
|
178
|
+
>>> endpoint = segmentation_model.model_deployment.url + "/predict"
|
179
|
+
>>> headers = {"Content-Type": "application/octet-stream"}
|
180
|
+
>>> requests.post(endpoint, data=image_bytes, auth=auth, headers=headers).json()
|
181
|
+
|
182
|
+
Examples
|
183
|
+
--------
|
184
|
+
>>> # Zero Shot Image Classification
|
185
|
+
>>> from transformers import pipeline
|
186
|
+
>>> import tempfile
|
187
|
+
>>> import PIL.Image
|
188
|
+
>>> import ads
|
189
|
+
>>> import requests
|
190
|
+
>>> import cloudpickle
|
191
|
+
>>> ## Download the image data
|
192
|
+
>>> image_url = "https://huggingface.co/datasets/Narsil/image_dummy/raw/main/parrots.png"
|
193
|
+
>>> image = PIL.Image.open(requests.get(image_link, stream=True).raw)
|
194
|
+
>>> image_bytes = cloudpickle.dumps(image)
|
195
|
+
>>> ## Download a pretrained model
|
196
|
+
>>> classifier = pipeline(model="openai/clip-vit-large-patch14")
|
197
|
+
>>> classifier(
|
198
|
+
images=image,
|
199
|
+
candidate_labels=["animals", "humans", "landscape"],
|
200
|
+
)
|
201
|
+
>>> ## Initiate a HuggingFacePipelineModel instance
|
202
|
+
>>> zero_shot_image_classification_model = HuggingFacePipelineModel(classifier, artifact_dir=empfile.mkdtemp())
|
203
|
+
>>> conda = "oci://bucket@namespace/path/to/conda/pack"
|
204
|
+
>>> python_version = "3.8"
|
205
|
+
>>> ## Prepare
|
206
|
+
>>> zero_shot_image_classification_model.prepare(inference_conda_env=conda, inference_python_version = python_version, force_overwrite=True)
|
207
|
+
>>> data = {"images": image, "candidate_labels": ["animals", "humans", "landscape"]}
|
208
|
+
>>> body = cloudpickle.dumps(data) # convert image to bytes
|
209
|
+
>>> ## Verify
|
210
|
+
>>> zero_shot_image_classification_model.verify(data=data)
|
211
|
+
>>> zero_shot_image_classification_model.verify(data=body)
|
212
|
+
>>> ## Save
|
213
|
+
>>> zero_shot_image_classification_model.save()
|
214
|
+
>>> ## Deploy
|
215
|
+
>>> log_group_id = "<log_group_id>"
|
216
|
+
>>> log_id = "<log_id>"
|
217
|
+
>>> zero_shot_image_classification_model.deploy(deployment_bandwidth_mbps=1000,
|
218
|
+
wait_for_completion=False,
|
219
|
+
deployment_log_group_id = log_group_id,
|
220
|
+
deployment_access_log_id = log_id,
|
221
|
+
deployment_predict_log_id = log_id)
|
222
|
+
>>> ## Predict from endpoint
|
223
|
+
>>> zero_shot_image_classification_model.predict(image)
|
224
|
+
>>> zero_shot_image_classification_model.predict(body)
|
225
|
+
>>> ### Invoke the model
|
226
|
+
>>> auth = ads.common.auth.default_signer()['signer']
|
227
|
+
>>> endpoint = zero_shot_image_classification_model.model_deployment.url + "/predict"
|
228
|
+
>>> headers = {"Content-Type": "application/octet-stream"}
|
229
|
+
>>> requests.post(endpoint, data=body, auth=auth, headers=headers).json()
|
230
|
+
"""
|
231
|
+
|
232
|
+
_PREFIX = "huggingface"
|
233
|
+
model_save_serializer_type = HuggingFaceSerializerType
|
234
|
+
|
235
|
+
@runtime_dependency(
|
236
|
+
module="transformers", install_from=OptionalDependency.HUGGINGFACE
|
237
|
+
)
|
238
|
+
def __init__(
|
239
|
+
self,
|
240
|
+
estimator: Callable,
|
241
|
+
artifact_dir: Optional[str] = None,
|
242
|
+
properties: Optional[ModelProperties] = None,
|
243
|
+
auth: Dict = None,
|
244
|
+
model_save_serializer: Optional[SERDE] = model_save_serializer_type.HUGGINGFACE,
|
245
|
+
model_input_serializer: Optional[SERDE] = ModelSerializerType.CLOUDPICKLE,
|
246
|
+
**kwargs,
|
247
|
+
):
|
248
|
+
"""
|
249
|
+
Initiates a HuggingFacePipelineModel instance.
|
250
|
+
|
251
|
+
Parameters
|
252
|
+
----------
|
253
|
+
estimator: Callable
|
254
|
+
HuggingFacePipeline Model
|
255
|
+
artifact_dir: str
|
256
|
+
Directory for generate artifact.
|
257
|
+
properties: (ModelProperties, optional). Defaults to None.
|
258
|
+
ModelProperties object required to save and deploy model.
|
259
|
+
For more details, check https://accelerated-data-science.readthedocs.io/en/latest/ads.model.html#module-ads.model.model_properties.
|
260
|
+
auth :(Dict, optional). Defaults to None.
|
261
|
+
The default authetication is set using `ads.set_auth` API. If you need to override the
|
262
|
+
default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
|
263
|
+
authentication signer and kwargs required to instantiate IdentityClient object.
|
264
|
+
model_save_serializer: (SERDE or str, optional). Defaults to None.
|
265
|
+
Instance of ads.model.SERDE. Used for serialize/deserialize model.
|
266
|
+
model_input_serializer: (SERDE, optional). Defaults to None.
|
267
|
+
Instance of ads.model.SERDE. Used for serialize/deserialize data.
|
268
|
+
|
269
|
+
Returns
|
270
|
+
-------
|
271
|
+
HuggingFacePipelineModel
|
272
|
+
HuggingFacePipelineModel instance.
|
273
|
+
|
274
|
+
Examples
|
275
|
+
--------
|
276
|
+
>>> from transformers import pipeline
|
277
|
+
>>> import tempfile
|
278
|
+
>>> import PIL.Image
|
279
|
+
>>> import ads
|
280
|
+
>>> import requests
|
281
|
+
>>> import cloudpickle
|
282
|
+
>>> ## download the image
|
283
|
+
>>> image_url = "https://huggingface.co/datasets/Narsil/image_dummy/raw/main/parrots.png"
|
284
|
+
>>> image = PIL.Image.open(requests.get(image_link, stream=True).raw)
|
285
|
+
>>> image_bytes = cloudpickle.dumps(image)
|
286
|
+
>>> ## download the pretrained model
|
287
|
+
>>> classifier = pipeline(model="openai/clip-vit-large-patch14")
|
288
|
+
>>> classifier(
|
289
|
+
images=image,
|
290
|
+
candidate_labels=["animals", "humans", "landscape"],
|
291
|
+
)
|
292
|
+
>>> ## Initiate a HuggingFacePipelineModel instance
|
293
|
+
>>> zero_shot_image_classification_model = HuggingFacePipelineModel(classifier, artifact_dir=empfile.mkdtemp())
|
294
|
+
>>> ## Prepare a model artifact
|
295
|
+
>>> conda = "oci://bucket@namespace/path/to/conda/pack"
|
296
|
+
>>> python_version = "3.8"
|
297
|
+
>>> zero_shot_image_classification_model.prepare(inference_conda_env=conda, inference_python_version = python_version, force_overwrite=True)
|
298
|
+
>>> ## Test data
|
299
|
+
>>> data = {"images": image, "candidate_labels": ["animals", "humans", "landscape"]}
|
300
|
+
>>> body = cloudpickle.dumps(data) # convert image to bytes
|
301
|
+
>>> ## Verify
|
302
|
+
>>> zero_shot_image_classification_model.verify(data=data)
|
303
|
+
>>> zero_shot_image_classification_model.verify(data=body)
|
304
|
+
>>> ## Save
|
305
|
+
>>> zero_shot_image_classification_model.save()
|
306
|
+
>>> ## Deploy
|
307
|
+
>>> log_group_id = "<log_group_id>"
|
308
|
+
>>> log_id = "<log_id>"
|
309
|
+
>>> zero_shot_image_classification_model.deploy(deployment_bandwidth_mbps=100,
|
310
|
+
wait_for_completion=False,
|
311
|
+
deployment_log_group_id = log_group_id,
|
312
|
+
deployment_access_log_id = log_id,
|
313
|
+
deployment_predict_log_id = log_id)
|
314
|
+
>>> zero_shot_image_classification_model.predict(image)
|
315
|
+
>>> zero_shot_image_classification_model.predict(body)
|
316
|
+
>>> ### Invoke the model by sending bytes
|
317
|
+
>>> auth = ads.common.auth.default_signer()['signer']
|
318
|
+
>>> endpoint = zero_shot_image_classification_model.model_deployment.url + "/predict"
|
319
|
+
>>> headers = {"Content-Type": "application/octet-stream"}
|
320
|
+
>>> requests.post(endpoint, data=body, auth=auth, headers=headers).json()
|
321
|
+
"""
|
322
|
+
if not isinstance(estimator, transformers.pipelines.base.Pipeline):
|
323
|
+
raise TypeError(
|
324
|
+
f"{str(type(estimator))} is not supported in HuggingFacePipelineModel."
|
325
|
+
)
|
326
|
+
super().__init__(
|
327
|
+
estimator=estimator,
|
328
|
+
artifact_dir=artifact_dir,
|
329
|
+
properties=properties,
|
330
|
+
auth=auth,
|
331
|
+
model_save_serializer=model_save_serializer,
|
332
|
+
model_input_serializer=model_input_serializer,
|
333
|
+
**kwargs,
|
334
|
+
)
|
335
|
+
self._extractor = HuggingFaceExtractor(estimator)
|
336
|
+
self.framework = self._extractor.framework
|
337
|
+
self.algorithm = self._extractor.algorithm
|
338
|
+
self.version = self._extractor.version
|
339
|
+
self.hyperparameter = self._extractor.hyperparameter
|
340
|
+
self.task = self.estimator.task
|
341
|
+
self._score_args["task"] = self.estimator.task
|
342
|
+
|
343
|
+
def _handle_model_file_name(
|
344
|
+
self, as_onnx: bool = False, model_file_name: str = None
|
345
|
+
):
|
346
|
+
"""
|
347
|
+
The artifact directory to store model files.
|
348
|
+
|
349
|
+
Parameters
|
350
|
+
----------
|
351
|
+
as_onnx: bool. Defaults to False
|
352
|
+
If set as True, it will be ignored as onnx conversion is not supported.
|
353
|
+
model_file_name: str
|
354
|
+
Will be ignored as huggingface pipeline requires to folder to store the model
|
355
|
+
files and those files will be stored at the artifact directory.
|
356
|
+
|
357
|
+
Returns
|
358
|
+
-------
|
359
|
+
str
|
360
|
+
The artifact directory.
|
361
|
+
"""
|
362
|
+
return self.artifact_dir
|
363
|
+
|
364
|
+
def serialize_model(
|
365
|
+
self,
|
366
|
+
as_onnx: bool = False,
|
367
|
+
force_overwrite: bool = False,
|
368
|
+
X_sample: Optional[Union[Dict, str, List, PIL.Image.Image]] = None,
|
369
|
+
**kwargs,
|
370
|
+
) -> None:
|
371
|
+
"""
|
372
|
+
Serialize and save HuggingFace model using model specific method.
|
373
|
+
|
374
|
+
Parameters
|
375
|
+
----------
|
376
|
+
as_onnx: (bool, optional). Defaults to False.
|
377
|
+
If set as True, convert into ONNX model.
|
378
|
+
force_overwrite: (bool, optional). Defaults to False.
|
379
|
+
If set as True, overwrite serialized model if exists.
|
380
|
+
X_sample: Union[Dict, str, List, PIL.Image.Image]. Defaults to None.
|
381
|
+
A sample of input data that will be used to generate input schema and detect onnx_args.
|
382
|
+
|
383
|
+
Returns
|
384
|
+
-------
|
385
|
+
None
|
386
|
+
Nothing.
|
387
|
+
"""
|
388
|
+
|
389
|
+
if as_onnx:
|
390
|
+
raise NotImplementedError(
|
391
|
+
"HuggingFace Pipeline to onnx conversion is not supported."
|
392
|
+
)
|
393
|
+
|
394
|
+
super().serialize_model(
|
395
|
+
as_onnx=False,
|
396
|
+
force_overwrite=force_overwrite,
|
397
|
+
X_sample=X_sample,
|
398
|
+
**kwargs,
|
399
|
+
)
|
@@ -0,0 +1,266 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*--
|
3
|
+
|
4
|
+
# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
|
5
|
+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
|
6
|
+
|
7
|
+
|
8
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
9
|
+
|
10
|
+
import numpy as np
|
11
|
+
import pandas as pd
|
12
|
+
from ads.common import logger
|
13
|
+
from ads.model.extractor.lightgbm_extractor import LightgbmExtractor
|
14
|
+
from ads.model.generic_model import FrameworkSpecificModel
|
15
|
+
from ads.model.model_properties import ModelProperties
|
16
|
+
from ads.model.serde.model_serializer import LightGBMModelSerializerType
|
17
|
+
from ads.model.common.utils import DEPRECATE_AS_ONNX_WARNING
|
18
|
+
from ads.model.serde.common import SERDE
|
19
|
+
|
20
|
+
|
21
|
+
class LightGBMModel(FrameworkSpecificModel):
|
22
|
+
"""LightGBMModel class for estimators from Lightgbm framework.
|
23
|
+
|
24
|
+
Attributes
|
25
|
+
----------
|
26
|
+
algorithm: str
|
27
|
+
The algorithm of the model.
|
28
|
+
artifact_dir: str
|
29
|
+
Artifact directory to store the files needed for deployment.
|
30
|
+
auth: Dict
|
31
|
+
Default authentication is set using the `ads.set_auth` API. To override the
|
32
|
+
default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create
|
33
|
+
an authentication signer to instantiate an IdentityClient object.
|
34
|
+
estimator: Callable
|
35
|
+
A trained lightgbm estimator/model using Lightgbm.
|
36
|
+
framework: str
|
37
|
+
"lightgbm", the framework name of the model.
|
38
|
+
hyperparameter: dict
|
39
|
+
The hyperparameters of the estimator.
|
40
|
+
metadata_custom: ModelCustomMetadata
|
41
|
+
The model custom metadata.
|
42
|
+
metadata_provenance: ModelProvenanceMetadata
|
43
|
+
The model provenance metadata.
|
44
|
+
metadata_taxonomy: ModelTaxonomyMetadata
|
45
|
+
The model taxonomy metadata.
|
46
|
+
model_artifact: ModelArtifact
|
47
|
+
This is built by calling prepare.
|
48
|
+
model_deployment: ModelDeployment
|
49
|
+
A ModelDeployment instance.
|
50
|
+
model_file_name: str
|
51
|
+
Name of the serialized model.
|
52
|
+
model_id: str
|
53
|
+
The model ID.
|
54
|
+
properties: ModelProperties
|
55
|
+
ModelProperties object required to save and deploy model.
|
56
|
+
For more details, check https://accelerated-data-science.readthedocs.io/en/latest/ads.model.html#module-ads.model.model_properties.
|
57
|
+
runtime_info: RuntimeInfo
|
58
|
+
A RuntimeInfo instance.
|
59
|
+
schema_input: Schema
|
60
|
+
Schema describes the structure of the input data.
|
61
|
+
schema_output: Schema
|
62
|
+
Schema describes the structure of the output data.
|
63
|
+
serialize: bool
|
64
|
+
Whether to serialize the model to pkl file by default. If False, you need to serialize the model manually,
|
65
|
+
save it under artifact_dir and update the score.py manually.
|
66
|
+
version: str
|
67
|
+
The framework version of the model.
|
68
|
+
|
69
|
+
Methods
|
70
|
+
-------
|
71
|
+
delete_deployment(...)
|
72
|
+
Deletes the current model deployment.
|
73
|
+
deploy(..., **kwargs)
|
74
|
+
Deploys a model.
|
75
|
+
from_model_artifact(uri, model_file_name, artifact_dir, ..., **kwargs)
|
76
|
+
Loads model from the specified folder, or zip/tar archive.
|
77
|
+
from_model_catalog(model_id, model_file_name, artifact_dir, ..., **kwargs)
|
78
|
+
Loads model from model catalog.
|
79
|
+
introspect(...)
|
80
|
+
Runs model introspection.
|
81
|
+
predict(data, ...)
|
82
|
+
Returns prediction of input data run against the model deployment endpoint.
|
83
|
+
prepare(..., **kwargs)
|
84
|
+
Prepare and save the score.py, serialized model and runtime.yaml file.
|
85
|
+
reload(...)
|
86
|
+
Reloads the model artifact files: `score.py` and the `runtime.yaml`.
|
87
|
+
save(..., **kwargs)
|
88
|
+
Saves model artifacts to the model catalog.
|
89
|
+
summary_status(...)
|
90
|
+
Gets a summary table of the current status.
|
91
|
+
verify(data, ...)
|
92
|
+
Tests if deployment works in local environment.
|
93
|
+
|
94
|
+
Examples
|
95
|
+
--------
|
96
|
+
>>> import lightgbm as lgb
|
97
|
+
>>> import tempfile
|
98
|
+
>>> from sklearn.model_selection import train_test_split
|
99
|
+
>>> from sklearn.datasets import load_iris
|
100
|
+
>>> from ads.model.framework.lightgbm_model import LightGBMModel
|
101
|
+
|
102
|
+
>>> iris = load_iris()
|
103
|
+
>>> X, y = iris.data, iris.target
|
104
|
+
|
105
|
+
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)
|
106
|
+
>>> train = lgb.Dataset(X_train, label=y_train)
|
107
|
+
>>> param = {
|
108
|
+
... 'objective': 'multiclass', 'num_class': 3,
|
109
|
+
... }
|
110
|
+
>>> lightgbm_estimator = lgb.train(param, train)
|
111
|
+
|
112
|
+
>>> lightgbm_model = LightGBMModel(estimator=lightgbm_estimator,
|
113
|
+
... artifact_dir=tempfile.mkdtemp())
|
114
|
+
|
115
|
+
>>> lightgbm_model.prepare(inference_conda_env="generalml_p37_cpu_v1", force_overwrite=True)
|
116
|
+
>>> lightgbm_model.reload()
|
117
|
+
>>> lightgbm_model.verify(X_test)
|
118
|
+
>>> lightgbm_model.save()
|
119
|
+
>>> model_deployment = lightgbm_model.deploy(wait_for_completion=False)
|
120
|
+
>>> lightgbm_model.predict(X_test)
|
121
|
+
"""
|
122
|
+
|
123
|
+
_PREFIX = "lightgbm"
|
124
|
+
model_save_serializer_type = LightGBMModelSerializerType
|
125
|
+
|
126
|
+
def __init__(
|
127
|
+
self,
|
128
|
+
estimator: Callable,
|
129
|
+
artifact_dir: Optional[str] = None,
|
130
|
+
properties: Optional[ModelProperties] = None,
|
131
|
+
auth: Dict = None,
|
132
|
+
model_save_serializer: Optional[SERDE] = None,
|
133
|
+
model_input_serializer: Optional[SERDE] = None,
|
134
|
+
**kwargs,
|
135
|
+
):
|
136
|
+
"""
|
137
|
+
Initiates a LightGBMModel instance. This class wraps the Lightgbm model as estimator.
|
138
|
+
It's primary purpose is to hold the trained model and do serialization.
|
139
|
+
|
140
|
+
Parameters
|
141
|
+
----------
|
142
|
+
estimator:
|
143
|
+
any model object generated by Lightgbm framework
|
144
|
+
artifact_dir: str
|
145
|
+
Directory for generate artifact.
|
146
|
+
properties: (ModelProperties, optional). Defaults to None.
|
147
|
+
ModelProperties object required to save and deploy model.
|
148
|
+
auth :(Dict, optional). Defaults to None.
|
149
|
+
The default authetication is set using `ads.set_auth` API. If you need to override the
|
150
|
+
default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
|
151
|
+
authentication signer and kwargs required to instantiate IdentityClient object.
|
152
|
+
model_save_serializer: (SERDE or str, optional). Defaults to None.
|
153
|
+
Instance of ads.model.SERDE. Used for serialize/deserialize model.
|
154
|
+
model_input_serializer: (SERDE, optional). Defaults to None.
|
155
|
+
Instance of ads.model.SERDE. Used for serialize/deserialize data.
|
156
|
+
|
157
|
+
Returns
|
158
|
+
-------
|
159
|
+
LightGBMModel
|
160
|
+
LightGBMModel instance.
|
161
|
+
|
162
|
+
Raises
|
163
|
+
------
|
164
|
+
TypeError: If the input model is not a Lightgbm model or not supported for serialization.
|
165
|
+
|
166
|
+
|
167
|
+
Examples
|
168
|
+
--------
|
169
|
+
>>> import lightgbm as lgb
|
170
|
+
>>> import tempfile
|
171
|
+
>>> from sklearn.model_selection import train_test_split
|
172
|
+
>>> from sklearn.datasets import load_iris
|
173
|
+
>>> from ads.model.framework.lightgbm_model import LightGBMModel
|
174
|
+
>>> iris = load_iris()
|
175
|
+
>>> X, y = iris.data, iris.target
|
176
|
+
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)
|
177
|
+
>>> train = lgb.Dataset(X_train, label=y_train)
|
178
|
+
>>> param = {
|
179
|
+
... 'objective': 'multiclass', 'num_class': 3,
|
180
|
+
... }
|
181
|
+
>>> lightgbm_estimator = lgb.train(param, train)
|
182
|
+
>>> lightgbm_model = LightGBMModel(estimator=lightgbm_estimator, artifact_dir=tempfile.mkdtemp())
|
183
|
+
>>> lightgbm_model.prepare(inference_conda_env="generalml_p37_cpu_v1")
|
184
|
+
>>> lightgbm_model.verify(X_test)
|
185
|
+
>>> lightgbm_model.save()
|
186
|
+
>>> model_deployment = lightgbm_model.deploy()
|
187
|
+
>>> lightgbm_model.predict(X_test)
|
188
|
+
>>> lightgbm_model.delete_deployment()
|
189
|
+
"""
|
190
|
+
model_type = str(type(estimator))
|
191
|
+
if not (
|
192
|
+
model_type.startswith("<class 'lightgbm.basic.")
|
193
|
+
or model_type.startswith("<class 'lightgbm.sklearn.")
|
194
|
+
or model_type.startswith("<class 'onnxruntime.")
|
195
|
+
):
|
196
|
+
raise TypeError(f"{model_type} is not supported in LightGBMModel.")
|
197
|
+
|
198
|
+
default_model_save_serializer = "joblib"
|
199
|
+
if model_type.startswith("<class 'lightgbm.basic."):
|
200
|
+
default_model_save_serializer = "lightgbm"
|
201
|
+
|
202
|
+
super().__init__(
|
203
|
+
estimator=estimator,
|
204
|
+
artifact_dir=artifact_dir,
|
205
|
+
properties=properties,
|
206
|
+
auth=auth,
|
207
|
+
model_save_serializer=model_save_serializer
|
208
|
+
or default_model_save_serializer,
|
209
|
+
model_input_serializer=model_input_serializer,
|
210
|
+
**kwargs,
|
211
|
+
)
|
212
|
+
self._extractor = LightgbmExtractor(estimator)
|
213
|
+
self.framework = self._extractor.framework
|
214
|
+
self.algorithm = self._extractor.algorithm
|
215
|
+
self.version = self._extractor.version
|
216
|
+
self.hyperparameter = self._extractor.hyperparameter
|
217
|
+
|
218
|
+
def serialize_model(
|
219
|
+
self,
|
220
|
+
as_onnx: bool = False,
|
221
|
+
initial_types: List[Tuple] = None,
|
222
|
+
force_overwrite: bool = False,
|
223
|
+
X_sample: Optional[
|
224
|
+
Union[
|
225
|
+
Dict,
|
226
|
+
str,
|
227
|
+
List,
|
228
|
+
Tuple,
|
229
|
+
np.ndarray,
|
230
|
+
pd.core.series.Series,
|
231
|
+
pd.core.frame.DataFrame,
|
232
|
+
]
|
233
|
+
] = None,
|
234
|
+
**kwargs: Dict,
|
235
|
+
):
|
236
|
+
"""
|
237
|
+
Serialize and save Lightgbm model.
|
238
|
+
|
239
|
+
Parameters
|
240
|
+
----------
|
241
|
+
as_onnx: (boolean, optional). Defaults to False.
|
242
|
+
If set as True, provide `initial_types` or `X_sample` to convert into ONNX.
|
243
|
+
initial_types: (List[Tuple], optional). Defaults to None.
|
244
|
+
Each element is a tuple of a variable name and a type.
|
245
|
+
force_overwrite: (boolean, optional). Defaults to False.
|
246
|
+
If set as True, overwrite serialized model if exists.
|
247
|
+
X_sample: Union[Dict, str, List, np.ndarray, pd.core.series.Series, pd.core.frame.DataFrame,]. Defaults to None.
|
248
|
+
Contains model inputs such that model(`X_sample`) is a valid invocation of the model.
|
249
|
+
Used to generate `initial_types`.
|
250
|
+
|
251
|
+
Returns
|
252
|
+
-------
|
253
|
+
None
|
254
|
+
Nothing.
|
255
|
+
"""
|
256
|
+
if as_onnx:
|
257
|
+
logger.warning(DEPRECATE_AS_ONNX_WARNING)
|
258
|
+
self.set_model_save_serializer("lightgbm_onnx")
|
259
|
+
|
260
|
+
super().serialize_model(
|
261
|
+
as_onnx=as_onnx,
|
262
|
+
initial_types=initial_types,
|
263
|
+
force_overwrite=force_overwrite,
|
264
|
+
X_sample=X_sample,
|
265
|
+
**kwargs,
|
266
|
+
)
|