oracle-ads 2.13.8__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (857) hide show
  1. {oracle_ads-2.13.8.dist-info → oracle_ads-2.13.9rc0.dist-info}/METADATA +151 -151
  2. oracle_ads-2.13.9rc0.dist-info/RECORD +9 -0
  3. {oracle_ads-2.13.8.dist-info → oracle_ads-2.13.9rc0.dist-info}/WHEEL +2 -1
  4. {oracle_ads-2.13.8.dist-info → oracle_ads-2.13.9rc0.dist-info}/entry_points.txt +1 -2
  5. oracle_ads-2.13.9rc0.dist-info/top_level.txt +1 -0
  6. ads/aqua/__init__.py +0 -40
  7. ads/aqua/app.py +0 -506
  8. ads/aqua/cli.py +0 -96
  9. ads/aqua/client/__init__.py +0 -3
  10. ads/aqua/client/client.py +0 -836
  11. ads/aqua/client/openai_client.py +0 -305
  12. ads/aqua/common/__init__.py +0 -5
  13. ads/aqua/common/decorator.py +0 -125
  14. ads/aqua/common/entities.py +0 -266
  15. ads/aqua/common/enums.py +0 -122
  16. ads/aqua/common/errors.py +0 -109
  17. ads/aqua/common/utils.py +0 -1285
  18. ads/aqua/config/__init__.py +0 -4
  19. ads/aqua/config/container_config.py +0 -248
  20. ads/aqua/config/evaluation/__init__.py +0 -4
  21. ads/aqua/config/evaluation/evaluation_service_config.py +0 -147
  22. ads/aqua/config/utils/__init__.py +0 -4
  23. ads/aqua/config/utils/serializer.py +0 -339
  24. ads/aqua/constants.py +0 -114
  25. ads/aqua/data.py +0 -14
  26. ads/aqua/dummy_data/icon.txt +0 -1
  27. ads/aqua/dummy_data/oci_model_deployments.json +0 -56
  28. ads/aqua/dummy_data/oci_models.json +0 -1
  29. ads/aqua/dummy_data/readme.md +0 -26
  30. ads/aqua/evaluation/__init__.py +0 -8
  31. ads/aqua/evaluation/constants.py +0 -53
  32. ads/aqua/evaluation/entities.py +0 -186
  33. ads/aqua/evaluation/errors.py +0 -70
  34. ads/aqua/evaluation/evaluation.py +0 -1814
  35. ads/aqua/extension/__init__.py +0 -42
  36. ads/aqua/extension/aqua_ws_msg_handler.py +0 -76
  37. ads/aqua/extension/base_handler.py +0 -90
  38. ads/aqua/extension/common_handler.py +0 -121
  39. ads/aqua/extension/common_ws_msg_handler.py +0 -36
  40. ads/aqua/extension/deployment_handler.py +0 -298
  41. ads/aqua/extension/deployment_ws_msg_handler.py +0 -54
  42. ads/aqua/extension/errors.py +0 -30
  43. ads/aqua/extension/evaluation_handler.py +0 -129
  44. ads/aqua/extension/evaluation_ws_msg_handler.py +0 -61
  45. ads/aqua/extension/finetune_handler.py +0 -96
  46. ads/aqua/extension/model_handler.py +0 -390
  47. ads/aqua/extension/models/__init__.py +0 -0
  48. ads/aqua/extension/models/ws_models.py +0 -145
  49. ads/aqua/extension/models_ws_msg_handler.py +0 -50
  50. ads/aqua/extension/ui_handler.py +0 -282
  51. ads/aqua/extension/ui_websocket_handler.py +0 -130
  52. ads/aqua/extension/utils.py +0 -133
  53. ads/aqua/finetuning/__init__.py +0 -7
  54. ads/aqua/finetuning/constants.py +0 -23
  55. ads/aqua/finetuning/entities.py +0 -181
  56. ads/aqua/finetuning/finetuning.py +0 -731
  57. ads/aqua/model/__init__.py +0 -8
  58. ads/aqua/model/constants.py +0 -60
  59. ads/aqua/model/entities.py +0 -306
  60. ads/aqua/model/enums.py +0 -30
  61. ads/aqua/model/model.py +0 -2080
  62. ads/aqua/modeldeployment/__init__.py +0 -8
  63. ads/aqua/modeldeployment/constants.py +0 -10
  64. ads/aqua/modeldeployment/deployment.py +0 -1324
  65. ads/aqua/modeldeployment/entities.py +0 -653
  66. ads/aqua/modeldeployment/inference.py +0 -74
  67. ads/aqua/modeldeployment/utils.py +0 -543
  68. ads/aqua/resources/gpu_shapes_index.json +0 -94
  69. ads/aqua/server/__init__.py +0 -4
  70. ads/aqua/server/__main__.py +0 -24
  71. ads/aqua/server/app.py +0 -47
  72. ads/aqua/server/aqua_spec.yml +0 -1291
  73. ads/aqua/training/__init__.py +0 -4
  74. ads/aqua/training/exceptions.py +0 -476
  75. ads/aqua/ui.py +0 -499
  76. ads/automl/__init__.py +0 -9
  77. ads/automl/driver.py +0 -330
  78. ads/automl/provider.py +0 -975
  79. ads/bds/__init__.py +0 -5
  80. ads/bds/auth.py +0 -127
  81. ads/bds/big_data_service.py +0 -255
  82. ads/catalog/__init__.py +0 -19
  83. ads/catalog/model.py +0 -1576
  84. ads/catalog/notebook.py +0 -461
  85. ads/catalog/project.py +0 -468
  86. ads/catalog/summary.py +0 -178
  87. ads/common/__init__.py +0 -11
  88. ads/common/analyzer.py +0 -65
  89. ads/common/artifact/.model-ignore +0 -63
  90. ads/common/artifact/__init__.py +0 -10
  91. ads/common/auth.py +0 -1122
  92. ads/common/card_identifier.py +0 -83
  93. ads/common/config.py +0 -647
  94. ads/common/data.py +0 -165
  95. ads/common/decorator/__init__.py +0 -9
  96. ads/common/decorator/argument_to_case.py +0 -88
  97. ads/common/decorator/deprecate.py +0 -69
  98. ads/common/decorator/require_nonempty_arg.py +0 -65
  99. ads/common/decorator/runtime_dependency.py +0 -178
  100. ads/common/decorator/threaded.py +0 -97
  101. ads/common/decorator/utils.py +0 -35
  102. ads/common/dsc_file_system.py +0 -303
  103. ads/common/error.py +0 -14
  104. ads/common/extended_enum.py +0 -81
  105. ads/common/function/__init__.py +0 -5
  106. ads/common/function/fn_util.py +0 -142
  107. ads/common/function/func_conf.yaml +0 -25
  108. ads/common/ipython.py +0 -76
  109. ads/common/model.py +0 -679
  110. ads/common/model_artifact.py +0 -1759
  111. ads/common/model_artifact_schema.json +0 -107
  112. ads/common/model_export_util.py +0 -664
  113. ads/common/model_metadata.py +0 -24
  114. ads/common/object_storage_details.py +0 -296
  115. ads/common/oci_client.py +0 -175
  116. ads/common/oci_datascience.py +0 -46
  117. ads/common/oci_logging.py +0 -1144
  118. ads/common/oci_mixin.py +0 -957
  119. ads/common/oci_resource.py +0 -136
  120. ads/common/serializer.py +0 -559
  121. ads/common/utils.py +0 -1852
  122. ads/common/word_lists.py +0 -1491
  123. ads/common/work_request.py +0 -189
  124. ads/data_labeling/__init__.py +0 -13
  125. ads/data_labeling/boundingbox.py +0 -253
  126. ads/data_labeling/constants.py +0 -47
  127. ads/data_labeling/data_labeling_service.py +0 -244
  128. ads/data_labeling/interface/__init__.py +0 -5
  129. ads/data_labeling/interface/loader.py +0 -16
  130. ads/data_labeling/interface/parser.py +0 -16
  131. ads/data_labeling/interface/reader.py +0 -23
  132. ads/data_labeling/loader/__init__.py +0 -5
  133. ads/data_labeling/loader/file_loader.py +0 -241
  134. ads/data_labeling/metadata.py +0 -110
  135. ads/data_labeling/mixin/__init__.py +0 -5
  136. ads/data_labeling/mixin/data_labeling.py +0 -232
  137. ads/data_labeling/ner.py +0 -129
  138. ads/data_labeling/parser/__init__.py +0 -5
  139. ads/data_labeling/parser/dls_record_parser.py +0 -388
  140. ads/data_labeling/parser/export_metadata_parser.py +0 -94
  141. ads/data_labeling/parser/export_record_parser.py +0 -473
  142. ads/data_labeling/reader/__init__.py +0 -5
  143. ads/data_labeling/reader/dataset_reader.py +0 -574
  144. ads/data_labeling/reader/dls_record_reader.py +0 -121
  145. ads/data_labeling/reader/export_record_reader.py +0 -62
  146. ads/data_labeling/reader/jsonl_reader.py +0 -75
  147. ads/data_labeling/reader/metadata_reader.py +0 -203
  148. ads/data_labeling/reader/record_reader.py +0 -263
  149. ads/data_labeling/record.py +0 -52
  150. ads/data_labeling/visualizer/__init__.py +0 -5
  151. ads/data_labeling/visualizer/image_visualizer.py +0 -525
  152. ads/data_labeling/visualizer/text_visualizer.py +0 -357
  153. ads/database/__init__.py +0 -5
  154. ads/database/connection.py +0 -338
  155. ads/dataset/__init__.py +0 -10
  156. ads/dataset/capabilities.md +0 -51
  157. ads/dataset/classification_dataset.py +0 -339
  158. ads/dataset/correlation.py +0 -226
  159. ads/dataset/correlation_plot.py +0 -563
  160. ads/dataset/dask_series.py +0 -173
  161. ads/dataset/dataframe_transformer.py +0 -110
  162. ads/dataset/dataset.py +0 -1979
  163. ads/dataset/dataset_browser.py +0 -360
  164. ads/dataset/dataset_with_target.py +0 -995
  165. ads/dataset/exception.py +0 -25
  166. ads/dataset/factory.py +0 -987
  167. ads/dataset/feature_engineering_transformer.py +0 -35
  168. ads/dataset/feature_selection.py +0 -107
  169. ads/dataset/forecasting_dataset.py +0 -26
  170. ads/dataset/helper.py +0 -1450
  171. ads/dataset/label_encoder.py +0 -99
  172. ads/dataset/mixin/__init__.py +0 -5
  173. ads/dataset/mixin/dataset_accessor.py +0 -134
  174. ads/dataset/pipeline.py +0 -58
  175. ads/dataset/plot.py +0 -710
  176. ads/dataset/progress.py +0 -86
  177. ads/dataset/recommendation.py +0 -297
  178. ads/dataset/recommendation_transformer.py +0 -502
  179. ads/dataset/regression_dataset.py +0 -14
  180. ads/dataset/sampled_dataset.py +0 -1050
  181. ads/dataset/target.py +0 -98
  182. ads/dataset/timeseries.py +0 -18
  183. ads/dbmixin/__init__.py +0 -5
  184. ads/dbmixin/db_pandas_accessor.py +0 -153
  185. ads/environment/__init__.py +0 -9
  186. ads/environment/ml_runtime.py +0 -66
  187. ads/evaluations/README.md +0 -14
  188. ads/evaluations/__init__.py +0 -109
  189. ads/evaluations/evaluation_plot.py +0 -983
  190. ads/evaluations/evaluator.py +0 -1334
  191. ads/evaluations/statistical_metrics.py +0 -543
  192. ads/experiments/__init__.py +0 -9
  193. ads/experiments/capabilities.md +0 -0
  194. ads/explanations/__init__.py +0 -21
  195. ads/explanations/base_explainer.py +0 -142
  196. ads/explanations/capabilities.md +0 -83
  197. ads/explanations/explainer.py +0 -190
  198. ads/explanations/mlx_global_explainer.py +0 -1050
  199. ads/explanations/mlx_interface.py +0 -386
  200. ads/explanations/mlx_local_explainer.py +0 -287
  201. ads/explanations/mlx_whatif_explainer.py +0 -201
  202. ads/feature_engineering/__init__.py +0 -20
  203. ads/feature_engineering/accessor/__init__.py +0 -5
  204. ads/feature_engineering/accessor/dataframe_accessor.py +0 -535
  205. ads/feature_engineering/accessor/mixin/__init__.py +0 -5
  206. ads/feature_engineering/accessor/mixin/correlation.py +0 -166
  207. ads/feature_engineering/accessor/mixin/eda_mixin.py +0 -266
  208. ads/feature_engineering/accessor/mixin/eda_mixin_series.py +0 -85
  209. ads/feature_engineering/accessor/mixin/feature_types_mixin.py +0 -211
  210. ads/feature_engineering/accessor/mixin/utils.py +0 -65
  211. ads/feature_engineering/accessor/series_accessor.py +0 -431
  212. ads/feature_engineering/adsimage/__init__.py +0 -5
  213. ads/feature_engineering/adsimage/image.py +0 -192
  214. ads/feature_engineering/adsimage/image_reader.py +0 -170
  215. ads/feature_engineering/adsimage/interface/__init__.py +0 -5
  216. ads/feature_engineering/adsimage/interface/reader.py +0 -19
  217. ads/feature_engineering/adsstring/__init__.py +0 -7
  218. ads/feature_engineering/adsstring/oci_language/__init__.py +0 -8
  219. ads/feature_engineering/adsstring/string/__init__.py +0 -8
  220. ads/feature_engineering/data_schema.json +0 -57
  221. ads/feature_engineering/dataset/__init__.py +0 -5
  222. ads/feature_engineering/dataset/zip_code_data.py +0 -42062
  223. ads/feature_engineering/exceptions.py +0 -40
  224. ads/feature_engineering/feature_type/__init__.py +0 -133
  225. ads/feature_engineering/feature_type/address.py +0 -184
  226. ads/feature_engineering/feature_type/adsstring/__init__.py +0 -5
  227. ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +0 -164
  228. ads/feature_engineering/feature_type/adsstring/oci_language.py +0 -93
  229. ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +0 -5
  230. ads/feature_engineering/feature_type/adsstring/parsers/base.py +0 -47
  231. ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +0 -96
  232. ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +0 -221
  233. ads/feature_engineering/feature_type/adsstring/string.py +0 -258
  234. ads/feature_engineering/feature_type/base.py +0 -58
  235. ads/feature_engineering/feature_type/boolean.py +0 -183
  236. ads/feature_engineering/feature_type/category.py +0 -146
  237. ads/feature_engineering/feature_type/constant.py +0 -137
  238. ads/feature_engineering/feature_type/continuous.py +0 -151
  239. ads/feature_engineering/feature_type/creditcard.py +0 -314
  240. ads/feature_engineering/feature_type/datetime.py +0 -190
  241. ads/feature_engineering/feature_type/discrete.py +0 -134
  242. ads/feature_engineering/feature_type/document.py +0 -43
  243. ads/feature_engineering/feature_type/gis.py +0 -251
  244. ads/feature_engineering/feature_type/handler/__init__.py +0 -5
  245. ads/feature_engineering/feature_type/handler/feature_validator.py +0 -524
  246. ads/feature_engineering/feature_type/handler/feature_warning.py +0 -319
  247. ads/feature_engineering/feature_type/handler/warnings.py +0 -128
  248. ads/feature_engineering/feature_type/integer.py +0 -142
  249. ads/feature_engineering/feature_type/ip_address.py +0 -144
  250. ads/feature_engineering/feature_type/ip_address_v4.py +0 -138
  251. ads/feature_engineering/feature_type/ip_address_v6.py +0 -138
  252. ads/feature_engineering/feature_type/lat_long.py +0 -256
  253. ads/feature_engineering/feature_type/object.py +0 -43
  254. ads/feature_engineering/feature_type/ordinal.py +0 -132
  255. ads/feature_engineering/feature_type/phone_number.py +0 -135
  256. ads/feature_engineering/feature_type/string.py +0 -171
  257. ads/feature_engineering/feature_type/text.py +0 -93
  258. ads/feature_engineering/feature_type/unknown.py +0 -43
  259. ads/feature_engineering/feature_type/zip_code.py +0 -164
  260. ads/feature_engineering/feature_type_manager.py +0 -406
  261. ads/feature_engineering/schema.py +0 -795
  262. ads/feature_engineering/utils.py +0 -245
  263. ads/feature_store/.readthedocs.yaml +0 -19
  264. ads/feature_store/README.md +0 -65
  265. ads/feature_store/__init__.py +0 -9
  266. ads/feature_store/common/__init__.py +0 -0
  267. ads/feature_store/common/enums.py +0 -339
  268. ads/feature_store/common/exceptions.py +0 -18
  269. ads/feature_store/common/spark_session_singleton.py +0 -125
  270. ads/feature_store/common/utils/__init__.py +0 -0
  271. ads/feature_store/common/utils/base64_encoder_decoder.py +0 -72
  272. ads/feature_store/common/utils/feature_schema_mapper.py +0 -283
  273. ads/feature_store/common/utils/transformation_utils.py +0 -82
  274. ads/feature_store/common/utils/utility.py +0 -403
  275. ads/feature_store/data_validation/__init__.py +0 -0
  276. ads/feature_store/data_validation/great_expectation.py +0 -129
  277. ads/feature_store/dataset.py +0 -1230
  278. ads/feature_store/dataset_job.py +0 -530
  279. ads/feature_store/docs/Dockerfile +0 -7
  280. ads/feature_store/docs/Makefile +0 -44
  281. ads/feature_store/docs/conf.py +0 -28
  282. ads/feature_store/docs/requirements.txt +0 -14
  283. ads/feature_store/docs/source/ads.feature_store.query.rst +0 -20
  284. ads/feature_store/docs/source/cicd.rst +0 -137
  285. ads/feature_store/docs/source/conf.py +0 -86
  286. ads/feature_store/docs/source/data_versioning.rst +0 -33
  287. ads/feature_store/docs/source/dataset.rst +0 -388
  288. ads/feature_store/docs/source/dataset_job.rst +0 -27
  289. ads/feature_store/docs/source/demo.rst +0 -70
  290. ads/feature_store/docs/source/entity.rst +0 -78
  291. ads/feature_store/docs/source/feature_group.rst +0 -624
  292. ads/feature_store/docs/source/feature_group_job.rst +0 -29
  293. ads/feature_store/docs/source/feature_store.rst +0 -122
  294. ads/feature_store/docs/source/feature_store_class.rst +0 -123
  295. ads/feature_store/docs/source/feature_validation.rst +0 -66
  296. ads/feature_store/docs/source/figures/cicd.png +0 -0
  297. ads/feature_store/docs/source/figures/data_validation.png +0 -0
  298. ads/feature_store/docs/source/figures/data_versioning.png +0 -0
  299. ads/feature_store/docs/source/figures/dataset.gif +0 -0
  300. ads/feature_store/docs/source/figures/dataset.png +0 -0
  301. ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
  302. ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
  303. ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
  304. ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
  305. ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
  306. ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
  307. ads/feature_store/docs/source/figures/entity.png +0 -0
  308. ads/feature_store/docs/source/figures/feature_group.png +0 -0
  309. ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
  310. ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
  311. ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
  312. ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
  313. ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
  314. ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
  315. ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
  316. ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
  317. ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
  318. ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
  319. ads/feature_store/docs/source/figures/overview.png +0 -0
  320. ads/feature_store/docs/source/figures/resource_manager.png +0 -0
  321. ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
  322. ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
  323. ads/feature_store/docs/source/figures/stats_1.png +0 -0
  324. ads/feature_store/docs/source/figures/stats_2.png +0 -0
  325. ads/feature_store/docs/source/figures/stats_d.png +0 -0
  326. ads/feature_store/docs/source/figures/stats_fg.png +0 -0
  327. ads/feature_store/docs/source/figures/transformation.png +0 -0
  328. ads/feature_store/docs/source/figures/transformations.gif +0 -0
  329. ads/feature_store/docs/source/figures/validation.png +0 -0
  330. ads/feature_store/docs/source/figures/validation_fg.png +0 -0
  331. ads/feature_store/docs/source/figures/validation_results.png +0 -0
  332. ads/feature_store/docs/source/figures/validation_summary.png +0 -0
  333. ads/feature_store/docs/source/index.rst +0 -81
  334. ads/feature_store/docs/source/module.rst +0 -8
  335. ads/feature_store/docs/source/notebook.rst +0 -94
  336. ads/feature_store/docs/source/overview.rst +0 -47
  337. ads/feature_store/docs/source/quickstart.rst +0 -176
  338. ads/feature_store/docs/source/release_notes.rst +0 -194
  339. ads/feature_store/docs/source/setup_feature_store.rst +0 -81
  340. ads/feature_store/docs/source/statistics.rst +0 -58
  341. ads/feature_store/docs/source/transformation.rst +0 -199
  342. ads/feature_store/docs/source/ui.rst +0 -65
  343. ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +0 -66
  344. ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +0 -192
  345. ads/feature_store/docs/source/user_guides.setup.terraform.rst +0 -338
  346. ads/feature_store/entity.py +0 -718
  347. ads/feature_store/execution_strategy/__init__.py +0 -0
  348. ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
  349. ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +0 -375
  350. ads/feature_store/execution_strategy/engine/__init__.py +0 -0
  351. ads/feature_store/execution_strategy/engine/spark_engine.py +0 -316
  352. ads/feature_store/execution_strategy/execution_strategy.py +0 -113
  353. ads/feature_store/execution_strategy/execution_strategy_provider.py +0 -47
  354. ads/feature_store/execution_strategy/spark/__init__.py +0 -0
  355. ads/feature_store/execution_strategy/spark/spark_execution.py +0 -618
  356. ads/feature_store/feature.py +0 -192
  357. ads/feature_store/feature_group.py +0 -1494
  358. ads/feature_store/feature_group_expectation.py +0 -346
  359. ads/feature_store/feature_group_job.py +0 -602
  360. ads/feature_store/feature_lineage/__init__.py +0 -0
  361. ads/feature_store/feature_lineage/graphviz_service.py +0 -180
  362. ads/feature_store/feature_option_details.py +0 -50
  363. ads/feature_store/feature_statistics/__init__.py +0 -0
  364. ads/feature_store/feature_statistics/statistics_service.py +0 -99
  365. ads/feature_store/feature_store.py +0 -699
  366. ads/feature_store/feature_store_registrar.py +0 -518
  367. ads/feature_store/input_feature_detail.py +0 -149
  368. ads/feature_store/mixin/__init__.py +0 -4
  369. ads/feature_store/mixin/oci_feature_store.py +0 -145
  370. ads/feature_store/model_details.py +0 -73
  371. ads/feature_store/query/__init__.py +0 -0
  372. ads/feature_store/query/filter.py +0 -266
  373. ads/feature_store/query/generator/__init__.py +0 -0
  374. ads/feature_store/query/generator/query_generator.py +0 -298
  375. ads/feature_store/query/join.py +0 -161
  376. ads/feature_store/query/query.py +0 -403
  377. ads/feature_store/query/validator/__init__.py +0 -0
  378. ads/feature_store/query/validator/query_validator.py +0 -57
  379. ads/feature_store/response/__init__.py +0 -0
  380. ads/feature_store/response/response_builder.py +0 -68
  381. ads/feature_store/service/__init__.py +0 -0
  382. ads/feature_store/service/oci_dataset.py +0 -139
  383. ads/feature_store/service/oci_dataset_job.py +0 -199
  384. ads/feature_store/service/oci_entity.py +0 -125
  385. ads/feature_store/service/oci_feature_group.py +0 -164
  386. ads/feature_store/service/oci_feature_group_job.py +0 -214
  387. ads/feature_store/service/oci_feature_store.py +0 -182
  388. ads/feature_store/service/oci_lineage.py +0 -87
  389. ads/feature_store/service/oci_transformation.py +0 -104
  390. ads/feature_store/statistics/__init__.py +0 -0
  391. ads/feature_store/statistics/abs_feature_value.py +0 -49
  392. ads/feature_store/statistics/charts/__init__.py +0 -0
  393. ads/feature_store/statistics/charts/abstract_feature_plot.py +0 -37
  394. ads/feature_store/statistics/charts/box_plot.py +0 -148
  395. ads/feature_store/statistics/charts/frequency_distribution.py +0 -65
  396. ads/feature_store/statistics/charts/probability_distribution.py +0 -68
  397. ads/feature_store/statistics/charts/top_k_frequent_elements.py +0 -98
  398. ads/feature_store/statistics/feature_stat.py +0 -126
  399. ads/feature_store/statistics/generic_feature_value.py +0 -33
  400. ads/feature_store/statistics/statistics.py +0 -41
  401. ads/feature_store/statistics_config.py +0 -101
  402. ads/feature_store/templates/feature_store_template.yaml +0 -45
  403. ads/feature_store/transformation.py +0 -499
  404. ads/feature_store/validation_output.py +0 -57
  405. ads/hpo/__init__.py +0 -9
  406. ads/hpo/_imports.py +0 -91
  407. ads/hpo/ads_search_space.py +0 -439
  408. ads/hpo/distributions.py +0 -325
  409. ads/hpo/objective.py +0 -280
  410. ads/hpo/search_cv.py +0 -1657
  411. ads/hpo/stopping_criterion.py +0 -75
  412. ads/hpo/tuner_artifact.py +0 -413
  413. ads/hpo/utils.py +0 -91
  414. ads/hpo/validation.py +0 -140
  415. ads/hpo/visualization/__init__.py +0 -5
  416. ads/hpo/visualization/_contour.py +0 -23
  417. ads/hpo/visualization/_edf.py +0 -20
  418. ads/hpo/visualization/_intermediate_values.py +0 -21
  419. ads/hpo/visualization/_optimization_history.py +0 -25
  420. ads/hpo/visualization/_parallel_coordinate.py +0 -169
  421. ads/hpo/visualization/_param_importances.py +0 -26
  422. ads/jobs/__init__.py +0 -53
  423. ads/jobs/ads_job.py +0 -663
  424. ads/jobs/builders/__init__.py +0 -5
  425. ads/jobs/builders/base.py +0 -156
  426. ads/jobs/builders/infrastructure/__init__.py +0 -6
  427. ads/jobs/builders/infrastructure/base.py +0 -165
  428. ads/jobs/builders/infrastructure/dataflow.py +0 -1252
  429. ads/jobs/builders/infrastructure/dsc_job.py +0 -1894
  430. ads/jobs/builders/infrastructure/dsc_job_runtime.py +0 -1233
  431. ads/jobs/builders/infrastructure/utils.py +0 -65
  432. ads/jobs/builders/runtimes/__init__.py +0 -5
  433. ads/jobs/builders/runtimes/artifact.py +0 -338
  434. ads/jobs/builders/runtimes/base.py +0 -325
  435. ads/jobs/builders/runtimes/container_runtime.py +0 -242
  436. ads/jobs/builders/runtimes/python_runtime.py +0 -1016
  437. ads/jobs/builders/runtimes/pytorch_runtime.py +0 -204
  438. ads/jobs/cli.py +0 -104
  439. ads/jobs/env_var_parser.py +0 -131
  440. ads/jobs/extension.py +0 -160
  441. ads/jobs/schema/__init__.py +0 -5
  442. ads/jobs/schema/infrastructure_schema.json +0 -116
  443. ads/jobs/schema/job_schema.json +0 -42
  444. ads/jobs/schema/runtime_schema.json +0 -183
  445. ads/jobs/schema/validator.py +0 -141
  446. ads/jobs/serializer.py +0 -296
  447. ads/jobs/templates/__init__.py +0 -5
  448. ads/jobs/templates/container.py +0 -6
  449. ads/jobs/templates/driver_notebook.py +0 -177
  450. ads/jobs/templates/driver_oci.py +0 -500
  451. ads/jobs/templates/driver_python.py +0 -48
  452. ads/jobs/templates/driver_pytorch.py +0 -852
  453. ads/jobs/templates/driver_utils.py +0 -615
  454. ads/jobs/templates/hostname_from_env.c +0 -55
  455. ads/jobs/templates/oci_metrics.py +0 -181
  456. ads/jobs/utils.py +0 -104
  457. ads/llm/__init__.py +0 -28
  458. ads/llm/autogen/__init__.py +0 -2
  459. ads/llm/autogen/constants.py +0 -15
  460. ads/llm/autogen/reports/__init__.py +0 -2
  461. ads/llm/autogen/reports/base.py +0 -67
  462. ads/llm/autogen/reports/data.py +0 -103
  463. ads/llm/autogen/reports/session.py +0 -526
  464. ads/llm/autogen/reports/templates/chat_box.html +0 -13
  465. ads/llm/autogen/reports/templates/chat_box_lt.html +0 -5
  466. ads/llm/autogen/reports/templates/chat_box_rt.html +0 -6
  467. ads/llm/autogen/reports/utils.py +0 -56
  468. ads/llm/autogen/v02/__init__.py +0 -4
  469. ads/llm/autogen/v02/client.py +0 -295
  470. ads/llm/autogen/v02/log_handlers/__init__.py +0 -2
  471. ads/llm/autogen/v02/log_handlers/oci_file_handler.py +0 -83
  472. ads/llm/autogen/v02/loggers/__init__.py +0 -6
  473. ads/llm/autogen/v02/loggers/metric_logger.py +0 -320
  474. ads/llm/autogen/v02/loggers/session_logger.py +0 -580
  475. ads/llm/autogen/v02/loggers/utils.py +0 -86
  476. ads/llm/autogen/v02/runtime_logging.py +0 -163
  477. ads/llm/chain.py +0 -268
  478. ads/llm/chat_template.py +0 -31
  479. ads/llm/deploy.py +0 -63
  480. ads/llm/guardrails/__init__.py +0 -5
  481. ads/llm/guardrails/base.py +0 -442
  482. ads/llm/guardrails/huggingface.py +0 -44
  483. ads/llm/langchain/__init__.py +0 -5
  484. ads/llm/langchain/plugins/__init__.py +0 -5
  485. ads/llm/langchain/plugins/chat_models/__init__.py +0 -5
  486. ads/llm/langchain/plugins/chat_models/oci_data_science.py +0 -1027
  487. ads/llm/langchain/plugins/embeddings/__init__.py +0 -4
  488. ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +0 -184
  489. ads/llm/langchain/plugins/llms/__init__.py +0 -5
  490. ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +0 -979
  491. ads/llm/requirements.txt +0 -3
  492. ads/llm/serialize.py +0 -219
  493. ads/llm/serializers/__init__.py +0 -0
  494. ads/llm/serializers/retrieval_qa.py +0 -153
  495. ads/llm/serializers/runnable_parallel.py +0 -27
  496. ads/llm/templates/score_chain.jinja2 +0 -155
  497. ads/llm/templates/tool_chat_template_hermes.jinja +0 -130
  498. ads/llm/templates/tool_chat_template_mistral_parallel.jinja +0 -94
  499. ads/model/__init__.py +0 -52
  500. ads/model/artifact.py +0 -573
  501. ads/model/artifact_downloader.py +0 -254
  502. ads/model/artifact_uploader.py +0 -267
  503. ads/model/base_properties.py +0 -238
  504. ads/model/common/.model-ignore +0 -66
  505. ads/model/common/__init__.py +0 -5
  506. ads/model/common/utils.py +0 -142
  507. ads/model/datascience_model.py +0 -2635
  508. ads/model/deployment/__init__.py +0 -20
  509. ads/model/deployment/common/__init__.py +0 -5
  510. ads/model/deployment/common/utils.py +0 -308
  511. ads/model/deployment/model_deployer.py +0 -466
  512. ads/model/deployment/model_deployment.py +0 -1846
  513. ads/model/deployment/model_deployment_infrastructure.py +0 -671
  514. ads/model/deployment/model_deployment_properties.py +0 -493
  515. ads/model/deployment/model_deployment_runtime.py +0 -838
  516. ads/model/extractor/__init__.py +0 -5
  517. ads/model/extractor/automl_extractor.py +0 -74
  518. ads/model/extractor/embedding_onnx_extractor.py +0 -80
  519. ads/model/extractor/huggingface_extractor.py +0 -88
  520. ads/model/extractor/keras_extractor.py +0 -84
  521. ads/model/extractor/lightgbm_extractor.py +0 -93
  522. ads/model/extractor/model_info_extractor.py +0 -114
  523. ads/model/extractor/model_info_extractor_factory.py +0 -105
  524. ads/model/extractor/pytorch_extractor.py +0 -87
  525. ads/model/extractor/sklearn_extractor.py +0 -112
  526. ads/model/extractor/spark_extractor.py +0 -89
  527. ads/model/extractor/tensorflow_extractor.py +0 -85
  528. ads/model/extractor/xgboost_extractor.py +0 -94
  529. ads/model/framework/__init__.py +0 -5
  530. ads/model/framework/automl_model.py +0 -178
  531. ads/model/framework/embedding_onnx_model.py +0 -438
  532. ads/model/framework/huggingface_model.py +0 -399
  533. ads/model/framework/lightgbm_model.py +0 -266
  534. ads/model/framework/pytorch_model.py +0 -266
  535. ads/model/framework/sklearn_model.py +0 -250
  536. ads/model/framework/spark_model.py +0 -326
  537. ads/model/framework/tensorflow_model.py +0 -254
  538. ads/model/framework/xgboost_model.py +0 -258
  539. ads/model/generic_model.py +0 -3518
  540. ads/model/model_artifact_boilerplate/README.md +0 -381
  541. ads/model/model_artifact_boilerplate/__init__.py +0 -5
  542. ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +0 -5
  543. ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +0 -427
  544. ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +0 -2
  545. ads/model/model_artifact_boilerplate/runtime.yaml +0 -7
  546. ads/model/model_artifact_boilerplate/score.py +0 -61
  547. ads/model/model_file_description_schema.json +0 -68
  548. ads/model/model_introspect.py +0 -331
  549. ads/model/model_metadata.py +0 -1810
  550. ads/model/model_metadata_mixin.py +0 -460
  551. ads/model/model_properties.py +0 -63
  552. ads/model/model_version_set.py +0 -739
  553. ads/model/runtime/__init__.py +0 -5
  554. ads/model/runtime/env_info.py +0 -306
  555. ads/model/runtime/model_deployment_details.py +0 -37
  556. ads/model/runtime/model_provenance_details.py +0 -58
  557. ads/model/runtime/runtime_info.py +0 -81
  558. ads/model/runtime/schemas/inference_env_info_schema.yaml +0 -16
  559. ads/model/runtime/schemas/model_provenance_schema.yaml +0 -36
  560. ads/model/runtime/schemas/training_env_info_schema.yaml +0 -16
  561. ads/model/runtime/utils.py +0 -201
  562. ads/model/serde/__init__.py +0 -5
  563. ads/model/serde/common.py +0 -40
  564. ads/model/serde/model_input.py +0 -547
  565. ads/model/serde/model_serializer.py +0 -1184
  566. ads/model/service/__init__.py +0 -5
  567. ads/model/service/oci_datascience_model.py +0 -1076
  568. ads/model/service/oci_datascience_model_deployment.py +0 -500
  569. ads/model/service/oci_datascience_model_version_set.py +0 -176
  570. ads/model/transformer/__init__.py +0 -5
  571. ads/model/transformer/onnx_transformer.py +0 -324
  572. ads/mysqldb/__init__.py +0 -5
  573. ads/mysqldb/mysql_db.py +0 -227
  574. ads/opctl/__init__.py +0 -18
  575. ads/opctl/anomaly_detection.py +0 -11
  576. ads/opctl/backend/__init__.py +0 -5
  577. ads/opctl/backend/ads_dataflow.py +0 -353
  578. ads/opctl/backend/ads_ml_job.py +0 -710
  579. ads/opctl/backend/ads_ml_pipeline.py +0 -164
  580. ads/opctl/backend/ads_model_deployment.py +0 -209
  581. ads/opctl/backend/base.py +0 -146
  582. ads/opctl/backend/local.py +0 -1053
  583. ads/opctl/backend/marketplace/__init__.py +0 -9
  584. ads/opctl/backend/marketplace/helm_helper.py +0 -173
  585. ads/opctl/backend/marketplace/local_marketplace.py +0 -271
  586. ads/opctl/backend/marketplace/marketplace_backend_runner.py +0 -71
  587. ads/opctl/backend/marketplace/marketplace_operator_interface.py +0 -44
  588. ads/opctl/backend/marketplace/marketplace_operator_runner.py +0 -24
  589. ads/opctl/backend/marketplace/marketplace_utils.py +0 -212
  590. ads/opctl/backend/marketplace/models/__init__.py +0 -5
  591. ads/opctl/backend/marketplace/models/bearer_token.py +0 -94
  592. ads/opctl/backend/marketplace/models/marketplace_type.py +0 -70
  593. ads/opctl/backend/marketplace/models/ocir_details.py +0 -56
  594. ads/opctl/backend/marketplace/prerequisite_checker.py +0 -238
  595. ads/opctl/cli.py +0 -707
  596. ads/opctl/cmds.py +0 -869
  597. ads/opctl/conda/__init__.py +0 -5
  598. ads/opctl/conda/cli.py +0 -193
  599. ads/opctl/conda/cmds.py +0 -749
  600. ads/opctl/conda/config.yaml +0 -34
  601. ads/opctl/conda/manifest_template.yaml +0 -13
  602. ads/opctl/conda/multipart_uploader.py +0 -188
  603. ads/opctl/conda/pack.py +0 -89
  604. ads/opctl/config/__init__.py +0 -5
  605. ads/opctl/config/base.py +0 -57
  606. ads/opctl/config/diagnostics/__init__.py +0 -5
  607. ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +0 -62
  608. ads/opctl/config/merger.py +0 -255
  609. ads/opctl/config/resolver.py +0 -297
  610. ads/opctl/config/utils.py +0 -79
  611. ads/opctl/config/validator.py +0 -17
  612. ads/opctl/config/versioner.py +0 -68
  613. ads/opctl/config/yaml_parsers/__init__.py +0 -7
  614. ads/opctl/config/yaml_parsers/base.py +0 -58
  615. ads/opctl/config/yaml_parsers/distributed/__init__.py +0 -7
  616. ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +0 -201
  617. ads/opctl/constants.py +0 -66
  618. ads/opctl/decorator/__init__.py +0 -5
  619. ads/opctl/decorator/common.py +0 -129
  620. ads/opctl/diagnostics/__init__.py +0 -5
  621. ads/opctl/diagnostics/__main__.py +0 -25
  622. ads/opctl/diagnostics/check_distributed_job_requirements.py +0 -212
  623. ads/opctl/diagnostics/check_requirements.py +0 -144
  624. ads/opctl/diagnostics/requirement_exception.py +0 -9
  625. ads/opctl/distributed/README.md +0 -109
  626. ads/opctl/distributed/__init__.py +0 -5
  627. ads/opctl/distributed/certificates.py +0 -32
  628. ads/opctl/distributed/cli.py +0 -207
  629. ads/opctl/distributed/cmds.py +0 -731
  630. ads/opctl/distributed/common/__init__.py +0 -5
  631. ads/opctl/distributed/common/abstract_cluster_provider.py +0 -449
  632. ads/opctl/distributed/common/abstract_framework_spec_builder.py +0 -88
  633. ads/opctl/distributed/common/cluster_config_helper.py +0 -103
  634. ads/opctl/distributed/common/cluster_provider_factory.py +0 -21
  635. ads/opctl/distributed/common/cluster_runner.py +0 -54
  636. ads/opctl/distributed/common/framework_factory.py +0 -29
  637. ads/opctl/docker/Dockerfile.job +0 -103
  638. ads/opctl/docker/Dockerfile.job.arm +0 -107
  639. ads/opctl/docker/Dockerfile.job.gpu +0 -175
  640. ads/opctl/docker/base-env.yaml +0 -13
  641. ads/opctl/docker/cuda.repo +0 -6
  642. ads/opctl/docker/operator/.dockerignore +0 -0
  643. ads/opctl/docker/operator/Dockerfile +0 -41
  644. ads/opctl/docker/operator/Dockerfile.gpu +0 -85
  645. ads/opctl/docker/operator/cuda.repo +0 -6
  646. ads/opctl/docker/operator/environment.yaml +0 -8
  647. ads/opctl/forecast.py +0 -11
  648. ads/opctl/index.yaml +0 -3
  649. ads/opctl/model/__init__.py +0 -5
  650. ads/opctl/model/cli.py +0 -65
  651. ads/opctl/model/cmds.py +0 -73
  652. ads/opctl/operator/README.md +0 -4
  653. ads/opctl/operator/__init__.py +0 -31
  654. ads/opctl/operator/cli.py +0 -344
  655. ads/opctl/operator/cmd.py +0 -596
  656. ads/opctl/operator/common/__init__.py +0 -5
  657. ads/opctl/operator/common/backend_factory.py +0 -460
  658. ads/opctl/operator/common/const.py +0 -27
  659. ads/opctl/operator/common/data/synthetic.csv +0 -16001
  660. ads/opctl/operator/common/dictionary_merger.py +0 -148
  661. ads/opctl/operator/common/errors.py +0 -42
  662. ads/opctl/operator/common/operator_config.py +0 -99
  663. ads/opctl/operator/common/operator_loader.py +0 -811
  664. ads/opctl/operator/common/operator_schema.yaml +0 -130
  665. ads/opctl/operator/common/operator_yaml_generator.py +0 -152
  666. ads/opctl/operator/common/utils.py +0 -208
  667. ads/opctl/operator/lowcode/__init__.py +0 -5
  668. ads/opctl/operator/lowcode/anomaly/MLoperator +0 -16
  669. ads/opctl/operator/lowcode/anomaly/README.md +0 -207
  670. ads/opctl/operator/lowcode/anomaly/__init__.py +0 -5
  671. ads/opctl/operator/lowcode/anomaly/__main__.py +0 -103
  672. ads/opctl/operator/lowcode/anomaly/cmd.py +0 -35
  673. ads/opctl/operator/lowcode/anomaly/const.py +0 -167
  674. ads/opctl/operator/lowcode/anomaly/environment.yaml +0 -10
  675. ads/opctl/operator/lowcode/anomaly/model/__init__.py +0 -5
  676. ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +0 -146
  677. ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +0 -162
  678. ads/opctl/operator/lowcode/anomaly/model/automlx.py +0 -99
  679. ads/opctl/operator/lowcode/anomaly/model/autots.py +0 -115
  680. ads/opctl/operator/lowcode/anomaly/model/base_model.py +0 -404
  681. ads/opctl/operator/lowcode/anomaly/model/factory.py +0 -110
  682. ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +0 -78
  683. ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +0 -78
  684. ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +0 -120
  685. ads/opctl/operator/lowcode/anomaly/model/tods.py +0 -119
  686. ads/opctl/operator/lowcode/anomaly/operator_config.py +0 -127
  687. ads/opctl/operator/lowcode/anomaly/schema.yaml +0 -401
  688. ads/opctl/operator/lowcode/anomaly/utils.py +0 -88
  689. ads/opctl/operator/lowcode/common/__init__.py +0 -5
  690. ads/opctl/operator/lowcode/common/const.py +0 -10
  691. ads/opctl/operator/lowcode/common/data.py +0 -116
  692. ads/opctl/operator/lowcode/common/errors.py +0 -47
  693. ads/opctl/operator/lowcode/common/transformations.py +0 -296
  694. ads/opctl/operator/lowcode/common/utils.py +0 -293
  695. ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +0 -13
  696. ads/opctl/operator/lowcode/feature_store_marketplace/README.md +0 -30
  697. ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +0 -5
  698. ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +0 -116
  699. ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +0 -85
  700. ads/opctl/operator/lowcode/feature_store_marketplace/const.py +0 -15
  701. ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
  702. ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +0 -4
  703. ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +0 -32
  704. ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +0 -43
  705. ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +0 -120
  706. ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +0 -34
  707. ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +0 -386
  708. ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +0 -160
  709. ads/opctl/operator/lowcode/forecast/MLoperator +0 -25
  710. ads/opctl/operator/lowcode/forecast/README.md +0 -209
  711. ads/opctl/operator/lowcode/forecast/__init__.py +0 -5
  712. ads/opctl/operator/lowcode/forecast/__main__.py +0 -89
  713. ads/opctl/operator/lowcode/forecast/cmd.py +0 -40
  714. ads/opctl/operator/lowcode/forecast/const.py +0 -92
  715. ads/opctl/operator/lowcode/forecast/environment.yaml +0 -20
  716. ads/opctl/operator/lowcode/forecast/errors.py +0 -26
  717. ads/opctl/operator/lowcode/forecast/model/__init__.py +0 -5
  718. ads/opctl/operator/lowcode/forecast/model/arima.py +0 -279
  719. ads/opctl/operator/lowcode/forecast/model/automlx.py +0 -542
  720. ads/opctl/operator/lowcode/forecast/model/autots.py +0 -312
  721. ads/opctl/operator/lowcode/forecast/model/base_model.py +0 -863
  722. ads/opctl/operator/lowcode/forecast/model/factory.py +0 -106
  723. ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +0 -492
  724. ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +0 -243
  725. ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +0 -486
  726. ads/opctl/operator/lowcode/forecast/model/prophet.py +0 -445
  727. ads/opctl/operator/lowcode/forecast/model_evaluator.py +0 -244
  728. ads/opctl/operator/lowcode/forecast/operator_config.py +0 -234
  729. ads/opctl/operator/lowcode/forecast/schema.yaml +0 -506
  730. ads/opctl/operator/lowcode/forecast/utils.py +0 -413
  731. ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +0 -7
  732. ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +0 -285
  733. ads/opctl/operator/lowcode/forecast/whatifserve/score.py +0 -246
  734. ads/opctl/operator/lowcode/pii/MLoperator +0 -17
  735. ads/opctl/operator/lowcode/pii/README.md +0 -208
  736. ads/opctl/operator/lowcode/pii/__init__.py +0 -5
  737. ads/opctl/operator/lowcode/pii/__main__.py +0 -78
  738. ads/opctl/operator/lowcode/pii/cmd.py +0 -39
  739. ads/opctl/operator/lowcode/pii/constant.py +0 -84
  740. ads/opctl/operator/lowcode/pii/environment.yaml +0 -17
  741. ads/opctl/operator/lowcode/pii/errors.py +0 -27
  742. ads/opctl/operator/lowcode/pii/model/__init__.py +0 -5
  743. ads/opctl/operator/lowcode/pii/model/factory.py +0 -82
  744. ads/opctl/operator/lowcode/pii/model/guardrails.py +0 -167
  745. ads/opctl/operator/lowcode/pii/model/pii.py +0 -145
  746. ads/opctl/operator/lowcode/pii/model/processor/__init__.py +0 -34
  747. ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +0 -34
  748. ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +0 -35
  749. ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +0 -225
  750. ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +0 -73
  751. ads/opctl/operator/lowcode/pii/model/processor/remover.py +0 -26
  752. ads/opctl/operator/lowcode/pii/model/report.py +0 -487
  753. ads/opctl/operator/lowcode/pii/operator_config.py +0 -95
  754. ads/opctl/operator/lowcode/pii/schema.yaml +0 -108
  755. ads/opctl/operator/lowcode/pii/utils.py +0 -43
  756. ads/opctl/operator/lowcode/recommender/MLoperator +0 -16
  757. ads/opctl/operator/lowcode/recommender/README.md +0 -206
  758. ads/opctl/operator/lowcode/recommender/__init__.py +0 -5
  759. ads/opctl/operator/lowcode/recommender/__main__.py +0 -82
  760. ads/opctl/operator/lowcode/recommender/cmd.py +0 -33
  761. ads/opctl/operator/lowcode/recommender/constant.py +0 -30
  762. ads/opctl/operator/lowcode/recommender/environment.yaml +0 -11
  763. ads/opctl/operator/lowcode/recommender/model/base_model.py +0 -212
  764. ads/opctl/operator/lowcode/recommender/model/factory.py +0 -56
  765. ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +0 -25
  766. ads/opctl/operator/lowcode/recommender/model/svd.py +0 -106
  767. ads/opctl/operator/lowcode/recommender/operator_config.py +0 -81
  768. ads/opctl/operator/lowcode/recommender/schema.yaml +0 -265
  769. ads/opctl/operator/lowcode/recommender/utils.py +0 -13
  770. ads/opctl/operator/runtime/__init__.py +0 -5
  771. ads/opctl/operator/runtime/const.py +0 -17
  772. ads/opctl/operator/runtime/container_runtime_schema.yaml +0 -50
  773. ads/opctl/operator/runtime/marketplace_runtime.py +0 -50
  774. ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +0 -21
  775. ads/opctl/operator/runtime/python_runtime_schema.yaml +0 -21
  776. ads/opctl/operator/runtime/runtime.py +0 -115
  777. ads/opctl/schema.yaml.yml +0 -36
  778. ads/opctl/script.py +0 -40
  779. ads/opctl/spark/__init__.py +0 -5
  780. ads/opctl/spark/cli.py +0 -43
  781. ads/opctl/spark/cmds.py +0 -147
  782. ads/opctl/templates/diagnostic_report_template.jinja2 +0 -102
  783. ads/opctl/utils.py +0 -344
  784. ads/oracledb/__init__.py +0 -5
  785. ads/oracledb/oracle_db.py +0 -346
  786. ads/pipeline/__init__.py +0 -39
  787. ads/pipeline/ads_pipeline.py +0 -2279
  788. ads/pipeline/ads_pipeline_run.py +0 -772
  789. ads/pipeline/ads_pipeline_step.py +0 -605
  790. ads/pipeline/builders/__init__.py +0 -5
  791. ads/pipeline/builders/infrastructure/__init__.py +0 -5
  792. ads/pipeline/builders/infrastructure/custom_script.py +0 -32
  793. ads/pipeline/cli.py +0 -119
  794. ads/pipeline/extension.py +0 -291
  795. ads/pipeline/schema/__init__.py +0 -5
  796. ads/pipeline/schema/cs_step_schema.json +0 -35
  797. ads/pipeline/schema/ml_step_schema.json +0 -31
  798. ads/pipeline/schema/pipeline_schema.json +0 -71
  799. ads/pipeline/visualizer/__init__.py +0 -5
  800. ads/pipeline/visualizer/base.py +0 -570
  801. ads/pipeline/visualizer/graph_renderer.py +0 -272
  802. ads/pipeline/visualizer/text_renderer.py +0 -84
  803. ads/secrets/__init__.py +0 -11
  804. ads/secrets/adb.py +0 -386
  805. ads/secrets/auth_token.py +0 -86
  806. ads/secrets/big_data_service.py +0 -365
  807. ads/secrets/mysqldb.py +0 -149
  808. ads/secrets/oracledb.py +0 -160
  809. ads/secrets/secrets.py +0 -407
  810. ads/telemetry/__init__.py +0 -7
  811. ads/telemetry/base.py +0 -69
  812. ads/telemetry/client.py +0 -125
  813. ads/telemetry/telemetry.py +0 -257
  814. ads/templates/dataflow_pyspark.jinja2 +0 -13
  815. ads/templates/dataflow_sparksql.jinja2 +0 -22
  816. ads/templates/func.jinja2 +0 -20
  817. ads/templates/schemas/openapi.json +0 -1740
  818. ads/templates/score-pkl.jinja2 +0 -173
  819. ads/templates/score.jinja2 +0 -322
  820. ads/templates/score_embedding_onnx.jinja2 +0 -202
  821. ads/templates/score_generic.jinja2 +0 -165
  822. ads/templates/score_huggingface_pipeline.jinja2 +0 -217
  823. ads/templates/score_lightgbm.jinja2 +0 -185
  824. ads/templates/score_onnx.jinja2 +0 -407
  825. ads/templates/score_onnx_new.jinja2 +0 -473
  826. ads/templates/score_oracle_automl.jinja2 +0 -185
  827. ads/templates/score_pyspark.jinja2 +0 -154
  828. ads/templates/score_pytorch.jinja2 +0 -219
  829. ads/templates/score_scikit-learn.jinja2 +0 -184
  830. ads/templates/score_tensorflow.jinja2 +0 -184
  831. ads/templates/score_xgboost.jinja2 +0 -178
  832. ads/text_dataset/__init__.py +0 -5
  833. ads/text_dataset/backends.py +0 -211
  834. ads/text_dataset/dataset.py +0 -445
  835. ads/text_dataset/extractor.py +0 -207
  836. ads/text_dataset/options.py +0 -53
  837. ads/text_dataset/udfs.py +0 -22
  838. ads/text_dataset/utils.py +0 -49
  839. ads/type_discovery/__init__.py +0 -9
  840. ads/type_discovery/abstract_detector.py +0 -21
  841. ads/type_discovery/constant_detector.py +0 -41
  842. ads/type_discovery/continuous_detector.py +0 -54
  843. ads/type_discovery/credit_card_detector.py +0 -99
  844. ads/type_discovery/datetime_detector.py +0 -92
  845. ads/type_discovery/discrete_detector.py +0 -118
  846. ads/type_discovery/document_detector.py +0 -146
  847. ads/type_discovery/ip_detector.py +0 -68
  848. ads/type_discovery/latlon_detector.py +0 -90
  849. ads/type_discovery/phone_number_detector.py +0 -63
  850. ads/type_discovery/type_discovery_driver.py +0 -87
  851. ads/type_discovery/typed_feature.py +0 -594
  852. ads/type_discovery/unknown_detector.py +0 -41
  853. ads/type_discovery/zipcode_detector.py +0 -48
  854. ads/vault/__init__.py +0 -7
  855. ads/vault/vault.py +0 -237
  856. oracle_ads-2.13.8.dist-info/RECORD +0 -858
  857. {oracle_ads-2.13.8.dist-info → oracle_ads-2.13.9rc0.dist-info}/licenses/LICENSE.txt +0 -0
ads/catalog/model.py DELETED
@@ -1,1576 +0,0 @@
1
- #!/usr/bin/env python
2
- # -*- coding: utf-8; -*-
3
-
4
- # Copyright (c) 2020, 2024 Oracle and/or its affiliates.
5
- # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
6
-
7
- import warnings
8
-
9
- warnings.warn(
10
- (
11
- "The `ads.catalog.model` is deprecated in `oracle-ads 2.6.9` and will be removed in `oracle-ads 3.0`. "
12
- "Use framework specific Model utility class for saving and deploying model. "
13
- "Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html"
14
- ),
15
- DeprecationWarning,
16
- stacklevel=2,
17
- )
18
-
19
- import json
20
- import os
21
- import shutil
22
- import tempfile
23
- import time
24
- import uuid
25
- from typing import Dict, Optional, Union
26
- from zipfile import ZipFile
27
-
28
- import pandas as pd
29
- import yaml
30
- from ads.catalog.summary import SummaryList
31
- from ads.common import auth, logger, oci_client, utils
32
- from ads.common.decorator.deprecate import deprecated
33
- from ads.common.decorator.runtime_dependency import (
34
- runtime_dependency,
35
- OptionalDependency,
36
- )
37
- from ads.common.model_artifact import ConflictStrategy, ModelArtifact
38
- from ads.model.model_metadata import (
39
- METADATA_SIZE_LIMIT,
40
- MetadataSizeTooLarge,
41
- ModelCustomMetadata,
42
- ModelTaxonomyMetadata,
43
- )
44
- from ads.common.object_storage_details import ObjectStorageDetails
45
- from ads.common.oci_resource import SEARCH_TYPE, OCIResource
46
- from ads.config import (
47
- NB_SESSION_COMPARTMENT_OCID,
48
- OCI_ODSC_SERVICE_ENDPOINT,
49
- OCI_REGION_METADATA,
50
- PROJECT_OCID,
51
- )
52
- from ads.dataset.progress import TqdmProgressBar
53
- from ads.feature_engineering.schema import Schema
54
- from ads.model.model_version_set import ModelVersionSet, _extract_model_version_set_id
55
- from ads.model.deployment.model_deployer import ModelDeployer
56
- from oci.data_science.data_science_client import DataScienceClient
57
- from oci.data_science.models import (
58
- ArtifactExportDetailsObjectStorage,
59
- ArtifactImportDetailsObjectStorage,
60
- CreateModelDetails,
61
- ExportModelArtifactDetails,
62
- ImportModelArtifactDetails,
63
- )
64
- from oci.data_science.models import Model as OCIModel
65
- from oci.data_science.models import ModelSummary, WorkRequest
66
- from oci.data_science.models.model_provenance import ModelProvenance
67
- from oci.data_science.models.update_model_details import UpdateModelDetails
68
- from oci.exceptions import ServiceError
69
- from oci.identity import IdentityClient
70
-
71
- _UPDATE_MODEL_DETAILS_ATTRIBUTES = [
72
- "display_name",
73
- "description",
74
- "freeform_tags",
75
- "defined_tags",
76
- "model_version_set_id",
77
- "version_label",
78
- ]
79
- _MODEL_PROVENANCE_ATTRIBUTES = ModelProvenance().swagger_types.keys()
80
- _ETAG_KEY = "ETag"
81
- _MAX_ARTIFACT_SIZE_IN_BYTES = 2147483648 # 2GB
82
- _WORK_REQUEST_INTERVAL_IN_SEC = 3
83
-
84
-
85
- class ModelWithActiveDeploymentError(Exception): # pragma: no cover
86
- pass
87
-
88
-
89
- class ModelArtifactSizeError(Exception): # pragma: no cover
90
- def __init__(self, max_artifact_size: str):
91
- super().__init__(
92
- f"The model artifacts size is greater than `{max_artifact_size}`. "
93
- "The `bucket_uri` needs to be specified to "
94
- "copy artifacts to the object storage bucket. "
95
- "Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`"
96
- )
97
-
98
-
99
- def _get_etag(response) -> str:
100
- """Gets etag from the response."""
101
- if _ETAG_KEY in response.headers:
102
- return response.headers[_ETAG_KEY].split("--")[0]
103
- return None
104
-
105
-
106
- class ModelSummaryList(SummaryList):
107
- """Model Summary List which represents a list of Model Object.
108
-
109
- Methods
110
- -------
111
- sort_by(self, columns, reverse=False)
112
- Performs a multi-key sort on a particular set of columns and returns the sorted ModelSummaryList.
113
- Results are listed in a descending order by default.
114
- filter(self, selection, instance=None)
115
- Filters the model list according to a lambda filter function, or list comprehension.
116
- """
117
-
118
- @deprecated(
119
- "2.6.6",
120
- details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
121
- )
122
- def __init__(
123
- self,
124
- model_catalog,
125
- model_list,
126
- response=None,
127
- datetime_format=utils.date_format,
128
- ):
129
- super(ModelSummaryList, self).__init__(
130
- model_list, datetime_format=datetime_format
131
- )
132
- self.mc = model_catalog
133
- self.response = response
134
-
135
- def __add__(self, rhs):
136
- return ModelSummaryList(
137
- self.mc, list.__add__(self, rhs), datetime_format=self.datetime_format
138
- )
139
-
140
- def __getitem__(self, item):
141
- return self.mc.get_model(super(ModelSummaryList, self).__getitem__(item).id)
142
-
143
- def sort_by(self, columns, reverse=False):
144
- """
145
- Performs a multi-key sort on a particular set of columns and returns the sorted ModelSummaryList.
146
- Results are listed in a descending order by default.
147
-
148
- Parameters
149
- ----------
150
- columns: List of string
151
- A list of columns which are provided to sort on
152
- reverse: Boolean (defaults to false)
153
- If you'd like to reverse the results (for example, to get ascending instead of descending results)
154
-
155
- Returns
156
- -------
157
- ModelSummaryList: A sorted ModelSummaryList
158
- """
159
- return ModelSummaryList(
160
- self.mc,
161
- self._sort_by(columns, reverse=reverse),
162
- datetime_format=self.datetime_format,
163
- )
164
-
165
- def filter(self, selection, instance=None):
166
- """
167
- Filters the model list according to a lambda filter function, or list comprehension.
168
-
169
- Parameters
170
- ----------
171
- selection: lambda function filtering model instances, or a list-comprehension
172
- function of list filtering projects
173
- instance: list, optional
174
- list to filter, optional, defaults to self
175
-
176
- Returns
177
- -------
178
- ModelSummaryList: A filtered ModelSummaryList
179
- """
180
- instance = instance if instance is not None else self
181
-
182
- if callable(selection):
183
- res = list(filter(selection, instance))
184
- # lambda filtering
185
- if len(res) == 0:
186
- print("No models found")
187
- return
188
- return ModelSummaryList(self.mc, res, datetime_format=self.datetime_format)
189
- elif isinstance(selection, list):
190
- # list comprehension
191
- if len(selection) == 0:
192
- print("No models found")
193
- return
194
- return ModelSummaryList(
195
- self.mc, selection, datetime_format=self.datetime_format
196
- )
197
- else:
198
- raise ValueError(
199
- "Filter selection must be a function or a ProjectSummaryList"
200
- )
201
-
202
-
203
- class Model:
204
- """Class that represents the ADS implementation of model catalog item.
205
- Converts the metadata and schema from OCI implememtation to ADS implementation.
206
-
207
- Methods
208
- -------
209
- to_dataframe
210
- Converts model to dataframe format.
211
- show_in_notebook
212
- Shows model in the notebook in dataframe or YAML representation.
213
- activate
214
- Activates model.
215
- deactivate
216
- Deactivates model.
217
- commit
218
- Commits the changes made to the model.
219
- rollback
220
- Rollbacks the changes made to the model.
221
- load_model
222
- Loads the model from the model catalog based on model ID.
223
- """
224
-
225
- _FIELDS_TO_DECORATE = [
226
- "schema_input",
227
- "schema_output",
228
- "metadata_custom",
229
- "metadata_taxonomy",
230
- ]
231
-
232
- _NEW_ATTRIBUTES = [
233
- "input_schema",
234
- "output_schema",
235
- "custom_metadata_list",
236
- "defined_metadata_list",
237
- ]
238
-
239
- @deprecated(
240
- "2.6.6",
241
- details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
242
- )
243
- def __init__(
244
- self,
245
- model: OCIModel,
246
- model_etag: str,
247
- provenance_metadata: ModelProvenance,
248
- provenance_etag: str,
249
- ds_client: DataScienceClient,
250
- identity_client: IdentityClient,
251
- ) -> None:
252
- """Initializes the Model.
253
-
254
- Parameters
255
- ----------
256
- model: OCIModel
257
- The OCI model object.
258
- model_etag: str
259
- The model ETag.
260
- provenance_metadata: ModelProvenance
261
- The model provenance metadata.
262
- provenance_etag: str
263
- The model provenance metadata ETag.
264
- ds_client: DataScienceClient
265
- The Oracle DataScience client.
266
- identity_client: IdentityClient
267
- The Orcale Identity Service Client.
268
- """
269
- self.ds_client = ds_client
270
- self.identity_client = identity_client
271
- self.user_name = ""
272
- self._etag = model_etag
273
- self._provenance_metadata_etag = provenance_etag
274
- self.provenance_metadata = provenance_metadata
275
- self._extract_oci_model(model)
276
- self._extract_user_name(model)
277
-
278
- def _extract_oci_model(self, model: OCIModel) -> None:
279
- """Extracts the model information from OCI model."""
280
- for key in model.swagger_types.keys():
281
- if key not in self._NEW_ATTRIBUTES:
282
- val = getattr(model, key)
283
- setattr(self, key, val)
284
- self.schema_input = self._extract_schema("input_schema", model)
285
- self.schema_output = self._extract_schema("output_schema", model)
286
- self.metadata_custom = self._extract_metadata_custom(model)
287
- self.metadata_taxonomy = self._extract_metadata_taxonomy(model)
288
- self.swagger_types = model.swagger_types
289
- self.lifecycle_state = model.lifecycle_state
290
-
291
- def _validate_metadata(self):
292
- self.metadata_custom.validate()
293
- self.metadata_taxonomy.validate()
294
- total_size = self.metadata_custom.size() + self.metadata_taxonomy.size()
295
- if total_size > METADATA_SIZE_LIMIT:
296
- raise MetadataSizeTooLarge(total_size)
297
- return True
298
-
299
- def _extract_user_name(self, model: OCIModel) -> None:
300
- try:
301
- user = self.identity_client.get_user(model.created_by)
302
- self.user_name = user.data.name
303
- except:
304
- pass
305
-
306
- @staticmethod
307
- def _extract_schema(key, model):
308
- """Extracts the input and output schema."""
309
- schema = Schema()
310
- if hasattr(model, key):
311
- try:
312
- schema = (
313
- Schema.from_dict(json.loads(getattr(model, key)))
314
- if getattr(model, key)
315
- else Schema()
316
- )
317
- except Exception as e:
318
- logger.warning(str(e))
319
- return schema
320
-
321
- @staticmethod
322
- def _extract_metadata_taxonomy(model):
323
- """Extracts the taxonomy metadata."""
324
- metadata_taxonomy = ModelTaxonomyMetadata()
325
- if hasattr(model, "defined_metadata_list"):
326
- try:
327
- metadata_taxonomy = ModelTaxonomyMetadata._from_oci_metadata(
328
- model.defined_metadata_list
329
- )
330
- except Exception as e:
331
- logger.warning(str(e))
332
- return metadata_taxonomy
333
-
334
- @staticmethod
335
- def _extract_metadata_custom(model):
336
- """Extracts the custom metadata."""
337
- metadata_custom = ModelCustomMetadata()
338
- if hasattr(model, "custom_metadata_list"):
339
- try:
340
- metadata_custom = ModelCustomMetadata._from_oci_metadata(
341
- model.custom_metadata_list
342
- )
343
- except Exception as e:
344
- logger.warning(str(e))
345
- return metadata_custom
346
-
347
- def _to_dict(self):
348
- """Converts the model attributes to dictionary format."""
349
- attributes = {}
350
- for key in _UPDATE_MODEL_DETAILS_ATTRIBUTES:
351
- if hasattr(self, key):
352
- attributes[key] = getattr(self, key)
353
-
354
- if self.provenance_metadata is not None:
355
- attributes.update(
356
- {
357
- key: getattr(self.provenance_metadata, key)
358
- for key in _MODEL_PROVENANCE_ATTRIBUTES
359
- }
360
- )
361
- for field in self._FIELDS_TO_DECORATE:
362
- attributes[field] = getattr(self, field).to_dict()
363
- return attributes
364
-
365
- def _to_yaml(self):
366
- """Converts the model attributes to yaml format."""
367
- attributes = self._to_dict()
368
- return yaml.safe_dump(attributes)
369
-
370
- def to_dataframe(self) -> pd.DataFrame:
371
- """
372
- Converts the model to dataframe format.
373
-
374
- Returns
375
- -------
376
- panadas.DataFrame
377
- Pandas dataframe.
378
- """
379
- attributes = self._to_dict()
380
- df = pd.DataFrame.from_dict(attributes, orient="index", columns=[""]).dropna()
381
- return df
382
-
383
- @runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
384
- def show_in_notebook(self, display_format: str = "dataframe") -> None:
385
- """Shows model in dataframe or yaml format.
386
- Supported formats: `dataframe` and `yaml`. Defaults to dataframe format.
387
-
388
- Returns
389
- -------
390
- None
391
- Nothing.
392
- """
393
- if display_format == "dataframe":
394
- from IPython.core.display import display
395
-
396
- display(self.to_dataframe())
397
- elif display_format == "yaml":
398
- print(self._to_yaml())
399
- else:
400
- NotImplementedError(
401
- "`display_format` is not supported. Choose 'dataframe' or 'yaml'"
402
- )
403
-
404
- def _repr_html_(self):
405
- """Shows model in dataframe format."""
406
- return (
407
- self.to_dataframe().style.set_properties(**{"margin-left": "0px"}).to_html()
408
- )
409
-
410
- def __repr__(self):
411
- """Shows model in dataframe format."""
412
- return (
413
- self.to_dataframe().style.set_properties(**{"margin-left": "0px"}).to_html()
414
- )
415
-
416
- def activate(self) -> None:
417
- """Activates model.
418
-
419
- Returns
420
- -------
421
- None
422
- Nothing.
423
- """
424
- self.lifecycle_state = OCIModel.LIFECYCLE_STATE_ACTIVE
425
-
426
- def deactivate(self) -> None:
427
- """Deactivates model.
428
-
429
- Returns
430
- -------
431
- None
432
- Nothing.
433
- """
434
- self.lifecycle_state = OCIModel.LIFECYCLE_STATE_INACTIVE
435
-
436
- def commit(self, force: bool = True) -> None:
437
- """Commits model changes.
438
-
439
- Parameters
440
- ----------
441
- force: bool
442
- If True, any remote changes on this model would be lost.
443
-
444
- Returns
445
- -------
446
- None
447
- Nothing.
448
- """
449
- self._validate_metadata()
450
- attributes = {
451
- key: getattr(self, key) for key in _UPDATE_MODEL_DETAILS_ATTRIBUTES
452
- }
453
-
454
- if hasattr(self, "metadata_custom"):
455
- attributes["custom_metadata_list"] = self.metadata_custom._to_oci_metadata()
456
- if hasattr(self, "metadata_taxonomy"):
457
- attributes[
458
- "defined_metadata_list"
459
- ] = self.metadata_taxonomy._to_oci_metadata()
460
-
461
- update_model_details = UpdateModelDetails(**attributes)
462
- # freeform_tags=self._model.freeform_tags, defined_tags=self._model.defined_tags)
463
-
464
- # update model
465
- # https://docs.oracle.com/en-us/iaas/Content/API/Concepts/usingapi.htm#eleven
466
- # The API supports etags for the purposes of optimistic concurrency control.
467
- # The GET and POST calls return an etag response header with a value you should store.
468
- # When you later want to update or delete the resource, set the if-match header to the ETag
469
- # you received for the resource. The resource will then be updated or deleted
470
- # only if the ETag you provide matches the current value of that resource's ETag.
471
- kwargs = {}
472
- if not force:
473
- kwargs["if_match"] = self._etag
474
-
475
- self.ds_client.update_model(
476
- self.id, update_model_details=update_model_details, **kwargs
477
- )
478
- # store the lifecycle status, as updating the model will delete info not included in "update_model_details"
479
- lifecycle_status = self.lifecycle_state
480
- self.__dict__.update(self._load_model().__dict__)
481
- self.lifecycle_state = lifecycle_status
482
-
483
- # update model state
484
- if not force:
485
- kwargs["if_match"] = self._etag
486
- if self.lifecycle_state == OCIModel.LIFECYCLE_STATE_ACTIVE:
487
- self.ds_client.activate_model(self.id, **kwargs)
488
- elif self.lifecycle_state == OCIModel.LIFECYCLE_STATE_INACTIVE:
489
- self.ds_client.deactivate_model(self.id, **kwargs)
490
- self.__dict__.update(self._load_model().__dict__)
491
-
492
- # update model provenance
493
- if self.provenance_metadata != ModelProvenance():
494
- if not force:
495
- kwargs["if_match"] = self._provenance_metadata_etag
496
- response = self.ds_client.update_model_provenance(
497
- self.id, self.provenance_metadata, **kwargs
498
- )
499
- # get model etag again, as updating model provenance changes it
500
- self.__dict__.update(self._load_model().__dict__)
501
-
502
- @staticmethod
503
- def _get_provenance_metadata(ds_client: DataScienceClient, model_id: str):
504
- """Gets provenance information for specified model."""
505
- try:
506
- provenance_response = ds_client.get_model_provenance(model_id)
507
- except ServiceError as e:
508
- if e.status == 404:
509
- try:
510
- provenance_response = ds_client.create_model_provenance(
511
- model_id, ModelProvenance()
512
- )
513
- except ServiceError as e2:
514
- raise e2
515
- elif e.status == 409:
516
- print("The model has been deleted.")
517
- raise e
518
- else:
519
- raise e
520
- return provenance_response
521
-
522
- @classmethod
523
- def load_model(
524
- cls,
525
- ds_client: DataScienceClient,
526
- identity_client: IdentityClient,
527
- model_id: str,
528
- ) -> "Model":
529
- """Loads the model from the model catalog based on model ID.
530
-
531
- Parameters
532
- ----------
533
- ds_client: DataScienceClient
534
- The Oracle DataScience client.
535
- identity_client: IdentityClient
536
- The Orcale Identity Service Client.
537
- model_id: str
538
- The model ID.
539
-
540
- Returns
541
- -------
542
- Model
543
- The ADS model catalog item.
544
-
545
- Raises
546
- ------
547
- ServiceError: If error occures while getting model from server.
548
- KeyError: If model not found.
549
- ValueError: If error occures while getting model provenance mettadata from server.
550
- """
551
-
552
- try:
553
- model_response = ds_client.get_model(model_id)
554
- except ServiceError as e:
555
- if e.status == 404:
556
- raise KeyError(e.message) from e
557
- raise e
558
-
559
- try:
560
- provenance_response = cls._get_provenance_metadata(ds_client, model_id)
561
- except Exception as e:
562
- raise ValueError(
563
- f"Unable to fetch model provenance metadata for model {model_id}"
564
- )
565
-
566
- return cls(
567
- model_response.data,
568
- _get_etag(model_response),
569
- provenance_response.data,
570
- _get_etag(provenance_response),
571
- ds_client,
572
- identity_client,
573
- )
574
-
575
- def _load_model(self):
576
- """Loads the model from model catalog."""
577
- return self.load_model(self.ds_client, self.identity_client, self.id)
578
-
579
- def rollback(self) -> None:
580
- """Rollbacks the changes made to the model.
581
-
582
- Returns
583
- -------
584
- None
585
- Nothing.
586
- """
587
- self.__dict__.update(self._load_model().__dict__)
588
-
589
-
590
- class ModelCatalog:
591
- """
592
- Allows to list, load, update, download, upload and delete models from model catalog.
593
-
594
- Methods
595
- -------
596
- get_model(self, model_id)
597
- Loads the model from the model catalog based on model_id.
598
- list_models(self, project_id=None, include_deleted=False, datetime_format=utils.date_format, **kwargs)
599
- Lists all models in a given compartment, or in the current project if project_id is specified.
600
- list_model_deployment(self, model_id, config=None, tenant_id=None, limit=500, page=None, **kwargs)
601
- Gets the list of model deployments by model Id across the compartments.
602
- update_model(self, model_id, update_model_details=None, **kwargs)
603
- Updates a model with given model_id, using the provided update data.
604
- delete_model(self, model, **kwargs)
605
- Deletes the model based on model_id.
606
- download_model(self, model_id, target_dir, force_overwrite=False, install_libs=False, conflict_strategy=ConflictStrategy.IGNORE)
607
- Downloads the model from model_dir to target_dir based on model_id.
608
- upload_model(self, model_artifact, provenance_metadata=None, project_id=None, display_name=None, description=None)
609
- Uploads the model artifact to cloud storage.
610
- """
611
-
612
- @deprecated(
613
- "2.6.6",
614
- details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
615
- )
616
- def __init__(
617
- self,
618
- compartment_id: Optional[str] = None,
619
- ds_client_auth: Optional[dict] = None,
620
- identity_client_auth: Optional[dict] = None,
621
- timeout: Optional[int] = None,
622
- ds_client: Optional[DataScienceClient] = None,
623
- identity_client: Optional[IdentityClient] = None,
624
- ):
625
- """Initializes model catalog instance.
626
-
627
- Parameters
628
- ----------
629
- compartment_id : (str, optional). Defaults to None.
630
- Model compartment OCID. If `None`, the `config.NB_SESSION_COMPARTMENT_OCID` would be used.
631
- ds_client_auth : (dict, optional). Defaults to None.
632
- The default authetication is set using `ads.set_auth` API. If you need to override the
633
- default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
634
- authentication signer and kwargs required to instantiate DataScienceClient object.
635
- identity_client_auth : (dict, optional). Defaults to None.
636
- The default authetication is set using `ads.set_auth` API. If you need to override the
637
- default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
638
- authentication signer and kwargs required to instantiate IdentityClient object.
639
- timeout: (int, optional). Defaults to 10 seconds.
640
- The connection timeout in seconds for the client.
641
- ds_client: DataScienceClient
642
- The Oracle DataScience client.
643
- identity_client: IdentityClient
644
- The Orcale Identity Service Client.
645
-
646
- Raises
647
- ------
648
- ValueError
649
- If compartment ID not specified.
650
- TypeError
651
- If timeout not an integer.
652
- """
653
- self.compartment_id = (
654
- NB_SESSION_COMPARTMENT_OCID if compartment_id is None else compartment_id
655
- )
656
- if self.compartment_id is None:
657
- raise ValueError("compartment_id needs to be specified.")
658
-
659
- if timeout and not isinstance(timeout, int):
660
- raise TypeError("Timeout must be an integer.")
661
-
662
- self.ds_client_auth = ds_client_auth
663
- self.identity_client_auth = identity_client_auth
664
- self.ds_client = ds_client
665
- self.identity_client = identity_client
666
-
667
- if not self.ds_client:
668
- self.ds_client_auth = (
669
- ds_client_auth
670
- if ds_client_auth
671
- else auth.default_signer(
672
- {"service_endpoint": OCI_ODSC_SERVICE_ENDPOINT}
673
- )
674
- )
675
- if timeout:
676
- if not self.ds_client_auth.get("client_kwargs"):
677
- self.ds_client_auth["client_kwargs"] = {}
678
- self.ds_client_auth["client_kwargs"]["timeout"] = timeout
679
- self.ds_client = oci_client.OCIClientFactory(
680
- **self.ds_client_auth
681
- ).data_science
682
-
683
- if not self.identity_client:
684
- self.identity_client_auth = (
685
- identity_client_auth
686
- if identity_client_auth
687
- else auth.default_signer(
688
- {"service_endpoint": OCI_ODSC_SERVICE_ENDPOINT}
689
- )
690
- )
691
- if timeout:
692
- if not self.identity_client_auth.get("client_kwargs"):
693
- self.identity_client_auth["client_kwargs"] = {}
694
- self.identity_client_auth["client_kwargs"]["timeout"] = timeout
695
- self.identity_client = oci_client.OCIClientFactory(
696
- **self.identity_client_auth
697
- ).identity
698
-
699
- self.short_id_index = {}
700
-
701
- def __getitem__(self, model_id): # pragma: no cover
702
- return self.get_model(model_id)
703
-
704
- def __contains__(self, model_id): # pragma: no cover
705
- try:
706
- return self.get_model(model_id) is not None
707
- except KeyError:
708
- return False
709
- except Exception:
710
- raise
711
-
712
- def __iter__(self): # pragma: no cover
713
- return self.list_models().__iter__()
714
-
715
- def __len__(self): # pragma: no cover
716
- return len(self.list_models())
717
-
718
- def get_model(self, model_id):
719
- """
720
- Loads the model from the model catalog based on model_id.
721
-
722
- Parameters
723
- ----------
724
- model_id: str, required
725
- The model ID.
726
-
727
- Returns
728
- -------
729
- ads.catalog.Model
730
- The ads.catalog.Model with the matching ID.
731
- """
732
- if not model_id.startswith("ocid"):
733
- model_id = self.short_id_index[model_id]
734
- self.id = model_id
735
- return Model.load_model(self.ds_client, self.identity_client, model_id)
736
-
737
- def list_models(
738
- self,
739
- project_id: str = None,
740
- include_deleted: bool = False,
741
- datetime_format: str = utils.date_format,
742
- **kwargs,
743
- ):
744
- """
745
- Lists all models in a given compartment, or in the current project if project_id is specified.
746
-
747
- Parameters
748
- ----------
749
- project_id: str
750
- The project_id of model.
751
- include_deleted: bool, optional, default=False
752
- Whether to include deleted models in the returned list.
753
- datetime_format: str, optional, default: '%Y-%m-%d %H:%M:%S'
754
- Change format for date time fields.
755
-
756
- Returns
757
- -------
758
- ModelSummaryList
759
- A list of models.
760
- """
761
- try:
762
- list_models_response = self.ds_client.list_models(
763
- self.compartment_id, project_id=project_id, **kwargs
764
- )
765
- if list_models_response.data is None or len(list_models_response.data) == 0:
766
- print("No model found.")
767
- return
768
- except ServiceError as se:
769
- if se.status == 404:
770
- raise KeyError(se.message) from se
771
- else:
772
- raise
773
-
774
- model_list_filtered = [
775
- Model(
776
- model=model,
777
- model_etag=None,
778
- provenance_metadata=None,
779
- provenance_etag=None,
780
- ds_client=self.ds_client,
781
- identity_client=self.identity_client,
782
- )
783
- for model in list_models_response.data
784
- if include_deleted
785
- or model.lifecycle_state != ModelSummary.LIFECYCLE_STATE_DELETED
786
- ]
787
- # handle empty list
788
- if model_list_filtered is None or len(model_list_filtered) == 0:
789
- print("No model found.")
790
- return []
791
-
792
- msl = ModelSummaryList(
793
- self,
794
- model_list_filtered,
795
- list_models_response,
796
- datetime_format=datetime_format,
797
- )
798
- self.short_id_index.update(msl.short_id_index)
799
- return msl
800
-
801
- def list_model_deployment(
802
- self,
803
- model_id: str,
804
- config: dict = None,
805
- tenant_id: str = None,
806
- limit: int = 500,
807
- page: str = None,
808
- **kwargs,
809
- ):
810
- """
811
- Gets the list of model deployments by model Id across the compartments.
812
-
813
- Parameters
814
- ----------
815
- model_id: str
816
- The model ID.
817
- config: dict (optional)
818
- Configuration keys and values as per SDK and Tool Configuration.
819
- The from_file() method can be used to load configuration from a file.
820
- Alternatively, a dict can be passed. You can validate_config the dict
821
- using validate_config(). Defaults to None.
822
- tenant_id: str (optional)
823
- The tenancy ID, which can be used to specify a different tenancy
824
- (for cross-tenancy authorization) when searching for resources in
825
- a different tenancy. Defaults to None.
826
- limit: int (optional)
827
- The maximum number of items to return. The value must be between
828
- 1 and 1000. Defaults to 500.
829
- page: str (optional)
830
- The page at which to start retrieving results.
831
-
832
- Returns
833
- -------
834
- The list of model deployments.
835
- """
836
- query = f"query datasciencemodeldeployment resources where ModelId='{model_id}'"
837
- return OCIResource.search(
838
- query,
839
- type=SEARCH_TYPE.STRUCTURED,
840
- config=config,
841
- tenant_id=tenant_id,
842
- limit=limit,
843
- page=page,
844
- **kwargs,
845
- )
846
-
847
- def update_model(self, model_id, update_model_details=None, **kwargs) -> Model:
848
- """
849
- Updates a model with given model_id, using the provided update data.
850
-
851
- Parameters
852
- ----------
853
- model_id: str
854
- The model ID.
855
- update_model_details: UpdateModelDetails
856
- Contains the update model details data to apply.
857
- Mandatory unless kwargs are supplied.
858
- kwargs: dict, optional
859
- Update model details can be supplied instead as kwargs.
860
-
861
- Returns
862
- -------
863
- Model
864
- The ads.catalog.Model with the matching ID.
865
- """
866
- if not model_id.startswith("ocid"):
867
- model_id = self.short_id_index[model_id]
868
- if update_model_details is None:
869
- update_model_details = UpdateModelDetails(
870
- **{
871
- k: v
872
- for k, v in kwargs.items()
873
- if k in _UPDATE_MODEL_DETAILS_ATTRIBUTES
874
- }
875
- )
876
- update_model_details.compartment_id = self.compartment_id
877
- # filter kwargs removing used keys
878
- kwargs = {
879
- k: v
880
- for k, v in kwargs.items()
881
- if k not in _UPDATE_MODEL_DETAILS_ATTRIBUTES
882
- }
883
- update_model_response = self.ds_client.update_model(
884
- model_id, update_model_details, **kwargs
885
- )
886
- provenance_response = Model._get_provenance_metadata(self.ds_client, model_id)
887
- return Model(
888
- model=update_model_response.data,
889
- model_etag=_get_etag(update_model_response),
890
- provenance_metadata=provenance_response.data,
891
- provenance_etag=_get_etag(provenance_response),
892
- ds_client=self.ds_client,
893
- identity_client=self.identity_client,
894
- )
895
-
896
- def delete_model(self, model: Union[str, "ads.catalog.Model"], **kwargs) -> bool:
897
- """
898
- Deletes the model from Model Catalog.
899
-
900
- Parameters
901
- ----------
902
- model: Union[str, "ads.catalog.Model"]
903
- The OCID of the model to delete as a string, or a `ads.catalog.Model` instance.
904
-
905
- kwargs:
906
- delete_associated_model_deployment: (bool, optional). Defaults to `False`.
907
- Whether associated model deployments need to be deletet or not.
908
-
909
- Returns
910
- -------
911
- bool
912
- `True` if the model was successfully deleted.
913
-
914
- Raises
915
- ------
916
- ModelWithActiveDeploymentError
917
- If model has active model deployments ant inout attribute
918
- `delete_associated_model_deployment` set to `False`.
919
- """
920
- model_id = (
921
- model.id
922
- if isinstance(model, Model)
923
- else self.short_id_index[model]
924
- if not model.startswith("ocid")
925
- else model
926
- )
927
- delete_associated_model_deployment = kwargs.pop(
928
- "delete_associated_model_deployment", None
929
- )
930
- active_deployments = tuple(
931
- item
932
- for item in self.list_model_deployment(model_id)
933
- if item.lifecycle_state == "ACTIVE"
934
- )
935
-
936
- if len(active_deployments) > 0:
937
- if not delete_associated_model_deployment:
938
- raise ModelWithActiveDeploymentError(
939
- f"The model `{model_id}` has active model deployments: "
940
- f"{[item.identifier for item in active_deployments]}. "
941
- "Delete associated model deployments before deleting the model or "
942
- "set the `delete_associated_model_deployment` attribute to `True`."
943
- )
944
-
945
- logger.info(
946
- f"Deleting model deployments associated with the model `{model_id}`."
947
- )
948
- for oci_model_deployment in active_deployments:
949
- (
950
- ModelDeployer(config=self.ds_client_auth)
951
- .get_model_deployment(oci_model_deployment.identifier)
952
- .delete(wait_for_completion=True)
953
- )
954
-
955
- logger.info(f"Deleting model `{model_id}`.")
956
- self.ds_client.delete_model(model_id, **kwargs)
957
- return True
958
-
959
- def _download_artifact(
960
- self,
961
- model_id: str,
962
- target_dir: str,
963
- force_overwrite: Optional[bool] = False,
964
- bucket_uri: Optional[str] = None,
965
- remove_existing_artifact: Optional[bool] = True,
966
- ) -> None:
967
- """
968
- Downloads the model artifacts from model catalog to target_dir based on `model_id`.
969
-
970
- Parameters
971
- ----------
972
- model_id: str
973
- The OCID of the model to download.
974
- target_dir: str
975
- The target location of model artifacts.
976
- force_overwrite: (bool, optional). Defaults to `False`.
977
- Overwrite target directory if exists.
978
- bucket_uri: (str, optional). Defaults to None.
979
- The OCI Object Storage URI where model artifacts will be copied to.
980
- The `bucket_uri` is only necessary for downloading large artifacts with
981
- size is greater than 2GB. Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`.
982
- remove_existing_artifact: (bool, optional). Defaults to `True`.
983
- Whether artifacts uploaded to object storage bucket need to be removed or not.
984
-
985
- Raises
986
- ------
987
- ValueError
988
- If targeted directory does not exist.
989
- KeyError
990
- If model id not found.
991
-
992
- Returns
993
- -------
994
- None
995
- Nothing
996
- """
997
- if os.path.exists(target_dir) and os.listdir(target_dir):
998
- if not force_overwrite:
999
- raise ValueError(
1000
- f"The `{target_dir}` directory already exists. "
1001
- "Set `force_overwrite` to `True` if you wish to overwrite."
1002
- )
1003
- shutil.rmtree(target_dir)
1004
-
1005
- with utils.get_progress_bar(6) as progress:
1006
- progress.update("Getting information about model artifacts")
1007
-
1008
- # If the size of artifacts greater than 2GB, then artifacts
1009
- # need to be imported to OS bucket at first.
1010
- try:
1011
- model_artifact_info = self.ds_client.head_model_artifact(
1012
- model_id=model_id
1013
- ).headers
1014
- except ServiceError as ex:
1015
- if ex.status == 404:
1016
- raise KeyError(f"The model `{model_id}` not found.") from ex
1017
- raise
1018
-
1019
- artifact_size = int(model_artifact_info.get("content-length"))
1020
-
1021
- if bucket_uri or artifact_size > _MAX_ARTIFACT_SIZE_IN_BYTES:
1022
- if not bucket_uri:
1023
- raise ModelArtifactSizeError(
1024
- utils.human_size(_MAX_ARTIFACT_SIZE_IN_BYTES)
1025
- )
1026
-
1027
- self._download_large_artifact(
1028
- model_id=model_id,
1029
- target_dir=target_dir,
1030
- bucket_uri=os.path.join(bucket_uri, f"{model_id}.zip"),
1031
- progress=progress,
1032
- remove_existing_artifact=remove_existing_artifact,
1033
- )
1034
- else:
1035
- self._download_small_artifact(
1036
- model_id=model_id, target_dir=target_dir, progress=progress
1037
- )
1038
- progress.update()
1039
-
1040
- progress.update("Done")
1041
-
1042
- def _download_large_artifact(
1043
- self,
1044
- model_id: str,
1045
- target_dir: str,
1046
- bucket_uri: str,
1047
- progress: TqdmProgressBar,
1048
- remove_existing_artifact: Optional[bool] = True,
1049
- ) -> None:
1050
- """
1051
- Downloads the model artifacts from model catalog to target_dir based on `model_id`.
1052
- This method is used for artifacts with size greater than 2GB.
1053
-
1054
- Parameters
1055
- ----------
1056
- model_id: str
1057
- The OCID of the model to download.
1058
- target_dir: str
1059
- The target location of model artifacts.
1060
- bucket_uri: str
1061
- The OCI Object Storage URI where model artifacts will be copied to.
1062
- The `bucket_uri` is only necessary for downloading large artifacts with
1063
- size is greater than 2GB. Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`.
1064
- progress: TqdmProgressBar
1065
- The progress bar.
1066
- remove_existing_artifact: (bool, optional). Defaults to `True`.
1067
- Whether artifacts uploaded to object storage bucket need to be removed or not.
1068
-
1069
- Returns
1070
- -------
1071
- None
1072
- Nothing.
1073
- """
1074
- progress.update(f"Importing model artifacts from model catalog")
1075
- self._import_model_artifact(model_id=model_id, bucket_uri=bucket_uri)
1076
-
1077
- progress.update("Copying model artifacts to the artifact directory")
1078
- with tempfile.TemporaryDirectory() as temp_dir:
1079
- zip_file_path = os.path.join(temp_dir, f"{str(uuid.uuid4())}.zip")
1080
- zip_file_path = utils.copy_file(
1081
- uri_src=bucket_uri,
1082
- uri_dst=zip_file_path,
1083
- progressbar_description="Copying model artifacts to the artifact directory",
1084
- )
1085
-
1086
- progress.update("Extracting model artifacts")
1087
- with ZipFile(zip_file_path) as zip_file:
1088
- zip_file.extractall(target_dir)
1089
-
1090
- if remove_existing_artifact:
1091
- progress.update("Removing temporary artifacts")
1092
- utils.remove_file(bucket_uri, self.ds_client_auth)
1093
- else:
1094
- progress.update()
1095
-
1096
- def _import_model_artifact(
1097
- self,
1098
- model_id: str,
1099
- bucket_uri: str,
1100
- ):
1101
- """Imports model artifact from the model catalog to the object storage bucket.
1102
- This method is used for the case when the artifact size is greater than 2GB.
1103
- """
1104
- bucket_details = ObjectStorageDetails.from_path(bucket_uri)
1105
- response = self.ds_client.import_model_artifact(
1106
- model_id=model_id,
1107
- import_model_artifact_details=ImportModelArtifactDetails(
1108
- artifact_import_details=ArtifactImportDetailsObjectStorage(
1109
- namespace=bucket_details.namespace,
1110
- destination_bucket=bucket_details.bucket,
1111
- destination_object_name=bucket_details.filepath,
1112
- destination_region=self._region,
1113
- )
1114
- ),
1115
- )
1116
-
1117
- self._wait_for_work_request(
1118
- response=response,
1119
- first_step_description="Preparing to import model artifacts from the model catalog",
1120
- num_steps=3,
1121
- )
1122
-
1123
- def _download_small_artifact(
1124
- self,
1125
- model_id: str,
1126
- target_dir: str,
1127
- progress: TqdmProgressBar,
1128
- ) -> None:
1129
- """
1130
- Downloads the model artifacts from model catalog to target_dir based on `model_id`.
1131
- This method is used for the artifacts with size less than 2GB.
1132
-
1133
- Parameters
1134
- ----------
1135
- model_id: str
1136
- The OCID of the model to download.
1137
- target_dir: str
1138
- The target location of model artifacts.
1139
- progress: TqdmProgressBar
1140
- The progress bar.
1141
-
1142
- Returns
1143
- -------
1144
- None
1145
- Nothing
1146
- """
1147
- progress.update("Importing model artifacts from catalog")
1148
- try:
1149
- zip_contents = self.ds_client.get_model_artifact_content(
1150
- model_id
1151
- ).data.content
1152
- except ServiceError as ex:
1153
- if ex.status == 404:
1154
- raise KeyError(ex.message) from ex
1155
- raise
1156
-
1157
- with tempfile.TemporaryDirectory() as temp_dir:
1158
- progress.update("Copying model artifacts to the artifact directory")
1159
- zip_file_path = os.path.join(temp_dir, f"{str(uuid.uuid4())}.zip")
1160
- with open(zip_file_path, "wb") as zip_file:
1161
- zip_file.write(zip_contents)
1162
- progress.update("Extracting model artifacts")
1163
- with ZipFile(zip_file_path) as zip_file:
1164
- zip_file.extractall(target_dir)
1165
-
1166
- @deprecated(
1167
- "2.5.9",
1168
- details="Instead use `ads.common.model_artifact.ModelArtifact.from_model_catalog()`.",
1169
- )
1170
- def download_model(
1171
- self,
1172
- model_id: str,
1173
- target_dir: str,
1174
- force_overwrite: bool = False,
1175
- install_libs: bool = False,
1176
- conflict_strategy=ConflictStrategy.IGNORE,
1177
- bucket_uri: Optional[str] = None,
1178
- remove_existing_artifact: Optional[bool] = True,
1179
- ):
1180
- """
1181
- Downloads the model from model_dir to target_dir based on model_id.
1182
-
1183
- Parameters
1184
- ----------
1185
- model_id: str
1186
- The OCID of the model to download.
1187
- target_dir: str
1188
- The target location of model after download.
1189
- force_overwrite: bool
1190
- Overwrite target_dir if exists.
1191
- install_libs: bool, default: False
1192
- Install the libraries specified in ds-requirements.txt which are missing in the current environment.
1193
- conflict_strategy: ConflictStrategy, default: IGNORE
1194
- Determines how to handle version conflicts between the current environment and requirements of
1195
- model artifact.
1196
- Valid values: "IGNORE", "UPDATE" or ConflictStrategy.
1197
- IGNORE: Use the installed version in case of conflict
1198
- UPDATE: Force update dependency to the version required by model artifact in case of conflict
1199
- bucket_uri: (str, optional). Defaults to None.
1200
- The OCI Object Storage URI where model artifacts will be copied to.
1201
- The `bucket_uri` is only necessary for downloading large artifacts with
1202
- size is greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
1203
- remove_existing_artifact: (bool, optional). Defaults to `True`.
1204
- Whether artifacts uploaded to object storage bucket need to be removed or not.
1205
-
1206
- Returns
1207
- -------
1208
- ModelArtifact
1209
- A ModelArtifact instance.
1210
- """
1211
- self._download_artifact(
1212
- model_id,
1213
- target_dir,
1214
- force_overwrite,
1215
- bucket_uri=bucket_uri,
1216
- remove_existing_artifact=remove_existing_artifact,
1217
- )
1218
-
1219
- result = ModelArtifact(
1220
- target_dir,
1221
- conflict_strategy=conflict_strategy,
1222
- install_libs=install_libs,
1223
- reload=False,
1224
- )
1225
-
1226
- try:
1227
- model_response = self.ds_client.get_model(model_id)
1228
- except ServiceError as e:
1229
- if e.status == 404:
1230
- raise KeyError(e.message) from e
1231
- raise e
1232
-
1233
- if hasattr(model_response.data, "custom_metadata_list"):
1234
- try:
1235
- result.metadata_custom = ModelCustomMetadata._from_oci_metadata(
1236
- model_response.data.custom_metadata_list
1237
- )
1238
- except:
1239
- result.metadata_custom = ModelCustomMetadata()
1240
- if hasattr(model_response.data, "defined_metadata_list"):
1241
- try:
1242
- result.metadata_taxonomy = ModelTaxonomyMetadata._from_oci_metadata(
1243
- model_response.data.defined_metadata_list
1244
- )
1245
- except:
1246
- result.metadata_taxonomy = ModelTaxonomyMetadata()
1247
- if hasattr(model_response.data, "input_schema"):
1248
- try:
1249
- result.schema_input = Schema.from_dict(
1250
- json.loads(model_response.data.input_schema)
1251
- if model_response.data.input_schema != ""
1252
- else Schema()
1253
- )
1254
- except:
1255
- result.schema_input = Schema()
1256
- if hasattr(model_response.data, "output_schema"):
1257
- try:
1258
- result.schema_output = Schema.from_dict(
1259
- json.loads(model_response.data.output_schema)
1260
- if model_response.data.output_schema != ""
1261
- else Schema()
1262
- )
1263
- except:
1264
- result.schema_output = Schema()
1265
-
1266
- if not install_libs:
1267
- logger.warning(
1268
- "Libraries in `ds-requirements.txt` were not installed. "
1269
- "Use `install_requirements()` to install the required dependencies."
1270
- )
1271
- return result
1272
-
1273
- @deprecated(
1274
- "2.6.6",
1275
- details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
1276
- )
1277
- def upload_model(
1278
- self,
1279
- model_artifact: ModelArtifact,
1280
- provenance_metadata: Optional[ModelProvenance] = None,
1281
- project_id: Optional[str] = None,
1282
- display_name: Optional[str] = None,
1283
- description: Optional[str] = None,
1284
- freeform_tags: Optional[Dict[str, Dict[str, object]]] = None,
1285
- defined_tags: Optional[Dict[str, Dict[str, object]]] = None,
1286
- bucket_uri: Optional[str] = None,
1287
- remove_existing_artifact: Optional[bool] = True,
1288
- overwrite_existing_artifact: Optional[bool] = True,
1289
- model_version_set: Optional[Union[str, ModelVersionSet]] = None,
1290
- version_label: Optional[str] = None,
1291
- ):
1292
- """
1293
- Uploads the model artifact to cloud storage.
1294
-
1295
- Parameters
1296
- ----------
1297
- model_artifact: Union[ModelArtifact, GenericModel]
1298
- The model artifacts or generic model instance.
1299
- provenance_metadata: (ModelProvenance, optional). Defaults to None.
1300
- Model provenance gives data scientists information about the origin of their model.
1301
- This information allows data scientists to reproduce
1302
- the development environment in which the model was trained.
1303
- project_id: (str, optional). Defaults to None.
1304
- The project_id of model.
1305
- display_name: (str, optional). Defaults to None.
1306
- The name of model. If a display_name is not provided, a randomly generated easy to remember name
1307
- with timestamp will be generated, like 'strange-spider-2022-08-17-23:55.02'.
1308
- description: (str, optional). Defaults to None.
1309
- The description of model.
1310
- freeform_tags: (Dict[str, str], optional). Defaults to None.
1311
- Freeform tags for the model, by default None
1312
- defined_tags: (Dict[str, dict[str, object]], optional). Defaults to None.
1313
- Defined tags for the model, by default None.
1314
- bucket_uri: (str, optional). Defaults to None.
1315
- The OCI Object Storage URI where model artifacts will be copied to.
1316
- The `bucket_uri` is only necessary for uploading large artifacts which
1317
- size greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
1318
- remove_existing_artifact: (bool, optional). Defaults to `True`.
1319
- Whether artifacts uploaded to object storage bucket need to be removed or not.
1320
- overwrite_existing_artifact: (bool, optional). Defaults to `True`.
1321
- Overwrite target bucket artifact if exists.
1322
- model_version_set: (Union[str, ModelVersionSet], optional). Defaults to None.
1323
- The Model version set OCID, or name, or `ModelVersionSet` instance.
1324
- version_label: (str, optional). Defaults to None.
1325
- The model version label.
1326
-
1327
- Returns
1328
- -------
1329
- ads.catalog.Model
1330
- The ads.catalog.Model with the matching ID.
1331
- """
1332
- project_id = project_id or PROJECT_OCID
1333
- if not project_id:
1334
- raise ValueError("`project_id` needs to be specified.")
1335
-
1336
- copy_artifact_to_os = False
1337
- if (
1338
- bucket_uri
1339
- or utils.folder_size(model_artifact.artifact_dir)
1340
- > _MAX_ARTIFACT_SIZE_IN_BYTES
1341
- ):
1342
- if not bucket_uri:
1343
- raise ValueError(
1344
- f"The model artifacts size is greater than `{utils.human_size(_MAX_ARTIFACT_SIZE_IN_BYTES)}`. "
1345
- "The `bucket_uri` needs to be specified to "
1346
- "copy artifacts to the object storage bucket. "
1347
- "Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`"
1348
- )
1349
- copy_artifact_to_os = True
1350
-
1351
- # extract model_version_set_id from model_version_set attribute or environment
1352
- # variables in case of saving model in context of model version set.
1353
- model_version_set_id = _extract_model_version_set_id(model_version_set)
1354
- # Set default display_name if not specified - randomly generated easy to remember name generated
1355
- display_name = display_name or utils.get_random_name_for_resource()
1356
-
1357
- with utils.get_progress_bar(5) as progress:
1358
- project_id = PROJECT_OCID if project_id is None else project_id
1359
- if project_id is None:
1360
- raise ValueError("project_id needs to be specified.")
1361
- schema_file = os.path.join(model_artifact.artifact_dir, "schema.json")
1362
- if os.path.exists(schema_file):
1363
- with open(schema_file, "r") as schema:
1364
- metadata = json.load(schema)
1365
- freeform_tags = {"problem_type": metadata["problem_type"]}
1366
-
1367
- progress.update("Saving model in the model catalog")
1368
- create_model_details = CreateModelDetails(
1369
- display_name=display_name,
1370
- description=description,
1371
- project_id=project_id,
1372
- compartment_id=self.compartment_id,
1373
- custom_metadata_list=model_artifact.metadata_custom._to_oci_metadata()
1374
- if model_artifact.metadata_custom is not None
1375
- else [],
1376
- defined_metadata_list=model_artifact.metadata_taxonomy._to_oci_metadata()
1377
- if model_artifact.metadata_taxonomy is not None
1378
- else [],
1379
- input_schema=model_artifact.schema_input.to_json()
1380
- if model_artifact.schema_input is not None
1381
- else '{"schema": []}',
1382
- output_schema=model_artifact.schema_output.to_json()
1383
- if model_artifact.schema_output is not None
1384
- else '{"schema": []}',
1385
- freeform_tags=freeform_tags,
1386
- defined_tags=defined_tags,
1387
- model_version_set_id=model_version_set_id,
1388
- version_label=version_label,
1389
- )
1390
- model = self.ds_client.create_model(create_model_details)
1391
-
1392
- if provenance_metadata is not None:
1393
- progress.update("Saving provenance metadata")
1394
- self.ds_client.create_model_provenance(
1395
- model.data.id, provenance_metadata
1396
- )
1397
- else:
1398
- progress.update()
1399
-
1400
- # if the model artifact size greater than 2GB then export function
1401
- # needs to be used instead of upload. The export function will copy
1402
- # model artifacts to the OS bucket at first and then will upload
1403
- # the artifacts to the model catalog.
1404
- if copy_artifact_to_os:
1405
- self._export_model_artifact(
1406
- model_id=model.data.id,
1407
- model_artifact=model_artifact,
1408
- progress=progress,
1409
- bucket_uri=bucket_uri,
1410
- remove_existing_artifact=remove_existing_artifact,
1411
- overwrite_existing_artifact=overwrite_existing_artifact,
1412
- )
1413
- else:
1414
- self._upload_model_artifact(
1415
- model_id=model.data.id,
1416
- model_artifact=model_artifact,
1417
- progress=progress,
1418
- )
1419
- progress.update()
1420
-
1421
- progress.update("Done")
1422
- return self.get_model(model.data.id)
1423
-
1424
- def _prepare_model_artifact(self, model_artifact, progress: TqdmProgressBar) -> str:
1425
- """Prepares model artifacts to save in the Model Catalog.
1426
-
1427
- Returns
1428
- -------
1429
- str
1430
- The path to the model artifact zip archive.
1431
- """
1432
- progress.update("Preparing model artifacts zip")
1433
- files_to_upload = model_artifact._get_files()
1434
- artifact_path = "/tmp/saved_model_" + str(uuid.uuid4()) + ".zip"
1435
- zf = ZipFile(artifact_path, "w")
1436
- for matched_file in files_to_upload:
1437
- zf.write(
1438
- os.path.join(model_artifact.artifact_dir, matched_file),
1439
- arcname=matched_file,
1440
- )
1441
- zf.close()
1442
- return artifact_path
1443
-
1444
- def _upload_model_artifact(self, model_id, model_artifact, progress):
1445
- """Uploads model artifact to the model catalog.
1446
- This method can be used only if the size of model artifact is less than 2GB.
1447
- For the artifacts with size greater than 2 GB the `_export_model_artifact`
1448
- method should be used instead.
1449
- """
1450
- artifact_zip_path = self._prepare_model_artifact(model_artifact, progress)
1451
- progress.update("Uploading model artifacts to the catalog")
1452
- with open(artifact_zip_path, "rb") as file_data:
1453
- bytes_content = file_data.read()
1454
- self.ds_client.create_model_artifact(
1455
- model_id,
1456
- bytes_content,
1457
- content_disposition=f'attachment; filename="{model_id}.zip"',
1458
- )
1459
- os.remove(artifact_zip_path)
1460
- progress.update()
1461
-
1462
- def _export_model_artifact(
1463
- self,
1464
- model_id: str,
1465
- model_artifact: ModelArtifact,
1466
- bucket_uri: str,
1467
- progress,
1468
- remove_existing_artifact: Optional[bool] = True,
1469
- overwrite_existing_artifact: Optional[bool] = True,
1470
- ):
1471
- """Exports model artifact to the model catalog.
1472
- This method is used for the case when the artifact size is greater than 2GB.
1473
- 1. Archive model artifact.
1474
- 2. Copies the artifact to the object storage bucket.
1475
- 3. Exports artifact from the user's object storage bucket to the system one.
1476
- """
1477
- artifact_zip_path = self._prepare_model_artifact(model_artifact, progress)
1478
- progress.update(f"Copying model artifact to the Object Storage bucket")
1479
-
1480
- try:
1481
- bucket_uri_file_name = os.path.basename(bucket_uri)
1482
- if not bucket_uri_file_name:
1483
- bucket_uri = os.path.join(bucket_uri, f"{model_id}.zip")
1484
- elif not bucket_uri.lower().endswith(".zip"):
1485
- bucket_uri = f"{bucket_uri}.zip"
1486
-
1487
- bucket_file_name = utils.copy_file(
1488
- artifact_zip_path,
1489
- bucket_uri,
1490
- force_overwrite=overwrite_existing_artifact,
1491
- auth=self.ds_client_auth,
1492
- progressbar_description="Copying model artifact to the Object Storage bucket",
1493
- )
1494
- except FileExistsError:
1495
- raise FileExistsError(
1496
- f"The `{bucket_uri}` exists. Please use a new file name or "
1497
- "set `overwrite_existing_artifact` to `True` if you wish to overwrite."
1498
- )
1499
-
1500
- os.remove(artifact_zip_path)
1501
-
1502
- progress.update("Exporting model artifact to the model catalog")
1503
- bucket_details = ObjectStorageDetails.from_path(bucket_file_name)
1504
- response = self.ds_client.export_model_artifact(
1505
- model_id=model_id,
1506
- export_model_artifact_details=ExportModelArtifactDetails(
1507
- artifact_export_details=ArtifactExportDetailsObjectStorage(
1508
- namespace=bucket_details.namespace,
1509
- source_bucket=bucket_details.bucket,
1510
- source_object_name=bucket_details.filepath,
1511
- source_region=self._region,
1512
- )
1513
- ),
1514
- )
1515
-
1516
- self._wait_for_work_request(
1517
- response=response,
1518
- first_step_description="Preparing to export model artifact to the model catalog",
1519
- num_steps=4,
1520
- )
1521
-
1522
- if remove_existing_artifact:
1523
- progress.update(
1524
- "Removing temporary model artifact from the Object Storage bucket"
1525
- )
1526
- utils.remove_file(bucket_file_name, self.ds_client_auth)
1527
- else:
1528
- progress.update()
1529
-
1530
- @property
1531
- def _region(self):
1532
- """Gets current region."""
1533
- if "region" in self.ds_client_auth.get("config", {}):
1534
- return self.ds_client_auth["config"]["region"]
1535
- return json.loads(OCI_REGION_METADATA)["regionIdentifier"]
1536
-
1537
- def _wait_for_work_request(
1538
- self, response, first_step_description: str = "", num_steps=4
1539
- ):
1540
- """Waits for the work request to be completed."""
1541
- STOP_STATE = (
1542
- WorkRequest.STATUS_SUCCEEDED,
1543
- WorkRequest.STATUS_CANCELED,
1544
- WorkRequest.STATUS_CANCELING,
1545
- WorkRequest.STATUS_FAILED,
1546
- )
1547
- work_request_id = response.headers["opc-work-request-id"]
1548
- work_request_logs = None
1549
-
1550
- i = 0
1551
- with utils.get_progress_bar(num_steps) as progress:
1552
- progress.update(first_step_description)
1553
- while not work_request_logs or len(work_request_logs) < num_steps:
1554
- time.sleep(_WORK_REQUEST_INTERVAL_IN_SEC)
1555
- work_request = self.ds_client.get_work_request(work_request_id)
1556
- work_request_logs = self.ds_client.list_work_request_logs(
1557
- work_request_id
1558
- ).data
1559
- if work_request_logs:
1560
- new_work_request_logs = work_request_logs[i:]
1561
-
1562
- for wr_item in new_work_request_logs:
1563
- progress.update(wr_item.message)
1564
- i += 1
1565
-
1566
- if work_request.data.status in STOP_STATE:
1567
- if work_request.data.status != WorkRequest.STATUS_SUCCEEDED:
1568
- if work_request_logs:
1569
- raise Exception(work_request_logs[-1].message)
1570
- else:
1571
- raise Exception(
1572
- "An error occurred in attempt to perform the operation. Check the service logs to get more details."
1573
- )
1574
- else:
1575
- break
1576
- return work_request