oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (857) hide show
  1. {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/METADATA +151 -151
  2. oracle_ads-2.13.9rc0.dist-info/RECORD +9 -0
  3. {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/WHEEL +2 -1
  4. {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/entry_points.txt +1 -2
  5. oracle_ads-2.13.9rc0.dist-info/top_level.txt +1 -0
  6. ads/aqua/__init__.py +0 -40
  7. ads/aqua/app.py +0 -506
  8. ads/aqua/cli.py +0 -96
  9. ads/aqua/client/__init__.py +0 -3
  10. ads/aqua/client/client.py +0 -836
  11. ads/aqua/client/openai_client.py +0 -305
  12. ads/aqua/common/__init__.py +0 -5
  13. ads/aqua/common/decorator.py +0 -125
  14. ads/aqua/common/entities.py +0 -266
  15. ads/aqua/common/enums.py +0 -122
  16. ads/aqua/common/errors.py +0 -109
  17. ads/aqua/common/utils.py +0 -1285
  18. ads/aqua/config/__init__.py +0 -4
  19. ads/aqua/config/container_config.py +0 -248
  20. ads/aqua/config/evaluation/__init__.py +0 -4
  21. ads/aqua/config/evaluation/evaluation_service_config.py +0 -147
  22. ads/aqua/config/utils/__init__.py +0 -4
  23. ads/aqua/config/utils/serializer.py +0 -339
  24. ads/aqua/constants.py +0 -114
  25. ads/aqua/data.py +0 -14
  26. ads/aqua/dummy_data/icon.txt +0 -1
  27. ads/aqua/dummy_data/oci_model_deployments.json +0 -56
  28. ads/aqua/dummy_data/oci_models.json +0 -1
  29. ads/aqua/dummy_data/readme.md +0 -26
  30. ads/aqua/evaluation/__init__.py +0 -8
  31. ads/aqua/evaluation/constants.py +0 -53
  32. ads/aqua/evaluation/entities.py +0 -186
  33. ads/aqua/evaluation/errors.py +0 -70
  34. ads/aqua/evaluation/evaluation.py +0 -1814
  35. ads/aqua/extension/__init__.py +0 -42
  36. ads/aqua/extension/aqua_ws_msg_handler.py +0 -76
  37. ads/aqua/extension/base_handler.py +0 -90
  38. ads/aqua/extension/common_handler.py +0 -121
  39. ads/aqua/extension/common_ws_msg_handler.py +0 -36
  40. ads/aqua/extension/deployment_handler.py +0 -298
  41. ads/aqua/extension/deployment_ws_msg_handler.py +0 -54
  42. ads/aqua/extension/errors.py +0 -30
  43. ads/aqua/extension/evaluation_handler.py +0 -129
  44. ads/aqua/extension/evaluation_ws_msg_handler.py +0 -61
  45. ads/aqua/extension/finetune_handler.py +0 -96
  46. ads/aqua/extension/model_handler.py +0 -390
  47. ads/aqua/extension/models/__init__.py +0 -0
  48. ads/aqua/extension/models/ws_models.py +0 -145
  49. ads/aqua/extension/models_ws_msg_handler.py +0 -50
  50. ads/aqua/extension/ui_handler.py +0 -282
  51. ads/aqua/extension/ui_websocket_handler.py +0 -130
  52. ads/aqua/extension/utils.py +0 -133
  53. ads/aqua/finetuning/__init__.py +0 -7
  54. ads/aqua/finetuning/constants.py +0 -23
  55. ads/aqua/finetuning/entities.py +0 -181
  56. ads/aqua/finetuning/finetuning.py +0 -731
  57. ads/aqua/model/__init__.py +0 -8
  58. ads/aqua/model/constants.py +0 -60
  59. ads/aqua/model/entities.py +0 -306
  60. ads/aqua/model/enums.py +0 -30
  61. ads/aqua/model/model.py +0 -2079
  62. ads/aqua/modeldeployment/__init__.py +0 -8
  63. ads/aqua/modeldeployment/constants.py +0 -10
  64. ads/aqua/modeldeployment/deployment.py +0 -1324
  65. ads/aqua/modeldeployment/entities.py +0 -653
  66. ads/aqua/modeldeployment/inference.py +0 -74
  67. ads/aqua/modeldeployment/utils.py +0 -543
  68. ads/aqua/resources/gpu_shapes_index.json +0 -94
  69. ads/aqua/server/__init__.py +0 -4
  70. ads/aqua/server/__main__.py +0 -24
  71. ads/aqua/server/app.py +0 -47
  72. ads/aqua/server/aqua_spec.yml +0 -1291
  73. ads/aqua/training/__init__.py +0 -4
  74. ads/aqua/training/exceptions.py +0 -476
  75. ads/aqua/ui.py +0 -499
  76. ads/automl/__init__.py +0 -9
  77. ads/automl/driver.py +0 -330
  78. ads/automl/provider.py +0 -975
  79. ads/bds/__init__.py +0 -5
  80. ads/bds/auth.py +0 -127
  81. ads/bds/big_data_service.py +0 -255
  82. ads/catalog/__init__.py +0 -19
  83. ads/catalog/model.py +0 -1576
  84. ads/catalog/notebook.py +0 -461
  85. ads/catalog/project.py +0 -468
  86. ads/catalog/summary.py +0 -178
  87. ads/common/__init__.py +0 -11
  88. ads/common/analyzer.py +0 -65
  89. ads/common/artifact/.model-ignore +0 -63
  90. ads/common/artifact/__init__.py +0 -10
  91. ads/common/auth.py +0 -1122
  92. ads/common/card_identifier.py +0 -83
  93. ads/common/config.py +0 -647
  94. ads/common/data.py +0 -165
  95. ads/common/decorator/__init__.py +0 -9
  96. ads/common/decorator/argument_to_case.py +0 -88
  97. ads/common/decorator/deprecate.py +0 -69
  98. ads/common/decorator/require_nonempty_arg.py +0 -65
  99. ads/common/decorator/runtime_dependency.py +0 -178
  100. ads/common/decorator/threaded.py +0 -97
  101. ads/common/decorator/utils.py +0 -35
  102. ads/common/dsc_file_system.py +0 -303
  103. ads/common/error.py +0 -14
  104. ads/common/extended_enum.py +0 -81
  105. ads/common/function/__init__.py +0 -5
  106. ads/common/function/fn_util.py +0 -142
  107. ads/common/function/func_conf.yaml +0 -25
  108. ads/common/ipython.py +0 -76
  109. ads/common/model.py +0 -679
  110. ads/common/model_artifact.py +0 -1759
  111. ads/common/model_artifact_schema.json +0 -107
  112. ads/common/model_export_util.py +0 -664
  113. ads/common/model_metadata.py +0 -24
  114. ads/common/object_storage_details.py +0 -296
  115. ads/common/oci_client.py +0 -175
  116. ads/common/oci_datascience.py +0 -46
  117. ads/common/oci_logging.py +0 -1144
  118. ads/common/oci_mixin.py +0 -957
  119. ads/common/oci_resource.py +0 -136
  120. ads/common/serializer.py +0 -559
  121. ads/common/utils.py +0 -1852
  122. ads/common/word_lists.py +0 -1491
  123. ads/common/work_request.py +0 -189
  124. ads/data_labeling/__init__.py +0 -13
  125. ads/data_labeling/boundingbox.py +0 -253
  126. ads/data_labeling/constants.py +0 -47
  127. ads/data_labeling/data_labeling_service.py +0 -244
  128. ads/data_labeling/interface/__init__.py +0 -5
  129. ads/data_labeling/interface/loader.py +0 -16
  130. ads/data_labeling/interface/parser.py +0 -16
  131. ads/data_labeling/interface/reader.py +0 -23
  132. ads/data_labeling/loader/__init__.py +0 -5
  133. ads/data_labeling/loader/file_loader.py +0 -241
  134. ads/data_labeling/metadata.py +0 -110
  135. ads/data_labeling/mixin/__init__.py +0 -5
  136. ads/data_labeling/mixin/data_labeling.py +0 -232
  137. ads/data_labeling/ner.py +0 -129
  138. ads/data_labeling/parser/__init__.py +0 -5
  139. ads/data_labeling/parser/dls_record_parser.py +0 -388
  140. ads/data_labeling/parser/export_metadata_parser.py +0 -94
  141. ads/data_labeling/parser/export_record_parser.py +0 -473
  142. ads/data_labeling/reader/__init__.py +0 -5
  143. ads/data_labeling/reader/dataset_reader.py +0 -574
  144. ads/data_labeling/reader/dls_record_reader.py +0 -121
  145. ads/data_labeling/reader/export_record_reader.py +0 -62
  146. ads/data_labeling/reader/jsonl_reader.py +0 -75
  147. ads/data_labeling/reader/metadata_reader.py +0 -203
  148. ads/data_labeling/reader/record_reader.py +0 -263
  149. ads/data_labeling/record.py +0 -52
  150. ads/data_labeling/visualizer/__init__.py +0 -5
  151. ads/data_labeling/visualizer/image_visualizer.py +0 -525
  152. ads/data_labeling/visualizer/text_visualizer.py +0 -357
  153. ads/database/__init__.py +0 -5
  154. ads/database/connection.py +0 -338
  155. ads/dataset/__init__.py +0 -10
  156. ads/dataset/capabilities.md +0 -51
  157. ads/dataset/classification_dataset.py +0 -339
  158. ads/dataset/correlation.py +0 -226
  159. ads/dataset/correlation_plot.py +0 -563
  160. ads/dataset/dask_series.py +0 -173
  161. ads/dataset/dataframe_transformer.py +0 -110
  162. ads/dataset/dataset.py +0 -1979
  163. ads/dataset/dataset_browser.py +0 -360
  164. ads/dataset/dataset_with_target.py +0 -995
  165. ads/dataset/exception.py +0 -25
  166. ads/dataset/factory.py +0 -987
  167. ads/dataset/feature_engineering_transformer.py +0 -35
  168. ads/dataset/feature_selection.py +0 -107
  169. ads/dataset/forecasting_dataset.py +0 -26
  170. ads/dataset/helper.py +0 -1450
  171. ads/dataset/label_encoder.py +0 -99
  172. ads/dataset/mixin/__init__.py +0 -5
  173. ads/dataset/mixin/dataset_accessor.py +0 -134
  174. ads/dataset/pipeline.py +0 -58
  175. ads/dataset/plot.py +0 -710
  176. ads/dataset/progress.py +0 -86
  177. ads/dataset/recommendation.py +0 -297
  178. ads/dataset/recommendation_transformer.py +0 -502
  179. ads/dataset/regression_dataset.py +0 -14
  180. ads/dataset/sampled_dataset.py +0 -1050
  181. ads/dataset/target.py +0 -98
  182. ads/dataset/timeseries.py +0 -18
  183. ads/dbmixin/__init__.py +0 -5
  184. ads/dbmixin/db_pandas_accessor.py +0 -153
  185. ads/environment/__init__.py +0 -9
  186. ads/environment/ml_runtime.py +0 -66
  187. ads/evaluations/README.md +0 -14
  188. ads/evaluations/__init__.py +0 -109
  189. ads/evaluations/evaluation_plot.py +0 -983
  190. ads/evaluations/evaluator.py +0 -1334
  191. ads/evaluations/statistical_metrics.py +0 -543
  192. ads/experiments/__init__.py +0 -9
  193. ads/experiments/capabilities.md +0 -0
  194. ads/explanations/__init__.py +0 -21
  195. ads/explanations/base_explainer.py +0 -142
  196. ads/explanations/capabilities.md +0 -83
  197. ads/explanations/explainer.py +0 -190
  198. ads/explanations/mlx_global_explainer.py +0 -1050
  199. ads/explanations/mlx_interface.py +0 -386
  200. ads/explanations/mlx_local_explainer.py +0 -287
  201. ads/explanations/mlx_whatif_explainer.py +0 -201
  202. ads/feature_engineering/__init__.py +0 -20
  203. ads/feature_engineering/accessor/__init__.py +0 -5
  204. ads/feature_engineering/accessor/dataframe_accessor.py +0 -535
  205. ads/feature_engineering/accessor/mixin/__init__.py +0 -5
  206. ads/feature_engineering/accessor/mixin/correlation.py +0 -166
  207. ads/feature_engineering/accessor/mixin/eda_mixin.py +0 -266
  208. ads/feature_engineering/accessor/mixin/eda_mixin_series.py +0 -85
  209. ads/feature_engineering/accessor/mixin/feature_types_mixin.py +0 -211
  210. ads/feature_engineering/accessor/mixin/utils.py +0 -65
  211. ads/feature_engineering/accessor/series_accessor.py +0 -431
  212. ads/feature_engineering/adsimage/__init__.py +0 -5
  213. ads/feature_engineering/adsimage/image.py +0 -192
  214. ads/feature_engineering/adsimage/image_reader.py +0 -170
  215. ads/feature_engineering/adsimage/interface/__init__.py +0 -5
  216. ads/feature_engineering/adsimage/interface/reader.py +0 -19
  217. ads/feature_engineering/adsstring/__init__.py +0 -7
  218. ads/feature_engineering/adsstring/oci_language/__init__.py +0 -8
  219. ads/feature_engineering/adsstring/string/__init__.py +0 -8
  220. ads/feature_engineering/data_schema.json +0 -57
  221. ads/feature_engineering/dataset/__init__.py +0 -5
  222. ads/feature_engineering/dataset/zip_code_data.py +0 -42062
  223. ads/feature_engineering/exceptions.py +0 -40
  224. ads/feature_engineering/feature_type/__init__.py +0 -133
  225. ads/feature_engineering/feature_type/address.py +0 -184
  226. ads/feature_engineering/feature_type/adsstring/__init__.py +0 -5
  227. ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +0 -164
  228. ads/feature_engineering/feature_type/adsstring/oci_language.py +0 -93
  229. ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +0 -5
  230. ads/feature_engineering/feature_type/adsstring/parsers/base.py +0 -47
  231. ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +0 -96
  232. ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +0 -221
  233. ads/feature_engineering/feature_type/adsstring/string.py +0 -258
  234. ads/feature_engineering/feature_type/base.py +0 -58
  235. ads/feature_engineering/feature_type/boolean.py +0 -183
  236. ads/feature_engineering/feature_type/category.py +0 -146
  237. ads/feature_engineering/feature_type/constant.py +0 -137
  238. ads/feature_engineering/feature_type/continuous.py +0 -151
  239. ads/feature_engineering/feature_type/creditcard.py +0 -314
  240. ads/feature_engineering/feature_type/datetime.py +0 -190
  241. ads/feature_engineering/feature_type/discrete.py +0 -134
  242. ads/feature_engineering/feature_type/document.py +0 -43
  243. ads/feature_engineering/feature_type/gis.py +0 -251
  244. ads/feature_engineering/feature_type/handler/__init__.py +0 -5
  245. ads/feature_engineering/feature_type/handler/feature_validator.py +0 -524
  246. ads/feature_engineering/feature_type/handler/feature_warning.py +0 -319
  247. ads/feature_engineering/feature_type/handler/warnings.py +0 -128
  248. ads/feature_engineering/feature_type/integer.py +0 -142
  249. ads/feature_engineering/feature_type/ip_address.py +0 -144
  250. ads/feature_engineering/feature_type/ip_address_v4.py +0 -138
  251. ads/feature_engineering/feature_type/ip_address_v6.py +0 -138
  252. ads/feature_engineering/feature_type/lat_long.py +0 -256
  253. ads/feature_engineering/feature_type/object.py +0 -43
  254. ads/feature_engineering/feature_type/ordinal.py +0 -132
  255. ads/feature_engineering/feature_type/phone_number.py +0 -135
  256. ads/feature_engineering/feature_type/string.py +0 -171
  257. ads/feature_engineering/feature_type/text.py +0 -93
  258. ads/feature_engineering/feature_type/unknown.py +0 -43
  259. ads/feature_engineering/feature_type/zip_code.py +0 -164
  260. ads/feature_engineering/feature_type_manager.py +0 -406
  261. ads/feature_engineering/schema.py +0 -795
  262. ads/feature_engineering/utils.py +0 -245
  263. ads/feature_store/.readthedocs.yaml +0 -19
  264. ads/feature_store/README.md +0 -65
  265. ads/feature_store/__init__.py +0 -9
  266. ads/feature_store/common/__init__.py +0 -0
  267. ads/feature_store/common/enums.py +0 -339
  268. ads/feature_store/common/exceptions.py +0 -18
  269. ads/feature_store/common/spark_session_singleton.py +0 -125
  270. ads/feature_store/common/utils/__init__.py +0 -0
  271. ads/feature_store/common/utils/base64_encoder_decoder.py +0 -72
  272. ads/feature_store/common/utils/feature_schema_mapper.py +0 -283
  273. ads/feature_store/common/utils/transformation_utils.py +0 -82
  274. ads/feature_store/common/utils/utility.py +0 -403
  275. ads/feature_store/data_validation/__init__.py +0 -0
  276. ads/feature_store/data_validation/great_expectation.py +0 -129
  277. ads/feature_store/dataset.py +0 -1230
  278. ads/feature_store/dataset_job.py +0 -530
  279. ads/feature_store/docs/Dockerfile +0 -7
  280. ads/feature_store/docs/Makefile +0 -44
  281. ads/feature_store/docs/conf.py +0 -28
  282. ads/feature_store/docs/requirements.txt +0 -14
  283. ads/feature_store/docs/source/ads.feature_store.query.rst +0 -20
  284. ads/feature_store/docs/source/cicd.rst +0 -137
  285. ads/feature_store/docs/source/conf.py +0 -86
  286. ads/feature_store/docs/source/data_versioning.rst +0 -33
  287. ads/feature_store/docs/source/dataset.rst +0 -388
  288. ads/feature_store/docs/source/dataset_job.rst +0 -27
  289. ads/feature_store/docs/source/demo.rst +0 -70
  290. ads/feature_store/docs/source/entity.rst +0 -78
  291. ads/feature_store/docs/source/feature_group.rst +0 -624
  292. ads/feature_store/docs/source/feature_group_job.rst +0 -29
  293. ads/feature_store/docs/source/feature_store.rst +0 -122
  294. ads/feature_store/docs/source/feature_store_class.rst +0 -123
  295. ads/feature_store/docs/source/feature_validation.rst +0 -66
  296. ads/feature_store/docs/source/figures/cicd.png +0 -0
  297. ads/feature_store/docs/source/figures/data_validation.png +0 -0
  298. ads/feature_store/docs/source/figures/data_versioning.png +0 -0
  299. ads/feature_store/docs/source/figures/dataset.gif +0 -0
  300. ads/feature_store/docs/source/figures/dataset.png +0 -0
  301. ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
  302. ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
  303. ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
  304. ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
  305. ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
  306. ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
  307. ads/feature_store/docs/source/figures/entity.png +0 -0
  308. ads/feature_store/docs/source/figures/feature_group.png +0 -0
  309. ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
  310. ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
  311. ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
  312. ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
  313. ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
  314. ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
  315. ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
  316. ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
  317. ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
  318. ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
  319. ads/feature_store/docs/source/figures/overview.png +0 -0
  320. ads/feature_store/docs/source/figures/resource_manager.png +0 -0
  321. ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
  322. ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
  323. ads/feature_store/docs/source/figures/stats_1.png +0 -0
  324. ads/feature_store/docs/source/figures/stats_2.png +0 -0
  325. ads/feature_store/docs/source/figures/stats_d.png +0 -0
  326. ads/feature_store/docs/source/figures/stats_fg.png +0 -0
  327. ads/feature_store/docs/source/figures/transformation.png +0 -0
  328. ads/feature_store/docs/source/figures/transformations.gif +0 -0
  329. ads/feature_store/docs/source/figures/validation.png +0 -0
  330. ads/feature_store/docs/source/figures/validation_fg.png +0 -0
  331. ads/feature_store/docs/source/figures/validation_results.png +0 -0
  332. ads/feature_store/docs/source/figures/validation_summary.png +0 -0
  333. ads/feature_store/docs/source/index.rst +0 -81
  334. ads/feature_store/docs/source/module.rst +0 -8
  335. ads/feature_store/docs/source/notebook.rst +0 -94
  336. ads/feature_store/docs/source/overview.rst +0 -47
  337. ads/feature_store/docs/source/quickstart.rst +0 -176
  338. ads/feature_store/docs/source/release_notes.rst +0 -194
  339. ads/feature_store/docs/source/setup_feature_store.rst +0 -81
  340. ads/feature_store/docs/source/statistics.rst +0 -58
  341. ads/feature_store/docs/source/transformation.rst +0 -199
  342. ads/feature_store/docs/source/ui.rst +0 -65
  343. ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +0 -66
  344. ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +0 -192
  345. ads/feature_store/docs/source/user_guides.setup.terraform.rst +0 -338
  346. ads/feature_store/entity.py +0 -718
  347. ads/feature_store/execution_strategy/__init__.py +0 -0
  348. ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
  349. ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +0 -375
  350. ads/feature_store/execution_strategy/engine/__init__.py +0 -0
  351. ads/feature_store/execution_strategy/engine/spark_engine.py +0 -316
  352. ads/feature_store/execution_strategy/execution_strategy.py +0 -113
  353. ads/feature_store/execution_strategy/execution_strategy_provider.py +0 -47
  354. ads/feature_store/execution_strategy/spark/__init__.py +0 -0
  355. ads/feature_store/execution_strategy/spark/spark_execution.py +0 -618
  356. ads/feature_store/feature.py +0 -192
  357. ads/feature_store/feature_group.py +0 -1494
  358. ads/feature_store/feature_group_expectation.py +0 -346
  359. ads/feature_store/feature_group_job.py +0 -602
  360. ads/feature_store/feature_lineage/__init__.py +0 -0
  361. ads/feature_store/feature_lineage/graphviz_service.py +0 -180
  362. ads/feature_store/feature_option_details.py +0 -50
  363. ads/feature_store/feature_statistics/__init__.py +0 -0
  364. ads/feature_store/feature_statistics/statistics_service.py +0 -99
  365. ads/feature_store/feature_store.py +0 -699
  366. ads/feature_store/feature_store_registrar.py +0 -518
  367. ads/feature_store/input_feature_detail.py +0 -149
  368. ads/feature_store/mixin/__init__.py +0 -4
  369. ads/feature_store/mixin/oci_feature_store.py +0 -145
  370. ads/feature_store/model_details.py +0 -73
  371. ads/feature_store/query/__init__.py +0 -0
  372. ads/feature_store/query/filter.py +0 -266
  373. ads/feature_store/query/generator/__init__.py +0 -0
  374. ads/feature_store/query/generator/query_generator.py +0 -298
  375. ads/feature_store/query/join.py +0 -161
  376. ads/feature_store/query/query.py +0 -403
  377. ads/feature_store/query/validator/__init__.py +0 -0
  378. ads/feature_store/query/validator/query_validator.py +0 -57
  379. ads/feature_store/response/__init__.py +0 -0
  380. ads/feature_store/response/response_builder.py +0 -68
  381. ads/feature_store/service/__init__.py +0 -0
  382. ads/feature_store/service/oci_dataset.py +0 -139
  383. ads/feature_store/service/oci_dataset_job.py +0 -199
  384. ads/feature_store/service/oci_entity.py +0 -125
  385. ads/feature_store/service/oci_feature_group.py +0 -164
  386. ads/feature_store/service/oci_feature_group_job.py +0 -214
  387. ads/feature_store/service/oci_feature_store.py +0 -182
  388. ads/feature_store/service/oci_lineage.py +0 -87
  389. ads/feature_store/service/oci_transformation.py +0 -104
  390. ads/feature_store/statistics/__init__.py +0 -0
  391. ads/feature_store/statistics/abs_feature_value.py +0 -49
  392. ads/feature_store/statistics/charts/__init__.py +0 -0
  393. ads/feature_store/statistics/charts/abstract_feature_plot.py +0 -37
  394. ads/feature_store/statistics/charts/box_plot.py +0 -148
  395. ads/feature_store/statistics/charts/frequency_distribution.py +0 -65
  396. ads/feature_store/statistics/charts/probability_distribution.py +0 -68
  397. ads/feature_store/statistics/charts/top_k_frequent_elements.py +0 -98
  398. ads/feature_store/statistics/feature_stat.py +0 -126
  399. ads/feature_store/statistics/generic_feature_value.py +0 -33
  400. ads/feature_store/statistics/statistics.py +0 -41
  401. ads/feature_store/statistics_config.py +0 -101
  402. ads/feature_store/templates/feature_store_template.yaml +0 -45
  403. ads/feature_store/transformation.py +0 -499
  404. ads/feature_store/validation_output.py +0 -57
  405. ads/hpo/__init__.py +0 -9
  406. ads/hpo/_imports.py +0 -91
  407. ads/hpo/ads_search_space.py +0 -439
  408. ads/hpo/distributions.py +0 -325
  409. ads/hpo/objective.py +0 -280
  410. ads/hpo/search_cv.py +0 -1657
  411. ads/hpo/stopping_criterion.py +0 -75
  412. ads/hpo/tuner_artifact.py +0 -413
  413. ads/hpo/utils.py +0 -91
  414. ads/hpo/validation.py +0 -140
  415. ads/hpo/visualization/__init__.py +0 -5
  416. ads/hpo/visualization/_contour.py +0 -23
  417. ads/hpo/visualization/_edf.py +0 -20
  418. ads/hpo/visualization/_intermediate_values.py +0 -21
  419. ads/hpo/visualization/_optimization_history.py +0 -25
  420. ads/hpo/visualization/_parallel_coordinate.py +0 -169
  421. ads/hpo/visualization/_param_importances.py +0 -26
  422. ads/jobs/__init__.py +0 -53
  423. ads/jobs/ads_job.py +0 -663
  424. ads/jobs/builders/__init__.py +0 -5
  425. ads/jobs/builders/base.py +0 -156
  426. ads/jobs/builders/infrastructure/__init__.py +0 -6
  427. ads/jobs/builders/infrastructure/base.py +0 -165
  428. ads/jobs/builders/infrastructure/dataflow.py +0 -1252
  429. ads/jobs/builders/infrastructure/dsc_job.py +0 -1894
  430. ads/jobs/builders/infrastructure/dsc_job_runtime.py +0 -1233
  431. ads/jobs/builders/infrastructure/utils.py +0 -65
  432. ads/jobs/builders/runtimes/__init__.py +0 -5
  433. ads/jobs/builders/runtimes/artifact.py +0 -338
  434. ads/jobs/builders/runtimes/base.py +0 -325
  435. ads/jobs/builders/runtimes/container_runtime.py +0 -242
  436. ads/jobs/builders/runtimes/python_runtime.py +0 -1016
  437. ads/jobs/builders/runtimes/pytorch_runtime.py +0 -204
  438. ads/jobs/cli.py +0 -104
  439. ads/jobs/env_var_parser.py +0 -131
  440. ads/jobs/extension.py +0 -160
  441. ads/jobs/schema/__init__.py +0 -5
  442. ads/jobs/schema/infrastructure_schema.json +0 -116
  443. ads/jobs/schema/job_schema.json +0 -42
  444. ads/jobs/schema/runtime_schema.json +0 -183
  445. ads/jobs/schema/validator.py +0 -141
  446. ads/jobs/serializer.py +0 -296
  447. ads/jobs/templates/__init__.py +0 -5
  448. ads/jobs/templates/container.py +0 -6
  449. ads/jobs/templates/driver_notebook.py +0 -177
  450. ads/jobs/templates/driver_oci.py +0 -500
  451. ads/jobs/templates/driver_python.py +0 -48
  452. ads/jobs/templates/driver_pytorch.py +0 -852
  453. ads/jobs/templates/driver_utils.py +0 -615
  454. ads/jobs/templates/hostname_from_env.c +0 -55
  455. ads/jobs/templates/oci_metrics.py +0 -181
  456. ads/jobs/utils.py +0 -104
  457. ads/llm/__init__.py +0 -28
  458. ads/llm/autogen/__init__.py +0 -2
  459. ads/llm/autogen/constants.py +0 -15
  460. ads/llm/autogen/reports/__init__.py +0 -2
  461. ads/llm/autogen/reports/base.py +0 -67
  462. ads/llm/autogen/reports/data.py +0 -103
  463. ads/llm/autogen/reports/session.py +0 -526
  464. ads/llm/autogen/reports/templates/chat_box.html +0 -13
  465. ads/llm/autogen/reports/templates/chat_box_lt.html +0 -5
  466. ads/llm/autogen/reports/templates/chat_box_rt.html +0 -6
  467. ads/llm/autogen/reports/utils.py +0 -56
  468. ads/llm/autogen/v02/__init__.py +0 -4
  469. ads/llm/autogen/v02/client.py +0 -295
  470. ads/llm/autogen/v02/log_handlers/__init__.py +0 -2
  471. ads/llm/autogen/v02/log_handlers/oci_file_handler.py +0 -83
  472. ads/llm/autogen/v02/loggers/__init__.py +0 -6
  473. ads/llm/autogen/v02/loggers/metric_logger.py +0 -320
  474. ads/llm/autogen/v02/loggers/session_logger.py +0 -580
  475. ads/llm/autogen/v02/loggers/utils.py +0 -86
  476. ads/llm/autogen/v02/runtime_logging.py +0 -163
  477. ads/llm/chain.py +0 -268
  478. ads/llm/chat_template.py +0 -31
  479. ads/llm/deploy.py +0 -63
  480. ads/llm/guardrails/__init__.py +0 -5
  481. ads/llm/guardrails/base.py +0 -442
  482. ads/llm/guardrails/huggingface.py +0 -44
  483. ads/llm/langchain/__init__.py +0 -5
  484. ads/llm/langchain/plugins/__init__.py +0 -5
  485. ads/llm/langchain/plugins/chat_models/__init__.py +0 -5
  486. ads/llm/langchain/plugins/chat_models/oci_data_science.py +0 -1027
  487. ads/llm/langchain/plugins/embeddings/__init__.py +0 -4
  488. ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +0 -184
  489. ads/llm/langchain/plugins/llms/__init__.py +0 -5
  490. ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +0 -979
  491. ads/llm/requirements.txt +0 -3
  492. ads/llm/serialize.py +0 -219
  493. ads/llm/serializers/__init__.py +0 -0
  494. ads/llm/serializers/retrieval_qa.py +0 -153
  495. ads/llm/serializers/runnable_parallel.py +0 -27
  496. ads/llm/templates/score_chain.jinja2 +0 -155
  497. ads/llm/templates/tool_chat_template_hermes.jinja +0 -130
  498. ads/llm/templates/tool_chat_template_mistral_parallel.jinja +0 -94
  499. ads/model/__init__.py +0 -52
  500. ads/model/artifact.py +0 -573
  501. ads/model/artifact_downloader.py +0 -254
  502. ads/model/artifact_uploader.py +0 -267
  503. ads/model/base_properties.py +0 -238
  504. ads/model/common/.model-ignore +0 -66
  505. ads/model/common/__init__.py +0 -5
  506. ads/model/common/utils.py +0 -142
  507. ads/model/datascience_model.py +0 -2635
  508. ads/model/deployment/__init__.py +0 -20
  509. ads/model/deployment/common/__init__.py +0 -5
  510. ads/model/deployment/common/utils.py +0 -308
  511. ads/model/deployment/model_deployer.py +0 -466
  512. ads/model/deployment/model_deployment.py +0 -1846
  513. ads/model/deployment/model_deployment_infrastructure.py +0 -671
  514. ads/model/deployment/model_deployment_properties.py +0 -493
  515. ads/model/deployment/model_deployment_runtime.py +0 -838
  516. ads/model/extractor/__init__.py +0 -5
  517. ads/model/extractor/automl_extractor.py +0 -74
  518. ads/model/extractor/embedding_onnx_extractor.py +0 -80
  519. ads/model/extractor/huggingface_extractor.py +0 -88
  520. ads/model/extractor/keras_extractor.py +0 -84
  521. ads/model/extractor/lightgbm_extractor.py +0 -93
  522. ads/model/extractor/model_info_extractor.py +0 -114
  523. ads/model/extractor/model_info_extractor_factory.py +0 -105
  524. ads/model/extractor/pytorch_extractor.py +0 -87
  525. ads/model/extractor/sklearn_extractor.py +0 -112
  526. ads/model/extractor/spark_extractor.py +0 -89
  527. ads/model/extractor/tensorflow_extractor.py +0 -85
  528. ads/model/extractor/xgboost_extractor.py +0 -94
  529. ads/model/framework/__init__.py +0 -5
  530. ads/model/framework/automl_model.py +0 -178
  531. ads/model/framework/embedding_onnx_model.py +0 -438
  532. ads/model/framework/huggingface_model.py +0 -399
  533. ads/model/framework/lightgbm_model.py +0 -266
  534. ads/model/framework/pytorch_model.py +0 -266
  535. ads/model/framework/sklearn_model.py +0 -250
  536. ads/model/framework/spark_model.py +0 -326
  537. ads/model/framework/tensorflow_model.py +0 -254
  538. ads/model/framework/xgboost_model.py +0 -258
  539. ads/model/generic_model.py +0 -3518
  540. ads/model/model_artifact_boilerplate/README.md +0 -381
  541. ads/model/model_artifact_boilerplate/__init__.py +0 -5
  542. ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +0 -5
  543. ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +0 -427
  544. ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +0 -2
  545. ads/model/model_artifact_boilerplate/runtime.yaml +0 -7
  546. ads/model/model_artifact_boilerplate/score.py +0 -61
  547. ads/model/model_file_description_schema.json +0 -68
  548. ads/model/model_introspect.py +0 -331
  549. ads/model/model_metadata.py +0 -1810
  550. ads/model/model_metadata_mixin.py +0 -460
  551. ads/model/model_properties.py +0 -63
  552. ads/model/model_version_set.py +0 -739
  553. ads/model/runtime/__init__.py +0 -5
  554. ads/model/runtime/env_info.py +0 -306
  555. ads/model/runtime/model_deployment_details.py +0 -37
  556. ads/model/runtime/model_provenance_details.py +0 -58
  557. ads/model/runtime/runtime_info.py +0 -81
  558. ads/model/runtime/schemas/inference_env_info_schema.yaml +0 -16
  559. ads/model/runtime/schemas/model_provenance_schema.yaml +0 -36
  560. ads/model/runtime/schemas/training_env_info_schema.yaml +0 -16
  561. ads/model/runtime/utils.py +0 -201
  562. ads/model/serde/__init__.py +0 -5
  563. ads/model/serde/common.py +0 -40
  564. ads/model/serde/model_input.py +0 -547
  565. ads/model/serde/model_serializer.py +0 -1184
  566. ads/model/service/__init__.py +0 -5
  567. ads/model/service/oci_datascience_model.py +0 -1076
  568. ads/model/service/oci_datascience_model_deployment.py +0 -500
  569. ads/model/service/oci_datascience_model_version_set.py +0 -176
  570. ads/model/transformer/__init__.py +0 -5
  571. ads/model/transformer/onnx_transformer.py +0 -324
  572. ads/mysqldb/__init__.py +0 -5
  573. ads/mysqldb/mysql_db.py +0 -227
  574. ads/opctl/__init__.py +0 -18
  575. ads/opctl/anomaly_detection.py +0 -11
  576. ads/opctl/backend/__init__.py +0 -5
  577. ads/opctl/backend/ads_dataflow.py +0 -353
  578. ads/opctl/backend/ads_ml_job.py +0 -710
  579. ads/opctl/backend/ads_ml_pipeline.py +0 -164
  580. ads/opctl/backend/ads_model_deployment.py +0 -209
  581. ads/opctl/backend/base.py +0 -146
  582. ads/opctl/backend/local.py +0 -1053
  583. ads/opctl/backend/marketplace/__init__.py +0 -9
  584. ads/opctl/backend/marketplace/helm_helper.py +0 -173
  585. ads/opctl/backend/marketplace/local_marketplace.py +0 -271
  586. ads/opctl/backend/marketplace/marketplace_backend_runner.py +0 -71
  587. ads/opctl/backend/marketplace/marketplace_operator_interface.py +0 -44
  588. ads/opctl/backend/marketplace/marketplace_operator_runner.py +0 -24
  589. ads/opctl/backend/marketplace/marketplace_utils.py +0 -212
  590. ads/opctl/backend/marketplace/models/__init__.py +0 -5
  591. ads/opctl/backend/marketplace/models/bearer_token.py +0 -94
  592. ads/opctl/backend/marketplace/models/marketplace_type.py +0 -70
  593. ads/opctl/backend/marketplace/models/ocir_details.py +0 -56
  594. ads/opctl/backend/marketplace/prerequisite_checker.py +0 -238
  595. ads/opctl/cli.py +0 -707
  596. ads/opctl/cmds.py +0 -869
  597. ads/opctl/conda/__init__.py +0 -5
  598. ads/opctl/conda/cli.py +0 -193
  599. ads/opctl/conda/cmds.py +0 -749
  600. ads/opctl/conda/config.yaml +0 -34
  601. ads/opctl/conda/manifest_template.yaml +0 -13
  602. ads/opctl/conda/multipart_uploader.py +0 -188
  603. ads/opctl/conda/pack.py +0 -89
  604. ads/opctl/config/__init__.py +0 -5
  605. ads/opctl/config/base.py +0 -57
  606. ads/opctl/config/diagnostics/__init__.py +0 -5
  607. ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +0 -62
  608. ads/opctl/config/merger.py +0 -255
  609. ads/opctl/config/resolver.py +0 -297
  610. ads/opctl/config/utils.py +0 -79
  611. ads/opctl/config/validator.py +0 -17
  612. ads/opctl/config/versioner.py +0 -68
  613. ads/opctl/config/yaml_parsers/__init__.py +0 -7
  614. ads/opctl/config/yaml_parsers/base.py +0 -58
  615. ads/opctl/config/yaml_parsers/distributed/__init__.py +0 -7
  616. ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +0 -201
  617. ads/opctl/constants.py +0 -66
  618. ads/opctl/decorator/__init__.py +0 -5
  619. ads/opctl/decorator/common.py +0 -129
  620. ads/opctl/diagnostics/__init__.py +0 -5
  621. ads/opctl/diagnostics/__main__.py +0 -25
  622. ads/opctl/diagnostics/check_distributed_job_requirements.py +0 -212
  623. ads/opctl/diagnostics/check_requirements.py +0 -144
  624. ads/opctl/diagnostics/requirement_exception.py +0 -9
  625. ads/opctl/distributed/README.md +0 -109
  626. ads/opctl/distributed/__init__.py +0 -5
  627. ads/opctl/distributed/certificates.py +0 -32
  628. ads/opctl/distributed/cli.py +0 -207
  629. ads/opctl/distributed/cmds.py +0 -731
  630. ads/opctl/distributed/common/__init__.py +0 -5
  631. ads/opctl/distributed/common/abstract_cluster_provider.py +0 -449
  632. ads/opctl/distributed/common/abstract_framework_spec_builder.py +0 -88
  633. ads/opctl/distributed/common/cluster_config_helper.py +0 -103
  634. ads/opctl/distributed/common/cluster_provider_factory.py +0 -21
  635. ads/opctl/distributed/common/cluster_runner.py +0 -54
  636. ads/opctl/distributed/common/framework_factory.py +0 -29
  637. ads/opctl/docker/Dockerfile.job +0 -103
  638. ads/opctl/docker/Dockerfile.job.arm +0 -107
  639. ads/opctl/docker/Dockerfile.job.gpu +0 -175
  640. ads/opctl/docker/base-env.yaml +0 -13
  641. ads/opctl/docker/cuda.repo +0 -6
  642. ads/opctl/docker/operator/.dockerignore +0 -0
  643. ads/opctl/docker/operator/Dockerfile +0 -41
  644. ads/opctl/docker/operator/Dockerfile.gpu +0 -85
  645. ads/opctl/docker/operator/cuda.repo +0 -6
  646. ads/opctl/docker/operator/environment.yaml +0 -8
  647. ads/opctl/forecast.py +0 -11
  648. ads/opctl/index.yaml +0 -3
  649. ads/opctl/model/__init__.py +0 -5
  650. ads/opctl/model/cli.py +0 -65
  651. ads/opctl/model/cmds.py +0 -73
  652. ads/opctl/operator/README.md +0 -4
  653. ads/opctl/operator/__init__.py +0 -31
  654. ads/opctl/operator/cli.py +0 -344
  655. ads/opctl/operator/cmd.py +0 -596
  656. ads/opctl/operator/common/__init__.py +0 -5
  657. ads/opctl/operator/common/backend_factory.py +0 -460
  658. ads/opctl/operator/common/const.py +0 -27
  659. ads/opctl/operator/common/data/synthetic.csv +0 -16001
  660. ads/opctl/operator/common/dictionary_merger.py +0 -148
  661. ads/opctl/operator/common/errors.py +0 -42
  662. ads/opctl/operator/common/operator_config.py +0 -99
  663. ads/opctl/operator/common/operator_loader.py +0 -811
  664. ads/opctl/operator/common/operator_schema.yaml +0 -130
  665. ads/opctl/operator/common/operator_yaml_generator.py +0 -152
  666. ads/opctl/operator/common/utils.py +0 -208
  667. ads/opctl/operator/lowcode/__init__.py +0 -5
  668. ads/opctl/operator/lowcode/anomaly/MLoperator +0 -16
  669. ads/opctl/operator/lowcode/anomaly/README.md +0 -207
  670. ads/opctl/operator/lowcode/anomaly/__init__.py +0 -5
  671. ads/opctl/operator/lowcode/anomaly/__main__.py +0 -103
  672. ads/opctl/operator/lowcode/anomaly/cmd.py +0 -35
  673. ads/opctl/operator/lowcode/anomaly/const.py +0 -167
  674. ads/opctl/operator/lowcode/anomaly/environment.yaml +0 -10
  675. ads/opctl/operator/lowcode/anomaly/model/__init__.py +0 -5
  676. ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +0 -146
  677. ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +0 -162
  678. ads/opctl/operator/lowcode/anomaly/model/automlx.py +0 -99
  679. ads/opctl/operator/lowcode/anomaly/model/autots.py +0 -115
  680. ads/opctl/operator/lowcode/anomaly/model/base_model.py +0 -404
  681. ads/opctl/operator/lowcode/anomaly/model/factory.py +0 -110
  682. ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +0 -78
  683. ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +0 -78
  684. ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +0 -120
  685. ads/opctl/operator/lowcode/anomaly/model/tods.py +0 -119
  686. ads/opctl/operator/lowcode/anomaly/operator_config.py +0 -127
  687. ads/opctl/operator/lowcode/anomaly/schema.yaml +0 -401
  688. ads/opctl/operator/lowcode/anomaly/utils.py +0 -88
  689. ads/opctl/operator/lowcode/common/__init__.py +0 -5
  690. ads/opctl/operator/lowcode/common/const.py +0 -10
  691. ads/opctl/operator/lowcode/common/data.py +0 -116
  692. ads/opctl/operator/lowcode/common/errors.py +0 -47
  693. ads/opctl/operator/lowcode/common/transformations.py +0 -296
  694. ads/opctl/operator/lowcode/common/utils.py +0 -293
  695. ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +0 -13
  696. ads/opctl/operator/lowcode/feature_store_marketplace/README.md +0 -30
  697. ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +0 -5
  698. ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +0 -116
  699. ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +0 -85
  700. ads/opctl/operator/lowcode/feature_store_marketplace/const.py +0 -15
  701. ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
  702. ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +0 -4
  703. ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +0 -32
  704. ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +0 -43
  705. ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +0 -120
  706. ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +0 -34
  707. ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +0 -386
  708. ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +0 -160
  709. ads/opctl/operator/lowcode/forecast/MLoperator +0 -25
  710. ads/opctl/operator/lowcode/forecast/README.md +0 -209
  711. ads/opctl/operator/lowcode/forecast/__init__.py +0 -5
  712. ads/opctl/operator/lowcode/forecast/__main__.py +0 -89
  713. ads/opctl/operator/lowcode/forecast/cmd.py +0 -40
  714. ads/opctl/operator/lowcode/forecast/const.py +0 -92
  715. ads/opctl/operator/lowcode/forecast/environment.yaml +0 -20
  716. ads/opctl/operator/lowcode/forecast/errors.py +0 -26
  717. ads/opctl/operator/lowcode/forecast/model/__init__.py +0 -5
  718. ads/opctl/operator/lowcode/forecast/model/arima.py +0 -279
  719. ads/opctl/operator/lowcode/forecast/model/automlx.py +0 -542
  720. ads/opctl/operator/lowcode/forecast/model/autots.py +0 -312
  721. ads/opctl/operator/lowcode/forecast/model/base_model.py +0 -863
  722. ads/opctl/operator/lowcode/forecast/model/factory.py +0 -106
  723. ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +0 -492
  724. ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +0 -243
  725. ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +0 -486
  726. ads/opctl/operator/lowcode/forecast/model/prophet.py +0 -445
  727. ads/opctl/operator/lowcode/forecast/model_evaluator.py +0 -244
  728. ads/opctl/operator/lowcode/forecast/operator_config.py +0 -234
  729. ads/opctl/operator/lowcode/forecast/schema.yaml +0 -506
  730. ads/opctl/operator/lowcode/forecast/utils.py +0 -413
  731. ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +0 -7
  732. ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +0 -285
  733. ads/opctl/operator/lowcode/forecast/whatifserve/score.py +0 -246
  734. ads/opctl/operator/lowcode/pii/MLoperator +0 -17
  735. ads/opctl/operator/lowcode/pii/README.md +0 -208
  736. ads/opctl/operator/lowcode/pii/__init__.py +0 -5
  737. ads/opctl/operator/lowcode/pii/__main__.py +0 -78
  738. ads/opctl/operator/lowcode/pii/cmd.py +0 -39
  739. ads/opctl/operator/lowcode/pii/constant.py +0 -84
  740. ads/opctl/operator/lowcode/pii/environment.yaml +0 -17
  741. ads/opctl/operator/lowcode/pii/errors.py +0 -27
  742. ads/opctl/operator/lowcode/pii/model/__init__.py +0 -5
  743. ads/opctl/operator/lowcode/pii/model/factory.py +0 -82
  744. ads/opctl/operator/lowcode/pii/model/guardrails.py +0 -167
  745. ads/opctl/operator/lowcode/pii/model/pii.py +0 -145
  746. ads/opctl/operator/lowcode/pii/model/processor/__init__.py +0 -34
  747. ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +0 -34
  748. ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +0 -35
  749. ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +0 -225
  750. ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +0 -73
  751. ads/opctl/operator/lowcode/pii/model/processor/remover.py +0 -26
  752. ads/opctl/operator/lowcode/pii/model/report.py +0 -487
  753. ads/opctl/operator/lowcode/pii/operator_config.py +0 -95
  754. ads/opctl/operator/lowcode/pii/schema.yaml +0 -108
  755. ads/opctl/operator/lowcode/pii/utils.py +0 -43
  756. ads/opctl/operator/lowcode/recommender/MLoperator +0 -16
  757. ads/opctl/operator/lowcode/recommender/README.md +0 -206
  758. ads/opctl/operator/lowcode/recommender/__init__.py +0 -5
  759. ads/opctl/operator/lowcode/recommender/__main__.py +0 -82
  760. ads/opctl/operator/lowcode/recommender/cmd.py +0 -33
  761. ads/opctl/operator/lowcode/recommender/constant.py +0 -30
  762. ads/opctl/operator/lowcode/recommender/environment.yaml +0 -11
  763. ads/opctl/operator/lowcode/recommender/model/base_model.py +0 -212
  764. ads/opctl/operator/lowcode/recommender/model/factory.py +0 -56
  765. ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +0 -25
  766. ads/opctl/operator/lowcode/recommender/model/svd.py +0 -106
  767. ads/opctl/operator/lowcode/recommender/operator_config.py +0 -81
  768. ads/opctl/operator/lowcode/recommender/schema.yaml +0 -265
  769. ads/opctl/operator/lowcode/recommender/utils.py +0 -13
  770. ads/opctl/operator/runtime/__init__.py +0 -5
  771. ads/opctl/operator/runtime/const.py +0 -17
  772. ads/opctl/operator/runtime/container_runtime_schema.yaml +0 -50
  773. ads/opctl/operator/runtime/marketplace_runtime.py +0 -50
  774. ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +0 -21
  775. ads/opctl/operator/runtime/python_runtime_schema.yaml +0 -21
  776. ads/opctl/operator/runtime/runtime.py +0 -115
  777. ads/opctl/schema.yaml.yml +0 -36
  778. ads/opctl/script.py +0 -40
  779. ads/opctl/spark/__init__.py +0 -5
  780. ads/opctl/spark/cli.py +0 -43
  781. ads/opctl/spark/cmds.py +0 -147
  782. ads/opctl/templates/diagnostic_report_template.jinja2 +0 -102
  783. ads/opctl/utils.py +0 -344
  784. ads/oracledb/__init__.py +0 -5
  785. ads/oracledb/oracle_db.py +0 -346
  786. ads/pipeline/__init__.py +0 -39
  787. ads/pipeline/ads_pipeline.py +0 -2279
  788. ads/pipeline/ads_pipeline_run.py +0 -772
  789. ads/pipeline/ads_pipeline_step.py +0 -605
  790. ads/pipeline/builders/__init__.py +0 -5
  791. ads/pipeline/builders/infrastructure/__init__.py +0 -5
  792. ads/pipeline/builders/infrastructure/custom_script.py +0 -32
  793. ads/pipeline/cli.py +0 -119
  794. ads/pipeline/extension.py +0 -291
  795. ads/pipeline/schema/__init__.py +0 -5
  796. ads/pipeline/schema/cs_step_schema.json +0 -35
  797. ads/pipeline/schema/ml_step_schema.json +0 -31
  798. ads/pipeline/schema/pipeline_schema.json +0 -71
  799. ads/pipeline/visualizer/__init__.py +0 -5
  800. ads/pipeline/visualizer/base.py +0 -570
  801. ads/pipeline/visualizer/graph_renderer.py +0 -272
  802. ads/pipeline/visualizer/text_renderer.py +0 -84
  803. ads/secrets/__init__.py +0 -11
  804. ads/secrets/adb.py +0 -386
  805. ads/secrets/auth_token.py +0 -86
  806. ads/secrets/big_data_service.py +0 -365
  807. ads/secrets/mysqldb.py +0 -149
  808. ads/secrets/oracledb.py +0 -160
  809. ads/secrets/secrets.py +0 -407
  810. ads/telemetry/__init__.py +0 -7
  811. ads/telemetry/base.py +0 -69
  812. ads/telemetry/client.py +0 -125
  813. ads/telemetry/telemetry.py +0 -257
  814. ads/templates/dataflow_pyspark.jinja2 +0 -13
  815. ads/templates/dataflow_sparksql.jinja2 +0 -22
  816. ads/templates/func.jinja2 +0 -20
  817. ads/templates/schemas/openapi.json +0 -1740
  818. ads/templates/score-pkl.jinja2 +0 -173
  819. ads/templates/score.jinja2 +0 -322
  820. ads/templates/score_embedding_onnx.jinja2 +0 -202
  821. ads/templates/score_generic.jinja2 +0 -165
  822. ads/templates/score_huggingface_pipeline.jinja2 +0 -217
  823. ads/templates/score_lightgbm.jinja2 +0 -185
  824. ads/templates/score_onnx.jinja2 +0 -407
  825. ads/templates/score_onnx_new.jinja2 +0 -473
  826. ads/templates/score_oracle_automl.jinja2 +0 -185
  827. ads/templates/score_pyspark.jinja2 +0 -154
  828. ads/templates/score_pytorch.jinja2 +0 -219
  829. ads/templates/score_scikit-learn.jinja2 +0 -184
  830. ads/templates/score_tensorflow.jinja2 +0 -184
  831. ads/templates/score_xgboost.jinja2 +0 -178
  832. ads/text_dataset/__init__.py +0 -5
  833. ads/text_dataset/backends.py +0 -211
  834. ads/text_dataset/dataset.py +0 -445
  835. ads/text_dataset/extractor.py +0 -207
  836. ads/text_dataset/options.py +0 -53
  837. ads/text_dataset/udfs.py +0 -22
  838. ads/text_dataset/utils.py +0 -49
  839. ads/type_discovery/__init__.py +0 -9
  840. ads/type_discovery/abstract_detector.py +0 -21
  841. ads/type_discovery/constant_detector.py +0 -41
  842. ads/type_discovery/continuous_detector.py +0 -54
  843. ads/type_discovery/credit_card_detector.py +0 -99
  844. ads/type_discovery/datetime_detector.py +0 -92
  845. ads/type_discovery/discrete_detector.py +0 -118
  846. ads/type_discovery/document_detector.py +0 -146
  847. ads/type_discovery/ip_detector.py +0 -68
  848. ads/type_discovery/latlon_detector.py +0 -90
  849. ads/type_discovery/phone_number_detector.py +0 -63
  850. ads/type_discovery/type_discovery_driver.py +0 -87
  851. ads/type_discovery/typed_feature.py +0 -594
  852. ads/type_discovery/unknown_detector.py +0 -41
  853. ads/type_discovery/zipcode_detector.py +0 -48
  854. ads/vault/__init__.py +0 -7
  855. ads/vault/vault.py +0 -237
  856. oracle_ads-2.13.7.dist-info/RECORD +0 -858
  857. {oracle_ads-2.13.7.dist-info → oracle_ads-2.13.9rc0.dist-info}/licenses/LICENSE.txt +0 -0
ads/aqua/model/model.py DELETED
@@ -1,2079 +0,0 @@
1
- #!/usr/bin/env python
2
- # Copyright (c) 2024, 2025 Oracle and/or its affiliates.
3
- # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
4
- import json
5
- import os
6
- import pathlib
7
- from datetime import datetime, timedelta
8
- from threading import Lock
9
- from typing import Any, Dict, List, Optional, Set, Union
10
-
11
- import oci
12
- from cachetools import TTLCache
13
- from huggingface_hub import snapshot_download
14
- from oci.data_science.models import JobRun, Metadata, Model, UpdateModelDetails
15
-
16
- from ads.aqua import logger
17
- from ads.aqua.app import AquaApp
18
- from ads.aqua.common.entities import AquaMultiModelRef
19
- from ads.aqua.common.enums import (
20
- ConfigFolder,
21
- CustomInferenceContainerTypeFamily,
22
- FineTuningContainerTypeFamily,
23
- InferenceContainerTypeFamily,
24
- ModelFormat,
25
- Platform,
26
- Tags,
27
- )
28
- from ads.aqua.common.errors import (
29
- AquaFileNotFoundError,
30
- AquaRuntimeError,
31
- AquaValueError,
32
- )
33
- from ads.aqua.common.utils import (
34
- LifecycleStatus,
35
- _build_resource_identifier,
36
- cleanup_local_hf_model_artifact,
37
- create_word_icon,
38
- generate_tei_cmd_var,
39
- get_artifact_path,
40
- get_hf_model_info,
41
- get_preferred_compatible_family,
42
- list_os_files_with_extension,
43
- load_config,
44
- upload_folder,
45
- )
46
- from ads.aqua.config.container_config import AquaContainerConfig, Usage
47
- from ads.aqua.constants import (
48
- AQUA_MODEL_ARTIFACT_CONFIG,
49
- AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME,
50
- AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE,
51
- AQUA_MODEL_ARTIFACT_FILE,
52
- AQUA_MODEL_TOKENIZER_CONFIG,
53
- AQUA_MODEL_TYPE_CUSTOM,
54
- HF_METADATA_FOLDER,
55
- LICENSE,
56
- MODEL_BY_REFERENCE_OSS_PATH_KEY,
57
- README,
58
- READY_TO_DEPLOY_STATUS,
59
- READY_TO_FINE_TUNE_STATUS,
60
- READY_TO_IMPORT_STATUS,
61
- TRAINING_METRICS_FINAL,
62
- TRINING_METRICS,
63
- VALIDATION_METRICS,
64
- VALIDATION_METRICS_FINAL,
65
- )
66
- from ads.aqua.model.constants import (
67
- AquaModelMetadataKeys,
68
- FineTuningCustomMetadata,
69
- FineTuningMetricCategories,
70
- ModelCustomMetadataFields,
71
- ModelType,
72
- )
73
- from ads.aqua.model.entities import (
74
- AquaFineTuneModel,
75
- AquaFineTuningMetric,
76
- AquaModel,
77
- AquaModelLicense,
78
- AquaModelReadme,
79
- AquaModelSummary,
80
- ImportModelDetails,
81
- ModelValidationResult,
82
- )
83
- from ads.aqua.model.enums import MultiModelSupportedTaskType
84
- from ads.common.auth import default_signer
85
- from ads.common.oci_resource import SEARCH_TYPE, OCIResource
86
- from ads.common.utils import (
87
- UNKNOWN,
88
- get_console_link,
89
- is_path_exists,
90
- read_file,
91
- )
92
- from ads.config import (
93
- AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME,
94
- AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,
95
- AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME,
96
- AQUA_EVALUATION_CONTAINER_METADATA_NAME,
97
- AQUA_FINETUNING_CONTAINER_METADATA_NAME,
98
- AQUA_SERVICE_MODELS,
99
- COMPARTMENT_OCID,
100
- PROJECT_OCID,
101
- SERVICE,
102
- TENANCY_OCID,
103
- USER,
104
- )
105
- from ads.model import DataScienceModel
106
- from ads.model.common.utils import MetadataArtifactPathType
107
- from ads.model.model_metadata import (
108
- MetadataCustomCategory,
109
- ModelCustomMetadata,
110
- ModelCustomMetadataItem,
111
- )
112
- from ads.telemetry import telemetry
113
-
114
-
115
- class AquaModelApp(AquaApp):
116
- """Provides a suite of APIs to interact with Aqua models within the Oracle
117
- Cloud Infrastructure Data Science service, serving as an interface for
118
- managing machine learning models.
119
-
120
-
121
- Methods
122
- -------
123
- create(model_id: str, project_id: str, compartment_id: str = None, **kwargs) -> "AquaModel"
124
- Creates custom aqua model from service model.
125
- get(model_id: str) -> AquaModel:
126
- Retrieves details of an Aqua model by its unique identifier.
127
- list(compartment_id: str = None, project_id: str = None, **kwargs) -> List[AquaModelSummary]:
128
- Lists all Aqua models within a specified compartment and/or project.
129
- clear_model_list_cache()
130
- Allows clear list model cache items from the service models compartment.
131
- register(model: str, os_path: str, local_dir: str = None)
132
-
133
- Note:
134
- This class is designed to work within the Oracle Cloud Infrastructure
135
- and requires proper configuration and authentication set up to interact
136
- with OCI services.
137
- """
138
-
139
- _service_models_cache = TTLCache(
140
- maxsize=10, ttl=timedelta(hours=5), timer=datetime.now
141
- )
142
- # Used for saving service model details
143
- _service_model_details_cache = TTLCache(
144
- maxsize=10, ttl=timedelta(hours=5), timer=datetime.now
145
- )
146
- _cache_lock = Lock()
147
-
148
- @telemetry(entry_point="plugin=model&action=create", name="aqua")
149
- def create(
150
- self,
151
- model_id: Union[str, AquaMultiModelRef],
152
- project_id: Optional[str] = None,
153
- compartment_id: Optional[str] = None,
154
- freeform_tags: Optional[Dict] = None,
155
- defined_tags: Optional[Dict] = None,
156
- **kwargs,
157
- ) -> DataScienceModel:
158
- """
159
- Creates a custom Aqua model from a service model.
160
-
161
- Parameters
162
- ----------
163
- model_id : Union[str, AquaMultiModelRef]
164
- The model ID as a string or a AquaMultiModelRef instance to be deployed.
165
- project_id : Optional[str]
166
- The project ID for the custom model.
167
- compartment_id : Optional[str]
168
- The compartment ID for the custom model. Defaults to None.
169
- If not provided, the compartment ID will be fetched from environment variables.
170
- freeform_tags : Optional[Dict]
171
- Freeform tags for the model.
172
- defined_tags : Optional[Dict]
173
- Defined tags for the model.
174
-
175
- Returns
176
- -------
177
- DataScienceModel
178
- The instance of DataScienceModel.
179
- """
180
- model_id = (
181
- model_id.model_id if isinstance(model_id, AquaMultiModelRef) else model_id
182
- )
183
- service_model = DataScienceModel.from_id(model_id)
184
- target_project = project_id or PROJECT_OCID
185
- target_compartment = compartment_id or COMPARTMENT_OCID
186
-
187
- # Skip model copying if it is registered model
188
- if service_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None) is not None:
189
- logger.info(
190
- f"Aqua Model {model_id} already exists in the user's compartment."
191
- "Skipped copying."
192
- )
193
- return service_model
194
-
195
- # combine tags
196
- combined_freeform_tags = {
197
- **(service_model.freeform_tags or {}),
198
- **(freeform_tags or {}),
199
- }
200
- combined_defined_tags = {
201
- **(service_model.defined_tags or {}),
202
- **(defined_tags or {}),
203
- }
204
-
205
- custom_model = (
206
- DataScienceModel()
207
- .with_compartment_id(target_compartment)
208
- .with_project_id(target_project)
209
- .with_model_file_description(json_dict=service_model.model_file_description)
210
- .with_display_name(service_model.display_name)
211
- .with_description(service_model.description)
212
- .with_freeform_tags(**combined_freeform_tags)
213
- .with_defined_tags(**combined_defined_tags)
214
- .with_custom_metadata_list(service_model.custom_metadata_list)
215
- .with_defined_metadata_list(service_model.defined_metadata_list)
216
- .with_provenance_metadata(service_model.provenance_metadata)
217
- .create(model_by_reference=True, **kwargs)
218
- )
219
- logger.info(
220
- f"Aqua Model {custom_model.id} created with the service model {model_id}."
221
- )
222
-
223
- # Track unique models that were created in the user's compartment
224
- self.telemetry.record_event_async(
225
- category="aqua/service/model",
226
- action="create",
227
- detail=service_model.display_name,
228
- )
229
-
230
- return custom_model
231
-
232
- @telemetry(entry_point="plugin=model&action=create", name="aqua")
233
- def create_multi(
234
- self,
235
- models: List[AquaMultiModelRef],
236
- project_id: Optional[str] = None,
237
- compartment_id: Optional[str] = None,
238
- freeform_tags: Optional[Dict] = None,
239
- defined_tags: Optional[Dict] = None,
240
- **kwargs, # noqa: ARG002
241
- ) -> DataScienceModel:
242
- """
243
- Creates a multi-model grouping using the provided model list.
244
-
245
- Parameters
246
- ----------
247
- models : List[AquaMultiModelRef]
248
- List of AquaMultiModelRef instances for creating a multi-model group.
249
- project_id : Optional[str]
250
- The project ID for the multi-model group.
251
- compartment_id : Optional[str]
252
- The compartment ID for the multi-model group.
253
- freeform_tags : Optional[Dict]
254
- Freeform tags for the model.
255
- defined_tags : Optional[Dict]
256
- Defined tags for the model.
257
-
258
- Returns
259
- -------
260
- DataScienceModel
261
- Instance of DataScienceModel object.
262
- """
263
-
264
- if not models:
265
- raise AquaValueError(
266
- "Model list cannot be empty. Please provide at least one model for deployment."
267
- )
268
-
269
- artifact_list = []
270
- display_name_list = []
271
- model_custom_metadata = ModelCustomMetadata()
272
-
273
- service_inference_containers = (
274
- self.get_container_config().to_dict().get("inference")
275
- )
276
-
277
- supported_container_families = [
278
- container_config_item.family
279
- for container_config_item in service_inference_containers
280
- if any(
281
- usage.upper() in container_config_item.usages
282
- for usage in [Usage.MULTI_MODEL, Usage.OTHER]
283
- )
284
- ]
285
-
286
- if not supported_container_families:
287
- raise AquaValueError(
288
- "Currently, there are no containers that support multi-model deployment."
289
- )
290
-
291
- selected_models_deployment_containers = set()
292
-
293
- # Process each model
294
- for model in models:
295
- source_model = DataScienceModel.from_id(model.model_id)
296
- display_name = source_model.display_name
297
- # Update model name in user's input model
298
- model.model_name = model.model_name or display_name
299
-
300
- # TODO Uncomment the section below, if only service models should be allowed for multi-model deployment
301
- # if not source_model.freeform_tags.get(Tags.AQUA_SERVICE_MODEL_TAG, UNKNOWN):
302
- # raise AquaValueError(
303
- # f"Invalid selected model {display_name}. "
304
- # "Currently only service models are supported for multi model deployment."
305
- # )
306
-
307
- if (
308
- source_model.freeform_tags.get(Tags.TASK, UNKNOWN).lower()
309
- not in MultiModelSupportedTaskType
310
- ):
311
- raise AquaValueError(
312
- f"Invalid or missing {Tags.TASK} tag for selected model {display_name}. "
313
- f"Currently only `{MultiModelSupportedTaskType.values()}` models are supported for multi model deployment."
314
- )
315
-
316
- display_name_list.append(display_name)
317
-
318
- # Retrieve model artifact
319
- model_artifact_path = source_model.artifact
320
- if not model_artifact_path:
321
- raise AquaValueError(
322
- f"Model '{display_name}' (ID: {model.model_id}) has no artifacts. "
323
- "Please register the model first."
324
- )
325
-
326
- # Update model artifact location in user's input model
327
- model.artifact_location = model_artifact_path
328
-
329
- artifact_list.append(model_artifact_path)
330
-
331
- # Validate deployment container consistency
332
- deployment_container = source_model.custom_metadata_list.get(
333
- ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
334
- ModelCustomMetadataItem(
335
- key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER
336
- ),
337
- ).value
338
-
339
- if deployment_container not in supported_container_families:
340
- raise AquaValueError(
341
- f"Unsupported deployment container '{deployment_container}' for model '{source_model.id}'. "
342
- f"Only '{supported_container_families}' are supported for multi-model deployments."
343
- )
344
-
345
- selected_models_deployment_containers.add(deployment_container)
346
-
347
- if not selected_models_deployment_containers:
348
- raise AquaValueError(
349
- "None of the selected models are associated with a recognized container family. "
350
- "Please review the selected models, or select a different group of models."
351
- )
352
-
353
- # Check if the all models in the group shares same container family
354
- if len(selected_models_deployment_containers) > 1:
355
- deployment_container = get_preferred_compatible_family(
356
- selected_families=selected_models_deployment_containers
357
- )
358
- if not deployment_container:
359
- raise AquaValueError(
360
- "The selected models are associated with different container families: "
361
- f"{list(selected_models_deployment_containers)}."
362
- "For multi-model deployment, all models in the group must share the same container family."
363
- )
364
- else:
365
- deployment_container = selected_models_deployment_containers.pop()
366
-
367
- # Generate model group details
368
- timestamp = datetime.now().strftime("%Y%m%d")
369
- model_group_display_name = f"model_group_{timestamp}"
370
- combined_models = ", ".join(display_name_list)
371
- model_group_description = f"Multi-model grouping using {combined_models}."
372
-
373
- # Add global metadata
374
- model_custom_metadata.add(
375
- key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
376
- value=deployment_container,
377
- description=f"Inference container mapping for {model_group_display_name}",
378
- category="Other",
379
- )
380
- model_custom_metadata.add(
381
- key=ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT,
382
- value=str(len(models)),
383
- description="Number of models in the group.",
384
- category="Other",
385
- )
386
-
387
- # Combine tags. The `Tags.AQUA_TAG` has been excluded, because we don't want to show
388
- # the models created for multi-model purpose in the AQUA models list.
389
- tags = {
390
- # Tags.AQUA_TAG: "active",
391
- Tags.MULTIMODEL_TYPE_TAG: "true",
392
- **(freeform_tags or {}),
393
- }
394
-
395
- # Create multi-model group
396
- custom_model = (
397
- DataScienceModel()
398
- .with_compartment_id(compartment_id)
399
- .with_project_id(project_id)
400
- .with_display_name(model_group_display_name)
401
- .with_description(model_group_description)
402
- .with_freeform_tags(**tags)
403
- .with_defined_tags(**(defined_tags or {}))
404
- .with_custom_metadata_list(model_custom_metadata)
405
- )
406
-
407
- # Attach artifacts
408
- for artifact in artifact_list:
409
- custom_model.add_artifact(uri=artifact)
410
-
411
- # Finalize creation
412
- custom_model.create(model_by_reference=True)
413
-
414
- logger.info(
415
- f"Aqua Model '{custom_model.id}' created with models: {', '.join(display_name_list)}."
416
- )
417
-
418
- # Create custom metadata for multi model metadata
419
- custom_model.create_custom_metadata_artifact(
420
- metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA,
421
- artifact_path_or_content=json.dumps(
422
- [model.model_dump() for model in models]
423
- ).encode(),
424
- path_type=MetadataArtifactPathType.CONTENT,
425
- )
426
-
427
- logger.debug(
428
- f"Multi model metadata uploaded for Aqua model: {custom_model.id}."
429
- )
430
-
431
- # Track telemetry event
432
- self.telemetry.record_event_async(
433
- category="aqua/multimodel",
434
- action="create",
435
- detail=combined_models,
436
- )
437
-
438
- return custom_model
439
-
440
- @telemetry(entry_point="plugin=model&action=get", name="aqua")
441
- def get(self, model_id: str) -> "AquaModel":
442
- """Gets the information of an Aqua model.
443
-
444
- Parameters
445
- ----------
446
- model_id: str
447
- The model OCID.
448
- load_model_card: (bool, optional). Defaults to `True`.
449
- Whether to load model card from artifacts or not.
450
-
451
- Returns
452
- -------
453
- AquaModel:
454
- The instance of AquaModel.
455
- """
456
-
457
- cached_item = self._service_model_details_cache.get(model_id)
458
- if cached_item:
459
- logger.info(f"Fetching model details for model {model_id} from cache.")
460
- return cached_item
461
-
462
- logger.info(f"Fetching model details for model {model_id}.")
463
- ds_model = DataScienceModel.from_id(model_id)
464
-
465
- if not self._if_show(ds_model):
466
- raise AquaRuntimeError(
467
- f"Target model `{ds_model.id} `is not an Aqua model as it does not contain "
468
- f"{Tags.AQUA_TAG} tag."
469
- )
470
-
471
- is_fine_tuned_model = bool(
472
- ds_model.freeform_tags
473
- and ds_model.freeform_tags.get(Tags.AQUA_FINE_TUNED_MODEL_TAG)
474
- )
475
-
476
- inference_container = ds_model.custom_metadata_list.get(
477
- ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
478
- ModelCustomMetadataItem(key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER),
479
- ).value
480
- inference_container_uri = ds_model.custom_metadata_list.get(
481
- ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI,
482
- ModelCustomMetadataItem(
483
- key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI
484
- ),
485
- ).value
486
- evaluation_container = ds_model.custom_metadata_list.get(
487
- ModelCustomMetadataFields.EVALUATION_CONTAINER,
488
- ModelCustomMetadataItem(key=ModelCustomMetadataFields.EVALUATION_CONTAINER),
489
- ).value
490
- finetuning_container: str = ds_model.custom_metadata_list.get(
491
- ModelCustomMetadataFields.FINETUNE_CONTAINER,
492
- ModelCustomMetadataItem(key=ModelCustomMetadataFields.FINETUNE_CONTAINER),
493
- ).value
494
- artifact_location = ds_model.custom_metadata_list.get(
495
- ModelCustomMetadataFields.ARTIFACT_LOCATION,
496
- ModelCustomMetadataItem(key=ModelCustomMetadataFields.ARTIFACT_LOCATION),
497
- ).value
498
-
499
- aqua_model_attributes = dict(
500
- **self._process_model(ds_model, self.region),
501
- project_id=ds_model.project_id,
502
- inference_container=inference_container,
503
- inference_container_uri=inference_container_uri,
504
- finetuning_container=finetuning_container,
505
- evaluation_container=evaluation_container,
506
- artifact_location=artifact_location,
507
- )
508
-
509
- if not is_fine_tuned_model:
510
- model_details = AquaModel(**aqua_model_attributes)
511
- self._service_model_details_cache.__setitem__(
512
- key=model_id, value=model_details
513
- )
514
-
515
- else:
516
- try:
517
- jobrun_ocid = ds_model.provenance_metadata.training_id
518
- jobrun = self.ds_client.get_job_run(jobrun_ocid).data
519
- except Exception as e:
520
- logger.debug(
521
- f"Missing jobrun information in the provenance metadata of the given model {model_id}."
522
- f"\nError: {str(e)}"
523
- )
524
- jobrun = None
525
-
526
- try:
527
- source_id = ds_model.custom_metadata_list.get(
528
- FineTuningCustomMetadata.FT_SOURCE
529
- ).value
530
- except ValueError as e:
531
- logger.debug(
532
- f"Custom metadata is missing {FineTuningCustomMetadata.FT_SOURCE} key for "
533
- f"model {model_id}.\nError: {str(e)}"
534
- )
535
- source_id = UNKNOWN
536
-
537
- try:
538
- source_name = ds_model.custom_metadata_list.get(
539
- FineTuningCustomMetadata.FT_SOURCE_NAME
540
- ).value
541
- except ValueError as e:
542
- logger.debug(
543
- f"Custom metadata is missing {FineTuningCustomMetadata.FT_SOURCE_NAME} key for "
544
- f"model {model_id}.\nError: {str(e)}"
545
- )
546
- source_name = UNKNOWN
547
-
548
- source_identifier = _build_resource_identifier(
549
- id=source_id,
550
- name=source_name,
551
- region=self.region,
552
- )
553
-
554
- ft_metrics = self._build_ft_metrics(ds_model.custom_metadata_list)
555
-
556
- job_run_status = (
557
- jobrun.lifecycle_state
558
- if jobrun and jobrun.lifecycle_state != JobRun.LIFECYCLE_STATE_DELETED
559
- else (
560
- JobRun.LIFECYCLE_STATE_SUCCEEDED
561
- if self.if_artifact_exist(ds_model.id)
562
- else JobRun.LIFECYCLE_STATE_FAILED
563
- )
564
- )
565
- # TODO: change the argument's name.
566
- lifecycle_state = LifecycleStatus.get_status(
567
- evaluation_status=ds_model.lifecycle_state,
568
- job_run_status=job_run_status,
569
- )
570
-
571
- model_details = AquaFineTuneModel(
572
- **aqua_model_attributes,
573
- source=source_identifier,
574
- lifecycle_state=(
575
- Model.LIFECYCLE_STATE_ACTIVE
576
- if lifecycle_state == JobRun.LIFECYCLE_STATE_SUCCEEDED
577
- else lifecycle_state
578
- ),
579
- metrics=ft_metrics,
580
- model=ds_model,
581
- jobrun=jobrun,
582
- region=self.region,
583
- )
584
-
585
- return model_details
586
-
587
- @telemetry(entry_point="plugin=model&action=delete", name="aqua")
588
- def delete_model(self, model_id):
589
- ds_model = DataScienceModel.from_id(model_id)
590
- is_registered_model = ds_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None)
591
- is_fine_tuned_model = ds_model.freeform_tags.get(
592
- Tags.AQUA_FINE_TUNED_MODEL_TAG, None
593
- )
594
- if is_registered_model or is_fine_tuned_model:
595
- logger.info(f"Deleting model {model_id}.")
596
- return ds_model.delete()
597
- else:
598
- raise AquaRuntimeError(
599
- f"Failed to delete model:{model_id}. Only registered models or finetuned model can be deleted."
600
- )
601
-
602
- @telemetry(entry_point="plugin=model&action=edit", name="aqua")
603
- def edit_registered_model(
604
- self, id, inference_container, inference_container_uri, enable_finetuning, task
605
- ):
606
- """Edits the default config of unverified registered model.
607
-
608
- Parameters
609
- ----------
610
- id: str
611
- The model OCID.
612
- inference_container: str.
613
- The inference container family name
614
- inference_container_uri: str
615
- The inference container uri for embedding models
616
- enable_finetuning: str
617
- Flag to enable or disable finetuning over the model. Defaults to None
618
- task:
619
- The usecase type of the model. e.g , text-generation , text_embedding etc.
620
-
621
- Returns
622
- -------
623
- Model:
624
- The instance of oci.data_science.models.Model.
625
-
626
- """
627
- ds_model = DataScienceModel.from_id(id)
628
- if ds_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None):
629
- if ds_model.freeform_tags.get(Tags.AQUA_SERVICE_MODEL_TAG, None):
630
- raise AquaRuntimeError(
631
- "Only registered unverified models can be edited."
632
- )
633
- else:
634
- custom_metadata_list = ds_model.custom_metadata_list
635
- freeform_tags = ds_model.freeform_tags
636
- if inference_container:
637
- if (
638
- inference_container in CustomInferenceContainerTypeFamily
639
- and inference_container_uri is None
640
- ):
641
- raise AquaRuntimeError(
642
- "Inference container URI must be provided."
643
- )
644
- else:
645
- custom_metadata_list.add(
646
- key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
647
- value=inference_container,
648
- category=MetadataCustomCategory.OTHER,
649
- description="Deployment container mapping for SMC",
650
- replace=True,
651
- )
652
- if inference_container_uri:
653
- if (
654
- inference_container in CustomInferenceContainerTypeFamily
655
- or inference_container is None
656
- ):
657
- custom_metadata_list.add(
658
- key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI,
659
- value=inference_container_uri,
660
- category=MetadataCustomCategory.OTHER,
661
- description=f"Inference container URI for {ds_model.display_name}",
662
- replace=True,
663
- )
664
- else:
665
- raise AquaRuntimeError(
666
- f"Inference container URI can be edited only with container values: {CustomInferenceContainerTypeFamily.values()}"
667
- )
668
-
669
- if enable_finetuning is not None:
670
- if enable_finetuning.lower() == "true":
671
- custom_metadata_list.add(
672
- key=ModelCustomMetadataFields.FINETUNE_CONTAINER,
673
- value=FineTuningContainerTypeFamily.AQUA_FINETUNING_CONTAINER_FAMILY,
674
- category=MetadataCustomCategory.OTHER,
675
- description="Fine-tuning container mapping for SMC",
676
- replace=True,
677
- )
678
- freeform_tags.update({Tags.READY_TO_FINE_TUNE: "true"})
679
- elif enable_finetuning.lower() == "false":
680
- try:
681
- custom_metadata_list.remove(
682
- ModelCustomMetadataFields.FINETUNE_CONTAINER
683
- )
684
- freeform_tags.pop(Tags.READY_TO_FINE_TUNE)
685
- except Exception as ex:
686
- raise AquaRuntimeError(
687
- f"The given model already doesn't support finetuning: {ex}"
688
- ) from ex
689
-
690
- custom_metadata_list.remove("modelDescription")
691
- if task:
692
- freeform_tags.update({Tags.TASK: task})
693
- updated_custom_metadata_list = [
694
- Metadata(**metadata)
695
- for metadata in custom_metadata_list.to_dict()["data"]
696
- ]
697
- update_model_details = UpdateModelDetails(
698
- custom_metadata_list=updated_custom_metadata_list,
699
- freeform_tags=freeform_tags,
700
- )
701
- AquaApp().update_model(id, update_model_details)
702
- logger.info(f"Updated model details for the model {id}.")
703
- else:
704
- raise AquaRuntimeError("Only registered unverified models can be edited.")
705
-
706
- def _fetch_metric_from_metadata(
707
- self,
708
- custom_metadata_list: ModelCustomMetadata,
709
- target: str,
710
- category: str,
711
- metric_name: str,
712
- ) -> AquaFineTuningMetric:
713
- """Gets target metric from `ads.model.model_metadata.ModelCustomMetadata`."""
714
- try:
715
- scores = []
716
- for custom_metadata in custom_metadata_list._items:
717
- # We use description to group metrics
718
- if custom_metadata.description == target:
719
- scores.append(custom_metadata.value)
720
- if metric_name.endswith("final"):
721
- break
722
-
723
- return AquaFineTuningMetric(
724
- name=metric_name,
725
- category=category,
726
- scores=scores,
727
- )
728
- except Exception:
729
- return AquaFineTuningMetric(name=metric_name, category=category, scores=[])
730
-
731
- def _build_ft_metrics(
732
- self, custom_metadata_list: ModelCustomMetadata
733
- ) -> List[AquaFineTuningMetric]:
734
- """Builds Fine Tuning metrics."""
735
-
736
- validation_metrics = self._fetch_metric_from_metadata(
737
- custom_metadata_list=custom_metadata_list,
738
- target=FineTuningCustomMetadata.VALIDATION_METRICS_EPOCH,
739
- category=FineTuningMetricCategories.VALIDATION,
740
- metric_name=VALIDATION_METRICS,
741
- )
742
-
743
- training_metrics = self._fetch_metric_from_metadata(
744
- custom_metadata_list=custom_metadata_list,
745
- target=FineTuningCustomMetadata.TRAINING_METRICS_EPOCH,
746
- category=FineTuningMetricCategories.TRAINING,
747
- metric_name=TRINING_METRICS,
748
- )
749
-
750
- validation_final = self._fetch_metric_from_metadata(
751
- custom_metadata_list=custom_metadata_list,
752
- target=FineTuningCustomMetadata.VALIDATION_METRICS_FINAL,
753
- category=FineTuningMetricCategories.VALIDATION,
754
- metric_name=VALIDATION_METRICS_FINAL,
755
- )
756
-
757
- training_final = self._fetch_metric_from_metadata(
758
- custom_metadata_list=custom_metadata_list,
759
- target=FineTuningCustomMetadata.TRAINING_METRICS_FINAL,
760
- category=FineTuningMetricCategories.TRAINING,
761
- metric_name=TRAINING_METRICS_FINAL,
762
- )
763
-
764
- return [
765
- validation_metrics,
766
- training_metrics,
767
- validation_final,
768
- training_final,
769
- ]
770
-
771
- def get_hf_tokenizer_config(self, model_id):
772
- """
773
- Gets the default model tokenizer config for the given Aqua model.
774
- Returns the content of tokenizer_config.json stored in model artifact.
775
-
776
- Parameters
777
- ----------
778
- model_id: str
779
- The OCID of the Aqua model.
780
-
781
- Returns
782
- -------
783
- Dict:
784
- Model tokenizer config.
785
- """
786
- config = self.get_config(
787
- model_id, AQUA_MODEL_TOKENIZER_CONFIG, ConfigFolder.ARTIFACT
788
- ).config
789
- if not config:
790
- logger.debug(
791
- f"{AQUA_MODEL_TOKENIZER_CONFIG} is not available for the model: {model_id}. "
792
- f"Check if the custom metadata has the artifact path set."
793
- )
794
- return config
795
-
796
- return config
797
-
798
- @staticmethod
799
- def to_aqua_model(
800
- model: Union[
801
- DataScienceModel,
802
- oci.data_science.models.model.Model,
803
- oci.data_science.models.ModelSummary,
804
- oci.resource_search.models.ResourceSummary,
805
- ],
806
- region: str,
807
- ) -> AquaModel:
808
- """Converts a model to an Aqua model."""
809
- return AquaModel(**AquaModelApp._process_model(model, region))
810
-
811
- @staticmethod
812
- def _process_model(
813
- model: Union[
814
- DataScienceModel,
815
- oci.data_science.models.model.Model,
816
- oci.data_science.models.ModelSummary,
817
- oci.resource_search.models.ResourceSummary,
818
- ],
819
- region: str,
820
- inference_containers: Optional[List[Any]] = None,
821
- ) -> dict:
822
- """Constructs required fields for AquaModelSummary."""
823
-
824
- # todo: revisit icon generation code
825
- # icon = self._load_icon(model.display_name)
826
- icon = ""
827
-
828
- tags = {}
829
- tags.update(model.defined_tags or {})
830
- tags.update(model.freeform_tags or {})
831
-
832
- model_id = (
833
- model.identifier
834
- if isinstance(model, oci.resource_search.models.ResourceSummary)
835
- else model.id
836
- )
837
-
838
- console_link = get_console_link(
839
- resource="models",
840
- ocid=model_id,
841
- region=region,
842
- )
843
-
844
- description = ""
845
- if isinstance(model, (DataScienceModel, oci.data_science.models.model.Model)):
846
- description = model.description
847
- elif isinstance(model, oci.resource_search.models.ResourceSummary):
848
- description = model.additional_details.get("description")
849
-
850
- search_text = (
851
- AquaModelApp._build_search_text(tags=tags, description=description)
852
- if tags
853
- else UNKNOWN
854
- )
855
-
856
- freeform_tags = model.freeform_tags or {}
857
- is_fine_tuned_model = Tags.AQUA_FINE_TUNED_MODEL_TAG in freeform_tags
858
- ready_to_deploy = (
859
- freeform_tags.get(Tags.AQUA_TAG, "").upper() == READY_TO_DEPLOY_STATUS
860
- )
861
-
862
- ready_to_finetune = (
863
- freeform_tags.get(Tags.READY_TO_FINE_TUNE, "").upper()
864
- == READY_TO_FINE_TUNE_STATUS
865
- )
866
- ready_to_import = (
867
- freeform_tags.get(Tags.READY_TO_IMPORT, "").upper()
868
- == READY_TO_IMPORT_STATUS
869
- )
870
-
871
- try:
872
- model_file = model.custom_metadata_list.get(AQUA_MODEL_ARTIFACT_FILE).value
873
- except Exception:
874
- model_file = UNKNOWN
875
-
876
- if not inference_containers:
877
- inference_containers = (
878
- AquaApp().get_container_config().to_dict().get("inference")
879
- )
880
-
881
- model_formats_str = freeform_tags.get(
882
- Tags.MODEL_FORMAT, ModelFormat.SAFETENSORS
883
- ).upper()
884
- model_formats = model_formats_str.split(",")
885
-
886
- supported_platform: Set[str] = set()
887
-
888
- for container in inference_containers:
889
- for model_format in model_formats:
890
- if model_format in container.model_formats:
891
- supported_platform.update(container.platforms)
892
-
893
- nvidia_gpu_supported = Platform.NVIDIA_GPU in supported_platform
894
- arm_cpu_supported = Platform.ARM_CPU in supported_platform
895
-
896
- return {
897
- "compartment_id": model.compartment_id,
898
- "icon": icon or UNKNOWN,
899
- "id": model_id,
900
- "license": freeform_tags.get(Tags.LICENSE, UNKNOWN),
901
- "name": model.display_name,
902
- "organization": freeform_tags.get(Tags.ORGANIZATION, UNKNOWN),
903
- "task": freeform_tags.get(Tags.TASK, UNKNOWN),
904
- "time_created": str(model.time_created),
905
- "is_fine_tuned_model": is_fine_tuned_model,
906
- "tags": tags,
907
- "console_link": console_link,
908
- "search_text": search_text,
909
- "ready_to_deploy": ready_to_deploy,
910
- "ready_to_finetune": ready_to_finetune,
911
- "ready_to_import": ready_to_import,
912
- "nvidia_gpu_supported": nvidia_gpu_supported,
913
- "arm_cpu_supported": arm_cpu_supported,
914
- "model_file": model_file,
915
- "model_formats": model_formats,
916
- }
917
-
918
- @telemetry(entry_point="plugin=model&action=list", name="aqua")
919
- def list(
920
- self,
921
- compartment_id: str = None,
922
- category: str = None,
923
- project_id: str = None,
924
- model_type: str = None,
925
- **kwargs,
926
- ) -> List["AquaModelSummary"]:
927
- """Lists all Aqua models within a specified compartment and/or project.
928
- If `category` is not specified, the method defaults to returning
929
- the service models within the pre-configured default compartment. By default, the list
930
- of models in the service compartment are cached. Use clear_model_list_cache() to invalidate
931
- the cache.
932
-
933
- Parameters
934
- ----------
935
- compartment_id: (str, optional). Defaults to `None`.
936
- The compartment OCID.
937
- category: (str,optional). Defaults to `SERVICE`
938
- The category of the models to fetch. Can be either `USER` or `SERVICE`
939
- project_id: (str, optional). Defaults to `None`.
940
- The project OCID.
941
- model_type: (str, optional). Defaults to `None`.
942
- Model type represents the type of model in the user compartment, can be either FT or BASE.
943
- **kwargs:
944
- Additional keyword arguments that can be used to filter the results.
945
-
946
- Returns
947
- -------
948
- List[AquaModelSummary]:
949
- The list of the `ads.aqua.model.AquaModelSummary`.
950
- """
951
-
952
- category = category or kwargs.pop("category", SERVICE)
953
- compartment_id = compartment_id or COMPARTMENT_OCID
954
- if category == USER:
955
- # tracks number of times custom model listing was called
956
- self.telemetry.record_event_async(
957
- category="aqua/custom/model", action="list"
958
- )
959
-
960
- logger.info(f"Fetching custom models from compartment_id={compartment_id}.")
961
- model_type = model_type.upper() if model_type else ModelType.FT
962
- models = self._rqs(compartment_id, model_type=model_type)
963
- logger.info(
964
- f"Fetched {len(models)} models from {compartment_id or COMPARTMENT_OCID}."
965
- )
966
- else:
967
- # tracks number of times service model listing was called
968
- self.telemetry.record_event_async(
969
- category="aqua/service/model", action="list"
970
- )
971
-
972
- if AQUA_SERVICE_MODELS in self._service_models_cache:
973
- logger.info("Returning service models list from cache.")
974
- return self._service_models_cache.get(AQUA_SERVICE_MODELS)
975
- lifecycle_state = kwargs.pop(
976
- "lifecycle_state", Model.LIFECYCLE_STATE_ACTIVE
977
- )
978
-
979
- models = self.list_resource(
980
- self.ds_client.list_models,
981
- compartment_id=compartment_id,
982
- lifecycle_state=lifecycle_state,
983
- category=category,
984
- **kwargs,
985
- )
986
- logger.info(f"Fetched {len(models)} service models.")
987
-
988
- aqua_models = []
989
- inference_containers = self.get_container_config().to_dict().get("inference")
990
- for model in models:
991
- aqua_models.append(
992
- AquaModelSummary(
993
- **self._process_model(
994
- model=model,
995
- region=self.region,
996
- inference_containers=inference_containers,
997
- ),
998
- project_id=project_id or UNKNOWN,
999
- )
1000
- )
1001
- if category == SERVICE:
1002
- self._service_models_cache.__setitem__(
1003
- key=AQUA_SERVICE_MODELS, value=aqua_models
1004
- )
1005
-
1006
- return aqua_models
1007
-
1008
- def clear_model_list_cache(
1009
- self,
1010
- ):
1011
- """
1012
- Allows user to clear list model cache items from the service models compartment.
1013
- Returns
1014
- -------
1015
- dict with the key used, and True if cache has the key that needs to be deleted.
1016
- """
1017
- res = {}
1018
- with self._cache_lock:
1019
- if AQUA_SERVICE_MODELS in self._service_models_cache:
1020
- self._service_models_cache.pop(key=AQUA_SERVICE_MODELS)
1021
- logger.info("Cleared models cache for service compartment.")
1022
- res = {
1023
- "cache_deleted": True,
1024
- }
1025
- return res
1026
-
1027
- def clear_model_details_cache(self, model_id):
1028
- """
1029
- Allows user to clear model details cache item
1030
- Returns
1031
- -------
1032
- dict with the key used, and True if cache has the key that needs to be deleted.
1033
- """
1034
- res = {}
1035
- with self._cache_lock:
1036
- if model_id in self._service_model_details_cache:
1037
- self._service_model_details_cache.pop(key=model_id)
1038
- logger.info(f"Clearing model details cache for model {model_id}.")
1039
- res = {"key": {"model_id": model_id}, "cache_deleted": True}
1040
-
1041
- return res
1042
-
1043
- @staticmethod
1044
- def list_valid_inference_containers():
1045
- containers = AquaApp().get_container_config().to_dict().get("inference")
1046
- family_values = [item.family for item in containers]
1047
- return family_values
1048
-
1049
- @telemetry(
1050
- entry_point="plugin=model&action=get_defined_metadata_artifact_content",
1051
- name="aqua",
1052
- )
1053
- def get_defined_metadata_artifact_content(self, model_id: str, metadata_key: str):
1054
- """
1055
- Gets the defined metadata artifact content for the given model
1056
-
1057
- Args:
1058
- model_id: str
1059
- model ocid for which defined metadata artifact needs to be created
1060
- metadata_key: str
1061
- defined metadata key like Readme , License , DeploymentConfiguration , FinetuningConfiguration
1062
- Returns:
1063
- The model defined metadata artifact content. Can be either str or Dict
1064
-
1065
- """
1066
-
1067
- content = self.get_config(model_id, metadata_key)
1068
- if not content:
1069
- logger.debug(
1070
- f"Defined metadata artifact {metadata_key} for model: {model_id} is not available."
1071
- )
1072
- return content
1073
-
1074
- @telemetry(
1075
- entry_point="plugin=model&action=create_defined_metadata_artifact", name="aqua"
1076
- )
1077
- def create_defined_metadata_artifact(
1078
- self,
1079
- model_id: str,
1080
- metadata_key: str,
1081
- path_type: MetadataArtifactPathType,
1082
- artifact_path_or_content: str,
1083
- ) -> None:
1084
- """
1085
- Creates defined metadata artifact for the registered unverified model
1086
-
1087
- Args:
1088
- model_id: str
1089
- model ocid for which defined metadata artifact needs to be created
1090
- metadata_key: str
1091
- defined metadata key like Readme , License , DeploymentConfiguration , FinetuningConfiguration
1092
- path_type: str
1093
- path type of the given defined metadata can be local , oss or the content itself
1094
- artifact_path_or_content: str
1095
- It can be local path or oss path or the actual content itself
1096
- Returns:
1097
- None
1098
- """
1099
-
1100
- ds_model = DataScienceModel.from_id(model_id)
1101
- oci_aqua = ds_model.freeform_tags.get(Tags.AQUA_TAG, None)
1102
- if not oci_aqua:
1103
- raise AquaRuntimeError(f"Target model {model_id} is not an Aqua model.")
1104
- is_registered_model = ds_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None)
1105
- is_verified_model = ds_model.freeform_tags.get(
1106
- Tags.AQUA_SERVICE_MODEL_TAG, None
1107
- )
1108
- if is_registered_model and not is_verified_model:
1109
- try:
1110
- ds_model.create_defined_metadata_artifact(
1111
- metadata_key_name=metadata_key,
1112
- artifact_path_or_content=artifact_path_or_content,
1113
- path_type=path_type,
1114
- )
1115
- except Exception as ex:
1116
- raise AquaRuntimeError(
1117
- f"Error occurred in creating defined metadata artifact for model {model_id}: {ex}"
1118
- ) from ex
1119
- else:
1120
- raise AquaRuntimeError(
1121
- f"Cannot create defined metadata artifact for model {model_id}"
1122
- )
1123
-
1124
- def _create_model_catalog_entry(
1125
- self,
1126
- os_path: str,
1127
- model_name: str,
1128
- inference_container: str,
1129
- finetuning_container: str,
1130
- verified_model: DataScienceModel,
1131
- validation_result: ModelValidationResult,
1132
- compartment_id: Optional[str],
1133
- project_id: Optional[str],
1134
- inference_container_uri: Optional[str],
1135
- freeform_tags: Optional[dict] = None,
1136
- defined_tags: Optional[dict] = None,
1137
- ) -> DataScienceModel:
1138
- """Create model by reference from the object storage path
1139
-
1140
- Args:
1141
- os_path (str): OCI where the model is uploaded - oci://bucket@namespace/prefix
1142
- model_name (str): name of the model
1143
- inference_container (str): selects service defaults
1144
- finetuning_container (str): selects service defaults
1145
- verified_model (DataScienceModel): If set, then copies all the tags and custom metadata information from the service verified model
1146
- compartment_id (Optional[str]): Compartment Id of the compartment where the model has to be created
1147
- project_id (Optional[str]): Project id of the project where the model has to be created
1148
- inference_container_uri (Optional[str]): Inference container uri for BYOC
1149
- freeform_tags (dict): Freeform tags for the model
1150
- defined_tags (dict): Defined tags for the model
1151
-
1152
- Returns:
1153
- DataScienceModel: Returns Datascience model instance.
1154
- """
1155
- model = DataScienceModel()
1156
- tags: Dict[str, str] = (
1157
- {
1158
- **verified_model.freeform_tags,
1159
- Tags.AQUA_SERVICE_MODEL_TAG: verified_model.id,
1160
- }
1161
- if verified_model
1162
- else {
1163
- Tags.AQUA_TAG: "active",
1164
- Tags.BASE_MODEL_CUSTOM: "true",
1165
- }
1166
- )
1167
- tags.update({Tags.BASE_MODEL_CUSTOM: "true"})
1168
-
1169
- if validation_result and validation_result.model_formats:
1170
- tags.update(
1171
- {
1172
- Tags.MODEL_FORMAT: ",".join(
1173
- model_format for model_format in validation_result.model_formats
1174
- )
1175
- }
1176
- )
1177
-
1178
- # Remove `ready_to_import` tag that might get copied from service model.
1179
- tags.pop(Tags.READY_TO_IMPORT, None)
1180
- defined_metadata_dict = {}
1181
- readme_file_path = os_path.rstrip("/") + "/" + README
1182
- license_file_path = os_path.rstrip("/") + "/" + LICENSE
1183
- if verified_model:
1184
- # Verified model is a model in the service catalog that either has no artifacts but contains all the necessary metadata for deploying and fine tuning.
1185
- # If set, then we copy all the model metadata.
1186
- metadata = verified_model.custom_metadata_list
1187
- if verified_model.model_file_description:
1188
- model = model.with_model_file_description(
1189
- json_dict=verified_model.model_file_description
1190
- )
1191
- defined_metadata_list = (
1192
- verified_model.defined_metadata_list._to_oci_metadata()
1193
- )
1194
- for defined_metadata in defined_metadata_list:
1195
- if defined_metadata.has_artifact:
1196
- content = (
1197
- self.ds_client.get_model_defined_metadatum_artifact_content(
1198
- verified_model.id, defined_metadata.key
1199
- ).data.content
1200
- )
1201
- defined_metadata_dict[defined_metadata.key] = content
1202
- else:
1203
- metadata = ModelCustomMetadata()
1204
- if not inference_container:
1205
- raise AquaRuntimeError(
1206
- f"Require Inference container information. Model: {model_name} does not have associated inference "
1207
- f"container defaults. Check docs for more information on how to pass inference container."
1208
- )
1209
- metadata.add(
1210
- key=AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,
1211
- value=inference_container,
1212
- description=f"Inference container mapping for {model_name}",
1213
- category="Other",
1214
- )
1215
- if inference_container_uri:
1216
- metadata.add(
1217
- key=AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME,
1218
- value=inference_container_uri,
1219
- description=f"Inference container URI for {model_name}",
1220
- category="Other",
1221
- )
1222
-
1223
- inference_containers = (
1224
- AquaContainerConfig.from_service_config(
1225
- service_containers=self.list_service_containers()
1226
- )
1227
- .to_dict()
1228
- .get("inference")
1229
- )
1230
- smc_container_set = {container.family for container in inference_containers}
1231
- # only add cmd vars if inference container is not an SMC
1232
- if (
1233
- inference_container not in smc_container_set
1234
- and inference_container in CustomInferenceContainerTypeFamily.values()
1235
- ):
1236
- cmd_vars = generate_tei_cmd_var(os_path)
1237
- metadata.add(
1238
- key=AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME,
1239
- value=" ".join(cmd_vars),
1240
- description=f"Inference container cmd vars for {model_name}",
1241
- category="Other",
1242
- )
1243
-
1244
- if finetuning_container:
1245
- tags[Tags.READY_TO_FINE_TUNE] = "true"
1246
- metadata.add(
1247
- key=AQUA_FINETUNING_CONTAINER_METADATA_NAME,
1248
- value=finetuning_container,
1249
- description=f"Fine-tuning container mapping for {model_name}",
1250
- category="Other",
1251
- )
1252
- else:
1253
- logger.warn(
1254
- "Proceeding with model registration without the fine-tuning container information. "
1255
- "This model will not be available for fine tuning."
1256
- )
1257
- if validation_result and validation_result.model_file:
1258
- metadata.add(
1259
- key=AQUA_MODEL_ARTIFACT_FILE,
1260
- value=validation_result.model_file,
1261
- description=f"The model file for {model_name}",
1262
- category="Other",
1263
- )
1264
-
1265
- metadata.add(
1266
- key=AQUA_EVALUATION_CONTAINER_METADATA_NAME,
1267
- value="odsc-llm-evaluate",
1268
- description="Evaluation container mapping for SMC",
1269
- category="Other",
1270
- )
1271
-
1272
- if validation_result and validation_result.tags:
1273
- tags[Tags.TASK] = validation_result.tags.get(Tags.TASK, UNKNOWN)
1274
- tags[Tags.ORGANIZATION] = validation_result.tags.get(
1275
- Tags.ORGANIZATION, UNKNOWN
1276
- )
1277
- tags[Tags.LICENSE] = validation_result.tags.get(Tags.LICENSE, UNKNOWN)
1278
-
1279
- # Set artifact location to user bucket, and replace existing key if present.
1280
- metadata.add(
1281
- key=MODEL_BY_REFERENCE_OSS_PATH_KEY,
1282
- value=os_path,
1283
- description="artifact location",
1284
- category="Other",
1285
- replace=True,
1286
- )
1287
- # override tags with freeform tags if set
1288
- tags = {**tags, **(freeform_tags or {})}
1289
- model = (
1290
- model.with_custom_metadata_list(metadata)
1291
- .with_compartment_id(compartment_id or COMPARTMENT_OCID)
1292
- .with_project_id(project_id or PROJECT_OCID)
1293
- .with_artifact(os_path)
1294
- .with_display_name(model_name)
1295
- .with_freeform_tags(**tags)
1296
- .with_defined_tags(**(defined_tags or {}))
1297
- ).create(model_by_reference=True)
1298
- logger.debug(f"Created model catalog entry for the model:\n{model}")
1299
- for key, value in defined_metadata_dict.items():
1300
- model.create_defined_metadata_artifact(
1301
- key, value, MetadataArtifactPathType.CONTENT
1302
- )
1303
-
1304
- if is_path_exists(readme_file_path):
1305
- try:
1306
- model.create_defined_metadata_artifact(
1307
- AquaModelMetadataKeys.README,
1308
- readme_file_path,
1309
- MetadataArtifactPathType.OSS,
1310
- )
1311
- except Exception as ex:
1312
- logger.error(
1313
- f"Error Uploading Readme in defined metadata for model: {model.id} : {str(ex)}"
1314
- )
1315
- if not verified_model and is_path_exists(license_file_path):
1316
- try:
1317
- model.create_defined_metadata_artifact(
1318
- AquaModelMetadataKeys.LICENSE,
1319
- license_file_path,
1320
- MetadataArtifactPathType.OSS,
1321
- )
1322
- except Exception as ex:
1323
- logger.error(
1324
- f"Error Uploading License in defined metadata for model: {model.id} : {str(ex)}"
1325
- )
1326
- return model
1327
-
1328
- @staticmethod
1329
- def get_model_files(os_path: str, model_format: str) -> List[str]:
1330
- """
1331
- Get a list of model files based on the given OS path and model format.
1332
-
1333
- Args:
1334
- os_path (str): The OS path where the model files are located.
1335
- model_format (str): The format of the model files.
1336
-
1337
- Returns:
1338
- List[str]: A list of model file names.
1339
-
1340
- """
1341
- model_files: List[str] = []
1342
- # todo: revisit this logic to account for .bin files. In the current state, .bin and .safetensor models
1343
- # are grouped in one category and validation checks for config.json files only.
1344
- if model_format == ModelFormat.SAFETENSORS:
1345
- model_files.extend(
1346
- list_os_files_with_extension(oss_path=os_path, extension=".safetensors")
1347
- )
1348
- try:
1349
- load_config(
1350
- file_path=os_path,
1351
- config_file_name=AQUA_MODEL_ARTIFACT_CONFIG,
1352
- )
1353
- except Exception as ex:
1354
- message = (
1355
- f"The model path {os_path} does not contain the file config.json. "
1356
- f"Please check if the path is correct or the model artifacts are available at this location."
1357
- )
1358
- logger.warning(
1359
- f"{message}\n"
1360
- f"Details: {ex.reason if isinstance(ex, AquaFileNotFoundError) else str(ex)}\n"
1361
- )
1362
- else:
1363
- model_files.append(AQUA_MODEL_ARTIFACT_CONFIG)
1364
-
1365
- if model_format == ModelFormat.GGUF:
1366
- model_files.extend(
1367
- list_os_files_with_extension(oss_path=os_path, extension=".gguf")
1368
- )
1369
- logger.debug(
1370
- f"Fetched {len(model_files)} model files from {os_path} for model format {model_format}."
1371
- )
1372
- return model_files
1373
-
1374
- @staticmethod
1375
- def get_hf_model_files(model_name: str, model_format: str) -> List[str]:
1376
- """
1377
- Get a list of model files based on the given OS path and model format.
1378
-
1379
- Args:
1380
- model_name (str): The huggingface model name.
1381
- model_format (str): The format of the model files.
1382
-
1383
- Returns:
1384
- List[str]: A list of model file names.
1385
-
1386
- """
1387
- model_files: List[str] = []
1388
-
1389
- # todo: revisit this logic to account for .bin files. In the current state, .bin and .safetensor models
1390
- # are grouped in one category and returns config.json file only.
1391
-
1392
- try:
1393
- model_siblings = get_hf_model_info(repo_id=model_name).siblings
1394
- except Exception as e:
1395
- huggingface_err_message = str(e)
1396
- raise AquaValueError(
1397
- f"Could not get the model files of {model_name} from https://huggingface.co. "
1398
- f"Error: {huggingface_err_message}."
1399
- ) from e
1400
-
1401
- if not model_siblings:
1402
- raise AquaValueError(
1403
- f"Failed to fetch the model files of {model_name} from https://huggingface.co."
1404
- )
1405
-
1406
- for model_sibling in model_siblings:
1407
- extension = pathlib.Path(model_sibling.rfilename).suffix[1:].upper()
1408
- if (
1409
- model_format == ModelFormat.SAFETENSORS
1410
- and model_sibling.rfilename == AQUA_MODEL_ARTIFACT_CONFIG
1411
- ):
1412
- model_files.append(model_sibling.rfilename)
1413
- if extension == model_format:
1414
- model_files.append(model_sibling.rfilename)
1415
-
1416
- logger.debug(
1417
- f"Fetched {len(model_files)} model files for the model {model_name} for model format {model_format}."
1418
- )
1419
- return model_files
1420
-
1421
- def _validate_model(
1422
- self,
1423
- import_model_details: ImportModelDetails = None,
1424
- model_name: str = None,
1425
- verified_model: DataScienceModel = None,
1426
- ) -> ModelValidationResult:
1427
- """
1428
- Validates the model configuration and returns the model format telemetry model name.
1429
-
1430
- Args:
1431
- import_model_details (ImportModelDetails): Model details for importing the model.
1432
- model_name (str): name of the model
1433
- verified_model (DataScienceModel): If set, then copies all the tags and custom metadata information from
1434
- the service verified model
1435
-
1436
- Returns:
1437
- ModelValidationResult: The result of the model validation.
1438
-
1439
- Raises:
1440
- AquaRuntimeError: If there is an error while loading the config file or if the model path is incorrect.
1441
- AquaValueError: If the model format is not supported by AQUA.
1442
- """
1443
- model_formats = []
1444
- validation_result: ModelValidationResult = ModelValidationResult()
1445
-
1446
- hf_download_config_present = False
1447
-
1448
- if import_model_details.download_from_hf:
1449
- safetensors_model_files = self.get_hf_model_files(
1450
- model_name, ModelFormat.SAFETENSORS
1451
- )
1452
- if (
1453
- safetensors_model_files
1454
- and AQUA_MODEL_ARTIFACT_CONFIG in safetensors_model_files
1455
- ):
1456
- hf_download_config_present = True
1457
- gguf_model_files = self.get_hf_model_files(model_name, ModelFormat.GGUF)
1458
- else:
1459
- safetensors_model_files = self.get_model_files(
1460
- import_model_details.os_path, ModelFormat.SAFETENSORS
1461
- )
1462
- gguf_model_files = self.get_model_files(
1463
- import_model_details.os_path, ModelFormat.GGUF
1464
- )
1465
-
1466
- if not (safetensors_model_files or gguf_model_files):
1467
- raise AquaRuntimeError(
1468
- f"The model {model_name} does not contain either {ModelFormat.SAFETENSORS} "
1469
- f"or {ModelFormat.GGUF} files in {import_model_details.os_path} or Hugging Face repository. "
1470
- f"Please check if the path is correct or the model artifacts are available at this location."
1471
- )
1472
-
1473
- if verified_model:
1474
- aqua_model = self.to_aqua_model(verified_model, self.region)
1475
- model_formats = aqua_model.model_formats
1476
- else:
1477
- if safetensors_model_files:
1478
- model_formats.append(ModelFormat.SAFETENSORS)
1479
- if gguf_model_files:
1480
- model_formats.append(ModelFormat.GGUF)
1481
-
1482
- # get tags for models from hf
1483
- if import_model_details.download_from_hf:
1484
- model_info = get_hf_model_info(repo_id=model_name)
1485
-
1486
- try:
1487
- license_value = UNKNOWN
1488
- if model_info.tags:
1489
- license_tag = next(
1490
- (
1491
- tag
1492
- for tag in model_info.tags
1493
- if tag.startswith("license:")
1494
- ),
1495
- UNKNOWN,
1496
- )
1497
- license_value = (
1498
- license_tag.split(":")[1] if license_tag else UNKNOWN
1499
- )
1500
-
1501
- hf_tags = {
1502
- Tags.TASK: (model_info and model_info.pipeline_tag) or UNKNOWN,
1503
- Tags.ORGANIZATION: (
1504
- model_info.author
1505
- if model_info and hasattr(model_info, "author")
1506
- else UNKNOWN
1507
- ),
1508
- Tags.LICENSE: license_value,
1509
- }
1510
- validation_result.tags = hf_tags
1511
- except Exception as ex:
1512
- logger.debug(
1513
- f"An error occurred while getting tag information for model {model_name}. "
1514
- f"Error: {str(ex)}"
1515
- )
1516
-
1517
- validation_result.model_formats = model_formats
1518
-
1519
- # now as we know that at least one type of model files exist, validate the content of oss path.
1520
- # for safetensors, we check if config.json files exist, and for gguf format we check if files with
1521
- # gguf extension exist.
1522
- if {ModelFormat.SAFETENSORS, ModelFormat.GGUF}.issubset(set(model_formats)):
1523
- if (
1524
- import_model_details.inference_container.lower()
1525
- == InferenceContainerTypeFamily.AQUA_LLAMA_CPP_CONTAINER_FAMILY
1526
- ):
1527
- self._validate_gguf_format(
1528
- import_model_details=import_model_details,
1529
- verified_model=verified_model,
1530
- gguf_model_files=gguf_model_files,
1531
- validation_result=validation_result,
1532
- model_name=model_name,
1533
- )
1534
- else:
1535
- self._validate_safetensor_format(
1536
- import_model_details=import_model_details,
1537
- verified_model=verified_model,
1538
- validation_result=validation_result,
1539
- hf_download_config_present=hf_download_config_present,
1540
- model_name=model_name,
1541
- )
1542
- elif ModelFormat.SAFETENSORS in model_formats:
1543
- self._validate_safetensor_format(
1544
- import_model_details=import_model_details,
1545
- verified_model=verified_model,
1546
- validation_result=validation_result,
1547
- hf_download_config_present=hf_download_config_present,
1548
- model_name=model_name,
1549
- )
1550
- elif ModelFormat.GGUF in model_formats:
1551
- self._validate_gguf_format(
1552
- import_model_details=import_model_details,
1553
- verified_model=verified_model,
1554
- gguf_model_files=gguf_model_files,
1555
- validation_result=validation_result,
1556
- model_name=model_name,
1557
- )
1558
-
1559
- return validation_result
1560
-
1561
- @staticmethod
1562
- def _validate_safetensor_format(
1563
- import_model_details: ImportModelDetails = None,
1564
- verified_model: DataScienceModel = None,
1565
- validation_result: ModelValidationResult = None,
1566
- hf_download_config_present: bool = None,
1567
- model_name: str = None,
1568
- ):
1569
- if import_model_details.download_from_hf:
1570
- # validates config.json exists for safetensors model from huggingface
1571
- if not (
1572
- hf_download_config_present
1573
- or import_model_details.ignore_model_artifact_check
1574
- ):
1575
- raise AquaRuntimeError(
1576
- f"The model {model_name} does not contain {AQUA_MODEL_ARTIFACT_CONFIG} file as required "
1577
- f"by {ModelFormat.SAFETENSORS} format model."
1578
- f" Please check if the model name is correct in Hugging Face repository."
1579
- )
1580
- validation_result.telemetry_model_name = model_name
1581
- else:
1582
- # validate if config.json is available from object storage, and get model name for telemetry
1583
- model_config = None
1584
- try:
1585
- model_config = load_config(
1586
- file_path=import_model_details.os_path,
1587
- config_file_name=AQUA_MODEL_ARTIFACT_CONFIG,
1588
- )
1589
- except Exception as ex:
1590
- message = (
1591
- f"The model path {import_model_details.os_path} does not contain the file config.json. "
1592
- f"Please check if the path is correct or the model artifacts are available at this location."
1593
- )
1594
- if not import_model_details.ignore_model_artifact_check:
1595
- logger.error(
1596
- f"{message}\n"
1597
- f"Details: {ex.reason if isinstance(ex, AquaFileNotFoundError) else str(ex)}"
1598
- )
1599
- raise AquaRuntimeError(message) from ex
1600
- else:
1601
- logger.warning(
1602
- f"{message}\n"
1603
- f"Proceeding with model registration as ignore_model_artifact_check field is set."
1604
- )
1605
-
1606
- if verified_model:
1607
- # model_type validation, log message if metadata field doesn't match.
1608
- try:
1609
- metadata_model_type = verified_model.custom_metadata_list.get(
1610
- AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE
1611
- ).value
1612
- if metadata_model_type and model_config is not None:
1613
- if AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE in model_config:
1614
- if (
1615
- model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE]
1616
- != metadata_model_type
1617
- ):
1618
- logger.debug(
1619
- f"The {AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE} attribute in {AQUA_MODEL_ARTIFACT_CONFIG}"
1620
- f" at {import_model_details.os_path} is invalid, expected {metadata_model_type} for "
1621
- f"the model {model_name}. Please check if the path is correct or "
1622
- f"the correct model artifacts are available at this location."
1623
- f""
1624
- )
1625
- else:
1626
- logger.debug(
1627
- f"Could not find {AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE} attribute in "
1628
- f"{AQUA_MODEL_ARTIFACT_CONFIG}. Proceeding with model registration."
1629
- )
1630
- except Exception as ex:
1631
- # todo: raise exception if model_type doesn't match. Currently log message and pass since service
1632
- # models do not have this metadata.
1633
- logger.debug(
1634
- f"Error occurred while processing metadata for model {model_name}. "
1635
- f"Exception: {str(ex)}"
1636
- )
1637
- validation_result.telemetry_model_name = verified_model.display_name
1638
- elif (
1639
- model_config is not None
1640
- and AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME in model_config
1641
- ):
1642
- validation_result.telemetry_model_name = f"{AQUA_MODEL_TYPE_CUSTOM}_{model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME]}"
1643
- elif (
1644
- model_config is not None
1645
- and AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE in model_config
1646
- ):
1647
- validation_result.telemetry_model_name = f"{AQUA_MODEL_TYPE_CUSTOM}_{model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE]}"
1648
- else:
1649
- validation_result.telemetry_model_name = AQUA_MODEL_TYPE_CUSTOM
1650
-
1651
- @staticmethod
1652
- def _validate_gguf_format(
1653
- import_model_details: ImportModelDetails = None,
1654
- verified_model: DataScienceModel = None,
1655
- gguf_model_files: List[str] = None,
1656
- validation_result: ModelValidationResult = None,
1657
- model_name: str = None,
1658
- ):
1659
- if import_model_details.finetuning_container:
1660
- raise AquaValueError(
1661
- "Fine-tuning is currently not supported with GGUF model format."
1662
- )
1663
- if verified_model:
1664
- try:
1665
- model_file = verified_model.custom_metadata_list.get(
1666
- AQUA_MODEL_ARTIFACT_FILE
1667
- ).value
1668
- except ValueError as err:
1669
- raise AquaRuntimeError(
1670
- f"The model {verified_model.display_name} does not contain the custom metadata {AQUA_MODEL_ARTIFACT_FILE}. "
1671
- f"Please check if the model has the valid metadata."
1672
- ) from err
1673
- else:
1674
- model_file = import_model_details.model_file
1675
-
1676
- model_files = gguf_model_files
1677
- # todo: have a separate error validation class for different type of error messages.
1678
- if model_file:
1679
- if model_file not in model_files:
1680
- raise AquaRuntimeError(
1681
- f"The model path {import_model_details.os_path} or the Hugging Face "
1682
- f"model repository for {model_name} does not contain the file "
1683
- f"{model_file}. Please check if the path is correct or the model "
1684
- f"artifacts are available at this location."
1685
- )
1686
- else:
1687
- validation_result.model_file = model_file
1688
- elif len(model_files) == 0:
1689
- raise AquaRuntimeError(
1690
- f"The model path {import_model_details.os_path} or the Hugging Face model "
1691
- f"repository for {model_name} does not contain any GGUF format files. "
1692
- f"Please check if the path is correct or the model artifacts are available "
1693
- f"at this location."
1694
- )
1695
- elif len(model_files) > 1:
1696
- raise AquaRuntimeError(
1697
- f"The model path {import_model_details.os_path} or the Hugging Face model "
1698
- f"repository for {model_name} contains multiple GGUF format files. "
1699
- f"Please specify the file that needs to be deployed using the model_file "
1700
- f"parameter."
1701
- )
1702
- else:
1703
- validation_result.model_file = model_files[0]
1704
-
1705
- if verified_model:
1706
- validation_result.telemetry_model_name = verified_model.display_name
1707
- elif import_model_details.download_from_hf:
1708
- validation_result.telemetry_model_name = model_name
1709
- else:
1710
- validation_result.telemetry_model_name = AQUA_MODEL_TYPE_CUSTOM
1711
-
1712
- @staticmethod
1713
- def _download_model_from_hf(
1714
- model_name: str,
1715
- os_path: str,
1716
- local_dir: str = None,
1717
- allow_patterns: List[str] = None,
1718
- ignore_patterns: List[str] = None,
1719
- ) -> str:
1720
- """This helper function downloads the model artifact from Hugging Face to a local folder, then uploads
1721
- to object storage location.
1722
-
1723
- Parameters
1724
- ----------
1725
- model_name (str): The huggingface model name.
1726
- os_path (str): The OS path where the model files are located.
1727
- local_dir (str): The local temp dir to store the huggingface model.
1728
- allow_patterns (list): Model files matching at least one pattern are downloaded.
1729
- Example: ["*.json"] will download all .json files. ["folder/*"] will download all files under `folder`.
1730
- Patterns are Standard Wildcards (globbing patterns) and rules can be found here: https://docs.python.org/3/library/fnmatch.html
1731
- ignore_patterns (list): Model files matching any of the patterns are not downloaded.
1732
- Example: ["*.json"] will ignore all .json files. ["folder/*"] will ignore all files under `folder`.
1733
- Patterns are Standard Wildcards (globbing patterns) and rules can be found here: https://docs.python.org/3/library/fnmatch.html
1734
-
1735
- Returns
1736
- -------
1737
- model_artifact_path (str): Location where the model artifacts are downloaded.
1738
- """
1739
- # Download the model from hub
1740
- if local_dir:
1741
- local_dir = os.path.join(local_dir, model_name)
1742
- os.makedirs(local_dir, exist_ok=True)
1743
-
1744
- # if local_dir is not set, the return value points to the cached data folder
1745
- local_dir = snapshot_download(
1746
- repo_id=model_name,
1747
- local_dir=local_dir,
1748
- allow_patterns=allow_patterns,
1749
- ignore_patterns=ignore_patterns,
1750
- )
1751
- # Upload to object storage and skip .cache/huggingface/ folder
1752
- logger.debug(
1753
- f"Uploading local artifacts from local directory {local_dir} to {os_path}."
1754
- )
1755
- # Upload to object storage
1756
- model_artifact_path = upload_folder(
1757
- os_path=os_path,
1758
- local_dir=local_dir,
1759
- model_name=model_name,
1760
- exclude_pattern=f"{HF_METADATA_FOLDER}*",
1761
- )
1762
-
1763
- return model_artifact_path
1764
-
1765
- def register(
1766
- self, import_model_details: ImportModelDetails = None, **kwargs
1767
- ) -> AquaModel:
1768
- """Loads the model from object storage and registers as Model in Data Science Model catalog
1769
- The inference container and finetuning container could be of type Service Managed Container(SMC) or custom.
1770
- If it is custom, full container URI is expected. If it of type SMC, only the container family name is expected.\n
1771
- For detailed information about CLI flags see: https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/ai-quick-actions/cli-tips.md#register-model
1772
-
1773
- Args:
1774
- import_model_details (ImportModelDetails): Model details for importing the model.
1775
- kwargs:
1776
- model (str): name of the model or OCID of the service model that has inference and finetuning information
1777
- os_path (str): Object storage destination URI to store the downloaded model. Format: oci://bucket-name@namespace/prefix
1778
- inference_container (str): selects service defaults
1779
- finetuning_container (str): selects service defaults
1780
- allow_patterns (list): Model files matching at least one pattern are downloaded.
1781
- Example: ["*.json"] will download all .json files. ["folder/*"] will download all files under `folder`.
1782
- Patterns are Standard Wildcards (globbing patterns) and rules can be found here: https://docs.python.org/3/library/fnmatch.html
1783
- ignore_patterns (list): Model files matching any of the patterns are not downloaded.
1784
- Example: ["*.json"] will ignore all .json files. ["folder/*"] will ignore all files under `folder`.
1785
- Patterns are Standard Wildcards (globbing patterns) and rules can be found here: https://docs.python.org/3/library/fnmatch.html
1786
- cleanup_model_cache (bool): Deletes downloaded files from local machine after model is successfully
1787
- registered. Set to True by default.
1788
-
1789
- Returns:
1790
- AquaModel:
1791
- The registered model as a AquaModel object.
1792
- """
1793
- if not import_model_details:
1794
- import_model_details = ImportModelDetails(**kwargs)
1795
-
1796
- # If OCID of a model is passed, we need to copy the defaults for Tags and metadata from the service model.
1797
- verified_model: Optional[DataScienceModel] = None
1798
- if (
1799
- import_model_details.model.startswith("ocid")
1800
- and "datasciencemodel" in import_model_details.model
1801
- ):
1802
- logger.info(f"Fetching details for model {import_model_details.model}.")
1803
- verified_model = DataScienceModel.from_id(import_model_details.model)
1804
- else:
1805
- # If users passes model name, check if there is model with the same name in the service model catalog. If it is there, then use that model
1806
- model_service_id = self._find_matching_aqua_model(
1807
- import_model_details.model
1808
- )
1809
- if model_service_id:
1810
- logger.info(
1811
- f"Found service model for {import_model_details.model}: {model_service_id}"
1812
- )
1813
- verified_model = DataScienceModel.from_id(model_service_id)
1814
-
1815
- # Copy the model name from the service model if `model` is ocid
1816
- model_name = (
1817
- verified_model.display_name
1818
- if verified_model
1819
- else import_model_details.model
1820
- )
1821
-
1822
- # validate model and artifact
1823
- validation_result = self._validate_model(
1824
- import_model_details=import_model_details,
1825
- model_name=model_name,
1826
- verified_model=verified_model,
1827
- )
1828
-
1829
- # download model from hugginface if indicates
1830
- if import_model_details.download_from_hf:
1831
- artifact_path = self._download_model_from_hf(
1832
- model_name=model_name,
1833
- os_path=import_model_details.os_path,
1834
- local_dir=import_model_details.local_dir,
1835
- allow_patterns=import_model_details.allow_patterns,
1836
- ignore_patterns=import_model_details.ignore_patterns,
1837
- ).rstrip("/")
1838
- else:
1839
- artifact_path = import_model_details.os_path.rstrip("/")
1840
-
1841
- # Create Model catalog entry with pass by reference
1842
- ds_model = self._create_model_catalog_entry(
1843
- os_path=artifact_path,
1844
- model_name=model_name,
1845
- inference_container=import_model_details.inference_container,
1846
- finetuning_container=import_model_details.finetuning_container,
1847
- verified_model=verified_model,
1848
- validation_result=validation_result,
1849
- compartment_id=import_model_details.compartment_id,
1850
- project_id=import_model_details.project_id,
1851
- inference_container_uri=import_model_details.inference_container_uri,
1852
- freeform_tags=import_model_details.freeform_tags,
1853
- defined_tags=import_model_details.defined_tags,
1854
- )
1855
- # registered model will always have inference and evaluation container, but
1856
- # fine-tuning container may be not set
1857
- inference_container = ds_model.custom_metadata_list.get(
1858
- ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
1859
- ModelCustomMetadataItem(key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER),
1860
- ).value
1861
- inference_container_uri = ds_model.custom_metadata_list.get(
1862
- ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI,
1863
- ModelCustomMetadataItem(
1864
- key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI
1865
- ),
1866
- ).value
1867
- evaluation_container = ds_model.custom_metadata_list.get(
1868
- ModelCustomMetadataFields.EVALUATION_CONTAINER,
1869
- ModelCustomMetadataItem(key=ModelCustomMetadataFields.EVALUATION_CONTAINER),
1870
- ).value
1871
- finetuning_container: str = ds_model.custom_metadata_list.get(
1872
- ModelCustomMetadataFields.FINETUNE_CONTAINER,
1873
- ModelCustomMetadataItem(key=ModelCustomMetadataFields.FINETUNE_CONTAINER),
1874
- ).value
1875
-
1876
- aqua_model_attributes = dict(
1877
- **self._process_model(ds_model, self.region),
1878
- project_id=ds_model.project_id,
1879
- inference_container=inference_container,
1880
- inference_container_uri=inference_container_uri,
1881
- finetuning_container=finetuning_container,
1882
- evaluation_container=evaluation_container,
1883
- artifact_location=artifact_path,
1884
- )
1885
-
1886
- self.telemetry.record_event_async(
1887
- category="aqua/model",
1888
- action="register",
1889
- detail=validation_result.telemetry_model_name,
1890
- )
1891
-
1892
- if (
1893
- import_model_details.download_from_hf
1894
- and import_model_details.cleanup_model_cache
1895
- ):
1896
- cleanup_local_hf_model_artifact(
1897
- model_name=model_name, local_dir=import_model_details.local_dir
1898
- )
1899
-
1900
- return AquaModel(**aqua_model_attributes)
1901
-
1902
- def _if_show(self, model: DataScienceModel) -> bool:
1903
- """Determine if the given model should be return by `list`."""
1904
- if model.freeform_tags is None:
1905
- return False
1906
-
1907
- TARGET_TAGS = model.freeform_tags.keys()
1908
- return Tags.AQUA_TAG in TARGET_TAGS or Tags.AQUA_TAG.lower() in TARGET_TAGS
1909
-
1910
- def _load_icon(self, model_name: str) -> str:
1911
- """Loads icon."""
1912
-
1913
- # TODO: switch to the official logo
1914
- try:
1915
- return create_word_icon(model_name, return_as_datauri=True)
1916
- except Exception as e:
1917
- logger.debug(f"Failed to load icon for the model={model_name}: {str(e)}.")
1918
- return None
1919
-
1920
- def _rqs(self, compartment_id: str, model_type="FT", **kwargs):
1921
- """Use RQS to fetch models in the user tenancy."""
1922
- if model_type == ModelType.FT:
1923
- filter_tag = Tags.AQUA_FINE_TUNED_MODEL_TAG
1924
- elif model_type == ModelType.BASE:
1925
- filter_tag = Tags.BASE_MODEL_CUSTOM
1926
- # elif model_type == ModelType.MULTIMODEL:
1927
- # filter_tag = Tags.MULTIMODEL_TYPE_TAG
1928
- else:
1929
- raise AquaValueError(
1930
- f"Model of type {model_type} is unknown. The values should be in {ModelType.values()}"
1931
- )
1932
-
1933
- condition_tags = f"&& (freeformTags.key = '{Tags.AQUA_TAG}' && freeformTags.key = '{filter_tag}')"
1934
- condition_lifecycle = "&& lifecycleState = 'ACTIVE'"
1935
- query = f"query datasciencemodel resources where (compartmentId = '{compartment_id}' {condition_lifecycle} {condition_tags})"
1936
- logger.info(query)
1937
- logger.info(f"tenant_id={TENANCY_OCID}")
1938
- return OCIResource.search(
1939
- query, type=SEARCH_TYPE.STRUCTURED, tenant_id=TENANCY_OCID, **kwargs
1940
- )
1941
-
1942
- @staticmethod
1943
- def _build_search_text(tags: dict, description: str = None) -> str:
1944
- """Constructs search_text field in response."""
1945
- description = description or ""
1946
- tags_text = (
1947
- ",".join(str(v) for v in tags.values()) if isinstance(tags, dict) else ""
1948
- )
1949
- separator = " " if description else ""
1950
- return f"{description}{separator}{tags_text}"
1951
-
1952
- @telemetry(entry_point="plugin=model&action=load_readme", name="aqua")
1953
- def load_readme(self, model_id: str) -> AquaModelReadme:
1954
- """Loads the readme or the model card for the given model.
1955
-
1956
- Parameters
1957
- ----------
1958
- model_id: str
1959
- The model id.
1960
-
1961
- Returns
1962
- -------
1963
- AquaModelReadme:
1964
- The instance of AquaModelReadme.
1965
- """
1966
- oci_model = self.ds_client.get_model(model_id).data
1967
- artifact_path = get_artifact_path(oci_model.custom_metadata_list)
1968
- if not artifact_path:
1969
- raise AquaRuntimeError(
1970
- f"Readme could not be loaded. Failed to get artifact path from custom metadata for"
1971
- f"the model {model_id}."
1972
- )
1973
-
1974
- content = ""
1975
- try:
1976
- content = self.ds_client.get_model_defined_metadatum_artifact_content(
1977
- model_id, AquaModelMetadataKeys.README
1978
- ).data.content.decode("utf-8", errors="ignore")
1979
- logger.info(f"Fetched {README} from defined metadata for model: {model_id}")
1980
- except Exception as ex:
1981
- logger.error(
1982
- f"Readme could not be found for model: {model_id} in defined metadata : {str(ex)}"
1983
- )
1984
- artifact_path = get_artifact_path(oci_model.custom_metadata_list)
1985
- readme_path = os.path.join(os.path.dirname(artifact_path), "artifact")
1986
- if not is_path_exists(readme_path):
1987
- readme_path = os.path.join(artifact_path.rstrip("/"), "artifact")
1988
- if not is_path_exists(readme_path):
1989
- readme_path = f"{artifact_path.rstrip('/')}/"
1990
-
1991
- readme_file_path = os.path.join(readme_path, README)
1992
- logger.info(f"Fetching {README} from {readme_file_path}")
1993
- if is_path_exists(readme_file_path):
1994
- try:
1995
- content = str(read_file(readme_file_path, auth=default_signer()))
1996
- except Exception as e:
1997
- logger.debug(
1998
- f"Error occurred while fetching config {README} at path {readme_file_path} : {str(e)}"
1999
- )
2000
- return AquaModelReadme(id=model_id, model_card=content)
2001
-
2002
- @telemetry(entry_point="plugin=model&action=load_license", name="aqua")
2003
- def load_license(self, model_id: str) -> AquaModelLicense:
2004
- """Loads the license full text for the given model.
2005
-
2006
- Parameters
2007
- ----------
2008
- model_id: str
2009
- The model id.
2010
-
2011
- Returns
2012
- -------
2013
- AquaModelLicense:
2014
- The instance of AquaModelLicense.
2015
- """
2016
- oci_model = self.ds_client.get_model(model_id).data
2017
- artifact_path = get_artifact_path(oci_model.custom_metadata_list)
2018
- if not artifact_path:
2019
- raise AquaRuntimeError(
2020
- f"License could not be loaded. Failed to get artifact path from custom metadata for"
2021
- f"the model {model_id}."
2022
- )
2023
-
2024
- content = ""
2025
- try:
2026
- content = self.ds_client.get_model_defined_metadatum_artifact_content(
2027
- model_id, AquaModelMetadataKeys.LICENSE
2028
- ).data.content.decode("utf-8", errors="ignore")
2029
- logger.info(
2030
- f"Fetched {LICENSE} from defined metadata for model: {model_id}"
2031
- )
2032
- except Exception as ex:
2033
- logger.error(
2034
- f"License could not be found for model: {model_id} in defined metadata : {str(ex)}"
2035
- )
2036
- artifact_path = get_artifact_path(oci_model.custom_metadata_list)
2037
- license_path = os.path.join(os.path.dirname(artifact_path), "config")
2038
- if not is_path_exists(license_path):
2039
- license_path = os.path.join(artifact_path.rstrip("/"), "config")
2040
- if not is_path_exists(license_path):
2041
- license_path = f"{artifact_path.rstrip('/')}/"
2042
-
2043
- license_file_path = os.path.join(license_path, LICENSE)
2044
- logger.info(f"Fetching {LICENSE} from {license_file_path}")
2045
- if is_path_exists(license_file_path):
2046
- try:
2047
- content = str(read_file(license_file_path, auth=default_signer()))
2048
- except Exception as e:
2049
- logger.debug(
2050
- f"Error occurred while fetching config {LICENSE} at path {license_path} : {str(e)}"
2051
- )
2052
- return AquaModelLicense(id=model_id, license=content)
2053
-
2054
- def _find_matching_aqua_model(self, model_id: str) -> Optional[str]:
2055
- """
2056
- Finds a matching model in AQUA based on the model ID from list of verified models.
2057
-
2058
- Parameters
2059
- ----------
2060
- model_id (str): Verified model ID to match.
2061
-
2062
- Returns
2063
- -------
2064
- Optional[str]
2065
- Returns model ocid that matches the model in the service catalog else returns None.
2066
- """
2067
- # Convert the model ID to lowercase once
2068
- model_id_lower = model_id.lower()
2069
-
2070
- aqua_model_list = self.list()
2071
-
2072
- for aqua_model_summary in aqua_model_list:
2073
- if aqua_model_summary.name.lower() == model_id_lower:
2074
- logger.info(
2075
- f"Found matching verified model id {aqua_model_summary.id} for the model {model_id}"
2076
- )
2077
- return aqua_model_summary.id
2078
-
2079
- return None