oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (857) hide show
  1. ads/aqua/__init__.py +40 -0
  2. ads/aqua/app.py +506 -0
  3. ads/aqua/cli.py +96 -0
  4. ads/aqua/client/__init__.py +3 -0
  5. ads/aqua/client/client.py +836 -0
  6. ads/aqua/client/openai_client.py +305 -0
  7. ads/aqua/common/__init__.py +5 -0
  8. ads/aqua/common/decorator.py +125 -0
  9. ads/aqua/common/entities.py +269 -0
  10. ads/aqua/common/enums.py +122 -0
  11. ads/aqua/common/errors.py +109 -0
  12. ads/aqua/common/utils.py +1285 -0
  13. ads/aqua/config/__init__.py +4 -0
  14. ads/aqua/config/container_config.py +248 -0
  15. ads/aqua/config/evaluation/__init__.py +4 -0
  16. ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
  17. ads/aqua/config/utils/__init__.py +4 -0
  18. ads/aqua/config/utils/serializer.py +339 -0
  19. ads/aqua/constants.py +116 -0
  20. ads/aqua/data.py +14 -0
  21. ads/aqua/dummy_data/icon.txt +1 -0
  22. ads/aqua/dummy_data/oci_model_deployments.json +56 -0
  23. ads/aqua/dummy_data/oci_models.json +1 -0
  24. ads/aqua/dummy_data/readme.md +26 -0
  25. ads/aqua/evaluation/__init__.py +8 -0
  26. ads/aqua/evaluation/constants.py +53 -0
  27. ads/aqua/evaluation/entities.py +186 -0
  28. ads/aqua/evaluation/errors.py +70 -0
  29. ads/aqua/evaluation/evaluation.py +1814 -0
  30. ads/aqua/extension/__init__.py +42 -0
  31. ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
  32. ads/aqua/extension/base_handler.py +90 -0
  33. ads/aqua/extension/common_handler.py +121 -0
  34. ads/aqua/extension/common_ws_msg_handler.py +36 -0
  35. ads/aqua/extension/deployment_handler.py +298 -0
  36. ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
  37. ads/aqua/extension/errors.py +30 -0
  38. ads/aqua/extension/evaluation_handler.py +129 -0
  39. ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
  40. ads/aqua/extension/finetune_handler.py +96 -0
  41. ads/aqua/extension/model_handler.py +390 -0
  42. ads/aqua/extension/models/__init__.py +0 -0
  43. ads/aqua/extension/models/ws_models.py +145 -0
  44. ads/aqua/extension/models_ws_msg_handler.py +50 -0
  45. ads/aqua/extension/ui_handler.py +282 -0
  46. ads/aqua/extension/ui_websocket_handler.py +130 -0
  47. ads/aqua/extension/utils.py +133 -0
  48. ads/aqua/finetuning/__init__.py +7 -0
  49. ads/aqua/finetuning/constants.py +23 -0
  50. ads/aqua/finetuning/entities.py +181 -0
  51. ads/aqua/finetuning/finetuning.py +749 -0
  52. ads/aqua/model/__init__.py +8 -0
  53. ads/aqua/model/constants.py +60 -0
  54. ads/aqua/model/entities.py +385 -0
  55. ads/aqua/model/enums.py +32 -0
  56. ads/aqua/model/model.py +2114 -0
  57. ads/aqua/modeldeployment/__init__.py +8 -0
  58. ads/aqua/modeldeployment/constants.py +10 -0
  59. ads/aqua/modeldeployment/deployment.py +1326 -0
  60. ads/aqua/modeldeployment/entities.py +653 -0
  61. ads/aqua/modeldeployment/inference.py +74 -0
  62. ads/aqua/modeldeployment/utils.py +543 -0
  63. ads/aqua/resources/gpu_shapes_index.json +94 -0
  64. ads/aqua/server/__init__.py +4 -0
  65. ads/aqua/server/__main__.py +24 -0
  66. ads/aqua/server/app.py +47 -0
  67. ads/aqua/server/aqua_spec.yml +1291 -0
  68. ads/aqua/training/__init__.py +4 -0
  69. ads/aqua/training/exceptions.py +476 -0
  70. ads/aqua/ui.py +499 -0
  71. ads/automl/__init__.py +9 -0
  72. ads/automl/driver.py +330 -0
  73. ads/automl/provider.py +975 -0
  74. ads/bds/__init__.py +5 -0
  75. ads/bds/auth.py +127 -0
  76. ads/bds/big_data_service.py +255 -0
  77. ads/catalog/__init__.py +19 -0
  78. ads/catalog/model.py +1576 -0
  79. ads/catalog/notebook.py +461 -0
  80. ads/catalog/project.py +468 -0
  81. ads/catalog/summary.py +178 -0
  82. ads/common/__init__.py +11 -0
  83. ads/common/analyzer.py +65 -0
  84. ads/common/artifact/.model-ignore +63 -0
  85. ads/common/artifact/__init__.py +10 -0
  86. ads/common/auth.py +1122 -0
  87. ads/common/card_identifier.py +83 -0
  88. ads/common/config.py +647 -0
  89. ads/common/data.py +165 -0
  90. ads/common/decorator/__init__.py +9 -0
  91. ads/common/decorator/argument_to_case.py +88 -0
  92. ads/common/decorator/deprecate.py +69 -0
  93. ads/common/decorator/require_nonempty_arg.py +65 -0
  94. ads/common/decorator/runtime_dependency.py +178 -0
  95. ads/common/decorator/threaded.py +97 -0
  96. ads/common/decorator/utils.py +35 -0
  97. ads/common/dsc_file_system.py +303 -0
  98. ads/common/error.py +14 -0
  99. ads/common/extended_enum.py +81 -0
  100. ads/common/function/__init__.py +5 -0
  101. ads/common/function/fn_util.py +142 -0
  102. ads/common/function/func_conf.yaml +25 -0
  103. ads/common/ipython.py +76 -0
  104. ads/common/model.py +679 -0
  105. ads/common/model_artifact.py +1759 -0
  106. ads/common/model_artifact_schema.json +107 -0
  107. ads/common/model_export_util.py +664 -0
  108. ads/common/model_metadata.py +24 -0
  109. ads/common/object_storage_details.py +296 -0
  110. ads/common/oci_client.py +175 -0
  111. ads/common/oci_datascience.py +46 -0
  112. ads/common/oci_logging.py +1144 -0
  113. ads/common/oci_mixin.py +957 -0
  114. ads/common/oci_resource.py +136 -0
  115. ads/common/serializer.py +559 -0
  116. ads/common/utils.py +1852 -0
  117. ads/common/word_lists.py +1491 -0
  118. ads/common/work_request.py +189 -0
  119. ads/data_labeling/__init__.py +13 -0
  120. ads/data_labeling/boundingbox.py +253 -0
  121. ads/data_labeling/constants.py +47 -0
  122. ads/data_labeling/data_labeling_service.py +244 -0
  123. ads/data_labeling/interface/__init__.py +5 -0
  124. ads/data_labeling/interface/loader.py +16 -0
  125. ads/data_labeling/interface/parser.py +16 -0
  126. ads/data_labeling/interface/reader.py +23 -0
  127. ads/data_labeling/loader/__init__.py +5 -0
  128. ads/data_labeling/loader/file_loader.py +241 -0
  129. ads/data_labeling/metadata.py +110 -0
  130. ads/data_labeling/mixin/__init__.py +5 -0
  131. ads/data_labeling/mixin/data_labeling.py +232 -0
  132. ads/data_labeling/ner.py +129 -0
  133. ads/data_labeling/parser/__init__.py +5 -0
  134. ads/data_labeling/parser/dls_record_parser.py +388 -0
  135. ads/data_labeling/parser/export_metadata_parser.py +94 -0
  136. ads/data_labeling/parser/export_record_parser.py +473 -0
  137. ads/data_labeling/reader/__init__.py +5 -0
  138. ads/data_labeling/reader/dataset_reader.py +574 -0
  139. ads/data_labeling/reader/dls_record_reader.py +121 -0
  140. ads/data_labeling/reader/export_record_reader.py +62 -0
  141. ads/data_labeling/reader/jsonl_reader.py +75 -0
  142. ads/data_labeling/reader/metadata_reader.py +203 -0
  143. ads/data_labeling/reader/record_reader.py +263 -0
  144. ads/data_labeling/record.py +52 -0
  145. ads/data_labeling/visualizer/__init__.py +5 -0
  146. ads/data_labeling/visualizer/image_visualizer.py +525 -0
  147. ads/data_labeling/visualizer/text_visualizer.py +357 -0
  148. ads/database/__init__.py +5 -0
  149. ads/database/connection.py +338 -0
  150. ads/dataset/__init__.py +10 -0
  151. ads/dataset/capabilities.md +51 -0
  152. ads/dataset/classification_dataset.py +339 -0
  153. ads/dataset/correlation.py +226 -0
  154. ads/dataset/correlation_plot.py +563 -0
  155. ads/dataset/dask_series.py +173 -0
  156. ads/dataset/dataframe_transformer.py +110 -0
  157. ads/dataset/dataset.py +1979 -0
  158. ads/dataset/dataset_browser.py +360 -0
  159. ads/dataset/dataset_with_target.py +995 -0
  160. ads/dataset/exception.py +25 -0
  161. ads/dataset/factory.py +987 -0
  162. ads/dataset/feature_engineering_transformer.py +35 -0
  163. ads/dataset/feature_selection.py +107 -0
  164. ads/dataset/forecasting_dataset.py +26 -0
  165. ads/dataset/helper.py +1450 -0
  166. ads/dataset/label_encoder.py +99 -0
  167. ads/dataset/mixin/__init__.py +5 -0
  168. ads/dataset/mixin/dataset_accessor.py +134 -0
  169. ads/dataset/pipeline.py +58 -0
  170. ads/dataset/plot.py +710 -0
  171. ads/dataset/progress.py +86 -0
  172. ads/dataset/recommendation.py +297 -0
  173. ads/dataset/recommendation_transformer.py +502 -0
  174. ads/dataset/regression_dataset.py +14 -0
  175. ads/dataset/sampled_dataset.py +1050 -0
  176. ads/dataset/target.py +98 -0
  177. ads/dataset/timeseries.py +18 -0
  178. ads/dbmixin/__init__.py +5 -0
  179. ads/dbmixin/db_pandas_accessor.py +153 -0
  180. ads/environment/__init__.py +9 -0
  181. ads/environment/ml_runtime.py +66 -0
  182. ads/evaluations/README.md +14 -0
  183. ads/evaluations/__init__.py +109 -0
  184. ads/evaluations/evaluation_plot.py +983 -0
  185. ads/evaluations/evaluator.py +1334 -0
  186. ads/evaluations/statistical_metrics.py +543 -0
  187. ads/experiments/__init__.py +9 -0
  188. ads/experiments/capabilities.md +0 -0
  189. ads/explanations/__init__.py +21 -0
  190. ads/explanations/base_explainer.py +142 -0
  191. ads/explanations/capabilities.md +83 -0
  192. ads/explanations/explainer.py +190 -0
  193. ads/explanations/mlx_global_explainer.py +1050 -0
  194. ads/explanations/mlx_interface.py +386 -0
  195. ads/explanations/mlx_local_explainer.py +287 -0
  196. ads/explanations/mlx_whatif_explainer.py +201 -0
  197. ads/feature_engineering/__init__.py +20 -0
  198. ads/feature_engineering/accessor/__init__.py +5 -0
  199. ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
  200. ads/feature_engineering/accessor/mixin/__init__.py +5 -0
  201. ads/feature_engineering/accessor/mixin/correlation.py +166 -0
  202. ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
  203. ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
  204. ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
  205. ads/feature_engineering/accessor/mixin/utils.py +65 -0
  206. ads/feature_engineering/accessor/series_accessor.py +431 -0
  207. ads/feature_engineering/adsimage/__init__.py +5 -0
  208. ads/feature_engineering/adsimage/image.py +192 -0
  209. ads/feature_engineering/adsimage/image_reader.py +170 -0
  210. ads/feature_engineering/adsimage/interface/__init__.py +5 -0
  211. ads/feature_engineering/adsimage/interface/reader.py +19 -0
  212. ads/feature_engineering/adsstring/__init__.py +7 -0
  213. ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
  214. ads/feature_engineering/adsstring/string/__init__.py +8 -0
  215. ads/feature_engineering/data_schema.json +57 -0
  216. ads/feature_engineering/dataset/__init__.py +5 -0
  217. ads/feature_engineering/dataset/zip_code_data.py +42062 -0
  218. ads/feature_engineering/exceptions.py +40 -0
  219. ads/feature_engineering/feature_type/__init__.py +133 -0
  220. ads/feature_engineering/feature_type/address.py +184 -0
  221. ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
  222. ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
  223. ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
  224. ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
  225. ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
  226. ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
  227. ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
  228. ads/feature_engineering/feature_type/adsstring/string.py +258 -0
  229. ads/feature_engineering/feature_type/base.py +58 -0
  230. ads/feature_engineering/feature_type/boolean.py +183 -0
  231. ads/feature_engineering/feature_type/category.py +146 -0
  232. ads/feature_engineering/feature_type/constant.py +137 -0
  233. ads/feature_engineering/feature_type/continuous.py +151 -0
  234. ads/feature_engineering/feature_type/creditcard.py +314 -0
  235. ads/feature_engineering/feature_type/datetime.py +190 -0
  236. ads/feature_engineering/feature_type/discrete.py +134 -0
  237. ads/feature_engineering/feature_type/document.py +43 -0
  238. ads/feature_engineering/feature_type/gis.py +251 -0
  239. ads/feature_engineering/feature_type/handler/__init__.py +5 -0
  240. ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
  241. ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
  242. ads/feature_engineering/feature_type/handler/warnings.py +128 -0
  243. ads/feature_engineering/feature_type/integer.py +142 -0
  244. ads/feature_engineering/feature_type/ip_address.py +144 -0
  245. ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
  246. ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
  247. ads/feature_engineering/feature_type/lat_long.py +256 -0
  248. ads/feature_engineering/feature_type/object.py +43 -0
  249. ads/feature_engineering/feature_type/ordinal.py +132 -0
  250. ads/feature_engineering/feature_type/phone_number.py +135 -0
  251. ads/feature_engineering/feature_type/string.py +171 -0
  252. ads/feature_engineering/feature_type/text.py +93 -0
  253. ads/feature_engineering/feature_type/unknown.py +43 -0
  254. ads/feature_engineering/feature_type/zip_code.py +164 -0
  255. ads/feature_engineering/feature_type_manager.py +406 -0
  256. ads/feature_engineering/schema.py +795 -0
  257. ads/feature_engineering/utils.py +245 -0
  258. ads/feature_store/.readthedocs.yaml +19 -0
  259. ads/feature_store/README.md +65 -0
  260. ads/feature_store/__init__.py +9 -0
  261. ads/feature_store/common/__init__.py +0 -0
  262. ads/feature_store/common/enums.py +339 -0
  263. ads/feature_store/common/exceptions.py +18 -0
  264. ads/feature_store/common/spark_session_singleton.py +125 -0
  265. ads/feature_store/common/utils/__init__.py +0 -0
  266. ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
  267. ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
  268. ads/feature_store/common/utils/transformation_utils.py +82 -0
  269. ads/feature_store/common/utils/utility.py +403 -0
  270. ads/feature_store/data_validation/__init__.py +0 -0
  271. ads/feature_store/data_validation/great_expectation.py +129 -0
  272. ads/feature_store/dataset.py +1230 -0
  273. ads/feature_store/dataset_job.py +530 -0
  274. ads/feature_store/docs/Dockerfile +7 -0
  275. ads/feature_store/docs/Makefile +44 -0
  276. ads/feature_store/docs/conf.py +28 -0
  277. ads/feature_store/docs/requirements.txt +14 -0
  278. ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
  279. ads/feature_store/docs/source/cicd.rst +137 -0
  280. ads/feature_store/docs/source/conf.py +86 -0
  281. ads/feature_store/docs/source/data_versioning.rst +33 -0
  282. ads/feature_store/docs/source/dataset.rst +388 -0
  283. ads/feature_store/docs/source/dataset_job.rst +27 -0
  284. ads/feature_store/docs/source/demo.rst +70 -0
  285. ads/feature_store/docs/source/entity.rst +78 -0
  286. ads/feature_store/docs/source/feature_group.rst +624 -0
  287. ads/feature_store/docs/source/feature_group_job.rst +29 -0
  288. ads/feature_store/docs/source/feature_store.rst +122 -0
  289. ads/feature_store/docs/source/feature_store_class.rst +123 -0
  290. ads/feature_store/docs/source/feature_validation.rst +66 -0
  291. ads/feature_store/docs/source/figures/cicd.png +0 -0
  292. ads/feature_store/docs/source/figures/data_validation.png +0 -0
  293. ads/feature_store/docs/source/figures/data_versioning.png +0 -0
  294. ads/feature_store/docs/source/figures/dataset.gif +0 -0
  295. ads/feature_store/docs/source/figures/dataset.png +0 -0
  296. ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
  297. ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
  298. ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
  299. ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
  300. ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
  301. ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
  302. ads/feature_store/docs/source/figures/entity.png +0 -0
  303. ads/feature_store/docs/source/figures/feature_group.png +0 -0
  304. ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
  305. ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
  306. ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
  307. ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
  308. ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
  309. ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
  310. ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
  311. ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
  312. ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
  313. ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
  314. ads/feature_store/docs/source/figures/overview.png +0 -0
  315. ads/feature_store/docs/source/figures/resource_manager.png +0 -0
  316. ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
  317. ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
  318. ads/feature_store/docs/source/figures/stats_1.png +0 -0
  319. ads/feature_store/docs/source/figures/stats_2.png +0 -0
  320. ads/feature_store/docs/source/figures/stats_d.png +0 -0
  321. ads/feature_store/docs/source/figures/stats_fg.png +0 -0
  322. ads/feature_store/docs/source/figures/transformation.png +0 -0
  323. ads/feature_store/docs/source/figures/transformations.gif +0 -0
  324. ads/feature_store/docs/source/figures/validation.png +0 -0
  325. ads/feature_store/docs/source/figures/validation_fg.png +0 -0
  326. ads/feature_store/docs/source/figures/validation_results.png +0 -0
  327. ads/feature_store/docs/source/figures/validation_summary.png +0 -0
  328. ads/feature_store/docs/source/index.rst +81 -0
  329. ads/feature_store/docs/source/module.rst +8 -0
  330. ads/feature_store/docs/source/notebook.rst +94 -0
  331. ads/feature_store/docs/source/overview.rst +47 -0
  332. ads/feature_store/docs/source/quickstart.rst +176 -0
  333. ads/feature_store/docs/source/release_notes.rst +194 -0
  334. ads/feature_store/docs/source/setup_feature_store.rst +81 -0
  335. ads/feature_store/docs/source/statistics.rst +58 -0
  336. ads/feature_store/docs/source/transformation.rst +199 -0
  337. ads/feature_store/docs/source/ui.rst +65 -0
  338. ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
  339. ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
  340. ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
  341. ads/feature_store/entity.py +718 -0
  342. ads/feature_store/execution_strategy/__init__.py +0 -0
  343. ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
  344. ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
  345. ads/feature_store/execution_strategy/engine/__init__.py +0 -0
  346. ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
  347. ads/feature_store/execution_strategy/execution_strategy.py +113 -0
  348. ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
  349. ads/feature_store/execution_strategy/spark/__init__.py +0 -0
  350. ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
  351. ads/feature_store/feature.py +192 -0
  352. ads/feature_store/feature_group.py +1494 -0
  353. ads/feature_store/feature_group_expectation.py +346 -0
  354. ads/feature_store/feature_group_job.py +602 -0
  355. ads/feature_store/feature_lineage/__init__.py +0 -0
  356. ads/feature_store/feature_lineage/graphviz_service.py +180 -0
  357. ads/feature_store/feature_option_details.py +50 -0
  358. ads/feature_store/feature_statistics/__init__.py +0 -0
  359. ads/feature_store/feature_statistics/statistics_service.py +99 -0
  360. ads/feature_store/feature_store.py +699 -0
  361. ads/feature_store/feature_store_registrar.py +518 -0
  362. ads/feature_store/input_feature_detail.py +149 -0
  363. ads/feature_store/mixin/__init__.py +4 -0
  364. ads/feature_store/mixin/oci_feature_store.py +145 -0
  365. ads/feature_store/model_details.py +73 -0
  366. ads/feature_store/query/__init__.py +0 -0
  367. ads/feature_store/query/filter.py +266 -0
  368. ads/feature_store/query/generator/__init__.py +0 -0
  369. ads/feature_store/query/generator/query_generator.py +298 -0
  370. ads/feature_store/query/join.py +161 -0
  371. ads/feature_store/query/query.py +403 -0
  372. ads/feature_store/query/validator/__init__.py +0 -0
  373. ads/feature_store/query/validator/query_validator.py +57 -0
  374. ads/feature_store/response/__init__.py +0 -0
  375. ads/feature_store/response/response_builder.py +68 -0
  376. ads/feature_store/service/__init__.py +0 -0
  377. ads/feature_store/service/oci_dataset.py +139 -0
  378. ads/feature_store/service/oci_dataset_job.py +199 -0
  379. ads/feature_store/service/oci_entity.py +125 -0
  380. ads/feature_store/service/oci_feature_group.py +164 -0
  381. ads/feature_store/service/oci_feature_group_job.py +214 -0
  382. ads/feature_store/service/oci_feature_store.py +182 -0
  383. ads/feature_store/service/oci_lineage.py +87 -0
  384. ads/feature_store/service/oci_transformation.py +104 -0
  385. ads/feature_store/statistics/__init__.py +0 -0
  386. ads/feature_store/statistics/abs_feature_value.py +49 -0
  387. ads/feature_store/statistics/charts/__init__.py +0 -0
  388. ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
  389. ads/feature_store/statistics/charts/box_plot.py +148 -0
  390. ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
  391. ads/feature_store/statistics/charts/probability_distribution.py +68 -0
  392. ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
  393. ads/feature_store/statistics/feature_stat.py +126 -0
  394. ads/feature_store/statistics/generic_feature_value.py +33 -0
  395. ads/feature_store/statistics/statistics.py +41 -0
  396. ads/feature_store/statistics_config.py +101 -0
  397. ads/feature_store/templates/feature_store_template.yaml +45 -0
  398. ads/feature_store/transformation.py +499 -0
  399. ads/feature_store/validation_output.py +57 -0
  400. ads/hpo/__init__.py +9 -0
  401. ads/hpo/_imports.py +91 -0
  402. ads/hpo/ads_search_space.py +439 -0
  403. ads/hpo/distributions.py +325 -0
  404. ads/hpo/objective.py +280 -0
  405. ads/hpo/search_cv.py +1657 -0
  406. ads/hpo/stopping_criterion.py +75 -0
  407. ads/hpo/tuner_artifact.py +413 -0
  408. ads/hpo/utils.py +91 -0
  409. ads/hpo/validation.py +140 -0
  410. ads/hpo/visualization/__init__.py +5 -0
  411. ads/hpo/visualization/_contour.py +23 -0
  412. ads/hpo/visualization/_edf.py +20 -0
  413. ads/hpo/visualization/_intermediate_values.py +21 -0
  414. ads/hpo/visualization/_optimization_history.py +25 -0
  415. ads/hpo/visualization/_parallel_coordinate.py +169 -0
  416. ads/hpo/visualization/_param_importances.py +26 -0
  417. ads/jobs/__init__.py +53 -0
  418. ads/jobs/ads_job.py +663 -0
  419. ads/jobs/builders/__init__.py +5 -0
  420. ads/jobs/builders/base.py +156 -0
  421. ads/jobs/builders/infrastructure/__init__.py +6 -0
  422. ads/jobs/builders/infrastructure/base.py +165 -0
  423. ads/jobs/builders/infrastructure/dataflow.py +1252 -0
  424. ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
  425. ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
  426. ads/jobs/builders/infrastructure/utils.py +65 -0
  427. ads/jobs/builders/runtimes/__init__.py +5 -0
  428. ads/jobs/builders/runtimes/artifact.py +338 -0
  429. ads/jobs/builders/runtimes/base.py +325 -0
  430. ads/jobs/builders/runtimes/container_runtime.py +242 -0
  431. ads/jobs/builders/runtimes/python_runtime.py +1016 -0
  432. ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
  433. ads/jobs/cli.py +104 -0
  434. ads/jobs/env_var_parser.py +131 -0
  435. ads/jobs/extension.py +160 -0
  436. ads/jobs/schema/__init__.py +5 -0
  437. ads/jobs/schema/infrastructure_schema.json +116 -0
  438. ads/jobs/schema/job_schema.json +42 -0
  439. ads/jobs/schema/runtime_schema.json +183 -0
  440. ads/jobs/schema/validator.py +141 -0
  441. ads/jobs/serializer.py +296 -0
  442. ads/jobs/templates/__init__.py +5 -0
  443. ads/jobs/templates/container.py +6 -0
  444. ads/jobs/templates/driver_notebook.py +177 -0
  445. ads/jobs/templates/driver_oci.py +500 -0
  446. ads/jobs/templates/driver_python.py +48 -0
  447. ads/jobs/templates/driver_pytorch.py +852 -0
  448. ads/jobs/templates/driver_utils.py +615 -0
  449. ads/jobs/templates/hostname_from_env.c +55 -0
  450. ads/jobs/templates/oci_metrics.py +181 -0
  451. ads/jobs/utils.py +104 -0
  452. ads/llm/__init__.py +28 -0
  453. ads/llm/autogen/__init__.py +2 -0
  454. ads/llm/autogen/constants.py +15 -0
  455. ads/llm/autogen/reports/__init__.py +2 -0
  456. ads/llm/autogen/reports/base.py +67 -0
  457. ads/llm/autogen/reports/data.py +103 -0
  458. ads/llm/autogen/reports/session.py +526 -0
  459. ads/llm/autogen/reports/templates/chat_box.html +13 -0
  460. ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
  461. ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
  462. ads/llm/autogen/reports/utils.py +56 -0
  463. ads/llm/autogen/v02/__init__.py +4 -0
  464. ads/llm/autogen/v02/client.py +295 -0
  465. ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
  466. ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
  467. ads/llm/autogen/v02/loggers/__init__.py +6 -0
  468. ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
  469. ads/llm/autogen/v02/loggers/session_logger.py +580 -0
  470. ads/llm/autogen/v02/loggers/utils.py +86 -0
  471. ads/llm/autogen/v02/runtime_logging.py +163 -0
  472. ads/llm/chain.py +268 -0
  473. ads/llm/chat_template.py +31 -0
  474. ads/llm/deploy.py +63 -0
  475. ads/llm/guardrails/__init__.py +5 -0
  476. ads/llm/guardrails/base.py +442 -0
  477. ads/llm/guardrails/huggingface.py +44 -0
  478. ads/llm/langchain/__init__.py +5 -0
  479. ads/llm/langchain/plugins/__init__.py +5 -0
  480. ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
  481. ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
  482. ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
  483. ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
  484. ads/llm/langchain/plugins/llms/__init__.py +5 -0
  485. ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
  486. ads/llm/requirements.txt +3 -0
  487. ads/llm/serialize.py +219 -0
  488. ads/llm/serializers/__init__.py +0 -0
  489. ads/llm/serializers/retrieval_qa.py +153 -0
  490. ads/llm/serializers/runnable_parallel.py +27 -0
  491. ads/llm/templates/score_chain.jinja2 +155 -0
  492. ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
  493. ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
  494. ads/model/__init__.py +52 -0
  495. ads/model/artifact.py +573 -0
  496. ads/model/artifact_downloader.py +254 -0
  497. ads/model/artifact_uploader.py +267 -0
  498. ads/model/base_properties.py +238 -0
  499. ads/model/common/.model-ignore +66 -0
  500. ads/model/common/__init__.py +5 -0
  501. ads/model/common/utils.py +142 -0
  502. ads/model/datascience_model.py +2635 -0
  503. ads/model/deployment/__init__.py +20 -0
  504. ads/model/deployment/common/__init__.py +5 -0
  505. ads/model/deployment/common/utils.py +308 -0
  506. ads/model/deployment/model_deployer.py +466 -0
  507. ads/model/deployment/model_deployment.py +1846 -0
  508. ads/model/deployment/model_deployment_infrastructure.py +671 -0
  509. ads/model/deployment/model_deployment_properties.py +493 -0
  510. ads/model/deployment/model_deployment_runtime.py +838 -0
  511. ads/model/extractor/__init__.py +5 -0
  512. ads/model/extractor/automl_extractor.py +74 -0
  513. ads/model/extractor/embedding_onnx_extractor.py +80 -0
  514. ads/model/extractor/huggingface_extractor.py +88 -0
  515. ads/model/extractor/keras_extractor.py +84 -0
  516. ads/model/extractor/lightgbm_extractor.py +93 -0
  517. ads/model/extractor/model_info_extractor.py +114 -0
  518. ads/model/extractor/model_info_extractor_factory.py +105 -0
  519. ads/model/extractor/pytorch_extractor.py +87 -0
  520. ads/model/extractor/sklearn_extractor.py +112 -0
  521. ads/model/extractor/spark_extractor.py +89 -0
  522. ads/model/extractor/tensorflow_extractor.py +85 -0
  523. ads/model/extractor/xgboost_extractor.py +94 -0
  524. ads/model/framework/__init__.py +5 -0
  525. ads/model/framework/automl_model.py +178 -0
  526. ads/model/framework/embedding_onnx_model.py +438 -0
  527. ads/model/framework/huggingface_model.py +399 -0
  528. ads/model/framework/lightgbm_model.py +266 -0
  529. ads/model/framework/pytorch_model.py +266 -0
  530. ads/model/framework/sklearn_model.py +250 -0
  531. ads/model/framework/spark_model.py +326 -0
  532. ads/model/framework/tensorflow_model.py +254 -0
  533. ads/model/framework/xgboost_model.py +258 -0
  534. ads/model/generic_model.py +3518 -0
  535. ads/model/model_artifact_boilerplate/README.md +381 -0
  536. ads/model/model_artifact_boilerplate/__init__.py +5 -0
  537. ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
  538. ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
  539. ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
  540. ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
  541. ads/model/model_artifact_boilerplate/score.py +61 -0
  542. ads/model/model_file_description_schema.json +68 -0
  543. ads/model/model_introspect.py +331 -0
  544. ads/model/model_metadata.py +1810 -0
  545. ads/model/model_metadata_mixin.py +460 -0
  546. ads/model/model_properties.py +63 -0
  547. ads/model/model_version_set.py +739 -0
  548. ads/model/runtime/__init__.py +5 -0
  549. ads/model/runtime/env_info.py +306 -0
  550. ads/model/runtime/model_deployment_details.py +37 -0
  551. ads/model/runtime/model_provenance_details.py +58 -0
  552. ads/model/runtime/runtime_info.py +81 -0
  553. ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
  554. ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
  555. ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
  556. ads/model/runtime/utils.py +201 -0
  557. ads/model/serde/__init__.py +5 -0
  558. ads/model/serde/common.py +40 -0
  559. ads/model/serde/model_input.py +547 -0
  560. ads/model/serde/model_serializer.py +1184 -0
  561. ads/model/service/__init__.py +5 -0
  562. ads/model/service/oci_datascience_model.py +1076 -0
  563. ads/model/service/oci_datascience_model_deployment.py +500 -0
  564. ads/model/service/oci_datascience_model_version_set.py +176 -0
  565. ads/model/transformer/__init__.py +5 -0
  566. ads/model/transformer/onnx_transformer.py +324 -0
  567. ads/mysqldb/__init__.py +5 -0
  568. ads/mysqldb/mysql_db.py +227 -0
  569. ads/opctl/__init__.py +18 -0
  570. ads/opctl/anomaly_detection.py +11 -0
  571. ads/opctl/backend/__init__.py +5 -0
  572. ads/opctl/backend/ads_dataflow.py +353 -0
  573. ads/opctl/backend/ads_ml_job.py +710 -0
  574. ads/opctl/backend/ads_ml_pipeline.py +164 -0
  575. ads/opctl/backend/ads_model_deployment.py +209 -0
  576. ads/opctl/backend/base.py +146 -0
  577. ads/opctl/backend/local.py +1053 -0
  578. ads/opctl/backend/marketplace/__init__.py +9 -0
  579. ads/opctl/backend/marketplace/helm_helper.py +173 -0
  580. ads/opctl/backend/marketplace/local_marketplace.py +271 -0
  581. ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
  582. ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
  583. ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
  584. ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
  585. ads/opctl/backend/marketplace/models/__init__.py +5 -0
  586. ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
  587. ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
  588. ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
  589. ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
  590. ads/opctl/cli.py +707 -0
  591. ads/opctl/cmds.py +869 -0
  592. ads/opctl/conda/__init__.py +5 -0
  593. ads/opctl/conda/cli.py +193 -0
  594. ads/opctl/conda/cmds.py +749 -0
  595. ads/opctl/conda/config.yaml +34 -0
  596. ads/opctl/conda/manifest_template.yaml +13 -0
  597. ads/opctl/conda/multipart_uploader.py +188 -0
  598. ads/opctl/conda/pack.py +89 -0
  599. ads/opctl/config/__init__.py +5 -0
  600. ads/opctl/config/base.py +57 -0
  601. ads/opctl/config/diagnostics/__init__.py +5 -0
  602. ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
  603. ads/opctl/config/merger.py +255 -0
  604. ads/opctl/config/resolver.py +297 -0
  605. ads/opctl/config/utils.py +79 -0
  606. ads/opctl/config/validator.py +17 -0
  607. ads/opctl/config/versioner.py +68 -0
  608. ads/opctl/config/yaml_parsers/__init__.py +7 -0
  609. ads/opctl/config/yaml_parsers/base.py +58 -0
  610. ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
  611. ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
  612. ads/opctl/constants.py +66 -0
  613. ads/opctl/decorator/__init__.py +5 -0
  614. ads/opctl/decorator/common.py +129 -0
  615. ads/opctl/diagnostics/__init__.py +5 -0
  616. ads/opctl/diagnostics/__main__.py +25 -0
  617. ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
  618. ads/opctl/diagnostics/check_requirements.py +144 -0
  619. ads/opctl/diagnostics/requirement_exception.py +9 -0
  620. ads/opctl/distributed/README.md +109 -0
  621. ads/opctl/distributed/__init__.py +5 -0
  622. ads/opctl/distributed/certificates.py +32 -0
  623. ads/opctl/distributed/cli.py +207 -0
  624. ads/opctl/distributed/cmds.py +731 -0
  625. ads/opctl/distributed/common/__init__.py +5 -0
  626. ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
  627. ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
  628. ads/opctl/distributed/common/cluster_config_helper.py +103 -0
  629. ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
  630. ads/opctl/distributed/common/cluster_runner.py +54 -0
  631. ads/opctl/distributed/common/framework_factory.py +29 -0
  632. ads/opctl/docker/Dockerfile.job +103 -0
  633. ads/opctl/docker/Dockerfile.job.arm +107 -0
  634. ads/opctl/docker/Dockerfile.job.gpu +175 -0
  635. ads/opctl/docker/base-env.yaml +13 -0
  636. ads/opctl/docker/cuda.repo +6 -0
  637. ads/opctl/docker/operator/.dockerignore +0 -0
  638. ads/opctl/docker/operator/Dockerfile +41 -0
  639. ads/opctl/docker/operator/Dockerfile.gpu +85 -0
  640. ads/opctl/docker/operator/cuda.repo +6 -0
  641. ads/opctl/docker/operator/environment.yaml +8 -0
  642. ads/opctl/forecast.py +11 -0
  643. ads/opctl/index.yaml +3 -0
  644. ads/opctl/model/__init__.py +5 -0
  645. ads/opctl/model/cli.py +65 -0
  646. ads/opctl/model/cmds.py +73 -0
  647. ads/opctl/operator/README.md +4 -0
  648. ads/opctl/operator/__init__.py +31 -0
  649. ads/opctl/operator/cli.py +344 -0
  650. ads/opctl/operator/cmd.py +596 -0
  651. ads/opctl/operator/common/__init__.py +5 -0
  652. ads/opctl/operator/common/backend_factory.py +460 -0
  653. ads/opctl/operator/common/const.py +27 -0
  654. ads/opctl/operator/common/data/synthetic.csv +16001 -0
  655. ads/opctl/operator/common/dictionary_merger.py +148 -0
  656. ads/opctl/operator/common/errors.py +42 -0
  657. ads/opctl/operator/common/operator_config.py +99 -0
  658. ads/opctl/operator/common/operator_loader.py +811 -0
  659. ads/opctl/operator/common/operator_schema.yaml +130 -0
  660. ads/opctl/operator/common/operator_yaml_generator.py +152 -0
  661. ads/opctl/operator/common/utils.py +208 -0
  662. ads/opctl/operator/lowcode/__init__.py +5 -0
  663. ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
  664. ads/opctl/operator/lowcode/anomaly/README.md +207 -0
  665. ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
  666. ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
  667. ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
  668. ads/opctl/operator/lowcode/anomaly/const.py +167 -0
  669. ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
  670. ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
  671. ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
  672. ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
  673. ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
  674. ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
  675. ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
  676. ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
  677. ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
  678. ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
  679. ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
  680. ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
  681. ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
  682. ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
  683. ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
  684. ads/opctl/operator/lowcode/common/__init__.py +5 -0
  685. ads/opctl/operator/lowcode/common/const.py +10 -0
  686. ads/opctl/operator/lowcode/common/data.py +116 -0
  687. ads/opctl/operator/lowcode/common/errors.py +47 -0
  688. ads/opctl/operator/lowcode/common/transformations.py +296 -0
  689. ads/opctl/operator/lowcode/common/utils.py +384 -0
  690. ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
  691. ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
  692. ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
  693. ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
  694. ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
  695. ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
  696. ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
  697. ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
  698. ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
  699. ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
  700. ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
  701. ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
  702. ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
  703. ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
  704. ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
  705. ads/opctl/operator/lowcode/forecast/README.md +209 -0
  706. ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
  707. ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
  708. ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
  709. ads/opctl/operator/lowcode/forecast/const.py +92 -0
  710. ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
  711. ads/opctl/operator/lowcode/forecast/errors.py +26 -0
  712. ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
  713. ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
  714. ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
  715. ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
  716. ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
  717. ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
  718. ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
  719. ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
  720. ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
  721. ads/opctl/operator/lowcode/forecast/model/prophet.py +445 -0
  722. ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
  723. ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
  724. ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
  725. ads/opctl/operator/lowcode/forecast/utils.py +397 -0
  726. ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
  727. ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
  728. ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
  729. ads/opctl/operator/lowcode/pii/MLoperator +17 -0
  730. ads/opctl/operator/lowcode/pii/README.md +208 -0
  731. ads/opctl/operator/lowcode/pii/__init__.py +5 -0
  732. ads/opctl/operator/lowcode/pii/__main__.py +78 -0
  733. ads/opctl/operator/lowcode/pii/cmd.py +39 -0
  734. ads/opctl/operator/lowcode/pii/constant.py +84 -0
  735. ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
  736. ads/opctl/operator/lowcode/pii/errors.py +27 -0
  737. ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
  738. ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
  739. ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
  740. ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
  741. ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
  742. ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
  743. ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
  744. ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
  745. ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
  746. ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
  747. ads/opctl/operator/lowcode/pii/model/report.py +487 -0
  748. ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
  749. ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
  750. ads/opctl/operator/lowcode/pii/utils.py +43 -0
  751. ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
  752. ads/opctl/operator/lowcode/recommender/README.md +206 -0
  753. ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
  754. ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
  755. ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
  756. ads/opctl/operator/lowcode/recommender/constant.py +30 -0
  757. ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
  758. ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
  759. ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
  760. ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
  761. ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
  762. ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
  763. ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
  764. ads/opctl/operator/lowcode/recommender/utils.py +13 -0
  765. ads/opctl/operator/runtime/__init__.py +5 -0
  766. ads/opctl/operator/runtime/const.py +17 -0
  767. ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
  768. ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
  769. ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
  770. ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
  771. ads/opctl/operator/runtime/runtime.py +115 -0
  772. ads/opctl/schema.yaml.yml +36 -0
  773. ads/opctl/script.py +40 -0
  774. ads/opctl/spark/__init__.py +5 -0
  775. ads/opctl/spark/cli.py +43 -0
  776. ads/opctl/spark/cmds.py +147 -0
  777. ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
  778. ads/opctl/utils.py +344 -0
  779. ads/oracledb/__init__.py +5 -0
  780. ads/oracledb/oracle_db.py +346 -0
  781. ads/pipeline/__init__.py +39 -0
  782. ads/pipeline/ads_pipeline.py +2279 -0
  783. ads/pipeline/ads_pipeline_run.py +772 -0
  784. ads/pipeline/ads_pipeline_step.py +605 -0
  785. ads/pipeline/builders/__init__.py +5 -0
  786. ads/pipeline/builders/infrastructure/__init__.py +5 -0
  787. ads/pipeline/builders/infrastructure/custom_script.py +32 -0
  788. ads/pipeline/cli.py +119 -0
  789. ads/pipeline/extension.py +291 -0
  790. ads/pipeline/schema/__init__.py +5 -0
  791. ads/pipeline/schema/cs_step_schema.json +35 -0
  792. ads/pipeline/schema/ml_step_schema.json +31 -0
  793. ads/pipeline/schema/pipeline_schema.json +71 -0
  794. ads/pipeline/visualizer/__init__.py +5 -0
  795. ads/pipeline/visualizer/base.py +570 -0
  796. ads/pipeline/visualizer/graph_renderer.py +272 -0
  797. ads/pipeline/visualizer/text_renderer.py +84 -0
  798. ads/secrets/__init__.py +11 -0
  799. ads/secrets/adb.py +386 -0
  800. ads/secrets/auth_token.py +86 -0
  801. ads/secrets/big_data_service.py +365 -0
  802. ads/secrets/mysqldb.py +149 -0
  803. ads/secrets/oracledb.py +160 -0
  804. ads/secrets/secrets.py +407 -0
  805. ads/telemetry/__init__.py +7 -0
  806. ads/telemetry/base.py +69 -0
  807. ads/telemetry/client.py +125 -0
  808. ads/telemetry/telemetry.py +257 -0
  809. ads/templates/dataflow_pyspark.jinja2 +13 -0
  810. ads/templates/dataflow_sparksql.jinja2 +22 -0
  811. ads/templates/func.jinja2 +20 -0
  812. ads/templates/schemas/openapi.json +1740 -0
  813. ads/templates/score-pkl.jinja2 +173 -0
  814. ads/templates/score.jinja2 +322 -0
  815. ads/templates/score_embedding_onnx.jinja2 +202 -0
  816. ads/templates/score_generic.jinja2 +165 -0
  817. ads/templates/score_huggingface_pipeline.jinja2 +217 -0
  818. ads/templates/score_lightgbm.jinja2 +185 -0
  819. ads/templates/score_onnx.jinja2 +407 -0
  820. ads/templates/score_onnx_new.jinja2 +473 -0
  821. ads/templates/score_oracle_automl.jinja2 +185 -0
  822. ads/templates/score_pyspark.jinja2 +154 -0
  823. ads/templates/score_pytorch.jinja2 +219 -0
  824. ads/templates/score_scikit-learn.jinja2 +184 -0
  825. ads/templates/score_tensorflow.jinja2 +184 -0
  826. ads/templates/score_xgboost.jinja2 +178 -0
  827. ads/text_dataset/__init__.py +5 -0
  828. ads/text_dataset/backends.py +211 -0
  829. ads/text_dataset/dataset.py +445 -0
  830. ads/text_dataset/extractor.py +207 -0
  831. ads/text_dataset/options.py +53 -0
  832. ads/text_dataset/udfs.py +22 -0
  833. ads/text_dataset/utils.py +49 -0
  834. ads/type_discovery/__init__.py +9 -0
  835. ads/type_discovery/abstract_detector.py +21 -0
  836. ads/type_discovery/constant_detector.py +41 -0
  837. ads/type_discovery/continuous_detector.py +54 -0
  838. ads/type_discovery/credit_card_detector.py +99 -0
  839. ads/type_discovery/datetime_detector.py +92 -0
  840. ads/type_discovery/discrete_detector.py +118 -0
  841. ads/type_discovery/document_detector.py +146 -0
  842. ads/type_discovery/ip_detector.py +68 -0
  843. ads/type_discovery/latlon_detector.py +90 -0
  844. ads/type_discovery/phone_number_detector.py +63 -0
  845. ads/type_discovery/type_discovery_driver.py +87 -0
  846. ads/type_discovery/typed_feature.py +594 -0
  847. ads/type_discovery/unknown_detector.py +41 -0
  848. ads/type_discovery/zipcode_detector.py +48 -0
  849. ads/vault/__init__.py +7 -0
  850. ads/vault/vault.py +237 -0
  851. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.9rc1.dist-info}/METADATA +150 -150
  852. oracle_ads-2.13.9rc1.dist-info/RECORD +858 -0
  853. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.9rc1.dist-info}/WHEEL +1 -2
  854. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.9rc1.dist-info}/entry_points.txt +2 -1
  855. oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
  856. oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
  857. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.9rc1.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,2114 @@
1
+ #!/usr/bin/env python
2
+ # Copyright (c) 2024, 2025 Oracle and/or its affiliates.
3
+ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
4
+ import json
5
+ import os
6
+ import pathlib
7
+ import re
8
+ from datetime import datetime, timedelta
9
+ from threading import Lock
10
+ from typing import Any, Dict, List, Optional, Set, Union
11
+
12
+ import oci
13
+ from cachetools import TTLCache
14
+ from huggingface_hub import snapshot_download
15
+ from oci.data_science.models import JobRun, Metadata, Model, UpdateModelDetails
16
+
17
+ from ads.aqua import logger
18
+ from ads.aqua.app import AquaApp
19
+ from ads.aqua.common.entities import AquaMultiModelRef
20
+ from ads.aqua.common.enums import (
21
+ ConfigFolder,
22
+ CustomInferenceContainerTypeFamily,
23
+ FineTuningContainerTypeFamily,
24
+ InferenceContainerTypeFamily,
25
+ ModelFormat,
26
+ Platform,
27
+ Tags,
28
+ )
29
+ from ads.aqua.common.errors import (
30
+ AquaFileNotFoundError,
31
+ AquaRuntimeError,
32
+ AquaValueError,
33
+ )
34
+ from ads.aqua.common.utils import (
35
+ LifecycleStatus,
36
+ _build_resource_identifier,
37
+ cleanup_local_hf_model_artifact,
38
+ create_word_icon,
39
+ generate_tei_cmd_var,
40
+ get_artifact_path,
41
+ get_hf_model_info,
42
+ get_preferred_compatible_family,
43
+ list_os_files_with_extension,
44
+ load_config,
45
+ upload_folder,
46
+ )
47
+ from ads.aqua.config.container_config import AquaContainerConfig, Usage
48
+ from ads.aqua.constants import (
49
+ AQUA_MODEL_ARTIFACT_CONFIG,
50
+ AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME,
51
+ AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE,
52
+ AQUA_MODEL_ARTIFACT_FILE,
53
+ AQUA_MODEL_TOKENIZER_CONFIG,
54
+ AQUA_MODEL_TYPE_CUSTOM,
55
+ HF_METADATA_FOLDER,
56
+ LICENSE,
57
+ MODEL_BY_REFERENCE_OSS_PATH_KEY,
58
+ README,
59
+ READY_TO_DEPLOY_STATUS,
60
+ READY_TO_FINE_TUNE_STATUS,
61
+ READY_TO_IMPORT_STATUS,
62
+ TRAINING_METRICS_FINAL,
63
+ TRINING_METRICS,
64
+ VALIDATION_METRICS,
65
+ VALIDATION_METRICS_FINAL,
66
+ )
67
+ from ads.aqua.model.constants import (
68
+ AquaModelMetadataKeys,
69
+ FineTuningCustomMetadata,
70
+ FineTuningMetricCategories,
71
+ ModelCustomMetadataFields,
72
+ ModelType,
73
+ )
74
+ from ads.aqua.model.entities import (
75
+ AquaFineTuneModel,
76
+ AquaFineTuningMetric,
77
+ AquaModel,
78
+ AquaModelLicense,
79
+ AquaModelReadme,
80
+ AquaModelSummary,
81
+ ImportModelDetails,
82
+ ModelFileDescription,
83
+ ModelValidationResult,
84
+ )
85
+ from ads.aqua.model.enums import MultiModelSupportedTaskType
86
+ from ads.common.auth import default_signer
87
+ from ads.common.oci_resource import SEARCH_TYPE, OCIResource
88
+ from ads.common.utils import (
89
+ UNKNOWN,
90
+ get_console_link,
91
+ is_path_exists,
92
+ read_file,
93
+ )
94
+ from ads.config import (
95
+ AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME,
96
+ AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,
97
+ AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME,
98
+ AQUA_EVALUATION_CONTAINER_METADATA_NAME,
99
+ AQUA_FINETUNING_CONTAINER_METADATA_NAME,
100
+ AQUA_SERVICE_MODELS,
101
+ COMPARTMENT_OCID,
102
+ PROJECT_OCID,
103
+ SERVICE,
104
+ TENANCY_OCID,
105
+ USER,
106
+ )
107
+ from ads.model import DataScienceModel
108
+ from ads.model.common.utils import MetadataArtifactPathType
109
+ from ads.model.model_metadata import (
110
+ MetadataCustomCategory,
111
+ ModelCustomMetadata,
112
+ ModelCustomMetadataItem,
113
+ )
114
+ from ads.telemetry import telemetry
115
+
116
+
117
+ class AquaModelApp(AquaApp):
118
+ """Provides a suite of APIs to interact with Aqua models within the Oracle
119
+ Cloud Infrastructure Data Science service, serving as an interface for
120
+ managing machine learning models.
121
+
122
+
123
+ Methods
124
+ -------
125
+ create(model_id: str, project_id: str, compartment_id: str = None, **kwargs) -> "AquaModel"
126
+ Creates custom aqua model from service model.
127
+ get(model_id: str) -> AquaModel:
128
+ Retrieves details of an Aqua model by its unique identifier.
129
+ list(compartment_id: str = None, project_id: str = None, **kwargs) -> List[AquaModelSummary]:
130
+ Lists all Aqua models within a specified compartment and/or project.
131
+ clear_model_list_cache()
132
+ Allows clear list model cache items from the service models compartment.
133
+ register(model: str, os_path: str, local_dir: str = None)
134
+
135
+ Note:
136
+ This class is designed to work within the Oracle Cloud Infrastructure
137
+ and requires proper configuration and authentication set up to interact
138
+ with OCI services.
139
+ """
140
+
141
+ _service_models_cache = TTLCache(
142
+ maxsize=10, ttl=timedelta(hours=5), timer=datetime.now
143
+ )
144
+ # Used for saving service model details
145
+ _service_model_details_cache = TTLCache(
146
+ maxsize=10, ttl=timedelta(hours=5), timer=datetime.now
147
+ )
148
+ _cache_lock = Lock()
149
+
150
+ @telemetry(entry_point="plugin=model&action=create", name="aqua")
151
+ def create(
152
+ self,
153
+ model_id: Union[str, AquaMultiModelRef],
154
+ project_id: Optional[str] = None,
155
+ compartment_id: Optional[str] = None,
156
+ freeform_tags: Optional[Dict] = None,
157
+ defined_tags: Optional[Dict] = None,
158
+ **kwargs,
159
+ ) -> DataScienceModel:
160
+ """
161
+ Creates a custom Aqua model from a service model.
162
+
163
+ Parameters
164
+ ----------
165
+ model_id : Union[str, AquaMultiModelRef]
166
+ The model ID as a string or a AquaMultiModelRef instance to be deployed.
167
+ project_id : Optional[str]
168
+ The project ID for the custom model.
169
+ compartment_id : Optional[str]
170
+ The compartment ID for the custom model. Defaults to None.
171
+ If not provided, the compartment ID will be fetched from environment variables.
172
+ freeform_tags : Optional[Dict]
173
+ Freeform tags for the model.
174
+ defined_tags : Optional[Dict]
175
+ Defined tags for the model.
176
+
177
+ Returns
178
+ -------
179
+ DataScienceModel
180
+ The instance of DataScienceModel.
181
+ """
182
+ model_id = (
183
+ model_id.model_id if isinstance(model_id, AquaMultiModelRef) else model_id
184
+ )
185
+ service_model = DataScienceModel.from_id(model_id)
186
+ target_project = project_id or PROJECT_OCID
187
+ target_compartment = compartment_id or COMPARTMENT_OCID
188
+
189
+ # Skip model copying if it is registered model or fine-tuned model
190
+ if (
191
+ service_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None) is not None
192
+ or service_model.freeform_tags.get(Tags.AQUA_FINE_TUNED_MODEL_TAG)
193
+ is not None
194
+ ):
195
+ logger.info(
196
+ f"Aqua Model {model_id} already exists in the user's compartment."
197
+ "Skipped copying."
198
+ )
199
+ return service_model
200
+
201
+ # combine tags
202
+ combined_freeform_tags = {
203
+ **(service_model.freeform_tags or {}),
204
+ **(freeform_tags or {}),
205
+ }
206
+ combined_defined_tags = {
207
+ **(service_model.defined_tags or {}),
208
+ **(defined_tags or {}),
209
+ }
210
+
211
+ custom_model = (
212
+ DataScienceModel()
213
+ .with_compartment_id(target_compartment)
214
+ .with_project_id(target_project)
215
+ .with_model_file_description(json_dict=service_model.model_file_description)
216
+ .with_display_name(service_model.display_name)
217
+ .with_description(service_model.description)
218
+ .with_freeform_tags(**combined_freeform_tags)
219
+ .with_defined_tags(**combined_defined_tags)
220
+ .with_custom_metadata_list(service_model.custom_metadata_list)
221
+ .with_defined_metadata_list(service_model.defined_metadata_list)
222
+ .with_provenance_metadata(service_model.provenance_metadata)
223
+ .create(model_by_reference=True, **kwargs)
224
+ )
225
+ logger.info(
226
+ f"Aqua Model {custom_model.id} created with the service model {model_id}."
227
+ )
228
+
229
+ # Track unique models that were created in the user's compartment
230
+ self.telemetry.record_event_async(
231
+ category="aqua/service/model",
232
+ action="create",
233
+ detail=service_model.display_name,
234
+ )
235
+
236
+ return custom_model
237
+
238
+ @telemetry(entry_point="plugin=model&action=create", name="aqua")
239
+ def create_multi(
240
+ self,
241
+ models: List[AquaMultiModelRef],
242
+ project_id: Optional[str] = None,
243
+ compartment_id: Optional[str] = None,
244
+ freeform_tags: Optional[Dict] = None,
245
+ defined_tags: Optional[Dict] = None,
246
+ **kwargs, # noqa: ARG002
247
+ ) -> DataScienceModel:
248
+ """
249
+ Creates a multi-model grouping using the provided model list.
250
+
251
+ Parameters
252
+ ----------
253
+ models : List[AquaMultiModelRef]
254
+ List of AquaMultiModelRef instances for creating a multi-model group.
255
+ project_id : Optional[str]
256
+ The project ID for the multi-model group.
257
+ compartment_id : Optional[str]
258
+ The compartment ID for the multi-model group.
259
+ freeform_tags : Optional[Dict]
260
+ Freeform tags for the model.
261
+ defined_tags : Optional[Dict]
262
+ Defined tags for the model.
263
+
264
+ Returns
265
+ -------
266
+ DataScienceModel
267
+ Instance of DataScienceModel object.
268
+ """
269
+
270
+ if not models:
271
+ raise AquaValueError(
272
+ "Model list cannot be empty. Please provide at least one model for deployment."
273
+ )
274
+
275
+ display_name_list = []
276
+ model_file_description_list: List[ModelFileDescription] = []
277
+ model_custom_metadata = ModelCustomMetadata()
278
+
279
+ service_inference_containers = (
280
+ self.get_container_config().to_dict().get("inference")
281
+ )
282
+
283
+ supported_container_families = [
284
+ container_config_item.family
285
+ for container_config_item in service_inference_containers
286
+ if any(
287
+ usage.upper() in container_config_item.usages
288
+ for usage in [Usage.MULTI_MODEL, Usage.OTHER]
289
+ )
290
+ ]
291
+
292
+ if not supported_container_families:
293
+ raise AquaValueError(
294
+ "Currently, there are no containers that support multi-model deployment."
295
+ )
296
+
297
+ selected_models_deployment_containers = set()
298
+
299
+ # Process each model
300
+ for model in models:
301
+ source_model = DataScienceModel.from_id(model.model_id)
302
+ display_name = source_model.display_name
303
+ model_file_description = source_model.model_file_description
304
+ # Update model name in user's input model
305
+ model.model_name = model.model_name or display_name
306
+
307
+ # TODO Uncomment the section below, if only service models should be allowed for multi-model deployment
308
+ # if not source_model.freeform_tags.get(Tags.AQUA_SERVICE_MODEL_TAG, UNKNOWN):
309
+ # raise AquaValueError(
310
+ # f"Invalid selected model {display_name}. "
311
+ # "Currently only service models are supported for multi model deployment."
312
+ # )
313
+
314
+ display_name_list.append(display_name)
315
+
316
+ self._extract_model_task(model, source_model)
317
+
318
+ # Retrieve model artifact
319
+ model_artifact_path = source_model.artifact
320
+ if not model_artifact_path:
321
+ raise AquaValueError(
322
+ f"Model '{display_name}' (ID: {model.model_id}) has no artifacts. "
323
+ "Please register the model first."
324
+ )
325
+
326
+ # Update model artifact location in user's input model
327
+ model.artifact_location = model_artifact_path
328
+
329
+ if not model_file_description:
330
+ raise AquaValueError(
331
+ f"Model '{display_name}' (ID: {model.model_id}) has no file description. "
332
+ "Please register the model first."
333
+ )
334
+
335
+ model_file_description_list.append(
336
+ ModelFileDescription(**model_file_description)
337
+ )
338
+
339
+ # Validate deployment container consistency
340
+ deployment_container = source_model.custom_metadata_list.get(
341
+ ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
342
+ ModelCustomMetadataItem(
343
+ key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER
344
+ ),
345
+ ).value
346
+
347
+ if deployment_container not in supported_container_families:
348
+ raise AquaValueError(
349
+ f"Unsupported deployment container '{deployment_container}' for model '{source_model.id}'. "
350
+ f"Only '{supported_container_families}' are supported for multi-model deployments."
351
+ )
352
+
353
+ selected_models_deployment_containers.add(deployment_container)
354
+
355
+ if not selected_models_deployment_containers:
356
+ raise AquaValueError(
357
+ "None of the selected models are associated with a recognized container family. "
358
+ "Please review the selected models, or select a different group of models."
359
+ )
360
+
361
+ # Check if the all models in the group shares same container family
362
+ if len(selected_models_deployment_containers) > 1:
363
+ deployment_container = get_preferred_compatible_family(
364
+ selected_families=selected_models_deployment_containers
365
+ )
366
+ if not deployment_container:
367
+ raise AquaValueError(
368
+ "The selected models are associated with different container families: "
369
+ f"{list(selected_models_deployment_containers)}."
370
+ "For multi-model deployment, all models in the group must share the same container family."
371
+ )
372
+ else:
373
+ deployment_container = selected_models_deployment_containers.pop()
374
+
375
+ # Generate model group details
376
+ timestamp = datetime.now().strftime("%Y%m%d")
377
+ model_group_display_name = f"model_group_{timestamp}"
378
+ combined_models = ", ".join(display_name_list)
379
+ model_group_description = f"Multi-model grouping using {combined_models}."
380
+
381
+ # Add global metadata
382
+ model_custom_metadata.add(
383
+ key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
384
+ value=deployment_container,
385
+ description=f"Inference container mapping for {model_group_display_name}",
386
+ category="Other",
387
+ )
388
+ model_custom_metadata.add(
389
+ key=ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT,
390
+ value=str(len(models)),
391
+ description="Number of models in the group.",
392
+ category="Other",
393
+ )
394
+
395
+ # Combine tags. The `Tags.AQUA_TAG` has been excluded, because we don't want to show
396
+ # the models created for multi-model purpose in the AQUA models list.
397
+ tags = {
398
+ # Tags.AQUA_TAG: "active",
399
+ Tags.MULTIMODEL_TYPE_TAG: "true",
400
+ **(freeform_tags or {}),
401
+ }
402
+
403
+ # Create multi-model group
404
+ custom_model = (
405
+ DataScienceModel()
406
+ .with_compartment_id(compartment_id)
407
+ .with_project_id(project_id)
408
+ .with_display_name(model_group_display_name)
409
+ .with_description(model_group_description)
410
+ .with_freeform_tags(**tags)
411
+ .with_defined_tags(**(defined_tags or {}))
412
+ .with_custom_metadata_list(model_custom_metadata)
413
+ )
414
+
415
+ # Update multi model file description to attach artifacts
416
+ custom_model.with_model_file_description(
417
+ json_dict=ModelFileDescription(
418
+ models=[
419
+ models
420
+ for model_file_description in model_file_description_list
421
+ for models in model_file_description.models
422
+ ]
423
+ ).model_dump(by_alias=True)
424
+ )
425
+
426
+ # Finalize creation
427
+ custom_model.create(model_by_reference=True)
428
+
429
+ logger.info(
430
+ f"Aqua Model '{custom_model.id}' created with models: {', '.join(display_name_list)}."
431
+ )
432
+
433
+ # Create custom metadata for multi model metadata
434
+ custom_model.create_custom_metadata_artifact(
435
+ metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA,
436
+ artifact_path_or_content=json.dumps(
437
+ [model.model_dump() for model in models]
438
+ ).encode(),
439
+ path_type=MetadataArtifactPathType.CONTENT,
440
+ )
441
+
442
+ logger.debug(
443
+ f"Multi model metadata uploaded for Aqua model: {custom_model.id}."
444
+ )
445
+
446
+ # Track telemetry event
447
+ self.telemetry.record_event_async(
448
+ category="aqua/multimodel",
449
+ action="create",
450
+ detail=combined_models,
451
+ )
452
+
453
+ return custom_model
454
+
455
+ @telemetry(entry_point="plugin=model&action=get", name="aqua")
456
+ def get(self, model_id: str) -> "AquaModel":
457
+ """Gets the information of an Aqua model.
458
+
459
+ Parameters
460
+ ----------
461
+ model_id: str
462
+ The model OCID.
463
+ load_model_card: (bool, optional). Defaults to `True`.
464
+ Whether to load model card from artifacts or not.
465
+
466
+ Returns
467
+ -------
468
+ AquaModel:
469
+ The instance of AquaModel.
470
+ """
471
+
472
+ cached_item = self._service_model_details_cache.get(model_id)
473
+ if cached_item:
474
+ logger.info(f"Fetching model details for model {model_id} from cache.")
475
+ return cached_item
476
+
477
+ logger.info(f"Fetching model details for model {model_id}.")
478
+ ds_model = DataScienceModel.from_id(model_id)
479
+
480
+ if not self._if_show(ds_model):
481
+ raise AquaRuntimeError(
482
+ f"Target model `{ds_model.id} `is not an Aqua model as it does not contain "
483
+ f"{Tags.AQUA_TAG} tag."
484
+ )
485
+
486
+ is_fine_tuned_model = bool(
487
+ ds_model.freeform_tags
488
+ and ds_model.freeform_tags.get(Tags.AQUA_FINE_TUNED_MODEL_TAG)
489
+ )
490
+
491
+ inference_container = ds_model.custom_metadata_list.get(
492
+ ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
493
+ ModelCustomMetadataItem(key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER),
494
+ ).value
495
+ inference_container_uri = ds_model.custom_metadata_list.get(
496
+ ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI,
497
+ ModelCustomMetadataItem(
498
+ key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI
499
+ ),
500
+ ).value
501
+ evaluation_container = ds_model.custom_metadata_list.get(
502
+ ModelCustomMetadataFields.EVALUATION_CONTAINER,
503
+ ModelCustomMetadataItem(key=ModelCustomMetadataFields.EVALUATION_CONTAINER),
504
+ ).value
505
+ finetuning_container: str = ds_model.custom_metadata_list.get(
506
+ ModelCustomMetadataFields.FINETUNE_CONTAINER,
507
+ ModelCustomMetadataItem(key=ModelCustomMetadataFields.FINETUNE_CONTAINER),
508
+ ).value
509
+ artifact_location = ds_model.custom_metadata_list.get(
510
+ ModelCustomMetadataFields.ARTIFACT_LOCATION,
511
+ ModelCustomMetadataItem(key=ModelCustomMetadataFields.ARTIFACT_LOCATION),
512
+ ).value
513
+
514
+ aqua_model_attributes = dict(
515
+ **self._process_model(ds_model, self.region),
516
+ project_id=ds_model.project_id,
517
+ inference_container=inference_container,
518
+ inference_container_uri=inference_container_uri,
519
+ finetuning_container=finetuning_container,
520
+ evaluation_container=evaluation_container,
521
+ artifact_location=artifact_location,
522
+ )
523
+
524
+ if not is_fine_tuned_model:
525
+ model_details = AquaModel(**aqua_model_attributes)
526
+ self._service_model_details_cache.__setitem__(
527
+ key=model_id, value=model_details
528
+ )
529
+
530
+ else:
531
+ try:
532
+ jobrun_ocid = ds_model.provenance_metadata.training_id
533
+ jobrun = self.ds_client.get_job_run(jobrun_ocid).data
534
+ except Exception as e:
535
+ logger.debug(
536
+ f"Missing jobrun information in the provenance metadata of the given model {model_id}."
537
+ f"\nError: {str(e)}"
538
+ )
539
+ jobrun = None
540
+
541
+ try:
542
+ source_id = ds_model.custom_metadata_list.get(
543
+ FineTuningCustomMetadata.FT_SOURCE
544
+ ).value
545
+ except ValueError as e:
546
+ logger.debug(
547
+ f"Custom metadata is missing {FineTuningCustomMetadata.FT_SOURCE} key for "
548
+ f"model {model_id}.\nError: {str(e)}"
549
+ )
550
+ source_id = UNKNOWN
551
+
552
+ try:
553
+ source_name = ds_model.custom_metadata_list.get(
554
+ FineTuningCustomMetadata.FT_SOURCE_NAME
555
+ ).value
556
+ except ValueError as e:
557
+ logger.debug(
558
+ f"Custom metadata is missing {FineTuningCustomMetadata.FT_SOURCE_NAME} key for "
559
+ f"model {model_id}.\nError: {str(e)}"
560
+ )
561
+ source_name = UNKNOWN
562
+
563
+ source_identifier = _build_resource_identifier(
564
+ id=source_id,
565
+ name=source_name,
566
+ region=self.region,
567
+ )
568
+
569
+ ft_metrics = self._build_ft_metrics(ds_model.custom_metadata_list)
570
+
571
+ job_run_status = (
572
+ jobrun.lifecycle_state
573
+ if jobrun and jobrun.lifecycle_state != JobRun.LIFECYCLE_STATE_DELETED
574
+ else (
575
+ JobRun.LIFECYCLE_STATE_SUCCEEDED
576
+ if self.if_artifact_exist(ds_model.id)
577
+ else JobRun.LIFECYCLE_STATE_FAILED
578
+ )
579
+ )
580
+ # TODO: change the argument's name.
581
+ lifecycle_state = LifecycleStatus.get_status(
582
+ evaluation_status=ds_model.lifecycle_state,
583
+ job_run_status=job_run_status,
584
+ )
585
+
586
+ model_details = AquaFineTuneModel(
587
+ **aqua_model_attributes,
588
+ source=source_identifier,
589
+ lifecycle_state=(
590
+ Model.LIFECYCLE_STATE_ACTIVE
591
+ if lifecycle_state == JobRun.LIFECYCLE_STATE_SUCCEEDED
592
+ else lifecycle_state
593
+ ),
594
+ metrics=ft_metrics,
595
+ model=ds_model,
596
+ jobrun=jobrun,
597
+ region=self.region,
598
+ )
599
+
600
+ return model_details
601
+
602
+ @telemetry(entry_point="plugin=model&action=delete", name="aqua")
603
+ def delete_model(self, model_id):
604
+ ds_model = DataScienceModel.from_id(model_id)
605
+ is_registered_model = ds_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None)
606
+ is_fine_tuned_model = ds_model.freeform_tags.get(
607
+ Tags.AQUA_FINE_TUNED_MODEL_TAG, None
608
+ )
609
+ if is_registered_model or is_fine_tuned_model:
610
+ logger.info(f"Deleting model {model_id}.")
611
+ return ds_model.delete()
612
+ else:
613
+ raise AquaRuntimeError(
614
+ f"Failed to delete model:{model_id}. Only registered models or finetuned model can be deleted."
615
+ )
616
+
617
+ @telemetry(entry_point="plugin=model&action=edit", name="aqua")
618
+ def edit_registered_model(
619
+ self, id, inference_container, inference_container_uri, enable_finetuning, task
620
+ ):
621
+ """Edits the default config of unverified registered model.
622
+
623
+ Parameters
624
+ ----------
625
+ id: str
626
+ The model OCID.
627
+ inference_container: str.
628
+ The inference container family name
629
+ inference_container_uri: str
630
+ The inference container uri for embedding models
631
+ enable_finetuning: str
632
+ Flag to enable or disable finetuning over the model. Defaults to None
633
+ task:
634
+ The usecase type of the model. e.g , text-generation , text_embedding etc.
635
+
636
+ Returns
637
+ -------
638
+ Model:
639
+ The instance of oci.data_science.models.Model.
640
+
641
+ """
642
+ ds_model = DataScienceModel.from_id(id)
643
+ if ds_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None):
644
+ if ds_model.freeform_tags.get(Tags.AQUA_SERVICE_MODEL_TAG, None):
645
+ raise AquaRuntimeError(
646
+ "Only registered unverified models can be edited."
647
+ )
648
+ else:
649
+ custom_metadata_list = ds_model.custom_metadata_list
650
+ freeform_tags = ds_model.freeform_tags
651
+ if inference_container:
652
+ if (
653
+ inference_container in CustomInferenceContainerTypeFamily
654
+ and inference_container_uri is None
655
+ ):
656
+ raise AquaRuntimeError(
657
+ "Inference container URI must be provided."
658
+ )
659
+ else:
660
+ custom_metadata_list.add(
661
+ key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
662
+ value=inference_container,
663
+ category=MetadataCustomCategory.OTHER,
664
+ description="Deployment container mapping for SMC",
665
+ replace=True,
666
+ )
667
+ if inference_container_uri:
668
+ if (
669
+ inference_container in CustomInferenceContainerTypeFamily
670
+ or inference_container is None
671
+ ):
672
+ custom_metadata_list.add(
673
+ key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI,
674
+ value=inference_container_uri,
675
+ category=MetadataCustomCategory.OTHER,
676
+ description=f"Inference container URI for {ds_model.display_name}",
677
+ replace=True,
678
+ )
679
+ else:
680
+ raise AquaRuntimeError(
681
+ f"Inference container URI can be edited only with container values: {CustomInferenceContainerTypeFamily.values()}"
682
+ )
683
+
684
+ if enable_finetuning is not None:
685
+ if enable_finetuning.lower() == "true":
686
+ custom_metadata_list.add(
687
+ key=ModelCustomMetadataFields.FINETUNE_CONTAINER,
688
+ value=FineTuningContainerTypeFamily.AQUA_FINETUNING_CONTAINER_FAMILY,
689
+ category=MetadataCustomCategory.OTHER,
690
+ description="Fine-tuning container mapping for SMC",
691
+ replace=True,
692
+ )
693
+ freeform_tags.update({Tags.READY_TO_FINE_TUNE: "true"})
694
+ elif enable_finetuning.lower() == "false":
695
+ try:
696
+ custom_metadata_list.remove(
697
+ ModelCustomMetadataFields.FINETUNE_CONTAINER
698
+ )
699
+ freeform_tags.pop(Tags.READY_TO_FINE_TUNE)
700
+ except Exception as ex:
701
+ raise AquaRuntimeError(
702
+ f"The given model already doesn't support finetuning: {ex}"
703
+ ) from ex
704
+
705
+ custom_metadata_list.remove("modelDescription")
706
+ if task:
707
+ freeform_tags.update({Tags.TASK: task})
708
+ updated_custom_metadata_list = [
709
+ Metadata(**metadata)
710
+ for metadata in custom_metadata_list.to_dict()["data"]
711
+ ]
712
+ update_model_details = UpdateModelDetails(
713
+ custom_metadata_list=updated_custom_metadata_list,
714
+ freeform_tags=freeform_tags,
715
+ )
716
+ AquaApp().update_model(id, update_model_details)
717
+ logger.info(f"Updated model details for the model {id}.")
718
+ else:
719
+ raise AquaRuntimeError("Only registered unverified models can be edited.")
720
+
721
+ def _extract_model_task(
722
+ self,
723
+ model: AquaMultiModelRef,
724
+ source_model: DataScienceModel,
725
+ ) -> None:
726
+ """In a Multi Model Deployment, will set model_task parameter in AquaMultiModelRef from freeform tags or user"""
727
+ # user does not supply model task, we extract from model metadata
728
+ if not model.model_task:
729
+ model.model_task = source_model.freeform_tags.get(Tags.TASK, UNKNOWN)
730
+
731
+ task_tag = re.sub(r"-", "_", model.model_task).lower()
732
+ # re-visit logic when more model task types are supported
733
+ if task_tag in MultiModelSupportedTaskType:
734
+ model.model_task = task_tag
735
+ else:
736
+ raise AquaValueError(
737
+ f"Invalid or missing {task_tag} tag for selected model {source_model.display_name}. "
738
+ f"Currently only `{MultiModelSupportedTaskType.values()}` models are supported for multi model deployment."
739
+ )
740
+
741
+ def _fetch_metric_from_metadata(
742
+ self,
743
+ custom_metadata_list: ModelCustomMetadata,
744
+ target: str,
745
+ category: str,
746
+ metric_name: str,
747
+ ) -> AquaFineTuningMetric:
748
+ """Gets target metric from `ads.model.model_metadata.ModelCustomMetadata`."""
749
+ try:
750
+ scores = []
751
+ for custom_metadata in custom_metadata_list._items:
752
+ # We use description to group metrics
753
+ if custom_metadata.description == target:
754
+ scores.append(custom_metadata.value)
755
+ if metric_name.endswith("final"):
756
+ break
757
+
758
+ return AquaFineTuningMetric(
759
+ name=metric_name,
760
+ category=category,
761
+ scores=scores,
762
+ )
763
+ except Exception:
764
+ return AquaFineTuningMetric(name=metric_name, category=category, scores=[])
765
+
766
+ def _build_ft_metrics(
767
+ self, custom_metadata_list: ModelCustomMetadata
768
+ ) -> List[AquaFineTuningMetric]:
769
+ """Builds Fine Tuning metrics."""
770
+
771
+ validation_metrics = self._fetch_metric_from_metadata(
772
+ custom_metadata_list=custom_metadata_list,
773
+ target=FineTuningCustomMetadata.VALIDATION_METRICS_EPOCH,
774
+ category=FineTuningMetricCategories.VALIDATION,
775
+ metric_name=VALIDATION_METRICS,
776
+ )
777
+
778
+ training_metrics = self._fetch_metric_from_metadata(
779
+ custom_metadata_list=custom_metadata_list,
780
+ target=FineTuningCustomMetadata.TRAINING_METRICS_EPOCH,
781
+ category=FineTuningMetricCategories.TRAINING,
782
+ metric_name=TRINING_METRICS,
783
+ )
784
+
785
+ validation_final = self._fetch_metric_from_metadata(
786
+ custom_metadata_list=custom_metadata_list,
787
+ target=FineTuningCustomMetadata.VALIDATION_METRICS_FINAL,
788
+ category=FineTuningMetricCategories.VALIDATION,
789
+ metric_name=VALIDATION_METRICS_FINAL,
790
+ )
791
+
792
+ training_final = self._fetch_metric_from_metadata(
793
+ custom_metadata_list=custom_metadata_list,
794
+ target=FineTuningCustomMetadata.TRAINING_METRICS_FINAL,
795
+ category=FineTuningMetricCategories.TRAINING,
796
+ metric_name=TRAINING_METRICS_FINAL,
797
+ )
798
+
799
+ return [
800
+ validation_metrics,
801
+ training_metrics,
802
+ validation_final,
803
+ training_final,
804
+ ]
805
+
806
+ def get_hf_tokenizer_config(self, model_id):
807
+ """
808
+ Gets the default model tokenizer config for the given Aqua model.
809
+ Returns the content of tokenizer_config.json stored in model artifact.
810
+
811
+ Parameters
812
+ ----------
813
+ model_id: str
814
+ The OCID of the Aqua model.
815
+
816
+ Returns
817
+ -------
818
+ Dict:
819
+ Model tokenizer config.
820
+ """
821
+ config = self.get_config(
822
+ model_id, AQUA_MODEL_TOKENIZER_CONFIG, ConfigFolder.ARTIFACT
823
+ ).config
824
+ if not config:
825
+ logger.debug(
826
+ f"{AQUA_MODEL_TOKENIZER_CONFIG} is not available for the model: {model_id}. "
827
+ f"Check if the custom metadata has the artifact path set."
828
+ )
829
+ return config
830
+
831
+ return config
832
+
833
+ @staticmethod
834
+ def to_aqua_model(
835
+ model: Union[
836
+ DataScienceModel,
837
+ oci.data_science.models.model.Model,
838
+ oci.data_science.models.ModelSummary,
839
+ oci.resource_search.models.ResourceSummary,
840
+ ],
841
+ region: str,
842
+ ) -> AquaModel:
843
+ """Converts a model to an Aqua model."""
844
+ return AquaModel(**AquaModelApp._process_model(model, region))
845
+
846
+ @staticmethod
847
+ def _process_model(
848
+ model: Union[
849
+ DataScienceModel,
850
+ oci.data_science.models.model.Model,
851
+ oci.data_science.models.ModelSummary,
852
+ oci.resource_search.models.ResourceSummary,
853
+ ],
854
+ region: str,
855
+ inference_containers: Optional[List[Any]] = None,
856
+ ) -> dict:
857
+ """Constructs required fields for AquaModelSummary."""
858
+
859
+ # todo: revisit icon generation code
860
+ # icon = self._load_icon(model.display_name)
861
+ icon = ""
862
+
863
+ tags = {}
864
+ tags.update(model.defined_tags or {})
865
+ tags.update(model.freeform_tags or {})
866
+
867
+ model_id = (
868
+ model.identifier
869
+ if isinstance(model, oci.resource_search.models.ResourceSummary)
870
+ else model.id
871
+ )
872
+
873
+ console_link = get_console_link(
874
+ resource="models",
875
+ ocid=model_id,
876
+ region=region,
877
+ )
878
+
879
+ description = ""
880
+ if isinstance(model, (DataScienceModel, oci.data_science.models.model.Model)):
881
+ description = model.description
882
+ elif isinstance(model, oci.resource_search.models.ResourceSummary):
883
+ description = model.additional_details.get("description")
884
+
885
+ search_text = (
886
+ AquaModelApp._build_search_text(tags=tags, description=description)
887
+ if tags
888
+ else UNKNOWN
889
+ )
890
+
891
+ freeform_tags = model.freeform_tags or {}
892
+ is_fine_tuned_model = Tags.AQUA_FINE_TUNED_MODEL_TAG in freeform_tags
893
+ ready_to_deploy = (
894
+ freeform_tags.get(Tags.AQUA_TAG, "").upper() == READY_TO_DEPLOY_STATUS
895
+ )
896
+
897
+ ready_to_finetune = (
898
+ freeform_tags.get(Tags.READY_TO_FINE_TUNE, "").upper()
899
+ == READY_TO_FINE_TUNE_STATUS
900
+ )
901
+ ready_to_import = (
902
+ freeform_tags.get(Tags.READY_TO_IMPORT, "").upper()
903
+ == READY_TO_IMPORT_STATUS
904
+ )
905
+
906
+ try:
907
+ model_file = model.custom_metadata_list.get(AQUA_MODEL_ARTIFACT_FILE).value
908
+ except Exception:
909
+ model_file = UNKNOWN
910
+
911
+ if not inference_containers:
912
+ inference_containers = (
913
+ AquaApp().get_container_config().to_dict().get("inference")
914
+ )
915
+
916
+ model_formats_str = freeform_tags.get(
917
+ Tags.MODEL_FORMAT, ModelFormat.SAFETENSORS
918
+ ).upper()
919
+ model_formats = model_formats_str.split(",")
920
+
921
+ supported_platform: Set[str] = set()
922
+
923
+ for container in inference_containers:
924
+ for model_format in model_formats:
925
+ if model_format in container.model_formats:
926
+ supported_platform.update(container.platforms)
927
+
928
+ nvidia_gpu_supported = Platform.NVIDIA_GPU in supported_platform
929
+ arm_cpu_supported = Platform.ARM_CPU in supported_platform
930
+
931
+ return {
932
+ "compartment_id": model.compartment_id,
933
+ "icon": icon or UNKNOWN,
934
+ "id": model_id,
935
+ "license": freeform_tags.get(Tags.LICENSE, UNKNOWN),
936
+ "name": model.display_name,
937
+ "organization": freeform_tags.get(Tags.ORGANIZATION, UNKNOWN),
938
+ "task": freeform_tags.get(Tags.TASK, UNKNOWN),
939
+ "time_created": str(model.time_created),
940
+ "is_fine_tuned_model": is_fine_tuned_model,
941
+ "tags": tags,
942
+ "console_link": console_link,
943
+ "search_text": search_text,
944
+ "ready_to_deploy": ready_to_deploy,
945
+ "ready_to_finetune": ready_to_finetune,
946
+ "ready_to_import": ready_to_import,
947
+ "nvidia_gpu_supported": nvidia_gpu_supported,
948
+ "arm_cpu_supported": arm_cpu_supported,
949
+ "model_file": model_file,
950
+ "model_formats": model_formats,
951
+ }
952
+
953
+ @telemetry(entry_point="plugin=model&action=list", name="aqua")
954
+ def list(
955
+ self,
956
+ compartment_id: str = None,
957
+ category: str = None,
958
+ project_id: str = None,
959
+ model_type: str = None,
960
+ **kwargs,
961
+ ) -> List["AquaModelSummary"]:
962
+ """Lists all Aqua models within a specified compartment and/or project.
963
+ If `category` is not specified, the method defaults to returning
964
+ the service models within the pre-configured default compartment. By default, the list
965
+ of models in the service compartment are cached. Use clear_model_list_cache() to invalidate
966
+ the cache.
967
+
968
+ Parameters
969
+ ----------
970
+ compartment_id: (str, optional). Defaults to `None`.
971
+ The compartment OCID.
972
+ category: (str,optional). Defaults to `SERVICE`
973
+ The category of the models to fetch. Can be either `USER` or `SERVICE`
974
+ project_id: (str, optional). Defaults to `None`.
975
+ The project OCID.
976
+ model_type: (str, optional). Defaults to `None`.
977
+ Model type represents the type of model in the user compartment, can be either FT or BASE.
978
+ **kwargs:
979
+ Additional keyword arguments that can be used to filter the results.
980
+
981
+ Returns
982
+ -------
983
+ List[AquaModelSummary]:
984
+ The list of the `ads.aqua.model.AquaModelSummary`.
985
+ """
986
+
987
+ category = category or kwargs.pop("category", SERVICE)
988
+ compartment_id = compartment_id or COMPARTMENT_OCID
989
+ if category == USER:
990
+ # tracks number of times custom model listing was called
991
+ self.telemetry.record_event_async(
992
+ category="aqua/custom/model", action="list"
993
+ )
994
+
995
+ logger.info(f"Fetching custom models from compartment_id={compartment_id}.")
996
+ model_type = model_type.upper() if model_type else ModelType.FT
997
+ models = self._rqs(compartment_id, model_type=model_type)
998
+ logger.info(
999
+ f"Fetched {len(models)} models from {compartment_id or COMPARTMENT_OCID}."
1000
+ )
1001
+ else:
1002
+ # tracks number of times service model listing was called
1003
+ self.telemetry.record_event_async(
1004
+ category="aqua/service/model", action="list"
1005
+ )
1006
+
1007
+ if AQUA_SERVICE_MODELS in self._service_models_cache:
1008
+ logger.info("Returning service models list from cache.")
1009
+ return self._service_models_cache.get(AQUA_SERVICE_MODELS)
1010
+ lifecycle_state = kwargs.pop(
1011
+ "lifecycle_state", Model.LIFECYCLE_STATE_ACTIVE
1012
+ )
1013
+
1014
+ models = self.list_resource(
1015
+ self.ds_client.list_models,
1016
+ compartment_id=compartment_id,
1017
+ lifecycle_state=lifecycle_state,
1018
+ category=category,
1019
+ **kwargs,
1020
+ )
1021
+ logger.info(f"Fetched {len(models)} service models.")
1022
+
1023
+ aqua_models = []
1024
+ inference_containers = self.get_container_config().to_dict().get("inference")
1025
+ for model in models:
1026
+ aqua_models.append(
1027
+ AquaModelSummary(
1028
+ **self._process_model(
1029
+ model=model,
1030
+ region=self.region,
1031
+ inference_containers=inference_containers,
1032
+ ),
1033
+ project_id=project_id or UNKNOWN,
1034
+ )
1035
+ )
1036
+ if category == SERVICE:
1037
+ self._service_models_cache.__setitem__(
1038
+ key=AQUA_SERVICE_MODELS, value=aqua_models
1039
+ )
1040
+
1041
+ return aqua_models
1042
+
1043
+ def clear_model_list_cache(
1044
+ self,
1045
+ ):
1046
+ """
1047
+ Allows user to clear list model cache items from the service models compartment.
1048
+ Returns
1049
+ -------
1050
+ dict with the key used, and True if cache has the key that needs to be deleted.
1051
+ """
1052
+ res = {}
1053
+ with self._cache_lock:
1054
+ if AQUA_SERVICE_MODELS in self._service_models_cache:
1055
+ self._service_models_cache.pop(key=AQUA_SERVICE_MODELS)
1056
+ logger.info("Cleared models cache for service compartment.")
1057
+ res = {
1058
+ "cache_deleted": True,
1059
+ }
1060
+ return res
1061
+
1062
+ def clear_model_details_cache(self, model_id):
1063
+ """
1064
+ Allows user to clear model details cache item
1065
+ Returns
1066
+ -------
1067
+ dict with the key used, and True if cache has the key that needs to be deleted.
1068
+ """
1069
+ res = {}
1070
+ with self._cache_lock:
1071
+ if model_id in self._service_model_details_cache:
1072
+ self._service_model_details_cache.pop(key=model_id)
1073
+ logger.info(f"Clearing model details cache for model {model_id}.")
1074
+ res = {"key": {"model_id": model_id}, "cache_deleted": True}
1075
+
1076
+ return res
1077
+
1078
+ @staticmethod
1079
+ def list_valid_inference_containers():
1080
+ containers = AquaApp().get_container_config().to_dict().get("inference")
1081
+ family_values = [item.family for item in containers]
1082
+ return family_values
1083
+
1084
+ @telemetry(
1085
+ entry_point="plugin=model&action=get_defined_metadata_artifact_content",
1086
+ name="aqua",
1087
+ )
1088
+ def get_defined_metadata_artifact_content(self, model_id: str, metadata_key: str):
1089
+ """
1090
+ Gets the defined metadata artifact content for the given model
1091
+
1092
+ Args:
1093
+ model_id: str
1094
+ model ocid for which defined metadata artifact needs to be created
1095
+ metadata_key: str
1096
+ defined metadata key like Readme , License , DeploymentConfiguration , FinetuningConfiguration
1097
+ Returns:
1098
+ The model defined metadata artifact content. Can be either str or Dict
1099
+
1100
+ """
1101
+
1102
+ content = self.get_config(model_id, metadata_key)
1103
+ if not content:
1104
+ logger.debug(
1105
+ f"Defined metadata artifact {metadata_key} for model: {model_id} is not available."
1106
+ )
1107
+ return content
1108
+
1109
+ @telemetry(
1110
+ entry_point="plugin=model&action=create_defined_metadata_artifact", name="aqua"
1111
+ )
1112
+ def create_defined_metadata_artifact(
1113
+ self,
1114
+ model_id: str,
1115
+ metadata_key: str,
1116
+ path_type: MetadataArtifactPathType,
1117
+ artifact_path_or_content: str,
1118
+ ) -> None:
1119
+ """
1120
+ Creates defined metadata artifact for the registered unverified model
1121
+
1122
+ Args:
1123
+ model_id: str
1124
+ model ocid for which defined metadata artifact needs to be created
1125
+ metadata_key: str
1126
+ defined metadata key like Readme , License , DeploymentConfiguration , FinetuningConfiguration
1127
+ path_type: str
1128
+ path type of the given defined metadata can be local , oss or the content itself
1129
+ artifact_path_or_content: str
1130
+ It can be local path or oss path or the actual content itself
1131
+ Returns:
1132
+ None
1133
+ """
1134
+
1135
+ ds_model = DataScienceModel.from_id(model_id)
1136
+ oci_aqua = ds_model.freeform_tags.get(Tags.AQUA_TAG, None)
1137
+ if not oci_aqua:
1138
+ raise AquaRuntimeError(f"Target model {model_id} is not an Aqua model.")
1139
+ is_registered_model = ds_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None)
1140
+ is_verified_model = ds_model.freeform_tags.get(
1141
+ Tags.AQUA_SERVICE_MODEL_TAG, None
1142
+ )
1143
+ if is_registered_model and not is_verified_model:
1144
+ try:
1145
+ ds_model.create_defined_metadata_artifact(
1146
+ metadata_key_name=metadata_key,
1147
+ artifact_path_or_content=artifact_path_or_content,
1148
+ path_type=path_type,
1149
+ )
1150
+ except Exception as ex:
1151
+ raise AquaRuntimeError(
1152
+ f"Error occurred in creating defined metadata artifact for model {model_id}: {ex}"
1153
+ ) from ex
1154
+ else:
1155
+ raise AquaRuntimeError(
1156
+ f"Cannot create defined metadata artifact for model {model_id}"
1157
+ )
1158
+
1159
+ def _create_model_catalog_entry(
1160
+ self,
1161
+ os_path: str,
1162
+ model_name: str,
1163
+ inference_container: str,
1164
+ finetuning_container: str,
1165
+ verified_model: DataScienceModel,
1166
+ validation_result: ModelValidationResult,
1167
+ compartment_id: Optional[str],
1168
+ project_id: Optional[str],
1169
+ inference_container_uri: Optional[str],
1170
+ freeform_tags: Optional[dict] = None,
1171
+ defined_tags: Optional[dict] = None,
1172
+ ) -> DataScienceModel:
1173
+ """Create model by reference from the object storage path
1174
+
1175
+ Args:
1176
+ os_path (str): OCI where the model is uploaded - oci://bucket@namespace/prefix
1177
+ model_name (str): name of the model
1178
+ inference_container (str): selects service defaults
1179
+ finetuning_container (str): selects service defaults
1180
+ verified_model (DataScienceModel): If set, then copies all the tags and custom metadata information from the service verified model
1181
+ compartment_id (Optional[str]): Compartment Id of the compartment where the model has to be created
1182
+ project_id (Optional[str]): Project id of the project where the model has to be created
1183
+ inference_container_uri (Optional[str]): Inference container uri for BYOC
1184
+ freeform_tags (dict): Freeform tags for the model
1185
+ defined_tags (dict): Defined tags for the model
1186
+
1187
+ Returns:
1188
+ DataScienceModel: Returns Datascience model instance.
1189
+ """
1190
+ model = DataScienceModel()
1191
+ tags: Dict[str, str] = (
1192
+ {
1193
+ **verified_model.freeform_tags,
1194
+ Tags.AQUA_SERVICE_MODEL_TAG: verified_model.id,
1195
+ }
1196
+ if verified_model
1197
+ else {
1198
+ Tags.AQUA_TAG: "active",
1199
+ Tags.BASE_MODEL_CUSTOM: "true",
1200
+ }
1201
+ )
1202
+ tags.update({Tags.BASE_MODEL_CUSTOM: "true"})
1203
+
1204
+ if validation_result and validation_result.model_formats:
1205
+ tags.update(
1206
+ {
1207
+ Tags.MODEL_FORMAT: ",".join(
1208
+ model_format for model_format in validation_result.model_formats
1209
+ )
1210
+ }
1211
+ )
1212
+
1213
+ # Remove `ready_to_import` tag that might get copied from service model.
1214
+ tags.pop(Tags.READY_TO_IMPORT, None)
1215
+ defined_metadata_dict = {}
1216
+ readme_file_path = os_path.rstrip("/") + "/" + README
1217
+ license_file_path = os_path.rstrip("/") + "/" + LICENSE
1218
+ if verified_model:
1219
+ # Verified model is a model in the service catalog that either has no artifacts but contains all the necessary metadata for deploying and fine tuning.
1220
+ # If set, then we copy all the model metadata.
1221
+ metadata = verified_model.custom_metadata_list
1222
+ if verified_model.model_file_description:
1223
+ model = model.with_model_file_description(
1224
+ json_dict=verified_model.model_file_description
1225
+ )
1226
+ defined_metadata_list = (
1227
+ verified_model.defined_metadata_list._to_oci_metadata()
1228
+ )
1229
+ for defined_metadata in defined_metadata_list:
1230
+ if defined_metadata.has_artifact:
1231
+ content = (
1232
+ self.ds_client.get_model_defined_metadatum_artifact_content(
1233
+ verified_model.id, defined_metadata.key
1234
+ ).data.content
1235
+ )
1236
+ defined_metadata_dict[defined_metadata.key] = content
1237
+ else:
1238
+ metadata = ModelCustomMetadata()
1239
+ if not inference_container:
1240
+ raise AquaRuntimeError(
1241
+ f"Require Inference container information. Model: {model_name} does not have associated inference "
1242
+ f"container defaults. Check docs for more information on how to pass inference container."
1243
+ )
1244
+ metadata.add(
1245
+ key=AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,
1246
+ value=inference_container,
1247
+ description=f"Inference container mapping for {model_name}",
1248
+ category="Other",
1249
+ )
1250
+ if inference_container_uri:
1251
+ metadata.add(
1252
+ key=AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME,
1253
+ value=inference_container_uri,
1254
+ description=f"Inference container URI for {model_name}",
1255
+ category="Other",
1256
+ )
1257
+
1258
+ inference_containers = (
1259
+ AquaContainerConfig.from_service_config(
1260
+ service_containers=self.list_service_containers()
1261
+ )
1262
+ .to_dict()
1263
+ .get("inference")
1264
+ )
1265
+ smc_container_set = {container.family for container in inference_containers}
1266
+ # only add cmd vars if inference container is not an SMC
1267
+ if (
1268
+ inference_container not in smc_container_set
1269
+ and inference_container in CustomInferenceContainerTypeFamily.values()
1270
+ ):
1271
+ cmd_vars = generate_tei_cmd_var(os_path)
1272
+ metadata.add(
1273
+ key=AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME,
1274
+ value=" ".join(cmd_vars),
1275
+ description=f"Inference container cmd vars for {model_name}",
1276
+ category="Other",
1277
+ )
1278
+
1279
+ if finetuning_container:
1280
+ tags[Tags.READY_TO_FINE_TUNE] = "true"
1281
+ metadata.add(
1282
+ key=AQUA_FINETUNING_CONTAINER_METADATA_NAME,
1283
+ value=finetuning_container,
1284
+ description=f"Fine-tuning container mapping for {model_name}",
1285
+ category="Other",
1286
+ )
1287
+ else:
1288
+ logger.warn(
1289
+ "Proceeding with model registration without the fine-tuning container information. "
1290
+ "This model will not be available for fine tuning."
1291
+ )
1292
+ if validation_result and validation_result.model_file:
1293
+ metadata.add(
1294
+ key=AQUA_MODEL_ARTIFACT_FILE,
1295
+ value=validation_result.model_file,
1296
+ description=f"The model file for {model_name}",
1297
+ category="Other",
1298
+ )
1299
+
1300
+ metadata.add(
1301
+ key=AQUA_EVALUATION_CONTAINER_METADATA_NAME,
1302
+ value="odsc-llm-evaluate",
1303
+ description="Evaluation container mapping for SMC",
1304
+ category="Other",
1305
+ )
1306
+
1307
+ if validation_result and validation_result.tags:
1308
+ tags[Tags.TASK] = validation_result.tags.get(Tags.TASK, UNKNOWN)
1309
+ tags[Tags.ORGANIZATION] = validation_result.tags.get(
1310
+ Tags.ORGANIZATION, UNKNOWN
1311
+ )
1312
+ tags[Tags.LICENSE] = validation_result.tags.get(Tags.LICENSE, UNKNOWN)
1313
+
1314
+ # Set artifact location to user bucket, and replace existing key if present.
1315
+ metadata.add(
1316
+ key=MODEL_BY_REFERENCE_OSS_PATH_KEY,
1317
+ value=os_path,
1318
+ description="artifact location",
1319
+ category="Other",
1320
+ replace=True,
1321
+ )
1322
+ # override tags with freeform tags if set
1323
+ tags = {**tags, **(freeform_tags or {})}
1324
+ model = (
1325
+ model.with_custom_metadata_list(metadata)
1326
+ .with_compartment_id(compartment_id or COMPARTMENT_OCID)
1327
+ .with_project_id(project_id or PROJECT_OCID)
1328
+ .with_artifact(os_path)
1329
+ .with_display_name(model_name)
1330
+ .with_freeform_tags(**tags)
1331
+ .with_defined_tags(**(defined_tags or {}))
1332
+ ).create(model_by_reference=True)
1333
+ logger.debug(f"Created model catalog entry for the model:\n{model}")
1334
+ for key, value in defined_metadata_dict.items():
1335
+ model.create_defined_metadata_artifact(
1336
+ key, value, MetadataArtifactPathType.CONTENT
1337
+ )
1338
+
1339
+ if is_path_exists(readme_file_path):
1340
+ try:
1341
+ model.create_defined_metadata_artifact(
1342
+ AquaModelMetadataKeys.README,
1343
+ readme_file_path,
1344
+ MetadataArtifactPathType.OSS,
1345
+ )
1346
+ except Exception as ex:
1347
+ logger.error(
1348
+ f"Error Uploading Readme in defined metadata for model: {model.id} : {str(ex)}"
1349
+ )
1350
+ if not verified_model and is_path_exists(license_file_path):
1351
+ try:
1352
+ model.create_defined_metadata_artifact(
1353
+ AquaModelMetadataKeys.LICENSE,
1354
+ license_file_path,
1355
+ MetadataArtifactPathType.OSS,
1356
+ )
1357
+ except Exception as ex:
1358
+ logger.error(
1359
+ f"Error Uploading License in defined metadata for model: {model.id} : {str(ex)}"
1360
+ )
1361
+ return model
1362
+
1363
+ @staticmethod
1364
+ def get_model_files(os_path: str, model_format: str) -> List[str]:
1365
+ """
1366
+ Get a list of model files based on the given OS path and model format.
1367
+
1368
+ Args:
1369
+ os_path (str): The OS path where the model files are located.
1370
+ model_format (str): The format of the model files.
1371
+
1372
+ Returns:
1373
+ List[str]: A list of model file names.
1374
+
1375
+ """
1376
+ model_files: List[str] = []
1377
+ # todo: revisit this logic to account for .bin files. In the current state, .bin and .safetensor models
1378
+ # are grouped in one category and validation checks for config.json files only.
1379
+ if model_format == ModelFormat.SAFETENSORS:
1380
+ model_files.extend(
1381
+ list_os_files_with_extension(oss_path=os_path, extension=".safetensors")
1382
+ )
1383
+ try:
1384
+ load_config(
1385
+ file_path=os_path,
1386
+ config_file_name=AQUA_MODEL_ARTIFACT_CONFIG,
1387
+ )
1388
+ except Exception as ex:
1389
+ message = (
1390
+ f"The model path {os_path} does not contain the file config.json. "
1391
+ f"Please check if the path is correct or the model artifacts are available at this location."
1392
+ )
1393
+ logger.warning(
1394
+ f"{message}\n"
1395
+ f"Details: {ex.reason if isinstance(ex, AquaFileNotFoundError) else str(ex)}\n"
1396
+ )
1397
+ else:
1398
+ model_files.append(AQUA_MODEL_ARTIFACT_CONFIG)
1399
+
1400
+ if model_format == ModelFormat.GGUF:
1401
+ model_files.extend(
1402
+ list_os_files_with_extension(oss_path=os_path, extension=".gguf")
1403
+ )
1404
+ logger.debug(
1405
+ f"Fetched {len(model_files)} model files from {os_path} for model format {model_format}."
1406
+ )
1407
+ return model_files
1408
+
1409
+ @staticmethod
1410
+ def get_hf_model_files(model_name: str, model_format: str) -> List[str]:
1411
+ """
1412
+ Get a list of model files based on the given OS path and model format.
1413
+
1414
+ Args:
1415
+ model_name (str): The huggingface model name.
1416
+ model_format (str): The format of the model files.
1417
+
1418
+ Returns:
1419
+ List[str]: A list of model file names.
1420
+
1421
+ """
1422
+ model_files: List[str] = []
1423
+
1424
+ # todo: revisit this logic to account for .bin files. In the current state, .bin and .safetensor models
1425
+ # are grouped in one category and returns config.json file only.
1426
+
1427
+ try:
1428
+ model_siblings = get_hf_model_info(repo_id=model_name).siblings
1429
+ except Exception as e:
1430
+ huggingface_err_message = str(e)
1431
+ raise AquaValueError(
1432
+ f"Could not get the model files of {model_name} from https://huggingface.co. "
1433
+ f"Error: {huggingface_err_message}."
1434
+ ) from e
1435
+
1436
+ if not model_siblings:
1437
+ raise AquaValueError(
1438
+ f"Failed to fetch the model files of {model_name} from https://huggingface.co."
1439
+ )
1440
+
1441
+ for model_sibling in model_siblings:
1442
+ extension = pathlib.Path(model_sibling.rfilename).suffix[1:].upper()
1443
+ if (
1444
+ model_format == ModelFormat.SAFETENSORS
1445
+ and model_sibling.rfilename == AQUA_MODEL_ARTIFACT_CONFIG
1446
+ ):
1447
+ model_files.append(model_sibling.rfilename)
1448
+ if extension == model_format:
1449
+ model_files.append(model_sibling.rfilename)
1450
+
1451
+ logger.debug(
1452
+ f"Fetched {len(model_files)} model files for the model {model_name} for model format {model_format}."
1453
+ )
1454
+ return model_files
1455
+
1456
+ def _validate_model(
1457
+ self,
1458
+ import_model_details: ImportModelDetails = None,
1459
+ model_name: str = None,
1460
+ verified_model: DataScienceModel = None,
1461
+ ) -> ModelValidationResult:
1462
+ """
1463
+ Validates the model configuration and returns the model format telemetry model name.
1464
+
1465
+ Args:
1466
+ import_model_details (ImportModelDetails): Model details for importing the model.
1467
+ model_name (str): name of the model
1468
+ verified_model (DataScienceModel): If set, then copies all the tags and custom metadata information from
1469
+ the service verified model
1470
+
1471
+ Returns:
1472
+ ModelValidationResult: The result of the model validation.
1473
+
1474
+ Raises:
1475
+ AquaRuntimeError: If there is an error while loading the config file or if the model path is incorrect.
1476
+ AquaValueError: If the model format is not supported by AQUA.
1477
+ """
1478
+ model_formats = []
1479
+ validation_result: ModelValidationResult = ModelValidationResult()
1480
+
1481
+ hf_download_config_present = False
1482
+
1483
+ if import_model_details.download_from_hf:
1484
+ safetensors_model_files = self.get_hf_model_files(
1485
+ model_name, ModelFormat.SAFETENSORS
1486
+ )
1487
+ if (
1488
+ safetensors_model_files
1489
+ and AQUA_MODEL_ARTIFACT_CONFIG in safetensors_model_files
1490
+ ):
1491
+ hf_download_config_present = True
1492
+ gguf_model_files = self.get_hf_model_files(model_name, ModelFormat.GGUF)
1493
+ else:
1494
+ safetensors_model_files = self.get_model_files(
1495
+ import_model_details.os_path, ModelFormat.SAFETENSORS
1496
+ )
1497
+ gguf_model_files = self.get_model_files(
1498
+ import_model_details.os_path, ModelFormat.GGUF
1499
+ )
1500
+
1501
+ if not (safetensors_model_files or gguf_model_files):
1502
+ raise AquaRuntimeError(
1503
+ f"The model {model_name} does not contain either {ModelFormat.SAFETENSORS} "
1504
+ f"or {ModelFormat.GGUF} files in {import_model_details.os_path} or Hugging Face repository. "
1505
+ f"Please check if the path is correct or the model artifacts are available at this location."
1506
+ )
1507
+
1508
+ if verified_model:
1509
+ aqua_model = self.to_aqua_model(verified_model, self.region)
1510
+ model_formats = aqua_model.model_formats
1511
+ else:
1512
+ if safetensors_model_files:
1513
+ model_formats.append(ModelFormat.SAFETENSORS)
1514
+ if gguf_model_files:
1515
+ model_formats.append(ModelFormat.GGUF)
1516
+
1517
+ # get tags for models from hf
1518
+ if import_model_details.download_from_hf:
1519
+ model_info = get_hf_model_info(repo_id=model_name)
1520
+
1521
+ try:
1522
+ license_value = UNKNOWN
1523
+ if model_info.tags:
1524
+ license_tag = next(
1525
+ (
1526
+ tag
1527
+ for tag in model_info.tags
1528
+ if tag.startswith("license:")
1529
+ ),
1530
+ UNKNOWN,
1531
+ )
1532
+ license_value = (
1533
+ license_tag.split(":")[1] if license_tag else UNKNOWN
1534
+ )
1535
+
1536
+ hf_tags = {
1537
+ Tags.TASK: (model_info and model_info.pipeline_tag) or UNKNOWN,
1538
+ Tags.ORGANIZATION: (
1539
+ model_info.author
1540
+ if model_info and hasattr(model_info, "author")
1541
+ else UNKNOWN
1542
+ ),
1543
+ Tags.LICENSE: license_value,
1544
+ }
1545
+ validation_result.tags = hf_tags
1546
+ except Exception as ex:
1547
+ logger.debug(
1548
+ f"An error occurred while getting tag information for model {model_name}. "
1549
+ f"Error: {str(ex)}"
1550
+ )
1551
+
1552
+ validation_result.model_formats = model_formats
1553
+
1554
+ # now as we know that at least one type of model files exist, validate the content of oss path.
1555
+ # for safetensors, we check if config.json files exist, and for gguf format we check if files with
1556
+ # gguf extension exist.
1557
+ if {ModelFormat.SAFETENSORS, ModelFormat.GGUF}.issubset(set(model_formats)):
1558
+ if (
1559
+ import_model_details.inference_container.lower()
1560
+ == InferenceContainerTypeFamily.AQUA_LLAMA_CPP_CONTAINER_FAMILY
1561
+ ):
1562
+ self._validate_gguf_format(
1563
+ import_model_details=import_model_details,
1564
+ verified_model=verified_model,
1565
+ gguf_model_files=gguf_model_files,
1566
+ validation_result=validation_result,
1567
+ model_name=model_name,
1568
+ )
1569
+ else:
1570
+ self._validate_safetensor_format(
1571
+ import_model_details=import_model_details,
1572
+ verified_model=verified_model,
1573
+ validation_result=validation_result,
1574
+ hf_download_config_present=hf_download_config_present,
1575
+ model_name=model_name,
1576
+ )
1577
+ elif ModelFormat.SAFETENSORS in model_formats:
1578
+ self._validate_safetensor_format(
1579
+ import_model_details=import_model_details,
1580
+ verified_model=verified_model,
1581
+ validation_result=validation_result,
1582
+ hf_download_config_present=hf_download_config_present,
1583
+ model_name=model_name,
1584
+ )
1585
+ elif ModelFormat.GGUF in model_formats:
1586
+ self._validate_gguf_format(
1587
+ import_model_details=import_model_details,
1588
+ verified_model=verified_model,
1589
+ gguf_model_files=gguf_model_files,
1590
+ validation_result=validation_result,
1591
+ model_name=model_name,
1592
+ )
1593
+
1594
+ return validation_result
1595
+
1596
+ @staticmethod
1597
+ def _validate_safetensor_format(
1598
+ import_model_details: ImportModelDetails = None,
1599
+ verified_model: DataScienceModel = None,
1600
+ validation_result: ModelValidationResult = None,
1601
+ hf_download_config_present: bool = None,
1602
+ model_name: str = None,
1603
+ ):
1604
+ if import_model_details.download_from_hf:
1605
+ # validates config.json exists for safetensors model from huggingface
1606
+ if not (
1607
+ hf_download_config_present
1608
+ or import_model_details.ignore_model_artifact_check
1609
+ ):
1610
+ raise AquaRuntimeError(
1611
+ f"The model {model_name} does not contain {AQUA_MODEL_ARTIFACT_CONFIG} file as required "
1612
+ f"by {ModelFormat.SAFETENSORS} format model."
1613
+ f" Please check if the model name is correct in Hugging Face repository."
1614
+ )
1615
+ validation_result.telemetry_model_name = model_name
1616
+ else:
1617
+ # validate if config.json is available from object storage, and get model name for telemetry
1618
+ model_config = None
1619
+ try:
1620
+ model_config = load_config(
1621
+ file_path=import_model_details.os_path,
1622
+ config_file_name=AQUA_MODEL_ARTIFACT_CONFIG,
1623
+ )
1624
+ except Exception as ex:
1625
+ message = (
1626
+ f"The model path {import_model_details.os_path} does not contain the file config.json. "
1627
+ f"Please check if the path is correct or the model artifacts are available at this location."
1628
+ )
1629
+ if not import_model_details.ignore_model_artifact_check:
1630
+ logger.error(
1631
+ f"{message}\n"
1632
+ f"Details: {ex.reason if isinstance(ex, AquaFileNotFoundError) else str(ex)}"
1633
+ )
1634
+ raise AquaRuntimeError(message) from ex
1635
+ else:
1636
+ logger.warning(
1637
+ f"{message}\n"
1638
+ f"Proceeding with model registration as ignore_model_artifact_check field is set."
1639
+ )
1640
+
1641
+ if verified_model:
1642
+ # model_type validation, log message if metadata field doesn't match.
1643
+ try:
1644
+ metadata_model_type = verified_model.custom_metadata_list.get(
1645
+ AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE
1646
+ ).value
1647
+ if metadata_model_type and model_config is not None:
1648
+ if AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE in model_config:
1649
+ if (
1650
+ model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE]
1651
+ != metadata_model_type
1652
+ ):
1653
+ logger.debug(
1654
+ f"The {AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE} attribute in {AQUA_MODEL_ARTIFACT_CONFIG}"
1655
+ f" at {import_model_details.os_path} is invalid, expected {metadata_model_type} for "
1656
+ f"the model {model_name}. Please check if the path is correct or "
1657
+ f"the correct model artifacts are available at this location."
1658
+ f""
1659
+ )
1660
+ else:
1661
+ logger.debug(
1662
+ f"Could not find {AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE} attribute in "
1663
+ f"{AQUA_MODEL_ARTIFACT_CONFIG}. Proceeding with model registration."
1664
+ )
1665
+ except Exception as ex:
1666
+ # todo: raise exception if model_type doesn't match. Currently log message and pass since service
1667
+ # models do not have this metadata.
1668
+ logger.debug(
1669
+ f"Error occurred while processing metadata for model {model_name}. "
1670
+ f"Exception: {str(ex)}"
1671
+ )
1672
+ validation_result.telemetry_model_name = verified_model.display_name
1673
+ elif (
1674
+ model_config is not None
1675
+ and AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME in model_config
1676
+ ):
1677
+ validation_result.telemetry_model_name = f"{AQUA_MODEL_TYPE_CUSTOM}_{model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_NAME]}"
1678
+ elif (
1679
+ model_config is not None
1680
+ and AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE in model_config
1681
+ ):
1682
+ validation_result.telemetry_model_name = f"{AQUA_MODEL_TYPE_CUSTOM}_{model_config[AQUA_MODEL_ARTIFACT_CONFIG_MODEL_TYPE]}"
1683
+ else:
1684
+ validation_result.telemetry_model_name = AQUA_MODEL_TYPE_CUSTOM
1685
+
1686
+ @staticmethod
1687
+ def _validate_gguf_format(
1688
+ import_model_details: ImportModelDetails = None,
1689
+ verified_model: DataScienceModel = None,
1690
+ gguf_model_files: List[str] = None,
1691
+ validation_result: ModelValidationResult = None,
1692
+ model_name: str = None,
1693
+ ):
1694
+ if import_model_details.finetuning_container:
1695
+ raise AquaValueError(
1696
+ "Fine-tuning is currently not supported with GGUF model format."
1697
+ )
1698
+ if verified_model:
1699
+ try:
1700
+ model_file = verified_model.custom_metadata_list.get(
1701
+ AQUA_MODEL_ARTIFACT_FILE
1702
+ ).value
1703
+ except ValueError as err:
1704
+ raise AquaRuntimeError(
1705
+ f"The model {verified_model.display_name} does not contain the custom metadata {AQUA_MODEL_ARTIFACT_FILE}. "
1706
+ f"Please check if the model has the valid metadata."
1707
+ ) from err
1708
+ else:
1709
+ model_file = import_model_details.model_file
1710
+
1711
+ model_files = gguf_model_files
1712
+ # todo: have a separate error validation class for different type of error messages.
1713
+ if model_file:
1714
+ if model_file not in model_files:
1715
+ raise AquaRuntimeError(
1716
+ f"The model path {import_model_details.os_path} or the Hugging Face "
1717
+ f"model repository for {model_name} does not contain the file "
1718
+ f"{model_file}. Please check if the path is correct or the model "
1719
+ f"artifacts are available at this location."
1720
+ )
1721
+ else:
1722
+ validation_result.model_file = model_file
1723
+ elif len(model_files) == 0:
1724
+ raise AquaRuntimeError(
1725
+ f"The model path {import_model_details.os_path} or the Hugging Face model "
1726
+ f"repository for {model_name} does not contain any GGUF format files. "
1727
+ f"Please check if the path is correct or the model artifacts are available "
1728
+ f"at this location."
1729
+ )
1730
+ elif len(model_files) > 1:
1731
+ raise AquaRuntimeError(
1732
+ f"The model path {import_model_details.os_path} or the Hugging Face model "
1733
+ f"repository for {model_name} contains multiple GGUF format files. "
1734
+ f"Please specify the file that needs to be deployed using the model_file "
1735
+ f"parameter."
1736
+ )
1737
+ else:
1738
+ validation_result.model_file = model_files[0]
1739
+
1740
+ if verified_model:
1741
+ validation_result.telemetry_model_name = verified_model.display_name
1742
+ elif import_model_details.download_from_hf:
1743
+ validation_result.telemetry_model_name = model_name
1744
+ else:
1745
+ validation_result.telemetry_model_name = AQUA_MODEL_TYPE_CUSTOM
1746
+
1747
+ @staticmethod
1748
+ def _download_model_from_hf(
1749
+ model_name: str,
1750
+ os_path: str,
1751
+ local_dir: str = None,
1752
+ allow_patterns: List[str] = None,
1753
+ ignore_patterns: List[str] = None,
1754
+ ) -> str:
1755
+ """This helper function downloads the model artifact from Hugging Face to a local folder, then uploads
1756
+ to object storage location.
1757
+
1758
+ Parameters
1759
+ ----------
1760
+ model_name (str): The huggingface model name.
1761
+ os_path (str): The OS path where the model files are located.
1762
+ local_dir (str): The local temp dir to store the huggingface model.
1763
+ allow_patterns (list): Model files matching at least one pattern are downloaded.
1764
+ Example: ["*.json"] will download all .json files. ["folder/*"] will download all files under `folder`.
1765
+ Patterns are Standard Wildcards (globbing patterns) and rules can be found here: https://docs.python.org/3/library/fnmatch.html
1766
+ ignore_patterns (list): Model files matching any of the patterns are not downloaded.
1767
+ Example: ["*.json"] will ignore all .json files. ["folder/*"] will ignore all files under `folder`.
1768
+ Patterns are Standard Wildcards (globbing patterns) and rules can be found here: https://docs.python.org/3/library/fnmatch.html
1769
+
1770
+ Returns
1771
+ -------
1772
+ model_artifact_path (str): Location where the model artifacts are downloaded.
1773
+ """
1774
+ # Download the model from hub
1775
+ if local_dir:
1776
+ local_dir = os.path.join(local_dir, model_name)
1777
+ os.makedirs(local_dir, exist_ok=True)
1778
+
1779
+ # if local_dir is not set, the return value points to the cached data folder
1780
+ local_dir = snapshot_download(
1781
+ repo_id=model_name,
1782
+ local_dir=local_dir,
1783
+ allow_patterns=allow_patterns,
1784
+ ignore_patterns=ignore_patterns,
1785
+ )
1786
+ # Upload to object storage and skip .cache/huggingface/ folder
1787
+ logger.debug(
1788
+ f"Uploading local artifacts from local directory {local_dir} to {os_path}."
1789
+ )
1790
+ # Upload to object storage
1791
+ model_artifact_path = upload_folder(
1792
+ os_path=os_path,
1793
+ local_dir=local_dir,
1794
+ model_name=model_name,
1795
+ exclude_pattern=f"{HF_METADATA_FOLDER}*",
1796
+ )
1797
+
1798
+ return model_artifact_path
1799
+
1800
+ def register(
1801
+ self, import_model_details: ImportModelDetails = None, **kwargs
1802
+ ) -> AquaModel:
1803
+ """Loads the model from object storage and registers as Model in Data Science Model catalog
1804
+ The inference container and finetuning container could be of type Service Managed Container(SMC) or custom.
1805
+ If it is custom, full container URI is expected. If it of type SMC, only the container family name is expected.\n
1806
+ For detailed information about CLI flags see: https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/ai-quick-actions/cli-tips.md#register-model
1807
+
1808
+ Args:
1809
+ import_model_details (ImportModelDetails): Model details for importing the model.
1810
+ kwargs:
1811
+ model (str): name of the model or OCID of the service model that has inference and finetuning information
1812
+ os_path (str): Object storage destination URI to store the downloaded model. Format: oci://bucket-name@namespace/prefix
1813
+ inference_container (str): selects service defaults
1814
+ finetuning_container (str): selects service defaults
1815
+ allow_patterns (list): Model files matching at least one pattern are downloaded.
1816
+ Example: ["*.json"] will download all .json files. ["folder/*"] will download all files under `folder`.
1817
+ Patterns are Standard Wildcards (globbing patterns) and rules can be found here: https://docs.python.org/3/library/fnmatch.html
1818
+ ignore_patterns (list): Model files matching any of the patterns are not downloaded.
1819
+ Example: ["*.json"] will ignore all .json files. ["folder/*"] will ignore all files under `folder`.
1820
+ Patterns are Standard Wildcards (globbing patterns) and rules can be found here: https://docs.python.org/3/library/fnmatch.html
1821
+ cleanup_model_cache (bool): Deletes downloaded files from local machine after model is successfully
1822
+ registered. Set to True by default.
1823
+
1824
+ Returns:
1825
+ AquaModel:
1826
+ The registered model as a AquaModel object.
1827
+ """
1828
+ if not import_model_details:
1829
+ import_model_details = ImportModelDetails(**kwargs)
1830
+
1831
+ # If OCID of a model is passed, we need to copy the defaults for Tags and metadata from the service model.
1832
+ verified_model: Optional[DataScienceModel] = None
1833
+ if (
1834
+ import_model_details.model.startswith("ocid")
1835
+ and "datasciencemodel" in import_model_details.model
1836
+ ):
1837
+ logger.info(f"Fetching details for model {import_model_details.model}.")
1838
+ verified_model = DataScienceModel.from_id(import_model_details.model)
1839
+ else:
1840
+ # If users passes model name, check if there is model with the same name in the service model catalog. If it is there, then use that model
1841
+ model_service_id = self._find_matching_aqua_model(
1842
+ import_model_details.model
1843
+ )
1844
+ if model_service_id:
1845
+ logger.info(
1846
+ f"Found service model for {import_model_details.model}: {model_service_id}"
1847
+ )
1848
+ verified_model = DataScienceModel.from_id(model_service_id)
1849
+
1850
+ # Copy the model name from the service model if `model` is ocid
1851
+ model_name = (
1852
+ verified_model.display_name
1853
+ if verified_model
1854
+ else import_model_details.model
1855
+ )
1856
+
1857
+ # validate model and artifact
1858
+ validation_result = self._validate_model(
1859
+ import_model_details=import_model_details,
1860
+ model_name=model_name,
1861
+ verified_model=verified_model,
1862
+ )
1863
+
1864
+ # download model from hugginface if indicates
1865
+ if import_model_details.download_from_hf:
1866
+ artifact_path = self._download_model_from_hf(
1867
+ model_name=model_name,
1868
+ os_path=import_model_details.os_path,
1869
+ local_dir=import_model_details.local_dir,
1870
+ allow_patterns=import_model_details.allow_patterns,
1871
+ ignore_patterns=import_model_details.ignore_patterns,
1872
+ ).rstrip("/")
1873
+ else:
1874
+ artifact_path = import_model_details.os_path.rstrip("/")
1875
+
1876
+ # Create Model catalog entry with pass by reference
1877
+ ds_model = self._create_model_catalog_entry(
1878
+ os_path=artifact_path,
1879
+ model_name=model_name,
1880
+ inference_container=import_model_details.inference_container,
1881
+ finetuning_container=import_model_details.finetuning_container,
1882
+ verified_model=verified_model,
1883
+ validation_result=validation_result,
1884
+ compartment_id=import_model_details.compartment_id,
1885
+ project_id=import_model_details.project_id,
1886
+ inference_container_uri=import_model_details.inference_container_uri,
1887
+ freeform_tags=import_model_details.freeform_tags,
1888
+ defined_tags=import_model_details.defined_tags,
1889
+ )
1890
+ # registered model will always have inference and evaluation container, but
1891
+ # fine-tuning container may be not set
1892
+ inference_container = ds_model.custom_metadata_list.get(
1893
+ ModelCustomMetadataFields.DEPLOYMENT_CONTAINER,
1894
+ ModelCustomMetadataItem(key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER),
1895
+ ).value
1896
+ inference_container_uri = ds_model.custom_metadata_list.get(
1897
+ ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI,
1898
+ ModelCustomMetadataItem(
1899
+ key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER_URI
1900
+ ),
1901
+ ).value
1902
+ evaluation_container = ds_model.custom_metadata_list.get(
1903
+ ModelCustomMetadataFields.EVALUATION_CONTAINER,
1904
+ ModelCustomMetadataItem(key=ModelCustomMetadataFields.EVALUATION_CONTAINER),
1905
+ ).value
1906
+ finetuning_container: str = ds_model.custom_metadata_list.get(
1907
+ ModelCustomMetadataFields.FINETUNE_CONTAINER,
1908
+ ModelCustomMetadataItem(key=ModelCustomMetadataFields.FINETUNE_CONTAINER),
1909
+ ).value
1910
+
1911
+ aqua_model_attributes = dict(
1912
+ **self._process_model(ds_model, self.region),
1913
+ project_id=ds_model.project_id,
1914
+ inference_container=inference_container,
1915
+ inference_container_uri=inference_container_uri,
1916
+ finetuning_container=finetuning_container,
1917
+ evaluation_container=evaluation_container,
1918
+ artifact_location=artifact_path,
1919
+ )
1920
+
1921
+ self.telemetry.record_event_async(
1922
+ category="aqua/model",
1923
+ action="register",
1924
+ detail=validation_result.telemetry_model_name,
1925
+ )
1926
+
1927
+ if (
1928
+ import_model_details.download_from_hf
1929
+ and import_model_details.cleanup_model_cache
1930
+ ):
1931
+ cleanup_local_hf_model_artifact(
1932
+ model_name=model_name, local_dir=import_model_details.local_dir
1933
+ )
1934
+
1935
+ return AquaModel(**aqua_model_attributes)
1936
+
1937
+ def _if_show(self, model: DataScienceModel) -> bool:
1938
+ """Determine if the given model should be return by `list`."""
1939
+ if model.freeform_tags is None:
1940
+ return False
1941
+
1942
+ TARGET_TAGS = model.freeform_tags.keys()
1943
+ return Tags.AQUA_TAG in TARGET_TAGS or Tags.AQUA_TAG.lower() in TARGET_TAGS
1944
+
1945
+ def _load_icon(self, model_name: str) -> str:
1946
+ """Loads icon."""
1947
+
1948
+ # TODO: switch to the official logo
1949
+ try:
1950
+ return create_word_icon(model_name, return_as_datauri=True)
1951
+ except Exception as e:
1952
+ logger.debug(f"Failed to load icon for the model={model_name}: {str(e)}.")
1953
+ return None
1954
+
1955
+ def _rqs(self, compartment_id: str, model_type="FT", **kwargs):
1956
+ """Use RQS to fetch models in the user tenancy."""
1957
+ if model_type == ModelType.FT:
1958
+ filter_tag = Tags.AQUA_FINE_TUNED_MODEL_TAG
1959
+ elif model_type == ModelType.BASE:
1960
+ filter_tag = Tags.BASE_MODEL_CUSTOM
1961
+ # elif model_type == ModelType.MULTIMODEL:
1962
+ # filter_tag = Tags.MULTIMODEL_TYPE_TAG
1963
+ else:
1964
+ raise AquaValueError(
1965
+ f"Model of type {model_type} is unknown. The values should be in {ModelType.values()}"
1966
+ )
1967
+
1968
+ condition_tags = f"&& (freeformTags.key = '{Tags.AQUA_TAG}' && freeformTags.key = '{filter_tag}')"
1969
+ condition_lifecycle = "&& lifecycleState = 'ACTIVE'"
1970
+ query = f"query datasciencemodel resources where (compartmentId = '{compartment_id}' {condition_lifecycle} {condition_tags})"
1971
+ logger.info(query)
1972
+ logger.info(f"tenant_id={TENANCY_OCID}")
1973
+ return OCIResource.search(
1974
+ query, type=SEARCH_TYPE.STRUCTURED, tenant_id=TENANCY_OCID, **kwargs
1975
+ )
1976
+
1977
+ @staticmethod
1978
+ def _build_search_text(tags: dict, description: str = None) -> str:
1979
+ """Constructs search_text field in response."""
1980
+ description = description or ""
1981
+ tags_text = (
1982
+ ",".join(str(v) for v in tags.values()) if isinstance(tags, dict) else ""
1983
+ )
1984
+ separator = " " if description else ""
1985
+ return f"{description}{separator}{tags_text}"
1986
+
1987
+ @telemetry(entry_point="plugin=model&action=load_readme", name="aqua")
1988
+ def load_readme(self, model_id: str) -> AquaModelReadme:
1989
+ """Loads the readme or the model card for the given model.
1990
+
1991
+ Parameters
1992
+ ----------
1993
+ model_id: str
1994
+ The model id.
1995
+
1996
+ Returns
1997
+ -------
1998
+ AquaModelReadme:
1999
+ The instance of AquaModelReadme.
2000
+ """
2001
+ oci_model = self.ds_client.get_model(model_id).data
2002
+ artifact_path = get_artifact_path(oci_model.custom_metadata_list)
2003
+ if not artifact_path:
2004
+ raise AquaRuntimeError(
2005
+ f"Readme could not be loaded. Failed to get artifact path from custom metadata for"
2006
+ f"the model {model_id}."
2007
+ )
2008
+
2009
+ content = ""
2010
+ try:
2011
+ content = self.ds_client.get_model_defined_metadatum_artifact_content(
2012
+ model_id, AquaModelMetadataKeys.README
2013
+ ).data.content.decode("utf-8", errors="ignore")
2014
+ logger.info(f"Fetched {README} from defined metadata for model: {model_id}")
2015
+ except Exception as ex:
2016
+ logger.error(
2017
+ f"Readme could not be found for model: {model_id} in defined metadata : {str(ex)}"
2018
+ )
2019
+ artifact_path = get_artifact_path(oci_model.custom_metadata_list)
2020
+ readme_path = os.path.join(os.path.dirname(artifact_path), "artifact")
2021
+ if not is_path_exists(readme_path):
2022
+ readme_path = os.path.join(artifact_path.rstrip("/"), "artifact")
2023
+ if not is_path_exists(readme_path):
2024
+ readme_path = f"{artifact_path.rstrip('/')}/"
2025
+
2026
+ readme_file_path = os.path.join(readme_path, README)
2027
+ logger.info(f"Fetching {README} from {readme_file_path}")
2028
+ if is_path_exists(readme_file_path):
2029
+ try:
2030
+ content = str(read_file(readme_file_path, auth=default_signer()))
2031
+ except Exception as e:
2032
+ logger.debug(
2033
+ f"Error occurred while fetching config {README} at path {readme_file_path} : {str(e)}"
2034
+ )
2035
+ return AquaModelReadme(id=model_id, model_card=content)
2036
+
2037
+ @telemetry(entry_point="plugin=model&action=load_license", name="aqua")
2038
+ def load_license(self, model_id: str) -> AquaModelLicense:
2039
+ """Loads the license full text for the given model.
2040
+
2041
+ Parameters
2042
+ ----------
2043
+ model_id: str
2044
+ The model id.
2045
+
2046
+ Returns
2047
+ -------
2048
+ AquaModelLicense:
2049
+ The instance of AquaModelLicense.
2050
+ """
2051
+ oci_model = self.ds_client.get_model(model_id).data
2052
+ artifact_path = get_artifact_path(oci_model.custom_metadata_list)
2053
+ if not artifact_path:
2054
+ raise AquaRuntimeError(
2055
+ f"License could not be loaded. Failed to get artifact path from custom metadata for"
2056
+ f"the model {model_id}."
2057
+ )
2058
+
2059
+ content = ""
2060
+ try:
2061
+ content = self.ds_client.get_model_defined_metadatum_artifact_content(
2062
+ model_id, AquaModelMetadataKeys.LICENSE
2063
+ ).data.content.decode("utf-8", errors="ignore")
2064
+ logger.info(
2065
+ f"Fetched {LICENSE} from defined metadata for model: {model_id}"
2066
+ )
2067
+ except Exception as ex:
2068
+ logger.error(
2069
+ f"License could not be found for model: {model_id} in defined metadata : {str(ex)}"
2070
+ )
2071
+ artifact_path = get_artifact_path(oci_model.custom_metadata_list)
2072
+ license_path = os.path.join(os.path.dirname(artifact_path), "config")
2073
+ if not is_path_exists(license_path):
2074
+ license_path = os.path.join(artifact_path.rstrip("/"), "config")
2075
+ if not is_path_exists(license_path):
2076
+ license_path = f"{artifact_path.rstrip('/')}/"
2077
+
2078
+ license_file_path = os.path.join(license_path, LICENSE)
2079
+ logger.info(f"Fetching {LICENSE} from {license_file_path}")
2080
+ if is_path_exists(license_file_path):
2081
+ try:
2082
+ content = str(read_file(license_file_path, auth=default_signer()))
2083
+ except Exception as e:
2084
+ logger.debug(
2085
+ f"Error occurred while fetching config {LICENSE} at path {license_path} : {str(e)}"
2086
+ )
2087
+ return AquaModelLicense(id=model_id, license=content)
2088
+
2089
+ def _find_matching_aqua_model(self, model_id: str) -> Optional[str]:
2090
+ """
2091
+ Finds a matching model in AQUA based on the model ID from list of verified models.
2092
+
2093
+ Parameters
2094
+ ----------
2095
+ model_id (str): Verified model ID to match.
2096
+
2097
+ Returns
2098
+ -------
2099
+ Optional[str]
2100
+ Returns model ocid that matches the model in the service catalog else returns None.
2101
+ """
2102
+ # Convert the model ID to lowercase once
2103
+ model_id_lower = model_id.lower()
2104
+
2105
+ aqua_model_list = self.list()
2106
+
2107
+ for aqua_model_summary in aqua_model_list:
2108
+ if aqua_model_summary.name.lower() == model_id_lower:
2109
+ logger.info(
2110
+ f"Found matching verified model id {aqua_model_summary.id} for the model {model_id}"
2111
+ )
2112
+ return aqua_model_summary.id
2113
+
2114
+ return None