oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (858) hide show
  1. ads/aqua/__init__.py +40 -0
  2. ads/aqua/app.py +507 -0
  3. ads/aqua/cli.py +96 -0
  4. ads/aqua/client/__init__.py +3 -0
  5. ads/aqua/client/client.py +836 -0
  6. ads/aqua/client/openai_client.py +305 -0
  7. ads/aqua/common/__init__.py +5 -0
  8. ads/aqua/common/decorator.py +125 -0
  9. ads/aqua/common/entities.py +274 -0
  10. ads/aqua/common/enums.py +134 -0
  11. ads/aqua/common/errors.py +109 -0
  12. ads/aqua/common/utils.py +1295 -0
  13. ads/aqua/config/__init__.py +4 -0
  14. ads/aqua/config/container_config.py +247 -0
  15. ads/aqua/config/evaluation/__init__.py +4 -0
  16. ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
  17. ads/aqua/config/utils/__init__.py +4 -0
  18. ads/aqua/config/utils/serializer.py +339 -0
  19. ads/aqua/constants.py +116 -0
  20. ads/aqua/data.py +14 -0
  21. ads/aqua/dummy_data/icon.txt +1 -0
  22. ads/aqua/dummy_data/oci_model_deployments.json +56 -0
  23. ads/aqua/dummy_data/oci_models.json +1 -0
  24. ads/aqua/dummy_data/readme.md +26 -0
  25. ads/aqua/evaluation/__init__.py +8 -0
  26. ads/aqua/evaluation/constants.py +53 -0
  27. ads/aqua/evaluation/entities.py +186 -0
  28. ads/aqua/evaluation/errors.py +70 -0
  29. ads/aqua/evaluation/evaluation.py +1814 -0
  30. ads/aqua/extension/__init__.py +42 -0
  31. ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
  32. ads/aqua/extension/base_handler.py +90 -0
  33. ads/aqua/extension/common_handler.py +121 -0
  34. ads/aqua/extension/common_ws_msg_handler.py +36 -0
  35. ads/aqua/extension/deployment_handler.py +381 -0
  36. ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
  37. ads/aqua/extension/errors.py +30 -0
  38. ads/aqua/extension/evaluation_handler.py +129 -0
  39. ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
  40. ads/aqua/extension/finetune_handler.py +96 -0
  41. ads/aqua/extension/model_handler.py +390 -0
  42. ads/aqua/extension/models/__init__.py +0 -0
  43. ads/aqua/extension/models/ws_models.py +145 -0
  44. ads/aqua/extension/models_ws_msg_handler.py +50 -0
  45. ads/aqua/extension/ui_handler.py +300 -0
  46. ads/aqua/extension/ui_websocket_handler.py +130 -0
  47. ads/aqua/extension/utils.py +133 -0
  48. ads/aqua/finetuning/__init__.py +7 -0
  49. ads/aqua/finetuning/constants.py +23 -0
  50. ads/aqua/finetuning/entities.py +181 -0
  51. ads/aqua/finetuning/finetuning.py +749 -0
  52. ads/aqua/model/__init__.py +8 -0
  53. ads/aqua/model/constants.py +60 -0
  54. ads/aqua/model/entities.py +385 -0
  55. ads/aqua/model/enums.py +32 -0
  56. ads/aqua/model/model.py +2134 -0
  57. ads/aqua/model/utils.py +52 -0
  58. ads/aqua/modeldeployment/__init__.py +6 -0
  59. ads/aqua/modeldeployment/constants.py +10 -0
  60. ads/aqua/modeldeployment/deployment.py +1315 -0
  61. ads/aqua/modeldeployment/entities.py +653 -0
  62. ads/aqua/modeldeployment/utils.py +543 -0
  63. ads/aqua/resources/gpu_shapes_index.json +94 -0
  64. ads/aqua/server/__init__.py +4 -0
  65. ads/aqua/server/__main__.py +24 -0
  66. ads/aqua/server/app.py +47 -0
  67. ads/aqua/server/aqua_spec.yml +1291 -0
  68. ads/aqua/training/__init__.py +4 -0
  69. ads/aqua/training/exceptions.py +476 -0
  70. ads/aqua/ui.py +519 -0
  71. ads/automl/__init__.py +9 -0
  72. ads/automl/driver.py +330 -0
  73. ads/automl/provider.py +975 -0
  74. ads/bds/__init__.py +5 -0
  75. ads/bds/auth.py +127 -0
  76. ads/bds/big_data_service.py +255 -0
  77. ads/catalog/__init__.py +19 -0
  78. ads/catalog/model.py +1576 -0
  79. ads/catalog/notebook.py +461 -0
  80. ads/catalog/project.py +468 -0
  81. ads/catalog/summary.py +178 -0
  82. ads/common/__init__.py +11 -0
  83. ads/common/analyzer.py +65 -0
  84. ads/common/artifact/.model-ignore +63 -0
  85. ads/common/artifact/__init__.py +10 -0
  86. ads/common/auth.py +1122 -0
  87. ads/common/card_identifier.py +83 -0
  88. ads/common/config.py +647 -0
  89. ads/common/data.py +165 -0
  90. ads/common/decorator/__init__.py +9 -0
  91. ads/common/decorator/argument_to_case.py +88 -0
  92. ads/common/decorator/deprecate.py +69 -0
  93. ads/common/decorator/require_nonempty_arg.py +65 -0
  94. ads/common/decorator/runtime_dependency.py +178 -0
  95. ads/common/decorator/threaded.py +97 -0
  96. ads/common/decorator/utils.py +35 -0
  97. ads/common/dsc_file_system.py +303 -0
  98. ads/common/error.py +14 -0
  99. ads/common/extended_enum.py +81 -0
  100. ads/common/function/__init__.py +5 -0
  101. ads/common/function/fn_util.py +142 -0
  102. ads/common/function/func_conf.yaml +25 -0
  103. ads/common/ipython.py +76 -0
  104. ads/common/model.py +679 -0
  105. ads/common/model_artifact.py +1759 -0
  106. ads/common/model_artifact_schema.json +107 -0
  107. ads/common/model_export_util.py +664 -0
  108. ads/common/model_metadata.py +24 -0
  109. ads/common/object_storage_details.py +296 -0
  110. ads/common/oci_client.py +179 -0
  111. ads/common/oci_datascience.py +46 -0
  112. ads/common/oci_logging.py +1144 -0
  113. ads/common/oci_mixin.py +957 -0
  114. ads/common/oci_resource.py +136 -0
  115. ads/common/serializer.py +559 -0
  116. ads/common/utils.py +1852 -0
  117. ads/common/word_lists.py +1491 -0
  118. ads/common/work_request.py +189 -0
  119. ads/config.py +1 -0
  120. ads/data_labeling/__init__.py +13 -0
  121. ads/data_labeling/boundingbox.py +253 -0
  122. ads/data_labeling/constants.py +47 -0
  123. ads/data_labeling/data_labeling_service.py +244 -0
  124. ads/data_labeling/interface/__init__.py +5 -0
  125. ads/data_labeling/interface/loader.py +16 -0
  126. ads/data_labeling/interface/parser.py +16 -0
  127. ads/data_labeling/interface/reader.py +23 -0
  128. ads/data_labeling/loader/__init__.py +5 -0
  129. ads/data_labeling/loader/file_loader.py +241 -0
  130. ads/data_labeling/metadata.py +110 -0
  131. ads/data_labeling/mixin/__init__.py +5 -0
  132. ads/data_labeling/mixin/data_labeling.py +232 -0
  133. ads/data_labeling/ner.py +129 -0
  134. ads/data_labeling/parser/__init__.py +5 -0
  135. ads/data_labeling/parser/dls_record_parser.py +388 -0
  136. ads/data_labeling/parser/export_metadata_parser.py +94 -0
  137. ads/data_labeling/parser/export_record_parser.py +473 -0
  138. ads/data_labeling/reader/__init__.py +5 -0
  139. ads/data_labeling/reader/dataset_reader.py +574 -0
  140. ads/data_labeling/reader/dls_record_reader.py +121 -0
  141. ads/data_labeling/reader/export_record_reader.py +62 -0
  142. ads/data_labeling/reader/jsonl_reader.py +75 -0
  143. ads/data_labeling/reader/metadata_reader.py +203 -0
  144. ads/data_labeling/reader/record_reader.py +263 -0
  145. ads/data_labeling/record.py +52 -0
  146. ads/data_labeling/visualizer/__init__.py +5 -0
  147. ads/data_labeling/visualizer/image_visualizer.py +525 -0
  148. ads/data_labeling/visualizer/text_visualizer.py +357 -0
  149. ads/database/__init__.py +5 -0
  150. ads/database/connection.py +338 -0
  151. ads/dataset/__init__.py +10 -0
  152. ads/dataset/capabilities.md +51 -0
  153. ads/dataset/classification_dataset.py +339 -0
  154. ads/dataset/correlation.py +226 -0
  155. ads/dataset/correlation_plot.py +563 -0
  156. ads/dataset/dask_series.py +173 -0
  157. ads/dataset/dataframe_transformer.py +110 -0
  158. ads/dataset/dataset.py +1979 -0
  159. ads/dataset/dataset_browser.py +360 -0
  160. ads/dataset/dataset_with_target.py +995 -0
  161. ads/dataset/exception.py +25 -0
  162. ads/dataset/factory.py +987 -0
  163. ads/dataset/feature_engineering_transformer.py +35 -0
  164. ads/dataset/feature_selection.py +107 -0
  165. ads/dataset/forecasting_dataset.py +26 -0
  166. ads/dataset/helper.py +1450 -0
  167. ads/dataset/label_encoder.py +99 -0
  168. ads/dataset/mixin/__init__.py +5 -0
  169. ads/dataset/mixin/dataset_accessor.py +134 -0
  170. ads/dataset/pipeline.py +58 -0
  171. ads/dataset/plot.py +710 -0
  172. ads/dataset/progress.py +86 -0
  173. ads/dataset/recommendation.py +297 -0
  174. ads/dataset/recommendation_transformer.py +502 -0
  175. ads/dataset/regression_dataset.py +14 -0
  176. ads/dataset/sampled_dataset.py +1050 -0
  177. ads/dataset/target.py +98 -0
  178. ads/dataset/timeseries.py +18 -0
  179. ads/dbmixin/__init__.py +5 -0
  180. ads/dbmixin/db_pandas_accessor.py +153 -0
  181. ads/environment/__init__.py +9 -0
  182. ads/environment/ml_runtime.py +66 -0
  183. ads/evaluations/README.md +14 -0
  184. ads/evaluations/__init__.py +109 -0
  185. ads/evaluations/evaluation_plot.py +983 -0
  186. ads/evaluations/evaluator.py +1334 -0
  187. ads/evaluations/statistical_metrics.py +543 -0
  188. ads/experiments/__init__.py +9 -0
  189. ads/experiments/capabilities.md +0 -0
  190. ads/explanations/__init__.py +21 -0
  191. ads/explanations/base_explainer.py +142 -0
  192. ads/explanations/capabilities.md +83 -0
  193. ads/explanations/explainer.py +190 -0
  194. ads/explanations/mlx_global_explainer.py +1050 -0
  195. ads/explanations/mlx_interface.py +386 -0
  196. ads/explanations/mlx_local_explainer.py +287 -0
  197. ads/explanations/mlx_whatif_explainer.py +201 -0
  198. ads/feature_engineering/__init__.py +20 -0
  199. ads/feature_engineering/accessor/__init__.py +5 -0
  200. ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
  201. ads/feature_engineering/accessor/mixin/__init__.py +5 -0
  202. ads/feature_engineering/accessor/mixin/correlation.py +166 -0
  203. ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
  204. ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
  205. ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
  206. ads/feature_engineering/accessor/mixin/utils.py +65 -0
  207. ads/feature_engineering/accessor/series_accessor.py +431 -0
  208. ads/feature_engineering/adsimage/__init__.py +5 -0
  209. ads/feature_engineering/adsimage/image.py +192 -0
  210. ads/feature_engineering/adsimage/image_reader.py +170 -0
  211. ads/feature_engineering/adsimage/interface/__init__.py +5 -0
  212. ads/feature_engineering/adsimage/interface/reader.py +19 -0
  213. ads/feature_engineering/adsstring/__init__.py +7 -0
  214. ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
  215. ads/feature_engineering/adsstring/string/__init__.py +8 -0
  216. ads/feature_engineering/data_schema.json +57 -0
  217. ads/feature_engineering/dataset/__init__.py +5 -0
  218. ads/feature_engineering/dataset/zip_code_data.py +42062 -0
  219. ads/feature_engineering/exceptions.py +40 -0
  220. ads/feature_engineering/feature_type/__init__.py +133 -0
  221. ads/feature_engineering/feature_type/address.py +184 -0
  222. ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
  223. ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
  224. ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
  225. ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
  226. ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
  227. ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
  228. ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
  229. ads/feature_engineering/feature_type/adsstring/string.py +258 -0
  230. ads/feature_engineering/feature_type/base.py +58 -0
  231. ads/feature_engineering/feature_type/boolean.py +183 -0
  232. ads/feature_engineering/feature_type/category.py +146 -0
  233. ads/feature_engineering/feature_type/constant.py +137 -0
  234. ads/feature_engineering/feature_type/continuous.py +151 -0
  235. ads/feature_engineering/feature_type/creditcard.py +314 -0
  236. ads/feature_engineering/feature_type/datetime.py +190 -0
  237. ads/feature_engineering/feature_type/discrete.py +134 -0
  238. ads/feature_engineering/feature_type/document.py +43 -0
  239. ads/feature_engineering/feature_type/gis.py +251 -0
  240. ads/feature_engineering/feature_type/handler/__init__.py +5 -0
  241. ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
  242. ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
  243. ads/feature_engineering/feature_type/handler/warnings.py +128 -0
  244. ads/feature_engineering/feature_type/integer.py +142 -0
  245. ads/feature_engineering/feature_type/ip_address.py +144 -0
  246. ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
  247. ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
  248. ads/feature_engineering/feature_type/lat_long.py +256 -0
  249. ads/feature_engineering/feature_type/object.py +43 -0
  250. ads/feature_engineering/feature_type/ordinal.py +132 -0
  251. ads/feature_engineering/feature_type/phone_number.py +135 -0
  252. ads/feature_engineering/feature_type/string.py +171 -0
  253. ads/feature_engineering/feature_type/text.py +93 -0
  254. ads/feature_engineering/feature_type/unknown.py +43 -0
  255. ads/feature_engineering/feature_type/zip_code.py +164 -0
  256. ads/feature_engineering/feature_type_manager.py +406 -0
  257. ads/feature_engineering/schema.py +795 -0
  258. ads/feature_engineering/utils.py +245 -0
  259. ads/feature_store/.readthedocs.yaml +19 -0
  260. ads/feature_store/README.md +65 -0
  261. ads/feature_store/__init__.py +9 -0
  262. ads/feature_store/common/__init__.py +0 -0
  263. ads/feature_store/common/enums.py +339 -0
  264. ads/feature_store/common/exceptions.py +18 -0
  265. ads/feature_store/common/spark_session_singleton.py +125 -0
  266. ads/feature_store/common/utils/__init__.py +0 -0
  267. ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
  268. ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
  269. ads/feature_store/common/utils/transformation_utils.py +82 -0
  270. ads/feature_store/common/utils/utility.py +403 -0
  271. ads/feature_store/data_validation/__init__.py +0 -0
  272. ads/feature_store/data_validation/great_expectation.py +129 -0
  273. ads/feature_store/dataset.py +1230 -0
  274. ads/feature_store/dataset_job.py +530 -0
  275. ads/feature_store/docs/Dockerfile +7 -0
  276. ads/feature_store/docs/Makefile +44 -0
  277. ads/feature_store/docs/conf.py +28 -0
  278. ads/feature_store/docs/requirements.txt +14 -0
  279. ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
  280. ads/feature_store/docs/source/cicd.rst +137 -0
  281. ads/feature_store/docs/source/conf.py +86 -0
  282. ads/feature_store/docs/source/data_versioning.rst +33 -0
  283. ads/feature_store/docs/source/dataset.rst +388 -0
  284. ads/feature_store/docs/source/dataset_job.rst +27 -0
  285. ads/feature_store/docs/source/demo.rst +70 -0
  286. ads/feature_store/docs/source/entity.rst +78 -0
  287. ads/feature_store/docs/source/feature_group.rst +624 -0
  288. ads/feature_store/docs/source/feature_group_job.rst +29 -0
  289. ads/feature_store/docs/source/feature_store.rst +122 -0
  290. ads/feature_store/docs/source/feature_store_class.rst +123 -0
  291. ads/feature_store/docs/source/feature_validation.rst +66 -0
  292. ads/feature_store/docs/source/figures/cicd.png +0 -0
  293. ads/feature_store/docs/source/figures/data_validation.png +0 -0
  294. ads/feature_store/docs/source/figures/data_versioning.png +0 -0
  295. ads/feature_store/docs/source/figures/dataset.gif +0 -0
  296. ads/feature_store/docs/source/figures/dataset.png +0 -0
  297. ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
  298. ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
  299. ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
  300. ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
  301. ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
  302. ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
  303. ads/feature_store/docs/source/figures/entity.png +0 -0
  304. ads/feature_store/docs/source/figures/feature_group.png +0 -0
  305. ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
  306. ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
  307. ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
  308. ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
  309. ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
  310. ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
  311. ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
  312. ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
  313. ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
  314. ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
  315. ads/feature_store/docs/source/figures/overview.png +0 -0
  316. ads/feature_store/docs/source/figures/resource_manager.png +0 -0
  317. ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
  318. ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
  319. ads/feature_store/docs/source/figures/stats_1.png +0 -0
  320. ads/feature_store/docs/source/figures/stats_2.png +0 -0
  321. ads/feature_store/docs/source/figures/stats_d.png +0 -0
  322. ads/feature_store/docs/source/figures/stats_fg.png +0 -0
  323. ads/feature_store/docs/source/figures/transformation.png +0 -0
  324. ads/feature_store/docs/source/figures/transformations.gif +0 -0
  325. ads/feature_store/docs/source/figures/validation.png +0 -0
  326. ads/feature_store/docs/source/figures/validation_fg.png +0 -0
  327. ads/feature_store/docs/source/figures/validation_results.png +0 -0
  328. ads/feature_store/docs/source/figures/validation_summary.png +0 -0
  329. ads/feature_store/docs/source/index.rst +81 -0
  330. ads/feature_store/docs/source/module.rst +8 -0
  331. ads/feature_store/docs/source/notebook.rst +94 -0
  332. ads/feature_store/docs/source/overview.rst +47 -0
  333. ads/feature_store/docs/source/quickstart.rst +176 -0
  334. ads/feature_store/docs/source/release_notes.rst +194 -0
  335. ads/feature_store/docs/source/setup_feature_store.rst +81 -0
  336. ads/feature_store/docs/source/statistics.rst +58 -0
  337. ads/feature_store/docs/source/transformation.rst +199 -0
  338. ads/feature_store/docs/source/ui.rst +65 -0
  339. ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
  340. ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
  341. ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
  342. ads/feature_store/entity.py +718 -0
  343. ads/feature_store/execution_strategy/__init__.py +0 -0
  344. ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
  345. ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
  346. ads/feature_store/execution_strategy/engine/__init__.py +0 -0
  347. ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
  348. ads/feature_store/execution_strategy/execution_strategy.py +113 -0
  349. ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
  350. ads/feature_store/execution_strategy/spark/__init__.py +0 -0
  351. ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
  352. ads/feature_store/feature.py +192 -0
  353. ads/feature_store/feature_group.py +1494 -0
  354. ads/feature_store/feature_group_expectation.py +346 -0
  355. ads/feature_store/feature_group_job.py +602 -0
  356. ads/feature_store/feature_lineage/__init__.py +0 -0
  357. ads/feature_store/feature_lineage/graphviz_service.py +180 -0
  358. ads/feature_store/feature_option_details.py +50 -0
  359. ads/feature_store/feature_statistics/__init__.py +0 -0
  360. ads/feature_store/feature_statistics/statistics_service.py +99 -0
  361. ads/feature_store/feature_store.py +699 -0
  362. ads/feature_store/feature_store_registrar.py +518 -0
  363. ads/feature_store/input_feature_detail.py +149 -0
  364. ads/feature_store/mixin/__init__.py +4 -0
  365. ads/feature_store/mixin/oci_feature_store.py +145 -0
  366. ads/feature_store/model_details.py +73 -0
  367. ads/feature_store/query/__init__.py +0 -0
  368. ads/feature_store/query/filter.py +266 -0
  369. ads/feature_store/query/generator/__init__.py +0 -0
  370. ads/feature_store/query/generator/query_generator.py +298 -0
  371. ads/feature_store/query/join.py +161 -0
  372. ads/feature_store/query/query.py +403 -0
  373. ads/feature_store/query/validator/__init__.py +0 -0
  374. ads/feature_store/query/validator/query_validator.py +57 -0
  375. ads/feature_store/response/__init__.py +0 -0
  376. ads/feature_store/response/response_builder.py +68 -0
  377. ads/feature_store/service/__init__.py +0 -0
  378. ads/feature_store/service/oci_dataset.py +139 -0
  379. ads/feature_store/service/oci_dataset_job.py +199 -0
  380. ads/feature_store/service/oci_entity.py +125 -0
  381. ads/feature_store/service/oci_feature_group.py +164 -0
  382. ads/feature_store/service/oci_feature_group_job.py +214 -0
  383. ads/feature_store/service/oci_feature_store.py +182 -0
  384. ads/feature_store/service/oci_lineage.py +87 -0
  385. ads/feature_store/service/oci_transformation.py +104 -0
  386. ads/feature_store/statistics/__init__.py +0 -0
  387. ads/feature_store/statistics/abs_feature_value.py +49 -0
  388. ads/feature_store/statistics/charts/__init__.py +0 -0
  389. ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
  390. ads/feature_store/statistics/charts/box_plot.py +148 -0
  391. ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
  392. ads/feature_store/statistics/charts/probability_distribution.py +68 -0
  393. ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
  394. ads/feature_store/statistics/feature_stat.py +126 -0
  395. ads/feature_store/statistics/generic_feature_value.py +33 -0
  396. ads/feature_store/statistics/statistics.py +41 -0
  397. ads/feature_store/statistics_config.py +101 -0
  398. ads/feature_store/templates/feature_store_template.yaml +45 -0
  399. ads/feature_store/transformation.py +499 -0
  400. ads/feature_store/validation_output.py +57 -0
  401. ads/hpo/__init__.py +9 -0
  402. ads/hpo/_imports.py +91 -0
  403. ads/hpo/ads_search_space.py +439 -0
  404. ads/hpo/distributions.py +325 -0
  405. ads/hpo/objective.py +280 -0
  406. ads/hpo/search_cv.py +1657 -0
  407. ads/hpo/stopping_criterion.py +75 -0
  408. ads/hpo/tuner_artifact.py +413 -0
  409. ads/hpo/utils.py +91 -0
  410. ads/hpo/validation.py +140 -0
  411. ads/hpo/visualization/__init__.py +5 -0
  412. ads/hpo/visualization/_contour.py +23 -0
  413. ads/hpo/visualization/_edf.py +20 -0
  414. ads/hpo/visualization/_intermediate_values.py +21 -0
  415. ads/hpo/visualization/_optimization_history.py +25 -0
  416. ads/hpo/visualization/_parallel_coordinate.py +169 -0
  417. ads/hpo/visualization/_param_importances.py +26 -0
  418. ads/jobs/__init__.py +53 -0
  419. ads/jobs/ads_job.py +663 -0
  420. ads/jobs/builders/__init__.py +5 -0
  421. ads/jobs/builders/base.py +156 -0
  422. ads/jobs/builders/infrastructure/__init__.py +6 -0
  423. ads/jobs/builders/infrastructure/base.py +165 -0
  424. ads/jobs/builders/infrastructure/dataflow.py +1252 -0
  425. ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
  426. ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
  427. ads/jobs/builders/infrastructure/utils.py +65 -0
  428. ads/jobs/builders/runtimes/__init__.py +5 -0
  429. ads/jobs/builders/runtimes/artifact.py +338 -0
  430. ads/jobs/builders/runtimes/base.py +325 -0
  431. ads/jobs/builders/runtimes/container_runtime.py +242 -0
  432. ads/jobs/builders/runtimes/python_runtime.py +1016 -0
  433. ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
  434. ads/jobs/cli.py +104 -0
  435. ads/jobs/env_var_parser.py +131 -0
  436. ads/jobs/extension.py +160 -0
  437. ads/jobs/schema/__init__.py +5 -0
  438. ads/jobs/schema/infrastructure_schema.json +116 -0
  439. ads/jobs/schema/job_schema.json +42 -0
  440. ads/jobs/schema/runtime_schema.json +183 -0
  441. ads/jobs/schema/validator.py +141 -0
  442. ads/jobs/serializer.py +296 -0
  443. ads/jobs/templates/__init__.py +5 -0
  444. ads/jobs/templates/container.py +6 -0
  445. ads/jobs/templates/driver_notebook.py +177 -0
  446. ads/jobs/templates/driver_oci.py +500 -0
  447. ads/jobs/templates/driver_python.py +48 -0
  448. ads/jobs/templates/driver_pytorch.py +852 -0
  449. ads/jobs/templates/driver_utils.py +615 -0
  450. ads/jobs/templates/hostname_from_env.c +55 -0
  451. ads/jobs/templates/oci_metrics.py +181 -0
  452. ads/jobs/utils.py +104 -0
  453. ads/llm/__init__.py +28 -0
  454. ads/llm/autogen/__init__.py +2 -0
  455. ads/llm/autogen/constants.py +15 -0
  456. ads/llm/autogen/reports/__init__.py +2 -0
  457. ads/llm/autogen/reports/base.py +67 -0
  458. ads/llm/autogen/reports/data.py +103 -0
  459. ads/llm/autogen/reports/session.py +526 -0
  460. ads/llm/autogen/reports/templates/chat_box.html +13 -0
  461. ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
  462. ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
  463. ads/llm/autogen/reports/utils.py +56 -0
  464. ads/llm/autogen/v02/__init__.py +4 -0
  465. ads/llm/autogen/v02/client.py +295 -0
  466. ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
  467. ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
  468. ads/llm/autogen/v02/loggers/__init__.py +6 -0
  469. ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
  470. ads/llm/autogen/v02/loggers/session_logger.py +580 -0
  471. ads/llm/autogen/v02/loggers/utils.py +86 -0
  472. ads/llm/autogen/v02/runtime_logging.py +163 -0
  473. ads/llm/chain.py +268 -0
  474. ads/llm/chat_template.py +31 -0
  475. ads/llm/deploy.py +63 -0
  476. ads/llm/guardrails/__init__.py +5 -0
  477. ads/llm/guardrails/base.py +442 -0
  478. ads/llm/guardrails/huggingface.py +44 -0
  479. ads/llm/langchain/__init__.py +5 -0
  480. ads/llm/langchain/plugins/__init__.py +5 -0
  481. ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
  482. ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
  483. ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
  484. ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
  485. ads/llm/langchain/plugins/llms/__init__.py +5 -0
  486. ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
  487. ads/llm/requirements.txt +3 -0
  488. ads/llm/serialize.py +219 -0
  489. ads/llm/serializers/__init__.py +0 -0
  490. ads/llm/serializers/retrieval_qa.py +153 -0
  491. ads/llm/serializers/runnable_parallel.py +27 -0
  492. ads/llm/templates/score_chain.jinja2 +155 -0
  493. ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
  494. ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
  495. ads/model/__init__.py +52 -0
  496. ads/model/artifact.py +573 -0
  497. ads/model/artifact_downloader.py +254 -0
  498. ads/model/artifact_uploader.py +267 -0
  499. ads/model/base_properties.py +238 -0
  500. ads/model/common/.model-ignore +66 -0
  501. ads/model/common/__init__.py +5 -0
  502. ads/model/common/utils.py +142 -0
  503. ads/model/datascience_model.py +2635 -0
  504. ads/model/deployment/__init__.py +20 -0
  505. ads/model/deployment/common/__init__.py +5 -0
  506. ads/model/deployment/common/utils.py +308 -0
  507. ads/model/deployment/model_deployer.py +466 -0
  508. ads/model/deployment/model_deployment.py +1846 -0
  509. ads/model/deployment/model_deployment_infrastructure.py +671 -0
  510. ads/model/deployment/model_deployment_properties.py +493 -0
  511. ads/model/deployment/model_deployment_runtime.py +838 -0
  512. ads/model/extractor/__init__.py +5 -0
  513. ads/model/extractor/automl_extractor.py +74 -0
  514. ads/model/extractor/embedding_onnx_extractor.py +80 -0
  515. ads/model/extractor/huggingface_extractor.py +88 -0
  516. ads/model/extractor/keras_extractor.py +84 -0
  517. ads/model/extractor/lightgbm_extractor.py +93 -0
  518. ads/model/extractor/model_info_extractor.py +114 -0
  519. ads/model/extractor/model_info_extractor_factory.py +105 -0
  520. ads/model/extractor/pytorch_extractor.py +87 -0
  521. ads/model/extractor/sklearn_extractor.py +112 -0
  522. ads/model/extractor/spark_extractor.py +89 -0
  523. ads/model/extractor/tensorflow_extractor.py +85 -0
  524. ads/model/extractor/xgboost_extractor.py +94 -0
  525. ads/model/framework/__init__.py +5 -0
  526. ads/model/framework/automl_model.py +178 -0
  527. ads/model/framework/embedding_onnx_model.py +438 -0
  528. ads/model/framework/huggingface_model.py +399 -0
  529. ads/model/framework/lightgbm_model.py +266 -0
  530. ads/model/framework/pytorch_model.py +266 -0
  531. ads/model/framework/sklearn_model.py +250 -0
  532. ads/model/framework/spark_model.py +326 -0
  533. ads/model/framework/tensorflow_model.py +254 -0
  534. ads/model/framework/xgboost_model.py +258 -0
  535. ads/model/generic_model.py +3518 -0
  536. ads/model/model_artifact_boilerplate/README.md +381 -0
  537. ads/model/model_artifact_boilerplate/__init__.py +5 -0
  538. ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
  539. ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
  540. ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
  541. ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
  542. ads/model/model_artifact_boilerplate/score.py +61 -0
  543. ads/model/model_file_description_schema.json +68 -0
  544. ads/model/model_introspect.py +331 -0
  545. ads/model/model_metadata.py +1810 -0
  546. ads/model/model_metadata_mixin.py +460 -0
  547. ads/model/model_properties.py +63 -0
  548. ads/model/model_version_set.py +739 -0
  549. ads/model/runtime/__init__.py +5 -0
  550. ads/model/runtime/env_info.py +306 -0
  551. ads/model/runtime/model_deployment_details.py +37 -0
  552. ads/model/runtime/model_provenance_details.py +58 -0
  553. ads/model/runtime/runtime_info.py +81 -0
  554. ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
  555. ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
  556. ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
  557. ads/model/runtime/utils.py +201 -0
  558. ads/model/serde/__init__.py +5 -0
  559. ads/model/serde/common.py +40 -0
  560. ads/model/serde/model_input.py +547 -0
  561. ads/model/serde/model_serializer.py +1184 -0
  562. ads/model/service/__init__.py +5 -0
  563. ads/model/service/oci_datascience_model.py +1076 -0
  564. ads/model/service/oci_datascience_model_deployment.py +500 -0
  565. ads/model/service/oci_datascience_model_version_set.py +176 -0
  566. ads/model/transformer/__init__.py +5 -0
  567. ads/model/transformer/onnx_transformer.py +324 -0
  568. ads/mysqldb/__init__.py +5 -0
  569. ads/mysqldb/mysql_db.py +227 -0
  570. ads/opctl/__init__.py +18 -0
  571. ads/opctl/anomaly_detection.py +11 -0
  572. ads/opctl/backend/__init__.py +5 -0
  573. ads/opctl/backend/ads_dataflow.py +353 -0
  574. ads/opctl/backend/ads_ml_job.py +710 -0
  575. ads/opctl/backend/ads_ml_pipeline.py +164 -0
  576. ads/opctl/backend/ads_model_deployment.py +209 -0
  577. ads/opctl/backend/base.py +146 -0
  578. ads/opctl/backend/local.py +1053 -0
  579. ads/opctl/backend/marketplace/__init__.py +9 -0
  580. ads/opctl/backend/marketplace/helm_helper.py +173 -0
  581. ads/opctl/backend/marketplace/local_marketplace.py +271 -0
  582. ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
  583. ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
  584. ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
  585. ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
  586. ads/opctl/backend/marketplace/models/__init__.py +5 -0
  587. ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
  588. ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
  589. ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
  590. ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
  591. ads/opctl/cli.py +707 -0
  592. ads/opctl/cmds.py +869 -0
  593. ads/opctl/conda/__init__.py +5 -0
  594. ads/opctl/conda/cli.py +193 -0
  595. ads/opctl/conda/cmds.py +749 -0
  596. ads/opctl/conda/config.yaml +34 -0
  597. ads/opctl/conda/manifest_template.yaml +13 -0
  598. ads/opctl/conda/multipart_uploader.py +188 -0
  599. ads/opctl/conda/pack.py +89 -0
  600. ads/opctl/config/__init__.py +5 -0
  601. ads/opctl/config/base.py +57 -0
  602. ads/opctl/config/diagnostics/__init__.py +5 -0
  603. ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
  604. ads/opctl/config/merger.py +255 -0
  605. ads/opctl/config/resolver.py +297 -0
  606. ads/opctl/config/utils.py +79 -0
  607. ads/opctl/config/validator.py +17 -0
  608. ads/opctl/config/versioner.py +68 -0
  609. ads/opctl/config/yaml_parsers/__init__.py +7 -0
  610. ads/opctl/config/yaml_parsers/base.py +58 -0
  611. ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
  612. ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
  613. ads/opctl/constants.py +66 -0
  614. ads/opctl/decorator/__init__.py +5 -0
  615. ads/opctl/decorator/common.py +129 -0
  616. ads/opctl/diagnostics/__init__.py +5 -0
  617. ads/opctl/diagnostics/__main__.py +25 -0
  618. ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
  619. ads/opctl/diagnostics/check_requirements.py +144 -0
  620. ads/opctl/diagnostics/requirement_exception.py +9 -0
  621. ads/opctl/distributed/README.md +109 -0
  622. ads/opctl/distributed/__init__.py +5 -0
  623. ads/opctl/distributed/certificates.py +32 -0
  624. ads/opctl/distributed/cli.py +207 -0
  625. ads/opctl/distributed/cmds.py +731 -0
  626. ads/opctl/distributed/common/__init__.py +5 -0
  627. ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
  628. ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
  629. ads/opctl/distributed/common/cluster_config_helper.py +103 -0
  630. ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
  631. ads/opctl/distributed/common/cluster_runner.py +54 -0
  632. ads/opctl/distributed/common/framework_factory.py +29 -0
  633. ads/opctl/docker/Dockerfile.job +103 -0
  634. ads/opctl/docker/Dockerfile.job.arm +107 -0
  635. ads/opctl/docker/Dockerfile.job.gpu +175 -0
  636. ads/opctl/docker/base-env.yaml +13 -0
  637. ads/opctl/docker/cuda.repo +6 -0
  638. ads/opctl/docker/operator/.dockerignore +0 -0
  639. ads/opctl/docker/operator/Dockerfile +41 -0
  640. ads/opctl/docker/operator/Dockerfile.gpu +85 -0
  641. ads/opctl/docker/operator/cuda.repo +6 -0
  642. ads/opctl/docker/operator/environment.yaml +8 -0
  643. ads/opctl/forecast.py +11 -0
  644. ads/opctl/index.yaml +3 -0
  645. ads/opctl/model/__init__.py +5 -0
  646. ads/opctl/model/cli.py +65 -0
  647. ads/opctl/model/cmds.py +73 -0
  648. ads/opctl/operator/README.md +4 -0
  649. ads/opctl/operator/__init__.py +31 -0
  650. ads/opctl/operator/cli.py +344 -0
  651. ads/opctl/operator/cmd.py +596 -0
  652. ads/opctl/operator/common/__init__.py +5 -0
  653. ads/opctl/operator/common/backend_factory.py +460 -0
  654. ads/opctl/operator/common/const.py +27 -0
  655. ads/opctl/operator/common/data/synthetic.csv +16001 -0
  656. ads/opctl/operator/common/dictionary_merger.py +148 -0
  657. ads/opctl/operator/common/errors.py +42 -0
  658. ads/opctl/operator/common/operator_config.py +99 -0
  659. ads/opctl/operator/common/operator_loader.py +811 -0
  660. ads/opctl/operator/common/operator_schema.yaml +130 -0
  661. ads/opctl/operator/common/operator_yaml_generator.py +152 -0
  662. ads/opctl/operator/common/utils.py +208 -0
  663. ads/opctl/operator/lowcode/__init__.py +5 -0
  664. ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
  665. ads/opctl/operator/lowcode/anomaly/README.md +207 -0
  666. ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
  667. ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
  668. ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
  669. ads/opctl/operator/lowcode/anomaly/const.py +167 -0
  670. ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
  671. ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
  672. ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
  673. ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
  674. ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
  675. ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
  676. ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
  677. ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
  678. ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
  679. ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
  680. ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
  681. ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
  682. ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
  683. ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
  684. ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
  685. ads/opctl/operator/lowcode/common/__init__.py +5 -0
  686. ads/opctl/operator/lowcode/common/const.py +10 -0
  687. ads/opctl/operator/lowcode/common/data.py +116 -0
  688. ads/opctl/operator/lowcode/common/errors.py +47 -0
  689. ads/opctl/operator/lowcode/common/transformations.py +296 -0
  690. ads/opctl/operator/lowcode/common/utils.py +384 -0
  691. ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
  692. ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
  693. ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
  694. ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
  695. ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
  696. ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
  697. ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
  698. ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
  699. ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
  700. ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
  701. ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
  702. ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
  703. ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
  704. ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
  705. ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
  706. ads/opctl/operator/lowcode/forecast/README.md +209 -0
  707. ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
  708. ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
  709. ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
  710. ads/opctl/operator/lowcode/forecast/const.py +92 -0
  711. ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
  712. ads/opctl/operator/lowcode/forecast/errors.py +26 -0
  713. ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
  714. ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
  715. ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
  716. ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
  717. ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
  718. ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
  719. ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
  720. ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
  721. ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
  722. ads/opctl/operator/lowcode/forecast/model/prophet.py +450 -0
  723. ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
  724. ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
  725. ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
  726. ads/opctl/operator/lowcode/forecast/utils.py +397 -0
  727. ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
  728. ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
  729. ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
  730. ads/opctl/operator/lowcode/pii/MLoperator +17 -0
  731. ads/opctl/operator/lowcode/pii/README.md +208 -0
  732. ads/opctl/operator/lowcode/pii/__init__.py +5 -0
  733. ads/opctl/operator/lowcode/pii/__main__.py +78 -0
  734. ads/opctl/operator/lowcode/pii/cmd.py +39 -0
  735. ads/opctl/operator/lowcode/pii/constant.py +84 -0
  736. ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
  737. ads/opctl/operator/lowcode/pii/errors.py +27 -0
  738. ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
  739. ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
  740. ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
  741. ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
  742. ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
  743. ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
  744. ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
  745. ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
  746. ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
  747. ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
  748. ads/opctl/operator/lowcode/pii/model/report.py +487 -0
  749. ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
  750. ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
  751. ads/opctl/operator/lowcode/pii/utils.py +43 -0
  752. ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
  753. ads/opctl/operator/lowcode/recommender/README.md +206 -0
  754. ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
  755. ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
  756. ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
  757. ads/opctl/operator/lowcode/recommender/constant.py +30 -0
  758. ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
  759. ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
  760. ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
  761. ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
  762. ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
  763. ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
  764. ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
  765. ads/opctl/operator/lowcode/recommender/utils.py +13 -0
  766. ads/opctl/operator/runtime/__init__.py +5 -0
  767. ads/opctl/operator/runtime/const.py +17 -0
  768. ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
  769. ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
  770. ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
  771. ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
  772. ads/opctl/operator/runtime/runtime.py +115 -0
  773. ads/opctl/schema.yaml.yml +36 -0
  774. ads/opctl/script.py +40 -0
  775. ads/opctl/spark/__init__.py +5 -0
  776. ads/opctl/spark/cli.py +43 -0
  777. ads/opctl/spark/cmds.py +147 -0
  778. ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
  779. ads/opctl/utils.py +344 -0
  780. ads/oracledb/__init__.py +5 -0
  781. ads/oracledb/oracle_db.py +346 -0
  782. ads/pipeline/__init__.py +39 -0
  783. ads/pipeline/ads_pipeline.py +2279 -0
  784. ads/pipeline/ads_pipeline_run.py +772 -0
  785. ads/pipeline/ads_pipeline_step.py +605 -0
  786. ads/pipeline/builders/__init__.py +5 -0
  787. ads/pipeline/builders/infrastructure/__init__.py +5 -0
  788. ads/pipeline/builders/infrastructure/custom_script.py +32 -0
  789. ads/pipeline/cli.py +119 -0
  790. ads/pipeline/extension.py +291 -0
  791. ads/pipeline/schema/__init__.py +5 -0
  792. ads/pipeline/schema/cs_step_schema.json +35 -0
  793. ads/pipeline/schema/ml_step_schema.json +31 -0
  794. ads/pipeline/schema/pipeline_schema.json +71 -0
  795. ads/pipeline/visualizer/__init__.py +5 -0
  796. ads/pipeline/visualizer/base.py +570 -0
  797. ads/pipeline/visualizer/graph_renderer.py +272 -0
  798. ads/pipeline/visualizer/text_renderer.py +84 -0
  799. ads/secrets/__init__.py +11 -0
  800. ads/secrets/adb.py +386 -0
  801. ads/secrets/auth_token.py +86 -0
  802. ads/secrets/big_data_service.py +365 -0
  803. ads/secrets/mysqldb.py +149 -0
  804. ads/secrets/oracledb.py +160 -0
  805. ads/secrets/secrets.py +407 -0
  806. ads/telemetry/__init__.py +7 -0
  807. ads/telemetry/base.py +69 -0
  808. ads/telemetry/client.py +122 -0
  809. ads/telemetry/telemetry.py +257 -0
  810. ads/templates/dataflow_pyspark.jinja2 +13 -0
  811. ads/templates/dataflow_sparksql.jinja2 +22 -0
  812. ads/templates/func.jinja2 +20 -0
  813. ads/templates/schemas/openapi.json +1740 -0
  814. ads/templates/score-pkl.jinja2 +173 -0
  815. ads/templates/score.jinja2 +322 -0
  816. ads/templates/score_embedding_onnx.jinja2 +202 -0
  817. ads/templates/score_generic.jinja2 +165 -0
  818. ads/templates/score_huggingface_pipeline.jinja2 +217 -0
  819. ads/templates/score_lightgbm.jinja2 +185 -0
  820. ads/templates/score_onnx.jinja2 +407 -0
  821. ads/templates/score_onnx_new.jinja2 +473 -0
  822. ads/templates/score_oracle_automl.jinja2 +185 -0
  823. ads/templates/score_pyspark.jinja2 +154 -0
  824. ads/templates/score_pytorch.jinja2 +219 -0
  825. ads/templates/score_scikit-learn.jinja2 +184 -0
  826. ads/templates/score_tensorflow.jinja2 +184 -0
  827. ads/templates/score_xgboost.jinja2 +178 -0
  828. ads/text_dataset/__init__.py +5 -0
  829. ads/text_dataset/backends.py +211 -0
  830. ads/text_dataset/dataset.py +445 -0
  831. ads/text_dataset/extractor.py +207 -0
  832. ads/text_dataset/options.py +53 -0
  833. ads/text_dataset/udfs.py +22 -0
  834. ads/text_dataset/utils.py +49 -0
  835. ads/type_discovery/__init__.py +9 -0
  836. ads/type_discovery/abstract_detector.py +21 -0
  837. ads/type_discovery/constant_detector.py +41 -0
  838. ads/type_discovery/continuous_detector.py +54 -0
  839. ads/type_discovery/credit_card_detector.py +99 -0
  840. ads/type_discovery/datetime_detector.py +92 -0
  841. ads/type_discovery/discrete_detector.py +118 -0
  842. ads/type_discovery/document_detector.py +146 -0
  843. ads/type_discovery/ip_detector.py +68 -0
  844. ads/type_discovery/latlon_detector.py +90 -0
  845. ads/type_discovery/phone_number_detector.py +63 -0
  846. ads/type_discovery/type_discovery_driver.py +87 -0
  847. ads/type_discovery/typed_feature.py +594 -0
  848. ads/type_discovery/unknown_detector.py +41 -0
  849. ads/type_discovery/zipcode_detector.py +48 -0
  850. ads/vault/__init__.py +7 -0
  851. ads/vault/vault.py +237 -0
  852. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/METADATA +150 -149
  853. oracle_ads-2.13.10rc0.dist-info/RECORD +858 -0
  854. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/WHEEL +1 -2
  855. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/entry_points.txt +2 -1
  856. oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
  857. oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
  858. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,2635 @@
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) 2022, 2025 Oracle and/or its affiliates.
4
+ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
5
+
6
+ import json
7
+ import logging
8
+ import os
9
+ import shutil
10
+ import tempfile
11
+ from copy import deepcopy
12
+ from typing import Dict, List, Optional, Tuple, Union
13
+
14
+ import pandas
15
+ import yaml
16
+ from jsonschema import ValidationError, validate
17
+
18
+ from ads.common import oci_client as oc
19
+ from ads.common import utils
20
+ from ads.common.extended_enum import ExtendedEnum
21
+ from ads.common.object_storage_details import ObjectStorageDetails
22
+ from ads.common.utils import is_path_exists
23
+ from ads.config import (
24
+ AQUA_SERVICE_MODELS_BUCKET as SERVICE_MODELS_BUCKET,
25
+ )
26
+ from ads.config import (
27
+ COMPARTMENT_OCID,
28
+ PROJECT_OCID,
29
+ USER,
30
+ )
31
+ from ads.feature_engineering.schema import Schema
32
+ from ads.jobs.builders.base import Builder
33
+ from ads.model.artifact_downloader import (
34
+ LargeArtifactDownloader,
35
+ SmallArtifactDownloader,
36
+ )
37
+ from ads.model.artifact_uploader import LargeArtifactUploader, SmallArtifactUploader
38
+ from ads.model.common.utils import MetadataArtifactPathType
39
+ from ads.model.model_metadata import (
40
+ MetadataCustomCategory,
41
+ ModelCustomMetadata,
42
+ ModelCustomMetadataItem,
43
+ ModelProvenanceMetadata,
44
+ ModelTaxonomyMetadata,
45
+ )
46
+ from ads.model.service.oci_datascience_model import (
47
+ ModelMetadataArtifactDetails,
48
+ ModelProvenanceNotFoundError,
49
+ OCIDataScienceModel,
50
+ )
51
+
52
+ logger = logging.getLogger(__name__)
53
+
54
+
55
+ _MAX_ARTIFACT_SIZE_IN_BYTES = 2147483648 # 2GB
56
+ MODEL_BY_REFERENCE_VERSION = "1.0"
57
+ MODEL_BY_REFERENCE_JSON_FILE_NAME = "model_description.json"
58
+
59
+
60
+ class ModelArtifactSizeError(Exception): # pragma: no cover
61
+ def __init__(self, max_artifact_size: str):
62
+ super().__init__(
63
+ f"The model artifacts size is greater than `{max_artifact_size}`. "
64
+ "The `bucket_uri` needs to be specified to "
65
+ "copy artifacts to the object storage bucket. "
66
+ "Example: `bucket_uri=oci://<bucket_name>@<namespace>/prefix/`"
67
+ )
68
+
69
+
70
+ class BucketNotVersionedError(Exception): # pragma: no cover
71
+ def __init__(
72
+ self,
73
+ msg="Model artifact bucket is not versioned. Enable versioning on the bucket to proceed with model creation by reference.",
74
+ ):
75
+ super().__init__(msg)
76
+
77
+
78
+ class PathNotFoundError(Exception):
79
+ def __init__(self, msg="The given path doesn't exist."):
80
+ super().__init__(msg)
81
+
82
+
83
+ class ModelFileDescriptionError(Exception): # pragma: no cover
84
+ def __init__(self, msg="Model File Description file is not set up."):
85
+ super().__init__(msg)
86
+
87
+
88
+ class InvalidArtifactType(Exception): # pragma: no cover
89
+ pass
90
+
91
+
92
+ class InvalidArtifactPathTypeOrContentError(Exception): # pragma: no cover
93
+ def __init__(self, msg="Invalid type of Metdata artifact content"):
94
+ super().__init__(msg)
95
+
96
+
97
+ class CustomerNotificationType(ExtendedEnum):
98
+ NONE = "NONE"
99
+ ALL = "ALL"
100
+ ON_FAILURE = "ON_FAILURE"
101
+ ON_SUCCESS = "ON_SUCCESS"
102
+
103
+
104
+ class SettingStatus(ExtendedEnum):
105
+ """Enum to represent the status of retention settings."""
106
+
107
+ PENDING = "PENDING"
108
+ SUCCEEDED = "SUCCEEDED"
109
+ FAILED = "FAILED"
110
+
111
+
112
+ class ModelBackupSetting:
113
+ """
114
+ Class that represents Model Backup Setting Details Metadata.
115
+
116
+ Methods
117
+ -------
118
+ to_dict(self) -> Dict:
119
+ Serializes the backup settings into a dictionary.
120
+ from_dict(cls, data: Dict) -> 'ModelBackupSetting':
121
+ Constructs backup settings from a dictionary.
122
+ to_json(self) -> str:
123
+ Serializes the backup settings into a JSON string.
124
+ from_json(cls, json_str: str) -> 'ModelBackupSetting':
125
+ Constructs backup settings from a JSON string.
126
+ to_yaml(self) -> str:
127
+ Serializes the backup settings into a YAML string.
128
+ validate(self) -> bool:
129
+ Validates the backup settings details.
130
+ """
131
+
132
+ def __init__(
133
+ self,
134
+ is_backup_enabled: Optional[bool] = None,
135
+ backup_region: Optional[str] = None,
136
+ customer_notification_type: Optional[CustomerNotificationType] = None,
137
+ ):
138
+ self.is_backup_enabled = (
139
+ is_backup_enabled if is_backup_enabled is not None else False
140
+ )
141
+ self.backup_region = backup_region
142
+ self.customer_notification_type = (
143
+ customer_notification_type or CustomerNotificationType.NONE
144
+ )
145
+
146
+ def to_dict(self) -> Dict:
147
+ """Serializes the backup settings into a dictionary."""
148
+ return {
149
+ "is_backup_enabled": self.is_backup_enabled,
150
+ "backup_region": self.backup_region,
151
+ "customer_notification_type": self.customer_notification_type,
152
+ }
153
+
154
+ @classmethod
155
+ def from_dict(cls, data: Dict) -> "ModelBackupSetting":
156
+ """Constructs backup settings from a dictionary."""
157
+ return cls(
158
+ is_backup_enabled=data.get("is_backup_enabled"),
159
+ backup_region=data.get("backup_region"),
160
+ customer_notification_type=data.get("customer_notification_type") or None,
161
+ )
162
+
163
+ def to_json(self) -> str:
164
+ """Serializes the backup settings into a JSON string."""
165
+ return json.dumps(self.to_dict())
166
+
167
+ @classmethod
168
+ def from_json(cls, json_str) -> "ModelBackupSetting":
169
+ """Constructs backup settings from a JSON string or dictionary."""
170
+ data = json.loads(json_str) if isinstance(json_str, str) else json_str
171
+
172
+ return cls.from_dict(data)
173
+
174
+ def to_yaml(self) -> str:
175
+ """Serializes the backup settings into a YAML string."""
176
+ return yaml.dump(self.to_dict())
177
+
178
+ def validate(self) -> bool:
179
+ """Validates the backup settings details. Returns True if valid, False otherwise."""
180
+ return all(
181
+ [
182
+ isinstance(self.is_backup_enabled, bool),
183
+ not self.backup_region or isinstance(self.backup_region, str),
184
+ isinstance(self.customer_notification_type, str)
185
+ and self.customer_notification_type
186
+ in CustomerNotificationType.values(),
187
+ ]
188
+ )
189
+
190
+ def __repr__(self):
191
+ return self.to_yaml()
192
+
193
+
194
+ class ModelRetentionSetting:
195
+ """
196
+ Class that represents Model Retention Setting Details Metadata.
197
+
198
+ Methods
199
+ -------
200
+ to_dict(self) -> Dict:
201
+ Serializes the retention settings into a dictionary.
202
+ from_dict(cls, data: Dict) -> 'ModelRetentionSetting':
203
+ Constructs retention settings from a dictionary.
204
+ to_json(self) -> str:
205
+ Serializes the retention settings into a JSON string.
206
+ from_json(cls, json_str: str) -> 'ModelRetentionSetting':
207
+ Constructs retention settings from a JSON string.
208
+ to_yaml(self) -> str:
209
+ Serializes the retention settings into a YAML string.
210
+ validate(self) -> bool:
211
+ Validates the retention settings details.
212
+ """
213
+
214
+ def __init__(
215
+ self,
216
+ archive_after_days: Optional[int] = None,
217
+ delete_after_days: Optional[int] = None,
218
+ customer_notification_type: Optional[CustomerNotificationType] = None,
219
+ ):
220
+ self.archive_after_days = archive_after_days
221
+ self.delete_after_days = delete_after_days
222
+ self.customer_notification_type = (
223
+ customer_notification_type or CustomerNotificationType.NONE
224
+ )
225
+
226
+ def to_dict(self) -> Dict:
227
+ """Serializes the retention settings into a dictionary."""
228
+ return {
229
+ "archive_after_days": self.archive_after_days,
230
+ "delete_after_days": self.delete_after_days,
231
+ "customer_notification_type": self.customer_notification_type,
232
+ }
233
+
234
+ @classmethod
235
+ def from_dict(cls, data: Dict) -> "ModelRetentionSetting":
236
+ """Constructs retention settings from a dictionary."""
237
+ return cls(
238
+ archive_after_days=data.get("archive_after_days"),
239
+ delete_after_days=data.get("delete_after_days"),
240
+ customer_notification_type=data.get("customer_notification_type") or None,
241
+ )
242
+
243
+ def to_json(self) -> str:
244
+ """Serializes the retention settings into a JSON string."""
245
+ return json.dumps(self.to_dict())
246
+
247
+ @classmethod
248
+ def from_json(cls, json_str) -> "ModelRetentionSetting":
249
+ """Constructs retention settings from a JSON string."""
250
+ data = json.loads(json_str) if isinstance(json_str, str) else json_str
251
+ return cls.from_dict(data)
252
+
253
+ def to_yaml(self) -> str:
254
+ """Serializes the retention settings into a YAML string."""
255
+ return yaml.dump(self.to_dict())
256
+
257
+ def validate(self) -> bool:
258
+ """Validates the retention settings details. Returns True if valid, False otherwise."""
259
+ return all(
260
+ [
261
+ self.archive_after_days is None
262
+ or (
263
+ isinstance(self.archive_after_days, int)
264
+ and self.archive_after_days >= 0
265
+ ),
266
+ self.delete_after_days is None
267
+ or (
268
+ isinstance(self.delete_after_days, int)
269
+ and self.delete_after_days >= 0
270
+ ),
271
+ isinstance(self.customer_notification_type, str)
272
+ and self.customer_notification_type
273
+ in CustomerNotificationType.values(),
274
+ ]
275
+ )
276
+
277
+ def __repr__(self):
278
+ return self.to_yaml()
279
+
280
+
281
+ class ModelRetentionOperationDetails:
282
+ """
283
+ Class that represents Model Retention Operation Details Metadata.
284
+
285
+ Methods
286
+ -------
287
+ to_dict(self) -> Dict:
288
+ Serializes the retention operation details into a dictionary.
289
+ from_dict(cls, data: Dict) -> 'ModelRetentionOperationDetails':
290
+ Constructs retention operation details from a dictionary.
291
+ to_json(self) -> str:
292
+ Serializes the retention operation details into a JSON string.
293
+ from_json(cls, json_str: str) -> 'ModelRetentionOperationDetails':
294
+ Constructs retention operation details from a JSON string.
295
+ to_yaml(self) -> str:
296
+ Serializes the retention operation details into a YAML string.
297
+ validate(self) -> bool:
298
+ Validates the retention operation details.
299
+ """
300
+
301
+ def __init__(
302
+ self,
303
+ archive_state: Optional[SettingStatus] = None,
304
+ archive_state_details: Optional[str] = None,
305
+ delete_state: Optional[SettingStatus] = None,
306
+ delete_state_details: Optional[str] = None,
307
+ time_archival_scheduled: Optional[int] = None,
308
+ time_deletion_scheduled: Optional[int] = None,
309
+ ):
310
+ self.archive_state = archive_state
311
+ self.archive_state_details = archive_state_details
312
+ self.delete_state = delete_state
313
+ self.delete_state_details = delete_state_details
314
+ self.time_archival_scheduled = time_archival_scheduled
315
+ self.time_deletion_scheduled = time_deletion_scheduled
316
+
317
+ def to_dict(self) -> Dict:
318
+ """Serializes the retention operation details into a dictionary."""
319
+ return {
320
+ "archive_state": self.archive_state or None,
321
+ "archive_state_details": self.archive_state_details,
322
+ "delete_state": self.delete_state or None,
323
+ "delete_state_details": self.delete_state_details,
324
+ "time_archival_scheduled": self.time_archival_scheduled,
325
+ "time_deletion_scheduled": self.time_deletion_scheduled,
326
+ }
327
+
328
+ @classmethod
329
+ def from_dict(cls, data: Dict) -> "ModelRetentionOperationDetails":
330
+ """Constructs retention operation details from a dictionary."""
331
+ return cls(
332
+ archive_state=data.get("archive_state") or None,
333
+ archive_state_details=data.get("archive_state_details"),
334
+ delete_state=data.get("delete_state") or None,
335
+ delete_state_details=data.get("delete_state_details"),
336
+ time_archival_scheduled=data.get("time_archival_scheduled"),
337
+ time_deletion_scheduled=data.get("time_deletion_scheduled"),
338
+ )
339
+
340
+ def to_json(self) -> str:
341
+ """Serializes the retention operation details into a JSON string."""
342
+ return json.dumps(self.to_dict())
343
+
344
+ @classmethod
345
+ def from_json(cls, json_str: str) -> "ModelRetentionOperationDetails":
346
+ """Constructs retention operation details from a JSON string."""
347
+ data = json.loads(json_str)
348
+ return cls.from_dict(data)
349
+
350
+ def to_yaml(self) -> str:
351
+ """Serializes the retention operation details into a YAML string."""
352
+ return yaml.dump(self.to_dict())
353
+
354
+ def validate(self) -> bool:
355
+ """Validates the retention operation details."""
356
+ return all(
357
+ [
358
+ self.archive_state is None
359
+ or self.archive_state in SettingStatus.values(),
360
+ self.delete_state is None
361
+ or self.delete_state in SettingStatus.values(),
362
+ self.time_archival_scheduled is None
363
+ or isinstance(self.time_archival_scheduled, int),
364
+ self.time_deletion_scheduled is None
365
+ or isinstance(self.time_deletion_scheduled, int),
366
+ ]
367
+ )
368
+
369
+ def __repr__(self):
370
+ return self.to_yaml()
371
+
372
+
373
+ class ModelBackupOperationDetails:
374
+ """
375
+ Class that represents Model Backup Operation Details Metadata.
376
+
377
+ Methods
378
+ -------
379
+ to_dict(self) -> Dict:
380
+ Serializes the backup operation details into a dictionary.
381
+ from_dict(cls, data: Dict) -> 'ModelBackupOperationDetails':
382
+ Constructs backup operation details from a dictionary.
383
+ to_json(self) -> str:
384
+ Serializes the backup operation details into a JSON string.
385
+ from_json(cls, json_str: str) -> 'ModelBackupOperationDetails':
386
+ Constructs backup operation details from a JSON string.
387
+ to_yaml(self) -> str:
388
+ Serializes the backup operation details into a YAML string.
389
+ validate(self) -> bool:
390
+ Validates the backup operation details.
391
+ """
392
+
393
+ def __init__(
394
+ self,
395
+ backup_state: Optional[SettingStatus] = None,
396
+ backup_state_details: Optional[str] = None,
397
+ time_last_backup: Optional[int] = None,
398
+ ):
399
+ self.backup_state = backup_state
400
+ self.backup_state_details = backup_state_details
401
+ self.time_last_backup = time_last_backup
402
+
403
+ def to_dict(self) -> Dict:
404
+ """Serializes the backup operation details into a dictionary."""
405
+ return {
406
+ "backup_state": self.backup_state or None,
407
+ "backup_state_details": self.backup_state_details,
408
+ "time_last_backup": self.time_last_backup,
409
+ }
410
+
411
+ @classmethod
412
+ def from_dict(cls, data: Dict) -> "ModelBackupOperationDetails":
413
+ """Constructs backup operation details from a dictionary."""
414
+ return cls(
415
+ backup_state=data.get("backup_state") or None,
416
+ backup_state_details=data.get("backup_state_details"),
417
+ time_last_backup=data.get("time_last_backup"),
418
+ )
419
+
420
+ def to_json(self) -> str:
421
+ """Serializes the backup operation details into a JSON string."""
422
+ return json.dumps(self.to_dict())
423
+
424
+ @classmethod
425
+ def from_json(cls, json_str: str) -> "ModelBackupOperationDetails":
426
+ """Constructs backup operation details from a JSON string."""
427
+ data = json.loads(json_str)
428
+ return cls.from_dict(data)
429
+
430
+ def to_yaml(self) -> str:
431
+ """Serializes the backup operation details into a YAML string."""
432
+ return yaml.dump(self.to_dict())
433
+
434
+ def validate(self) -> bool:
435
+ """Validates the backup operation details."""
436
+ return not (
437
+ (
438
+ self.backup_state is not None
439
+ and self.backup_state not in SettingStatus.values()
440
+ )
441
+ or (
442
+ self.time_last_backup is not None
443
+ and not isinstance(self.time_last_backup, int)
444
+ )
445
+ )
446
+
447
+ def __repr__(self):
448
+ return self.to_yaml()
449
+
450
+
451
+ class DataScienceModel(Builder):
452
+ """Represents a Data Science Model.
453
+
454
+ Attributes
455
+ ----------
456
+ id: str
457
+ Model ID.
458
+ project_id: str
459
+ Project OCID.
460
+ compartment_id: str
461
+ Compartment OCID.
462
+ name: str
463
+ Model name.
464
+ description: str
465
+ Model description.
466
+ freeform_tags: Dict[str, str]
467
+ Model freeform tags.
468
+ defined_tags: Dict[str, Dict[str, object]]
469
+ Model defined tags.
470
+ input_schema: ads.feature_engineering.Schema
471
+ Model input schema.
472
+ output_schema: ads.feature_engineering.Schema, Dict
473
+ Model output schema.
474
+ defined_metadata_list: ModelTaxonomyMetadata
475
+ Model defined metadata.
476
+ custom_metadata_list: ModelCustomMetadata
477
+ Model custom metadata.
478
+ provenance_metadata: ModelProvenanceMetadata
479
+ Model provenance metadata.
480
+ artifact: str
481
+ The artifact location. Can be either path to folder with artifacts or
482
+ path to zip archive.
483
+ status: Union[str, None]
484
+ Model status.
485
+ model_version_set_id: str
486
+ Model version set ID
487
+ version_label: str
488
+ Model version label
489
+ version_id: str
490
+ Model version id
491
+ model_file_description: dict
492
+ Contains object path details for models created by reference.
493
+ backup_setting: ModelBackupSetting
494
+ The value to assign to the backup_setting property of this CreateModelDetails.
495
+ retention_setting: ModelRetentionSetting
496
+ The value to assign to the retention_setting property of this CreateModelDetails.
497
+ retention_operation_details: ModelRetentionOperationDetails
498
+ The value to assign to the retention_operation_details property for the Model.
499
+ backup_operation_details: ModelBackupOperationDetails
500
+ The value to assign to the backup_operation_details property for the Model.
501
+
502
+ Methods
503
+ -------
504
+ create(self, **kwargs) -> "DataScienceModel"
505
+ Creates model.
506
+ delete(self, delete_associated_model_deployment: Optional[bool] = False) -> "DataScienceModel":
507
+ Removes model.
508
+ to_dict(self) -> dict
509
+ Serializes model to a dictionary.
510
+ from_id(cls, id: str) -> "DataScienceModel"
511
+ Gets an existing model by OCID.
512
+ from_dict(cls, config: dict) -> "DataScienceModel"
513
+ Loads model instance from a dictionary of configurations.
514
+ upload_artifact(self, ...) -> None
515
+ Uploads model artifacts to the model catalog.
516
+ download_artifact(self, ...) -> None
517
+ Downloads model artifacts from the model catalog.
518
+ update(self, **kwargs) -> "DataScienceModel"
519
+ Updates datascience model in model catalog.
520
+ list(cls, compartment_id: str = None, **kwargs) -> List["DataScienceModel"]
521
+ Lists datascience models in a given compartment.
522
+ sync(self):
523
+ Sync up a datascience model with OCI datascience model.
524
+ with_project_id(self, project_id: str) -> "DataScienceModel"
525
+ Sets the project ID.
526
+ with_description(self, description: str) -> "DataScienceModel"
527
+ Sets the description.
528
+ with_compartment_id(self, compartment_id: str) -> "DataScienceModel"
529
+ Sets the compartment ID.
530
+ with_display_name(self, name: str) -> "DataScienceModel"
531
+ Sets the name.
532
+ with_freeform_tags(self, **kwargs: Dict[str, str]) -> "DataScienceModel"
533
+ Sets freeform tags.
534
+ with_defined_tags(self, **kwargs: Dict[str, Dict[str, object]]) -> "DataScienceModel"
535
+ Sets defined tags.
536
+ with_input_schema(self, schema: Union[Schema, Dict]) -> "DataScienceModel"
537
+ Sets the model input schema.
538
+ with_output_schema(self, schema: Union[Schema, Dict]) -> "DataScienceModel"
539
+ Sets the model output schema.
540
+ with_defined_metadata_list(self, metadata: Union[ModelTaxonomyMetadata, Dict]) -> "DataScienceModel"
541
+ Sets model taxonomy (defined) metadata.
542
+ with_custom_metadata_list(self, metadata: Union[ModelCustomMetadata, Dict]) -> "DataScienceModel"
543
+ Sets model custom metadata.
544
+ with_provenance_metadata(self, metadata: Union[ModelProvenanceMetadata, Dict]) -> "DataScienceModel"
545
+ Sets model provenance metadata.
546
+ with_artifact(self, *uri: str)
547
+ Sets the artifact location. Can be a local. For models created by reference, uri can take in single arg or multiple args in case
548
+ of a fine-tuned or multimodel setting.
549
+ with_model_version_set_id(self, model_version_set_id: str):
550
+ Sets the model version set ID.
551
+ with_version_label(self, version_label: str):
552
+ Sets the model version label.
553
+ with_version_id(self, version_id: str):
554
+ Sets the model version id.
555
+ with_model_file_description: dict
556
+ Sets path details for models created by reference. Input can be either a dict, string or json file and
557
+ the schema is dictated by model_file_description_schema.json
558
+
559
+
560
+ Examples
561
+ --------
562
+ >>> ds_model = (DataScienceModel()
563
+ ... .with_compartment_id(os.environ["NB_SESSION_COMPARTMENT_OCID"])
564
+ ... .with_project_id(os.environ["PROJECT_OCID"])
565
+ ... .with_display_name("TestModel")
566
+ ... .with_description("Testing the test model")
567
+ ... .with_freeform_tags(tag1="val1", tag2="val2")
568
+ ... .with_artifact("/path/to/the/model/artifacts/"))
569
+ >>> ds_model.create()
570
+ >>> ds_model.status()
571
+ >>> ds_model.with_description("new description").update()
572
+ >>> ds_model.download_artifact("/path/to/dst/folder/")
573
+ >>> ds_model.delete()
574
+ >>> DataScienceModel.list()
575
+ """
576
+
577
+ _PREFIX = "datascience_model"
578
+
579
+ CONST_ID = "id"
580
+ CONST_PROJECT_ID = "projectId"
581
+ CONST_COMPARTMENT_ID = "compartmentId"
582
+ CONST_DISPLAY_NAME = "displayName"
583
+ CONST_DESCRIPTION = "description"
584
+ CONST_FREEFORM_TAG = "freeformTags"
585
+ CONST_DEFINED_TAG = "definedTags"
586
+ CONST_INPUT_SCHEMA = "inputSchema"
587
+ CONST_OUTPUT_SCHEMA = "outputSchema"
588
+ CONST_CUSTOM_METADATA = "customMetadataList"
589
+ CONST_DEFINED_METADATA = "definedMetadataList"
590
+ CONST_PROVENANCE_METADATA = "provenanceMetadata"
591
+ CONST_ARTIFACT = "artifact"
592
+ CONST_MODEL_VERSION_SET_ID = "modelVersionSetId"
593
+ CONST_MODEL_VERSION_SET_NAME = "modelVersionSetName"
594
+ CONST_MODEL_VERSION_LABEL = "versionLabel"
595
+ CONST_MODEL_VERSION_ID = "versionId"
596
+ CONST_TIME_CREATED = "timeCreated"
597
+ CONST_LIFECYCLE_STATE = "lifecycleState"
598
+ CONST_LIFECYCLE_DETAILS = "lifecycleDetails"
599
+ CONST_MODEL_FILE_DESCRIPTION = "modelDescription"
600
+ CONST_BACKUP_SETTING = "backupSetting"
601
+ CONST_RETENTION_SETTING = "retentionSetting"
602
+ CONST_BACKUP_OPERATION_DETAILS = "backupOperationDetails"
603
+ CONST_RETENTION_OPERATION_DETAILS = "retentionOperationDetails"
604
+
605
+ attribute_map = {
606
+ CONST_ID: "id",
607
+ CONST_PROJECT_ID: "project_id",
608
+ CONST_COMPARTMENT_ID: "compartment_id",
609
+ CONST_DISPLAY_NAME: "display_name",
610
+ CONST_DESCRIPTION: "description",
611
+ CONST_FREEFORM_TAG: "freeform_tags",
612
+ CONST_DEFINED_TAG: "defined_tags",
613
+ CONST_INPUT_SCHEMA: "input_schema",
614
+ CONST_OUTPUT_SCHEMA: "output_schema",
615
+ CONST_CUSTOM_METADATA: "custom_metadata_list",
616
+ CONST_DEFINED_METADATA: "defined_metadata_list",
617
+ CONST_PROVENANCE_METADATA: "provenance_metadata",
618
+ CONST_ARTIFACT: "artifact",
619
+ CONST_MODEL_VERSION_SET_ID: "model_version_set_id",
620
+ CONST_MODEL_VERSION_SET_NAME: "model_version_set_name",
621
+ CONST_MODEL_VERSION_LABEL: "version_label",
622
+ CONST_MODEL_VERSION_ID: "version_id",
623
+ CONST_TIME_CREATED: "time_created",
624
+ CONST_LIFECYCLE_STATE: "lifecycle_state",
625
+ CONST_LIFECYCLE_DETAILS: "lifecycle_details",
626
+ CONST_MODEL_FILE_DESCRIPTION: "model_description",
627
+ CONST_BACKUP_SETTING: "backup_setting",
628
+ CONST_RETENTION_SETTING: "retention_setting",
629
+ CONST_BACKUP_OPERATION_DETAILS: "backup_operation_details",
630
+ CONST_RETENTION_OPERATION_DETAILS: "retention_operation_details",
631
+ }
632
+
633
+ def __init__(self, spec: Dict = None, **kwargs) -> None:
634
+ """Initializes datascience model.
635
+
636
+ Parameters
637
+ ----------
638
+ spec: (Dict, optional). Defaults to None.
639
+ Object specification.
640
+
641
+ kwargs: Dict
642
+ Specification as keyword arguments.
643
+ If 'spec' contains the same key as the one in kwargs,
644
+ the value from kwargs will be used.
645
+
646
+ - project_id: str
647
+ - compartment_id: str
648
+ - name: str
649
+ - description: str
650
+ - defined_tags: Dict[str, Dict[str, object]]
651
+ - freeform_tags: Dict[str, str]
652
+ - input_schema: Union[ads.feature_engineering.Schema, Dict]
653
+ - output_schema: Union[ads.feature_engineering.Schema, Dict]
654
+ - defined_metadata_list: Union[ModelTaxonomyMetadata, Dict]
655
+ - custom_metadata_list: Union[ModelCustomMetadata, Dict]
656
+ - provenance_metadata: Union[ModelProvenanceMetadata, Dict]
657
+ - artifact: str
658
+ """
659
+ super().__init__(spec=spec, **deepcopy(kwargs))
660
+ # Reinitiate complex attributes
661
+ self._init_complex_attributes()
662
+ # Specify oci datascience model instance
663
+ self.dsc_model = self._to_oci_dsc_model()
664
+ self.local_copy_dir = None
665
+
666
+ @property
667
+ def id(self) -> Optional[str]:
668
+ """The model OCID."""
669
+ if self.dsc_model:
670
+ return self.dsc_model.id
671
+ return None
672
+
673
+ @property
674
+ def status(self) -> Union[str, None]:
675
+ """Status of the model.
676
+
677
+ Returns
678
+ -------
679
+ str
680
+ Status of the model.
681
+ """
682
+ if self.dsc_model:
683
+ return self.dsc_model.status
684
+ return None
685
+
686
+ @property
687
+ def lifecycle_state(self) -> Union[str, None]:
688
+ """Status of the model.
689
+
690
+ Returns
691
+ -------
692
+ str
693
+ Status of the model.
694
+ """
695
+ if self.dsc_model:
696
+ return self.dsc_model.status
697
+ return None
698
+
699
+ @property
700
+ def lifecycle_details(self) -> str:
701
+ """
702
+ Gets the lifecycle_details of this DataScienceModel.
703
+ Details about the lifecycle state of the model.
704
+
705
+ :return: The lifecycle_details of this DataScienceModel.
706
+ :rtype: str
707
+ """
708
+ return self.get_spec(self.CONST_LIFECYCLE_DETAILS)
709
+
710
+ @lifecycle_details.setter
711
+ def lifecycle_details(self, lifecycle_details: str) -> "DataScienceModel":
712
+ """
713
+ Sets the lifecycle_details of this DataScienceModel.
714
+ Details about the lifecycle state of the model.
715
+
716
+ :param lifecycle_details: The lifecycle_details of this DataScienceModel.
717
+ :type: str
718
+ """
719
+ return self.set_spec(self.CONST_LIFECYCLE_DETAILS, lifecycle_details)
720
+
721
+ @property
722
+ def kind(self) -> str:
723
+ """The kind of the object as showing in a YAML."""
724
+ return "datascienceModel"
725
+
726
+ @property
727
+ def project_id(self) -> str:
728
+ return self.get_spec(self.CONST_PROJECT_ID)
729
+
730
+ def with_project_id(self, project_id: str) -> "DataScienceModel":
731
+ """Sets the project ID.
732
+
733
+ Parameters
734
+ ----------
735
+ project_id: str
736
+ The project ID.
737
+
738
+ Returns
739
+ -------
740
+ DataScienceModel
741
+ The DataScienceModel instance (self)
742
+ """
743
+ return self.set_spec(self.CONST_PROJECT_ID, project_id)
744
+
745
+ @property
746
+ def time_created(self) -> str:
747
+ return self.get_spec(self.CONST_TIME_CREATED)
748
+
749
+ @property
750
+ def description(self) -> str:
751
+ return self.get_spec(self.CONST_DESCRIPTION)
752
+
753
+ def with_description(self, description: str) -> "DataScienceModel":
754
+ """Sets the description.
755
+
756
+ Parameters
757
+ ----------
758
+ description: str
759
+ The description of the model.
760
+
761
+ Returns
762
+ -------
763
+ DataScienceModel
764
+ The DataScienceModel instance (self)
765
+ """
766
+ return self.set_spec(self.CONST_DESCRIPTION, description)
767
+
768
+ @property
769
+ def compartment_id(self) -> str:
770
+ return self.get_spec(self.CONST_COMPARTMENT_ID)
771
+
772
+ def with_compartment_id(self, compartment_id: str) -> "DataScienceModel":
773
+ """Sets the compartment ID.
774
+
775
+ Parameters
776
+ ----------
777
+ compartment_id: str
778
+ The compartment ID.
779
+
780
+ Returns
781
+ -------
782
+ DataScienceModel
783
+ The DataScienceModel instance (self)
784
+ """
785
+ return self.set_spec(self.CONST_COMPARTMENT_ID, compartment_id)
786
+
787
+ @property
788
+ def display_name(self) -> str:
789
+ return self.get_spec(self.CONST_DISPLAY_NAME)
790
+
791
+ @display_name.setter
792
+ def display_name(self, name: str) -> "DataScienceModel":
793
+ return self.set_spec(self.CONST_DISPLAY_NAME, name)
794
+
795
+ def with_display_name(self, name: str) -> "DataScienceModel":
796
+ """Sets the name.
797
+
798
+ Parameters
799
+ ----------
800
+ name: str
801
+ The name.
802
+
803
+ Returns
804
+ -------
805
+ DataScienceModel
806
+ The DataScienceModel instance (self)
807
+ """
808
+ return self.set_spec(self.CONST_DISPLAY_NAME, name)
809
+
810
+ @property
811
+ def freeform_tags(self) -> Dict[str, str]:
812
+ return self.get_spec(self.CONST_FREEFORM_TAG)
813
+
814
+ def with_freeform_tags(self, **kwargs: Dict[str, str]) -> "DataScienceModel":
815
+ """Sets freeform tags.
816
+
817
+ Returns
818
+ -------
819
+ DataScienceModel
820
+ The DataScienceModel instance (self)
821
+ """
822
+ return self.set_spec(self.CONST_FREEFORM_TAG, kwargs)
823
+
824
+ @property
825
+ def defined_tags(self) -> Dict[str, Dict[str, object]]:
826
+ return self.get_spec(self.CONST_DEFINED_TAG)
827
+
828
+ def with_defined_tags(
829
+ self, **kwargs: Dict[str, Dict[str, object]]
830
+ ) -> "DataScienceModel":
831
+ """Sets defined tags.
832
+
833
+ Returns
834
+ -------
835
+ DataScienceModel
836
+ The DataScienceModel instance (self)
837
+ """
838
+ return self.set_spec(self.CONST_DEFINED_TAG, kwargs)
839
+
840
+ @property
841
+ def input_schema(self) -> Union[Schema, Dict]:
842
+ """Returns model input schema.
843
+
844
+ Returns
845
+ -------
846
+ ads.feature_engineering.Schema
847
+ Model input schema.
848
+ """
849
+ return self.get_spec(self.CONST_INPUT_SCHEMA)
850
+
851
+ def with_input_schema(self, schema: Union[Schema, Dict]) -> "DataScienceModel":
852
+ """Sets the model input schema.
853
+
854
+ Parameters
855
+ ----------
856
+ schema: Union[ads.feature_engineering.Schema, Dict]
857
+ The model input schema.
858
+
859
+ Returns
860
+ -------
861
+ DataScienceModel
862
+ The DataScienceModel instance (self)
863
+ """
864
+ if schema and isinstance(schema, Dict):
865
+ try:
866
+ schema = Schema.from_dict(schema)
867
+ except Exception as err:
868
+ logger.warn(err)
869
+
870
+ return self.set_spec(self.CONST_INPUT_SCHEMA, schema)
871
+
872
+ @property
873
+ def output_schema(self) -> Union[Schema, Dict]:
874
+ """Returns model output schema.
875
+
876
+ Returns
877
+ -------
878
+ ads.feature_engineering.Schema
879
+ Model output schema.
880
+ """
881
+ return self.get_spec(self.CONST_OUTPUT_SCHEMA)
882
+
883
+ def with_output_schema(self, schema: Union[Schema, Dict]) -> "DataScienceModel":
884
+ """Sets the model output schema.
885
+
886
+ Parameters
887
+ ----------
888
+ schema: Union[ads.feature_engineering.Schema, Dict]
889
+ The model output schema.
890
+
891
+ Returns
892
+ -------
893
+ DataScienceModel
894
+ The DataScienceModel instance (self)
895
+ """
896
+ if schema and isinstance(schema, Dict):
897
+ try:
898
+ schema = Schema.from_dict(schema)
899
+ except Exception as err:
900
+ logger.warn(err)
901
+
902
+ return self.set_spec(self.CONST_OUTPUT_SCHEMA, schema)
903
+
904
+ @property
905
+ def defined_metadata_list(self) -> ModelTaxonomyMetadata:
906
+ """Returns model taxonomy (defined) metadatda."""
907
+ return self.get_spec(self.CONST_DEFINED_METADATA)
908
+
909
+ def with_defined_metadata_list(
910
+ self, metadata: Union[ModelTaxonomyMetadata, Dict]
911
+ ) -> "DataScienceModel":
912
+ """Sets model taxonomy (defined) metadata.
913
+
914
+ Parameters
915
+ ----------
916
+ metadata: Union[ModelTaxonomyMetadata, Dict]
917
+ The defined metadata.
918
+
919
+ Returns
920
+ -------
921
+ DataScienceModel
922
+ The DataScienceModel instance (self)
923
+ """
924
+ if metadata and isinstance(metadata, Dict):
925
+ metadata = ModelTaxonomyMetadata.from_dict(metadata)
926
+ return self.set_spec(self.CONST_DEFINED_METADATA, metadata)
927
+
928
+ @property
929
+ def custom_metadata_list(self) -> ModelCustomMetadata:
930
+ """Returns model custom metadatda."""
931
+ return self.get_spec(self.CONST_CUSTOM_METADATA)
932
+
933
+ def with_custom_metadata_list(
934
+ self, metadata: Union[ModelCustomMetadata, Dict]
935
+ ) -> "DataScienceModel":
936
+ """Sets model custom metadata.
937
+
938
+ Parameters
939
+ ----------
940
+ metadata: Union[ModelCustomMetadata, Dict]
941
+ The custom metadata.
942
+
943
+ Returns
944
+ -------
945
+ DataScienceModel
946
+ The DataScienceModel instance (self)
947
+ """
948
+ if metadata and isinstance(metadata, Dict):
949
+ metadata = ModelCustomMetadata.from_dict(metadata)
950
+ return self.set_spec(self.CONST_CUSTOM_METADATA, metadata)
951
+
952
+ @property
953
+ def provenance_metadata(self) -> ModelProvenanceMetadata:
954
+ """Returns model provenance metadatda."""
955
+ return self.get_spec(self.CONST_PROVENANCE_METADATA)
956
+
957
+ def with_provenance_metadata(
958
+ self, metadata: Union[ModelProvenanceMetadata, Dict]
959
+ ) -> "DataScienceModel":
960
+ """Sets model provenance metadata.
961
+
962
+ Parameters
963
+ ----------
964
+ provenance_metadata: Union[ModelProvenanceMetadata, Dict]
965
+ The provenance metadata.
966
+
967
+ Returns
968
+ -------
969
+ DataScienceModel
970
+ The DataScienceModel instance (self)
971
+ """
972
+ if metadata and isinstance(metadata, Dict):
973
+ metadata = ModelProvenanceMetadata.from_dict(metadata)
974
+ return self.set_spec(self.CONST_PROVENANCE_METADATA, metadata)
975
+
976
+ @property
977
+ def artifact(self) -> Union[str, list]:
978
+ return self.get_spec(self.CONST_ARTIFACT)
979
+
980
+ def with_artifact(self, uri: str, *args):
981
+ """Sets the artifact location. Can be a local.
982
+
983
+ Parameters
984
+ ----------
985
+ uri: str
986
+ Path to artifact directory or to the ZIP archive.
987
+ It could contain a serialized model(required) as well as any files needed for deployment.
988
+ The content of the source folder will be zipped and uploaded to the model catalog.
989
+ For models created by reference, uri can take in single arg or multiple args in case of a fine-tuned or
990
+ multimodel setting.
991
+ Examples
992
+ --------
993
+ >>> .with_artifact(uri="./model1/")
994
+ >>> .with_artifact(uri="./model1.zip")
995
+ >>> .with_artifact("./model1", "./model2")
996
+ """
997
+
998
+ return self.set_spec(self.CONST_ARTIFACT, [uri] + list(args) if args else uri)
999
+
1000
+ @property
1001
+ def model_version_set_id(self) -> str:
1002
+ return self.get_spec(self.CONST_MODEL_VERSION_SET_ID)
1003
+
1004
+ def with_model_version_set_id(self, model_version_set_id: str):
1005
+ """Sets the model version set ID.
1006
+
1007
+ Parameters
1008
+ ----------
1009
+ urmodel_version_set_idi: str
1010
+ The Model version set OCID.
1011
+ """
1012
+ return self.set_spec(self.CONST_MODEL_VERSION_SET_ID, model_version_set_id)
1013
+
1014
+ @property
1015
+ def model_version_set_name(self) -> str:
1016
+ return self.get_spec(self.CONST_MODEL_VERSION_SET_NAME)
1017
+
1018
+ @property
1019
+ def version_label(self) -> str:
1020
+ return self.get_spec(self.CONST_MODEL_VERSION_LABEL)
1021
+
1022
+ def with_version_label(self, version_label: str):
1023
+ """Sets the model version label.
1024
+
1025
+ Parameters
1026
+ ----------
1027
+ version_label: str
1028
+ The model version label.
1029
+ """
1030
+ return self.set_spec(self.CONST_MODEL_VERSION_LABEL, version_label)
1031
+
1032
+ @property
1033
+ def version_id(self) -> str:
1034
+ return self.get_spec(self.CONST_MODEL_VERSION_ID)
1035
+
1036
+ def with_version_id(self, version_id: str):
1037
+ """Sets the model version id.
1038
+
1039
+ Parameters
1040
+ ----------
1041
+ version_id: str
1042
+ The model version id.
1043
+ """
1044
+ return self.set_spec(self.CONST_MODEL_VERSION_ID, version_id)
1045
+
1046
+ @property
1047
+ def model_file_description(self) -> dict:
1048
+ return self.get_spec(self.CONST_MODEL_FILE_DESCRIPTION)
1049
+
1050
+ def with_model_file_description(
1051
+ self, json_dict: dict = None, json_string: str = None, json_uri: str = None
1052
+ ):
1053
+ """Sets the json file description for model passed by reference
1054
+ Parameters
1055
+ ----------
1056
+ json_dict : dict, optional
1057
+ json dict, by default None
1058
+ json_string : str, optional
1059
+ json string, by default None
1060
+ json_uri : str, optional
1061
+ URI location of file containing json, by default None
1062
+
1063
+ Examples
1064
+ --------
1065
+ >>> DataScienceModel().with_model_file_description(json_string="<json_string>")
1066
+ >>> DataScienceModel().with_model_file_description(json_dict=dict())
1067
+ >>> DataScienceModel().with_model_file_description(json_uri="./model_description.json")
1068
+ """
1069
+ if json_dict:
1070
+ json_data = json_dict
1071
+ elif json_string:
1072
+ json_data = json.loads(json_string)
1073
+ elif json_uri:
1074
+ with open(json_uri) as json_file:
1075
+ json_data = json.load(json_file)
1076
+ else:
1077
+ raise ValueError("Must provide either a valid json string or URI location.")
1078
+
1079
+ schema_file_path = os.path.join(
1080
+ os.path.dirname(os.path.abspath(__file__)),
1081
+ "model_file_description_schema.json",
1082
+ )
1083
+ with open(schema_file_path, encoding="utf-8") as schema_file:
1084
+ schema = json.load(schema_file)
1085
+
1086
+ try:
1087
+ validate(json_data, schema)
1088
+ except ValidationError as ve:
1089
+ message = (
1090
+ f"model_file_description_schema.json validation failed. "
1091
+ f"See Exception: {ve}"
1092
+ )
1093
+ logging.error(message)
1094
+ raise ModelFileDescriptionError(message)
1095
+
1096
+ return self.set_spec(self.CONST_MODEL_FILE_DESCRIPTION, json_data)
1097
+
1098
+ @property
1099
+ def retention_setting(self) -> ModelRetentionSetting:
1100
+ """
1101
+ Gets the retention_setting of this model.
1102
+
1103
+ :return: The retention_setting of this model.
1104
+ :rtype: RetentionSetting
1105
+ """
1106
+ return self.get_spec(self.CONST_RETENTION_SETTING)
1107
+
1108
+ def with_retention_setting(
1109
+ self, retention_setting: Union[Dict, ModelRetentionSetting]
1110
+ ) -> "DataScienceModel":
1111
+ """
1112
+ Sets the retention setting details for the model.
1113
+
1114
+ Parameters
1115
+ ----------
1116
+ retention_setting : Union[Dict, RetentionSetting]
1117
+ The retention setting details for the model. Can be provided as either a dictionary or
1118
+ an instance of the `RetentionSetting` class.
1119
+
1120
+ Returns
1121
+ -------
1122
+ DataScienceModel
1123
+ The `DataScienceModel` instance (self) for method chaining.
1124
+ """
1125
+ return self.set_spec(self.CONST_RETENTION_SETTING, retention_setting)
1126
+
1127
+ @property
1128
+ def backup_setting(self) -> ModelBackupSetting:
1129
+ """
1130
+ Gets the backup_setting of this model.
1131
+
1132
+ :return: The backup_setting of this model.
1133
+ :rtype: BackupSetting
1134
+ """
1135
+ return self.get_spec(self.CONST_BACKUP_SETTING)
1136
+
1137
+ def with_backup_setting(
1138
+ self, backup_setting: Union[Dict, ModelBackupSetting]
1139
+ ) -> "DataScienceModel":
1140
+ """
1141
+ Sets the model's backup setting details.
1142
+
1143
+ Parameters
1144
+ ----------
1145
+ backup_setting : Union[Dict, BackupSetting]
1146
+ The backup setting details for the model. This can be passed as either a dictionary or
1147
+ an instance of the `BackupSetting` class.
1148
+
1149
+ Returns
1150
+ -------
1151
+ DataScienceModel
1152
+ The `DataScienceModel` instance (self) for method chaining.
1153
+ """
1154
+
1155
+ return self.set_spec(self.CONST_BACKUP_SETTING, backup_setting)
1156
+
1157
+ @property
1158
+ def retention_operation_details(self) -> ModelRetentionOperationDetails:
1159
+ """
1160
+ Gets the retention_operation_details of this Model using the spec constant.
1161
+
1162
+ :return: The retention_operation_details of this Model.
1163
+ :rtype: ModelRetentionOperationDetails
1164
+ """
1165
+ return self.get_spec(self.CONST_RETENTION_OPERATION_DETAILS)
1166
+
1167
+ @property
1168
+ def backup_operation_details(self) -> "ModelBackupOperationDetails":
1169
+ """
1170
+ Gets the backup_operation_details of this Model using the spec constant.
1171
+
1172
+ :return: The backup_operation_details of this Model.
1173
+ :rtype: ModelBackupOperationDetails
1174
+ """
1175
+ return self.get_spec(self.CONST_BACKUP_OPERATION_DETAILS)
1176
+
1177
+ def create(self, **kwargs) -> "DataScienceModel":
1178
+ """Creates datascience model.
1179
+
1180
+ Parameters
1181
+ ----------
1182
+ kwargs
1183
+ Additional kwargs arguments.
1184
+ Can be any attribute that `oci.data_science.models.Model` accepts.
1185
+
1186
+ In addition can be also provided the attributes listed below.
1187
+
1188
+ bucket_uri: (str, optional). Defaults to None.
1189
+ The OCI Object Storage URI where model artifacts will be copied to.
1190
+ The `bucket_uri` is only necessary for uploading large artifacts which
1191
+ size is greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
1192
+
1193
+ .. versionadded:: 2.8.10
1194
+
1195
+ If `artifact` is provided as an object storage path to a zip archive, `bucket_uri` will be ignored.
1196
+
1197
+ overwrite_existing_artifact: (bool, optional). Defaults to `True`.
1198
+ Overwrite target bucket artifact if exists.
1199
+ remove_existing_artifact: (bool, optional). Defaults to `True`.
1200
+ Wether artifacts uploaded to object storage bucket need to be removed or not.
1201
+ region: (str, optional). Defaults to `None`.
1202
+ The destination Object Storage bucket region.
1203
+ By default the value will be extracted from the `OCI_REGION_METADATA` environment variable.
1204
+ auth: (Dict, optional). Defaults to `None`.
1205
+ The default authentication is set using `ads.set_auth` API.
1206
+ If you need to override the default, use the `ads.common.auth.api_keys` or
1207
+ `ads.common.auth.resource_principal` to create appropriate authentication signer
1208
+ and kwargs required to instantiate IdentityClient object.
1209
+ timeout: (int, optional). Defaults to 10 seconds.
1210
+ The connection timeout in seconds for the client.
1211
+ parallel_process_count: (int, optional).
1212
+ The number of worker processes to use in parallel for uploading individual parts of a multipart upload.
1213
+ model_by_reference: (bool, optional)
1214
+ Whether model artifact is made available to Model Store by reference. Requires artifact location to be
1215
+ provided using with_artifact method.
1216
+
1217
+ Returns
1218
+ -------
1219
+ DataScienceModel
1220
+ The DataScienceModel instance (self)
1221
+
1222
+ Raises
1223
+ ------
1224
+ ValueError
1225
+ If compartment id not provided.
1226
+ If project id not provided.
1227
+ """
1228
+ if not self.compartment_id:
1229
+ raise ValueError("Compartment id must be provided.")
1230
+
1231
+ if not self.project_id:
1232
+ raise ValueError("Project id must be provided.")
1233
+
1234
+ if not self.display_name:
1235
+ self.display_name = self._random_display_name()
1236
+
1237
+ model_by_reference = kwargs.pop("model_by_reference", False)
1238
+ if model_by_reference:
1239
+ # Update custom metadata
1240
+ logger.info("Update custom metadata field with model by reference flag.")
1241
+ metadata_item = ModelCustomMetadataItem(
1242
+ key=self.CONST_MODEL_FILE_DESCRIPTION,
1243
+ value="true",
1244
+ description="model by reference flag",
1245
+ category=MetadataCustomCategory.OTHER,
1246
+ )
1247
+ if self.custom_metadata_list:
1248
+ self.custom_metadata_list._add(metadata_item, replace=True)
1249
+ else:
1250
+ custom_metadata = ModelCustomMetadata()
1251
+ custom_metadata._add(metadata_item)
1252
+ self.with_custom_metadata_list(custom_metadata)
1253
+
1254
+ payload = deepcopy(self._spec)
1255
+ payload.pop("id", None)
1256
+ logger.debug(f"Creating a model with payload {payload}")
1257
+
1258
+ # Create model in the model catalog
1259
+ logger.info("Saving model to the Model Catalog.")
1260
+ self.dsc_model = self._to_oci_dsc_model(**kwargs).create()
1261
+
1262
+ # Create model provenance
1263
+ if self.provenance_metadata:
1264
+ logger.info("Saving model provenance metadata.")
1265
+ self.dsc_model.create_model_provenance(
1266
+ self.provenance_metadata._to_oci_metadata()
1267
+ )
1268
+
1269
+ # Upload artifacts
1270
+ logger.info("Uploading model artifacts.")
1271
+ self.upload_artifact(
1272
+ bucket_uri=kwargs.pop("bucket_uri", None),
1273
+ overwrite_existing_artifact=kwargs.pop("overwrite_existing_artifact", True),
1274
+ remove_existing_artifact=kwargs.pop("remove_existing_artifact", True),
1275
+ region=kwargs.pop("region", None),
1276
+ auth=kwargs.pop("auth", None),
1277
+ timeout=kwargs.pop("timeout", None),
1278
+ parallel_process_count=kwargs.pop("parallel_process_count", None),
1279
+ model_by_reference=model_by_reference,
1280
+ )
1281
+
1282
+ # Sync up model
1283
+ self.sync()
1284
+ logger.info(f"Model {self.id} has been successfully saved.")
1285
+
1286
+ return self
1287
+
1288
+ def upload_artifact(
1289
+ self,
1290
+ bucket_uri: Optional[str] = None,
1291
+ auth: Optional[Dict] = None,
1292
+ region: Optional[str] = None,
1293
+ overwrite_existing_artifact: Optional[bool] = True,
1294
+ remove_existing_artifact: Optional[bool] = True,
1295
+ timeout: Optional[int] = None,
1296
+ parallel_process_count: int = utils.DEFAULT_PARALLEL_PROCESS_COUNT,
1297
+ model_by_reference: Optional[bool] = False,
1298
+ ) -> None:
1299
+ """Uploads model artifacts to the model catalog.
1300
+
1301
+ Parameters
1302
+ ----------
1303
+ bucket_uri: (str, optional). Defaults to None.
1304
+ The OCI Object Storage URI where model artifacts will be copied to.
1305
+ The `bucket_uri` is only necessary for uploading large artifacts which
1306
+ size is greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
1307
+
1308
+ .. versionadded:: 2.8.10
1309
+
1310
+ If `artifact` is provided as an object storage path to a zip archive, `bucket_uri` will be ignored.
1311
+
1312
+ auth: (Dict, optional). Defaults to `None`.
1313
+ The default authentication is set using `ads.set_auth` API.
1314
+ If you need to override the default, use the `ads.common.auth.api_keys` or
1315
+ `ads.common.auth.resource_principal` to create appropriate authentication signer
1316
+ and kwargs required to instantiate IdentityClient object.
1317
+ region: (str, optional). Defaults to `None`.
1318
+ The destination Object Storage bucket region.
1319
+ By default the value will be extracted from the `OCI_REGION_METADATA` environment variables.
1320
+ overwrite_existing_artifact: (bool, optional). Defaults to `True`.
1321
+ Overwrite target bucket artifact if exists.
1322
+ remove_existing_artifact: (bool, optional). Defaults to `True`.
1323
+ Wether artifacts uploaded to object storage bucket need to be removed or not.
1324
+ timeout: (int, optional). Defaults to 10 seconds.
1325
+ The connection timeout in seconds for the client.
1326
+ parallel_process_count: (int, optional)
1327
+ The number of worker processes to use in parallel for uploading individual parts of a multipart upload.
1328
+ model_by_reference: (bool, optional)
1329
+ Whether model artifact is made available to Model Store by reference.
1330
+ """
1331
+ # Upload artifact to the model catalog
1332
+ if model_by_reference and self.model_file_description:
1333
+ logger.info(
1334
+ "Model artifact will be uploaded using model_file_description contents, "
1335
+ "artifact location will not be used."
1336
+ )
1337
+ elif not self.artifact:
1338
+ logger.warn(
1339
+ "Model artifact location not provided. "
1340
+ "Provide the artifact location to upload artifacts to the model catalog."
1341
+ )
1342
+ return
1343
+
1344
+ if timeout:
1345
+ self.dsc_model._client = None
1346
+ self.dsc_model.__class__.kwargs = {
1347
+ **(self.dsc_model.__class__.kwargs or {}),
1348
+ "timeout": timeout,
1349
+ }
1350
+
1351
+ if model_by_reference:
1352
+ self._validate_prepare_file_description_artifact()
1353
+ else:
1354
+ if isinstance(self.artifact, list):
1355
+ raise InvalidArtifactType(
1356
+ "Multiple artifacts are only allowed for models created by reference."
1357
+ )
1358
+
1359
+ if ObjectStorageDetails.is_oci_path(self.artifact):
1360
+ if bucket_uri and bucket_uri != self.artifact:
1361
+ logger.warn(
1362
+ "The `bucket_uri` will be ignored and the value of `self.artifact` will be used instead."
1363
+ )
1364
+ bucket_uri = self.artifact
1365
+
1366
+ if not model_by_reference and (
1367
+ bucket_uri or utils.folder_size(self.artifact) > _MAX_ARTIFACT_SIZE_IN_BYTES
1368
+ ):
1369
+ if not bucket_uri:
1370
+ raise ModelArtifactSizeError(
1371
+ max_artifact_size=utils.human_size(_MAX_ARTIFACT_SIZE_IN_BYTES)
1372
+ )
1373
+
1374
+ artifact_uploader = LargeArtifactUploader(
1375
+ dsc_model=self.dsc_model,
1376
+ artifact_path=self.artifact,
1377
+ auth=auth,
1378
+ region=region,
1379
+ bucket_uri=bucket_uri,
1380
+ overwrite_existing_artifact=overwrite_existing_artifact,
1381
+ remove_existing_artifact=remove_existing_artifact,
1382
+ parallel_process_count=parallel_process_count,
1383
+ )
1384
+ else:
1385
+ artifact_uploader = SmallArtifactUploader(
1386
+ dsc_model=self.dsc_model,
1387
+ artifact_path=self.artifact,
1388
+ )
1389
+ artifact_uploader.upload()
1390
+
1391
+ self._remove_file_description_artifact()
1392
+
1393
+ def _remove_file_description_artifact(self):
1394
+ """Removes temporary model file description artifact for model by reference."""
1395
+ # delete if local copy directory was created
1396
+ if self.local_copy_dir:
1397
+ shutil.rmtree(self.local_copy_dir, ignore_errors=True)
1398
+
1399
+ def restore_model(
1400
+ self,
1401
+ restore_model_for_hours_specified: Optional[int] = None,
1402
+ ) -> None:
1403
+ """
1404
+ Restore archived model artifact.
1405
+
1406
+ Parameters
1407
+ ----------
1408
+
1409
+ restore_model_for_hours_specified : Optional[int]
1410
+ Duration in hours for which the archived model is available for access.
1411
+
1412
+ Returns
1413
+ -------
1414
+ None
1415
+
1416
+ Raises
1417
+ ------
1418
+ ValueError
1419
+ If the model ID is invalid or if any parameters are incorrect.
1420
+ """
1421
+ # Validate model_id
1422
+ if not self.id:
1423
+ logger.warn(
1424
+ "Model needs to be saved to the model catalog before it can be restored."
1425
+ )
1426
+ return
1427
+
1428
+ # Optional: Validate restore_model_for_hours_specified
1429
+ if restore_model_for_hours_specified is not None and (
1430
+ not isinstance(restore_model_for_hours_specified, int)
1431
+ or restore_model_for_hours_specified <= 0
1432
+ ):
1433
+ raise ValueError(
1434
+ "restore_model_for_hours_specified must be a positive integer."
1435
+ )
1436
+
1437
+ self.dsc_model.restore_archived_model_artifact(
1438
+ restore_model_for_hours_specified=restore_model_for_hours_specified,
1439
+ )
1440
+
1441
+ def download_artifact(
1442
+ self,
1443
+ target_dir: str,
1444
+ auth: Optional[Dict] = None,
1445
+ force_overwrite: Optional[bool] = False,
1446
+ bucket_uri: Optional[str] = None,
1447
+ region: Optional[str] = None,
1448
+ overwrite_existing_artifact: Optional[bool] = True,
1449
+ remove_existing_artifact: Optional[bool] = True,
1450
+ timeout: Optional[int] = None,
1451
+ ):
1452
+ """Downloads model artifacts from the model catalog.
1453
+
1454
+ Parameters
1455
+ ----------
1456
+ target_dir: str
1457
+ The target location of model artifacts.
1458
+ auth: (Dict, optional). Defaults to `None`.
1459
+ The default authentication is set using `ads.set_auth` API.
1460
+ If you need to override the default, use the `ads.common.auth.api_keys` or
1461
+ `ads.common.auth.resource_principal` to create appropriate authentication signer
1462
+ and kwargs required to instantiate IdentityClient object.
1463
+ force_overwrite: (bool, optional). Defaults to `False`.
1464
+ Overwrite target directory if exists.
1465
+ bucket_uri: (str, optional). Defaults to None.
1466
+ The OCI Object Storage URI where model artifacts will be copied to.
1467
+ The `bucket_uri` is only necessary for uploading large artifacts which
1468
+ size is greater than 2GB. Example: `oci://<bucket_name>@<namespace>/prefix/`.
1469
+ region: (str, optional). Defaults to `None`.
1470
+ The destination Object Storage bucket region.
1471
+ By default the value will be extracted from the `OCI_REGION_METADATA` environment variables.
1472
+ overwrite_existing_artifact: (bool, optional). Defaults to `True`.
1473
+ Overwrite target bucket artifact if exists.
1474
+ remove_existing_artifact: (bool, optional). Defaults to `True`.
1475
+ Wether artifacts uploaded to object storage bucket need to be removed or not.
1476
+ timeout: (int, optional). Defaults to 10 seconds.
1477
+ The connection timeout in seconds for the client.
1478
+
1479
+ Raises
1480
+ ------
1481
+ ModelArtifactSizeError
1482
+ If model artifacts size greater than 2GB and temporary OS bucket uri not provided.
1483
+ """
1484
+ # Upload artifact to the model catalog
1485
+ if not self.artifact:
1486
+ logger.warn(
1487
+ "Model doesn't contain an artifact. "
1488
+ "The artifact needs to be uploaded to the model catalog at first. "
1489
+ )
1490
+ return
1491
+
1492
+ if timeout:
1493
+ self.dsc_model._client = None
1494
+ self.dsc_model.__class__.kwargs = {
1495
+ **(self.dsc_model.__class__.kwargs or {}),
1496
+ "timeout": timeout,
1497
+ }
1498
+ try:
1499
+ model_by_reference = self.custom_metadata_list.get(
1500
+ self.CONST_MODEL_FILE_DESCRIPTION
1501
+ ).value
1502
+ logging.info(
1503
+ f"modelDescription tag found in custom metadata list with value {model_by_reference}"
1504
+ )
1505
+ except ValueError:
1506
+ model_by_reference = False
1507
+
1508
+ if model_by_reference:
1509
+ _, artifact_size = self._download_file_description_artifact()
1510
+ logging.warning(
1511
+ f"Model {self.dsc_model.id} was created by reference, artifacts will be downloaded from the bucket {bucket_uri}"
1512
+ )
1513
+ # artifacts will be downloaded from model_file_description
1514
+ bucket_uri = None
1515
+ else:
1516
+ artifact_info = self.dsc_model.get_artifact_info()
1517
+ artifact_size = int(artifact_info.get("content-length"))
1518
+
1519
+ if not bucket_uri and artifact_size > _MAX_ARTIFACT_SIZE_IN_BYTES:
1520
+ raise ModelArtifactSizeError(
1521
+ utils.human_size(_MAX_ARTIFACT_SIZE_IN_BYTES)
1522
+ )
1523
+
1524
+ if (
1525
+ artifact_size > _MAX_ARTIFACT_SIZE_IN_BYTES
1526
+ or bucket_uri
1527
+ or model_by_reference
1528
+ ):
1529
+ artifact_downloader = LargeArtifactDownloader(
1530
+ dsc_model=self.dsc_model,
1531
+ target_dir=target_dir,
1532
+ auth=auth,
1533
+ force_overwrite=force_overwrite,
1534
+ region=region,
1535
+ bucket_uri=bucket_uri,
1536
+ overwrite_existing_artifact=overwrite_existing_artifact,
1537
+ remove_existing_artifact=remove_existing_artifact,
1538
+ model_file_description=self.model_file_description,
1539
+ )
1540
+ else:
1541
+ artifact_downloader = SmallArtifactDownloader(
1542
+ dsc_model=self.dsc_model,
1543
+ target_dir=target_dir,
1544
+ force_overwrite=force_overwrite,
1545
+ )
1546
+ artifact_downloader.download()
1547
+
1548
+ def update(self, **kwargs) -> "DataScienceModel":
1549
+ """Updates datascience model in model catalog.
1550
+
1551
+ Parameters
1552
+ ----------
1553
+ kwargs
1554
+ Additional kwargs arguments.
1555
+ Can be any attribute that `oci.data_science.models.Model` accepts.
1556
+
1557
+ Returns
1558
+ -------
1559
+ DataScienceModel
1560
+ The DataScienceModel instance (self).
1561
+ """
1562
+ if not self.id:
1563
+ logger.warn(
1564
+ "Model needs to be saved to the model catalog before it can be updated."
1565
+ )
1566
+ return
1567
+
1568
+ logger.debug(f"Updating a model with payload {self._spec}")
1569
+ logger.info(f"Updating model {self.id} in the Model Catalog.")
1570
+ self.dsc_model = self._to_oci_dsc_model(**kwargs).update()
1571
+
1572
+ logger.debug(f"Updating a model provenance metadata {self.provenance_metadata}")
1573
+ if self.provenance_metadata:
1574
+ try:
1575
+ self.dsc_model.get_model_provenance()
1576
+ self.dsc_model.update_model_provenance(
1577
+ self.provenance_metadata._to_oci_metadata()
1578
+ )
1579
+ except ModelProvenanceNotFoundError:
1580
+ self.dsc_model.create_model_provenance(
1581
+ self.provenance_metadata._to_oci_metadata()
1582
+ )
1583
+
1584
+ return self.sync()
1585
+
1586
+ def delete(
1587
+ self,
1588
+ delete_associated_model_deployment: Optional[bool] = False,
1589
+ ) -> "DataScienceModel":
1590
+ """Removes model from the model catalog.
1591
+
1592
+ Parameters
1593
+ ----------
1594
+ delete_associated_model_deployment: (bool, optional). Defaults to `False`.
1595
+ Whether associated model deployments need to be deleted or not.
1596
+
1597
+ Returns
1598
+ -------
1599
+ DataScienceModel
1600
+ The DataScienceModel instance (self).
1601
+ """
1602
+ self.dsc_model.delete(delete_associated_model_deployment)
1603
+ return self.sync()
1604
+
1605
+ @classmethod
1606
+ def list(
1607
+ cls,
1608
+ compartment_id: str = None,
1609
+ project_id: str = None,
1610
+ category: str = USER,
1611
+ **kwargs,
1612
+ ) -> List["DataScienceModel"]:
1613
+ """Lists datascience models in a given compartment.
1614
+
1615
+ Parameters
1616
+ ----------
1617
+ compartment_id: (str, optional). Defaults to `None`.
1618
+ The compartment OCID.
1619
+ project_id: (str, optional). Defaults to `None`.
1620
+ The project OCID.
1621
+ category: (str, optional). Defaults to `USER`.
1622
+ The category of Model. Allowed values are: "USER", "SERVICE"
1623
+ kwargs
1624
+ Additional keyword arguments for filtering models.
1625
+
1626
+ Returns
1627
+ -------
1628
+ List[DataScienceModel]
1629
+ The list of the datascience models.
1630
+ """
1631
+ return [
1632
+ cls()._update_from_oci_dsc_model(model)
1633
+ for model in OCIDataScienceModel.list_resource(
1634
+ compartment_id, project_id=project_id, category=category, **kwargs
1635
+ )
1636
+ ]
1637
+
1638
+ @classmethod
1639
+ def list_df(
1640
+ cls,
1641
+ compartment_id: str = None,
1642
+ project_id: str = None,
1643
+ category: str = USER,
1644
+ **kwargs,
1645
+ ) -> "pandas.DataFrame":
1646
+ """Lists datascience models in a given compartment.
1647
+
1648
+ Parameters
1649
+ ----------
1650
+ compartment_id: (str, optional). Defaults to `None`.
1651
+ The compartment OCID.
1652
+ project_id: (str, optional). Defaults to `None`.
1653
+ The project OCID.
1654
+ category: (str, optional). Defaults to `None`.
1655
+ The category of Model.
1656
+ kwargs
1657
+ Additional keyword arguments for filtering models.
1658
+
1659
+ Returns
1660
+ -------
1661
+ pandas.DataFrame
1662
+ The list of the datascience models in a pandas dataframe format.
1663
+ """
1664
+ records = []
1665
+ for model in OCIDataScienceModel.list_resource(
1666
+ compartment_id, project_id=project_id, category=category, **kwargs
1667
+ ):
1668
+ records.append(
1669
+ {
1670
+ "id": f"...{model.id[-6:]}",
1671
+ "display_name": model.display_name,
1672
+ "description": model.description,
1673
+ "time_created": model.time_created.strftime(utils.date_format),
1674
+ "lifecycle_state": model.lifecycle_state,
1675
+ "created_by": f"...{model.created_by[-6:]}",
1676
+ "compartment_id": f"...{model.compartment_id[-6:]}",
1677
+ "project_id": f"...{model.project_id[-6:]}",
1678
+ }
1679
+ )
1680
+ return pandas.DataFrame.from_records(records)
1681
+
1682
+ @classmethod
1683
+ def from_id(cls, id: str) -> "DataScienceModel":
1684
+ """Gets an existing model by OCID.
1685
+
1686
+ Parameters
1687
+ ----------
1688
+ id: str
1689
+ The model OCID.
1690
+
1691
+ Returns
1692
+ -------
1693
+ DataScienceModel
1694
+ An instance of DataScienceModel.
1695
+ """
1696
+ return cls()._update_from_oci_dsc_model(OCIDataScienceModel.from_id(id))
1697
+
1698
+ def sync(self):
1699
+ """Sync up a datascience model with OCI datascience model."""
1700
+ return self._update_from_oci_dsc_model(OCIDataScienceModel.from_id(self.id))
1701
+
1702
+ def _init_complex_attributes(self):
1703
+ """Initiates complex attributes."""
1704
+ self.with_custom_metadata_list(self.custom_metadata_list)
1705
+ self.with_defined_metadata_list(self.defined_metadata_list)
1706
+ self.with_provenance_metadata(self.provenance_metadata)
1707
+ self.with_input_schema(self.input_schema)
1708
+ self.with_output_schema(self.output_schema)
1709
+
1710
+ def _to_oci_dsc_model(self, **kwargs):
1711
+ """Creates an `OCIDataScienceModel` instance from the `DataScienceModel`.
1712
+
1713
+ kwargs
1714
+ Additional kwargs arguments.
1715
+ Can be any attribute that `oci.data_science.models.Model` accepts.
1716
+
1717
+ Returns
1718
+ -------
1719
+ OCIDataScienceModel
1720
+ The instance of the OCIDataScienceModel.
1721
+ """
1722
+ COMPLEX_ATTRIBUTES_CONVERTER = {
1723
+ self.CONST_INPUT_SCHEMA: "to_json",
1724
+ self.CONST_OUTPUT_SCHEMA: "to_json",
1725
+ self.CONST_CUSTOM_METADATA: "_to_oci_metadata",
1726
+ self.CONST_DEFINED_METADATA: "_to_oci_metadata",
1727
+ self.CONST_PROVENANCE_METADATA: "_to_oci_metadata",
1728
+ }
1729
+ dsc_spec = {}
1730
+ for infra_attr, dsc_attr in self.attribute_map.items():
1731
+ value = self.get_spec(infra_attr)
1732
+ if infra_attr in COMPLEX_ATTRIBUTES_CONVERTER and value:
1733
+ if isinstance(value, dict):
1734
+ dsc_spec[dsc_attr] = json.dumps(value)
1735
+ else:
1736
+ dsc_spec[dsc_attr] = getattr(
1737
+ self.get_spec(infra_attr),
1738
+ COMPLEX_ATTRIBUTES_CONVERTER[infra_attr],
1739
+ )()
1740
+ else:
1741
+ dsc_spec[dsc_attr] = value
1742
+
1743
+ dsc_spec.update(**kwargs)
1744
+ return OCIDataScienceModel(**dsc_spec)
1745
+
1746
+ def _update_from_oci_dsc_model(
1747
+ self, dsc_model: OCIDataScienceModel
1748
+ ) -> "DataScienceModel":
1749
+ """Update the properties from an OCIDataScienceModel object.
1750
+
1751
+ Parameters
1752
+ ----------
1753
+ dsc_model: OCIDataScienceModel
1754
+ An instance of OCIDataScienceModel.
1755
+
1756
+ Returns
1757
+ -------
1758
+ DataScienceModel
1759
+ The DataScienceModel instance (self).
1760
+ """
1761
+ COMPLEX_ATTRIBUTES_CONVERTER = {
1762
+ self.CONST_INPUT_SCHEMA: [Schema.from_json, json.loads],
1763
+ self.CONST_OUTPUT_SCHEMA: [Schema.from_json, json.loads],
1764
+ self.CONST_CUSTOM_METADATA: ModelCustomMetadata._from_oci_metadata,
1765
+ self.CONST_DEFINED_METADATA: ModelTaxonomyMetadata._from_oci_metadata,
1766
+ self.CONST_BACKUP_SETTING: ModelBackupSetting.to_dict,
1767
+ self.CONST_RETENTION_SETTING: ModelRetentionSetting.to_dict,
1768
+ self.CONST_BACKUP_OPERATION_DETAILS: ModelBackupOperationDetails.to_dict,
1769
+ self.CONST_RETENTION_OPERATION_DETAILS: ModelRetentionOperationDetails.to_dict,
1770
+ }
1771
+
1772
+ # Update the main properties
1773
+ self.dsc_model = dsc_model
1774
+ for infra_attr, dsc_attr in self.attribute_map.items():
1775
+ value = utils.get_value(dsc_model, dsc_attr)
1776
+ if value:
1777
+ if infra_attr in COMPLEX_ATTRIBUTES_CONVERTER:
1778
+ converter = COMPLEX_ATTRIBUTES_CONVERTER[infra_attr]
1779
+ if isinstance(converter, List):
1780
+ for converter_item in converter:
1781
+ try:
1782
+ value = converter_item(value)
1783
+ except Exception as err:
1784
+ logger.warn(err)
1785
+ pass
1786
+ else:
1787
+ value = converter(value)
1788
+ self.set_spec(infra_attr, value)
1789
+
1790
+ # Update provenance metadata
1791
+ try:
1792
+ self.set_spec(
1793
+ self.CONST_PROVENANCE_METADATA,
1794
+ ModelProvenanceMetadata._from_oci_metadata(
1795
+ self.dsc_model.get_model_provenance()
1796
+ ),
1797
+ )
1798
+ except ModelProvenanceNotFoundError:
1799
+ pass
1800
+
1801
+ # Update artifact info
1802
+ try:
1803
+ artifact_info = self.dsc_model.get_artifact_info()
1804
+ _, file_name_info = utils.parse_content_disposition(
1805
+ artifact_info["Content-Disposition"]
1806
+ )
1807
+
1808
+ if self.dsc_model.is_model_created_by_reference():
1809
+ _, file_extension = os.path.splitext(file_name_info["filename"])
1810
+ if file_extension.lower() == ".json":
1811
+ bucket_uri, _ = self._download_file_description_artifact()
1812
+ self.set_spec(self.CONST_ARTIFACT, bucket_uri)
1813
+ else:
1814
+ self.set_spec(self.CONST_ARTIFACT, file_name_info["filename"])
1815
+ except:
1816
+ pass
1817
+ return self
1818
+
1819
+ def to_dict(self) -> Dict:
1820
+ """Serializes model to a dictionary.
1821
+
1822
+ Returns
1823
+ -------
1824
+ dict
1825
+ The model serialized as a dictionary.
1826
+ """
1827
+ spec = deepcopy(self._spec)
1828
+ for key, value in spec.items():
1829
+ if hasattr(value, "to_dict"):
1830
+ value = value.to_dict()
1831
+ spec[key] = value
1832
+
1833
+ return {
1834
+ "kind": self.kind,
1835
+ "type": self.type,
1836
+ "spec": utils.batch_convert_case(spec, "camel"),
1837
+ }
1838
+
1839
+ @classmethod
1840
+ def from_dict(cls, config: Dict) -> "DataScienceModel":
1841
+ """Loads model instance from a dictionary of configurations.
1842
+
1843
+ Parameters
1844
+ ----------
1845
+ config: Dict
1846
+ A dictionary of configurations.
1847
+
1848
+ Returns
1849
+ -------
1850
+ DataScienceModel
1851
+ The model instance.
1852
+ """
1853
+ return cls(spec=utils.batch_convert_case(deepcopy(config["spec"]), "snake"))
1854
+
1855
+ def _random_display_name(self):
1856
+ """Generates a random display name."""
1857
+ return f"{self._PREFIX}-{utils.get_random_name_for_resource()}"
1858
+
1859
+ def _load_default_properties(self) -> Dict:
1860
+ """Load default properties from environment variables, notebook session, etc.
1861
+
1862
+ Returns
1863
+ -------
1864
+ Dict
1865
+ A dictionary of default properties.
1866
+ """
1867
+ defaults = super()._load_default_properties()
1868
+ compartment_ocid = COMPARTMENT_OCID
1869
+ if compartment_ocid:
1870
+ defaults[self.CONST_COMPARTMENT_ID] = compartment_ocid
1871
+ if PROJECT_OCID:
1872
+ defaults[self.CONST_PROJECT_ID] = PROJECT_OCID
1873
+ defaults[self.CONST_DISPLAY_NAME] = self._random_display_name()
1874
+
1875
+ return defaults
1876
+
1877
+ def __getattr__(self, item):
1878
+ if f"with_{item}" in self.__dir__():
1879
+ return self.get_spec(item)
1880
+ raise AttributeError(f"Attribute {item} not found.")
1881
+
1882
+ def _validate_prepare_file_description_artifact(self):
1883
+ """This helper method validates the path to check if the buckets are versioned and if the OSS location and
1884
+ the files exist. Next, it creates a json dict with the path information and sets it as the artifact to be
1885
+ uploaded."""
1886
+
1887
+ if not self.model_file_description:
1888
+ bucket_uri = self.artifact
1889
+ if isinstance(bucket_uri, str):
1890
+ bucket_uri = [bucket_uri]
1891
+
1892
+ for uri in bucket_uri:
1893
+ os_path = ObjectStorageDetails.from_path(uri)
1894
+ # for aqua use case, user may not have access to the service bucket.
1895
+ if os_path.bucket == SERVICE_MODELS_BUCKET:
1896
+ continue
1897
+ if not os_path.is_bucket_versioned():
1898
+ message = f"Model artifact bucket {uri} is not versioned. Enable versioning on the bucket to proceed with model creation by reference."
1899
+ logger.error(message)
1900
+ raise BucketNotVersionedError(message)
1901
+
1902
+ json_data = self._prepare_file_description_artifact(bucket_uri)
1903
+ self.with_model_file_description(json_dict=json_data)
1904
+
1905
+ self.local_copy_dir = tempfile.mkdtemp()
1906
+ # create temp directory for model description file
1907
+ json_file_path = os.path.join(
1908
+ self.local_copy_dir, MODEL_BY_REFERENCE_JSON_FILE_NAME
1909
+ )
1910
+ with open(json_file_path, "w") as outfile:
1911
+ json.dump(self.model_file_description, outfile, indent=2)
1912
+
1913
+ self.with_artifact(json_file_path)
1914
+
1915
+ @staticmethod
1916
+ def _prepare_file_description_artifact(bucket_uri: list) -> dict:
1917
+ """Prepares yaml file config if model is passed by reference and uploaded to catalog.
1918
+
1919
+ Returns
1920
+ -------
1921
+ dict
1922
+ json dict with the model by reference artifact details
1923
+ """
1924
+
1925
+ # create json content
1926
+ content = dict()
1927
+ content["version"] = MODEL_BY_REFERENCE_VERSION
1928
+ content["type"] = "modelOSSReferenceDescription"
1929
+ content["models"] = []
1930
+
1931
+ for uri in bucket_uri:
1932
+ if not ObjectStorageDetails.is_oci_path(uri) or uri.endswith(".zip"):
1933
+ msg = "Artifact path cannot be a zip file or local directory for model creation by reference."
1934
+ logging.error(msg)
1935
+ raise InvalidArtifactType(msg)
1936
+
1937
+ # read list from objects from artifact location
1938
+ oss_details = ObjectStorageDetails.from_path(uri)
1939
+
1940
+ # first retrieve the etag and version id
1941
+ object_versions = oss_details.list_object_versions(fields="etag")
1942
+ version_dict = {
1943
+ obj.etag: obj.version_id
1944
+ for obj in object_versions
1945
+ if obj.etag is not None
1946
+ }
1947
+
1948
+ # add version id based on etag for each object
1949
+ objects = oss_details.list_objects(fields="name,etag,size").objects
1950
+
1951
+ if len(objects) == 0:
1952
+ raise ModelFileDescriptionError(
1953
+ f"The path {oss_details.path} does not exist or no objects were found in the path. "
1954
+ )
1955
+
1956
+ object_list = []
1957
+ for obj in objects:
1958
+ object_list.append(
1959
+ {
1960
+ "name": obj.name,
1961
+ "version": version_dict[obj.etag],
1962
+ "sizeInBytes": obj.size,
1963
+ }
1964
+ )
1965
+ content["models"].extend(
1966
+ [
1967
+ {
1968
+ "namespace": oss_details.namespace,
1969
+ "bucketName": oss_details.bucket,
1970
+ "prefix": oss_details.filepath,
1971
+ "objects": object_list,
1972
+ }
1973
+ ]
1974
+ )
1975
+
1976
+ return content
1977
+
1978
+ def _download_file_description_artifact(self) -> Tuple[Union[str, List[str]], int]:
1979
+ """Loads the json file from model artifact, updates the
1980
+ model file description property, and returns the bucket uri and artifact size details.
1981
+
1982
+ Returns
1983
+ -------
1984
+ bucket_uri: Union[str, List[str]]
1985
+ Location(s) of bucket where model artifacts are present
1986
+ artifact_size: int
1987
+ estimated size of the model files in bytes
1988
+
1989
+ """
1990
+ if not self.model_file_description:
1991
+ # get model file description from model artifact json
1992
+ with tempfile.TemporaryDirectory() as temp_dir:
1993
+ artifact_downloader = SmallArtifactDownloader(
1994
+ dsc_model=self.dsc_model,
1995
+ target_dir=temp_dir,
1996
+ )
1997
+ artifact_downloader.download()
1998
+ # create temp directory for model description file
1999
+ json_file_path = os.path.join(
2000
+ temp_dir, MODEL_BY_REFERENCE_JSON_FILE_NAME
2001
+ )
2002
+ self.with_model_file_description(json_uri=json_file_path)
2003
+
2004
+ model_file_desc_dict = self.model_file_description
2005
+ models = model_file_desc_dict["models"]
2006
+
2007
+ bucket_uri = list()
2008
+ artifact_size = 0
2009
+ for model in models:
2010
+ namespace = model["namespace"]
2011
+ bucket_name = model["bucketName"]
2012
+ prefix = model["prefix"]
2013
+ objects = model["objects"]
2014
+ uri = f"oci://{bucket_name}@{namespace}/{prefix}"
2015
+ artifact_size += sum([obj["sizeInBytes"] for obj in objects])
2016
+ bucket_uri.append(uri)
2017
+
2018
+ return bucket_uri[0] if len(bucket_uri) == 1 else bucket_uri, artifact_size
2019
+
2020
+ def add_artifact(
2021
+ self,
2022
+ uri: Optional[str] = None,
2023
+ namespace: Optional[str] = None,
2024
+ bucket: Optional[str] = None,
2025
+ prefix: Optional[str] = None,
2026
+ files: Optional[List[str]] = None,
2027
+ ):
2028
+ """
2029
+ Adds information about objects in a specified bucket to the model description JSON.
2030
+
2031
+ Parameters
2032
+ ----------
2033
+ uri : str, optional
2034
+ The URI representing the location of the artifact in OCI object storage.
2035
+ namespace : str, optional
2036
+ The namespace of the bucket containing the objects. Required if `uri` is not provided.
2037
+ bucket : str, optional
2038
+ The name of the bucket containing the objects. Required if `uri` is not provided.
2039
+ prefix : str, optional
2040
+ The prefix of the objects to add. Defaults to None. Cannot be provided if `files` is provided.
2041
+ files : list of str, optional
2042
+ A list of file names to include in the model description. If provided, only objects with matching file names will be included. Cannot be provided if `prefix` is provided.
2043
+
2044
+ Returns
2045
+ -------
2046
+ None
2047
+
2048
+ Raises
2049
+ ------
2050
+ ValueError
2051
+ - If both `uri` and (`namespace` and `bucket`) are provided.
2052
+ - If neither `uri` nor both `namespace` and `bucket` are provided.
2053
+ - If both `prefix` and `files` are provided.
2054
+ - If no files are found to add to the model description.
2055
+
2056
+ Note
2057
+ ----
2058
+ - If `files` is not provided, it retrieves information about all objects in the bucket.
2059
+ - If `files` is provided, it only retrieves information about objects with matching file names.
2060
+ - If no objects are found to add to the model description, a ValueError is raised.
2061
+ """
2062
+
2063
+ if uri and (namespace or bucket):
2064
+ raise ValueError(
2065
+ "Either 'uri' must be provided or both 'namespace' and 'bucket' must be provided."
2066
+ )
2067
+ if uri:
2068
+ object_storage_details = ObjectStorageDetails.from_path(uri)
2069
+ bucket = object_storage_details.bucket
2070
+ namespace = object_storage_details.namespace
2071
+ prefix = (
2072
+ None
2073
+ if object_storage_details.filepath == ""
2074
+ else object_storage_details.filepath
2075
+ )
2076
+ if (not namespace) or (not bucket):
2077
+ raise ValueError("Both 'namespace' and 'bucket' must be provided.")
2078
+
2079
+ # Check if both prefix and files are provided
2080
+ if prefix is not None and files is not None:
2081
+ raise ValueError(
2082
+ "Both 'prefix' and 'files' cannot be provided. Please provide only one."
2083
+ )
2084
+
2085
+ if self.model_file_description is None:
2086
+ self.empty_json = {
2087
+ "version": "1.0",
2088
+ "type": "modelOSSReferenceDescription",
2089
+ "models": [],
2090
+ }
2091
+ self.set_spec(self.CONST_MODEL_FILE_DESCRIPTION, self.empty_json)
2092
+
2093
+ # Get object storage client
2094
+ self.object_storage_client = oc.OCIClientFactory(
2095
+ **(self.dsc_model.auth)
2096
+ ).object_storage
2097
+
2098
+ # Remove if the model already exists
2099
+ self.remove_artifact(namespace=namespace, bucket=bucket, prefix=prefix)
2100
+
2101
+ def check_if_file_exists(fileName):
2102
+ isExists = False
2103
+ try:
2104
+ headResponse = self.object_storage_client.head_object(
2105
+ namespace, bucket, object_name=fileName
2106
+ )
2107
+ if headResponse.status == 200:
2108
+ isExists = True
2109
+ except Exception as e:
2110
+ if hasattr(e, "status") and e.status == 404:
2111
+ logger.error(f"File not found in bucket: {fileName}")
2112
+ else:
2113
+ logger.error(f"An error occured: {e}")
2114
+ return isExists
2115
+
2116
+ # Function to un-paginate the api call with while loop
2117
+ def list_obj_versions_unpaginated():
2118
+ objectStorageList = []
2119
+ has_next_page, opc_next_page = True, None
2120
+ while has_next_page:
2121
+ response = self.object_storage_client.list_object_versions(
2122
+ namespace_name=namespace,
2123
+ bucket_name=bucket,
2124
+ prefix=prefix,
2125
+ fields="name,size",
2126
+ page=opc_next_page,
2127
+ )
2128
+ objectStorageList.extend(response.data.items)
2129
+ has_next_page = response.has_next_page
2130
+ opc_next_page = response.next_page
2131
+ return objectStorageList
2132
+
2133
+ # Fetch object details and put it into the objects variable
2134
+ objectStorageList = []
2135
+ if files is None:
2136
+ objectStorageList = list_obj_versions_unpaginated()
2137
+ else:
2138
+ for fileName in files:
2139
+ if check_if_file_exists(fileName=fileName):
2140
+ objectStorageList.append(
2141
+ self.object_storage_client.list_object_versions(
2142
+ namespace_name=namespace,
2143
+ bucket_name=bucket,
2144
+ prefix=fileName,
2145
+ fields="name,size",
2146
+ ).data.items[0]
2147
+ )
2148
+
2149
+ objects = [
2150
+ {"name": obj.name, "version": obj.version_id, "sizeInBytes": obj.size}
2151
+ for obj in objectStorageList
2152
+ if obj.size > 0
2153
+ ]
2154
+
2155
+ if len(objects) == 0:
2156
+ error_message = (
2157
+ f"No files to add in the bucket: {bucket} with namespace: {namespace} "
2158
+ f"and prefix: {prefix}. File names: {files}"
2159
+ )
2160
+ logger.error(error_message)
2161
+ raise ValueError(error_message)
2162
+
2163
+ tmp_model_file_description = self.model_file_description
2164
+ tmp_model_file_description["models"].append(
2165
+ {
2166
+ "namespace": namespace,
2167
+ "bucketName": bucket,
2168
+ "prefix": "" if not prefix else prefix,
2169
+ "objects": objects,
2170
+ }
2171
+ )
2172
+ self.set_spec(self.CONST_MODEL_FILE_DESCRIPTION, tmp_model_file_description)
2173
+
2174
+ def remove_artifact(
2175
+ self,
2176
+ uri: Optional[str] = None,
2177
+ namespace: Optional[str] = None,
2178
+ bucket: Optional[str] = None,
2179
+ prefix: Optional[str] = None,
2180
+ ):
2181
+ """
2182
+ Removes information about objects in a specified bucket or using a specified URI from the model description JSON.
2183
+
2184
+ Parameters
2185
+ ----------
2186
+ uri : str, optional
2187
+ The URI representing the location of the artifact in OCI object storage.
2188
+ namespace : str, optional
2189
+ The namespace of the bucket containing the objects. Required if `uri` is not provided.
2190
+ bucket : str, optional
2191
+ The name of the bucket containing the objects. Required if `uri` is not provided.
2192
+ prefix : str, optional
2193
+ The prefix of the objects to remove. Defaults to None.
2194
+
2195
+ Returns
2196
+ -------
2197
+ None
2198
+
2199
+ Raises
2200
+ ------
2201
+ ValueError
2202
+ - If both 'uri' and ('namespace' and 'bucket') are provided.
2203
+ - If neither 'uri' nor both 'namespace' and 'bucket' are provided.
2204
+ - If the model description JSON is None.
2205
+ """
2206
+
2207
+ if uri and (namespace or bucket):
2208
+ raise ValueError(
2209
+ "Either 'uri' must be provided or both 'namespace' and 'bucket' must be provided."
2210
+ )
2211
+ if uri:
2212
+ object_storage_details = ObjectStorageDetails.from_path(uri)
2213
+ bucket = object_storage_details.bucket
2214
+ namespace = object_storage_details.namespace
2215
+ prefix = (
2216
+ None
2217
+ if object_storage_details.filepath == ""
2218
+ else object_storage_details.filepath
2219
+ )
2220
+ if (not namespace) or (not bucket):
2221
+ raise ValueError("Both 'namespace' and 'bucket' must be provided.")
2222
+
2223
+ def find_model_idx():
2224
+ for idx, model in enumerate(self.model_file_description["models"]):
2225
+ if (
2226
+ model["namespace"],
2227
+ model["bucketName"],
2228
+ (model["prefix"] if ("prefix" in model) else None),
2229
+ ) == (namespace, bucket, "" if not prefix else prefix):
2230
+ return idx
2231
+ return -1
2232
+
2233
+ if self.model_file_description is None:
2234
+ return
2235
+
2236
+ modelSearchIdx = find_model_idx()
2237
+ if modelSearchIdx == -1:
2238
+ return
2239
+ else:
2240
+ # model found case
2241
+ self.model_file_description["models"].pop(modelSearchIdx)
2242
+
2243
+ def if_model_custom_metadata_artifact_exist(
2244
+ self, metadata_key_name: str, **kwargs
2245
+ ) -> bool:
2246
+ """Checks if the custom metadata artifact exists for the model.
2247
+
2248
+ Parameters
2249
+ ----------
2250
+ metadata_key_name: str
2251
+ Custom metadata key name
2252
+ **kwargs :
2253
+ Additional keyword arguments passed in head_model_artifact.
2254
+
2255
+ Returns
2256
+ -------
2257
+ bool
2258
+ Whether the artifact exists.
2259
+ """
2260
+
2261
+ try:
2262
+ response = self.dsc_model.head_custom_metadata_artifact(
2263
+ metadata_key_name=metadata_key_name, **kwargs
2264
+ )
2265
+ return int(response.status) == 200
2266
+ except Exception as ex:
2267
+ logger.info(
2268
+ f"Error fetching custom metadata: {metadata_key_name} for model {self.id}. {ex}"
2269
+ )
2270
+ return False
2271
+
2272
+ def create_custom_metadata_artifact(
2273
+ self,
2274
+ metadata_key_name: str,
2275
+ artifact_path_or_content: Union[str, bytes],
2276
+ path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
2277
+ ) -> ModelMetadataArtifactDetails:
2278
+ """Creates model custom metadata artifact for specified model.
2279
+
2280
+ Parameters
2281
+ ----------
2282
+ metadata_key_name: str
2283
+ The name of the model custom metadata key
2284
+
2285
+ artifact_path_or_content: Union[str,bytes]
2286
+ The model custom metadata artifact path to be uploaded. It can also be the actual content of the custom metadata artifact
2287
+ The type is string when it represents local path or oss path.
2288
+ The type is bytes when it represents content itself
2289
+
2290
+ path_type: MetadataArtifactPathType
2291
+ Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
2292
+ Specifies what type of path is to be provided for metadata artifact.
2293
+
2294
+ Example:
2295
+ >>> ds_model=DataScienceModel.from_id("ocid1.datasciencemodel.iad.xxyxz...")
2296
+ >>> ds_model.create_custom_metadata_artifact(
2297
+ ... "README",
2298
+ ... artifact_path_or_content="/Users/<username>/Downloads/README.md",
2299
+ ... path_type=MetadataArtifactPathType.LOCAL
2300
+ ... )
2301
+
2302
+ Returns
2303
+ -------
2304
+ ModelMetadataArtifactDetails
2305
+ The model custom metadata artifact creation info.
2306
+ Example:
2307
+ {
2308
+ 'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
2309
+ 'opc-request-id': 'E4F7',
2310
+ 'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
2311
+ 'X-Content-Type-Options': 'nosniff',
2312
+ 'Content-Length': '4029958',
2313
+ 'Vary': 'Origin',
2314
+ 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
2315
+ 'status': 204
2316
+ }
2317
+
2318
+ """
2319
+ if path_type == MetadataArtifactPathType.CONTENT and not isinstance(
2320
+ artifact_path_or_content, bytes
2321
+ ):
2322
+ raise InvalidArtifactPathTypeOrContentError(
2323
+ f"Invalid type of artifact content: {type(artifact_path_or_content)}. It should be bytes."
2324
+ )
2325
+
2326
+ return self.dsc_model.create_custom_metadata_artifact(
2327
+ metadata_key_name=metadata_key_name,
2328
+ artifact_path_or_content=artifact_path_or_content,
2329
+ path_type=path_type,
2330
+ )
2331
+
2332
+ def create_defined_metadata_artifact(
2333
+ self,
2334
+ metadata_key_name: str,
2335
+ artifact_path_or_content: Union[str, bytes],
2336
+ path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
2337
+ ) -> ModelMetadataArtifactDetails:
2338
+ """Creates model defined metadata artifact for specified model.
2339
+
2340
+ Parameters
2341
+ ----------
2342
+ metadata_key_name: str
2343
+ The name of the model defined metadata key
2344
+
2345
+ artifact_path_or_content: Union[str,bytes]
2346
+ The model defined metadata artifact path to be uploaded. It can also be the actual content of the defined metadata
2347
+ The type is string when it represents local path or oss path.
2348
+ The type is bytes when it represents content itself
2349
+
2350
+ path_type: MetadataArtifactPathType
2351
+ Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
2352
+ Specifies what type of path is to be provided for metadata artifact.
2353
+ Can be either local , oss or the actual content itself
2354
+
2355
+ Example:
2356
+ >>> ds_model=DataScienceModel.from_id("ocid1.datasciencemodel.iad.xxyxz...")
2357
+ >>> ds_model.create_defined_metadata_artifact(
2358
+ ... "README",
2359
+ ... artifact_path_or_content="oci://path/to/bucket/README.md",
2360
+ ... path_type=MetadataArtifactPathType.OSS
2361
+ ... )
2362
+
2363
+ Returns
2364
+ -------
2365
+ ModelMetadataArtifactDetails
2366
+ The model defined metadata artifact creation info.
2367
+ Example:
2368
+ {
2369
+ 'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
2370
+ 'opc-request-id': 'E4F7',
2371
+ 'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
2372
+ 'X-Content-Type-Options': 'nosniff',
2373
+ 'Content-Length': '4029958',
2374
+ 'Vary': 'Origin',
2375
+ 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
2376
+ 'status': 204
2377
+ }
2378
+
2379
+ """
2380
+ if path_type == MetadataArtifactPathType.CONTENT and not isinstance(
2381
+ artifact_path_or_content, bytes
2382
+ ):
2383
+ raise InvalidArtifactPathTypeOrContentError(
2384
+ f"Invalid type of artifact content: {type(artifact_path_or_content)}. It should be bytes."
2385
+ )
2386
+
2387
+ return self.dsc_model.create_defined_metadata_artifact(
2388
+ metadata_key_name=metadata_key_name,
2389
+ artifact_path_or_content=artifact_path_or_content,
2390
+ path_type=path_type,
2391
+ )
2392
+
2393
+ def update_custom_metadata_artifact(
2394
+ self,
2395
+ metadata_key_name: str,
2396
+ artifact_path_or_content: Union[str, bytes],
2397
+ path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
2398
+ ) -> ModelMetadataArtifactDetails:
2399
+ """Update model custom metadata artifact for specified model.
2400
+
2401
+ Parameters
2402
+ ----------
2403
+ metadata_key_name: str
2404
+ The name of the model custom metadata key
2405
+
2406
+ artifact_path_or_content: Union[str,bytes]
2407
+ The model custom metadata artifact path to be uploaded. It can also be the actual content of the custom metadata
2408
+ The type is string when it represents local path or oss path.
2409
+ The type is bytes when it represents content itself
2410
+
2411
+ path_type: MetadataArtifactPathType
2412
+ Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
2413
+ Specifies what type of path is to be provided for metadata artifact.
2414
+ Can be either local , oss or the actual content itself
2415
+
2416
+ Returns
2417
+ -------
2418
+ ModelMetadataArtifactDetails
2419
+ The model custom metadata artifact update info.
2420
+ Example:
2421
+ {
2422
+ 'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
2423
+ 'opc-request-id': 'E4F7',
2424
+ 'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
2425
+ 'X-Content-Type-Options': 'nosniff',
2426
+ 'Content-Length': '4029958',
2427
+ 'Vary': 'Origin',
2428
+ 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
2429
+ 'status': 204
2430
+ }
2431
+
2432
+ """
2433
+ if path_type == MetadataArtifactPathType.CONTENT and not isinstance(
2434
+ artifact_path_or_content, bytes
2435
+ ):
2436
+ raise InvalidArtifactPathTypeOrContentError(
2437
+ f"Invalid type of artifact content: {type(artifact_path_or_content)}. It should be bytes."
2438
+ )
2439
+
2440
+ return self.dsc_model.update_custom_metadata_artifact(
2441
+ metadata_key_name=metadata_key_name,
2442
+ artifact_path_or_content=artifact_path_or_content,
2443
+ path_type=path_type,
2444
+ )
2445
+
2446
+ def update_defined_metadata_artifact(
2447
+ self,
2448
+ metadata_key_name: str,
2449
+ artifact_path_or_content: Union[str, bytes],
2450
+ path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
2451
+ ) -> ModelMetadataArtifactDetails:
2452
+ """Update model defined metadata artifact for specified model.
2453
+
2454
+ Parameters
2455
+ ----------
2456
+ metadata_key_name: str
2457
+ The name of the model defined metadata key
2458
+
2459
+ artifact_path_or_content: Union[str,bytes]
2460
+ The model defined metadata artifact path to be uploaded. It can also be the actual content of the defined metadata
2461
+ The type is string when it represents local path or oss path.
2462
+ The type is bytes when it represents content itself
2463
+
2464
+ path_type: MetadataArtifactPathType
2465
+ Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
2466
+ Specifies what type of path is to be provided for metadata artifact.
2467
+ Can be either local , oss or the actual content itself
2468
+
2469
+ Returns
2470
+ -------
2471
+ ModelMetadataArtifactDetails
2472
+ The model defined metadata artifact update info.
2473
+ Example:
2474
+ {
2475
+ 'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
2476
+ 'opc-request-id': 'E4F7',
2477
+ 'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
2478
+ 'X-Content-Type-Options': 'nosniff',
2479
+ 'Content-Length': '4029958',
2480
+ 'Vary': 'Origin',
2481
+ 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
2482
+ 'status': 204
2483
+ }
2484
+
2485
+ """
2486
+ if path_type == MetadataArtifactPathType.CONTENT and not isinstance(
2487
+ artifact_path_or_content, bytes
2488
+ ):
2489
+ raise InvalidArtifactPathTypeOrContentError(
2490
+ f"Invalid type of artifact content: {type(artifact_path_or_content)}. It should be bytes."
2491
+ )
2492
+
2493
+ return self.dsc_model.update_defined_metadata_artifact(
2494
+ metadata_key_name=metadata_key_name,
2495
+ artifact_path_or_content=artifact_path_or_content,
2496
+ path_type=path_type,
2497
+ )
2498
+
2499
+ def get_custom_metadata_artifact(
2500
+ self, metadata_key_name: str, target_dir: str, override: bool = False
2501
+ ) -> bytes:
2502
+ """Downloads model custom metadata artifact content for specified model metadata key.
2503
+
2504
+ Parameters
2505
+ ----------
2506
+ metadata_key_name: str
2507
+ The name of the custom metadata key of the model
2508
+
2509
+ target_dir: str
2510
+ The local file path where downloaded model custom metadata artifact will be saved.
2511
+
2512
+ override: bool
2513
+ A boolean flag that controls downloaded metadata artifact file overwriting
2514
+ - If True, overwrites the file if it already exists.
2515
+ - If False (default), raises a `FileExistsError` if the file exists.
2516
+ Returns
2517
+ -------
2518
+ bytes
2519
+ File content of the custom metadata artifact
2520
+
2521
+ """
2522
+ if not is_path_exists(target_dir):
2523
+ raise PathNotFoundError(f"Path : {target_dir} does not exist")
2524
+
2525
+ file_content = self.dsc_model.get_custom_metadata_artifact(
2526
+ metadata_key_name=metadata_key_name
2527
+ )
2528
+ artifact_file_path = os.path.join(target_dir, f"{metadata_key_name}")
2529
+
2530
+ if not override and is_path_exists(artifact_file_path):
2531
+ raise FileExistsError(
2532
+ f"File already exists: {artifact_file_path}. Please use boolean override parameter to override the file content."
2533
+ )
2534
+
2535
+ with open(artifact_file_path, "wb") as _file:
2536
+ _file.write(file_content)
2537
+ logger.debug(f"Artifact downloaded to location - {artifact_file_path}")
2538
+ return file_content
2539
+
2540
+ def get_defined_metadata_artifact(
2541
+ self, metadata_key_name: str, target_dir: str, override: bool = False
2542
+ ) -> bytes:
2543
+ """Downloads model defined metadata artifact content for specified model metadata key.
2544
+
2545
+ Parameters
2546
+ ----------
2547
+ metadata_key_name: str
2548
+ The name of the model metadatum in the metadata.
2549
+
2550
+ target_dir: str
2551
+ The local file path where downloaded model defined metadata artifact will be saved.
2552
+
2553
+ override: bool
2554
+ A boolean flag that controls downloaded metadata artifact file overwriting
2555
+ - If True, overwrites the file if it already exists.
2556
+ - If False (default), raises a `FileExistsError` if the file exists.
2557
+ Returns
2558
+ -------
2559
+ bytes
2560
+ File content of the custom metadata artifact
2561
+
2562
+ """
2563
+ if not is_path_exists(target_dir):
2564
+ raise PathNotFoundError(f"Path : {target_dir} does not exist")
2565
+
2566
+ file_content = self.dsc_model.get_defined_metadata_artifact(
2567
+ metadata_key_name=metadata_key_name
2568
+ )
2569
+ artifact_file_path = os.path.join(target_dir, f"{metadata_key_name}")
2570
+
2571
+ if not override and is_path_exists(artifact_file_path):
2572
+ raise FileExistsError(
2573
+ f"File already exists: {artifact_file_path}. Please use boolean override parameter to override the file content."
2574
+ )
2575
+
2576
+ with open(artifact_file_path, "wb") as _file:
2577
+ _file.write(file_content)
2578
+ logger.debug(f"Artifact downloaded to location - {artifact_file_path}")
2579
+ return file_content
2580
+
2581
+ def delete_custom_metadata_artifact(
2582
+ self, metadata_key_name: str
2583
+ ) -> ModelMetadataArtifactDetails:
2584
+ """Deletes model custom metadata artifact for specified model metadata key.
2585
+
2586
+ Parameters
2587
+ ----------
2588
+ metadata_key_name: str
2589
+ The name of the model metadatum in the metadata.
2590
+ Returns
2591
+ -------
2592
+ ModelMetadataArtifactDetails
2593
+ The model custom metadata artifact delete call info.
2594
+ Example:
2595
+ {
2596
+ 'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
2597
+ 'opc-request-id': 'E4F7',
2598
+ 'X-Content-Type-Options': 'nosniff',
2599
+ 'Vary': 'Origin',
2600
+ 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
2601
+ 'status': 204
2602
+ }
2603
+
2604
+ """
2605
+ return self.dsc_model.delete_custom_metadata_artifact(
2606
+ metadata_key_name=metadata_key_name
2607
+ )
2608
+
2609
+ def delete_defined_metadata_artifact(
2610
+ self, metadata_key_name: str
2611
+ ) -> ModelMetadataArtifactDetails:
2612
+ """Deletes model defined metadata artifact for specified model metadata key.
2613
+
2614
+ Parameters
2615
+ ----------
2616
+ metadata_key_name: str
2617
+ The name of the model metadatum in the metadata.
2618
+ Returns
2619
+ -------
2620
+ ModelMetadataArtifactDetails
2621
+ The model defined metadata artifact delete call info.
2622
+ Example:
2623
+ {
2624
+ 'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
2625
+ 'opc-request-id': 'E4F7',
2626
+ 'X-Content-Type-Options': 'nosniff',
2627
+ 'Vary': 'Origin',
2628
+ 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
2629
+ 'status': 204
2630
+ }
2631
+
2632
+ """
2633
+ return self.dsc_model.delete_defined_metadata_artifact(
2634
+ metadata_key_name=metadata_key_name
2635
+ )