oracle-ads 2.13.8__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (857) hide show
  1. {oracle_ads-2.13.8.dist-info → oracle_ads-2.13.9rc0.dist-info}/METADATA +151 -151
  2. oracle_ads-2.13.9rc0.dist-info/RECORD +9 -0
  3. {oracle_ads-2.13.8.dist-info → oracle_ads-2.13.9rc0.dist-info}/WHEEL +2 -1
  4. {oracle_ads-2.13.8.dist-info → oracle_ads-2.13.9rc0.dist-info}/entry_points.txt +1 -2
  5. oracle_ads-2.13.9rc0.dist-info/top_level.txt +1 -0
  6. ads/aqua/__init__.py +0 -40
  7. ads/aqua/app.py +0 -506
  8. ads/aqua/cli.py +0 -96
  9. ads/aqua/client/__init__.py +0 -3
  10. ads/aqua/client/client.py +0 -836
  11. ads/aqua/client/openai_client.py +0 -305
  12. ads/aqua/common/__init__.py +0 -5
  13. ads/aqua/common/decorator.py +0 -125
  14. ads/aqua/common/entities.py +0 -266
  15. ads/aqua/common/enums.py +0 -122
  16. ads/aqua/common/errors.py +0 -109
  17. ads/aqua/common/utils.py +0 -1285
  18. ads/aqua/config/__init__.py +0 -4
  19. ads/aqua/config/container_config.py +0 -248
  20. ads/aqua/config/evaluation/__init__.py +0 -4
  21. ads/aqua/config/evaluation/evaluation_service_config.py +0 -147
  22. ads/aqua/config/utils/__init__.py +0 -4
  23. ads/aqua/config/utils/serializer.py +0 -339
  24. ads/aqua/constants.py +0 -114
  25. ads/aqua/data.py +0 -14
  26. ads/aqua/dummy_data/icon.txt +0 -1
  27. ads/aqua/dummy_data/oci_model_deployments.json +0 -56
  28. ads/aqua/dummy_data/oci_models.json +0 -1
  29. ads/aqua/dummy_data/readme.md +0 -26
  30. ads/aqua/evaluation/__init__.py +0 -8
  31. ads/aqua/evaluation/constants.py +0 -53
  32. ads/aqua/evaluation/entities.py +0 -186
  33. ads/aqua/evaluation/errors.py +0 -70
  34. ads/aqua/evaluation/evaluation.py +0 -1814
  35. ads/aqua/extension/__init__.py +0 -42
  36. ads/aqua/extension/aqua_ws_msg_handler.py +0 -76
  37. ads/aqua/extension/base_handler.py +0 -90
  38. ads/aqua/extension/common_handler.py +0 -121
  39. ads/aqua/extension/common_ws_msg_handler.py +0 -36
  40. ads/aqua/extension/deployment_handler.py +0 -298
  41. ads/aqua/extension/deployment_ws_msg_handler.py +0 -54
  42. ads/aqua/extension/errors.py +0 -30
  43. ads/aqua/extension/evaluation_handler.py +0 -129
  44. ads/aqua/extension/evaluation_ws_msg_handler.py +0 -61
  45. ads/aqua/extension/finetune_handler.py +0 -96
  46. ads/aqua/extension/model_handler.py +0 -390
  47. ads/aqua/extension/models/__init__.py +0 -0
  48. ads/aqua/extension/models/ws_models.py +0 -145
  49. ads/aqua/extension/models_ws_msg_handler.py +0 -50
  50. ads/aqua/extension/ui_handler.py +0 -282
  51. ads/aqua/extension/ui_websocket_handler.py +0 -130
  52. ads/aqua/extension/utils.py +0 -133
  53. ads/aqua/finetuning/__init__.py +0 -7
  54. ads/aqua/finetuning/constants.py +0 -23
  55. ads/aqua/finetuning/entities.py +0 -181
  56. ads/aqua/finetuning/finetuning.py +0 -731
  57. ads/aqua/model/__init__.py +0 -8
  58. ads/aqua/model/constants.py +0 -60
  59. ads/aqua/model/entities.py +0 -306
  60. ads/aqua/model/enums.py +0 -30
  61. ads/aqua/model/model.py +0 -2080
  62. ads/aqua/modeldeployment/__init__.py +0 -8
  63. ads/aqua/modeldeployment/constants.py +0 -10
  64. ads/aqua/modeldeployment/deployment.py +0 -1324
  65. ads/aqua/modeldeployment/entities.py +0 -653
  66. ads/aqua/modeldeployment/inference.py +0 -74
  67. ads/aqua/modeldeployment/utils.py +0 -543
  68. ads/aqua/resources/gpu_shapes_index.json +0 -94
  69. ads/aqua/server/__init__.py +0 -4
  70. ads/aqua/server/__main__.py +0 -24
  71. ads/aqua/server/app.py +0 -47
  72. ads/aqua/server/aqua_spec.yml +0 -1291
  73. ads/aqua/training/__init__.py +0 -4
  74. ads/aqua/training/exceptions.py +0 -476
  75. ads/aqua/ui.py +0 -499
  76. ads/automl/__init__.py +0 -9
  77. ads/automl/driver.py +0 -330
  78. ads/automl/provider.py +0 -975
  79. ads/bds/__init__.py +0 -5
  80. ads/bds/auth.py +0 -127
  81. ads/bds/big_data_service.py +0 -255
  82. ads/catalog/__init__.py +0 -19
  83. ads/catalog/model.py +0 -1576
  84. ads/catalog/notebook.py +0 -461
  85. ads/catalog/project.py +0 -468
  86. ads/catalog/summary.py +0 -178
  87. ads/common/__init__.py +0 -11
  88. ads/common/analyzer.py +0 -65
  89. ads/common/artifact/.model-ignore +0 -63
  90. ads/common/artifact/__init__.py +0 -10
  91. ads/common/auth.py +0 -1122
  92. ads/common/card_identifier.py +0 -83
  93. ads/common/config.py +0 -647
  94. ads/common/data.py +0 -165
  95. ads/common/decorator/__init__.py +0 -9
  96. ads/common/decorator/argument_to_case.py +0 -88
  97. ads/common/decorator/deprecate.py +0 -69
  98. ads/common/decorator/require_nonempty_arg.py +0 -65
  99. ads/common/decorator/runtime_dependency.py +0 -178
  100. ads/common/decorator/threaded.py +0 -97
  101. ads/common/decorator/utils.py +0 -35
  102. ads/common/dsc_file_system.py +0 -303
  103. ads/common/error.py +0 -14
  104. ads/common/extended_enum.py +0 -81
  105. ads/common/function/__init__.py +0 -5
  106. ads/common/function/fn_util.py +0 -142
  107. ads/common/function/func_conf.yaml +0 -25
  108. ads/common/ipython.py +0 -76
  109. ads/common/model.py +0 -679
  110. ads/common/model_artifact.py +0 -1759
  111. ads/common/model_artifact_schema.json +0 -107
  112. ads/common/model_export_util.py +0 -664
  113. ads/common/model_metadata.py +0 -24
  114. ads/common/object_storage_details.py +0 -296
  115. ads/common/oci_client.py +0 -175
  116. ads/common/oci_datascience.py +0 -46
  117. ads/common/oci_logging.py +0 -1144
  118. ads/common/oci_mixin.py +0 -957
  119. ads/common/oci_resource.py +0 -136
  120. ads/common/serializer.py +0 -559
  121. ads/common/utils.py +0 -1852
  122. ads/common/word_lists.py +0 -1491
  123. ads/common/work_request.py +0 -189
  124. ads/data_labeling/__init__.py +0 -13
  125. ads/data_labeling/boundingbox.py +0 -253
  126. ads/data_labeling/constants.py +0 -47
  127. ads/data_labeling/data_labeling_service.py +0 -244
  128. ads/data_labeling/interface/__init__.py +0 -5
  129. ads/data_labeling/interface/loader.py +0 -16
  130. ads/data_labeling/interface/parser.py +0 -16
  131. ads/data_labeling/interface/reader.py +0 -23
  132. ads/data_labeling/loader/__init__.py +0 -5
  133. ads/data_labeling/loader/file_loader.py +0 -241
  134. ads/data_labeling/metadata.py +0 -110
  135. ads/data_labeling/mixin/__init__.py +0 -5
  136. ads/data_labeling/mixin/data_labeling.py +0 -232
  137. ads/data_labeling/ner.py +0 -129
  138. ads/data_labeling/parser/__init__.py +0 -5
  139. ads/data_labeling/parser/dls_record_parser.py +0 -388
  140. ads/data_labeling/parser/export_metadata_parser.py +0 -94
  141. ads/data_labeling/parser/export_record_parser.py +0 -473
  142. ads/data_labeling/reader/__init__.py +0 -5
  143. ads/data_labeling/reader/dataset_reader.py +0 -574
  144. ads/data_labeling/reader/dls_record_reader.py +0 -121
  145. ads/data_labeling/reader/export_record_reader.py +0 -62
  146. ads/data_labeling/reader/jsonl_reader.py +0 -75
  147. ads/data_labeling/reader/metadata_reader.py +0 -203
  148. ads/data_labeling/reader/record_reader.py +0 -263
  149. ads/data_labeling/record.py +0 -52
  150. ads/data_labeling/visualizer/__init__.py +0 -5
  151. ads/data_labeling/visualizer/image_visualizer.py +0 -525
  152. ads/data_labeling/visualizer/text_visualizer.py +0 -357
  153. ads/database/__init__.py +0 -5
  154. ads/database/connection.py +0 -338
  155. ads/dataset/__init__.py +0 -10
  156. ads/dataset/capabilities.md +0 -51
  157. ads/dataset/classification_dataset.py +0 -339
  158. ads/dataset/correlation.py +0 -226
  159. ads/dataset/correlation_plot.py +0 -563
  160. ads/dataset/dask_series.py +0 -173
  161. ads/dataset/dataframe_transformer.py +0 -110
  162. ads/dataset/dataset.py +0 -1979
  163. ads/dataset/dataset_browser.py +0 -360
  164. ads/dataset/dataset_with_target.py +0 -995
  165. ads/dataset/exception.py +0 -25
  166. ads/dataset/factory.py +0 -987
  167. ads/dataset/feature_engineering_transformer.py +0 -35
  168. ads/dataset/feature_selection.py +0 -107
  169. ads/dataset/forecasting_dataset.py +0 -26
  170. ads/dataset/helper.py +0 -1450
  171. ads/dataset/label_encoder.py +0 -99
  172. ads/dataset/mixin/__init__.py +0 -5
  173. ads/dataset/mixin/dataset_accessor.py +0 -134
  174. ads/dataset/pipeline.py +0 -58
  175. ads/dataset/plot.py +0 -710
  176. ads/dataset/progress.py +0 -86
  177. ads/dataset/recommendation.py +0 -297
  178. ads/dataset/recommendation_transformer.py +0 -502
  179. ads/dataset/regression_dataset.py +0 -14
  180. ads/dataset/sampled_dataset.py +0 -1050
  181. ads/dataset/target.py +0 -98
  182. ads/dataset/timeseries.py +0 -18
  183. ads/dbmixin/__init__.py +0 -5
  184. ads/dbmixin/db_pandas_accessor.py +0 -153
  185. ads/environment/__init__.py +0 -9
  186. ads/environment/ml_runtime.py +0 -66
  187. ads/evaluations/README.md +0 -14
  188. ads/evaluations/__init__.py +0 -109
  189. ads/evaluations/evaluation_plot.py +0 -983
  190. ads/evaluations/evaluator.py +0 -1334
  191. ads/evaluations/statistical_metrics.py +0 -543
  192. ads/experiments/__init__.py +0 -9
  193. ads/experiments/capabilities.md +0 -0
  194. ads/explanations/__init__.py +0 -21
  195. ads/explanations/base_explainer.py +0 -142
  196. ads/explanations/capabilities.md +0 -83
  197. ads/explanations/explainer.py +0 -190
  198. ads/explanations/mlx_global_explainer.py +0 -1050
  199. ads/explanations/mlx_interface.py +0 -386
  200. ads/explanations/mlx_local_explainer.py +0 -287
  201. ads/explanations/mlx_whatif_explainer.py +0 -201
  202. ads/feature_engineering/__init__.py +0 -20
  203. ads/feature_engineering/accessor/__init__.py +0 -5
  204. ads/feature_engineering/accessor/dataframe_accessor.py +0 -535
  205. ads/feature_engineering/accessor/mixin/__init__.py +0 -5
  206. ads/feature_engineering/accessor/mixin/correlation.py +0 -166
  207. ads/feature_engineering/accessor/mixin/eda_mixin.py +0 -266
  208. ads/feature_engineering/accessor/mixin/eda_mixin_series.py +0 -85
  209. ads/feature_engineering/accessor/mixin/feature_types_mixin.py +0 -211
  210. ads/feature_engineering/accessor/mixin/utils.py +0 -65
  211. ads/feature_engineering/accessor/series_accessor.py +0 -431
  212. ads/feature_engineering/adsimage/__init__.py +0 -5
  213. ads/feature_engineering/adsimage/image.py +0 -192
  214. ads/feature_engineering/adsimage/image_reader.py +0 -170
  215. ads/feature_engineering/adsimage/interface/__init__.py +0 -5
  216. ads/feature_engineering/adsimage/interface/reader.py +0 -19
  217. ads/feature_engineering/adsstring/__init__.py +0 -7
  218. ads/feature_engineering/adsstring/oci_language/__init__.py +0 -8
  219. ads/feature_engineering/adsstring/string/__init__.py +0 -8
  220. ads/feature_engineering/data_schema.json +0 -57
  221. ads/feature_engineering/dataset/__init__.py +0 -5
  222. ads/feature_engineering/dataset/zip_code_data.py +0 -42062
  223. ads/feature_engineering/exceptions.py +0 -40
  224. ads/feature_engineering/feature_type/__init__.py +0 -133
  225. ads/feature_engineering/feature_type/address.py +0 -184
  226. ads/feature_engineering/feature_type/adsstring/__init__.py +0 -5
  227. ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +0 -164
  228. ads/feature_engineering/feature_type/adsstring/oci_language.py +0 -93
  229. ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +0 -5
  230. ads/feature_engineering/feature_type/adsstring/parsers/base.py +0 -47
  231. ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +0 -96
  232. ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +0 -221
  233. ads/feature_engineering/feature_type/adsstring/string.py +0 -258
  234. ads/feature_engineering/feature_type/base.py +0 -58
  235. ads/feature_engineering/feature_type/boolean.py +0 -183
  236. ads/feature_engineering/feature_type/category.py +0 -146
  237. ads/feature_engineering/feature_type/constant.py +0 -137
  238. ads/feature_engineering/feature_type/continuous.py +0 -151
  239. ads/feature_engineering/feature_type/creditcard.py +0 -314
  240. ads/feature_engineering/feature_type/datetime.py +0 -190
  241. ads/feature_engineering/feature_type/discrete.py +0 -134
  242. ads/feature_engineering/feature_type/document.py +0 -43
  243. ads/feature_engineering/feature_type/gis.py +0 -251
  244. ads/feature_engineering/feature_type/handler/__init__.py +0 -5
  245. ads/feature_engineering/feature_type/handler/feature_validator.py +0 -524
  246. ads/feature_engineering/feature_type/handler/feature_warning.py +0 -319
  247. ads/feature_engineering/feature_type/handler/warnings.py +0 -128
  248. ads/feature_engineering/feature_type/integer.py +0 -142
  249. ads/feature_engineering/feature_type/ip_address.py +0 -144
  250. ads/feature_engineering/feature_type/ip_address_v4.py +0 -138
  251. ads/feature_engineering/feature_type/ip_address_v6.py +0 -138
  252. ads/feature_engineering/feature_type/lat_long.py +0 -256
  253. ads/feature_engineering/feature_type/object.py +0 -43
  254. ads/feature_engineering/feature_type/ordinal.py +0 -132
  255. ads/feature_engineering/feature_type/phone_number.py +0 -135
  256. ads/feature_engineering/feature_type/string.py +0 -171
  257. ads/feature_engineering/feature_type/text.py +0 -93
  258. ads/feature_engineering/feature_type/unknown.py +0 -43
  259. ads/feature_engineering/feature_type/zip_code.py +0 -164
  260. ads/feature_engineering/feature_type_manager.py +0 -406
  261. ads/feature_engineering/schema.py +0 -795
  262. ads/feature_engineering/utils.py +0 -245
  263. ads/feature_store/.readthedocs.yaml +0 -19
  264. ads/feature_store/README.md +0 -65
  265. ads/feature_store/__init__.py +0 -9
  266. ads/feature_store/common/__init__.py +0 -0
  267. ads/feature_store/common/enums.py +0 -339
  268. ads/feature_store/common/exceptions.py +0 -18
  269. ads/feature_store/common/spark_session_singleton.py +0 -125
  270. ads/feature_store/common/utils/__init__.py +0 -0
  271. ads/feature_store/common/utils/base64_encoder_decoder.py +0 -72
  272. ads/feature_store/common/utils/feature_schema_mapper.py +0 -283
  273. ads/feature_store/common/utils/transformation_utils.py +0 -82
  274. ads/feature_store/common/utils/utility.py +0 -403
  275. ads/feature_store/data_validation/__init__.py +0 -0
  276. ads/feature_store/data_validation/great_expectation.py +0 -129
  277. ads/feature_store/dataset.py +0 -1230
  278. ads/feature_store/dataset_job.py +0 -530
  279. ads/feature_store/docs/Dockerfile +0 -7
  280. ads/feature_store/docs/Makefile +0 -44
  281. ads/feature_store/docs/conf.py +0 -28
  282. ads/feature_store/docs/requirements.txt +0 -14
  283. ads/feature_store/docs/source/ads.feature_store.query.rst +0 -20
  284. ads/feature_store/docs/source/cicd.rst +0 -137
  285. ads/feature_store/docs/source/conf.py +0 -86
  286. ads/feature_store/docs/source/data_versioning.rst +0 -33
  287. ads/feature_store/docs/source/dataset.rst +0 -388
  288. ads/feature_store/docs/source/dataset_job.rst +0 -27
  289. ads/feature_store/docs/source/demo.rst +0 -70
  290. ads/feature_store/docs/source/entity.rst +0 -78
  291. ads/feature_store/docs/source/feature_group.rst +0 -624
  292. ads/feature_store/docs/source/feature_group_job.rst +0 -29
  293. ads/feature_store/docs/source/feature_store.rst +0 -122
  294. ads/feature_store/docs/source/feature_store_class.rst +0 -123
  295. ads/feature_store/docs/source/feature_validation.rst +0 -66
  296. ads/feature_store/docs/source/figures/cicd.png +0 -0
  297. ads/feature_store/docs/source/figures/data_validation.png +0 -0
  298. ads/feature_store/docs/source/figures/data_versioning.png +0 -0
  299. ads/feature_store/docs/source/figures/dataset.gif +0 -0
  300. ads/feature_store/docs/source/figures/dataset.png +0 -0
  301. ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
  302. ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
  303. ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
  304. ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
  305. ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
  306. ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
  307. ads/feature_store/docs/source/figures/entity.png +0 -0
  308. ads/feature_store/docs/source/figures/feature_group.png +0 -0
  309. ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
  310. ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
  311. ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
  312. ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
  313. ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
  314. ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
  315. ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
  316. ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
  317. ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
  318. ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
  319. ads/feature_store/docs/source/figures/overview.png +0 -0
  320. ads/feature_store/docs/source/figures/resource_manager.png +0 -0
  321. ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
  322. ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
  323. ads/feature_store/docs/source/figures/stats_1.png +0 -0
  324. ads/feature_store/docs/source/figures/stats_2.png +0 -0
  325. ads/feature_store/docs/source/figures/stats_d.png +0 -0
  326. ads/feature_store/docs/source/figures/stats_fg.png +0 -0
  327. ads/feature_store/docs/source/figures/transformation.png +0 -0
  328. ads/feature_store/docs/source/figures/transformations.gif +0 -0
  329. ads/feature_store/docs/source/figures/validation.png +0 -0
  330. ads/feature_store/docs/source/figures/validation_fg.png +0 -0
  331. ads/feature_store/docs/source/figures/validation_results.png +0 -0
  332. ads/feature_store/docs/source/figures/validation_summary.png +0 -0
  333. ads/feature_store/docs/source/index.rst +0 -81
  334. ads/feature_store/docs/source/module.rst +0 -8
  335. ads/feature_store/docs/source/notebook.rst +0 -94
  336. ads/feature_store/docs/source/overview.rst +0 -47
  337. ads/feature_store/docs/source/quickstart.rst +0 -176
  338. ads/feature_store/docs/source/release_notes.rst +0 -194
  339. ads/feature_store/docs/source/setup_feature_store.rst +0 -81
  340. ads/feature_store/docs/source/statistics.rst +0 -58
  341. ads/feature_store/docs/source/transformation.rst +0 -199
  342. ads/feature_store/docs/source/ui.rst +0 -65
  343. ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +0 -66
  344. ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +0 -192
  345. ads/feature_store/docs/source/user_guides.setup.terraform.rst +0 -338
  346. ads/feature_store/entity.py +0 -718
  347. ads/feature_store/execution_strategy/__init__.py +0 -0
  348. ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
  349. ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +0 -375
  350. ads/feature_store/execution_strategy/engine/__init__.py +0 -0
  351. ads/feature_store/execution_strategy/engine/spark_engine.py +0 -316
  352. ads/feature_store/execution_strategy/execution_strategy.py +0 -113
  353. ads/feature_store/execution_strategy/execution_strategy_provider.py +0 -47
  354. ads/feature_store/execution_strategy/spark/__init__.py +0 -0
  355. ads/feature_store/execution_strategy/spark/spark_execution.py +0 -618
  356. ads/feature_store/feature.py +0 -192
  357. ads/feature_store/feature_group.py +0 -1494
  358. ads/feature_store/feature_group_expectation.py +0 -346
  359. ads/feature_store/feature_group_job.py +0 -602
  360. ads/feature_store/feature_lineage/__init__.py +0 -0
  361. ads/feature_store/feature_lineage/graphviz_service.py +0 -180
  362. ads/feature_store/feature_option_details.py +0 -50
  363. ads/feature_store/feature_statistics/__init__.py +0 -0
  364. ads/feature_store/feature_statistics/statistics_service.py +0 -99
  365. ads/feature_store/feature_store.py +0 -699
  366. ads/feature_store/feature_store_registrar.py +0 -518
  367. ads/feature_store/input_feature_detail.py +0 -149
  368. ads/feature_store/mixin/__init__.py +0 -4
  369. ads/feature_store/mixin/oci_feature_store.py +0 -145
  370. ads/feature_store/model_details.py +0 -73
  371. ads/feature_store/query/__init__.py +0 -0
  372. ads/feature_store/query/filter.py +0 -266
  373. ads/feature_store/query/generator/__init__.py +0 -0
  374. ads/feature_store/query/generator/query_generator.py +0 -298
  375. ads/feature_store/query/join.py +0 -161
  376. ads/feature_store/query/query.py +0 -403
  377. ads/feature_store/query/validator/__init__.py +0 -0
  378. ads/feature_store/query/validator/query_validator.py +0 -57
  379. ads/feature_store/response/__init__.py +0 -0
  380. ads/feature_store/response/response_builder.py +0 -68
  381. ads/feature_store/service/__init__.py +0 -0
  382. ads/feature_store/service/oci_dataset.py +0 -139
  383. ads/feature_store/service/oci_dataset_job.py +0 -199
  384. ads/feature_store/service/oci_entity.py +0 -125
  385. ads/feature_store/service/oci_feature_group.py +0 -164
  386. ads/feature_store/service/oci_feature_group_job.py +0 -214
  387. ads/feature_store/service/oci_feature_store.py +0 -182
  388. ads/feature_store/service/oci_lineage.py +0 -87
  389. ads/feature_store/service/oci_transformation.py +0 -104
  390. ads/feature_store/statistics/__init__.py +0 -0
  391. ads/feature_store/statistics/abs_feature_value.py +0 -49
  392. ads/feature_store/statistics/charts/__init__.py +0 -0
  393. ads/feature_store/statistics/charts/abstract_feature_plot.py +0 -37
  394. ads/feature_store/statistics/charts/box_plot.py +0 -148
  395. ads/feature_store/statistics/charts/frequency_distribution.py +0 -65
  396. ads/feature_store/statistics/charts/probability_distribution.py +0 -68
  397. ads/feature_store/statistics/charts/top_k_frequent_elements.py +0 -98
  398. ads/feature_store/statistics/feature_stat.py +0 -126
  399. ads/feature_store/statistics/generic_feature_value.py +0 -33
  400. ads/feature_store/statistics/statistics.py +0 -41
  401. ads/feature_store/statistics_config.py +0 -101
  402. ads/feature_store/templates/feature_store_template.yaml +0 -45
  403. ads/feature_store/transformation.py +0 -499
  404. ads/feature_store/validation_output.py +0 -57
  405. ads/hpo/__init__.py +0 -9
  406. ads/hpo/_imports.py +0 -91
  407. ads/hpo/ads_search_space.py +0 -439
  408. ads/hpo/distributions.py +0 -325
  409. ads/hpo/objective.py +0 -280
  410. ads/hpo/search_cv.py +0 -1657
  411. ads/hpo/stopping_criterion.py +0 -75
  412. ads/hpo/tuner_artifact.py +0 -413
  413. ads/hpo/utils.py +0 -91
  414. ads/hpo/validation.py +0 -140
  415. ads/hpo/visualization/__init__.py +0 -5
  416. ads/hpo/visualization/_contour.py +0 -23
  417. ads/hpo/visualization/_edf.py +0 -20
  418. ads/hpo/visualization/_intermediate_values.py +0 -21
  419. ads/hpo/visualization/_optimization_history.py +0 -25
  420. ads/hpo/visualization/_parallel_coordinate.py +0 -169
  421. ads/hpo/visualization/_param_importances.py +0 -26
  422. ads/jobs/__init__.py +0 -53
  423. ads/jobs/ads_job.py +0 -663
  424. ads/jobs/builders/__init__.py +0 -5
  425. ads/jobs/builders/base.py +0 -156
  426. ads/jobs/builders/infrastructure/__init__.py +0 -6
  427. ads/jobs/builders/infrastructure/base.py +0 -165
  428. ads/jobs/builders/infrastructure/dataflow.py +0 -1252
  429. ads/jobs/builders/infrastructure/dsc_job.py +0 -1894
  430. ads/jobs/builders/infrastructure/dsc_job_runtime.py +0 -1233
  431. ads/jobs/builders/infrastructure/utils.py +0 -65
  432. ads/jobs/builders/runtimes/__init__.py +0 -5
  433. ads/jobs/builders/runtimes/artifact.py +0 -338
  434. ads/jobs/builders/runtimes/base.py +0 -325
  435. ads/jobs/builders/runtimes/container_runtime.py +0 -242
  436. ads/jobs/builders/runtimes/python_runtime.py +0 -1016
  437. ads/jobs/builders/runtimes/pytorch_runtime.py +0 -204
  438. ads/jobs/cli.py +0 -104
  439. ads/jobs/env_var_parser.py +0 -131
  440. ads/jobs/extension.py +0 -160
  441. ads/jobs/schema/__init__.py +0 -5
  442. ads/jobs/schema/infrastructure_schema.json +0 -116
  443. ads/jobs/schema/job_schema.json +0 -42
  444. ads/jobs/schema/runtime_schema.json +0 -183
  445. ads/jobs/schema/validator.py +0 -141
  446. ads/jobs/serializer.py +0 -296
  447. ads/jobs/templates/__init__.py +0 -5
  448. ads/jobs/templates/container.py +0 -6
  449. ads/jobs/templates/driver_notebook.py +0 -177
  450. ads/jobs/templates/driver_oci.py +0 -500
  451. ads/jobs/templates/driver_python.py +0 -48
  452. ads/jobs/templates/driver_pytorch.py +0 -852
  453. ads/jobs/templates/driver_utils.py +0 -615
  454. ads/jobs/templates/hostname_from_env.c +0 -55
  455. ads/jobs/templates/oci_metrics.py +0 -181
  456. ads/jobs/utils.py +0 -104
  457. ads/llm/__init__.py +0 -28
  458. ads/llm/autogen/__init__.py +0 -2
  459. ads/llm/autogen/constants.py +0 -15
  460. ads/llm/autogen/reports/__init__.py +0 -2
  461. ads/llm/autogen/reports/base.py +0 -67
  462. ads/llm/autogen/reports/data.py +0 -103
  463. ads/llm/autogen/reports/session.py +0 -526
  464. ads/llm/autogen/reports/templates/chat_box.html +0 -13
  465. ads/llm/autogen/reports/templates/chat_box_lt.html +0 -5
  466. ads/llm/autogen/reports/templates/chat_box_rt.html +0 -6
  467. ads/llm/autogen/reports/utils.py +0 -56
  468. ads/llm/autogen/v02/__init__.py +0 -4
  469. ads/llm/autogen/v02/client.py +0 -295
  470. ads/llm/autogen/v02/log_handlers/__init__.py +0 -2
  471. ads/llm/autogen/v02/log_handlers/oci_file_handler.py +0 -83
  472. ads/llm/autogen/v02/loggers/__init__.py +0 -6
  473. ads/llm/autogen/v02/loggers/metric_logger.py +0 -320
  474. ads/llm/autogen/v02/loggers/session_logger.py +0 -580
  475. ads/llm/autogen/v02/loggers/utils.py +0 -86
  476. ads/llm/autogen/v02/runtime_logging.py +0 -163
  477. ads/llm/chain.py +0 -268
  478. ads/llm/chat_template.py +0 -31
  479. ads/llm/deploy.py +0 -63
  480. ads/llm/guardrails/__init__.py +0 -5
  481. ads/llm/guardrails/base.py +0 -442
  482. ads/llm/guardrails/huggingface.py +0 -44
  483. ads/llm/langchain/__init__.py +0 -5
  484. ads/llm/langchain/plugins/__init__.py +0 -5
  485. ads/llm/langchain/plugins/chat_models/__init__.py +0 -5
  486. ads/llm/langchain/plugins/chat_models/oci_data_science.py +0 -1027
  487. ads/llm/langchain/plugins/embeddings/__init__.py +0 -4
  488. ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +0 -184
  489. ads/llm/langchain/plugins/llms/__init__.py +0 -5
  490. ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +0 -979
  491. ads/llm/requirements.txt +0 -3
  492. ads/llm/serialize.py +0 -219
  493. ads/llm/serializers/__init__.py +0 -0
  494. ads/llm/serializers/retrieval_qa.py +0 -153
  495. ads/llm/serializers/runnable_parallel.py +0 -27
  496. ads/llm/templates/score_chain.jinja2 +0 -155
  497. ads/llm/templates/tool_chat_template_hermes.jinja +0 -130
  498. ads/llm/templates/tool_chat_template_mistral_parallel.jinja +0 -94
  499. ads/model/__init__.py +0 -52
  500. ads/model/artifact.py +0 -573
  501. ads/model/artifact_downloader.py +0 -254
  502. ads/model/artifact_uploader.py +0 -267
  503. ads/model/base_properties.py +0 -238
  504. ads/model/common/.model-ignore +0 -66
  505. ads/model/common/__init__.py +0 -5
  506. ads/model/common/utils.py +0 -142
  507. ads/model/datascience_model.py +0 -2635
  508. ads/model/deployment/__init__.py +0 -20
  509. ads/model/deployment/common/__init__.py +0 -5
  510. ads/model/deployment/common/utils.py +0 -308
  511. ads/model/deployment/model_deployer.py +0 -466
  512. ads/model/deployment/model_deployment.py +0 -1846
  513. ads/model/deployment/model_deployment_infrastructure.py +0 -671
  514. ads/model/deployment/model_deployment_properties.py +0 -493
  515. ads/model/deployment/model_deployment_runtime.py +0 -838
  516. ads/model/extractor/__init__.py +0 -5
  517. ads/model/extractor/automl_extractor.py +0 -74
  518. ads/model/extractor/embedding_onnx_extractor.py +0 -80
  519. ads/model/extractor/huggingface_extractor.py +0 -88
  520. ads/model/extractor/keras_extractor.py +0 -84
  521. ads/model/extractor/lightgbm_extractor.py +0 -93
  522. ads/model/extractor/model_info_extractor.py +0 -114
  523. ads/model/extractor/model_info_extractor_factory.py +0 -105
  524. ads/model/extractor/pytorch_extractor.py +0 -87
  525. ads/model/extractor/sklearn_extractor.py +0 -112
  526. ads/model/extractor/spark_extractor.py +0 -89
  527. ads/model/extractor/tensorflow_extractor.py +0 -85
  528. ads/model/extractor/xgboost_extractor.py +0 -94
  529. ads/model/framework/__init__.py +0 -5
  530. ads/model/framework/automl_model.py +0 -178
  531. ads/model/framework/embedding_onnx_model.py +0 -438
  532. ads/model/framework/huggingface_model.py +0 -399
  533. ads/model/framework/lightgbm_model.py +0 -266
  534. ads/model/framework/pytorch_model.py +0 -266
  535. ads/model/framework/sklearn_model.py +0 -250
  536. ads/model/framework/spark_model.py +0 -326
  537. ads/model/framework/tensorflow_model.py +0 -254
  538. ads/model/framework/xgboost_model.py +0 -258
  539. ads/model/generic_model.py +0 -3518
  540. ads/model/model_artifact_boilerplate/README.md +0 -381
  541. ads/model/model_artifact_boilerplate/__init__.py +0 -5
  542. ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +0 -5
  543. ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +0 -427
  544. ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +0 -2
  545. ads/model/model_artifact_boilerplate/runtime.yaml +0 -7
  546. ads/model/model_artifact_boilerplate/score.py +0 -61
  547. ads/model/model_file_description_schema.json +0 -68
  548. ads/model/model_introspect.py +0 -331
  549. ads/model/model_metadata.py +0 -1810
  550. ads/model/model_metadata_mixin.py +0 -460
  551. ads/model/model_properties.py +0 -63
  552. ads/model/model_version_set.py +0 -739
  553. ads/model/runtime/__init__.py +0 -5
  554. ads/model/runtime/env_info.py +0 -306
  555. ads/model/runtime/model_deployment_details.py +0 -37
  556. ads/model/runtime/model_provenance_details.py +0 -58
  557. ads/model/runtime/runtime_info.py +0 -81
  558. ads/model/runtime/schemas/inference_env_info_schema.yaml +0 -16
  559. ads/model/runtime/schemas/model_provenance_schema.yaml +0 -36
  560. ads/model/runtime/schemas/training_env_info_schema.yaml +0 -16
  561. ads/model/runtime/utils.py +0 -201
  562. ads/model/serde/__init__.py +0 -5
  563. ads/model/serde/common.py +0 -40
  564. ads/model/serde/model_input.py +0 -547
  565. ads/model/serde/model_serializer.py +0 -1184
  566. ads/model/service/__init__.py +0 -5
  567. ads/model/service/oci_datascience_model.py +0 -1076
  568. ads/model/service/oci_datascience_model_deployment.py +0 -500
  569. ads/model/service/oci_datascience_model_version_set.py +0 -176
  570. ads/model/transformer/__init__.py +0 -5
  571. ads/model/transformer/onnx_transformer.py +0 -324
  572. ads/mysqldb/__init__.py +0 -5
  573. ads/mysqldb/mysql_db.py +0 -227
  574. ads/opctl/__init__.py +0 -18
  575. ads/opctl/anomaly_detection.py +0 -11
  576. ads/opctl/backend/__init__.py +0 -5
  577. ads/opctl/backend/ads_dataflow.py +0 -353
  578. ads/opctl/backend/ads_ml_job.py +0 -710
  579. ads/opctl/backend/ads_ml_pipeline.py +0 -164
  580. ads/opctl/backend/ads_model_deployment.py +0 -209
  581. ads/opctl/backend/base.py +0 -146
  582. ads/opctl/backend/local.py +0 -1053
  583. ads/opctl/backend/marketplace/__init__.py +0 -9
  584. ads/opctl/backend/marketplace/helm_helper.py +0 -173
  585. ads/opctl/backend/marketplace/local_marketplace.py +0 -271
  586. ads/opctl/backend/marketplace/marketplace_backend_runner.py +0 -71
  587. ads/opctl/backend/marketplace/marketplace_operator_interface.py +0 -44
  588. ads/opctl/backend/marketplace/marketplace_operator_runner.py +0 -24
  589. ads/opctl/backend/marketplace/marketplace_utils.py +0 -212
  590. ads/opctl/backend/marketplace/models/__init__.py +0 -5
  591. ads/opctl/backend/marketplace/models/bearer_token.py +0 -94
  592. ads/opctl/backend/marketplace/models/marketplace_type.py +0 -70
  593. ads/opctl/backend/marketplace/models/ocir_details.py +0 -56
  594. ads/opctl/backend/marketplace/prerequisite_checker.py +0 -238
  595. ads/opctl/cli.py +0 -707
  596. ads/opctl/cmds.py +0 -869
  597. ads/opctl/conda/__init__.py +0 -5
  598. ads/opctl/conda/cli.py +0 -193
  599. ads/opctl/conda/cmds.py +0 -749
  600. ads/opctl/conda/config.yaml +0 -34
  601. ads/opctl/conda/manifest_template.yaml +0 -13
  602. ads/opctl/conda/multipart_uploader.py +0 -188
  603. ads/opctl/conda/pack.py +0 -89
  604. ads/opctl/config/__init__.py +0 -5
  605. ads/opctl/config/base.py +0 -57
  606. ads/opctl/config/diagnostics/__init__.py +0 -5
  607. ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +0 -62
  608. ads/opctl/config/merger.py +0 -255
  609. ads/opctl/config/resolver.py +0 -297
  610. ads/opctl/config/utils.py +0 -79
  611. ads/opctl/config/validator.py +0 -17
  612. ads/opctl/config/versioner.py +0 -68
  613. ads/opctl/config/yaml_parsers/__init__.py +0 -7
  614. ads/opctl/config/yaml_parsers/base.py +0 -58
  615. ads/opctl/config/yaml_parsers/distributed/__init__.py +0 -7
  616. ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +0 -201
  617. ads/opctl/constants.py +0 -66
  618. ads/opctl/decorator/__init__.py +0 -5
  619. ads/opctl/decorator/common.py +0 -129
  620. ads/opctl/diagnostics/__init__.py +0 -5
  621. ads/opctl/diagnostics/__main__.py +0 -25
  622. ads/opctl/diagnostics/check_distributed_job_requirements.py +0 -212
  623. ads/opctl/diagnostics/check_requirements.py +0 -144
  624. ads/opctl/diagnostics/requirement_exception.py +0 -9
  625. ads/opctl/distributed/README.md +0 -109
  626. ads/opctl/distributed/__init__.py +0 -5
  627. ads/opctl/distributed/certificates.py +0 -32
  628. ads/opctl/distributed/cli.py +0 -207
  629. ads/opctl/distributed/cmds.py +0 -731
  630. ads/opctl/distributed/common/__init__.py +0 -5
  631. ads/opctl/distributed/common/abstract_cluster_provider.py +0 -449
  632. ads/opctl/distributed/common/abstract_framework_spec_builder.py +0 -88
  633. ads/opctl/distributed/common/cluster_config_helper.py +0 -103
  634. ads/opctl/distributed/common/cluster_provider_factory.py +0 -21
  635. ads/opctl/distributed/common/cluster_runner.py +0 -54
  636. ads/opctl/distributed/common/framework_factory.py +0 -29
  637. ads/opctl/docker/Dockerfile.job +0 -103
  638. ads/opctl/docker/Dockerfile.job.arm +0 -107
  639. ads/opctl/docker/Dockerfile.job.gpu +0 -175
  640. ads/opctl/docker/base-env.yaml +0 -13
  641. ads/opctl/docker/cuda.repo +0 -6
  642. ads/opctl/docker/operator/.dockerignore +0 -0
  643. ads/opctl/docker/operator/Dockerfile +0 -41
  644. ads/opctl/docker/operator/Dockerfile.gpu +0 -85
  645. ads/opctl/docker/operator/cuda.repo +0 -6
  646. ads/opctl/docker/operator/environment.yaml +0 -8
  647. ads/opctl/forecast.py +0 -11
  648. ads/opctl/index.yaml +0 -3
  649. ads/opctl/model/__init__.py +0 -5
  650. ads/opctl/model/cli.py +0 -65
  651. ads/opctl/model/cmds.py +0 -73
  652. ads/opctl/operator/README.md +0 -4
  653. ads/opctl/operator/__init__.py +0 -31
  654. ads/opctl/operator/cli.py +0 -344
  655. ads/opctl/operator/cmd.py +0 -596
  656. ads/opctl/operator/common/__init__.py +0 -5
  657. ads/opctl/operator/common/backend_factory.py +0 -460
  658. ads/opctl/operator/common/const.py +0 -27
  659. ads/opctl/operator/common/data/synthetic.csv +0 -16001
  660. ads/opctl/operator/common/dictionary_merger.py +0 -148
  661. ads/opctl/operator/common/errors.py +0 -42
  662. ads/opctl/operator/common/operator_config.py +0 -99
  663. ads/opctl/operator/common/operator_loader.py +0 -811
  664. ads/opctl/operator/common/operator_schema.yaml +0 -130
  665. ads/opctl/operator/common/operator_yaml_generator.py +0 -152
  666. ads/opctl/operator/common/utils.py +0 -208
  667. ads/opctl/operator/lowcode/__init__.py +0 -5
  668. ads/opctl/operator/lowcode/anomaly/MLoperator +0 -16
  669. ads/opctl/operator/lowcode/anomaly/README.md +0 -207
  670. ads/opctl/operator/lowcode/anomaly/__init__.py +0 -5
  671. ads/opctl/operator/lowcode/anomaly/__main__.py +0 -103
  672. ads/opctl/operator/lowcode/anomaly/cmd.py +0 -35
  673. ads/opctl/operator/lowcode/anomaly/const.py +0 -167
  674. ads/opctl/operator/lowcode/anomaly/environment.yaml +0 -10
  675. ads/opctl/operator/lowcode/anomaly/model/__init__.py +0 -5
  676. ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +0 -146
  677. ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +0 -162
  678. ads/opctl/operator/lowcode/anomaly/model/automlx.py +0 -99
  679. ads/opctl/operator/lowcode/anomaly/model/autots.py +0 -115
  680. ads/opctl/operator/lowcode/anomaly/model/base_model.py +0 -404
  681. ads/opctl/operator/lowcode/anomaly/model/factory.py +0 -110
  682. ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +0 -78
  683. ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +0 -78
  684. ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +0 -120
  685. ads/opctl/operator/lowcode/anomaly/model/tods.py +0 -119
  686. ads/opctl/operator/lowcode/anomaly/operator_config.py +0 -127
  687. ads/opctl/operator/lowcode/anomaly/schema.yaml +0 -401
  688. ads/opctl/operator/lowcode/anomaly/utils.py +0 -88
  689. ads/opctl/operator/lowcode/common/__init__.py +0 -5
  690. ads/opctl/operator/lowcode/common/const.py +0 -10
  691. ads/opctl/operator/lowcode/common/data.py +0 -116
  692. ads/opctl/operator/lowcode/common/errors.py +0 -47
  693. ads/opctl/operator/lowcode/common/transformations.py +0 -296
  694. ads/opctl/operator/lowcode/common/utils.py +0 -293
  695. ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +0 -13
  696. ads/opctl/operator/lowcode/feature_store_marketplace/README.md +0 -30
  697. ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +0 -5
  698. ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +0 -116
  699. ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +0 -85
  700. ads/opctl/operator/lowcode/feature_store_marketplace/const.py +0 -15
  701. ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
  702. ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +0 -4
  703. ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +0 -32
  704. ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +0 -43
  705. ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +0 -120
  706. ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +0 -34
  707. ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +0 -386
  708. ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +0 -160
  709. ads/opctl/operator/lowcode/forecast/MLoperator +0 -25
  710. ads/opctl/operator/lowcode/forecast/README.md +0 -209
  711. ads/opctl/operator/lowcode/forecast/__init__.py +0 -5
  712. ads/opctl/operator/lowcode/forecast/__main__.py +0 -89
  713. ads/opctl/operator/lowcode/forecast/cmd.py +0 -40
  714. ads/opctl/operator/lowcode/forecast/const.py +0 -92
  715. ads/opctl/operator/lowcode/forecast/environment.yaml +0 -20
  716. ads/opctl/operator/lowcode/forecast/errors.py +0 -26
  717. ads/opctl/operator/lowcode/forecast/model/__init__.py +0 -5
  718. ads/opctl/operator/lowcode/forecast/model/arima.py +0 -279
  719. ads/opctl/operator/lowcode/forecast/model/automlx.py +0 -542
  720. ads/opctl/operator/lowcode/forecast/model/autots.py +0 -312
  721. ads/opctl/operator/lowcode/forecast/model/base_model.py +0 -863
  722. ads/opctl/operator/lowcode/forecast/model/factory.py +0 -106
  723. ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +0 -492
  724. ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +0 -243
  725. ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +0 -486
  726. ads/opctl/operator/lowcode/forecast/model/prophet.py +0 -445
  727. ads/opctl/operator/lowcode/forecast/model_evaluator.py +0 -244
  728. ads/opctl/operator/lowcode/forecast/operator_config.py +0 -234
  729. ads/opctl/operator/lowcode/forecast/schema.yaml +0 -506
  730. ads/opctl/operator/lowcode/forecast/utils.py +0 -413
  731. ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +0 -7
  732. ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +0 -285
  733. ads/opctl/operator/lowcode/forecast/whatifserve/score.py +0 -246
  734. ads/opctl/operator/lowcode/pii/MLoperator +0 -17
  735. ads/opctl/operator/lowcode/pii/README.md +0 -208
  736. ads/opctl/operator/lowcode/pii/__init__.py +0 -5
  737. ads/opctl/operator/lowcode/pii/__main__.py +0 -78
  738. ads/opctl/operator/lowcode/pii/cmd.py +0 -39
  739. ads/opctl/operator/lowcode/pii/constant.py +0 -84
  740. ads/opctl/operator/lowcode/pii/environment.yaml +0 -17
  741. ads/opctl/operator/lowcode/pii/errors.py +0 -27
  742. ads/opctl/operator/lowcode/pii/model/__init__.py +0 -5
  743. ads/opctl/operator/lowcode/pii/model/factory.py +0 -82
  744. ads/opctl/operator/lowcode/pii/model/guardrails.py +0 -167
  745. ads/opctl/operator/lowcode/pii/model/pii.py +0 -145
  746. ads/opctl/operator/lowcode/pii/model/processor/__init__.py +0 -34
  747. ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +0 -34
  748. ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +0 -35
  749. ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +0 -225
  750. ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +0 -73
  751. ads/opctl/operator/lowcode/pii/model/processor/remover.py +0 -26
  752. ads/opctl/operator/lowcode/pii/model/report.py +0 -487
  753. ads/opctl/operator/lowcode/pii/operator_config.py +0 -95
  754. ads/opctl/operator/lowcode/pii/schema.yaml +0 -108
  755. ads/opctl/operator/lowcode/pii/utils.py +0 -43
  756. ads/opctl/operator/lowcode/recommender/MLoperator +0 -16
  757. ads/opctl/operator/lowcode/recommender/README.md +0 -206
  758. ads/opctl/operator/lowcode/recommender/__init__.py +0 -5
  759. ads/opctl/operator/lowcode/recommender/__main__.py +0 -82
  760. ads/opctl/operator/lowcode/recommender/cmd.py +0 -33
  761. ads/opctl/operator/lowcode/recommender/constant.py +0 -30
  762. ads/opctl/operator/lowcode/recommender/environment.yaml +0 -11
  763. ads/opctl/operator/lowcode/recommender/model/base_model.py +0 -212
  764. ads/opctl/operator/lowcode/recommender/model/factory.py +0 -56
  765. ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +0 -25
  766. ads/opctl/operator/lowcode/recommender/model/svd.py +0 -106
  767. ads/opctl/operator/lowcode/recommender/operator_config.py +0 -81
  768. ads/opctl/operator/lowcode/recommender/schema.yaml +0 -265
  769. ads/opctl/operator/lowcode/recommender/utils.py +0 -13
  770. ads/opctl/operator/runtime/__init__.py +0 -5
  771. ads/opctl/operator/runtime/const.py +0 -17
  772. ads/opctl/operator/runtime/container_runtime_schema.yaml +0 -50
  773. ads/opctl/operator/runtime/marketplace_runtime.py +0 -50
  774. ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +0 -21
  775. ads/opctl/operator/runtime/python_runtime_schema.yaml +0 -21
  776. ads/opctl/operator/runtime/runtime.py +0 -115
  777. ads/opctl/schema.yaml.yml +0 -36
  778. ads/opctl/script.py +0 -40
  779. ads/opctl/spark/__init__.py +0 -5
  780. ads/opctl/spark/cli.py +0 -43
  781. ads/opctl/spark/cmds.py +0 -147
  782. ads/opctl/templates/diagnostic_report_template.jinja2 +0 -102
  783. ads/opctl/utils.py +0 -344
  784. ads/oracledb/__init__.py +0 -5
  785. ads/oracledb/oracle_db.py +0 -346
  786. ads/pipeline/__init__.py +0 -39
  787. ads/pipeline/ads_pipeline.py +0 -2279
  788. ads/pipeline/ads_pipeline_run.py +0 -772
  789. ads/pipeline/ads_pipeline_step.py +0 -605
  790. ads/pipeline/builders/__init__.py +0 -5
  791. ads/pipeline/builders/infrastructure/__init__.py +0 -5
  792. ads/pipeline/builders/infrastructure/custom_script.py +0 -32
  793. ads/pipeline/cli.py +0 -119
  794. ads/pipeline/extension.py +0 -291
  795. ads/pipeline/schema/__init__.py +0 -5
  796. ads/pipeline/schema/cs_step_schema.json +0 -35
  797. ads/pipeline/schema/ml_step_schema.json +0 -31
  798. ads/pipeline/schema/pipeline_schema.json +0 -71
  799. ads/pipeline/visualizer/__init__.py +0 -5
  800. ads/pipeline/visualizer/base.py +0 -570
  801. ads/pipeline/visualizer/graph_renderer.py +0 -272
  802. ads/pipeline/visualizer/text_renderer.py +0 -84
  803. ads/secrets/__init__.py +0 -11
  804. ads/secrets/adb.py +0 -386
  805. ads/secrets/auth_token.py +0 -86
  806. ads/secrets/big_data_service.py +0 -365
  807. ads/secrets/mysqldb.py +0 -149
  808. ads/secrets/oracledb.py +0 -160
  809. ads/secrets/secrets.py +0 -407
  810. ads/telemetry/__init__.py +0 -7
  811. ads/telemetry/base.py +0 -69
  812. ads/telemetry/client.py +0 -125
  813. ads/telemetry/telemetry.py +0 -257
  814. ads/templates/dataflow_pyspark.jinja2 +0 -13
  815. ads/templates/dataflow_sparksql.jinja2 +0 -22
  816. ads/templates/func.jinja2 +0 -20
  817. ads/templates/schemas/openapi.json +0 -1740
  818. ads/templates/score-pkl.jinja2 +0 -173
  819. ads/templates/score.jinja2 +0 -322
  820. ads/templates/score_embedding_onnx.jinja2 +0 -202
  821. ads/templates/score_generic.jinja2 +0 -165
  822. ads/templates/score_huggingface_pipeline.jinja2 +0 -217
  823. ads/templates/score_lightgbm.jinja2 +0 -185
  824. ads/templates/score_onnx.jinja2 +0 -407
  825. ads/templates/score_onnx_new.jinja2 +0 -473
  826. ads/templates/score_oracle_automl.jinja2 +0 -185
  827. ads/templates/score_pyspark.jinja2 +0 -154
  828. ads/templates/score_pytorch.jinja2 +0 -219
  829. ads/templates/score_scikit-learn.jinja2 +0 -184
  830. ads/templates/score_tensorflow.jinja2 +0 -184
  831. ads/templates/score_xgboost.jinja2 +0 -178
  832. ads/text_dataset/__init__.py +0 -5
  833. ads/text_dataset/backends.py +0 -211
  834. ads/text_dataset/dataset.py +0 -445
  835. ads/text_dataset/extractor.py +0 -207
  836. ads/text_dataset/options.py +0 -53
  837. ads/text_dataset/udfs.py +0 -22
  838. ads/text_dataset/utils.py +0 -49
  839. ads/type_discovery/__init__.py +0 -9
  840. ads/type_discovery/abstract_detector.py +0 -21
  841. ads/type_discovery/constant_detector.py +0 -41
  842. ads/type_discovery/continuous_detector.py +0 -54
  843. ads/type_discovery/credit_card_detector.py +0 -99
  844. ads/type_discovery/datetime_detector.py +0 -92
  845. ads/type_discovery/discrete_detector.py +0 -118
  846. ads/type_discovery/document_detector.py +0 -146
  847. ads/type_discovery/ip_detector.py +0 -68
  848. ads/type_discovery/latlon_detector.py +0 -90
  849. ads/type_discovery/phone_number_detector.py +0 -63
  850. ads/type_discovery/type_discovery_driver.py +0 -87
  851. ads/type_discovery/typed_feature.py +0 -594
  852. ads/type_discovery/unknown_detector.py +0 -41
  853. ads/type_discovery/zipcode_detector.py +0 -48
  854. ads/vault/__init__.py +0 -7
  855. ads/vault/vault.py +0 -237
  856. oracle_ads-2.13.8.dist-info/RECORD +0 -858
  857. {oracle_ads-2.13.8.dist-info → oracle_ads-2.13.9rc0.dist-info}/licenses/LICENSE.txt +0 -0
ads/opctl/conda/cmds.py DELETED
@@ -1,749 +0,0 @@
1
- #!/usr/bin/env python
2
- # -*- coding: utf-8; -*-
3
-
4
- # Copyright (c) 2022 Oracle and/or its affiliates.
5
- # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
6
-
7
- import datetime
8
- import os
9
- import shutil
10
- import subprocess
11
- import shlex
12
- import json
13
- import glob
14
- from typing import Dict
15
-
16
- import click
17
- import yaml
18
-
19
- from datetime import datetime
20
- import ocifs
21
-
22
- from ads.common.oci_client import OCIClientFactory
23
- from ads.common.auth import create_signer
24
- from ads.common.decorator.runtime_dependency import (
25
- runtime_dependency,
26
- OptionalDependency,
27
- )
28
- from ads.config import NO_CONTAINER
29
-
30
- from ads.opctl.constants import (
31
- ML_JOB_GPU_IMAGE,
32
- ML_JOB_IMAGE,
33
- DEFAULT_IMAGE_HOME_DIR,
34
- DEFAULT_IMAGE_CONDA_DIR,
35
- DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR,
36
- )
37
- from ads.opctl import logger
38
- from ads.opctl.config.utils import read_from_ini
39
- from ads.opctl.utils import (
40
- parse_conda_uri,
41
- run_container,
42
- get_docker_client,
43
- is_in_notebook_session,
44
- run_command,
45
- )
46
- from ads.opctl.config.base import ConfigProcessor
47
- from ads.opctl.config.merger import ConfigMerger
48
- from ads.opctl.conda.multipart_uploader import MultiPartUploader
49
- import tempfile
50
-
51
-
52
- def _fetch_manifest_template() -> Dict:
53
- manifest_template_file = os.path.join(
54
- os.path.dirname(os.path.abspath(__file__)), "manifest_template.yaml"
55
- )
56
- with open(manifest_template_file) as manifest_file:
57
- manifest_template = yaml.safe_load(manifest_file)
58
- return manifest_template
59
-
60
-
61
- @runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
62
- def _check_job_image_exists(gpu: bool) -> None:
63
- if gpu:
64
- image = ML_JOB_GPU_IMAGE
65
- else:
66
- image = ML_JOB_IMAGE
67
- try:
68
- client = get_docker_client()
69
- client.api.inspect_image(image)
70
- except docker.errors.ImageNotFound:
71
- if gpu:
72
- cmd = "`ads opctl build-image -g job-local`"
73
- else:
74
- cmd = "`ads opctl build-image job-local`"
75
- raise RuntimeError(
76
- f"Please run {cmd} to build a local image for Jobs development first."
77
- )
78
-
79
-
80
- def _get_name(name: str, env_file: str) -> str:
81
- if not name and env_file:
82
- with open(env_file) as f:
83
- name = yaml.safe_load(f.read()).get("manifest").get("name", None)
84
- if not name:
85
- raise ValueError(
86
- "Either specify environment name in environment yaml or with `--name`."
87
- )
88
- return name
89
-
90
-
91
- def create(**kwargs) -> str:
92
- p = ConfigProcessor().step(ConfigMerger, **kwargs)
93
- exec_config = p.config["execution"]
94
- name = _get_name(exec_config.get("name"), exec_config.get("environment_file"))
95
- _create(
96
- name=name,
97
- version=exec_config["version"],
98
- env_file=exec_config["environment_file"],
99
- conda_pack_folder=exec_config["conda_pack_folder"],
100
- gpu=exec_config.get("gpu", False),
101
- overwrite=exec_config.get("overwrite", False),
102
- )
103
-
104
-
105
- def _create(
106
- name: str,
107
- version: str,
108
- env_file: str,
109
- conda_pack_folder: str,
110
- gpu: bool,
111
- overwrite: bool,
112
- prepare_publish: bool = False,
113
- ) -> str:
114
- """Create a conda pack given an environment yaml file under conda pack folder specified.
115
-
116
- Parameters
117
- ----------
118
- slug : str
119
- slug of the conda pack
120
- env_file : str
121
- path to conda environment yaml
122
- conda_pack_folder : str
123
- path to conda pack folder
124
- gpu : bool
125
- whether to build against GPU image
126
- overwrite : bool
127
- whether to overwrite existing pack of the same slug
128
- prepare_pubish : bool
129
- whether to create conda pack archive after conda pack is created
130
-
131
- Raises
132
- ------
133
- FileNotFoundError
134
- Environment file not found
135
- RuntimeError
136
- Environment file may be inproperly removed
137
- RuntimeError
138
- Creating pack failed
139
-
140
- Return
141
- ------
142
- str:
143
- slug of the environment
144
- """
145
-
146
- if not os.path.exists(env_file):
147
- raise FileNotFoundError(f"Environment file {env_file} is not found.")
148
-
149
- conda_dep = None
150
- with open(env_file) as mfile:
151
- conda_dep = yaml.safe_load(mfile.read())
152
- # If manifest exists in the environment.yaml file, use that
153
- manifest = conda_dep.get("manifest", {})
154
- slug = manifest.get(
155
- "slug", f"{name}_v{version}".replace(" ", "").replace(".", "_").lower()
156
- )
157
- pack_folder_path = os.path.join(
158
- os.path.abspath(os.path.expanduser(conda_pack_folder)), slug
159
- )
160
-
161
- if os.path.exists(pack_folder_path):
162
- overwrite = overwrite or click.confirm(
163
- f"Conda pack with slug {slug} already exists. Do you wish to overwrite?"
164
- )
165
- if overwrite:
166
- if (
167
- os.path.commonpath(
168
- [pack_folder_path, os.path.abspath(os.path.expanduser(env_file))]
169
- )
170
- == pack_folder_path
171
- ):
172
- raise RuntimeError(
173
- f"Environment file {os.path.abspath(os.path.expanduser(env_file))} is in {pack_folder_path}. Overwriting this folder will remove the file."
174
- )
175
- shutil.rmtree(pack_folder_path)
176
- else:
177
- return
178
-
179
- os.makedirs(pack_folder_path, exist_ok=True)
180
-
181
- logger.info(
182
- f"Preparing manifest. Manifest in the environment: {conda_dep.get('manifest')}"
183
- )
184
- manifest_template = _fetch_manifest_template()
185
- if "name" not in manifest:
186
- manifest_template["manifest"]["name"] = name
187
- manifest_template["manifest"]["slug"] = slug
188
- if "type" not in manifest:
189
- logger.info("Setting manifest to published")
190
- manifest_template["manifest"]["type"] = "published"
191
- if "version" not in manifest:
192
- manifest_template["manifest"]["version"] = version
193
- manifest_template["manifest"]["arch_type"] = "GPU" if gpu else "CPU"
194
-
195
- manifest_template["manifest"]["create_date"] = datetime.utcnow().strftime(
196
- "%a, %b %d, %Y, %H:%M:%S %Z UTC"
197
- )
198
-
199
- if not "manifest_version" in manifest:
200
- manifest_template["manifest"]["manifest_version"] = "1.0"
201
-
202
- logger.info(f"Creating conda environment {slug}")
203
- manifest_dict = {
204
- k: manifest_template["manifest"][k]
205
- for k in manifest_template["manifest"]
206
- if manifest_template["manifest"][k]
207
- }
208
- if "manifest" in conda_dep:
209
- conda_dep["manifest"].update(manifest_dict)
210
- else:
211
- conda_dep["manifest"] = manifest_dict
212
- logger.info(f"Updated conda environment manifest: {conda_dep.get('manifest')}")
213
-
214
- if is_in_notebook_session() or NO_CONTAINER:
215
- command = f"conda env create --prefix {pack_folder_path} --file {os.path.abspath(os.path.expanduser(env_file))}"
216
- proc = run_command(command, shell=True)
217
- if proc.returncode != 0:
218
- raise RuntimeError(
219
- f"Failed to create conda environment. (exit code {proc.returncode})"
220
- )
221
- else:
222
- _check_job_image_exists(gpu)
223
- docker_pack_folder_path = os.path.join(DEFAULT_IMAGE_HOME_DIR, slug)
224
- docker_env_file_path = os.path.join(
225
- DEFAULT_IMAGE_HOME_DIR, os.path.basename(env_file)
226
- )
227
-
228
- create_command = f"conda env create --prefix {docker_pack_folder_path} --file {docker_env_file_path}"
229
-
230
- volumes = {
231
- pack_folder_path: {"bind": docker_pack_folder_path},
232
- os.path.abspath(os.path.expanduser(env_file)): {
233
- "bind": docker_env_file_path
234
- },
235
- }
236
-
237
- if gpu:
238
- image = ML_JOB_GPU_IMAGE
239
- else:
240
- image = ML_JOB_IMAGE
241
- try:
242
- if prepare_publish:
243
- tmp_file = tempfile.NamedTemporaryFile(suffix=".yaml")
244
- # Save the manifest in the temp file that can be mounted inside the container so that archiving will work
245
- with open(tmp_file.name, "w") as f:
246
- yaml.safe_dump(conda_dep, f)
247
-
248
- pack_script = os.path.join(
249
- os.path.dirname(os.path.abspath(__file__)), "pack.py"
250
- )
251
- pack_command = f"python {os.path.join(DEFAULT_IMAGE_HOME_DIR, 'pack.py')} --conda-path {docker_pack_folder_path} --manifest-location {os.path.join(DEFAULT_IMAGE_HOME_DIR, 'manifest.yaml')}"
252
-
253
- # add pack script and manifest file to the mount so that archive can be created in the same container run
254
- condapack_script = {
255
- pack_script: {
256
- "bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, "pack.py")
257
- },
258
- tmp_file.name: {
259
- "bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, "manifest.yaml")
260
- },
261
- }
262
- volumes = {
263
- **volumes,
264
- **condapack_script,
265
- } # | not supported in python 3.8
266
-
267
- run_container(
268
- image=image,
269
- bind_volumes=volumes,
270
- entrypoint="/bin/bash -c ",
271
- env_vars={},
272
- command=f" '{create_command} && {pack_command}'",
273
- )
274
- else:
275
- run_container(
276
- image=image,
277
- bind_volumes=volumes,
278
- env_vars={},
279
- command=create_command,
280
- )
281
- except Exception:
282
- if os.path.exists(pack_folder_path):
283
- shutil.rmtree(pack_folder_path)
284
- raise RuntimeError(f"Could not create environment {slug}.")
285
-
286
- # Save the manifest file inside the host machine, where the conda environment is saved.
287
- manifest_location = f"{os.path.join(pack_folder_path, slug)}_manifest.yaml"
288
- with open(manifest_location, "w") as mfile:
289
- yaml.safe_dump(conda_dep, mfile)
290
-
291
- logger.info(f"Environment `{slug}` setup complete.")
292
- print(f"Pack {slug} created under {pack_folder_path}.")
293
-
294
- return slug
295
-
296
-
297
- def _fetch_pack_path(
298
- slug: str,
299
- conda_pack_os_prefix: str,
300
- oci_config: str,
301
- oci_profile: str,
302
- auth_type: str,
303
- ) -> str:
304
- oci_auth = create_signer(auth_type, oci_config, oci_profile)
305
- fs = ocifs.OCIFileSystem(**oci_auth)
306
- fnames = fs.ls(conda_pack_os_prefix, detail=True, refresh=True)
307
- for f in fnames:
308
- if os.path.basename(f["name"]) == slug:
309
- return f"oci://{f['name']}"
310
- elif f["type"] == "directory":
311
- path = _fetch_pack_path(
312
- slug, f"oci://{f['name']}", oci_config, oci_profile, auth_type
313
- )
314
- if path:
315
- return path
316
- return None
317
-
318
-
319
- def install(**kwargs) -> None:
320
- p = ConfigProcessor().step(ConfigMerger, **kwargs)
321
- exec_config = p.config["execution"]
322
- if exec_config.get("slug") and exec_config.get("conda_pack_os_prefix"):
323
- conda_uri = _fetch_pack_path(
324
- exec_config["slug"],
325
- exec_config["conda_pack_os_prefix"],
326
- exec_config.get("oci_config"),
327
- exec_config.get("oci_profile"),
328
- exec_config.get("auth"),
329
- )
330
- if not conda_uri:
331
- raise FileNotFoundError(
332
- f"{exec_config['slug']} not found under {exec_config['conda_pack_os_prefix']}"
333
- )
334
- elif exec_config.get("conda_uri"):
335
- conda_uri = exec_config["conda_uri"]
336
- else:
337
- raise ValueError(
338
- "Either `--conda-uri`, or `--slug` and `--conda-pack-os-prefix` has to be specified."
339
- )
340
- _install(
341
- conda_uri,
342
- os.path.abspath(os.path.expanduser(exec_config["conda_pack_folder"])),
343
- exec_config.get("oci_config"),
344
- exec_config.get("oci_profile"),
345
- overwrite=exec_config.get("overwrite", False),
346
- debug=kwargs.get("debug", False),
347
- auth_type=exec_config.get("auth"),
348
- )
349
-
350
-
351
- def _install(
352
- conda_uri: str,
353
- conda_pack_folder: str,
354
- oci_config: str = None,
355
- oci_profile: str = None,
356
- overwrite: bool = False,
357
- debug: bool = False,
358
- auth_type: str = None,
359
- ) -> None:
360
- """
361
- Install conda pack.
362
-
363
- Parameters
364
- ----------
365
- conda_uri: str
366
- OCI object storage uri to conda pack
367
- conda_pack_folder: str
368
- local folder to save conda packs
369
- oci_config: str
370
- path to OCI config file
371
- oci_profile: str
372
- OCI config profile
373
- overwrite: bool
374
- whether to overwrite existing pack
375
- debug: bool
376
- whether to turn on debug mode
377
- auth_type : str
378
- authentication method
379
- Returns
380
- -------
381
- None
382
- """
383
- ns, bucket, path, slug = parse_conda_uri(conda_uri)
384
- if bucket == "service-conda-packs":
385
- raise ValueError(
386
- "Download service conda pack is not allowed. Only custom conda pack can be downloaded to local machine. You need to publish it to your own bucket first."
387
- )
388
- os.makedirs(conda_pack_folder, exist_ok=True)
389
- pack_path = os.path.join(os.path.expanduser(conda_pack_folder), slug + ".tar.gz")
390
- pack_folder_path = os.path.join(os.path.expanduser(conda_pack_folder), slug)
391
-
392
- if not (is_in_notebook_session() or NO_CONTAINER):
393
- _check_job_image_exists(gpu=False)
394
-
395
- while os.path.exists(pack_folder_path):
396
- if overwrite:
397
- break
398
- else:
399
- ans = click.prompt(
400
- f"conda pack with slug {slug} already exists. Enter a new name or 'o' for overwrite:",
401
- default="o",
402
- )
403
- if ans == "o":
404
- overwrite = True
405
- break
406
- else:
407
- slug = ans
408
- pack_path = os.path.join(
409
- os.path.expanduser(conda_pack_folder), slug + ".tar.gz"
410
- )
411
- pack_folder_path = os.path.join(
412
- os.path.expanduser(conda_pack_folder), slug
413
- )
414
-
415
- if oci_config is None or oci_profile is None:
416
- download_command = [
417
- "oci",
418
- "os",
419
- "object",
420
- "get",
421
- "--name",
422
- f"{path}",
423
- "-bn",
424
- bucket,
425
- "-ns",
426
- ns,
427
- "--file",
428
- pack_path,
429
- "--auth",
430
- auth_type,
431
- ]
432
- else:
433
- oci_config = os.path.abspath(os.path.expanduser(oci_config))
434
- parser = read_from_ini(os.path.expanduser(oci_config))
435
- if oci_profile not in parser:
436
- raise ValueError(f"PROFILE {oci_profile} not found in {oci_config}.")
437
- download_command = [
438
- "oci",
439
- "os",
440
- "object",
441
- "get",
442
- "--name",
443
- f"{path}",
444
- "-bn",
445
- bucket,
446
- "-ns",
447
- ns,
448
- "--file",
449
- pack_path,
450
- "--profile",
451
- oci_profile,
452
- "--config-file",
453
- oci_config,
454
- ]
455
- if debug:
456
- download_command.append("-d")
457
- print(" ".join([shlex.quote(c) for c in download_command]))
458
- process = subprocess.Popen(download_command)
459
- process.communicate()
460
- if process.returncode != 0:
461
- if os.path.exists(pack_path):
462
- os.remove(pack_path)
463
- raise RuntimeError(
464
- f"Downloading pack failed with return code {process.returncode}. Please double check the path and make sure you have access."
465
- )
466
- else:
467
- print(f"Download {conda_uri} completed")
468
-
469
- print(f"Start unpacking {pack_path}")
470
- os.makedirs(pack_folder_path, exist_ok=True)
471
-
472
- process = subprocess.Popen(["tar", "-xf", pack_path, "-C", pack_folder_path])
473
- process.communicate()
474
- if process.returncode != 0:
475
- shutil.rmtree(pack_folder_path)
476
- raise Exception(
477
- f"Error extracting {pack_path} to {pack_folder_path}. Please try again."
478
- )
479
-
480
- # Run the conda-unpack script to clean up prefixes
481
- # This will fix problems related to repeated "placehold" paths.
482
- # See https://conda.github.io/conda-pack/#commandline-usage
483
- unpack_script = os.path.join(pack_folder_path, "bin", "conda-unpack")
484
- if os.path.exists(unpack_script):
485
- if is_in_notebook_session() or NO_CONTAINER:
486
- run_command(unpack_script)
487
- else:
488
- volumes = {
489
- pack_folder_path: {"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, slug)}
490
- }
491
- try:
492
- run_container(
493
- image=ML_JOB_IMAGE,
494
- bind_volumes=volumes,
495
- env_vars={},
496
- command=os.path.join(
497
- DEFAULT_IMAGE_HOME_DIR, slug, "bin/conda-unpack"
498
- ),
499
- )
500
- except Exception:
501
- raise RuntimeError(f"Error unpacking environment {slug}.")
502
- if os.path.exists(os.path.join(pack_folder_path, "spark-defaults.conf")):
503
- if is_in_notebook_session() or NO_CONTAINER:
504
- if not os.path.exists(DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR):
505
- os.makedirs(DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR)
506
- shutil.copy(
507
- os.path.join(pack_folder_path, "spark-defaults.conf"),
508
- DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR,
509
- )
510
- for file in os.listdir(pack_folder_path):
511
- if os.path.splitext(file)[1] == ".jar":
512
- shutil.copy(
513
- os.path.join(pack_folder_path, file),
514
- DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR,
515
- )
516
- else:
517
- with open(os.path.join(pack_folder_path, "spark-defaults.conf")) as f:
518
- confs = f.read()
519
- confs = confs.replace(
520
- DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR,
521
- os.path.join(DEFAULT_IMAGE_CONDA_DIR, slug),
522
- )
523
- with open(
524
- os.path.join(pack_folder_path, "spark-defaults.conf"), "w"
525
- ) as f:
526
- f.write(confs)
527
-
528
- os.remove(pack_path)
529
- manifest_path = glob.glob(os.path.join(pack_folder_path, "*_manifest.yaml"))[0]
530
- with open(manifest_path) as f:
531
- env = yaml.safe_load(f.read())
532
- env["manifest"]["pack_uri"] = conda_uri
533
- with open(manifest_path, "w") as f:
534
- yaml.safe_dump(env, f)
535
- print(f"{slug} set up at {pack_folder_path}.")
536
-
537
-
538
- def publish(**kwargs) -> None:
539
- p = ConfigProcessor().step(ConfigMerger, **kwargs)
540
- exec_config = p.config["execution"]
541
- skip_archive = False
542
- if exec_config.get("environment_file", None):
543
- name = _get_name(exec_config.get("name"), exec_config.get("environment_file"))
544
- slug = _create(
545
- name=name,
546
- version=exec_config["version"],
547
- env_file=exec_config["environment_file"],
548
- conda_pack_folder=exec_config["conda_pack_folder"],
549
- gpu=exec_config.get("gpu", False),
550
- overwrite=exec_config["overwrite"],
551
- prepare_publish=True,
552
- )
553
- skip_archive = (
554
- True # The conda pack archive is already created during create process.
555
- )
556
- else:
557
- slug = exec_config.get("slug")
558
- if not slug:
559
- raise ValueError("Please specify slug of the conda pack via `--slug`.")
560
- if not exec_config.get("conda_pack_os_prefix"):
561
- raise ValueError(
562
- "Please specify object storage path to save conda pack either via `ads opctl configure` or `--conda-pack-os-prefix`."
563
- )
564
- _publish(
565
- conda_slug=slug,
566
- conda_uri_prefix=exec_config["conda_pack_os_prefix"],
567
- conda_pack_folder=exec_config["conda_pack_folder"],
568
- oci_config=exec_config.get("oci_config"),
569
- oci_profile=exec_config.get("oci_profile"),
570
- overwrite=exec_config["overwrite"],
571
- auth_type=exec_config["auth"],
572
- skip_archive=skip_archive,
573
- )
574
-
575
-
576
- def _publish(
577
- conda_slug: str,
578
- conda_uri_prefix: str,
579
- conda_pack_folder: str,
580
- oci_config: str,
581
- oci_profile: str,
582
- overwrite: bool,
583
- auth_type: str,
584
- skip_archive: bool = False,
585
- ) -> None:
586
- """Publish a local conda pack to object storage location
587
-
588
- Parameters
589
- ----------
590
- conda_slug : str
591
- slug of conda pack
592
- conda_uri_prefix : str
593
- object storage prefix to save conda pack
594
- conda_pack_folder : str
595
- path to local conda folder
596
- oci_config : str
597
- oci config file path
598
- oci_profile : str
599
- oci config profile
600
- overwrite : bool
601
- whether to overwrite existing pack of the same slug
602
- auth_type : str
603
- authentication method
604
-
605
- Raises
606
- ------
607
- FileNotFoundError
608
- local conda pack folder not found
609
- FileNotFoundError
610
- manifest file not found inside conda pack
611
- RuntimeError
612
- IP packages found inside conda pack
613
- RuntimeError
614
- Packing conda failed
615
- RuntimeError
616
- Pack file not found
617
- """
618
- ns, bucket, prefix, _ = parse_conda_uri(conda_uri_prefix)
619
-
620
- # ======= check if conda pack and manifest exists ==============
621
- pack_folder_path = os.path.abspath(
622
- os.path.expanduser(os.path.join(conda_pack_folder, conda_slug))
623
- )
624
- if not os.path.exists(pack_folder_path):
625
- raise FileNotFoundError(
626
- f"Could not find environment {conda_slug} in {conda_pack_folder}."
627
- )
628
- paths = glob.glob(os.path.join(pack_folder_path, "*_manifest.yaml"))
629
- if len(paths) != 1:
630
- raise FileNotFoundError(
631
- "Could not locate manifest file in the provided environment."
632
- )
633
- else:
634
- manifest_location = paths[0]
635
-
636
- # ===== check if pack contains IP packages =========
637
- print(f"Loading environment information from {manifest_location}.")
638
- with open(manifest_location) as mlf:
639
- env = yaml.safe_load(mlf.read())
640
- if "IP" in env["manifest"] and env["manifest"]["IP"].lower() == "y":
641
- raise RuntimeError("This environment has IP restricted packages.")
642
-
643
- oci_auth = create_signer(auth_type, oci_config, oci_profile)
644
- client = OCIClientFactory(**oci_auth).object_storage
645
- publish_slug = conda_slug
646
- if not overwrite:
647
- existing_packs = client.list_objects(ns, bucket, prefix=prefix).data.objects
648
- pack_names = [os.path.basename(pack.name) for pack in existing_packs]
649
- while publish_slug in pack_names:
650
- ans = click.prompt(
651
- f"{conda_slug} exists in {conda_uri_prefix}. Enter a new name or overwrite (o)",
652
- default="o",
653
- )
654
- if ans == "o":
655
- break
656
- else:
657
- publish_slug = "_".join(ans.lower().split(" "))
658
-
659
- pack_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pack.py")
660
- if not skip_archive:
661
- if is_in_notebook_session() or NO_CONTAINER:
662
- # Set the CONDA_PUBLISH_TYPE environment variable so that the `type` attribute inside the manifest is not changed
663
- publish_type = os.environ.get("CONDA_PUBLISH_TYPE")
664
- command = f"python {pack_script} --conda-path {pack_folder_path}"
665
- if publish_type:
666
- command = f"CONDA_PUBLISH_TYPE={publish_type} {command}"
667
- proc = run_command(command, shell=True)
668
- if proc.returncode != 0:
669
- raise RuntimeError(
670
- f"Failed to archive the conda environment. (exit code {proc.returncode})"
671
- )
672
- else:
673
- volumes = {
674
- pack_folder_path: {
675
- "bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, conda_slug)
676
- },
677
- pack_script: {"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, "pack.py")},
678
- }
679
- command = f"python {os.path.join(DEFAULT_IMAGE_HOME_DIR, 'pack.py')} --conda-path {os.path.join(DEFAULT_IMAGE_HOME_DIR, conda_slug)}"
680
- gpu = env["manifest"]["arch_type"] == "GPU"
681
- _check_job_image_exists(gpu)
682
- if gpu:
683
- image = ML_JOB_GPU_IMAGE
684
- else:
685
- image = ML_JOB_IMAGE
686
- try:
687
- run_container(
688
- image=image, bind_volumes=volumes, env_vars={}, command=command
689
- )
690
- except Exception:
691
- raise RuntimeError(f"Could not pack environment {conda_slug}.")
692
- NOT_ALLOWED_CHARS = "@#$%^&*/"
693
-
694
- if any(chr in conda_slug for chr in NOT_ALLOWED_CHARS):
695
- raise ValueError(
696
- f"Invalid conda_slug. Found {NOT_ALLOWED_CHARS} in slug name. Please use a different slug name."
697
- )
698
- pack_file = os.path.join(pack_folder_path, f"{conda_slug}.tar.gz")
699
- if not os.path.exists(pack_file):
700
- raise RuntimeError(f"Pack {pack_file} was not created.")
701
- pack_size = round(os.path.getsize(pack_file) / 2**20, 2)
702
-
703
- with open(manifest_location) as mlf:
704
- env = yaml.safe_load(mlf.read())
705
- manifest = env["manifest"]
706
- manifest["slug"] = conda_slug
707
- manifest["create_date"] = datetime.utcnow().strftime(
708
- "%a, %b %d, %Y, %H:%M:%S %Z UTC"
709
- )
710
- manifest["size_mb"] = pack_size
711
- ns, bucket, prefix, _ = parse_conda_uri(conda_uri_prefix)
712
-
713
- pack_uri = os.path.join(
714
- conda_uri_prefix,
715
- manifest.get("arch_type", "CPU").lower(),
716
- manifest["name"],
717
- str(manifest["version"]),
718
- publish_slug,
719
- )
720
- if os.environ.get("CONDA_PUBLISH_TYPE") != "service":
721
- # Set these values only for published conda pack
722
- manifest["pack_path"] = os.path.join(
723
- prefix,
724
- manifest.get("arch_type", "CPU").lower(),
725
- manifest["name"],
726
- str(manifest["version"]),
727
- publish_slug,
728
- )
729
- manifest["pack_uri"] = pack_uri
730
- else:
731
- manifest["type"] = "published"
732
-
733
- with open(manifest_location, "w") as f:
734
- yaml.safe_dump(env, f)
735
- if pack_size > 100:
736
- MultiPartUploader(
737
- pack_file, pack_uri, 10, oci_config, oci_profile, auth_type
738
- ).upload(manifest)
739
- else:
740
- with open(pack_file, "rb") as pkf:
741
- client.put_object(
742
- ns,
743
- bucket,
744
- manifest["pack_path"],
745
- pkf,
746
- opc_meta={"manifest": json.dumps(manifest)},
747
- )
748
- print(f"Conda pack {pack_uri} published.")
749
- os.remove(pack_file)