oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (858) hide show
  1. ads/aqua/__init__.py +40 -0
  2. ads/aqua/app.py +507 -0
  3. ads/aqua/cli.py +96 -0
  4. ads/aqua/client/__init__.py +3 -0
  5. ads/aqua/client/client.py +836 -0
  6. ads/aqua/client/openai_client.py +305 -0
  7. ads/aqua/common/__init__.py +5 -0
  8. ads/aqua/common/decorator.py +125 -0
  9. ads/aqua/common/entities.py +274 -0
  10. ads/aqua/common/enums.py +134 -0
  11. ads/aqua/common/errors.py +109 -0
  12. ads/aqua/common/utils.py +1295 -0
  13. ads/aqua/config/__init__.py +4 -0
  14. ads/aqua/config/container_config.py +246 -0
  15. ads/aqua/config/evaluation/__init__.py +4 -0
  16. ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
  17. ads/aqua/config/utils/__init__.py +4 -0
  18. ads/aqua/config/utils/serializer.py +339 -0
  19. ads/aqua/constants.py +116 -0
  20. ads/aqua/data.py +14 -0
  21. ads/aqua/dummy_data/icon.txt +1 -0
  22. ads/aqua/dummy_data/oci_model_deployments.json +56 -0
  23. ads/aqua/dummy_data/oci_models.json +1 -0
  24. ads/aqua/dummy_data/readme.md +26 -0
  25. ads/aqua/evaluation/__init__.py +8 -0
  26. ads/aqua/evaluation/constants.py +53 -0
  27. ads/aqua/evaluation/entities.py +186 -0
  28. ads/aqua/evaluation/errors.py +70 -0
  29. ads/aqua/evaluation/evaluation.py +1814 -0
  30. ads/aqua/extension/__init__.py +42 -0
  31. ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
  32. ads/aqua/extension/base_handler.py +90 -0
  33. ads/aqua/extension/common_handler.py +121 -0
  34. ads/aqua/extension/common_ws_msg_handler.py +36 -0
  35. ads/aqua/extension/deployment_handler.py +381 -0
  36. ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
  37. ads/aqua/extension/errors.py +30 -0
  38. ads/aqua/extension/evaluation_handler.py +129 -0
  39. ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
  40. ads/aqua/extension/finetune_handler.py +96 -0
  41. ads/aqua/extension/model_handler.py +390 -0
  42. ads/aqua/extension/models/__init__.py +0 -0
  43. ads/aqua/extension/models/ws_models.py +145 -0
  44. ads/aqua/extension/models_ws_msg_handler.py +50 -0
  45. ads/aqua/extension/ui_handler.py +300 -0
  46. ads/aqua/extension/ui_websocket_handler.py +130 -0
  47. ads/aqua/extension/utils.py +133 -0
  48. ads/aqua/finetuning/__init__.py +7 -0
  49. ads/aqua/finetuning/constants.py +23 -0
  50. ads/aqua/finetuning/entities.py +181 -0
  51. ads/aqua/finetuning/finetuning.py +749 -0
  52. ads/aqua/model/__init__.py +8 -0
  53. ads/aqua/model/constants.py +60 -0
  54. ads/aqua/model/entities.py +385 -0
  55. ads/aqua/model/enums.py +32 -0
  56. ads/aqua/model/model.py +2134 -0
  57. ads/aqua/model/utils.py +52 -0
  58. ads/aqua/modeldeployment/__init__.py +6 -0
  59. ads/aqua/modeldeployment/constants.py +10 -0
  60. ads/aqua/modeldeployment/deployment.py +1315 -0
  61. ads/aqua/modeldeployment/entities.py +653 -0
  62. ads/aqua/modeldeployment/utils.py +543 -0
  63. ads/aqua/resources/gpu_shapes_index.json +94 -0
  64. ads/aqua/server/__init__.py +4 -0
  65. ads/aqua/server/__main__.py +24 -0
  66. ads/aqua/server/app.py +47 -0
  67. ads/aqua/server/aqua_spec.yml +1291 -0
  68. ads/aqua/training/__init__.py +4 -0
  69. ads/aqua/training/exceptions.py +476 -0
  70. ads/aqua/ui.py +519 -0
  71. ads/automl/__init__.py +9 -0
  72. ads/automl/driver.py +330 -0
  73. ads/automl/provider.py +975 -0
  74. ads/bds/__init__.py +5 -0
  75. ads/bds/auth.py +127 -0
  76. ads/bds/big_data_service.py +255 -0
  77. ads/catalog/__init__.py +19 -0
  78. ads/catalog/model.py +1576 -0
  79. ads/catalog/notebook.py +461 -0
  80. ads/catalog/project.py +468 -0
  81. ads/catalog/summary.py +178 -0
  82. ads/common/__init__.py +11 -0
  83. ads/common/analyzer.py +65 -0
  84. ads/common/artifact/.model-ignore +63 -0
  85. ads/common/artifact/__init__.py +10 -0
  86. ads/common/auth.py +1122 -0
  87. ads/common/card_identifier.py +83 -0
  88. ads/common/config.py +647 -0
  89. ads/common/data.py +165 -0
  90. ads/common/decorator/__init__.py +9 -0
  91. ads/common/decorator/argument_to_case.py +88 -0
  92. ads/common/decorator/deprecate.py +69 -0
  93. ads/common/decorator/require_nonempty_arg.py +65 -0
  94. ads/common/decorator/runtime_dependency.py +178 -0
  95. ads/common/decorator/threaded.py +97 -0
  96. ads/common/decorator/utils.py +35 -0
  97. ads/common/dsc_file_system.py +303 -0
  98. ads/common/error.py +14 -0
  99. ads/common/extended_enum.py +81 -0
  100. ads/common/function/__init__.py +5 -0
  101. ads/common/function/fn_util.py +142 -0
  102. ads/common/function/func_conf.yaml +25 -0
  103. ads/common/ipython.py +76 -0
  104. ads/common/model.py +679 -0
  105. ads/common/model_artifact.py +1759 -0
  106. ads/common/model_artifact_schema.json +107 -0
  107. ads/common/model_export_util.py +664 -0
  108. ads/common/model_metadata.py +24 -0
  109. ads/common/object_storage_details.py +296 -0
  110. ads/common/oci_client.py +179 -0
  111. ads/common/oci_datascience.py +46 -0
  112. ads/common/oci_logging.py +1144 -0
  113. ads/common/oci_mixin.py +957 -0
  114. ads/common/oci_resource.py +136 -0
  115. ads/common/serializer.py +559 -0
  116. ads/common/utils.py +1852 -0
  117. ads/common/word_lists.py +1491 -0
  118. ads/common/work_request.py +189 -0
  119. ads/config.py +1 -0
  120. ads/data_labeling/__init__.py +13 -0
  121. ads/data_labeling/boundingbox.py +253 -0
  122. ads/data_labeling/constants.py +47 -0
  123. ads/data_labeling/data_labeling_service.py +244 -0
  124. ads/data_labeling/interface/__init__.py +5 -0
  125. ads/data_labeling/interface/loader.py +16 -0
  126. ads/data_labeling/interface/parser.py +16 -0
  127. ads/data_labeling/interface/reader.py +23 -0
  128. ads/data_labeling/loader/__init__.py +5 -0
  129. ads/data_labeling/loader/file_loader.py +241 -0
  130. ads/data_labeling/metadata.py +110 -0
  131. ads/data_labeling/mixin/__init__.py +5 -0
  132. ads/data_labeling/mixin/data_labeling.py +232 -0
  133. ads/data_labeling/ner.py +129 -0
  134. ads/data_labeling/parser/__init__.py +5 -0
  135. ads/data_labeling/parser/dls_record_parser.py +388 -0
  136. ads/data_labeling/parser/export_metadata_parser.py +94 -0
  137. ads/data_labeling/parser/export_record_parser.py +473 -0
  138. ads/data_labeling/reader/__init__.py +5 -0
  139. ads/data_labeling/reader/dataset_reader.py +574 -0
  140. ads/data_labeling/reader/dls_record_reader.py +121 -0
  141. ads/data_labeling/reader/export_record_reader.py +62 -0
  142. ads/data_labeling/reader/jsonl_reader.py +75 -0
  143. ads/data_labeling/reader/metadata_reader.py +203 -0
  144. ads/data_labeling/reader/record_reader.py +263 -0
  145. ads/data_labeling/record.py +52 -0
  146. ads/data_labeling/visualizer/__init__.py +5 -0
  147. ads/data_labeling/visualizer/image_visualizer.py +525 -0
  148. ads/data_labeling/visualizer/text_visualizer.py +357 -0
  149. ads/database/__init__.py +5 -0
  150. ads/database/connection.py +338 -0
  151. ads/dataset/__init__.py +10 -0
  152. ads/dataset/capabilities.md +51 -0
  153. ads/dataset/classification_dataset.py +339 -0
  154. ads/dataset/correlation.py +226 -0
  155. ads/dataset/correlation_plot.py +563 -0
  156. ads/dataset/dask_series.py +173 -0
  157. ads/dataset/dataframe_transformer.py +110 -0
  158. ads/dataset/dataset.py +1979 -0
  159. ads/dataset/dataset_browser.py +360 -0
  160. ads/dataset/dataset_with_target.py +995 -0
  161. ads/dataset/exception.py +25 -0
  162. ads/dataset/factory.py +987 -0
  163. ads/dataset/feature_engineering_transformer.py +35 -0
  164. ads/dataset/feature_selection.py +107 -0
  165. ads/dataset/forecasting_dataset.py +26 -0
  166. ads/dataset/helper.py +1450 -0
  167. ads/dataset/label_encoder.py +99 -0
  168. ads/dataset/mixin/__init__.py +5 -0
  169. ads/dataset/mixin/dataset_accessor.py +134 -0
  170. ads/dataset/pipeline.py +58 -0
  171. ads/dataset/plot.py +710 -0
  172. ads/dataset/progress.py +86 -0
  173. ads/dataset/recommendation.py +297 -0
  174. ads/dataset/recommendation_transformer.py +502 -0
  175. ads/dataset/regression_dataset.py +14 -0
  176. ads/dataset/sampled_dataset.py +1050 -0
  177. ads/dataset/target.py +98 -0
  178. ads/dataset/timeseries.py +18 -0
  179. ads/dbmixin/__init__.py +5 -0
  180. ads/dbmixin/db_pandas_accessor.py +153 -0
  181. ads/environment/__init__.py +9 -0
  182. ads/environment/ml_runtime.py +66 -0
  183. ads/evaluations/README.md +14 -0
  184. ads/evaluations/__init__.py +109 -0
  185. ads/evaluations/evaluation_plot.py +983 -0
  186. ads/evaluations/evaluator.py +1334 -0
  187. ads/evaluations/statistical_metrics.py +543 -0
  188. ads/experiments/__init__.py +9 -0
  189. ads/experiments/capabilities.md +0 -0
  190. ads/explanations/__init__.py +21 -0
  191. ads/explanations/base_explainer.py +142 -0
  192. ads/explanations/capabilities.md +83 -0
  193. ads/explanations/explainer.py +190 -0
  194. ads/explanations/mlx_global_explainer.py +1050 -0
  195. ads/explanations/mlx_interface.py +386 -0
  196. ads/explanations/mlx_local_explainer.py +287 -0
  197. ads/explanations/mlx_whatif_explainer.py +201 -0
  198. ads/feature_engineering/__init__.py +20 -0
  199. ads/feature_engineering/accessor/__init__.py +5 -0
  200. ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
  201. ads/feature_engineering/accessor/mixin/__init__.py +5 -0
  202. ads/feature_engineering/accessor/mixin/correlation.py +166 -0
  203. ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
  204. ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
  205. ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
  206. ads/feature_engineering/accessor/mixin/utils.py +65 -0
  207. ads/feature_engineering/accessor/series_accessor.py +431 -0
  208. ads/feature_engineering/adsimage/__init__.py +5 -0
  209. ads/feature_engineering/adsimage/image.py +192 -0
  210. ads/feature_engineering/adsimage/image_reader.py +170 -0
  211. ads/feature_engineering/adsimage/interface/__init__.py +5 -0
  212. ads/feature_engineering/adsimage/interface/reader.py +19 -0
  213. ads/feature_engineering/adsstring/__init__.py +7 -0
  214. ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
  215. ads/feature_engineering/adsstring/string/__init__.py +8 -0
  216. ads/feature_engineering/data_schema.json +57 -0
  217. ads/feature_engineering/dataset/__init__.py +5 -0
  218. ads/feature_engineering/dataset/zip_code_data.py +42062 -0
  219. ads/feature_engineering/exceptions.py +40 -0
  220. ads/feature_engineering/feature_type/__init__.py +133 -0
  221. ads/feature_engineering/feature_type/address.py +184 -0
  222. ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
  223. ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
  224. ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
  225. ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
  226. ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
  227. ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
  228. ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
  229. ads/feature_engineering/feature_type/adsstring/string.py +258 -0
  230. ads/feature_engineering/feature_type/base.py +58 -0
  231. ads/feature_engineering/feature_type/boolean.py +183 -0
  232. ads/feature_engineering/feature_type/category.py +146 -0
  233. ads/feature_engineering/feature_type/constant.py +137 -0
  234. ads/feature_engineering/feature_type/continuous.py +151 -0
  235. ads/feature_engineering/feature_type/creditcard.py +314 -0
  236. ads/feature_engineering/feature_type/datetime.py +190 -0
  237. ads/feature_engineering/feature_type/discrete.py +134 -0
  238. ads/feature_engineering/feature_type/document.py +43 -0
  239. ads/feature_engineering/feature_type/gis.py +251 -0
  240. ads/feature_engineering/feature_type/handler/__init__.py +5 -0
  241. ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
  242. ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
  243. ads/feature_engineering/feature_type/handler/warnings.py +128 -0
  244. ads/feature_engineering/feature_type/integer.py +142 -0
  245. ads/feature_engineering/feature_type/ip_address.py +144 -0
  246. ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
  247. ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
  248. ads/feature_engineering/feature_type/lat_long.py +256 -0
  249. ads/feature_engineering/feature_type/object.py +43 -0
  250. ads/feature_engineering/feature_type/ordinal.py +132 -0
  251. ads/feature_engineering/feature_type/phone_number.py +135 -0
  252. ads/feature_engineering/feature_type/string.py +171 -0
  253. ads/feature_engineering/feature_type/text.py +93 -0
  254. ads/feature_engineering/feature_type/unknown.py +43 -0
  255. ads/feature_engineering/feature_type/zip_code.py +164 -0
  256. ads/feature_engineering/feature_type_manager.py +406 -0
  257. ads/feature_engineering/schema.py +795 -0
  258. ads/feature_engineering/utils.py +245 -0
  259. ads/feature_store/.readthedocs.yaml +19 -0
  260. ads/feature_store/README.md +65 -0
  261. ads/feature_store/__init__.py +9 -0
  262. ads/feature_store/common/__init__.py +0 -0
  263. ads/feature_store/common/enums.py +339 -0
  264. ads/feature_store/common/exceptions.py +18 -0
  265. ads/feature_store/common/spark_session_singleton.py +125 -0
  266. ads/feature_store/common/utils/__init__.py +0 -0
  267. ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
  268. ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
  269. ads/feature_store/common/utils/transformation_utils.py +82 -0
  270. ads/feature_store/common/utils/utility.py +403 -0
  271. ads/feature_store/data_validation/__init__.py +0 -0
  272. ads/feature_store/data_validation/great_expectation.py +129 -0
  273. ads/feature_store/dataset.py +1230 -0
  274. ads/feature_store/dataset_job.py +530 -0
  275. ads/feature_store/docs/Dockerfile +7 -0
  276. ads/feature_store/docs/Makefile +44 -0
  277. ads/feature_store/docs/conf.py +28 -0
  278. ads/feature_store/docs/requirements.txt +14 -0
  279. ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
  280. ads/feature_store/docs/source/cicd.rst +137 -0
  281. ads/feature_store/docs/source/conf.py +86 -0
  282. ads/feature_store/docs/source/data_versioning.rst +33 -0
  283. ads/feature_store/docs/source/dataset.rst +388 -0
  284. ads/feature_store/docs/source/dataset_job.rst +27 -0
  285. ads/feature_store/docs/source/demo.rst +70 -0
  286. ads/feature_store/docs/source/entity.rst +78 -0
  287. ads/feature_store/docs/source/feature_group.rst +624 -0
  288. ads/feature_store/docs/source/feature_group_job.rst +29 -0
  289. ads/feature_store/docs/source/feature_store.rst +122 -0
  290. ads/feature_store/docs/source/feature_store_class.rst +123 -0
  291. ads/feature_store/docs/source/feature_validation.rst +66 -0
  292. ads/feature_store/docs/source/figures/cicd.png +0 -0
  293. ads/feature_store/docs/source/figures/data_validation.png +0 -0
  294. ads/feature_store/docs/source/figures/data_versioning.png +0 -0
  295. ads/feature_store/docs/source/figures/dataset.gif +0 -0
  296. ads/feature_store/docs/source/figures/dataset.png +0 -0
  297. ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
  298. ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
  299. ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
  300. ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
  301. ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
  302. ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
  303. ads/feature_store/docs/source/figures/entity.png +0 -0
  304. ads/feature_store/docs/source/figures/feature_group.png +0 -0
  305. ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
  306. ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
  307. ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
  308. ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
  309. ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
  310. ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
  311. ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
  312. ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
  313. ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
  314. ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
  315. ads/feature_store/docs/source/figures/overview.png +0 -0
  316. ads/feature_store/docs/source/figures/resource_manager.png +0 -0
  317. ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
  318. ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
  319. ads/feature_store/docs/source/figures/stats_1.png +0 -0
  320. ads/feature_store/docs/source/figures/stats_2.png +0 -0
  321. ads/feature_store/docs/source/figures/stats_d.png +0 -0
  322. ads/feature_store/docs/source/figures/stats_fg.png +0 -0
  323. ads/feature_store/docs/source/figures/transformation.png +0 -0
  324. ads/feature_store/docs/source/figures/transformations.gif +0 -0
  325. ads/feature_store/docs/source/figures/validation.png +0 -0
  326. ads/feature_store/docs/source/figures/validation_fg.png +0 -0
  327. ads/feature_store/docs/source/figures/validation_results.png +0 -0
  328. ads/feature_store/docs/source/figures/validation_summary.png +0 -0
  329. ads/feature_store/docs/source/index.rst +81 -0
  330. ads/feature_store/docs/source/module.rst +8 -0
  331. ads/feature_store/docs/source/notebook.rst +94 -0
  332. ads/feature_store/docs/source/overview.rst +47 -0
  333. ads/feature_store/docs/source/quickstart.rst +176 -0
  334. ads/feature_store/docs/source/release_notes.rst +194 -0
  335. ads/feature_store/docs/source/setup_feature_store.rst +81 -0
  336. ads/feature_store/docs/source/statistics.rst +58 -0
  337. ads/feature_store/docs/source/transformation.rst +199 -0
  338. ads/feature_store/docs/source/ui.rst +65 -0
  339. ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
  340. ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
  341. ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
  342. ads/feature_store/entity.py +718 -0
  343. ads/feature_store/execution_strategy/__init__.py +0 -0
  344. ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
  345. ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
  346. ads/feature_store/execution_strategy/engine/__init__.py +0 -0
  347. ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
  348. ads/feature_store/execution_strategy/execution_strategy.py +113 -0
  349. ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
  350. ads/feature_store/execution_strategy/spark/__init__.py +0 -0
  351. ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
  352. ads/feature_store/feature.py +192 -0
  353. ads/feature_store/feature_group.py +1494 -0
  354. ads/feature_store/feature_group_expectation.py +346 -0
  355. ads/feature_store/feature_group_job.py +602 -0
  356. ads/feature_store/feature_lineage/__init__.py +0 -0
  357. ads/feature_store/feature_lineage/graphviz_service.py +180 -0
  358. ads/feature_store/feature_option_details.py +50 -0
  359. ads/feature_store/feature_statistics/__init__.py +0 -0
  360. ads/feature_store/feature_statistics/statistics_service.py +99 -0
  361. ads/feature_store/feature_store.py +699 -0
  362. ads/feature_store/feature_store_registrar.py +518 -0
  363. ads/feature_store/input_feature_detail.py +149 -0
  364. ads/feature_store/mixin/__init__.py +4 -0
  365. ads/feature_store/mixin/oci_feature_store.py +145 -0
  366. ads/feature_store/model_details.py +73 -0
  367. ads/feature_store/query/__init__.py +0 -0
  368. ads/feature_store/query/filter.py +266 -0
  369. ads/feature_store/query/generator/__init__.py +0 -0
  370. ads/feature_store/query/generator/query_generator.py +298 -0
  371. ads/feature_store/query/join.py +161 -0
  372. ads/feature_store/query/query.py +403 -0
  373. ads/feature_store/query/validator/__init__.py +0 -0
  374. ads/feature_store/query/validator/query_validator.py +57 -0
  375. ads/feature_store/response/__init__.py +0 -0
  376. ads/feature_store/response/response_builder.py +68 -0
  377. ads/feature_store/service/__init__.py +0 -0
  378. ads/feature_store/service/oci_dataset.py +139 -0
  379. ads/feature_store/service/oci_dataset_job.py +199 -0
  380. ads/feature_store/service/oci_entity.py +125 -0
  381. ads/feature_store/service/oci_feature_group.py +164 -0
  382. ads/feature_store/service/oci_feature_group_job.py +214 -0
  383. ads/feature_store/service/oci_feature_store.py +182 -0
  384. ads/feature_store/service/oci_lineage.py +87 -0
  385. ads/feature_store/service/oci_transformation.py +104 -0
  386. ads/feature_store/statistics/__init__.py +0 -0
  387. ads/feature_store/statistics/abs_feature_value.py +49 -0
  388. ads/feature_store/statistics/charts/__init__.py +0 -0
  389. ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
  390. ads/feature_store/statistics/charts/box_plot.py +148 -0
  391. ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
  392. ads/feature_store/statistics/charts/probability_distribution.py +68 -0
  393. ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
  394. ads/feature_store/statistics/feature_stat.py +126 -0
  395. ads/feature_store/statistics/generic_feature_value.py +33 -0
  396. ads/feature_store/statistics/statistics.py +41 -0
  397. ads/feature_store/statistics_config.py +101 -0
  398. ads/feature_store/templates/feature_store_template.yaml +45 -0
  399. ads/feature_store/transformation.py +499 -0
  400. ads/feature_store/validation_output.py +57 -0
  401. ads/hpo/__init__.py +9 -0
  402. ads/hpo/_imports.py +91 -0
  403. ads/hpo/ads_search_space.py +439 -0
  404. ads/hpo/distributions.py +325 -0
  405. ads/hpo/objective.py +280 -0
  406. ads/hpo/search_cv.py +1657 -0
  407. ads/hpo/stopping_criterion.py +75 -0
  408. ads/hpo/tuner_artifact.py +413 -0
  409. ads/hpo/utils.py +91 -0
  410. ads/hpo/validation.py +140 -0
  411. ads/hpo/visualization/__init__.py +5 -0
  412. ads/hpo/visualization/_contour.py +23 -0
  413. ads/hpo/visualization/_edf.py +20 -0
  414. ads/hpo/visualization/_intermediate_values.py +21 -0
  415. ads/hpo/visualization/_optimization_history.py +25 -0
  416. ads/hpo/visualization/_parallel_coordinate.py +169 -0
  417. ads/hpo/visualization/_param_importances.py +26 -0
  418. ads/jobs/__init__.py +53 -0
  419. ads/jobs/ads_job.py +663 -0
  420. ads/jobs/builders/__init__.py +5 -0
  421. ads/jobs/builders/base.py +156 -0
  422. ads/jobs/builders/infrastructure/__init__.py +6 -0
  423. ads/jobs/builders/infrastructure/base.py +165 -0
  424. ads/jobs/builders/infrastructure/dataflow.py +1252 -0
  425. ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
  426. ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
  427. ads/jobs/builders/infrastructure/utils.py +65 -0
  428. ads/jobs/builders/runtimes/__init__.py +5 -0
  429. ads/jobs/builders/runtimes/artifact.py +338 -0
  430. ads/jobs/builders/runtimes/base.py +325 -0
  431. ads/jobs/builders/runtimes/container_runtime.py +242 -0
  432. ads/jobs/builders/runtimes/python_runtime.py +1016 -0
  433. ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
  434. ads/jobs/cli.py +104 -0
  435. ads/jobs/env_var_parser.py +131 -0
  436. ads/jobs/extension.py +160 -0
  437. ads/jobs/schema/__init__.py +5 -0
  438. ads/jobs/schema/infrastructure_schema.json +116 -0
  439. ads/jobs/schema/job_schema.json +42 -0
  440. ads/jobs/schema/runtime_schema.json +183 -0
  441. ads/jobs/schema/validator.py +141 -0
  442. ads/jobs/serializer.py +296 -0
  443. ads/jobs/templates/__init__.py +5 -0
  444. ads/jobs/templates/container.py +6 -0
  445. ads/jobs/templates/driver_notebook.py +177 -0
  446. ads/jobs/templates/driver_oci.py +500 -0
  447. ads/jobs/templates/driver_python.py +48 -0
  448. ads/jobs/templates/driver_pytorch.py +852 -0
  449. ads/jobs/templates/driver_utils.py +615 -0
  450. ads/jobs/templates/hostname_from_env.c +55 -0
  451. ads/jobs/templates/oci_metrics.py +181 -0
  452. ads/jobs/utils.py +104 -0
  453. ads/llm/__init__.py +28 -0
  454. ads/llm/autogen/__init__.py +2 -0
  455. ads/llm/autogen/constants.py +15 -0
  456. ads/llm/autogen/reports/__init__.py +2 -0
  457. ads/llm/autogen/reports/base.py +67 -0
  458. ads/llm/autogen/reports/data.py +103 -0
  459. ads/llm/autogen/reports/session.py +526 -0
  460. ads/llm/autogen/reports/templates/chat_box.html +13 -0
  461. ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
  462. ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
  463. ads/llm/autogen/reports/utils.py +56 -0
  464. ads/llm/autogen/v02/__init__.py +4 -0
  465. ads/llm/autogen/v02/client.py +295 -0
  466. ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
  467. ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
  468. ads/llm/autogen/v02/loggers/__init__.py +6 -0
  469. ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
  470. ads/llm/autogen/v02/loggers/session_logger.py +580 -0
  471. ads/llm/autogen/v02/loggers/utils.py +86 -0
  472. ads/llm/autogen/v02/runtime_logging.py +163 -0
  473. ads/llm/chain.py +268 -0
  474. ads/llm/chat_template.py +31 -0
  475. ads/llm/deploy.py +63 -0
  476. ads/llm/guardrails/__init__.py +5 -0
  477. ads/llm/guardrails/base.py +442 -0
  478. ads/llm/guardrails/huggingface.py +44 -0
  479. ads/llm/langchain/__init__.py +5 -0
  480. ads/llm/langchain/plugins/__init__.py +5 -0
  481. ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
  482. ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
  483. ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
  484. ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
  485. ads/llm/langchain/plugins/llms/__init__.py +5 -0
  486. ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
  487. ads/llm/requirements.txt +3 -0
  488. ads/llm/serialize.py +219 -0
  489. ads/llm/serializers/__init__.py +0 -0
  490. ads/llm/serializers/retrieval_qa.py +153 -0
  491. ads/llm/serializers/runnable_parallel.py +27 -0
  492. ads/llm/templates/score_chain.jinja2 +155 -0
  493. ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
  494. ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
  495. ads/model/__init__.py +52 -0
  496. ads/model/artifact.py +573 -0
  497. ads/model/artifact_downloader.py +254 -0
  498. ads/model/artifact_uploader.py +267 -0
  499. ads/model/base_properties.py +238 -0
  500. ads/model/common/.model-ignore +66 -0
  501. ads/model/common/__init__.py +5 -0
  502. ads/model/common/utils.py +142 -0
  503. ads/model/datascience_model.py +2635 -0
  504. ads/model/deployment/__init__.py +20 -0
  505. ads/model/deployment/common/__init__.py +5 -0
  506. ads/model/deployment/common/utils.py +308 -0
  507. ads/model/deployment/model_deployer.py +466 -0
  508. ads/model/deployment/model_deployment.py +1846 -0
  509. ads/model/deployment/model_deployment_infrastructure.py +671 -0
  510. ads/model/deployment/model_deployment_properties.py +493 -0
  511. ads/model/deployment/model_deployment_runtime.py +838 -0
  512. ads/model/extractor/__init__.py +5 -0
  513. ads/model/extractor/automl_extractor.py +74 -0
  514. ads/model/extractor/embedding_onnx_extractor.py +80 -0
  515. ads/model/extractor/huggingface_extractor.py +88 -0
  516. ads/model/extractor/keras_extractor.py +84 -0
  517. ads/model/extractor/lightgbm_extractor.py +93 -0
  518. ads/model/extractor/model_info_extractor.py +114 -0
  519. ads/model/extractor/model_info_extractor_factory.py +105 -0
  520. ads/model/extractor/pytorch_extractor.py +87 -0
  521. ads/model/extractor/sklearn_extractor.py +112 -0
  522. ads/model/extractor/spark_extractor.py +89 -0
  523. ads/model/extractor/tensorflow_extractor.py +85 -0
  524. ads/model/extractor/xgboost_extractor.py +94 -0
  525. ads/model/framework/__init__.py +5 -0
  526. ads/model/framework/automl_model.py +178 -0
  527. ads/model/framework/embedding_onnx_model.py +438 -0
  528. ads/model/framework/huggingface_model.py +399 -0
  529. ads/model/framework/lightgbm_model.py +266 -0
  530. ads/model/framework/pytorch_model.py +266 -0
  531. ads/model/framework/sklearn_model.py +250 -0
  532. ads/model/framework/spark_model.py +326 -0
  533. ads/model/framework/tensorflow_model.py +254 -0
  534. ads/model/framework/xgboost_model.py +258 -0
  535. ads/model/generic_model.py +3518 -0
  536. ads/model/model_artifact_boilerplate/README.md +381 -0
  537. ads/model/model_artifact_boilerplate/__init__.py +5 -0
  538. ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
  539. ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
  540. ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
  541. ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
  542. ads/model/model_artifact_boilerplate/score.py +61 -0
  543. ads/model/model_file_description_schema.json +68 -0
  544. ads/model/model_introspect.py +331 -0
  545. ads/model/model_metadata.py +1810 -0
  546. ads/model/model_metadata_mixin.py +460 -0
  547. ads/model/model_properties.py +63 -0
  548. ads/model/model_version_set.py +739 -0
  549. ads/model/runtime/__init__.py +5 -0
  550. ads/model/runtime/env_info.py +306 -0
  551. ads/model/runtime/model_deployment_details.py +37 -0
  552. ads/model/runtime/model_provenance_details.py +58 -0
  553. ads/model/runtime/runtime_info.py +81 -0
  554. ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
  555. ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
  556. ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
  557. ads/model/runtime/utils.py +201 -0
  558. ads/model/serde/__init__.py +5 -0
  559. ads/model/serde/common.py +40 -0
  560. ads/model/serde/model_input.py +547 -0
  561. ads/model/serde/model_serializer.py +1184 -0
  562. ads/model/service/__init__.py +5 -0
  563. ads/model/service/oci_datascience_model.py +1076 -0
  564. ads/model/service/oci_datascience_model_deployment.py +500 -0
  565. ads/model/service/oci_datascience_model_version_set.py +176 -0
  566. ads/model/transformer/__init__.py +5 -0
  567. ads/model/transformer/onnx_transformer.py +324 -0
  568. ads/mysqldb/__init__.py +5 -0
  569. ads/mysqldb/mysql_db.py +227 -0
  570. ads/opctl/__init__.py +18 -0
  571. ads/opctl/anomaly_detection.py +11 -0
  572. ads/opctl/backend/__init__.py +5 -0
  573. ads/opctl/backend/ads_dataflow.py +353 -0
  574. ads/opctl/backend/ads_ml_job.py +710 -0
  575. ads/opctl/backend/ads_ml_pipeline.py +164 -0
  576. ads/opctl/backend/ads_model_deployment.py +209 -0
  577. ads/opctl/backend/base.py +146 -0
  578. ads/opctl/backend/local.py +1053 -0
  579. ads/opctl/backend/marketplace/__init__.py +9 -0
  580. ads/opctl/backend/marketplace/helm_helper.py +173 -0
  581. ads/opctl/backend/marketplace/local_marketplace.py +271 -0
  582. ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
  583. ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
  584. ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
  585. ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
  586. ads/opctl/backend/marketplace/models/__init__.py +5 -0
  587. ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
  588. ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
  589. ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
  590. ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
  591. ads/opctl/cli.py +707 -0
  592. ads/opctl/cmds.py +869 -0
  593. ads/opctl/conda/__init__.py +5 -0
  594. ads/opctl/conda/cli.py +193 -0
  595. ads/opctl/conda/cmds.py +749 -0
  596. ads/opctl/conda/config.yaml +34 -0
  597. ads/opctl/conda/manifest_template.yaml +13 -0
  598. ads/opctl/conda/multipart_uploader.py +188 -0
  599. ads/opctl/conda/pack.py +89 -0
  600. ads/opctl/config/__init__.py +5 -0
  601. ads/opctl/config/base.py +57 -0
  602. ads/opctl/config/diagnostics/__init__.py +5 -0
  603. ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
  604. ads/opctl/config/merger.py +255 -0
  605. ads/opctl/config/resolver.py +297 -0
  606. ads/opctl/config/utils.py +79 -0
  607. ads/opctl/config/validator.py +17 -0
  608. ads/opctl/config/versioner.py +68 -0
  609. ads/opctl/config/yaml_parsers/__init__.py +7 -0
  610. ads/opctl/config/yaml_parsers/base.py +58 -0
  611. ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
  612. ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
  613. ads/opctl/constants.py +66 -0
  614. ads/opctl/decorator/__init__.py +5 -0
  615. ads/opctl/decorator/common.py +129 -0
  616. ads/opctl/diagnostics/__init__.py +5 -0
  617. ads/opctl/diagnostics/__main__.py +25 -0
  618. ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
  619. ads/opctl/diagnostics/check_requirements.py +144 -0
  620. ads/opctl/diagnostics/requirement_exception.py +9 -0
  621. ads/opctl/distributed/README.md +109 -0
  622. ads/opctl/distributed/__init__.py +5 -0
  623. ads/opctl/distributed/certificates.py +32 -0
  624. ads/opctl/distributed/cli.py +207 -0
  625. ads/opctl/distributed/cmds.py +731 -0
  626. ads/opctl/distributed/common/__init__.py +5 -0
  627. ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
  628. ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
  629. ads/opctl/distributed/common/cluster_config_helper.py +103 -0
  630. ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
  631. ads/opctl/distributed/common/cluster_runner.py +54 -0
  632. ads/opctl/distributed/common/framework_factory.py +29 -0
  633. ads/opctl/docker/Dockerfile.job +103 -0
  634. ads/opctl/docker/Dockerfile.job.arm +107 -0
  635. ads/opctl/docker/Dockerfile.job.gpu +175 -0
  636. ads/opctl/docker/base-env.yaml +13 -0
  637. ads/opctl/docker/cuda.repo +6 -0
  638. ads/opctl/docker/operator/.dockerignore +0 -0
  639. ads/opctl/docker/operator/Dockerfile +41 -0
  640. ads/opctl/docker/operator/Dockerfile.gpu +85 -0
  641. ads/opctl/docker/operator/cuda.repo +6 -0
  642. ads/opctl/docker/operator/environment.yaml +8 -0
  643. ads/opctl/forecast.py +11 -0
  644. ads/opctl/index.yaml +3 -0
  645. ads/opctl/model/__init__.py +5 -0
  646. ads/opctl/model/cli.py +65 -0
  647. ads/opctl/model/cmds.py +73 -0
  648. ads/opctl/operator/README.md +4 -0
  649. ads/opctl/operator/__init__.py +31 -0
  650. ads/opctl/operator/cli.py +344 -0
  651. ads/opctl/operator/cmd.py +596 -0
  652. ads/opctl/operator/common/__init__.py +5 -0
  653. ads/opctl/operator/common/backend_factory.py +460 -0
  654. ads/opctl/operator/common/const.py +27 -0
  655. ads/opctl/operator/common/data/synthetic.csv +16001 -0
  656. ads/opctl/operator/common/dictionary_merger.py +148 -0
  657. ads/opctl/operator/common/errors.py +42 -0
  658. ads/opctl/operator/common/operator_config.py +99 -0
  659. ads/opctl/operator/common/operator_loader.py +811 -0
  660. ads/opctl/operator/common/operator_schema.yaml +130 -0
  661. ads/opctl/operator/common/operator_yaml_generator.py +152 -0
  662. ads/opctl/operator/common/utils.py +208 -0
  663. ads/opctl/operator/lowcode/__init__.py +5 -0
  664. ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
  665. ads/opctl/operator/lowcode/anomaly/README.md +207 -0
  666. ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
  667. ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
  668. ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
  669. ads/opctl/operator/lowcode/anomaly/const.py +167 -0
  670. ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
  671. ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
  672. ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
  673. ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
  674. ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
  675. ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
  676. ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
  677. ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
  678. ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
  679. ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
  680. ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
  681. ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
  682. ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
  683. ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
  684. ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
  685. ads/opctl/operator/lowcode/common/__init__.py +5 -0
  686. ads/opctl/operator/lowcode/common/const.py +10 -0
  687. ads/opctl/operator/lowcode/common/data.py +116 -0
  688. ads/opctl/operator/lowcode/common/errors.py +47 -0
  689. ads/opctl/operator/lowcode/common/transformations.py +296 -0
  690. ads/opctl/operator/lowcode/common/utils.py +384 -0
  691. ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
  692. ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
  693. ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
  694. ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
  695. ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
  696. ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
  697. ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
  698. ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
  699. ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
  700. ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
  701. ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
  702. ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
  703. ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
  704. ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
  705. ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
  706. ads/opctl/operator/lowcode/forecast/README.md +209 -0
  707. ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
  708. ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
  709. ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
  710. ads/opctl/operator/lowcode/forecast/const.py +92 -0
  711. ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
  712. ads/opctl/operator/lowcode/forecast/errors.py +26 -0
  713. ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
  714. ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
  715. ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
  716. ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
  717. ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
  718. ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
  719. ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
  720. ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
  721. ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
  722. ads/opctl/operator/lowcode/forecast/model/prophet.py +450 -0
  723. ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
  724. ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
  725. ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
  726. ads/opctl/operator/lowcode/forecast/utils.py +397 -0
  727. ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
  728. ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
  729. ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
  730. ads/opctl/operator/lowcode/pii/MLoperator +17 -0
  731. ads/opctl/operator/lowcode/pii/README.md +208 -0
  732. ads/opctl/operator/lowcode/pii/__init__.py +5 -0
  733. ads/opctl/operator/lowcode/pii/__main__.py +78 -0
  734. ads/opctl/operator/lowcode/pii/cmd.py +39 -0
  735. ads/opctl/operator/lowcode/pii/constant.py +84 -0
  736. ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
  737. ads/opctl/operator/lowcode/pii/errors.py +27 -0
  738. ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
  739. ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
  740. ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
  741. ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
  742. ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
  743. ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
  744. ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
  745. ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
  746. ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
  747. ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
  748. ads/opctl/operator/lowcode/pii/model/report.py +487 -0
  749. ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
  750. ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
  751. ads/opctl/operator/lowcode/pii/utils.py +43 -0
  752. ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
  753. ads/opctl/operator/lowcode/recommender/README.md +206 -0
  754. ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
  755. ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
  756. ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
  757. ads/opctl/operator/lowcode/recommender/constant.py +30 -0
  758. ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
  759. ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
  760. ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
  761. ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
  762. ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
  763. ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
  764. ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
  765. ads/opctl/operator/lowcode/recommender/utils.py +13 -0
  766. ads/opctl/operator/runtime/__init__.py +5 -0
  767. ads/opctl/operator/runtime/const.py +17 -0
  768. ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
  769. ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
  770. ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
  771. ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
  772. ads/opctl/operator/runtime/runtime.py +115 -0
  773. ads/opctl/schema.yaml.yml +36 -0
  774. ads/opctl/script.py +40 -0
  775. ads/opctl/spark/__init__.py +5 -0
  776. ads/opctl/spark/cli.py +43 -0
  777. ads/opctl/spark/cmds.py +147 -0
  778. ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
  779. ads/opctl/utils.py +344 -0
  780. ads/oracledb/__init__.py +5 -0
  781. ads/oracledb/oracle_db.py +346 -0
  782. ads/pipeline/__init__.py +39 -0
  783. ads/pipeline/ads_pipeline.py +2279 -0
  784. ads/pipeline/ads_pipeline_run.py +772 -0
  785. ads/pipeline/ads_pipeline_step.py +605 -0
  786. ads/pipeline/builders/__init__.py +5 -0
  787. ads/pipeline/builders/infrastructure/__init__.py +5 -0
  788. ads/pipeline/builders/infrastructure/custom_script.py +32 -0
  789. ads/pipeline/cli.py +119 -0
  790. ads/pipeline/extension.py +291 -0
  791. ads/pipeline/schema/__init__.py +5 -0
  792. ads/pipeline/schema/cs_step_schema.json +35 -0
  793. ads/pipeline/schema/ml_step_schema.json +31 -0
  794. ads/pipeline/schema/pipeline_schema.json +71 -0
  795. ads/pipeline/visualizer/__init__.py +5 -0
  796. ads/pipeline/visualizer/base.py +570 -0
  797. ads/pipeline/visualizer/graph_renderer.py +272 -0
  798. ads/pipeline/visualizer/text_renderer.py +84 -0
  799. ads/secrets/__init__.py +11 -0
  800. ads/secrets/adb.py +386 -0
  801. ads/secrets/auth_token.py +86 -0
  802. ads/secrets/big_data_service.py +365 -0
  803. ads/secrets/mysqldb.py +149 -0
  804. ads/secrets/oracledb.py +160 -0
  805. ads/secrets/secrets.py +407 -0
  806. ads/telemetry/__init__.py +7 -0
  807. ads/telemetry/base.py +69 -0
  808. ads/telemetry/client.py +122 -0
  809. ads/telemetry/telemetry.py +257 -0
  810. ads/templates/dataflow_pyspark.jinja2 +13 -0
  811. ads/templates/dataflow_sparksql.jinja2 +22 -0
  812. ads/templates/func.jinja2 +20 -0
  813. ads/templates/schemas/openapi.json +1740 -0
  814. ads/templates/score-pkl.jinja2 +173 -0
  815. ads/templates/score.jinja2 +322 -0
  816. ads/templates/score_embedding_onnx.jinja2 +202 -0
  817. ads/templates/score_generic.jinja2 +165 -0
  818. ads/templates/score_huggingface_pipeline.jinja2 +217 -0
  819. ads/templates/score_lightgbm.jinja2 +185 -0
  820. ads/templates/score_onnx.jinja2 +407 -0
  821. ads/templates/score_onnx_new.jinja2 +473 -0
  822. ads/templates/score_oracle_automl.jinja2 +185 -0
  823. ads/templates/score_pyspark.jinja2 +154 -0
  824. ads/templates/score_pytorch.jinja2 +219 -0
  825. ads/templates/score_scikit-learn.jinja2 +184 -0
  826. ads/templates/score_tensorflow.jinja2 +184 -0
  827. ads/templates/score_xgboost.jinja2 +178 -0
  828. ads/text_dataset/__init__.py +5 -0
  829. ads/text_dataset/backends.py +211 -0
  830. ads/text_dataset/dataset.py +445 -0
  831. ads/text_dataset/extractor.py +207 -0
  832. ads/text_dataset/options.py +53 -0
  833. ads/text_dataset/udfs.py +22 -0
  834. ads/text_dataset/utils.py +49 -0
  835. ads/type_discovery/__init__.py +9 -0
  836. ads/type_discovery/abstract_detector.py +21 -0
  837. ads/type_discovery/constant_detector.py +41 -0
  838. ads/type_discovery/continuous_detector.py +54 -0
  839. ads/type_discovery/credit_card_detector.py +99 -0
  840. ads/type_discovery/datetime_detector.py +92 -0
  841. ads/type_discovery/discrete_detector.py +118 -0
  842. ads/type_discovery/document_detector.py +146 -0
  843. ads/type_discovery/ip_detector.py +68 -0
  844. ads/type_discovery/latlon_detector.py +90 -0
  845. ads/type_discovery/phone_number_detector.py +63 -0
  846. ads/type_discovery/type_discovery_driver.py +87 -0
  847. ads/type_discovery/typed_feature.py +594 -0
  848. ads/type_discovery/unknown_detector.py +41 -0
  849. ads/type_discovery/zipcode_detector.py +48 -0
  850. ads/vault/__init__.py +7 -0
  851. ads/vault/vault.py +237 -0
  852. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/METADATA +150 -149
  853. oracle_ads-2.13.10.dist-info/RECORD +858 -0
  854. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/WHEEL +1 -2
  855. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/entry_points.txt +2 -1
  856. oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
  857. oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
  858. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,664 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8; -*-
3
+
4
+ # Copyright (c) 2020, 2023 Oracle and/or its affiliates.
5
+ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
6
+ import warnings
7
+
8
+ warnings.warn(
9
+ (
10
+ "The `ads.common.model_export_util` is deprecated in `oracle-ads 2.6.9` and will be removed in `oracle-ads 3.0`. "
11
+ "Use framework specific Model utility class for saving and deploying model. "
12
+ "Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html"
13
+ ),
14
+ DeprecationWarning,
15
+ stacklevel=2,
16
+ )
17
+
18
+ import json
19
+ import logging
20
+ import os
21
+ import sys
22
+ from typing import Any, Dict, Union
23
+
24
+ import cloudpickle
25
+ import numpy as np
26
+ import pandas as pd
27
+ from ads.common import logger, utils
28
+ from ads.common.data import ADSData
29
+ from ads.common.decorator.runtime_dependency import (
30
+ OptionalDependency,
31
+ runtime_dependency,
32
+ )
33
+ from ads.common.decorator.deprecate import deprecated
34
+ from ads.common.function.fn_util import (
35
+ generate_fn_artifacts,
36
+ get_function_config,
37
+ write_score,
38
+ )
39
+ from ads.common.model_artifact import ModelArtifact
40
+ from ads.model.model_metadata import UseCaseType
41
+ from ads.feature_engineering.schema import DataSizeTooWide
42
+ from pkg_resources import DistributionNotFound, get_distribution
43
+
44
+ from ads.model.transformer.onnx_transformer import (
45
+ ONNXTransformer,
46
+ ) # Do not remove this line till ADS 3.0 wll be deployed.
47
+
48
+ pd.options.mode.chained_assignment = None
49
+
50
+ logging.basicConfig(stream=sys.stdout, level=logging.INFO)
51
+ logger = logging.getLogger("ADS")
52
+
53
+ OnnxConvertibleModels = [
54
+ "sklearn",
55
+ "automl",
56
+ "lightgbm",
57
+ "xgboost",
58
+ "torch",
59
+ "tensorflow",
60
+ "keras",
61
+ "mxnet",
62
+ ]
63
+ NoVerifyModels = ["automl", "torch", "mxnet", "lightgbm", "xgboost"]
64
+ AlreadyWrittenModels = ["torch", "mxnet", "automl"]
65
+ TransformableData = ["automl"]
66
+ Progress_Steps_W_Fn = 6
67
+ Progress_Steps_Wo_Fn = 4
68
+
69
+
70
+ @deprecated(
71
+ "2.6.6",
72
+ details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
73
+ )
74
+ def prepare_generic_model(
75
+ model_path: str,
76
+ fn_artifact_files_included: bool = False,
77
+ fn_name: str = "model_api",
78
+ force_overwrite: bool = False,
79
+ model: Any = None,
80
+ data_sample: ADSData = None,
81
+ use_case_type=None,
82
+ X_sample: Union[
83
+ list,
84
+ tuple,
85
+ pd.Series,
86
+ np.ndarray,
87
+ pd.DataFrame,
88
+ ] = None,
89
+ y_sample: Union[
90
+ list,
91
+ tuple,
92
+ pd.Series,
93
+ np.ndarray,
94
+ pd.DataFrame,
95
+ ] = None,
96
+ **kwargs,
97
+ ) -> ModelArtifact:
98
+ """
99
+ Generates template files to aid model deployment.
100
+ The model could be accompanied by other artifacts all of which can be dumped at `model_path`.
101
+ Following files are generated:
102
+ * func.yaml
103
+ * func.py
104
+ * requirements.txt
105
+ * score.py
106
+
107
+ Parameters
108
+ ----------
109
+ model_path : str
110
+ Path where the artifacts must be saved.
111
+ The serialized model object and any other associated files/objects must
112
+ be saved in the `model_path` directory
113
+ fn_artifact_files_included : bool
114
+ Default is False, if turned off, function artifacts are not generated.
115
+ fn_name : str
116
+ Opional parameter to specify the function name
117
+ force_overwrite : bool
118
+ Opional parameter to specify if the model_artifact should overwrite the existing model_path (if it exists)
119
+ model : (Any, optional). Defaults to None.
120
+ This is an optional model object which is only used to extract taxonomy metadata.
121
+ Supported models: automl, keras, lightgbm, pytorch, sklearn, tensorflow, and xgboost.
122
+ If the model is not under supported frameworks, then extracting taxonomy metadata will be skipped.
123
+ The alternative way is using `atifact.populate_metadata(model=model, usecase_type=UseCaseType.REGRESSION)`.
124
+ data_sample : ADSData
125
+ A sample of the test data that will be provided to predict() API of scoring script
126
+ Used to generate schema_input and schema_output
127
+ use_case_type: str
128
+ The use case type of the model
129
+ X_sample : Union[list, tuple, pd.Series, np.ndarray, pd.DataFrame, dask.dataframe.core.Series, dask.dataframe.core.DataFrame]
130
+ A sample of input data that will be provided to predict() API of scoring script
131
+ Used to generate input schema.
132
+ y_sample : Union[list, tuple, pd.Series, np.ndarray, pd.DataFrame, dask.dataframe.core.Series, dask.dataframe.core.DataFrame]
133
+ A sample of output data that is expected to be returned by predict() API of scoring script,
134
+ corresponding to X_sample
135
+ Used to generate output schema.
136
+
137
+ **kwargs
138
+ ________
139
+ data_science_env : bool, default: False
140
+ If set to True, the datascience environment represented by the slug in the training conda environment will be used.
141
+ inference_conda_env : str, default: None
142
+ Conda environment to use within the model deployment service for inferencing. For example, oci://bucketname@namespace/path/to/conda/env
143
+ ignore_deployment_error : bool, default: False
144
+ If set to True, the prepare method will ignore all the errors that may impact model deployment.
145
+ underlying_model : str, default: 'UNKNOWN'
146
+ Underlying Model Type, could be "automl", "sklearn", "h2o", "lightgbm", "xgboost", "torch", "mxnet", "tensorflow", "keras", "pyod" and etc.
147
+ model_libs : dict, default: {}
148
+ Model required libraries where the key is the library names and the value is the library versions.
149
+ For example, {numpy: 1.21.1}.
150
+ progress : int, default: None
151
+ max number of progress.
152
+ inference_python_version: str, default:None.
153
+ If provided will be added to the generated runtime yaml
154
+ max_col_num: (int, optional). Defaults to utils.DATA_SCHEMA_MAX_COL_NUM.
155
+ The maximum column size of the data that allows to auto generate schema.
156
+
157
+ Examples
158
+ --------
159
+ >>> import cloudpickle
160
+ >>> import os
161
+ >>> from sklearn.linear_model import LogisticRegression
162
+ >>> from sklearn.datasets import make_classification
163
+ >>> import ads
164
+ >>> from ads.common.model_export_util import prepare_generic_model
165
+ >>> import yaml
166
+ >>> import oci
167
+ >>>
168
+ >>> ads.set_auth('api_key', oci_config_location=oci.config.DEFAULT_LOCATION, profile='DEFAULT')
169
+ >>> model_artifact_location = os.path.expanduser('~/myusecase/model/')
170
+ >>> inference_conda_env="oci://my-bucket@namespace/conda_environments/cpu/Data_Exploration_and_Manipulation_for_CPU_Python_3.7/2.0/dataexpl_p37_cpu_v2"
171
+ >>> inference_python_version = "3.7"
172
+ >>> if not os.path.exists(model_artifact_location):
173
+ ... os.makedirs(model_artifact_location)
174
+ >>> X, y = make_classification(n_samples=100, n_features=20, n_classes=2)
175
+ >>> lrmodel = LogisticRegression().fit(X, y)
176
+ >>> with open(os.path.join(model_artifact_location, 'model.pkl'), "wb") as mfile:
177
+ ... cloudpickle.dump(lrmodel, mfile)
178
+ >>> modelartifact = prepare_generic_model(
179
+ ... model_artifact_location,
180
+ ... model = lrmodel,
181
+ ... force_overwrite=True,
182
+ ... inference_conda_env=inference_conda_env,
183
+ ... ignore_deployment_error=True,
184
+ ... inference_python_version=inference_python_version
185
+ ... )
186
+ >>> modelartifact.reload() # Call reload to update the ModelArtifact object with the generated score.py
187
+ >>> assert len(modelartifact.predict(X[:5])['prediction']) == 5 #Test the generated score.py works. This may require customization.
188
+ >>> with open(os.path.join(model_artifact_location, "runtime.yaml")) as rf:
189
+ ... content = yaml.load(rf, Loader=yaml.FullLoader)
190
+ ... assert content['MODEL_DEPLOYMENT']['INFERENCE_CONDA_ENV']['INFERENCE_ENV_PATH'] == inference_conda_env
191
+ ... assert content['MODEL_DEPLOYMENT']['INFERENCE_CONDA_ENV']['INFERENCE_PYTHON_VERSION'] == inference_python_version
192
+ >>> # Save Model to model artifact
193
+ >>> ocimodel = modelartifact.save(
194
+ ... project_id="oci1......", # OCID of the project to which the model to be associated
195
+ ... compartment_id="oci1......", # OCID of the compartment where the model will reside
196
+ ... display_name="LRModel_01",
197
+ ... description="My Logistic Regression Model",
198
+ ... ignore_pending_changes=True,
199
+ ... timeout=100,
200
+ ... ignore_introspection=True,
201
+ ... )
202
+ >>> print(f"The OCID of the model is: {ocimodel.id}")
203
+
204
+ Returns
205
+ -------
206
+ model_artifact: ads.model_artifact.model_artifact
207
+ A generic model artifact
208
+ """
209
+ if "function_artifacts" in kwargs:
210
+ if (
211
+ fn_artifact_files_included
212
+ or fn_artifact_files_included != kwargs["function_artifacts"]
213
+ ):
214
+ raise ValueError(
215
+ "Parameter 'function_artifacts' and 'fn_artifact_files_included' cannot be used at "
216
+ "the same time. Parameter 'function_artifacts' is deprecated and removed in future releases."
217
+ )
218
+ else:
219
+ logger.warning(
220
+ "Parameter 'function_artifacts' is deprecated and removed in future releases. Use 'fn_artifact_files_included' instead."
221
+ )
222
+ fn_artifact_files_included = kwargs["function_artifacts"]
223
+
224
+ assert model_path, "Required folder path for saving artifacts"
225
+
226
+ # If this is being called from ADSModel.prepare, grab relevant data
227
+ underlying_model = kwargs.get("underlying_model", "UNKNOWN")
228
+ model_libs = kwargs.get(
229
+ "model_libs", {}
230
+ ) # utils.extract_lib_dependencies_from_model(self.est)
231
+ progress = kwargs.get("progress", None)
232
+ max_col_num = kwargs.get("max_col_num", utils.DATA_SCHEMA_MAX_COL_NUM)
233
+ artifact_type_generic = progress is not None
234
+
235
+ from ads.common.model import ADSModel
236
+
237
+ if isinstance(model, ADSModel) and underlying_model != "automl":
238
+ raise ValueError(
239
+ "Only generic model can be used to generate generic model artifact."
240
+ )
241
+
242
+ if use_case_type and use_case_type not in UseCaseType:
243
+ raise ValueError(f"Invalid usecase type. Choose from {UseCaseType.values()}")
244
+
245
+ with progress if artifact_type_generic else utils.get_progress_bar(
246
+ Progress_Steps_W_Fn if fn_artifact_files_included else Progress_Steps_Wo_Fn
247
+ ) as progress:
248
+ if not artifact_type_generic:
249
+ progress.update("Preparing Model Artifact Directory")
250
+ if os.path.exists(model_path):
251
+ if force_overwrite:
252
+ logger.warning(
253
+ f"As force_overwrite is set to True, all the existing files in the {model_path} will be removed"
254
+ )
255
+ else:
256
+ raise ValueError(
257
+ "Directory already exists, set force_overwrite to True if you wish to overwrite."
258
+ )
259
+
260
+ os.makedirs(model_path, exist_ok=True)
261
+
262
+ progress.update("Updating requirements.txt")
263
+ if fn_artifact_files_included:
264
+ # fdk removed from dependency list in pyproject.toml (fn deployments deprecated)
265
+ # before we request versions we want to check if fdk installed by user
266
+ # and provide support in error message, if not installed
267
+ try:
268
+ get_distribution("fdk")
269
+ except Exception as e:
270
+ if isinstance(e, DistributionNotFound):
271
+ error_message = (
272
+ "fdk library not installed in current environment, it is required "
273
+ "for deployment with fn. Install fdk with 'pip install fdk'."
274
+ )
275
+ logger.error(str(error_message))
276
+ raise
277
+ else:
278
+ required_fn_libs = get_function_config()["requires"]["functions"]
279
+ [
280
+ model_libs.update({lib: get_distribution(lib).version})
281
+ for lib in required_fn_libs
282
+ ]
283
+ required_model_libs = get_function_config()["requires"][
284
+ kwargs.get("serializer", "default")
285
+ ]
286
+ [
287
+ model_libs.update({lib: get_distribution(lib).version})
288
+ for lib in required_model_libs
289
+ ]
290
+ utils.generate_requirement_file(
291
+ requirements=model_libs, file_path=model_path
292
+ )
293
+
294
+ model_artifact_args = {}
295
+ if "inference_conda_env" in kwargs:
296
+ model_artifact_args["inference_conda_env"] = kwargs["inference_conda_env"]
297
+ if "inference_python_version" in kwargs:
298
+ model_artifact_args["inference_python_version"] = kwargs[
299
+ "inference_python_version"
300
+ ]
301
+ if "data_science_env" in kwargs:
302
+ model_artifact_args["data_science_env"] = kwargs["data_science_env"]
303
+ if "ignore_deployment_error" in kwargs:
304
+ model_artifact_args["ignore_deployment_error"] = kwargs[
305
+ "ignore_deployment_error"
306
+ ]
307
+ model_artifact = ModelArtifact(
308
+ model_path,
309
+ reload=False,
310
+ create=True,
311
+ progress=progress,
312
+ **model_artifact_args,
313
+ )
314
+
315
+ model_kwargs = {"_underlying_model": underlying_model, "progress": progress}
316
+ kwargs.update(model_kwargs)
317
+ if fn_artifact_files_included:
318
+ generate_fn_artifacts(
319
+ model_path,
320
+ fn_name,
321
+ artifact_type_generic=artifact_type_generic,
322
+ **kwargs,
323
+ )
324
+ if progress:
325
+ progress.update("Writing score.py")
326
+ write_score(model_path, **kwargs)
327
+ if model is None:
328
+ logger.warning(
329
+ "Taxonomy metadata was not extracted. "
330
+ "To auto-populate taxonomy metadata the model must be provided. "
331
+ "Pass the model as a parameter to .prepare_generic_model(model=model, usecase_type=UseCaseType.REGRESSION). "
332
+ "Alternative way is using atifact.populate_metadata(model=model, usecase_type=UseCaseType.REGRESSION)."
333
+ )
334
+ try:
335
+ model_artifact.populate_metadata(model, use_case_type)
336
+ except:
337
+ logger.warning("Failed to populate the custom and taxonomy metadata.")
338
+
339
+ try:
340
+ model_artifact.populate_schema(
341
+ data_sample,
342
+ X_sample,
343
+ y_sample,
344
+ max_col_num,
345
+ )
346
+ except DataSizeTooWide:
347
+ logger.warning(
348
+ f"The data has too many columns and "
349
+ f"the maximum allowable number of columns is `{max_col_num}`. "
350
+ "The schema was not auto generated. increase allowable number of columns."
351
+ )
352
+
353
+ return model_artifact
354
+
355
+
356
+ @deprecated(
357
+ "2.6.6",
358
+ details="Use framework specific Model utility class for saving and deploying model. Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html",
359
+ )
360
+ def serialize_model(
361
+ model=None, target_dir=None, X=None, y=None, model_type=None, **kwargs
362
+ ):
363
+ """
364
+ Parameters
365
+ ----------
366
+ model : ads.Model
367
+ A model to be serialized
368
+ target_dir : str, optional
369
+ directory to output the serialized model
370
+ X : Union[pandas.DataFrame, pandas.Series]
371
+ The X data
372
+ y : Union[list, pandas.DataFrame, pandas.Series]
373
+ Tbe Y data
374
+ model_type : str, optional
375
+ A string corresponding to the model type
376
+
377
+ Returns
378
+ -------
379
+ model_kwargs: Dict
380
+ A dictionary of model kwargs for the serialized model
381
+ """
382
+ model_kwargs = {}
383
+ try:
384
+ assert model_type in OnnxConvertibleModels
385
+ assert X is not None, (
386
+ "WARNING: In order to convert model to onnnx format, you will need to provide a data "
387
+ "sample "
388
+ )
389
+ # Try to serialize the model based off of type
390
+ onx = None
391
+ if model_type == "sklearn":
392
+ onx = _sklearn_to_onnx(model=model)
393
+ elif model_type == "automl":
394
+ _automl_to_pkl(model=model, target_dir=target_dir)
395
+ elif model_type == "lightgbm":
396
+ onx = _lightgbm_to_onnx(model=model, X=X, y=y)
397
+ elif model_type == "torch":
398
+ _torch_to_onnx(model=model, target_dir=target_dir, X=X, y=y)
399
+ elif model_type == "xgboost":
400
+ onx = _xgboost_to_onnx(model=model, X=X, y=y)
401
+ elif model_type == "tensorflow":
402
+ onx = _tf_to_onnx(model=model, X=X, y=y)
403
+ elif model_type == "keras":
404
+ onx = _keras_to_onnx(model=model, X=X, y=y)
405
+ elif model_type == "mxnet":
406
+ _mxnet_to_onnx(model=model, X=X, y=y)
407
+ else:
408
+ raise Exception(
409
+ f"ADS does not have method to convert models of type: {model_type} to onnx"
410
+ )
411
+
412
+ # Save the model to the target file
413
+ if model_type not in AlreadyWrittenModels:
414
+ with open(os.path.join(target_dir, "model.onnx"), "wb") as f:
415
+ f.write(onx.SerializeToString())
416
+
417
+ # Lastly, set the necessary env variables for the remaining workflow
418
+ if model_type == "automl":
419
+ model_kwargs["model_name"] = "model.pkl"
420
+ model_kwargs["serializer"] = "pkl"
421
+ else:
422
+ model_kwargs["model_name"] = "model.onnx"
423
+ model_kwargs["serializer"] = "onnx"
424
+ except Exception as e:
425
+ logger.error(
426
+ f"Failed to serialize the model as ONNX returned the error message::{e}."
427
+ )
428
+ return model_kwargs
429
+
430
+
431
+ @runtime_dependency(module="skl2onnx", install_from=OptionalDependency.ONNX)
432
+ @runtime_dependency(module="onnxmltools", install_from=OptionalDependency.ONNX)
433
+ @runtime_dependency(module="onnx", install_from=OptionalDependency.ONNX)
434
+ def _sklearn_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
435
+ from skl2onnx.common.data_types import FloatTensorType
436
+
437
+ n_cols = model.est.n_features_in_
438
+ initial_types = [("input", FloatTensorType([None, n_cols]))]
439
+ return onnxmltools.convert_sklearn(
440
+ model.est,
441
+ name=None,
442
+ initial_types=initial_types,
443
+ doc_string="",
444
+ target_opset=None,
445
+ targeted_onnx=onnx.__version__,
446
+ custom_conversion_functions=None,
447
+ custom_shape_calculators=None,
448
+ )
449
+
450
+
451
+ def _automl_to_pkl(model=None, target_dir=None, **kwargs):
452
+ with open(os.path.join(target_dir, "model.pkl"), "wb") as outfile:
453
+ cloudpickle.dump(model, outfile)
454
+
455
+
456
+ @runtime_dependency(module="lightgbm", install_from=OptionalDependency.BOOSTED)
457
+ @runtime_dependency(module="skl2onnx", install_from=OptionalDependency.ONNX)
458
+ @runtime_dependency(module="onnxmltools", install_from=OptionalDependency.ONNX)
459
+ @runtime_dependency(module="onnx", install_from=OptionalDependency.ONNX)
460
+ def _lightgbm_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
461
+ from skl2onnx.common.data_types import FloatTensorType
462
+
463
+ in_types = [("input", FloatTensorType([None, len(X.columns)]))]
464
+
465
+ if str(type(model.est)) == "<class 'sklearn.pipeline.Pipeline'>":
466
+ model_est_types = [type(val[1]) for val in model.est.steps]
467
+ else:
468
+ model_est_types = [type(model.est)]
469
+
470
+ if lightgbm.sklearn.LGBMClassifier in model_est_types:
471
+ from onnxmltools.convert.lightgbm.operator_converters.LightGbm import (
472
+ convert_lightgbm,
473
+ )
474
+ from skl2onnx.common.shape_calculator import (
475
+ calculate_linear_classifier_output_shapes,
476
+ )
477
+
478
+ skl2onnx.update_registered_converter(
479
+ lightgbm.LGBMClassifier,
480
+ "LightGbmLGBMClassifier",
481
+ calculate_linear_classifier_output_shapes,
482
+ convert_lightgbm,
483
+ options={"nocl": [True, False], "zipmap": [True, False]},
484
+ )
485
+ elif lightgbm.sklearn.LGBMRegressor in model_est_types:
486
+ from onnxmltools.convert.lightgbm.operator_converters.LightGbm import (
487
+ convert_lightgbm,
488
+ )
489
+ from skl2onnx.common.shape_calculator import (
490
+ calculate_linear_regressor_output_shapes,
491
+ )
492
+
493
+ skl2onnx.update_registered_converter(
494
+ lightgbm.LGBMRegressor,
495
+ "LightGbmLGBMRegressor",
496
+ calculate_linear_regressor_output_shapes,
497
+ convert_lightgbm,
498
+ options={"nocl": [True, False], "zipmap": [True, False]},
499
+ )
500
+ else:
501
+ return onnxmltools.convert_lightgbm(
502
+ model.est,
503
+ name=None,
504
+ initial_types=in_types,
505
+ doc_string="",
506
+ target_opset=None,
507
+ targeted_onnx=onnx.__version__,
508
+ custom_conversion_functions=None,
509
+ custom_shape_calculators=None,
510
+ )
511
+ return skl2onnx.convert_sklearn(model.est, initial_types=in_types)
512
+
513
+
514
+ @runtime_dependency(module="xgboost", install_from=OptionalDependency.BOOSTED)
515
+ @runtime_dependency(module="skl2onnx", install_from=OptionalDependency.ONNX)
516
+ @runtime_dependency(module="onnxmltools", install_from=OptionalDependency.ONNX)
517
+ @runtime_dependency(module="onnx", install_from=OptionalDependency.ONNX)
518
+ def _xgboost_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
519
+ from skl2onnx.common.data_types import FloatTensorType
520
+
521
+ in_types = [("input", FloatTensorType([None, len(X.columns)]))]
522
+
523
+ if str(type(model.est)) == "<class 'sklearn.pipeline.Pipeline'>":
524
+ model_est_types = [type(val[1]) for val in model.est.steps]
525
+ else:
526
+ model_est_types = [type(model.est)]
527
+ if xgboost.sklearn.XGBClassifier in model_est_types:
528
+ from onnxmltools.convert.xgboost.operator_converters.XGBoost import (
529
+ convert_xgboost,
530
+ )
531
+ from skl2onnx.common.shape_calculator import (
532
+ calculate_linear_classifier_output_shapes,
533
+ )
534
+
535
+ skl2onnx.update_registered_converter(
536
+ xgboost.XGBClassifier,
537
+ "XGBoostXGBClassifier",
538
+ calculate_linear_classifier_output_shapes,
539
+ convert_xgboost,
540
+ options={"nocl": [True, False], "zipmap": [True, False]},
541
+ )
542
+ elif xgboost.sklearn.XGBRegressor in model_est_types:
543
+ from onnxmltools.convert.xgboost.operator_converters.XGBoost import (
544
+ convert_xgboost,
545
+ )
546
+ from skl2onnx.common.shape_calculator import (
547
+ calculate_linear_regressor_output_shapes,
548
+ )
549
+
550
+ skl2onnx.update_registered_converter(
551
+ xgboost.XGBRegressor,
552
+ "XGBoostXGBRegressor",
553
+ calculate_linear_regressor_output_shapes,
554
+ convert_xgboost,
555
+ options={"nocl": [True, False], "zipmap": [True, False]},
556
+ )
557
+ else:
558
+ from onnxmltools.convert import convert_xgboost
559
+
560
+ return convert_xgboost(
561
+ model,
562
+ name=None,
563
+ initial_types=in_types,
564
+ doc_string="",
565
+ target_opset=None,
566
+ targeted_onnx=onnx.__version__,
567
+ custom_conversion_functions=None,
568
+ custom_shape_calculators=None,
569
+ )
570
+ return skl2onnx.convert_sklearn(model.est, initial_types=in_types)
571
+
572
+
573
+ @runtime_dependency(module="torch", install_from=OptionalDependency.PYTORCH)
574
+ @runtime_dependency(module="onnx", install_from=OptionalDependency.ONNX)
575
+ def _torch_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
576
+ import torch
577
+
578
+ assert hasattr(torch, "onnx"), (
579
+ f"This version of pytorch {torch.__version__} does not appear to support onnx "
580
+ f"conversion "
581
+ )
582
+ # Add variable batch size to the beginning of the shape
583
+ sample_input = torch.randn([1] + [X[:1].shape[1]], requires_grad=True)
584
+ torch.onnx.export(
585
+ model.est,
586
+ sample_input,
587
+ f=os.path.join(target_dir, "model.onnx"),
588
+ input_names=["input"],
589
+ output_names=["output"],
590
+ dynamic_axes={"input": {0: "batch_size"}, "output": {0: "batch_size"}},
591
+ )
592
+
593
+
594
+ @runtime_dependency(module="onnxmltools", install_from=OptionalDependency.ONNX)
595
+ def _tf_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
596
+ try:
597
+ from onnxmltools.convert import convert_tensorflow
598
+
599
+ return convert_tensorflow(
600
+ model,
601
+ name=None,
602
+ input_names=None,
603
+ output_names=None,
604
+ doc_string="",
605
+ target_opset=None,
606
+ channel_first_inputs=None,
607
+ debug_mode=False,
608
+ custom_op_conversions=None,
609
+ )
610
+ except Exception as e:
611
+ _log_automatic_serialization_keras(e)
612
+
613
+
614
+ @runtime_dependency(module="onnxmltools", install_from=OptionalDependency.ONNX)
615
+ @runtime_dependency(module="onnx", install_from=OptionalDependency.ONNX)
616
+ def _keras_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
617
+ try:
618
+ from onnxmltools.convert import convert_keras
619
+
620
+ return convert_keras(
621
+ model,
622
+ name=None,
623
+ initial_types=None,
624
+ doc_string="",
625
+ target_opset=None,
626
+ targeted_onnx=onnx.__version__,
627
+ channel_first_inputs=None,
628
+ custom_conversion_functions=None,
629
+ custom_shape_calculators=None,
630
+ default_batch_size=1,
631
+ )
632
+ except Exception as e:
633
+ _log_automatic_serialization_keras(e)
634
+
635
+
636
+ @runtime_dependency(module="onnx", install_from=OptionalDependency.ONNX)
637
+ def _mxnet_to_onnx(model=None, target_dir=None, X=None, y=None, **kwargs):
638
+ assert onnx.__version__ == "1.3.0", "Mxnet can only export to onnx version 1.3.0"
639
+ from mxnet.contrib import onnx as onnx_mxnet
640
+
641
+ sym = kwargs.get("sym", None)
642
+ params = kwargs.get("params", None)
643
+ assert sym is not None and params is not None, (
644
+ "Pass `sym` [Path to the json file or Symbol object] and "
645
+ "`params` [Path to the params file or params dictionary. "
646
+ "(Including both arg_params and aux_params)] to kwargs"
647
+ )
648
+ input_shape = X.shape
649
+ onnx_file = "model.onnx"
650
+ onnx_mxnet.export_model(
651
+ sym=sym,
652
+ params=params,
653
+ input_shape=[input_shape],
654
+ input_type=np.float32,
655
+ onnx_file_path=onnx_file,
656
+ )
657
+
658
+
659
+ def _log_automatic_serialization_keras(e):
660
+ logger.error(
661
+ f"The following error occured: {e}. "
662
+ f"This may be because automatic serialization for Keras models is not supported. "
663
+ f"Use `prepare_generic_model()` method to manually serialize the model."
664
+ )
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*--
3
+
4
+ # Copyright (c) 2022 Oracle and/or its affiliates.
5
+ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
6
+
7
+ """
8
+ The module created for the back compatability.
9
+ The original `model_metadata` was moved to the `ads.model` package.
10
+ """
11
+
12
+ import warnings
13
+
14
+ warnings.warn(
15
+ (
16
+ "The `ads.common.model_metadata` is deprecated in `oracle-ads 2.6.8` "
17
+ "and will be removed in future release. "
18
+ "Use the `ads.model.model_metadata` instead."
19
+ ),
20
+ DeprecationWarning,
21
+ stacklevel=2,
22
+ )
23
+
24
+ from ads.model.model_metadata import *