oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (858) hide show
  1. ads/aqua/__init__.py +40 -0
  2. ads/aqua/app.py +507 -0
  3. ads/aqua/cli.py +96 -0
  4. ads/aqua/client/__init__.py +3 -0
  5. ads/aqua/client/client.py +836 -0
  6. ads/aqua/client/openai_client.py +305 -0
  7. ads/aqua/common/__init__.py +5 -0
  8. ads/aqua/common/decorator.py +125 -0
  9. ads/aqua/common/entities.py +274 -0
  10. ads/aqua/common/enums.py +134 -0
  11. ads/aqua/common/errors.py +109 -0
  12. ads/aqua/common/utils.py +1295 -0
  13. ads/aqua/config/__init__.py +4 -0
  14. ads/aqua/config/container_config.py +247 -0
  15. ads/aqua/config/evaluation/__init__.py +4 -0
  16. ads/aqua/config/evaluation/evaluation_service_config.py +147 -0
  17. ads/aqua/config/utils/__init__.py +4 -0
  18. ads/aqua/config/utils/serializer.py +339 -0
  19. ads/aqua/constants.py +116 -0
  20. ads/aqua/data.py +14 -0
  21. ads/aqua/dummy_data/icon.txt +1 -0
  22. ads/aqua/dummy_data/oci_model_deployments.json +56 -0
  23. ads/aqua/dummy_data/oci_models.json +1 -0
  24. ads/aqua/dummy_data/readme.md +26 -0
  25. ads/aqua/evaluation/__init__.py +8 -0
  26. ads/aqua/evaluation/constants.py +53 -0
  27. ads/aqua/evaluation/entities.py +186 -0
  28. ads/aqua/evaluation/errors.py +70 -0
  29. ads/aqua/evaluation/evaluation.py +1814 -0
  30. ads/aqua/extension/__init__.py +42 -0
  31. ads/aqua/extension/aqua_ws_msg_handler.py +76 -0
  32. ads/aqua/extension/base_handler.py +90 -0
  33. ads/aqua/extension/common_handler.py +121 -0
  34. ads/aqua/extension/common_ws_msg_handler.py +36 -0
  35. ads/aqua/extension/deployment_handler.py +381 -0
  36. ads/aqua/extension/deployment_ws_msg_handler.py +54 -0
  37. ads/aqua/extension/errors.py +30 -0
  38. ads/aqua/extension/evaluation_handler.py +129 -0
  39. ads/aqua/extension/evaluation_ws_msg_handler.py +61 -0
  40. ads/aqua/extension/finetune_handler.py +96 -0
  41. ads/aqua/extension/model_handler.py +390 -0
  42. ads/aqua/extension/models/__init__.py +0 -0
  43. ads/aqua/extension/models/ws_models.py +145 -0
  44. ads/aqua/extension/models_ws_msg_handler.py +50 -0
  45. ads/aqua/extension/ui_handler.py +300 -0
  46. ads/aqua/extension/ui_websocket_handler.py +130 -0
  47. ads/aqua/extension/utils.py +133 -0
  48. ads/aqua/finetuning/__init__.py +7 -0
  49. ads/aqua/finetuning/constants.py +23 -0
  50. ads/aqua/finetuning/entities.py +181 -0
  51. ads/aqua/finetuning/finetuning.py +749 -0
  52. ads/aqua/model/__init__.py +8 -0
  53. ads/aqua/model/constants.py +60 -0
  54. ads/aqua/model/entities.py +385 -0
  55. ads/aqua/model/enums.py +32 -0
  56. ads/aqua/model/model.py +2134 -0
  57. ads/aqua/model/utils.py +52 -0
  58. ads/aqua/modeldeployment/__init__.py +6 -0
  59. ads/aqua/modeldeployment/constants.py +10 -0
  60. ads/aqua/modeldeployment/deployment.py +1315 -0
  61. ads/aqua/modeldeployment/entities.py +653 -0
  62. ads/aqua/modeldeployment/utils.py +543 -0
  63. ads/aqua/resources/gpu_shapes_index.json +94 -0
  64. ads/aqua/server/__init__.py +4 -0
  65. ads/aqua/server/__main__.py +24 -0
  66. ads/aqua/server/app.py +47 -0
  67. ads/aqua/server/aqua_spec.yml +1291 -0
  68. ads/aqua/training/__init__.py +4 -0
  69. ads/aqua/training/exceptions.py +476 -0
  70. ads/aqua/ui.py +519 -0
  71. ads/automl/__init__.py +9 -0
  72. ads/automl/driver.py +330 -0
  73. ads/automl/provider.py +975 -0
  74. ads/bds/__init__.py +5 -0
  75. ads/bds/auth.py +127 -0
  76. ads/bds/big_data_service.py +255 -0
  77. ads/catalog/__init__.py +19 -0
  78. ads/catalog/model.py +1576 -0
  79. ads/catalog/notebook.py +461 -0
  80. ads/catalog/project.py +468 -0
  81. ads/catalog/summary.py +178 -0
  82. ads/common/__init__.py +11 -0
  83. ads/common/analyzer.py +65 -0
  84. ads/common/artifact/.model-ignore +63 -0
  85. ads/common/artifact/__init__.py +10 -0
  86. ads/common/auth.py +1122 -0
  87. ads/common/card_identifier.py +83 -0
  88. ads/common/config.py +647 -0
  89. ads/common/data.py +165 -0
  90. ads/common/decorator/__init__.py +9 -0
  91. ads/common/decorator/argument_to_case.py +88 -0
  92. ads/common/decorator/deprecate.py +69 -0
  93. ads/common/decorator/require_nonempty_arg.py +65 -0
  94. ads/common/decorator/runtime_dependency.py +178 -0
  95. ads/common/decorator/threaded.py +97 -0
  96. ads/common/decorator/utils.py +35 -0
  97. ads/common/dsc_file_system.py +303 -0
  98. ads/common/error.py +14 -0
  99. ads/common/extended_enum.py +81 -0
  100. ads/common/function/__init__.py +5 -0
  101. ads/common/function/fn_util.py +142 -0
  102. ads/common/function/func_conf.yaml +25 -0
  103. ads/common/ipython.py +76 -0
  104. ads/common/model.py +679 -0
  105. ads/common/model_artifact.py +1759 -0
  106. ads/common/model_artifact_schema.json +107 -0
  107. ads/common/model_export_util.py +664 -0
  108. ads/common/model_metadata.py +24 -0
  109. ads/common/object_storage_details.py +296 -0
  110. ads/common/oci_client.py +179 -0
  111. ads/common/oci_datascience.py +46 -0
  112. ads/common/oci_logging.py +1144 -0
  113. ads/common/oci_mixin.py +957 -0
  114. ads/common/oci_resource.py +136 -0
  115. ads/common/serializer.py +559 -0
  116. ads/common/utils.py +1852 -0
  117. ads/common/word_lists.py +1491 -0
  118. ads/common/work_request.py +189 -0
  119. ads/config.py +1 -0
  120. ads/data_labeling/__init__.py +13 -0
  121. ads/data_labeling/boundingbox.py +253 -0
  122. ads/data_labeling/constants.py +47 -0
  123. ads/data_labeling/data_labeling_service.py +244 -0
  124. ads/data_labeling/interface/__init__.py +5 -0
  125. ads/data_labeling/interface/loader.py +16 -0
  126. ads/data_labeling/interface/parser.py +16 -0
  127. ads/data_labeling/interface/reader.py +23 -0
  128. ads/data_labeling/loader/__init__.py +5 -0
  129. ads/data_labeling/loader/file_loader.py +241 -0
  130. ads/data_labeling/metadata.py +110 -0
  131. ads/data_labeling/mixin/__init__.py +5 -0
  132. ads/data_labeling/mixin/data_labeling.py +232 -0
  133. ads/data_labeling/ner.py +129 -0
  134. ads/data_labeling/parser/__init__.py +5 -0
  135. ads/data_labeling/parser/dls_record_parser.py +388 -0
  136. ads/data_labeling/parser/export_metadata_parser.py +94 -0
  137. ads/data_labeling/parser/export_record_parser.py +473 -0
  138. ads/data_labeling/reader/__init__.py +5 -0
  139. ads/data_labeling/reader/dataset_reader.py +574 -0
  140. ads/data_labeling/reader/dls_record_reader.py +121 -0
  141. ads/data_labeling/reader/export_record_reader.py +62 -0
  142. ads/data_labeling/reader/jsonl_reader.py +75 -0
  143. ads/data_labeling/reader/metadata_reader.py +203 -0
  144. ads/data_labeling/reader/record_reader.py +263 -0
  145. ads/data_labeling/record.py +52 -0
  146. ads/data_labeling/visualizer/__init__.py +5 -0
  147. ads/data_labeling/visualizer/image_visualizer.py +525 -0
  148. ads/data_labeling/visualizer/text_visualizer.py +357 -0
  149. ads/database/__init__.py +5 -0
  150. ads/database/connection.py +338 -0
  151. ads/dataset/__init__.py +10 -0
  152. ads/dataset/capabilities.md +51 -0
  153. ads/dataset/classification_dataset.py +339 -0
  154. ads/dataset/correlation.py +226 -0
  155. ads/dataset/correlation_plot.py +563 -0
  156. ads/dataset/dask_series.py +173 -0
  157. ads/dataset/dataframe_transformer.py +110 -0
  158. ads/dataset/dataset.py +1979 -0
  159. ads/dataset/dataset_browser.py +360 -0
  160. ads/dataset/dataset_with_target.py +995 -0
  161. ads/dataset/exception.py +25 -0
  162. ads/dataset/factory.py +987 -0
  163. ads/dataset/feature_engineering_transformer.py +35 -0
  164. ads/dataset/feature_selection.py +107 -0
  165. ads/dataset/forecasting_dataset.py +26 -0
  166. ads/dataset/helper.py +1450 -0
  167. ads/dataset/label_encoder.py +99 -0
  168. ads/dataset/mixin/__init__.py +5 -0
  169. ads/dataset/mixin/dataset_accessor.py +134 -0
  170. ads/dataset/pipeline.py +58 -0
  171. ads/dataset/plot.py +710 -0
  172. ads/dataset/progress.py +86 -0
  173. ads/dataset/recommendation.py +297 -0
  174. ads/dataset/recommendation_transformer.py +502 -0
  175. ads/dataset/regression_dataset.py +14 -0
  176. ads/dataset/sampled_dataset.py +1050 -0
  177. ads/dataset/target.py +98 -0
  178. ads/dataset/timeseries.py +18 -0
  179. ads/dbmixin/__init__.py +5 -0
  180. ads/dbmixin/db_pandas_accessor.py +153 -0
  181. ads/environment/__init__.py +9 -0
  182. ads/environment/ml_runtime.py +66 -0
  183. ads/evaluations/README.md +14 -0
  184. ads/evaluations/__init__.py +109 -0
  185. ads/evaluations/evaluation_plot.py +983 -0
  186. ads/evaluations/evaluator.py +1334 -0
  187. ads/evaluations/statistical_metrics.py +543 -0
  188. ads/experiments/__init__.py +9 -0
  189. ads/experiments/capabilities.md +0 -0
  190. ads/explanations/__init__.py +21 -0
  191. ads/explanations/base_explainer.py +142 -0
  192. ads/explanations/capabilities.md +83 -0
  193. ads/explanations/explainer.py +190 -0
  194. ads/explanations/mlx_global_explainer.py +1050 -0
  195. ads/explanations/mlx_interface.py +386 -0
  196. ads/explanations/mlx_local_explainer.py +287 -0
  197. ads/explanations/mlx_whatif_explainer.py +201 -0
  198. ads/feature_engineering/__init__.py +20 -0
  199. ads/feature_engineering/accessor/__init__.py +5 -0
  200. ads/feature_engineering/accessor/dataframe_accessor.py +535 -0
  201. ads/feature_engineering/accessor/mixin/__init__.py +5 -0
  202. ads/feature_engineering/accessor/mixin/correlation.py +166 -0
  203. ads/feature_engineering/accessor/mixin/eda_mixin.py +266 -0
  204. ads/feature_engineering/accessor/mixin/eda_mixin_series.py +85 -0
  205. ads/feature_engineering/accessor/mixin/feature_types_mixin.py +211 -0
  206. ads/feature_engineering/accessor/mixin/utils.py +65 -0
  207. ads/feature_engineering/accessor/series_accessor.py +431 -0
  208. ads/feature_engineering/adsimage/__init__.py +5 -0
  209. ads/feature_engineering/adsimage/image.py +192 -0
  210. ads/feature_engineering/adsimage/image_reader.py +170 -0
  211. ads/feature_engineering/adsimage/interface/__init__.py +5 -0
  212. ads/feature_engineering/adsimage/interface/reader.py +19 -0
  213. ads/feature_engineering/adsstring/__init__.py +7 -0
  214. ads/feature_engineering/adsstring/oci_language/__init__.py +8 -0
  215. ads/feature_engineering/adsstring/string/__init__.py +8 -0
  216. ads/feature_engineering/data_schema.json +57 -0
  217. ads/feature_engineering/dataset/__init__.py +5 -0
  218. ads/feature_engineering/dataset/zip_code_data.py +42062 -0
  219. ads/feature_engineering/exceptions.py +40 -0
  220. ads/feature_engineering/feature_type/__init__.py +133 -0
  221. ads/feature_engineering/feature_type/address.py +184 -0
  222. ads/feature_engineering/feature_type/adsstring/__init__.py +5 -0
  223. ads/feature_engineering/feature_type/adsstring/common_regex_mixin.py +164 -0
  224. ads/feature_engineering/feature_type/adsstring/oci_language.py +93 -0
  225. ads/feature_engineering/feature_type/adsstring/parsers/__init__.py +5 -0
  226. ads/feature_engineering/feature_type/adsstring/parsers/base.py +47 -0
  227. ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py +96 -0
  228. ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py +221 -0
  229. ads/feature_engineering/feature_type/adsstring/string.py +258 -0
  230. ads/feature_engineering/feature_type/base.py +58 -0
  231. ads/feature_engineering/feature_type/boolean.py +183 -0
  232. ads/feature_engineering/feature_type/category.py +146 -0
  233. ads/feature_engineering/feature_type/constant.py +137 -0
  234. ads/feature_engineering/feature_type/continuous.py +151 -0
  235. ads/feature_engineering/feature_type/creditcard.py +314 -0
  236. ads/feature_engineering/feature_type/datetime.py +190 -0
  237. ads/feature_engineering/feature_type/discrete.py +134 -0
  238. ads/feature_engineering/feature_type/document.py +43 -0
  239. ads/feature_engineering/feature_type/gis.py +251 -0
  240. ads/feature_engineering/feature_type/handler/__init__.py +5 -0
  241. ads/feature_engineering/feature_type/handler/feature_validator.py +524 -0
  242. ads/feature_engineering/feature_type/handler/feature_warning.py +319 -0
  243. ads/feature_engineering/feature_type/handler/warnings.py +128 -0
  244. ads/feature_engineering/feature_type/integer.py +142 -0
  245. ads/feature_engineering/feature_type/ip_address.py +144 -0
  246. ads/feature_engineering/feature_type/ip_address_v4.py +138 -0
  247. ads/feature_engineering/feature_type/ip_address_v6.py +138 -0
  248. ads/feature_engineering/feature_type/lat_long.py +256 -0
  249. ads/feature_engineering/feature_type/object.py +43 -0
  250. ads/feature_engineering/feature_type/ordinal.py +132 -0
  251. ads/feature_engineering/feature_type/phone_number.py +135 -0
  252. ads/feature_engineering/feature_type/string.py +171 -0
  253. ads/feature_engineering/feature_type/text.py +93 -0
  254. ads/feature_engineering/feature_type/unknown.py +43 -0
  255. ads/feature_engineering/feature_type/zip_code.py +164 -0
  256. ads/feature_engineering/feature_type_manager.py +406 -0
  257. ads/feature_engineering/schema.py +795 -0
  258. ads/feature_engineering/utils.py +245 -0
  259. ads/feature_store/.readthedocs.yaml +19 -0
  260. ads/feature_store/README.md +65 -0
  261. ads/feature_store/__init__.py +9 -0
  262. ads/feature_store/common/__init__.py +0 -0
  263. ads/feature_store/common/enums.py +339 -0
  264. ads/feature_store/common/exceptions.py +18 -0
  265. ads/feature_store/common/spark_session_singleton.py +125 -0
  266. ads/feature_store/common/utils/__init__.py +0 -0
  267. ads/feature_store/common/utils/base64_encoder_decoder.py +72 -0
  268. ads/feature_store/common/utils/feature_schema_mapper.py +283 -0
  269. ads/feature_store/common/utils/transformation_utils.py +82 -0
  270. ads/feature_store/common/utils/utility.py +403 -0
  271. ads/feature_store/data_validation/__init__.py +0 -0
  272. ads/feature_store/data_validation/great_expectation.py +129 -0
  273. ads/feature_store/dataset.py +1230 -0
  274. ads/feature_store/dataset_job.py +530 -0
  275. ads/feature_store/docs/Dockerfile +7 -0
  276. ads/feature_store/docs/Makefile +44 -0
  277. ads/feature_store/docs/conf.py +28 -0
  278. ads/feature_store/docs/requirements.txt +14 -0
  279. ads/feature_store/docs/source/ads.feature_store.query.rst +20 -0
  280. ads/feature_store/docs/source/cicd.rst +137 -0
  281. ads/feature_store/docs/source/conf.py +86 -0
  282. ads/feature_store/docs/source/data_versioning.rst +33 -0
  283. ads/feature_store/docs/source/dataset.rst +388 -0
  284. ads/feature_store/docs/source/dataset_job.rst +27 -0
  285. ads/feature_store/docs/source/demo.rst +70 -0
  286. ads/feature_store/docs/source/entity.rst +78 -0
  287. ads/feature_store/docs/source/feature_group.rst +624 -0
  288. ads/feature_store/docs/source/feature_group_job.rst +29 -0
  289. ads/feature_store/docs/source/feature_store.rst +122 -0
  290. ads/feature_store/docs/source/feature_store_class.rst +123 -0
  291. ads/feature_store/docs/source/feature_validation.rst +66 -0
  292. ads/feature_store/docs/source/figures/cicd.png +0 -0
  293. ads/feature_store/docs/source/figures/data_validation.png +0 -0
  294. ads/feature_store/docs/source/figures/data_versioning.png +0 -0
  295. ads/feature_store/docs/source/figures/dataset.gif +0 -0
  296. ads/feature_store/docs/source/figures/dataset.png +0 -0
  297. ads/feature_store/docs/source/figures/dataset_lineage.png +0 -0
  298. ads/feature_store/docs/source/figures/dataset_statistics.png +0 -0
  299. ads/feature_store/docs/source/figures/dataset_statistics_viz.png +0 -0
  300. ads/feature_store/docs/source/figures/dataset_validation_results.png +0 -0
  301. ads/feature_store/docs/source/figures/dataset_validation_summary.png +0 -0
  302. ads/feature_store/docs/source/figures/drift_monitoring.png +0 -0
  303. ads/feature_store/docs/source/figures/entity.png +0 -0
  304. ads/feature_store/docs/source/figures/feature_group.png +0 -0
  305. ads/feature_store/docs/source/figures/feature_group_lineage.png +0 -0
  306. ads/feature_store/docs/source/figures/feature_group_statistics_viz.png +0 -0
  307. ads/feature_store/docs/source/figures/feature_store_deployment.png +0 -0
  308. ads/feature_store/docs/source/figures/feature_store_overview.png +0 -0
  309. ads/feature_store/docs/source/figures/featuregroup.gif +0 -0
  310. ads/feature_store/docs/source/figures/lineage_d1.png +0 -0
  311. ads/feature_store/docs/source/figures/lineage_d2.png +0 -0
  312. ads/feature_store/docs/source/figures/lineage_fg.png +0 -0
  313. ads/feature_store/docs/source/figures/logo-dark-mode.png +0 -0
  314. ads/feature_store/docs/source/figures/logo-light-mode.png +0 -0
  315. ads/feature_store/docs/source/figures/overview.png +0 -0
  316. ads/feature_store/docs/source/figures/resource_manager.png +0 -0
  317. ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png +0 -0
  318. ads/feature_store/docs/source/figures/resource_manager_home.png +0 -0
  319. ads/feature_store/docs/source/figures/stats_1.png +0 -0
  320. ads/feature_store/docs/source/figures/stats_2.png +0 -0
  321. ads/feature_store/docs/source/figures/stats_d.png +0 -0
  322. ads/feature_store/docs/source/figures/stats_fg.png +0 -0
  323. ads/feature_store/docs/source/figures/transformation.png +0 -0
  324. ads/feature_store/docs/source/figures/transformations.gif +0 -0
  325. ads/feature_store/docs/source/figures/validation.png +0 -0
  326. ads/feature_store/docs/source/figures/validation_fg.png +0 -0
  327. ads/feature_store/docs/source/figures/validation_results.png +0 -0
  328. ads/feature_store/docs/source/figures/validation_summary.png +0 -0
  329. ads/feature_store/docs/source/index.rst +81 -0
  330. ads/feature_store/docs/source/module.rst +8 -0
  331. ads/feature_store/docs/source/notebook.rst +94 -0
  332. ads/feature_store/docs/source/overview.rst +47 -0
  333. ads/feature_store/docs/source/quickstart.rst +176 -0
  334. ads/feature_store/docs/source/release_notes.rst +194 -0
  335. ads/feature_store/docs/source/setup_feature_store.rst +81 -0
  336. ads/feature_store/docs/source/statistics.rst +58 -0
  337. ads/feature_store/docs/source/transformation.rst +199 -0
  338. ads/feature_store/docs/source/ui.rst +65 -0
  339. ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst +66 -0
  340. ads/feature_store/docs/source/user_guides.setup.helm_chart.rst +192 -0
  341. ads/feature_store/docs/source/user_guides.setup.terraform.rst +338 -0
  342. ads/feature_store/entity.py +718 -0
  343. ads/feature_store/execution_strategy/__init__.py +0 -0
  344. ads/feature_store/execution_strategy/delta_lake/__init__.py +0 -0
  345. ads/feature_store/execution_strategy/delta_lake/delta_lake_service.py +375 -0
  346. ads/feature_store/execution_strategy/engine/__init__.py +0 -0
  347. ads/feature_store/execution_strategy/engine/spark_engine.py +316 -0
  348. ads/feature_store/execution_strategy/execution_strategy.py +113 -0
  349. ads/feature_store/execution_strategy/execution_strategy_provider.py +47 -0
  350. ads/feature_store/execution_strategy/spark/__init__.py +0 -0
  351. ads/feature_store/execution_strategy/spark/spark_execution.py +618 -0
  352. ads/feature_store/feature.py +192 -0
  353. ads/feature_store/feature_group.py +1494 -0
  354. ads/feature_store/feature_group_expectation.py +346 -0
  355. ads/feature_store/feature_group_job.py +602 -0
  356. ads/feature_store/feature_lineage/__init__.py +0 -0
  357. ads/feature_store/feature_lineage/graphviz_service.py +180 -0
  358. ads/feature_store/feature_option_details.py +50 -0
  359. ads/feature_store/feature_statistics/__init__.py +0 -0
  360. ads/feature_store/feature_statistics/statistics_service.py +99 -0
  361. ads/feature_store/feature_store.py +699 -0
  362. ads/feature_store/feature_store_registrar.py +518 -0
  363. ads/feature_store/input_feature_detail.py +149 -0
  364. ads/feature_store/mixin/__init__.py +4 -0
  365. ads/feature_store/mixin/oci_feature_store.py +145 -0
  366. ads/feature_store/model_details.py +73 -0
  367. ads/feature_store/query/__init__.py +0 -0
  368. ads/feature_store/query/filter.py +266 -0
  369. ads/feature_store/query/generator/__init__.py +0 -0
  370. ads/feature_store/query/generator/query_generator.py +298 -0
  371. ads/feature_store/query/join.py +161 -0
  372. ads/feature_store/query/query.py +403 -0
  373. ads/feature_store/query/validator/__init__.py +0 -0
  374. ads/feature_store/query/validator/query_validator.py +57 -0
  375. ads/feature_store/response/__init__.py +0 -0
  376. ads/feature_store/response/response_builder.py +68 -0
  377. ads/feature_store/service/__init__.py +0 -0
  378. ads/feature_store/service/oci_dataset.py +139 -0
  379. ads/feature_store/service/oci_dataset_job.py +199 -0
  380. ads/feature_store/service/oci_entity.py +125 -0
  381. ads/feature_store/service/oci_feature_group.py +164 -0
  382. ads/feature_store/service/oci_feature_group_job.py +214 -0
  383. ads/feature_store/service/oci_feature_store.py +182 -0
  384. ads/feature_store/service/oci_lineage.py +87 -0
  385. ads/feature_store/service/oci_transformation.py +104 -0
  386. ads/feature_store/statistics/__init__.py +0 -0
  387. ads/feature_store/statistics/abs_feature_value.py +49 -0
  388. ads/feature_store/statistics/charts/__init__.py +0 -0
  389. ads/feature_store/statistics/charts/abstract_feature_plot.py +37 -0
  390. ads/feature_store/statistics/charts/box_plot.py +148 -0
  391. ads/feature_store/statistics/charts/frequency_distribution.py +65 -0
  392. ads/feature_store/statistics/charts/probability_distribution.py +68 -0
  393. ads/feature_store/statistics/charts/top_k_frequent_elements.py +98 -0
  394. ads/feature_store/statistics/feature_stat.py +126 -0
  395. ads/feature_store/statistics/generic_feature_value.py +33 -0
  396. ads/feature_store/statistics/statistics.py +41 -0
  397. ads/feature_store/statistics_config.py +101 -0
  398. ads/feature_store/templates/feature_store_template.yaml +45 -0
  399. ads/feature_store/transformation.py +499 -0
  400. ads/feature_store/validation_output.py +57 -0
  401. ads/hpo/__init__.py +9 -0
  402. ads/hpo/_imports.py +91 -0
  403. ads/hpo/ads_search_space.py +439 -0
  404. ads/hpo/distributions.py +325 -0
  405. ads/hpo/objective.py +280 -0
  406. ads/hpo/search_cv.py +1657 -0
  407. ads/hpo/stopping_criterion.py +75 -0
  408. ads/hpo/tuner_artifact.py +413 -0
  409. ads/hpo/utils.py +91 -0
  410. ads/hpo/validation.py +140 -0
  411. ads/hpo/visualization/__init__.py +5 -0
  412. ads/hpo/visualization/_contour.py +23 -0
  413. ads/hpo/visualization/_edf.py +20 -0
  414. ads/hpo/visualization/_intermediate_values.py +21 -0
  415. ads/hpo/visualization/_optimization_history.py +25 -0
  416. ads/hpo/visualization/_parallel_coordinate.py +169 -0
  417. ads/hpo/visualization/_param_importances.py +26 -0
  418. ads/jobs/__init__.py +53 -0
  419. ads/jobs/ads_job.py +663 -0
  420. ads/jobs/builders/__init__.py +5 -0
  421. ads/jobs/builders/base.py +156 -0
  422. ads/jobs/builders/infrastructure/__init__.py +6 -0
  423. ads/jobs/builders/infrastructure/base.py +165 -0
  424. ads/jobs/builders/infrastructure/dataflow.py +1252 -0
  425. ads/jobs/builders/infrastructure/dsc_job.py +1894 -0
  426. ads/jobs/builders/infrastructure/dsc_job_runtime.py +1233 -0
  427. ads/jobs/builders/infrastructure/utils.py +65 -0
  428. ads/jobs/builders/runtimes/__init__.py +5 -0
  429. ads/jobs/builders/runtimes/artifact.py +338 -0
  430. ads/jobs/builders/runtimes/base.py +325 -0
  431. ads/jobs/builders/runtimes/container_runtime.py +242 -0
  432. ads/jobs/builders/runtimes/python_runtime.py +1016 -0
  433. ads/jobs/builders/runtimes/pytorch_runtime.py +204 -0
  434. ads/jobs/cli.py +104 -0
  435. ads/jobs/env_var_parser.py +131 -0
  436. ads/jobs/extension.py +160 -0
  437. ads/jobs/schema/__init__.py +5 -0
  438. ads/jobs/schema/infrastructure_schema.json +116 -0
  439. ads/jobs/schema/job_schema.json +42 -0
  440. ads/jobs/schema/runtime_schema.json +183 -0
  441. ads/jobs/schema/validator.py +141 -0
  442. ads/jobs/serializer.py +296 -0
  443. ads/jobs/templates/__init__.py +5 -0
  444. ads/jobs/templates/container.py +6 -0
  445. ads/jobs/templates/driver_notebook.py +177 -0
  446. ads/jobs/templates/driver_oci.py +500 -0
  447. ads/jobs/templates/driver_python.py +48 -0
  448. ads/jobs/templates/driver_pytorch.py +852 -0
  449. ads/jobs/templates/driver_utils.py +615 -0
  450. ads/jobs/templates/hostname_from_env.c +55 -0
  451. ads/jobs/templates/oci_metrics.py +181 -0
  452. ads/jobs/utils.py +104 -0
  453. ads/llm/__init__.py +28 -0
  454. ads/llm/autogen/__init__.py +2 -0
  455. ads/llm/autogen/constants.py +15 -0
  456. ads/llm/autogen/reports/__init__.py +2 -0
  457. ads/llm/autogen/reports/base.py +67 -0
  458. ads/llm/autogen/reports/data.py +103 -0
  459. ads/llm/autogen/reports/session.py +526 -0
  460. ads/llm/autogen/reports/templates/chat_box.html +13 -0
  461. ads/llm/autogen/reports/templates/chat_box_lt.html +5 -0
  462. ads/llm/autogen/reports/templates/chat_box_rt.html +6 -0
  463. ads/llm/autogen/reports/utils.py +56 -0
  464. ads/llm/autogen/v02/__init__.py +4 -0
  465. ads/llm/autogen/v02/client.py +295 -0
  466. ads/llm/autogen/v02/log_handlers/__init__.py +2 -0
  467. ads/llm/autogen/v02/log_handlers/oci_file_handler.py +83 -0
  468. ads/llm/autogen/v02/loggers/__init__.py +6 -0
  469. ads/llm/autogen/v02/loggers/metric_logger.py +320 -0
  470. ads/llm/autogen/v02/loggers/session_logger.py +580 -0
  471. ads/llm/autogen/v02/loggers/utils.py +86 -0
  472. ads/llm/autogen/v02/runtime_logging.py +163 -0
  473. ads/llm/chain.py +268 -0
  474. ads/llm/chat_template.py +31 -0
  475. ads/llm/deploy.py +63 -0
  476. ads/llm/guardrails/__init__.py +5 -0
  477. ads/llm/guardrails/base.py +442 -0
  478. ads/llm/guardrails/huggingface.py +44 -0
  479. ads/llm/langchain/__init__.py +5 -0
  480. ads/llm/langchain/plugins/__init__.py +5 -0
  481. ads/llm/langchain/plugins/chat_models/__init__.py +5 -0
  482. ads/llm/langchain/plugins/chat_models/oci_data_science.py +1027 -0
  483. ads/llm/langchain/plugins/embeddings/__init__.py +4 -0
  484. ads/llm/langchain/plugins/embeddings/oci_data_science_model_deployment_endpoint.py +184 -0
  485. ads/llm/langchain/plugins/llms/__init__.py +5 -0
  486. ads/llm/langchain/plugins/llms/oci_data_science_model_deployment_endpoint.py +979 -0
  487. ads/llm/requirements.txt +3 -0
  488. ads/llm/serialize.py +219 -0
  489. ads/llm/serializers/__init__.py +0 -0
  490. ads/llm/serializers/retrieval_qa.py +153 -0
  491. ads/llm/serializers/runnable_parallel.py +27 -0
  492. ads/llm/templates/score_chain.jinja2 +155 -0
  493. ads/llm/templates/tool_chat_template_hermes.jinja +130 -0
  494. ads/llm/templates/tool_chat_template_mistral_parallel.jinja +94 -0
  495. ads/model/__init__.py +52 -0
  496. ads/model/artifact.py +573 -0
  497. ads/model/artifact_downloader.py +254 -0
  498. ads/model/artifact_uploader.py +267 -0
  499. ads/model/base_properties.py +238 -0
  500. ads/model/common/.model-ignore +66 -0
  501. ads/model/common/__init__.py +5 -0
  502. ads/model/common/utils.py +142 -0
  503. ads/model/datascience_model.py +2635 -0
  504. ads/model/deployment/__init__.py +20 -0
  505. ads/model/deployment/common/__init__.py +5 -0
  506. ads/model/deployment/common/utils.py +308 -0
  507. ads/model/deployment/model_deployer.py +466 -0
  508. ads/model/deployment/model_deployment.py +1846 -0
  509. ads/model/deployment/model_deployment_infrastructure.py +671 -0
  510. ads/model/deployment/model_deployment_properties.py +493 -0
  511. ads/model/deployment/model_deployment_runtime.py +838 -0
  512. ads/model/extractor/__init__.py +5 -0
  513. ads/model/extractor/automl_extractor.py +74 -0
  514. ads/model/extractor/embedding_onnx_extractor.py +80 -0
  515. ads/model/extractor/huggingface_extractor.py +88 -0
  516. ads/model/extractor/keras_extractor.py +84 -0
  517. ads/model/extractor/lightgbm_extractor.py +93 -0
  518. ads/model/extractor/model_info_extractor.py +114 -0
  519. ads/model/extractor/model_info_extractor_factory.py +105 -0
  520. ads/model/extractor/pytorch_extractor.py +87 -0
  521. ads/model/extractor/sklearn_extractor.py +112 -0
  522. ads/model/extractor/spark_extractor.py +89 -0
  523. ads/model/extractor/tensorflow_extractor.py +85 -0
  524. ads/model/extractor/xgboost_extractor.py +94 -0
  525. ads/model/framework/__init__.py +5 -0
  526. ads/model/framework/automl_model.py +178 -0
  527. ads/model/framework/embedding_onnx_model.py +438 -0
  528. ads/model/framework/huggingface_model.py +399 -0
  529. ads/model/framework/lightgbm_model.py +266 -0
  530. ads/model/framework/pytorch_model.py +266 -0
  531. ads/model/framework/sklearn_model.py +250 -0
  532. ads/model/framework/spark_model.py +326 -0
  533. ads/model/framework/tensorflow_model.py +254 -0
  534. ads/model/framework/xgboost_model.py +258 -0
  535. ads/model/generic_model.py +3518 -0
  536. ads/model/model_artifact_boilerplate/README.md +381 -0
  537. ads/model/model_artifact_boilerplate/__init__.py +5 -0
  538. ads/model/model_artifact_boilerplate/artifact_introspection_test/__init__.py +5 -0
  539. ads/model/model_artifact_boilerplate/artifact_introspection_test/model_artifact_validate.py +427 -0
  540. ads/model/model_artifact_boilerplate/artifact_introspection_test/requirements.txt +2 -0
  541. ads/model/model_artifact_boilerplate/runtime.yaml +7 -0
  542. ads/model/model_artifact_boilerplate/score.py +61 -0
  543. ads/model/model_file_description_schema.json +68 -0
  544. ads/model/model_introspect.py +331 -0
  545. ads/model/model_metadata.py +1810 -0
  546. ads/model/model_metadata_mixin.py +460 -0
  547. ads/model/model_properties.py +63 -0
  548. ads/model/model_version_set.py +739 -0
  549. ads/model/runtime/__init__.py +5 -0
  550. ads/model/runtime/env_info.py +306 -0
  551. ads/model/runtime/model_deployment_details.py +37 -0
  552. ads/model/runtime/model_provenance_details.py +58 -0
  553. ads/model/runtime/runtime_info.py +81 -0
  554. ads/model/runtime/schemas/inference_env_info_schema.yaml +16 -0
  555. ads/model/runtime/schemas/model_provenance_schema.yaml +36 -0
  556. ads/model/runtime/schemas/training_env_info_schema.yaml +16 -0
  557. ads/model/runtime/utils.py +201 -0
  558. ads/model/serde/__init__.py +5 -0
  559. ads/model/serde/common.py +40 -0
  560. ads/model/serde/model_input.py +547 -0
  561. ads/model/serde/model_serializer.py +1184 -0
  562. ads/model/service/__init__.py +5 -0
  563. ads/model/service/oci_datascience_model.py +1076 -0
  564. ads/model/service/oci_datascience_model_deployment.py +500 -0
  565. ads/model/service/oci_datascience_model_version_set.py +176 -0
  566. ads/model/transformer/__init__.py +5 -0
  567. ads/model/transformer/onnx_transformer.py +324 -0
  568. ads/mysqldb/__init__.py +5 -0
  569. ads/mysqldb/mysql_db.py +227 -0
  570. ads/opctl/__init__.py +18 -0
  571. ads/opctl/anomaly_detection.py +11 -0
  572. ads/opctl/backend/__init__.py +5 -0
  573. ads/opctl/backend/ads_dataflow.py +353 -0
  574. ads/opctl/backend/ads_ml_job.py +710 -0
  575. ads/opctl/backend/ads_ml_pipeline.py +164 -0
  576. ads/opctl/backend/ads_model_deployment.py +209 -0
  577. ads/opctl/backend/base.py +146 -0
  578. ads/opctl/backend/local.py +1053 -0
  579. ads/opctl/backend/marketplace/__init__.py +9 -0
  580. ads/opctl/backend/marketplace/helm_helper.py +173 -0
  581. ads/opctl/backend/marketplace/local_marketplace.py +271 -0
  582. ads/opctl/backend/marketplace/marketplace_backend_runner.py +71 -0
  583. ads/opctl/backend/marketplace/marketplace_operator_interface.py +44 -0
  584. ads/opctl/backend/marketplace/marketplace_operator_runner.py +24 -0
  585. ads/opctl/backend/marketplace/marketplace_utils.py +212 -0
  586. ads/opctl/backend/marketplace/models/__init__.py +5 -0
  587. ads/opctl/backend/marketplace/models/bearer_token.py +94 -0
  588. ads/opctl/backend/marketplace/models/marketplace_type.py +70 -0
  589. ads/opctl/backend/marketplace/models/ocir_details.py +56 -0
  590. ads/opctl/backend/marketplace/prerequisite_checker.py +238 -0
  591. ads/opctl/cli.py +707 -0
  592. ads/opctl/cmds.py +869 -0
  593. ads/opctl/conda/__init__.py +5 -0
  594. ads/opctl/conda/cli.py +193 -0
  595. ads/opctl/conda/cmds.py +749 -0
  596. ads/opctl/conda/config.yaml +34 -0
  597. ads/opctl/conda/manifest_template.yaml +13 -0
  598. ads/opctl/conda/multipart_uploader.py +188 -0
  599. ads/opctl/conda/pack.py +89 -0
  600. ads/opctl/config/__init__.py +5 -0
  601. ads/opctl/config/base.py +57 -0
  602. ads/opctl/config/diagnostics/__init__.py +5 -0
  603. ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml +62 -0
  604. ads/opctl/config/merger.py +255 -0
  605. ads/opctl/config/resolver.py +297 -0
  606. ads/opctl/config/utils.py +79 -0
  607. ads/opctl/config/validator.py +17 -0
  608. ads/opctl/config/versioner.py +68 -0
  609. ads/opctl/config/yaml_parsers/__init__.py +7 -0
  610. ads/opctl/config/yaml_parsers/base.py +58 -0
  611. ads/opctl/config/yaml_parsers/distributed/__init__.py +7 -0
  612. ads/opctl/config/yaml_parsers/distributed/yaml_parser.py +201 -0
  613. ads/opctl/constants.py +66 -0
  614. ads/opctl/decorator/__init__.py +5 -0
  615. ads/opctl/decorator/common.py +129 -0
  616. ads/opctl/diagnostics/__init__.py +5 -0
  617. ads/opctl/diagnostics/__main__.py +25 -0
  618. ads/opctl/diagnostics/check_distributed_job_requirements.py +212 -0
  619. ads/opctl/diagnostics/check_requirements.py +144 -0
  620. ads/opctl/diagnostics/requirement_exception.py +9 -0
  621. ads/opctl/distributed/README.md +109 -0
  622. ads/opctl/distributed/__init__.py +5 -0
  623. ads/opctl/distributed/certificates.py +32 -0
  624. ads/opctl/distributed/cli.py +207 -0
  625. ads/opctl/distributed/cmds.py +731 -0
  626. ads/opctl/distributed/common/__init__.py +5 -0
  627. ads/opctl/distributed/common/abstract_cluster_provider.py +449 -0
  628. ads/opctl/distributed/common/abstract_framework_spec_builder.py +88 -0
  629. ads/opctl/distributed/common/cluster_config_helper.py +103 -0
  630. ads/opctl/distributed/common/cluster_provider_factory.py +21 -0
  631. ads/opctl/distributed/common/cluster_runner.py +54 -0
  632. ads/opctl/distributed/common/framework_factory.py +29 -0
  633. ads/opctl/docker/Dockerfile.job +103 -0
  634. ads/opctl/docker/Dockerfile.job.arm +107 -0
  635. ads/opctl/docker/Dockerfile.job.gpu +175 -0
  636. ads/opctl/docker/base-env.yaml +13 -0
  637. ads/opctl/docker/cuda.repo +6 -0
  638. ads/opctl/docker/operator/.dockerignore +0 -0
  639. ads/opctl/docker/operator/Dockerfile +41 -0
  640. ads/opctl/docker/operator/Dockerfile.gpu +85 -0
  641. ads/opctl/docker/operator/cuda.repo +6 -0
  642. ads/opctl/docker/operator/environment.yaml +8 -0
  643. ads/opctl/forecast.py +11 -0
  644. ads/opctl/index.yaml +3 -0
  645. ads/opctl/model/__init__.py +5 -0
  646. ads/opctl/model/cli.py +65 -0
  647. ads/opctl/model/cmds.py +73 -0
  648. ads/opctl/operator/README.md +4 -0
  649. ads/opctl/operator/__init__.py +31 -0
  650. ads/opctl/operator/cli.py +344 -0
  651. ads/opctl/operator/cmd.py +596 -0
  652. ads/opctl/operator/common/__init__.py +5 -0
  653. ads/opctl/operator/common/backend_factory.py +460 -0
  654. ads/opctl/operator/common/const.py +27 -0
  655. ads/opctl/operator/common/data/synthetic.csv +16001 -0
  656. ads/opctl/operator/common/dictionary_merger.py +148 -0
  657. ads/opctl/operator/common/errors.py +42 -0
  658. ads/opctl/operator/common/operator_config.py +99 -0
  659. ads/opctl/operator/common/operator_loader.py +811 -0
  660. ads/opctl/operator/common/operator_schema.yaml +130 -0
  661. ads/opctl/operator/common/operator_yaml_generator.py +152 -0
  662. ads/opctl/operator/common/utils.py +208 -0
  663. ads/opctl/operator/lowcode/__init__.py +5 -0
  664. ads/opctl/operator/lowcode/anomaly/MLoperator +16 -0
  665. ads/opctl/operator/lowcode/anomaly/README.md +207 -0
  666. ads/opctl/operator/lowcode/anomaly/__init__.py +5 -0
  667. ads/opctl/operator/lowcode/anomaly/__main__.py +103 -0
  668. ads/opctl/operator/lowcode/anomaly/cmd.py +35 -0
  669. ads/opctl/operator/lowcode/anomaly/const.py +167 -0
  670. ads/opctl/operator/lowcode/anomaly/environment.yaml +10 -0
  671. ads/opctl/operator/lowcode/anomaly/model/__init__.py +5 -0
  672. ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py +146 -0
  673. ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py +162 -0
  674. ads/opctl/operator/lowcode/anomaly/model/automlx.py +99 -0
  675. ads/opctl/operator/lowcode/anomaly/model/autots.py +115 -0
  676. ads/opctl/operator/lowcode/anomaly/model/base_model.py +404 -0
  677. ads/opctl/operator/lowcode/anomaly/model/factory.py +110 -0
  678. ads/opctl/operator/lowcode/anomaly/model/isolationforest.py +78 -0
  679. ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py +78 -0
  680. ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py +120 -0
  681. ads/opctl/operator/lowcode/anomaly/model/tods.py +119 -0
  682. ads/opctl/operator/lowcode/anomaly/operator_config.py +127 -0
  683. ads/opctl/operator/lowcode/anomaly/schema.yaml +401 -0
  684. ads/opctl/operator/lowcode/anomaly/utils.py +88 -0
  685. ads/opctl/operator/lowcode/common/__init__.py +5 -0
  686. ads/opctl/operator/lowcode/common/const.py +10 -0
  687. ads/opctl/operator/lowcode/common/data.py +116 -0
  688. ads/opctl/operator/lowcode/common/errors.py +47 -0
  689. ads/opctl/operator/lowcode/common/transformations.py +296 -0
  690. ads/opctl/operator/lowcode/common/utils.py +384 -0
  691. ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator +13 -0
  692. ads/opctl/operator/lowcode/feature_store_marketplace/README.md +30 -0
  693. ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py +5 -0
  694. ads/opctl/operator/lowcode/feature_store_marketplace/__main__.py +116 -0
  695. ads/opctl/operator/lowcode/feature_store_marketplace/cmd.py +85 -0
  696. ads/opctl/operator/lowcode/feature_store_marketplace/const.py +15 -0
  697. ads/opctl/operator/lowcode/feature_store_marketplace/environment.yaml +0 -0
  698. ads/opctl/operator/lowcode/feature_store_marketplace/models/__init__.py +4 -0
  699. ads/opctl/operator/lowcode/feature_store_marketplace/models/apigw_config.py +32 -0
  700. ads/opctl/operator/lowcode/feature_store_marketplace/models/db_config.py +43 -0
  701. ads/opctl/operator/lowcode/feature_store_marketplace/models/mysql_config.py +120 -0
  702. ads/opctl/operator/lowcode/feature_store_marketplace/models/serializable_yaml_model.py +34 -0
  703. ads/opctl/operator/lowcode/feature_store_marketplace/operator_utils.py +386 -0
  704. ads/opctl/operator/lowcode/feature_store_marketplace/schema.yaml +160 -0
  705. ads/opctl/operator/lowcode/forecast/MLoperator +25 -0
  706. ads/opctl/operator/lowcode/forecast/README.md +209 -0
  707. ads/opctl/operator/lowcode/forecast/__init__.py +5 -0
  708. ads/opctl/operator/lowcode/forecast/__main__.py +89 -0
  709. ads/opctl/operator/lowcode/forecast/cmd.py +40 -0
  710. ads/opctl/operator/lowcode/forecast/const.py +92 -0
  711. ads/opctl/operator/lowcode/forecast/environment.yaml +20 -0
  712. ads/opctl/operator/lowcode/forecast/errors.py +26 -0
  713. ads/opctl/operator/lowcode/forecast/model/__init__.py +5 -0
  714. ads/opctl/operator/lowcode/forecast/model/arima.py +279 -0
  715. ads/opctl/operator/lowcode/forecast/model/automlx.py +553 -0
  716. ads/opctl/operator/lowcode/forecast/model/autots.py +312 -0
  717. ads/opctl/operator/lowcode/forecast/model/base_model.py +875 -0
  718. ads/opctl/operator/lowcode/forecast/model/factory.py +106 -0
  719. ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py +492 -0
  720. ads/opctl/operator/lowcode/forecast/model/ml_forecast.py +243 -0
  721. ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +482 -0
  722. ads/opctl/operator/lowcode/forecast/model/prophet.py +450 -0
  723. ads/opctl/operator/lowcode/forecast/model_evaluator.py +244 -0
  724. ads/opctl/operator/lowcode/forecast/operator_config.py +234 -0
  725. ads/opctl/operator/lowcode/forecast/schema.yaml +506 -0
  726. ads/opctl/operator/lowcode/forecast/utils.py +397 -0
  727. ads/opctl/operator/lowcode/forecast/whatifserve/__init__.py +7 -0
  728. ads/opctl/operator/lowcode/forecast/whatifserve/deployment_manager.py +285 -0
  729. ads/opctl/operator/lowcode/forecast/whatifserve/score.py +246 -0
  730. ads/opctl/operator/lowcode/pii/MLoperator +17 -0
  731. ads/opctl/operator/lowcode/pii/README.md +208 -0
  732. ads/opctl/operator/lowcode/pii/__init__.py +5 -0
  733. ads/opctl/operator/lowcode/pii/__main__.py +78 -0
  734. ads/opctl/operator/lowcode/pii/cmd.py +39 -0
  735. ads/opctl/operator/lowcode/pii/constant.py +84 -0
  736. ads/opctl/operator/lowcode/pii/environment.yaml +17 -0
  737. ads/opctl/operator/lowcode/pii/errors.py +27 -0
  738. ads/opctl/operator/lowcode/pii/model/__init__.py +5 -0
  739. ads/opctl/operator/lowcode/pii/model/factory.py +82 -0
  740. ads/opctl/operator/lowcode/pii/model/guardrails.py +167 -0
  741. ads/opctl/operator/lowcode/pii/model/pii.py +145 -0
  742. ads/opctl/operator/lowcode/pii/model/processor/__init__.py +34 -0
  743. ads/opctl/operator/lowcode/pii/model/processor/email_replacer.py +34 -0
  744. ads/opctl/operator/lowcode/pii/model/processor/mbi_replacer.py +35 -0
  745. ads/opctl/operator/lowcode/pii/model/processor/name_replacer.py +225 -0
  746. ads/opctl/operator/lowcode/pii/model/processor/number_replacer.py +73 -0
  747. ads/opctl/operator/lowcode/pii/model/processor/remover.py +26 -0
  748. ads/opctl/operator/lowcode/pii/model/report.py +487 -0
  749. ads/opctl/operator/lowcode/pii/operator_config.py +95 -0
  750. ads/opctl/operator/lowcode/pii/schema.yaml +108 -0
  751. ads/opctl/operator/lowcode/pii/utils.py +43 -0
  752. ads/opctl/operator/lowcode/recommender/MLoperator +16 -0
  753. ads/opctl/operator/lowcode/recommender/README.md +206 -0
  754. ads/opctl/operator/lowcode/recommender/__init__.py +5 -0
  755. ads/opctl/operator/lowcode/recommender/__main__.py +82 -0
  756. ads/opctl/operator/lowcode/recommender/cmd.py +33 -0
  757. ads/opctl/operator/lowcode/recommender/constant.py +30 -0
  758. ads/opctl/operator/lowcode/recommender/environment.yaml +11 -0
  759. ads/opctl/operator/lowcode/recommender/model/base_model.py +212 -0
  760. ads/opctl/operator/lowcode/recommender/model/factory.py +56 -0
  761. ads/opctl/operator/lowcode/recommender/model/recommender_dataset.py +25 -0
  762. ads/opctl/operator/lowcode/recommender/model/svd.py +106 -0
  763. ads/opctl/operator/lowcode/recommender/operator_config.py +81 -0
  764. ads/opctl/operator/lowcode/recommender/schema.yaml +265 -0
  765. ads/opctl/operator/lowcode/recommender/utils.py +13 -0
  766. ads/opctl/operator/runtime/__init__.py +5 -0
  767. ads/opctl/operator/runtime/const.py +17 -0
  768. ads/opctl/operator/runtime/container_runtime_schema.yaml +50 -0
  769. ads/opctl/operator/runtime/marketplace_runtime.py +50 -0
  770. ads/opctl/operator/runtime/python_marketplace_runtime_schema.yaml +21 -0
  771. ads/opctl/operator/runtime/python_runtime_schema.yaml +21 -0
  772. ads/opctl/operator/runtime/runtime.py +115 -0
  773. ads/opctl/schema.yaml.yml +36 -0
  774. ads/opctl/script.py +40 -0
  775. ads/opctl/spark/__init__.py +5 -0
  776. ads/opctl/spark/cli.py +43 -0
  777. ads/opctl/spark/cmds.py +147 -0
  778. ads/opctl/templates/diagnostic_report_template.jinja2 +102 -0
  779. ads/opctl/utils.py +344 -0
  780. ads/oracledb/__init__.py +5 -0
  781. ads/oracledb/oracle_db.py +346 -0
  782. ads/pipeline/__init__.py +39 -0
  783. ads/pipeline/ads_pipeline.py +2279 -0
  784. ads/pipeline/ads_pipeline_run.py +772 -0
  785. ads/pipeline/ads_pipeline_step.py +605 -0
  786. ads/pipeline/builders/__init__.py +5 -0
  787. ads/pipeline/builders/infrastructure/__init__.py +5 -0
  788. ads/pipeline/builders/infrastructure/custom_script.py +32 -0
  789. ads/pipeline/cli.py +119 -0
  790. ads/pipeline/extension.py +291 -0
  791. ads/pipeline/schema/__init__.py +5 -0
  792. ads/pipeline/schema/cs_step_schema.json +35 -0
  793. ads/pipeline/schema/ml_step_schema.json +31 -0
  794. ads/pipeline/schema/pipeline_schema.json +71 -0
  795. ads/pipeline/visualizer/__init__.py +5 -0
  796. ads/pipeline/visualizer/base.py +570 -0
  797. ads/pipeline/visualizer/graph_renderer.py +272 -0
  798. ads/pipeline/visualizer/text_renderer.py +84 -0
  799. ads/secrets/__init__.py +11 -0
  800. ads/secrets/adb.py +386 -0
  801. ads/secrets/auth_token.py +86 -0
  802. ads/secrets/big_data_service.py +365 -0
  803. ads/secrets/mysqldb.py +149 -0
  804. ads/secrets/oracledb.py +160 -0
  805. ads/secrets/secrets.py +407 -0
  806. ads/telemetry/__init__.py +7 -0
  807. ads/telemetry/base.py +69 -0
  808. ads/telemetry/client.py +122 -0
  809. ads/telemetry/telemetry.py +257 -0
  810. ads/templates/dataflow_pyspark.jinja2 +13 -0
  811. ads/templates/dataflow_sparksql.jinja2 +22 -0
  812. ads/templates/func.jinja2 +20 -0
  813. ads/templates/schemas/openapi.json +1740 -0
  814. ads/templates/score-pkl.jinja2 +173 -0
  815. ads/templates/score.jinja2 +322 -0
  816. ads/templates/score_embedding_onnx.jinja2 +202 -0
  817. ads/templates/score_generic.jinja2 +165 -0
  818. ads/templates/score_huggingface_pipeline.jinja2 +217 -0
  819. ads/templates/score_lightgbm.jinja2 +185 -0
  820. ads/templates/score_onnx.jinja2 +407 -0
  821. ads/templates/score_onnx_new.jinja2 +473 -0
  822. ads/templates/score_oracle_automl.jinja2 +185 -0
  823. ads/templates/score_pyspark.jinja2 +154 -0
  824. ads/templates/score_pytorch.jinja2 +219 -0
  825. ads/templates/score_scikit-learn.jinja2 +184 -0
  826. ads/templates/score_tensorflow.jinja2 +184 -0
  827. ads/templates/score_xgboost.jinja2 +178 -0
  828. ads/text_dataset/__init__.py +5 -0
  829. ads/text_dataset/backends.py +211 -0
  830. ads/text_dataset/dataset.py +445 -0
  831. ads/text_dataset/extractor.py +207 -0
  832. ads/text_dataset/options.py +53 -0
  833. ads/text_dataset/udfs.py +22 -0
  834. ads/text_dataset/utils.py +49 -0
  835. ads/type_discovery/__init__.py +9 -0
  836. ads/type_discovery/abstract_detector.py +21 -0
  837. ads/type_discovery/constant_detector.py +41 -0
  838. ads/type_discovery/continuous_detector.py +54 -0
  839. ads/type_discovery/credit_card_detector.py +99 -0
  840. ads/type_discovery/datetime_detector.py +92 -0
  841. ads/type_discovery/discrete_detector.py +118 -0
  842. ads/type_discovery/document_detector.py +146 -0
  843. ads/type_discovery/ip_detector.py +68 -0
  844. ads/type_discovery/latlon_detector.py +90 -0
  845. ads/type_discovery/phone_number_detector.py +63 -0
  846. ads/type_discovery/type_discovery_driver.py +87 -0
  847. ads/type_discovery/typed_feature.py +594 -0
  848. ads/type_discovery/unknown_detector.py +41 -0
  849. ads/type_discovery/zipcode_detector.py +48 -0
  850. ads/vault/__init__.py +7 -0
  851. ads/vault/vault.py +237 -0
  852. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/METADATA +150 -149
  853. oracle_ads-2.13.10rc0.dist-info/RECORD +858 -0
  854. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/WHEEL +1 -2
  855. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/entry_points.txt +2 -1
  856. oracle_ads-2.13.9rc0.dist-info/RECORD +0 -9
  857. oracle_ads-2.13.9rc0.dist-info/top_level.txt +0 -1
  858. {oracle_ads-2.13.9rc0.dist-info → oracle_ads-2.13.10rc0.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,404 @@
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
4
+ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
5
+
6
+ import logging
7
+ import os
8
+ import tempfile
9
+ import time
10
+ from abc import ABC, abstractmethod
11
+ from typing import Tuple
12
+
13
+ import fsspec
14
+ import numpy as np
15
+ import pandas as pd
16
+ import report_creator as rc
17
+ from sklearn import linear_model
18
+
19
+ from ads.common.object_storage_details import ObjectStorageDetails
20
+ from ads.opctl import logger
21
+ from ads.opctl.operator.lowcode.anomaly.const import (
22
+ SUBSAMPLE_THRESHOLD,
23
+ OutputColumns,
24
+ SupportedMetrics,
25
+ )
26
+ from ads.opctl.operator.lowcode.anomaly.utils import _build_metrics_df, default_signer
27
+ from ads.opctl.operator.lowcode.common.utils import (
28
+ disable_print,
29
+ enable_print,
30
+ human_time_friendly,
31
+ write_data,
32
+ )
33
+
34
+ from ..const import NonTimeADSupportedModels, SupportedModels
35
+ from ..operator_config import AnomalyOperatorConfig, AnomalyOperatorSpec
36
+ from .anomaly_dataset import AnomalyDatasets, AnomalyOutput, TestData
37
+
38
+ logging.getLogger("report_creator").setLevel(logging.WARNING)
39
+
40
+
41
+ class AnomalyOperatorBaseModel(ABC):
42
+ """The base class for the anomaly detection operator models."""
43
+
44
+ def __init__(self, config: AnomalyOperatorConfig, datasets: AnomalyDatasets):
45
+ """Instantiates the AnomalyOperatorBaseModel instance.
46
+
47
+ Properties
48
+ ----------
49
+ config: AnomalyOperatorConfig
50
+ The anomaly detection operator configuration.
51
+ """
52
+
53
+ self.config: AnomalyOperatorConfig = config
54
+ self.spec: AnomalyOperatorSpec = config.spec
55
+ self.datasets = datasets
56
+
57
+ if self.spec.validation_data is not None:
58
+ self.X_valid_dict = self.datasets.valid_data.X_valid_dict
59
+ self.y_valid_dict = self.datasets.valid_data.y_valid_dict
60
+ else:
61
+ self.X_valid_dict = None
62
+ self.y_valid_dict = None
63
+
64
+ def generate_report(self):
65
+ """Generates the report."""
66
+ import matplotlib.pyplot as plt
67
+
68
+ plt.rcParams.update({"figure.max_open_warning": 0})
69
+
70
+ start_time = time.time()
71
+ # fallback using sklearn oneclasssvm when the sub model _build_model fails
72
+ try:
73
+ anomaly_output = self._build_model()
74
+ except Exception as e:
75
+ logger.warning(f"Found exception: {e}")
76
+ if self.spec.datetime_column:
77
+ anomaly_output = self._fallback_build_model()
78
+
79
+ elapsed_time = time.time() - start_time
80
+
81
+ summary_metrics = None
82
+ total_metrics = None
83
+ test_data = None
84
+
85
+ if self.spec.test_data:
86
+ test_data = TestData(self.spec)
87
+ total_metrics, summary_metrics = self._test_data_evaluate_metrics(
88
+ anomaly_output, test_data, elapsed_time
89
+ )
90
+ table_blocks = [
91
+ rc.DataTable(
92
+ df.head(SUBSAMPLE_THRESHOLD)
93
+ if self.spec.subsample_report_data and len(df) > SUBSAMPLE_THRESHOLD
94
+ else df,
95
+ label=col,
96
+ index=True,
97
+ )
98
+ for col, df in self.datasets.full_data_dict.items()
99
+ ]
100
+ data_table = rc.Select(blocks=table_blocks)
101
+ date_column = (
102
+ self.spec.datetime_column.name if self.spec.datetime_column else "index"
103
+ )
104
+
105
+ (
106
+ model_description,
107
+ other_sections,
108
+ ) = self._generate_report()
109
+
110
+ blocks = []
111
+ for target, df in self.datasets.full_data_dict.items():
112
+ if target in anomaly_output.list_categories():
113
+ figure_blocks = []
114
+ time_col = df[date_column].reset_index(drop=True)
115
+ anomaly_col = anomaly_output.get_anomalies_by_cat(category=target)[
116
+ OutputColumns.ANOMALY_COL
117
+ ]
118
+ anomaly_indices = [
119
+ i for i, index in enumerate(anomaly_col) if index == 1
120
+ ]
121
+ downsampled_time_col = time_col
122
+ selected_indices = list(range(len(time_col)))
123
+ if self.spec.subsample_report_data:
124
+ non_anomaly_indices = [
125
+ i for i in range(len(time_col)) if i not in anomaly_indices
126
+ ]
127
+ # Downsample non-anomalous data if it exceeds the threshold (1000)
128
+ if len(non_anomaly_indices) > SUBSAMPLE_THRESHOLD:
129
+ downsampled_non_anomaly_indices = non_anomaly_indices[
130
+ :: len(non_anomaly_indices) // SUBSAMPLE_THRESHOLD
131
+ ]
132
+ selected_indices = (
133
+ anomaly_indices + downsampled_non_anomaly_indices
134
+ )
135
+ selected_indices.sort()
136
+ downsampled_time_col = time_col[selected_indices]
137
+
138
+ columns = set(df.columns).difference({date_column})
139
+ for col in columns:
140
+ y = df[col].reset_index(drop=True)
141
+
142
+ downsampled_y = y[selected_indices]
143
+
144
+ fig, ax = plt.subplots(figsize=(8, 3), layout="constrained")
145
+ ax.grid()
146
+ ax.plot(downsampled_time_col, downsampled_y, color="black")
147
+ # Plot anomalies
148
+ for i in anomaly_indices:
149
+ ax.scatter(time_col[i], y[i], color="red", marker="o")
150
+ plt.xlabel(date_column)
151
+ plt.ylabel(col)
152
+ plt.title(f"`{col}` with reference to anomalies")
153
+ figure_blocks.append(rc.Widget(ax))
154
+ else:
155
+ figure_blocks = None
156
+
157
+ blocks.append(
158
+ rc.Group(*figure_blocks, label=target)
159
+ ) if figure_blocks else None
160
+ plots = rc.Select(blocks)
161
+
162
+ report_sections = []
163
+ title_text = rc.Heading("Anomaly Detection Report", level=1)
164
+
165
+ yaml_appendix_title = rc.Heading("Reference: YAML File", level=2)
166
+ yaml_appendix = rc.Yaml(self.config.to_dict())
167
+ summary = rc.Block(
168
+ rc.Group(
169
+ rc.Text(f"You selected the **`{self.spec.model}`** model.\n"),
170
+ model_description,
171
+ rc.Text(
172
+ "Based on your dataset, you could have also selected "
173
+ f"any of the models: `{'`, `'.join(SupportedModels.keys() if self.spec.datetime_column else NonTimeADSupportedModels.keys())}`."
174
+ ),
175
+ rc.Metric(
176
+ heading="Analysis was completed in ",
177
+ value=human_time_friendly(elapsed_time),
178
+ ),
179
+ label="Summary",
180
+ )
181
+ )
182
+ sec_text = rc.Heading("Train Evaluation Metrics", level=2)
183
+ sec = rc.DataTable(self._evaluation_metrics(anomaly_output), index=True)
184
+ evaluation_metrics_sec = [sec_text, sec]
185
+
186
+ test_metrics_sections = []
187
+ if total_metrics is not None and not total_metrics.empty:
188
+ sec_text = rc.Heading("Test Data Evaluation Metrics", level=2)
189
+ sec = rc.DataTable(total_metrics, index=True)
190
+ test_metrics_sections = test_metrics_sections + [sec_text, sec]
191
+
192
+ if summary_metrics is not None and not summary_metrics.empty:
193
+ sec_text = rc.Heading("Test Data Summary Metrics", level=2)
194
+ sec = rc.DataTable(summary_metrics, index=True)
195
+ test_metrics_sections = test_metrics_sections + [sec_text, sec]
196
+
197
+ report_sections = (
198
+ [title_text, summary]
199
+ + [plots]
200
+ + [data_table]
201
+ + evaluation_metrics_sec
202
+ + test_metrics_sections
203
+ + [yaml_appendix_title, yaml_appendix]
204
+ )
205
+
206
+ # save the report and result CSV
207
+ self._save_report(
208
+ report_sections=report_sections,
209
+ anomaly_output=anomaly_output,
210
+ test_metrics=total_metrics,
211
+ )
212
+
213
+ def _evaluation_metrics(self, anomaly_output):
214
+ total_metrics = pd.DataFrame()
215
+ for cat in anomaly_output.list_categories():
216
+ num_anomalies = anomaly_output.get_num_anomalies_by_cat(cat)
217
+ metrics_df = pd.DataFrame.from_dict(
218
+ {"Num of Anomalies": num_anomalies}, orient="index", columns=[cat]
219
+ )
220
+ total_metrics = pd.concat([total_metrics, metrics_df], axis=1)
221
+ return total_metrics
222
+
223
+ def _test_data_evaluate_metrics(self, anomaly_output, test_data, elapsed_time):
224
+ total_metrics = pd.DataFrame()
225
+ summary_metrics = pd.DataFrame()
226
+
227
+ for cat in anomaly_output.list_categories():
228
+ output = anomaly_output.category_map[cat][0]
229
+ date_col = (
230
+ self.spec.datetime_column.name if self.spec.datetime_column else "index"
231
+ )
232
+
233
+ test_data_i = test_data.get_data_for_series(cat)
234
+
235
+ dates = output[output[date_col].isin(test_data_i[date_col])][date_col]
236
+
237
+ metrics_df = _build_metrics_df(
238
+ test_data_i[test_data_i[date_col].isin(dates)][
239
+ OutputColumns.ANOMALY_COL
240
+ ].values,
241
+ output[output[date_col].isin(dates)][OutputColumns.ANOMALY_COL].values,
242
+ cat,
243
+ )
244
+ total_metrics = pd.concat([total_metrics, metrics_df], axis=1)
245
+
246
+ if total_metrics.empty:
247
+ return total_metrics, summary_metrics
248
+
249
+ summary_metrics = pd.DataFrame(
250
+ {
251
+ SupportedMetrics.MEAN_RECALL: np.mean(
252
+ total_metrics.loc[SupportedMetrics.RECALL]
253
+ ),
254
+ SupportedMetrics.MEDIAN_RECALL: np.median(
255
+ total_metrics.loc[SupportedMetrics.RECALL]
256
+ ),
257
+ SupportedMetrics.MEAN_PRECISION: np.mean(
258
+ total_metrics.loc[SupportedMetrics.PRECISION]
259
+ ),
260
+ SupportedMetrics.MEDIAN_PRECISION: np.median(
261
+ total_metrics.loc[SupportedMetrics.PRECISION]
262
+ ),
263
+ SupportedMetrics.MEAN_ACCURACY: np.mean(
264
+ total_metrics.loc[SupportedMetrics.ACCURACY]
265
+ ),
266
+ SupportedMetrics.MEDIAN_ACCURACY: np.median(
267
+ total_metrics.loc[SupportedMetrics.ACCURACY]
268
+ ),
269
+ SupportedMetrics.MEAN_F1_SCORE: np.mean(
270
+ total_metrics.loc[SupportedMetrics.F1_SCORE]
271
+ ),
272
+ SupportedMetrics.MEDIAN_F1_SCORE: np.median(
273
+ total_metrics.loc[SupportedMetrics.F1_SCORE]
274
+ ),
275
+ SupportedMetrics.MEAN_ROC_AUC: np.mean(
276
+ total_metrics.loc[SupportedMetrics.ROC_AUC]
277
+ ),
278
+ SupportedMetrics.MEDIAN_ROC_AUC: np.median(
279
+ total_metrics.loc[SupportedMetrics.ROC_AUC]
280
+ ),
281
+ SupportedMetrics.MEAN_PRC_AUC: np.mean(
282
+ total_metrics.loc[SupportedMetrics.PRC_AUC]
283
+ ),
284
+ SupportedMetrics.MEDIAN_PRC_AUC: np.median(
285
+ total_metrics.loc[SupportedMetrics.PRC_AUC]
286
+ ),
287
+ SupportedMetrics.ELAPSED_TIME: elapsed_time,
288
+ },
289
+ index=["All Targets"],
290
+ )
291
+
292
+ return total_metrics, summary_metrics
293
+
294
+ def _save_report(
295
+ self,
296
+ report_sections: Tuple,
297
+ anomaly_output: AnomalyOutput,
298
+ test_metrics: pd.DataFrame,
299
+ ):
300
+ """Saves resulting reports to the given folder."""
301
+ unique_output_dir = self.spec.output_directory.url
302
+
303
+ if ObjectStorageDetails.is_oci_path(unique_output_dir):
304
+ storage_options = default_signer()
305
+ else:
306
+ storage_options = {}
307
+
308
+ # report-creator html report
309
+ with tempfile.TemporaryDirectory() as temp_dir:
310
+ report_local_path = os.path.join(temp_dir, "___report.html")
311
+ disable_print()
312
+ with rc.ReportCreator("My Report") as report:
313
+ report.save(rc.Block(*report_sections), report_local_path)
314
+ enable_print()
315
+ with open(report_local_path) as f1:
316
+ with fsspec.open(
317
+ os.path.join(unique_output_dir, self.spec.report_file_name),
318
+ "w",
319
+ **storage_options,
320
+ ) as f2:
321
+ f2.write(f1.read())
322
+
323
+ if self.spec.generate_inliers:
324
+ inliers = anomaly_output.get_inliers(self.datasets)
325
+ write_data(
326
+ data=inliers,
327
+ filename=os.path.join(unique_output_dir, self.spec.inliers_filename),
328
+ format="csv",
329
+ storage_options=storage_options,
330
+ )
331
+
332
+ outliers = anomaly_output.get_outliers(self.datasets)
333
+ write_data(
334
+ data=outliers,
335
+ filename=os.path.join(unique_output_dir, self.spec.outliers_filename),
336
+ format="csv",
337
+ storage_options=storage_options,
338
+ )
339
+
340
+ if test_metrics is not None and not test_metrics.empty:
341
+ write_data(
342
+ data=test_metrics.rename_axis("metrics").reset_index(),
343
+ filename=os.path.join(
344
+ unique_output_dir, self.spec.test_metrics_filename
345
+ ),
346
+ format="csv",
347
+ storage_options=storage_options,
348
+ )
349
+
350
+ logger.warning(
351
+ f"The report has been successfully "
352
+ f"generated and placed to the: {unique_output_dir}."
353
+ )
354
+
355
+ def _fallback_build_model(self):
356
+ """
357
+ Fallback method for the sub model _build_model method.
358
+ """
359
+ logger.warning(
360
+ f"The build_model method has failed for the model: {self.spec.model}. "
361
+ "A fallback model will be built."
362
+ )
363
+
364
+ date_column = self.spec.datetime_column.name
365
+
366
+ anomaly_output = AnomalyOutput(date_column=date_column)
367
+
368
+ # map the output as per anomaly dataset class, 1: outlier, 0: inlier
369
+ self.outlier_map = {1: 0, -1: 1}
370
+
371
+ # Iterate over the full_data_dict items
372
+ for target, df in self.datasets.full_data_dict.items():
373
+ est = linear_model.SGDOneClassSVM(random_state=42)
374
+ est.fit(df[self.spec.target_column].fillna(0).values.reshape(-1, 1))
375
+ y_pred = np.vectorize(self.outlier_map.get)(
376
+ est.predict(df[self.spec.target_column].fillna(0).values.reshape(-1, 1))
377
+ )
378
+ scores = est.score_samples(
379
+ df[self.spec.target_column].fillna(0).values.reshape(-1, 1)
380
+ )
381
+
382
+ anomaly = pd.DataFrame(
383
+ {date_column: df[date_column], OutputColumns.ANOMALY_COL: y_pred}
384
+ ).reset_index(drop=True)
385
+ score = pd.DataFrame(
386
+ {date_column: df[date_column], OutputColumns.SCORE_COL: scores}
387
+ ).reset_index(drop=True)
388
+ anomaly_output.add_output(target, anomaly, score)
389
+
390
+ return anomaly_output
391
+
392
+ @abstractmethod
393
+ def _generate_report(self):
394
+ """
395
+ Generates the report for the particular model.
396
+ The method that needs to be implemented on the particular model level.
397
+ """
398
+
399
+ @abstractmethod
400
+ def _build_model(self) -> pd.DataFrame:
401
+ """
402
+ Build the model.
403
+ The method that needs to be implemented on the particular model level.
404
+ """
@@ -0,0 +1,110 @@
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
4
+ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
5
+
6
+ from ads.opctl.operator.lowcode.anomaly.utils import select_auto_model
7
+
8
+ from ..const import NonTimeADSupportedModels, SupportedModels
9
+ from ..operator_config import AnomalyOperatorConfig
10
+ from .anomaly_dataset import AnomalyDatasets
11
+ from .anomaly_merlion import AnomalyMerlionOperatorModel
12
+ from .autots import AutoTSOperatorModel
13
+ from .base_model import AnomalyOperatorBaseModel
14
+ from .isolationforest import IsolationForestOperatorModel
15
+ from .oneclasssvm import OneClassSVMOperatorModel
16
+ from .randomcutforest import RandomCutForestOperatorModel
17
+
18
+
19
+ class UnSupportedModelError(Exception):
20
+ """Exception raised when the model is not supported.
21
+
22
+ Attributes:
23
+ operator_config (AnomalyOperatorConfig): The operator configuration.
24
+ model_type (str): The type of the unsupported model.
25
+ """
26
+
27
+ def __init__(self, operator_config: AnomalyOperatorConfig, model_type: str):
28
+ supported_models = (
29
+ SupportedModels.values()
30
+ if operator_config.spec.datetime_column
31
+ else NonTimeADSupportedModels.values()
32
+ )
33
+ message = (
34
+ f"Model: `{model_type}` is not supported. "
35
+ f"Supported models: {supported_models}"
36
+ )
37
+ super().__init__(message)
38
+
39
+
40
+ class AnomalyOperatorModelFactory:
41
+ """
42
+ The factory class helps to instantiate proper model operator based on the model type.
43
+ """
44
+
45
+ _MAP = {
46
+ SupportedModels.AutoTS: AutoTSOperatorModel,
47
+ SupportedModels.IQR: AutoTSOperatorModel,
48
+ SupportedModels.LOF: AutoTSOperatorModel,
49
+ SupportedModels.ISOLATIONFOREST: AutoTSOperatorModel,
50
+ SupportedModels.ZSCORE: AutoTSOperatorModel,
51
+ SupportedModels.ROLLING_ZSCORE: AutoTSOperatorModel,
52
+ SupportedModels.EE: AutoTSOperatorModel,
53
+ SupportedModels.MAD: AutoTSOperatorModel,
54
+ SupportedModels.DAGMM: AnomalyMerlionOperatorModel,
55
+ SupportedModels.DEEP_POINT_ANOMALY_DETECTOR: AnomalyMerlionOperatorModel,
56
+ SupportedModels.LSTM_ED: AnomalyMerlionOperatorModel,
57
+ SupportedModels.SPECTRAL_RESIDUAL: AnomalyMerlionOperatorModel,
58
+ SupportedModels.VAE: AnomalyMerlionOperatorModel,
59
+ SupportedModels.ARIMA: AnomalyMerlionOperatorModel,
60
+ SupportedModels.ETS: AnomalyMerlionOperatorModel,
61
+ SupportedModels.PROPHET: AnomalyMerlionOperatorModel,
62
+ SupportedModels.SARIMA: AnomalyMerlionOperatorModel,
63
+ SupportedModels.BOCPD: AnomalyMerlionOperatorModel,
64
+ }
65
+
66
+ _NonTime_MAP = {
67
+ NonTimeADSupportedModels.OneClassSVM: OneClassSVMOperatorModel,
68
+ NonTimeADSupportedModels.IsolationForest: IsolationForestOperatorModel,
69
+ NonTimeADSupportedModels.RandomCutForest: RandomCutForestOperatorModel,
70
+ # TODO: Add DBScan model for non time based anomaly
71
+ # NonTimeADSupportedModels.DBScan: DBScanOperatorModel,
72
+ }
73
+
74
+ @classmethod
75
+ def get_model(
76
+ cls, operator_config: AnomalyOperatorConfig, datasets: AnomalyDatasets
77
+ ) -> AnomalyOperatorBaseModel:
78
+ """
79
+ Gets the operator model based on the model type.
80
+
81
+ Parameters
82
+ ----------
83
+ operator_config: AnomalyOperatorConfig
84
+ The anomaly detection operator config.
85
+
86
+ datasets: AnomalyDatasets
87
+ Datasets for finding anomaly
88
+
89
+ Returns
90
+ -------
91
+ AnomalyOperatorBaseModel
92
+ The anomaly detection operator model.
93
+
94
+ Raises
95
+ ------
96
+ UnSupportedModelError
97
+ In case of not supported model.
98
+ """
99
+ model_type = operator_config.spec.model
100
+ if model_type == "auto":
101
+ model_type = select_auto_model(operator_config)
102
+
103
+ model_map = (
104
+ cls._MAP if operator_config.spec.datetime_column else cls._NonTime_MAP
105
+ )
106
+
107
+ if model_type not in model_map:
108
+ raise UnSupportedModelError(operator_config, model_type)
109
+
110
+ return model_map[model_type](config=operator_config, datasets=datasets)
@@ -0,0 +1,78 @@
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
4
+ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
5
+
6
+ import logging
7
+
8
+ import numpy as np
9
+ import pandas as pd
10
+ import report_creator as rc
11
+
12
+ from ads.common.decorator.runtime_dependency import runtime_dependency
13
+ from ads.opctl.operator.lowcode.anomaly.const import OutputColumns
14
+
15
+ from .anomaly_dataset import AnomalyOutput
16
+ from .base_model import AnomalyOperatorBaseModel
17
+
18
+ logging.getLogger("report_creator").setLevel(logging.WARNING)
19
+
20
+
21
+ class IsolationForestOperatorModel(AnomalyOperatorBaseModel):
22
+ """Class representing OneClassSVM Anomaly Detection operator model."""
23
+
24
+ @runtime_dependency(
25
+ module="sklearn",
26
+ err_msg=(
27
+ "Please run `pip3 install scikit-learn` to "
28
+ "install the required dependencies for OneClassSVM."
29
+ ),
30
+ )
31
+ def _build_model(self) -> AnomalyOutput:
32
+ from sklearn.ensemble import IsolationForest
33
+
34
+ model_kwargs = self.spec.model_kwargs
35
+ # map the output as per anomaly dataset class, 1: outlier, 0: inlier
36
+ self.outlier_map = {1: 0, -1: 1}
37
+
38
+ anomaly_output = AnomalyOutput(date_column="index")
39
+
40
+ for target, df in self.datasets.full_data_dict.items():
41
+ model = IsolationForest(**model_kwargs)
42
+ model.fit(df)
43
+ y_pred = np.vectorize(self.outlier_map.get)(model.predict(df))
44
+
45
+ scores = model.score_samples(df)
46
+
47
+ index_col = df.columns[0]
48
+
49
+ anomaly = pd.DataFrame(
50
+ {index_col: df[index_col], OutputColumns.ANOMALY_COL: y_pred}
51
+ ).reset_index(drop=True)
52
+ score = pd.DataFrame(
53
+ {"index": df[index_col], OutputColumns.SCORE_COL: scores}
54
+ ).reset_index(drop=True)
55
+
56
+ anomaly_output.add_output(target, anomaly, score)
57
+
58
+ return anomaly_output
59
+
60
+ def _generate_report(self):
61
+ """Generates the report."""
62
+
63
+ other_sections = [
64
+ rc.Heading("Selected Models Overview", level=2),
65
+ rc.Text(
66
+ "The following tables provide information regarding the chosen model."
67
+ ),
68
+ ]
69
+
70
+ model_description = rc.Text(
71
+ "The Isolation Forest is an ensemble of “Isolation Trees” that “isolate” observations by recursive random partitioning"
72
+ " which can be represented by a tree structure. The number of splittings required to isolate a sample is lower for outliers and higher for inliers."
73
+ )
74
+
75
+ return (
76
+ model_description,
77
+ other_sections,
78
+ )
@@ -0,0 +1,78 @@
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
4
+ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
5
+
6
+ import logging
7
+
8
+ import numpy as np
9
+ import pandas as pd
10
+ import report_creator as rc
11
+
12
+ from ads.common.decorator.runtime_dependency import runtime_dependency
13
+ from ads.opctl.operator.lowcode.anomaly.const import OutputColumns
14
+
15
+ from .anomaly_dataset import AnomalyOutput
16
+ from .base_model import AnomalyOperatorBaseModel
17
+
18
+ logging.getLogger("report_creator").setLevel(logging.WARNING)
19
+
20
+
21
+ class OneClassSVMOperatorModel(AnomalyOperatorBaseModel):
22
+ """Class representing OneClassSVM Anomaly Detection operator model."""
23
+
24
+ @runtime_dependency(
25
+ module="sklearn",
26
+ err_msg=(
27
+ "Please run `pip3 install scikit-learn` to "
28
+ "install the required dependencies for OneClassSVM."
29
+ ),
30
+ )
31
+ def _build_model(self) -> AnomalyOutput:
32
+ from sklearn.svm import OneClassSVM
33
+
34
+ model_kwargs = self.spec.model_kwargs
35
+ # map the output as per anomaly dataset class, 1: outlier, 0: inlier
36
+ self.outlier_map = {1: 0, -1: 1}
37
+
38
+ anomaly_output = AnomalyOutput(date_column="index")
39
+
40
+ for target, df in self.datasets.full_data_dict.items():
41
+ model = OneClassSVM(**model_kwargs)
42
+ model.fit(df)
43
+ y_pred = np.vectorize(self.outlier_map.get)(model.predict(df))
44
+
45
+ scores = model.score_samples(df)
46
+
47
+ index_col = df.columns[0]
48
+
49
+ anomaly = pd.DataFrame(
50
+ {index_col: df[index_col], OutputColumns.ANOMALY_COL: y_pred}
51
+ ).reset_index(drop=True)
52
+ score = pd.DataFrame(
53
+ {"index": df[index_col], OutputColumns.SCORE_COL: scores}
54
+ ).reset_index(drop=True)
55
+
56
+ anomaly_output.add_output(target, anomaly, score)
57
+
58
+ return anomaly_output
59
+
60
+ def _generate_report(self):
61
+ """Generates the report."""
62
+
63
+ other_sections = [
64
+ rc.Heading("Selected Models Overview", level=2),
65
+ rc.Text(
66
+ "The following tables provide information regarding the chosen model."
67
+ ),
68
+ ]
69
+
70
+ model_description = rc.Text(
71
+ "The oneclasssvm model is a full-stack automated machine learning system for outlier detection. "
72
+ "It is best suited for novelty detection when the training set is not contaminated by outliers"
73
+ )
74
+
75
+ return (
76
+ model_description,
77
+ other_sections,
78
+ )