mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (444) hide show
  1. mlrun/__init__.py +3 -3
  2. mlrun/__main__.py +79 -37
  3. mlrun/api/__init__.py +1 -1
  4. mlrun/api/api/__init__.py +1 -1
  5. mlrun/api/api/api.py +4 -4
  6. mlrun/api/api/deps.py +10 -21
  7. mlrun/api/api/endpoints/__init__.py +1 -1
  8. mlrun/api/api/endpoints/artifacts.py +64 -36
  9. mlrun/api/api/endpoints/auth.py +4 -4
  10. mlrun/api/api/endpoints/background_tasks.py +11 -11
  11. mlrun/api/api/endpoints/client_spec.py +5 -5
  12. mlrun/api/api/endpoints/clusterization_spec.py +6 -4
  13. mlrun/api/api/endpoints/feature_store.py +124 -115
  14. mlrun/api/api/endpoints/files.py +22 -14
  15. mlrun/api/api/endpoints/frontend_spec.py +28 -21
  16. mlrun/api/api/endpoints/functions.py +142 -87
  17. mlrun/api/api/endpoints/grafana_proxy.py +89 -442
  18. mlrun/api/api/endpoints/healthz.py +20 -7
  19. mlrun/api/api/endpoints/hub.py +320 -0
  20. mlrun/api/api/endpoints/internal/__init__.py +1 -1
  21. mlrun/api/api/endpoints/internal/config.py +1 -1
  22. mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
  23. mlrun/api/api/endpoints/logs.py +11 -11
  24. mlrun/api/api/endpoints/model_endpoints.py +74 -70
  25. mlrun/api/api/endpoints/operations.py +13 -9
  26. mlrun/api/api/endpoints/pipelines.py +93 -88
  27. mlrun/api/api/endpoints/projects.py +35 -35
  28. mlrun/api/api/endpoints/runs.py +69 -27
  29. mlrun/api/api/endpoints/runtime_resources.py +28 -28
  30. mlrun/api/api/endpoints/schedules.py +98 -41
  31. mlrun/api/api/endpoints/secrets.py +37 -32
  32. mlrun/api/api/endpoints/submit.py +12 -12
  33. mlrun/api/api/endpoints/tags.py +20 -22
  34. mlrun/api/api/utils.py +251 -42
  35. mlrun/api/constants.py +1 -1
  36. mlrun/api/crud/__init__.py +18 -15
  37. mlrun/api/crud/artifacts.py +10 -10
  38. mlrun/api/crud/client_spec.py +4 -4
  39. mlrun/api/crud/clusterization_spec.py +3 -3
  40. mlrun/api/crud/feature_store.py +54 -46
  41. mlrun/api/crud/functions.py +3 -3
  42. mlrun/api/crud/hub.py +312 -0
  43. mlrun/api/crud/logs.py +11 -9
  44. mlrun/api/crud/model_monitoring/__init__.py +3 -3
  45. mlrun/api/crud/model_monitoring/grafana.py +435 -0
  46. mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
  47. mlrun/api/crud/notifications.py +149 -0
  48. mlrun/api/crud/pipelines.py +67 -52
  49. mlrun/api/crud/projects.py +51 -23
  50. mlrun/api/crud/runs.py +7 -5
  51. mlrun/api/crud/runtime_resources.py +13 -13
  52. mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
  53. mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
  54. mlrun/api/crud/runtimes/nuclio/function.py +505 -0
  55. mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
  56. mlrun/api/crud/secrets.py +88 -46
  57. mlrun/api/crud/tags.py +5 -5
  58. mlrun/api/db/__init__.py +1 -1
  59. mlrun/api/db/base.py +102 -54
  60. mlrun/api/db/init_db.py +2 -3
  61. mlrun/api/db/session.py +4 -12
  62. mlrun/api/db/sqldb/__init__.py +1 -1
  63. mlrun/api/db/sqldb/db.py +439 -196
  64. mlrun/api/db/sqldb/helpers.py +1 -1
  65. mlrun/api/db/sqldb/models/__init__.py +3 -3
  66. mlrun/api/db/sqldb/models/models_mysql.py +82 -64
  67. mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
  68. mlrun/api/db/sqldb/session.py +27 -20
  69. mlrun/api/initial_data.py +82 -24
  70. mlrun/api/launcher.py +196 -0
  71. mlrun/api/main.py +91 -22
  72. mlrun/api/middlewares.py +6 -5
  73. mlrun/api/migrations_mysql/env.py +1 -1
  74. mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
  75. mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
  76. mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
  77. mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
  78. mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
  79. mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
  80. mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
  81. mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
  82. mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
  83. mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
  84. mlrun/api/migrations_sqlite/env.py +1 -1
  85. mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
  86. mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
  87. mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
  88. mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
  89. mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
  90. mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
  91. mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
  92. mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
  93. mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
  94. mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
  95. mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
  96. mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
  97. mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
  98. mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
  99. mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
  100. mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
  101. mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
  102. mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
  103. mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
  104. mlrun/api/schemas/__init__.py +216 -138
  105. mlrun/api/utils/__init__.py +1 -1
  106. mlrun/api/utils/asyncio.py +1 -1
  107. mlrun/api/utils/auth/__init__.py +1 -1
  108. mlrun/api/utils/auth/providers/__init__.py +1 -1
  109. mlrun/api/utils/auth/providers/base.py +7 -7
  110. mlrun/api/utils/auth/providers/nop.py +6 -7
  111. mlrun/api/utils/auth/providers/opa.py +17 -17
  112. mlrun/api/utils/auth/verifier.py +36 -34
  113. mlrun/api/utils/background_tasks.py +24 -24
  114. mlrun/{builder.py → api/utils/builder.py} +216 -123
  115. mlrun/api/utils/clients/__init__.py +1 -1
  116. mlrun/api/utils/clients/chief.py +19 -4
  117. mlrun/api/utils/clients/iguazio.py +106 -60
  118. mlrun/api/utils/clients/log_collector.py +1 -1
  119. mlrun/api/utils/clients/nuclio.py +23 -23
  120. mlrun/api/utils/clients/protocols/grpc.py +2 -2
  121. mlrun/api/utils/db/__init__.py +1 -1
  122. mlrun/api/utils/db/alembic.py +1 -1
  123. mlrun/api/utils/db/backup.py +1 -1
  124. mlrun/api/utils/db/mysql.py +24 -25
  125. mlrun/api/utils/db/sql_collation.py +1 -1
  126. mlrun/api/utils/db/sqlite_migration.py +2 -2
  127. mlrun/api/utils/events/__init__.py +14 -0
  128. mlrun/api/utils/events/base.py +57 -0
  129. mlrun/api/utils/events/events_factory.py +41 -0
  130. mlrun/api/utils/events/iguazio.py +217 -0
  131. mlrun/api/utils/events/nop.py +55 -0
  132. mlrun/api/utils/helpers.py +16 -13
  133. mlrun/api/utils/memory_reports.py +1 -1
  134. mlrun/api/utils/periodic.py +6 -3
  135. mlrun/api/utils/projects/__init__.py +1 -1
  136. mlrun/api/utils/projects/follower.py +33 -33
  137. mlrun/api/utils/projects/leader.py +36 -34
  138. mlrun/api/utils/projects/member.py +27 -27
  139. mlrun/api/utils/projects/remotes/__init__.py +1 -1
  140. mlrun/api/utils/projects/remotes/follower.py +13 -13
  141. mlrun/api/utils/projects/remotes/leader.py +10 -10
  142. mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
  143. mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
  144. mlrun/api/utils/scheduler.py +140 -51
  145. mlrun/api/utils/singletons/__init__.py +1 -1
  146. mlrun/api/utils/singletons/db.py +9 -15
  147. mlrun/api/utils/singletons/k8s.py +677 -5
  148. mlrun/api/utils/singletons/logs_dir.py +1 -1
  149. mlrun/api/utils/singletons/project_member.py +1 -1
  150. mlrun/api/utils/singletons/scheduler.py +1 -1
  151. mlrun/artifacts/__init__.py +2 -2
  152. mlrun/artifacts/base.py +8 -2
  153. mlrun/artifacts/dataset.py +5 -3
  154. mlrun/artifacts/manager.py +7 -1
  155. mlrun/artifacts/model.py +15 -4
  156. mlrun/artifacts/plots.py +1 -1
  157. mlrun/common/__init__.py +1 -1
  158. mlrun/common/constants.py +15 -0
  159. mlrun/common/model_monitoring.py +209 -0
  160. mlrun/common/schemas/__init__.py +167 -0
  161. mlrun/{api → common}/schemas/artifact.py +13 -14
  162. mlrun/{api → common}/schemas/auth.py +10 -8
  163. mlrun/{api → common}/schemas/background_task.py +3 -3
  164. mlrun/{api → common}/schemas/client_spec.py +1 -1
  165. mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
  166. mlrun/{api → common}/schemas/constants.py +21 -8
  167. mlrun/common/schemas/events.py +36 -0
  168. mlrun/{api → common}/schemas/feature_store.py +2 -1
  169. mlrun/{api → common}/schemas/frontend_spec.py +7 -6
  170. mlrun/{api → common}/schemas/function.py +5 -5
  171. mlrun/{api → common}/schemas/http.py +3 -3
  172. mlrun/common/schemas/hub.py +134 -0
  173. mlrun/{api → common}/schemas/k8s.py +3 -3
  174. mlrun/{api → common}/schemas/memory_reports.py +1 -1
  175. mlrun/common/schemas/model_endpoints.py +342 -0
  176. mlrun/common/schemas/notification.py +57 -0
  177. mlrun/{api → common}/schemas/object.py +6 -6
  178. mlrun/{api → common}/schemas/pipeline.py +3 -3
  179. mlrun/{api → common}/schemas/project.py +6 -5
  180. mlrun/common/schemas/regex.py +24 -0
  181. mlrun/common/schemas/runs.py +30 -0
  182. mlrun/{api → common}/schemas/runtime_resource.py +3 -3
  183. mlrun/{api → common}/schemas/schedule.py +19 -7
  184. mlrun/{api → common}/schemas/secret.py +3 -3
  185. mlrun/{api → common}/schemas/tag.py +2 -2
  186. mlrun/common/types.py +25 -0
  187. mlrun/config.py +152 -20
  188. mlrun/data_types/__init__.py +7 -2
  189. mlrun/data_types/data_types.py +4 -2
  190. mlrun/data_types/infer.py +1 -1
  191. mlrun/data_types/spark.py +10 -3
  192. mlrun/datastore/__init__.py +10 -3
  193. mlrun/datastore/azure_blob.py +1 -1
  194. mlrun/datastore/base.py +185 -53
  195. mlrun/datastore/datastore.py +1 -1
  196. mlrun/datastore/filestore.py +1 -1
  197. mlrun/datastore/google_cloud_storage.py +1 -1
  198. mlrun/datastore/inmem.py +4 -1
  199. mlrun/datastore/redis.py +1 -1
  200. mlrun/datastore/s3.py +1 -1
  201. mlrun/datastore/sources.py +192 -70
  202. mlrun/datastore/spark_udf.py +44 -0
  203. mlrun/datastore/store_resources.py +4 -4
  204. mlrun/datastore/targets.py +115 -45
  205. mlrun/datastore/utils.py +127 -5
  206. mlrun/datastore/v3io.py +1 -1
  207. mlrun/datastore/wasbfs/__init__.py +1 -1
  208. mlrun/datastore/wasbfs/fs.py +1 -1
  209. mlrun/db/__init__.py +7 -5
  210. mlrun/db/base.py +112 -68
  211. mlrun/db/httpdb.py +445 -277
  212. mlrun/db/nopdb.py +491 -0
  213. mlrun/db/sqldb.py +112 -65
  214. mlrun/errors.py +6 -1
  215. mlrun/execution.py +44 -22
  216. mlrun/feature_store/__init__.py +1 -1
  217. mlrun/feature_store/api.py +143 -95
  218. mlrun/feature_store/common.py +16 -20
  219. mlrun/feature_store/feature_set.py +42 -12
  220. mlrun/feature_store/feature_vector.py +32 -21
  221. mlrun/feature_store/ingestion.py +9 -12
  222. mlrun/feature_store/retrieval/__init__.py +3 -2
  223. mlrun/feature_store/retrieval/base.py +388 -66
  224. mlrun/feature_store/retrieval/dask_merger.py +63 -151
  225. mlrun/feature_store/retrieval/job.py +30 -12
  226. mlrun/feature_store/retrieval/local_merger.py +40 -133
  227. mlrun/feature_store/retrieval/spark_merger.py +129 -127
  228. mlrun/feature_store/retrieval/storey_merger.py +173 -0
  229. mlrun/feature_store/steps.py +132 -15
  230. mlrun/features.py +8 -3
  231. mlrun/frameworks/__init__.py +1 -1
  232. mlrun/frameworks/_common/__init__.py +1 -1
  233. mlrun/frameworks/_common/artifacts_library.py +1 -1
  234. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  235. mlrun/frameworks/_common/model_handler.py +1 -1
  236. mlrun/frameworks/_common/plan.py +1 -1
  237. mlrun/frameworks/_common/producer.py +1 -1
  238. mlrun/frameworks/_common/utils.py +1 -1
  239. mlrun/frameworks/_dl_common/__init__.py +1 -1
  240. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
  241. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  242. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  243. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  244. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  245. mlrun/frameworks/_dl_common/utils.py +1 -1
  246. mlrun/frameworks/_ml_common/__init__.py +1 -1
  247. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  248. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
  249. mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
  250. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  251. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  252. mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
  253. mlrun/frameworks/_ml_common/plan.py +1 -1
  254. mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
  255. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
  256. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
  257. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  258. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  259. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  260. mlrun/frameworks/_ml_common/producer.py +1 -1
  261. mlrun/frameworks/_ml_common/utils.py +1 -1
  262. mlrun/frameworks/auto_mlrun/__init__.py +1 -1
  263. mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
  264. mlrun/frameworks/huggingface/__init__.py +1 -1
  265. mlrun/frameworks/huggingface/model_server.py +1 -1
  266. mlrun/frameworks/lgbm/__init__.py +1 -1
  267. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
  268. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  269. mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
  270. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
  271. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
  272. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  273. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  274. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  275. mlrun/frameworks/lgbm/model_handler.py +1 -1
  276. mlrun/frameworks/lgbm/model_server.py +1 -1
  277. mlrun/frameworks/lgbm/utils.py +1 -1
  278. mlrun/frameworks/onnx/__init__.py +1 -1
  279. mlrun/frameworks/onnx/dataset.py +1 -1
  280. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  281. mlrun/frameworks/onnx/model_handler.py +1 -1
  282. mlrun/frameworks/onnx/model_server.py +1 -1
  283. mlrun/frameworks/parallel_coordinates.py +1 -1
  284. mlrun/frameworks/pytorch/__init__.py +1 -1
  285. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
  286. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  287. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  288. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  289. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  290. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  291. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  292. mlrun/frameworks/pytorch/model_handler.py +1 -1
  293. mlrun/frameworks/pytorch/model_server.py +1 -1
  294. mlrun/frameworks/pytorch/utils.py +1 -1
  295. mlrun/frameworks/sklearn/__init__.py +1 -1
  296. mlrun/frameworks/sklearn/estimator.py +1 -1
  297. mlrun/frameworks/sklearn/metric.py +1 -1
  298. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  299. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  300. mlrun/frameworks/sklearn/model_handler.py +1 -1
  301. mlrun/frameworks/sklearn/utils.py +1 -1
  302. mlrun/frameworks/tf_keras/__init__.py +1 -1
  303. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
  304. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  305. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  306. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  307. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  308. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  309. mlrun/frameworks/tf_keras/model_server.py +1 -1
  310. mlrun/frameworks/tf_keras/utils.py +1 -1
  311. mlrun/frameworks/xgboost/__init__.py +1 -1
  312. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  313. mlrun/frameworks/xgboost/model_handler.py +1 -1
  314. mlrun/frameworks/xgboost/utils.py +1 -1
  315. mlrun/k8s_utils.py +14 -765
  316. mlrun/kfpops.py +14 -17
  317. mlrun/launcher/__init__.py +13 -0
  318. mlrun/launcher/base.py +406 -0
  319. mlrun/launcher/client.py +159 -0
  320. mlrun/launcher/factory.py +50 -0
  321. mlrun/launcher/local.py +276 -0
  322. mlrun/launcher/remote.py +178 -0
  323. mlrun/lists.py +10 -2
  324. mlrun/mlutils/__init__.py +1 -1
  325. mlrun/mlutils/data.py +1 -1
  326. mlrun/mlutils/models.py +1 -1
  327. mlrun/mlutils/plots.py +1 -1
  328. mlrun/model.py +252 -14
  329. mlrun/model_monitoring/__init__.py +41 -0
  330. mlrun/model_monitoring/features_drift_table.py +1 -1
  331. mlrun/model_monitoring/helpers.py +123 -38
  332. mlrun/model_monitoring/model_endpoint.py +144 -0
  333. mlrun/model_monitoring/model_monitoring_batch.py +310 -259
  334. mlrun/model_monitoring/stores/__init__.py +106 -0
  335. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
  336. mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
  337. mlrun/model_monitoring/stores/models/__init__.py +23 -0
  338. mlrun/model_monitoring/stores/models/base.py +18 -0
  339. mlrun/model_monitoring/stores/models/mysql.py +100 -0
  340. mlrun/model_monitoring/stores/models/sqlite.py +98 -0
  341. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
  342. mlrun/model_monitoring/stream_processing_fs.py +239 -271
  343. mlrun/package/__init__.py +163 -0
  344. mlrun/package/context_handler.py +325 -0
  345. mlrun/package/errors.py +47 -0
  346. mlrun/package/packager.py +298 -0
  347. mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
  348. mlrun/package/packagers/default_packager.py +422 -0
  349. mlrun/package/packagers/numpy_packagers.py +612 -0
  350. mlrun/package/packagers/pandas_packagers.py +968 -0
  351. mlrun/package/packagers/python_standard_library_packagers.py +616 -0
  352. mlrun/package/packagers_manager.py +786 -0
  353. mlrun/package/utils/__init__.py +53 -0
  354. mlrun/package/utils/_archiver.py +226 -0
  355. mlrun/package/utils/_formatter.py +211 -0
  356. mlrun/package/utils/_pickler.py +234 -0
  357. mlrun/package/utils/_supported_format.py +71 -0
  358. mlrun/package/utils/log_hint_utils.py +93 -0
  359. mlrun/package/utils/type_hint_utils.py +298 -0
  360. mlrun/platforms/__init__.py +1 -1
  361. mlrun/platforms/iguazio.py +34 -2
  362. mlrun/platforms/other.py +1 -1
  363. mlrun/projects/__init__.py +1 -1
  364. mlrun/projects/operations.py +14 -9
  365. mlrun/projects/pipelines.py +31 -13
  366. mlrun/projects/project.py +762 -238
  367. mlrun/render.py +49 -19
  368. mlrun/run.py +57 -326
  369. mlrun/runtimes/__init__.py +3 -9
  370. mlrun/runtimes/base.py +247 -784
  371. mlrun/runtimes/constants.py +1 -1
  372. mlrun/runtimes/daskjob.py +45 -41
  373. mlrun/runtimes/funcdoc.py +43 -7
  374. mlrun/runtimes/function.py +66 -656
  375. mlrun/runtimes/function_reference.py +1 -1
  376. mlrun/runtimes/generators.py +1 -1
  377. mlrun/runtimes/kubejob.py +99 -116
  378. mlrun/runtimes/local.py +59 -66
  379. mlrun/runtimes/mpijob/__init__.py +1 -1
  380. mlrun/runtimes/mpijob/abstract.py +13 -15
  381. mlrun/runtimes/mpijob/v1.py +3 -1
  382. mlrun/runtimes/mpijob/v1alpha1.py +1 -1
  383. mlrun/runtimes/nuclio.py +1 -1
  384. mlrun/runtimes/pod.py +51 -26
  385. mlrun/runtimes/remotesparkjob.py +3 -1
  386. mlrun/runtimes/serving.py +12 -4
  387. mlrun/runtimes/sparkjob/__init__.py +1 -2
  388. mlrun/runtimes/sparkjob/abstract.py +44 -31
  389. mlrun/runtimes/sparkjob/spark3job.py +11 -9
  390. mlrun/runtimes/utils.py +61 -42
  391. mlrun/secrets.py +16 -18
  392. mlrun/serving/__init__.py +3 -2
  393. mlrun/serving/merger.py +1 -1
  394. mlrun/serving/remote.py +1 -1
  395. mlrun/serving/routers.py +39 -42
  396. mlrun/serving/server.py +23 -13
  397. mlrun/serving/serving_wrapper.py +1 -1
  398. mlrun/serving/states.py +172 -39
  399. mlrun/serving/utils.py +1 -1
  400. mlrun/serving/v1_serving.py +1 -1
  401. mlrun/serving/v2_serving.py +29 -21
  402. mlrun/utils/__init__.py +1 -2
  403. mlrun/utils/async_http.py +8 -1
  404. mlrun/utils/azure_vault.py +1 -1
  405. mlrun/utils/clones.py +2 -2
  406. mlrun/utils/condition_evaluator.py +65 -0
  407. mlrun/utils/db.py +52 -0
  408. mlrun/utils/helpers.py +188 -13
  409. mlrun/utils/http.py +89 -54
  410. mlrun/utils/logger.py +48 -8
  411. mlrun/utils/model_monitoring.py +132 -100
  412. mlrun/utils/notifications/__init__.py +1 -1
  413. mlrun/utils/notifications/notification/__init__.py +8 -6
  414. mlrun/utils/notifications/notification/base.py +20 -14
  415. mlrun/utils/notifications/notification/console.py +7 -4
  416. mlrun/utils/notifications/notification/git.py +36 -19
  417. mlrun/utils/notifications/notification/ipython.py +10 -8
  418. mlrun/utils/notifications/notification/slack.py +18 -13
  419. mlrun/utils/notifications/notification_pusher.py +377 -56
  420. mlrun/utils/regex.py +6 -1
  421. mlrun/utils/singleton.py +1 -1
  422. mlrun/utils/v3io_clients.py +1 -1
  423. mlrun/utils/vault.py +270 -269
  424. mlrun/utils/version/__init__.py +1 -1
  425. mlrun/utils/version/version.json +2 -2
  426. mlrun/utils/version/version.py +1 -1
  427. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
  428. mlrun-1.4.0.dist-info/RECORD +434 -0
  429. mlrun/api/api/endpoints/marketplace.py +0 -257
  430. mlrun/api/crud/marketplace.py +0 -221
  431. mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
  432. mlrun/api/db/filedb/db.py +0 -518
  433. mlrun/api/schemas/marketplace.py +0 -128
  434. mlrun/api/schemas/model_endpoints.py +0 -185
  435. mlrun/db/filedb.py +0 -891
  436. mlrun/feature_store/retrieval/online.py +0 -92
  437. mlrun/model_monitoring/constants.py +0 -67
  438. mlrun/runtimes/package/context_handler.py +0 -711
  439. mlrun/runtimes/sparkjob/spark2job.py +0 -59
  440. mlrun-1.3.3.dist-info/RECORD +0 -381
  441. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
  442. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
  443. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
  444. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
@@ -1,711 +0,0 @@
1
- # Copyright 2018 Iguazio
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- #
15
- import inspect
16
- import os
17
- import shutil
18
- from collections import OrderedDict
19
- from enum import Enum
20
- from pathlib import Path
21
- from typing import Any, Callable, Dict, List, Type, Union
22
-
23
- import cloudpickle
24
- import numpy as np
25
- import pandas as pd
26
-
27
- from mlrun.datastore import DataItem
28
- from mlrun.errors import MLRunInvalidArgumentError, MLRunRuntimeError
29
- from mlrun.execution import MLClientCtx
30
- from mlrun.utils import logger
31
-
32
-
33
- # TODO: Move the `ArtifactType` to constants.py
34
- class ArtifactType(Enum):
35
- """
36
- Possible artifact types to log using the MLRun `context` decorator.
37
- """
38
-
39
- # Types:
40
- DATASET = "dataset"
41
- DIRECTORY = "directory"
42
- FILE = "file"
43
- OBJECT = "object"
44
- PLOT = "plot"
45
- RESULT = "result"
46
-
47
- # Constants:
48
- DEFAULT = RESULT
49
-
50
-
51
- class InputsParser:
52
- """
53
- A static class to hold all the common parsing functions - functions for parsing MLRun DataItem to the user desired
54
- type.
55
- """
56
-
57
- @staticmethod
58
- def parse_pandas_dataframe(data_item: DataItem) -> pd.DataFrame:
59
- """
60
- Parse an MLRun `DataItem` to a `pandas.DataFrame`.
61
-
62
- :param data_item: The `DataItem` to parse.
63
-
64
- :returns: The `DataItem` as a `pandas.DataFrame`.
65
- """
66
- return data_item.as_df()
67
-
68
- @staticmethod
69
- def parse_numpy_array(data_item: DataItem) -> np.ndarray:
70
- """
71
- Parse an MLRun `DataItem` to a `numpy.ndarray`.
72
-
73
- :param data_item: The `DataItem` to parse.
74
-
75
- :returns: The `DataItem` as a `numpy.ndarray`.
76
- """
77
- return data_item.as_df().to_numpy()
78
-
79
- @staticmethod
80
- def parse_dict(data_item: DataItem) -> dict:
81
- """
82
- Parse an MLRun `DataItem` to a `dict`.
83
-
84
- :param data_item: The `DataItem` to parse.
85
-
86
- :returns: The `DataItem` as a `dict`.
87
- """
88
- return data_item.as_df().to_dict()
89
-
90
- @staticmethod
91
- def parse_list(data_item: DataItem) -> list:
92
- """
93
- Parse an MLRun `DataItem` to a `list`.
94
-
95
- :param data_item: The `DataItem` to parse.
96
-
97
- :returns: The `DataItem` as a `list`.
98
- """
99
- return data_item.as_df().to_numpy().tolist()
100
-
101
- @staticmethod
102
- def parse_object(data_item: DataItem) -> object:
103
- """
104
- Parse an MLRun `DataItem` to its unpickled object. The pickle file will be downloaded to a local temp
105
- directory and then loaded.
106
-
107
- :param data_item: The `DataItem` to parse.
108
-
109
- :returns: The `DataItem` as the original object that was pickled once it was logged.
110
- """
111
- object_file = data_item.local()
112
- with open(object_file, "rb") as pickle_file:
113
- obj = cloudpickle.load(pickle_file)
114
- return obj
115
-
116
-
117
- class OutputsLogger:
118
- """
119
- A static class to hold all the common logging functions - functions for logging different objects by artifact type
120
- to MLRun.
121
- """
122
-
123
- @staticmethod
124
- def log_dataset(
125
- ctx: MLClientCtx,
126
- obj: Union[pd.DataFrame, np.ndarray, pd.Series, dict, list],
127
- key: str,
128
- logging_kwargs: dict,
129
- ):
130
- """
131
- Log an object as a dataset. The dataset wil lbe cast to a `pandas.DataFrame`. Supporting casting from
132
- `pandas.Series`, `numpy.ndarray`, `dict` and `list`.
133
-
134
- :param ctx: The MLRun context to log with.
135
- :param obj: The data to log.
136
- :param key: The key of the artifact.
137
- :param logging_kwargs: Additional keyword arguments to pass to the `context.log_dataset`
138
-
139
- :raise MLRunInvalidArgumentError: If the type is not supported for being cast to `pandas.DataFrame`.
140
- """
141
- # Check for the object type:
142
- if not isinstance(obj, pd.DataFrame):
143
- if isinstance(obj, (np.ndarray, pd.Series, dict, list)):
144
- obj = pd.DataFrame(obj)
145
- else:
146
- raise MLRunInvalidArgumentError(
147
- f"The value requested to be logged as a dataset artifact is of type '{type(obj)}' and it "
148
- f"cannot be logged as a dataset. Please parse it in your code into one `numpy.ndarray`, "
149
- f"`pandas.DataFrame`, `pandas.Series`, `dict`, `list` before returning it so we can log it."
150
- )
151
-
152
- # Log the DataFrame object as a dataset:
153
- ctx.log_dataset(**logging_kwargs, key=key, df=obj)
154
-
155
- @staticmethod
156
- def log_directory(
157
- ctx: MLClientCtx,
158
- obj: Union[str, Path],
159
- key: str,
160
- logging_kwargs: dict,
161
- ):
162
- """
163
- Log a directory as a zip file. The zip file will be created at the current working directory. Once logged,
164
- it will be deleted.
165
-
166
- :param ctx: The MLRun context to log with.
167
- :param obj: The directory to zip path.
168
- :param key: The key of the artifact.
169
- :param logging_kwargs: Additional keyword arguments to pass to the `context.log_artifact` method.
170
-
171
- :raises MLRunInvalidArgumentError: In case the given path is not of a directory or do not exist.
172
- """
173
- # In case it is a `pathlib` path, parse to str:
174
- obj = str(obj)
175
-
176
- # Verify the path is of an existing directory:
177
- if not os.path.isdir(obj):
178
- raise MLRunInvalidArgumentError(
179
- f"The given path is not a directory: '{obj}'"
180
- )
181
- if not os.path.exists(obj):
182
- raise MLRunInvalidArgumentError(
183
- f"The given directory path do not exist: '{obj}'"
184
- )
185
-
186
- # Zip the directory:
187
- directory_zip_path = shutil.make_archive(
188
- base_name=key,
189
- format="zip",
190
- root_dir=os.path.abspath(obj),
191
- )
192
-
193
- # Log the zip file:
194
- ctx.log_artifact(**logging_kwargs, item=key, local_path=directory_zip_path)
195
-
196
- # Delete the zip file:
197
- os.remove(directory_zip_path)
198
-
199
- @staticmethod
200
- def log_file(
201
- ctx: MLClientCtx,
202
- obj: Union[str, Path],
203
- key: str,
204
- logging_kwargs: dict,
205
- ):
206
- """
207
- Log a file to MLRun.
208
-
209
- :param ctx: The MLRun context to log with.
210
- :param obj: The path of the file to log.
211
- :param key: The key of the artifact.
212
- :param logging_kwargs: Additional keyword arguments to pass to the `context.log_artifact` method.
213
-
214
- :raises MLRunInvalidArgumentError: In case the given path is not of a file or do not exist.
215
- """
216
- # In case it is a `pathlib` path, parse to str:
217
- obj = str(obj)
218
-
219
- # Verify the path is of an existing directory:
220
- if not os.path.isfile(obj):
221
- raise MLRunInvalidArgumentError(f"The given path is not a file: '{obj}'")
222
- if not os.path.exists(obj):
223
- raise MLRunInvalidArgumentError(
224
- f"The given directory path do not exist: '{obj}'"
225
- )
226
-
227
- # Log the zip file:
228
- ctx.log_artifact(**logging_kwargs, item=key, local_path=os.path.abspath(obj))
229
-
230
- @staticmethod
231
- def log_object(ctx: MLClientCtx, obj, key: str, logging_kwargs: dict):
232
- """
233
- Log an object as a pickle.
234
-
235
- :param ctx: The MLRun context to log with.
236
- :param obj: The object to log.
237
- :param key: The key of the artifact.
238
- :param logging_kwargs: Additional keyword arguments to pass to the `context.log_artifact` method.
239
- """
240
- ctx.log_artifact(
241
- **logging_kwargs,
242
- item=key,
243
- body=obj if isinstance(obj, (bytes, bytearray)) else cloudpickle.dumps(obj),
244
- format="pkl",
245
- )
246
-
247
- @staticmethod
248
- def log_plot(ctx: MLClientCtx, obj, key: str, logging_kwargs: dict):
249
- """
250
- Log an object as a plot. Currently, supporting plots produced by one the following modules: `matplotlib`,
251
- `seaborn`, `plotly` and `bokeh`.
252
-
253
- :param ctx: The MLRun context to log with.
254
- :param obj: The plot to log.
255
- :param key: The key of the artifact.
256
- :param logging_kwargs: Additional keyword arguments to pass to the `context.log_artifact`.
257
-
258
- :raise MLRunInvalidArgumentError: If the object type is not supported (meaning the plot was not produced by
259
- one of the supported modules).
260
- """
261
- # Create the plot artifact according to the module produced the object:
262
- artifact = None
263
-
264
- # `matplotlib` and `seaborn`:
265
- try:
266
- import matplotlib.pyplot as plt
267
-
268
- from mlrun.artifacts import PlotArtifact
269
-
270
- # Get the figure:
271
- figure = None
272
- if isinstance(obj, plt.Figure):
273
- figure = obj
274
- elif isinstance(obj, plt.Axes):
275
- if hasattr(obj, "get_figure"):
276
- figure = obj.get_figure()
277
- elif hasattr(obj, "figure"):
278
- figure = obj.figure
279
- elif hasattr(obj, "fig"):
280
- figure = obj.fig
281
-
282
- # Create the artifact:
283
- if figure is not None:
284
- artifact = PlotArtifact(key=key, body=figure)
285
- except ModuleNotFoundError:
286
- pass
287
-
288
- # `plotly`:
289
- if artifact is None:
290
- try:
291
- import plotly
292
-
293
- from mlrun.artifacts import PlotlyArtifact
294
-
295
- if isinstance(obj, plotly.graph_objs.Figure):
296
- artifact = PlotlyArtifact(key=key, figure=obj)
297
- except ModuleNotFoundError:
298
- pass
299
-
300
- # `bokeh`:
301
- if artifact is None:
302
- try:
303
- import bokeh.plotting as bokeh_plt
304
-
305
- from mlrun.artifacts import BokehArtifact
306
-
307
- if isinstance(obj, bokeh_plt.Figure):
308
- artifact = BokehArtifact(key=key, figure=obj)
309
- except ModuleNotFoundError:
310
- pass
311
- except ImportError:
312
- logger.warn(
313
- "Bokeh installation is ignored. If needed, "
314
- "make sure you have the required version with `pip install mlrun[bokeh]`"
315
- )
316
-
317
- # Log the artifact:
318
- if artifact is None:
319
- raise MLRunInvalidArgumentError(
320
- f"The given plot is of type `{type(obj)}`. We currently support logging plots produced by one of "
321
- f"the following modules: `matplotlib`, `seaborn`, `plotly` and `bokeh`. You may try to save the "
322
- f"plot to file and log it as a file instead."
323
- )
324
- ctx.log_artifact(**logging_kwargs, item=artifact)
325
-
326
- @staticmethod
327
- def log_result(
328
- ctx: MLClientCtx,
329
- obj: Union[int, float, str, list, tuple, dict, np.ndarray],
330
- key: str,
331
- logging_kwargs: dict,
332
- ):
333
- """
334
- Log an object as a result. The objects value will be cast to a serializable version of itself. Supporting:
335
- int, float, str, list, tuple, dict, numpy.ndarray
336
-
337
- :param ctx: The MLRun context to log with.
338
- :param obj: The value to log.
339
- :param key: The key of the artifact.
340
- :param logging_kwargs: Additional keyword arguments to pass to the `context.log_result` method.
341
- """
342
- ctx.log_result(**logging_kwargs, key=key, value=obj)
343
-
344
-
345
- class ContextHandler:
346
- """
347
- Private class for handling an MLRun context of a function that is wrapped in MLRun's `handler` decorator.
348
-
349
- The context handler have 3 duties:
350
- 1. Check if the user used MLRun to run the wrapped function and if so, get the MLRun context.
351
- 2. Parse the user's inputs (MLRun `DataItem`) to the function.
352
- 3. Log the function's outputs to MLRun.
353
-
354
- The context handler use dictionaries to map objects to their logging / parsing function. The maps can be edited
355
- using the relevant `update_X` class method. If needed to add additional artifacts types, the `ArtifactType` class
356
- can be inherited and replaced as well using the `update_artifact_type_class` class method.
357
- """
358
-
359
- # The artifact type enum class to use:
360
- _ARTIFACT_TYPE_CLASS = ArtifactType
361
- # The map to use to get default artifact types of objects:
362
- _DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP = None
363
- # The map to use for logging an object by its type:
364
- _OUTPUTS_LOGGING_MAP = None
365
- # The map to use for parsing an object by its type:
366
- _INPUTS_PARSING_MAP = None
367
-
368
- @classmethod
369
- def update_artifact_type_class(cls, artifact_type_class: Type[ArtifactType]):
370
- """
371
- Update the artifact type enum class that the handler will use to specify new artifact types to log and parse.
372
-
373
- :param artifact_type_class: An enum inheriting from the `ArtifactType` enum.
374
- """
375
- cls._ARTIFACT_TYPE_CLASS = artifact_type_class
376
-
377
- @classmethod
378
- def update_default_objects_artifact_types_map(
379
- cls, updates: Dict[type, ArtifactType]
380
- ):
381
- """
382
- Enrich the default objects artifact types map with new objects types to support.
383
-
384
- :param updates: New objects types to artifact types to support.
385
- """
386
- if cls._DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP is None:
387
- cls._init_default_objects_artifact_types_map()
388
- cls._DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP.update(updates)
389
-
390
- @classmethod
391
- def update_outputs_logging_map(
392
- cls,
393
- updates: Dict[ArtifactType, Callable[[MLClientCtx, Any, str, dict], None]],
394
- ):
395
- """
396
- Enrich the outputs logging map with new artifact types to support. The outputs logging map is a dictionary of
397
- artifact type enum as key, and a function that will handle the given output. The function must accept 4 keyword
398
- arguments
399
-
400
- * ctx: `mlrun.MLClientCtx` - The MLRun context to log with.
401
- * obj: `Any` - The value / object to log.
402
- * key: `str` - The key of the artifact.
403
- * logging_kwargs: `dict` - Keyword arguments the user can pass in the instructions tuple.
404
-
405
- :param updates: New artifact types to support - a dictionary of artifact type enum as key, and a function that
406
- will handle the given output to update the current map.
407
- """
408
- if cls._OUTPUTS_LOGGING_MAP is None:
409
- cls._init_outputs_logging_map()
410
- cls._OUTPUTS_LOGGING_MAP.update(updates)
411
-
412
- @classmethod
413
- def update_inputs_parsing_map(cls, updates: Dict[type, Callable[[DataItem], Any]]):
414
- """
415
- Enrich the inputs parsing map with new objects to support. The inputs parsing map is a dictionary of object
416
- types as key, and a function that will handle the given input. The function must accept 1 keyword argument
417
- (data_item: `mlrun.DataItem`) and return the relevant parsed object.
418
-
419
- :param updates: New object types to support - a dictionary of artifact type enum as key, and a function that
420
- will handle the given input to update the current map.
421
- """
422
- if cls._INPUTS_PARSING_MAP is None:
423
- cls._init_inputs_parsing_map()
424
- cls._INPUTS_PARSING_MAP.update(updates)
425
-
426
- def __init__(self):
427
- """
428
- Initialize a context handler.
429
- """
430
- # Initialize the maps:
431
- if self._DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP is None:
432
- self._init_default_objects_artifact_types_map()
433
- if self._OUTPUTS_LOGGING_MAP is None:
434
- self._init_outputs_logging_map()
435
- if self._INPUTS_PARSING_MAP is None:
436
- self._init_inputs_parsing_map()
437
-
438
- # Set up a variable to hold the context:
439
- self._context: MLClientCtx = None
440
-
441
- def look_for_context(self, args: tuple, kwargs: dict):
442
- """
443
- Look for an MLRun context (`mlrun.MLClientCtx`). The handler will look for a context in the given order:
444
- 1. The given arguments.
445
- 2. The given keyword arguments.
446
- 3. If an MLRun RunTime was used the context will be located via the `mlrun.get_or_create_ctx` method.
447
-
448
- :param args: The arguments tuple passed to the function.
449
- :param kwargs: The keyword arguments dictionary passed to the function.
450
- """
451
- # Search in the given arguments:
452
- for argument in args:
453
- if isinstance(argument, MLClientCtx):
454
- self._context = argument
455
- return
456
-
457
- # Search in the given keyword arguments:
458
- for argument_name, argument_value in kwargs.items():
459
- if isinstance(argument_value, MLClientCtx):
460
- self._context = argument_value
461
- return
462
-
463
- # Search if the function was triggered from an MLRun RunTime object by looking at the call stack:
464
- # Index 0: the current frame.
465
- # Index 1: the decorator's frame.
466
- # Index 2-...: If it is from mlrun.runtimes we can be sure it ran via MLRun, otherwise not.
467
- for callstack_frame in inspect.getouterframes(inspect.currentframe()):
468
- if os.path.join("mlrun", "runtimes", "local") in callstack_frame.filename:
469
- import mlrun
470
-
471
- self._context = mlrun.get_or_create_ctx("context")
472
- break
473
-
474
- def is_context_available(self) -> bool:
475
- """
476
- Check if a context was found by the method `look_for_context`.
477
-
478
- :returns: True if a context was found and False otherwise.
479
- """
480
- return self._context is not None
481
-
482
- def parse_inputs(
483
- self,
484
- args: tuple,
485
- kwargs: dict,
486
- type_hints: OrderedDict,
487
- ) -> tuple:
488
- """
489
- Parse the given arguments and keyword arguments data items to the expected types.
490
-
491
- :param args: The arguments tuple passed to the function.
492
- :param kwargs: The keyword arguments dictionary passed to the function.
493
- :param type_hints: An ordered dictionary of the expected types of arguments.
494
-
495
- :returns: The parsed args (kwargs are parsed inplace).
496
- """
497
- # Parse the arguments:
498
- parsed_args = []
499
- type_hints_keys = list(type_hints.keys())
500
- for i, argument in enumerate(args):
501
- if (
502
- isinstance(argument, DataItem)
503
- and type_hints[type_hints_keys[i]] != inspect._empty
504
- ):
505
- parsed_args.append(
506
- self._parse_input(
507
- data_item=argument,
508
- type_hint=type_hints[type_hints_keys[i]],
509
- )
510
- )
511
- continue
512
- parsed_args.append(argument)
513
- parsed_args = tuple(parsed_args) # `args` is expected to be a tuple.
514
-
515
- # Parse the keyword arguments:
516
- for key in kwargs.keys():
517
- if isinstance(kwargs[key], DataItem) and type_hints[key] not in [
518
- inspect._empty,
519
- DataItem,
520
- ]:
521
- kwargs[key] = self._parse_input(
522
- data_item=kwargs[key], type_hint=type_hints[key]
523
- )
524
-
525
- return parsed_args
526
-
527
- def log_outputs(
528
- self,
529
- outputs: list,
530
- log_hints: List[Union[Dict[str, str], None]],
531
- ):
532
- """
533
- Log the given outputs as artifacts with the stored context.
534
-
535
- :param outputs: List of outputs to log.
536
- :param log_hints: List of logging configurations to use.
537
- """
538
- for obj, log_hint in zip(outputs, log_hints):
539
- # Check if needed to log (not None):
540
- if log_hint is None:
541
- continue
542
- # Parse the instructions:
543
- artifact_type = self._DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP.get(
544
- type(obj), self._ARTIFACT_TYPE_CLASS.DEFAULT
545
- ).value
546
- key = log_hint.pop("key")
547
- artifact_type = log_hint.pop("artifact_type", artifact_type)
548
- # Check if the object to log is None (None values are only logged if the artifact type is Result):
549
- if obj is None and artifact_type != ArtifactType.RESULT.value:
550
- continue
551
- # Log:
552
- self._log_output(
553
- obj=obj,
554
- artifact_type=artifact_type,
555
- key=key,
556
- logging_kwargs=log_hint,
557
- )
558
-
559
- def set_labels(self, labels: Dict[str, str]):
560
- """
561
- Set the given labels with the stored context.
562
-
563
- :param labels: The labels to set.
564
- """
565
- for key, value in labels.items():
566
- self._context.set_label(key=key, value=value)
567
-
568
- @classmethod
569
- def _init_default_objects_artifact_types_map(cls):
570
- """
571
- Initialize the default objects artifact types map with the basic classes supported by MLRun. In addition, it
572
- will try to support further common packages that are not required in MLRun.
573
- """
574
- # Initialize the map with the default classes:
575
- cls._DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP = {
576
- pd.DataFrame: ArtifactType.DATASET,
577
- pd.Series: ArtifactType.DATASET,
578
- np.ndarray: ArtifactType.DATASET,
579
- dict: ArtifactType.RESULT,
580
- list: ArtifactType.RESULT,
581
- tuple: ArtifactType.RESULT,
582
- str: ArtifactType.RESULT,
583
- int: ArtifactType.RESULT,
584
- float: ArtifactType.RESULT,
585
- bytes: ArtifactType.OBJECT,
586
- bytearray: ArtifactType.OBJECT,
587
- }
588
-
589
- # Try to enrich it with further classes according ot the user's environment:
590
- try:
591
- import matplotlib.pyplot as plt
592
-
593
- cls._DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP[plt.Figure] = ArtifactType.PLOT
594
- cls._DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP[plt.Axes] = ArtifactType.PLOT
595
- except ModuleNotFoundError:
596
- pass
597
- try:
598
- import plotly
599
-
600
- cls._DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP[
601
- plotly.graph_objs.Figure
602
- ] = ArtifactType.PLOT
603
- except ModuleNotFoundError:
604
- pass
605
- try:
606
- import bokeh.plotting as bokeh_plt
607
-
608
- cls._DEFAULT_OBJECTS_ARTIFACT_TYPES_MAP[
609
- bokeh_plt.Figure
610
- ] = ArtifactType.PLOT
611
- except ModuleNotFoundError:
612
- pass
613
- except ImportError:
614
- logger.warn(
615
- "Bokeh installation is ignored. If needed, "
616
- "make sure you have the required version with `pip install mlrun[bokeh]`"
617
- )
618
-
619
- @classmethod
620
- def _init_outputs_logging_map(cls):
621
- """
622
- Initialize the outputs logging map for the basic artifact types supported by MLRun.
623
- """
624
- cls._OUTPUTS_LOGGING_MAP = {
625
- ArtifactType.DATASET: OutputsLogger.log_dataset,
626
- ArtifactType.DIRECTORY: OutputsLogger.log_directory,
627
- ArtifactType.FILE: OutputsLogger.log_file,
628
- ArtifactType.OBJECT: OutputsLogger.log_object,
629
- ArtifactType.PLOT: OutputsLogger.log_plot,
630
- ArtifactType.RESULT: OutputsLogger.log_result,
631
- }
632
-
633
- @classmethod
634
- def _init_inputs_parsing_map(cls):
635
- """
636
- Initialize the inputs parsing map with the basic classes supported by MLRun.
637
- """
638
- cls._INPUTS_PARSING_MAP = {
639
- pd.DataFrame: InputsParser.parse_pandas_dataframe,
640
- np.ndarray: InputsParser.parse_numpy_array,
641
- dict: InputsParser.parse_dict,
642
- list: InputsParser.parse_list,
643
- object: InputsParser.parse_object,
644
- }
645
-
646
- def _parse_input(self, data_item: DataItem, type_hint: type) -> Any:
647
- """
648
- Parse the given data frame to the expected type. By default, it will be parsed to an object (will be treated as
649
- a pickle).
650
-
651
- :param data_item: The data item to parse.
652
- :param type_hint: The expected type to parse to.
653
-
654
- :returns: The parsed data item.
655
-
656
- :raises MLRunRuntimeError: If an error was raised during the parsing function.
657
- """
658
- if str(type_hint).startswith("typing."):
659
- return data_item
660
- try:
661
- return self._INPUTS_PARSING_MAP.get(
662
- type_hint, self._INPUTS_PARSING_MAP[object]
663
- )(data_item=data_item)
664
- except Exception as exception:
665
- raise MLRunRuntimeError(
666
- f"MLRun tried to parse a `DataItem` of type '{type_hint}' but failed. Be sure the item was "
667
- f"logged correctly - as the type you are trying to parse it back to. In general, python objects should "
668
- f"be logged as pickles."
669
- ) from exception
670
-
671
- def _log_output(
672
- self,
673
- obj,
674
- artifact_type: Union[ArtifactType, str],
675
- key: str,
676
- logging_kwargs: Dict[str, Any],
677
- ):
678
- """
679
- Log the given object to MLRun as the given artifact type with the provided key. The key can be part of a
680
- logging keyword arguments to pass to the relevant context logging function.
681
-
682
- :param obj: The object to log.
683
- :param artifact_type: The artifact type to log the object as.
684
- :param key: The key (name) of the artifact or a logging kwargs to use when logging the artifact.
685
-
686
- :raises MLRunInvalidArgumentError: If a key was provided in the logging kwargs.
687
- :raises MLRunRuntimeError: If an error was raised during the logging function.
688
- """
689
- # Get the artifact type (will also verify the artifact type is valid):
690
- artifact_type = self._ARTIFACT_TYPE_CLASS(artifact_type)
691
-
692
- # Check if 'key' or 'item' were given the logging kwargs:
693
- if "key" in logging_kwargs or "item" in logging_kwargs:
694
- raise MLRunInvalidArgumentError(
695
- "When passing logging keyword arguments, both 'key' and 'item' (according to the context method) "
696
- "cannot be added to the dictionary as the key is given on its own."
697
- )
698
-
699
- # Use the logging map to log the object:
700
- try:
701
- self._OUTPUTS_LOGGING_MAP[artifact_type](
702
- ctx=self._context,
703
- obj=obj,
704
- key=key,
705
- logging_kwargs=logging_kwargs,
706
- )
707
- except Exception as exception:
708
- raise MLRunRuntimeError(
709
- f"MLRun tried to log '{key}' as '{artifact_type.value}' but failed. If you didn't provide the artifact "
710
- f"type and the default one does not fit, try to select the correct type from the enum `ArtifactType`."
711
- ) from exception