mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (444) hide show
  1. mlrun/__init__.py +3 -3
  2. mlrun/__main__.py +79 -37
  3. mlrun/api/__init__.py +1 -1
  4. mlrun/api/api/__init__.py +1 -1
  5. mlrun/api/api/api.py +4 -4
  6. mlrun/api/api/deps.py +10 -21
  7. mlrun/api/api/endpoints/__init__.py +1 -1
  8. mlrun/api/api/endpoints/artifacts.py +64 -36
  9. mlrun/api/api/endpoints/auth.py +4 -4
  10. mlrun/api/api/endpoints/background_tasks.py +11 -11
  11. mlrun/api/api/endpoints/client_spec.py +5 -5
  12. mlrun/api/api/endpoints/clusterization_spec.py +6 -4
  13. mlrun/api/api/endpoints/feature_store.py +124 -115
  14. mlrun/api/api/endpoints/files.py +22 -14
  15. mlrun/api/api/endpoints/frontend_spec.py +28 -21
  16. mlrun/api/api/endpoints/functions.py +142 -87
  17. mlrun/api/api/endpoints/grafana_proxy.py +89 -442
  18. mlrun/api/api/endpoints/healthz.py +20 -7
  19. mlrun/api/api/endpoints/hub.py +320 -0
  20. mlrun/api/api/endpoints/internal/__init__.py +1 -1
  21. mlrun/api/api/endpoints/internal/config.py +1 -1
  22. mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
  23. mlrun/api/api/endpoints/logs.py +11 -11
  24. mlrun/api/api/endpoints/model_endpoints.py +74 -70
  25. mlrun/api/api/endpoints/operations.py +13 -9
  26. mlrun/api/api/endpoints/pipelines.py +93 -88
  27. mlrun/api/api/endpoints/projects.py +35 -35
  28. mlrun/api/api/endpoints/runs.py +69 -27
  29. mlrun/api/api/endpoints/runtime_resources.py +28 -28
  30. mlrun/api/api/endpoints/schedules.py +98 -41
  31. mlrun/api/api/endpoints/secrets.py +37 -32
  32. mlrun/api/api/endpoints/submit.py +12 -12
  33. mlrun/api/api/endpoints/tags.py +20 -22
  34. mlrun/api/api/utils.py +251 -42
  35. mlrun/api/constants.py +1 -1
  36. mlrun/api/crud/__init__.py +18 -15
  37. mlrun/api/crud/artifacts.py +10 -10
  38. mlrun/api/crud/client_spec.py +4 -4
  39. mlrun/api/crud/clusterization_spec.py +3 -3
  40. mlrun/api/crud/feature_store.py +54 -46
  41. mlrun/api/crud/functions.py +3 -3
  42. mlrun/api/crud/hub.py +312 -0
  43. mlrun/api/crud/logs.py +11 -9
  44. mlrun/api/crud/model_monitoring/__init__.py +3 -3
  45. mlrun/api/crud/model_monitoring/grafana.py +435 -0
  46. mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
  47. mlrun/api/crud/notifications.py +149 -0
  48. mlrun/api/crud/pipelines.py +67 -52
  49. mlrun/api/crud/projects.py +51 -23
  50. mlrun/api/crud/runs.py +7 -5
  51. mlrun/api/crud/runtime_resources.py +13 -13
  52. mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
  53. mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
  54. mlrun/api/crud/runtimes/nuclio/function.py +505 -0
  55. mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
  56. mlrun/api/crud/secrets.py +88 -46
  57. mlrun/api/crud/tags.py +5 -5
  58. mlrun/api/db/__init__.py +1 -1
  59. mlrun/api/db/base.py +102 -54
  60. mlrun/api/db/init_db.py +2 -3
  61. mlrun/api/db/session.py +4 -12
  62. mlrun/api/db/sqldb/__init__.py +1 -1
  63. mlrun/api/db/sqldb/db.py +439 -196
  64. mlrun/api/db/sqldb/helpers.py +1 -1
  65. mlrun/api/db/sqldb/models/__init__.py +3 -3
  66. mlrun/api/db/sqldb/models/models_mysql.py +82 -64
  67. mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
  68. mlrun/api/db/sqldb/session.py +27 -20
  69. mlrun/api/initial_data.py +82 -24
  70. mlrun/api/launcher.py +196 -0
  71. mlrun/api/main.py +91 -22
  72. mlrun/api/middlewares.py +6 -5
  73. mlrun/api/migrations_mysql/env.py +1 -1
  74. mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
  75. mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
  76. mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
  77. mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
  78. mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
  79. mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
  80. mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
  81. mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
  82. mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
  83. mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
  84. mlrun/api/migrations_sqlite/env.py +1 -1
  85. mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
  86. mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
  87. mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
  88. mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
  89. mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
  90. mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
  91. mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
  92. mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
  93. mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
  94. mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
  95. mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
  96. mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
  97. mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
  98. mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
  99. mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
  100. mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
  101. mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
  102. mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
  103. mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
  104. mlrun/api/schemas/__init__.py +216 -138
  105. mlrun/api/utils/__init__.py +1 -1
  106. mlrun/api/utils/asyncio.py +1 -1
  107. mlrun/api/utils/auth/__init__.py +1 -1
  108. mlrun/api/utils/auth/providers/__init__.py +1 -1
  109. mlrun/api/utils/auth/providers/base.py +7 -7
  110. mlrun/api/utils/auth/providers/nop.py +6 -7
  111. mlrun/api/utils/auth/providers/opa.py +17 -17
  112. mlrun/api/utils/auth/verifier.py +36 -34
  113. mlrun/api/utils/background_tasks.py +24 -24
  114. mlrun/{builder.py → api/utils/builder.py} +216 -123
  115. mlrun/api/utils/clients/__init__.py +1 -1
  116. mlrun/api/utils/clients/chief.py +19 -4
  117. mlrun/api/utils/clients/iguazio.py +106 -60
  118. mlrun/api/utils/clients/log_collector.py +1 -1
  119. mlrun/api/utils/clients/nuclio.py +23 -23
  120. mlrun/api/utils/clients/protocols/grpc.py +2 -2
  121. mlrun/api/utils/db/__init__.py +1 -1
  122. mlrun/api/utils/db/alembic.py +1 -1
  123. mlrun/api/utils/db/backup.py +1 -1
  124. mlrun/api/utils/db/mysql.py +24 -25
  125. mlrun/api/utils/db/sql_collation.py +1 -1
  126. mlrun/api/utils/db/sqlite_migration.py +2 -2
  127. mlrun/api/utils/events/__init__.py +14 -0
  128. mlrun/api/utils/events/base.py +57 -0
  129. mlrun/api/utils/events/events_factory.py +41 -0
  130. mlrun/api/utils/events/iguazio.py +217 -0
  131. mlrun/api/utils/events/nop.py +55 -0
  132. mlrun/api/utils/helpers.py +16 -13
  133. mlrun/api/utils/memory_reports.py +1 -1
  134. mlrun/api/utils/periodic.py +6 -3
  135. mlrun/api/utils/projects/__init__.py +1 -1
  136. mlrun/api/utils/projects/follower.py +33 -33
  137. mlrun/api/utils/projects/leader.py +36 -34
  138. mlrun/api/utils/projects/member.py +27 -27
  139. mlrun/api/utils/projects/remotes/__init__.py +1 -1
  140. mlrun/api/utils/projects/remotes/follower.py +13 -13
  141. mlrun/api/utils/projects/remotes/leader.py +10 -10
  142. mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
  143. mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
  144. mlrun/api/utils/scheduler.py +140 -51
  145. mlrun/api/utils/singletons/__init__.py +1 -1
  146. mlrun/api/utils/singletons/db.py +9 -15
  147. mlrun/api/utils/singletons/k8s.py +677 -5
  148. mlrun/api/utils/singletons/logs_dir.py +1 -1
  149. mlrun/api/utils/singletons/project_member.py +1 -1
  150. mlrun/api/utils/singletons/scheduler.py +1 -1
  151. mlrun/artifacts/__init__.py +2 -2
  152. mlrun/artifacts/base.py +8 -2
  153. mlrun/artifacts/dataset.py +5 -3
  154. mlrun/artifacts/manager.py +7 -1
  155. mlrun/artifacts/model.py +15 -4
  156. mlrun/artifacts/plots.py +1 -1
  157. mlrun/common/__init__.py +1 -1
  158. mlrun/common/constants.py +15 -0
  159. mlrun/common/model_monitoring.py +209 -0
  160. mlrun/common/schemas/__init__.py +167 -0
  161. mlrun/{api → common}/schemas/artifact.py +13 -14
  162. mlrun/{api → common}/schemas/auth.py +10 -8
  163. mlrun/{api → common}/schemas/background_task.py +3 -3
  164. mlrun/{api → common}/schemas/client_spec.py +1 -1
  165. mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
  166. mlrun/{api → common}/schemas/constants.py +21 -8
  167. mlrun/common/schemas/events.py +36 -0
  168. mlrun/{api → common}/schemas/feature_store.py +2 -1
  169. mlrun/{api → common}/schemas/frontend_spec.py +7 -6
  170. mlrun/{api → common}/schemas/function.py +5 -5
  171. mlrun/{api → common}/schemas/http.py +3 -3
  172. mlrun/common/schemas/hub.py +134 -0
  173. mlrun/{api → common}/schemas/k8s.py +3 -3
  174. mlrun/{api → common}/schemas/memory_reports.py +1 -1
  175. mlrun/common/schemas/model_endpoints.py +342 -0
  176. mlrun/common/schemas/notification.py +57 -0
  177. mlrun/{api → common}/schemas/object.py +6 -6
  178. mlrun/{api → common}/schemas/pipeline.py +3 -3
  179. mlrun/{api → common}/schemas/project.py +6 -5
  180. mlrun/common/schemas/regex.py +24 -0
  181. mlrun/common/schemas/runs.py +30 -0
  182. mlrun/{api → common}/schemas/runtime_resource.py +3 -3
  183. mlrun/{api → common}/schemas/schedule.py +19 -7
  184. mlrun/{api → common}/schemas/secret.py +3 -3
  185. mlrun/{api → common}/schemas/tag.py +2 -2
  186. mlrun/common/types.py +25 -0
  187. mlrun/config.py +152 -20
  188. mlrun/data_types/__init__.py +7 -2
  189. mlrun/data_types/data_types.py +4 -2
  190. mlrun/data_types/infer.py +1 -1
  191. mlrun/data_types/spark.py +10 -3
  192. mlrun/datastore/__init__.py +10 -3
  193. mlrun/datastore/azure_blob.py +1 -1
  194. mlrun/datastore/base.py +185 -53
  195. mlrun/datastore/datastore.py +1 -1
  196. mlrun/datastore/filestore.py +1 -1
  197. mlrun/datastore/google_cloud_storage.py +1 -1
  198. mlrun/datastore/inmem.py +4 -1
  199. mlrun/datastore/redis.py +1 -1
  200. mlrun/datastore/s3.py +1 -1
  201. mlrun/datastore/sources.py +192 -70
  202. mlrun/datastore/spark_udf.py +44 -0
  203. mlrun/datastore/store_resources.py +4 -4
  204. mlrun/datastore/targets.py +115 -45
  205. mlrun/datastore/utils.py +127 -5
  206. mlrun/datastore/v3io.py +1 -1
  207. mlrun/datastore/wasbfs/__init__.py +1 -1
  208. mlrun/datastore/wasbfs/fs.py +1 -1
  209. mlrun/db/__init__.py +7 -5
  210. mlrun/db/base.py +112 -68
  211. mlrun/db/httpdb.py +445 -277
  212. mlrun/db/nopdb.py +491 -0
  213. mlrun/db/sqldb.py +112 -65
  214. mlrun/errors.py +6 -1
  215. mlrun/execution.py +44 -22
  216. mlrun/feature_store/__init__.py +1 -1
  217. mlrun/feature_store/api.py +143 -95
  218. mlrun/feature_store/common.py +16 -20
  219. mlrun/feature_store/feature_set.py +42 -12
  220. mlrun/feature_store/feature_vector.py +32 -21
  221. mlrun/feature_store/ingestion.py +9 -12
  222. mlrun/feature_store/retrieval/__init__.py +3 -2
  223. mlrun/feature_store/retrieval/base.py +388 -66
  224. mlrun/feature_store/retrieval/dask_merger.py +63 -151
  225. mlrun/feature_store/retrieval/job.py +30 -12
  226. mlrun/feature_store/retrieval/local_merger.py +40 -133
  227. mlrun/feature_store/retrieval/spark_merger.py +129 -127
  228. mlrun/feature_store/retrieval/storey_merger.py +173 -0
  229. mlrun/feature_store/steps.py +132 -15
  230. mlrun/features.py +8 -3
  231. mlrun/frameworks/__init__.py +1 -1
  232. mlrun/frameworks/_common/__init__.py +1 -1
  233. mlrun/frameworks/_common/artifacts_library.py +1 -1
  234. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  235. mlrun/frameworks/_common/model_handler.py +1 -1
  236. mlrun/frameworks/_common/plan.py +1 -1
  237. mlrun/frameworks/_common/producer.py +1 -1
  238. mlrun/frameworks/_common/utils.py +1 -1
  239. mlrun/frameworks/_dl_common/__init__.py +1 -1
  240. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
  241. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  242. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  243. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  244. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  245. mlrun/frameworks/_dl_common/utils.py +1 -1
  246. mlrun/frameworks/_ml_common/__init__.py +1 -1
  247. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  248. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
  249. mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
  250. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  251. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  252. mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
  253. mlrun/frameworks/_ml_common/plan.py +1 -1
  254. mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
  255. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
  256. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
  257. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  258. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  259. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  260. mlrun/frameworks/_ml_common/producer.py +1 -1
  261. mlrun/frameworks/_ml_common/utils.py +1 -1
  262. mlrun/frameworks/auto_mlrun/__init__.py +1 -1
  263. mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
  264. mlrun/frameworks/huggingface/__init__.py +1 -1
  265. mlrun/frameworks/huggingface/model_server.py +1 -1
  266. mlrun/frameworks/lgbm/__init__.py +1 -1
  267. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
  268. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  269. mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
  270. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
  271. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
  272. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  273. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  274. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  275. mlrun/frameworks/lgbm/model_handler.py +1 -1
  276. mlrun/frameworks/lgbm/model_server.py +1 -1
  277. mlrun/frameworks/lgbm/utils.py +1 -1
  278. mlrun/frameworks/onnx/__init__.py +1 -1
  279. mlrun/frameworks/onnx/dataset.py +1 -1
  280. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  281. mlrun/frameworks/onnx/model_handler.py +1 -1
  282. mlrun/frameworks/onnx/model_server.py +1 -1
  283. mlrun/frameworks/parallel_coordinates.py +1 -1
  284. mlrun/frameworks/pytorch/__init__.py +1 -1
  285. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
  286. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  287. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  288. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  289. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  290. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  291. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  292. mlrun/frameworks/pytorch/model_handler.py +1 -1
  293. mlrun/frameworks/pytorch/model_server.py +1 -1
  294. mlrun/frameworks/pytorch/utils.py +1 -1
  295. mlrun/frameworks/sklearn/__init__.py +1 -1
  296. mlrun/frameworks/sklearn/estimator.py +1 -1
  297. mlrun/frameworks/sklearn/metric.py +1 -1
  298. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  299. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  300. mlrun/frameworks/sklearn/model_handler.py +1 -1
  301. mlrun/frameworks/sklearn/utils.py +1 -1
  302. mlrun/frameworks/tf_keras/__init__.py +1 -1
  303. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
  304. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  305. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  306. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  307. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  308. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  309. mlrun/frameworks/tf_keras/model_server.py +1 -1
  310. mlrun/frameworks/tf_keras/utils.py +1 -1
  311. mlrun/frameworks/xgboost/__init__.py +1 -1
  312. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  313. mlrun/frameworks/xgboost/model_handler.py +1 -1
  314. mlrun/frameworks/xgboost/utils.py +1 -1
  315. mlrun/k8s_utils.py +14 -765
  316. mlrun/kfpops.py +14 -17
  317. mlrun/launcher/__init__.py +13 -0
  318. mlrun/launcher/base.py +406 -0
  319. mlrun/launcher/client.py +159 -0
  320. mlrun/launcher/factory.py +50 -0
  321. mlrun/launcher/local.py +276 -0
  322. mlrun/launcher/remote.py +178 -0
  323. mlrun/lists.py +10 -2
  324. mlrun/mlutils/__init__.py +1 -1
  325. mlrun/mlutils/data.py +1 -1
  326. mlrun/mlutils/models.py +1 -1
  327. mlrun/mlutils/plots.py +1 -1
  328. mlrun/model.py +252 -14
  329. mlrun/model_monitoring/__init__.py +41 -0
  330. mlrun/model_monitoring/features_drift_table.py +1 -1
  331. mlrun/model_monitoring/helpers.py +123 -38
  332. mlrun/model_monitoring/model_endpoint.py +144 -0
  333. mlrun/model_monitoring/model_monitoring_batch.py +310 -259
  334. mlrun/model_monitoring/stores/__init__.py +106 -0
  335. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
  336. mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
  337. mlrun/model_monitoring/stores/models/__init__.py +23 -0
  338. mlrun/model_monitoring/stores/models/base.py +18 -0
  339. mlrun/model_monitoring/stores/models/mysql.py +100 -0
  340. mlrun/model_monitoring/stores/models/sqlite.py +98 -0
  341. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
  342. mlrun/model_monitoring/stream_processing_fs.py +239 -271
  343. mlrun/package/__init__.py +163 -0
  344. mlrun/package/context_handler.py +325 -0
  345. mlrun/package/errors.py +47 -0
  346. mlrun/package/packager.py +298 -0
  347. mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
  348. mlrun/package/packagers/default_packager.py +422 -0
  349. mlrun/package/packagers/numpy_packagers.py +612 -0
  350. mlrun/package/packagers/pandas_packagers.py +968 -0
  351. mlrun/package/packagers/python_standard_library_packagers.py +616 -0
  352. mlrun/package/packagers_manager.py +786 -0
  353. mlrun/package/utils/__init__.py +53 -0
  354. mlrun/package/utils/_archiver.py +226 -0
  355. mlrun/package/utils/_formatter.py +211 -0
  356. mlrun/package/utils/_pickler.py +234 -0
  357. mlrun/package/utils/_supported_format.py +71 -0
  358. mlrun/package/utils/log_hint_utils.py +93 -0
  359. mlrun/package/utils/type_hint_utils.py +298 -0
  360. mlrun/platforms/__init__.py +1 -1
  361. mlrun/platforms/iguazio.py +34 -2
  362. mlrun/platforms/other.py +1 -1
  363. mlrun/projects/__init__.py +1 -1
  364. mlrun/projects/operations.py +14 -9
  365. mlrun/projects/pipelines.py +31 -13
  366. mlrun/projects/project.py +762 -238
  367. mlrun/render.py +49 -19
  368. mlrun/run.py +57 -326
  369. mlrun/runtimes/__init__.py +3 -9
  370. mlrun/runtimes/base.py +247 -784
  371. mlrun/runtimes/constants.py +1 -1
  372. mlrun/runtimes/daskjob.py +45 -41
  373. mlrun/runtimes/funcdoc.py +43 -7
  374. mlrun/runtimes/function.py +66 -656
  375. mlrun/runtimes/function_reference.py +1 -1
  376. mlrun/runtimes/generators.py +1 -1
  377. mlrun/runtimes/kubejob.py +99 -116
  378. mlrun/runtimes/local.py +59 -66
  379. mlrun/runtimes/mpijob/__init__.py +1 -1
  380. mlrun/runtimes/mpijob/abstract.py +13 -15
  381. mlrun/runtimes/mpijob/v1.py +3 -1
  382. mlrun/runtimes/mpijob/v1alpha1.py +1 -1
  383. mlrun/runtimes/nuclio.py +1 -1
  384. mlrun/runtimes/pod.py +51 -26
  385. mlrun/runtimes/remotesparkjob.py +3 -1
  386. mlrun/runtimes/serving.py +12 -4
  387. mlrun/runtimes/sparkjob/__init__.py +1 -2
  388. mlrun/runtimes/sparkjob/abstract.py +44 -31
  389. mlrun/runtimes/sparkjob/spark3job.py +11 -9
  390. mlrun/runtimes/utils.py +61 -42
  391. mlrun/secrets.py +16 -18
  392. mlrun/serving/__init__.py +3 -2
  393. mlrun/serving/merger.py +1 -1
  394. mlrun/serving/remote.py +1 -1
  395. mlrun/serving/routers.py +39 -42
  396. mlrun/serving/server.py +23 -13
  397. mlrun/serving/serving_wrapper.py +1 -1
  398. mlrun/serving/states.py +172 -39
  399. mlrun/serving/utils.py +1 -1
  400. mlrun/serving/v1_serving.py +1 -1
  401. mlrun/serving/v2_serving.py +29 -21
  402. mlrun/utils/__init__.py +1 -2
  403. mlrun/utils/async_http.py +8 -1
  404. mlrun/utils/azure_vault.py +1 -1
  405. mlrun/utils/clones.py +2 -2
  406. mlrun/utils/condition_evaluator.py +65 -0
  407. mlrun/utils/db.py +52 -0
  408. mlrun/utils/helpers.py +188 -13
  409. mlrun/utils/http.py +89 -54
  410. mlrun/utils/logger.py +48 -8
  411. mlrun/utils/model_monitoring.py +132 -100
  412. mlrun/utils/notifications/__init__.py +1 -1
  413. mlrun/utils/notifications/notification/__init__.py +8 -6
  414. mlrun/utils/notifications/notification/base.py +20 -14
  415. mlrun/utils/notifications/notification/console.py +7 -4
  416. mlrun/utils/notifications/notification/git.py +36 -19
  417. mlrun/utils/notifications/notification/ipython.py +10 -8
  418. mlrun/utils/notifications/notification/slack.py +18 -13
  419. mlrun/utils/notifications/notification_pusher.py +377 -56
  420. mlrun/utils/regex.py +6 -1
  421. mlrun/utils/singleton.py +1 -1
  422. mlrun/utils/v3io_clients.py +1 -1
  423. mlrun/utils/vault.py +270 -269
  424. mlrun/utils/version/__init__.py +1 -1
  425. mlrun/utils/version/version.json +2 -2
  426. mlrun/utils/version/version.py +1 -1
  427. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
  428. mlrun-1.4.0.dist-info/RECORD +434 -0
  429. mlrun/api/api/endpoints/marketplace.py +0 -257
  430. mlrun/api/crud/marketplace.py +0 -221
  431. mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
  432. mlrun/api/db/filedb/db.py +0 -518
  433. mlrun/api/schemas/marketplace.py +0 -128
  434. mlrun/api/schemas/model_endpoints.py +0 -185
  435. mlrun/db/filedb.py +0 -891
  436. mlrun/feature_store/retrieval/online.py +0 -92
  437. mlrun/model_monitoring/constants.py +0 -67
  438. mlrun/runtimes/package/context_handler.py +0 -711
  439. mlrun/runtimes/sparkjob/spark2job.py +0 -59
  440. mlrun-1.3.3.dist-info/RECORD +0 -381
  441. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
  442. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
  443. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
  444. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
mlrun/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@ from .db import get_run_db
35
35
  from .errors import MLRunInvalidArgumentError, MLRunNotFoundError
36
36
  from .execution import MLClientCtx
37
37
  from .model import RunObject, RunTemplate, new_task
38
+ from .package import ArtifactType, DefaultPackager, Packager, handler
38
39
  from .platforms import (
39
40
  VolumeMount,
40
41
  auto_mount,
@@ -62,14 +63,13 @@ from .run import (
62
63
  get_object,
63
64
  get_or_create_ctx,
64
65
  get_pipeline,
65
- handler,
66
66
  import_function,
67
67
  new_function,
68
68
  run_local,
69
69
  run_pipeline,
70
70
  wait_for_pipeline_completion,
71
71
  )
72
- from .runtimes import ArtifactType, new_model_server
72
+ from .runtimes import new_model_server
73
73
  from .secrets import get_secret_or_env
74
74
  from .utils.version import Version
75
75
 
mlrun/__main__.py CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env python
2
2
 
3
- # Copyright 2018 Iguazio
3
+ # Copyright 2023 Iguazio
4
4
  #
5
5
  # Licensed under the Apache License, Version 2.0 (the "License");
6
6
  # you may not use this file except in compliance with the License.
@@ -34,11 +34,9 @@ from tabulate import tabulate
34
34
 
35
35
  import mlrun
36
36
 
37
- from .builder import upload_tarball
38
37
  from .config import config as mlconf
39
38
  from .db import get_run_db
40
39
  from .errors import err_to_str
41
- from .k8s_utils import K8sHelper
42
40
  from .model import RunTemplate
43
41
  from .platforms import auto_mount as auto_mount_modifier
44
42
  from .projects import load_project
@@ -545,7 +543,7 @@ def build(
545
543
  logger.info(f"uploading data from {src} to {archive}")
546
544
  target = archive if archive.endswith("/") else archive + "/"
547
545
  target += f"src-{meta.project}-{meta.name}-{meta.tag or 'latest'}.tar.gz"
548
- upload_tarball(src, target)
546
+ mlrun.datastore.utils.upload_tarball(src, target)
549
547
  # todo: replace function.yaml inside the tar
550
548
  b.source = target
551
549
 
@@ -700,20 +698,6 @@ def deploy(
700
698
  fp.write(function.status.nuclio_name)
701
699
 
702
700
 
703
- @main.command(context_settings=dict(ignore_unknown_options=True))
704
- @click.argument("pod", type=str, callback=validate_base_argument)
705
- @click.option("--namespace", "-n", help="kubernetes namespace")
706
- @click.option(
707
- "--timeout", "-t", default=600, show_default=True, help="timeout in seconds"
708
- )
709
- def watch(pod, namespace, timeout):
710
- """Read current or previous task (pod) logs."""
711
- print("This command will be deprecated in future version !!!\n")
712
- k8s = K8sHelper(namespace)
713
- status = k8s.watch(pod, namespace, timeout)
714
- print(f"Pod {pod} last status is: {status}")
715
-
716
-
717
701
  @main.command(context_settings=dict(ignore_unknown_options=True))
718
702
  @click.argument("kind", type=str, callback=validate_base_argument)
719
703
  @click.argument(
@@ -779,7 +763,7 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
779
763
  name, project=project, tag=tag, labels=selector
780
764
  )
781
765
  df = artifacts.to_df()[
782
- ["tree", "key", "iter", "kind", "path", "hash", "updated"]
766
+ ["tree", "key", "iter", "kind", "path", "hash", "updated", "uri"]
783
767
  ]
784
768
  df["tree"] = df["tree"].apply(lambda x: f"..{x[-8:]}")
785
769
  df["hash"] = df["hash"].apply(lambda x: f"..{x[-6:]}")
@@ -1052,6 +1036,15 @@ def logs(uid, project, offset, db, watch):
1052
1036
  is_flag=True,
1053
1037
  help="Store the project secrets as k8s secrets",
1054
1038
  )
1039
+ @click.option(
1040
+ "--notifications",
1041
+ "--notification",
1042
+ "-nt",
1043
+ multiple=True,
1044
+ help="To have a notification for the run set notification file "
1045
+ "destination define: file=notification.json or a "
1046
+ 'dictionary configuration e.g \'{"slack":{"webhook":"<webhook>"}}\'',
1047
+ )
1055
1048
  def project(
1056
1049
  context,
1057
1050
  name,
@@ -1077,6 +1070,7 @@ def project(
1077
1070
  timeout,
1078
1071
  ensure_project,
1079
1072
  schedule,
1073
+ notifications,
1080
1074
  overwrite_schedule,
1081
1075
  save_secrets,
1082
1076
  save,
@@ -1095,25 +1089,30 @@ def project(
1095
1089
  if db:
1096
1090
  mlconf.dbpath = db
1097
1091
 
1098
- proj = load_project(context, url, name, init_git=init_git, clone=clone, save=save)
1092
+ # set the CLI/GIT parameters in load_project() so they can be used by project setup scripts
1093
+ parameters = fill_params(param) if param else {}
1094
+ if git_repo:
1095
+ parameters["git_repo"] = git_repo
1096
+ if git_issue:
1097
+ parameters["git_issue"] = git_issue
1098
+ commit = environ.get("GITHUB_SHA") or environ.get("CI_COMMIT_SHA")
1099
+ if commit and not parameters.get("commit_id"):
1100
+ parameters["commit_id"] = commit
1101
+
1102
+ proj = load_project(
1103
+ context,
1104
+ url,
1105
+ name,
1106
+ init_git=init_git,
1107
+ clone=clone,
1108
+ save=save,
1109
+ parameters=parameters,
1110
+ )
1099
1111
  url_str = " from " + url if url else ""
1100
1112
  print(f"Loading project {proj.name}{url_str} into {context}:\n")
1101
1113
 
1102
1114
  if is_relative_path(artifact_path):
1103
1115
  artifact_path = path.abspath(artifact_path)
1104
- if param:
1105
- proj.spec.params = fill_params(param, proj.spec.params)
1106
- if git_repo:
1107
- proj.spec.params["git_repo"] = git_repo
1108
- if git_issue:
1109
- proj.spec.params["git_issue"] = git_issue
1110
- commit = (
1111
- proj.get_param("commit_id")
1112
- or environ.get("GITHUB_SHA")
1113
- or environ.get("CI_COMMIT_SHA")
1114
- )
1115
- if commit:
1116
- proj.spec.params["commit_id"] = commit
1117
1116
  if secrets:
1118
1117
  secrets = line2keylist(secrets, "kind", "source")
1119
1118
  secret_store = SecretsStore.from_list(secrets)
@@ -1152,6 +1151,8 @@ def project(
1152
1151
  "token": proj.get_param("GIT_TOKEN"),
1153
1152
  },
1154
1153
  )
1154
+ if notifications:
1155
+ load_notification(notifications, proj)
1155
1156
  try:
1156
1157
  proj.run(
1157
1158
  name=run,
@@ -1169,11 +1170,9 @@ def project(
1169
1170
  timeout=timeout,
1170
1171
  overwrite=overwrite_schedule,
1171
1172
  )
1172
-
1173
- except Exception as exc:
1173
+ except Exception as err:
1174
1174
  print(traceback.format_exc())
1175
- message = f"failed to run pipeline, {err_to_str(exc)}"
1176
- proj.notifiers.push(message, "error")
1175
+ send_workflow_error_notification(run, proj, err)
1177
1176
  exit(1)
1178
1177
 
1179
1178
  elif sync:
@@ -1450,5 +1449,48 @@ def func_url_to_runtime(func_url, ensure_project: bool = False):
1450
1449
  return runtime
1451
1450
 
1452
1451
 
1452
+ def load_notification(notifications: str, project: mlrun.projects.MlrunProject):
1453
+ """
1454
+ A dictionary or json file containing notification dictionaries can be used by the user to set notifications.
1455
+ Each notification is stored in a tuple called notifications.
1456
+ The code then goes through each value in the notifications tuple and check
1457
+ if the notification starts with "file=", such as "file=notification.json," in those cases it loads the
1458
+ notification.json file and uses add_notification_to_project to add the notifications from the file to
1459
+ the project. If not, it adds the notification dictionary to the project.
1460
+ :param notifications: Notifications file or a dictionary to be added to the project
1461
+ :param project: The object to which the notifications will be added
1462
+ :return:
1463
+ """
1464
+ for notification in notifications:
1465
+ if notification.startswith("file="):
1466
+ file_path = notification.split("=")[-1]
1467
+ notification = open(file_path, "r")
1468
+ notification = json.load(notification)
1469
+ else:
1470
+ notification = json.loads(notification)
1471
+ add_notification_to_project(notification, project)
1472
+
1473
+
1474
+ def add_notification_to_project(
1475
+ notification: str, project: mlrun.projects.MlrunProject
1476
+ ):
1477
+ for notification_type, notification_params in notification.items():
1478
+ project.notifiers.add_notification(
1479
+ notification_type=notification_type, params=notification_params
1480
+ )
1481
+
1482
+
1483
+ def send_workflow_error_notification(
1484
+ run_id: str, project: mlrun.projects.MlrunProject, error: KeyError
1485
+ ):
1486
+ message = (
1487
+ f":x: Failed to run scheduled workflow {run_id} in Project {project.name} !\n"
1488
+ f"error: ```{err_to_str(error)}```"
1489
+ )
1490
+ project.notifiers.push(
1491
+ message=message, severity=mlrun.common.schemas.NotificationSeverity.ERROR
1492
+ )
1493
+
1494
+
1453
1495
  if __name__ == "__main__":
1454
1496
  main()
mlrun/api/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
mlrun/api/api/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
mlrun/api/api/api.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -28,9 +28,9 @@ from mlrun.api.api.endpoints import (
28
28
  functions,
29
29
  grafana_proxy,
30
30
  healthz,
31
+ hub,
31
32
  internal,
32
33
  logs,
33
- marketplace,
34
34
  model_endpoints,
35
35
  operations,
36
36
  pipelines,
@@ -125,8 +125,8 @@ api_router.include_router(
125
125
  api_router.include_router(grafana_proxy.router, tags=["grafana", "model-endpoints"])
126
126
  api_router.include_router(model_endpoints.router, tags=["model-endpoints"])
127
127
  api_router.include_router(
128
- marketplace.router,
129
- tags=["marketplace"],
128
+ hub.router,
129
+ tags=["hub"],
130
130
  dependencies=[Depends(mlrun.api.api.deps.authenticate_request)],
131
131
  )
132
132
  api_router.include_router(
mlrun/api/api/deps.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -20,9 +20,9 @@ from sqlalchemy.orm import Session
20
20
 
21
21
  import mlrun
22
22
  import mlrun.api.db.session
23
- import mlrun.api.schemas
24
23
  import mlrun.api.utils.auth.verifier
25
24
  import mlrun.api.utils.clients.iguazio
25
+ import mlrun.common.schemas
26
26
 
27
27
 
28
28
  def get_db_session() -> typing.Generator[Session, None, None]:
@@ -35,7 +35,7 @@ def get_db_session() -> typing.Generator[Session, None, None]:
35
35
  mlrun.api.db.session.close_session(db_session)
36
36
 
37
37
 
38
- async def authenticate_request(request: Request) -> mlrun.api.schemas.AuthInfo:
38
+ async def authenticate_request(request: Request) -> mlrun.common.schemas.AuthInfo:
39
39
  return await mlrun.api.utils.auth.verifier.AuthVerifier().authenticate_request(
40
40
  request
41
41
  )
@@ -46,7 +46,7 @@ def verify_api_state(request: Request):
46
46
  request.scope
47
47
  )
48
48
  path = path_with_query_string.split("?")[0]
49
- if mlrun.mlconf.httpdb.state == mlrun.api.schemas.APIStates.offline:
49
+ if mlrun.mlconf.httpdb.state == mlrun.common.schemas.APIStates.offline:
50
50
  enabled_endpoints = [
51
51
  # we want to stay healthy
52
52
  "healthz",
@@ -56,10 +56,10 @@ def verify_api_state(request: Request):
56
56
  if not any(enabled_endpoint in path for enabled_endpoint in enabled_endpoints):
57
57
  raise mlrun.errors.MLRunPreconditionFailedError("API is in offline state")
58
58
  if mlrun.mlconf.httpdb.state in [
59
- mlrun.api.schemas.APIStates.waiting_for_migrations,
60
- mlrun.api.schemas.APIStates.migrations_in_progress,
61
- mlrun.api.schemas.APIStates.migrations_failed,
62
- mlrun.api.schemas.APIStates.waiting_for_chief,
59
+ mlrun.common.schemas.APIStates.waiting_for_migrations,
60
+ mlrun.common.schemas.APIStates.migrations_in_progress,
61
+ mlrun.common.schemas.APIStates.migrations_failed,
62
+ mlrun.common.schemas.APIStates.waiting_for_chief,
63
63
  ]:
64
64
  enabled_endpoints = [
65
65
  "healthz",
@@ -70,20 +70,9 @@ def verify_api_state(request: Request):
70
70
  "memory-reports",
71
71
  ]
72
72
  if not any(enabled_endpoint in path for enabled_endpoint in enabled_endpoints):
73
- message = (
74
- "API is waiting for migrations to be triggered. Send POST request to /api/operations/migrations to"
75
- " trigger it"
76
- )
77
- if (
78
- mlrun.mlconf.httpdb.state
79
- == mlrun.api.schemas.APIStates.migrations_in_progress
80
- ):
81
- message = "Migrations are in progress"
82
- elif (
73
+ message = mlrun.common.schemas.APIStates.description(
83
74
  mlrun.mlconf.httpdb.state
84
- == mlrun.api.schemas.APIStates.migrations_failed
85
- ):
86
- message = "Migrations failed, API can't be started"
75
+ )
87
76
  raise mlrun.errors.MLRunPreconditionFailedError(message)
88
77
 
89
78
 
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -22,18 +22,23 @@ from sqlalchemy.orm import Session
22
22
  import mlrun.api.crud
23
23
  import mlrun.api.utils.auth.verifier
24
24
  import mlrun.api.utils.singletons.project_member
25
- from mlrun.api import schemas
25
+ import mlrun.common.schemas
26
26
  from mlrun.api.api import deps
27
27
  from mlrun.api.api.utils import log_and_raise
28
- from mlrun.api.schemas.artifact import ArtifactsFormat
28
+ from mlrun.common.schemas.artifact import ArtifactsFormat
29
29
  from mlrun.config import config
30
- from mlrun.utils import is_legacy_artifact
30
+ from mlrun.utils import is_legacy_artifact, logger
31
31
 
32
32
  router = APIRouter()
33
33
 
34
34
 
35
- # TODO /artifact/{project}/{uid}/{key:path} should be deprecated in 1.4
36
- @router.post("/artifact/{project}/{uid}/{key:path}")
35
+ # TODO: remove /artifact/{project}/{uid}/{key:path} in 1.6.0
36
+ @router.post(
37
+ "/artifact/{project}/{uid}/{key:path}",
38
+ deprecated=True,
39
+ description="/artifact/{project}/{uid}/{key:path} is deprecated in 1.4.0 and will be removed in 1.6.0, "
40
+ "use /projects/{project}/artifacts/{uid}/{key:path} instead",
41
+ )
37
42
  @router.post("/projects/{project}/artifacts/{uid}/{key:path}")
38
43
  async def store_artifact(
39
44
  request: Request,
@@ -42,7 +47,7 @@ async def store_artifact(
42
47
  key: str,
43
48
  tag: str = "",
44
49
  iter: int = 0,
45
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
50
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
46
51
  db_session: Session = Depends(deps.get_db_session),
47
52
  ):
48
53
  await run_in_threadpool(
@@ -52,10 +57,10 @@ async def store_artifact(
52
57
  auth_info=auth_info,
53
58
  )
54
59
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
55
- mlrun.api.schemas.AuthorizationResourceTypes.artifact,
60
+ mlrun.common.schemas.AuthorizationResourceTypes.artifact,
56
61
  project,
57
62
  key,
58
- mlrun.api.schemas.AuthorizationAction.store,
63
+ mlrun.common.schemas.AuthorizationAction.store,
59
64
  auth_info,
60
65
  )
61
66
 
@@ -65,6 +70,9 @@ async def store_artifact(
65
70
  except ValueError:
66
71
  log_and_raise(HTTPStatus.BAD_REQUEST.value, reason="bad JSON body")
67
72
 
73
+ logger.debug(
74
+ "Storing artifact", project=project, uid=uid, key=key, tag=tag, iter=iter
75
+ )
68
76
  await run_in_threadpool(
69
77
  mlrun.api.crud.Artifacts().store_artifact,
70
78
  db_session,
@@ -81,13 +89,13 @@ async def store_artifact(
81
89
  @router.get("/projects/{project}/artifact-tags")
82
90
  async def list_artifact_tags(
83
91
  project: str,
84
- category: schemas.ArtifactCategories = None,
85
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
92
+ category: mlrun.common.schemas.ArtifactCategories = None,
93
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
86
94
  db_session: Session = Depends(deps.get_db_session),
87
95
  ):
88
96
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
89
97
  project,
90
- mlrun.api.schemas.AuthorizationAction.read,
98
+ mlrun.common.schemas.AuthorizationAction.read,
91
99
  auth_info,
92
100
  )
93
101
  tag_tuples = await run_in_threadpool(
@@ -95,7 +103,7 @@ async def list_artifact_tags(
95
103
  )
96
104
  artifact_key_to_tag = {tag_tuple[1]: tag_tuple[2] for tag_tuple in tag_tuples}
97
105
  allowed_artifact_keys = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
98
- mlrun.api.schemas.AuthorizationResourceTypes.artifact,
106
+ mlrun.common.schemas.AuthorizationResourceTypes.artifact,
99
107
  list(artifact_key_to_tag.keys()),
100
108
  lambda artifact_key: (
101
109
  project,
@@ -115,8 +123,13 @@ async def list_artifact_tags(
115
123
  }
116
124
 
117
125
 
118
- # TODO /projects/{project}/artifact/{key:path} should be deprecated in 1.4
119
- @router.get("/projects/{project}/artifact/{key:path}")
126
+ # TODO: remove /projects/{project}/artifact/{key:path} in 1.6.0
127
+ @router.get(
128
+ "/projects/{project}/artifact/{key:path}",
129
+ deprecated=True,
130
+ description="/projects/{project}/artifact/{key:path} is deprecated in 1.4.0 and will be removed in 1.6.0, "
131
+ "use /projects/{project}/artifacts/{key:path} instead",
132
+ )
120
133
  @router.get("/projects/{project}/artifacts/{key:path}")
121
134
  async def get_artifact(
122
135
  project: str,
@@ -124,7 +137,7 @@ async def get_artifact(
124
137
  tag: str = "latest",
125
138
  iter: int = 0,
126
139
  format_: ArtifactsFormat = Query(ArtifactsFormat.full, alias="format"),
127
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
140
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
128
141
  db_session: Session = Depends(deps.get_db_session),
129
142
  ):
130
143
  data = await run_in_threadpool(
@@ -137,10 +150,10 @@ async def get_artifact(
137
150
  format_,
138
151
  )
139
152
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
140
- mlrun.api.schemas.AuthorizationResourceTypes.artifact,
153
+ mlrun.common.schemas.AuthorizationResourceTypes.artifact,
141
154
  project,
142
155
  key,
143
- mlrun.api.schemas.AuthorizationAction.read,
156
+ mlrun.common.schemas.AuthorizationAction.read,
144
157
  auth_info,
145
158
  )
146
159
  return {
@@ -148,22 +161,27 @@ async def get_artifact(
148
161
  }
149
162
 
150
163
 
151
- # TODO /artifact/{project}/{uid} should be deprecated in 1.4
152
- @router.delete("/artifact/{project}/{uid}")
164
+ # TODO: remove /artifact/{project}/{uid} in 1.6.0
165
+ @router.delete(
166
+ "/artifact/{project}/{uid}",
167
+ deprecated=True,
168
+ description="/artifact/{project}/{uid} is deprecated in 1.4.0 and will be removed in 1.6.0, "
169
+ "use /projects/{project}/artifacts/{uid} instead",
170
+ )
153
171
  @router.delete("/projects/{project}/artifacts/{uid}")
154
172
  async def delete_artifact(
155
173
  project: str,
156
174
  uid: str,
157
175
  key: str,
158
176
  tag: str = "",
159
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
177
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
160
178
  db_session: Session = Depends(deps.get_db_session),
161
179
  ):
162
180
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
163
- mlrun.api.schemas.AuthorizationResourceTypes.artifact,
181
+ mlrun.common.schemas.AuthorizationResourceTypes.artifact,
164
182
  project,
165
183
  key,
166
- mlrun.api.schemas.AuthorizationAction.delete,
184
+ mlrun.common.schemas.AuthorizationAction.delete,
167
185
  auth_info,
168
186
  )
169
187
  await run_in_threadpool(
@@ -172,27 +190,32 @@ async def delete_artifact(
172
190
  return {}
173
191
 
174
192
 
175
- # TODO /artifacts should be deprecated in 1.4
176
- @router.get("/artifacts")
193
+ # TODO: remove /artifacts in 1.6.0
194
+ @router.get(
195
+ "/artifacts",
196
+ deprecated=True,
197
+ description="/artifacts is deprecated in 1.4.0 and will be removed in 1.6.0, "
198
+ "use /projects/{project}/artifacts instead",
199
+ )
177
200
  @router.get("/projects/{project}/artifacts")
178
201
  async def list_artifacts(
179
202
  project: str = None,
180
203
  name: str = None,
181
204
  tag: str = None,
182
205
  kind: str = None,
183
- category: schemas.ArtifactCategories = None,
206
+ category: mlrun.common.schemas.ArtifactCategories = None,
184
207
  labels: List[str] = Query([], alias="label"),
185
208
  iter: int = Query(None, ge=0),
186
209
  best_iteration: bool = Query(False, alias="best-iteration"),
187
210
  format_: ArtifactsFormat = Query(ArtifactsFormat.full, alias="format"),
188
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
211
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
189
212
  db_session: Session = Depends(deps.get_db_session),
190
213
  ):
191
214
  if project is None:
192
215
  project = config.default_project
193
216
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
194
217
  project,
195
- mlrun.api.schemas.AuthorizationAction.read,
218
+ mlrun.common.schemas.AuthorizationAction.read,
196
219
  auth_info,
197
220
  )
198
221
 
@@ -211,7 +234,7 @@ async def list_artifacts(
211
234
  )
212
235
 
213
236
  artifacts = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
214
- mlrun.api.schemas.AuthorizationResourceTypes.artifact,
237
+ mlrun.common.schemas.AuthorizationResourceTypes.artifact,
215
238
  artifacts,
216
239
  _artifact_project_and_resource_name_extractor,
217
240
  auth_info,
@@ -221,14 +244,19 @@ async def list_artifacts(
221
244
  }
222
245
 
223
246
 
224
- # TODO /artifacts should be deprecated in 1.4
225
- @router.delete("/artifacts")
247
+ # TODO: remove /artifacts in 1.6.0
248
+ @router.delete(
249
+ "/artifacts",
250
+ deprecated=True,
251
+ description="/artifacts is deprecated in 1.4.0 and will be removed in 1.6.0, "
252
+ "use /projects/{project}/artifacts instead",
253
+ )
226
254
  async def delete_artifacts_legacy(
227
255
  project: str = mlrun.mlconf.default_project,
228
256
  name: str = "",
229
257
  tag: str = "",
230
258
  labels: List[str] = Query([], alias="label"),
231
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
259
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
232
260
  db_session: Session = Depends(deps.get_db_session),
233
261
  ):
234
262
  return await _delete_artifacts(
@@ -247,7 +275,7 @@ async def delete_artifacts(
247
275
  name: str = "",
248
276
  tag: str = "",
249
277
  labels: List[str] = Query([], alias="label"),
250
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
278
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
251
279
  db_session: Session = Depends(deps.get_db_session),
252
280
  ):
253
281
  return await _delete_artifacts(
@@ -265,7 +293,7 @@ async def _delete_artifacts(
265
293
  name: str = None,
266
294
  tag: str = None,
267
295
  labels: List[str] = None,
268
- auth_info: mlrun.api.schemas.AuthInfo = None,
296
+ auth_info: mlrun.common.schemas.AuthInfo = None,
269
297
  db_session: Session = None,
270
298
  ):
271
299
  artifacts = await run_in_threadpool(
@@ -277,10 +305,10 @@ async def _delete_artifacts(
277
305
  labels,
278
306
  )
279
307
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resources_permissions(
280
- mlrun.api.schemas.AuthorizationResourceTypes.artifact,
308
+ mlrun.common.schemas.AuthorizationResourceTypes.artifact,
281
309
  artifacts,
282
310
  _artifact_project_and_resource_name_extractor,
283
- mlrun.api.schemas.AuthorizationAction.delete,
311
+ mlrun.common.schemas.AuthorizationAction.delete,
284
312
  auth_info,
285
313
  )
286
314
  await run_in_threadpool(
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -15,16 +15,16 @@
15
15
  import fastapi
16
16
 
17
17
  import mlrun.api.api.deps
18
- import mlrun.api.schemas
19
18
  import mlrun.api.utils.auth.verifier
19
+ import mlrun.common.schemas
20
20
 
21
21
  router = fastapi.APIRouter()
22
22
 
23
23
 
24
24
  @router.post("/authorization/verifications")
25
25
  async def verify_authorization(
26
- authorization_verification_input: mlrun.api.schemas.AuthorizationVerificationInput,
27
- auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
26
+ authorization_verification_input: mlrun.common.schemas.AuthorizationVerificationInput,
27
+ auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
28
28
  mlrun.api.api.deps.authenticate_request
29
29
  ),
30
30
  ):