mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (444) hide show
  1. mlrun/__init__.py +3 -3
  2. mlrun/__main__.py +79 -37
  3. mlrun/api/__init__.py +1 -1
  4. mlrun/api/api/__init__.py +1 -1
  5. mlrun/api/api/api.py +4 -4
  6. mlrun/api/api/deps.py +10 -21
  7. mlrun/api/api/endpoints/__init__.py +1 -1
  8. mlrun/api/api/endpoints/artifacts.py +64 -36
  9. mlrun/api/api/endpoints/auth.py +4 -4
  10. mlrun/api/api/endpoints/background_tasks.py +11 -11
  11. mlrun/api/api/endpoints/client_spec.py +5 -5
  12. mlrun/api/api/endpoints/clusterization_spec.py +6 -4
  13. mlrun/api/api/endpoints/feature_store.py +124 -115
  14. mlrun/api/api/endpoints/files.py +22 -14
  15. mlrun/api/api/endpoints/frontend_spec.py +28 -21
  16. mlrun/api/api/endpoints/functions.py +142 -87
  17. mlrun/api/api/endpoints/grafana_proxy.py +89 -442
  18. mlrun/api/api/endpoints/healthz.py +20 -7
  19. mlrun/api/api/endpoints/hub.py +320 -0
  20. mlrun/api/api/endpoints/internal/__init__.py +1 -1
  21. mlrun/api/api/endpoints/internal/config.py +1 -1
  22. mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
  23. mlrun/api/api/endpoints/logs.py +11 -11
  24. mlrun/api/api/endpoints/model_endpoints.py +74 -70
  25. mlrun/api/api/endpoints/operations.py +13 -9
  26. mlrun/api/api/endpoints/pipelines.py +93 -88
  27. mlrun/api/api/endpoints/projects.py +35 -35
  28. mlrun/api/api/endpoints/runs.py +69 -27
  29. mlrun/api/api/endpoints/runtime_resources.py +28 -28
  30. mlrun/api/api/endpoints/schedules.py +98 -41
  31. mlrun/api/api/endpoints/secrets.py +37 -32
  32. mlrun/api/api/endpoints/submit.py +12 -12
  33. mlrun/api/api/endpoints/tags.py +20 -22
  34. mlrun/api/api/utils.py +251 -42
  35. mlrun/api/constants.py +1 -1
  36. mlrun/api/crud/__init__.py +18 -15
  37. mlrun/api/crud/artifacts.py +10 -10
  38. mlrun/api/crud/client_spec.py +4 -4
  39. mlrun/api/crud/clusterization_spec.py +3 -3
  40. mlrun/api/crud/feature_store.py +54 -46
  41. mlrun/api/crud/functions.py +3 -3
  42. mlrun/api/crud/hub.py +312 -0
  43. mlrun/api/crud/logs.py +11 -9
  44. mlrun/api/crud/model_monitoring/__init__.py +3 -3
  45. mlrun/api/crud/model_monitoring/grafana.py +435 -0
  46. mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
  47. mlrun/api/crud/notifications.py +149 -0
  48. mlrun/api/crud/pipelines.py +67 -52
  49. mlrun/api/crud/projects.py +51 -23
  50. mlrun/api/crud/runs.py +7 -5
  51. mlrun/api/crud/runtime_resources.py +13 -13
  52. mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
  53. mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
  54. mlrun/api/crud/runtimes/nuclio/function.py +505 -0
  55. mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
  56. mlrun/api/crud/secrets.py +88 -46
  57. mlrun/api/crud/tags.py +5 -5
  58. mlrun/api/db/__init__.py +1 -1
  59. mlrun/api/db/base.py +102 -54
  60. mlrun/api/db/init_db.py +2 -3
  61. mlrun/api/db/session.py +4 -12
  62. mlrun/api/db/sqldb/__init__.py +1 -1
  63. mlrun/api/db/sqldb/db.py +439 -196
  64. mlrun/api/db/sqldb/helpers.py +1 -1
  65. mlrun/api/db/sqldb/models/__init__.py +3 -3
  66. mlrun/api/db/sqldb/models/models_mysql.py +82 -64
  67. mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
  68. mlrun/api/db/sqldb/session.py +27 -20
  69. mlrun/api/initial_data.py +82 -24
  70. mlrun/api/launcher.py +196 -0
  71. mlrun/api/main.py +91 -22
  72. mlrun/api/middlewares.py +6 -5
  73. mlrun/api/migrations_mysql/env.py +1 -1
  74. mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
  75. mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
  76. mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
  77. mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
  78. mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
  79. mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
  80. mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
  81. mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
  82. mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
  83. mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
  84. mlrun/api/migrations_sqlite/env.py +1 -1
  85. mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
  86. mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
  87. mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
  88. mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
  89. mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
  90. mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
  91. mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
  92. mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
  93. mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
  94. mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
  95. mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
  96. mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
  97. mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
  98. mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
  99. mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
  100. mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
  101. mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
  102. mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
  103. mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
  104. mlrun/api/schemas/__init__.py +216 -138
  105. mlrun/api/utils/__init__.py +1 -1
  106. mlrun/api/utils/asyncio.py +1 -1
  107. mlrun/api/utils/auth/__init__.py +1 -1
  108. mlrun/api/utils/auth/providers/__init__.py +1 -1
  109. mlrun/api/utils/auth/providers/base.py +7 -7
  110. mlrun/api/utils/auth/providers/nop.py +6 -7
  111. mlrun/api/utils/auth/providers/opa.py +17 -17
  112. mlrun/api/utils/auth/verifier.py +36 -34
  113. mlrun/api/utils/background_tasks.py +24 -24
  114. mlrun/{builder.py → api/utils/builder.py} +216 -123
  115. mlrun/api/utils/clients/__init__.py +1 -1
  116. mlrun/api/utils/clients/chief.py +19 -4
  117. mlrun/api/utils/clients/iguazio.py +106 -60
  118. mlrun/api/utils/clients/log_collector.py +1 -1
  119. mlrun/api/utils/clients/nuclio.py +23 -23
  120. mlrun/api/utils/clients/protocols/grpc.py +2 -2
  121. mlrun/api/utils/db/__init__.py +1 -1
  122. mlrun/api/utils/db/alembic.py +1 -1
  123. mlrun/api/utils/db/backup.py +1 -1
  124. mlrun/api/utils/db/mysql.py +24 -25
  125. mlrun/api/utils/db/sql_collation.py +1 -1
  126. mlrun/api/utils/db/sqlite_migration.py +2 -2
  127. mlrun/api/utils/events/__init__.py +14 -0
  128. mlrun/api/utils/events/base.py +57 -0
  129. mlrun/api/utils/events/events_factory.py +41 -0
  130. mlrun/api/utils/events/iguazio.py +217 -0
  131. mlrun/api/utils/events/nop.py +55 -0
  132. mlrun/api/utils/helpers.py +16 -13
  133. mlrun/api/utils/memory_reports.py +1 -1
  134. mlrun/api/utils/periodic.py +6 -3
  135. mlrun/api/utils/projects/__init__.py +1 -1
  136. mlrun/api/utils/projects/follower.py +33 -33
  137. mlrun/api/utils/projects/leader.py +36 -34
  138. mlrun/api/utils/projects/member.py +27 -27
  139. mlrun/api/utils/projects/remotes/__init__.py +1 -1
  140. mlrun/api/utils/projects/remotes/follower.py +13 -13
  141. mlrun/api/utils/projects/remotes/leader.py +10 -10
  142. mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
  143. mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
  144. mlrun/api/utils/scheduler.py +140 -51
  145. mlrun/api/utils/singletons/__init__.py +1 -1
  146. mlrun/api/utils/singletons/db.py +9 -15
  147. mlrun/api/utils/singletons/k8s.py +677 -5
  148. mlrun/api/utils/singletons/logs_dir.py +1 -1
  149. mlrun/api/utils/singletons/project_member.py +1 -1
  150. mlrun/api/utils/singletons/scheduler.py +1 -1
  151. mlrun/artifacts/__init__.py +2 -2
  152. mlrun/artifacts/base.py +8 -2
  153. mlrun/artifacts/dataset.py +5 -3
  154. mlrun/artifacts/manager.py +7 -1
  155. mlrun/artifacts/model.py +15 -4
  156. mlrun/artifacts/plots.py +1 -1
  157. mlrun/common/__init__.py +1 -1
  158. mlrun/common/constants.py +15 -0
  159. mlrun/common/model_monitoring.py +209 -0
  160. mlrun/common/schemas/__init__.py +167 -0
  161. mlrun/{api → common}/schemas/artifact.py +13 -14
  162. mlrun/{api → common}/schemas/auth.py +10 -8
  163. mlrun/{api → common}/schemas/background_task.py +3 -3
  164. mlrun/{api → common}/schemas/client_spec.py +1 -1
  165. mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
  166. mlrun/{api → common}/schemas/constants.py +21 -8
  167. mlrun/common/schemas/events.py +36 -0
  168. mlrun/{api → common}/schemas/feature_store.py +2 -1
  169. mlrun/{api → common}/schemas/frontend_spec.py +7 -6
  170. mlrun/{api → common}/schemas/function.py +5 -5
  171. mlrun/{api → common}/schemas/http.py +3 -3
  172. mlrun/common/schemas/hub.py +134 -0
  173. mlrun/{api → common}/schemas/k8s.py +3 -3
  174. mlrun/{api → common}/schemas/memory_reports.py +1 -1
  175. mlrun/common/schemas/model_endpoints.py +342 -0
  176. mlrun/common/schemas/notification.py +57 -0
  177. mlrun/{api → common}/schemas/object.py +6 -6
  178. mlrun/{api → common}/schemas/pipeline.py +3 -3
  179. mlrun/{api → common}/schemas/project.py +6 -5
  180. mlrun/common/schemas/regex.py +24 -0
  181. mlrun/common/schemas/runs.py +30 -0
  182. mlrun/{api → common}/schemas/runtime_resource.py +3 -3
  183. mlrun/{api → common}/schemas/schedule.py +19 -7
  184. mlrun/{api → common}/schemas/secret.py +3 -3
  185. mlrun/{api → common}/schemas/tag.py +2 -2
  186. mlrun/common/types.py +25 -0
  187. mlrun/config.py +152 -20
  188. mlrun/data_types/__init__.py +7 -2
  189. mlrun/data_types/data_types.py +4 -2
  190. mlrun/data_types/infer.py +1 -1
  191. mlrun/data_types/spark.py +10 -3
  192. mlrun/datastore/__init__.py +10 -3
  193. mlrun/datastore/azure_blob.py +1 -1
  194. mlrun/datastore/base.py +185 -53
  195. mlrun/datastore/datastore.py +1 -1
  196. mlrun/datastore/filestore.py +1 -1
  197. mlrun/datastore/google_cloud_storage.py +1 -1
  198. mlrun/datastore/inmem.py +4 -1
  199. mlrun/datastore/redis.py +1 -1
  200. mlrun/datastore/s3.py +1 -1
  201. mlrun/datastore/sources.py +192 -70
  202. mlrun/datastore/spark_udf.py +44 -0
  203. mlrun/datastore/store_resources.py +4 -4
  204. mlrun/datastore/targets.py +115 -45
  205. mlrun/datastore/utils.py +127 -5
  206. mlrun/datastore/v3io.py +1 -1
  207. mlrun/datastore/wasbfs/__init__.py +1 -1
  208. mlrun/datastore/wasbfs/fs.py +1 -1
  209. mlrun/db/__init__.py +7 -5
  210. mlrun/db/base.py +112 -68
  211. mlrun/db/httpdb.py +445 -277
  212. mlrun/db/nopdb.py +491 -0
  213. mlrun/db/sqldb.py +112 -65
  214. mlrun/errors.py +6 -1
  215. mlrun/execution.py +44 -22
  216. mlrun/feature_store/__init__.py +1 -1
  217. mlrun/feature_store/api.py +143 -95
  218. mlrun/feature_store/common.py +16 -20
  219. mlrun/feature_store/feature_set.py +42 -12
  220. mlrun/feature_store/feature_vector.py +32 -21
  221. mlrun/feature_store/ingestion.py +9 -12
  222. mlrun/feature_store/retrieval/__init__.py +3 -2
  223. mlrun/feature_store/retrieval/base.py +388 -66
  224. mlrun/feature_store/retrieval/dask_merger.py +63 -151
  225. mlrun/feature_store/retrieval/job.py +30 -12
  226. mlrun/feature_store/retrieval/local_merger.py +40 -133
  227. mlrun/feature_store/retrieval/spark_merger.py +129 -127
  228. mlrun/feature_store/retrieval/storey_merger.py +173 -0
  229. mlrun/feature_store/steps.py +132 -15
  230. mlrun/features.py +8 -3
  231. mlrun/frameworks/__init__.py +1 -1
  232. mlrun/frameworks/_common/__init__.py +1 -1
  233. mlrun/frameworks/_common/artifacts_library.py +1 -1
  234. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  235. mlrun/frameworks/_common/model_handler.py +1 -1
  236. mlrun/frameworks/_common/plan.py +1 -1
  237. mlrun/frameworks/_common/producer.py +1 -1
  238. mlrun/frameworks/_common/utils.py +1 -1
  239. mlrun/frameworks/_dl_common/__init__.py +1 -1
  240. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
  241. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  242. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  243. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  244. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  245. mlrun/frameworks/_dl_common/utils.py +1 -1
  246. mlrun/frameworks/_ml_common/__init__.py +1 -1
  247. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  248. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
  249. mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
  250. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  251. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  252. mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
  253. mlrun/frameworks/_ml_common/plan.py +1 -1
  254. mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
  255. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
  256. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
  257. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  258. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  259. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  260. mlrun/frameworks/_ml_common/producer.py +1 -1
  261. mlrun/frameworks/_ml_common/utils.py +1 -1
  262. mlrun/frameworks/auto_mlrun/__init__.py +1 -1
  263. mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
  264. mlrun/frameworks/huggingface/__init__.py +1 -1
  265. mlrun/frameworks/huggingface/model_server.py +1 -1
  266. mlrun/frameworks/lgbm/__init__.py +1 -1
  267. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
  268. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  269. mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
  270. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
  271. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
  272. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  273. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  274. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  275. mlrun/frameworks/lgbm/model_handler.py +1 -1
  276. mlrun/frameworks/lgbm/model_server.py +1 -1
  277. mlrun/frameworks/lgbm/utils.py +1 -1
  278. mlrun/frameworks/onnx/__init__.py +1 -1
  279. mlrun/frameworks/onnx/dataset.py +1 -1
  280. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  281. mlrun/frameworks/onnx/model_handler.py +1 -1
  282. mlrun/frameworks/onnx/model_server.py +1 -1
  283. mlrun/frameworks/parallel_coordinates.py +1 -1
  284. mlrun/frameworks/pytorch/__init__.py +1 -1
  285. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
  286. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  287. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  288. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  289. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  290. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  291. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  292. mlrun/frameworks/pytorch/model_handler.py +1 -1
  293. mlrun/frameworks/pytorch/model_server.py +1 -1
  294. mlrun/frameworks/pytorch/utils.py +1 -1
  295. mlrun/frameworks/sklearn/__init__.py +1 -1
  296. mlrun/frameworks/sklearn/estimator.py +1 -1
  297. mlrun/frameworks/sklearn/metric.py +1 -1
  298. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  299. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  300. mlrun/frameworks/sklearn/model_handler.py +1 -1
  301. mlrun/frameworks/sklearn/utils.py +1 -1
  302. mlrun/frameworks/tf_keras/__init__.py +1 -1
  303. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
  304. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  305. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  306. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  307. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  308. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  309. mlrun/frameworks/tf_keras/model_server.py +1 -1
  310. mlrun/frameworks/tf_keras/utils.py +1 -1
  311. mlrun/frameworks/xgboost/__init__.py +1 -1
  312. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  313. mlrun/frameworks/xgboost/model_handler.py +1 -1
  314. mlrun/frameworks/xgboost/utils.py +1 -1
  315. mlrun/k8s_utils.py +14 -765
  316. mlrun/kfpops.py +14 -17
  317. mlrun/launcher/__init__.py +13 -0
  318. mlrun/launcher/base.py +406 -0
  319. mlrun/launcher/client.py +159 -0
  320. mlrun/launcher/factory.py +50 -0
  321. mlrun/launcher/local.py +276 -0
  322. mlrun/launcher/remote.py +178 -0
  323. mlrun/lists.py +10 -2
  324. mlrun/mlutils/__init__.py +1 -1
  325. mlrun/mlutils/data.py +1 -1
  326. mlrun/mlutils/models.py +1 -1
  327. mlrun/mlutils/plots.py +1 -1
  328. mlrun/model.py +252 -14
  329. mlrun/model_monitoring/__init__.py +41 -0
  330. mlrun/model_monitoring/features_drift_table.py +1 -1
  331. mlrun/model_monitoring/helpers.py +123 -38
  332. mlrun/model_monitoring/model_endpoint.py +144 -0
  333. mlrun/model_monitoring/model_monitoring_batch.py +310 -259
  334. mlrun/model_monitoring/stores/__init__.py +106 -0
  335. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
  336. mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
  337. mlrun/model_monitoring/stores/models/__init__.py +23 -0
  338. mlrun/model_monitoring/stores/models/base.py +18 -0
  339. mlrun/model_monitoring/stores/models/mysql.py +100 -0
  340. mlrun/model_monitoring/stores/models/sqlite.py +98 -0
  341. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
  342. mlrun/model_monitoring/stream_processing_fs.py +239 -271
  343. mlrun/package/__init__.py +163 -0
  344. mlrun/package/context_handler.py +325 -0
  345. mlrun/package/errors.py +47 -0
  346. mlrun/package/packager.py +298 -0
  347. mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
  348. mlrun/package/packagers/default_packager.py +422 -0
  349. mlrun/package/packagers/numpy_packagers.py +612 -0
  350. mlrun/package/packagers/pandas_packagers.py +968 -0
  351. mlrun/package/packagers/python_standard_library_packagers.py +616 -0
  352. mlrun/package/packagers_manager.py +786 -0
  353. mlrun/package/utils/__init__.py +53 -0
  354. mlrun/package/utils/_archiver.py +226 -0
  355. mlrun/package/utils/_formatter.py +211 -0
  356. mlrun/package/utils/_pickler.py +234 -0
  357. mlrun/package/utils/_supported_format.py +71 -0
  358. mlrun/package/utils/log_hint_utils.py +93 -0
  359. mlrun/package/utils/type_hint_utils.py +298 -0
  360. mlrun/platforms/__init__.py +1 -1
  361. mlrun/platforms/iguazio.py +34 -2
  362. mlrun/platforms/other.py +1 -1
  363. mlrun/projects/__init__.py +1 -1
  364. mlrun/projects/operations.py +14 -9
  365. mlrun/projects/pipelines.py +31 -13
  366. mlrun/projects/project.py +762 -238
  367. mlrun/render.py +49 -19
  368. mlrun/run.py +57 -326
  369. mlrun/runtimes/__init__.py +3 -9
  370. mlrun/runtimes/base.py +247 -784
  371. mlrun/runtimes/constants.py +1 -1
  372. mlrun/runtimes/daskjob.py +45 -41
  373. mlrun/runtimes/funcdoc.py +43 -7
  374. mlrun/runtimes/function.py +66 -656
  375. mlrun/runtimes/function_reference.py +1 -1
  376. mlrun/runtimes/generators.py +1 -1
  377. mlrun/runtimes/kubejob.py +99 -116
  378. mlrun/runtimes/local.py +59 -66
  379. mlrun/runtimes/mpijob/__init__.py +1 -1
  380. mlrun/runtimes/mpijob/abstract.py +13 -15
  381. mlrun/runtimes/mpijob/v1.py +3 -1
  382. mlrun/runtimes/mpijob/v1alpha1.py +1 -1
  383. mlrun/runtimes/nuclio.py +1 -1
  384. mlrun/runtimes/pod.py +51 -26
  385. mlrun/runtimes/remotesparkjob.py +3 -1
  386. mlrun/runtimes/serving.py +12 -4
  387. mlrun/runtimes/sparkjob/__init__.py +1 -2
  388. mlrun/runtimes/sparkjob/abstract.py +44 -31
  389. mlrun/runtimes/sparkjob/spark3job.py +11 -9
  390. mlrun/runtimes/utils.py +61 -42
  391. mlrun/secrets.py +16 -18
  392. mlrun/serving/__init__.py +3 -2
  393. mlrun/serving/merger.py +1 -1
  394. mlrun/serving/remote.py +1 -1
  395. mlrun/serving/routers.py +39 -42
  396. mlrun/serving/server.py +23 -13
  397. mlrun/serving/serving_wrapper.py +1 -1
  398. mlrun/serving/states.py +172 -39
  399. mlrun/serving/utils.py +1 -1
  400. mlrun/serving/v1_serving.py +1 -1
  401. mlrun/serving/v2_serving.py +29 -21
  402. mlrun/utils/__init__.py +1 -2
  403. mlrun/utils/async_http.py +8 -1
  404. mlrun/utils/azure_vault.py +1 -1
  405. mlrun/utils/clones.py +2 -2
  406. mlrun/utils/condition_evaluator.py +65 -0
  407. mlrun/utils/db.py +52 -0
  408. mlrun/utils/helpers.py +188 -13
  409. mlrun/utils/http.py +89 -54
  410. mlrun/utils/logger.py +48 -8
  411. mlrun/utils/model_monitoring.py +132 -100
  412. mlrun/utils/notifications/__init__.py +1 -1
  413. mlrun/utils/notifications/notification/__init__.py +8 -6
  414. mlrun/utils/notifications/notification/base.py +20 -14
  415. mlrun/utils/notifications/notification/console.py +7 -4
  416. mlrun/utils/notifications/notification/git.py +36 -19
  417. mlrun/utils/notifications/notification/ipython.py +10 -8
  418. mlrun/utils/notifications/notification/slack.py +18 -13
  419. mlrun/utils/notifications/notification_pusher.py +377 -56
  420. mlrun/utils/regex.py +6 -1
  421. mlrun/utils/singleton.py +1 -1
  422. mlrun/utils/v3io_clients.py +1 -1
  423. mlrun/utils/vault.py +270 -269
  424. mlrun/utils/version/__init__.py +1 -1
  425. mlrun/utils/version/version.json +2 -2
  426. mlrun/utils/version/version.py +1 -1
  427. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
  428. mlrun-1.4.0.dist-info/RECORD +434 -0
  429. mlrun/api/api/endpoints/marketplace.py +0 -257
  430. mlrun/api/crud/marketplace.py +0 -221
  431. mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
  432. mlrun/api/db/filedb/db.py +0 -518
  433. mlrun/api/schemas/marketplace.py +0 -128
  434. mlrun/api/schemas/model_endpoints.py +0 -185
  435. mlrun/db/filedb.py +0 -891
  436. mlrun/feature_store/retrieval/online.py +0 -92
  437. mlrun/model_monitoring/constants.py +0 -67
  438. mlrun/runtimes/package/context_handler.py +0 -711
  439. mlrun/runtimes/sparkjob/spark2job.py +0 -59
  440. mlrun-1.3.3rc1.dist-info/RECORD +0 -381
  441. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
  442. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
  443. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
  444. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -20,23 +20,24 @@ from fastapi.concurrency import run_in_threadpool
20
20
  from sqlalchemy.orm import Session
21
21
 
22
22
  import mlrun.api.api.utils
23
+ import mlrun.api.crud
23
24
  import mlrun.api.utils.auth.verifier
24
25
  import mlrun.api.utils.clients.chief
25
26
  import mlrun.api.utils.singletons.project_member
26
- from mlrun.api import schemas
27
+ import mlrun.common.schemas
27
28
  from mlrun.api.api import deps
28
29
  from mlrun.api.utils.singletons.scheduler import get_scheduler
29
30
  from mlrun.utils import logger
30
31
 
31
- router = APIRouter()
32
+ router = APIRouter(prefix="/projects/{project}/schedules")
32
33
 
33
34
 
34
- @router.post("/projects/{project}/schedules")
35
+ @router.post("")
35
36
  async def create_schedule(
36
37
  project: str,
37
- schedule: schemas.ScheduleInput,
38
+ schedule: mlrun.common.schemas.ScheduleInput,
38
39
  request: fastapi.Request,
39
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
40
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
40
41
  db_session: Session = Depends(deps.get_db_session),
41
42
  ):
42
43
  await run_in_threadpool(
@@ -46,16 +47,16 @@ async def create_schedule(
46
47
  auth_info=auth_info,
47
48
  )
48
49
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
49
- mlrun.api.schemas.AuthorizationResourceTypes.schedule,
50
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
50
51
  project,
51
52
  schedule.name,
52
- mlrun.api.schemas.AuthorizationAction.create,
53
+ mlrun.common.schemas.AuthorizationAction.create,
53
54
  auth_info,
54
55
  )
55
56
  # to reduce redundant load on the chief, we re-route the request only if the user has permissions
56
57
  if (
57
58
  mlrun.mlconf.httpdb.clusterization.role
58
- != mlrun.api.schemas.ClusterizationRole.chief
59
+ != mlrun.common.schemas.ClusterizationRole.chief
59
60
  ):
60
61
  logger.info(
61
62
  "Requesting to create schedule, re-routing to chief",
@@ -86,26 +87,26 @@ async def create_schedule(
86
87
  return Response(status_code=HTTPStatus.CREATED.value)
87
88
 
88
89
 
89
- @router.put("/projects/{project}/schedules/{name}")
90
+ @router.put("/{name}")
90
91
  async def update_schedule(
91
92
  project: str,
92
93
  name: str,
93
- schedule: schemas.ScheduleUpdate,
94
+ schedule: mlrun.common.schemas.ScheduleUpdate,
94
95
  request: fastapi.Request,
95
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
96
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
96
97
  db_session: Session = Depends(deps.get_db_session),
97
98
  ):
98
99
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
99
- mlrun.api.schemas.AuthorizationResourceTypes.schedule,
100
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
100
101
  project,
101
102
  name,
102
- mlrun.api.schemas.AuthorizationAction.update,
103
+ mlrun.common.schemas.AuthorizationAction.update,
103
104
  auth_info,
104
105
  )
105
106
  # to reduce redundant load on the chief, we re-route the request only if the user has permissions
106
107
  if (
107
108
  mlrun.mlconf.httpdb.clusterization.role
108
- != mlrun.api.schemas.ClusterizationRole.chief
109
+ != mlrun.common.schemas.ClusterizationRole.chief
109
110
  ):
110
111
  logger.info(
111
112
  "Requesting to update schedule, re-routing to chief",
@@ -136,20 +137,20 @@ async def update_schedule(
136
137
  return Response(status_code=HTTPStatus.OK.value)
137
138
 
138
139
 
139
- @router.get("/projects/{project}/schedules", response_model=schemas.SchedulesOutput)
140
+ @router.get("", response_model=mlrun.common.schemas.SchedulesOutput)
140
141
  async def list_schedules(
141
142
  project: str,
142
143
  name: str = None,
143
144
  labels: str = None,
144
- kind: schemas.ScheduleKinds = None,
145
+ kind: mlrun.common.schemas.ScheduleKinds = None,
145
146
  include_last_run: bool = False,
146
147
  include_credentials: bool = fastapi.Query(False, alias="include-credentials"),
147
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
148
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
148
149
  db_session: Session = Depends(deps.get_db_session),
149
150
  ):
150
151
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
151
152
  project,
152
- mlrun.api.schemas.AuthorizationAction.read,
153
+ mlrun.common.schemas.AuthorizationAction.read,
153
154
  auth_info,
154
155
  )
155
156
  schedules = await run_in_threadpool(
@@ -163,7 +164,7 @@ async def list_schedules(
163
164
  include_credentials,
164
165
  )
165
166
  filtered_schedules = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
166
- mlrun.api.schemas.AuthorizationResourceTypes.schedule,
167
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
167
168
  schedules.schedules,
168
169
  lambda schedule: (
169
170
  schedule.project,
@@ -176,14 +177,15 @@ async def list_schedules(
176
177
 
177
178
 
178
179
  @router.get(
179
- "/projects/{project}/schedules/{name}", response_model=schemas.ScheduleOutput
180
+ "/{name}",
181
+ response_model=mlrun.common.schemas.ScheduleOutput,
180
182
  )
181
183
  async def get_schedule(
182
184
  project: str,
183
185
  name: str,
184
186
  include_last_run: bool = False,
185
187
  include_credentials: bool = fastapi.Query(False, alias="include-credentials"),
186
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
188
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
187
189
  db_session: Session = Depends(deps.get_db_session),
188
190
  ):
189
191
  schedule = await run_in_threadpool(
@@ -195,34 +197,34 @@ async def get_schedule(
195
197
  include_credentials,
196
198
  )
197
199
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
198
- mlrun.api.schemas.AuthorizationResourceTypes.schedule,
200
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
199
201
  project,
200
202
  name,
201
- mlrun.api.schemas.AuthorizationAction.read,
203
+ mlrun.common.schemas.AuthorizationAction.read,
202
204
  auth_info,
203
205
  )
204
206
  return schedule
205
207
 
206
208
 
207
- @router.post("/projects/{project}/schedules/{name}/invoke")
209
+ @router.post("/{name}/invoke")
208
210
  async def invoke_schedule(
209
211
  project: str,
210
212
  name: str,
211
213
  request: fastapi.Request,
212
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
214
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
213
215
  db_session: Session = Depends(deps.get_db_session),
214
216
  ):
215
217
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
216
- mlrun.api.schemas.AuthorizationResourceTypes.schedule,
218
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
217
219
  project,
218
220
  name,
219
- mlrun.api.schemas.AuthorizationAction.update,
221
+ mlrun.common.schemas.AuthorizationAction.update,
220
222
  auth_info,
221
223
  )
222
224
  # to reduce redundant load on the chief, we re-route the request only if the user has permissions
223
225
  if (
224
226
  mlrun.mlconf.httpdb.clusterization.role
225
- != mlrun.api.schemas.ClusterizationRole.chief
227
+ != mlrun.common.schemas.ClusterizationRole.chief
226
228
  ):
227
229
  logger.info(
228
230
  "Requesting to invoke schedule, re-routing to chief",
@@ -237,27 +239,25 @@ async def invoke_schedule(
237
239
  return await get_scheduler().invoke_schedule(db_session, auth_info, project, name)
238
240
 
239
241
 
240
- @router.delete(
241
- "/projects/{project}/schedules/{name}", status_code=HTTPStatus.NO_CONTENT.value
242
- )
242
+ @router.delete("/{name}", status_code=HTTPStatus.NO_CONTENT.value)
243
243
  async def delete_schedule(
244
244
  project: str,
245
245
  name: str,
246
246
  request: fastapi.Request,
247
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
247
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
248
248
  db_session: Session = Depends(deps.get_db_session),
249
249
  ):
250
250
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
251
- mlrun.api.schemas.AuthorizationResourceTypes.schedule,
251
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
252
252
  project,
253
253
  name,
254
- mlrun.api.schemas.AuthorizationAction.delete,
254
+ mlrun.common.schemas.AuthorizationAction.delete,
255
255
  auth_info,
256
256
  )
257
257
  # to reduce redundant load on the chief, we re-route the request only if the user has permissions
258
258
  if (
259
259
  mlrun.mlconf.httpdb.clusterization.role
260
- != mlrun.api.schemas.ClusterizationRole.chief
260
+ != mlrun.common.schemas.ClusterizationRole.chief
261
261
  ):
262
262
  logger.info(
263
263
  "Requesting to delete schedule, re-routing to chief",
@@ -273,11 +273,11 @@ async def delete_schedule(
273
273
  return Response(status_code=HTTPStatus.NO_CONTENT.value)
274
274
 
275
275
 
276
- @router.delete("/projects/{project}/schedules", status_code=HTTPStatus.NO_CONTENT.value)
276
+ @router.delete("", status_code=HTTPStatus.NO_CONTENT.value)
277
277
  async def delete_schedules(
278
278
  project: str,
279
279
  request: fastapi.Request,
280
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
280
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
281
281
  db_session: Session = Depends(deps.get_db_session),
282
282
  ):
283
283
  schedules = await run_in_threadpool(
@@ -286,16 +286,16 @@ async def delete_schedules(
286
286
  project,
287
287
  )
288
288
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resources_permissions(
289
- mlrun.api.schemas.AuthorizationResourceTypes.schedule,
289
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
290
290
  schedules.schedules,
291
291
  lambda schedule: (schedule.project, schedule.name),
292
- mlrun.api.schemas.AuthorizationAction.delete,
292
+ mlrun.common.schemas.AuthorizationAction.delete,
293
293
  auth_info,
294
294
  )
295
295
  # to reduce redundant load on the chief, we re-route the request only if the user has permissions
296
296
  if (
297
297
  mlrun.mlconf.httpdb.clusterization.role
298
- != mlrun.api.schemas.ClusterizationRole.chief
298
+ != mlrun.common.schemas.ClusterizationRole.chief
299
299
  ):
300
300
  logger.info(
301
301
  "Requesting to delete all project schedules, re-routing to chief",
@@ -306,3 +306,60 @@ async def delete_schedules(
306
306
 
307
307
  await run_in_threadpool(get_scheduler().delete_schedules, db_session, project)
308
308
  return Response(status_code=HTTPStatus.NO_CONTENT.value)
309
+
310
+
311
+ @router.put("/{name}/notifications", status_code=HTTPStatus.OK.value)
312
+ async def set_schedule_notifications(
313
+ project: str,
314
+ name: str,
315
+ request: fastapi.Request,
316
+ set_notifications_request: mlrun.common.schemas.SetNotificationRequest = fastapi.Body(
317
+ ...
318
+ ),
319
+ auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
320
+ mlrun.api.api.deps.authenticate_request
321
+ ),
322
+ db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
323
+ ):
324
+ await fastapi.concurrency.run_in_threadpool(
325
+ mlrun.api.utils.singletons.project_member.get_project_member().ensure_project,
326
+ db_session,
327
+ project,
328
+ auth_info=auth_info,
329
+ )
330
+
331
+ # check permission per object type
332
+ await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
333
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
334
+ project,
335
+ resource_name=name,
336
+ action=mlrun.common.schemas.AuthorizationAction.update,
337
+ auth_info=auth_info,
338
+ )
339
+
340
+ if (
341
+ mlrun.mlconf.httpdb.clusterization.role
342
+ != mlrun.common.schemas.ClusterizationRole.chief
343
+ ):
344
+ logger.info(
345
+ "Requesting to set schedule notifications, re-routing to chief",
346
+ project=project,
347
+ schedule=set_notifications_request.dict(),
348
+ )
349
+ chief_client = mlrun.api.utils.clients.chief.Client()
350
+ return await chief_client.set_schedule_notifications(
351
+ project=project,
352
+ schedule_name=name,
353
+ request=request,
354
+ json=set_notifications_request.dict(),
355
+ )
356
+
357
+ await fastapi.concurrency.run_in_threadpool(
358
+ mlrun.api.crud.Notifications().set_object_notifications,
359
+ db_session,
360
+ auth_info,
361
+ project,
362
+ set_notifications_request.notifications,
363
+ mlrun.common.schemas.ScheduleIdentifier(name=name),
364
+ )
365
+ return fastapi.Response(status_code=HTTPStatus.OK.value)
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -23,9 +23,8 @@ import mlrun.api.api.deps
23
23
  import mlrun.api.crud
24
24
  import mlrun.api.utils.auth.verifier
25
25
  import mlrun.api.utils.singletons.project_member
26
+ import mlrun.common.schemas
26
27
  import mlrun.errors
27
- from mlrun.api import schemas
28
- from mlrun.utils.vault import add_vault_user_secrets
29
28
 
30
29
  router = fastapi.APIRouter()
31
30
 
@@ -33,8 +32,8 @@ router = fastapi.APIRouter()
33
32
  @router.post("/projects/{project}/secrets", status_code=HTTPStatus.CREATED.value)
34
33
  async def store_project_secrets(
35
34
  project: str,
36
- secrets: schemas.SecretsData,
37
- auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
35
+ secrets: mlrun.common.schemas.SecretsData,
36
+ auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
38
37
  mlrun.api.api.deps.authenticate_request
39
38
  ),
40
39
  db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
@@ -50,10 +49,10 @@ async def store_project_secrets(
50
49
  )
51
50
 
52
51
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
53
- mlrun.api.schemas.AuthorizationResourceTypes.secret,
52
+ mlrun.common.schemas.AuthorizationResourceTypes.secret,
54
53
  project,
55
54
  secrets.provider,
56
- mlrun.api.schemas.AuthorizationAction.create,
55
+ mlrun.common.schemas.AuthorizationAction.create,
57
56
  auth_info,
58
57
  )
59
58
  await run_in_threadpool(
@@ -66,9 +65,9 @@ async def store_project_secrets(
66
65
  @router.delete("/projects/{project}/secrets", status_code=HTTPStatus.NO_CONTENT.value)
67
66
  async def delete_project_secrets(
68
67
  project: str,
69
- provider: schemas.SecretProviderName = schemas.SecretProviderName.kubernetes,
68
+ provider: mlrun.common.schemas.SecretProviderName = mlrun.common.schemas.SecretProviderName.kubernetes,
70
69
  secrets: List[str] = fastapi.Query(None, alias="secret"),
71
- auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
70
+ auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
72
71
  mlrun.api.api.deps.authenticate_request
73
72
  ),
74
73
  db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
@@ -81,10 +80,10 @@ async def delete_project_secrets(
81
80
  )
82
81
 
83
82
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
84
- mlrun.api.schemas.AuthorizationResourceTypes.secret,
83
+ mlrun.common.schemas.AuthorizationResourceTypes.secret,
85
84
  project,
86
85
  provider,
87
- mlrun.api.schemas.AuthorizationAction.delete,
86
+ mlrun.common.schemas.AuthorizationAction.delete,
88
87
  auth_info,
89
88
  )
90
89
  await run_in_threadpool(
@@ -94,12 +93,17 @@ async def delete_project_secrets(
94
93
  return fastapi.Response(status_code=HTTPStatus.NO_CONTENT.value)
95
94
 
96
95
 
97
- @router.get("/projects/{project}/secret-keys", response_model=schemas.SecretKeysData)
96
+ @router.get(
97
+ "/projects/{project}/secret-keys",
98
+ response_model=mlrun.common.schemas.SecretKeysData,
99
+ )
98
100
  async def list_project_secret_keys(
99
101
  project: str,
100
- provider: schemas.SecretProviderName = schemas.SecretProviderName.kubernetes,
101
- token: str = fastapi.Header(None, alias=schemas.HeaderNames.secret_store_token),
102
- auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
102
+ provider: mlrun.common.schemas.SecretProviderName = mlrun.common.schemas.SecretProviderName.kubernetes,
103
+ token: str = fastapi.Header(
104
+ None, alias=mlrun.common.schemas.HeaderNames.secret_store_token
105
+ ),
106
+ auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
103
107
  mlrun.api.api.deps.authenticate_request
104
108
  ),
105
109
  db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
@@ -111,10 +115,10 @@ async def list_project_secret_keys(
111
115
  auth_info.session,
112
116
  )
113
117
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
114
- mlrun.api.schemas.AuthorizationResourceTypes.secret,
118
+ mlrun.common.schemas.AuthorizationResourceTypes.secret,
115
119
  project,
116
120
  provider,
117
- mlrun.api.schemas.AuthorizationAction.read,
121
+ mlrun.common.schemas.AuthorizationAction.read,
118
122
  auth_info,
119
123
  )
120
124
  return await run_in_threadpool(
@@ -122,13 +126,17 @@ async def list_project_secret_keys(
122
126
  )
123
127
 
124
128
 
125
- @router.get("/projects/{project}/secrets", response_model=schemas.SecretsData)
129
+ @router.get(
130
+ "/projects/{project}/secrets", response_model=mlrun.common.schemas.SecretsData
131
+ )
126
132
  async def list_project_secrets(
127
133
  project: str,
128
134
  secrets: List[str] = fastapi.Query(None, alias="secret"),
129
- provider: schemas.SecretProviderName = schemas.SecretProviderName.kubernetes,
130
- token: str = fastapi.Header(None, alias=schemas.HeaderNames.secret_store_token),
131
- auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
135
+ provider: mlrun.common.schemas.SecretProviderName = mlrun.common.schemas.SecretProviderName.kubernetes,
136
+ token: str = fastapi.Header(
137
+ None, alias=mlrun.common.schemas.HeaderNames.secret_store_token
138
+ ),
139
+ auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
132
140
  mlrun.api.api.deps.authenticate_request
133
141
  ),
134
142
  db_session: Session = fastapi.Depends(mlrun.api.api.deps.get_db_session),
@@ -140,10 +148,10 @@ async def list_project_secrets(
140
148
  auth_info.session,
141
149
  )
142
150
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
143
- mlrun.api.schemas.AuthorizationResourceTypes.secret,
151
+ mlrun.common.schemas.AuthorizationResourceTypes.secret,
144
152
  project,
145
153
  provider,
146
- mlrun.api.schemas.AuthorizationAction.read,
154
+ mlrun.common.schemas.AuthorizationAction.read,
147
155
  auth_info,
148
156
  )
149
157
  return await run_in_threadpool(
@@ -153,13 +161,10 @@ async def list_project_secrets(
153
161
 
154
162
  @router.post("/user-secrets", status_code=HTTPStatus.CREATED.value)
155
163
  def add_user_secrets(
156
- secrets: schemas.UserSecretCreationRequest,
164
+ secrets: mlrun.common.schemas.UserSecretCreationRequest,
157
165
  ):
158
- if secrets.provider != schemas.SecretProviderName.vault:
159
- return fastapi.Response(
160
- status_code=HTTPStatus.BAD_REQUEST.vault,
161
- content=f"Invalid secrets provider {secrets.provider}",
162
- )
163
-
164
- add_vault_user_secrets(secrets.user, secrets.secrets)
165
- return fastapi.Response(status_code=HTTPStatus.CREATED.value)
166
+ # vault is not used
167
+ return fastapi.Response(
168
+ status_code=HTTPStatus.BAD_REQUEST.value,
169
+ content=f"Invalid secrets provider {secrets.provider}",
170
+ )
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -20,10 +20,10 @@ from fastapi import APIRouter, Depends, Header, Request
20
20
  from sqlalchemy.orm import Session
21
21
 
22
22
  import mlrun.api.api.utils
23
- import mlrun.api.schemas
24
23
  import mlrun.api.utils.auth.verifier
25
24
  import mlrun.api.utils.clients.chief
26
25
  import mlrun.api.utils.singletons.project_member
26
+ import mlrun.common.schemas
27
27
  import mlrun.utils.helpers
28
28
  from mlrun.api.api import deps
29
29
  from mlrun.utils import logger
@@ -38,13 +38,13 @@ router = APIRouter()
38
38
  async def submit_job(
39
39
  request: Request,
40
40
  username: Optional[str] = Header(None, alias="x-remote-user"),
41
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
41
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
42
42
  db_session: Session = Depends(deps.get_db_session),
43
43
  client_version: Optional[str] = Header(
44
- None, alias=mlrun.api.schemas.HeaderNames.client_version
44
+ None, alias=mlrun.common.schemas.HeaderNames.client_version
45
45
  ),
46
46
  client_python_version: Optional[str] = Header(
47
- None, alias=mlrun.api.schemas.HeaderNames.python_version
47
+ None, alias=mlrun.common.schemas.HeaderNames.python_version
48
48
  ),
49
49
  ):
50
50
  data = None
@@ -70,18 +70,18 @@ async def submit_job(
70
70
  _,
71
71
  ) = mlrun.utils.helpers.parse_versioned_object_uri(function_url)
72
72
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
73
- mlrun.api.schemas.AuthorizationResourceTypes.function,
73
+ mlrun.common.schemas.AuthorizationResourceTypes.function,
74
74
  function_project,
75
75
  function_name,
76
- mlrun.api.schemas.AuthorizationAction.read,
76
+ mlrun.common.schemas.AuthorizationAction.read,
77
77
  auth_info,
78
78
  )
79
79
  if data.get("schedule"):
80
80
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
81
- mlrun.api.schemas.AuthorizationResourceTypes.schedule,
81
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
82
82
  data["task"]["metadata"]["project"],
83
83
  data["task"]["metadata"]["name"],
84
- mlrun.api.schemas.AuthorizationAction.create,
84
+ mlrun.common.schemas.AuthorizationAction.create,
85
85
  auth_info,
86
86
  )
87
87
  # schedules are meant to be run solely by the chief, then if run is configured to run as scheduled
@@ -89,7 +89,7 @@ async def submit_job(
89
89
  # to reduce redundant load on the chief, we re-route the request only if the user has permissions
90
90
  if (
91
91
  mlrun.mlconf.httpdb.clusterization.role
92
- != mlrun.api.schemas.ClusterizationRole.chief
92
+ != mlrun.common.schemas.ClusterizationRole.chief
93
93
  ):
94
94
  logger.info(
95
95
  "Requesting to submit job with schedules, re-routing to chief",
@@ -102,10 +102,10 @@ async def submit_job(
102
102
 
103
103
  else:
104
104
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
105
- mlrun.api.schemas.AuthorizationResourceTypes.run,
105
+ mlrun.common.schemas.AuthorizationResourceTypes.run,
106
106
  data["task"]["metadata"]["project"],
107
107
  "",
108
- mlrun.api.schemas.AuthorizationAction.create,
108
+ mlrun.common.schemas.AuthorizationAction.create,
109
109
  auth_info,
110
110
  )
111
111
 
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -20,20 +20,20 @@ import sqlalchemy.orm
20
20
 
21
21
  import mlrun.api.api.deps
22
22
  import mlrun.api.crud.tags
23
- import mlrun.api.schemas
24
23
  import mlrun.api.utils.auth.verifier
25
24
  import mlrun.api.utils.singletons.project_member
25
+ import mlrun.common.schemas
26
26
  from mlrun.utils.helpers import tag_name_regex_as_string
27
27
 
28
- router = fastapi.APIRouter()
28
+ router = fastapi.APIRouter(prefix="/projects/{project}/tags")
29
29
 
30
30
 
31
- @router.post("/projects/{project}/tags/{tag}", response_model=mlrun.api.schemas.Tag)
31
+ @router.post("/{tag}", response_model=mlrun.common.schemas.Tag)
32
32
  async def overwrite_object_tags_with_tag(
33
33
  project: str,
34
34
  tag: str = fastapi.Path(..., regex=tag_name_regex_as_string()),
35
- tag_objects: mlrun.api.schemas.TagObjects = fastapi.Body(...),
36
- auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
35
+ tag_objects: mlrun.common.schemas.TagObjects = fastapi.Body(...),
36
+ auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
37
37
  mlrun.api.api.deps.authenticate_request
38
38
  ),
39
39
  db_session: sqlalchemy.orm.Session = fastapi.Depends(
@@ -49,11 +49,11 @@ async def overwrite_object_tags_with_tag(
49
49
 
50
50
  # check permission per object type
51
51
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
52
- getattr(mlrun.api.schemas.AuthorizationResourceTypes, tag_objects.kind),
52
+ getattr(mlrun.common.schemas.AuthorizationResourceTypes, tag_objects.kind),
53
53
  project,
54
54
  resource_name="",
55
55
  # not actually overwriting objects, just overwriting the objects tags
56
- action=mlrun.api.schemas.AuthorizationAction.update,
56
+ action=mlrun.common.schemas.AuthorizationAction.update,
57
57
  auth_info=auth_info,
58
58
  )
59
59
 
@@ -64,15 +64,15 @@ async def overwrite_object_tags_with_tag(
64
64
  tag,
65
65
  tag_objects,
66
66
  )
67
- return mlrun.api.schemas.Tag(name=tag, project=project)
67
+ return mlrun.common.schemas.Tag(name=tag, project=project)
68
68
 
69
69
 
70
- @router.put("/projects/{project}/tags/{tag}", response_model=mlrun.api.schemas.Tag)
70
+ @router.put("/{tag}", response_model=mlrun.common.schemas.Tag)
71
71
  async def append_tag_to_objects(
72
72
  project: str,
73
73
  tag: str = fastapi.Path(..., regex=tag_name_regex_as_string()),
74
- tag_objects: mlrun.api.schemas.TagObjects = fastapi.Body(...),
75
- auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
74
+ tag_objects: mlrun.common.schemas.TagObjects = fastapi.Body(...),
75
+ auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
76
76
  mlrun.api.api.deps.authenticate_request
77
77
  ),
78
78
  db_session: sqlalchemy.orm.Session = fastapi.Depends(
@@ -87,10 +87,10 @@ async def append_tag_to_objects(
87
87
  )
88
88
 
89
89
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
90
- getattr(mlrun.api.schemas.AuthorizationResourceTypes, tag_objects.kind),
90
+ getattr(mlrun.common.schemas.AuthorizationResourceTypes, tag_objects.kind),
91
91
  project,
92
92
  resource_name="",
93
- action=mlrun.api.schemas.AuthorizationAction.update,
93
+ action=mlrun.common.schemas.AuthorizationAction.update,
94
94
  auth_info=auth_info,
95
95
  )
96
96
 
@@ -101,17 +101,15 @@ async def append_tag_to_objects(
101
101
  tag,
102
102
  tag_objects,
103
103
  )
104
- return mlrun.api.schemas.Tag(name=tag, project=project)
104
+ return mlrun.common.schemas.Tag(name=tag, project=project)
105
105
 
106
106
 
107
- @router.delete(
108
- "/projects/{project}/tags/{tag}", status_code=http.HTTPStatus.NO_CONTENT.value
109
- )
107
+ @router.delete("/{tag}", status_code=http.HTTPStatus.NO_CONTENT.value)
110
108
  async def delete_tag_from_objects(
111
109
  project: str,
112
110
  tag: str,
113
- tag_objects: mlrun.api.schemas.TagObjects,
114
- auth_info: mlrun.api.schemas.AuthInfo = fastapi.Depends(
111
+ tag_objects: mlrun.common.schemas.TagObjects,
112
+ auth_info: mlrun.common.schemas.AuthInfo = fastapi.Depends(
115
113
  mlrun.api.api.deps.authenticate_request
116
114
  ),
117
115
  db_session: sqlalchemy.orm.Session = fastapi.Depends(
@@ -126,11 +124,11 @@ async def delete_tag_from_objects(
126
124
  )
127
125
 
128
126
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
129
- getattr(mlrun.api.schemas.AuthorizationResourceTypes, tag_objects.kind),
127
+ getattr(mlrun.common.schemas.AuthorizationResourceTypes, tag_objects.kind),
130
128
  project,
131
129
  resource_name="",
132
130
  # not actually deleting objects, just deleting the objects tags
133
- action=mlrun.api.schemas.AuthorizationAction.update,
131
+ action=mlrun.common.schemas.AuthorizationAction.update,
134
132
  auth_info=auth_info,
135
133
  )
136
134