mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of mlrun might be problematic. Click here for more details.

Files changed (444) hide show
  1. mlrun/__init__.py +3 -3
  2. mlrun/__main__.py +79 -37
  3. mlrun/api/__init__.py +1 -1
  4. mlrun/api/api/__init__.py +1 -1
  5. mlrun/api/api/api.py +4 -4
  6. mlrun/api/api/deps.py +10 -21
  7. mlrun/api/api/endpoints/__init__.py +1 -1
  8. mlrun/api/api/endpoints/artifacts.py +64 -36
  9. mlrun/api/api/endpoints/auth.py +4 -4
  10. mlrun/api/api/endpoints/background_tasks.py +11 -11
  11. mlrun/api/api/endpoints/client_spec.py +5 -5
  12. mlrun/api/api/endpoints/clusterization_spec.py +6 -4
  13. mlrun/api/api/endpoints/feature_store.py +124 -115
  14. mlrun/api/api/endpoints/files.py +22 -14
  15. mlrun/api/api/endpoints/frontend_spec.py +28 -21
  16. mlrun/api/api/endpoints/functions.py +142 -87
  17. mlrun/api/api/endpoints/grafana_proxy.py +89 -442
  18. mlrun/api/api/endpoints/healthz.py +20 -7
  19. mlrun/api/api/endpoints/hub.py +320 -0
  20. mlrun/api/api/endpoints/internal/__init__.py +1 -1
  21. mlrun/api/api/endpoints/internal/config.py +1 -1
  22. mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
  23. mlrun/api/api/endpoints/logs.py +11 -11
  24. mlrun/api/api/endpoints/model_endpoints.py +74 -70
  25. mlrun/api/api/endpoints/operations.py +13 -9
  26. mlrun/api/api/endpoints/pipelines.py +93 -88
  27. mlrun/api/api/endpoints/projects.py +35 -35
  28. mlrun/api/api/endpoints/runs.py +69 -27
  29. mlrun/api/api/endpoints/runtime_resources.py +28 -28
  30. mlrun/api/api/endpoints/schedules.py +98 -41
  31. mlrun/api/api/endpoints/secrets.py +37 -32
  32. mlrun/api/api/endpoints/submit.py +12 -12
  33. mlrun/api/api/endpoints/tags.py +20 -22
  34. mlrun/api/api/utils.py +251 -42
  35. mlrun/api/constants.py +1 -1
  36. mlrun/api/crud/__init__.py +18 -15
  37. mlrun/api/crud/artifacts.py +10 -10
  38. mlrun/api/crud/client_spec.py +4 -4
  39. mlrun/api/crud/clusterization_spec.py +3 -3
  40. mlrun/api/crud/feature_store.py +54 -46
  41. mlrun/api/crud/functions.py +3 -3
  42. mlrun/api/crud/hub.py +312 -0
  43. mlrun/api/crud/logs.py +11 -9
  44. mlrun/api/crud/model_monitoring/__init__.py +3 -3
  45. mlrun/api/crud/model_monitoring/grafana.py +435 -0
  46. mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
  47. mlrun/api/crud/notifications.py +149 -0
  48. mlrun/api/crud/pipelines.py +67 -52
  49. mlrun/api/crud/projects.py +51 -23
  50. mlrun/api/crud/runs.py +7 -5
  51. mlrun/api/crud/runtime_resources.py +13 -13
  52. mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
  53. mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
  54. mlrun/api/crud/runtimes/nuclio/function.py +505 -0
  55. mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
  56. mlrun/api/crud/secrets.py +88 -46
  57. mlrun/api/crud/tags.py +5 -5
  58. mlrun/api/db/__init__.py +1 -1
  59. mlrun/api/db/base.py +102 -54
  60. mlrun/api/db/init_db.py +2 -3
  61. mlrun/api/db/session.py +4 -12
  62. mlrun/api/db/sqldb/__init__.py +1 -1
  63. mlrun/api/db/sqldb/db.py +439 -196
  64. mlrun/api/db/sqldb/helpers.py +1 -1
  65. mlrun/api/db/sqldb/models/__init__.py +3 -3
  66. mlrun/api/db/sqldb/models/models_mysql.py +82 -64
  67. mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
  68. mlrun/api/db/sqldb/session.py +27 -20
  69. mlrun/api/initial_data.py +82 -24
  70. mlrun/api/launcher.py +196 -0
  71. mlrun/api/main.py +91 -22
  72. mlrun/api/middlewares.py +6 -5
  73. mlrun/api/migrations_mysql/env.py +1 -1
  74. mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
  75. mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
  76. mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
  77. mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
  78. mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
  79. mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
  80. mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
  81. mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
  82. mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
  83. mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
  84. mlrun/api/migrations_sqlite/env.py +1 -1
  85. mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
  86. mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
  87. mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
  88. mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
  89. mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
  90. mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
  91. mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
  92. mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
  93. mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
  94. mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
  95. mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
  96. mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
  97. mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
  98. mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
  99. mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
  100. mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
  101. mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
  102. mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
  103. mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
  104. mlrun/api/schemas/__init__.py +216 -138
  105. mlrun/api/utils/__init__.py +1 -1
  106. mlrun/api/utils/asyncio.py +1 -1
  107. mlrun/api/utils/auth/__init__.py +1 -1
  108. mlrun/api/utils/auth/providers/__init__.py +1 -1
  109. mlrun/api/utils/auth/providers/base.py +7 -7
  110. mlrun/api/utils/auth/providers/nop.py +6 -7
  111. mlrun/api/utils/auth/providers/opa.py +17 -17
  112. mlrun/api/utils/auth/verifier.py +36 -34
  113. mlrun/api/utils/background_tasks.py +24 -24
  114. mlrun/{builder.py → api/utils/builder.py} +216 -123
  115. mlrun/api/utils/clients/__init__.py +1 -1
  116. mlrun/api/utils/clients/chief.py +19 -4
  117. mlrun/api/utils/clients/iguazio.py +106 -60
  118. mlrun/api/utils/clients/log_collector.py +1 -1
  119. mlrun/api/utils/clients/nuclio.py +23 -23
  120. mlrun/api/utils/clients/protocols/grpc.py +2 -2
  121. mlrun/api/utils/db/__init__.py +1 -1
  122. mlrun/api/utils/db/alembic.py +1 -1
  123. mlrun/api/utils/db/backup.py +1 -1
  124. mlrun/api/utils/db/mysql.py +24 -25
  125. mlrun/api/utils/db/sql_collation.py +1 -1
  126. mlrun/api/utils/db/sqlite_migration.py +2 -2
  127. mlrun/api/utils/events/__init__.py +14 -0
  128. mlrun/api/utils/events/base.py +57 -0
  129. mlrun/api/utils/events/events_factory.py +41 -0
  130. mlrun/api/utils/events/iguazio.py +217 -0
  131. mlrun/api/utils/events/nop.py +55 -0
  132. mlrun/api/utils/helpers.py +16 -13
  133. mlrun/api/utils/memory_reports.py +1 -1
  134. mlrun/api/utils/periodic.py +6 -3
  135. mlrun/api/utils/projects/__init__.py +1 -1
  136. mlrun/api/utils/projects/follower.py +33 -33
  137. mlrun/api/utils/projects/leader.py +36 -34
  138. mlrun/api/utils/projects/member.py +27 -27
  139. mlrun/api/utils/projects/remotes/__init__.py +1 -1
  140. mlrun/api/utils/projects/remotes/follower.py +13 -13
  141. mlrun/api/utils/projects/remotes/leader.py +10 -10
  142. mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
  143. mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
  144. mlrun/api/utils/scheduler.py +140 -51
  145. mlrun/api/utils/singletons/__init__.py +1 -1
  146. mlrun/api/utils/singletons/db.py +9 -15
  147. mlrun/api/utils/singletons/k8s.py +677 -5
  148. mlrun/api/utils/singletons/logs_dir.py +1 -1
  149. mlrun/api/utils/singletons/project_member.py +1 -1
  150. mlrun/api/utils/singletons/scheduler.py +1 -1
  151. mlrun/artifacts/__init__.py +2 -2
  152. mlrun/artifacts/base.py +8 -2
  153. mlrun/artifacts/dataset.py +5 -3
  154. mlrun/artifacts/manager.py +7 -1
  155. mlrun/artifacts/model.py +15 -4
  156. mlrun/artifacts/plots.py +1 -1
  157. mlrun/common/__init__.py +1 -1
  158. mlrun/common/constants.py +15 -0
  159. mlrun/common/model_monitoring.py +209 -0
  160. mlrun/common/schemas/__init__.py +167 -0
  161. mlrun/{api → common}/schemas/artifact.py +13 -14
  162. mlrun/{api → common}/schemas/auth.py +10 -8
  163. mlrun/{api → common}/schemas/background_task.py +3 -3
  164. mlrun/{api → common}/schemas/client_spec.py +1 -1
  165. mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
  166. mlrun/{api → common}/schemas/constants.py +21 -8
  167. mlrun/common/schemas/events.py +36 -0
  168. mlrun/{api → common}/schemas/feature_store.py +2 -1
  169. mlrun/{api → common}/schemas/frontend_spec.py +7 -6
  170. mlrun/{api → common}/schemas/function.py +5 -5
  171. mlrun/{api → common}/schemas/http.py +3 -3
  172. mlrun/common/schemas/hub.py +134 -0
  173. mlrun/{api → common}/schemas/k8s.py +3 -3
  174. mlrun/{api → common}/schemas/memory_reports.py +1 -1
  175. mlrun/common/schemas/model_endpoints.py +342 -0
  176. mlrun/common/schemas/notification.py +57 -0
  177. mlrun/{api → common}/schemas/object.py +6 -6
  178. mlrun/{api → common}/schemas/pipeline.py +3 -3
  179. mlrun/{api → common}/schemas/project.py +6 -5
  180. mlrun/common/schemas/regex.py +24 -0
  181. mlrun/common/schemas/runs.py +30 -0
  182. mlrun/{api → common}/schemas/runtime_resource.py +3 -3
  183. mlrun/{api → common}/schemas/schedule.py +19 -7
  184. mlrun/{api → common}/schemas/secret.py +3 -3
  185. mlrun/{api → common}/schemas/tag.py +2 -2
  186. mlrun/common/types.py +25 -0
  187. mlrun/config.py +152 -20
  188. mlrun/data_types/__init__.py +7 -2
  189. mlrun/data_types/data_types.py +4 -2
  190. mlrun/data_types/infer.py +1 -1
  191. mlrun/data_types/spark.py +10 -3
  192. mlrun/datastore/__init__.py +10 -3
  193. mlrun/datastore/azure_blob.py +1 -1
  194. mlrun/datastore/base.py +185 -53
  195. mlrun/datastore/datastore.py +1 -1
  196. mlrun/datastore/filestore.py +1 -1
  197. mlrun/datastore/google_cloud_storage.py +1 -1
  198. mlrun/datastore/inmem.py +4 -1
  199. mlrun/datastore/redis.py +1 -1
  200. mlrun/datastore/s3.py +1 -1
  201. mlrun/datastore/sources.py +192 -70
  202. mlrun/datastore/spark_udf.py +44 -0
  203. mlrun/datastore/store_resources.py +4 -4
  204. mlrun/datastore/targets.py +115 -45
  205. mlrun/datastore/utils.py +127 -5
  206. mlrun/datastore/v3io.py +1 -1
  207. mlrun/datastore/wasbfs/__init__.py +1 -1
  208. mlrun/datastore/wasbfs/fs.py +1 -1
  209. mlrun/db/__init__.py +7 -5
  210. mlrun/db/base.py +112 -68
  211. mlrun/db/httpdb.py +445 -277
  212. mlrun/db/nopdb.py +491 -0
  213. mlrun/db/sqldb.py +112 -65
  214. mlrun/errors.py +6 -1
  215. mlrun/execution.py +44 -22
  216. mlrun/feature_store/__init__.py +1 -1
  217. mlrun/feature_store/api.py +143 -95
  218. mlrun/feature_store/common.py +16 -20
  219. mlrun/feature_store/feature_set.py +42 -12
  220. mlrun/feature_store/feature_vector.py +32 -21
  221. mlrun/feature_store/ingestion.py +9 -12
  222. mlrun/feature_store/retrieval/__init__.py +3 -2
  223. mlrun/feature_store/retrieval/base.py +388 -66
  224. mlrun/feature_store/retrieval/dask_merger.py +63 -151
  225. mlrun/feature_store/retrieval/job.py +30 -12
  226. mlrun/feature_store/retrieval/local_merger.py +40 -133
  227. mlrun/feature_store/retrieval/spark_merger.py +129 -127
  228. mlrun/feature_store/retrieval/storey_merger.py +173 -0
  229. mlrun/feature_store/steps.py +132 -15
  230. mlrun/features.py +8 -3
  231. mlrun/frameworks/__init__.py +1 -1
  232. mlrun/frameworks/_common/__init__.py +1 -1
  233. mlrun/frameworks/_common/artifacts_library.py +1 -1
  234. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  235. mlrun/frameworks/_common/model_handler.py +1 -1
  236. mlrun/frameworks/_common/plan.py +1 -1
  237. mlrun/frameworks/_common/producer.py +1 -1
  238. mlrun/frameworks/_common/utils.py +1 -1
  239. mlrun/frameworks/_dl_common/__init__.py +1 -1
  240. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
  241. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  242. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  243. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  244. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  245. mlrun/frameworks/_dl_common/utils.py +1 -1
  246. mlrun/frameworks/_ml_common/__init__.py +1 -1
  247. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  248. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
  249. mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
  250. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  251. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  252. mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
  253. mlrun/frameworks/_ml_common/plan.py +1 -1
  254. mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
  255. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
  256. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
  257. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  258. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  259. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  260. mlrun/frameworks/_ml_common/producer.py +1 -1
  261. mlrun/frameworks/_ml_common/utils.py +1 -1
  262. mlrun/frameworks/auto_mlrun/__init__.py +1 -1
  263. mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
  264. mlrun/frameworks/huggingface/__init__.py +1 -1
  265. mlrun/frameworks/huggingface/model_server.py +1 -1
  266. mlrun/frameworks/lgbm/__init__.py +1 -1
  267. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
  268. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  269. mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
  270. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
  271. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
  272. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  273. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  274. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  275. mlrun/frameworks/lgbm/model_handler.py +1 -1
  276. mlrun/frameworks/lgbm/model_server.py +1 -1
  277. mlrun/frameworks/lgbm/utils.py +1 -1
  278. mlrun/frameworks/onnx/__init__.py +1 -1
  279. mlrun/frameworks/onnx/dataset.py +1 -1
  280. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  281. mlrun/frameworks/onnx/model_handler.py +1 -1
  282. mlrun/frameworks/onnx/model_server.py +1 -1
  283. mlrun/frameworks/parallel_coordinates.py +1 -1
  284. mlrun/frameworks/pytorch/__init__.py +1 -1
  285. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
  286. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  287. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  288. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  289. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  290. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  291. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  292. mlrun/frameworks/pytorch/model_handler.py +1 -1
  293. mlrun/frameworks/pytorch/model_server.py +1 -1
  294. mlrun/frameworks/pytorch/utils.py +1 -1
  295. mlrun/frameworks/sklearn/__init__.py +1 -1
  296. mlrun/frameworks/sklearn/estimator.py +1 -1
  297. mlrun/frameworks/sklearn/metric.py +1 -1
  298. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  299. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  300. mlrun/frameworks/sklearn/model_handler.py +1 -1
  301. mlrun/frameworks/sklearn/utils.py +1 -1
  302. mlrun/frameworks/tf_keras/__init__.py +1 -1
  303. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
  304. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  305. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  306. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  307. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  308. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  309. mlrun/frameworks/tf_keras/model_server.py +1 -1
  310. mlrun/frameworks/tf_keras/utils.py +1 -1
  311. mlrun/frameworks/xgboost/__init__.py +1 -1
  312. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  313. mlrun/frameworks/xgboost/model_handler.py +1 -1
  314. mlrun/frameworks/xgboost/utils.py +1 -1
  315. mlrun/k8s_utils.py +14 -765
  316. mlrun/kfpops.py +14 -17
  317. mlrun/launcher/__init__.py +13 -0
  318. mlrun/launcher/base.py +406 -0
  319. mlrun/launcher/client.py +159 -0
  320. mlrun/launcher/factory.py +50 -0
  321. mlrun/launcher/local.py +276 -0
  322. mlrun/launcher/remote.py +178 -0
  323. mlrun/lists.py +10 -2
  324. mlrun/mlutils/__init__.py +1 -1
  325. mlrun/mlutils/data.py +1 -1
  326. mlrun/mlutils/models.py +1 -1
  327. mlrun/mlutils/plots.py +1 -1
  328. mlrun/model.py +252 -14
  329. mlrun/model_monitoring/__init__.py +41 -0
  330. mlrun/model_monitoring/features_drift_table.py +1 -1
  331. mlrun/model_monitoring/helpers.py +123 -38
  332. mlrun/model_monitoring/model_endpoint.py +144 -0
  333. mlrun/model_monitoring/model_monitoring_batch.py +310 -259
  334. mlrun/model_monitoring/stores/__init__.py +106 -0
  335. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
  336. mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
  337. mlrun/model_monitoring/stores/models/__init__.py +23 -0
  338. mlrun/model_monitoring/stores/models/base.py +18 -0
  339. mlrun/model_monitoring/stores/models/mysql.py +100 -0
  340. mlrun/model_monitoring/stores/models/sqlite.py +98 -0
  341. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
  342. mlrun/model_monitoring/stream_processing_fs.py +239 -271
  343. mlrun/package/__init__.py +163 -0
  344. mlrun/package/context_handler.py +325 -0
  345. mlrun/package/errors.py +47 -0
  346. mlrun/package/packager.py +298 -0
  347. mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
  348. mlrun/package/packagers/default_packager.py +422 -0
  349. mlrun/package/packagers/numpy_packagers.py +612 -0
  350. mlrun/package/packagers/pandas_packagers.py +968 -0
  351. mlrun/package/packagers/python_standard_library_packagers.py +616 -0
  352. mlrun/package/packagers_manager.py +786 -0
  353. mlrun/package/utils/__init__.py +53 -0
  354. mlrun/package/utils/_archiver.py +226 -0
  355. mlrun/package/utils/_formatter.py +211 -0
  356. mlrun/package/utils/_pickler.py +234 -0
  357. mlrun/package/utils/_supported_format.py +71 -0
  358. mlrun/package/utils/log_hint_utils.py +93 -0
  359. mlrun/package/utils/type_hint_utils.py +298 -0
  360. mlrun/platforms/__init__.py +1 -1
  361. mlrun/platforms/iguazio.py +34 -2
  362. mlrun/platforms/other.py +1 -1
  363. mlrun/projects/__init__.py +1 -1
  364. mlrun/projects/operations.py +14 -9
  365. mlrun/projects/pipelines.py +31 -13
  366. mlrun/projects/project.py +762 -238
  367. mlrun/render.py +49 -19
  368. mlrun/run.py +57 -326
  369. mlrun/runtimes/__init__.py +3 -9
  370. mlrun/runtimes/base.py +247 -784
  371. mlrun/runtimes/constants.py +1 -1
  372. mlrun/runtimes/daskjob.py +45 -41
  373. mlrun/runtimes/funcdoc.py +43 -7
  374. mlrun/runtimes/function.py +66 -656
  375. mlrun/runtimes/function_reference.py +1 -1
  376. mlrun/runtimes/generators.py +1 -1
  377. mlrun/runtimes/kubejob.py +99 -116
  378. mlrun/runtimes/local.py +59 -66
  379. mlrun/runtimes/mpijob/__init__.py +1 -1
  380. mlrun/runtimes/mpijob/abstract.py +13 -15
  381. mlrun/runtimes/mpijob/v1.py +3 -1
  382. mlrun/runtimes/mpijob/v1alpha1.py +1 -1
  383. mlrun/runtimes/nuclio.py +1 -1
  384. mlrun/runtimes/pod.py +51 -26
  385. mlrun/runtimes/remotesparkjob.py +3 -1
  386. mlrun/runtimes/serving.py +12 -4
  387. mlrun/runtimes/sparkjob/__init__.py +1 -2
  388. mlrun/runtimes/sparkjob/abstract.py +44 -31
  389. mlrun/runtimes/sparkjob/spark3job.py +11 -9
  390. mlrun/runtimes/utils.py +61 -42
  391. mlrun/secrets.py +16 -18
  392. mlrun/serving/__init__.py +3 -2
  393. mlrun/serving/merger.py +1 -1
  394. mlrun/serving/remote.py +1 -1
  395. mlrun/serving/routers.py +39 -42
  396. mlrun/serving/server.py +23 -13
  397. mlrun/serving/serving_wrapper.py +1 -1
  398. mlrun/serving/states.py +172 -39
  399. mlrun/serving/utils.py +1 -1
  400. mlrun/serving/v1_serving.py +1 -1
  401. mlrun/serving/v2_serving.py +29 -21
  402. mlrun/utils/__init__.py +1 -2
  403. mlrun/utils/async_http.py +8 -1
  404. mlrun/utils/azure_vault.py +1 -1
  405. mlrun/utils/clones.py +2 -2
  406. mlrun/utils/condition_evaluator.py +65 -0
  407. mlrun/utils/db.py +52 -0
  408. mlrun/utils/helpers.py +188 -13
  409. mlrun/utils/http.py +89 -54
  410. mlrun/utils/logger.py +48 -8
  411. mlrun/utils/model_monitoring.py +132 -100
  412. mlrun/utils/notifications/__init__.py +1 -1
  413. mlrun/utils/notifications/notification/__init__.py +8 -6
  414. mlrun/utils/notifications/notification/base.py +20 -14
  415. mlrun/utils/notifications/notification/console.py +7 -4
  416. mlrun/utils/notifications/notification/git.py +36 -19
  417. mlrun/utils/notifications/notification/ipython.py +10 -8
  418. mlrun/utils/notifications/notification/slack.py +18 -13
  419. mlrun/utils/notifications/notification_pusher.py +377 -56
  420. mlrun/utils/regex.py +6 -1
  421. mlrun/utils/singleton.py +1 -1
  422. mlrun/utils/v3io_clients.py +1 -1
  423. mlrun/utils/vault.py +270 -269
  424. mlrun/utils/version/__init__.py +1 -1
  425. mlrun/utils/version/version.json +2 -2
  426. mlrun/utils/version/version.py +1 -1
  427. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
  428. mlrun-1.4.0.dist-info/RECORD +434 -0
  429. mlrun/api/api/endpoints/marketplace.py +0 -257
  430. mlrun/api/crud/marketplace.py +0 -221
  431. mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
  432. mlrun/api/db/filedb/db.py +0 -518
  433. mlrun/api/schemas/marketplace.py +0 -128
  434. mlrun/api/schemas/model_endpoints.py +0 -185
  435. mlrun/db/filedb.py +0 -891
  436. mlrun/feature_store/retrieval/online.py +0 -92
  437. mlrun/model_monitoring/constants.py +0 -67
  438. mlrun/runtimes/package/context_handler.py +0 -711
  439. mlrun/runtimes/sparkjob/spark2job.py +0 -59
  440. mlrun-1.3.3rc1.dist-info/RECORD +0 -381
  441. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
  442. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
  443. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
  444. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
mlrun/api/utils/projects/follower.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@ import sqlalchemy.orm
23
23
 
24
24
  import mlrun.api.crud
25
25
  import mlrun.api.db.session
26
- import mlrun.api.schemas
27
26
  import mlrun.api.utils.auth.verifier
28
27
  import mlrun.api.utils.clients.iguazio
29
28
  import mlrun.api.utils.clients.nuclio
@@ -31,6 +30,7 @@ import mlrun.api.utils.periodic
31
30
  import mlrun.api.utils.projects.member
32
31
  import mlrun.api.utils.projects.remotes.leader
33
32
  import mlrun.api.utils.projects.remotes.nop_leader
33
+ import mlrun.common.schemas
34
34
  import mlrun.config
35
35
  import mlrun.errors
36
36
  import mlrun.utils
@@ -78,7 +78,7 @@ class Member(
78
78
  # we're doing a full_sync on every initialization
79
79
  full_sync = (
80
80
  mlrun.mlconf.httpdb.clusterization.role
81
- == mlrun.api.schemas.ClusterizationRole.chief
81
+ == mlrun.common.schemas.ClusterizationRole.chief
82
82
  )
83
83
  self._sync_projects(full_sync=full_sync)
84
84
  except Exception as exc:
@@ -96,12 +96,12 @@ class Member(
96
96
  def create_project(
97
97
  self,
98
98
  db_session: sqlalchemy.orm.Session,
99
- project: mlrun.api.schemas.Project,
100
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
99
+ project: mlrun.common.schemas.Project,
100
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
101
101
  leader_session: typing.Optional[str] = None,
102
102
  wait_for_completion: bool = True,
103
103
  commit_before_get: bool = False,
104
- ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
104
+ ) -> typing.Tuple[typing.Optional[mlrun.common.schemas.Project], bool]:
105
105
  if self._is_request_from_leader(projects_role):
106
106
  mlrun.api.crud.Projects().create_project(db_session, project)
107
107
  return project, False
@@ -137,11 +137,11 @@ class Member(
137
137
  self,
138
138
  db_session: sqlalchemy.orm.Session,
139
139
  name: str,
140
- project: mlrun.api.schemas.Project,
141
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
140
+ project: mlrun.common.schemas.Project,
141
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
142
142
  leader_session: typing.Optional[str] = None,
143
143
  wait_for_completion: bool = True,
144
- ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
144
+ ) -> typing.Tuple[typing.Optional[mlrun.common.schemas.Project], bool]:
145
145
  if self._is_request_from_leader(projects_role):
146
146
  mlrun.api.crud.Projects().store_project(db_session, name, project)
147
147
  return project, False
@@ -166,11 +166,11 @@ class Member(
166
166
  db_session: sqlalchemy.orm.Session,
167
167
  name: str,
168
168
  project: dict,
169
- patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
170
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
169
+ patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
170
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
171
171
  leader_session: typing.Optional[str] = None,
172
172
  wait_for_completion: bool = True,
173
- ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
173
+ ) -> typing.Tuple[typing.Optional[mlrun.common.schemas.Project], bool]:
174
174
  if self._is_request_from_leader(projects_role):
175
175
  # No real scenario for this to be useful currently - in iguazio patch is transformed to store request
176
176
  raise NotImplementedError("Patch operation not supported from leader")
@@ -179,7 +179,7 @@ class Member(
179
179
  strategy = patch_mode.to_mergedeep_strategy()
180
180
  current_project_dict = current_project.dict(exclude_unset=True)
181
181
  mergedeep.merge(current_project_dict, project, strategy=strategy)
182
- patched_project = mlrun.api.schemas.Project(**current_project_dict)
182
+ patched_project = mlrun.common.schemas.Project(**current_project_dict)
183
183
  return self.store_project(
184
184
  db_session,
185
185
  name,
@@ -193,9 +193,9 @@ class Member(
193
193
  self,
194
194
  db_session: sqlalchemy.orm.Session,
195
195
  name: str,
196
- deletion_strategy: mlrun.api.schemas.DeletionStrategy = mlrun.api.schemas.DeletionStrategy.default(),
197
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
198
- auth_info: mlrun.api.schemas.AuthInfo = mlrun.api.schemas.AuthInfo(),
196
+ deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
197
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
198
+ auth_info: mlrun.common.schemas.AuthInfo = mlrun.common.schemas.AuthInfo(),
199
199
  wait_for_completion: bool = True,
200
200
  ) -> bool:
201
201
  if self._is_request_from_leader(projects_role):
@@ -216,30 +216,30 @@ class Member(
216
216
  db_session: sqlalchemy.orm.Session,
217
217
  name: str,
218
218
  leader_session: typing.Optional[str] = None,
219
- ) -> mlrun.api.schemas.Project:
219
+ ) -> mlrun.common.schemas.Project:
220
220
  return mlrun.api.crud.Projects().get_project(db_session, name)
221
221
 
222
222
  def get_project_owner(
223
223
  self,
224
224
  db_session: sqlalchemy.orm.Session,
225
225
  name: str,
226
- ) -> mlrun.api.schemas.ProjectOwner:
226
+ ) -> mlrun.common.schemas.ProjectOwner:
227
227
  return self._leader_client.get_project_owner(self._sync_session, name)
228
228
 
229
229
  def list_projects(
230
230
  self,
231
231
  db_session: sqlalchemy.orm.Session,
232
232
  owner: str = None,
233
- format_: mlrun.api.schemas.ProjectsFormat = mlrun.api.schemas.ProjectsFormat.full,
233
+ format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.full,
234
234
  labels: typing.List[str] = None,
235
- state: mlrun.api.schemas.ProjectState = None,
235
+ state: mlrun.common.schemas.ProjectState = None,
236
236
  # needed only for external usage when requesting leader format
237
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
237
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
238
238
  leader_session: typing.Optional[str] = None,
239
239
  names: typing.Optional[typing.List[str]] = None,
240
- ) -> mlrun.api.schemas.ProjectsOutput:
240
+ ) -> mlrun.common.schemas.ProjectsOutput:
241
241
  if (
242
- format_ == mlrun.api.schemas.ProjectsFormat.leader
242
+ format_ == mlrun.common.schemas.ProjectsFormat.leader
243
243
  and not self._is_request_from_leader(projects_role)
244
244
  ):
245
245
  raise mlrun.errors.MLRunAccessDeniedError(
@@ -249,7 +249,7 @@ class Member(
249
249
  projects_output = mlrun.api.crud.Projects().list_projects(
250
250
  db_session, owner, format_, labels, state, names
251
251
  )
252
- if format_ == mlrun.api.schemas.ProjectsFormat.leader:
252
+ if format_ == mlrun.common.schemas.ProjectsFormat.leader:
253
253
  leader_projects = [
254
254
  self._leader_client.format_as_leader_project(project)
255
255
  for project in projects_output.projects
@@ -262,11 +262,11 @@ class Member(
262
262
  db_session: sqlalchemy.orm.Session,
263
263
  owner: str = None,
264
264
  labels: typing.List[str] = None,
265
- state: mlrun.api.schemas.ProjectState = None,
266
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
265
+ state: mlrun.common.schemas.ProjectState = None,
266
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
267
267
  leader_session: typing.Optional[str] = None,
268
268
  names: typing.Optional[typing.List[str]] = None,
269
- ) -> mlrun.api.schemas.ProjectSummariesOutput:
269
+ ) -> mlrun.common.schemas.ProjectSummariesOutput:
270
270
  return await mlrun.api.crud.Projects().list_project_summaries(
271
271
  db_session, owner, labels, state, names
272
272
  )
@@ -276,7 +276,7 @@ class Member(
276
276
  db_session: sqlalchemy.orm.Session,
277
277
  name: str,
278
278
  leader_session: typing.Optional[str] = None,
279
- ) -> mlrun.api.schemas.ProjectSummary:
279
+ ) -> mlrun.common.schemas.ProjectSummary:
280
280
  return await mlrun.api.crud.Projects().get_project_summary(db_session, name)
281
281
 
282
282
  def _start_periodic_sync(self):
@@ -316,14 +316,14 @@ class Member(
316
316
  db_session = mlrun.api.db.session.create_session()
317
317
  try:
318
318
  db_projects = mlrun.api.crud.Projects().list_projects(
319
- db_session, format_=mlrun.api.schemas.ProjectsFormat.name_only
319
+ db_session, format_=mlrun.common.schemas.ProjectsFormat.name_only
320
320
  )
321
321
  # Don't add projects in non terminal state if they didn't exist before to prevent race conditions
322
322
  filtered_projects = []
323
323
  for leader_project in leader_projects:
324
324
  if (
325
325
  leader_project.status.state
326
- not in mlrun.api.schemas.ProjectState.terminal_states()
326
+ not in mlrun.common.schemas.ProjectState.terminal_states()
327
327
  and leader_project.metadata.name not in db_projects.projects
328
328
  ):
329
329
  continue
@@ -349,7 +349,7 @@ class Member(
349
349
  mlrun.api.crud.Projects().delete_project(
350
350
  db_session,
351
351
  project_to_remove,
352
- mlrun.api.schemas.DeletionStrategy.cascading,
352
+ mlrun.common.schemas.DeletionStrategy.cascading,
353
353
  )
354
354
  if latest_updated_at:
355
355
 
@@ -363,7 +363,7 @@ class Member(
363
363
  mlrun.api.db.session.close_session(db_session)
364
364
 
365
365
  def _is_request_from_leader(
366
- self, projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole]
366
+ self, projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole]
367
367
  ) -> bool:
368
368
  if projects_role and projects_role.value == self._leader_name:
369
369
  return True
@@ -371,7 +371,7 @@ class Member(
371
371
 
372
372
  @staticmethod
373
373
  def _is_project_matching_labels(
374
- labels: typing.List[str], project: mlrun.api.schemas.Project
374
+ labels: typing.List[str], project: mlrun.common.schemas.Project
375
375
  ):
376
376
  if not project.metadata.labels:
377
377
  return False
mlrun/api/utils/projects/leader.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -20,12 +20,12 @@ import humanfriendly
20
20
  import sqlalchemy.orm
21
21
 
22
22
  import mlrun.api.db.session
23
- import mlrun.api.schemas
24
23
  import mlrun.api.utils.clients.nuclio
25
24
  import mlrun.api.utils.periodic
26
25
  import mlrun.api.utils.projects.member
27
26
  import mlrun.api.utils.projects.remotes.follower
28
27
  import mlrun.api.utils.projects.remotes.nop_follower
28
+ import mlrun.common.schemas
29
29
  import mlrun.config
30
30
  import mlrun.errors
31
31
  import mlrun.utils
@@ -58,12 +58,12 @@ class Member(
58
58
  def create_project(
59
59
  self,
60
60
  db_session: sqlalchemy.orm.Session,
61
- project: mlrun.api.schemas.Project,
62
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
61
+ project: mlrun.common.schemas.Project,
62
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
63
63
  leader_session: typing.Optional[str] = None,
64
64
  wait_for_completion: bool = True,
65
65
  commit_before_get: bool = False,
66
- ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
66
+ ) -> typing.Tuple[typing.Optional[mlrun.common.schemas.Project], bool]:
67
67
  self._enrich_and_validate_before_creation(project)
68
68
  self._run_on_all_followers(True, "create_project", db_session, project)
69
69
  return self.get_project(db_session, project.metadata.name), False
@@ -72,11 +72,11 @@ class Member(
72
72
  self,
73
73
  db_session: sqlalchemy.orm.Session,
74
74
  name: str,
75
- project: mlrun.api.schemas.Project,
76
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
75
+ project: mlrun.common.schemas.Project,
76
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
77
77
  leader_session: typing.Optional[str] = None,
78
78
  wait_for_completion: bool = True,
79
- ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
79
+ ) -> typing.Tuple[typing.Optional[mlrun.common.schemas.Project], bool]:
80
80
  self._enrich_project(project)
81
81
  mlrun.projects.ProjectMetadata.validate_project_name(name)
82
82
  self._validate_body_and_path_names_matches(name, project)
@@ -88,11 +88,11 @@ class Member(
88
88
  db_session: sqlalchemy.orm.Session,
89
89
  name: str,
90
90
  project: dict,
91
- patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
92
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
91
+ patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
92
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
93
93
  leader_session: typing.Optional[str] = None,
94
94
  wait_for_completion: bool = True,
95
- ) -> typing.Tuple[mlrun.api.schemas.Project, bool]:
95
+ ) -> typing.Tuple[mlrun.common.schemas.Project, bool]:
96
96
  self._enrich_project_patch(project)
97
97
  self._validate_body_and_path_names_matches(name, project)
98
98
  self._run_on_all_followers(
@@ -104,9 +104,9 @@ class Member(
104
104
  self,
105
105
  db_session: sqlalchemy.orm.Session,
106
106
  name: str,
107
- deletion_strategy: mlrun.api.schemas.DeletionStrategy = mlrun.api.schemas.DeletionStrategy.default(),
108
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
109
- auth_info: mlrun.api.schemas.AuthInfo = mlrun.api.schemas.AuthInfo(),
107
+ deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
108
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
109
+ auth_info: mlrun.common.schemas.AuthInfo = mlrun.common.schemas.AuthInfo(),
110
110
  wait_for_completion: bool = True,
111
111
  ) -> bool:
112
112
  self._projects_in_deletion.add(name)
@@ -123,20 +123,20 @@ class Member(
123
123
  db_session: sqlalchemy.orm.Session,
124
124
  name: str,
125
125
  leader_session: typing.Optional[str] = None,
126
- ) -> mlrun.api.schemas.Project:
126
+ ) -> mlrun.common.schemas.Project:
127
127
  return self._leader_follower.get_project(db_session, name)
128
128
 
129
129
  def list_projects(
130
130
  self,
131
131
  db_session: sqlalchemy.orm.Session,
132
132
  owner: str = None,
133
- format_: mlrun.api.schemas.ProjectsFormat = mlrun.api.schemas.ProjectsFormat.full,
133
+ format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.full,
134
134
  labels: typing.List[str] = None,
135
- state: mlrun.api.schemas.ProjectState = None,
136
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
135
+ state: mlrun.common.schemas.ProjectState = None,
136
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
137
137
  leader_session: typing.Optional[str] = None,
138
138
  names: typing.Optional[typing.List[str]] = None,
139
- ) -> mlrun.api.schemas.ProjectsOutput:
139
+ ) -> mlrun.common.schemas.ProjectsOutput:
140
140
  return self._leader_follower.list_projects(
141
141
  db_session, owner, format_, labels, state, names
142
142
  )
@@ -146,11 +146,11 @@ class Member(
146
146
  db_session: sqlalchemy.orm.Session,
147
147
  owner: str = None,
148
148
  labels: typing.List[str] = None,
149
- state: mlrun.api.schemas.ProjectState = None,
150
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
149
+ state: mlrun.common.schemas.ProjectState = None,
150
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
151
151
  leader_session: typing.Optional[str] = None,
152
152
  names: typing.Optional[typing.List[str]] = None,
153
- ) -> mlrun.api.schemas.ProjectSummariesOutput:
153
+ ) -> mlrun.common.schemas.ProjectSummariesOutput:
154
154
  return await self._leader_follower.list_project_summaries(
155
155
  db_session, owner, labels, state, names
156
156
  )
@@ -160,14 +160,14 @@ class Member(
160
160
  db_session: sqlalchemy.orm.Session,
161
161
  name: str,
162
162
  leader_session: typing.Optional[str] = None,
163
- ) -> mlrun.api.schemas.ProjectSummary:
163
+ ) -> mlrun.common.schemas.ProjectSummary:
164
164
  return await self._leader_follower.get_project_summary(db_session, name)
165
165
 
166
166
  def get_project_owner(
167
167
  self,
168
168
  db_session: sqlalchemy.orm.Session,
169
169
  name: str,
170
- ) -> mlrun.api.schemas.ProjectOwner:
170
+ ) -> mlrun.common.schemas.ProjectOwner:
171
171
  raise NotImplementedError()
172
172
 
173
173
  def _start_periodic_sync(self):
@@ -192,8 +192,8 @@ class Member(
192
192
  db_session = mlrun.api.db.session.create_session()
193
193
  try:
194
194
  # re-generating all of the maps every time since _ensure_follower_projects_synced might cause changes
195
- leader_projects: mlrun.api.schemas.ProjectsOutput
196
- follower_projects_map: typing.Dict[str, mlrun.api.schemas.ProjectsOutput]
195
+ leader_projects: mlrun.common.schemas.ProjectsOutput
196
+ follower_projects_map: typing.Dict[str, mlrun.common.schemas.ProjectsOutput]
197
197
  leader_projects, follower_projects_map = self._run_on_all_followers(
198
198
  True, "list_projects", db_session
199
199
  )
@@ -245,9 +245,9 @@ class Member(
245
245
  follower_names: typing.Set[str],
246
246
  project_name: str,
247
247
  followers_projects_map: typing.Dict[
248
- str, typing.Dict[str, mlrun.api.schemas.Project]
248
+ str, typing.Dict[str, mlrun.common.schemas.Project]
249
249
  ],
250
- leader_projects_map: typing.Dict[str, mlrun.api.schemas.Project],
250
+ leader_projects_map: typing.Dict[str, mlrun.common.schemas.Project],
251
251
  ):
252
252
  # FIXME: This function only handles syncing project existence, i.e. if a user updates a project attribute
253
253
  # through one of the followers this change won't be synced and the projects will be left with this discrepancy
@@ -307,7 +307,7 @@ class Member(
307
307
  db_session: sqlalchemy.orm.Session,
308
308
  follower_names: typing.Set[str],
309
309
  project_name: str,
310
- project: mlrun.api.schemas.Project,
310
+ project: mlrun.common.schemas.Project,
311
311
  ):
312
312
  for follower_name in follower_names:
313
313
  logger.debug(
@@ -338,7 +338,7 @@ class Member(
338
338
  # the name of the follower which we took the missing project from
339
339
  project_follower_name: str,
340
340
  project_name: str,
341
- project: mlrun.api.schemas.Project,
341
+ project: mlrun.common.schemas.Project,
342
342
  ):
343
343
  for missing_follower in missing_followers:
344
344
  logger.debug(
@@ -423,12 +423,14 @@ class Member(
423
423
  raise ValueError(f"Unknown follower name: {name}")
424
424
  return followers_classes_map[name]
425
425
 
426
- def _enrich_and_validate_before_creation(self, project: mlrun.api.schemas.Project):
426
+ def _enrich_and_validate_before_creation(
427
+ self, project: mlrun.common.schemas.Project
428
+ ):
427
429
  self._enrich_project(project)
428
430
  mlrun.projects.ProjectMetadata.validate_project_name(project.metadata.name)
429
431
 
430
432
  @staticmethod
431
- def _enrich_project(project: mlrun.api.schemas.Project):
433
+ def _enrich_project(project: mlrun.common.schemas.Project):
432
434
  project.status.state = project.spec.desired_state
433
435
 
434
436
  @staticmethod
@@ -452,9 +454,9 @@ class Member(
452
454
 
453
455
  @staticmethod
454
456
  def _validate_body_and_path_names_matches(
455
- path_name: str, project: typing.Union[mlrun.api.schemas.Project, dict]
457
+ path_name: str, project: typing.Union[mlrun.common.schemas.Project, dict]
456
458
  ):
457
- if isinstance(project, mlrun.api.schemas.Project):
459
+ if isinstance(project, mlrun.common.schemas.Project):
458
460
  body_name = project.metadata.name
459
461
  elif isinstance(project, dict):
460
462
  body_name = project.get("metadata", {}).get("name")
mlrun/api/utils/projects/member.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -19,8 +19,8 @@ import sqlalchemy.orm
19
19
 
20
20
  import mlrun.api.crud
21
21
  import mlrun.api.db.session
22
- import mlrun.api.schemas
23
22
  import mlrun.api.utils.clients.log_collector
23
+ import mlrun.common.schemas
24
24
  import mlrun.utils.singleton
25
25
  from mlrun.utils import logger
26
26
 
@@ -39,11 +39,11 @@ class Member(abc.ABC):
39
39
  db_session: sqlalchemy.orm.Session,
40
40
  name: str,
41
41
  wait_for_completion: bool = True,
42
- auth_info: mlrun.api.schemas.AuthInfo = mlrun.api.schemas.AuthInfo(),
42
+ auth_info: mlrun.common.schemas.AuthInfo = mlrun.common.schemas.AuthInfo(),
43
43
  ):
44
44
  project_names = self.list_projects(
45
45
  db_session,
46
- format_=mlrun.api.schemas.ProjectsFormat.name_only,
46
+ format_=mlrun.common.schemas.ProjectsFormat.name_only,
47
47
  leader_session=auth_info.session,
48
48
  )
49
49
  if name not in project_names.projects:
@@ -53,12 +53,12 @@ class Member(abc.ABC):
53
53
  def create_project(
54
54
  self,
55
55
  db_session: sqlalchemy.orm.Session,
56
- project: mlrun.api.schemas.Project,
57
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
56
+ project: mlrun.common.schemas.Project,
57
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
58
58
  leader_session: typing.Optional[str] = None,
59
59
  wait_for_completion: bool = True,
60
60
  commit_before_get: bool = False,
61
- ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
61
+ ) -> typing.Tuple[typing.Optional[mlrun.common.schemas.Project], bool]:
62
62
  pass
63
63
 
64
64
  @abc.abstractmethod
@@ -66,11 +66,11 @@ class Member(abc.ABC):
66
66
  self,
67
67
  db_session: sqlalchemy.orm.Session,
68
68
  name: str,
69
- project: mlrun.api.schemas.Project,
70
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
69
+ project: mlrun.common.schemas.Project,
70
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
71
71
  leader_session: typing.Optional[str] = None,
72
72
  wait_for_completion: bool = True,
73
- ) -> typing.Tuple[typing.Optional[mlrun.api.schemas.Project], bool]:
73
+ ) -> typing.Tuple[typing.Optional[mlrun.common.schemas.Project], bool]:
74
74
  pass
75
75
 
76
76
  @abc.abstractmethod
@@ -79,11 +79,11 @@ class Member(abc.ABC):
79
79
  db_session: sqlalchemy.orm.Session,
80
80
  name: str,
81
81
  project: dict,
82
- patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
83
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
82
+ patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
83
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
84
84
  leader_session: typing.Optional[str] = None,
85
85
  wait_for_completion: bool = True,
86
- ) -> typing.Tuple[mlrun.api.schemas.Project, bool]:
86
+ ) -> typing.Tuple[mlrun.common.schemas.Project, bool]:
87
87
  pass
88
88
 
89
89
  @abc.abstractmethod
@@ -91,9 +91,9 @@ class Member(abc.ABC):
91
91
  self,
92
92
  db_session: sqlalchemy.orm.Session,
93
93
  name: str,
94
- deletion_strategy: mlrun.api.schemas.DeletionStrategy = mlrun.api.schemas.DeletionStrategy.default(),
95
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
96
- auth_info: mlrun.api.schemas.AuthInfo = mlrun.api.schemas.AuthInfo(),
94
+ deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
95
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
96
+ auth_info: mlrun.common.schemas.AuthInfo = mlrun.common.schemas.AuthInfo(),
97
97
  wait_for_completion: bool = True,
98
98
  ) -> bool:
99
99
  pass
@@ -104,7 +104,7 @@ class Member(abc.ABC):
104
104
  db_session: sqlalchemy.orm.Session,
105
105
  name: str,
106
106
  leader_session: typing.Optional[str] = None,
107
- ) -> mlrun.api.schemas.Project:
107
+ ) -> mlrun.common.schemas.Project:
108
108
  pass
109
109
 
110
110
  @abc.abstractmethod
@@ -112,13 +112,13 @@ class Member(abc.ABC):
112
112
  self,
113
113
  db_session: sqlalchemy.orm.Session,
114
114
  owner: str = None,
115
- format_: mlrun.api.schemas.ProjectsFormat = mlrun.api.schemas.ProjectsFormat.full,
115
+ format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.full,
116
116
  labels: typing.List[str] = None,
117
- state: mlrun.api.schemas.ProjectState = None,
118
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
117
+ state: mlrun.common.schemas.ProjectState = None,
118
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
119
119
  leader_session: typing.Optional[str] = None,
120
120
  names: typing.Optional[typing.List[str]] = None,
121
- ) -> mlrun.api.schemas.ProjectsOutput:
121
+ ) -> mlrun.common.schemas.ProjectsOutput:
122
122
  pass
123
123
 
124
124
  @abc.abstractmethod
@@ -127,7 +127,7 @@ class Member(abc.ABC):
127
127
  db_session: sqlalchemy.orm.Session,
128
128
  name: str,
129
129
  leader_session: typing.Optional[str] = None,
130
- ) -> mlrun.api.schemas.ProjectSummary:
130
+ ) -> mlrun.common.schemas.ProjectSummary:
131
131
  pass
132
132
 
133
133
  @abc.abstractmethod
@@ -136,11 +136,11 @@ class Member(abc.ABC):
136
136
  db_session: sqlalchemy.orm.Session,
137
137
  owner: str = None,
138
138
  labels: typing.List[str] = None,
139
- state: mlrun.api.schemas.ProjectState = None,
140
- projects_role: typing.Optional[mlrun.api.schemas.ProjectsRole] = None,
139
+ state: mlrun.common.schemas.ProjectState = None,
140
+ projects_role: typing.Optional[mlrun.common.schemas.ProjectsRole] = None,
141
141
  leader_session: typing.Optional[str] = None,
142
142
  names: typing.Optional[typing.List[str]] = None,
143
- ) -> mlrun.api.schemas.ProjectSummariesOutput:
143
+ ) -> mlrun.common.schemas.ProjectSummariesOutput:
144
144
  pass
145
145
 
146
146
  @abc.abstractmethod
@@ -148,7 +148,7 @@ class Member(abc.ABC):
148
148
  self,
149
149
  db_session: sqlalchemy.orm.Session,
150
150
  name: str,
151
- ) -> mlrun.api.schemas.ProjectOwner:
151
+ ) -> mlrun.common.schemas.ProjectOwner:
152
152
  pass
153
153
 
154
154
  async def post_delete_project(
@@ -157,7 +157,7 @@ class Member(abc.ABC):
157
157
  ):
158
158
  if (
159
159
  mlrun.mlconf.log_collector.mode
160
- != mlrun.api.schemas.LogsCollectorMode.legacy
160
+ != mlrun.common.schemas.LogsCollectorMode.legacy
161
161
  ):
162
162
  await self._stop_logs_for_project(project_name)
163
163
  await self._delete_project_logs(project_name)
mlrun/api/utils/projects/remotes/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
mlrun/api/utils/projects/remotes/follower.py CHANGED
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -17,13 +17,13 @@ import typing
17
17
 
18
18
  import sqlalchemy.orm
19
19
 
20
- import mlrun.api.schemas
20
+ import mlrun.common.schemas
21
21
 
22
22
 
23
23
  class Member(abc.ABC):
24
24
  @abc.abstractmethod
25
25
  def create_project(
26
- self, session: sqlalchemy.orm.Session, project: mlrun.api.schemas.Project
26
+ self, session: sqlalchemy.orm.Session, project: mlrun.common.schemas.Project
27
27
  ):
28
28
  pass
29
29
 
@@ -32,7 +32,7 @@ class Member(abc.ABC):
32
32
  self,
33
33
  session: sqlalchemy.orm.Session,
34
34
  name: str,
35
- project: mlrun.api.schemas.Project,
35
+ project: mlrun.common.schemas.Project,
36
36
  ):
37
37
  pass
38
38
 
@@ -42,7 +42,7 @@ class Member(abc.ABC):
42
42
  session: sqlalchemy.orm.Session,
43
43
  name: str,
44
44
  project: dict,
45
- patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
45
+ patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
46
46
  ):
47
47
  pass
48
48
 
@@ -51,14 +51,14 @@ class Member(abc.ABC):
51
51
  self,
52
52
  session: sqlalchemy.orm.Session,
53
53
  name: str,
54
- deletion_strategy: mlrun.api.schemas.DeletionStrategy = mlrun.api.schemas.DeletionStrategy.default(),
54
+ deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
55
55
  ):
56
56
  pass
57
57
 
58
58
  @abc.abstractmethod
59
59
  def get_project(
60
60
  self, session: sqlalchemy.orm.Session, name: str
61
- ) -> mlrun.api.schemas.Project:
61
+ ) -> mlrun.common.schemas.Project:
62
62
  pass
63
63
 
64
64
  @abc.abstractmethod
@@ -66,11 +66,11 @@ class Member(abc.ABC):
66
66
  self,
67
67
  session: sqlalchemy.orm.Session,
68
68
  owner: str = None,
69
- format_: mlrun.api.schemas.ProjectsFormat = mlrun.api.schemas.ProjectsFormat.full,
69
+ format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.full,
70
70
  labels: typing.List[str] = None,
71
- state: mlrun.api.schemas.ProjectState = None,
71
+ state: mlrun.common.schemas.ProjectState = None,
72
72
  names: typing.Optional[typing.List[str]] = None,
73
- ) -> mlrun.api.schemas.ProjectsOutput:
73
+ ) -> mlrun.common.schemas.ProjectsOutput:
74
74
  pass
75
75
 
76
76
  @abc.abstractmethod
@@ -79,13 +79,13 @@ class Member(abc.ABC):
79
79
  session: sqlalchemy.orm.Session,
80
80
  owner: str = None,
81
81
  labels: typing.List[str] = None,
82
- state: mlrun.api.schemas.ProjectState = None,
82
+ state: mlrun.common.schemas.ProjectState = None,
83
83
  names: typing.Optional[typing.List[str]] = None,
84
- ) -> mlrun.api.schemas.ProjectSummariesOutput:
84
+ ) -> mlrun.common.schemas.ProjectSummariesOutput:
85
85
  pass
86
86
 
87
87
  @abc.abstractmethod
88
88
  def get_project_summary(
89
89
  self, session: sqlalchemy.orm.Session, name: str
90
- ) -> mlrun.api.schemas.ProjectSummary:
90
+ ) -> mlrun.common.schemas.ProjectSummary:
91
91
  pass