mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (444) hide show
  1. mlrun/__init__.py +3 -3
  2. mlrun/__main__.py +79 -37
  3. mlrun/api/__init__.py +1 -1
  4. mlrun/api/api/__init__.py +1 -1
  5. mlrun/api/api/api.py +4 -4
  6. mlrun/api/api/deps.py +10 -21
  7. mlrun/api/api/endpoints/__init__.py +1 -1
  8. mlrun/api/api/endpoints/artifacts.py +64 -36
  9. mlrun/api/api/endpoints/auth.py +4 -4
  10. mlrun/api/api/endpoints/background_tasks.py +11 -11
  11. mlrun/api/api/endpoints/client_spec.py +5 -5
  12. mlrun/api/api/endpoints/clusterization_spec.py +6 -4
  13. mlrun/api/api/endpoints/feature_store.py +124 -115
  14. mlrun/api/api/endpoints/files.py +22 -14
  15. mlrun/api/api/endpoints/frontend_spec.py +28 -21
  16. mlrun/api/api/endpoints/functions.py +142 -87
  17. mlrun/api/api/endpoints/grafana_proxy.py +89 -442
  18. mlrun/api/api/endpoints/healthz.py +20 -7
  19. mlrun/api/api/endpoints/hub.py +320 -0
  20. mlrun/api/api/endpoints/internal/__init__.py +1 -1
  21. mlrun/api/api/endpoints/internal/config.py +1 -1
  22. mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
  23. mlrun/api/api/endpoints/logs.py +11 -11
  24. mlrun/api/api/endpoints/model_endpoints.py +74 -70
  25. mlrun/api/api/endpoints/operations.py +13 -9
  26. mlrun/api/api/endpoints/pipelines.py +93 -88
  27. mlrun/api/api/endpoints/projects.py +35 -35
  28. mlrun/api/api/endpoints/runs.py +69 -27
  29. mlrun/api/api/endpoints/runtime_resources.py +28 -28
  30. mlrun/api/api/endpoints/schedules.py +98 -41
  31. mlrun/api/api/endpoints/secrets.py +37 -32
  32. mlrun/api/api/endpoints/submit.py +12 -12
  33. mlrun/api/api/endpoints/tags.py +20 -22
  34. mlrun/api/api/utils.py +251 -42
  35. mlrun/api/constants.py +1 -1
  36. mlrun/api/crud/__init__.py +18 -15
  37. mlrun/api/crud/artifacts.py +10 -10
  38. mlrun/api/crud/client_spec.py +4 -4
  39. mlrun/api/crud/clusterization_spec.py +3 -3
  40. mlrun/api/crud/feature_store.py +54 -46
  41. mlrun/api/crud/functions.py +3 -3
  42. mlrun/api/crud/hub.py +312 -0
  43. mlrun/api/crud/logs.py +11 -9
  44. mlrun/api/crud/model_monitoring/__init__.py +3 -3
  45. mlrun/api/crud/model_monitoring/grafana.py +435 -0
  46. mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
  47. mlrun/api/crud/notifications.py +149 -0
  48. mlrun/api/crud/pipelines.py +67 -52
  49. mlrun/api/crud/projects.py +51 -23
  50. mlrun/api/crud/runs.py +7 -5
  51. mlrun/api/crud/runtime_resources.py +13 -13
  52. mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
  53. mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
  54. mlrun/api/crud/runtimes/nuclio/function.py +505 -0
  55. mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
  56. mlrun/api/crud/secrets.py +88 -46
  57. mlrun/api/crud/tags.py +5 -5
  58. mlrun/api/db/__init__.py +1 -1
  59. mlrun/api/db/base.py +102 -54
  60. mlrun/api/db/init_db.py +2 -3
  61. mlrun/api/db/session.py +4 -12
  62. mlrun/api/db/sqldb/__init__.py +1 -1
  63. mlrun/api/db/sqldb/db.py +439 -196
  64. mlrun/api/db/sqldb/helpers.py +1 -1
  65. mlrun/api/db/sqldb/models/__init__.py +3 -3
  66. mlrun/api/db/sqldb/models/models_mysql.py +82 -64
  67. mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
  68. mlrun/api/db/sqldb/session.py +27 -20
  69. mlrun/api/initial_data.py +82 -24
  70. mlrun/api/launcher.py +196 -0
  71. mlrun/api/main.py +91 -22
  72. mlrun/api/middlewares.py +6 -5
  73. mlrun/api/migrations_mysql/env.py +1 -1
  74. mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
  75. mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
  76. mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
  77. mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
  78. mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
  79. mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
  80. mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
  81. mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
  82. mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
  83. mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
  84. mlrun/api/migrations_sqlite/env.py +1 -1
  85. mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
  86. mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
  87. mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
  88. mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
  89. mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
  90. mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
  91. mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
  92. mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
  93. mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
  94. mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
  95. mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
  96. mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
  97. mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
  98. mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
  99. mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
  100. mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
  101. mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
  102. mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
  103. mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
  104. mlrun/api/schemas/__init__.py +216 -138
  105. mlrun/api/utils/__init__.py +1 -1
  106. mlrun/api/utils/asyncio.py +1 -1
  107. mlrun/api/utils/auth/__init__.py +1 -1
  108. mlrun/api/utils/auth/providers/__init__.py +1 -1
  109. mlrun/api/utils/auth/providers/base.py +7 -7
  110. mlrun/api/utils/auth/providers/nop.py +6 -7
  111. mlrun/api/utils/auth/providers/opa.py +17 -17
  112. mlrun/api/utils/auth/verifier.py +36 -34
  113. mlrun/api/utils/background_tasks.py +24 -24
  114. mlrun/{builder.py → api/utils/builder.py} +216 -123
  115. mlrun/api/utils/clients/__init__.py +1 -1
  116. mlrun/api/utils/clients/chief.py +19 -4
  117. mlrun/api/utils/clients/iguazio.py +106 -60
  118. mlrun/api/utils/clients/log_collector.py +1 -1
  119. mlrun/api/utils/clients/nuclio.py +23 -23
  120. mlrun/api/utils/clients/protocols/grpc.py +2 -2
  121. mlrun/api/utils/db/__init__.py +1 -1
  122. mlrun/api/utils/db/alembic.py +1 -1
  123. mlrun/api/utils/db/backup.py +1 -1
  124. mlrun/api/utils/db/mysql.py +24 -25
  125. mlrun/api/utils/db/sql_collation.py +1 -1
  126. mlrun/api/utils/db/sqlite_migration.py +2 -2
  127. mlrun/api/utils/events/__init__.py +14 -0
  128. mlrun/api/utils/events/base.py +57 -0
  129. mlrun/api/utils/events/events_factory.py +41 -0
  130. mlrun/api/utils/events/iguazio.py +217 -0
  131. mlrun/api/utils/events/nop.py +55 -0
  132. mlrun/api/utils/helpers.py +16 -13
  133. mlrun/api/utils/memory_reports.py +1 -1
  134. mlrun/api/utils/periodic.py +6 -3
  135. mlrun/api/utils/projects/__init__.py +1 -1
  136. mlrun/api/utils/projects/follower.py +33 -33
  137. mlrun/api/utils/projects/leader.py +36 -34
  138. mlrun/api/utils/projects/member.py +27 -27
  139. mlrun/api/utils/projects/remotes/__init__.py +1 -1
  140. mlrun/api/utils/projects/remotes/follower.py +13 -13
  141. mlrun/api/utils/projects/remotes/leader.py +10 -10
  142. mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
  143. mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
  144. mlrun/api/utils/scheduler.py +140 -51
  145. mlrun/api/utils/singletons/__init__.py +1 -1
  146. mlrun/api/utils/singletons/db.py +9 -15
  147. mlrun/api/utils/singletons/k8s.py +677 -5
  148. mlrun/api/utils/singletons/logs_dir.py +1 -1
  149. mlrun/api/utils/singletons/project_member.py +1 -1
  150. mlrun/api/utils/singletons/scheduler.py +1 -1
  151. mlrun/artifacts/__init__.py +2 -2
  152. mlrun/artifacts/base.py +8 -2
  153. mlrun/artifacts/dataset.py +5 -3
  154. mlrun/artifacts/manager.py +7 -1
  155. mlrun/artifacts/model.py +15 -4
  156. mlrun/artifacts/plots.py +1 -1
  157. mlrun/common/__init__.py +1 -1
  158. mlrun/common/constants.py +15 -0
  159. mlrun/common/model_monitoring.py +209 -0
  160. mlrun/common/schemas/__init__.py +167 -0
  161. mlrun/{api → common}/schemas/artifact.py +13 -14
  162. mlrun/{api → common}/schemas/auth.py +10 -8
  163. mlrun/{api → common}/schemas/background_task.py +3 -3
  164. mlrun/{api → common}/schemas/client_spec.py +1 -1
  165. mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
  166. mlrun/{api → common}/schemas/constants.py +21 -8
  167. mlrun/common/schemas/events.py +36 -0
  168. mlrun/{api → common}/schemas/feature_store.py +2 -1
  169. mlrun/{api → common}/schemas/frontend_spec.py +7 -6
  170. mlrun/{api → common}/schemas/function.py +5 -5
  171. mlrun/{api → common}/schemas/http.py +3 -3
  172. mlrun/common/schemas/hub.py +134 -0
  173. mlrun/{api → common}/schemas/k8s.py +3 -3
  174. mlrun/{api → common}/schemas/memory_reports.py +1 -1
  175. mlrun/common/schemas/model_endpoints.py +342 -0
  176. mlrun/common/schemas/notification.py +57 -0
  177. mlrun/{api → common}/schemas/object.py +6 -6
  178. mlrun/{api → common}/schemas/pipeline.py +3 -3
  179. mlrun/{api → common}/schemas/project.py +6 -5
  180. mlrun/common/schemas/regex.py +24 -0
  181. mlrun/common/schemas/runs.py +30 -0
  182. mlrun/{api → common}/schemas/runtime_resource.py +3 -3
  183. mlrun/{api → common}/schemas/schedule.py +19 -7
  184. mlrun/{api → common}/schemas/secret.py +3 -3
  185. mlrun/{api → common}/schemas/tag.py +2 -2
  186. mlrun/common/types.py +25 -0
  187. mlrun/config.py +152 -20
  188. mlrun/data_types/__init__.py +7 -2
  189. mlrun/data_types/data_types.py +4 -2
  190. mlrun/data_types/infer.py +1 -1
  191. mlrun/data_types/spark.py +10 -3
  192. mlrun/datastore/__init__.py +10 -3
  193. mlrun/datastore/azure_blob.py +1 -1
  194. mlrun/datastore/base.py +185 -53
  195. mlrun/datastore/datastore.py +1 -1
  196. mlrun/datastore/filestore.py +1 -1
  197. mlrun/datastore/google_cloud_storage.py +1 -1
  198. mlrun/datastore/inmem.py +4 -1
  199. mlrun/datastore/redis.py +1 -1
  200. mlrun/datastore/s3.py +1 -1
  201. mlrun/datastore/sources.py +192 -70
  202. mlrun/datastore/spark_udf.py +44 -0
  203. mlrun/datastore/store_resources.py +4 -4
  204. mlrun/datastore/targets.py +115 -45
  205. mlrun/datastore/utils.py +127 -5
  206. mlrun/datastore/v3io.py +1 -1
  207. mlrun/datastore/wasbfs/__init__.py +1 -1
  208. mlrun/datastore/wasbfs/fs.py +1 -1
  209. mlrun/db/__init__.py +7 -5
  210. mlrun/db/base.py +112 -68
  211. mlrun/db/httpdb.py +445 -277
  212. mlrun/db/nopdb.py +491 -0
  213. mlrun/db/sqldb.py +112 -65
  214. mlrun/errors.py +6 -1
  215. mlrun/execution.py +44 -22
  216. mlrun/feature_store/__init__.py +1 -1
  217. mlrun/feature_store/api.py +143 -95
  218. mlrun/feature_store/common.py +16 -20
  219. mlrun/feature_store/feature_set.py +42 -12
  220. mlrun/feature_store/feature_vector.py +32 -21
  221. mlrun/feature_store/ingestion.py +9 -12
  222. mlrun/feature_store/retrieval/__init__.py +3 -2
  223. mlrun/feature_store/retrieval/base.py +388 -66
  224. mlrun/feature_store/retrieval/dask_merger.py +63 -151
  225. mlrun/feature_store/retrieval/job.py +30 -12
  226. mlrun/feature_store/retrieval/local_merger.py +40 -133
  227. mlrun/feature_store/retrieval/spark_merger.py +129 -127
  228. mlrun/feature_store/retrieval/storey_merger.py +173 -0
  229. mlrun/feature_store/steps.py +132 -15
  230. mlrun/features.py +8 -3
  231. mlrun/frameworks/__init__.py +1 -1
  232. mlrun/frameworks/_common/__init__.py +1 -1
  233. mlrun/frameworks/_common/artifacts_library.py +1 -1
  234. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  235. mlrun/frameworks/_common/model_handler.py +1 -1
  236. mlrun/frameworks/_common/plan.py +1 -1
  237. mlrun/frameworks/_common/producer.py +1 -1
  238. mlrun/frameworks/_common/utils.py +1 -1
  239. mlrun/frameworks/_dl_common/__init__.py +1 -1
  240. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
  241. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  242. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  243. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  244. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  245. mlrun/frameworks/_dl_common/utils.py +1 -1
  246. mlrun/frameworks/_ml_common/__init__.py +1 -1
  247. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  248. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
  249. mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
  250. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  251. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  252. mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
  253. mlrun/frameworks/_ml_common/plan.py +1 -1
  254. mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
  255. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
  256. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
  257. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  258. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  259. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  260. mlrun/frameworks/_ml_common/producer.py +1 -1
  261. mlrun/frameworks/_ml_common/utils.py +1 -1
  262. mlrun/frameworks/auto_mlrun/__init__.py +1 -1
  263. mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
  264. mlrun/frameworks/huggingface/__init__.py +1 -1
  265. mlrun/frameworks/huggingface/model_server.py +1 -1
  266. mlrun/frameworks/lgbm/__init__.py +1 -1
  267. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
  268. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  269. mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
  270. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
  271. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
  272. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  273. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  274. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  275. mlrun/frameworks/lgbm/model_handler.py +1 -1
  276. mlrun/frameworks/lgbm/model_server.py +1 -1
  277. mlrun/frameworks/lgbm/utils.py +1 -1
  278. mlrun/frameworks/onnx/__init__.py +1 -1
  279. mlrun/frameworks/onnx/dataset.py +1 -1
  280. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  281. mlrun/frameworks/onnx/model_handler.py +1 -1
  282. mlrun/frameworks/onnx/model_server.py +1 -1
  283. mlrun/frameworks/parallel_coordinates.py +1 -1
  284. mlrun/frameworks/pytorch/__init__.py +1 -1
  285. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
  286. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  287. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  288. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  289. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  290. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  291. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  292. mlrun/frameworks/pytorch/model_handler.py +1 -1
  293. mlrun/frameworks/pytorch/model_server.py +1 -1
  294. mlrun/frameworks/pytorch/utils.py +1 -1
  295. mlrun/frameworks/sklearn/__init__.py +1 -1
  296. mlrun/frameworks/sklearn/estimator.py +1 -1
  297. mlrun/frameworks/sklearn/metric.py +1 -1
  298. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  299. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  300. mlrun/frameworks/sklearn/model_handler.py +1 -1
  301. mlrun/frameworks/sklearn/utils.py +1 -1
  302. mlrun/frameworks/tf_keras/__init__.py +1 -1
  303. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
  304. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  305. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  306. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  307. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  308. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  309. mlrun/frameworks/tf_keras/model_server.py +1 -1
  310. mlrun/frameworks/tf_keras/utils.py +1 -1
  311. mlrun/frameworks/xgboost/__init__.py +1 -1
  312. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  313. mlrun/frameworks/xgboost/model_handler.py +1 -1
  314. mlrun/frameworks/xgboost/utils.py +1 -1
  315. mlrun/k8s_utils.py +14 -765
  316. mlrun/kfpops.py +14 -17
  317. mlrun/launcher/__init__.py +13 -0
  318. mlrun/launcher/base.py +406 -0
  319. mlrun/launcher/client.py +159 -0
  320. mlrun/launcher/factory.py +50 -0
  321. mlrun/launcher/local.py +276 -0
  322. mlrun/launcher/remote.py +178 -0
  323. mlrun/lists.py +10 -2
  324. mlrun/mlutils/__init__.py +1 -1
  325. mlrun/mlutils/data.py +1 -1
  326. mlrun/mlutils/models.py +1 -1
  327. mlrun/mlutils/plots.py +1 -1
  328. mlrun/model.py +252 -14
  329. mlrun/model_monitoring/__init__.py +41 -0
  330. mlrun/model_monitoring/features_drift_table.py +1 -1
  331. mlrun/model_monitoring/helpers.py +123 -38
  332. mlrun/model_monitoring/model_endpoint.py +144 -0
  333. mlrun/model_monitoring/model_monitoring_batch.py +310 -259
  334. mlrun/model_monitoring/stores/__init__.py +106 -0
  335. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
  336. mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
  337. mlrun/model_monitoring/stores/models/__init__.py +23 -0
  338. mlrun/model_monitoring/stores/models/base.py +18 -0
  339. mlrun/model_monitoring/stores/models/mysql.py +100 -0
  340. mlrun/model_monitoring/stores/models/sqlite.py +98 -0
  341. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
  342. mlrun/model_monitoring/stream_processing_fs.py +239 -271
  343. mlrun/package/__init__.py +163 -0
  344. mlrun/package/context_handler.py +325 -0
  345. mlrun/package/errors.py +47 -0
  346. mlrun/package/packager.py +298 -0
  347. mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
  348. mlrun/package/packagers/default_packager.py +422 -0
  349. mlrun/package/packagers/numpy_packagers.py +612 -0
  350. mlrun/package/packagers/pandas_packagers.py +968 -0
  351. mlrun/package/packagers/python_standard_library_packagers.py +616 -0
  352. mlrun/package/packagers_manager.py +786 -0
  353. mlrun/package/utils/__init__.py +53 -0
  354. mlrun/package/utils/_archiver.py +226 -0
  355. mlrun/package/utils/_formatter.py +211 -0
  356. mlrun/package/utils/_pickler.py +234 -0
  357. mlrun/package/utils/_supported_format.py +71 -0
  358. mlrun/package/utils/log_hint_utils.py +93 -0
  359. mlrun/package/utils/type_hint_utils.py +298 -0
  360. mlrun/platforms/__init__.py +1 -1
  361. mlrun/platforms/iguazio.py +34 -2
  362. mlrun/platforms/other.py +1 -1
  363. mlrun/projects/__init__.py +1 -1
  364. mlrun/projects/operations.py +14 -9
  365. mlrun/projects/pipelines.py +31 -13
  366. mlrun/projects/project.py +762 -238
  367. mlrun/render.py +49 -19
  368. mlrun/run.py +57 -326
  369. mlrun/runtimes/__init__.py +3 -9
  370. mlrun/runtimes/base.py +247 -784
  371. mlrun/runtimes/constants.py +1 -1
  372. mlrun/runtimes/daskjob.py +45 -41
  373. mlrun/runtimes/funcdoc.py +43 -7
  374. mlrun/runtimes/function.py +66 -656
  375. mlrun/runtimes/function_reference.py +1 -1
  376. mlrun/runtimes/generators.py +1 -1
  377. mlrun/runtimes/kubejob.py +99 -116
  378. mlrun/runtimes/local.py +59 -66
  379. mlrun/runtimes/mpijob/__init__.py +1 -1
  380. mlrun/runtimes/mpijob/abstract.py +13 -15
  381. mlrun/runtimes/mpijob/v1.py +3 -1
  382. mlrun/runtimes/mpijob/v1alpha1.py +1 -1
  383. mlrun/runtimes/nuclio.py +1 -1
  384. mlrun/runtimes/pod.py +51 -26
  385. mlrun/runtimes/remotesparkjob.py +3 -1
  386. mlrun/runtimes/serving.py +12 -4
  387. mlrun/runtimes/sparkjob/__init__.py +1 -2
  388. mlrun/runtimes/sparkjob/abstract.py +44 -31
  389. mlrun/runtimes/sparkjob/spark3job.py +11 -9
  390. mlrun/runtimes/utils.py +61 -42
  391. mlrun/secrets.py +16 -18
  392. mlrun/serving/__init__.py +3 -2
  393. mlrun/serving/merger.py +1 -1
  394. mlrun/serving/remote.py +1 -1
  395. mlrun/serving/routers.py +39 -42
  396. mlrun/serving/server.py +23 -13
  397. mlrun/serving/serving_wrapper.py +1 -1
  398. mlrun/serving/states.py +172 -39
  399. mlrun/serving/utils.py +1 -1
  400. mlrun/serving/v1_serving.py +1 -1
  401. mlrun/serving/v2_serving.py +29 -21
  402. mlrun/utils/__init__.py +1 -2
  403. mlrun/utils/async_http.py +8 -1
  404. mlrun/utils/azure_vault.py +1 -1
  405. mlrun/utils/clones.py +2 -2
  406. mlrun/utils/condition_evaluator.py +65 -0
  407. mlrun/utils/db.py +52 -0
  408. mlrun/utils/helpers.py +188 -13
  409. mlrun/utils/http.py +89 -54
  410. mlrun/utils/logger.py +48 -8
  411. mlrun/utils/model_monitoring.py +132 -100
  412. mlrun/utils/notifications/__init__.py +1 -1
  413. mlrun/utils/notifications/notification/__init__.py +8 -6
  414. mlrun/utils/notifications/notification/base.py +20 -14
  415. mlrun/utils/notifications/notification/console.py +7 -4
  416. mlrun/utils/notifications/notification/git.py +36 -19
  417. mlrun/utils/notifications/notification/ipython.py +10 -8
  418. mlrun/utils/notifications/notification/slack.py +18 -13
  419. mlrun/utils/notifications/notification_pusher.py +377 -56
  420. mlrun/utils/regex.py +6 -1
  421. mlrun/utils/singleton.py +1 -1
  422. mlrun/utils/v3io_clients.py +1 -1
  423. mlrun/utils/vault.py +270 -269
  424. mlrun/utils/version/__init__.py +1 -1
  425. mlrun/utils/version/version.json +2 -2
  426. mlrun/utils/version/version.py +1 -1
  427. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
  428. mlrun-1.4.0.dist-info/RECORD +434 -0
  429. mlrun/api/api/endpoints/marketplace.py +0 -257
  430. mlrun/api/crud/marketplace.py +0 -221
  431. mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
  432. mlrun/api/db/filedb/db.py +0 -518
  433. mlrun/api/schemas/marketplace.py +0 -128
  434. mlrun/api/schemas/model_endpoints.py +0 -185
  435. mlrun/db/filedb.py +0 -891
  436. mlrun/feature_store/retrieval/online.py +0 -92
  437. mlrun/model_monitoring/constants.py +0 -67
  438. mlrun/runtimes/package/context_handler.py +0 -711
  439. mlrun/runtimes/sparkjob/spark2job.py +0 -59
  440. mlrun-1.3.3.dist-info/RECORD +0 -381
  441. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
  442. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
  443. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
  444. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -23,10 +23,10 @@ from sqlalchemy.orm import Session
23
23
  import mlrun.api.crud
24
24
  import mlrun.api.utils.auth.verifier
25
25
  import mlrun.api.utils.singletons.project_member
26
+ import mlrun.common.schemas
26
27
  import mlrun.errors
27
28
  import mlrun.feature_store
28
29
  from mlrun import v3io_cred
29
- from mlrun.api import schemas
30
30
  from mlrun.api.api import deps
31
31
  from mlrun.api.api.utils import log_and_raise, parse_reference
32
32
  from mlrun.data_types import InferOptions
@@ -34,15 +34,15 @@ from mlrun.datastore.targets import get_default_prefix_for_target
34
34
  from mlrun.feature_store.api import RunConfig, ingest
35
35
  from mlrun.model import DataSource, DataTargetBase
36
36
 
37
- router = APIRouter()
37
+ router = APIRouter(prefix="/projects/{project}")
38
38
 
39
39
 
40
- @router.post("/projects/{project}/feature-sets", response_model=schemas.FeatureSet)
40
+ @router.post("/feature-sets", response_model=mlrun.common.schemas.FeatureSet)
41
41
  async def create_feature_set(
42
42
  project: str,
43
- feature_set: schemas.FeatureSet,
43
+ feature_set: mlrun.common.schemas.FeatureSet,
44
44
  versioned: bool = True,
45
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
45
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
46
46
  db_session: Session = Depends(deps.get_db_session),
47
47
  ):
48
48
  await run_in_threadpool(
@@ -52,10 +52,10 @@ async def create_feature_set(
52
52
  auth_info=auth_info,
53
53
  )
54
54
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
55
- mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
55
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
56
56
  project,
57
57
  feature_set.metadata.name,
58
- mlrun.api.schemas.AuthorizationAction.create,
58
+ mlrun.common.schemas.AuthorizationAction.create,
59
59
  auth_info,
60
60
  )
61
61
  feature_set_uid = await run_in_threadpool(
@@ -77,16 +77,16 @@ async def create_feature_set(
77
77
 
78
78
 
79
79
  @router.put(
80
- "/projects/{project}/feature-sets/{name}/references/{reference}",
81
- response_model=schemas.FeatureSet,
80
+ "/feature-sets/{name}/references/{reference}",
81
+ response_model=mlrun.common.schemas.FeatureSet,
82
82
  )
83
83
  async def store_feature_set(
84
84
  project: str,
85
85
  name: str,
86
86
  reference: str,
87
- feature_set: schemas.FeatureSet,
87
+ feature_set: mlrun.common.schemas.FeatureSet,
88
88
  versioned: bool = True,
89
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
89
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
90
90
  db_session: Session = Depends(deps.get_db_session),
91
91
  ):
92
92
  await run_in_threadpool(
@@ -96,10 +96,10 @@ async def store_feature_set(
96
96
  auth_info=auth_info,
97
97
  )
98
98
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
99
- mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
99
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
100
100
  project,
101
101
  name,
102
- mlrun.api.schemas.AuthorizationAction.store,
102
+ mlrun.common.schemas.AuthorizationAction.store,
103
103
  auth_info,
104
104
  )
105
105
  tag, uid = parse_reference(reference)
@@ -123,23 +123,24 @@ async def store_feature_set(
123
123
  )
124
124
 
125
125
 
126
- @router.patch("/projects/{project}/feature-sets/{name}/references/{reference}")
126
+ @router.patch("/feature-sets/{name}/references/{reference}")
127
127
  async def patch_feature_set(
128
128
  project: str,
129
129
  name: str,
130
130
  feature_set_update: dict,
131
131
  reference: str,
132
- patch_mode: schemas.PatchMode = Header(
133
- schemas.PatchMode.replace, alias=schemas.HeaderNames.patch_mode
132
+ patch_mode: mlrun.common.schemas.PatchMode = Header(
133
+ mlrun.common.schemas.PatchMode.replace,
134
+ alias=mlrun.common.schemas.HeaderNames.patch_mode,
134
135
  ),
135
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
136
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
136
137
  db_session: Session = Depends(deps.get_db_session),
137
138
  ):
138
139
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
139
- mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
140
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
140
141
  project,
141
142
  name,
142
- mlrun.api.schemas.AuthorizationAction.update,
143
+ mlrun.common.schemas.AuthorizationAction.update,
143
144
  auth_info,
144
145
  )
145
146
  tag, uid = parse_reference(reference)
@@ -157,14 +158,14 @@ async def patch_feature_set(
157
158
 
158
159
 
159
160
  @router.get(
160
- "/projects/{project}/feature-sets/{name}/references/{reference}",
161
- response_model=schemas.FeatureSet,
161
+ "/feature-sets/{name}/references/{reference}",
162
+ response_model=mlrun.common.schemas.FeatureSet,
162
163
  )
163
164
  async def get_feature_set(
164
165
  project: str,
165
166
  name: str,
166
167
  reference: str,
167
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
168
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
168
169
  db_session: Session = Depends(deps.get_db_session),
169
170
  ):
170
171
  tag, uid = parse_reference(reference)
@@ -177,29 +178,29 @@ async def get_feature_set(
177
178
  uid,
178
179
  )
179
180
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
180
- mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
181
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
181
182
  project,
182
183
  name,
183
- mlrun.api.schemas.AuthorizationAction.read,
184
+ mlrun.common.schemas.AuthorizationAction.read,
184
185
  auth_info,
185
186
  )
186
187
  return feature_set
187
188
 
188
189
 
189
- @router.delete("/projects/{project}/feature-sets/{name}")
190
- @router.delete("/projects/{project}/feature-sets/{name}/references/{reference}")
190
+ @router.delete("/feature-sets/{name}")
191
+ @router.delete("/feature-sets/{name}/references/{reference}")
191
192
  async def delete_feature_set(
192
193
  project: str,
193
194
  name: str,
194
195
  reference: str = None,
195
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
196
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
196
197
  db_session: Session = Depends(deps.get_db_session),
197
198
  ):
198
199
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
199
- mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
200
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
200
201
  project,
201
202
  name,
202
- mlrun.api.schemas.AuthorizationAction.delete,
203
+ mlrun.common.schemas.AuthorizationAction.delete,
203
204
  auth_info,
204
205
  )
205
206
  tag = uid = None
@@ -217,7 +218,8 @@ async def delete_feature_set(
217
218
 
218
219
 
219
220
  @router.get(
220
- "/projects/{project}/feature-sets", response_model=schemas.FeatureSetsOutput
221
+ "/feature-sets",
222
+ response_model=mlrun.common.schemas.FeatureSetsOutput,
221
223
  )
222
224
  async def list_feature_sets(
223
225
  project: str,
@@ -227,20 +229,22 @@ async def list_feature_sets(
227
229
  entities: List[str] = Query(None, alias="entity"),
228
230
  features: List[str] = Query(None, alias="feature"),
229
231
  labels: List[str] = Query(None, alias="label"),
230
- partition_by: schemas.FeatureStorePartitionByField = Query(
232
+ partition_by: mlrun.common.schemas.FeatureStorePartitionByField = Query(
231
233
  None, alias="partition-by"
232
234
  ),
233
235
  rows_per_partition: int = Query(1, alias="rows-per-partition", gt=0),
234
- partition_sort_by: schemas.SortField = Query(None, alias="partition-sort-by"),
235
- partition_order: schemas.OrderType = Query(
236
- schemas.OrderType.desc, alias="partition-order"
236
+ partition_sort_by: mlrun.common.schemas.SortField = Query(
237
+ None, alias="partition-sort-by"
237
238
  ),
238
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
239
+ partition_order: mlrun.common.schemas.OrderType = Query(
240
+ mlrun.common.schemas.OrderType.desc, alias="partition-order"
241
+ ),
242
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
239
243
  db_session: Session = Depends(deps.get_db_session),
240
244
  ):
241
245
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
242
246
  project,
243
- mlrun.api.schemas.AuthorizationAction.read,
247
+ mlrun.common.schemas.AuthorizationAction.read,
244
248
  auth_info,
245
249
  )
246
250
  feature_sets = await run_in_threadpool(
@@ -259,7 +263,7 @@ async def list_feature_sets(
259
263
  partition_order,
260
264
  )
261
265
  feature_sets = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
262
- mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
266
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
263
267
  feature_sets.feature_sets,
264
268
  lambda feature_set: (
265
269
  feature_set.metadata.project,
@@ -267,17 +271,17 @@ async def list_feature_sets(
267
271
  ),
268
272
  auth_info,
269
273
  )
270
- return mlrun.api.schemas.FeatureSetsOutput(feature_sets=feature_sets)
274
+ return mlrun.common.schemas.FeatureSetsOutput(feature_sets=feature_sets)
271
275
 
272
276
 
273
277
  @router.get(
274
- "/projects/{project}/feature-sets/{name}/tags",
275
- response_model=schemas.FeatureSetsTagsOutput,
278
+ "/feature-sets/{name}/tags",
279
+ response_model=mlrun.common.schemas.FeatureSetsTagsOutput,
276
280
  )
277
281
  async def list_feature_sets_tags(
278
282
  project: str,
279
283
  name: str,
280
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
284
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
281
285
  db_session: Session = Depends(deps.get_db_session),
282
286
  ):
283
287
  if name != "*":
@@ -286,7 +290,7 @@ async def list_feature_sets_tags(
286
290
  )
287
291
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
288
292
  project,
289
- mlrun.api.schemas.AuthorizationAction.read,
293
+ mlrun.common.schemas.AuthorizationAction.read,
290
294
  auth_info,
291
295
  )
292
296
  tag_tuples = await run_in_threadpool(
@@ -298,7 +302,7 @@ async def list_feature_sets_tags(
298
302
  auth_verifier = mlrun.api.utils.auth.verifier.AuthVerifier()
299
303
  allowed_feature_set_names = (
300
304
  await auth_verifier.filter_project_resources_by_permissions(
301
- mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
305
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
302
306
  list(feature_set_name_to_tag.keys()),
303
307
  lambda feature_set_name: (
304
308
  project,
@@ -312,7 +316,7 @@ async def list_feature_sets_tags(
312
316
  for tag_tuple in tag_tuples
313
317
  if tag_tuple[1] in allowed_feature_set_names
314
318
  }
315
- return mlrun.api.schemas.FeatureSetsTagsOutput(tags=list(tags))
319
+ return mlrun.common.schemas.FeatureSetsTagsOutput(tags=list(tags))
316
320
 
317
321
 
318
322
  def _has_v3io_path(data_source, data_targets, feature_set):
@@ -341,8 +345,8 @@ def _has_v3io_path(data_source, data_targets, feature_set):
341
345
 
342
346
 
343
347
  @router.post(
344
- "/projects/{project}/feature-sets/{name}/references/{reference}/ingest",
345
- response_model=schemas.FeatureSetIngestOutput,
348
+ "/feature-sets/{name}/references/{reference}/ingest",
349
+ response_model=mlrun.common.schemas.FeatureSetIngestOutput,
346
350
  status_code=HTTPStatus.ACCEPTED.value,
347
351
  )
348
352
  async def ingest_feature_set(
@@ -350,10 +354,10 @@ async def ingest_feature_set(
350
354
  name: str,
351
355
  reference: str,
352
356
  ingest_parameters: Optional[
353
- schemas.FeatureSetIngestInput
354
- ] = schemas.FeatureSetIngestInput(),
357
+ mlrun.common.schemas.FeatureSetIngestInput
358
+ ] = mlrun.common.schemas.FeatureSetIngestInput(),
355
359
  username: str = Header(None, alias="x-remote-user"),
356
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
360
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
357
361
  db_session: Session = Depends(deps.get_db_session),
358
362
  ):
359
363
  """
@@ -361,17 +365,17 @@ async def ingest_feature_set(
361
365
  that already being happen on client side
362
366
  """
363
367
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
364
- mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
368
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
365
369
  project,
366
370
  name,
367
- mlrun.api.schemas.AuthorizationAction.update,
371
+ mlrun.common.schemas.AuthorizationAction.update,
368
372
  auth_info,
369
373
  )
370
374
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
371
- mlrun.api.schemas.AuthorizationResourceTypes.run,
375
+ mlrun.common.schemas.AuthorizationResourceTypes.run,
372
376
  project,
373
377
  "",
374
- mlrun.api.schemas.AuthorizationAction.create,
378
+ mlrun.common.schemas.AuthorizationAction.create,
375
379
  auth_info,
376
380
  )
377
381
  data_source = data_targets = None
@@ -379,10 +383,10 @@ async def ingest_feature_set(
379
383
  data_source = DataSource.from_dict(ingest_parameters.source.dict())
380
384
  if data_source.schedule:
381
385
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
382
- mlrun.api.schemas.AuthorizationResourceTypes.schedule,
386
+ mlrun.common.schemas.AuthorizationResourceTypes.schedule,
383
387
  project,
384
388
  "",
385
- mlrun.api.schemas.AuthorizationAction.create,
389
+ mlrun.common.schemas.AuthorizationAction.create,
386
390
  auth_info,
387
391
  )
388
392
  tag, uid = parse_reference(reference)
@@ -398,10 +402,10 @@ async def ingest_feature_set(
398
402
  if feature_set.spec.function and feature_set.spec.function.function_object:
399
403
  function = feature_set.spec.function.function_object
400
404
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
401
- mlrun.api.schemas.AuthorizationResourceTypes.function,
405
+ mlrun.common.schemas.AuthorizationResourceTypes.function,
402
406
  function.metadata.project,
403
407
  function.metadata.name,
404
- mlrun.api.schemas.AuthorizationAction.read,
408
+ mlrun.common.schemas.AuthorizationAction.read,
405
409
  auth_info,
406
410
  )
407
411
  # Need to override the default rundb since we're in the server.
@@ -447,25 +451,25 @@ async def ingest_feature_set(
447
451
  run_config=run_config,
448
452
  )
449
453
  # ingest may modify the feature-set contents, so returning the updated feature-set.
450
- result_feature_set = schemas.FeatureSet(**feature_set.to_dict())
451
- return schemas.FeatureSetIngestOutput(
454
+ result_feature_set = mlrun.common.schemas.FeatureSet(**feature_set.to_dict())
455
+ return mlrun.common.schemas.FeatureSetIngestOutput(
452
456
  feature_set=result_feature_set, run_object=run_params.to_dict()
453
457
  )
454
458
 
455
459
 
456
- @router.get("/projects/{project}/features", response_model=schemas.FeaturesOutput)
460
+ @router.get("/features", response_model=mlrun.common.schemas.FeaturesOutput)
457
461
  async def list_features(
458
462
  project: str,
459
463
  name: str = None,
460
464
  tag: str = None,
461
465
  entities: List[str] = Query(None, alias="entity"),
462
466
  labels: List[str] = Query(None, alias="label"),
463
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
467
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
464
468
  db_session: Session = Depends(deps.get_db_session),
465
469
  ):
466
470
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
467
471
  project,
468
- mlrun.api.schemas.AuthorizationAction.read,
472
+ mlrun.common.schemas.AuthorizationAction.read,
469
473
  auth_info,
470
474
  )
471
475
  features = await run_in_threadpool(
@@ -478,7 +482,7 @@ async def list_features(
478
482
  labels,
479
483
  )
480
484
  features = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
481
- mlrun.api.schemas.AuthorizationResourceTypes.feature,
485
+ mlrun.common.schemas.AuthorizationResourceTypes.feature,
482
486
  features.features,
483
487
  lambda feature_list_output: (
484
488
  feature_list_output.feature_set_digest.metadata.project,
@@ -486,21 +490,21 @@ async def list_features(
486
490
  ),
487
491
  auth_info,
488
492
  )
489
- return mlrun.api.schemas.FeaturesOutput(features=features)
493
+ return mlrun.common.schemas.FeaturesOutput(features=features)
490
494
 
491
495
 
492
- @router.get("/projects/{project}/entities", response_model=schemas.EntitiesOutput)
496
+ @router.get("/entities", response_model=mlrun.common.schemas.EntitiesOutput)
493
497
  async def list_entities(
494
498
  project: str,
495
499
  name: str = None,
496
500
  tag: str = None,
497
501
  labels: List[str] = Query(None, alias="label"),
498
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
502
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
499
503
  db_session: Session = Depends(deps.get_db_session),
500
504
  ):
501
505
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
502
506
  project,
503
- mlrun.api.schemas.AuthorizationAction.read,
507
+ mlrun.common.schemas.AuthorizationAction.read,
504
508
  auth_info,
505
509
  )
506
510
  entities = await run_in_threadpool(
@@ -512,7 +516,7 @@ async def list_entities(
512
516
  labels,
513
517
  )
514
518
  entities = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
515
- mlrun.api.schemas.AuthorizationResourceTypes.entity,
519
+ mlrun.common.schemas.AuthorizationResourceTypes.entity,
516
520
  entities.entities,
517
521
  lambda entity_list_output: (
518
522
  entity_list_output.feature_set_digest.metadata.project,
@@ -520,17 +524,18 @@ async def list_entities(
520
524
  ),
521
525
  auth_info,
522
526
  )
523
- return mlrun.api.schemas.EntitiesOutput(entities=entities)
527
+ return mlrun.common.schemas.EntitiesOutput(entities=entities)
524
528
 
525
529
 
526
530
  @router.post(
527
- "/projects/{project}/feature-vectors", response_model=schemas.FeatureVector
531
+ "/feature-vectors",
532
+ response_model=mlrun.common.schemas.FeatureVector,
528
533
  )
529
534
  async def create_feature_vector(
530
535
  project: str,
531
- feature_vector: schemas.FeatureVector,
536
+ feature_vector: mlrun.common.schemas.FeatureVector,
532
537
  versioned: bool = True,
533
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
538
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
534
539
  db_session: Session = Depends(deps.get_db_session),
535
540
  ):
536
541
  await run_in_threadpool(
@@ -540,10 +545,10 @@ async def create_feature_vector(
540
545
  auth_info=auth_info,
541
546
  )
542
547
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
543
- mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
548
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
544
549
  project,
545
550
  feature_vector.metadata.name,
546
- mlrun.api.schemas.AuthorizationAction.create,
551
+ mlrun.common.schemas.AuthorizationAction.create,
547
552
  auth_info,
548
553
  )
549
554
  await _verify_feature_vector_features_permissions(
@@ -568,14 +573,14 @@ async def create_feature_vector(
568
573
 
569
574
 
570
575
  @router.get(
571
- "/projects/{project}/feature-vectors/{name}/references/{reference}",
572
- response_model=schemas.FeatureVector,
576
+ "/feature-vectors/{name}/references/{reference}",
577
+ response_model=mlrun.common.schemas.FeatureVector,
573
578
  )
574
579
  async def get_feature_vector(
575
580
  project: str,
576
581
  name: str,
577
582
  reference: str,
578
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
583
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
579
584
  db_session: Session = Depends(deps.get_db_session),
580
585
  ):
581
586
  tag, uid = parse_reference(reference)
@@ -588,10 +593,10 @@ async def get_feature_vector(
588
593
  uid,
589
594
  )
590
595
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
591
- mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
596
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
592
597
  project,
593
598
  name,
594
- mlrun.api.schemas.AuthorizationAction.read,
599
+ mlrun.common.schemas.AuthorizationAction.read,
595
600
  auth_info,
596
601
  )
597
602
  await _verify_feature_vector_features_permissions(
@@ -601,7 +606,8 @@ async def get_feature_vector(
601
606
 
602
607
 
603
608
  @router.get(
604
- "/projects/{project}/feature-vectors", response_model=schemas.FeatureVectorsOutput
609
+ "/feature-vectors",
610
+ response_model=mlrun.common.schemas.FeatureVectorsOutput,
605
611
  )
606
612
  async def list_feature_vectors(
607
613
  project: str,
@@ -609,20 +615,22 @@ async def list_feature_vectors(
609
615
  state: str = None,
610
616
  tag: str = None,
611
617
  labels: List[str] = Query(None, alias="label"),
612
- partition_by: schemas.FeatureStorePartitionByField = Query(
618
+ partition_by: mlrun.common.schemas.FeatureStorePartitionByField = Query(
613
619
  None, alias="partition-by"
614
620
  ),
615
621
  rows_per_partition: int = Query(1, alias="rows-per-partition", gt=0),
616
- partition_sort_by: schemas.SortField = Query(None, alias="partition-sort-by"),
617
- partition_order: schemas.OrderType = Query(
618
- schemas.OrderType.desc, alias="partition-order"
622
+ partition_sort_by: mlrun.common.schemas.SortField = Query(
623
+ None, alias="partition-sort-by"
624
+ ),
625
+ partition_order: mlrun.common.schemas.OrderType = Query(
626
+ mlrun.common.schemas.OrderType.desc, alias="partition-order"
619
627
  ),
620
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
628
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
621
629
  db_session: Session = Depends(deps.get_db_session),
622
630
  ):
623
631
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
624
632
  project,
625
- mlrun.api.schemas.AuthorizationAction.read,
633
+ mlrun.common.schemas.AuthorizationAction.read,
626
634
  auth_info,
627
635
  )
628
636
  feature_vectors = await run_in_threadpool(
@@ -639,7 +647,7 @@ async def list_feature_vectors(
639
647
  partition_order,
640
648
  )
641
649
  feature_vectors = await mlrun.api.utils.auth.verifier.AuthVerifier().filter_project_resources_by_permissions(
642
- mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
650
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
643
651
  feature_vectors.feature_vectors,
644
652
  lambda feature_vector: (
645
653
  feature_vector.metadata.project,
@@ -653,17 +661,17 @@ async def list_feature_vectors(
653
661
  for fv in feature_vectors
654
662
  ]
655
663
  )
656
- return mlrun.api.schemas.FeatureVectorsOutput(feature_vectors=feature_vectors)
664
+ return mlrun.common.schemas.FeatureVectorsOutput(feature_vectors=feature_vectors)
657
665
 
658
666
 
659
667
  @router.get(
660
- "/projects/{project}/feature-vectors/{name}/tags",
661
- response_model=schemas.FeatureVectorsTagsOutput,
668
+ "/feature-vectors/{name}/tags",
669
+ response_model=mlrun.common.schemas.FeatureVectorsTagsOutput,
662
670
  )
663
671
  async def list_feature_vectors_tags(
664
672
  project: str,
665
673
  name: str,
666
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
674
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
667
675
  db_session: Session = Depends(deps.get_db_session),
668
676
  ):
669
677
  if name != "*":
@@ -672,7 +680,7 @@ async def list_feature_vectors_tags(
672
680
  )
673
681
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_permissions(
674
682
  project,
675
- mlrun.api.schemas.AuthorizationAction.read,
683
+ mlrun.common.schemas.AuthorizationAction.read,
676
684
  auth_info,
677
685
  )
678
686
  tag_tuples = await run_in_threadpool(
@@ -686,7 +694,7 @@ async def list_feature_vectors_tags(
686
694
  auth_verifier = mlrun.api.utils.auth.verifier.AuthVerifier()
687
695
  allowed_feature_vector_names = (
688
696
  await auth_verifier.filter_project_resources_by_permissions(
689
- mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
697
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
690
698
  list(feature_vector_name_to_tag.keys()),
691
699
  lambda feature_vector_name: (
692
700
  project,
@@ -700,20 +708,20 @@ async def list_feature_vectors_tags(
700
708
  for tag_tuple in tag_tuples
701
709
  if tag_tuple[1] in allowed_feature_vector_names
702
710
  }
703
- return mlrun.api.schemas.FeatureVectorsTagsOutput(tags=list(tags))
711
+ return mlrun.common.schemas.FeatureVectorsTagsOutput(tags=list(tags))
704
712
 
705
713
 
706
714
  @router.put(
707
- "/projects/{project}/feature-vectors/{name}/references/{reference}",
708
- response_model=schemas.FeatureVector,
715
+ "/feature-vectors/{name}/references/{reference}",
716
+ response_model=mlrun.common.schemas.FeatureVector,
709
717
  )
710
718
  async def store_feature_vector(
711
719
  project: str,
712
720
  name: str,
713
721
  reference: str,
714
- feature_vector: schemas.FeatureVector,
722
+ feature_vector: mlrun.common.schemas.FeatureVector,
715
723
  versioned: bool = True,
716
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
724
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
717
725
  db_session: Session = Depends(deps.get_db_session),
718
726
  ):
719
727
  await run_in_threadpool(
@@ -723,10 +731,10 @@ async def store_feature_vector(
723
731
  auth_info=auth_info,
724
732
  )
725
733
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
726
- mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
734
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
727
735
  project,
728
736
  name,
729
- mlrun.api.schemas.AuthorizationAction.update,
737
+ mlrun.common.schemas.AuthorizationAction.update,
730
738
  auth_info,
731
739
  )
732
740
  await _verify_feature_vector_features_permissions(
@@ -754,23 +762,24 @@ async def store_feature_vector(
754
762
  )
755
763
 
756
764
 
757
- @router.patch("/projects/{project}/feature-vectors/{name}/references/{reference}")
765
+ @router.patch("/feature-vectors/{name}/references/{reference}")
758
766
  async def patch_feature_vector(
759
767
  project: str,
760
768
  name: str,
761
769
  feature_vector_patch: dict,
762
770
  reference: str,
763
- patch_mode: schemas.PatchMode = Header(
764
- schemas.PatchMode.replace, alias=schemas.HeaderNames.patch_mode
771
+ patch_mode: mlrun.common.schemas.PatchMode = Header(
772
+ mlrun.common.schemas.PatchMode.replace,
773
+ alias=mlrun.common.schemas.HeaderNames.patch_mode,
765
774
  ),
766
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
775
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
767
776
  db_session: Session = Depends(deps.get_db_session),
768
777
  ):
769
778
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
770
- mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
779
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
771
780
  project,
772
781
  name,
773
- mlrun.api.schemas.AuthorizationAction.update,
782
+ mlrun.common.schemas.AuthorizationAction.update,
774
783
  auth_info,
775
784
  )
776
785
  await _verify_feature_vector_features_permissions(
@@ -790,20 +799,20 @@ async def patch_feature_vector(
790
799
  return Response(status_code=HTTPStatus.OK.value)
791
800
 
792
801
 
793
- @router.delete("/projects/{project}/feature-vectors/{name}")
794
- @router.delete("/projects/{project}/feature-vectors/{name}/references/{reference}")
802
+ @router.delete("/feature-vectors/{name}")
803
+ @router.delete("/feature-vectors/{name}/references/{reference}")
795
804
  async def delete_feature_vector(
796
805
  project: str,
797
806
  name: str,
798
807
  reference: str = None,
799
- auth_info: mlrun.api.schemas.AuthInfo = Depends(deps.authenticate_request),
808
+ auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
800
809
  db_session: Session = Depends(deps.get_db_session),
801
810
  ):
802
811
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resource_permissions(
803
- mlrun.api.schemas.AuthorizationResourceTypes.feature_vector,
812
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_vector,
804
813
  project,
805
814
  name,
806
- mlrun.api.schemas.AuthorizationAction.delete,
815
+ mlrun.common.schemas.AuthorizationAction.delete,
807
816
  auth_info,
808
817
  )
809
818
  tag = uid = None
@@ -821,7 +830,7 @@ async def delete_feature_vector(
821
830
 
822
831
 
823
832
  async def _verify_feature_vector_features_permissions(
824
- auth_info: mlrun.api.schemas.AuthInfo, project: str, feature_vector: dict
833
+ auth_info: mlrun.common.schemas.AuthInfo, project: str, feature_vector: dict
825
834
  ):
826
835
  features = []
827
836
  if feature_vector.get("spec", {}).get("features"):
@@ -840,12 +849,12 @@ async def _verify_feature_vector_features_permissions(
840
849
  for name in names:
841
850
  feature_set_project_name_tuples.append((_project, name))
842
851
  await mlrun.api.utils.auth.verifier.AuthVerifier().query_project_resources_permissions(
843
- mlrun.api.schemas.AuthorizationResourceTypes.feature_set,
852
+ mlrun.common.schemas.AuthorizationResourceTypes.feature_set,
844
853
  feature_set_project_name_tuples,
845
854
  lambda feature_set_project_name_tuple: (
846
855
  feature_set_project_name_tuple[0],
847
856
  feature_set_project_name_tuple[1],
848
857
  ),
849
- mlrun.api.schemas.AuthorizationAction.read,
858
+ mlrun.common.schemas.AuthorizationAction.read,
850
859
  auth_info,
851
860
  )