mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (444) hide show
  1. mlrun/__init__.py +3 -3
  2. mlrun/__main__.py +79 -37
  3. mlrun/api/__init__.py +1 -1
  4. mlrun/api/api/__init__.py +1 -1
  5. mlrun/api/api/api.py +4 -4
  6. mlrun/api/api/deps.py +10 -21
  7. mlrun/api/api/endpoints/__init__.py +1 -1
  8. mlrun/api/api/endpoints/artifacts.py +64 -36
  9. mlrun/api/api/endpoints/auth.py +4 -4
  10. mlrun/api/api/endpoints/background_tasks.py +11 -11
  11. mlrun/api/api/endpoints/client_spec.py +5 -5
  12. mlrun/api/api/endpoints/clusterization_spec.py +6 -4
  13. mlrun/api/api/endpoints/feature_store.py +124 -115
  14. mlrun/api/api/endpoints/files.py +22 -14
  15. mlrun/api/api/endpoints/frontend_spec.py +28 -21
  16. mlrun/api/api/endpoints/functions.py +142 -87
  17. mlrun/api/api/endpoints/grafana_proxy.py +89 -442
  18. mlrun/api/api/endpoints/healthz.py +20 -7
  19. mlrun/api/api/endpoints/hub.py +320 -0
  20. mlrun/api/api/endpoints/internal/__init__.py +1 -1
  21. mlrun/api/api/endpoints/internal/config.py +1 -1
  22. mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
  23. mlrun/api/api/endpoints/logs.py +11 -11
  24. mlrun/api/api/endpoints/model_endpoints.py +74 -70
  25. mlrun/api/api/endpoints/operations.py +13 -9
  26. mlrun/api/api/endpoints/pipelines.py +93 -88
  27. mlrun/api/api/endpoints/projects.py +35 -35
  28. mlrun/api/api/endpoints/runs.py +69 -27
  29. mlrun/api/api/endpoints/runtime_resources.py +28 -28
  30. mlrun/api/api/endpoints/schedules.py +98 -41
  31. mlrun/api/api/endpoints/secrets.py +37 -32
  32. mlrun/api/api/endpoints/submit.py +12 -12
  33. mlrun/api/api/endpoints/tags.py +20 -22
  34. mlrun/api/api/utils.py +251 -42
  35. mlrun/api/constants.py +1 -1
  36. mlrun/api/crud/__init__.py +18 -15
  37. mlrun/api/crud/artifacts.py +10 -10
  38. mlrun/api/crud/client_spec.py +4 -4
  39. mlrun/api/crud/clusterization_spec.py +3 -3
  40. mlrun/api/crud/feature_store.py +54 -46
  41. mlrun/api/crud/functions.py +3 -3
  42. mlrun/api/crud/hub.py +312 -0
  43. mlrun/api/crud/logs.py +11 -9
  44. mlrun/api/crud/model_monitoring/__init__.py +3 -3
  45. mlrun/api/crud/model_monitoring/grafana.py +435 -0
  46. mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
  47. mlrun/api/crud/notifications.py +149 -0
  48. mlrun/api/crud/pipelines.py +67 -52
  49. mlrun/api/crud/projects.py +51 -23
  50. mlrun/api/crud/runs.py +7 -5
  51. mlrun/api/crud/runtime_resources.py +13 -13
  52. mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
  53. mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
  54. mlrun/api/crud/runtimes/nuclio/function.py +505 -0
  55. mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
  56. mlrun/api/crud/secrets.py +88 -46
  57. mlrun/api/crud/tags.py +5 -5
  58. mlrun/api/db/__init__.py +1 -1
  59. mlrun/api/db/base.py +102 -54
  60. mlrun/api/db/init_db.py +2 -3
  61. mlrun/api/db/session.py +4 -12
  62. mlrun/api/db/sqldb/__init__.py +1 -1
  63. mlrun/api/db/sqldb/db.py +439 -196
  64. mlrun/api/db/sqldb/helpers.py +1 -1
  65. mlrun/api/db/sqldb/models/__init__.py +3 -3
  66. mlrun/api/db/sqldb/models/models_mysql.py +82 -64
  67. mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
  68. mlrun/api/db/sqldb/session.py +27 -20
  69. mlrun/api/initial_data.py +82 -24
  70. mlrun/api/launcher.py +196 -0
  71. mlrun/api/main.py +91 -22
  72. mlrun/api/middlewares.py +6 -5
  73. mlrun/api/migrations_mysql/env.py +1 -1
  74. mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
  75. mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
  76. mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
  77. mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
  78. mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
  79. mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
  80. mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
  81. mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
  82. mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
  83. mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
  84. mlrun/api/migrations_sqlite/env.py +1 -1
  85. mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
  86. mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
  87. mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
  88. mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
  89. mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
  90. mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
  91. mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
  92. mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
  93. mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
  94. mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
  95. mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
  96. mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
  97. mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
  98. mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
  99. mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
  100. mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
  101. mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
  102. mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
  103. mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
  104. mlrun/api/schemas/__init__.py +216 -138
  105. mlrun/api/utils/__init__.py +1 -1
  106. mlrun/api/utils/asyncio.py +1 -1
  107. mlrun/api/utils/auth/__init__.py +1 -1
  108. mlrun/api/utils/auth/providers/__init__.py +1 -1
  109. mlrun/api/utils/auth/providers/base.py +7 -7
  110. mlrun/api/utils/auth/providers/nop.py +6 -7
  111. mlrun/api/utils/auth/providers/opa.py +17 -17
  112. mlrun/api/utils/auth/verifier.py +36 -34
  113. mlrun/api/utils/background_tasks.py +24 -24
  114. mlrun/{builder.py → api/utils/builder.py} +216 -123
  115. mlrun/api/utils/clients/__init__.py +1 -1
  116. mlrun/api/utils/clients/chief.py +19 -4
  117. mlrun/api/utils/clients/iguazio.py +106 -60
  118. mlrun/api/utils/clients/log_collector.py +1 -1
  119. mlrun/api/utils/clients/nuclio.py +23 -23
  120. mlrun/api/utils/clients/protocols/grpc.py +2 -2
  121. mlrun/api/utils/db/__init__.py +1 -1
  122. mlrun/api/utils/db/alembic.py +1 -1
  123. mlrun/api/utils/db/backup.py +1 -1
  124. mlrun/api/utils/db/mysql.py +24 -25
  125. mlrun/api/utils/db/sql_collation.py +1 -1
  126. mlrun/api/utils/db/sqlite_migration.py +2 -2
  127. mlrun/api/utils/events/__init__.py +14 -0
  128. mlrun/api/utils/events/base.py +57 -0
  129. mlrun/api/utils/events/events_factory.py +41 -0
  130. mlrun/api/utils/events/iguazio.py +217 -0
  131. mlrun/api/utils/events/nop.py +55 -0
  132. mlrun/api/utils/helpers.py +16 -13
  133. mlrun/api/utils/memory_reports.py +1 -1
  134. mlrun/api/utils/periodic.py +6 -3
  135. mlrun/api/utils/projects/__init__.py +1 -1
  136. mlrun/api/utils/projects/follower.py +33 -33
  137. mlrun/api/utils/projects/leader.py +36 -34
  138. mlrun/api/utils/projects/member.py +27 -27
  139. mlrun/api/utils/projects/remotes/__init__.py +1 -1
  140. mlrun/api/utils/projects/remotes/follower.py +13 -13
  141. mlrun/api/utils/projects/remotes/leader.py +10 -10
  142. mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
  143. mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
  144. mlrun/api/utils/scheduler.py +140 -51
  145. mlrun/api/utils/singletons/__init__.py +1 -1
  146. mlrun/api/utils/singletons/db.py +9 -15
  147. mlrun/api/utils/singletons/k8s.py +677 -5
  148. mlrun/api/utils/singletons/logs_dir.py +1 -1
  149. mlrun/api/utils/singletons/project_member.py +1 -1
  150. mlrun/api/utils/singletons/scheduler.py +1 -1
  151. mlrun/artifacts/__init__.py +2 -2
  152. mlrun/artifacts/base.py +8 -2
  153. mlrun/artifacts/dataset.py +5 -3
  154. mlrun/artifacts/manager.py +7 -1
  155. mlrun/artifacts/model.py +15 -4
  156. mlrun/artifacts/plots.py +1 -1
  157. mlrun/common/__init__.py +1 -1
  158. mlrun/common/constants.py +15 -0
  159. mlrun/common/model_monitoring.py +209 -0
  160. mlrun/common/schemas/__init__.py +167 -0
  161. mlrun/{api → common}/schemas/artifact.py +13 -14
  162. mlrun/{api → common}/schemas/auth.py +10 -8
  163. mlrun/{api → common}/schemas/background_task.py +3 -3
  164. mlrun/{api → common}/schemas/client_spec.py +1 -1
  165. mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
  166. mlrun/{api → common}/schemas/constants.py +21 -8
  167. mlrun/common/schemas/events.py +36 -0
  168. mlrun/{api → common}/schemas/feature_store.py +2 -1
  169. mlrun/{api → common}/schemas/frontend_spec.py +7 -6
  170. mlrun/{api → common}/schemas/function.py +5 -5
  171. mlrun/{api → common}/schemas/http.py +3 -3
  172. mlrun/common/schemas/hub.py +134 -0
  173. mlrun/{api → common}/schemas/k8s.py +3 -3
  174. mlrun/{api → common}/schemas/memory_reports.py +1 -1
  175. mlrun/common/schemas/model_endpoints.py +342 -0
  176. mlrun/common/schemas/notification.py +57 -0
  177. mlrun/{api → common}/schemas/object.py +6 -6
  178. mlrun/{api → common}/schemas/pipeline.py +3 -3
  179. mlrun/{api → common}/schemas/project.py +6 -5
  180. mlrun/common/schemas/regex.py +24 -0
  181. mlrun/common/schemas/runs.py +30 -0
  182. mlrun/{api → common}/schemas/runtime_resource.py +3 -3
  183. mlrun/{api → common}/schemas/schedule.py +19 -7
  184. mlrun/{api → common}/schemas/secret.py +3 -3
  185. mlrun/{api → common}/schemas/tag.py +2 -2
  186. mlrun/common/types.py +25 -0
  187. mlrun/config.py +152 -20
  188. mlrun/data_types/__init__.py +7 -2
  189. mlrun/data_types/data_types.py +4 -2
  190. mlrun/data_types/infer.py +1 -1
  191. mlrun/data_types/spark.py +10 -3
  192. mlrun/datastore/__init__.py +10 -3
  193. mlrun/datastore/azure_blob.py +1 -1
  194. mlrun/datastore/base.py +185 -53
  195. mlrun/datastore/datastore.py +1 -1
  196. mlrun/datastore/filestore.py +1 -1
  197. mlrun/datastore/google_cloud_storage.py +1 -1
  198. mlrun/datastore/inmem.py +4 -1
  199. mlrun/datastore/redis.py +1 -1
  200. mlrun/datastore/s3.py +1 -1
  201. mlrun/datastore/sources.py +192 -70
  202. mlrun/datastore/spark_udf.py +44 -0
  203. mlrun/datastore/store_resources.py +4 -4
  204. mlrun/datastore/targets.py +115 -45
  205. mlrun/datastore/utils.py +127 -5
  206. mlrun/datastore/v3io.py +1 -1
  207. mlrun/datastore/wasbfs/__init__.py +1 -1
  208. mlrun/datastore/wasbfs/fs.py +1 -1
  209. mlrun/db/__init__.py +7 -5
  210. mlrun/db/base.py +112 -68
  211. mlrun/db/httpdb.py +445 -277
  212. mlrun/db/nopdb.py +491 -0
  213. mlrun/db/sqldb.py +112 -65
  214. mlrun/errors.py +6 -1
  215. mlrun/execution.py +44 -22
  216. mlrun/feature_store/__init__.py +1 -1
  217. mlrun/feature_store/api.py +143 -95
  218. mlrun/feature_store/common.py +16 -20
  219. mlrun/feature_store/feature_set.py +42 -12
  220. mlrun/feature_store/feature_vector.py +32 -21
  221. mlrun/feature_store/ingestion.py +9 -12
  222. mlrun/feature_store/retrieval/__init__.py +3 -2
  223. mlrun/feature_store/retrieval/base.py +388 -66
  224. mlrun/feature_store/retrieval/dask_merger.py +63 -151
  225. mlrun/feature_store/retrieval/job.py +30 -12
  226. mlrun/feature_store/retrieval/local_merger.py +40 -133
  227. mlrun/feature_store/retrieval/spark_merger.py +129 -127
  228. mlrun/feature_store/retrieval/storey_merger.py +173 -0
  229. mlrun/feature_store/steps.py +132 -15
  230. mlrun/features.py +8 -3
  231. mlrun/frameworks/__init__.py +1 -1
  232. mlrun/frameworks/_common/__init__.py +1 -1
  233. mlrun/frameworks/_common/artifacts_library.py +1 -1
  234. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  235. mlrun/frameworks/_common/model_handler.py +1 -1
  236. mlrun/frameworks/_common/plan.py +1 -1
  237. mlrun/frameworks/_common/producer.py +1 -1
  238. mlrun/frameworks/_common/utils.py +1 -1
  239. mlrun/frameworks/_dl_common/__init__.py +1 -1
  240. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
  241. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  242. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  243. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  244. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  245. mlrun/frameworks/_dl_common/utils.py +1 -1
  246. mlrun/frameworks/_ml_common/__init__.py +1 -1
  247. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  248. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
  249. mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
  250. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  251. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  252. mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
  253. mlrun/frameworks/_ml_common/plan.py +1 -1
  254. mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
  255. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
  256. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
  257. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  258. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  259. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  260. mlrun/frameworks/_ml_common/producer.py +1 -1
  261. mlrun/frameworks/_ml_common/utils.py +1 -1
  262. mlrun/frameworks/auto_mlrun/__init__.py +1 -1
  263. mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
  264. mlrun/frameworks/huggingface/__init__.py +1 -1
  265. mlrun/frameworks/huggingface/model_server.py +1 -1
  266. mlrun/frameworks/lgbm/__init__.py +1 -1
  267. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
  268. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  269. mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
  270. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
  271. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
  272. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  273. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  274. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  275. mlrun/frameworks/lgbm/model_handler.py +1 -1
  276. mlrun/frameworks/lgbm/model_server.py +1 -1
  277. mlrun/frameworks/lgbm/utils.py +1 -1
  278. mlrun/frameworks/onnx/__init__.py +1 -1
  279. mlrun/frameworks/onnx/dataset.py +1 -1
  280. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  281. mlrun/frameworks/onnx/model_handler.py +1 -1
  282. mlrun/frameworks/onnx/model_server.py +1 -1
  283. mlrun/frameworks/parallel_coordinates.py +1 -1
  284. mlrun/frameworks/pytorch/__init__.py +1 -1
  285. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
  286. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  287. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  288. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  289. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  290. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  291. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  292. mlrun/frameworks/pytorch/model_handler.py +1 -1
  293. mlrun/frameworks/pytorch/model_server.py +1 -1
  294. mlrun/frameworks/pytorch/utils.py +1 -1
  295. mlrun/frameworks/sklearn/__init__.py +1 -1
  296. mlrun/frameworks/sklearn/estimator.py +1 -1
  297. mlrun/frameworks/sklearn/metric.py +1 -1
  298. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  299. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  300. mlrun/frameworks/sklearn/model_handler.py +1 -1
  301. mlrun/frameworks/sklearn/utils.py +1 -1
  302. mlrun/frameworks/tf_keras/__init__.py +1 -1
  303. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
  304. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  305. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  306. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  307. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  308. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  309. mlrun/frameworks/tf_keras/model_server.py +1 -1
  310. mlrun/frameworks/tf_keras/utils.py +1 -1
  311. mlrun/frameworks/xgboost/__init__.py +1 -1
  312. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  313. mlrun/frameworks/xgboost/model_handler.py +1 -1
  314. mlrun/frameworks/xgboost/utils.py +1 -1
  315. mlrun/k8s_utils.py +14 -765
  316. mlrun/kfpops.py +14 -17
  317. mlrun/launcher/__init__.py +13 -0
  318. mlrun/launcher/base.py +406 -0
  319. mlrun/launcher/client.py +159 -0
  320. mlrun/launcher/factory.py +50 -0
  321. mlrun/launcher/local.py +276 -0
  322. mlrun/launcher/remote.py +178 -0
  323. mlrun/lists.py +10 -2
  324. mlrun/mlutils/__init__.py +1 -1
  325. mlrun/mlutils/data.py +1 -1
  326. mlrun/mlutils/models.py +1 -1
  327. mlrun/mlutils/plots.py +1 -1
  328. mlrun/model.py +252 -14
  329. mlrun/model_monitoring/__init__.py +41 -0
  330. mlrun/model_monitoring/features_drift_table.py +1 -1
  331. mlrun/model_monitoring/helpers.py +123 -38
  332. mlrun/model_monitoring/model_endpoint.py +144 -0
  333. mlrun/model_monitoring/model_monitoring_batch.py +310 -259
  334. mlrun/model_monitoring/stores/__init__.py +106 -0
  335. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
  336. mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
  337. mlrun/model_monitoring/stores/models/__init__.py +23 -0
  338. mlrun/model_monitoring/stores/models/base.py +18 -0
  339. mlrun/model_monitoring/stores/models/mysql.py +100 -0
  340. mlrun/model_monitoring/stores/models/sqlite.py +98 -0
  341. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
  342. mlrun/model_monitoring/stream_processing_fs.py +239 -271
  343. mlrun/package/__init__.py +163 -0
  344. mlrun/package/context_handler.py +325 -0
  345. mlrun/package/errors.py +47 -0
  346. mlrun/package/packager.py +298 -0
  347. mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
  348. mlrun/package/packagers/default_packager.py +422 -0
  349. mlrun/package/packagers/numpy_packagers.py +612 -0
  350. mlrun/package/packagers/pandas_packagers.py +968 -0
  351. mlrun/package/packagers/python_standard_library_packagers.py +616 -0
  352. mlrun/package/packagers_manager.py +786 -0
  353. mlrun/package/utils/__init__.py +53 -0
  354. mlrun/package/utils/_archiver.py +226 -0
  355. mlrun/package/utils/_formatter.py +211 -0
  356. mlrun/package/utils/_pickler.py +234 -0
  357. mlrun/package/utils/_supported_format.py +71 -0
  358. mlrun/package/utils/log_hint_utils.py +93 -0
  359. mlrun/package/utils/type_hint_utils.py +298 -0
  360. mlrun/platforms/__init__.py +1 -1
  361. mlrun/platforms/iguazio.py +34 -2
  362. mlrun/platforms/other.py +1 -1
  363. mlrun/projects/__init__.py +1 -1
  364. mlrun/projects/operations.py +14 -9
  365. mlrun/projects/pipelines.py +31 -13
  366. mlrun/projects/project.py +762 -238
  367. mlrun/render.py +49 -19
  368. mlrun/run.py +57 -326
  369. mlrun/runtimes/__init__.py +3 -9
  370. mlrun/runtimes/base.py +247 -784
  371. mlrun/runtimes/constants.py +1 -1
  372. mlrun/runtimes/daskjob.py +45 -41
  373. mlrun/runtimes/funcdoc.py +43 -7
  374. mlrun/runtimes/function.py +66 -656
  375. mlrun/runtimes/function_reference.py +1 -1
  376. mlrun/runtimes/generators.py +1 -1
  377. mlrun/runtimes/kubejob.py +99 -116
  378. mlrun/runtimes/local.py +59 -66
  379. mlrun/runtimes/mpijob/__init__.py +1 -1
  380. mlrun/runtimes/mpijob/abstract.py +13 -15
  381. mlrun/runtimes/mpijob/v1.py +3 -1
  382. mlrun/runtimes/mpijob/v1alpha1.py +1 -1
  383. mlrun/runtimes/nuclio.py +1 -1
  384. mlrun/runtimes/pod.py +51 -26
  385. mlrun/runtimes/remotesparkjob.py +3 -1
  386. mlrun/runtimes/serving.py +12 -4
  387. mlrun/runtimes/sparkjob/__init__.py +1 -2
  388. mlrun/runtimes/sparkjob/abstract.py +44 -31
  389. mlrun/runtimes/sparkjob/spark3job.py +11 -9
  390. mlrun/runtimes/utils.py +61 -42
  391. mlrun/secrets.py +16 -18
  392. mlrun/serving/__init__.py +3 -2
  393. mlrun/serving/merger.py +1 -1
  394. mlrun/serving/remote.py +1 -1
  395. mlrun/serving/routers.py +39 -42
  396. mlrun/serving/server.py +23 -13
  397. mlrun/serving/serving_wrapper.py +1 -1
  398. mlrun/serving/states.py +172 -39
  399. mlrun/serving/utils.py +1 -1
  400. mlrun/serving/v1_serving.py +1 -1
  401. mlrun/serving/v2_serving.py +29 -21
  402. mlrun/utils/__init__.py +1 -2
  403. mlrun/utils/async_http.py +8 -1
  404. mlrun/utils/azure_vault.py +1 -1
  405. mlrun/utils/clones.py +2 -2
  406. mlrun/utils/condition_evaluator.py +65 -0
  407. mlrun/utils/db.py +52 -0
  408. mlrun/utils/helpers.py +188 -13
  409. mlrun/utils/http.py +89 -54
  410. mlrun/utils/logger.py +48 -8
  411. mlrun/utils/model_monitoring.py +132 -100
  412. mlrun/utils/notifications/__init__.py +1 -1
  413. mlrun/utils/notifications/notification/__init__.py +8 -6
  414. mlrun/utils/notifications/notification/base.py +20 -14
  415. mlrun/utils/notifications/notification/console.py +7 -4
  416. mlrun/utils/notifications/notification/git.py +36 -19
  417. mlrun/utils/notifications/notification/ipython.py +10 -8
  418. mlrun/utils/notifications/notification/slack.py +18 -13
  419. mlrun/utils/notifications/notification_pusher.py +377 -56
  420. mlrun/utils/regex.py +6 -1
  421. mlrun/utils/singleton.py +1 -1
  422. mlrun/utils/v3io_clients.py +1 -1
  423. mlrun/utils/vault.py +270 -269
  424. mlrun/utils/version/__init__.py +1 -1
  425. mlrun/utils/version/version.json +2 -2
  426. mlrun/utils/version/version.py +1 -1
  427. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
  428. mlrun-1.4.0.dist-info/RECORD +434 -0
  429. mlrun/api/api/endpoints/marketplace.py +0 -257
  430. mlrun/api/crud/marketplace.py +0 -221
  431. mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
  432. mlrun/api/db/filedb/db.py +0 -518
  433. mlrun/api/schemas/marketplace.py +0 -128
  434. mlrun/api/schemas/model_endpoints.py +0 -185
  435. mlrun/db/filedb.py +0 -891
  436. mlrun/feature_store/retrieval/online.py +0 -92
  437. mlrun/model_monitoring/constants.py +0 -67
  438. mlrun/runtimes/package/context_handler.py +0 -711
  439. mlrun/runtimes/sparkjob/spark2job.py +0 -59
  440. mlrun-1.3.3rc1.dist-info/RECORD +0 -381
  441. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
  442. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
  443. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
  444. {mlrun-1.3.3rc1.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
mlrun/db/nopdb.py ADDED
@@ -0,0 +1,491 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ import datetime
17
+ from typing import List, Optional, Union
18
+
19
+ import mlrun.common.schemas
20
+ import mlrun.errors
21
+
22
+ from ..config import config
23
+ from ..utils import logger
24
+ from .base import RunDBInterface
25
+
26
+
27
+ class NopDB(RunDBInterface):
28
+ def __init__(self, url=None, *args, **kwargs):
29
+ self.url = url
30
+
31
+ def __getattribute__(self, attr):
32
+ def nop(*args, **kwargs):
33
+ env_var_message = (
34
+ "MLRUN_DBPATH is not set. Set this environment variable to the URL of the API "
35
+ "server in order to connect"
36
+ )
37
+ if config.httpdb.nop_db.raise_error:
38
+ raise mlrun.errors.MLRunBadRequestError(env_var_message)
39
+
40
+ if config.httpdb.nop_db.verbose:
41
+ logger.warning(
42
+ "Could not detect path to API server, not connected to API server!"
43
+ )
44
+ logger.warning(env_var_message)
45
+
46
+ return
47
+
48
+ if attr == "connect":
49
+ return super().__getattribute__(attr)
50
+ else:
51
+ nop()
52
+ return super().__getattribute__(attr)
53
+
54
+ def connect(self, secrets=None):
55
+ pass
56
+
57
+ def store_log(self, uid, project="", body=None, append=False):
58
+ pass
59
+
60
+ def get_log(self, uid, project="", offset=0, size=0):
61
+ pass
62
+
63
+ def store_run(self, struct, uid, project="", iter=0):
64
+ pass
65
+
66
+ def update_run(self, updates: dict, uid, project="", iter=0):
67
+ pass
68
+
69
+ def abort_run(self, uid, project="", iter=0, timeout=45):
70
+ pass
71
+
72
+ def read_run(self, uid, project="", iter=0):
73
+ pass
74
+
75
+ def list_runs(
76
+ self,
77
+ name="",
78
+ uid: Optional[Union[str, List[str]]] = None,
79
+ project="",
80
+ labels=None,
81
+ state="",
82
+ sort=True,
83
+ last=0,
84
+ iter=False,
85
+ start_time_from: datetime.datetime = None,
86
+ start_time_to: datetime.datetime = None,
87
+ last_update_time_from: datetime.datetime = None,
88
+ last_update_time_to: datetime.datetime = None,
89
+ partition_by: Union[mlrun.common.schemas.RunPartitionByField, str] = None,
90
+ rows_per_partition: int = 1,
91
+ partition_sort_by: Union[mlrun.common.schemas.SortField, str] = None,
92
+ partition_order: Union[
93
+ mlrun.common.schemas.OrderType, str
94
+ ] = mlrun.common.schemas.OrderType.desc,
95
+ max_partitions: int = 0,
96
+ ):
97
+ pass
98
+
99
+ def del_run(self, uid, project="", iter=0):
100
+ pass
101
+
102
+ def del_runs(self, name="", project="", labels=None, state="", days_ago=0):
103
+ pass
104
+
105
+ def store_artifact(self, key, artifact, uid, iter=None, tag="", project=""):
106
+ pass
107
+
108
+ def read_artifact(self, key, tag="", iter=None, project=""):
109
+ pass
110
+
111
+ def list_artifacts(
112
+ self,
113
+ name="",
114
+ project="",
115
+ tag="",
116
+ labels=None,
117
+ since=None,
118
+ until=None,
119
+ iter: int = None,
120
+ best_iteration: bool = False,
121
+ kind: str = None,
122
+ category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
123
+ ):
124
+ pass
125
+
126
+ def del_artifact(self, key, tag="", project=""):
127
+ pass
128
+
129
+ def del_artifacts(self, name="", project="", tag="", labels=None):
130
+ pass
131
+
132
+ def store_function(self, function, name, project="", tag="", versioned=False):
133
+ pass
134
+
135
+ def get_function(self, name, project="", tag="", hash_key=""):
136
+ pass
137
+
138
+ def delete_function(self, name: str, project: str = ""):
139
+ pass
140
+
141
+ def list_functions(self, name=None, project="", tag="", labels=None):
142
+ pass
143
+
144
+ def tag_objects(
145
+ self,
146
+ project: str,
147
+ tag_name: str,
148
+ tag_objects: mlrun.common.schemas.TagObjects,
149
+ replace: bool = False,
150
+ ):
151
+ pass
152
+
153
+ def delete_objects_tag(
154
+ self, project: str, tag_name: str, tag_objects: mlrun.common.schemas.TagObjects
155
+ ):
156
+ pass
157
+
158
+ def tag_artifacts(
159
+ self, artifacts, project: str, tag_name: str, replace: bool = False
160
+ ):
161
+ pass
162
+
163
+ def delete_artifacts_tags(self, artifacts, project: str, tag_name: str):
164
+ pass
165
+
166
+ def delete_project(
167
+ self,
168
+ name: str,
169
+ deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
170
+ ):
171
+ pass
172
+
173
+ def store_project(
174
+ self, name: str, project: mlrun.common.schemas.Project
175
+ ) -> mlrun.common.schemas.Project:
176
+ pass
177
+
178
+ def patch_project(
179
+ self,
180
+ name: str,
181
+ project: dict,
182
+ patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
183
+ ) -> mlrun.common.schemas.Project:
184
+ pass
185
+
186
+ def create_project(
187
+ self, project: mlrun.common.schemas.Project
188
+ ) -> mlrun.common.schemas.Project:
189
+ pass
190
+
191
+ def list_projects(
192
+ self,
193
+ owner: str = None,
194
+ format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.full,
195
+ labels: List[str] = None,
196
+ state: mlrun.common.schemas.ProjectState = None,
197
+ ) -> mlrun.common.schemas.ProjectsOutput:
198
+ pass
199
+
200
+ def get_project(self, name: str) -> mlrun.common.schemas.Project:
201
+ pass
202
+
203
+ def list_artifact_tags(
204
+ self,
205
+ project=None,
206
+ category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
207
+ ):
208
+ pass
209
+
210
+ def create_feature_set(
211
+ self,
212
+ feature_set: Union[dict, mlrun.common.schemas.FeatureSet],
213
+ project="",
214
+ versioned=True,
215
+ ) -> dict:
216
+ pass
217
+
218
+ def get_feature_set(
219
+ self, name: str, project: str = "", tag: str = None, uid: str = None
220
+ ) -> dict:
221
+ pass
222
+
223
+ def list_features(
224
+ self,
225
+ project: str,
226
+ name: str = None,
227
+ tag: str = None,
228
+ entities: List[str] = None,
229
+ labels: List[str] = None,
230
+ ) -> mlrun.common.schemas.FeaturesOutput:
231
+ pass
232
+
233
+ def list_entities(
234
+ self, project: str, name: str = None, tag: str = None, labels: List[str] = None
235
+ ) -> mlrun.common.schemas.EntitiesOutput:
236
+ pass
237
+
238
+ def list_feature_sets(
239
+ self,
240
+ project: str = "",
241
+ name: str = None,
242
+ tag: str = None,
243
+ state: str = None,
244
+ entities: List[str] = None,
245
+ features: List[str] = None,
246
+ labels: List[str] = None,
247
+ partition_by: Union[
248
+ mlrun.common.schemas.FeatureStorePartitionByField, str
249
+ ] = None,
250
+ rows_per_partition: int = 1,
251
+ partition_sort_by: Union[mlrun.common.schemas.SortField, str] = None,
252
+ partition_order: Union[
253
+ mlrun.common.schemas.OrderType, str
254
+ ] = mlrun.common.schemas.OrderType.desc,
255
+ ) -> List[dict]:
256
+ pass
257
+
258
+ def store_feature_set(
259
+ self,
260
+ feature_set: Union[dict, mlrun.common.schemas.FeatureSet],
261
+ name=None,
262
+ project="",
263
+ tag=None,
264
+ uid=None,
265
+ versioned=True,
266
+ ):
267
+ pass
268
+
269
+ def patch_feature_set(
270
+ self,
271
+ name,
272
+ feature_set: dict,
273
+ project="",
274
+ tag=None,
275
+ uid=None,
276
+ patch_mode: Union[
277
+ str, mlrun.common.schemas.PatchMode
278
+ ] = mlrun.common.schemas.PatchMode.replace,
279
+ ):
280
+ pass
281
+
282
+ def delete_feature_set(self, name, project="", tag=None, uid=None):
283
+ pass
284
+
285
+ def create_feature_vector(
286
+ self,
287
+ feature_vector: Union[dict, mlrun.common.schemas.FeatureVector],
288
+ project="",
289
+ versioned=True,
290
+ ) -> dict:
291
+ pass
292
+
293
+ def get_feature_vector(
294
+ self, name: str, project: str = "", tag: str = None, uid: str = None
295
+ ) -> dict:
296
+ pass
297
+
298
+ def list_feature_vectors(
299
+ self,
300
+ project: str = "",
301
+ name: str = None,
302
+ tag: str = None,
303
+ state: str = None,
304
+ labels: List[str] = None,
305
+ partition_by: Union[
306
+ mlrun.common.schemas.FeatureStorePartitionByField, str
307
+ ] = None,
308
+ rows_per_partition: int = 1,
309
+ partition_sort_by: Union[mlrun.common.schemas.SortField, str] = None,
310
+ partition_order: Union[
311
+ mlrun.common.schemas.OrderType, str
312
+ ] = mlrun.common.schemas.OrderType.desc,
313
+ ) -> List[dict]:
314
+ pass
315
+
316
+ def store_feature_vector(
317
+ self,
318
+ feature_vector: Union[dict, mlrun.common.schemas.FeatureVector],
319
+ name=None,
320
+ project="",
321
+ tag=None,
322
+ uid=None,
323
+ versioned=True,
324
+ ):
325
+ pass
326
+
327
+ def patch_feature_vector(
328
+ self,
329
+ name,
330
+ feature_vector_update: dict,
331
+ project="",
332
+ tag=None,
333
+ uid=None,
334
+ patch_mode: Union[
335
+ str, mlrun.common.schemas.PatchMode
336
+ ] = mlrun.common.schemas.PatchMode.replace,
337
+ ):
338
+ pass
339
+
340
+ def delete_feature_vector(self, name, project="", tag=None, uid=None):
341
+ pass
342
+
343
+ def list_pipelines(
344
+ self,
345
+ project: str,
346
+ namespace: str = None,
347
+ sort_by: str = "",
348
+ page_token: str = "",
349
+ filter_: str = "",
350
+ format_: Union[
351
+ str, mlrun.common.schemas.PipelinesFormat
352
+ ] = mlrun.common.schemas.PipelinesFormat.metadata_only,
353
+ page_size: int = None,
354
+ ) -> mlrun.common.schemas.PipelinesOutput:
355
+ pass
356
+
357
+ def create_project_secrets(
358
+ self,
359
+ project: str,
360
+ provider: Union[
361
+ str, mlrun.common.schemas.SecretProviderName
362
+ ] = mlrun.common.schemas.SecretProviderName.kubernetes,
363
+ secrets: dict = None,
364
+ ):
365
+ pass
366
+
367
+ def list_project_secrets(
368
+ self,
369
+ project: str,
370
+ token: str,
371
+ provider: Union[
372
+ str, mlrun.common.schemas.SecretProviderName
373
+ ] = mlrun.common.schemas.SecretProviderName.kubernetes,
374
+ secrets: List[str] = None,
375
+ ) -> mlrun.common.schemas.SecretsData:
376
+ pass
377
+
378
+ def list_project_secret_keys(
379
+ self,
380
+ project: str,
381
+ provider: Union[
382
+ str, mlrun.common.schemas.SecretProviderName
383
+ ] = mlrun.common.schemas.SecretProviderName.kubernetes,
384
+ token: str = None,
385
+ ) -> mlrun.common.schemas.SecretKeysData:
386
+ pass
387
+
388
+ def delete_project_secrets(
389
+ self,
390
+ project: str,
391
+ provider: Union[
392
+ str, mlrun.common.schemas.SecretProviderName
393
+ ] = mlrun.common.schemas.SecretProviderName.kubernetes,
394
+ secrets: List[str] = None,
395
+ ):
396
+ pass
397
+
398
+ def create_user_secrets(
399
+ self,
400
+ user: str,
401
+ provider: Union[
402
+ str, mlrun.common.schemas.SecretProviderName
403
+ ] = mlrun.common.schemas.SecretProviderName.vault,
404
+ secrets: dict = None,
405
+ ):
406
+ pass
407
+
408
+ def create_model_endpoint(
409
+ self,
410
+ project: str,
411
+ endpoint_id: str,
412
+ model_endpoint: mlrun.common.schemas.ModelEndpoint,
413
+ ):
414
+ pass
415
+
416
+ def delete_model_endpoint(self, project: str, endpoint_id: str):
417
+ pass
418
+
419
+ def list_model_endpoints(
420
+ self,
421
+ project: str,
422
+ model: Optional[str] = None,
423
+ function: Optional[str] = None,
424
+ labels: List[str] = None,
425
+ start: str = "now-1h",
426
+ end: str = "now",
427
+ metrics: Optional[List[str]] = None,
428
+ ):
429
+ pass
430
+
431
+ def get_model_endpoint(
432
+ self,
433
+ project: str,
434
+ endpoint_id: str,
435
+ start: Optional[str] = None,
436
+ end: Optional[str] = None,
437
+ metrics: Optional[List[str]] = None,
438
+ features: bool = False,
439
+ ):
440
+ pass
441
+
442
+ def patch_model_endpoint(self, project: str, endpoint_id: str, attributes: dict):
443
+ pass
444
+
445
+ def create_hub_source(
446
+ self, source: Union[dict, mlrun.common.schemas.IndexedHubSource]
447
+ ):
448
+ pass
449
+
450
+ def store_hub_source(
451
+ self,
452
+ source_name: str,
453
+ source: Union[dict, mlrun.common.schemas.IndexedHubSource],
454
+ ):
455
+ pass
456
+
457
+ def list_hub_sources(self):
458
+ pass
459
+
460
+ def get_hub_source(self, source_name: str):
461
+ pass
462
+
463
+ def delete_hub_source(self, source_name: str):
464
+ pass
465
+
466
+ def get_hub_catalog(
467
+ self,
468
+ source_name: str,
469
+ channel: str = None,
470
+ version: str = None,
471
+ tag: str = None,
472
+ force_refresh: bool = False,
473
+ ):
474
+ pass
475
+
476
+ def get_hub_item(
477
+ self,
478
+ source_name: str,
479
+ item_name: str,
480
+ channel: str = "development",
481
+ version: str = None,
482
+ tag: str = "latest",
483
+ force_refresh: bool = False,
484
+ ):
485
+ pass
486
+
487
+ def verify_authorization(
488
+ self,
489
+ authorization_verification_input: mlrun.common.schemas.AuthorizationVerificationInput,
490
+ ):
491
+ pass