mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (444) hide show
  1. mlrun/__init__.py +3 -3
  2. mlrun/__main__.py +79 -37
  3. mlrun/api/__init__.py +1 -1
  4. mlrun/api/api/__init__.py +1 -1
  5. mlrun/api/api/api.py +4 -4
  6. mlrun/api/api/deps.py +10 -21
  7. mlrun/api/api/endpoints/__init__.py +1 -1
  8. mlrun/api/api/endpoints/artifacts.py +64 -36
  9. mlrun/api/api/endpoints/auth.py +4 -4
  10. mlrun/api/api/endpoints/background_tasks.py +11 -11
  11. mlrun/api/api/endpoints/client_spec.py +5 -5
  12. mlrun/api/api/endpoints/clusterization_spec.py +6 -4
  13. mlrun/api/api/endpoints/feature_store.py +124 -115
  14. mlrun/api/api/endpoints/files.py +22 -14
  15. mlrun/api/api/endpoints/frontend_spec.py +28 -21
  16. mlrun/api/api/endpoints/functions.py +142 -87
  17. mlrun/api/api/endpoints/grafana_proxy.py +89 -442
  18. mlrun/api/api/endpoints/healthz.py +20 -7
  19. mlrun/api/api/endpoints/hub.py +320 -0
  20. mlrun/api/api/endpoints/internal/__init__.py +1 -1
  21. mlrun/api/api/endpoints/internal/config.py +1 -1
  22. mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
  23. mlrun/api/api/endpoints/logs.py +11 -11
  24. mlrun/api/api/endpoints/model_endpoints.py +74 -70
  25. mlrun/api/api/endpoints/operations.py +13 -9
  26. mlrun/api/api/endpoints/pipelines.py +93 -88
  27. mlrun/api/api/endpoints/projects.py +35 -35
  28. mlrun/api/api/endpoints/runs.py +69 -27
  29. mlrun/api/api/endpoints/runtime_resources.py +28 -28
  30. mlrun/api/api/endpoints/schedules.py +98 -41
  31. mlrun/api/api/endpoints/secrets.py +37 -32
  32. mlrun/api/api/endpoints/submit.py +12 -12
  33. mlrun/api/api/endpoints/tags.py +20 -22
  34. mlrun/api/api/utils.py +251 -42
  35. mlrun/api/constants.py +1 -1
  36. mlrun/api/crud/__init__.py +18 -15
  37. mlrun/api/crud/artifacts.py +10 -10
  38. mlrun/api/crud/client_spec.py +4 -4
  39. mlrun/api/crud/clusterization_spec.py +3 -3
  40. mlrun/api/crud/feature_store.py +54 -46
  41. mlrun/api/crud/functions.py +3 -3
  42. mlrun/api/crud/hub.py +312 -0
  43. mlrun/api/crud/logs.py +11 -9
  44. mlrun/api/crud/model_monitoring/__init__.py +3 -3
  45. mlrun/api/crud/model_monitoring/grafana.py +435 -0
  46. mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
  47. mlrun/api/crud/notifications.py +149 -0
  48. mlrun/api/crud/pipelines.py +67 -52
  49. mlrun/api/crud/projects.py +51 -23
  50. mlrun/api/crud/runs.py +7 -5
  51. mlrun/api/crud/runtime_resources.py +13 -13
  52. mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
  53. mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
  54. mlrun/api/crud/runtimes/nuclio/function.py +505 -0
  55. mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
  56. mlrun/api/crud/secrets.py +88 -46
  57. mlrun/api/crud/tags.py +5 -5
  58. mlrun/api/db/__init__.py +1 -1
  59. mlrun/api/db/base.py +102 -54
  60. mlrun/api/db/init_db.py +2 -3
  61. mlrun/api/db/session.py +4 -12
  62. mlrun/api/db/sqldb/__init__.py +1 -1
  63. mlrun/api/db/sqldb/db.py +439 -196
  64. mlrun/api/db/sqldb/helpers.py +1 -1
  65. mlrun/api/db/sqldb/models/__init__.py +3 -3
  66. mlrun/api/db/sqldb/models/models_mysql.py +82 -64
  67. mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
  68. mlrun/api/db/sqldb/session.py +27 -20
  69. mlrun/api/initial_data.py +82 -24
  70. mlrun/api/launcher.py +196 -0
  71. mlrun/api/main.py +91 -22
  72. mlrun/api/middlewares.py +6 -5
  73. mlrun/api/migrations_mysql/env.py +1 -1
  74. mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
  75. mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
  76. mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
  77. mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
  78. mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
  79. mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
  80. mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
  81. mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
  82. mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
  83. mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
  84. mlrun/api/migrations_sqlite/env.py +1 -1
  85. mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
  86. mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
  87. mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
  88. mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
  89. mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
  90. mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
  91. mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
  92. mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
  93. mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
  94. mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
  95. mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
  96. mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
  97. mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
  98. mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
  99. mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
  100. mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
  101. mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
  102. mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
  103. mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
  104. mlrun/api/schemas/__init__.py +216 -138
  105. mlrun/api/utils/__init__.py +1 -1
  106. mlrun/api/utils/asyncio.py +1 -1
  107. mlrun/api/utils/auth/__init__.py +1 -1
  108. mlrun/api/utils/auth/providers/__init__.py +1 -1
  109. mlrun/api/utils/auth/providers/base.py +7 -7
  110. mlrun/api/utils/auth/providers/nop.py +6 -7
  111. mlrun/api/utils/auth/providers/opa.py +17 -17
  112. mlrun/api/utils/auth/verifier.py +36 -34
  113. mlrun/api/utils/background_tasks.py +24 -24
  114. mlrun/{builder.py → api/utils/builder.py} +216 -123
  115. mlrun/api/utils/clients/__init__.py +1 -1
  116. mlrun/api/utils/clients/chief.py +19 -4
  117. mlrun/api/utils/clients/iguazio.py +106 -60
  118. mlrun/api/utils/clients/log_collector.py +1 -1
  119. mlrun/api/utils/clients/nuclio.py +23 -23
  120. mlrun/api/utils/clients/protocols/grpc.py +2 -2
  121. mlrun/api/utils/db/__init__.py +1 -1
  122. mlrun/api/utils/db/alembic.py +1 -1
  123. mlrun/api/utils/db/backup.py +1 -1
  124. mlrun/api/utils/db/mysql.py +24 -25
  125. mlrun/api/utils/db/sql_collation.py +1 -1
  126. mlrun/api/utils/db/sqlite_migration.py +2 -2
  127. mlrun/api/utils/events/__init__.py +14 -0
  128. mlrun/api/utils/events/base.py +57 -0
  129. mlrun/api/utils/events/events_factory.py +41 -0
  130. mlrun/api/utils/events/iguazio.py +217 -0
  131. mlrun/api/utils/events/nop.py +55 -0
  132. mlrun/api/utils/helpers.py +16 -13
  133. mlrun/api/utils/memory_reports.py +1 -1
  134. mlrun/api/utils/periodic.py +6 -3
  135. mlrun/api/utils/projects/__init__.py +1 -1
  136. mlrun/api/utils/projects/follower.py +33 -33
  137. mlrun/api/utils/projects/leader.py +36 -34
  138. mlrun/api/utils/projects/member.py +27 -27
  139. mlrun/api/utils/projects/remotes/__init__.py +1 -1
  140. mlrun/api/utils/projects/remotes/follower.py +13 -13
  141. mlrun/api/utils/projects/remotes/leader.py +10 -10
  142. mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
  143. mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
  144. mlrun/api/utils/scheduler.py +140 -51
  145. mlrun/api/utils/singletons/__init__.py +1 -1
  146. mlrun/api/utils/singletons/db.py +9 -15
  147. mlrun/api/utils/singletons/k8s.py +677 -5
  148. mlrun/api/utils/singletons/logs_dir.py +1 -1
  149. mlrun/api/utils/singletons/project_member.py +1 -1
  150. mlrun/api/utils/singletons/scheduler.py +1 -1
  151. mlrun/artifacts/__init__.py +2 -2
  152. mlrun/artifacts/base.py +8 -2
  153. mlrun/artifacts/dataset.py +5 -3
  154. mlrun/artifacts/manager.py +7 -1
  155. mlrun/artifacts/model.py +15 -4
  156. mlrun/artifacts/plots.py +1 -1
  157. mlrun/common/__init__.py +1 -1
  158. mlrun/common/constants.py +15 -0
  159. mlrun/common/model_monitoring.py +209 -0
  160. mlrun/common/schemas/__init__.py +167 -0
  161. mlrun/{api → common}/schemas/artifact.py +13 -14
  162. mlrun/{api → common}/schemas/auth.py +10 -8
  163. mlrun/{api → common}/schemas/background_task.py +3 -3
  164. mlrun/{api → common}/schemas/client_spec.py +1 -1
  165. mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
  166. mlrun/{api → common}/schemas/constants.py +21 -8
  167. mlrun/common/schemas/events.py +36 -0
  168. mlrun/{api → common}/schemas/feature_store.py +2 -1
  169. mlrun/{api → common}/schemas/frontend_spec.py +7 -6
  170. mlrun/{api → common}/schemas/function.py +5 -5
  171. mlrun/{api → common}/schemas/http.py +3 -3
  172. mlrun/common/schemas/hub.py +134 -0
  173. mlrun/{api → common}/schemas/k8s.py +3 -3
  174. mlrun/{api → common}/schemas/memory_reports.py +1 -1
  175. mlrun/common/schemas/model_endpoints.py +342 -0
  176. mlrun/common/schemas/notification.py +57 -0
  177. mlrun/{api → common}/schemas/object.py +6 -6
  178. mlrun/{api → common}/schemas/pipeline.py +3 -3
  179. mlrun/{api → common}/schemas/project.py +6 -5
  180. mlrun/common/schemas/regex.py +24 -0
  181. mlrun/common/schemas/runs.py +30 -0
  182. mlrun/{api → common}/schemas/runtime_resource.py +3 -3
  183. mlrun/{api → common}/schemas/schedule.py +19 -7
  184. mlrun/{api → common}/schemas/secret.py +3 -3
  185. mlrun/{api → common}/schemas/tag.py +2 -2
  186. mlrun/common/types.py +25 -0
  187. mlrun/config.py +152 -20
  188. mlrun/data_types/__init__.py +7 -2
  189. mlrun/data_types/data_types.py +4 -2
  190. mlrun/data_types/infer.py +1 -1
  191. mlrun/data_types/spark.py +10 -3
  192. mlrun/datastore/__init__.py +10 -3
  193. mlrun/datastore/azure_blob.py +1 -1
  194. mlrun/datastore/base.py +185 -53
  195. mlrun/datastore/datastore.py +1 -1
  196. mlrun/datastore/filestore.py +1 -1
  197. mlrun/datastore/google_cloud_storage.py +1 -1
  198. mlrun/datastore/inmem.py +4 -1
  199. mlrun/datastore/redis.py +1 -1
  200. mlrun/datastore/s3.py +1 -1
  201. mlrun/datastore/sources.py +192 -70
  202. mlrun/datastore/spark_udf.py +44 -0
  203. mlrun/datastore/store_resources.py +4 -4
  204. mlrun/datastore/targets.py +115 -45
  205. mlrun/datastore/utils.py +127 -5
  206. mlrun/datastore/v3io.py +1 -1
  207. mlrun/datastore/wasbfs/__init__.py +1 -1
  208. mlrun/datastore/wasbfs/fs.py +1 -1
  209. mlrun/db/__init__.py +7 -5
  210. mlrun/db/base.py +112 -68
  211. mlrun/db/httpdb.py +445 -277
  212. mlrun/db/nopdb.py +491 -0
  213. mlrun/db/sqldb.py +112 -65
  214. mlrun/errors.py +6 -1
  215. mlrun/execution.py +44 -22
  216. mlrun/feature_store/__init__.py +1 -1
  217. mlrun/feature_store/api.py +143 -95
  218. mlrun/feature_store/common.py +16 -20
  219. mlrun/feature_store/feature_set.py +42 -12
  220. mlrun/feature_store/feature_vector.py +32 -21
  221. mlrun/feature_store/ingestion.py +9 -12
  222. mlrun/feature_store/retrieval/__init__.py +3 -2
  223. mlrun/feature_store/retrieval/base.py +388 -66
  224. mlrun/feature_store/retrieval/dask_merger.py +63 -151
  225. mlrun/feature_store/retrieval/job.py +30 -12
  226. mlrun/feature_store/retrieval/local_merger.py +40 -133
  227. mlrun/feature_store/retrieval/spark_merger.py +129 -127
  228. mlrun/feature_store/retrieval/storey_merger.py +173 -0
  229. mlrun/feature_store/steps.py +132 -15
  230. mlrun/features.py +8 -3
  231. mlrun/frameworks/__init__.py +1 -1
  232. mlrun/frameworks/_common/__init__.py +1 -1
  233. mlrun/frameworks/_common/artifacts_library.py +1 -1
  234. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  235. mlrun/frameworks/_common/model_handler.py +1 -1
  236. mlrun/frameworks/_common/plan.py +1 -1
  237. mlrun/frameworks/_common/producer.py +1 -1
  238. mlrun/frameworks/_common/utils.py +1 -1
  239. mlrun/frameworks/_dl_common/__init__.py +1 -1
  240. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
  241. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  242. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  243. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  244. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  245. mlrun/frameworks/_dl_common/utils.py +1 -1
  246. mlrun/frameworks/_ml_common/__init__.py +1 -1
  247. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  248. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
  249. mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
  250. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  251. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  252. mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
  253. mlrun/frameworks/_ml_common/plan.py +1 -1
  254. mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
  255. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
  256. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
  257. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  258. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  259. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  260. mlrun/frameworks/_ml_common/producer.py +1 -1
  261. mlrun/frameworks/_ml_common/utils.py +1 -1
  262. mlrun/frameworks/auto_mlrun/__init__.py +1 -1
  263. mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
  264. mlrun/frameworks/huggingface/__init__.py +1 -1
  265. mlrun/frameworks/huggingface/model_server.py +1 -1
  266. mlrun/frameworks/lgbm/__init__.py +1 -1
  267. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
  268. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  269. mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
  270. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
  271. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
  272. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  273. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  274. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  275. mlrun/frameworks/lgbm/model_handler.py +1 -1
  276. mlrun/frameworks/lgbm/model_server.py +1 -1
  277. mlrun/frameworks/lgbm/utils.py +1 -1
  278. mlrun/frameworks/onnx/__init__.py +1 -1
  279. mlrun/frameworks/onnx/dataset.py +1 -1
  280. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  281. mlrun/frameworks/onnx/model_handler.py +1 -1
  282. mlrun/frameworks/onnx/model_server.py +1 -1
  283. mlrun/frameworks/parallel_coordinates.py +1 -1
  284. mlrun/frameworks/pytorch/__init__.py +1 -1
  285. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
  286. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  287. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  288. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  289. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  290. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  291. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  292. mlrun/frameworks/pytorch/model_handler.py +1 -1
  293. mlrun/frameworks/pytorch/model_server.py +1 -1
  294. mlrun/frameworks/pytorch/utils.py +1 -1
  295. mlrun/frameworks/sklearn/__init__.py +1 -1
  296. mlrun/frameworks/sklearn/estimator.py +1 -1
  297. mlrun/frameworks/sklearn/metric.py +1 -1
  298. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  299. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  300. mlrun/frameworks/sklearn/model_handler.py +1 -1
  301. mlrun/frameworks/sklearn/utils.py +1 -1
  302. mlrun/frameworks/tf_keras/__init__.py +1 -1
  303. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
  304. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  305. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  306. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  307. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  308. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  309. mlrun/frameworks/tf_keras/model_server.py +1 -1
  310. mlrun/frameworks/tf_keras/utils.py +1 -1
  311. mlrun/frameworks/xgboost/__init__.py +1 -1
  312. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  313. mlrun/frameworks/xgboost/model_handler.py +1 -1
  314. mlrun/frameworks/xgboost/utils.py +1 -1
  315. mlrun/k8s_utils.py +14 -765
  316. mlrun/kfpops.py +14 -17
  317. mlrun/launcher/__init__.py +13 -0
  318. mlrun/launcher/base.py +406 -0
  319. mlrun/launcher/client.py +159 -0
  320. mlrun/launcher/factory.py +50 -0
  321. mlrun/launcher/local.py +276 -0
  322. mlrun/launcher/remote.py +178 -0
  323. mlrun/lists.py +10 -2
  324. mlrun/mlutils/__init__.py +1 -1
  325. mlrun/mlutils/data.py +1 -1
  326. mlrun/mlutils/models.py +1 -1
  327. mlrun/mlutils/plots.py +1 -1
  328. mlrun/model.py +252 -14
  329. mlrun/model_monitoring/__init__.py +41 -0
  330. mlrun/model_monitoring/features_drift_table.py +1 -1
  331. mlrun/model_monitoring/helpers.py +123 -38
  332. mlrun/model_monitoring/model_endpoint.py +144 -0
  333. mlrun/model_monitoring/model_monitoring_batch.py +310 -259
  334. mlrun/model_monitoring/stores/__init__.py +106 -0
  335. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
  336. mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
  337. mlrun/model_monitoring/stores/models/__init__.py +23 -0
  338. mlrun/model_monitoring/stores/models/base.py +18 -0
  339. mlrun/model_monitoring/stores/models/mysql.py +100 -0
  340. mlrun/model_monitoring/stores/models/sqlite.py +98 -0
  341. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
  342. mlrun/model_monitoring/stream_processing_fs.py +239 -271
  343. mlrun/package/__init__.py +163 -0
  344. mlrun/package/context_handler.py +325 -0
  345. mlrun/package/errors.py +47 -0
  346. mlrun/package/packager.py +298 -0
  347. mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
  348. mlrun/package/packagers/default_packager.py +422 -0
  349. mlrun/package/packagers/numpy_packagers.py +612 -0
  350. mlrun/package/packagers/pandas_packagers.py +968 -0
  351. mlrun/package/packagers/python_standard_library_packagers.py +616 -0
  352. mlrun/package/packagers_manager.py +786 -0
  353. mlrun/package/utils/__init__.py +53 -0
  354. mlrun/package/utils/_archiver.py +226 -0
  355. mlrun/package/utils/_formatter.py +211 -0
  356. mlrun/package/utils/_pickler.py +234 -0
  357. mlrun/package/utils/_supported_format.py +71 -0
  358. mlrun/package/utils/log_hint_utils.py +93 -0
  359. mlrun/package/utils/type_hint_utils.py +298 -0
  360. mlrun/platforms/__init__.py +1 -1
  361. mlrun/platforms/iguazio.py +34 -2
  362. mlrun/platforms/other.py +1 -1
  363. mlrun/projects/__init__.py +1 -1
  364. mlrun/projects/operations.py +14 -9
  365. mlrun/projects/pipelines.py +31 -13
  366. mlrun/projects/project.py +762 -238
  367. mlrun/render.py +49 -19
  368. mlrun/run.py +57 -326
  369. mlrun/runtimes/__init__.py +3 -9
  370. mlrun/runtimes/base.py +247 -784
  371. mlrun/runtimes/constants.py +1 -1
  372. mlrun/runtimes/daskjob.py +45 -41
  373. mlrun/runtimes/funcdoc.py +43 -7
  374. mlrun/runtimes/function.py +66 -656
  375. mlrun/runtimes/function_reference.py +1 -1
  376. mlrun/runtimes/generators.py +1 -1
  377. mlrun/runtimes/kubejob.py +99 -116
  378. mlrun/runtimes/local.py +59 -66
  379. mlrun/runtimes/mpijob/__init__.py +1 -1
  380. mlrun/runtimes/mpijob/abstract.py +13 -15
  381. mlrun/runtimes/mpijob/v1.py +3 -1
  382. mlrun/runtimes/mpijob/v1alpha1.py +1 -1
  383. mlrun/runtimes/nuclio.py +1 -1
  384. mlrun/runtimes/pod.py +51 -26
  385. mlrun/runtimes/remotesparkjob.py +3 -1
  386. mlrun/runtimes/serving.py +12 -4
  387. mlrun/runtimes/sparkjob/__init__.py +1 -2
  388. mlrun/runtimes/sparkjob/abstract.py +44 -31
  389. mlrun/runtimes/sparkjob/spark3job.py +11 -9
  390. mlrun/runtimes/utils.py +61 -42
  391. mlrun/secrets.py +16 -18
  392. mlrun/serving/__init__.py +3 -2
  393. mlrun/serving/merger.py +1 -1
  394. mlrun/serving/remote.py +1 -1
  395. mlrun/serving/routers.py +39 -42
  396. mlrun/serving/server.py +23 -13
  397. mlrun/serving/serving_wrapper.py +1 -1
  398. mlrun/serving/states.py +172 -39
  399. mlrun/serving/utils.py +1 -1
  400. mlrun/serving/v1_serving.py +1 -1
  401. mlrun/serving/v2_serving.py +29 -21
  402. mlrun/utils/__init__.py +1 -2
  403. mlrun/utils/async_http.py +8 -1
  404. mlrun/utils/azure_vault.py +1 -1
  405. mlrun/utils/clones.py +2 -2
  406. mlrun/utils/condition_evaluator.py +65 -0
  407. mlrun/utils/db.py +52 -0
  408. mlrun/utils/helpers.py +188 -13
  409. mlrun/utils/http.py +89 -54
  410. mlrun/utils/logger.py +48 -8
  411. mlrun/utils/model_monitoring.py +132 -100
  412. mlrun/utils/notifications/__init__.py +1 -1
  413. mlrun/utils/notifications/notification/__init__.py +8 -6
  414. mlrun/utils/notifications/notification/base.py +20 -14
  415. mlrun/utils/notifications/notification/console.py +7 -4
  416. mlrun/utils/notifications/notification/git.py +36 -19
  417. mlrun/utils/notifications/notification/ipython.py +10 -8
  418. mlrun/utils/notifications/notification/slack.py +18 -13
  419. mlrun/utils/notifications/notification_pusher.py +377 -56
  420. mlrun/utils/regex.py +6 -1
  421. mlrun/utils/singleton.py +1 -1
  422. mlrun/utils/v3io_clients.py +1 -1
  423. mlrun/utils/vault.py +270 -269
  424. mlrun/utils/version/__init__.py +1 -1
  425. mlrun/utils/version/version.json +2 -2
  426. mlrun/utils/version/version.py +1 -1
  427. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
  428. mlrun-1.4.0.dist-info/RECORD +434 -0
  429. mlrun/api/api/endpoints/marketplace.py +0 -257
  430. mlrun/api/crud/marketplace.py +0 -221
  431. mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
  432. mlrun/api/db/filedb/db.py +0 -518
  433. mlrun/api/schemas/marketplace.py +0 -128
  434. mlrun/api/schemas/model_endpoints.py +0 -185
  435. mlrun/db/filedb.py +0 -891
  436. mlrun/feature_store/retrieval/online.py +0 -92
  437. mlrun/model_monitoring/constants.py +0 -67
  438. mlrun/runtimes/package/context_handler.py +0 -711
  439. mlrun/runtimes/sparkjob/spark2job.py +0 -59
  440. mlrun-1.3.3.dist-info/RECORD +0 -381
  441. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
  442. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
  443. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
  444. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -24,11 +24,12 @@ import urllib.parse
24
24
 
25
25
  import aiohttp
26
26
  import fastapi
27
+ import igz_mgmt.schemas.manual_events
27
28
  import requests.adapters
28
29
  from fastapi.concurrency import run_in_threadpool
29
30
 
30
- import mlrun.api.schemas
31
31
  import mlrun.api.utils.projects.remotes.leader
32
+ import mlrun.common.schemas
32
33
  import mlrun.errors
33
34
  import mlrun.utils.helpers
34
35
  import mlrun.utils.singleton
@@ -79,7 +80,7 @@ class Client(
79
80
  super().__init__(*args, **kwargs)
80
81
  self._session = mlrun.utils.HTTPSessionWithRetry(
81
82
  retry_on_exception=mlrun.mlconf.httpdb.projects.retry_leader_request_on_exception
82
- == mlrun.api.schemas.HTTPSessionRetryMode.enabled.value,
83
+ == mlrun.common.schemas.HTTPSessionRetryMode.enabled.value,
83
84
  verbose=True,
84
85
  )
85
86
  self._api_url = mlrun.mlconf.iguazio_api_url
@@ -89,13 +90,15 @@ class Client(
89
90
  [[1, 10], [5, None]]
90
91
  )
91
92
  self._wait_for_project_terminal_state_retry_interval = 5
93
+ self._logger = logger.get_child("iguazio-client")
94
+ self._igz_clients = {}
92
95
 
93
96
  def try_get_grafana_service_url(self, session: str) -> typing.Optional[str]:
94
97
  """
95
98
  Try to find a ready grafana app service, and return its URL
96
99
  If nothing found, returns None
97
100
  """
98
- logger.debug("Getting grafana service url from Iguazio")
101
+ self._logger.debug("Getting grafana service url from Iguazio")
99
102
  response = self._send_request_to_api(
100
103
  "GET",
101
104
  "app_services_manifests",
@@ -123,7 +126,7 @@ class Client(
123
126
 
124
127
  def verify_request_session(
125
128
  self, request: fastapi.Request
126
- ) -> mlrun.api.schemas.AuthInfo:
129
+ ) -> mlrun.common.schemas.AuthInfo:
127
130
  """
128
131
  Proxy the request to one of the session verification endpoints (which will verify the session of the request)
129
132
  """
@@ -140,7 +143,7 @@ class Client(
140
143
  response.headers, response.json()
141
144
  )
142
145
 
143
- def verify_session(self, session: str) -> mlrun.api.schemas.AuthInfo:
146
+ def verify_session(self, session: str) -> mlrun.common.schemas.AuthInfo:
144
147
  response = self._send_request_to_api(
145
148
  "POST",
146
149
  mlrun.mlconf.httpdb.authentication.iguazio.session_verification_endpoint,
@@ -183,28 +186,32 @@ class Client(
183
186
  json=body,
184
187
  )
185
188
  if response.status_code == http.HTTPStatus.CREATED.value:
186
- logger.debug("Created access key in Iguazio", planes=planes)
189
+ self._logger.debug("Created access key in Iguazio", planes=planes)
187
190
  return response.json()["data"]["id"]
188
191
 
189
192
  def create_project(
190
193
  self,
191
194
  session: str,
192
- project: mlrun.api.schemas.Project,
195
+ project: mlrun.common.schemas.Project,
193
196
  wait_for_completion: bool = True,
194
197
  ) -> bool:
195
- logger.debug("Creating project in Iguazio", project=project)
198
+ self._logger.debug("Creating project in Iguazio", project=project.metadata.name)
196
199
  body = self._transform_mlrun_project_to_iguazio_project(project)
197
200
  return self._create_project_in_iguazio(
198
- session, project.metadata.name, body, wait_for_completion
201
+ session,
202
+ project.metadata.name,
203
+ body,
204
+ wait_for_completion,
205
+ timeout=60,
199
206
  )
200
207
 
201
208
  def update_project(
202
209
  self,
203
210
  session: str,
204
211
  name: str,
205
- project: mlrun.api.schemas.Project,
212
+ project: mlrun.common.schemas.Project,
206
213
  ):
207
- logger.debug("Updating project in Iguazio", name=name)
214
+ self._logger.debug("Updating project in Iguazio", name=name)
208
215
  body = self._transform_mlrun_project_to_iguazio_project(project)
209
216
  self._put_project_to_iguazio(session, name, body)
210
217
 
@@ -212,17 +219,17 @@ class Client(
212
219
  self,
213
220
  session: str,
214
221
  name: str,
215
- deletion_strategy: mlrun.api.schemas.DeletionStrategy = mlrun.api.schemas.DeletionStrategy.default(),
222
+ deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
216
223
  wait_for_completion: bool = True,
217
224
  ) -> bool:
218
- logger.debug(
225
+ self._logger.debug(
219
226
  "Deleting project in Iguazio",
220
227
  name=name,
221
228
  deletion_strategy=deletion_strategy,
222
229
  )
223
230
  body = self._transform_mlrun_project_to_iguazio_project(
224
- mlrun.api.schemas.Project(
225
- metadata=mlrun.api.schemas.ProjectMetadata(name=name)
231
+ mlrun.common.schemas.Project(
232
+ metadata=mlrun.common.schemas.ProjectMetadata(name=name)
226
233
  )
227
234
  )
228
235
  headers = {
@@ -240,7 +247,7 @@ class Client(
240
247
  except requests.HTTPError as exc:
241
248
  if exc.response.status_code != http.HTTPStatus.NOT_FOUND.value:
242
249
  raise
243
- logger.debug(
250
+ self._logger.debug(
244
251
  "Project not found in Iguazio. Considering deletion as successful",
245
252
  name=name,
246
253
  deletion_strategy=deletion_strategy,
@@ -249,7 +256,7 @@ class Client(
249
256
  else:
250
257
  if wait_for_completion:
251
258
  job_id = response.json()["data"]["id"]
252
- logger.debug(
259
+ self._logger.debug(
253
260
  "Waiting for project deletion job in Iguazio",
254
261
  name=name,
255
262
  job_id=job_id,
@@ -266,7 +273,7 @@ class Client(
266
273
  updated_after: typing.Optional[datetime.datetime] = None,
267
274
  page_size: typing.Optional[int] = None,
268
275
  ) -> typing.Tuple[
269
- typing.List[mlrun.api.schemas.Project], typing.Optional[datetime.datetime]
276
+ typing.List[mlrun.common.schemas.Project], typing.Optional[datetime.datetime]
270
277
  ]:
271
278
  project_names, latest_updated_at = self._list_project_names(
272
279
  session, updated_after, page_size
@@ -277,14 +284,14 @@ class Client(
277
284
  self,
278
285
  session: str,
279
286
  name: str,
280
- ) -> mlrun.api.schemas.Project:
287
+ ) -> mlrun.common.schemas.Project:
281
288
  return self._get_project_from_iguazio(session, name)
282
289
 
283
290
  def get_project_owner(
284
291
  self,
285
292
  session: str,
286
293
  name: str,
287
- ) -> mlrun.api.schemas.ProjectOwner:
294
+ ) -> mlrun.common.schemas.ProjectOwner:
288
295
  response = self._get_project_from_iguazio_without_parsing(
289
296
  session, name, enrich_owner_access_key=True
290
297
  )
@@ -300,15 +307,15 @@ class Client(
300
307
  f"Unable to enrich project owner for project {name},"
301
308
  f" because project has no owner configured"
302
309
  )
303
- return mlrun.api.schemas.ProjectOwner(
310
+ return mlrun.common.schemas.ProjectOwner(
304
311
  username=owner_username,
305
312
  access_key=owner_access_key,
306
313
  )
307
314
 
308
315
  def format_as_leader_project(
309
- self, project: mlrun.api.schemas.Project
310
- ) -> mlrun.api.schemas.IguazioProject:
311
- return mlrun.api.schemas.IguazioProject(
316
+ self, project: mlrun.common.schemas.Project
317
+ ) -> mlrun.common.schemas.IguazioProject:
318
+ return mlrun.common.schemas.IguazioProject(
312
319
  data=self._transform_mlrun_project_to_iguazio_project(project)["data"]
313
320
  )
314
321
 
@@ -319,6 +326,23 @@ class Client(
319
326
  """
320
327
  return True
321
328
 
329
+ def emit_manual_event(self, access_key: str, event: igz_mgmt.Event):
330
+ """
331
+ Emit a manual event to Iguazio
332
+ """
333
+ client = self._get_igz_client(access_key)
334
+ igz_mgmt.ManualEvents.emit(
335
+ http_client=client, event=event, audit_tenant_id=client.tenant_id
336
+ )
337
+
338
+ def _get_igz_client(self, access_key: str) -> igz_mgmt.Client:
339
+ if not self._igz_clients.get(access_key):
340
+ self._igz_clients[access_key] = igz_mgmt.Client(
341
+ endpoint=self._api_url,
342
+ access_key=access_key,
343
+ )
344
+ return self._igz_clients[access_key]
345
+
322
346
  def _list_project_names(
323
347
  self,
324
348
  session: str,
@@ -353,7 +377,7 @@ class Client(
353
377
 
354
378
  def _list_projects_data(
355
379
  self, session: str, project_names: typing.List[str]
356
- ) -> typing.List[mlrun.api.schemas.Project]:
380
+ ) -> typing.List[mlrun.common.schemas.Project]:
357
381
  return [
358
382
  self._get_project_from_iguazio(session, project_name)
359
383
  for project_name in project_names
@@ -364,26 +388,24 @@ class Client(
364
388
  ) -> typing.Optional[datetime.datetime]:
365
389
  latest_updated_at = None
366
390
  for iguazio_project in response_body["data"]:
367
-
368
- # iguazio project might be in creating mode, in which case it doesn't have an updated_at field yet
369
- updated_at_str = iguazio_project["attributes"].get("updated_at")
370
- if not updated_at_str:
371
- logger.debug(
372
- "Project is in creating mode, skipping",
373
- name=iguazio_project.get("attributes", {}).get("name", "unknown"),
374
- )
375
- continue
376
- updated_at = datetime.datetime.fromisoformat(updated_at_str)
391
+ updated_at = datetime.datetime.fromisoformat(
392
+ iguazio_project["attributes"]["updated_at"]
393
+ )
377
394
  if latest_updated_at is None or latest_updated_at < updated_at:
378
395
  latest_updated_at = updated_at
379
396
  return latest_updated_at
380
397
 
381
398
  def _create_project_in_iguazio(
382
- self, session: str, name: str, body: dict, wait_for_completion: bool
399
+ self,
400
+ session: str,
401
+ name: str,
402
+ body: dict,
403
+ wait_for_completion: bool,
404
+ **kwargs,
383
405
  ) -> bool:
384
- _, job_id = self._post_project_to_iguazio(session, body)
406
+ _, job_id = self._post_project_to_iguazio(session, body, **kwargs)
385
407
  if wait_for_completion:
386
- logger.debug(
408
+ self._logger.debug(
387
409
  "Waiting for project creation job in Iguazio",
388
410
  name=name,
389
411
  job_id=job_id,
@@ -391,14 +413,27 @@ class Client(
391
413
  self._wait_for_job_completion(
392
414
  session, job_id, "Project creation job failed"
393
415
  )
416
+ self._logger.debug(
417
+ "Successfully created project in Iguazio",
418
+ name=name,
419
+ job_id=job_id,
420
+ )
394
421
  return False
395
422
  return True
396
423
 
397
424
  def _post_project_to_iguazio(
398
- self, session: str, body: dict
399
- ) -> typing.Tuple[mlrun.api.schemas.Project, str]:
425
+ self,
426
+ session: str,
427
+ body: dict,
428
+ **kwargs,
429
+ ) -> typing.Tuple[mlrun.common.schemas.Project, str]:
400
430
  response = self._send_request_to_api(
401
- "POST", "projects", "Failed creating project in Iguazio", session, json=body
431
+ "POST",
432
+ "projects",
433
+ "Failed creating project in Iguazio",
434
+ session,
435
+ json=body,
436
+ **kwargs,
402
437
  )
403
438
  response_body = response.json()
404
439
  return (
@@ -407,14 +442,19 @@ class Client(
407
442
  )
408
443
 
409
444
  def _put_project_to_iguazio(
410
- self, session: str, name: str, body: dict
411
- ) -> mlrun.api.schemas.Project:
445
+ self,
446
+ session: str,
447
+ name: str,
448
+ body: dict,
449
+ **kwargs,
450
+ ) -> mlrun.common.schemas.Project:
412
451
  response = self._send_request_to_api(
413
452
  "PUT",
414
453
  f"projects/__name__/{name}",
415
454
  "Failed updating project in Iguazio",
416
455
  session,
417
456
  json=body,
457
+ **kwargs,
418
458
  )
419
459
  return self._transform_iguazio_project_to_mlrun_project(response.json()["data"])
420
460
 
@@ -434,7 +474,7 @@ class Client(
434
474
 
435
475
  def _get_project_from_iguazio(
436
476
  self, session: str, name: str, include_owner_session: bool = False
437
- ) -> mlrun.api.schemas.Project:
477
+ ) -> mlrun.common.schemas.Project:
438
478
  response = self._get_project_from_iguazio_without_parsing(session, name)
439
479
  return self._transform_iguazio_project_to_mlrun_project(response.json()["data"])
440
480
 
@@ -452,7 +492,7 @@ class Client(
452
492
  job_state, job_result = mlrun.utils.helpers.retry_until_successful(
453
493
  self._wait_for_job_completion_retry_interval,
454
494
  360,
455
- logger,
495
+ self._logger,
456
496
  False,
457
497
  _verify_job_in_terminal_state,
458
498
  )
@@ -469,6 +509,7 @@ class Client(
469
509
  if not status_code:
470
510
  raise mlrun.errors.MLRunRuntimeError(error_message)
471
511
  raise mlrun.errors.raise_for_status_code(status_code, error_message)
512
+ self._logger.debug("Job completed successfully", job_id=job_id)
472
513
 
473
514
  def _send_request_to_api(
474
515
  self, method, path, error_message: str, session=None, **kwargs
@@ -490,7 +531,7 @@ class Client(
490
531
  self,
491
532
  response_headers: typing.Mapping[str, typing.Any],
492
533
  response_body: typing.Mapping[typing.Any, typing.Any],
493
- ) -> mlrun.api.schemas.AuthInfo:
534
+ ) -> mlrun.common.schemas.AuthInfo:
494
535
 
495
536
  (
496
537
  username,
@@ -511,7 +552,7 @@ class Client(
511
552
  user_id = user_id_from_body or user_id
512
553
  group_ids = group_ids_from_body or group_ids
513
554
 
514
- auth_info = mlrun.api.schemas.AuthInfo(
555
+ auth_info = mlrun.common.schemas.AuthInfo(
515
556
  username=username,
516
557
  session=session,
517
558
  user_id=user_id,
@@ -570,7 +611,7 @@ class Client(
570
611
 
571
612
  @staticmethod
572
613
  def _transform_mlrun_project_to_iguazio_project(
573
- project: mlrun.api.schemas.Project,
614
+ project: mlrun.common.schemas.Project,
574
615
  ) -> dict:
575
616
  body = {
576
617
  "data": {
@@ -607,7 +648,7 @@ class Client(
607
648
 
608
649
  @staticmethod
609
650
  def _transform_mlrun_project_to_iguazio_mlrun_project_attribute(
610
- project: mlrun.api.schemas.Project,
651
+ project: mlrun.common.schemas.Project,
611
652
  ):
612
653
  project_dict = project.dict(
613
654
  exclude_unset=True,
@@ -641,7 +682,7 @@ class Client(
641
682
  @staticmethod
642
683
  def _transform_iguazio_project_to_mlrun_project(
643
684
  iguazio_project,
644
- ) -> mlrun.api.schemas.Project:
685
+ ) -> mlrun.common.schemas.Project:
645
686
  mlrun_project_without_common_fields = json.loads(
646
687
  iguazio_project["attributes"].get("mlrun_project", "{}")
647
688
  )
@@ -649,14 +690,16 @@ class Client(
649
690
  mlrun_project_without_common_fields.setdefault("metadata", {})[
650
691
  "name"
651
692
  ] = iguazio_project["attributes"]["name"]
652
- mlrun_project = mlrun.api.schemas.Project(**mlrun_project_without_common_fields)
693
+ mlrun_project = mlrun.common.schemas.Project(
694
+ **mlrun_project_without_common_fields
695
+ )
653
696
  mlrun_project.metadata.created = datetime.datetime.fromisoformat(
654
697
  iguazio_project["attributes"]["created_at"]
655
698
  )
656
- mlrun_project.spec.desired_state = mlrun.api.schemas.ProjectDesiredState(
699
+ mlrun_project.spec.desired_state = mlrun.common.schemas.ProjectDesiredState(
657
700
  iguazio_project["attributes"]["admin_status"]
658
701
  )
659
- mlrun_project.status.state = mlrun.api.schemas.ProjectState(
702
+ mlrun_project.status.state = mlrun.common.schemas.ProjectState(
660
703
  iguazio_project["attributes"]["operational_status"]
661
704
  )
662
705
  if iguazio_project["attributes"].get("description"):
@@ -701,11 +744,11 @@ class Client(
701
744
  if kwargs.get("timeout") is None:
702
745
  kwargs["timeout"] = 20
703
746
  if "projects" in path:
704
- if mlrun.api.schemas.HeaderNames.projects_role not in kwargs.get(
747
+ if mlrun.common.schemas.HeaderNames.projects_role not in kwargs.get(
705
748
  "headers", {}
706
749
  ):
707
750
  kwargs.setdefault("headers", {})[
708
- mlrun.api.schemas.HeaderNames.projects_role
751
+ mlrun.common.schemas.HeaderNames.projects_role
709
752
  ] = "mlrun"
710
753
 
711
754
  # requests no longer supports header values to be enum (https://github.com/psf/requests/pull/6154)
@@ -720,6 +763,9 @@ class Client(
720
763
  self, method, path, response, response_body, error_message, kwargs
721
764
  ):
722
765
  log_kwargs = copy.deepcopy(kwargs)
766
+
767
+ # this can be big and spammy
768
+ log_kwargs.pop("json", None)
723
769
  log_kwargs.update({"method": method, "path": path})
724
770
  try:
725
771
  ctx = response_body.get("meta", {}).get("ctx")
@@ -732,7 +778,7 @@ class Client(
732
778
  if errors or ctx:
733
779
  log_kwargs.update({"ctx": ctx, "errors": errors})
734
780
 
735
- logger.warning("Request to iguazio failed", **log_kwargs)
781
+ self._logger.warning("Request to iguazio failed", **log_kwargs)
736
782
  mlrun.errors.raise_for_status(response, error_message)
737
783
 
738
784
 
@@ -779,7 +825,7 @@ class AsyncClient(Client):
779
825
 
780
826
  async def verify_request_session(
781
827
  self, request: fastapi.Request
782
- ) -> mlrun.api.schemas.AuthInfo:
828
+ ) -> mlrun.common.schemas.AuthInfo:
783
829
  """
784
830
  Proxy the request to one of the session verification endpoints (which will verify the session of the request)
785
831
  """
@@ -796,7 +842,7 @@ class AsyncClient(Client):
796
842
  response.headers, await response.json()
797
843
  )
798
844
 
799
- async def verify_session(self, session: str) -> mlrun.api.schemas.AuthInfo:
845
+ async def verify_session(self, session: str) -> mlrun.common.schemas.AuthInfo:
800
846
  async with self._send_request_to_api_async(
801
847
  "POST",
802
848
  mlrun.mlconf.httpdb.authentication.iguazio.session_verification_endpoint,
@@ -836,6 +882,6 @@ class AsyncClient(Client):
836
882
  if not self._async_session:
837
883
  self._async_session = mlrun.utils.AsyncClientWithRetry(
838
884
  retry_on_exception=mlrun.mlconf.httpdb.projects.retry_leader_request_on_exception
839
- == mlrun.api.schemas.HTTPSessionRetryMode.enabled.value,
885
+ == mlrun.common.schemas.HTTPSessionRetryMode.enabled.value,
840
886
  logger=logger,
841
887
  )
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -20,8 +20,8 @@ import typing
20
20
  import requests.adapters
21
21
  import sqlalchemy.orm
22
22
 
23
- import mlrun.api.schemas
24
23
  import mlrun.api.utils.projects.remotes.follower
24
+ import mlrun.common.schemas
25
25
  import mlrun.errors
26
26
  import mlrun.utils.singleton
27
27
  from mlrun.utils import logger
@@ -37,7 +37,7 @@ class Client(
37
37
  self._api_url = mlrun.config.config.nuclio_dashboard_url
38
38
 
39
39
  def create_project(
40
- self, session: sqlalchemy.orm.Session, project: mlrun.api.schemas.Project
40
+ self, session: sqlalchemy.orm.Session, project: mlrun.common.schemas.Project
41
41
  ):
42
42
  logger.debug("Creating project in Nuclio", project=project)
43
43
  body = self._generate_request_body(project)
@@ -47,7 +47,7 @@ class Client(
47
47
  self,
48
48
  session: sqlalchemy.orm.Session,
49
49
  name: str,
50
- project: mlrun.api.schemas.Project,
50
+ project: mlrun.common.schemas.Project,
51
51
  ):
52
52
  logger.debug("Storing project in Nuclio", name=name, project=project)
53
53
  body = self._generate_request_body(project)
@@ -65,7 +65,7 @@ class Client(
65
65
  session: sqlalchemy.orm.Session,
66
66
  name: str,
67
67
  project: dict,
68
- patch_mode: mlrun.api.schemas.PatchMode = mlrun.api.schemas.PatchMode.replace,
68
+ patch_mode: mlrun.common.schemas.PatchMode = mlrun.common.schemas.PatchMode.replace,
69
69
  ):
70
70
  logger.debug(
71
71
  "Patching project in Nuclio",
@@ -93,14 +93,14 @@ class Client(
93
93
  self,
94
94
  session: sqlalchemy.orm.Session,
95
95
  name: str,
96
- deletion_strategy: mlrun.api.schemas.DeletionStrategy = mlrun.api.schemas.DeletionStrategy.default(),
96
+ deletion_strategy: mlrun.common.schemas.DeletionStrategy = mlrun.common.schemas.DeletionStrategy.default(),
97
97
  ):
98
98
  logger.debug(
99
99
  "Deleting project in Nuclio", name=name, deletion_strategy=deletion_strategy
100
100
  )
101
101
  body = self._generate_request_body(
102
- mlrun.api.schemas.Project(
103
- metadata=mlrun.api.schemas.ProjectMetadata(name=name)
102
+ mlrun.common.schemas.Project(
103
+ metadata=mlrun.common.schemas.ProjectMetadata(name=name)
104
104
  )
105
105
  )
106
106
  headers = {
@@ -119,7 +119,7 @@ class Client(
119
119
 
120
120
  def get_project(
121
121
  self, session: sqlalchemy.orm.Session, name: str
122
- ) -> mlrun.api.schemas.Project:
122
+ ) -> mlrun.common.schemas.Project:
123
123
  response = self._get_project_from_nuclio(name)
124
124
  response_body = response.json()
125
125
  return self._transform_nuclio_project_to_schema(response_body)
@@ -128,11 +128,11 @@ class Client(
128
128
  self,
129
129
  session: sqlalchemy.orm.Session,
130
130
  owner: str = None,
131
- format_: mlrun.api.schemas.ProjectsFormat = mlrun.api.schemas.ProjectsFormat.full,
131
+ format_: mlrun.common.schemas.ProjectsFormat = mlrun.common.schemas.ProjectsFormat.full,
132
132
  labels: typing.List[str] = None,
133
- state: mlrun.api.schemas.ProjectState = None,
133
+ state: mlrun.common.schemas.ProjectState = None,
134
134
  names: typing.Optional[typing.List[str]] = None,
135
- ) -> mlrun.api.schemas.ProjectsOutput:
135
+ ) -> mlrun.common.schemas.ProjectsOutput:
136
136
  if owner:
137
137
  raise NotImplementedError(
138
138
  "Listing nuclio projects by owner is currently not supported"
@@ -154,10 +154,10 @@ class Client(
154
154
  projects = []
155
155
  for nuclio_project in response_body.values():
156
156
  projects.append(self._transform_nuclio_project_to_schema(nuclio_project))
157
- if format_ == mlrun.api.schemas.ProjectsFormat.full:
158
- return mlrun.api.schemas.ProjectsOutput(projects=projects)
159
- elif format_ == mlrun.api.schemas.ProjectsFormat.name_only:
160
- return mlrun.api.schemas.ProjectsOutput(
157
+ if format_ == mlrun.common.schemas.ProjectsFormat.full:
158
+ return mlrun.common.schemas.ProjectsOutput(projects=projects)
159
+ elif format_ == mlrun.common.schemas.ProjectsFormat.name_only:
160
+ return mlrun.common.schemas.ProjectsOutput(
161
161
  projects=[project.metadata.name for project in projects]
162
162
  )
163
163
  else:
@@ -170,14 +170,14 @@ class Client(
170
170
  session: sqlalchemy.orm.Session,
171
171
  owner: str = None,
172
172
  labels: typing.List[str] = None,
173
- state: mlrun.api.schemas.ProjectState = None,
173
+ state: mlrun.common.schemas.ProjectState = None,
174
174
  names: typing.Optional[typing.List[str]] = None,
175
- ) -> mlrun.api.schemas.ProjectSummariesOutput:
175
+ ) -> mlrun.common.schemas.ProjectSummariesOutput:
176
176
  raise NotImplementedError("Listing project summaries is not supported")
177
177
 
178
178
  def get_project_summary(
179
179
  self, session: sqlalchemy.orm.Session, name: str
180
- ) -> mlrun.api.schemas.ProjectSummary:
180
+ ) -> mlrun.common.schemas.ProjectSummary:
181
181
  raise NotImplementedError("Get project summary is not supported")
182
182
 
183
183
  def get_dashboard_version(self) -> str:
@@ -226,7 +226,7 @@ class Client(
226
226
  return response
227
227
 
228
228
  @staticmethod
229
- def _generate_request_body(project: mlrun.api.schemas.Project):
229
+ def _generate_request_body(project: mlrun.common.schemas.Project):
230
230
  body = {
231
231
  "metadata": {"name": project.metadata.name},
232
232
  }
@@ -240,13 +240,13 @@ class Client(
240
240
 
241
241
  @staticmethod
242
242
  def _transform_nuclio_project_to_schema(nuclio_project):
243
- return mlrun.api.schemas.Project(
244
- metadata=mlrun.api.schemas.ProjectMetadata(
243
+ return mlrun.common.schemas.Project(
244
+ metadata=mlrun.common.schemas.ProjectMetadata(
245
245
  name=nuclio_project["metadata"]["name"],
246
246
  labels=nuclio_project["metadata"].get("labels"),
247
247
  annotations=nuclio_project["metadata"].get("annotations"),
248
248
  ),
249
- spec=mlrun.api.schemas.ProjectSpec(
249
+ spec=mlrun.common.schemas.ProjectSpec(
250
250
  description=nuclio_project["spec"].get("description")
251
251
  ),
252
252
  )
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
15
15
  import google.protobuf.reflection
16
16
  import grpc
17
17
 
18
- import mlrun.api.schemas
18
+ import mlrun.common.schemas
19
19
  import mlrun.config
20
20
  import mlrun.errors
21
21
 
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.