mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (444) hide show
  1. mlrun/__init__.py +3 -3
  2. mlrun/__main__.py +79 -37
  3. mlrun/api/__init__.py +1 -1
  4. mlrun/api/api/__init__.py +1 -1
  5. mlrun/api/api/api.py +4 -4
  6. mlrun/api/api/deps.py +10 -21
  7. mlrun/api/api/endpoints/__init__.py +1 -1
  8. mlrun/api/api/endpoints/artifacts.py +64 -36
  9. mlrun/api/api/endpoints/auth.py +4 -4
  10. mlrun/api/api/endpoints/background_tasks.py +11 -11
  11. mlrun/api/api/endpoints/client_spec.py +5 -5
  12. mlrun/api/api/endpoints/clusterization_spec.py +6 -4
  13. mlrun/api/api/endpoints/feature_store.py +124 -115
  14. mlrun/api/api/endpoints/files.py +22 -14
  15. mlrun/api/api/endpoints/frontend_spec.py +28 -21
  16. mlrun/api/api/endpoints/functions.py +142 -87
  17. mlrun/api/api/endpoints/grafana_proxy.py +89 -442
  18. mlrun/api/api/endpoints/healthz.py +20 -7
  19. mlrun/api/api/endpoints/hub.py +320 -0
  20. mlrun/api/api/endpoints/internal/__init__.py +1 -1
  21. mlrun/api/api/endpoints/internal/config.py +1 -1
  22. mlrun/api/api/endpoints/internal/memory_reports.py +9 -9
  23. mlrun/api/api/endpoints/logs.py +11 -11
  24. mlrun/api/api/endpoints/model_endpoints.py +74 -70
  25. mlrun/api/api/endpoints/operations.py +13 -9
  26. mlrun/api/api/endpoints/pipelines.py +93 -88
  27. mlrun/api/api/endpoints/projects.py +35 -35
  28. mlrun/api/api/endpoints/runs.py +69 -27
  29. mlrun/api/api/endpoints/runtime_resources.py +28 -28
  30. mlrun/api/api/endpoints/schedules.py +98 -41
  31. mlrun/api/api/endpoints/secrets.py +37 -32
  32. mlrun/api/api/endpoints/submit.py +12 -12
  33. mlrun/api/api/endpoints/tags.py +20 -22
  34. mlrun/api/api/utils.py +251 -42
  35. mlrun/api/constants.py +1 -1
  36. mlrun/api/crud/__init__.py +18 -15
  37. mlrun/api/crud/artifacts.py +10 -10
  38. mlrun/api/crud/client_spec.py +4 -4
  39. mlrun/api/crud/clusterization_spec.py +3 -3
  40. mlrun/api/crud/feature_store.py +54 -46
  41. mlrun/api/crud/functions.py +3 -3
  42. mlrun/api/crud/hub.py +312 -0
  43. mlrun/api/crud/logs.py +11 -9
  44. mlrun/api/crud/model_monitoring/__init__.py +3 -3
  45. mlrun/api/crud/model_monitoring/grafana.py +435 -0
  46. mlrun/api/crud/model_monitoring/model_endpoints.py +352 -129
  47. mlrun/api/crud/notifications.py +149 -0
  48. mlrun/api/crud/pipelines.py +67 -52
  49. mlrun/api/crud/projects.py +51 -23
  50. mlrun/api/crud/runs.py +7 -5
  51. mlrun/api/crud/runtime_resources.py +13 -13
  52. mlrun/api/{db/filedb → crud/runtimes}/__init__.py +1 -1
  53. mlrun/api/crud/runtimes/nuclio/__init__.py +14 -0
  54. mlrun/api/crud/runtimes/nuclio/function.py +505 -0
  55. mlrun/api/crud/runtimes/nuclio/helpers.py +310 -0
  56. mlrun/api/crud/secrets.py +88 -46
  57. mlrun/api/crud/tags.py +5 -5
  58. mlrun/api/db/__init__.py +1 -1
  59. mlrun/api/db/base.py +102 -54
  60. mlrun/api/db/init_db.py +2 -3
  61. mlrun/api/db/session.py +4 -12
  62. mlrun/api/db/sqldb/__init__.py +1 -1
  63. mlrun/api/db/sqldb/db.py +439 -196
  64. mlrun/api/db/sqldb/helpers.py +1 -1
  65. mlrun/api/db/sqldb/models/__init__.py +3 -3
  66. mlrun/api/db/sqldb/models/models_mysql.py +82 -64
  67. mlrun/api/db/sqldb/models/models_sqlite.py +76 -64
  68. mlrun/api/db/sqldb/session.py +27 -20
  69. mlrun/api/initial_data.py +82 -24
  70. mlrun/api/launcher.py +196 -0
  71. mlrun/api/main.py +91 -22
  72. mlrun/api/middlewares.py +6 -5
  73. mlrun/api/migrations_mysql/env.py +1 -1
  74. mlrun/api/migrations_mysql/versions/28383af526f3_market_place_to_hub.py +40 -0
  75. mlrun/api/migrations_mysql/versions/32bae1b0e29c_increase_timestamp_fields_precision.py +1 -1
  76. mlrun/api/migrations_mysql/versions/4903aef6a91d_tag_foreign_key_and_cascades.py +1 -1
  77. mlrun/api/migrations_mysql/versions/5f1351c88a19_adding_background_tasks_table.py +1 -1
  78. mlrun/api/migrations_mysql/versions/88e656800d6a_add_requested_logs_column_and_index_to_.py +1 -1
  79. mlrun/api/migrations_mysql/versions/9d16de5f03a7_adding_data_versions_table.py +1 -1
  80. mlrun/api/migrations_mysql/versions/b86f5b53f3d7_adding_name_and_updated_to_runs_table.py +1 -1
  81. mlrun/api/migrations_mysql/versions/c4af40b0bf61_init.py +1 -1
  82. mlrun/api/migrations_mysql/versions/c905d15bd91d_notifications.py +72 -0
  83. mlrun/api/migrations_mysql/versions/ee041e8fdaa0_adding_next_run_time_column_to_schedule_.py +1 -1
  84. mlrun/api/migrations_sqlite/env.py +1 -1
  85. mlrun/api/migrations_sqlite/versions/11f8dd2dc9fe_init.py +1 -1
  86. mlrun/api/migrations_sqlite/versions/1c954f8cb32d_schedule_last_run_uri.py +1 -1
  87. mlrun/api/migrations_sqlite/versions/2b6d23c715aa_adding_feature_sets.py +1 -1
  88. mlrun/api/migrations_sqlite/versions/4acd9430b093_market_place_to_hub.py +77 -0
  89. mlrun/api/migrations_sqlite/versions/6401142f2d7c_adding_next_run_time_column_to_schedule_.py +1 -1
  90. mlrun/api/migrations_sqlite/versions/64d90a1a69bc_adding_background_tasks_table.py +1 -1
  91. mlrun/api/migrations_sqlite/versions/803438ecd005_add_requested_logs_column_to_runs.py +1 -1
  92. mlrun/api/migrations_sqlite/versions/863114f0c659_refactoring_feature_set.py +1 -1
  93. mlrun/api/migrations_sqlite/versions/959ae00528ad_notifications.py +63 -0
  94. mlrun/api/migrations_sqlite/versions/accf9fc83d38_adding_data_versions_table.py +1 -1
  95. mlrun/api/migrations_sqlite/versions/b68e8e897a28_schedule_labels.py +1 -1
  96. mlrun/api/migrations_sqlite/versions/bcd0c1f9720c_adding_project_labels.py +1 -1
  97. mlrun/api/migrations_sqlite/versions/cf21882f938e_schedule_id.py +1 -1
  98. mlrun/api/migrations_sqlite/versions/d781f58f607f_tag_object_name_string.py +1 -1
  99. mlrun/api/migrations_sqlite/versions/deac06871ace_adding_marketplace_sources_table.py +1 -1
  100. mlrun/api/migrations_sqlite/versions/e1dd5983c06b_schedule_concurrency_limit.py +1 -1
  101. mlrun/api/migrations_sqlite/versions/e5594ed3ab53_adding_name_and_updated_to_runs_table.py +1 -1
  102. mlrun/api/migrations_sqlite/versions/f4249b4ba6fa_adding_feature_vectors.py +1 -1
  103. mlrun/api/migrations_sqlite/versions/f7b5a1a03629_adding_feature_labels.py +1 -1
  104. mlrun/api/schemas/__init__.py +216 -138
  105. mlrun/api/utils/__init__.py +1 -1
  106. mlrun/api/utils/asyncio.py +1 -1
  107. mlrun/api/utils/auth/__init__.py +1 -1
  108. mlrun/api/utils/auth/providers/__init__.py +1 -1
  109. mlrun/api/utils/auth/providers/base.py +7 -7
  110. mlrun/api/utils/auth/providers/nop.py +6 -7
  111. mlrun/api/utils/auth/providers/opa.py +17 -17
  112. mlrun/api/utils/auth/verifier.py +36 -34
  113. mlrun/api/utils/background_tasks.py +24 -24
  114. mlrun/{builder.py → api/utils/builder.py} +216 -123
  115. mlrun/api/utils/clients/__init__.py +1 -1
  116. mlrun/api/utils/clients/chief.py +19 -4
  117. mlrun/api/utils/clients/iguazio.py +106 -60
  118. mlrun/api/utils/clients/log_collector.py +1 -1
  119. mlrun/api/utils/clients/nuclio.py +23 -23
  120. mlrun/api/utils/clients/protocols/grpc.py +2 -2
  121. mlrun/api/utils/db/__init__.py +1 -1
  122. mlrun/api/utils/db/alembic.py +1 -1
  123. mlrun/api/utils/db/backup.py +1 -1
  124. mlrun/api/utils/db/mysql.py +24 -25
  125. mlrun/api/utils/db/sql_collation.py +1 -1
  126. mlrun/api/utils/db/sqlite_migration.py +2 -2
  127. mlrun/api/utils/events/__init__.py +14 -0
  128. mlrun/api/utils/events/base.py +57 -0
  129. mlrun/api/utils/events/events_factory.py +41 -0
  130. mlrun/api/utils/events/iguazio.py +217 -0
  131. mlrun/api/utils/events/nop.py +55 -0
  132. mlrun/api/utils/helpers.py +16 -13
  133. mlrun/api/utils/memory_reports.py +1 -1
  134. mlrun/api/utils/periodic.py +6 -3
  135. mlrun/api/utils/projects/__init__.py +1 -1
  136. mlrun/api/utils/projects/follower.py +33 -33
  137. mlrun/api/utils/projects/leader.py +36 -34
  138. mlrun/api/utils/projects/member.py +27 -27
  139. mlrun/api/utils/projects/remotes/__init__.py +1 -1
  140. mlrun/api/utils/projects/remotes/follower.py +13 -13
  141. mlrun/api/utils/projects/remotes/leader.py +10 -10
  142. mlrun/api/utils/projects/remotes/nop_follower.py +27 -21
  143. mlrun/api/utils/projects/remotes/nop_leader.py +17 -16
  144. mlrun/api/utils/scheduler.py +140 -51
  145. mlrun/api/utils/singletons/__init__.py +1 -1
  146. mlrun/api/utils/singletons/db.py +9 -15
  147. mlrun/api/utils/singletons/k8s.py +677 -5
  148. mlrun/api/utils/singletons/logs_dir.py +1 -1
  149. mlrun/api/utils/singletons/project_member.py +1 -1
  150. mlrun/api/utils/singletons/scheduler.py +1 -1
  151. mlrun/artifacts/__init__.py +2 -2
  152. mlrun/artifacts/base.py +8 -2
  153. mlrun/artifacts/dataset.py +5 -3
  154. mlrun/artifacts/manager.py +7 -1
  155. mlrun/artifacts/model.py +15 -4
  156. mlrun/artifacts/plots.py +1 -1
  157. mlrun/common/__init__.py +1 -1
  158. mlrun/common/constants.py +15 -0
  159. mlrun/common/model_monitoring.py +209 -0
  160. mlrun/common/schemas/__init__.py +167 -0
  161. mlrun/{api → common}/schemas/artifact.py +13 -14
  162. mlrun/{api → common}/schemas/auth.py +10 -8
  163. mlrun/{api → common}/schemas/background_task.py +3 -3
  164. mlrun/{api → common}/schemas/client_spec.py +1 -1
  165. mlrun/{api → common}/schemas/clusterization_spec.py +3 -3
  166. mlrun/{api → common}/schemas/constants.py +21 -8
  167. mlrun/common/schemas/events.py +36 -0
  168. mlrun/{api → common}/schemas/feature_store.py +2 -1
  169. mlrun/{api → common}/schemas/frontend_spec.py +7 -6
  170. mlrun/{api → common}/schemas/function.py +5 -5
  171. mlrun/{api → common}/schemas/http.py +3 -3
  172. mlrun/common/schemas/hub.py +134 -0
  173. mlrun/{api → common}/schemas/k8s.py +3 -3
  174. mlrun/{api → common}/schemas/memory_reports.py +1 -1
  175. mlrun/common/schemas/model_endpoints.py +342 -0
  176. mlrun/common/schemas/notification.py +57 -0
  177. mlrun/{api → common}/schemas/object.py +6 -6
  178. mlrun/{api → common}/schemas/pipeline.py +3 -3
  179. mlrun/{api → common}/schemas/project.py +6 -5
  180. mlrun/common/schemas/regex.py +24 -0
  181. mlrun/common/schemas/runs.py +30 -0
  182. mlrun/{api → common}/schemas/runtime_resource.py +3 -3
  183. mlrun/{api → common}/schemas/schedule.py +19 -7
  184. mlrun/{api → common}/schemas/secret.py +3 -3
  185. mlrun/{api → common}/schemas/tag.py +2 -2
  186. mlrun/common/types.py +25 -0
  187. mlrun/config.py +152 -20
  188. mlrun/data_types/__init__.py +7 -2
  189. mlrun/data_types/data_types.py +4 -2
  190. mlrun/data_types/infer.py +1 -1
  191. mlrun/data_types/spark.py +10 -3
  192. mlrun/datastore/__init__.py +10 -3
  193. mlrun/datastore/azure_blob.py +1 -1
  194. mlrun/datastore/base.py +185 -53
  195. mlrun/datastore/datastore.py +1 -1
  196. mlrun/datastore/filestore.py +1 -1
  197. mlrun/datastore/google_cloud_storage.py +1 -1
  198. mlrun/datastore/inmem.py +4 -1
  199. mlrun/datastore/redis.py +1 -1
  200. mlrun/datastore/s3.py +1 -1
  201. mlrun/datastore/sources.py +192 -70
  202. mlrun/datastore/spark_udf.py +44 -0
  203. mlrun/datastore/store_resources.py +4 -4
  204. mlrun/datastore/targets.py +115 -45
  205. mlrun/datastore/utils.py +127 -5
  206. mlrun/datastore/v3io.py +1 -1
  207. mlrun/datastore/wasbfs/__init__.py +1 -1
  208. mlrun/datastore/wasbfs/fs.py +1 -1
  209. mlrun/db/__init__.py +7 -5
  210. mlrun/db/base.py +112 -68
  211. mlrun/db/httpdb.py +445 -277
  212. mlrun/db/nopdb.py +491 -0
  213. mlrun/db/sqldb.py +112 -65
  214. mlrun/errors.py +6 -1
  215. mlrun/execution.py +44 -22
  216. mlrun/feature_store/__init__.py +1 -1
  217. mlrun/feature_store/api.py +143 -95
  218. mlrun/feature_store/common.py +16 -20
  219. mlrun/feature_store/feature_set.py +42 -12
  220. mlrun/feature_store/feature_vector.py +32 -21
  221. mlrun/feature_store/ingestion.py +9 -12
  222. mlrun/feature_store/retrieval/__init__.py +3 -2
  223. mlrun/feature_store/retrieval/base.py +388 -66
  224. mlrun/feature_store/retrieval/dask_merger.py +63 -151
  225. mlrun/feature_store/retrieval/job.py +30 -12
  226. mlrun/feature_store/retrieval/local_merger.py +40 -133
  227. mlrun/feature_store/retrieval/spark_merger.py +129 -127
  228. mlrun/feature_store/retrieval/storey_merger.py +173 -0
  229. mlrun/feature_store/steps.py +132 -15
  230. mlrun/features.py +8 -3
  231. mlrun/frameworks/__init__.py +1 -1
  232. mlrun/frameworks/_common/__init__.py +1 -1
  233. mlrun/frameworks/_common/artifacts_library.py +1 -1
  234. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  235. mlrun/frameworks/_common/model_handler.py +1 -1
  236. mlrun/frameworks/_common/plan.py +1 -1
  237. mlrun/frameworks/_common/producer.py +1 -1
  238. mlrun/frameworks/_common/utils.py +1 -1
  239. mlrun/frameworks/_dl_common/__init__.py +1 -1
  240. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -1
  241. mlrun/frameworks/_dl_common/loggers/logger.py +1 -1
  242. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  243. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  244. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  245. mlrun/frameworks/_dl_common/utils.py +1 -1
  246. mlrun/frameworks/_ml_common/__init__.py +1 -1
  247. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  248. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -1
  249. mlrun/frameworks/_ml_common/loggers/logger.py +1 -1
  250. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  251. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  252. mlrun/frameworks/_ml_common/pkl_model_server.py +13 -1
  253. mlrun/frameworks/_ml_common/plan.py +1 -1
  254. mlrun/frameworks/_ml_common/plans/__init__.py +1 -1
  255. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +1 -6
  256. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +1 -1
  257. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  258. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  259. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  260. mlrun/frameworks/_ml_common/producer.py +1 -1
  261. mlrun/frameworks/_ml_common/utils.py +1 -1
  262. mlrun/frameworks/auto_mlrun/__init__.py +1 -1
  263. mlrun/frameworks/auto_mlrun/auto_mlrun.py +1 -1
  264. mlrun/frameworks/huggingface/__init__.py +1 -1
  265. mlrun/frameworks/huggingface/model_server.py +1 -1
  266. mlrun/frameworks/lgbm/__init__.py +1 -1
  267. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -1
  268. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  269. mlrun/frameworks/lgbm/callbacks/logging_callback.py +1 -1
  270. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +1 -1
  271. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -1
  272. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  273. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  274. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  275. mlrun/frameworks/lgbm/model_handler.py +1 -1
  276. mlrun/frameworks/lgbm/model_server.py +1 -1
  277. mlrun/frameworks/lgbm/utils.py +1 -1
  278. mlrun/frameworks/onnx/__init__.py +1 -1
  279. mlrun/frameworks/onnx/dataset.py +1 -1
  280. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  281. mlrun/frameworks/onnx/model_handler.py +1 -1
  282. mlrun/frameworks/onnx/model_server.py +1 -1
  283. mlrun/frameworks/parallel_coordinates.py +1 -1
  284. mlrun/frameworks/pytorch/__init__.py +1 -1
  285. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -1
  286. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  287. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  288. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  289. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  290. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  291. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  292. mlrun/frameworks/pytorch/model_handler.py +1 -1
  293. mlrun/frameworks/pytorch/model_server.py +1 -1
  294. mlrun/frameworks/pytorch/utils.py +1 -1
  295. mlrun/frameworks/sklearn/__init__.py +1 -1
  296. mlrun/frameworks/sklearn/estimator.py +1 -1
  297. mlrun/frameworks/sklearn/metric.py +1 -1
  298. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  299. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  300. mlrun/frameworks/sklearn/model_handler.py +1 -1
  301. mlrun/frameworks/sklearn/utils.py +1 -1
  302. mlrun/frameworks/tf_keras/__init__.py +1 -1
  303. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -1
  304. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  305. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  306. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  307. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  308. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  309. mlrun/frameworks/tf_keras/model_server.py +1 -1
  310. mlrun/frameworks/tf_keras/utils.py +1 -1
  311. mlrun/frameworks/xgboost/__init__.py +1 -1
  312. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  313. mlrun/frameworks/xgboost/model_handler.py +1 -1
  314. mlrun/frameworks/xgboost/utils.py +1 -1
  315. mlrun/k8s_utils.py +14 -765
  316. mlrun/kfpops.py +14 -17
  317. mlrun/launcher/__init__.py +13 -0
  318. mlrun/launcher/base.py +406 -0
  319. mlrun/launcher/client.py +159 -0
  320. mlrun/launcher/factory.py +50 -0
  321. mlrun/launcher/local.py +276 -0
  322. mlrun/launcher/remote.py +178 -0
  323. mlrun/lists.py +10 -2
  324. mlrun/mlutils/__init__.py +1 -1
  325. mlrun/mlutils/data.py +1 -1
  326. mlrun/mlutils/models.py +1 -1
  327. mlrun/mlutils/plots.py +1 -1
  328. mlrun/model.py +252 -14
  329. mlrun/model_monitoring/__init__.py +41 -0
  330. mlrun/model_monitoring/features_drift_table.py +1 -1
  331. mlrun/model_monitoring/helpers.py +123 -38
  332. mlrun/model_monitoring/model_endpoint.py +144 -0
  333. mlrun/model_monitoring/model_monitoring_batch.py +310 -259
  334. mlrun/model_monitoring/stores/__init__.py +106 -0
  335. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +448 -0
  336. mlrun/model_monitoring/stores/model_endpoint_store.py +147 -0
  337. mlrun/model_monitoring/stores/models/__init__.py +23 -0
  338. mlrun/model_monitoring/stores/models/base.py +18 -0
  339. mlrun/model_monitoring/stores/models/mysql.py +100 -0
  340. mlrun/model_monitoring/stores/models/sqlite.py +98 -0
  341. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +370 -0
  342. mlrun/model_monitoring/stream_processing_fs.py +239 -271
  343. mlrun/package/__init__.py +163 -0
  344. mlrun/package/context_handler.py +325 -0
  345. mlrun/package/errors.py +47 -0
  346. mlrun/package/packager.py +298 -0
  347. mlrun/{runtimes/package → package/packagers}/__init__.py +3 -1
  348. mlrun/package/packagers/default_packager.py +422 -0
  349. mlrun/package/packagers/numpy_packagers.py +612 -0
  350. mlrun/package/packagers/pandas_packagers.py +968 -0
  351. mlrun/package/packagers/python_standard_library_packagers.py +616 -0
  352. mlrun/package/packagers_manager.py +786 -0
  353. mlrun/package/utils/__init__.py +53 -0
  354. mlrun/package/utils/_archiver.py +226 -0
  355. mlrun/package/utils/_formatter.py +211 -0
  356. mlrun/package/utils/_pickler.py +234 -0
  357. mlrun/package/utils/_supported_format.py +71 -0
  358. mlrun/package/utils/log_hint_utils.py +93 -0
  359. mlrun/package/utils/type_hint_utils.py +298 -0
  360. mlrun/platforms/__init__.py +1 -1
  361. mlrun/platforms/iguazio.py +34 -2
  362. mlrun/platforms/other.py +1 -1
  363. mlrun/projects/__init__.py +1 -1
  364. mlrun/projects/operations.py +14 -9
  365. mlrun/projects/pipelines.py +31 -13
  366. mlrun/projects/project.py +762 -238
  367. mlrun/render.py +49 -19
  368. mlrun/run.py +57 -326
  369. mlrun/runtimes/__init__.py +3 -9
  370. mlrun/runtimes/base.py +247 -784
  371. mlrun/runtimes/constants.py +1 -1
  372. mlrun/runtimes/daskjob.py +45 -41
  373. mlrun/runtimes/funcdoc.py +43 -7
  374. mlrun/runtimes/function.py +66 -656
  375. mlrun/runtimes/function_reference.py +1 -1
  376. mlrun/runtimes/generators.py +1 -1
  377. mlrun/runtimes/kubejob.py +99 -116
  378. mlrun/runtimes/local.py +59 -66
  379. mlrun/runtimes/mpijob/__init__.py +1 -1
  380. mlrun/runtimes/mpijob/abstract.py +13 -15
  381. mlrun/runtimes/mpijob/v1.py +3 -1
  382. mlrun/runtimes/mpijob/v1alpha1.py +1 -1
  383. mlrun/runtimes/nuclio.py +1 -1
  384. mlrun/runtimes/pod.py +51 -26
  385. mlrun/runtimes/remotesparkjob.py +3 -1
  386. mlrun/runtimes/serving.py +12 -4
  387. mlrun/runtimes/sparkjob/__init__.py +1 -2
  388. mlrun/runtimes/sparkjob/abstract.py +44 -31
  389. mlrun/runtimes/sparkjob/spark3job.py +11 -9
  390. mlrun/runtimes/utils.py +61 -42
  391. mlrun/secrets.py +16 -18
  392. mlrun/serving/__init__.py +3 -2
  393. mlrun/serving/merger.py +1 -1
  394. mlrun/serving/remote.py +1 -1
  395. mlrun/serving/routers.py +39 -42
  396. mlrun/serving/server.py +23 -13
  397. mlrun/serving/serving_wrapper.py +1 -1
  398. mlrun/serving/states.py +172 -39
  399. mlrun/serving/utils.py +1 -1
  400. mlrun/serving/v1_serving.py +1 -1
  401. mlrun/serving/v2_serving.py +29 -21
  402. mlrun/utils/__init__.py +1 -2
  403. mlrun/utils/async_http.py +8 -1
  404. mlrun/utils/azure_vault.py +1 -1
  405. mlrun/utils/clones.py +2 -2
  406. mlrun/utils/condition_evaluator.py +65 -0
  407. mlrun/utils/db.py +52 -0
  408. mlrun/utils/helpers.py +188 -13
  409. mlrun/utils/http.py +89 -54
  410. mlrun/utils/logger.py +48 -8
  411. mlrun/utils/model_monitoring.py +132 -100
  412. mlrun/utils/notifications/__init__.py +1 -1
  413. mlrun/utils/notifications/notification/__init__.py +8 -6
  414. mlrun/utils/notifications/notification/base.py +20 -14
  415. mlrun/utils/notifications/notification/console.py +7 -4
  416. mlrun/utils/notifications/notification/git.py +36 -19
  417. mlrun/utils/notifications/notification/ipython.py +10 -8
  418. mlrun/utils/notifications/notification/slack.py +18 -13
  419. mlrun/utils/notifications/notification_pusher.py +377 -56
  420. mlrun/utils/regex.py +6 -1
  421. mlrun/utils/singleton.py +1 -1
  422. mlrun/utils/v3io_clients.py +1 -1
  423. mlrun/utils/vault.py +270 -269
  424. mlrun/utils/version/__init__.py +1 -1
  425. mlrun/utils/version/version.json +2 -2
  426. mlrun/utils/version/version.py +1 -1
  427. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/METADATA +16 -10
  428. mlrun-1.4.0.dist-info/RECORD +434 -0
  429. mlrun/api/api/endpoints/marketplace.py +0 -257
  430. mlrun/api/crud/marketplace.py +0 -221
  431. mlrun/api/crud/model_monitoring/model_endpoint_store.py +0 -847
  432. mlrun/api/db/filedb/db.py +0 -518
  433. mlrun/api/schemas/marketplace.py +0 -128
  434. mlrun/api/schemas/model_endpoints.py +0 -185
  435. mlrun/db/filedb.py +0 -891
  436. mlrun/feature_store/retrieval/online.py +0 -92
  437. mlrun/model_monitoring/constants.py +0 -67
  438. mlrun/runtimes/package/context_handler.py +0 -711
  439. mlrun/runtimes/sparkjob/spark2job.py +0 -59
  440. mlrun-1.3.3.dist-info/RECORD +0 -381
  441. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/LICENSE +0 -0
  442. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/WHEEL +0 -0
  443. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/entry_points.txt +0 -0
  444. {mlrun-1.3.3.dist-info → mlrun-1.4.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,342 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ import enum
17
+ import json
18
+ import typing
19
+ from typing import Any, Dict, List, Optional, Tuple, Union
20
+
21
+ from pydantic import BaseModel, Field
22
+ from pydantic.main import Extra
23
+
24
+ import mlrun.common.model_monitoring
25
+ from mlrun.common.schemas.object import ObjectKind, ObjectSpec, ObjectStatus
26
+
27
+
28
+ class ModelMonitoringStoreKinds:
29
+ # TODO: do changes in examples & demos In 1.5.0 remove
30
+ ENDPOINTS = "endpoints"
31
+ EVENTS = "events"
32
+
33
+
34
+ class ModelEndpointMetadata(BaseModel):
35
+ project: Optional[str] = ""
36
+ labels: Optional[dict] = {}
37
+ uid: Optional[str] = ""
38
+
39
+ class Config:
40
+ extra = Extra.allow
41
+
42
+ @classmethod
43
+ def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
44
+ """Create a `ModelEndpointMetadata` object from an endpoint dictionary
45
+
46
+ :param endpoint_dict: Model endpoint dictionary.
47
+ :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
48
+ dictionary using json.loads().
49
+ """
50
+ new_object = cls()
51
+ if json_parse_values is None:
52
+ json_parse_values = [mlrun.common.model_monitoring.EventFieldType.LABELS]
53
+
54
+ return _mapping_attributes(
55
+ base_model=new_object,
56
+ flattened_dictionary=endpoint_dict,
57
+ json_parse_values=json_parse_values,
58
+ )
59
+
60
+
61
+ class ModelEndpointSpec(ObjectSpec):
62
+ function_uri: Optional[str] = "" # <project_name>/<function_name>:<tag>
63
+ model: Optional[str] = "" # <model_name>:<version>
64
+ model_class: Optional[str] = ""
65
+ model_uri: Optional[str] = ""
66
+ feature_names: Optional[List[str]] = []
67
+ label_names: Optional[List[str]] = []
68
+ stream_path: Optional[str] = ""
69
+ algorithm: Optional[str] = ""
70
+ monitor_configuration: Optional[dict] = {}
71
+ active: Optional[bool] = True
72
+ monitoring_mode: Optional[
73
+ mlrun.common.model_monitoring.ModelMonitoringMode
74
+ ] = mlrun.common.model_monitoring.ModelMonitoringMode.disabled.value
75
+
76
+ @classmethod
77
+ def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
78
+ """Create a `ModelEndpointSpec` object from an endpoint dictionary
79
+
80
+ :param endpoint_dict: Model endpoint dictionary.
81
+ :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
82
+ dictionary using json.loads().
83
+ """
84
+ new_object = cls()
85
+ if json_parse_values is None:
86
+ json_parse_values = [
87
+ mlrun.common.model_monitoring.EventFieldType.FEATURE_NAMES,
88
+ mlrun.common.model_monitoring.EventFieldType.LABEL_NAMES,
89
+ mlrun.common.model_monitoring.EventFieldType.MONITOR_CONFIGURATION,
90
+ ]
91
+ return _mapping_attributes(
92
+ base_model=new_object,
93
+ flattened_dictionary=endpoint_dict,
94
+ json_parse_values=json_parse_values,
95
+ )
96
+
97
+
98
+ class Histogram(BaseModel):
99
+ buckets: List[float]
100
+ counts: List[int]
101
+
102
+
103
+ class FeatureValues(BaseModel):
104
+ min: float
105
+ mean: float
106
+ max: float
107
+ histogram: Histogram
108
+
109
+ @classmethod
110
+ def from_dict(cls, stats: Optional[dict]):
111
+ if stats:
112
+ return FeatureValues(
113
+ min=stats["min"],
114
+ mean=stats["mean"],
115
+ max=stats["max"],
116
+ histogram=Histogram(buckets=stats["hist"][1], counts=stats["hist"][0]),
117
+ )
118
+ else:
119
+ return None
120
+
121
+
122
+ class Features(BaseModel):
123
+ name: str
124
+ weight: float
125
+ expected: Optional[FeatureValues]
126
+ actual: Optional[FeatureValues]
127
+
128
+ @classmethod
129
+ def new(
130
+ cls,
131
+ feature_name: str,
132
+ feature_stats: Optional[dict],
133
+ current_stats: Optional[dict],
134
+ ):
135
+ return cls(
136
+ name=feature_name,
137
+ weight=-1.0,
138
+ expected=FeatureValues.from_dict(feature_stats),
139
+ actual=FeatureValues.from_dict(current_stats),
140
+ )
141
+
142
+
143
+ class ModelEndpointStatus(ObjectStatus):
144
+ feature_stats: Optional[dict] = {}
145
+ current_stats: Optional[dict] = {}
146
+ first_request: Optional[str] = ""
147
+ last_request: Optional[str] = ""
148
+ error_count: Optional[int] = 0
149
+ drift_status: Optional[str] = ""
150
+ drift_measures: Optional[dict] = {}
151
+ metrics: Optional[Dict[str, Dict[str, Any]]] = {
152
+ mlrun.common.model_monitoring.EventKeyMetrics.GENERIC: {
153
+ mlrun.common.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
154
+ mlrun.common.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
155
+ }
156
+ }
157
+ features: Optional[List[Features]] = []
158
+ children: Optional[List[str]] = []
159
+ children_uids: Optional[List[str]] = []
160
+ endpoint_type: Optional[
161
+ mlrun.common.model_monitoring.EndpointType
162
+ ] = mlrun.common.model_monitoring.EndpointType.NODE_EP.value
163
+ monitoring_feature_set_uri: Optional[str] = ""
164
+ state: Optional[str] = ""
165
+
166
+ class Config:
167
+ extra = Extra.allow
168
+
169
+ @classmethod
170
+ def from_flat_dict(cls, endpoint_dict: dict, json_parse_values: typing.List = None):
171
+ """Create a `ModelEndpointStatus` object from an endpoint dictionary
172
+
173
+ :param endpoint_dict: Model endpoint dictionary.
174
+ :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
175
+ dictionary using json.loads().
176
+ """
177
+ new_object = cls()
178
+ if json_parse_values is None:
179
+ json_parse_values = [
180
+ mlrun.common.model_monitoring.EventFieldType.FEATURE_STATS,
181
+ mlrun.common.model_monitoring.EventFieldType.CURRENT_STATS,
182
+ mlrun.common.model_monitoring.EventFieldType.DRIFT_MEASURES,
183
+ mlrun.common.model_monitoring.EventFieldType.METRICS,
184
+ mlrun.common.model_monitoring.EventFieldType.CHILDREN,
185
+ mlrun.common.model_monitoring.EventFieldType.CHILDREN_UIDS,
186
+ mlrun.common.model_monitoring.EventFieldType.ENDPOINT_TYPE,
187
+ ]
188
+ return _mapping_attributes(
189
+ base_model=new_object,
190
+ flattened_dictionary=endpoint_dict,
191
+ json_parse_values=json_parse_values,
192
+ )
193
+
194
+
195
+ class ModelEndpoint(BaseModel):
196
+ kind: ObjectKind = Field(ObjectKind.model_endpoint, const=True)
197
+ metadata: ModelEndpointMetadata = ModelEndpointMetadata()
198
+ spec: ModelEndpointSpec = ModelEndpointSpec()
199
+ status: ModelEndpointStatus = ModelEndpointStatus()
200
+
201
+ class Config:
202
+ extra = Extra.allow
203
+
204
+ def __init__(self, **data: Any):
205
+ super().__init__(**data)
206
+ if self.metadata.uid is None:
207
+ uid = mlrun.common.model_monitoring.create_model_endpoint_uid(
208
+ function_uri=self.spec.function_uri,
209
+ versioned_model=self.spec.model,
210
+ )
211
+ self.metadata.uid = str(uid)
212
+
213
+ def flat_dict(self):
214
+ """Generate a flattened `ModelEndpoint` dictionary. The flattened dictionary result is important for storing
215
+ the model endpoint object in the database.
216
+
217
+ :return: Flattened `ModelEndpoint` dictionary.
218
+ """
219
+ # Convert the ModelEndpoint object into a dictionary using BaseModel dict() function
220
+ # In addition, remove the BaseModel kind as it is not required by the DB schema
221
+ model_endpoint_dictionary = self.dict(exclude={"kind"})
222
+
223
+ # Initialize a flattened dictionary that will be filled with the model endpoint dictionary attributes
224
+ flatten_dict = {}
225
+ for k_object in model_endpoint_dictionary:
226
+ for key in model_endpoint_dictionary[k_object]:
227
+ # Extract the value of the current field
228
+ current_value = model_endpoint_dictionary[k_object][key]
229
+
230
+ # If the value is not from type str or bool (e.g. dict), convert it into a JSON string
231
+ # for matching the database required format
232
+ if not isinstance(current_value, (str, bool, int)) or isinstance(
233
+ current_value, enum.IntEnum
234
+ ):
235
+ flatten_dict[key] = json.dumps(current_value)
236
+ else:
237
+ flatten_dict[key] = current_value
238
+
239
+ if mlrun.common.model_monitoring.EventFieldType.METRICS not in flatten_dict:
240
+ # Initialize metrics dictionary
241
+ flatten_dict[mlrun.common.model_monitoring.EventFieldType.METRICS] = {
242
+ mlrun.common.model_monitoring.EventKeyMetrics.GENERIC: {
243
+ mlrun.common.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
244
+ mlrun.common.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
245
+ }
246
+ }
247
+
248
+ # Remove the features from the dictionary as this field will be filled only within the feature analysis process
249
+ flatten_dict.pop(mlrun.common.model_monitoring.EventFieldType.FEATURES, None)
250
+ return flatten_dict
251
+
252
+ @classmethod
253
+ def from_flat_dict(cls, endpoint_dict: dict):
254
+ """Create a `ModelEndpoint` object from an endpoint flattened dictionary. Because the provided dictionary
255
+ is flattened, we pass it as is to the subclasses without splitting the keys into spec, metadata, and status.
256
+
257
+ :param endpoint_dict: Model endpoint dictionary.
258
+ """
259
+
260
+ return cls(
261
+ metadata=ModelEndpointMetadata.from_flat_dict(endpoint_dict=endpoint_dict),
262
+ spec=ModelEndpointSpec.from_flat_dict(endpoint_dict=endpoint_dict),
263
+ status=ModelEndpointStatus.from_flat_dict(endpoint_dict=endpoint_dict),
264
+ )
265
+
266
+
267
+ class ModelEndpointList(BaseModel):
268
+ endpoints: List[ModelEndpoint] = []
269
+
270
+
271
+ class GrafanaColumn(BaseModel):
272
+ text: str
273
+ type: str
274
+
275
+
276
+ class GrafanaNumberColumn(GrafanaColumn):
277
+ text: str
278
+ type: str = "number"
279
+
280
+
281
+ class GrafanaStringColumn(GrafanaColumn):
282
+ text: str
283
+ type: str = "string"
284
+
285
+
286
+ class GrafanaTable(BaseModel):
287
+ columns: List[GrafanaColumn]
288
+ rows: List[List[Optional[Union[float, int, str]]]] = []
289
+ type: str = "table"
290
+
291
+ def add_row(self, *args):
292
+ self.rows.append(list(args))
293
+
294
+
295
+ class GrafanaDataPoint(BaseModel):
296
+ value: float
297
+ timestamp: int # Unix timestamp in milliseconds
298
+
299
+
300
+ class GrafanaTimeSeriesTarget(BaseModel):
301
+ target: str
302
+ datapoints: List[Tuple[float, int]] = []
303
+
304
+ def add_data_point(self, data_point: GrafanaDataPoint):
305
+ self.datapoints.append((data_point.value, data_point.timestamp))
306
+
307
+
308
+ def _mapping_attributes(
309
+ base_model: BaseModel,
310
+ flattened_dictionary: dict,
311
+ json_parse_values: typing.List = None,
312
+ ):
313
+ """Generate a `BaseModel` object with the provided dictionary attributes.
314
+
315
+ :param base_model: `BaseModel` object (e.g. `ModelEndpointMetadata`).
316
+ :param flattened_dictionary: Flattened dictionary that contains the model endpoint attributes.
317
+ :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
318
+ dictionary using json.loads().
319
+ """
320
+ # Get the fields of the provided base model object. These fields will be used to filter to relevent keys
321
+ # from the flattened dictionary.
322
+ wanted_keys = base_model.__fields__.keys()
323
+
324
+ # Generate a filtered flattened dictionary that will be parsed into the BaseModel object
325
+ dict_to_parse = {}
326
+ for field_key in wanted_keys:
327
+ if field_key in flattened_dictionary:
328
+ if field_key in json_parse_values:
329
+ # Parse the JSON value into a valid dictionary
330
+ dict_to_parse[field_key] = _json_loads_if_not_none(
331
+ flattened_dictionary[field_key]
332
+ )
333
+ else:
334
+ dict_to_parse[field_key] = flattened_dictionary[field_key]
335
+
336
+ return base_model.parse_obj(dict_to_parse)
337
+
338
+
339
+ def _json_loads_if_not_none(field: Any) -> Any:
340
+ return (
341
+ json.loads(field) if field and field != "null" and field is not None else None
342
+ )
@@ -0,0 +1,57 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import datetime
16
+ import typing
17
+
18
+ import pydantic
19
+
20
+ import mlrun.common.types
21
+
22
+
23
+ class NotificationKind(mlrun.common.types.StrEnum):
24
+ console = "console"
25
+ git = "git"
26
+ ipython = "ipython"
27
+ slack = "slack"
28
+
29
+
30
+ class NotificationSeverity(mlrun.common.types.StrEnum):
31
+ INFO = "info"
32
+ DEBUG = "debug"
33
+ VERBOSE = "verbose"
34
+ WARNING = "warning"
35
+ ERROR = "error"
36
+
37
+
38
+ class NotificationStatus(mlrun.common.types.StrEnum):
39
+ PENDING = "pending"
40
+ SENT = "sent"
41
+ ERROR = "error"
42
+
43
+
44
+ class Notification(pydantic.BaseModel):
45
+ kind: NotificationKind
46
+ name: str
47
+ message: str
48
+ severity: NotificationSeverity
49
+ when: typing.List[str]
50
+ condition: str
51
+ params: typing.Dict[str, typing.Any] = None
52
+ status: NotificationStatus = None
53
+ sent_time: typing.Union[str, datetime.datetime] = None
54
+
55
+
56
+ class SetNotificationRequest(pydantic.BaseModel):
57
+ notifications: typing.List[Notification] = None
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@ from typing import List, Optional
17
17
 
18
18
  from pydantic import BaseModel, Extra
19
19
 
20
- import mlrun.api.utils.helpers
20
+ import mlrun.common.types
21
21
 
22
22
 
23
23
  class ObjectMetadata(BaseModel):
@@ -69,12 +69,12 @@ class ObjectRecord(BaseModel):
69
69
  orm_mode = True
70
70
 
71
71
 
72
- class ObjectKind(mlrun.api.utils.helpers.StrEnum):
72
+ class ObjectKind(mlrun.common.types.StrEnum):
73
73
  project = "project"
74
74
  feature_set = "FeatureSet"
75
75
  background_task = "BackgroundTask"
76
76
  feature_vector = "FeatureVector"
77
77
  model_endpoint = "model-endpoint"
78
- marketplace_source = "MarketplaceSource"
79
- marketplace_item = "MarketplaceItem"
80
- marketplace_catalog = "MarketplaceCatalog"
78
+ hub_source = "HubSource"
79
+ hub_item = "HubItem"
80
+ hub_catalog = "HubCatalog"
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -16,10 +16,10 @@ import typing
16
16
 
17
17
  import pydantic
18
18
 
19
- import mlrun.api.utils.helpers
19
+ import mlrun.common.types
20
20
 
21
21
 
22
- class PipelinesFormat(mlrun.api.utils.helpers.StrEnum):
22
+ class PipelinesFormat(mlrun.common.types.StrEnum):
23
23
  full = "full"
24
24
  metadata_only = "metadata_only"
25
25
  summary = "summary"
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -17,12 +17,12 @@ import typing
17
17
 
18
18
  import pydantic
19
19
 
20
- import mlrun.api.utils.helpers
20
+ import mlrun.common.types
21
21
 
22
22
  from .object import ObjectKind, ObjectStatus
23
23
 
24
24
 
25
- class ProjectsFormat(mlrun.api.utils.helpers.StrEnum):
25
+ class ProjectsFormat(mlrun.common.types.StrEnum):
26
26
  full = "full"
27
27
  name_only = "name_only"
28
28
  # minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
@@ -42,13 +42,13 @@ class ProjectMetadata(pydantic.BaseModel):
42
42
  extra = pydantic.Extra.allow
43
43
 
44
44
 
45
- class ProjectDesiredState(mlrun.api.utils.helpers.StrEnum):
45
+ class ProjectDesiredState(mlrun.common.types.StrEnum):
46
46
  online = "online"
47
47
  offline = "offline"
48
48
  archived = "archived"
49
49
 
50
50
 
51
- class ProjectState(mlrun.api.utils.helpers.StrEnum):
51
+ class ProjectState(mlrun.common.types.StrEnum):
52
52
  unknown = "unknown"
53
53
  creating = "creating"
54
54
  deleting = "deleting"
@@ -83,6 +83,7 @@ class ProjectSpec(pydantic.BaseModel):
83
83
  subpath: typing.Optional[str] = None
84
84
  origin_url: typing.Optional[str] = None
85
85
  desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
86
+ custom_packagers: typing.Optional[typing.List[typing.Tuple[str, bool]]] = None
86
87
 
87
88
  class Config:
88
89
  extra = pydantic.Extra.allow
@@ -0,0 +1,24 @@
1
+ # Copyright 2018 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ import mlrun.common.types
16
+
17
+
18
+ class RegexMatchModes(mlrun.common.types.StrEnum):
19
+ """Regex match modes"""
20
+
21
+ # all regexes must match
22
+ all = "all"
23
+ # any of the regexes must match
24
+ any = "any"
@@ -0,0 +1,30 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import typing
16
+
17
+ # TODO: When we remove support for python 3.7, we can use Literal from the typing package.
18
+ # Remove the following try/except block with import from typing_extensions.
19
+ try:
20
+ from typing import Literal
21
+ except ImportError:
22
+ from typing_extensions import Literal
23
+
24
+ import pydantic
25
+
26
+
27
+ class RunIdentifier(pydantic.BaseModel):
28
+ kind: Literal["run"] = "run"
29
+ uid: typing.Optional[str]
30
+ iter: typing.Optional[int]
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -16,10 +16,10 @@ import typing
16
16
 
17
17
  import pydantic
18
18
 
19
- import mlrun.api.utils.helpers
19
+ import mlrun.common.types
20
20
 
21
21
 
22
- class ListRuntimeResourcesGroupByField(mlrun.api.utils.helpers.StrEnum):
22
+ class ListRuntimeResourcesGroupByField(mlrun.common.types.StrEnum):
23
23
  job = "job"
24
24
  project = "project"
25
25
 
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -15,11 +15,18 @@
15
15
  from datetime import datetime
16
16
  from typing import Any, List, Optional, Union
17
17
 
18
+ # TODO: When we remove support for python 3.7, we can use Literal from the typing package.
19
+ # Remove the following try/except block with import from typing_extensions.
20
+ try:
21
+ from typing import Literal
22
+ except ImportError:
23
+ from typing_extensions import Literal
24
+
18
25
  from pydantic import BaseModel
19
26
 
20
- import mlrun.api.utils.helpers
21
- from mlrun.api.schemas.auth import Credentials
22
- from mlrun.api.schemas.object import LabelRecord
27
+ import mlrun.common.types
28
+ from mlrun.common.schemas.auth import Credentials
29
+ from mlrun.common.schemas.object import LabelRecord
23
30
 
24
31
 
25
32
  class ScheduleCronTrigger(BaseModel):
@@ -36,8 +43,8 @@ class ScheduleCronTrigger(BaseModel):
36
43
  hour: Optional[Union[int, str]]
37
44
  minute: Optional[Union[int, str]]
38
45
  second: Optional[Union[int, str]]
39
- start_date: Optional[Union[datetime, str]]
40
- end_date: Optional[Union[datetime, str]]
46
+ start_date: Union[datetime, str] = None
47
+ end_date: Union[datetime, str] = None
41
48
 
42
49
  # APScheduler also supports datetime.tzinfo type, but Pydantic doesn't - so we don't
43
50
  timezone: Optional[str]
@@ -78,7 +85,7 @@ class ScheduleCronTrigger(BaseModel):
78
85
  return f"{self.minute} {self.hour} {self.day} {self.month} {self.day_of_week}"
79
86
 
80
87
 
81
- class ScheduleKinds(mlrun.api.utils.helpers.StrEnum):
88
+ class ScheduleKinds(mlrun.common.types.StrEnum):
82
89
  job = "job"
83
90
  pipeline = "pipeline"
84
91
 
@@ -136,3 +143,8 @@ class ScheduleOutput(ScheduleRecord):
136
143
 
137
144
  class SchedulesOutput(BaseModel):
138
145
  schedules: List[ScheduleOutput]
146
+
147
+
148
+ class ScheduleIdentifier(BaseModel):
149
+ kind: Literal["schedule"] = "schedule"
150
+ name: str
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -16,10 +16,10 @@ from typing import Optional
16
16
 
17
17
  from pydantic import BaseModel, Field
18
18
 
19
- import mlrun.api.utils.helpers
19
+ import mlrun.common.types
20
20
 
21
21
 
22
- class SecretProviderName(mlrun.api.utils.helpers.StrEnum):
22
+ class SecretProviderName(mlrun.common.types.StrEnum):
23
23
  """Enum containing names of valid providers for secrets."""
24
24
 
25
25
  vault = "vault"
@@ -1,4 +1,4 @@
1
- # Copyright 2018 Iguazio
1
+ # Copyright 2023 Iguazio
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -29,4 +29,4 @@ class TagObjects(pydantic.BaseModel):
29
29
 
30
30
  kind: str
31
31
  # TODO: Add more types to the list for new supported tagged objects
32
- identifiers: typing.List[typing.Union[ArtifactIdentifier]]
32
+ identifiers: typing.List[ArtifactIdentifier]
mlrun/common/types.py ADDED
@@ -0,0 +1,25 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ import enum
17
+
18
+
19
+ # TODO: From python 3.11 StrEnum is built-in and this will not be needed
20
+ class StrEnum(str, enum.Enum):
21
+ def __str__(self):
22
+ return self.value
23
+
24
+ def __repr__(self):
25
+ return self.value