mlrun 1.6.4rc7__py3-none-any.whl → 1.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (305) hide show
  1. mlrun/__init__.py +11 -1
  2. mlrun/__main__.py +40 -122
  3. mlrun/alerts/__init__.py +15 -0
  4. mlrun/alerts/alert.py +248 -0
  5. mlrun/api/schemas/__init__.py +5 -4
  6. mlrun/artifacts/__init__.py +8 -3
  7. mlrun/artifacts/base.py +47 -257
  8. mlrun/artifacts/dataset.py +11 -192
  9. mlrun/artifacts/manager.py +79 -47
  10. mlrun/artifacts/model.py +31 -159
  11. mlrun/artifacts/plots.py +23 -380
  12. mlrun/common/constants.py +74 -1
  13. mlrun/common/db/sql_session.py +5 -5
  14. mlrun/common/formatters/__init__.py +21 -0
  15. mlrun/common/formatters/artifact.py +45 -0
  16. mlrun/common/formatters/base.py +113 -0
  17. mlrun/common/formatters/feature_set.py +33 -0
  18. mlrun/common/formatters/function.py +46 -0
  19. mlrun/common/formatters/pipeline.py +53 -0
  20. mlrun/common/formatters/project.py +51 -0
  21. mlrun/common/formatters/run.py +29 -0
  22. mlrun/common/helpers.py +12 -3
  23. mlrun/common/model_monitoring/helpers.py +9 -5
  24. mlrun/{runtimes → common/runtimes}/constants.py +37 -9
  25. mlrun/common/schemas/__init__.py +31 -5
  26. mlrun/common/schemas/alert.py +202 -0
  27. mlrun/common/schemas/api_gateway.py +196 -0
  28. mlrun/common/schemas/artifact.py +25 -4
  29. mlrun/common/schemas/auth.py +16 -5
  30. mlrun/common/schemas/background_task.py +1 -1
  31. mlrun/common/schemas/client_spec.py +4 -2
  32. mlrun/common/schemas/common.py +7 -4
  33. mlrun/common/schemas/constants.py +3 -0
  34. mlrun/common/schemas/feature_store.py +74 -44
  35. mlrun/common/schemas/frontend_spec.py +15 -7
  36. mlrun/common/schemas/function.py +12 -1
  37. mlrun/common/schemas/hub.py +11 -18
  38. mlrun/common/schemas/memory_reports.py +2 -2
  39. mlrun/common/schemas/model_monitoring/__init__.py +20 -4
  40. mlrun/common/schemas/model_monitoring/constants.py +123 -42
  41. mlrun/common/schemas/model_monitoring/grafana.py +13 -9
  42. mlrun/common/schemas/model_monitoring/model_endpoints.py +101 -54
  43. mlrun/common/schemas/notification.py +71 -14
  44. mlrun/common/schemas/object.py +2 -2
  45. mlrun/{model_monitoring/controller_handler.py → common/schemas/pagination.py} +9 -12
  46. mlrun/common/schemas/pipeline.py +8 -1
  47. mlrun/common/schemas/project.py +69 -18
  48. mlrun/common/schemas/runs.py +7 -1
  49. mlrun/common/schemas/runtime_resource.py +8 -12
  50. mlrun/common/schemas/schedule.py +4 -4
  51. mlrun/common/schemas/tag.py +1 -2
  52. mlrun/common/schemas/workflow.py +12 -4
  53. mlrun/common/types.py +14 -1
  54. mlrun/config.py +154 -69
  55. mlrun/data_types/data_types.py +6 -1
  56. mlrun/data_types/spark.py +2 -2
  57. mlrun/data_types/to_pandas.py +67 -37
  58. mlrun/datastore/__init__.py +6 -8
  59. mlrun/datastore/alibaba_oss.py +131 -0
  60. mlrun/datastore/azure_blob.py +143 -42
  61. mlrun/datastore/base.py +102 -58
  62. mlrun/datastore/datastore.py +34 -13
  63. mlrun/datastore/datastore_profile.py +146 -20
  64. mlrun/datastore/dbfs_store.py +3 -7
  65. mlrun/datastore/filestore.py +1 -4
  66. mlrun/datastore/google_cloud_storage.py +97 -33
  67. mlrun/datastore/hdfs.py +56 -0
  68. mlrun/datastore/inmem.py +6 -3
  69. mlrun/datastore/redis.py +7 -2
  70. mlrun/datastore/s3.py +34 -12
  71. mlrun/datastore/snowflake_utils.py +45 -0
  72. mlrun/datastore/sources.py +303 -111
  73. mlrun/datastore/spark_utils.py +31 -2
  74. mlrun/datastore/store_resources.py +9 -7
  75. mlrun/datastore/storeytargets.py +151 -0
  76. mlrun/datastore/targets.py +453 -176
  77. mlrun/datastore/utils.py +72 -58
  78. mlrun/datastore/v3io.py +6 -1
  79. mlrun/db/base.py +274 -41
  80. mlrun/db/factory.py +1 -1
  81. mlrun/db/httpdb.py +893 -225
  82. mlrun/db/nopdb.py +291 -33
  83. mlrun/errors.py +36 -6
  84. mlrun/execution.py +115 -42
  85. mlrun/feature_store/__init__.py +0 -2
  86. mlrun/feature_store/api.py +65 -73
  87. mlrun/feature_store/common.py +7 -12
  88. mlrun/feature_store/feature_set.py +76 -55
  89. mlrun/feature_store/feature_vector.py +39 -31
  90. mlrun/feature_store/ingestion.py +7 -6
  91. mlrun/feature_store/retrieval/base.py +16 -11
  92. mlrun/feature_store/retrieval/dask_merger.py +2 -0
  93. mlrun/feature_store/retrieval/job.py +13 -4
  94. mlrun/feature_store/retrieval/local_merger.py +2 -0
  95. mlrun/feature_store/retrieval/spark_merger.py +24 -32
  96. mlrun/feature_store/steps.py +45 -34
  97. mlrun/features.py +11 -21
  98. mlrun/frameworks/_common/artifacts_library.py +9 -9
  99. mlrun/frameworks/_common/mlrun_interface.py +5 -5
  100. mlrun/frameworks/_common/model_handler.py +48 -48
  101. mlrun/frameworks/_common/plan.py +5 -6
  102. mlrun/frameworks/_common/producer.py +3 -4
  103. mlrun/frameworks/_common/utils.py +5 -5
  104. mlrun/frameworks/_dl_common/loggers/logger.py +6 -7
  105. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +9 -9
  106. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +23 -47
  107. mlrun/frameworks/_ml_common/artifacts_library.py +1 -2
  108. mlrun/frameworks/_ml_common/loggers/logger.py +3 -4
  109. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +4 -5
  110. mlrun/frameworks/_ml_common/model_handler.py +24 -24
  111. mlrun/frameworks/_ml_common/pkl_model_server.py +2 -2
  112. mlrun/frameworks/_ml_common/plan.py +2 -2
  113. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +2 -3
  114. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +2 -3
  115. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  116. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +3 -3
  117. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  118. mlrun/frameworks/_ml_common/utils.py +4 -4
  119. mlrun/frameworks/auto_mlrun/auto_mlrun.py +9 -9
  120. mlrun/frameworks/huggingface/model_server.py +4 -4
  121. mlrun/frameworks/lgbm/__init__.py +33 -33
  122. mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
  123. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -5
  124. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -5
  125. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -3
  126. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +6 -6
  127. mlrun/frameworks/lgbm/model_handler.py +10 -10
  128. mlrun/frameworks/lgbm/model_server.py +6 -6
  129. mlrun/frameworks/lgbm/utils.py +5 -5
  130. mlrun/frameworks/onnx/dataset.py +8 -8
  131. mlrun/frameworks/onnx/mlrun_interface.py +3 -3
  132. mlrun/frameworks/onnx/model_handler.py +6 -6
  133. mlrun/frameworks/onnx/model_server.py +7 -7
  134. mlrun/frameworks/parallel_coordinates.py +6 -6
  135. mlrun/frameworks/pytorch/__init__.py +18 -18
  136. mlrun/frameworks/pytorch/callbacks/callback.py +4 -5
  137. mlrun/frameworks/pytorch/callbacks/logging_callback.py +17 -17
  138. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +11 -11
  139. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +23 -29
  140. mlrun/frameworks/pytorch/callbacks_handler.py +38 -38
  141. mlrun/frameworks/pytorch/mlrun_interface.py +20 -20
  142. mlrun/frameworks/pytorch/model_handler.py +17 -17
  143. mlrun/frameworks/pytorch/model_server.py +7 -7
  144. mlrun/frameworks/sklearn/__init__.py +13 -13
  145. mlrun/frameworks/sklearn/estimator.py +4 -4
  146. mlrun/frameworks/sklearn/metrics_library.py +14 -14
  147. mlrun/frameworks/sklearn/mlrun_interface.py +16 -9
  148. mlrun/frameworks/sklearn/model_handler.py +2 -2
  149. mlrun/frameworks/tf_keras/__init__.py +10 -7
  150. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +15 -15
  151. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +11 -11
  152. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +19 -23
  153. mlrun/frameworks/tf_keras/mlrun_interface.py +9 -11
  154. mlrun/frameworks/tf_keras/model_handler.py +14 -14
  155. mlrun/frameworks/tf_keras/model_server.py +6 -6
  156. mlrun/frameworks/xgboost/__init__.py +13 -13
  157. mlrun/frameworks/xgboost/model_handler.py +6 -6
  158. mlrun/k8s_utils.py +61 -17
  159. mlrun/launcher/__init__.py +1 -1
  160. mlrun/launcher/base.py +16 -15
  161. mlrun/launcher/client.py +13 -11
  162. mlrun/launcher/factory.py +1 -1
  163. mlrun/launcher/local.py +23 -13
  164. mlrun/launcher/remote.py +17 -10
  165. mlrun/lists.py +7 -6
  166. mlrun/model.py +478 -103
  167. mlrun/model_monitoring/__init__.py +1 -1
  168. mlrun/model_monitoring/api.py +163 -371
  169. mlrun/{runtimes/mpijob/v1alpha1.py → model_monitoring/applications/__init__.py} +9 -15
  170. mlrun/model_monitoring/applications/_application_steps.py +188 -0
  171. mlrun/model_monitoring/applications/base.py +108 -0
  172. mlrun/model_monitoring/applications/context.py +341 -0
  173. mlrun/model_monitoring/{evidently_application.py → applications/evidently_base.py} +27 -22
  174. mlrun/model_monitoring/applications/histogram_data_drift.py +354 -0
  175. mlrun/model_monitoring/applications/results.py +99 -0
  176. mlrun/model_monitoring/controller.py +131 -278
  177. mlrun/model_monitoring/db/__init__.py +18 -0
  178. mlrun/model_monitoring/db/stores/__init__.py +136 -0
  179. mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
  180. mlrun/model_monitoring/db/stores/base/store.py +213 -0
  181. mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
  182. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
  183. mlrun/model_monitoring/db/stores/sqldb/models/base.py +190 -0
  184. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +103 -0
  185. mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
  186. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +659 -0
  187. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
  188. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +726 -0
  189. mlrun/model_monitoring/db/tsdb/__init__.py +105 -0
  190. mlrun/model_monitoring/db/tsdb/base.py +448 -0
  191. mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
  192. mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
  193. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +279 -0
  194. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +42 -0
  195. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +507 -0
  196. mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
  197. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +158 -0
  198. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +849 -0
  199. mlrun/model_monitoring/features_drift_table.py +134 -106
  200. mlrun/model_monitoring/helpers.py +199 -55
  201. mlrun/model_monitoring/metrics/__init__.py +13 -0
  202. mlrun/model_monitoring/metrics/histogram_distance.py +127 -0
  203. mlrun/model_monitoring/model_endpoint.py +3 -2
  204. mlrun/model_monitoring/stream_processing.py +131 -398
  205. mlrun/model_monitoring/tracking_policy.py +9 -2
  206. mlrun/model_monitoring/writer.py +161 -125
  207. mlrun/package/__init__.py +6 -6
  208. mlrun/package/context_handler.py +5 -5
  209. mlrun/package/packager.py +7 -7
  210. mlrun/package/packagers/default_packager.py +8 -8
  211. mlrun/package/packagers/numpy_packagers.py +15 -15
  212. mlrun/package/packagers/pandas_packagers.py +5 -5
  213. mlrun/package/packagers/python_standard_library_packagers.py +10 -10
  214. mlrun/package/packagers_manager.py +19 -23
  215. mlrun/package/utils/_formatter.py +6 -6
  216. mlrun/package/utils/_pickler.py +2 -2
  217. mlrun/package/utils/_supported_format.py +4 -4
  218. mlrun/package/utils/log_hint_utils.py +2 -2
  219. mlrun/package/utils/type_hint_utils.py +4 -9
  220. mlrun/platforms/__init__.py +11 -10
  221. mlrun/platforms/iguazio.py +24 -203
  222. mlrun/projects/operations.py +52 -25
  223. mlrun/projects/pipelines.py +191 -197
  224. mlrun/projects/project.py +1227 -400
  225. mlrun/render.py +16 -19
  226. mlrun/run.py +209 -184
  227. mlrun/runtimes/__init__.py +83 -15
  228. mlrun/runtimes/base.py +51 -35
  229. mlrun/runtimes/daskjob.py +17 -10
  230. mlrun/runtimes/databricks_job/databricks_cancel_task.py +1 -1
  231. mlrun/runtimes/databricks_job/databricks_runtime.py +8 -7
  232. mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
  233. mlrun/runtimes/funcdoc.py +1 -29
  234. mlrun/runtimes/function_reference.py +1 -1
  235. mlrun/runtimes/kubejob.py +34 -128
  236. mlrun/runtimes/local.py +40 -11
  237. mlrun/runtimes/mpijob/__init__.py +0 -20
  238. mlrun/runtimes/mpijob/abstract.py +9 -10
  239. mlrun/runtimes/mpijob/v1.py +1 -1
  240. mlrun/{model_monitoring/stores/models/sqlite.py → runtimes/nuclio/__init__.py} +7 -9
  241. mlrun/runtimes/nuclio/api_gateway.py +769 -0
  242. mlrun/runtimes/nuclio/application/__init__.py +15 -0
  243. mlrun/runtimes/nuclio/application/application.py +758 -0
  244. mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
  245. mlrun/runtimes/{function.py → nuclio/function.py} +200 -83
  246. mlrun/runtimes/{nuclio.py → nuclio/nuclio.py} +6 -6
  247. mlrun/runtimes/{serving.py → nuclio/serving.py} +65 -68
  248. mlrun/runtimes/pod.py +281 -101
  249. mlrun/runtimes/remotesparkjob.py +12 -9
  250. mlrun/runtimes/sparkjob/spark3job.py +67 -51
  251. mlrun/runtimes/utils.py +41 -75
  252. mlrun/secrets.py +9 -5
  253. mlrun/serving/__init__.py +8 -1
  254. mlrun/serving/remote.py +2 -7
  255. mlrun/serving/routers.py +85 -69
  256. mlrun/serving/server.py +69 -44
  257. mlrun/serving/states.py +209 -36
  258. mlrun/serving/utils.py +22 -14
  259. mlrun/serving/v1_serving.py +6 -7
  260. mlrun/serving/v2_serving.py +129 -54
  261. mlrun/track/tracker.py +2 -1
  262. mlrun/track/tracker_manager.py +3 -3
  263. mlrun/track/trackers/mlflow_tracker.py +6 -2
  264. mlrun/utils/async_http.py +6 -8
  265. mlrun/utils/azure_vault.py +1 -1
  266. mlrun/utils/clones.py +1 -2
  267. mlrun/utils/condition_evaluator.py +3 -3
  268. mlrun/utils/db.py +21 -3
  269. mlrun/utils/helpers.py +405 -225
  270. mlrun/utils/http.py +3 -6
  271. mlrun/utils/logger.py +112 -16
  272. mlrun/utils/notifications/notification/__init__.py +17 -13
  273. mlrun/utils/notifications/notification/base.py +50 -2
  274. mlrun/utils/notifications/notification/console.py +2 -0
  275. mlrun/utils/notifications/notification/git.py +24 -1
  276. mlrun/utils/notifications/notification/ipython.py +3 -1
  277. mlrun/utils/notifications/notification/slack.py +96 -21
  278. mlrun/utils/notifications/notification/webhook.py +59 -2
  279. mlrun/utils/notifications/notification_pusher.py +149 -30
  280. mlrun/utils/regex.py +9 -0
  281. mlrun/utils/retryer.py +208 -0
  282. mlrun/utils/singleton.py +1 -1
  283. mlrun/utils/v3io_clients.py +4 -6
  284. mlrun/utils/version/version.json +2 -2
  285. mlrun/utils/version/version.py +2 -6
  286. mlrun-1.7.0.dist-info/METADATA +378 -0
  287. mlrun-1.7.0.dist-info/RECORD +351 -0
  288. {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/WHEEL +1 -1
  289. mlrun/feature_store/retrieval/conversion.py +0 -273
  290. mlrun/kfpops.py +0 -868
  291. mlrun/model_monitoring/application.py +0 -310
  292. mlrun/model_monitoring/batch.py +0 -1095
  293. mlrun/model_monitoring/prometheus.py +0 -219
  294. mlrun/model_monitoring/stores/__init__.py +0 -111
  295. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +0 -576
  296. mlrun/model_monitoring/stores/model_endpoint_store.py +0 -147
  297. mlrun/model_monitoring/stores/models/__init__.py +0 -27
  298. mlrun/model_monitoring/stores/models/base.py +0 -84
  299. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -384
  300. mlrun/platforms/other.py +0 -306
  301. mlrun-1.6.4rc7.dist-info/METADATA +0 -272
  302. mlrun-1.6.4rc7.dist-info/RECORD +0 -314
  303. {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/LICENSE +0 -0
  304. {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/entry_points.txt +0 -0
  305. {mlrun-1.6.4rc7.dist-info → mlrun-1.7.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,202 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ from datetime import datetime
16
+ from typing import Annotated, Optional, Union
17
+
18
+ import pydantic
19
+
20
+ from mlrun.common.schemas.notification import Notification
21
+ from mlrun.common.types import StrEnum
22
+
23
+
24
+ class EventEntityKind(StrEnum):
25
+ MODEL_ENDPOINT_RESULT = "model-endpoint-result"
26
+ MODEL_MONITORING_APPLICATION = "model-monitoring-application"
27
+ JOB = "job"
28
+
29
+
30
+ class EventEntities(pydantic.BaseModel):
31
+ kind: EventEntityKind
32
+ project: str
33
+ ids: pydantic.conlist(str, min_items=1, max_items=1)
34
+
35
+
36
+ class EventKind(StrEnum):
37
+ DATA_DRIFT_DETECTED = "data-drift-detected"
38
+ DATA_DRIFT_SUSPECTED = "data-drift-suspected"
39
+ CONCEPT_DRIFT_DETECTED = "concept-drift-detected"
40
+ CONCEPT_DRIFT_SUSPECTED = "concept-drift-suspected"
41
+ MODEL_PERFORMANCE_DETECTED = "model-performance-detected"
42
+ MODEL_PERFORMANCE_SUSPECTED = "model-performance-suspected"
43
+ SYSTEM_PERFORMANCE_DETECTED = "system-performance-detected"
44
+ SYSTEM_PERFORMANCE_SUSPECTED = "system-performance-suspected"
45
+ MM_APP_ANOMALY_DETECTED = "mm-app-anomaly-detected"
46
+ MM_APP_ANOMALY_SUSPECTED = "mm-app-anomaly-suspected"
47
+ MM_APP_FAILED = "mm-app-failed"
48
+ FAILED = "failed"
49
+
50
+
51
+ _event_kind_entity_map = {
52
+ EventKind.DATA_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
53
+ EventKind.DATA_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
54
+ EventKind.CONCEPT_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
55
+ EventKind.CONCEPT_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
56
+ EventKind.MODEL_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
57
+ EventKind.MODEL_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
58
+ EventKind.SYSTEM_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
59
+ EventKind.SYSTEM_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
60
+ EventKind.MM_APP_ANOMALY_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
61
+ EventKind.MM_APP_ANOMALY_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
62
+ EventKind.MM_APP_FAILED: [EventEntityKind.MODEL_MONITORING_APPLICATION],
63
+ EventKind.FAILED: [EventEntityKind.JOB],
64
+ }
65
+
66
+
67
+ class Event(pydantic.BaseModel):
68
+ kind: EventKind
69
+ timestamp: Union[str, datetime] = None # occurrence time
70
+ entity: EventEntities
71
+ value_dict: Optional[dict] = pydantic.Field(default_factory=dict)
72
+
73
+ def is_valid(self):
74
+ return self.entity.kind in _event_kind_entity_map[self.kind]
75
+
76
+
77
+ class AlertActiveState(StrEnum):
78
+ ACTIVE = "active"
79
+ INACTIVE = "inactive"
80
+
81
+
82
+ class AlertSeverity(StrEnum):
83
+ LOW = "low"
84
+ MEDIUM = "medium"
85
+ HIGH = "high"
86
+
87
+
88
+ # what should trigger the alert. must be either event (at least 1), or prometheus query
89
+ class AlertTrigger(pydantic.BaseModel):
90
+ events: list[EventKind] = []
91
+ prometheus_alert: str = None
92
+
93
+ def __eq__(self, other):
94
+ return (
95
+ self.prometheus_alert == other.prometheus_alert
96
+ and self.events == other.events
97
+ )
98
+
99
+
100
+ class AlertCriteria(pydantic.BaseModel):
101
+ count: Annotated[
102
+ int,
103
+ pydantic.Field(
104
+ description="Number of events to wait until notification is sent"
105
+ ),
106
+ ] = 1
107
+ period: Annotated[
108
+ str,
109
+ pydantic.Field(
110
+ description="Time period during which event occurred. e.g. 1d, 3h, 5m, 15s"
111
+ ),
112
+ ] = None
113
+
114
+ def __eq__(self, other):
115
+ return self.count == other.count and self.period == other.period
116
+
117
+
118
+ class ResetPolicy(StrEnum):
119
+ MANUAL = "manual"
120
+ AUTO = "auto"
121
+
122
+
123
+ class AlertNotification(pydantic.BaseModel):
124
+ notification: Notification
125
+ cooldown_period: Annotated[
126
+ str,
127
+ pydantic.Field(
128
+ description="Period during which notifications "
129
+ "will not be sent after initial send. The format of this would be in time."
130
+ " e.g. 1d, 3h, 5m, 15s"
131
+ ),
132
+ ] = None
133
+
134
+
135
+ class AlertConfig(pydantic.BaseModel):
136
+ project: str
137
+ id: int = None
138
+ name: str
139
+ description: Optional[str] = ""
140
+ summary: Annotated[
141
+ str,
142
+ pydantic.Field(
143
+ description=(
144
+ "String to be sent in the notifications generated."
145
+ "e.g. 'Model {{project}}/{{entity}} is drifting.'"
146
+ "Supported variables: project, entity, name"
147
+ )
148
+ ),
149
+ ]
150
+ created: Union[str, datetime] = None
151
+ severity: AlertSeverity
152
+ entities: EventEntities
153
+ trigger: AlertTrigger
154
+ criteria: Optional[AlertCriteria]
155
+ reset_policy: ResetPolicy = ResetPolicy.AUTO
156
+ notifications: pydantic.conlist(AlertNotification, min_items=1)
157
+ state: AlertActiveState = AlertActiveState.INACTIVE
158
+ count: Optional[int] = 0
159
+
160
+ def get_raw_notifications(self) -> list[Notification]:
161
+ return [
162
+ alert_notification.notification for alert_notification in self.notifications
163
+ ]
164
+
165
+
166
+ class AlertsModes(StrEnum):
167
+ enabled = "enabled"
168
+ disabled = "disabled"
169
+
170
+
171
+ class AlertTemplate(
172
+ pydantic.BaseModel
173
+ ): # Template fields that are not shared with created configs
174
+ template_id: int = None
175
+ template_name: str
176
+ template_description: Optional[str] = (
177
+ "String explaining the purpose of this template"
178
+ )
179
+
180
+ # A property that identifies templates that were created by the system and cannot be modified/deleted by the user
181
+ system_generated: bool = False
182
+
183
+ # AlertConfig fields that are pre-defined
184
+ summary: Optional[str] = (
185
+ "String to be sent in the generated notifications e.g. 'Model {{project}}/{{entity}} is drifting.'"
186
+ "See AlertConfig.summary description"
187
+ )
188
+ severity: AlertSeverity
189
+ trigger: AlertTrigger
190
+ criteria: Optional[AlertCriteria]
191
+ reset_policy: ResetPolicy = ResetPolicy.AUTO
192
+
193
+ # This is slightly different than __eq__ as it doesn't compare everything
194
+ def templates_differ(self, other):
195
+ return (
196
+ self.template_description != other.template_description
197
+ or self.summary != other.summary
198
+ or self.severity != other.severity
199
+ or self.trigger != other.trigger
200
+ or self.reset_policy != other.reset_policy
201
+ or self.criteria != other.criteria
202
+ )
@@ -0,0 +1,196 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ import typing
16
+ from typing import Optional
17
+
18
+ import pydantic
19
+
20
+ import mlrun.common.constants as mlrun_constants
21
+ import mlrun.common.types
22
+ from mlrun.common.constants import MLRUN_FUNCTIONS_ANNOTATION
23
+ from mlrun.common.helpers import generate_api_gateway_name
24
+
25
+
26
+ class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
27
+ basic = "basicAuth"
28
+ none = "none"
29
+ access_key = "accessKey"
30
+
31
+ @classmethod
32
+ def from_str(cls, authentication_mode: str):
33
+ if authentication_mode == "none":
34
+ return cls.none
35
+ elif authentication_mode == "basicAuth":
36
+ return cls.basic
37
+ elif authentication_mode == "accessKey":
38
+ return cls.access_key
39
+ else:
40
+ raise mlrun.errors.MLRunInvalidArgumentError(
41
+ f"Authentication mode `{authentication_mode}` is not supported",
42
+ )
43
+
44
+
45
+ class APIGatewayState(mlrun.common.types.StrEnum):
46
+ none = ""
47
+ ready = "ready"
48
+ error = "error"
49
+ waiting_for_provisioning = "waitingForProvisioning"
50
+
51
+
52
+ class _APIGatewayBaseModel(pydantic.BaseModel):
53
+ class Config:
54
+ extra = pydantic.Extra.allow
55
+
56
+
57
+ class APIGatewayMetadata(_APIGatewayBaseModel):
58
+ name: str
59
+ namespace: Optional[str]
60
+ labels: Optional[dict] = {}
61
+ annotations: Optional[dict] = {}
62
+
63
+
64
+ class APIGatewayBasicAuth(_APIGatewayBaseModel):
65
+ username: str
66
+ password: str
67
+
68
+
69
+ class APIGatewayUpstream(_APIGatewayBaseModel):
70
+ kind: Optional[str] = "nucliofunction"
71
+ nucliofunction: dict[str, str]
72
+ percentage: Optional[int] = 0
73
+ port: Optional[int] = 0
74
+
75
+
76
+ class APIGatewaySpec(_APIGatewayBaseModel):
77
+ name: str
78
+ description: Optional[str]
79
+ path: Optional[str] = "/"
80
+ authenticationMode: Optional[APIGatewayAuthenticationMode] = ( # noqa: N815 - for compatibility with Nuclio https://github.com/nuclio/nuclio/blob/672b8e36f9edd6e42b4685ec1d27cabae3c5f045/pkg/platform/types.go#L476
81
+ APIGatewayAuthenticationMode.none
82
+ )
83
+ upstreams: list[APIGatewayUpstream]
84
+ authentication: Optional[dict[str, Optional[APIGatewayBasicAuth]]]
85
+ host: Optional[str]
86
+
87
+
88
+ class APIGatewayStatus(_APIGatewayBaseModel):
89
+ name: Optional[str]
90
+ state: Optional[APIGatewayState]
91
+
92
+
93
+ class APIGateway(_APIGatewayBaseModel):
94
+ metadata: APIGatewayMetadata
95
+ spec: APIGatewaySpec
96
+ status: Optional[APIGatewayStatus]
97
+
98
+ def get_function_names(self):
99
+ return [
100
+ upstream.nucliofunction.get("name")
101
+ for upstream in self.spec.upstreams
102
+ if upstream.nucliofunction.get("name")
103
+ ]
104
+
105
+ def get_invoke_url(self):
106
+ if self.spec.host and self.spec.path:
107
+ return f"{self.spec.host.rstrip('/')}/{self.spec.path.lstrip('/')}".rstrip(
108
+ "/"
109
+ )
110
+ return self.spec.host.rstrip("/")
111
+
112
+ def enrich_mlrun_names(self):
113
+ self._enrich_api_gateway_mlrun_name()
114
+ self._enrich_mlrun_function_names()
115
+ return self
116
+
117
+ def replace_nuclio_names_with_mlrun_names(self):
118
+ self._replace_nuclio_api_gateway_name_with_mlrun_name()
119
+ self._replace_nuclio_function_names_with_mlrun_names()
120
+ return self
121
+
122
+ def _replace_nuclio_function_names_with_mlrun_names(self):
123
+ # replace function names from nuclio names to mlrun names
124
+ # and adds mlrun function URI's to an api gateway annotations
125
+ # so when we then get api gateway entity from nuclio, we are able to get mlrun function names
126
+ mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
127
+ if mlrun_functions:
128
+ mlrun_function_uris = (
129
+ mlrun_functions.split("&")
130
+ if "&" in mlrun_functions
131
+ else [mlrun_functions]
132
+ )
133
+ if len(mlrun_function_uris) != len(self.spec.upstreams):
134
+ raise mlrun.errors.MLRunValueError(
135
+ "Error when translating nuclio names to mlrun names in api gateway:"
136
+ " number of functions doesn't match the mlrun functions in annotation"
137
+ )
138
+ for i in range(len(mlrun_function_uris)):
139
+ self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
140
+ return self
141
+
142
+ def _replace_nuclio_api_gateway_name_with_mlrun_name(self):
143
+ # replace api gateway name
144
+ # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
145
+ # remove the project prefix from the name if it exists
146
+ project_name = self.metadata.labels.get(
147
+ mlrun_constants.MLRunInternalLabels.nuclio_project_name
148
+ )
149
+ if project_name and self.spec.name.startswith(f"{project_name}-"):
150
+ self.spec.name = self.spec.name[len(project_name) + 1 :]
151
+ self.metadata.name = self.spec.name
152
+ return self
153
+
154
+ def _enrich_mlrun_function_names(self):
155
+ # enrich mlrun names with nuclio prefixes
156
+ # and add mlrun function's URIs to Nuclio function annotations
157
+ upstream_with_nuclio_names = []
158
+ mlrun_function_uris = []
159
+ for upstream in self.spec.upstreams:
160
+ uri = upstream.nucliofunction.get("name")
161
+ project, function_name, tag, _ = (
162
+ mlrun.common.helpers.parse_versioned_object_uri(uri)
163
+ )
164
+ upstream.nucliofunction["name"] = (
165
+ mlrun.runtimes.nuclio.function.get_fullname(function_name, project, tag)
166
+ )
167
+
168
+ upstream_with_nuclio_names.append(upstream)
169
+ mlrun_function_uris.append(uri)
170
+
171
+ self.spec.upstreams = upstream_with_nuclio_names
172
+ if len(mlrun_function_uris) == 1:
173
+ self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = mlrun_function_uris[
174
+ 0
175
+ ]
176
+ elif len(mlrun_function_uris) == 2:
177
+ self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = "&".join(
178
+ mlrun_function_uris
179
+ )
180
+ return self
181
+
182
+ def _enrich_api_gateway_mlrun_name(self):
183
+ # replace api gateway name
184
+ # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
185
+ # add the project prefix to the name
186
+ project_name = self.metadata.labels.get(
187
+ mlrun_constants.MLRunInternalLabels.nuclio_project_name
188
+ )
189
+ if project_name:
190
+ self.spec.name = generate_api_gateway_name(project_name, self.spec.name)
191
+ self.metadata.name = self.spec.name
192
+ return self
193
+
194
+
195
+ class APIGatewaysOutput(_APIGatewayBaseModel):
196
+ api_gateways: typing.Optional[dict[str, APIGateway]] = {}
@@ -15,6 +15,7 @@
15
15
  import typing
16
16
 
17
17
  import pydantic
18
+ from deprecated import deprecated
18
19
 
19
20
  import mlrun.common.types
20
21
 
@@ -30,7 +31,7 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
30
31
  # and should not be used as such
31
32
  link = "link"
32
33
 
33
- def to_kinds_filter(self) -> typing.Tuple[typing.List[str], bool]:
34
+ def to_kinds_filter(self) -> tuple[list[str], bool]:
34
35
  link_kind = ArtifactCategories.link.value
35
36
 
36
37
  if self.value == ArtifactCategories.model.value:
@@ -58,8 +59,13 @@ class ArtifactIdentifier(pydantic.BaseModel):
58
59
  # hash: typing.Optional[str]
59
60
 
60
61
 
62
+ @deprecated(
63
+ version="1.7.0",
64
+ reason="mlrun.common.schemas.ArtifactsFormat is deprecated and will be removed in 1.9.0. "
65
+ "Use mlrun.common.formatters.ArtifactFormat instead.",
66
+ category=FutureWarning,
67
+ )
61
68
  class ArtifactsFormat(mlrun.common.types.StrEnum):
62
- # TODO: add a format that returns a minimal response
63
69
  full = "full"
64
70
 
65
71
 
@@ -81,8 +87,8 @@ class ArtifactSpec(pydantic.BaseModel):
81
87
  inline: typing.Optional[str]
82
88
  size: typing.Optional[int]
83
89
  db_key: typing.Optional[str]
84
- extra_data: typing.Optional[typing.Dict[str, typing.Any]]
85
- unpackaging_instructions: typing.Optional[typing.Dict[str, typing.Any]]
90
+ extra_data: typing.Optional[dict[str, typing.Any]]
91
+ unpackaging_instructions: typing.Optional[dict[str, typing.Any]]
86
92
 
87
93
  class Config:
88
94
  extra = pydantic.Extra.allow
@@ -93,3 +99,18 @@ class Artifact(pydantic.BaseModel):
93
99
  metadata: ArtifactMetadata
94
100
  spec: ArtifactSpec
95
101
  status: ObjectStatus
102
+
103
+
104
+ class ArtifactsDeletionStrategies(mlrun.common.types.StrEnum):
105
+ """Artifacts deletion strategies types."""
106
+
107
+ metadata_only = "metadata-only"
108
+ """Only removes the artifact db record, leaving all related artifact data in-place"""
109
+
110
+ data_optional = "data-optional"
111
+ """Delete the artifact data of the artifact as a best-effort.
112
+ If artifact data deletion fails still try to delete the artifact db record"""
113
+
114
+ data_force = "data-force"
115
+ """Delete the artifact data, and if cannot delete it fail the deletion
116
+ and don’t delete the artifact db record"""
@@ -58,8 +58,12 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
58
58
  pipeline = "pipeline"
59
59
  hub_source = "hub-source"
60
60
  workflow = "workflow"
61
+ alert = "alert"
62
+ alert_templates = "alert-templates"
63
+ event = "event"
61
64
  datastore_profile = "datastore-profile"
62
- api_gateways = "api-gateways"
65
+ api_gateway = "api-gateway"
66
+ project_summaries = "project-summaries"
63
67
 
64
68
  def to_resource_string(
65
69
  self,
@@ -69,6 +73,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
69
73
  return {
70
74
  # project is the resource itself, so no need for both resource_name and project_name
71
75
  AuthorizationResourceTypes.project: "/projects/{project_name}",
76
+ AuthorizationResourceTypes.project_summaries: "/projects/{project_name}/project-summaries/{resource_name}",
72
77
  AuthorizationResourceTypes.function: "/projects/{project_name}/functions/{resource_name}",
73
78
  AuthorizationResourceTypes.artifact: "/projects/{project_name}/artifacts/{resource_name}",
74
79
  AuthorizationResourceTypes.project_background_task: (
@@ -83,6 +88,9 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
83
88
  AuthorizationResourceTypes.schedule: "/projects/{project_name}/schedules/{resource_name}",
84
89
  AuthorizationResourceTypes.secret: "/projects/{project_name}/secrets/{resource_name}",
85
90
  AuthorizationResourceTypes.run: "/projects/{project_name}/runs/{resource_name}",
91
+ AuthorizationResourceTypes.event: "/projects/{project_name}/events/{resource_name}",
92
+ AuthorizationResourceTypes.alert: "/projects/{project_name}/alerts/{resource_name}",
93
+ AuthorizationResourceTypes.alert_templates: "/alert-templates/{resource_name}",
86
94
  # runtime resource doesn't have an identifier, we don't need any auth granularity behind project level
87
95
  AuthorizationResourceTypes.runtime_resource: "/projects/{project_name}/runtime-resources",
88
96
  AuthorizationResourceTypes.model_endpoint: "/projects/{project_name}/model-endpoints/{resource_name}",
@@ -94,7 +102,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
94
102
  AuthorizationResourceTypes.hub_source: "/marketplace/sources",
95
103
  # workflow define how to run a pipeline and can be considered as the specification of a pipeline.
96
104
  AuthorizationResourceTypes.workflow: "/projects/{project_name}/workflows/{resource_name}",
97
- AuthorizationResourceTypes.api_gateways: "/projects/{project_name}/api-gateways",
105
+ AuthorizationResourceTypes.api_gateway: "/projects/{project_name}/api-gateways/{resource_name}",
98
106
  }[self].format(project_name=project_name, resource_name=resource_name)
99
107
 
100
108
 
@@ -115,17 +123,17 @@ class AuthInfo(pydantic.BaseModel):
115
123
  data_session: typing.Optional[str] = None
116
124
  access_key: typing.Optional[str] = None
117
125
  user_id: typing.Optional[str] = None
118
- user_group_ids: typing.List[str] = []
126
+ user_group_ids: list[str] = []
119
127
  user_unix_id: typing.Optional[int] = None
120
128
  projects_role: typing.Optional[ProjectsRole] = None
121
- planes: typing.List[str] = []
129
+ planes: list[str] = []
122
130
 
123
131
  def to_nuclio_auth_info(self):
124
132
  if self.session != "":
125
133
  return NuclioAuthInfo(password=self.session, mode=NuclioAuthKinds.iguazio)
126
134
  return None
127
135
 
128
- def get_member_ids(self) -> typing.List[str]:
136
+ def get_member_ids(self) -> list[str]:
129
137
  member_ids = []
130
138
  if self.user_id:
131
139
  member_ids.append(self.user_id)
@@ -133,6 +141,9 @@ class AuthInfo(pydantic.BaseModel):
133
141
  member_ids.extend(self.user_group_ids)
134
142
  return member_ids
135
143
 
144
+ def get_session(self) -> str:
145
+ return self.data_session or self.session
146
+
136
147
 
137
148
  class Credentials(pydantic.BaseModel):
138
149
  access_key: typing.Optional[str]
@@ -61,4 +61,4 @@ class BackgroundTask(pydantic.BaseModel):
61
61
 
62
62
 
63
63
  class BackgroundTaskList(pydantic.BaseModel):
64
- background_tasks: typing.List[BackgroundTask]
64
+ background_tasks: list[BackgroundTask]
@@ -28,7 +28,8 @@ class ClientSpec(pydantic.BaseModel):
28
28
  mpijob_crd_version: typing.Optional[str]
29
29
  ui_url: typing.Optional[str]
30
30
  artifact_path: typing.Optional[str]
31
- feature_store_data_prefixes: typing.Optional[typing.Dict[str, str]]
31
+ feature_store_data_prefixes: typing.Optional[dict[str, str]]
32
+ feature_store_default_targets: typing.Optional[str]
32
33
  spark_app_image: typing.Optional[str]
33
34
  spark_app_image_tag: typing.Optional[str]
34
35
  spark_history_server_path: typing.Optional[str]
@@ -56,8 +57,8 @@ class ClientSpec(pydantic.BaseModel):
56
57
  redis_url: typing.Optional[str]
57
58
  redis_type: typing.Optional[str]
58
59
  sql_url: typing.Optional[str]
59
- model_endpoint_monitoring_store_type: typing.Optional[str]
60
60
  model_endpoint_monitoring_endpoint_store_connection: typing.Optional[str]
61
+ model_monitoring_tsdb_connection: typing.Optional[str]
61
62
  ce: typing.Optional[dict]
62
63
  # not passing them as one object as it possible client user would like to override only one of the params
63
64
  calculate_artifact_hash: typing.Optional[str]
@@ -65,3 +66,4 @@ class ClientSpec(pydantic.BaseModel):
65
66
  logs: typing.Optional[dict]
66
67
  packagers: typing.Optional[dict]
67
68
  external_platform_tracking: typing.Optional[dict]
69
+ alerts_mode: typing.Optional[str]
@@ -11,16 +11,16 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import typing
16
16
 
17
17
  import pydantic
18
18
 
19
19
 
20
20
  class ImageBuilder(pydantic.BaseModel):
21
- functionSourceCode: typing.Optional[str] = None
22
- codeEntryType: typing.Optional[str] = None
23
- codeEntryAttributes: typing.Optional[str] = None
21
+ functionSourceCode: typing.Optional[str] = None # noqa: N815
22
+ codeEntryType: typing.Optional[str] = None # noqa: N815
23
+ codeEntryAttributes: typing.Optional[str] = None # noqa: N815
24
24
  source: typing.Optional[str] = None
25
25
  code_origin: typing.Optional[str] = None
26
26
  origin_filename: typing.Optional[str] = None
@@ -38,3 +38,6 @@ class ImageBuilder(pydantic.BaseModel):
38
38
  build_pod: typing.Optional[str] = None
39
39
  requirements: typing.Optional[list] = None
40
40
  source_code_target_dir: typing.Optional[str] = None
41
+
42
+ class Config:
43
+ extra = pydantic.Extra.allow
@@ -120,10 +120,13 @@ class FeatureStorePartitionByField(mlrun.common.types.StrEnum):
120
120
 
121
121
  class RunPartitionByField(mlrun.common.types.StrEnum):
122
122
  name = "name" # Supported for runs objects
123
+ project_and_name = "project_and_name" # Supported for runs objects
123
124
 
124
125
  def to_partition_by_db_field(self, db_cls):
125
126
  if self.value == RunPartitionByField.name:
126
127
  return db_cls.name
128
+ elif self.value == RunPartitionByField.project_and_name:
129
+ return db_cls.project, db_cls.name
127
130
  else:
128
131
  raise mlrun.errors.MLRunInvalidArgumentError(
129
132
  f"Unknown group by field: {self.value}"