mlrun 1.7.1rc10__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (257) hide show
  1. mlrun/__init__.py +23 -21
  2. mlrun/__main__.py +3 -3
  3. mlrun/alerts/alert.py +148 -14
  4. mlrun/artifacts/__init__.py +1 -2
  5. mlrun/artifacts/base.py +46 -12
  6. mlrun/artifacts/dataset.py +16 -16
  7. mlrun/artifacts/document.py +334 -0
  8. mlrun/artifacts/manager.py +15 -13
  9. mlrun/artifacts/model.py +66 -53
  10. mlrun/common/constants.py +7 -0
  11. mlrun/common/formatters/__init__.py +1 -0
  12. mlrun/common/formatters/feature_set.py +1 -0
  13. mlrun/common/formatters/function.py +1 -0
  14. mlrun/{model_monitoring/db/stores/base/__init__.py → common/formatters/model_endpoint.py} +16 -1
  15. mlrun/common/formatters/pipeline.py +1 -2
  16. mlrun/common/formatters/project.py +9 -0
  17. mlrun/common/model_monitoring/__init__.py +0 -5
  18. mlrun/common/model_monitoring/helpers.py +1 -29
  19. mlrun/common/runtimes/constants.py +1 -2
  20. mlrun/common/schemas/__init__.py +6 -2
  21. mlrun/common/schemas/alert.py +111 -19
  22. mlrun/common/schemas/api_gateway.py +3 -3
  23. mlrun/common/schemas/artifact.py +11 -7
  24. mlrun/common/schemas/auth.py +6 -4
  25. mlrun/common/schemas/background_task.py +7 -7
  26. mlrun/common/schemas/client_spec.py +2 -3
  27. mlrun/common/schemas/clusterization_spec.py +2 -2
  28. mlrun/common/schemas/common.py +53 -3
  29. mlrun/common/schemas/constants.py +15 -0
  30. mlrun/common/schemas/datastore_profile.py +1 -1
  31. mlrun/common/schemas/feature_store.py +9 -9
  32. mlrun/common/schemas/frontend_spec.py +4 -4
  33. mlrun/common/schemas/function.py +10 -10
  34. mlrun/common/schemas/hub.py +1 -1
  35. mlrun/common/schemas/k8s.py +3 -3
  36. mlrun/common/schemas/memory_reports.py +3 -3
  37. mlrun/common/schemas/model_monitoring/__init__.py +2 -1
  38. mlrun/common/schemas/model_monitoring/constants.py +66 -14
  39. mlrun/common/schemas/model_monitoring/grafana.py +1 -1
  40. mlrun/common/schemas/model_monitoring/model_endpoints.py +91 -147
  41. mlrun/common/schemas/notification.py +24 -3
  42. mlrun/common/schemas/object.py +1 -1
  43. mlrun/common/schemas/pagination.py +4 -4
  44. mlrun/common/schemas/partition.py +137 -0
  45. mlrun/common/schemas/pipeline.py +2 -2
  46. mlrun/common/schemas/project.py +25 -17
  47. mlrun/common/schemas/runs.py +2 -2
  48. mlrun/common/schemas/runtime_resource.py +5 -5
  49. mlrun/common/schemas/schedule.py +1 -1
  50. mlrun/common/schemas/secret.py +1 -1
  51. mlrun/common/schemas/tag.py +3 -3
  52. mlrun/common/schemas/workflow.py +5 -5
  53. mlrun/config.py +67 -10
  54. mlrun/data_types/__init__.py +0 -2
  55. mlrun/data_types/infer.py +3 -1
  56. mlrun/data_types/spark.py +2 -1
  57. mlrun/datastore/__init__.py +0 -2
  58. mlrun/datastore/alibaba_oss.py +4 -1
  59. mlrun/datastore/azure_blob.py +4 -1
  60. mlrun/datastore/base.py +12 -4
  61. mlrun/datastore/datastore.py +9 -3
  62. mlrun/datastore/datastore_profile.py +79 -20
  63. mlrun/datastore/dbfs_store.py +4 -1
  64. mlrun/datastore/filestore.py +4 -1
  65. mlrun/datastore/google_cloud_storage.py +4 -1
  66. mlrun/datastore/hdfs.py +4 -1
  67. mlrun/datastore/inmem.py +4 -1
  68. mlrun/datastore/redis.py +4 -1
  69. mlrun/datastore/s3.py +4 -1
  70. mlrun/datastore/sources.py +52 -51
  71. mlrun/datastore/store_resources.py +0 -2
  72. mlrun/datastore/targets.py +21 -21
  73. mlrun/datastore/utils.py +2 -2
  74. mlrun/datastore/v3io.py +4 -1
  75. mlrun/datastore/vectorstore.py +194 -0
  76. mlrun/datastore/wasbfs/fs.py +13 -12
  77. mlrun/db/base.py +208 -82
  78. mlrun/db/factory.py +0 -3
  79. mlrun/db/httpdb.py +1237 -386
  80. mlrun/db/nopdb.py +201 -74
  81. mlrun/errors.py +2 -2
  82. mlrun/execution.py +136 -50
  83. mlrun/feature_store/__init__.py +0 -2
  84. mlrun/feature_store/api.py +41 -40
  85. mlrun/feature_store/common.py +9 -9
  86. mlrun/feature_store/feature_set.py +20 -18
  87. mlrun/feature_store/feature_vector.py +27 -24
  88. mlrun/feature_store/retrieval/base.py +14 -9
  89. mlrun/feature_store/retrieval/job.py +2 -1
  90. mlrun/feature_store/steps.py +2 -2
  91. mlrun/features.py +30 -13
  92. mlrun/frameworks/__init__.py +1 -2
  93. mlrun/frameworks/_common/__init__.py +1 -2
  94. mlrun/frameworks/_common/artifacts_library.py +2 -2
  95. mlrun/frameworks/_common/mlrun_interface.py +10 -6
  96. mlrun/frameworks/_common/model_handler.py +29 -27
  97. mlrun/frameworks/_common/producer.py +3 -1
  98. mlrun/frameworks/_dl_common/__init__.py +1 -2
  99. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
  100. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
  101. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
  102. mlrun/frameworks/_ml_common/__init__.py +1 -2
  103. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
  104. mlrun/frameworks/_ml_common/model_handler.py +21 -21
  105. mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
  106. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
  107. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  108. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  109. mlrun/frameworks/auto_mlrun/__init__.py +1 -2
  110. mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
  111. mlrun/frameworks/huggingface/__init__.py +1 -2
  112. mlrun/frameworks/huggingface/model_server.py +9 -9
  113. mlrun/frameworks/lgbm/__init__.py +47 -44
  114. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
  115. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
  116. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
  117. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
  118. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
  119. mlrun/frameworks/lgbm/model_handler.py +15 -11
  120. mlrun/frameworks/lgbm/model_server.py +11 -7
  121. mlrun/frameworks/lgbm/utils.py +2 -2
  122. mlrun/frameworks/onnx/__init__.py +1 -2
  123. mlrun/frameworks/onnx/dataset.py +3 -3
  124. mlrun/frameworks/onnx/mlrun_interface.py +2 -2
  125. mlrun/frameworks/onnx/model_handler.py +7 -5
  126. mlrun/frameworks/onnx/model_server.py +8 -6
  127. mlrun/frameworks/parallel_coordinates.py +11 -11
  128. mlrun/frameworks/pytorch/__init__.py +22 -23
  129. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
  130. mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
  131. mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
  132. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
  133. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
  134. mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
  135. mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
  136. mlrun/frameworks/pytorch/model_handler.py +21 -17
  137. mlrun/frameworks/pytorch/model_server.py +13 -9
  138. mlrun/frameworks/sklearn/__init__.py +19 -18
  139. mlrun/frameworks/sklearn/estimator.py +2 -2
  140. mlrun/frameworks/sklearn/metric.py +3 -3
  141. mlrun/frameworks/sklearn/metrics_library.py +8 -6
  142. mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
  143. mlrun/frameworks/sklearn/model_handler.py +4 -3
  144. mlrun/frameworks/tf_keras/__init__.py +11 -12
  145. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
  146. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
  147. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
  148. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
  149. mlrun/frameworks/tf_keras/model_handler.py +17 -13
  150. mlrun/frameworks/tf_keras/model_server.py +12 -8
  151. mlrun/frameworks/xgboost/__init__.py +19 -18
  152. mlrun/frameworks/xgboost/model_handler.py +13 -9
  153. mlrun/launcher/base.py +3 -4
  154. mlrun/launcher/local.py +1 -1
  155. mlrun/launcher/remote.py +1 -1
  156. mlrun/lists.py +4 -3
  157. mlrun/model.py +117 -46
  158. mlrun/model_monitoring/__init__.py +4 -4
  159. mlrun/model_monitoring/api.py +61 -59
  160. mlrun/model_monitoring/applications/_application_steps.py +17 -17
  161. mlrun/model_monitoring/applications/base.py +165 -6
  162. mlrun/model_monitoring/applications/context.py +88 -37
  163. mlrun/model_monitoring/applications/evidently_base.py +0 -1
  164. mlrun/model_monitoring/applications/histogram_data_drift.py +43 -21
  165. mlrun/model_monitoring/applications/results.py +55 -3
  166. mlrun/model_monitoring/controller.py +207 -239
  167. mlrun/model_monitoring/db/__init__.py +0 -2
  168. mlrun/model_monitoring/db/_schedules.py +156 -0
  169. mlrun/model_monitoring/db/_stats.py +189 -0
  170. mlrun/model_monitoring/db/tsdb/base.py +78 -25
  171. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +61 -6
  172. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
  173. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +255 -29
  174. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
  175. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +78 -17
  176. mlrun/model_monitoring/helpers.py +152 -49
  177. mlrun/model_monitoring/stream_processing.py +99 -283
  178. mlrun/model_monitoring/tracking_policy.py +10 -3
  179. mlrun/model_monitoring/writer.py +48 -36
  180. mlrun/package/__init__.py +3 -6
  181. mlrun/package/context_handler.py +1 -1
  182. mlrun/package/packager.py +12 -9
  183. mlrun/package/packagers/__init__.py +0 -2
  184. mlrun/package/packagers/default_packager.py +14 -11
  185. mlrun/package/packagers/numpy_packagers.py +16 -7
  186. mlrun/package/packagers/pandas_packagers.py +18 -18
  187. mlrun/package/packagers/python_standard_library_packagers.py +25 -11
  188. mlrun/package/packagers_manager.py +31 -14
  189. mlrun/package/utils/__init__.py +0 -3
  190. mlrun/package/utils/_pickler.py +6 -6
  191. mlrun/platforms/__init__.py +47 -16
  192. mlrun/platforms/iguazio.py +4 -1
  193. mlrun/projects/operations.py +27 -27
  194. mlrun/projects/pipelines.py +71 -36
  195. mlrun/projects/project.py +865 -206
  196. mlrun/run.py +53 -10
  197. mlrun/runtimes/__init__.py +1 -3
  198. mlrun/runtimes/base.py +15 -11
  199. mlrun/runtimes/daskjob.py +9 -9
  200. mlrun/runtimes/generators.py +2 -1
  201. mlrun/runtimes/kubejob.py +4 -5
  202. mlrun/runtimes/mounts.py +572 -0
  203. mlrun/runtimes/mpijob/__init__.py +0 -2
  204. mlrun/runtimes/mpijob/abstract.py +7 -6
  205. mlrun/runtimes/nuclio/api_gateway.py +7 -7
  206. mlrun/runtimes/nuclio/application/application.py +11 -11
  207. mlrun/runtimes/nuclio/function.py +19 -17
  208. mlrun/runtimes/nuclio/serving.py +18 -11
  209. mlrun/runtimes/pod.py +154 -45
  210. mlrun/runtimes/remotesparkjob.py +3 -2
  211. mlrun/runtimes/sparkjob/__init__.py +0 -2
  212. mlrun/runtimes/sparkjob/spark3job.py +21 -11
  213. mlrun/runtimes/utils.py +6 -5
  214. mlrun/serving/merger.py +6 -4
  215. mlrun/serving/remote.py +18 -17
  216. mlrun/serving/routers.py +185 -172
  217. mlrun/serving/server.py +7 -1
  218. mlrun/serving/states.py +97 -78
  219. mlrun/serving/utils.py +13 -2
  220. mlrun/serving/v1_serving.py +3 -2
  221. mlrun/serving/v2_serving.py +74 -65
  222. mlrun/track/__init__.py +1 -1
  223. mlrun/track/tracker.py +2 -2
  224. mlrun/track/trackers/mlflow_tracker.py +6 -5
  225. mlrun/utils/async_http.py +1 -1
  226. mlrun/utils/clones.py +1 -1
  227. mlrun/utils/helpers.py +54 -16
  228. mlrun/utils/logger.py +106 -4
  229. mlrun/utils/notifications/notification/__init__.py +22 -19
  230. mlrun/utils/notifications/notification/base.py +33 -14
  231. mlrun/utils/notifications/notification/console.py +6 -6
  232. mlrun/utils/notifications/notification/git.py +11 -11
  233. mlrun/utils/notifications/notification/ipython.py +10 -9
  234. mlrun/utils/notifications/notification/mail.py +176 -0
  235. mlrun/utils/notifications/notification/slack.py +6 -6
  236. mlrun/utils/notifications/notification/webhook.py +6 -6
  237. mlrun/utils/notifications/notification_pusher.py +86 -44
  238. mlrun/utils/regex.py +3 -1
  239. mlrun/utils/version/version.json +2 -2
  240. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/METADATA +21 -16
  241. mlrun-1.8.0rc8.dist-info/RECORD +347 -0
  242. mlrun/model_monitoring/db/stores/__init__.py +0 -136
  243. mlrun/model_monitoring/db/stores/base/store.py +0 -213
  244. mlrun/model_monitoring/db/stores/sqldb/__init__.py +0 -13
  245. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -71
  246. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -190
  247. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -103
  248. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -40
  249. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -659
  250. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +0 -13
  251. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -726
  252. mlrun/model_monitoring/model_endpoint.py +0 -118
  253. mlrun-1.7.1rc10.dist-info/RECORD +0 -351
  254. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/LICENSE +0 -0
  255. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/WHEEL +0 -0
  256. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/entry_points.txt +0 -0
  257. {mlrun-1.7.1rc10.dist-info → mlrun-1.8.0rc8.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,176 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ import re
15
+ import typing
16
+ from email.mime.multipart import MIMEMultipart
17
+ from email.mime.text import MIMEText
18
+
19
+ import aiosmtplib
20
+
21
+ import mlrun.common.schemas
22
+ import mlrun.lists
23
+ import mlrun.utils.helpers
24
+ import mlrun.utils.notifications.notification.base as base
25
+ import mlrun.utils.regex
26
+
27
+ DEFAULT_SMTP_PORT = 587
28
+
29
+
30
+ class MailNotification(base.NotificationBase):
31
+ """
32
+ API/Client notification for sending run statuses as a mail message
33
+ """
34
+
35
+ boolean_params = ["use_tls", "start_tls", "validate_certs"]
36
+
37
+ required_params = [
38
+ "server_host",
39
+ "server_port",
40
+ "sender_address",
41
+ "username",
42
+ "password",
43
+ "email_addresses",
44
+ ] + boolean_params
45
+
46
+ @classmethod
47
+ def validate_params(cls, params):
48
+ for required_param in cls.required_params:
49
+ if required_param not in params:
50
+ raise ValueError(
51
+ f"Parameter '{required_param}' is required for MailNotification"
52
+ )
53
+
54
+ for boolean_param in cls.boolean_params:
55
+ if not isinstance(params.get(boolean_param, None), bool):
56
+ raise ValueError(
57
+ f"Parameter '{boolean_param}' must be a boolean for MailNotification"
58
+ )
59
+
60
+ cls._validate_emails(params)
61
+
62
+ async def push(
63
+ self,
64
+ message: str,
65
+ severity: typing.Optional[
66
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
67
+ ] = mlrun.common.schemas.NotificationSeverity.INFO,
68
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
69
+ custom_html: typing.Optional[typing.Optional[str]] = None,
70
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
71
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
72
+ ):
73
+ self.params["subject"] = f"[{severity}] {message}"
74
+ message_body_override = self.params.get("message_body_override", None)
75
+
76
+ runs_html = self._get_html(
77
+ message, severity, runs, custom_html, alert, event_data
78
+ )
79
+ self.params["body"] = runs_html
80
+
81
+ if message_body_override:
82
+ self.params["body"] = message_body_override.replace(
83
+ "{{ runs }}", runs_html
84
+ ).replace("{{runs}}", runs_html)
85
+
86
+ await self._send_email(**self.params)
87
+
88
+ @classmethod
89
+ def enrich_default_params(
90
+ cls, params: dict, default_params: typing.Optional[dict] = None
91
+ ) -> dict:
92
+ params = super().enrich_default_params(params, default_params)
93
+ params.setdefault("use_tls", True)
94
+ params.setdefault("start_tls", False)
95
+ params.setdefault("validate_certs", True)
96
+ params.setdefault("server_port", DEFAULT_SMTP_PORT)
97
+
98
+ default_mail_address = params.pop("default_email_addresses", "")
99
+ params["email_addresses"] = cls._merge_mail_addresses(
100
+ default_mail_address, params.get("email_addresses", "")
101
+ )
102
+
103
+ return params
104
+
105
+ @classmethod
106
+ def _merge_mail_addresses(
107
+ cls,
108
+ default_mail_address: typing.Union[str, list],
109
+ email_addresses: typing.Union[str, list],
110
+ ) -> str:
111
+ if isinstance(default_mail_address, str):
112
+ default_mail_address = (
113
+ default_mail_address.split(",") if default_mail_address else []
114
+ )
115
+ if isinstance(email_addresses, str):
116
+ email_addresses = email_addresses.split(",") if email_addresses else []
117
+ email_addresses.extend(default_mail_address)
118
+ email_addresses_str = ",".join(email_addresses)
119
+ return email_addresses_str
120
+
121
+ @classmethod
122
+ def _validate_emails(cls, params):
123
+ cls._validate_email_address(params["sender_address"])
124
+
125
+ if not isinstance(params["email_addresses"], (str, list)):
126
+ raise ValueError(
127
+ "Parameter 'email_addresses' must be a string or a list of strings"
128
+ )
129
+
130
+ email_addresses = params["email_addresses"]
131
+ if isinstance(email_addresses, str):
132
+ email_addresses = email_addresses.split(",")
133
+ for email_address in email_addresses:
134
+ cls._validate_email_address(email_address)
135
+
136
+ @classmethod
137
+ def _validate_email_address(cls, email_address):
138
+ if not isinstance(email_address, str):
139
+ raise ValueError(f"Email address '{email_address}' must be a string")
140
+
141
+ if not re.match(mlrun.utils.regex.mail_regex, email_address):
142
+ raise ValueError(f"Invalid email address '{email_address}'")
143
+
144
+ @staticmethod
145
+ async def _send_email(
146
+ email_addresses: str,
147
+ sender_address: str,
148
+ server_host: str,
149
+ server_port: int,
150
+ username: str,
151
+ password: str,
152
+ use_tls: bool,
153
+ start_tls: bool,
154
+ validate_certs: bool,
155
+ subject: str,
156
+ body: str,
157
+ **kwargs,
158
+ ):
159
+ # Create the email message
160
+ message = MIMEMultipart("alternative")
161
+ message["From"] = sender_address
162
+ message["To"] = email_addresses
163
+ message["Subject"] = subject
164
+ message.attach(MIMEText(body, "html"))
165
+
166
+ # Send the email
167
+ await aiosmtplib.send(
168
+ message,
169
+ hostname=server_host,
170
+ port=server_port,
171
+ username=username,
172
+ password=password,
173
+ use_tls=use_tls,
174
+ validate_certs=validate_certs,
175
+ start_tls=start_tls,
176
+ )
@@ -46,13 +46,13 @@ class SlackNotification(NotificationBase):
46
46
  async def push(
47
47
  self,
48
48
  message: str,
49
- severity: typing.Union[
50
- mlrun.common.schemas.NotificationSeverity, str
49
+ severity: typing.Optional[
50
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
51
51
  ] = mlrun.common.schemas.NotificationSeverity.INFO,
52
- runs: typing.Union[mlrun.lists.RunList, list] = None,
53
- custom_html: str = None,
54
- alert: mlrun.common.schemas.AlertConfig = None,
55
- event_data: mlrun.common.schemas.Event = None,
52
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
53
+ custom_html: typing.Optional[typing.Optional[str]] = None,
54
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
55
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
56
56
  ):
57
57
  webhook = self.params.get("webhook", None) or mlrun.get_secret_or_env(
58
58
  "SLACK_WEBHOOK"
@@ -37,13 +37,13 @@ class WebhookNotification(NotificationBase):
37
37
  async def push(
38
38
  self,
39
39
  message: str,
40
- severity: typing.Union[
41
- mlrun.common.schemas.NotificationSeverity, str
40
+ severity: typing.Optional[
41
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
42
42
  ] = mlrun.common.schemas.NotificationSeverity.INFO,
43
- runs: typing.Union[mlrun.lists.RunList, list] = None,
44
- custom_html: str = None,
45
- alert: mlrun.common.schemas.AlertConfig = None,
46
- event_data: mlrun.common.schemas.Event = None,
43
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
44
+ custom_html: typing.Optional[typing.Optional[str]] = None,
45
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
46
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
47
47
  ):
48
48
  url = self.params.get("url", None)
49
49
  method = self.params.get("method", "post").lower()
@@ -20,12 +20,8 @@ import traceback
20
20
  import typing
21
21
  from concurrent.futures import ThreadPoolExecutor
22
22
 
23
- import mlrun_pipelines.common.ops
24
- import mlrun_pipelines.models
25
- import mlrun_pipelines.utils
26
-
27
23
  import mlrun.common.constants as mlrun_constants
28
- import mlrun.common.runtimes.constants
24
+ import mlrun.common.runtimes.constants as runtimes_constants
29
25
  import mlrun.common.schemas
30
26
  import mlrun.config
31
27
  import mlrun.db.base
@@ -33,11 +29,14 @@ import mlrun.errors
33
29
  import mlrun.lists
34
30
  import mlrun.model
35
31
  import mlrun.utils.helpers
32
+ import mlrun.utils.notifications.notification as notification_module
33
+ import mlrun.utils.notifications.notification.base as base
34
+ import mlrun_pipelines.common.ops
35
+ import mlrun_pipelines.models
36
+ import mlrun_pipelines.utils
36
37
  from mlrun.utils import logger
37
38
  from mlrun.utils.condition_evaluator import evaluate_condition_in_separate_process
38
39
 
39
- from .notification import NotificationBase, NotificationTypes
40
-
41
40
 
42
41
  class _NotificationPusherBase:
43
42
  def _push(
@@ -100,31 +99,56 @@ class NotificationPusher(_NotificationPusherBase):
100
99
  "aborted": "{resource} aborted",
101
100
  }
102
101
 
103
- def __init__(self, runs: typing.Union[mlrun.lists.RunList, list]):
102
+ def __init__(
103
+ self,
104
+ runs: typing.Union[mlrun.lists.RunList, list],
105
+ default_params: typing.Optional[dict] = None,
106
+ ):
104
107
  self._runs = runs
108
+ self._default_params = default_params or {}
105
109
  self._sync_notifications: list[
106
- tuple[NotificationBase, mlrun.model.RunObject, mlrun.model.Notification]
110
+ tuple[
111
+ base.NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
112
+ ]
107
113
  ] = []
108
114
  self._async_notifications: list[
109
- tuple[NotificationBase, mlrun.model.RunObject, mlrun.model.Notification]
115
+ tuple[
116
+ base.NotificationBase, mlrun.model.RunObject, mlrun.model.Notification
117
+ ]
110
118
  ] = []
111
119
 
112
120
  for run in self._runs:
113
- if isinstance(run, dict):
114
- run = mlrun.model.RunObject.from_dict(run)
121
+ try:
122
+ self._process_run(run)
123
+ except Exception as exc:
124
+ logger.warning(
125
+ "Failed to process run",
126
+ run_uid=run.metadata.uid,
127
+ error=mlrun.errors.err_to_str(exc),
128
+ )
115
129
 
116
- for notification in run.spec.notifications:
117
- try:
118
- notification.status = run.status.notifications.get(
119
- notification.name
120
- ).get("status", mlrun.common.schemas.NotificationStatus.PENDING)
121
- except (AttributeError, KeyError):
122
- notification.status = (
123
- mlrun.common.schemas.NotificationStatus.PENDING
124
- )
130
+ def _process_run(self, run):
131
+ if isinstance(run, dict):
132
+ run = mlrun.model.RunObject.from_dict(run)
133
+
134
+ for notification in run.spec.notifications:
135
+ try:
136
+ self._process_notification(notification, run)
137
+ except Exception as exc:
138
+ logger.warning(
139
+ "Failed to process notification",
140
+ run_uid=run.metadata.uid,
141
+ notification=notification,
142
+ error=mlrun.errors.err_to_str(exc),
143
+ )
125
144
 
126
- if self._should_notify(run, notification):
127
- self._load_notification(run, notification)
145
+ def _process_notification(self, notification, run):
146
+ notification.status = run.status.notifications.get(notification.name, {}).get(
147
+ "status",
148
+ mlrun.common.schemas.NotificationStatus.PENDING,
149
+ )
150
+ if self._should_notify(run, notification):
151
+ self._load_notification(run, notification)
128
152
 
129
153
  def push(self):
130
154
  """
@@ -168,6 +192,11 @@ class NotificationPusher(_NotificationPusherBase):
168
192
  logger.warning(
169
193
  "Failed to push notification async",
170
194
  error=mlrun.errors.err_to_str(result),
195
+ traceback=traceback.format_exception(
196
+ etype=type(result),
197
+ value=result,
198
+ tb=result.__traceback__,
199
+ ),
171
200
  )
172
201
 
173
202
  logger.debug(
@@ -197,7 +226,7 @@ class NotificationPusher(_NotificationPusherBase):
197
226
  for when_state in when_states:
198
227
  if when_state == run_state:
199
228
  if (
200
- run_state == "completed"
229
+ run_state == runtimes_constants.RunStates.completed
201
230
  and evaluate_condition_in_separate_process(
202
231
  notification.condition,
203
232
  context={
@@ -205,22 +234,29 @@ class NotificationPusher(_NotificationPusherBase):
205
234
  "notification": notification.to_dict(),
206
235
  },
207
236
  )
208
- ) or run_state in ["error", "aborted"]:
237
+ ) or run_state in [
238
+ runtimes_constants.RunStates.error,
239
+ runtimes_constants.RunStates.aborted,
240
+ runtimes_constants.RunStates.running,
241
+ ]:
209
242
  return True
210
243
 
211
244
  return False
212
245
 
213
246
  def _load_notification(
214
247
  self, run: mlrun.model.RunObject, notification_object: mlrun.model.Notification
215
- ) -> NotificationBase:
248
+ ) -> base.NotificationBase:
216
249
  name = notification_object.name
217
- notification_type = NotificationTypes(
218
- notification_object.kind or NotificationTypes.console
250
+ notification_type = notification_module.NotificationTypes(
251
+ notification_object.kind or notification_module.NotificationTypes.console
219
252
  )
220
253
  params = {}
221
254
  params.update(notification_object.secret_params)
222
255
  params.update(notification_object.params)
223
- notification = notification_type.get_notification()(name, params)
256
+ default_params = self._default_params.get(notification_type.value, {})
257
+ notification = notification_type.get_notification()(
258
+ name, params, default_params
259
+ )
224
260
  if notification.is_async:
225
261
  self._async_notifications.append((notification, run, notification_object))
226
262
  else:
@@ -260,7 +296,7 @@ class NotificationPusher(_NotificationPusherBase):
260
296
 
261
297
  def _push_notification_sync(
262
298
  self,
263
- notification: NotificationBase,
299
+ notification: base.NotificationBase,
264
300
  run: mlrun.model.RunObject,
265
301
  notification_object: mlrun.model.Notification,
266
302
  ):
@@ -308,7 +344,7 @@ class NotificationPusher(_NotificationPusherBase):
308
344
 
309
345
  async def _push_notification_async(
310
346
  self,
311
- notification: NotificationBase,
347
+ notification: base.NotificationBase,
312
348
  run: mlrun.model.RunObject,
313
349
  notification_object: mlrun.model.Notification,
314
350
  ):
@@ -361,7 +397,7 @@ class NotificationPusher(_NotificationPusherBase):
361
397
  run_uid: str,
362
398
  project: str,
363
399
  notification: mlrun.model.Notification,
364
- status: str = None,
400
+ status: typing.Optional[str] = None,
365
401
  sent_time: typing.Optional[datetime.datetime] = None,
366
402
  reason: typing.Optional[str] = None,
367
403
  ):
@@ -409,7 +445,7 @@ class NotificationPusher(_NotificationPusherBase):
409
445
  _run["step_kind"] = _step.step_type
410
446
  if _step.skipped:
411
447
  _run.setdefault("status", {})["state"] = (
412
- mlrun.common.runtimes.constants.RunStates.skipped
448
+ runtimes_constants.RunStates.skipped
413
449
  )
414
450
  steps.append(_run)
415
451
 
@@ -436,7 +472,7 @@ class NotificationPusher(_NotificationPusherBase):
436
472
  if _step.skipped:
437
473
  state = mlrun.common.schemas.FunctionState.skipped
438
474
  else:
439
- state = mlrun.common.runtimes.constants.PodPhases.pod_phase_to_run_state(
475
+ state = runtimes_constants.PodPhases.pod_phase_to_run_state(
440
476
  pod_phase
441
477
  )
442
478
  function["status"] = {"state": state}
@@ -516,9 +552,11 @@ class NotificationPusher(_NotificationPusherBase):
516
552
 
517
553
 
518
554
  class CustomNotificationPusher(_NotificationPusherBase):
519
- def __init__(self, notification_types: list[str] = None):
555
+ def __init__(self, notification_types: typing.Optional[list[str]] = None):
520
556
  notifications = {
521
- notification_type: NotificationTypes(notification_type).get_notification()()
557
+ notification_type: notification_module.NotificationTypes(
558
+ notification_type
559
+ ).get_notification()()
522
560
  for notification_type in notification_types
523
561
  }
524
562
  self._sync_notifications = {
@@ -545,7 +583,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
545
583
  mlrun.common.schemas.NotificationSeverity, str
546
584
  ] = mlrun.common.schemas.NotificationSeverity.INFO,
547
585
  runs: typing.Union[mlrun.lists.RunList, list] = None,
548
- custom_html: str = None,
586
+ custom_html: typing.Optional[str] = None,
549
587
  ):
550
588
  def sync_push():
551
589
  for notification_type, notification in self._sync_notifications.items():
@@ -567,14 +605,16 @@ class CustomNotificationPusher(_NotificationPusherBase):
567
605
  def add_notification(
568
606
  self,
569
607
  notification_type: str,
570
- params: dict[str, str] = None,
608
+ params: typing.Optional[dict[str, str]] = None,
571
609
  ):
572
610
  if notification_type in self._async_notifications:
573
611
  self._async_notifications[notification_type].load_notification(params)
574
612
  elif notification_type in self._sync_notifications:
575
613
  self._sync_notifications[notification_type].load_notification(params)
576
614
  else:
577
- notification = NotificationTypes(notification_type).get_notification()(
615
+ notification = notification_module.NotificationTypes(
616
+ notification_type
617
+ ).get_notification()(
578
618
  params=params,
579
619
  )
580
620
  if notification.is_async:
@@ -592,7 +632,9 @@ class CustomNotificationPusher(_NotificationPusherBase):
592
632
  else:
593
633
  logger.warning(f"No notification of type {notification_type} in project")
594
634
 
595
- def edit_notification(self, notification_type: str, params: dict[str, str] = None):
635
+ def edit_notification(
636
+ self, notification_type: str, params: typing.Optional[dict[str, str]] = None
637
+ ):
596
638
  self.remove_notification(notification_type)
597
639
  self.add_notification(notification_type, params)
598
640
 
@@ -606,7 +648,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
606
648
 
607
649
  # get notification's inverse dependencies, and only push the notification if
608
650
  # none of its inverse dependencies are being sent
609
- inverse_dependencies = NotificationTypes(
651
+ inverse_dependencies = notification_module.NotificationTypes(
610
652
  notification_type
611
653
  ).inverse_dependencies()
612
654
  for inverse_dependency in inverse_dependencies:
@@ -622,8 +664,8 @@ class CustomNotificationPusher(_NotificationPusherBase):
622
664
  def push_pipeline_start_message(
623
665
  self,
624
666
  project: str,
625
- commit_id: str = None,
626
- pipeline_id: str = None,
667
+ commit_id: typing.Optional[str] = None,
668
+ pipeline_id: typing.Optional[str] = None,
627
669
  has_workflow_url: bool = False,
628
670
  ):
629
671
  message = f"Workflow started in project {project}"
@@ -651,7 +693,7 @@ class CustomNotificationPusher(_NotificationPusherBase):
651
693
  self,
652
694
  runs: typing.Union[mlrun.lists.RunList, list],
653
695
  push_all: bool = False,
654
- state: str = None,
696
+ state: typing.Optional[str] = None,
655
697
  ):
656
698
  """
657
699
  push a structured table with run results to notification targets
mlrun/utils/regex.py CHANGED
@@ -86,7 +86,7 @@ tag_name = label_value
86
86
 
87
87
  secret_key = k8s_secret_and_config_map_key
88
88
 
89
- artifact_key = [r"[^\/\\]+$"]
89
+ artifact_key = [r"^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$"]
90
90
 
91
91
  # must not start with _
92
92
  # must be alphanumeric or _
@@ -101,3 +101,5 @@ artifact_uri_pattern = r"^((?P<project>.*)/)?(?P<key>.*?)(\#(?P<iteration>.*?))?
101
101
  artifact_producer_uri_pattern = (
102
102
  r"^((?P<project>.*)/)?(?P<uid>.*?)(\-(?P<iteration>.*?))?$"
103
103
  )
104
+
105
+ mail_regex = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$"
@@ -1,4 +1,4 @@
1
1
  {
2
- "git_commit": "837dd64ab75493fe34986c6203d884a0ae33843b",
3
- "version": "1.7.1-rc10"
2
+ "git_commit": "052529570e5bbae67e99d98a89713d70c6751607",
3
+ "version": "1.8.0-rc8"
4
4
  }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mlrun
3
- Version: 1.7.1rc10
3
+ Version: 1.8.0rc8
4
4
  Summary: Tracking and config of machine learning runs
5
5
  Home-page: https://github.com/mlrun/mlrun
6
6
  Author: Yaron Haviv
@@ -23,35 +23,38 @@ Description-Content-Type: text/markdown
23
23
  License-File: LICENSE
24
24
  Requires-Dist: urllib3<1.27,>=1.26.9
25
25
  Requires-Dist: GitPython>=3.1.41,~=3.1
26
- Requires-Dist: aiohttp<3.11,>=3.9
26
+ Requires-Dist: aiohttp~=3.10.0
27
27
  Requires-Dist: aiohttp-retry~=2.8.0
28
28
  Requires-Dist: click~=8.1
29
29
  Requires-Dist: nest-asyncio~=1.0
30
30
  Requires-Dist: ipython~=8.10
31
- Requires-Dist: nuclio-jupyter~=0.10.4
32
- Requires-Dist: numpy<1.27.0,>=1.16.5
31
+ Requires-Dist: nuclio-jupyter~=0.11.1
32
+ Requires-Dist: numpy<1.27.0,>=1.26.4
33
33
  Requires-Dist: pandas<2.2,>=1.2
34
34
  Requires-Dist: pyarrow<18,>=10.0
35
- Requires-Dist: pyyaml<7,>=5.4.1
35
+ Requires-Dist: pyyaml<7,>=6.0.2
36
36
  Requires-Dist: requests~=2.32
37
37
  Requires-Dist: tabulate~=0.8.6
38
38
  Requires-Dist: v3io~=0.6.9
39
- Requires-Dist: pydantic<1.10.15,>=1.10.8
39
+ Requires-Dist: pydantic>=1.10.15
40
40
  Requires-Dist: mergedeep~=1.3
41
- Requires-Dist: v3io-frames~=0.10.14
41
+ Requires-Dist: v3io-frames~=0.10.14; python_version < "3.11"
42
+ Requires-Dist: v3io-frames!=0.11.*,!=0.12.*,>=0.10.14; python_version >= "3.11"
42
43
  Requires-Dist: semver~=3.0
43
44
  Requires-Dist: dependency-injector~=4.41
44
45
  Requires-Dist: fsspec<2024.7,>=2023.9.2
45
46
  Requires-Dist: v3iofs~=0.1.17
46
- Requires-Dist: storey~=1.7.50
47
+ Requires-Dist: storey~=1.8.0
47
48
  Requires-Dist: inflection~=0.5.0
48
- Requires-Dist: python-dotenv~=0.17.0
49
- Requires-Dist: setuptools~=71.0
49
+ Requires-Dist: python-dotenv~=1.0
50
+ Requires-Dist: setuptools>=75.2
50
51
  Requires-Dist: deprecated~=1.2
51
52
  Requires-Dist: jinja2>=3.1.3,~=3.1
52
53
  Requires-Dist: orjson<4,>=3.9.15
53
- Requires-Dist: mlrun-pipelines-kfp-common~=0.1.9
54
- Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.1.6
54
+ Requires-Dist: mlrun-pipelines-kfp-common~=0.2.3
55
+ Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.2.3; python_version < "3.11"
56
+ Requires-Dist: docstring_parser~=0.16
57
+ Requires-Dist: aiosmtplib~=3.0
55
58
  Provides-Extra: s3
56
59
  Requires-Dist: boto3<1.36,>=1.28.0; extra == "s3"
57
60
  Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "s3"
@@ -102,9 +105,9 @@ Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
102
105
  Provides-Extra: api
103
106
  Requires-Dist: uvicorn~=0.27.1; extra == "api"
104
107
  Requires-Dist: dask-kubernetes~=0.11.0; extra == "api"
105
- Requires-Dist: apscheduler==3.10.3; extra == "api"
108
+ Requires-Dist: apscheduler<4,>=3.11; extra == "api"
106
109
  Requires-Dist: objgraph~=3.6; extra == "api"
107
- Requires-Dist: igz-mgmt~=0.2.0; extra == "api"
110
+ Requires-Dist: igz-mgmt~=0.4.1; extra == "api"
108
111
  Requires-Dist: humanfriendly~=10.0; extra == "api"
109
112
  Requires-Dist: fastapi~=0.110.0; extra == "api"
110
113
  Requires-Dist: sqlalchemy~=1.4; extra == "api"
@@ -112,6 +115,7 @@ Requires-Dist: pymysql~=1.0; extra == "api"
112
115
  Requires-Dist: alembic~=1.9; extra == "api"
113
116
  Requires-Dist: timelength~=1.1; extra == "api"
114
117
  Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "api"
118
+ Requires-Dist: aiosmtplib~=3.0; extra == "api"
115
119
  Provides-Extra: all
116
120
  Requires-Dist: adlfs==2023.9.0; extra == "all"
117
121
  Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "all"
@@ -176,8 +180,9 @@ Requires-Dist: taoswswrap~=0.2.0; extra == "complete"
176
180
  Provides-Extra: complete-api
177
181
  Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
178
182
  Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
183
+ Requires-Dist: aiosmtplib~=3.0; extra == "complete-api"
179
184
  Requires-Dist: alembic~=1.9; extra == "complete-api"
180
- Requires-Dist: apscheduler==3.10.3; extra == "complete-api"
185
+ Requires-Dist: apscheduler<4,>=3.11; extra == "complete-api"
181
186
  Requires-Dist: avro~=1.11; extra == "complete-api"
182
187
  Requires-Dist: azure-core~=1.24; extra == "complete-api"
183
188
  Requires-Dist: azure-identity~=1.5; extra == "complete-api"
@@ -195,7 +200,7 @@ Requires-Dist: google-cloud-storage==2.14.0; extra == "complete-api"
195
200
  Requires-Dist: google-cloud==0.34; extra == "complete-api"
196
201
  Requires-Dist: graphviz~=0.20.0; extra == "complete-api"
197
202
  Requires-Dist: humanfriendly~=10.0; extra == "complete-api"
198
- Requires-Dist: igz-mgmt~=0.2.0; extra == "complete-api"
203
+ Requires-Dist: igz-mgmt~=0.4.1; extra == "complete-api"
199
204
  Requires-Dist: kafka-python~=2.0; extra == "complete-api"
200
205
  Requires-Dist: memray~=1.12; sys_platform != "win32" and extra == "complete-api"
201
206
  Requires-Dist: mlflow~=2.8; extra == "complete-api"