mlrun 1.7.2rc4__py3-none-any.whl → 1.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (275) hide show
  1. mlrun/__init__.py +26 -22
  2. mlrun/__main__.py +15 -16
  3. mlrun/alerts/alert.py +150 -15
  4. mlrun/api/schemas/__init__.py +1 -9
  5. mlrun/artifacts/__init__.py +2 -3
  6. mlrun/artifacts/base.py +62 -19
  7. mlrun/artifacts/dataset.py +17 -17
  8. mlrun/artifacts/document.py +454 -0
  9. mlrun/artifacts/manager.py +28 -18
  10. mlrun/artifacts/model.py +91 -59
  11. mlrun/artifacts/plots.py +2 -2
  12. mlrun/common/constants.py +8 -0
  13. mlrun/common/formatters/__init__.py +1 -0
  14. mlrun/common/formatters/artifact.py +1 -1
  15. mlrun/common/formatters/feature_set.py +2 -0
  16. mlrun/common/formatters/function.py +1 -0
  17. mlrun/{model_monitoring/db/stores/v3io_kv/__init__.py → common/formatters/model_endpoint.py} +17 -0
  18. mlrun/common/formatters/pipeline.py +1 -2
  19. mlrun/common/formatters/project.py +9 -0
  20. mlrun/common/model_monitoring/__init__.py +0 -5
  21. mlrun/common/model_monitoring/helpers.py +12 -62
  22. mlrun/common/runtimes/constants.py +25 -4
  23. mlrun/common/schemas/__init__.py +9 -5
  24. mlrun/common/schemas/alert.py +114 -19
  25. mlrun/common/schemas/api_gateway.py +3 -3
  26. mlrun/common/schemas/artifact.py +22 -9
  27. mlrun/common/schemas/auth.py +8 -4
  28. mlrun/common/schemas/background_task.py +7 -7
  29. mlrun/common/schemas/client_spec.py +4 -4
  30. mlrun/common/schemas/clusterization_spec.py +2 -2
  31. mlrun/common/schemas/common.py +53 -3
  32. mlrun/common/schemas/constants.py +15 -0
  33. mlrun/common/schemas/datastore_profile.py +1 -1
  34. mlrun/common/schemas/feature_store.py +9 -9
  35. mlrun/common/schemas/frontend_spec.py +4 -4
  36. mlrun/common/schemas/function.py +10 -10
  37. mlrun/common/schemas/hub.py +1 -1
  38. mlrun/common/schemas/k8s.py +3 -3
  39. mlrun/common/schemas/memory_reports.py +3 -3
  40. mlrun/common/schemas/model_monitoring/__init__.py +4 -8
  41. mlrun/common/schemas/model_monitoring/constants.py +127 -46
  42. mlrun/common/schemas/model_monitoring/grafana.py +18 -12
  43. mlrun/common/schemas/model_monitoring/model_endpoints.py +154 -160
  44. mlrun/common/schemas/notification.py +24 -3
  45. mlrun/common/schemas/object.py +1 -1
  46. mlrun/common/schemas/pagination.py +4 -4
  47. mlrun/common/schemas/partition.py +142 -0
  48. mlrun/common/schemas/pipeline.py +3 -3
  49. mlrun/common/schemas/project.py +26 -18
  50. mlrun/common/schemas/runs.py +3 -3
  51. mlrun/common/schemas/runtime_resource.py +5 -5
  52. mlrun/common/schemas/schedule.py +1 -1
  53. mlrun/common/schemas/secret.py +1 -1
  54. mlrun/{model_monitoring/db/stores/sqldb/__init__.py → common/schemas/serving.py} +10 -1
  55. mlrun/common/schemas/tag.py +3 -3
  56. mlrun/common/schemas/workflow.py +6 -5
  57. mlrun/common/types.py +1 -0
  58. mlrun/config.py +157 -89
  59. mlrun/data_types/__init__.py +5 -3
  60. mlrun/data_types/infer.py +13 -3
  61. mlrun/data_types/spark.py +2 -1
  62. mlrun/datastore/__init__.py +59 -18
  63. mlrun/datastore/alibaba_oss.py +4 -1
  64. mlrun/datastore/azure_blob.py +4 -1
  65. mlrun/datastore/base.py +19 -24
  66. mlrun/datastore/datastore.py +10 -4
  67. mlrun/datastore/datastore_profile.py +178 -45
  68. mlrun/datastore/dbfs_store.py +4 -1
  69. mlrun/datastore/filestore.py +4 -1
  70. mlrun/datastore/google_cloud_storage.py +4 -1
  71. mlrun/datastore/hdfs.py +4 -1
  72. mlrun/datastore/inmem.py +4 -1
  73. mlrun/datastore/redis.py +4 -1
  74. mlrun/datastore/s3.py +14 -3
  75. mlrun/datastore/sources.py +89 -92
  76. mlrun/datastore/store_resources.py +7 -4
  77. mlrun/datastore/storeytargets.py +51 -16
  78. mlrun/datastore/targets.py +38 -31
  79. mlrun/datastore/utils.py +87 -4
  80. mlrun/datastore/v3io.py +4 -1
  81. mlrun/datastore/vectorstore.py +291 -0
  82. mlrun/datastore/wasbfs/fs.py +13 -12
  83. mlrun/db/base.py +286 -100
  84. mlrun/db/httpdb.py +1562 -490
  85. mlrun/db/nopdb.py +250 -83
  86. mlrun/errors.py +6 -2
  87. mlrun/execution.py +194 -50
  88. mlrun/feature_store/__init__.py +2 -10
  89. mlrun/feature_store/api.py +20 -458
  90. mlrun/feature_store/common.py +9 -9
  91. mlrun/feature_store/feature_set.py +20 -18
  92. mlrun/feature_store/feature_vector.py +105 -479
  93. mlrun/feature_store/feature_vector_utils.py +466 -0
  94. mlrun/feature_store/retrieval/base.py +15 -11
  95. mlrun/feature_store/retrieval/job.py +2 -1
  96. mlrun/feature_store/retrieval/storey_merger.py +1 -1
  97. mlrun/feature_store/steps.py +3 -3
  98. mlrun/features.py +30 -13
  99. mlrun/frameworks/__init__.py +1 -2
  100. mlrun/frameworks/_common/__init__.py +1 -2
  101. mlrun/frameworks/_common/artifacts_library.py +2 -2
  102. mlrun/frameworks/_common/mlrun_interface.py +10 -6
  103. mlrun/frameworks/_common/model_handler.py +31 -31
  104. mlrun/frameworks/_common/producer.py +3 -1
  105. mlrun/frameworks/_dl_common/__init__.py +1 -2
  106. mlrun/frameworks/_dl_common/loggers/__init__.py +1 -2
  107. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +4 -4
  108. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +3 -3
  109. mlrun/frameworks/_ml_common/__init__.py +1 -2
  110. mlrun/frameworks/_ml_common/loggers/__init__.py +1 -2
  111. mlrun/frameworks/_ml_common/model_handler.py +21 -21
  112. mlrun/frameworks/_ml_common/plans/__init__.py +1 -2
  113. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +3 -1
  114. mlrun/frameworks/_ml_common/plans/dataset_plan.py +3 -3
  115. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +4 -4
  116. mlrun/frameworks/auto_mlrun/__init__.py +1 -2
  117. mlrun/frameworks/auto_mlrun/auto_mlrun.py +22 -15
  118. mlrun/frameworks/huggingface/__init__.py +1 -2
  119. mlrun/frameworks/huggingface/model_server.py +9 -9
  120. mlrun/frameworks/lgbm/__init__.py +47 -44
  121. mlrun/frameworks/lgbm/callbacks/__init__.py +1 -2
  122. mlrun/frameworks/lgbm/callbacks/logging_callback.py +4 -2
  123. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +4 -2
  124. mlrun/frameworks/lgbm/mlrun_interfaces/__init__.py +1 -2
  125. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +5 -5
  126. mlrun/frameworks/lgbm/model_handler.py +15 -11
  127. mlrun/frameworks/lgbm/model_server.py +11 -7
  128. mlrun/frameworks/lgbm/utils.py +2 -2
  129. mlrun/frameworks/onnx/__init__.py +1 -2
  130. mlrun/frameworks/onnx/dataset.py +3 -3
  131. mlrun/frameworks/onnx/mlrun_interface.py +2 -2
  132. mlrun/frameworks/onnx/model_handler.py +7 -5
  133. mlrun/frameworks/onnx/model_server.py +8 -6
  134. mlrun/frameworks/parallel_coordinates.py +11 -11
  135. mlrun/frameworks/pytorch/__init__.py +22 -23
  136. mlrun/frameworks/pytorch/callbacks/__init__.py +1 -2
  137. mlrun/frameworks/pytorch/callbacks/callback.py +2 -1
  138. mlrun/frameworks/pytorch/callbacks/logging_callback.py +15 -8
  139. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +19 -12
  140. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +22 -15
  141. mlrun/frameworks/pytorch/callbacks_handler.py +36 -30
  142. mlrun/frameworks/pytorch/mlrun_interface.py +17 -17
  143. mlrun/frameworks/pytorch/model_handler.py +21 -17
  144. mlrun/frameworks/pytorch/model_server.py +13 -9
  145. mlrun/frameworks/sklearn/__init__.py +19 -18
  146. mlrun/frameworks/sklearn/estimator.py +2 -2
  147. mlrun/frameworks/sklearn/metric.py +3 -3
  148. mlrun/frameworks/sklearn/metrics_library.py +8 -6
  149. mlrun/frameworks/sklearn/mlrun_interface.py +3 -2
  150. mlrun/frameworks/sklearn/model_handler.py +4 -3
  151. mlrun/frameworks/tf_keras/__init__.py +11 -12
  152. mlrun/frameworks/tf_keras/callbacks/__init__.py +1 -2
  153. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +17 -14
  154. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +15 -12
  155. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +21 -18
  156. mlrun/frameworks/tf_keras/model_handler.py +17 -13
  157. mlrun/frameworks/tf_keras/model_server.py +12 -8
  158. mlrun/frameworks/xgboost/__init__.py +19 -18
  159. mlrun/frameworks/xgboost/model_handler.py +13 -9
  160. mlrun/k8s_utils.py +2 -5
  161. mlrun/launcher/base.py +3 -4
  162. mlrun/launcher/client.py +2 -2
  163. mlrun/launcher/local.py +6 -2
  164. mlrun/launcher/remote.py +1 -1
  165. mlrun/lists.py +8 -4
  166. mlrun/model.py +132 -46
  167. mlrun/model_monitoring/__init__.py +3 -5
  168. mlrun/model_monitoring/api.py +113 -98
  169. mlrun/model_monitoring/applications/__init__.py +0 -5
  170. mlrun/model_monitoring/applications/_application_steps.py +81 -50
  171. mlrun/model_monitoring/applications/base.py +467 -14
  172. mlrun/model_monitoring/applications/context.py +212 -134
  173. mlrun/model_monitoring/{db/stores/base → applications/evidently}/__init__.py +6 -2
  174. mlrun/model_monitoring/applications/evidently/base.py +146 -0
  175. mlrun/model_monitoring/applications/histogram_data_drift.py +89 -56
  176. mlrun/model_monitoring/applications/results.py +67 -15
  177. mlrun/model_monitoring/controller.py +701 -315
  178. mlrun/model_monitoring/db/__init__.py +0 -2
  179. mlrun/model_monitoring/db/_schedules.py +242 -0
  180. mlrun/model_monitoring/db/_stats.py +189 -0
  181. mlrun/model_monitoring/db/tsdb/__init__.py +33 -22
  182. mlrun/model_monitoring/db/tsdb/base.py +243 -49
  183. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +76 -36
  184. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +33 -0
  185. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connection.py +213 -0
  186. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +534 -88
  187. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +1 -0
  188. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +436 -106
  189. mlrun/model_monitoring/helpers.py +356 -114
  190. mlrun/model_monitoring/stream_processing.py +190 -345
  191. mlrun/model_monitoring/tracking_policy.py +11 -4
  192. mlrun/model_monitoring/writer.py +49 -90
  193. mlrun/package/__init__.py +3 -6
  194. mlrun/package/context_handler.py +2 -2
  195. mlrun/package/packager.py +12 -9
  196. mlrun/package/packagers/__init__.py +0 -2
  197. mlrun/package/packagers/default_packager.py +14 -11
  198. mlrun/package/packagers/numpy_packagers.py +16 -7
  199. mlrun/package/packagers/pandas_packagers.py +18 -18
  200. mlrun/package/packagers/python_standard_library_packagers.py +25 -11
  201. mlrun/package/packagers_manager.py +35 -32
  202. mlrun/package/utils/__init__.py +0 -3
  203. mlrun/package/utils/_pickler.py +6 -6
  204. mlrun/platforms/__init__.py +47 -16
  205. mlrun/platforms/iguazio.py +4 -1
  206. mlrun/projects/operations.py +30 -30
  207. mlrun/projects/pipelines.py +116 -47
  208. mlrun/projects/project.py +1292 -329
  209. mlrun/render.py +5 -9
  210. mlrun/run.py +57 -14
  211. mlrun/runtimes/__init__.py +1 -3
  212. mlrun/runtimes/base.py +30 -22
  213. mlrun/runtimes/daskjob.py +9 -9
  214. mlrun/runtimes/databricks_job/databricks_runtime.py +6 -5
  215. mlrun/runtimes/function_reference.py +5 -2
  216. mlrun/runtimes/generators.py +3 -2
  217. mlrun/runtimes/kubejob.py +6 -7
  218. mlrun/runtimes/mounts.py +574 -0
  219. mlrun/runtimes/mpijob/__init__.py +0 -2
  220. mlrun/runtimes/mpijob/abstract.py +7 -6
  221. mlrun/runtimes/nuclio/api_gateway.py +7 -7
  222. mlrun/runtimes/nuclio/application/application.py +11 -13
  223. mlrun/runtimes/nuclio/application/reverse_proxy.go +66 -64
  224. mlrun/runtimes/nuclio/function.py +127 -70
  225. mlrun/runtimes/nuclio/serving.py +105 -37
  226. mlrun/runtimes/pod.py +159 -54
  227. mlrun/runtimes/remotesparkjob.py +3 -2
  228. mlrun/runtimes/sparkjob/__init__.py +0 -2
  229. mlrun/runtimes/sparkjob/spark3job.py +22 -12
  230. mlrun/runtimes/utils.py +7 -6
  231. mlrun/secrets.py +2 -2
  232. mlrun/serving/__init__.py +8 -0
  233. mlrun/serving/merger.py +7 -5
  234. mlrun/serving/remote.py +35 -22
  235. mlrun/serving/routers.py +186 -240
  236. mlrun/serving/server.py +41 -10
  237. mlrun/serving/states.py +432 -118
  238. mlrun/serving/utils.py +13 -2
  239. mlrun/serving/v1_serving.py +3 -2
  240. mlrun/serving/v2_serving.py +161 -203
  241. mlrun/track/__init__.py +1 -1
  242. mlrun/track/tracker.py +2 -2
  243. mlrun/track/trackers/mlflow_tracker.py +6 -5
  244. mlrun/utils/async_http.py +35 -22
  245. mlrun/utils/clones.py +7 -4
  246. mlrun/utils/helpers.py +511 -58
  247. mlrun/utils/logger.py +119 -13
  248. mlrun/utils/notifications/notification/__init__.py +22 -19
  249. mlrun/utils/notifications/notification/base.py +39 -15
  250. mlrun/utils/notifications/notification/console.py +6 -6
  251. mlrun/utils/notifications/notification/git.py +11 -11
  252. mlrun/utils/notifications/notification/ipython.py +10 -9
  253. mlrun/utils/notifications/notification/mail.py +176 -0
  254. mlrun/utils/notifications/notification/slack.py +16 -8
  255. mlrun/utils/notifications/notification/webhook.py +24 -8
  256. mlrun/utils/notifications/notification_pusher.py +191 -200
  257. mlrun/utils/regex.py +12 -2
  258. mlrun/utils/version/version.json +2 -2
  259. {mlrun-1.7.2rc4.dist-info → mlrun-1.8.0.dist-info}/METADATA +69 -54
  260. mlrun-1.8.0.dist-info/RECORD +351 -0
  261. {mlrun-1.7.2rc4.dist-info → mlrun-1.8.0.dist-info}/WHEEL +1 -1
  262. mlrun/model_monitoring/applications/evidently_base.py +0 -137
  263. mlrun/model_monitoring/db/stores/__init__.py +0 -136
  264. mlrun/model_monitoring/db/stores/base/store.py +0 -213
  265. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +0 -71
  266. mlrun/model_monitoring/db/stores/sqldb/models/base.py +0 -190
  267. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +0 -103
  268. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +0 -40
  269. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +0 -659
  270. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +0 -726
  271. mlrun/model_monitoring/model_endpoint.py +0 -118
  272. mlrun-1.7.2rc4.dist-info/RECORD +0 -351
  273. {mlrun-1.7.2rc4.dist-info → mlrun-1.8.0.dist-info}/entry_points.txt +0 -0
  274. {mlrun-1.7.2rc4.dist-info → mlrun-1.8.0.dist-info/licenses}/LICENSE +0 -0
  275. {mlrun-1.7.2rc4.dist-info → mlrun-1.8.0.dist-info}/top_level.txt +0 -0
mlrun/utils/logger.py CHANGED
@@ -11,9 +11,12 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
-
14
+ import contextvars
15
+ import datetime
15
16
  import logging
16
17
  import os
18
+ import string
19
+ import sys
17
20
  import typing
18
21
  from enum import Enum
19
22
  from functools import cached_property
@@ -22,15 +25,18 @@ from traceback import format_exception
22
25
  from typing import IO, Optional, Union
23
26
 
24
27
  import orjson
25
- import pydantic
28
+ import pydantic.v1
26
29
 
30
+ from mlrun import errors
27
31
  from mlrun.config import config
28
32
 
33
+ context_id_var = contextvars.ContextVar("context_id", default=None)
34
+
29
35
 
30
36
  class _BaseFormatter(logging.Formatter):
31
37
  def _json_dump(self, json_object):
32
38
  def default(obj):
33
- if isinstance(obj, pydantic.BaseModel):
39
+ if isinstance(obj, pydantic.v1.BaseModel):
34
40
  return obj.dict()
35
41
 
36
42
  # EAFP all the way.
@@ -50,16 +56,24 @@ class _BaseFormatter(logging.Formatter):
50
56
  json_object,
51
57
  option=orjson.OPT_NAIVE_UTC
52
58
  | orjson.OPT_SERIALIZE_NUMPY
59
+ | orjson.OPT_NON_STR_KEYS
53
60
  | orjson.OPT_SORT_KEYS,
54
61
  default=default,
55
62
  ).decode()
56
63
 
57
-
58
- class JSONFormatter(_BaseFormatter):
59
- def format(self, record) -> str:
64
+ def _record_with(self, record):
60
65
  record_with = getattr(record, "with", {})
61
66
  if record.exc_info:
62
67
  record_with.update(exc_info=format_exception(*record.exc_info))
68
+ if "ctx" not in record_with:
69
+ if (ctx_id := context_id_var.get()) is not None:
70
+ record_with["ctx"] = ctx_id
71
+ return record_with
72
+
73
+
74
+ class JSONFormatter(_BaseFormatter):
75
+ def format(self, record) -> str:
76
+ record_with = self._record_with(record)
63
77
  record_fields = {
64
78
  "datetime": self.formatTime(record, self.datefmt),
65
79
  "level": record.levelname.lower(),
@@ -86,11 +100,97 @@ class HumanReadableFormatter(_BaseFormatter):
86
100
  more = f": {record_with_encoded}" if record_with_encoded else ""
87
101
  return more
88
102
 
89
- def _record_with(self, record):
90
- record_with = getattr(record, "with", {})
91
- if record.exc_info:
92
- record_with.update(exc_info=format_exception(*record.exc_info))
93
- return record_with
103
+
104
+ class CustomFormatter(HumanReadableFormatter):
105
+ """
106
+ To enable custom logger formatter, configure MLRun with the following env variables:
107
+ 1. "MLRUN_LOG_FORMATTER" = "custom" - change the default log formatter.
108
+ 2. "MLRUN_LOG_FORMAT_OVERRIDE" = "> {timestamp} [{level}] Running module: {module} {message} {more}" - logger format
109
+ * Note that your custom format must include those 4 fields - timestamp, level, message and more
110
+ If the custom format is not configured properly , MLRun will use the default logger (human format).
111
+ """
112
+
113
+ # This attribute is used to solve an issue
114
+ # that causes the warning to be written numerous times(for any log generation).
115
+ # We want to print the errors just once, not for each logger generation.
116
+ fail_on_format_configuration = False # for issues that relates to unrecognized keys
117
+ fail_on_missing_default_keys_key = (
118
+ False # for issues that relates to missing default keys
119
+ )
120
+
121
+ def format(self, record) -> str:
122
+ more = self._resolve_more(record)
123
+ custom_format = config.log_format_override
124
+ _custom_format = None
125
+ current_time = datetime.datetime.now()
126
+ formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S,%f")[:-3]
127
+ try:
128
+ if custom_format:
129
+ default_keys = ["timestamp", "level", "message", "more"]
130
+ formatter = string.Formatter()
131
+ custom_format_keys = [
132
+ key
133
+ for _, key, _, _ in formatter.parse(custom_format)
134
+ if key is not None
135
+ ]
136
+ missing_default_flags = list(
137
+ set(default_keys) - set(custom_format_keys)
138
+ )
139
+
140
+ if (
141
+ missing_default_flags
142
+ and not CustomFormatter.fail_on_missing_default_keys_key
143
+ ):
144
+ print(
145
+ f'> {formatted_time} [warning] Custom loggers must '
146
+ f'include those keys within the logger format, {", ".join(default_keys)} '
147
+ f'your format is missing: {", ".join(missing_default_flags)}',
148
+ file=sys.stderr,
149
+ )
150
+ CustomFormatter.fail_on_missing_default_keys_key = True
151
+ record_dict = record.__dict__
152
+ missing_format_configuraiton_keys = list(
153
+ set(custom_format_keys)
154
+ - set(default_keys)
155
+ - set(record_dict.keys())
156
+ )
157
+ if missing_format_configuraiton_keys:
158
+ if not CustomFormatter.fail_on_format_configuration:
159
+ print(
160
+ f"> {formatted_time} [warning] Failed to create custom logger due "
161
+ f'to missing format key in the log record: {", ".join(missing_format_configuraiton_keys)}',
162
+ file=sys.stderr,
163
+ )
164
+ CustomFormatter.fail_on_format_configuration = True
165
+ _format = (
166
+ f"> {self.formatTime(record, self.datefmt)} "
167
+ f"[{record.levelname.lower()}] "
168
+ f"{record.getMessage().rstrip()}"
169
+ f"{more}"
170
+ )
171
+ _custom_format = custom_format.format(
172
+ timestamp=self.formatTime(record, self.datefmt),
173
+ level=record.levelname.lower(),
174
+ message=record.getMessage().rstrip(),
175
+ more=more or "",
176
+ **record_dict,
177
+ )
178
+ CustomFormatter.fail_on_format_configuration = True
179
+ except Exception as e:
180
+ if not CustomFormatter.fail_on_format_configuration:
181
+ print(
182
+ f"> {formatted_time} [warning] Failed to create custom logger, "
183
+ f"see Exception: {errors.err_to_str(e)}",
184
+ file=sys.stderr,
185
+ )
186
+ CustomFormatter.fail_on_format_configuration = True
187
+ _format = _custom_format or (
188
+ f"> {self.formatTime(record, self.datefmt)} "
189
+ f"[{record.levelname.lower()}] "
190
+ f"{record.getMessage().rstrip()}"
191
+ f"{more}"
192
+ )
193
+ return _format
94
194
 
95
195
 
96
196
  class HumanReadableExtendedFormatter(HumanReadableFormatter):
@@ -258,7 +358,6 @@ class Logger:
258
358
  self, level, message, *args, exc_info=None, **kw_args
259
359
  ):
260
360
  kw_args.update(self._bound_variables)
261
-
262
361
  if kw_args:
263
362
  self._logger.log(
264
363
  level, message, *args, exc_info=exc_info, extra={"with": kw_args}
@@ -272,17 +371,24 @@ class FormatterKinds(Enum):
272
371
  HUMAN = "human"
273
372
  HUMAN_EXTENDED = "human_extended"
274
373
  JSON = "json"
374
+ CUSTOM = "custom"
275
375
 
276
376
 
277
377
  def resolve_formatter_by_kind(
278
378
  formatter_kind: FormatterKinds,
279
379
  ) -> type[
280
- typing.Union[HumanReadableFormatter, HumanReadableExtendedFormatter, JSONFormatter]
380
+ typing.Union[
381
+ HumanReadableFormatter,
382
+ HumanReadableExtendedFormatter,
383
+ JSONFormatter,
384
+ CustomFormatter,
385
+ ]
281
386
  ]:
282
387
  return {
283
388
  FormatterKinds.HUMAN: HumanReadableFormatter,
284
389
  FormatterKinds.HUMAN_EXTENDED: HumanReadableExtendedFormatter,
285
390
  FormatterKinds.JSON: JSONFormatter,
391
+ FormatterKinds.CUSTOM: CustomFormatter,
286
392
  }[formatter_kind]
287
393
 
288
394
 
@@ -14,30 +14,32 @@
14
14
 
15
15
  import enum
16
16
 
17
- from mlrun.common.schemas.notification import NotificationKind
18
-
19
- from .base import NotificationBase
20
- from .console import ConsoleNotification
21
- from .git import GitNotification
22
- from .ipython import IPythonNotification
23
- from .slack import SlackNotification
24
- from .webhook import WebhookNotification
17
+ import mlrun.common.schemas.notification as notifications
18
+ import mlrun.utils.notifications.notification.base as base
19
+ import mlrun.utils.notifications.notification.console as console
20
+ import mlrun.utils.notifications.notification.git as git
21
+ import mlrun.utils.notifications.notification.ipython as ipython
22
+ import mlrun.utils.notifications.notification.mail as mail
23
+ import mlrun.utils.notifications.notification.slack as slack
24
+ import mlrun.utils.notifications.notification.webhook as webhook
25
25
 
26
26
 
27
27
  class NotificationTypes(str, enum.Enum):
28
- console = NotificationKind.console.value
29
- git = NotificationKind.git.value
30
- ipython = NotificationKind.ipython.value
31
- slack = NotificationKind.slack.value
32
- webhook = NotificationKind.webhook.value
28
+ console = notifications.NotificationKind.console.value
29
+ git = notifications.NotificationKind.git.value
30
+ ipython = notifications.NotificationKind.ipython.value
31
+ slack = notifications.NotificationKind.slack.value
32
+ mail = notifications.NotificationKind.mail.value
33
+ webhook = notifications.NotificationKind.webhook.value
33
34
 
34
- def get_notification(self) -> type[NotificationBase]:
35
+ def get_notification(self) -> type[base.NotificationBase]:
35
36
  return {
36
- self.console: ConsoleNotification,
37
- self.git: GitNotification,
38
- self.ipython: IPythonNotification,
39
- self.slack: SlackNotification,
40
- self.webhook: WebhookNotification,
37
+ self.console: console.ConsoleNotification,
38
+ self.git: git.GitNotification,
39
+ self.ipython: ipython.IPythonNotification,
40
+ self.slack: slack.SlackNotification,
41
+ self.mail: mail.MailNotification,
42
+ self.webhook: webhook.WebhookNotification,
41
43
  }.get(self)
42
44
 
43
45
  def inverse_dependencies(self) -> list[str]:
@@ -64,5 +66,6 @@ class NotificationTypes(str, enum.Enum):
64
66
  cls.git,
65
67
  cls.ipython,
66
68
  cls.slack,
69
+ cls.mail,
67
70
  cls.webhook,
68
71
  ]
@@ -14,6 +14,7 @@
14
14
 
15
15
  import asyncio
16
16
  import typing
17
+ from copy import deepcopy
17
18
 
18
19
  import mlrun.common.schemas
19
20
  import mlrun.lists
@@ -22,11 +23,20 @@ import mlrun.lists
22
23
  class NotificationBase:
23
24
  def __init__(
24
25
  self,
25
- name: str = None,
26
- params: dict[str, str] = None,
26
+ name: typing.Optional[str] = None,
27
+ params: typing.Optional[dict[str, str]] = None,
28
+ default_params: typing.Optional[dict[str, str]] = None,
27
29
  ):
30
+ """
31
+ NotificationBase is the base class for all notification types.
32
+
33
+ :param name: The name of the notification.
34
+ :param params: The parameters of the notification.
35
+ :param default_params: The default parameters of the notification. Used for server-side enrichment purposes.
36
+ """
28
37
  self.name = name
29
38
  self.params = params or {}
39
+ self.params = self.enrich_default_params(self.params, default_params)
30
40
 
31
41
  @classmethod
32
42
  def validate_params(cls, params):
@@ -43,13 +53,13 @@ class NotificationBase:
43
53
  def push(
44
54
  self,
45
55
  message: str,
46
- severity: typing.Union[
47
- mlrun.common.schemas.NotificationSeverity, str
56
+ severity: typing.Optional[
57
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
48
58
  ] = mlrun.common.schemas.NotificationSeverity.INFO,
49
- runs: typing.Union[mlrun.lists.RunList, list] = None,
50
- custom_html: str = None,
51
- alert: mlrun.common.schemas.AlertConfig = None,
52
- event_data: mlrun.common.schemas.Event = None,
59
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
60
+ custom_html: typing.Optional[str] = None,
61
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
62
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
53
63
  ):
54
64
  raise NotImplementedError()
55
65
 
@@ -59,16 +69,25 @@ class NotificationBase:
59
69
  ) -> None:
60
70
  self.params = params or {}
61
71
 
72
+ @classmethod
73
+ def enrich_default_params(
74
+ cls, params: dict, default_params: typing.Optional[dict] = None
75
+ ) -> dict:
76
+ default_params = default_params or {}
77
+ returned_params = deepcopy(default_params)
78
+ returned_params.update(params)
79
+ return returned_params
80
+
62
81
  def _get_html(
63
82
  self,
64
83
  message: str,
65
- severity: typing.Union[
66
- mlrun.common.schemas.NotificationSeverity, str
84
+ severity: typing.Optional[
85
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
67
86
  ] = mlrun.common.schemas.NotificationSeverity.INFO,
68
- runs: typing.Union[mlrun.lists.RunList, list] = None,
69
- custom_html: str = None,
70
- alert: mlrun.common.schemas.AlertConfig = None,
71
- event_data: mlrun.common.schemas.Event = None,
87
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
88
+ custom_html: typing.Optional[typing.Optional[str]] = None,
89
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
90
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
72
91
  ) -> str:
73
92
  if custom_html:
74
93
  return custom_html
@@ -115,7 +134,12 @@ class NotificationBase:
115
134
  event_data.entity.kind == mlrun.common.schemas.alert.EventEntityKind.JOB
116
135
  ): # JOB entity
117
136
  uid = event_data.value_dict.get("uid")
118
- url = mlrun.utils.helpers.get_ui_url(alert.project, uid)
137
+ name = event_data.entity.ids[0]
138
+ url = mlrun.utils.helpers.get_run_url(
139
+ alert.project,
140
+ uid=uid,
141
+ name=name,
142
+ )
119
143
  overview_type = "Job overview"
120
144
  else: # MODEL entity
121
145
  model_name = event_data.value_dict.get("model")
@@ -31,13 +31,13 @@ class ConsoleNotification(NotificationBase):
31
31
  def push(
32
32
  self,
33
33
  message: str,
34
- severity: typing.Union[
35
- mlrun.common.schemas.NotificationSeverity, str
34
+ severity: typing.Optional[
35
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
36
36
  ] = mlrun.common.schemas.NotificationSeverity.INFO,
37
- runs: typing.Union[mlrun.lists.RunList, list] = None,
38
- custom_html: str = None,
39
- alert: mlrun.common.schemas.AlertConfig = None,
40
- event_data: mlrun.common.schemas.Event = None,
37
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
38
+ custom_html: typing.Optional[typing.Optional[str]] = None,
39
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
40
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
41
41
  ):
42
42
  severity = self._resolve_severity(severity)
43
43
  print(f"[{severity}] {message}")
@@ -54,13 +54,13 @@ class GitNotification(NotificationBase):
54
54
  async def push(
55
55
  self,
56
56
  message: str,
57
- severity: typing.Union[
58
- mlrun.common.schemas.NotificationSeverity, str
57
+ severity: typing.Optional[
58
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
59
59
  ] = mlrun.common.schemas.NotificationSeverity.INFO,
60
- runs: typing.Union[mlrun.lists.RunList, list] = None,
61
- custom_html: str = None,
62
- alert: mlrun.common.schemas.AlertConfig = None,
63
- event_data: mlrun.common.schemas.Event = None,
60
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
61
+ custom_html: typing.Optional[typing.Optional[str]] = None,
62
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
63
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
64
64
  ):
65
65
  git_repo = self.params.get("repo", None)
66
66
  git_issue = self.params.get("issue", None)
@@ -85,11 +85,11 @@ class GitNotification(NotificationBase):
85
85
  @staticmethod
86
86
  async def _pr_comment(
87
87
  message: str,
88
- repo: str = None,
89
- issue: int = None,
90
- merge_request: int = None,
91
- token: str = None,
92
- server: str = None,
88
+ repo: typing.Optional[str] = None,
89
+ issue: typing.Optional[int] = None,
90
+ merge_request: typing.Optional[int] = None,
91
+ token: typing.Optional[str] = None,
92
+ server: typing.Optional[str] = None,
93
93
  gitlab: bool = False,
94
94
  ) -> str:
95
95
  """push comment message to Git system PR/issue
@@ -28,10 +28,11 @@ class IPythonNotification(NotificationBase):
28
28
 
29
29
  def __init__(
30
30
  self,
31
- name: str = None,
32
- params: dict[str, str] = None,
31
+ name: typing.Optional[str] = None,
32
+ params: typing.Optional[dict[str, str]] = None,
33
+ default_params: typing.Optional[dict[str, str]] = None,
33
34
  ):
34
- super().__init__(name, params)
35
+ super().__init__(name, params, default_params)
35
36
  self._ipython = None
36
37
  try:
37
38
  import IPython
@@ -48,13 +49,13 @@ class IPythonNotification(NotificationBase):
48
49
  def push(
49
50
  self,
50
51
  message: str,
51
- severity: typing.Union[
52
- mlrun.common.schemas.NotificationSeverity, str
52
+ severity: typing.Optional[
53
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
53
54
  ] = mlrun.common.schemas.NotificationSeverity.INFO,
54
- runs: typing.Union[mlrun.lists.RunList, list] = None,
55
- custom_html: str = None,
56
- alert: mlrun.common.schemas.AlertConfig = None,
57
- event_data: mlrun.common.schemas.Event = None,
55
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
56
+ custom_html: typing.Optional[typing.Optional[str]] = None,
57
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
58
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
58
59
  ):
59
60
  if not self._ipython:
60
61
  mlrun.utils.helpers.logger.debug(
@@ -0,0 +1,176 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ import re
15
+ import typing
16
+ from email.mime.multipart import MIMEMultipart
17
+ from email.mime.text import MIMEText
18
+
19
+ import aiosmtplib
20
+
21
+ import mlrun.common.schemas
22
+ import mlrun.lists
23
+ import mlrun.utils.helpers
24
+ import mlrun.utils.notifications.notification.base as base
25
+ import mlrun.utils.regex
26
+
27
+ DEFAULT_SMTP_PORT = 587
28
+
29
+
30
+ class MailNotification(base.NotificationBase):
31
+ """
32
+ API/Client notification for sending run statuses as a mail message
33
+ """
34
+
35
+ boolean_params = ["use_tls", "start_tls", "validate_certs"]
36
+
37
+ required_params = [
38
+ "server_host",
39
+ "server_port",
40
+ "sender_address",
41
+ "username",
42
+ "password",
43
+ "email_addresses",
44
+ ] + boolean_params
45
+
46
+ @classmethod
47
+ def validate_params(cls, params):
48
+ for required_param in cls.required_params:
49
+ if required_param not in params:
50
+ raise ValueError(
51
+ f"Parameter '{required_param}' is required for MailNotification"
52
+ )
53
+
54
+ for boolean_param in cls.boolean_params:
55
+ if not isinstance(params.get(boolean_param, None), bool):
56
+ raise ValueError(
57
+ f"Parameter '{boolean_param}' must be a boolean for MailNotification"
58
+ )
59
+
60
+ cls._validate_emails(params)
61
+
62
+ async def push(
63
+ self,
64
+ message: str,
65
+ severity: typing.Optional[
66
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
67
+ ] = mlrun.common.schemas.NotificationSeverity.INFO,
68
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
69
+ custom_html: typing.Optional[typing.Optional[str]] = None,
70
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
71
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
72
+ ):
73
+ self.params["subject"] = f"[{severity}] {message}"
74
+ message_body_override = self.params.get("message_body_override", None)
75
+
76
+ runs_html = self._get_html(
77
+ message, severity, runs, custom_html, alert, event_data
78
+ )
79
+ self.params["body"] = runs_html
80
+
81
+ if message_body_override:
82
+ self.params["body"] = message_body_override.replace(
83
+ "{{ runs }}", runs_html
84
+ ).replace("{{runs}}", runs_html)
85
+
86
+ await self._send_email(**self.params)
87
+
88
+ @classmethod
89
+ def enrich_default_params(
90
+ cls, params: dict, default_params: typing.Optional[dict] = None
91
+ ) -> dict:
92
+ params = super().enrich_default_params(params, default_params)
93
+ params.setdefault("use_tls", True)
94
+ params.setdefault("start_tls", False)
95
+ params.setdefault("validate_certs", True)
96
+ params.setdefault("server_port", DEFAULT_SMTP_PORT)
97
+
98
+ default_mail_address = params.pop("default_email_addresses", "")
99
+ params["email_addresses"] = cls._merge_mail_addresses(
100
+ default_mail_address, params.get("email_addresses", "")
101
+ )
102
+
103
+ return params
104
+
105
+ @classmethod
106
+ def _merge_mail_addresses(
107
+ cls,
108
+ default_mail_address: typing.Union[str, list],
109
+ email_addresses: typing.Union[str, list],
110
+ ) -> str:
111
+ if isinstance(default_mail_address, str):
112
+ default_mail_address = (
113
+ default_mail_address.split(",") if default_mail_address else []
114
+ )
115
+ if isinstance(email_addresses, str):
116
+ email_addresses = email_addresses.split(",") if email_addresses else []
117
+ email_addresses.extend(default_mail_address)
118
+ email_addresses_str = ",".join(email_addresses)
119
+ return email_addresses_str
120
+
121
+ @classmethod
122
+ def _validate_emails(cls, params):
123
+ cls._validate_email_address(params["sender_address"])
124
+
125
+ if not isinstance(params["email_addresses"], (str, list)):
126
+ raise ValueError(
127
+ "Parameter 'email_addresses' must be a string or a list of strings"
128
+ )
129
+
130
+ email_addresses = params["email_addresses"]
131
+ if isinstance(email_addresses, str):
132
+ email_addresses = email_addresses.split(",")
133
+ for email_address in email_addresses:
134
+ cls._validate_email_address(email_address)
135
+
136
+ @classmethod
137
+ def _validate_email_address(cls, email_address):
138
+ if not isinstance(email_address, str):
139
+ raise ValueError(f"Email address '{email_address}' must be a string")
140
+
141
+ if not re.match(mlrun.utils.regex.mail_regex, email_address):
142
+ raise ValueError(f"Invalid email address '{email_address}'")
143
+
144
+ @staticmethod
145
+ async def _send_email(
146
+ email_addresses: str,
147
+ sender_address: str,
148
+ server_host: str,
149
+ server_port: int,
150
+ username: str,
151
+ password: str,
152
+ use_tls: bool,
153
+ start_tls: bool,
154
+ validate_certs: bool,
155
+ subject: str,
156
+ body: str,
157
+ **kwargs,
158
+ ):
159
+ # Create the email message
160
+ message = MIMEMultipart("alternative")
161
+ message["From"] = sender_address
162
+ message["To"] = email_addresses
163
+ message["Subject"] = subject
164
+ message.attach(MIMEText(body, "html"))
165
+
166
+ # Send the email
167
+ await aiosmtplib.send(
168
+ message,
169
+ hostname=server_host,
170
+ port=server_port,
171
+ username=username,
172
+ password=password,
173
+ use_tls=use_tls,
174
+ validate_certs=validate_certs,
175
+ start_tls=start_tls,
176
+ )
@@ -16,6 +16,7 @@ import typing
16
16
 
17
17
  import aiohttp
18
18
 
19
+ import mlrun.common.runtimes.constants as runtimes_constants
19
20
  import mlrun.common.schemas
20
21
  import mlrun.lists
21
22
  import mlrun.utils.helpers
@@ -46,13 +47,13 @@ class SlackNotification(NotificationBase):
46
47
  async def push(
47
48
  self,
48
49
  message: str,
49
- severity: typing.Union[
50
- mlrun.common.schemas.NotificationSeverity, str
50
+ severity: typing.Optional[
51
+ typing.Union[mlrun.common.schemas.NotificationSeverity, str]
51
52
  ] = mlrun.common.schemas.NotificationSeverity.INFO,
52
- runs: typing.Union[mlrun.lists.RunList, list] = None,
53
- custom_html: str = None,
54
- alert: mlrun.common.schemas.AlertConfig = None,
55
- event_data: mlrun.common.schemas.Event = None,
53
+ runs: typing.Optional[typing.Union[mlrun.lists.RunList, list]] = None,
54
+ custom_html: typing.Optional[typing.Optional[str]] = None,
55
+ alert: typing.Optional[mlrun.common.schemas.AlertConfig] = None,
56
+ event_data: typing.Optional[mlrun.common.schemas.Event] = None,
56
57
  ):
57
58
  webhook = self.params.get("webhook", None) or mlrun.get_secret_or_env(
58
59
  "SLACK_WEBHOOK"
@@ -168,12 +169,19 @@ class SlackNotification(NotificationBase):
168
169
 
169
170
  def _get_run_line(self, run: dict) -> dict:
170
171
  meta = run["metadata"]
171
- url = mlrun.utils.helpers.get_ui_url(meta.get("project"), meta.get("uid"))
172
+ url = mlrun.utils.helpers.get_run_url(
173
+ meta.get("project"),
174
+ uid=meta.get("uid"),
175
+ name=meta.get("name"),
176
+ )
172
177
 
173
178
  # Only show the URL if the run is not a function (serving or mlrun function)
174
179
  kind = run.get("step_kind")
175
180
  state = run["status"].get("state", "")
176
- if state != "skipped" and (url and not kind or kind == "run"):
181
+
182
+ if state != runtimes_constants.RunStates.skipped and (
183
+ url and not kind or kind == "run"
184
+ ):
177
185
  line = f'<{url}|*{meta.get("name")}*>'
178
186
  else:
179
187
  line = meta.get("name")