mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (184) hide show
  1. mlrun/__init__.py +2 -35
  2. mlrun/__main__.py +3 -41
  3. mlrun/api/api/api.py +6 -0
  4. mlrun/api/api/endpoints/feature_store.py +0 -4
  5. mlrun/api/api/endpoints/files.py +14 -2
  6. mlrun/api/api/endpoints/frontend_spec.py +2 -1
  7. mlrun/api/api/endpoints/functions.py +95 -59
  8. mlrun/api/api/endpoints/grafana_proxy.py +9 -9
  9. mlrun/api/api/endpoints/logs.py +17 -3
  10. mlrun/api/api/endpoints/model_endpoints.py +3 -2
  11. mlrun/api/api/endpoints/pipelines.py +1 -5
  12. mlrun/api/api/endpoints/projects.py +88 -0
  13. mlrun/api/api/endpoints/runs.py +48 -6
  14. mlrun/api/api/endpoints/submit.py +2 -1
  15. mlrun/api/api/endpoints/workflows.py +355 -0
  16. mlrun/api/api/utils.py +3 -4
  17. mlrun/api/crud/__init__.py +1 -0
  18. mlrun/api/crud/client_spec.py +6 -2
  19. mlrun/api/crud/feature_store.py +5 -0
  20. mlrun/api/crud/model_monitoring/__init__.py +1 -0
  21. mlrun/api/crud/model_monitoring/deployment.py +497 -0
  22. mlrun/api/crud/model_monitoring/grafana.py +96 -42
  23. mlrun/api/crud/model_monitoring/helpers.py +159 -0
  24. mlrun/api/crud/model_monitoring/model_endpoints.py +202 -476
  25. mlrun/api/crud/notifications.py +9 -4
  26. mlrun/api/crud/pipelines.py +6 -11
  27. mlrun/api/crud/projects.py +2 -2
  28. mlrun/api/crud/runtime_resources.py +4 -3
  29. mlrun/api/crud/runtimes/nuclio/helpers.py +5 -1
  30. mlrun/api/crud/secrets.py +21 -0
  31. mlrun/api/crud/workflows.py +352 -0
  32. mlrun/api/db/base.py +16 -1
  33. mlrun/api/db/init_db.py +2 -4
  34. mlrun/api/db/session.py +1 -1
  35. mlrun/api/db/sqldb/db.py +129 -31
  36. mlrun/api/db/sqldb/models/models_mysql.py +15 -1
  37. mlrun/api/db/sqldb/models/models_sqlite.py +16 -2
  38. mlrun/api/launcher.py +38 -6
  39. mlrun/api/main.py +3 -2
  40. mlrun/api/rundb/__init__.py +13 -0
  41. mlrun/{db → api/rundb}/sqldb.py +36 -84
  42. mlrun/api/runtime_handlers/__init__.py +56 -0
  43. mlrun/api/runtime_handlers/base.py +1247 -0
  44. mlrun/api/runtime_handlers/daskjob.py +209 -0
  45. mlrun/api/runtime_handlers/kubejob.py +37 -0
  46. mlrun/api/runtime_handlers/mpijob.py +147 -0
  47. mlrun/api/runtime_handlers/remotesparkjob.py +29 -0
  48. mlrun/api/runtime_handlers/sparkjob.py +148 -0
  49. mlrun/api/schemas/__init__.py +17 -6
  50. mlrun/api/utils/builder.py +1 -4
  51. mlrun/api/utils/clients/chief.py +14 -0
  52. mlrun/api/utils/clients/iguazio.py +33 -33
  53. mlrun/api/utils/clients/nuclio.py +2 -2
  54. mlrun/api/utils/periodic.py +9 -2
  55. mlrun/api/utils/projects/follower.py +14 -7
  56. mlrun/api/utils/projects/leader.py +2 -1
  57. mlrun/api/utils/projects/remotes/nop_follower.py +2 -2
  58. mlrun/api/utils/projects/remotes/nop_leader.py +2 -2
  59. mlrun/api/utils/runtimes/__init__.py +14 -0
  60. mlrun/api/utils/runtimes/nuclio.py +43 -0
  61. mlrun/api/utils/scheduler.py +98 -15
  62. mlrun/api/utils/singletons/db.py +5 -1
  63. mlrun/api/utils/singletons/project_member.py +4 -1
  64. mlrun/api/utils/singletons/scheduler.py +1 -1
  65. mlrun/artifacts/base.py +6 -6
  66. mlrun/artifacts/dataset.py +4 -4
  67. mlrun/artifacts/manager.py +2 -3
  68. mlrun/artifacts/model.py +2 -2
  69. mlrun/artifacts/plots.py +8 -8
  70. mlrun/common/db/__init__.py +14 -0
  71. mlrun/common/helpers.py +37 -0
  72. mlrun/{mlutils → common/model_monitoring}/__init__.py +3 -2
  73. mlrun/common/model_monitoring/helpers.py +69 -0
  74. mlrun/common/schemas/__init__.py +13 -1
  75. mlrun/common/schemas/auth.py +4 -1
  76. mlrun/common/schemas/client_spec.py +1 -1
  77. mlrun/common/schemas/function.py +17 -0
  78. mlrun/common/schemas/model_monitoring/__init__.py +48 -0
  79. mlrun/common/{model_monitoring.py → schemas/model_monitoring/constants.py} +11 -23
  80. mlrun/common/schemas/model_monitoring/grafana.py +55 -0
  81. mlrun/common/schemas/{model_endpoints.py → model_monitoring/model_endpoints.py} +32 -65
  82. mlrun/common/schemas/notification.py +1 -0
  83. mlrun/common/schemas/object.py +4 -0
  84. mlrun/common/schemas/project.py +1 -0
  85. mlrun/common/schemas/regex.py +1 -1
  86. mlrun/common/schemas/runs.py +1 -8
  87. mlrun/common/schemas/schedule.py +1 -8
  88. mlrun/common/schemas/workflow.py +54 -0
  89. mlrun/config.py +45 -42
  90. mlrun/datastore/__init__.py +21 -0
  91. mlrun/datastore/base.py +1 -1
  92. mlrun/datastore/datastore.py +9 -0
  93. mlrun/datastore/dbfs_store.py +168 -0
  94. mlrun/datastore/helpers.py +18 -0
  95. mlrun/datastore/sources.py +1 -0
  96. mlrun/datastore/store_resources.py +2 -5
  97. mlrun/datastore/v3io.py +1 -2
  98. mlrun/db/__init__.py +4 -68
  99. mlrun/db/base.py +12 -0
  100. mlrun/db/factory.py +65 -0
  101. mlrun/db/httpdb.py +175 -20
  102. mlrun/db/nopdb.py +4 -2
  103. mlrun/execution.py +4 -2
  104. mlrun/feature_store/__init__.py +1 -0
  105. mlrun/feature_store/api.py +1 -2
  106. mlrun/feature_store/common.py +2 -1
  107. mlrun/feature_store/feature_set.py +1 -11
  108. mlrun/feature_store/feature_vector.py +340 -2
  109. mlrun/feature_store/ingestion.py +5 -10
  110. mlrun/feature_store/retrieval/base.py +118 -104
  111. mlrun/feature_store/retrieval/dask_merger.py +17 -10
  112. mlrun/feature_store/retrieval/job.py +4 -1
  113. mlrun/feature_store/retrieval/local_merger.py +18 -18
  114. mlrun/feature_store/retrieval/spark_merger.py +21 -14
  115. mlrun/feature_store/retrieval/storey_merger.py +22 -16
  116. mlrun/kfpops.py +3 -9
  117. mlrun/launcher/base.py +57 -53
  118. mlrun/launcher/client.py +5 -4
  119. mlrun/launcher/factory.py +24 -13
  120. mlrun/launcher/local.py +6 -6
  121. mlrun/launcher/remote.py +4 -4
  122. mlrun/lists.py +0 -11
  123. mlrun/model.py +11 -17
  124. mlrun/model_monitoring/__init__.py +2 -22
  125. mlrun/model_monitoring/features_drift_table.py +1 -1
  126. mlrun/model_monitoring/helpers.py +22 -210
  127. mlrun/model_monitoring/model_endpoint.py +1 -1
  128. mlrun/model_monitoring/model_monitoring_batch.py +127 -50
  129. mlrun/model_monitoring/prometheus.py +219 -0
  130. mlrun/model_monitoring/stores/__init__.py +16 -11
  131. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +95 -23
  132. mlrun/model_monitoring/stores/models/mysql.py +47 -29
  133. mlrun/model_monitoring/stores/models/sqlite.py +47 -29
  134. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +31 -19
  135. mlrun/model_monitoring/{stream_processing_fs.py → stream_processing.py} +206 -64
  136. mlrun/model_monitoring/tracking_policy.py +104 -0
  137. mlrun/package/packager.py +6 -8
  138. mlrun/package/packagers/default_packager.py +121 -10
  139. mlrun/package/packagers/numpy_packagers.py +1 -1
  140. mlrun/platforms/__init__.py +0 -2
  141. mlrun/platforms/iguazio.py +0 -56
  142. mlrun/projects/pipelines.py +53 -159
  143. mlrun/projects/project.py +10 -37
  144. mlrun/render.py +1 -1
  145. mlrun/run.py +8 -124
  146. mlrun/runtimes/__init__.py +6 -42
  147. mlrun/runtimes/base.py +29 -1249
  148. mlrun/runtimes/daskjob.py +2 -198
  149. mlrun/runtimes/funcdoc.py +0 -9
  150. mlrun/runtimes/function.py +25 -29
  151. mlrun/runtimes/kubejob.py +5 -29
  152. mlrun/runtimes/local.py +1 -1
  153. mlrun/runtimes/mpijob/__init__.py +2 -2
  154. mlrun/runtimes/mpijob/abstract.py +10 -1
  155. mlrun/runtimes/mpijob/v1.py +0 -76
  156. mlrun/runtimes/mpijob/v1alpha1.py +1 -74
  157. mlrun/runtimes/nuclio.py +3 -2
  158. mlrun/runtimes/pod.py +28 -18
  159. mlrun/runtimes/remotesparkjob.py +1 -15
  160. mlrun/runtimes/serving.py +14 -6
  161. mlrun/runtimes/sparkjob/__init__.py +0 -1
  162. mlrun/runtimes/sparkjob/abstract.py +4 -131
  163. mlrun/runtimes/utils.py +0 -26
  164. mlrun/serving/routers.py +7 -7
  165. mlrun/serving/server.py +11 -8
  166. mlrun/serving/states.py +7 -1
  167. mlrun/serving/v2_serving.py +6 -6
  168. mlrun/utils/helpers.py +23 -42
  169. mlrun/utils/notifications/notification/__init__.py +4 -0
  170. mlrun/utils/notifications/notification/webhook.py +61 -0
  171. mlrun/utils/notifications/notification_pusher.py +5 -25
  172. mlrun/utils/regex.py +7 -2
  173. mlrun/utils/version/version.json +2 -2
  174. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/METADATA +26 -25
  175. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/RECORD +180 -158
  176. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/WHEEL +1 -1
  177. mlrun/mlutils/data.py +0 -160
  178. mlrun/mlutils/models.py +0 -78
  179. mlrun/mlutils/plots.py +0 -902
  180. mlrun/utils/model_monitoring.py +0 -249
  181. /mlrun/{api/db/sqldb/session.py → common/db/sql_session.py} +0 -0
  182. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/LICENSE +0 -0
  183. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/entry_points.txt +0 -0
  184. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/top_level.txt +0 -0
mlrun/utils/helpers.py CHANGED
@@ -35,13 +35,13 @@ import pandas
35
35
  import semver
36
36
  import yaml
37
37
  from dateutil import parser
38
- from deprecated import deprecated
39
38
  from pandas._libs.tslibs.timestamps import Timedelta, Timestamp
40
39
  from yaml.representer import RepresenterError
41
40
 
42
41
  import mlrun
43
42
  import mlrun.common.schemas
44
43
  import mlrun.errors
44
+ import mlrun.utils.regex
45
45
  import mlrun.utils.version.version
46
46
  from mlrun.errors import err_to_str
47
47
 
@@ -238,6 +238,17 @@ def validate_artifact_key_name(
238
238
  )
239
239
 
240
240
 
241
+ def validate_v3io_stream_consumer_group(
242
+ value: str, raise_on_failure: bool = True
243
+ ) -> bool:
244
+ return mlrun.utils.helpers.verify_field_regex(
245
+ "consumerGroup",
246
+ value,
247
+ mlrun.utils.regex.v3io_stream_consumer_group,
248
+ raise_on_failure=raise_on_failure,
249
+ )
250
+
251
+
241
252
  def get_regex_list_as_string(regex_list: List) -> str:
242
253
  """
243
254
  This function is used to combine a list of regex strings into a single regex,
@@ -625,26 +636,6 @@ def dict_to_json(struct):
625
636
  return json.dumps(struct, cls=MyEncoder)
626
637
 
627
638
 
628
- def parse_versioned_object_uri(uri, default_project=""):
629
- project = default_project
630
- tag = ""
631
- hash_key = ""
632
- if "/" in uri:
633
- loc = uri.find("/")
634
- project = uri[:loc]
635
- uri = uri[loc + 1 :]
636
- if ":" in uri:
637
- loc = uri.find(":")
638
- tag = uri[loc + 1 :]
639
- uri = uri[:loc]
640
- if "@" in uri:
641
- loc = uri.find("@")
642
- hash_key = uri[loc + 1 :]
643
- uri = uri[:loc]
644
-
645
- return project, uri, tag, hash_key
646
-
647
-
648
639
  def parse_artifact_uri(uri, default_project=""):
649
640
  uri_pattern = r"^((?P<project>.*)/)?(?P<key>.*?)(\#(?P<iteration>.*?))?(:(?P<tag>.*?))?(@(?P<uid>.*))?$"
650
641
  match = re.match(uri_pattern, uri)
@@ -773,18 +764,6 @@ def new_pipe_metadata(
773
764
  return conf
774
765
 
775
766
 
776
- # TODO: remove in 1.5.0
777
- @deprecated(
778
- version="1.3.0",
779
- reason="'new_pipe_meta' will be removed in 1.5.0",
780
- category=FutureWarning,
781
- )
782
- def new_pipe_meta(artifact_path=None, ttl=None, *args):
783
- return new_pipe_metadata(
784
- artifact_path=artifact_path, cleanup_ttl=ttl, op_transformers=args
785
- )
786
-
787
-
788
767
  def _convert_python_package_version_to_image_tag(version: typing.Optional[str]):
789
768
  return (
790
769
  version.replace("+", "-").replace("0.0.0-", "") if version is not None else None
@@ -848,14 +827,14 @@ def resolve_image_tag_suffix(
848
827
  return "-py37"
849
828
  return ""
850
829
 
851
- # For mlrun 1.3.0, we decided to support mlrun runtimes images with both python 3.7 and 3.9 images.
830
+ # For mlrun 1.3.x and 1.4.x, we support mlrun runtimes images with both python 3.7 and 3.9 images.
852
831
  # While the python 3.9 images will continue to have no suffix, the python 3.7 images will have a '-py37' suffix.
853
832
  # Python 3.8 images will not be supported for mlrun 1.3.0, meaning that if the user has client with python 3.8
854
833
  # and mlrun 1.3.x then the image will be pulled without a suffix (which is the python 3.9 image).
855
834
  # using semver (x.y.z-X) to include rc versions as well
856
- if semver.VersionInfo.parse(mlrun_version) >= semver.VersionInfo.parse(
857
- "1.3.0-X"
858
- ) and python_version.startswith("3.7"):
835
+ if semver.VersionInfo.parse("1.5.0-X") > semver.VersionInfo.parse(
836
+ mlrun_version
837
+ ) >= semver.VersionInfo.parse("1.3.0-X") and python_version.startswith("3.7"):
859
838
  return "-py37"
860
839
  return ""
861
840
 
@@ -945,15 +924,12 @@ def create_step_backoff(steps=None):
945
924
  while True:
946
925
  current_step_value, current_step_remain = step
947
926
  if current_step_remain == 0:
948
-
949
927
  # No more in this step, moving on
950
928
  step = next(steps)
951
929
  elif current_step_remain is None:
952
-
953
930
  # We are in the last step, staying here forever
954
931
  yield current_step_value
955
932
  elif current_step_remain > 0:
956
-
957
933
  # Still more remains in this step, just reduce the remaining number
958
934
  step[1] -= 1
959
935
  yield current_step_value
@@ -968,7 +944,6 @@ def create_exponential_backoff(base=2, max_value=120, scale_factor=1):
968
944
  """
969
945
  exponent = 1
970
946
  while True:
971
-
972
947
  # This "complex" implementation (unlike the one in linear backoff) is to avoid exponent growing too fast and
973
948
  # risking going behind max_int
974
949
  next_value = scale_factor * (base**exponent)
@@ -1401,7 +1376,6 @@ def as_number(field_name, field_value):
1401
1376
  def filter_warnings(action, category):
1402
1377
  def decorator(function):
1403
1378
  def wrapper(*args, **kwargs):
1404
-
1405
1379
  # context manager that copies and, upon exit, restores the warnings filter and the showwarning() function.
1406
1380
  with warnings.catch_warnings():
1407
1381
  warnings.simplefilter(action, category)
@@ -1470,3 +1444,10 @@ class DeprecationHelper(object):
1470
1444
  def __getattr__(self, attr):
1471
1445
  self._warn()
1472
1446
  return getattr(self._new_target, attr)
1447
+
1448
+
1449
+ def normalize_workflow_name(name, project_name):
1450
+ workflow_name = (
1451
+ name.lstrip(project_name).lstrip("-") if project_name in name else name
1452
+ )
1453
+ return workflow_name
@@ -22,6 +22,7 @@ from .console import ConsoleNotification
22
22
  from .git import GitNotification
23
23
  from .ipython import IPythonNotification
24
24
  from .slack import SlackNotification
25
+ from .webhook import WebhookNotification
25
26
 
26
27
 
27
28
  class NotificationTypes(str, enum.Enum):
@@ -29,6 +30,7 @@ class NotificationTypes(str, enum.Enum):
29
30
  git = NotificationKind.git.value
30
31
  ipython = NotificationKind.ipython.value
31
32
  slack = NotificationKind.slack.value
33
+ webhook = NotificationKind.webhook.value
32
34
 
33
35
  def get_notification(self) -> typing.Type[NotificationBase]:
34
36
  return {
@@ -36,6 +38,7 @@ class NotificationTypes(str, enum.Enum):
36
38
  self.git: GitNotification,
37
39
  self.ipython: IPythonNotification,
38
40
  self.slack: SlackNotification,
41
+ self.webhook: WebhookNotification,
39
42
  }.get(self)
40
43
 
41
44
  def inverse_dependencies(self) -> typing.List[str]:
@@ -56,5 +59,6 @@ class NotificationTypes(str, enum.Enum):
56
59
  cls.git,
57
60
  cls.ipython,
58
61
  cls.slack,
62
+ cls.webhook,
59
63
  ]
60
64
  )
@@ -0,0 +1,61 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import typing
16
+
17
+ import aiohttp
18
+
19
+ import mlrun.common.schemas
20
+ import mlrun.lists
21
+ import mlrun.utils.helpers
22
+
23
+ from .base import NotificationBase
24
+
25
+
26
+ class WebhookNotification(NotificationBase):
27
+ """
28
+ API/Client notification for sending run statuses in a http request
29
+ """
30
+
31
+ async def push(
32
+ self,
33
+ message: str,
34
+ severity: typing.Union[
35
+ mlrun.common.schemas.NotificationSeverity, str
36
+ ] = mlrun.common.schemas.NotificationSeverity.INFO,
37
+ runs: typing.Union[mlrun.lists.RunList, list] = None,
38
+ custom_html: str = None,
39
+ ):
40
+ url = self.params.get("url", None)
41
+ method = self.params.get("method", "post").lower()
42
+ headers = self.params.get("headers", {})
43
+ override_body = self.params.get("override_body", None)
44
+
45
+ request_body = {
46
+ "message": message,
47
+ "severity": severity,
48
+ "runs": runs,
49
+ }
50
+
51
+ if custom_html:
52
+ request_body["custom_html"] = custom_html
53
+
54
+ if override_body:
55
+ request_body = override_body
56
+
57
+ async with aiohttp.ClientSession() as session:
58
+ response = await getattr(session, method)(
59
+ url, headers=headers, json=request_body
60
+ )
61
+ response.raise_for_status()
@@ -20,10 +20,9 @@ from concurrent.futures import ThreadPoolExecutor
20
20
 
21
21
  from fastapi.concurrency import run_in_threadpool
22
22
 
23
- import mlrun.api.db.base
24
- import mlrun.api.db.session
25
23
  import mlrun.common.schemas
26
24
  import mlrun.config
25
+ import mlrun.db.base
27
26
  import mlrun.lists
28
27
  import mlrun.model
29
28
  import mlrun.utils.helpers
@@ -63,10 +62,7 @@ class NotificationPusher(object):
63
62
  if self._should_notify(run, notification):
64
63
  self._load_notification(run, notification)
65
64
 
66
- def push(
67
- self,
68
- db: mlrun.api.db.base.DBInterface = None,
69
- ):
65
+ def push(self):
70
66
  """
71
67
  Asynchronously push notifications for all runs in the initialized runs list (if they should be pushed).
72
68
  When running from a sync environment, the notifications will be pushed asynchronously however the function will
@@ -82,7 +78,6 @@ class NotificationPusher(object):
82
78
  notification_data[0],
83
79
  notification_data[1],
84
80
  notification_data[2],
85
- db,
86
81
  )
87
82
 
88
83
  async def _async_push():
@@ -93,7 +88,6 @@ class NotificationPusher(object):
93
88
  notification_data[0],
94
89
  notification_data[1],
95
90
  notification_data[2],
96
- db,
97
91
  )
98
92
  )
99
93
 
@@ -200,7 +194,6 @@ class NotificationPusher(object):
200
194
  notification: NotificationBase,
201
195
  run: mlrun.model.RunObject,
202
196
  notification_object: mlrun.model.Notification,
203
- db: mlrun.api.db.base.DBInterface,
204
197
  ):
205
198
  message, severity, runs = self._prepare_notification_args(
206
199
  run, notification_object
@@ -213,7 +206,6 @@ class NotificationPusher(object):
213
206
  try:
214
207
  notification.push(message, severity, runs)
215
208
  self._update_notification_status(
216
- db,
217
209
  run.metadata.uid,
218
210
  run.metadata.project,
219
211
  notification_object,
@@ -222,7 +214,6 @@ class NotificationPusher(object):
222
214
  )
223
215
  except Exception as exc:
224
216
  self._update_notification_status(
225
- db,
226
217
  run.metadata.uid,
227
218
  run.metadata.project,
228
219
  notification_object,
@@ -235,7 +226,6 @@ class NotificationPusher(object):
235
226
  notification: NotificationBase,
236
227
  run: mlrun.model.RunObject,
237
228
  notification_object: mlrun.model.Notification,
238
- db: mlrun.api.db.base.DBInterface,
239
229
  ):
240
230
  message, severity, runs = self._prepare_notification_args(
241
231
  run, notification_object
@@ -250,7 +240,6 @@ class NotificationPusher(object):
250
240
 
251
241
  await run_in_threadpool(
252
242
  self._update_notification_status,
253
- db,
254
243
  run.metadata.uid,
255
244
  run.metadata.project,
256
245
  notification_object,
@@ -260,7 +249,6 @@ class NotificationPusher(object):
260
249
  except Exception as exc:
261
250
  await run_in_threadpool(
262
251
  self._update_notification_status,
263
- db,
264
252
  run.metadata.uid,
265
253
  run.metadata.project,
266
254
  notification_object,
@@ -270,30 +258,22 @@ class NotificationPusher(object):
270
258
 
271
259
  @staticmethod
272
260
  def _update_notification_status(
273
- db: mlrun.api.db.base.DBInterface,
274
261
  run_uid: str,
275
262
  project: str,
276
263
  notification: mlrun.model.Notification,
277
264
  status: str = None,
278
265
  sent_time: typing.Optional[datetime.datetime] = None,
279
266
  ):
280
-
281
- # nothing to update if not running as api
282
- # note, the notification mechanism may run "locally" for certain runtimes
283
- if not mlrun.config.is_running_as_api():
284
- return
285
-
286
- # TODO: move to api side
287
- db_session = mlrun.api.db.session.create_session()
267
+ db = mlrun.get_run_db()
288
268
  notification.status = status or notification.status
289
269
  notification.sent_time = sent_time or notification.sent_time
290
270
 
291
- # store directly in db, no need to use crud as the secrets are already loaded
271
+ # There is no need to mask the params as the secrets are already loaded
292
272
  db.store_run_notifications(
293
- db_session,
294
273
  [notification],
295
274
  run_uid,
296
275
  project,
276
+ mask_params=False,
297
277
  )
298
278
 
299
279
 
mlrun/utils/regex.py CHANGED
@@ -72,8 +72,8 @@ sparkjob_name = label_value + sprakjob_length + sparkjob_service_name
72
72
  # A project name have the following restrictions:
73
73
  # It should be a valid Nuclio Project CRD name which is dns 1123 subdomain
74
74
  # It should be a valid k8s label value since Nuclio use the project name in labels of resources
75
- # It should be a valid namespace name (cause we plan to map it to one) which is dns 1123 label
76
- # of the 3 restrictions, dns 1123 label is the most strict, so we enforce only it
75
+ # It should be a valid namespace name (because we plan to map it to one) which is dns 1123 label
76
+ # of the 3 restrictions, dns 1123 label is strictest, so we enforce only it
77
77
  project_name = dns_1123_label
78
78
 
79
79
  # Special characters are not permitted in tag names because they can be included in the url and cause problems.
@@ -83,3 +83,8 @@ tag_name = label_value
83
83
  secret_key = k8s_secret_and_config_map_key
84
84
 
85
85
  artifact_key = [r"[^\/\\]+$"]
86
+
87
+ # must not start with _
88
+ # must be alphanumeric or _
89
+ # max 256 length
90
+ v3io_stream_consumer_group = [r"^(?!_)[a-zA-Z0-9_]{1,256}$"]
@@ -1,4 +1,4 @@
1
1
  {
2
- "git_commit": "baa0b41494385443a085434b75b927c0700b6a49",
3
- "version": "1.4.0-rc25"
2
+ "git_commit": "dcf0e7215136557c4f4d1159a1b494defce16d40",
3
+ "version": "1.5.0-rc2"
4
4
  }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mlrun
3
- Version: 1.4.0rc25
3
+ Version: 1.5.0rc2
4
4
  Summary: Tracking and config of machine learning runs
5
5
  Home-page: https://github.com/mlrun/mlrun
6
6
  Author: Yaron Haviv
@@ -14,8 +14,8 @@ Classifier: Operating System :: POSIX :: Linux
14
14
  Classifier: Operating System :: Microsoft :: Windows
15
15
  Classifier: Operating System :: MacOS
16
16
  Classifier: Programming Language :: Python :: 3
17
- Classifier: Programming Language :: Python :: 3.7
18
17
  Classifier: Programming Language :: Python :: 3.8
18
+ Classifier: Programming Language :: Python :: 3.9
19
19
  Classifier: Programming Language :: Python
20
20
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
21
21
  Classifier: Topic :: Software Development :: Libraries
@@ -48,11 +48,12 @@ Requires-Dist: semver (~=2.13)
48
48
  Requires-Dist: dask (~=2021.11.2)
49
49
  Requires-Dist: distributed (~=2021.11.2)
50
50
  Requires-Dist: kubernetes (~=12.0)
51
+ Requires-Dist: dependency-injector (~=4.41)
51
52
  Requires-Dist: humanfriendly (~=9.2)
52
53
  Requires-Dist: fastapi (~=0.95.2)
53
- Requires-Dist: fsspec (~=2023.1.0)
54
+ Requires-Dist: fsspec (~=2023.6.0)
54
55
  Requires-Dist: v3iofs (~=0.1.15)
55
- Requires-Dist: storey (~=1.4.4)
56
+ Requires-Dist: storey (~=1.5.2)
56
57
  Requires-Dist: deepdiff (~=5.0)
57
58
  Requires-Dist: pymysql (~=1.0)
58
59
  Requires-Dist: inflection (~=0.5.0)
@@ -61,16 +62,16 @@ Requires-Dist: setuptools (~=65.5)
61
62
  Requires-Dist: deprecated (~=1.2)
62
63
  Requires-Dist: jinja2 (~=3.1)
63
64
  Provides-Extra: all
64
- Requires-Dist: adlfs (~=2022.2.0) ; extra == 'all'
65
- Requires-Dist: aiobotocore (~=2.4.2) ; extra == 'all'
65
+ Requires-Dist: adlfs (~=2023.4.0) ; extra == 'all'
66
+ Requires-Dist: aiobotocore (~=2.5.2) ; extra == 'all'
66
67
  Requires-Dist: avro (~=1.11) ; extra == 'all'
67
68
  Requires-Dist: azure-core (~=1.24) ; extra == 'all'
68
69
  Requires-Dist: azure-identity (~=1.5) ; extra == 'all'
69
70
  Requires-Dist: azure-keyvault-secrets (~=4.2) ; extra == 'all'
70
71
  Requires-Dist: azure-storage-blob (~=12.13) ; extra == 'all'
71
72
  Requires-Dist: bokeh (>=2.4.2,~=2.4) ; extra == 'all'
72
- Requires-Dist: boto3 (~=1.24.59) ; extra == 'all'
73
- Requires-Dist: gcsfs (~=2023.1.0) ; extra == 'all'
73
+ Requires-Dist: boto3 (~=1.26.161) ; extra == 'all'
74
+ Requires-Dist: gcsfs (~=2023.6.0) ; extra == 'all'
74
75
  Requires-Dist: google-cloud-bigquery[bqstorage,pandas] (~=3.2) ; extra == 'all'
75
76
  Requires-Dist: google-cloud-storage (~=1.20) ; extra == 'all'
76
77
  Requires-Dist: google-cloud (~=0.34) ; extra == 'all'
@@ -80,7 +81,7 @@ Requires-Dist: msrest (~=0.6.21) ; extra == 'all'
80
81
  Requires-Dist: plotly (<5.12.0,~=5.4) ; extra == 'all'
81
82
  Requires-Dist: pyopenssl (>=23) ; extra == 'all'
82
83
  Requires-Dist: redis (~=4.3) ; extra == 'all'
83
- Requires-Dist: s3fs (~=2023.1.0) ; extra == 'all'
84
+ Requires-Dist: s3fs (~=2023.6.0) ; extra == 'all'
84
85
  Provides-Extra: api
85
86
  Requires-Dist: uvicorn (~=0.20.0) ; extra == 'api'
86
87
  Requires-Dist: dask-kubernetes (~=0.11.0) ; extra == 'api'
@@ -92,7 +93,7 @@ Provides-Extra: azure-blob-storage
92
93
  Requires-Dist: msrest (~=0.6.21) ; extra == 'azure-blob-storage'
93
94
  Requires-Dist: azure-core (~=1.24) ; extra == 'azure-blob-storage'
94
95
  Requires-Dist: azure-storage-blob (~=12.13) ; extra == 'azure-blob-storage'
95
- Requires-Dist: adlfs (~=2022.2.0) ; extra == 'azure-blob-storage'
96
+ Requires-Dist: adlfs (~=2023.4.0) ; extra == 'azure-blob-storage'
96
97
  Requires-Dist: pyopenssl (>=23) ; extra == 'azure-blob-storage'
97
98
  Provides-Extra: azure-key-vault
98
99
  Requires-Dist: azure-identity (~=1.5) ; extra == 'azure-key-vault'
@@ -101,15 +102,15 @@ Requires-Dist: pyopenssl (>=23) ; extra == 'azure-key-vault'
101
102
  Provides-Extra: bokeh
102
103
  Requires-Dist: bokeh (>=2.4.2,~=2.4) ; extra == 'bokeh'
103
104
  Provides-Extra: complete
104
- Requires-Dist: adlfs (~=2022.2.0) ; extra == 'complete'
105
- Requires-Dist: aiobotocore (~=2.4.2) ; extra == 'complete'
105
+ Requires-Dist: adlfs (~=2023.4.0) ; extra == 'complete'
106
+ Requires-Dist: aiobotocore (~=2.5.2) ; extra == 'complete'
106
107
  Requires-Dist: avro (~=1.11) ; extra == 'complete'
107
108
  Requires-Dist: azure-core (~=1.24) ; extra == 'complete'
108
109
  Requires-Dist: azure-identity (~=1.5) ; extra == 'complete'
109
110
  Requires-Dist: azure-keyvault-secrets (~=4.2) ; extra == 'complete'
110
111
  Requires-Dist: azure-storage-blob (~=12.13) ; extra == 'complete'
111
- Requires-Dist: boto3 (~=1.24.59) ; extra == 'complete'
112
- Requires-Dist: gcsfs (~=2023.1.0) ; extra == 'complete'
112
+ Requires-Dist: boto3 (~=1.26.161) ; extra == 'complete'
113
+ Requires-Dist: gcsfs (~=2023.6.0) ; extra == 'complete'
113
114
  Requires-Dist: google-cloud-bigquery[bqstorage,pandas] (~=3.2) ; extra == 'complete'
114
115
  Requires-Dist: graphviz (~=0.20.0) ; extra == 'complete'
115
116
  Requires-Dist: kafka-python (~=2.0) ; extra == 'complete'
@@ -117,19 +118,19 @@ Requires-Dist: msrest (~=0.6.21) ; extra == 'complete'
117
118
  Requires-Dist: plotly (<5.12.0,~=5.4) ; extra == 'complete'
118
119
  Requires-Dist: pyopenssl (>=23) ; extra == 'complete'
119
120
  Requires-Dist: redis (~=4.3) ; extra == 'complete'
120
- Requires-Dist: s3fs (~=2023.1.0) ; extra == 'complete'
121
+ Requires-Dist: s3fs (~=2023.6.0) ; extra == 'complete'
121
122
  Provides-Extra: complete-api
122
- Requires-Dist: adlfs (~=2022.2.0) ; extra == 'complete-api'
123
- Requires-Dist: aiobotocore (~=2.4.2) ; extra == 'complete-api'
123
+ Requires-Dist: adlfs (~=2023.4.0) ; extra == 'complete-api'
124
+ Requires-Dist: aiobotocore (~=2.5.2) ; extra == 'complete-api'
124
125
  Requires-Dist: apscheduler (~=3.6) ; extra == 'complete-api'
125
126
  Requires-Dist: avro (~=1.11) ; extra == 'complete-api'
126
127
  Requires-Dist: azure-core (~=1.24) ; extra == 'complete-api'
127
128
  Requires-Dist: azure-identity (~=1.5) ; extra == 'complete-api'
128
129
  Requires-Dist: azure-keyvault-secrets (~=4.2) ; extra == 'complete-api'
129
130
  Requires-Dist: azure-storage-blob (~=12.13) ; extra == 'complete-api'
130
- Requires-Dist: boto3 (~=1.24.59) ; extra == 'complete-api'
131
+ Requires-Dist: boto3 (~=1.26.161) ; extra == 'complete-api'
131
132
  Requires-Dist: dask-kubernetes (~=0.11.0) ; extra == 'complete-api'
132
- Requires-Dist: gcsfs (~=2023.1.0) ; extra == 'complete-api'
133
+ Requires-Dist: gcsfs (~=2023.6.0) ; extra == 'complete-api'
133
134
  Requires-Dist: google-cloud-bigquery[bqstorage,pandas] (~=3.2) ; extra == 'complete-api'
134
135
  Requires-Dist: graphviz (~=0.20.0) ; extra == 'complete-api'
135
136
  Requires-Dist: igz-mgmt (~=0.0.10) ; extra == 'complete-api'
@@ -139,7 +140,7 @@ Requires-Dist: objgraph (~=3.5) ; extra == 'complete-api'
139
140
  Requires-Dist: plotly (<5.12.0,~=5.4) ; extra == 'complete-api'
140
141
  Requires-Dist: pyopenssl (>=23) ; extra == 'complete-api'
141
142
  Requires-Dist: redis (~=4.3) ; extra == 'complete-api'
142
- Requires-Dist: s3fs (~=2023.1.0) ; extra == 'complete-api'
143
+ Requires-Dist: s3fs (~=2023.6.0) ; extra == 'complete-api'
143
144
  Requires-Dist: sqlite3-to-mysql (~=1.4) ; extra == 'complete-api'
144
145
  Requires-Dist: uvicorn (~=0.20.0) ; extra == 'complete-api'
145
146
  Provides-Extra: google-cloud
@@ -149,7 +150,7 @@ Requires-Dist: google-cloud (~=0.34) ; extra == 'google-cloud'
149
150
  Provides-Extra: google-cloud-bigquery
150
151
  Requires-Dist: google-cloud-bigquery[bqstorage,pandas] (~=3.2) ; extra == 'google-cloud-bigquery'
151
152
  Provides-Extra: google-cloud-storage
152
- Requires-Dist: gcsfs (~=2023.1.0) ; extra == 'google-cloud-storage'
153
+ Requires-Dist: gcsfs (~=2023.6.0) ; extra == 'google-cloud-storage'
153
154
  Provides-Extra: graphviz
154
155
  Requires-Dist: graphviz (~=0.20.0) ; extra == 'graphviz'
155
156
  Provides-Extra: kafka
@@ -160,9 +161,9 @@ Requires-Dist: plotly (<5.12.0,~=5.4) ; extra == 'plotly'
160
161
  Provides-Extra: redis
161
162
  Requires-Dist: redis (~=4.3) ; extra == 'redis'
162
163
  Provides-Extra: s3
163
- Requires-Dist: boto3 (~=1.24.59) ; extra == 's3'
164
- Requires-Dist: aiobotocore (~=2.4.2) ; extra == 's3'
165
- Requires-Dist: s3fs (~=2023.1.0) ; extra == 's3'
164
+ Requires-Dist: boto3 (~=1.26.161) ; extra == 's3'
165
+ Requires-Dist: aiobotocore (~=2.5.2) ; extra == 's3'
166
+ Requires-Dist: s3fs (~=2023.6.0) ; extra == 's3'
166
167
 
167
168
  <a id="top"></a>
168
169
  [![Build Status](https://github.com/mlrun/mlrun/workflows/CI/badge.svg)](https://github.com/mlrun/mlrun/actions)
@@ -172,7 +173,7 @@ Requires-Dist: s3fs (~=2023.1.0) ; extra == 's3'
172
173
  [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
173
174
  ![GitHub commit activity](https://img.shields.io/github/commit-activity/w/mlrun/mlrun)
174
175
  ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/mlrun/mlrun?sort=semver)
175
- [![Join MLOps Live](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](mlopslive.slack.com)
176
+ [![Join MLOps Live](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://mlopslive.slack.com)
176
177
 
177
178
  <p align="left"><img src="docs/_static/images/MLRun-logo.png" alt="MLRun logo" width="150"/></p>
178
179