mlrun 1.7.0rc3__py3-none-any.whl → 1.7.0rc5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (76) hide show
  1. mlrun/artifacts/manager.py +6 -1
  2. mlrun/common/constants.py +2 -0
  3. mlrun/common/model_monitoring/helpers.py +12 -6
  4. mlrun/common/schemas/__init__.py +11 -0
  5. mlrun/common/schemas/api_gateway.py +85 -0
  6. mlrun/common/schemas/auth.py +2 -2
  7. mlrun/common/schemas/client_spec.py +1 -0
  8. mlrun/common/schemas/common.py +40 -0
  9. mlrun/common/schemas/model_monitoring/constants.py +4 -1
  10. mlrun/common/schemas/project.py +2 -0
  11. mlrun/config.py +31 -17
  12. mlrun/datastore/azure_blob.py +22 -9
  13. mlrun/datastore/base.py +15 -25
  14. mlrun/datastore/datastore.py +19 -8
  15. mlrun/datastore/datastore_profile.py +47 -5
  16. mlrun/datastore/google_cloud_storage.py +10 -6
  17. mlrun/datastore/hdfs.py +51 -0
  18. mlrun/datastore/redis.py +4 -0
  19. mlrun/datastore/s3.py +4 -0
  20. mlrun/datastore/sources.py +29 -43
  21. mlrun/datastore/targets.py +59 -53
  22. mlrun/datastore/utils.py +2 -49
  23. mlrun/datastore/v3io.py +4 -0
  24. mlrun/db/base.py +50 -0
  25. mlrun/db/httpdb.py +121 -50
  26. mlrun/db/nopdb.py +13 -0
  27. mlrun/execution.py +3 -3
  28. mlrun/feature_store/feature_vector.py +2 -2
  29. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +3 -3
  30. mlrun/frameworks/tf_keras/model_handler.py +7 -7
  31. mlrun/k8s_utils.py +10 -5
  32. mlrun/kfpops.py +19 -10
  33. mlrun/model.py +5 -0
  34. mlrun/model_monitoring/api.py +3 -3
  35. mlrun/model_monitoring/application.py +1 -1
  36. mlrun/model_monitoring/applications/__init__.py +13 -0
  37. mlrun/model_monitoring/applications/histogram_data_drift.py +218 -0
  38. mlrun/model_monitoring/batch.py +9 -111
  39. mlrun/model_monitoring/controller.py +73 -55
  40. mlrun/model_monitoring/controller_handler.py +13 -5
  41. mlrun/model_monitoring/features_drift_table.py +62 -53
  42. mlrun/model_monitoring/helpers.py +30 -21
  43. mlrun/model_monitoring/metrics/__init__.py +13 -0
  44. mlrun/model_monitoring/metrics/histogram_distance.py +127 -0
  45. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +14 -14
  46. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -1
  47. mlrun/package/packagers/pandas_packagers.py +3 -3
  48. mlrun/package/utils/_archiver.py +3 -1
  49. mlrun/platforms/iguazio.py +8 -65
  50. mlrun/projects/pipelines.py +21 -11
  51. mlrun/projects/project.py +180 -42
  52. mlrun/run.py +1 -1
  53. mlrun/runtimes/base.py +25 -2
  54. mlrun/runtimes/kubejob.py +5 -3
  55. mlrun/runtimes/local.py +2 -2
  56. mlrun/runtimes/mpijob/abstract.py +6 -6
  57. mlrun/runtimes/nuclio/__init__.py +1 -0
  58. mlrun/runtimes/nuclio/api_gateway.py +300 -0
  59. mlrun/runtimes/nuclio/function.py +9 -9
  60. mlrun/runtimes/nuclio/serving.py +3 -3
  61. mlrun/runtimes/pod.py +3 -3
  62. mlrun/runtimes/sparkjob/spark3job.py +3 -3
  63. mlrun/serving/remote.py +4 -2
  64. mlrun/serving/server.py +2 -8
  65. mlrun/utils/async_http.py +3 -3
  66. mlrun/utils/helpers.py +27 -5
  67. mlrun/utils/http.py +3 -3
  68. mlrun/utils/logger.py +2 -2
  69. mlrun/utils/notifications/notification_pusher.py +6 -6
  70. mlrun/utils/version/version.json +2 -2
  71. {mlrun-1.7.0rc3.dist-info → mlrun-1.7.0rc5.dist-info}/METADATA +13 -16
  72. {mlrun-1.7.0rc3.dist-info → mlrun-1.7.0rc5.dist-info}/RECORD +76 -68
  73. {mlrun-1.7.0rc3.dist-info → mlrun-1.7.0rc5.dist-info}/WHEEL +1 -1
  74. {mlrun-1.7.0rc3.dist-info → mlrun-1.7.0rc5.dist-info}/LICENSE +0 -0
  75. {mlrun-1.7.0rc3.dist-info → mlrun-1.7.0rc5.dist-info}/entry_points.txt +0 -0
  76. {mlrun-1.7.0rc3.dist-info → mlrun-1.7.0rc5.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,300 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ import base64
15
+ from typing import Optional, Union
16
+ from urllib.parse import urljoin
17
+
18
+ import requests
19
+
20
+ import mlrun
21
+ import mlrun.common.schemas
22
+
23
+ from .function import RemoteRuntime
24
+ from .serving import ServingRuntime
25
+
26
+ NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH = "basicAuth"
27
+ NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_NONE = "none"
28
+ PROJECT_NAME_LABEL = "nuclio.io/project-name"
29
+
30
+
31
+ class APIGateway:
32
+ def __init__(
33
+ self,
34
+ project,
35
+ name: str,
36
+ functions: Union[
37
+ list[str],
38
+ Union[
39
+ list[
40
+ Union[
41
+ RemoteRuntime,
42
+ ServingRuntime,
43
+ ]
44
+ ],
45
+ Union[RemoteRuntime, ServingRuntime],
46
+ ],
47
+ ],
48
+ description: str = "",
49
+ path: str = "/",
50
+ authentication_mode: Optional[
51
+ str
52
+ ] = NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_NONE,
53
+ host: Optional[str] = None,
54
+ canary: Optional[list[int]] = None,
55
+ username: Optional[str] = None,
56
+ password: Optional[str] = None,
57
+ ):
58
+ self.functions = None
59
+ self._validate(
60
+ project=project,
61
+ functions=functions,
62
+ name=name,
63
+ canary=canary,
64
+ username=username,
65
+ password=password,
66
+ )
67
+ self.project = project
68
+ self.name = name
69
+ self.host = host
70
+
71
+ self.path = path
72
+ self.description = description
73
+ self.authentication_mode = (
74
+ authentication_mode
75
+ if authentication_mode
76
+ else self._enrich_authentication_mode(username=username, password=password)
77
+ )
78
+ self.canary = canary
79
+ self._username = username
80
+ self._password = password
81
+
82
+ def invoke(
83
+ self,
84
+ method="POST",
85
+ headers: dict = {},
86
+ auth: Optional[tuple[str, str]] = None,
87
+ **kwargs,
88
+ ):
89
+ if not self.invoke_url:
90
+ raise mlrun.errors.MLRunInvalidArgumentError(
91
+ "Invocation url is not set. Set up gateway's `invoke_url` attribute."
92
+ )
93
+ if (
94
+ self.authentication_mode
95
+ == NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
96
+ and not auth
97
+ ):
98
+ raise mlrun.errors.MLRunInvalidArgumentError(
99
+ "API Gateway invocation requires authentication. Please pass credentials"
100
+ )
101
+ if auth:
102
+ headers["Authorization"] = self._generate_basic_auth(*auth)
103
+ return requests.request(
104
+ method=method, url=self.invoke_url, headers=headers, **kwargs
105
+ )
106
+
107
+ @classmethod
108
+ def from_scheme(cls, api_gateway: mlrun.common.schemas.APIGateway):
109
+ project = api_gateway.metadata.labels.get(PROJECT_NAME_LABEL)
110
+ functions, canary = cls._resolve_canary(api_gateway.spec.upstreams)
111
+ return cls(
112
+ project=project,
113
+ description=api_gateway.spec.description,
114
+ name=api_gateway.spec.name,
115
+ host=api_gateway.spec.host,
116
+ path=api_gateway.spec.path,
117
+ authentication_mode=str(api_gateway.spec.authenticationMode),
118
+ functions=functions,
119
+ canary=canary,
120
+ )
121
+
122
+ def to_scheme(self) -> mlrun.common.schemas.APIGateway:
123
+ upstreams = (
124
+ [
125
+ mlrun.common.schemas.APIGatewayUpstream(
126
+ nucliofunction={"name": function_name},
127
+ percentage=percentage,
128
+ )
129
+ for function_name, percentage in zip(self.functions, self.canary)
130
+ ]
131
+ if self.canary
132
+ else [
133
+ mlrun.common.schemas.APIGatewayUpstream(
134
+ nucliofunction={"name": function_name},
135
+ )
136
+ for function_name in self.functions
137
+ ]
138
+ )
139
+ api_gateway = mlrun.common.schemas.APIGateway(
140
+ metadata=mlrun.common.schemas.APIGatewayMetadata(name=self.name, labels={}),
141
+ spec=mlrun.common.schemas.APIGatewaySpec(
142
+ name=self.name,
143
+ description=self.description,
144
+ path=self.path,
145
+ authentication_mode=mlrun.common.schemas.APIGatewayAuthenticationMode.from_str(
146
+ self.authentication_mode
147
+ ),
148
+ upstreams=upstreams,
149
+ ),
150
+ )
151
+ if (
152
+ self.authentication_mode
153
+ is NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
154
+ ):
155
+ api_gateway.spec.authentication = mlrun.common.schemas.APIGatewayBasicAuth(
156
+ username=self._username, password=self._password
157
+ )
158
+ return api_gateway
159
+
160
+ @property
161
+ def invoke_url(
162
+ self,
163
+ ):
164
+ return urljoin(self.host, self.path)
165
+
166
+ def _validate(
167
+ self,
168
+ name: str,
169
+ project: str,
170
+ functions: Union[
171
+ list[str],
172
+ Union[
173
+ list[
174
+ Union[
175
+ RemoteRuntime,
176
+ ServingRuntime,
177
+ ]
178
+ ],
179
+ Union[RemoteRuntime, ServingRuntime],
180
+ ],
181
+ ],
182
+ canary: Optional[list[int]] = None,
183
+ username: Optional[str] = None,
184
+ password: Optional[str] = None,
185
+ ):
186
+ if not name:
187
+ raise mlrun.errors.MLRunInvalidArgumentError(
188
+ "API Gateway name cannot be empty"
189
+ )
190
+
191
+ self.functions = self._validate_functions(project=project, functions=functions)
192
+
193
+ # validating canary
194
+ if canary:
195
+ if len(self.functions) != len(canary):
196
+ raise mlrun.errors.MLRunInvalidArgumentError(
197
+ "Function and canary lists lengths do not match"
198
+ )
199
+ for canary_percent in canary:
200
+ if canary_percent < 0 or canary_percent > 100:
201
+ raise mlrun.errors.MLRunInvalidArgumentError(
202
+ "The percentage value must be in the range from 0 to 100"
203
+ )
204
+ if sum(canary) != 100:
205
+ raise mlrun.errors.MLRunInvalidArgumentError(
206
+ "The sum of canary function percents should be equal to 100"
207
+ )
208
+
209
+ # validating auth
210
+ if username and not password:
211
+ raise mlrun.errors.MLRunInvalidArgumentError("Password is not specified")
212
+
213
+ if password and not username:
214
+ raise mlrun.errors.MLRunInvalidArgumentError("Username is not specified")
215
+
216
+ @staticmethod
217
+ def _validate_functions(
218
+ project: str,
219
+ functions: Union[
220
+ list[str],
221
+ Union[
222
+ list[
223
+ Union[
224
+ RemoteRuntime,
225
+ ServingRuntime,
226
+ ]
227
+ ],
228
+ Union[RemoteRuntime, ServingRuntime],
229
+ ],
230
+ ],
231
+ ):
232
+ if not isinstance(functions, list):
233
+ functions = [functions]
234
+
235
+ # validating functions
236
+ if not 1 <= len(functions) <= 2:
237
+ raise mlrun.errors.MLRunInvalidArgumentError(
238
+ f"Gateway can be created from one or two functions, "
239
+ f"the number of functions passed is {len(functions)}"
240
+ )
241
+
242
+ function_names = []
243
+ for func in functions:
244
+ if isinstance(func, str):
245
+ function_names.append(func)
246
+ continue
247
+
248
+ function_name = (
249
+ func.metadata.name if hasattr(func, "metadata") else func.name
250
+ )
251
+ if func.kind not in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
252
+ raise mlrun.errors.MLRunInvalidArgumentError(
253
+ f"Input function {function_name} is not a Nuclio function"
254
+ )
255
+ if func.metadata.project != project:
256
+ raise mlrun.errors.MLRunInvalidArgumentError(
257
+ f"input function {function_name} "
258
+ f"does not belong to this project"
259
+ )
260
+ function_names.append(func.uri)
261
+ return function_names
262
+
263
+ @staticmethod
264
+ def _enrich_authentication_mode(username, password):
265
+ return (
266
+ NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_NONE
267
+ if username is not None and password is not None
268
+ else NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
269
+ )
270
+
271
+ @staticmethod
272
+ def _generate_basic_auth(username: str, password: str):
273
+ token = base64.b64encode(f"{username}:{password}".encode()).decode()
274
+ return f"Basic {token}"
275
+
276
+ @staticmethod
277
+ def _resolve_canary(
278
+ upstreams: list[mlrun.common.schemas.APIGatewayUpstream],
279
+ ) -> tuple[Union[list[str], None], Union[list[int], None]]:
280
+ if len(upstreams) == 1:
281
+ return [upstreams[0].nucliofunction.get("name")], None
282
+ elif len(upstreams) == 2:
283
+ canary = [0, 0]
284
+ functions = [
285
+ upstreams[0].nucliofunction.get("name"),
286
+ upstreams[1].nucliofunction.get("name"),
287
+ ]
288
+ percentage_1 = upstreams[0].percentage
289
+ percentage_2 = upstreams[1].percentage
290
+
291
+ if not percentage_1 and percentage_2:
292
+ percentage_1 = 100 - percentage_2
293
+ if not percentage_2 and percentage_1:
294
+ percentage_2 = 100 - percentage_1
295
+ if percentage_1 and percentage_2:
296
+ canary = [percentage_1, percentage_2]
297
+ return functions, canary
298
+ else:
299
+ # Nuclio only supports 1 or 2 upstream functions
300
+ return None, None
@@ -431,15 +431,15 @@ class RemoteRuntime(KubeResource):
431
431
  raise ValueError(
432
432
  "gateway timeout must be greater than the worker timeout"
433
433
  )
434
- annotations[
435
- "nginx.ingress.kubernetes.io/proxy-connect-timeout"
436
- ] = f"{gateway_timeout}"
437
- annotations[
438
- "nginx.ingress.kubernetes.io/proxy-read-timeout"
439
- ] = f"{gateway_timeout}"
440
- annotations[
441
- "nginx.ingress.kubernetes.io/proxy-send-timeout"
442
- ] = f"{gateway_timeout}"
434
+ annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = (
435
+ f"{gateway_timeout}"
436
+ )
437
+ annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = (
438
+ f"{gateway_timeout}"
439
+ )
440
+ annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = (
441
+ f"{gateway_timeout}"
442
+ )
443
443
 
444
444
  trigger = nuclio.HttpTrigger(
445
445
  workers=workers,
@@ -523,9 +523,9 @@ class ServingRuntime(RemoteRuntime):
523
523
  function_object.metadata.tag = self.metadata.tag
524
524
 
525
525
  function_object.metadata.labels = function_object.metadata.labels or {}
526
- function_object.metadata.labels[
527
- "mlrun/parent-function"
528
- ] = self.metadata.name
526
+ function_object.metadata.labels["mlrun/parent-function"] = (
527
+ self.metadata.name
528
+ )
529
529
  function_object._is_child_function = True
530
530
  if not function_object.spec.graph:
531
531
  # copy the current graph only if the child doesnt have a graph of his own
mlrun/runtimes/pod.py CHANGED
@@ -501,9 +501,9 @@ class KubeResourceSpec(FunctionSpec):
501
501
  )
502
502
  is None
503
503
  ):
504
- resources[resource_requirement][
505
- resource_type
506
- ] = default_resources[resource_requirement][resource_type]
504
+ resources[resource_requirement][resource_type] = (
505
+ default_resources[resource_requirement][resource_type]
506
+ )
507
507
  # This enables the user to define that no defaults would be applied on the resources
508
508
  elif resources == {}:
509
509
  return resources
@@ -368,9 +368,9 @@ class Spark3JobSpec(KubeResourceSpec):
368
368
  )
369
369
  is None
370
370
  ):
371
- resources[resource_requirement][
372
- resource_type
373
- ] = default_resources[resource_requirement][resource_type]
371
+ resources[resource_requirement][resource_type] = (
372
+ default_resources[resource_requirement][resource_type]
373
+ )
374
374
  else:
375
375
  resources = default_resources
376
376
 
mlrun/serving/remote.py CHANGED
@@ -21,6 +21,7 @@ import storey
21
21
  from storey.flow import _ConcurrentJobExecution
22
22
 
23
23
  import mlrun
24
+ import mlrun.config
24
25
  from mlrun.errors import err_to_str
25
26
  from mlrun.utils import logger
26
27
 
@@ -171,7 +172,8 @@ class RemoteStep(storey.SendToHttp):
171
172
  if not self._session:
172
173
  self._session = mlrun.utils.HTTPSessionWithRetry(
173
174
  self.retries,
174
- self.backoff_factor or mlrun.mlconf.http_retry_defaults.backoff_factor,
175
+ self.backoff_factor
176
+ or mlrun.config.config.http_retry_defaults.backoff_factor,
175
177
  retry_on_exception=False,
176
178
  retry_on_status=self.retries > 0,
177
179
  retry_on_post=True,
@@ -183,7 +185,7 @@ class RemoteStep(storey.SendToHttp):
183
185
  resp = self._session.request(
184
186
  method,
185
187
  url,
186
- verify=False,
188
+ verify=mlrun.config.config.httpdb.http.verify,
187
189
  headers=headers,
188
190
  data=body,
189
191
  timeout=self.timeout,
mlrun/serving/server.py CHANGED
@@ -188,7 +188,6 @@ class GraphServer(ModelObj):
188
188
 
189
189
  def init_object(self, namespace):
190
190
  self.graph.init_object(self.context, namespace, self.load_mode, reset=True)
191
- return v2_serving_handler if self.context.is_mock else v2_serving_async_handler
192
191
 
193
192
  def test(
194
193
  self,
@@ -339,9 +338,9 @@ def v2_serving_init(context, namespace=None):
339
338
  **kwargs,
340
339
  )
341
340
  context.logger.info("Initializing graph steps")
342
- serving_handler = server.init_object(namespace or get_caller_globals())
341
+ server.init_object(namespace or get_caller_globals())
343
342
  # set the handler hook to point to our handler
344
- setattr(context, "mlrun_handler", serving_handler)
343
+ setattr(context, "mlrun_handler", v2_serving_handler)
345
344
  setattr(context, "_server", server)
346
345
  context.logger.info_with("Serving was initialized", verbose=server.verbose)
347
346
  if server.verbose:
@@ -389,11 +388,6 @@ def v2_serving_handler(context, event, get_body=False):
389
388
  return context._server.run(event, context, get_body)
390
389
 
391
390
 
392
- async def v2_serving_async_handler(context, event, get_body=False):
393
- """hook for nuclio handler()"""
394
- return await context._server.run(event, context, get_body)
395
-
396
-
397
391
  def create_graph_server(
398
392
  parameters={},
399
393
  load_mode=None,
mlrun/utils/async_http.py CHANGED
@@ -137,9 +137,9 @@ class _CustomRequestContext(_RequestContext):
137
137
 
138
138
  # enrich user agent
139
139
  # will help traceability and debugging
140
- headers[
141
- aiohttp.hdrs.USER_AGENT
142
- ] = f"{aiohttp.http.SERVER_SOFTWARE} mlrun/{config.version}"
140
+ headers[aiohttp.hdrs.USER_AGENT] = (
141
+ f"{aiohttp.http.SERVER_SOFTWARE} mlrun/{config.version}"
142
+ )
143
143
 
144
144
  response: typing.Optional[
145
145
  aiohttp.ClientResponse
mlrun/utils/helpers.py CHANGED
@@ -12,6 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
+ import asyncio
15
16
  import enum
16
17
  import functools
17
18
  import hashlib
@@ -30,7 +31,6 @@ from os import path
30
31
  from types import ModuleType
31
32
  from typing import Any, Optional
32
33
 
33
- import anyio
34
34
  import git
35
35
  import inflection
36
36
  import numpy as np
@@ -49,6 +49,7 @@ import mlrun.common.schemas
49
49
  import mlrun.errors
50
50
  import mlrun.utils.regex
51
51
  import mlrun.utils.version.version
52
+ from mlrun.common.constants import MYSQL_MEDIUMBLOB_SIZE_BYTES
52
53
  from mlrun.config import config
53
54
 
54
55
  from .logger import create_logger
@@ -270,6 +271,17 @@ def validate_artifact_key_name(
270
271
  )
271
272
 
272
273
 
274
+ def validate_inline_artifact_body_size(body: typing.Union[str, bytes, None]) -> None:
275
+ if body and len(body) > MYSQL_MEDIUMBLOB_SIZE_BYTES:
276
+ raise mlrun.errors.MLRunBadRequestError(
277
+ "The body of the artifact exceeds the maximum allowed size. "
278
+ "Avoid embedding the artifact body. "
279
+ "This increases the size of the project yaml file and could affect the project during loading and saving. "
280
+ "More information is available at"
281
+ "https://docs.mlrun.org/en/latest/projects/automate-project-git-source.html#setting-and-registering-the-project-artifacts"
282
+ )
283
+
284
+
273
285
  def validate_v3io_stream_consumer_group(
274
286
  value: str, raise_on_failure: bool = True
275
287
  ) -> bool:
@@ -1464,13 +1476,15 @@ def normalize_project_username(username: str):
1464
1476
  return username
1465
1477
 
1466
1478
 
1467
- # run_in threadpool is taken from fastapi to allow us to run sync functions in a threadpool
1468
- # without importing fastapi in the client
1469
1479
  async def run_in_threadpool(func, *args, **kwargs):
1480
+ """
1481
+ Run a sync-function in the loop default thread pool executor pool and await its result.
1482
+ Note that this function is not suitable for CPU-bound tasks, as it will block the event loop.
1483
+ """
1484
+ loop = asyncio.get_running_loop()
1470
1485
  if kwargs:
1471
- # run_sync doesn't accept 'kwargs', so bind them in here
1472
1486
  func = functools.partial(func, **kwargs)
1473
- return await anyio.to_thread.run_sync(func, *args)
1487
+ return await loop.run_in_executor(None, func, *args)
1474
1488
 
1475
1489
 
1476
1490
  def is_explicit_ack_supported(context):
@@ -1540,3 +1554,11 @@ def get_local_file_schema() -> list:
1540
1554
  # The expression `list(string.ascii_lowercase)` generates a list of lowercase alphabets,
1541
1555
  # which corresponds to drive letters in Windows file paths such as `C:/Windows/path`.
1542
1556
  return ["file"] + list(string.ascii_lowercase)
1557
+
1558
+
1559
+ def is_safe_path(base, filepath, is_symlink=False):
1560
+ # Avoid path traversal attacks by ensuring that the path is safe
1561
+ resolved_filepath = (
1562
+ os.path.abspath(filepath) if not is_symlink else os.path.realpath(filepath)
1563
+ )
1564
+ return base == os.path.commonpath((base, resolved_filepath))
mlrun/utils/http.py CHANGED
@@ -109,9 +109,9 @@ class HTTPSessionWithRetry(requests.Session):
109
109
  def request(self, method, url, **kwargs):
110
110
  retry_count = 0
111
111
  kwargs.setdefault("headers", {})
112
- kwargs["headers"][
113
- "User-Agent"
114
- ] = f"{requests.utils.default_user_agent()} mlrun/{config.version}"
112
+ kwargs["headers"]["User-Agent"] = (
113
+ f"{requests.utils.default_user_agent()} mlrun/{config.version}"
114
+ )
115
115
  while True:
116
116
  try:
117
117
  response = super().request(method, url, **kwargs)
mlrun/utils/logger.py CHANGED
@@ -221,7 +221,7 @@ class FormatterKinds(Enum):
221
221
  JSON = "json"
222
222
 
223
223
 
224
- def _create_formatter_instance(formatter_kind: FormatterKinds) -> logging.Formatter:
224
+ def create_formatter_instance(formatter_kind: FormatterKinds) -> logging.Formatter:
225
225
  return {
226
226
  FormatterKinds.HUMAN: HumanReadableFormatter(),
227
227
  FormatterKinds.HUMAN_EXTENDED: HumanReadableExtendedFormatter(),
@@ -243,7 +243,7 @@ def create_logger(
243
243
  logger_instance = Logger(level, name=name, propagate=False)
244
244
 
245
245
  # resolve formatter
246
- formatter_instance = _create_formatter_instance(
246
+ formatter_instance = create_formatter_instance(
247
247
  FormatterKinds(formatter_kind.lower())
248
248
  )
249
249
 
@@ -303,9 +303,9 @@ class NotificationPusher(_NotificationPusherBase):
303
303
  traceback=traceback.format_exc(),
304
304
  )
305
305
  update_notification_status_kwargs["reason"] = f"Exception error: {str(exc)}"
306
- update_notification_status_kwargs[
307
- "status"
308
- ] = mlrun.common.schemas.NotificationStatus.ERROR
306
+ update_notification_status_kwargs["status"] = (
307
+ mlrun.common.schemas.NotificationStatus.ERROR
308
+ )
309
309
  raise exc
310
310
  finally:
311
311
  self._update_notification_status(
@@ -352,9 +352,9 @@ class NotificationPusher(_NotificationPusherBase):
352
352
  traceback=traceback.format_exc(),
353
353
  )
354
354
  update_notification_status_kwargs["reason"] = f"Exception error: {str(exc)}"
355
- update_notification_status_kwargs[
356
- "status"
357
- ] = mlrun.common.schemas.NotificationStatus.ERROR
355
+ update_notification_status_kwargs["status"] = (
356
+ mlrun.common.schemas.NotificationStatus.ERROR
357
+ )
358
358
  raise exc
359
359
  finally:
360
360
  await mlrun.utils.helpers.run_in_threadpool(
@@ -1,4 +1,4 @@
1
1
  {
2
- "git_commit": "21f65b697536706b30bf9525d2aecc673b38a4eb",
3
- "version": "1.7.0-rc3"
2
+ "git_commit": "d3324e482f4a4182ee0c8eda4af0d312718b599d",
3
+ "version": "1.7.0-rc5"
4
4
  }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mlrun
3
- Version: 1.7.0rc3
3
+ Version: 1.7.0rc5
4
4
  Summary: Tracking and config of machine learning runs
5
5
  Home-page: https://github.com/mlrun/mlrun
6
6
  Author: Yaron Haviv
@@ -44,13 +44,12 @@ Requires-Dist: semver ~=3.0
44
44
  Requires-Dist: dependency-injector ~=4.41
45
45
  Requires-Dist: fsspec ==2023.9.2
46
46
  Requires-Dist: v3iofs ~=0.1.17
47
- Requires-Dist: storey ~=1.7.4
47
+ Requires-Dist: storey ~=1.7.5
48
48
  Requires-Dist: inflection ~=0.5.0
49
49
  Requires-Dist: python-dotenv ~=0.17.0
50
- Requires-Dist: setuptools ~=68.2
50
+ Requires-Dist: setuptools ~=69.1
51
51
  Requires-Dist: deprecated ~=1.2
52
52
  Requires-Dist: jinja2 >=3.1.3,~=3.1
53
- Requires-Dist: anyio ~=3.7
54
53
  Requires-Dist: orjson ~=3.9
55
54
  Provides-Extra: all
56
55
  Requires-Dist: adlfs ==2023.9.0 ; extra == 'all'
@@ -80,12 +79,11 @@ Requires-Dist: sqlalchemy ~=1.4 ; extra == 'all'
80
79
  Provides-Extra: api
81
80
  Requires-Dist: uvicorn ~=0.27.1 ; extra == 'api'
82
81
  Requires-Dist: dask-kubernetes ~=0.11.0 ; extra == 'api'
83
- Requires-Dist: apscheduler !=3.10.2,~=3.6 ; extra == 'api'
84
- Requires-Dist: sqlite3-to-mysql ~=1.4 ; extra == 'api'
85
- Requires-Dist: objgraph ~=3.5 ; extra == 'api'
86
- Requires-Dist: igz-mgmt ~=0.0.10 ; extra == 'api'
87
- Requires-Dist: humanfriendly ~=9.2 ; extra == 'api'
88
- Requires-Dist: fastapi ~=0.103.2 ; extra == 'api'
82
+ Requires-Dist: apscheduler <4,>=3.10.3 ; extra == 'api'
83
+ Requires-Dist: objgraph ~=3.6 ; extra == 'api'
84
+ Requires-Dist: igz-mgmt ~=0.1.0 ; extra == 'api'
85
+ Requires-Dist: humanfriendly ~=10.0 ; extra == 'api'
86
+ Requires-Dist: fastapi ~=0.110.0 ; extra == 'api'
89
87
  Requires-Dist: sqlalchemy ~=1.4 ; extra == 'api'
90
88
  Requires-Dist: pymysql ~=1.0 ; extra == 'api'
91
89
  Requires-Dist: alembic ~=1.9 ; extra == 'api'
@@ -127,7 +125,7 @@ Provides-Extra: complete-api
127
125
  Requires-Dist: adlfs ==2023.9.0 ; extra == 'complete-api'
128
126
  Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 'complete-api'
129
127
  Requires-Dist: alembic ~=1.9 ; extra == 'complete-api'
130
- Requires-Dist: apscheduler !=3.10.2,~=3.6 ; extra == 'complete-api'
128
+ Requires-Dist: apscheduler <4,>=3.10.3 ; extra == 'complete-api'
131
129
  Requires-Dist: avro ~=1.11 ; extra == 'complete-api'
132
130
  Requires-Dist: azure-core ~=1.24 ; extra == 'complete-api'
133
131
  Requires-Dist: azure-identity ~=1.5 ; extra == 'complete-api'
@@ -137,23 +135,22 @@ Requires-Dist: dask-kubernetes ~=0.11.0 ; extra == 'complete-api'
137
135
  Requires-Dist: dask ~=2023.9.0 ; extra == 'complete-api'
138
136
  Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'complete-api'
139
137
  Requires-Dist: distributed ~=2023.9.0 ; extra == 'complete-api'
140
- Requires-Dist: fastapi ~=0.103.2 ; extra == 'complete-api'
138
+ Requires-Dist: fastapi ~=0.110.0 ; extra == 'complete-api'
141
139
  Requires-Dist: gcsfs ==2023.9.2 ; extra == 'complete-api'
142
140
  Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'complete-api'
143
141
  Requires-Dist: graphviz ~=0.20.0 ; extra == 'complete-api'
144
- Requires-Dist: humanfriendly ~=9.2 ; extra == 'complete-api'
145
- Requires-Dist: igz-mgmt ~=0.0.10 ; extra == 'complete-api'
142
+ Requires-Dist: humanfriendly ~=10.0 ; extra == 'complete-api'
143
+ Requires-Dist: igz-mgmt ~=0.1.0 ; extra == 'complete-api'
146
144
  Requires-Dist: kafka-python ~=2.0 ; extra == 'complete-api'
147
145
  Requires-Dist: mlflow ~=2.8 ; extra == 'complete-api'
148
146
  Requires-Dist: msrest ~=0.6.21 ; extra == 'complete-api'
149
- Requires-Dist: objgraph ~=3.5 ; extra == 'complete-api'
147
+ Requires-Dist: objgraph ~=3.6 ; extra == 'complete-api'
150
148
  Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete-api'
151
149
  Requires-Dist: pymysql ~=1.0 ; extra == 'complete-api'
152
150
  Requires-Dist: pyopenssl >=23 ; extra == 'complete-api'
153
151
  Requires-Dist: redis ~=4.3 ; extra == 'complete-api'
154
152
  Requires-Dist: s3fs ==2023.9.2 ; extra == 'complete-api'
155
153
  Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
156
- Requires-Dist: sqlite3-to-mysql ~=1.4 ; extra == 'complete-api'
157
154
  Requires-Dist: timelength ~=1.1 ; extra == 'complete-api'
158
155
  Requires-Dist: uvicorn ~=0.27.1 ; extra == 'complete-api'
159
156
  Provides-Extra: dask