mlrun 1.7.0rc41__py3-none-any.whl → 1.7.0rc43__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/artifacts/manager.py +6 -1
- mlrun/common/db/sql_session.py +3 -2
- mlrun/common/schemas/__init__.py +1 -0
- mlrun/common/schemas/api_gateway.py +6 -6
- mlrun/common/schemas/common.py +4 -4
- mlrun/common/schemas/frontend_spec.py +7 -0
- mlrun/common/schemas/notification.py +32 -5
- mlrun/config.py +19 -1
- mlrun/data_types/to_pandas.py +3 -3
- mlrun/datastore/base.py +0 -3
- mlrun/datastore/storeytargets.py +2 -1
- mlrun/datastore/targets.py +17 -4
- mlrun/errors.py +7 -4
- mlrun/execution.py +4 -1
- mlrun/feature_store/feature_vector.py +3 -1
- mlrun/feature_store/retrieval/job.py +3 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +13 -3
- mlrun/k8s_utils.py +48 -2
- mlrun/model.py +3 -2
- mlrun/model_monitoring/db/stores/__init__.py +3 -3
- mlrun/model_monitoring/db/tsdb/__init__.py +3 -3
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +7 -7
- mlrun/model_monitoring/helpers.py +0 -7
- mlrun/model_monitoring/writer.py +5 -1
- mlrun/package/packagers/default_packager.py +2 -2
- mlrun/projects/project.py +125 -47
- mlrun/runtimes/funcdoc.py +1 -1
- mlrun/runtimes/local.py +4 -1
- mlrun/runtimes/nuclio/application/application.py +3 -2
- mlrun/runtimes/pod.py +2 -0
- mlrun/runtimes/sparkjob/spark3job.py +5 -1
- mlrun/utils/async_http.py +1 -1
- mlrun/utils/helpers.py +17 -0
- mlrun/utils/notifications/notification/__init__.py +0 -1
- mlrun/utils/v3io_clients.py +2 -2
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/METADATA +10 -10
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/RECORD +42 -42
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/WHEEL +0 -0
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/top_level.txt +0 -0
|
@@ -265,13 +265,6 @@ def calculate_inputs_statistics(
|
|
|
265
265
|
counts.tolist(),
|
|
266
266
|
bins.tolist(),
|
|
267
267
|
]
|
|
268
|
-
elif "hist" in inputs_statistics[feature]:
|
|
269
|
-
# Comply with the other common features' histogram length
|
|
270
|
-
mlrun.common.model_monitoring.helpers.pad_hist(
|
|
271
|
-
mlrun.common.model_monitoring.helpers.Histogram(
|
|
272
|
-
inputs_statistics[feature]["hist"]
|
|
273
|
-
)
|
|
274
|
-
)
|
|
275
268
|
else:
|
|
276
269
|
# If the feature is not in the sample set and doesn't have a histogram, remove it from the statistics:
|
|
277
270
|
inputs_statistics.pop(feature)
|
mlrun/model_monitoring/writer.py
CHANGED
|
@@ -258,9 +258,13 @@ class ModelMonitoringWriter(StepToDict):
|
|
|
258
258
|
"data drift app",
|
|
259
259
|
endpoint_id=endpoint_id,
|
|
260
260
|
)
|
|
261
|
+
attributes = json.loads(event[ResultData.RESULT_EXTRA_DATA])
|
|
262
|
+
attributes[EventFieldType.DRIFT_STATUS] = str(
|
|
263
|
+
attributes[EventFieldType.DRIFT_STATUS]
|
|
264
|
+
)
|
|
261
265
|
self._app_result_store.update_model_endpoint(
|
|
262
266
|
endpoint_id=endpoint_id,
|
|
263
|
-
attributes=
|
|
267
|
+
attributes=attributes,
|
|
264
268
|
)
|
|
265
269
|
|
|
266
270
|
logger.info("Model monitoring writer finished handling event")
|
|
@@ -34,7 +34,7 @@ class _DefaultPackagerMeta(ABCMeta):
|
|
|
34
34
|
dynamically generated docstring that will include a summary of the packager.
|
|
35
35
|
"""
|
|
36
36
|
|
|
37
|
-
def __new__(
|
|
37
|
+
def __new__(cls, name: str, bases: tuple, namespace: dict, **kwargs):
|
|
38
38
|
"""
|
|
39
39
|
Create a new DefaultPackager metaclass that saves the original packager docstring to another attribute named
|
|
40
40
|
`_packager_doc`.
|
|
@@ -48,7 +48,7 @@ class _DefaultPackagerMeta(ABCMeta):
|
|
|
48
48
|
namespace["_packager_doc"] = namespace.get("__doc__", "")
|
|
49
49
|
|
|
50
50
|
# Continue creating the metaclass:
|
|
51
|
-
return super().__new__(
|
|
51
|
+
return super().__new__(cls, name, bases, namespace, **kwargs)
|
|
52
52
|
|
|
53
53
|
@property
|
|
54
54
|
def __doc__(cls: type["DefaultPackager"]) -> str:
|
mlrun/projects/project.py
CHANGED
|
@@ -18,6 +18,7 @@ import glob
|
|
|
18
18
|
import http
|
|
19
19
|
import importlib.util as imputil
|
|
20
20
|
import json
|
|
21
|
+
import os
|
|
21
22
|
import pathlib
|
|
22
23
|
import shutil
|
|
23
24
|
import tempfile
|
|
@@ -25,6 +26,7 @@ import typing
|
|
|
25
26
|
import uuid
|
|
26
27
|
import warnings
|
|
27
28
|
import zipfile
|
|
29
|
+
from copy import deepcopy
|
|
28
30
|
from os import environ, makedirs, path
|
|
29
31
|
from typing import Callable, Optional, Union
|
|
30
32
|
|
|
@@ -251,8 +253,7 @@ def new_project(
|
|
|
251
253
|
project.spec.description = description
|
|
252
254
|
|
|
253
255
|
if default_function_node_selector:
|
|
254
|
-
|
|
255
|
-
project.spec.default_function_node_selector[key] = val
|
|
256
|
+
project.spec.default_function_node_selector = default_function_node_selector
|
|
256
257
|
|
|
257
258
|
if parameters:
|
|
258
259
|
# Enable setting project parameters at load time, can be used to customize the project_setup
|
|
@@ -874,7 +875,7 @@ class ProjectSpec(ModelObj):
|
|
|
874
875
|
# in a tuple where the first index is the packager module's path (str) and the second is a flag (bool) for
|
|
875
876
|
# whether it is mandatory for a run (raise exception on collection error) or not.
|
|
876
877
|
self.custom_packagers = custom_packagers or []
|
|
877
|
-
self.
|
|
878
|
+
self._default_function_node_selector = default_function_node_selector or None
|
|
878
879
|
|
|
879
880
|
@property
|
|
880
881
|
def source(self) -> str:
|
|
@@ -1049,6 +1050,14 @@ class ProjectSpec(ModelObj):
|
|
|
1049
1050
|
if key in self._artifacts:
|
|
1050
1051
|
del self._artifacts[key]
|
|
1051
1052
|
|
|
1053
|
+
@property
|
|
1054
|
+
def default_function_node_selector(self):
|
|
1055
|
+
return self._default_function_node_selector
|
|
1056
|
+
|
|
1057
|
+
@default_function_node_selector.setter
|
|
1058
|
+
def default_function_node_selector(self, node_selector: dict[str, str]):
|
|
1059
|
+
self._default_function_node_selector = deepcopy(node_selector)
|
|
1060
|
+
|
|
1052
1061
|
@property
|
|
1053
1062
|
def build(self) -> ImageBuilder:
|
|
1054
1063
|
return self._build
|
|
@@ -2325,31 +2334,51 @@ class MlrunProject(ModelObj):
|
|
|
2325
2334
|
requirements: typing.Union[str, list[str]] = None,
|
|
2326
2335
|
requirements_file: str = "",
|
|
2327
2336
|
) -> mlrun.runtimes.BaseRuntime:
|
|
2328
|
-
"""
|
|
2337
|
+
"""
|
|
2338
|
+
| Update or add a function object to the project.
|
|
2339
|
+
| Function can be provided as an object (func) or a .py/.ipynb/.yaml URL.
|
|
2329
2340
|
|
|
2330
|
-
function
|
|
2331
|
-
|
|
2341
|
+
| Creating a function from a single file is done by specifying ``func`` and disabling ``with_repo``.
|
|
2342
|
+
| Creating a function with project source (specify ``with_repo=True``):
|
|
2343
|
+
| 1. Specify a relative ``func`` path.
|
|
2344
|
+
| 2. Specify a module ``handler`` (e.g. ``handler=package.package.func``) without ``func``.
|
|
2345
|
+
| Creating a function with non project source is done by specifying a module ``handler`` and on the
|
|
2346
|
+
returned function set the source with ``function.with_source_archive(<source>)``.
|
|
2332
2347
|
|
|
2333
|
-
|
|
2334
|
-
MLRun DB e.g. db://project/func:ver
|
|
2335
|
-
functions hub/market: e.g. hub://auto-trainer:master
|
|
2348
|
+
Support URL prefixes:
|
|
2336
2349
|
|
|
2337
|
-
|
|
2350
|
+
| Object (s3://, v3io://, ..)
|
|
2351
|
+
| MLRun DB e.g. db://project/func:ver
|
|
2352
|
+
| Functions hub/market: e.g. hub://auto-trainer:master
|
|
2353
|
+
|
|
2354
|
+
Examples::
|
|
2338
2355
|
|
|
2339
2356
|
proj.set_function(func_object)
|
|
2340
|
-
proj.set_function(
|
|
2341
|
-
"./src/mycode.py", "ingest", image="myrepo/ing:latest", with_repo=True
|
|
2342
|
-
)
|
|
2343
2357
|
proj.set_function("http://.../mynb.ipynb", "train")
|
|
2344
2358
|
proj.set_function("./func.yaml")
|
|
2345
2359
|
proj.set_function("hub://get_toy_data", "getdata")
|
|
2346
2360
|
|
|
2347
|
-
#
|
|
2361
|
+
# Create a function from a single file
|
|
2362
|
+
proj.set_function("./src/mycode.py", "ingest")
|
|
2348
2363
|
|
|
2349
|
-
#
|
|
2364
|
+
# Creating a function with project source
|
|
2365
|
+
proj.set_function(
|
|
2366
|
+
"./src/mycode.py", "ingest", image="myrepo/ing:latest", with_repo=True
|
|
2367
|
+
)
|
|
2368
|
+
proj.set_function("ingest", handler="package.package.func", with_repo=True)
|
|
2369
|
+
|
|
2370
|
+
# Creating a function with non project source
|
|
2371
|
+
func = proj.set_function(
|
|
2372
|
+
"ingest", handler="package.package.func", with_repo=False
|
|
2373
|
+
)
|
|
2374
|
+
func.with_source_archive("git://github.com/mlrun/something.git")
|
|
2375
|
+
|
|
2376
|
+
# Set function requirements
|
|
2377
|
+
|
|
2378
|
+
# By providing a list of packages
|
|
2350
2379
|
proj.set_function("my.py", requirements=["requests", "pandas"])
|
|
2351
2380
|
|
|
2352
|
-
#
|
|
2381
|
+
# By providing a path to a pip requirements file
|
|
2353
2382
|
proj.set_function("my.py", requirements="requirements.txt")
|
|
2354
2383
|
|
|
2355
2384
|
:param func: Function object or spec/code url, None refers to current Notebook
|
|
@@ -2369,7 +2398,7 @@ class MlrunProject(ModelObj):
|
|
|
2369
2398
|
:param requirements: A list of python packages
|
|
2370
2399
|
:param requirements_file: Path to a python requirements file
|
|
2371
2400
|
|
|
2372
|
-
:returns:
|
|
2401
|
+
:returns: :py:class:`~mlrun.runtimes.BaseRuntime`
|
|
2373
2402
|
"""
|
|
2374
2403
|
(
|
|
2375
2404
|
resolved_function_name,
|
|
@@ -2801,47 +2830,92 @@ class MlrunProject(ModelObj):
|
|
|
2801
2830
|
secrets=secrets or {},
|
|
2802
2831
|
)
|
|
2803
2832
|
|
|
2804
|
-
def sync_functions(
|
|
2805
|
-
|
|
2833
|
+
def sync_functions(
|
|
2834
|
+
self,
|
|
2835
|
+
names: list = None,
|
|
2836
|
+
always: bool = True,
|
|
2837
|
+
save: bool = False,
|
|
2838
|
+
silent: bool = False,
|
|
2839
|
+
):
|
|
2840
|
+
"""
|
|
2841
|
+
Reload function objects from specs and files.
|
|
2842
|
+
The function objects are synced against the definitions spec in `self.spec._function_definitions`.
|
|
2843
|
+
Referenced files/URLs in the function spec will be reloaded.
|
|
2844
|
+
Function definitions are parsed by the following precedence:
|
|
2845
|
+
1. Contains runtime spec.
|
|
2846
|
+
2. Contains module in the project's context.
|
|
2847
|
+
3. Contains path to function definition (yaml, DB, Hub).
|
|
2848
|
+
4. Contains path to .ipynb or .py files.
|
|
2849
|
+
5. Contains a Nuclio/Serving function image / an 'Application' kind definition.
|
|
2850
|
+
If function definition is already an object, some project metadata updates will apply however,
|
|
2851
|
+
it will not be reloaded.
|
|
2852
|
+
|
|
2853
|
+
:param names: Names of functions to reload, defaults to `self.spec._function_definitions.keys()`.
|
|
2854
|
+
:param always: Force reloading the functions.
|
|
2855
|
+
:param save: Whether to save the loaded functions or not.
|
|
2856
|
+
:param silent: Whether to raise an exception when a function fails to load.
|
|
2857
|
+
|
|
2858
|
+
:returns: Dictionary of function objects
|
|
2859
|
+
"""
|
|
2806
2860
|
if self._initialized and not always:
|
|
2807
2861
|
return self.spec._function_objects
|
|
2808
2862
|
|
|
2809
|
-
|
|
2863
|
+
functions = self.spec._function_objects
|
|
2810
2864
|
if not names:
|
|
2811
2865
|
names = self.spec._function_definitions.keys()
|
|
2812
|
-
|
|
2866
|
+
functions = {}
|
|
2867
|
+
|
|
2813
2868
|
origin = mlrun.runtimes.utils.add_code_metadata(self.spec.context)
|
|
2814
2869
|
for name in names:
|
|
2815
|
-
|
|
2816
|
-
if not
|
|
2817
|
-
|
|
2870
|
+
function_definition = self.spec._function_definitions.get(name)
|
|
2871
|
+
if not function_definition:
|
|
2872
|
+
if silent:
|
|
2873
|
+
logger.warn(
|
|
2874
|
+
"Function definition was not found, skipping reload", name=name
|
|
2875
|
+
)
|
|
2876
|
+
continue
|
|
2877
|
+
|
|
2878
|
+
raise ValueError(f"Function named {name} not found")
|
|
2879
|
+
|
|
2880
|
+
function_object = self.spec._function_objects.get(name, None)
|
|
2881
|
+
is_base_runtime = isinstance(
|
|
2882
|
+
function_object, mlrun.runtimes.base.BaseRuntime
|
|
2883
|
+
)
|
|
2818
2884
|
# If this function is already available locally, don't recreate it unless always=True
|
|
2819
|
-
if
|
|
2820
|
-
|
|
2821
|
-
self.spec._function_objects.get(name, None),
|
|
2822
|
-
mlrun.runtimes.base.BaseRuntime,
|
|
2823
|
-
)
|
|
2824
|
-
and not always
|
|
2825
|
-
):
|
|
2826
|
-
funcs[name] = self.spec._function_objects[name]
|
|
2885
|
+
if is_base_runtime and not always:
|
|
2886
|
+
functions[name] = function_object
|
|
2827
2887
|
continue
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2888
|
+
|
|
2889
|
+
# Reload the function
|
|
2890
|
+
if hasattr(function_definition, "to_dict"):
|
|
2891
|
+
name, func = _init_function_from_obj(function_definition, self, name)
|
|
2892
|
+
elif isinstance(function_definition, dict):
|
|
2833
2893
|
try:
|
|
2834
|
-
name, func = _init_function_from_dict(
|
|
2894
|
+
name, func = _init_function_from_dict(
|
|
2895
|
+
function_definition, self, name
|
|
2896
|
+
)
|
|
2835
2897
|
except FileNotFoundError as exc:
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2898
|
+
message = f"File {exc.filename} not found while syncing project functions."
|
|
2899
|
+
if silent:
|
|
2900
|
+
message += " Skipping function reload"
|
|
2901
|
+
logger.warn(message, name=name)
|
|
2902
|
+
continue
|
|
2903
|
+
|
|
2904
|
+
raise mlrun.errors.MLRunMissingDependencyError(message) from exc
|
|
2905
|
+
else:
|
|
2906
|
+
message = f"Function {name} must be an object or dict."
|
|
2907
|
+
if silent:
|
|
2908
|
+
message += " Skipping function reload"
|
|
2909
|
+
logger.warn(message, name=name)
|
|
2910
|
+
continue
|
|
2911
|
+
raise ValueError(message)
|
|
2912
|
+
|
|
2839
2913
|
func.spec.build.code_origin = origin
|
|
2840
|
-
|
|
2914
|
+
functions[name] = func
|
|
2841
2915
|
if save:
|
|
2842
2916
|
func.save(versioned=False)
|
|
2843
2917
|
|
|
2844
|
-
self.spec._function_objects =
|
|
2918
|
+
self.spec._function_objects = functions
|
|
2845
2919
|
self._initialized = True
|
|
2846
2920
|
return self.spec._function_objects
|
|
2847
2921
|
|
|
@@ -3041,11 +3115,10 @@ class MlrunProject(ModelObj):
|
|
|
3041
3115
|
)
|
|
3042
3116
|
|
|
3043
3117
|
if engine not in ["remote"] and not schedule:
|
|
3044
|
-
# For remote/scheduled runs
|
|
3045
|
-
|
|
3046
|
-
self.sync_functions(always=sync)
|
|
3118
|
+
# For remote/scheduled runs there is no need to sync functions as they can be loaded dynamically during run
|
|
3119
|
+
self.sync_functions(always=sync, silent=True)
|
|
3047
3120
|
if not self.spec._function_objects:
|
|
3048
|
-
|
|
3121
|
+
logger.warn(
|
|
3049
3122
|
"There are no functions in the project."
|
|
3050
3123
|
" Make sure you've set your functions with project.set_function()."
|
|
3051
3124
|
)
|
|
@@ -4289,6 +4362,7 @@ class MlrunProject(ModelObj):
|
|
|
4289
4362
|
kind=producer_dict.get("kind", ""),
|
|
4290
4363
|
project=producer_project,
|
|
4291
4364
|
tag=producer_tag,
|
|
4365
|
+
owner=producer_dict.get("owner", ""),
|
|
4292
4366
|
), True
|
|
4293
4367
|
|
|
4294
4368
|
# do not retain the artifact's producer, replace it with the project as the producer
|
|
@@ -4298,6 +4372,7 @@ class MlrunProject(ModelObj):
|
|
|
4298
4372
|
name=self.metadata.name,
|
|
4299
4373
|
project=self.metadata.name,
|
|
4300
4374
|
tag=project_producer_tag,
|
|
4375
|
+
owner=self._resolve_artifact_owner(),
|
|
4301
4376
|
), False
|
|
4302
4377
|
|
|
4303
4378
|
def _resolve_existing_artifact(
|
|
@@ -4337,6 +4412,9 @@ class MlrunProject(ModelObj):
|
|
|
4337
4412
|
def _get_project_tag(self):
|
|
4338
4413
|
return self._get_hexsha() or str(uuid.uuid4())
|
|
4339
4414
|
|
|
4415
|
+
def _resolve_artifact_owner(self):
|
|
4416
|
+
return os.getenv("V3IO_USERNAME") or self.spec.owner
|
|
4417
|
+
|
|
4340
4418
|
|
|
4341
4419
|
def _set_as_current_default_project(project: MlrunProject):
|
|
4342
4420
|
mlrun.mlconf.default_project = project.metadata.name
|
mlrun/runtimes/funcdoc.py
CHANGED
mlrun/runtimes/local.py
CHANGED
|
@@ -145,7 +145,10 @@ class ParallelRunner:
|
|
|
145
145
|
if function_name and generator.options.teardown_dask:
|
|
146
146
|
logger.info("Tearing down the dask cluster..")
|
|
147
147
|
mlrun.get_run_db().delete_runtime_resources(
|
|
148
|
-
|
|
148
|
+
project=self.metadata.project,
|
|
149
|
+
kind=mlrun.runtimes.RuntimeKinds.dask,
|
|
150
|
+
object_id=function_name,
|
|
151
|
+
force=True,
|
|
149
152
|
)
|
|
150
153
|
|
|
151
154
|
return results
|
|
@@ -438,8 +438,9 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
438
438
|
"""
|
|
439
439
|
Create the application API gateway. Once the application is deployed, the API gateway can be created.
|
|
440
440
|
An application without an API gateway is not accessible.
|
|
441
|
-
:param name: The name of the API gateway
|
|
442
|
-
:param path: Optional path of the API gateway, default value is "/"
|
|
441
|
+
:param name: The name of the API gateway
|
|
442
|
+
:param path: Optional path of the API gateway, default value is "/".
|
|
443
|
+
The given path should be supported by the deployed application
|
|
443
444
|
:param direct_port_access: Set True to allow direct port access to the application sidecar
|
|
444
445
|
:param authentication_mode: API Gateway authentication mode
|
|
445
446
|
:param authentication_creds: API Gateway basic authentication credentials as a tuple (username, password)
|
mlrun/runtimes/pod.py
CHANGED
|
@@ -38,6 +38,7 @@ from ..k8s_utils import (
|
|
|
38
38
|
generate_preemptible_nodes_affinity_terms,
|
|
39
39
|
generate_preemptible_nodes_anti_affinity_terms,
|
|
40
40
|
generate_preemptible_tolerations,
|
|
41
|
+
validate_node_selectors,
|
|
41
42
|
)
|
|
42
43
|
from ..utils import logger, update_in
|
|
43
44
|
from .base import BaseRuntime, FunctionSpec, spec_fields
|
|
@@ -1175,6 +1176,7 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
|
|
|
1175
1176
|
if node_name:
|
|
1176
1177
|
self.spec.node_name = node_name
|
|
1177
1178
|
if node_selector is not None:
|
|
1179
|
+
validate_node_selectors(node_selectors=node_selector, raise_on_error=False)
|
|
1178
1180
|
self.spec.node_selector = node_selector
|
|
1179
1181
|
if affinity is not None:
|
|
1180
1182
|
self.spec.affinity = affinity
|
|
@@ -18,6 +18,7 @@ from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
|
|
|
18
18
|
|
|
19
19
|
import mlrun.common.schemas.function
|
|
20
20
|
import mlrun.errors
|
|
21
|
+
import mlrun.k8s_utils
|
|
21
22
|
import mlrun.runtimes.pod
|
|
22
23
|
from mlrun.config import config
|
|
23
24
|
|
|
@@ -451,7 +452,7 @@ class Spark3JobSpec(KubeResourceSpec):
|
|
|
451
452
|
class Spark3Runtime(KubejobRuntime):
|
|
452
453
|
group = "sparkoperator.k8s.io"
|
|
453
454
|
version = "v1beta2"
|
|
454
|
-
apiVersion = group + "/" + version
|
|
455
|
+
apiVersion = group + "/" + version # noqa: N815
|
|
455
456
|
kind = "spark"
|
|
456
457
|
plural = "sparkapplications"
|
|
457
458
|
|
|
@@ -505,6 +506,7 @@ class Spark3Runtime(KubejobRuntime):
|
|
|
505
506
|
raise NotImplementedError(
|
|
506
507
|
"Setting node name is not supported for spark runtime"
|
|
507
508
|
)
|
|
509
|
+
mlrun.k8s_utils.validate_node_selectors(node_selector, raise_on_error=False)
|
|
508
510
|
self.with_driver_node_selection(node_name, node_selector, affinity, tolerations)
|
|
509
511
|
self.with_executor_node_selection(
|
|
510
512
|
node_name, node_selector, affinity, tolerations
|
|
@@ -537,6 +539,7 @@ class Spark3Runtime(KubejobRuntime):
|
|
|
537
539
|
if affinity is not None:
|
|
538
540
|
self.spec.driver_affinity = affinity
|
|
539
541
|
if node_selector is not None:
|
|
542
|
+
mlrun.k8s_utils.validate_node_selectors(node_selector, raise_on_error=False)
|
|
540
543
|
self.spec.driver_node_selector = node_selector
|
|
541
544
|
if tolerations is not None:
|
|
542
545
|
self.spec.driver_tolerations = tolerations
|
|
@@ -568,6 +571,7 @@ class Spark3Runtime(KubejobRuntime):
|
|
|
568
571
|
if affinity is not None:
|
|
569
572
|
self.spec.executor_affinity = affinity
|
|
570
573
|
if node_selector is not None:
|
|
574
|
+
mlrun.k8s_utils.validate_node_selectors(node_selector, raise_on_error=False)
|
|
571
575
|
self.spec.executor_node_selector = node_selector
|
|
572
576
|
if tolerations is not None:
|
|
573
577
|
self.spec.executor_tolerations = tolerations
|
mlrun/utils/async_http.py
CHANGED
|
@@ -237,7 +237,7 @@ class _CustomRequestContext(_RequestContext):
|
|
|
237
237
|
retry_wait = self._retry_options.get_timeout(
|
|
238
238
|
attempt=current_attempt, response=None
|
|
239
239
|
)
|
|
240
|
-
self._logger.
|
|
240
|
+
self._logger.warning(
|
|
241
241
|
"Request failed on retryable exception, retrying",
|
|
242
242
|
retry_wait_secs=retry_wait,
|
|
243
243
|
method=params.method,
|
mlrun/utils/helpers.py
CHANGED
|
@@ -1007,6 +1007,23 @@ def get_workflow_url(project, id=None):
|
|
|
1007
1007
|
return url
|
|
1008
1008
|
|
|
1009
1009
|
|
|
1010
|
+
def get_kfp_project_filter(project_name: str) -> str:
|
|
1011
|
+
"""
|
|
1012
|
+
Generates a filter string for KFP runs, using a substring predicate
|
|
1013
|
+
on the run's 'name' field. This is used as a heuristic to retrieve runs that are associated
|
|
1014
|
+
with a specific project. The 'op: 9' operator indicates that the filter checks if the
|
|
1015
|
+
project name appears as a substring in the run's name, ensuring that we can identify
|
|
1016
|
+
runs belonging to the desired project.
|
|
1017
|
+
"""
|
|
1018
|
+
is_substring_op = 9
|
|
1019
|
+
project_name_filter = {
|
|
1020
|
+
"predicates": [
|
|
1021
|
+
{"key": "name", "op": is_substring_op, "string_value": project_name}
|
|
1022
|
+
]
|
|
1023
|
+
}
|
|
1024
|
+
return json.dumps(project_name_filter)
|
|
1025
|
+
|
|
1026
|
+
|
|
1010
1027
|
def are_strings_in_exception_chain_messages(
|
|
1011
1028
|
exception: Exception, strings_list: list[str]
|
|
1012
1029
|
) -> bool:
|
mlrun/utils/v3io_clients.py
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
from v3io.dataplane import Client as V3IOClient
|
|
16
|
-
from v3io_frames import Client as
|
|
16
|
+
from v3io_frames import Client as V3IOFramesClient
|
|
17
17
|
from v3io_frames.client import ClientBase
|
|
18
18
|
|
|
19
19
|
_v3io_clients: dict[frozenset, V3IOClient] = {}
|
|
@@ -24,7 +24,7 @@ def get_frames_client(**kwargs) -> ClientBase:
|
|
|
24
24
|
global _frames_clients
|
|
25
25
|
kw_set = frozenset(kwargs.items())
|
|
26
26
|
if kw_set not in _frames_clients:
|
|
27
|
-
_frames_clients[kw_set] =
|
|
27
|
+
_frames_clients[kw_set] = V3IOFramesClient(**kwargs)
|
|
28
28
|
|
|
29
29
|
return _frames_clients[kw_set]
|
|
30
30
|
|
mlrun/utils/version/version.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mlrun
|
|
3
|
-
Version: 1.7.
|
|
3
|
+
Version: 1.7.0rc43
|
|
4
4
|
Summary: Tracking and config of machine learning runs
|
|
5
5
|
Home-page: https://github.com/mlrun/mlrun
|
|
6
6
|
Author: Yaron Haviv
|
|
@@ -41,7 +41,7 @@ Requires-Dist: mergedeep ~=1.3
|
|
|
41
41
|
Requires-Dist: v3io-frames ~=0.10.14
|
|
42
42
|
Requires-Dist: semver ~=3.0
|
|
43
43
|
Requires-Dist: dependency-injector ~=4.41
|
|
44
|
-
Requires-Dist: fsspec <2024.
|
|
44
|
+
Requires-Dist: fsspec <2024.7,>=2023.9.2
|
|
45
45
|
Requires-Dist: v3iofs ~=0.1.17
|
|
46
46
|
Requires-Dist: storey ~=1.7.24
|
|
47
47
|
Requires-Dist: inflection ~=0.5.0
|
|
@@ -67,7 +67,7 @@ Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 'all'
|
|
|
67
67
|
Requires-Dist: dask ~=2023.9.0 ; extra == 'all'
|
|
68
68
|
Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'all'
|
|
69
69
|
Requires-Dist: distributed ~=2023.9.0 ; extra == 'all'
|
|
70
|
-
Requires-Dist: gcsfs <2024.
|
|
70
|
+
Requires-Dist: gcsfs <2024.7,>=2023.9.2 ; extra == 'all'
|
|
71
71
|
Requires-Dist: google-cloud-bigquery-storage ~=2.17 ; extra == 'all'
|
|
72
72
|
Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'all'
|
|
73
73
|
Requires-Dist: google-cloud-storage ==2.14.0 ; extra == 'all'
|
|
@@ -81,7 +81,7 @@ Requires-Dist: ossfs ==2023.12.0 ; extra == 'all'
|
|
|
81
81
|
Requires-Dist: plotly ~=5.23 ; extra == 'all'
|
|
82
82
|
Requires-Dist: pyopenssl >=23 ; extra == 'all'
|
|
83
83
|
Requires-Dist: redis ~=4.3 ; extra == 'all'
|
|
84
|
-
Requires-Dist: s3fs <2024.
|
|
84
|
+
Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 'all'
|
|
85
85
|
Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'all'
|
|
86
86
|
Requires-Dist: sqlalchemy ~=1.4 ; extra == 'all'
|
|
87
87
|
Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'all'
|
|
@@ -120,7 +120,7 @@ Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 'complete'
|
|
|
120
120
|
Requires-Dist: dask ~=2023.9.0 ; extra == 'complete'
|
|
121
121
|
Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'complete'
|
|
122
122
|
Requires-Dist: distributed ~=2023.9.0 ; extra == 'complete'
|
|
123
|
-
Requires-Dist: gcsfs <2024.
|
|
123
|
+
Requires-Dist: gcsfs <2024.7,>=2023.9.2 ; extra == 'complete'
|
|
124
124
|
Requires-Dist: google-cloud-bigquery-storage ~=2.17 ; extra == 'complete'
|
|
125
125
|
Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'complete'
|
|
126
126
|
Requires-Dist: google-cloud-storage ==2.14.0 ; extra == 'complete'
|
|
@@ -134,7 +134,7 @@ Requires-Dist: ossfs ==2023.12.0 ; extra == 'complete'
|
|
|
134
134
|
Requires-Dist: plotly ~=5.23 ; extra == 'complete'
|
|
135
135
|
Requires-Dist: pyopenssl >=23 ; extra == 'complete'
|
|
136
136
|
Requires-Dist: redis ~=4.3 ; extra == 'complete'
|
|
137
|
-
Requires-Dist: s3fs <2024.
|
|
137
|
+
Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 'complete'
|
|
138
138
|
Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete'
|
|
139
139
|
Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete'
|
|
140
140
|
Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete'
|
|
@@ -153,7 +153,7 @@ Requires-Dist: dask ~=2023.9.0 ; extra == 'complete-api'
|
|
|
153
153
|
Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'complete-api'
|
|
154
154
|
Requires-Dist: distributed ~=2023.9.0 ; extra == 'complete-api'
|
|
155
155
|
Requires-Dist: fastapi ~=0.110.0 ; extra == 'complete-api'
|
|
156
|
-
Requires-Dist: gcsfs <2024.
|
|
156
|
+
Requires-Dist: gcsfs <2024.7,>=2023.9.2 ; extra == 'complete-api'
|
|
157
157
|
Requires-Dist: google-cloud-bigquery-storage ~=2.17 ; extra == 'complete-api'
|
|
158
158
|
Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'complete-api'
|
|
159
159
|
Requires-Dist: google-cloud-storage ==2.14.0 ; extra == 'complete-api'
|
|
@@ -171,7 +171,7 @@ Requires-Dist: plotly ~=5.23 ; extra == 'complete-api'
|
|
|
171
171
|
Requires-Dist: pymysql ~=1.0 ; extra == 'complete-api'
|
|
172
172
|
Requires-Dist: pyopenssl >=23 ; extra == 'complete-api'
|
|
173
173
|
Requires-Dist: redis ~=4.3 ; extra == 'complete-api'
|
|
174
|
-
Requires-Dist: s3fs <2024.
|
|
174
|
+
Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 'complete-api'
|
|
175
175
|
Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete-api'
|
|
176
176
|
Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
|
|
177
177
|
Requires-Dist: taos-ws-py ~=0.3.2 ; extra == 'complete-api'
|
|
@@ -188,7 +188,7 @@ Requires-Dist: google-cloud-storage ==2.14.0 ; extra == 'google-cloud'
|
|
|
188
188
|
Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'google-cloud'
|
|
189
189
|
Requires-Dist: google-cloud-bigquery-storage ~=2.17 ; extra == 'google-cloud'
|
|
190
190
|
Requires-Dist: google-cloud ==0.34 ; extra == 'google-cloud'
|
|
191
|
-
Requires-Dist: gcsfs <2024.
|
|
191
|
+
Requires-Dist: gcsfs <2024.7,>=2023.9.2 ; extra == 'google-cloud'
|
|
192
192
|
Provides-Extra: graphviz
|
|
193
193
|
Requires-Dist: graphviz ~=0.20.0 ; extra == 'graphviz'
|
|
194
194
|
Provides-Extra: kafka
|
|
@@ -203,7 +203,7 @@ Requires-Dist: redis ~=4.3 ; extra == 'redis'
|
|
|
203
203
|
Provides-Extra: s3
|
|
204
204
|
Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 's3'
|
|
205
205
|
Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 's3'
|
|
206
|
-
Requires-Dist: s3fs <2024.
|
|
206
|
+
Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 's3'
|
|
207
207
|
Provides-Extra: snowflake
|
|
208
208
|
Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'snowflake'
|
|
209
209
|
Provides-Extra: sqlalchemy
|