mlrun 1.7.0rc15__py3-none-any.whl → 1.7.0rc16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +10 -1
- mlrun/__main__.py +18 -4
- mlrun/alerts/__init__.py +15 -0
- mlrun/alerts/alert.py +141 -0
- mlrun/artifacts/__init__.py +7 -1
- mlrun/artifacts/base.py +28 -3
- mlrun/artifacts/dataset.py +8 -0
- mlrun/artifacts/manager.py +18 -0
- mlrun/artifacts/model.py +7 -0
- mlrun/artifacts/plots.py +13 -0
- mlrun/common/schemas/__init__.py +4 -2
- mlrun/common/schemas/alert.py +46 -4
- mlrun/common/schemas/api_gateway.py +4 -0
- mlrun/common/schemas/artifact.py +15 -0
- mlrun/common/schemas/auth.py +2 -0
- mlrun/common/schemas/model_monitoring/__init__.py +4 -1
- mlrun/common/schemas/model_monitoring/constants.py +16 -1
- mlrun/common/schemas/model_monitoring/model_endpoints.py +60 -1
- mlrun/common/schemas/project.py +2 -0
- mlrun/config.py +4 -1
- mlrun/datastore/datastore_profile.py +10 -7
- mlrun/db/base.py +23 -3
- mlrun/db/httpdb.py +97 -43
- mlrun/db/nopdb.py +20 -2
- mlrun/errors.py +5 -0
- mlrun/launcher/base.py +3 -2
- mlrun/lists.py +2 -0
- mlrun/model.py +7 -2
- mlrun/model_monitoring/__init__.py +1 -1
- mlrun/model_monitoring/applications/_application_steps.py +1 -2
- mlrun/model_monitoring/applications/context.py +1 -1
- mlrun/model_monitoring/applications/histogram_data_drift.py +64 -38
- mlrun/model_monitoring/db/__init__.py +2 -0
- mlrun/model_monitoring/db/stores/base/store.py +9 -36
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +63 -110
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +56 -202
- mlrun/model_monitoring/db/tsdb/__init__.py +71 -0
- mlrun/model_monitoring/db/tsdb/base.py +135 -0
- mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +117 -0
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +404 -0
- mlrun/model_monitoring/db/v3io_tsdb_reader.py +134 -0
- mlrun/model_monitoring/stream_processing.py +46 -210
- mlrun/model_monitoring/writer.py +49 -99
- mlrun/platforms/__init__.py +10 -9
- mlrun/platforms/iguazio.py +19 -200
- mlrun/projects/operations.py +11 -7
- mlrun/projects/pipelines.py +13 -76
- mlrun/projects/project.py +55 -14
- mlrun/render.py +9 -3
- mlrun/run.py +5 -38
- mlrun/runtimes/base.py +3 -3
- mlrun/runtimes/kubejob.py +2 -1
- mlrun/runtimes/nuclio/api_gateway.py +75 -9
- mlrun/runtimes/nuclio/function.py +8 -34
- mlrun/runtimes/pod.py +16 -36
- mlrun/runtimes/remotesparkjob.py +1 -1
- mlrun/runtimes/sparkjob/spark3job.py +1 -1
- mlrun/runtimes/utils.py +0 -38
- mlrun/utils/helpers.py +45 -31
- mlrun/utils/notifications/notification/base.py +1 -1
- mlrun/utils/notifications/notification/slack.py +9 -4
- mlrun/utils/notifications/notification/webhook.py +1 -1
- mlrun/utils/notifications/notification_pusher.py +15 -14
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc15.dist-info → mlrun-1.7.0rc16.dist-info}/METADATA +3 -2
- {mlrun-1.7.0rc15.dist-info → mlrun-1.7.0rc16.dist-info}/RECORD +71 -65
- mlrun/kfpops.py +0 -860
- mlrun/platforms/other.py +0 -305
- {mlrun-1.7.0rc15.dist-info → mlrun-1.7.0rc16.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc15.dist-info → mlrun-1.7.0rc16.dist-info}/WHEEL +0 -0
- {mlrun-1.7.0rc15.dist-info → mlrun-1.7.0rc16.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc15.dist-info → mlrun-1.7.0rc16.dist-info}/top_level.txt +0 -0
mlrun/run.py
CHANGED
|
@@ -29,11 +29,13 @@ from typing import Optional, Union
|
|
|
29
29
|
import nuclio
|
|
30
30
|
import yaml
|
|
31
31
|
from kfp import Client
|
|
32
|
+
from mlrun_pipelines.common.models import RunStatuses
|
|
33
|
+
from mlrun_pipelines.common.ops import format_summary_from_kfp_run, show_kfp_run
|
|
34
|
+
from mlrun_pipelines.models import PipelineRun
|
|
32
35
|
|
|
33
36
|
import mlrun.common.schemas
|
|
34
37
|
import mlrun.errors
|
|
35
38
|
import mlrun.utils.helpers
|
|
36
|
-
from mlrun.kfpops import format_summary_from_kfp_run, show_kfp_run
|
|
37
39
|
|
|
38
40
|
from .common.helpers import parse_versioned_object_uri
|
|
39
41
|
from .config import config as mlconf
|
|
@@ -68,41 +70,6 @@ from .utils import (
|
|
|
68
70
|
)
|
|
69
71
|
|
|
70
72
|
|
|
71
|
-
class RunStatuses:
|
|
72
|
-
succeeded = "Succeeded"
|
|
73
|
-
failed = "Failed"
|
|
74
|
-
skipped = "Skipped"
|
|
75
|
-
error = "Error"
|
|
76
|
-
running = "Running"
|
|
77
|
-
|
|
78
|
-
@staticmethod
|
|
79
|
-
def all():
|
|
80
|
-
return [
|
|
81
|
-
RunStatuses.succeeded,
|
|
82
|
-
RunStatuses.failed,
|
|
83
|
-
RunStatuses.skipped,
|
|
84
|
-
RunStatuses.error,
|
|
85
|
-
RunStatuses.running,
|
|
86
|
-
]
|
|
87
|
-
|
|
88
|
-
@staticmethod
|
|
89
|
-
def stable_statuses():
|
|
90
|
-
return [
|
|
91
|
-
RunStatuses.succeeded,
|
|
92
|
-
RunStatuses.failed,
|
|
93
|
-
RunStatuses.skipped,
|
|
94
|
-
RunStatuses.error,
|
|
95
|
-
]
|
|
96
|
-
|
|
97
|
-
@staticmethod
|
|
98
|
-
def transient_statuses():
|
|
99
|
-
return [
|
|
100
|
-
status
|
|
101
|
-
for status in RunStatuses.all()
|
|
102
|
-
if status not in RunStatuses.stable_statuses()
|
|
103
|
-
]
|
|
104
|
-
|
|
105
|
-
|
|
106
73
|
def function_to_module(code="", workdir=None, secrets=None, silent=False):
|
|
107
74
|
"""Load code, notebook or mlrun function as .py module
|
|
108
75
|
this function can import a local/remote py file or notebook
|
|
@@ -1021,7 +988,7 @@ def get_pipeline(
|
|
|
1021
988
|
:param project: the project of the pipeline run
|
|
1022
989
|
:param remote: read kfp data from mlrun service (default=True)
|
|
1023
990
|
|
|
1024
|
-
:return: kfp run
|
|
991
|
+
:return: kfp run
|
|
1025
992
|
"""
|
|
1026
993
|
namespace = namespace or mlconf.namespace
|
|
1027
994
|
if remote:
|
|
@@ -1045,7 +1012,7 @@ def get_pipeline(
|
|
|
1045
1012
|
not format_
|
|
1046
1013
|
or format_ == mlrun.common.schemas.PipelinesFormat.summary.value
|
|
1047
1014
|
):
|
|
1048
|
-
resp = format_summary_from_kfp_run(resp)
|
|
1015
|
+
resp = format_summary_from_kfp_run(PipelineRun(resp))
|
|
1049
1016
|
|
|
1050
1017
|
show_kfp_run(resp)
|
|
1051
1018
|
return resp
|
mlrun/runtimes/base.py
CHANGED
|
@@ -21,6 +21,7 @@ from os import environ
|
|
|
21
21
|
from typing import Callable, Optional, Union
|
|
22
22
|
|
|
23
23
|
import requests.exceptions
|
|
24
|
+
from mlrun_pipelines.common.ops import mlrun_op
|
|
24
25
|
from nuclio.build import mlrun_footer
|
|
25
26
|
|
|
26
27
|
import mlrun.common.constants
|
|
@@ -37,7 +38,6 @@ from mlrun.utils.helpers import generate_object_uri, verify_field_regex
|
|
|
37
38
|
from ..config import config
|
|
38
39
|
from ..datastore import store_manager
|
|
39
40
|
from ..errors import err_to_str
|
|
40
|
-
from ..kfpops import mlrun_op
|
|
41
41
|
from ..lists import RunList
|
|
42
42
|
from ..model import BaseMetadata, HyperParamOptions, ImageBuilder, ModelObj, RunObject
|
|
43
43
|
from ..utils import (
|
|
@@ -707,11 +707,11 @@ class BaseRuntime(ModelObj):
|
|
|
707
707
|
"key": "the_key".
|
|
708
708
|
:param auto_build: when set to True and the function require build it will be built on the first
|
|
709
709
|
function run, use only if you dont plan on changing the build config between runs
|
|
710
|
-
:return:
|
|
710
|
+
:return: mlrun_pipelines.models.PipelineNodeWrapper
|
|
711
711
|
"""
|
|
712
712
|
|
|
713
713
|
# if the function contain KFP PipelineParams (futures) pass the full spec to the
|
|
714
|
-
#
|
|
714
|
+
# PipelineNodeWrapper this way KFP will substitute the params with previous step outputs
|
|
715
715
|
if use_db and not self._has_pipeline_param():
|
|
716
716
|
# if the same function is built as part of the pipeline we do not use the versioned function
|
|
717
717
|
# rather the latest function w the same tag so we can pick up the updated image/status
|
mlrun/runtimes/kubejob.py
CHANGED
|
@@ -14,11 +14,12 @@
|
|
|
14
14
|
|
|
15
15
|
import warnings
|
|
16
16
|
|
|
17
|
+
from mlrun_pipelines.common.ops import build_op
|
|
18
|
+
|
|
17
19
|
import mlrun.common.schemas
|
|
18
20
|
import mlrun.db
|
|
19
21
|
import mlrun.errors
|
|
20
22
|
|
|
21
|
-
from ..kfpops import build_op
|
|
22
23
|
from ..model import RunObject
|
|
23
24
|
from .pod import KubeResource
|
|
24
25
|
|
|
@@ -17,10 +17,12 @@ from typing import Optional, Union
|
|
|
17
17
|
from urllib.parse import urljoin
|
|
18
18
|
|
|
19
19
|
import requests
|
|
20
|
+
from nuclio.auth import AuthInfo as NuclioAuthInfo
|
|
20
21
|
from requests.auth import HTTPBasicAuth
|
|
21
22
|
|
|
22
23
|
import mlrun
|
|
23
24
|
import mlrun.common.schemas
|
|
25
|
+
from mlrun.platforms.iguazio import min_iguazio_versions
|
|
24
26
|
|
|
25
27
|
from ...model import ModelObj
|
|
26
28
|
from ..utils import logger
|
|
@@ -29,6 +31,7 @@ from .serving import ServingRuntime
|
|
|
29
31
|
|
|
30
32
|
NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH = "basicAuth"
|
|
31
33
|
NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_NONE = "none"
|
|
34
|
+
NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_ACCESS_KEY = "accessKey"
|
|
32
35
|
PROJECT_NAME_LABEL = "nuclio.io/project-name"
|
|
33
36
|
|
|
34
37
|
|
|
@@ -50,6 +53,11 @@ class APIGatewayAuthenticator(typing.Protocol):
|
|
|
50
53
|
)
|
|
51
54
|
else:
|
|
52
55
|
return BasicAuth()
|
|
56
|
+
elif (
|
|
57
|
+
api_gateway_spec.authenticationMode
|
|
58
|
+
== NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_ACCESS_KEY
|
|
59
|
+
):
|
|
60
|
+
return AccessKeyAuth()
|
|
53
61
|
else:
|
|
54
62
|
return NoneAuth()
|
|
55
63
|
|
|
@@ -93,6 +101,16 @@ class BasicAuth(APIGatewayAuthenticator):
|
|
|
93
101
|
}
|
|
94
102
|
|
|
95
103
|
|
|
104
|
+
class AccessKeyAuth(APIGatewayAuthenticator):
|
|
105
|
+
"""
|
|
106
|
+
An API gateway authenticator with access key authentication.
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def authentication_mode(self) -> str:
|
|
111
|
+
return NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_ACCESS_KEY
|
|
112
|
+
|
|
113
|
+
|
|
96
114
|
class APIGatewayMetadata(ModelObj):
|
|
97
115
|
_dict_fields = ["name", "namespace", "labels", "annotations", "creation_timestamp"]
|
|
98
116
|
|
|
@@ -156,6 +174,7 @@ class APIGatewaySpec(ModelObj):
|
|
|
156
174
|
path: str = "/",
|
|
157
175
|
authentication: Optional[APIGatewayAuthenticator] = NoneAuth(),
|
|
158
176
|
canary: Optional[list[int]] = None,
|
|
177
|
+
ports: Optional[list[int]] = None,
|
|
159
178
|
):
|
|
160
179
|
"""
|
|
161
180
|
:param functions: The list of functions associated with the API gateway
|
|
@@ -169,7 +188,9 @@ class APIGatewaySpec(ModelObj):
|
|
|
169
188
|
:param authentication: The authentication for the API gateway of type
|
|
170
189
|
:py:class:`~mlrun.runtimes.nuclio.api_gateway.BasicAuth`
|
|
171
190
|
:param host: The host of the API gateway (optional). If not set, it will be automatically generated
|
|
172
|
-
:param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80]
|
|
191
|
+
:param canary: The canary percents for the API gateway of type list[int]; for instance: [20,80] (optional)
|
|
192
|
+
:param ports: The ports of the API gateway, as a list of integers that correspond to the functions in the
|
|
193
|
+
functions list. for instance: [8050] or [8050, 8081] (optional)
|
|
173
194
|
"""
|
|
174
195
|
self.description = description
|
|
175
196
|
self.host = host
|
|
@@ -178,8 +199,9 @@ class APIGatewaySpec(ModelObj):
|
|
|
178
199
|
self.functions = functions
|
|
179
200
|
self.canary = canary
|
|
180
201
|
self.project = project
|
|
202
|
+
self.ports = ports
|
|
181
203
|
|
|
182
|
-
self.validate(project=project, functions=functions, canary=canary)
|
|
204
|
+
self.validate(project=project, functions=functions, canary=canary, ports=ports)
|
|
183
205
|
|
|
184
206
|
def validate(
|
|
185
207
|
self,
|
|
@@ -198,6 +220,7 @@ class APIGatewaySpec(ModelObj):
|
|
|
198
220
|
],
|
|
199
221
|
],
|
|
200
222
|
canary: Optional[list[int]] = None,
|
|
223
|
+
ports: Optional[list[int]] = None,
|
|
201
224
|
):
|
|
202
225
|
self.functions = self._validate_functions(project=project, functions=functions)
|
|
203
226
|
|
|
@@ -205,6 +228,10 @@ class APIGatewaySpec(ModelObj):
|
|
|
205
228
|
if canary:
|
|
206
229
|
self.canary = self._validate_canary(canary)
|
|
207
230
|
|
|
231
|
+
# validating ports
|
|
232
|
+
if ports:
|
|
233
|
+
self.ports = self._validate_ports(ports)
|
|
234
|
+
|
|
208
235
|
def _validate_canary(self, canary: list[int]):
|
|
209
236
|
if len(self.functions) != len(canary):
|
|
210
237
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
@@ -221,6 +248,14 @@ class APIGatewaySpec(ModelObj):
|
|
|
221
248
|
)
|
|
222
249
|
return canary
|
|
223
250
|
|
|
251
|
+
def _validate_ports(self, ports):
|
|
252
|
+
if len(self.functions) != len(ports):
|
|
253
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
254
|
+
"Function and port lists lengths do not match"
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
return ports
|
|
258
|
+
|
|
224
259
|
@staticmethod
|
|
225
260
|
def _validate_functions(
|
|
226
261
|
project: str,
|
|
@@ -312,7 +347,7 @@ class APIGateway(ModelObj):
|
|
|
312
347
|
def invoke(
|
|
313
348
|
self,
|
|
314
349
|
method="POST",
|
|
315
|
-
headers: dict =
|
|
350
|
+
headers: dict = None,
|
|
316
351
|
auth: Optional[tuple[str, str]] = None,
|
|
317
352
|
**kwargs,
|
|
318
353
|
):
|
|
@@ -341,17 +376,26 @@ class APIGateway(ModelObj):
|
|
|
341
376
|
if (
|
|
342
377
|
self.spec.authentication.authentication_mode
|
|
343
378
|
== NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_BASIC_AUTH
|
|
344
|
-
and not auth
|
|
345
379
|
):
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
380
|
+
if not auth:
|
|
381
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
382
|
+
"API Gateway invocation requires authentication. Please pass credentials"
|
|
383
|
+
)
|
|
384
|
+
auth = HTTPBasicAuth(*auth)
|
|
385
|
+
|
|
386
|
+
if (
|
|
387
|
+
self.spec.authentication.authentication_mode
|
|
388
|
+
== NUCLIO_API_GATEWAY_AUTHENTICATION_MODE_ACCESS_KEY
|
|
389
|
+
):
|
|
390
|
+
# inject access key from env
|
|
391
|
+
auth = NuclioAuthInfo().from_envvar().to_requests_auth()
|
|
392
|
+
|
|
349
393
|
return requests.request(
|
|
350
394
|
method=method,
|
|
351
395
|
url=self.invoke_url,
|
|
352
|
-
headers=headers,
|
|
396
|
+
headers=headers or {},
|
|
353
397
|
**kwargs,
|
|
354
|
-
auth=
|
|
398
|
+
auth=auth,
|
|
355
399
|
)
|
|
356
400
|
|
|
357
401
|
def wait_for_readiness(self, max_wait_time=90):
|
|
@@ -407,6 +451,13 @@ class APIGateway(ModelObj):
|
|
|
407
451
|
"""
|
|
408
452
|
self.spec.authentication = BasicAuth(username=username, password=password)
|
|
409
453
|
|
|
454
|
+
@min_iguazio_versions("3.5.5")
|
|
455
|
+
def with_access_key_auth(self):
|
|
456
|
+
"""
|
|
457
|
+
Set access key authentication for the API gateway.
|
|
458
|
+
"""
|
|
459
|
+
self.spec.authentication = AccessKeyAuth()
|
|
460
|
+
|
|
410
461
|
def with_canary(
|
|
411
462
|
self,
|
|
412
463
|
functions: Union[
|
|
@@ -440,6 +491,17 @@ class APIGateway(ModelObj):
|
|
|
440
491
|
project=self.spec.project, functions=functions, canary=canary
|
|
441
492
|
)
|
|
442
493
|
|
|
494
|
+
def with_ports(self, ports: list[int]):
|
|
495
|
+
"""
|
|
496
|
+
Set ports for the API gateway
|
|
497
|
+
|
|
498
|
+
:param ports: The ports of the API gateway, as a list of integers that correspond to the functions in the
|
|
499
|
+
functions list. for instance: [8050] or [8050, 8081]
|
|
500
|
+
"""
|
|
501
|
+
self.spec.validate(
|
|
502
|
+
project=self.spec.project, functions=self.spec.functions, ports=ports
|
|
503
|
+
)
|
|
504
|
+
|
|
443
505
|
@classmethod
|
|
444
506
|
def from_scheme(cls, api_gateway: mlrun.common.schemas.APIGateway):
|
|
445
507
|
project = api_gateway.metadata.labels.get(PROJECT_NAME_LABEL)
|
|
@@ -487,6 +549,10 @@ class APIGateway(ModelObj):
|
|
|
487
549
|
for function_name in self.spec.functions
|
|
488
550
|
]
|
|
489
551
|
)
|
|
552
|
+
if self.spec.ports:
|
|
553
|
+
for i, port in enumerate(self.spec.ports):
|
|
554
|
+
upstreams[i].port = port
|
|
555
|
+
|
|
490
556
|
api_gateway = mlrun.common.schemas.APIGateway(
|
|
491
557
|
metadata=mlrun.common.schemas.APIGatewayMetadata(
|
|
492
558
|
name=self.metadata.name, labels={}
|
|
@@ -22,9 +22,11 @@ from time import sleep
|
|
|
22
22
|
import nuclio
|
|
23
23
|
import nuclio.utils
|
|
24
24
|
import requests
|
|
25
|
-
import semver
|
|
26
25
|
from aiohttp.client import ClientSession
|
|
27
26
|
from kubernetes import client
|
|
27
|
+
from mlrun_pipelines.common.mounts import VolumeMount
|
|
28
|
+
from mlrun_pipelines.common.ops import deploy_op
|
|
29
|
+
from mlrun_pipelines.mounts import mount_v3io, v3io_cred
|
|
28
30
|
from nuclio.deploy import find_dashboard_url, get_deploy_status
|
|
29
31
|
from nuclio.triggers import V3IOStreamTrigger
|
|
30
32
|
|
|
@@ -36,15 +38,11 @@ import mlrun.utils.helpers
|
|
|
36
38
|
from mlrun.common.schemas import AuthInfo
|
|
37
39
|
from mlrun.config import config as mlconf
|
|
38
40
|
from mlrun.errors import err_to_str
|
|
39
|
-
from mlrun.kfpops import deploy_op
|
|
40
41
|
from mlrun.lists import RunList
|
|
41
42
|
from mlrun.model import RunObject
|
|
42
43
|
from mlrun.platforms.iguazio import (
|
|
43
|
-
VolumeMount,
|
|
44
|
-
mount_v3io,
|
|
45
44
|
parse_path,
|
|
46
45
|
split_path,
|
|
47
|
-
v3io_cred,
|
|
48
46
|
)
|
|
49
47
|
from mlrun.runtimes.base import FunctionStatus, RunError
|
|
50
48
|
from mlrun.runtimes.pod import KubeResource, KubeResourceSpec
|
|
@@ -56,33 +54,9 @@ def validate_nuclio_version_compatibility(*min_versions):
|
|
|
56
54
|
"""
|
|
57
55
|
:param min_versions: Valid minimum version(s) required, assuming no 2 versions has equal major and minor.
|
|
58
56
|
"""
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
try:
|
|
63
|
-
parsed_current_version = semver.VersionInfo.parse(mlconf.nuclio_version)
|
|
64
|
-
except ValueError:
|
|
65
|
-
# only log when version is set but invalid
|
|
66
|
-
if mlconf.nuclio_version:
|
|
67
|
-
logger.warning(
|
|
68
|
-
"Unable to parse nuclio version, assuming compatibility",
|
|
69
|
-
nuclio_version=mlconf.nuclio_version,
|
|
70
|
-
min_versions=min_versions,
|
|
71
|
-
)
|
|
72
|
-
return True
|
|
73
|
-
|
|
74
|
-
parsed_min_versions.sort(reverse=True)
|
|
75
|
-
for parsed_min_version in parsed_min_versions:
|
|
76
|
-
if (
|
|
77
|
-
parsed_current_version.major == parsed_min_version.major
|
|
78
|
-
and parsed_current_version.minor == parsed_min_version.minor
|
|
79
|
-
and parsed_current_version.patch < parsed_min_version.patch
|
|
80
|
-
):
|
|
81
|
-
return False
|
|
82
|
-
|
|
83
|
-
if parsed_current_version >= parsed_min_version:
|
|
84
|
-
return True
|
|
85
|
-
return False
|
|
57
|
+
return mlrun.utils.helpers.validate_component_version_compatibility(
|
|
58
|
+
"nuclio", *min_versions
|
|
59
|
+
)
|
|
86
60
|
|
|
87
61
|
|
|
88
62
|
def min_nuclio_versions(*versions):
|
|
@@ -994,11 +968,11 @@ class RemoteRuntime(KubeResource):
|
|
|
994
968
|
ports = mlrun.utils.helpers.as_list(ports)
|
|
995
969
|
sidecar["ports"] = [
|
|
996
970
|
{
|
|
997
|
-
"name": "
|
|
971
|
+
"name": f"{name}-{i}",
|
|
998
972
|
"containerPort": port,
|
|
999
973
|
"protocol": "TCP",
|
|
1000
974
|
}
|
|
1001
|
-
for port in ports
|
|
975
|
+
for i, port in enumerate(ports)
|
|
1002
976
|
]
|
|
1003
977
|
|
|
1004
978
|
if command:
|
mlrun/runtimes/pod.py
CHANGED
|
@@ -20,8 +20,9 @@ import typing
|
|
|
20
20
|
from enum import Enum
|
|
21
21
|
|
|
22
22
|
import dotenv
|
|
23
|
-
import kfp.dsl
|
|
24
23
|
import kubernetes.client as k8s_client
|
|
24
|
+
import mlrun_pipelines.mounts
|
|
25
|
+
from mlrun_pipelines.mixins import KfpAdapterMixin
|
|
25
26
|
|
|
26
27
|
import mlrun.errors
|
|
27
28
|
import mlrun.utils.regex
|
|
@@ -41,7 +42,6 @@ from ..k8s_utils import (
|
|
|
41
42
|
from ..utils import logger, update_in
|
|
42
43
|
from .base import BaseRuntime, FunctionSpec, spec_fields
|
|
43
44
|
from .utils import (
|
|
44
|
-
apply_kfp,
|
|
45
45
|
get_gpu_from_resource_requirement,
|
|
46
46
|
get_item_name,
|
|
47
47
|
set_named_item,
|
|
@@ -935,12 +935,12 @@ class AutoMountType(str, Enum):
|
|
|
935
935
|
@classmethod
|
|
936
936
|
def all_mount_modifiers(cls):
|
|
937
937
|
return [
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
938
|
+
mlrun_pipelines.mounts.v3io_cred.__name__,
|
|
939
|
+
mlrun_pipelines.mounts.mount_v3io.__name__,
|
|
940
|
+
mlrun_pipelines.mounts.mount_pvc.__name__,
|
|
941
|
+
mlrun_pipelines.mounts.auto_mount.__name__,
|
|
942
|
+
mlrun_pipelines.mounts.mount_s3.__name__,
|
|
943
|
+
mlrun_pipelines.mounts.set_env_variables.__name__,
|
|
944
944
|
]
|
|
945
945
|
|
|
946
946
|
@classmethod
|
|
@@ -957,27 +957,27 @@ class AutoMountType(str, Enum):
|
|
|
957
957
|
def _get_auto_modifier():
|
|
958
958
|
# If we're running on Iguazio - use v3io_cred
|
|
959
959
|
if mlconf.igz_version != "":
|
|
960
|
-
return
|
|
960
|
+
return mlrun_pipelines.mounts.v3io_cred
|
|
961
961
|
# Else, either pvc mount if it's configured or do nothing otherwise
|
|
962
962
|
pvc_configured = (
|
|
963
963
|
"MLRUN_PVC_MOUNT" in os.environ
|
|
964
964
|
or "pvc_name" in mlconf.get_storage_auto_mount_params()
|
|
965
965
|
)
|
|
966
|
-
return
|
|
966
|
+
return mlrun_pipelines.mounts.mount_pvc if pvc_configured else None
|
|
967
967
|
|
|
968
968
|
def get_modifier(self):
|
|
969
969
|
return {
|
|
970
970
|
AutoMountType.none: None,
|
|
971
|
-
AutoMountType.v3io_credentials:
|
|
972
|
-
AutoMountType.v3io_fuse:
|
|
973
|
-
AutoMountType.pvc:
|
|
971
|
+
AutoMountType.v3io_credentials: mlrun_pipelines.mounts.v3io_cred,
|
|
972
|
+
AutoMountType.v3io_fuse: mlrun_pipelines.mounts.mount_v3io,
|
|
973
|
+
AutoMountType.pvc: mlrun_pipelines.mounts.mount_pvc,
|
|
974
974
|
AutoMountType.auto: self._get_auto_modifier(),
|
|
975
|
-
AutoMountType.s3:
|
|
976
|
-
AutoMountType.env:
|
|
975
|
+
AutoMountType.s3: mlrun_pipelines.mounts.mount_s3,
|
|
976
|
+
AutoMountType.env: mlrun_pipelines.mounts.set_env_variables,
|
|
977
977
|
}[self]
|
|
978
978
|
|
|
979
979
|
|
|
980
|
-
class KubeResource(BaseRuntime):
|
|
980
|
+
class KubeResource(BaseRuntime, KfpAdapterMixin):
|
|
981
981
|
"""
|
|
982
982
|
A parent class for runtimes that generate k8s resources when executing.
|
|
983
983
|
"""
|
|
@@ -997,26 +997,6 @@ class KubeResource(BaseRuntime):
|
|
|
997
997
|
def spec(self, spec):
|
|
998
998
|
self._spec = self._verify_dict(spec, "spec", KubeResourceSpec)
|
|
999
999
|
|
|
1000
|
-
def apply(self, modify):
|
|
1001
|
-
"""
|
|
1002
|
-
Apply a modifier to the runtime which is used to change the runtimes k8s object's spec.
|
|
1003
|
-
Modifiers can be either KFP modifiers or MLRun modifiers (which are compatible with KFP). All modifiers accept
|
|
1004
|
-
a `kfp.dsl.ContainerOp` object, apply some changes on its spec and return it so modifiers can be chained
|
|
1005
|
-
one after the other.
|
|
1006
|
-
|
|
1007
|
-
:param modify: a modifier runnable object
|
|
1008
|
-
:return: the runtime (self) after the modifications
|
|
1009
|
-
"""
|
|
1010
|
-
|
|
1011
|
-
# Kubeflow pipeline have a hook to add the component to the DAG on ContainerOp init
|
|
1012
|
-
# we remove the hook to suppress kubeflow op registration and return it after the apply()
|
|
1013
|
-
old_op_handler = kfp.dsl._container_op._register_op_handler
|
|
1014
|
-
kfp.dsl._container_op._register_op_handler = lambda x: self.metadata.name
|
|
1015
|
-
cop = kfp.dsl.ContainerOp("name", "image")
|
|
1016
|
-
kfp.dsl._container_op._register_op_handler = old_op_handler
|
|
1017
|
-
|
|
1018
|
-
return apply_kfp(modify, cop, self)
|
|
1019
|
-
|
|
1020
1000
|
def set_env_from_secret(self, name, secret=None, secret_key=None):
|
|
1021
1001
|
"""set pod environment var from secret"""
|
|
1022
1002
|
secret_key = secret_key or name
|
mlrun/runtimes/remotesparkjob.py
CHANGED
|
@@ -15,11 +15,11 @@ import re
|
|
|
15
15
|
from subprocess import run
|
|
16
16
|
|
|
17
17
|
import kubernetes.client
|
|
18
|
+
from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
|
|
18
19
|
|
|
19
20
|
import mlrun.errors
|
|
20
21
|
from mlrun.config import config
|
|
21
22
|
|
|
22
|
-
from ..platforms.iguazio import mount_v3io, mount_v3iod
|
|
23
23
|
from .kubejob import KubejobRuntime
|
|
24
24
|
from .pod import KubeResourceSpec
|
|
25
25
|
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import typing
|
|
15
15
|
|
|
16
16
|
import kubernetes.client
|
|
17
|
+
from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
|
|
17
18
|
|
|
18
19
|
import mlrun.common.schemas.function
|
|
19
20
|
import mlrun.errors
|
|
@@ -22,7 +23,6 @@ from mlrun.config import config
|
|
|
22
23
|
|
|
23
24
|
from ...execution import MLClientCtx
|
|
24
25
|
from ...model import RunObject
|
|
25
|
-
from ...platforms.iguazio import mount_v3io, mount_v3iod
|
|
26
26
|
from ...utils import update_in, verify_field_regex
|
|
27
27
|
from ..kubejob import KubejobRuntime
|
|
28
28
|
from ..pod import KubeResourceSpec
|
mlrun/runtimes/utils.py
CHANGED
|
@@ -20,7 +20,6 @@ from io import StringIO
|
|
|
20
20
|
from sys import stderr
|
|
21
21
|
|
|
22
22
|
import pandas as pd
|
|
23
|
-
from kubernetes import client
|
|
24
23
|
|
|
25
24
|
import mlrun
|
|
26
25
|
import mlrun.common.constants
|
|
@@ -280,43 +279,6 @@ def get_item_name(item, attr="name"):
|
|
|
280
279
|
return getattr(item, attr, None)
|
|
281
280
|
|
|
282
281
|
|
|
283
|
-
def apply_kfp(modify, cop, runtime):
|
|
284
|
-
modify(cop)
|
|
285
|
-
|
|
286
|
-
# Have to do it here to avoid circular dependencies
|
|
287
|
-
from .pod import AutoMountType
|
|
288
|
-
|
|
289
|
-
if AutoMountType.is_auto_modifier(modify):
|
|
290
|
-
runtime.spec.disable_auto_mount = True
|
|
291
|
-
|
|
292
|
-
api = client.ApiClient()
|
|
293
|
-
for k, v in cop.pod_labels.items():
|
|
294
|
-
runtime.metadata.labels[k] = v
|
|
295
|
-
for k, v in cop.pod_annotations.items():
|
|
296
|
-
runtime.metadata.annotations[k] = v
|
|
297
|
-
if cop.container.env:
|
|
298
|
-
env_names = [
|
|
299
|
-
e.name if hasattr(e, "name") else e["name"] for e in runtime.spec.env
|
|
300
|
-
]
|
|
301
|
-
for e in api.sanitize_for_serialization(cop.container.env):
|
|
302
|
-
name = e["name"]
|
|
303
|
-
if name in env_names:
|
|
304
|
-
runtime.spec.env[env_names.index(name)] = e
|
|
305
|
-
else:
|
|
306
|
-
runtime.spec.env.append(e)
|
|
307
|
-
env_names.append(name)
|
|
308
|
-
cop.container.env.clear()
|
|
309
|
-
|
|
310
|
-
if cop.volumes and cop.container.volume_mounts:
|
|
311
|
-
vols = api.sanitize_for_serialization(cop.volumes)
|
|
312
|
-
mounts = api.sanitize_for_serialization(cop.container.volume_mounts)
|
|
313
|
-
runtime.spec.update_vols_and_mounts(vols, mounts)
|
|
314
|
-
cop.volumes.clear()
|
|
315
|
-
cop.container.volume_mounts.clear()
|
|
316
|
-
|
|
317
|
-
return runtime
|
|
318
|
-
|
|
319
|
-
|
|
320
282
|
def verify_limits(
|
|
321
283
|
resources_field_name,
|
|
322
284
|
mem=None,
|
mlrun/utils/helpers.py
CHANGED
|
@@ -39,6 +39,7 @@ import pandas
|
|
|
39
39
|
import semver
|
|
40
40
|
import yaml
|
|
41
41
|
from dateutil import parser
|
|
42
|
+
from mlrun_pipelines.models import PipelineRun
|
|
42
43
|
from pandas._libs.tslibs.timestamps import Timedelta, Timestamp
|
|
43
44
|
from yaml.representer import RepresenterError
|
|
44
45
|
|
|
@@ -786,34 +787,6 @@ def gen_html_table(header, rows=None):
|
|
|
786
787
|
return style + '<table class="tg">\n' + out + "</table>\n\n"
|
|
787
788
|
|
|
788
789
|
|
|
789
|
-
def new_pipe_metadata(
|
|
790
|
-
artifact_path: str = None,
|
|
791
|
-
cleanup_ttl: int = None,
|
|
792
|
-
op_transformers: list[typing.Callable] = None,
|
|
793
|
-
):
|
|
794
|
-
from kfp.dsl import PipelineConf
|
|
795
|
-
|
|
796
|
-
def _set_artifact_path(task):
|
|
797
|
-
from kubernetes import client as k8s_client
|
|
798
|
-
|
|
799
|
-
task.add_env_variable(
|
|
800
|
-
k8s_client.V1EnvVar(name="MLRUN_ARTIFACT_PATH", value=artifact_path)
|
|
801
|
-
)
|
|
802
|
-
return task
|
|
803
|
-
|
|
804
|
-
conf = PipelineConf()
|
|
805
|
-
cleanup_ttl = cleanup_ttl or int(config.kfp_ttl)
|
|
806
|
-
|
|
807
|
-
if cleanup_ttl:
|
|
808
|
-
conf.set_ttl_seconds_after_finished(cleanup_ttl)
|
|
809
|
-
if artifact_path:
|
|
810
|
-
conf.add_op_transformer(_set_artifact_path)
|
|
811
|
-
if op_transformers:
|
|
812
|
-
for op_transformer in op_transformers:
|
|
813
|
-
conf.add_op_transformer(op_transformer)
|
|
814
|
-
return conf
|
|
815
|
-
|
|
816
|
-
|
|
817
790
|
def _convert_python_package_version_to_image_tag(version: typing.Optional[str]):
|
|
818
791
|
return (
|
|
819
792
|
version.replace("+", "-").replace("0.0.0-", "") if version is not None else None
|
|
@@ -1011,7 +984,7 @@ def get_workflow_url(project, id=None):
|
|
|
1011
984
|
|
|
1012
985
|
|
|
1013
986
|
def are_strings_in_exception_chain_messages(
|
|
1014
|
-
exception: Exception, strings_list
|
|
987
|
+
exception: Exception, strings_list: list[str]
|
|
1015
988
|
) -> bool:
|
|
1016
989
|
while exception is not None:
|
|
1017
990
|
if any([string in str(exception) for string in strings_list]):
|
|
@@ -1286,7 +1259,7 @@ def is_link_artifact(artifact):
|
|
|
1286
1259
|
return artifact.kind == mlrun.common.schemas.ArtifactCategories.link.value
|
|
1287
1260
|
|
|
1288
1261
|
|
|
1289
|
-
def format_run(run:
|
|
1262
|
+
def format_run(run: PipelineRun, with_project=False) -> dict:
|
|
1290
1263
|
fields = [
|
|
1291
1264
|
"id",
|
|
1292
1265
|
"name",
|
|
@@ -1323,7 +1296,7 @@ def format_run(run: dict, with_project=False) -> dict:
|
|
|
1323
1296
|
# pipelines are yet to populate the status or workflow has failed
|
|
1324
1297
|
# as observed https://jira.iguazeng.com/browse/ML-5195
|
|
1325
1298
|
# set to unknown to ensure a status is returned
|
|
1326
|
-
if run
|
|
1299
|
+
if run.get("status", None) is None:
|
|
1327
1300
|
run["status"] = inflection.titleize(
|
|
1328
1301
|
mlrun.common.runtimes.constants.RunStates.unknown
|
|
1329
1302
|
)
|
|
@@ -1583,3 +1556,44 @@ def additional_filters_warning(additional_filters, class_name):
|
|
|
1583
1556
|
f"additional_filters parameter is not supported in {class_name},"
|
|
1584
1557
|
f" parameter has been ignored."
|
|
1585
1558
|
)
|
|
1559
|
+
|
|
1560
|
+
|
|
1561
|
+
def validate_component_version_compatibility(
|
|
1562
|
+
component_name: typing.Literal["iguazio", "nuclio"], *min_versions: str
|
|
1563
|
+
):
|
|
1564
|
+
"""
|
|
1565
|
+
:param component_name: Name of the component to validate compatibility for.
|
|
1566
|
+
:param min_versions: Valid minimum version(s) required, assuming no 2 versions has equal major and minor.
|
|
1567
|
+
"""
|
|
1568
|
+
parsed_min_versions = [
|
|
1569
|
+
semver.VersionInfo.parse(min_version) for min_version in min_versions
|
|
1570
|
+
]
|
|
1571
|
+
parsed_current_version = None
|
|
1572
|
+
component_current_version = None
|
|
1573
|
+
try:
|
|
1574
|
+
if component_name == "iguazio":
|
|
1575
|
+
parsed_current_version = mlrun.mlconf.get_parsed_igz_version()
|
|
1576
|
+
component_current_version = mlrun.mlconf.igz_version
|
|
1577
|
+
if component_name == "nuclio":
|
|
1578
|
+
parsed_current_version = semver.VersionInfo.parse(
|
|
1579
|
+
mlrun.mlconf.nuclio_version
|
|
1580
|
+
)
|
|
1581
|
+
component_current_version = mlrun.mlconf.nuclio_version
|
|
1582
|
+
if not parsed_current_version:
|
|
1583
|
+
return True
|
|
1584
|
+
except ValueError:
|
|
1585
|
+
# only log when version is set but invalid
|
|
1586
|
+
if component_current_version:
|
|
1587
|
+
logger.warning(
|
|
1588
|
+
"Unable to parse current version, assuming compatibility",
|
|
1589
|
+
component_name=component_name,
|
|
1590
|
+
current_version=component_current_version,
|
|
1591
|
+
min_versions=min_versions,
|
|
1592
|
+
)
|
|
1593
|
+
return True
|
|
1594
|
+
|
|
1595
|
+
parsed_min_versions.sort(reverse=True)
|
|
1596
|
+
for parsed_min_version in parsed_min_versions:
|
|
1597
|
+
if parsed_current_version < parsed_min_version:
|
|
1598
|
+
return False
|
|
1599
|
+
return True
|