mlrun 1.6.2rc3__py3-none-any.whl → 1.6.2rc6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/common/db/sql_session.py +0 -3
- mlrun/common/model_monitoring/helpers.py +2 -4
- mlrun/common/schemas/__init__.py +0 -1
- mlrun/common/schemas/project.py +0 -2
- mlrun/config.py +10 -29
- mlrun/datastore/azure_blob.py +9 -9
- mlrun/datastore/base.py +28 -0
- mlrun/datastore/google_cloud_storage.py +6 -6
- mlrun/datastore/v3io.py +46 -70
- mlrun/db/base.py +0 -18
- mlrun/db/httpdb.py +25 -28
- mlrun/execution.py +3 -3
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +3 -3
- mlrun/frameworks/tf_keras/model_handler.py +7 -7
- mlrun/k8s_utils.py +5 -10
- mlrun/kfpops.py +10 -19
- mlrun/model.py +0 -5
- mlrun/model_monitoring/api.py +8 -8
- mlrun/model_monitoring/batch.py +1 -1
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +13 -13
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +1 -0
- mlrun/package/packagers/pandas_packagers.py +3 -3
- mlrun/package/utils/_archiver.py +1 -3
- mlrun/platforms/iguazio.py +65 -6
- mlrun/projects/pipelines.py +11 -21
- mlrun/projects/project.py +46 -65
- mlrun/runtimes/base.py +1 -24
- mlrun/runtimes/function.py +9 -9
- mlrun/runtimes/kubejob.py +3 -5
- mlrun/runtimes/local.py +2 -2
- mlrun/runtimes/mpijob/abstract.py +6 -6
- mlrun/runtimes/pod.py +3 -3
- mlrun/runtimes/serving.py +3 -3
- mlrun/runtimes/sparkjob/spark3job.py +3 -3
- mlrun/serving/remote.py +2 -4
- mlrun/utils/async_http.py +3 -3
- mlrun/utils/helpers.py +0 -8
- mlrun/utils/http.py +3 -3
- mlrun/utils/logger.py +2 -2
- mlrun/utils/notifications/notification_pusher.py +6 -6
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/METADATA +16 -14
- {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/RECORD +47 -48
- {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/WHEEL +1 -1
- mlrun/common/schemas/common.py +0 -40
- {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/LICENSE +0 -0
- {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/entry_points.txt +0 -0
- {mlrun-1.6.2rc3.dist-info → mlrun-1.6.2rc6.dist-info}/top_level.txt +0 -0
mlrun/k8s_utils.py
CHANGED
|
@@ -134,13 +134,13 @@ def sanitize_label_value(value: str) -> str:
|
|
|
134
134
|
return re.sub(r"([^a-zA-Z0-9_.-]|^[^a-zA-Z0-9]|[^a-zA-Z0-9]$)", "-", value[:63])
|
|
135
135
|
|
|
136
136
|
|
|
137
|
-
def verify_label_key(key
|
|
138
|
-
"""
|
|
139
|
-
Verify that the label key is valid for Kubernetes.
|
|
140
|
-
Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
|
|
141
|
-
"""
|
|
137
|
+
def verify_label_key(key):
|
|
142
138
|
if not key:
|
|
143
139
|
raise mlrun.errors.MLRunInvalidArgumentError("label key cannot be empty")
|
|
140
|
+
if key.startswith("k8s.io") or key.startswith("kubernetes.io"):
|
|
141
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
142
|
+
"Labels cannot start with 'k8s.io' or 'kubernetes.io'"
|
|
143
|
+
)
|
|
144
144
|
|
|
145
145
|
mlrun.utils.helpers.verify_field_regex(
|
|
146
146
|
f"project.metadata.labels.'{key}'",
|
|
@@ -148,11 +148,6 @@ def verify_label_key(key: str):
|
|
|
148
148
|
mlrun.utils.regex.k8s_character_limit,
|
|
149
149
|
)
|
|
150
150
|
|
|
151
|
-
if key.startswith("k8s.io/") or key.startswith("kubernetes.io/"):
|
|
152
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
153
|
-
"Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
|
|
154
|
-
)
|
|
155
|
-
|
|
156
151
|
parts = key.split("/")
|
|
157
152
|
if len(parts) == 1:
|
|
158
153
|
name = parts[0]
|
mlrun/kfpops.py
CHANGED
|
@@ -41,8 +41,8 @@ from .utils import (
|
|
|
41
41
|
|
|
42
42
|
# default KFP artifacts and output (ui metadata, metrics etc.)
|
|
43
43
|
# directories to /tmp to allow running with security context
|
|
44
|
-
KFPMETA_DIR = "/tmp"
|
|
45
|
-
KFP_ARTIFACTS_DIR = "/tmp"
|
|
44
|
+
KFPMETA_DIR = os.environ.get("KFPMETA_OUT_DIR", "/tmp")
|
|
45
|
+
KFP_ARTIFACTS_DIR = os.environ.get("KFP_ARTIFACTS_DIR", "/tmp")
|
|
46
46
|
|
|
47
47
|
project_annotation = "mlrun/project"
|
|
48
48
|
run_annotation = "mlrun/pipeline-step-type"
|
|
@@ -71,7 +71,7 @@ def write_kfpmeta(struct):
|
|
|
71
71
|
{"name": k, "numberValue": v} for k, v in results.items() if is_num(v)
|
|
72
72
|
],
|
|
73
73
|
}
|
|
74
|
-
with open(
|
|
74
|
+
with open(KFPMETA_DIR + "/mlpipeline-metrics.json", "w") as f:
|
|
75
75
|
json.dump(metrics, f)
|
|
76
76
|
|
|
77
77
|
struct = deepcopy(struct)
|
|
@@ -91,14 +91,7 @@ def write_kfpmeta(struct):
|
|
|
91
91
|
elif key in results:
|
|
92
92
|
val = results[key]
|
|
93
93
|
try:
|
|
94
|
-
|
|
95
|
-
path = os.path.join(KFP_ARTIFACTS_DIR, key)
|
|
96
|
-
if not mlrun.utils.helpers.is_safe_path(KFP_ARTIFACTS_DIR, path):
|
|
97
|
-
logger.warning(
|
|
98
|
-
"Path traversal is not allowed ignoring", path=path, key=key
|
|
99
|
-
)
|
|
100
|
-
continue
|
|
101
|
-
path = os.path.abspath(path)
|
|
94
|
+
path = "/".join([KFP_ARTIFACTS_DIR, key])
|
|
102
95
|
logger.info("Writing artifact output", path=path, val=val)
|
|
103
96
|
with open(path, "w") as fp:
|
|
104
97
|
fp.write(str(val))
|
|
@@ -116,7 +109,7 @@ def write_kfpmeta(struct):
|
|
|
116
109
|
"outputs": output_artifacts
|
|
117
110
|
+ [{"type": "markdown", "storage": "inline", "source": text}]
|
|
118
111
|
}
|
|
119
|
-
with open(
|
|
112
|
+
with open(KFPMETA_DIR + "/mlpipeline-ui-metadata.json", "w") as f:
|
|
120
113
|
json.dump(metadata, f)
|
|
121
114
|
|
|
122
115
|
|
|
@@ -408,9 +401,9 @@ def mlrun_op(
|
|
|
408
401
|
cmd += ["--label", f"{label}={val}"]
|
|
409
402
|
for output in outputs:
|
|
410
403
|
cmd += ["-o", str(output)]
|
|
411
|
-
file_outputs[
|
|
412
|
-
|
|
413
|
-
|
|
404
|
+
file_outputs[
|
|
405
|
+
output.replace(".", "_")
|
|
406
|
+
] = f"/tmp/{output}" # not using path.join to avoid windows "\"
|
|
414
407
|
if project:
|
|
415
408
|
cmd += ["--project", project]
|
|
416
409
|
if handler:
|
|
@@ -457,10 +450,8 @@ def mlrun_op(
|
|
|
457
450
|
command=cmd + [command],
|
|
458
451
|
file_outputs=file_outputs,
|
|
459
452
|
output_artifact_paths={
|
|
460
|
-
"mlpipeline-ui-metadata":
|
|
461
|
-
|
|
462
|
-
),
|
|
463
|
-
"mlpipeline-metrics": os.path.join(KFPMETA_DIR, "mlpipeline-metrics.json"),
|
|
453
|
+
"mlpipeline-ui-metadata": KFPMETA_DIR + "/mlpipeline-ui-metadata.json",
|
|
454
|
+
"mlpipeline-metrics": KFPMETA_DIR + "/mlpipeline-metrics.json",
|
|
464
455
|
},
|
|
465
456
|
)
|
|
466
457
|
cop = add_default_function_resources(cop)
|
mlrun/model.py
CHANGED
|
@@ -359,7 +359,6 @@ class ImageBuilder(ModelObj):
|
|
|
359
359
|
requirements: list = None,
|
|
360
360
|
extra_args=None,
|
|
361
361
|
builder_env=None,
|
|
362
|
-
source_code_target_dir=None,
|
|
363
362
|
):
|
|
364
363
|
self.functionSourceCode = functionSourceCode #: functionSourceCode
|
|
365
364
|
self.codeEntryType = "" #: codeEntryType
|
|
@@ -380,7 +379,6 @@ class ImageBuilder(ModelObj):
|
|
|
380
379
|
self.auto_build = auto_build #: auto_build
|
|
381
380
|
self.build_pod = None
|
|
382
381
|
self.requirements = requirements or [] #: pip requirements
|
|
383
|
-
self.source_code_target_dir = source_code_target_dir or None
|
|
384
382
|
|
|
385
383
|
@property
|
|
386
384
|
def source(self):
|
|
@@ -417,7 +415,6 @@ class ImageBuilder(ModelObj):
|
|
|
417
415
|
overwrite=False,
|
|
418
416
|
builder_env=None,
|
|
419
417
|
extra_args=None,
|
|
420
|
-
source_code_target_dir=None,
|
|
421
418
|
):
|
|
422
419
|
if image:
|
|
423
420
|
self.image = image
|
|
@@ -443,8 +440,6 @@ class ImageBuilder(ModelObj):
|
|
|
443
440
|
self.builder_env = builder_env
|
|
444
441
|
if extra_args:
|
|
445
442
|
self.extra_args = extra_args
|
|
446
|
-
if source_code_target_dir:
|
|
447
|
-
self.source_code_target_dir = source_code_target_dir
|
|
448
443
|
|
|
449
444
|
def with_commands(
|
|
450
445
|
self,
|
mlrun/model_monitoring/api.py
CHANGED
|
@@ -436,9 +436,9 @@ def _generate_model_endpoint(
|
|
|
436
436
|
] = possible_drift_threshold
|
|
437
437
|
|
|
438
438
|
model_endpoint.spec.monitoring_mode = monitoring_mode
|
|
439
|
-
model_endpoint.status.first_request =
|
|
440
|
-
|
|
441
|
-
)
|
|
439
|
+
model_endpoint.status.first_request = (
|
|
440
|
+
model_endpoint.status.last_request
|
|
441
|
+
) = datetime_now().isoformat()
|
|
442
442
|
if sample_set_statistics:
|
|
443
443
|
model_endpoint.status.feature_stats = sample_set_statistics
|
|
444
444
|
|
|
@@ -476,11 +476,11 @@ def trigger_drift_batch_job(
|
|
|
476
476
|
db_session = mlrun.get_run_db()
|
|
477
477
|
|
|
478
478
|
# Register the monitoring batch job (do nothing if already exist) and get the job function as a dictionary
|
|
479
|
-
batch_function_dict: typing.Dict[
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
479
|
+
batch_function_dict: typing.Dict[
|
|
480
|
+
str, typing.Any
|
|
481
|
+
] = db_session.deploy_monitoring_batch_job(
|
|
482
|
+
project=project,
|
|
483
|
+
default_batch_image=default_batch_image,
|
|
484
484
|
)
|
|
485
485
|
|
|
486
486
|
# Prepare current run params
|
mlrun/model_monitoring/batch.py
CHANGED
|
@@ -540,24 +540,24 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
540
540
|
and endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.METRICS]
|
|
541
541
|
== "null"
|
|
542
542
|
):
|
|
543
|
-
endpoint[
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
543
|
+
endpoint[
|
|
544
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.METRICS
|
|
545
|
+
] = json.dumps(
|
|
546
|
+
{
|
|
547
|
+
mlrun.common.schemas.model_monitoring.EventKeyMetrics.GENERIC: {
|
|
548
|
+
mlrun.common.schemas.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
|
|
549
|
+
mlrun.common.schemas.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
|
|
550
550
|
}
|
|
551
|
-
|
|
551
|
+
}
|
|
552
552
|
)
|
|
553
553
|
# Validate key `uid` instead of `endpoint_id`
|
|
554
554
|
# For backwards compatibility reasons, we replace the `endpoint_id` with `uid` which is the updated key name
|
|
555
555
|
if mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID in endpoint:
|
|
556
|
-
endpoint[
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
556
|
+
endpoint[
|
|
557
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.UID
|
|
558
|
+
] = endpoint[
|
|
559
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
|
|
560
|
+
]
|
|
561
561
|
|
|
562
562
|
@staticmethod
|
|
563
563
|
def _encode_field(field: typing.Union[str, bytes]) -> bytes:
|
|
@@ -31,6 +31,7 @@ from .models import get_model_endpoints_table
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
class SQLModelEndpointStore(ModelEndpointStore):
|
|
34
|
+
|
|
34
35
|
"""
|
|
35
36
|
Handles the DB operations when the DB target is from type SQL. For the SQL operations, we use SQLAlchemy, a Python
|
|
36
37
|
SQL toolkit that handles the communication with the database. When using SQL for storing the model endpoints
|
|
@@ -838,9 +838,9 @@ class PandasDataFramePackager(DefaultPackager):
|
|
|
838
838
|
"""
|
|
839
839
|
if isinstance(obj, dict):
|
|
840
840
|
for key, value in obj.items():
|
|
841
|
-
obj[
|
|
842
|
-
PandasDataFramePackager._prepare_result(obj=
|
|
843
|
-
)
|
|
841
|
+
obj[
|
|
842
|
+
PandasDataFramePackager._prepare_result(obj=key)
|
|
843
|
+
] = PandasDataFramePackager._prepare_result(obj=value)
|
|
844
844
|
elif isinstance(obj, list):
|
|
845
845
|
for i, value in enumerate(obj):
|
|
846
846
|
obj[i] = PandasDataFramePackager._prepare_result(obj=value)
|
mlrun/package/utils/_archiver.py
CHANGED
|
@@ -179,9 +179,7 @@ class _TarArchiver(_Archiver):
|
|
|
179
179
|
|
|
180
180
|
# Extract:
|
|
181
181
|
with tarfile.open(archive_path, f"r:{cls._MODE_STRING}") as tar_file:
|
|
182
|
-
|
|
183
|
-
# see: https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractall
|
|
184
|
-
tar_file.extractall(directory_path, filter="data")
|
|
182
|
+
tar_file.extractall(directory_path)
|
|
185
183
|
|
|
186
184
|
return str(directory_path)
|
|
187
185
|
|
mlrun/platforms/iguazio.py
CHANGED
|
@@ -16,15 +16,19 @@ import json
|
|
|
16
16
|
import os
|
|
17
17
|
import urllib
|
|
18
18
|
from collections import namedtuple
|
|
19
|
+
from datetime import datetime
|
|
20
|
+
from http import HTTPStatus
|
|
19
21
|
from urllib.parse import urlparse
|
|
20
22
|
|
|
21
23
|
import kfp.dsl
|
|
22
24
|
import requests
|
|
23
25
|
import semver
|
|
26
|
+
import urllib3
|
|
24
27
|
import v3io
|
|
25
28
|
|
|
26
29
|
import mlrun.errors
|
|
27
30
|
from mlrun.config import config as mlconf
|
|
31
|
+
from mlrun.errors import err_to_str
|
|
28
32
|
from mlrun.utils import dict_to_json
|
|
29
33
|
|
|
30
34
|
_cached_control_session = None
|
|
@@ -484,6 +488,25 @@ class V3ioStreamClient:
|
|
|
484
488
|
return response.output.records
|
|
485
489
|
|
|
486
490
|
|
|
491
|
+
def create_control_session(url, username, password):
|
|
492
|
+
# for systems without production cert - silence no cert verification WARN
|
|
493
|
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
494
|
+
if not username or not password:
|
|
495
|
+
raise ValueError("cannot create session key, missing username or password")
|
|
496
|
+
|
|
497
|
+
session = requests.Session()
|
|
498
|
+
session.auth = (username, password)
|
|
499
|
+
try:
|
|
500
|
+
auth = session.post(f"{url}/api/sessions", verify=False)
|
|
501
|
+
except OSError as exc:
|
|
502
|
+
raise OSError(f"error: cannot connect to {url}: {err_to_str(exc)}")
|
|
503
|
+
|
|
504
|
+
if not auth.ok:
|
|
505
|
+
raise OSError(f"failed to create session: {url}, {auth.text}")
|
|
506
|
+
|
|
507
|
+
return auth.json()["data"]["id"]
|
|
508
|
+
|
|
509
|
+
|
|
487
510
|
def is_iguazio_endpoint(endpoint_url: str) -> bool:
|
|
488
511
|
# TODO: find a better heuristic
|
|
489
512
|
return ".default-tenant." in endpoint_url
|
|
@@ -510,6 +533,21 @@ def is_iguazio_session_cookie(session_cookie: str) -> bool:
|
|
|
510
533
|
return False
|
|
511
534
|
|
|
512
535
|
|
|
536
|
+
def is_iguazio_system_2_10_or_above(dashboard_url):
|
|
537
|
+
# for systems without production cert - silence no cert verification WARN
|
|
538
|
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
539
|
+
response = requests.get(f"{dashboard_url}/api/external_versions", verify=False)
|
|
540
|
+
|
|
541
|
+
if not response.ok:
|
|
542
|
+
if response.status_code == HTTPStatus.NOT_FOUND.value:
|
|
543
|
+
# in iguazio systems prior to 2.10 this endpoint didn't exist, so the api returns 404 cause endpoint not
|
|
544
|
+
# found
|
|
545
|
+
return False
|
|
546
|
+
response.raise_for_status()
|
|
547
|
+
|
|
548
|
+
return True
|
|
549
|
+
|
|
550
|
+
|
|
513
551
|
# we assign the control session or access key to the password since this is iguazio auth scheme
|
|
514
552
|
# (requests should be sent with username:control_session/access_key as auth header)
|
|
515
553
|
def add_or_refresh_credentials(
|
|
@@ -539,12 +577,33 @@ def add_or_refresh_credentials(
|
|
|
539
577
|
# (ideally if we could identify we're in enterprise we would have verify here that token and username have value)
|
|
540
578
|
if not is_iguazio_endpoint(api_url):
|
|
541
579
|
return "", "", token
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
580
|
+
iguazio_dashboard_url = "https://dashboard" + api_url[api_url.find(".") :]
|
|
581
|
+
|
|
582
|
+
# in 2.8 mlrun api is protected with control session, from 2.10 it's protected with access key
|
|
583
|
+
is_access_key_auth = is_iguazio_system_2_10_or_above(iguazio_dashboard_url)
|
|
584
|
+
if is_access_key_auth:
|
|
585
|
+
if not username or not token:
|
|
586
|
+
raise ValueError(
|
|
587
|
+
"username and access key required to authenticate against iguazio system"
|
|
588
|
+
)
|
|
589
|
+
return username, token, ""
|
|
590
|
+
|
|
591
|
+
if not username or not password:
|
|
592
|
+
raise ValueError("username and password needed to create session")
|
|
593
|
+
|
|
594
|
+
global _cached_control_session
|
|
595
|
+
now = datetime.now()
|
|
596
|
+
if _cached_control_session:
|
|
597
|
+
if (
|
|
598
|
+
_cached_control_session[2] == username
|
|
599
|
+
and _cached_control_session[3] == password
|
|
600
|
+
and (now - _cached_control_session[1]).seconds < 20 * 60 * 60
|
|
601
|
+
):
|
|
602
|
+
return _cached_control_session[2], _cached_control_session[0], ""
|
|
603
|
+
|
|
604
|
+
control_session = create_control_session(iguazio_dashboard_url, username, password)
|
|
605
|
+
_cached_control_session = (control_session, now, username, password)
|
|
606
|
+
return username, control_session, ""
|
|
548
607
|
|
|
549
608
|
|
|
550
609
|
def parse_path(url, suffix="/"):
|
mlrun/projects/pipelines.py
CHANGED
|
@@ -69,16 +69,16 @@ class WorkflowSpec(mlrun.model.ModelObj):
|
|
|
69
69
|
|
|
70
70
|
def __init__(
|
|
71
71
|
self,
|
|
72
|
-
engine
|
|
73
|
-
code
|
|
74
|
-
path
|
|
75
|
-
args
|
|
76
|
-
name
|
|
77
|
-
handler
|
|
78
|
-
args_schema:
|
|
72
|
+
engine=None,
|
|
73
|
+
code=None,
|
|
74
|
+
path=None,
|
|
75
|
+
args=None,
|
|
76
|
+
name=None,
|
|
77
|
+
handler=None,
|
|
78
|
+
args_schema: dict = None,
|
|
79
79
|
schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
|
|
80
|
-
cleanup_ttl:
|
|
81
|
-
image:
|
|
80
|
+
cleanup_ttl: int = None,
|
|
81
|
+
image: str = None,
|
|
82
82
|
):
|
|
83
83
|
self.engine = engine
|
|
84
84
|
self.code = code
|
|
@@ -401,9 +401,6 @@ def enrich_function_object(
|
|
|
401
401
|
else:
|
|
402
402
|
f.spec.build.source = project.spec.source
|
|
403
403
|
f.spec.build.load_source_on_run = project.spec.load_source_on_run
|
|
404
|
-
f.spec.build.source_code_target_dir = (
|
|
405
|
-
project.spec.build.source_code_target_dir
|
|
406
|
-
)
|
|
407
404
|
f.spec.workdir = project.spec.workdir or project.spec.subpath
|
|
408
405
|
f.prepare_image_for_deploy()
|
|
409
406
|
|
|
@@ -865,11 +862,6 @@ class _RemoteRunner(_PipelineRunner):
|
|
|
865
862
|
)
|
|
866
863
|
return
|
|
867
864
|
|
|
868
|
-
logger.debug(
|
|
869
|
-
"Workflow submitted, waiting for pipeline run to start",
|
|
870
|
-
workflow_name=workflow_response.name,
|
|
871
|
-
)
|
|
872
|
-
|
|
873
865
|
# Getting workflow id from run:
|
|
874
866
|
response = retry_until_successful(
|
|
875
867
|
1,
|
|
@@ -996,7 +988,6 @@ def load_and_run(
|
|
|
996
988
|
cleanup_ttl: int = None,
|
|
997
989
|
load_only: bool = False,
|
|
998
990
|
wait_for_completion: bool = False,
|
|
999
|
-
project_context: str = None,
|
|
1000
991
|
):
|
|
1001
992
|
"""
|
|
1002
993
|
Auxiliary function that the RemoteRunner run once or run every schedule.
|
|
@@ -1027,11 +1018,10 @@ def load_and_run(
|
|
|
1027
1018
|
workflow and all its resources are deleted)
|
|
1028
1019
|
:param load_only: for just loading the project, inner use.
|
|
1029
1020
|
:param wait_for_completion: wait for workflow completion before returning
|
|
1030
|
-
:param project_context: project context path (used for loading the project)
|
|
1031
1021
|
"""
|
|
1032
1022
|
try:
|
|
1033
1023
|
project = mlrun.load_project(
|
|
1034
|
-
context=
|
|
1024
|
+
context=f"./{project_name}",
|
|
1035
1025
|
url=url,
|
|
1036
1026
|
name=project_name,
|
|
1037
1027
|
init_git=init_git,
|
|
@@ -1063,7 +1053,7 @@ def load_and_run(
|
|
|
1063
1053
|
|
|
1064
1054
|
raise error
|
|
1065
1055
|
|
|
1066
|
-
context.logger.info(f"Loaded project {project.name} successfully")
|
|
1056
|
+
context.logger.info(f"Loaded project {project.name} from remote successfully")
|
|
1067
1057
|
|
|
1068
1058
|
if load_only:
|
|
1069
1059
|
return
|
mlrun/projects/project.py
CHANGED
|
@@ -24,7 +24,7 @@ import typing
|
|
|
24
24
|
import uuid
|
|
25
25
|
import warnings
|
|
26
26
|
import zipfile
|
|
27
|
-
from os import environ, makedirs, path
|
|
27
|
+
from os import environ, makedirs, path, remove
|
|
28
28
|
from typing import Callable, Dict, List, Optional, Union
|
|
29
29
|
|
|
30
30
|
import dotenv
|
|
@@ -605,14 +605,9 @@ def _load_project_dir(context, name="", subpath=""):
|
|
|
605
605
|
# If there is a setup script do not force having project.yaml file
|
|
606
606
|
project = MlrunProject()
|
|
607
607
|
else:
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
message,
|
|
611
|
-
context=context,
|
|
612
|
-
name=name,
|
|
613
|
-
subpath=subpath,
|
|
608
|
+
raise mlrun.errors.MLRunNotFoundError(
|
|
609
|
+
"project or function YAML not found in path"
|
|
614
610
|
)
|
|
615
|
-
raise mlrun.errors.MLRunNotFoundError(message)
|
|
616
611
|
|
|
617
612
|
project.spec.context = context
|
|
618
613
|
project.metadata.name = name or project.metadata.name
|
|
@@ -1240,20 +1235,20 @@ class MlrunProject(ModelObj):
|
|
|
1240
1235
|
self,
|
|
1241
1236
|
name,
|
|
1242
1237
|
workflow_path: str,
|
|
1243
|
-
embed
|
|
1244
|
-
engine
|
|
1245
|
-
args_schema:
|
|
1246
|
-
handler
|
|
1238
|
+
embed=False,
|
|
1239
|
+
engine=None,
|
|
1240
|
+
args_schema: typing.List[EntrypointParam] = None,
|
|
1241
|
+
handler=None,
|
|
1247
1242
|
schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
|
|
1248
|
-
ttl
|
|
1249
|
-
image:
|
|
1243
|
+
ttl=None,
|
|
1244
|
+
image: str = None,
|
|
1250
1245
|
**args,
|
|
1251
1246
|
):
|
|
1252
1247
|
"""Add or update a workflow, specify a name and the code path
|
|
1253
1248
|
|
|
1254
1249
|
:param name: Name of the workflow
|
|
1255
1250
|
:param workflow_path: URL (remote) / Path (absolute or relative to the project code path i.e.
|
|
1256
|
-
|
|
1251
|
+
<project.spec.get_code_path()>/<workflow_path>) for the workflow file.
|
|
1257
1252
|
:param embed: Add the workflow code into the project.yaml
|
|
1258
1253
|
:param engine: Workflow processing engine ("kfp", "local", "remote" or "remote:local")
|
|
1259
1254
|
:param args_schema: List of arg schema definitions (:py:class`~mlrun.model.EntrypointParam`)
|
|
@@ -2600,45 +2595,40 @@ class MlrunProject(ModelObj):
|
|
|
2600
2595
|
cleanup_ttl: int = None,
|
|
2601
2596
|
notifications: typing.List[mlrun.model.Notification] = None,
|
|
2602
2597
|
) -> _PipelineRunStatus:
|
|
2603
|
-
"""
|
|
2598
|
+
"""run a workflow using kubeflow pipelines
|
|
2604
2599
|
|
|
2605
|
-
:param name:
|
|
2600
|
+
:param name: name of the workflow
|
|
2606
2601
|
:param workflow_path:
|
|
2607
|
-
|
|
2602
|
+
url to a workflow file, if not a project workflow
|
|
2608
2603
|
:param arguments:
|
|
2609
|
-
|
|
2604
|
+
kubeflow pipelines arguments (parameters)
|
|
2610
2605
|
:param artifact_path:
|
|
2611
|
-
|
|
2606
|
+
target path/url for workflow artifacts, the string
|
|
2612
2607
|
'{{workflow.uid}}' will be replaced by workflow id
|
|
2613
2608
|
:param workflow_handler:
|
|
2614
|
-
|
|
2615
|
-
:param namespace:
|
|
2616
|
-
:param sync:
|
|
2617
|
-
:param watch:
|
|
2618
|
-
:param dirty:
|
|
2619
|
-
:param engine:
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
:param local:
|
|
2609
|
+
workflow function handler (for running workflow function directly)
|
|
2610
|
+
:param namespace: kubernetes namespace if other than default
|
|
2611
|
+
:param sync: force functions sync before run
|
|
2612
|
+
:param watch: wait for pipeline completion
|
|
2613
|
+
:param dirty: allow running the workflow when the git repo is dirty
|
|
2614
|
+
:param engine: workflow engine running the workflow.
|
|
2615
|
+
supported values are 'kfp' (default), 'local' or 'remote'.
|
|
2616
|
+
for setting engine for remote running use 'remote:local' or 'remote:kfp'.
|
|
2617
|
+
:param local: run local pipeline with local functions (set local=True in function.run())
|
|
2623
2618
|
:param schedule: ScheduleCronTrigger class instance or a standard crontab expression string
|
|
2624
2619
|
(which will be converted to the class using its `from_crontab` constructor),
|
|
2625
2620
|
see this link for help:
|
|
2626
2621
|
https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron
|
|
2627
2622
|
for using the pre-defined workflow's schedule, set `schedule=True`
|
|
2628
|
-
:param timeout:
|
|
2629
|
-
:param source:
|
|
2630
|
-
|
|
2631
|
-
1. Remote URL which is loaded dynamically to the workflow runner.
|
|
2632
|
-
2. A path to the project's context on the workflow runner's image.
|
|
2633
|
-
Path can be absolute or relative to `project.spec.build.source_code_target_dir` if defined
|
|
2634
|
-
(enriched when building a project image with source, see `MlrunProject.build_image`).
|
|
2635
|
-
For other engines the source is used to validate that the code is up-to-date.
|
|
2623
|
+
:param timeout: timeout in seconds to wait for pipeline completion (watch will be activated)
|
|
2624
|
+
:param source: remote source to use instead of the actual `project.spec.source` (used when engine is remote).
|
|
2625
|
+
for other engines the source is to validate that the code is up-to-date
|
|
2636
2626
|
:param cleanup_ttl:
|
|
2637
|
-
|
|
2638
|
-
|
|
2627
|
+
pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
|
|
2628
|
+
workflow and all its resources are deleted)
|
|
2639
2629
|
:param notifications:
|
|
2640
|
-
|
|
2641
|
-
:returns:
|
|
2630
|
+
list of notifications to send for workflow completion
|
|
2631
|
+
:returns: run id
|
|
2642
2632
|
"""
|
|
2643
2633
|
|
|
2644
2634
|
arguments = arguments or {}
|
|
@@ -2785,7 +2775,7 @@ class MlrunProject(ModelObj):
|
|
|
2785
2775
|
def export(self, filepath=None, include_files: str = None):
|
|
2786
2776
|
"""save the project object into a yaml file or zip archive (default to project.yaml)
|
|
2787
2777
|
|
|
2788
|
-
By default
|
|
2778
|
+
By default the project object is exported to a yaml file, when the filepath suffix is '.zip'
|
|
2789
2779
|
the project context dir (code files) are also copied into the zip, the archive path can include
|
|
2790
2780
|
DataItem urls (for remote object storage, e.g. s3://<bucket>/<path>).
|
|
2791
2781
|
|
|
@@ -2810,19 +2800,19 @@ class MlrunProject(ModelObj):
|
|
|
2810
2800
|
|
|
2811
2801
|
if archive_code:
|
|
2812
2802
|
files_filter = include_files or "**"
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2803
|
+
tmp_path = None
|
|
2804
|
+
if "://" in filepath:
|
|
2805
|
+
tmp_path = tempfile.mktemp(".zip")
|
|
2806
|
+
zipf = zipfile.ZipFile(tmp_path or filepath, "w")
|
|
2807
|
+
for file_path in glob.iglob(
|
|
2808
|
+
f"{project_dir}/{files_filter}", recursive=True
|
|
2809
|
+
):
|
|
2810
|
+
write_path = pathlib.Path(file_path)
|
|
2811
|
+
zipf.write(write_path, arcname=write_path.relative_to(project_dir))
|
|
2812
|
+
zipf.close()
|
|
2813
|
+
if tmp_path:
|
|
2814
|
+
mlrun.get_dataitem(filepath).upload(tmp_path)
|
|
2815
|
+
remove(tmp_path)
|
|
2826
2816
|
|
|
2827
2817
|
def set_model_monitoring_credentials(
|
|
2828
2818
|
self,
|
|
@@ -3037,7 +3027,6 @@ class MlrunProject(ModelObj):
|
|
|
3037
3027
|
requirements_file: str = None,
|
|
3038
3028
|
builder_env: dict = None,
|
|
3039
3029
|
extra_args: str = None,
|
|
3040
|
-
source_code_target_dir: str = None,
|
|
3041
3030
|
):
|
|
3042
3031
|
"""specify builder configuration for the project
|
|
3043
3032
|
|
|
@@ -3058,8 +3047,6 @@ class MlrunProject(ModelObj):
|
|
|
3058
3047
|
e.g. builder_env={"GIT_TOKEN": token}, does not work yet in KFP
|
|
3059
3048
|
:param extra_args: A string containing additional builder arguments in the format of command-line options,
|
|
3060
3049
|
e.g. extra_args="--skip-tls-verify --build-arg A=val"
|
|
3061
|
-
:param source_code_target_dir: Path on the image where source code would be extracted
|
|
3062
|
-
(by default `/home/mlrun_code`)
|
|
3063
3050
|
"""
|
|
3064
3051
|
if not overwrite_build_params:
|
|
3065
3052
|
# TODO: change overwrite_build_params default to True in 1.8.0
|
|
@@ -3083,7 +3070,6 @@ class MlrunProject(ModelObj):
|
|
|
3083
3070
|
overwrite=overwrite_build_params,
|
|
3084
3071
|
builder_env=builder_env,
|
|
3085
3072
|
extra_args=extra_args,
|
|
3086
|
-
source_code_target_dir=source_code_target_dir,
|
|
3087
3073
|
)
|
|
3088
3074
|
|
|
3089
3075
|
if set_as_default and image != self.default_image:
|
|
@@ -3130,7 +3116,7 @@ class MlrunProject(ModelObj):
|
|
|
3130
3116
|
* False: The new params are merged with the existing
|
|
3131
3117
|
* True: The existing params are replaced by the new ones
|
|
3132
3118
|
:param extra_args: A string containing additional builder arguments in the format of command-line options,
|
|
3133
|
-
e.g. extra_args="--skip-tls-verify --build-arg A=val"
|
|
3119
|
+
e.g. extra_args="--skip-tls-verify --build-arg A=val"r
|
|
3134
3120
|
:param target_dir: Path on the image where source code would be extracted (by default `/home/mlrun_code`)
|
|
3135
3121
|
"""
|
|
3136
3122
|
if not base_image:
|
|
@@ -3198,11 +3184,6 @@ class MlrunProject(ModelObj):
|
|
|
3198
3184
|
force_build=True,
|
|
3199
3185
|
)
|
|
3200
3186
|
|
|
3201
|
-
# Get the enriched target dir from the function
|
|
3202
|
-
self.spec.build.source_code_target_dir = (
|
|
3203
|
-
function.spec.build.source_code_target_dir
|
|
3204
|
-
)
|
|
3205
|
-
|
|
3206
3187
|
try:
|
|
3207
3188
|
mlrun.db.get_run_db(secrets=self._secrets).delete_function(
|
|
3208
3189
|
name=function.metadata.name
|