mlrun 1.6.2rc5__py3-none-any.whl → 1.6.2rc6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/common/db/sql_session.py +0 -3
- mlrun/common/model_monitoring/helpers.py +2 -4
- mlrun/common/schemas/__init__.py +0 -1
- mlrun/common/schemas/project.py +0 -2
- mlrun/config.py +11 -30
- mlrun/datastore/azure_blob.py +9 -9
- mlrun/datastore/base.py +44 -22
- mlrun/datastore/google_cloud_storage.py +6 -6
- mlrun/datastore/v3io.py +46 -70
- mlrun/db/base.py +0 -18
- mlrun/db/httpdb.py +25 -28
- mlrun/execution.py +3 -3
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +3 -3
- mlrun/frameworks/tf_keras/model_handler.py +7 -7
- mlrun/k8s_utils.py +5 -10
- mlrun/kfpops.py +10 -19
- mlrun/model.py +0 -5
- mlrun/model_monitoring/api.py +8 -8
- mlrun/model_monitoring/batch.py +1 -1
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +13 -13
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +1 -0
- mlrun/package/packagers/pandas_packagers.py +3 -3
- mlrun/package/utils/_archiver.py +1 -3
- mlrun/platforms/iguazio.py +65 -6
- mlrun/projects/pipelines.py +11 -21
- mlrun/projects/project.py +46 -65
- mlrun/runtimes/base.py +1 -24
- mlrun/runtimes/function.py +9 -9
- mlrun/runtimes/kubejob.py +3 -5
- mlrun/runtimes/local.py +2 -2
- mlrun/runtimes/mpijob/abstract.py +6 -6
- mlrun/runtimes/pod.py +3 -3
- mlrun/runtimes/serving.py +3 -3
- mlrun/runtimes/sparkjob/spark3job.py +3 -3
- mlrun/serving/remote.py +2 -4
- mlrun/utils/async_http.py +3 -3
- mlrun/utils/helpers.py +0 -8
- mlrun/utils/http.py +3 -3
- mlrun/utils/logger.py +2 -2
- mlrun/utils/notifications/notification_pusher.py +6 -6
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.6.2rc5.dist-info → mlrun-1.6.2rc6.dist-info}/METADATA +16 -14
- {mlrun-1.6.2rc5.dist-info → mlrun-1.6.2rc6.dist-info}/RECORD +47 -48
- mlrun/common/schemas/common.py +0 -40
- {mlrun-1.6.2rc5.dist-info → mlrun-1.6.2rc6.dist-info}/LICENSE +0 -0
- {mlrun-1.6.2rc5.dist-info → mlrun-1.6.2rc6.dist-info}/WHEEL +0 -0
- {mlrun-1.6.2rc5.dist-info → mlrun-1.6.2rc6.dist-info}/entry_points.txt +0 -0
- {mlrun-1.6.2rc5.dist-info → mlrun-1.6.2rc6.dist-info}/top_level.txt +0 -0
mlrun/db/httpdb.py
CHANGED
|
@@ -152,7 +152,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
152
152
|
@staticmethod
|
|
153
153
|
def get_api_path_prefix(version: str = None) -> str:
|
|
154
154
|
"""
|
|
155
|
-
:param version: API version to use, None (the default) will mean to use the default value from
|
|
155
|
+
:param version: API version to use, None (the default) will mean to use the default value from mlconf,
|
|
156
156
|
for un-versioned api set an empty string.
|
|
157
157
|
"""
|
|
158
158
|
if version is not None:
|
|
@@ -250,11 +250,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
250
250
|
|
|
251
251
|
try:
|
|
252
252
|
response = self.session.request(
|
|
253
|
-
method,
|
|
254
|
-
url,
|
|
255
|
-
timeout=timeout,
|
|
256
|
-
verify=config.httpdb.http.verify,
|
|
257
|
-
**kw,
|
|
253
|
+
method, url, timeout=timeout, verify=False, **kw
|
|
258
254
|
)
|
|
259
255
|
except requests.RequestException as exc:
|
|
260
256
|
error = f"{err_to_str(exc)}: {error}" if error else err_to_str(exc)
|
|
@@ -306,11 +302,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
306
302
|
|
|
307
303
|
def connect(self, secrets=None):
|
|
308
304
|
"""Connect to the MLRun API server. Must be called prior to executing any other method.
|
|
309
|
-
The code utilizes the URL for the API server from the configuration - ``
|
|
305
|
+
The code utilizes the URL for the API server from the configuration - ``mlconf.dbpath``.
|
|
310
306
|
|
|
311
307
|
For example::
|
|
312
308
|
|
|
313
|
-
|
|
309
|
+
mlconf.dbpath = mlconf.dbpath or 'http://mlrun-api:8080'
|
|
314
310
|
db = get_run_db().connect()
|
|
315
311
|
"""
|
|
316
312
|
# hack to allow unit tests to instantiate HTTPRunDB without a real server behind
|
|
@@ -504,7 +500,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
504
500
|
if offset < 0:
|
|
505
501
|
raise MLRunInvalidArgumentError("Offset cannot be negative")
|
|
506
502
|
if size is None:
|
|
507
|
-
size = int(
|
|
503
|
+
size = int(mlrun.mlconf.httpdb.logs.pull_logs_default_size_limit)
|
|
508
504
|
elif size == -1:
|
|
509
505
|
logger.warning(
|
|
510
506
|
"Retrieving all logs. This may be inefficient and can result in a large log."
|
|
@@ -550,23 +546,25 @@ class HTTPRunDB(RunDBInterface):
|
|
|
550
546
|
|
|
551
547
|
state, text = self.get_log(uid, project, offset=offset)
|
|
552
548
|
if text:
|
|
553
|
-
print(text.decode(errors=
|
|
549
|
+
print(text.decode(errors=mlrun.mlconf.httpdb.logs.decode.errors))
|
|
554
550
|
nil_resp = 0
|
|
555
551
|
while True:
|
|
556
552
|
offset += len(text)
|
|
557
553
|
# if we get 3 nil responses in a row, increase the sleep time to 10 seconds
|
|
558
554
|
# TODO: refactor this to use a conditional backoff mechanism
|
|
559
555
|
if nil_resp < 3:
|
|
560
|
-
time.sleep(int(
|
|
556
|
+
time.sleep(int(mlrun.mlconf.httpdb.logs.pull_logs_default_interval))
|
|
561
557
|
else:
|
|
562
558
|
time.sleep(
|
|
563
|
-
int(
|
|
559
|
+
int(
|
|
560
|
+
mlrun.mlconf.httpdb.logs.pull_logs_backoff_no_logs_default_interval
|
|
561
|
+
)
|
|
564
562
|
)
|
|
565
563
|
state, text = self.get_log(uid, project, offset=offset)
|
|
566
564
|
if text:
|
|
567
565
|
nil_resp = 0
|
|
568
566
|
print(
|
|
569
|
-
text.decode(errors=
|
|
567
|
+
text.decode(errors=mlrun.mlconf.httpdb.logs.decode.errors),
|
|
570
568
|
end="",
|
|
571
569
|
)
|
|
572
570
|
else:
|
|
@@ -1137,17 +1135,17 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1137
1135
|
structured_dict = {}
|
|
1138
1136
|
for project, job_runtime_resources_map in response.json().items():
|
|
1139
1137
|
for job_id, runtime_resources in job_runtime_resources_map.items():
|
|
1140
|
-
structured_dict.setdefault(project, {})[
|
|
1141
|
-
|
|
1142
|
-
)
|
|
1138
|
+
structured_dict.setdefault(project, {})[
|
|
1139
|
+
job_id
|
|
1140
|
+
] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
|
|
1143
1141
|
return structured_dict
|
|
1144
1142
|
elif group_by == mlrun.common.schemas.ListRuntimeResourcesGroupByField.project:
|
|
1145
1143
|
structured_dict = {}
|
|
1146
1144
|
for project, kind_runtime_resources_map in response.json().items():
|
|
1147
1145
|
for kind, runtime_resources in kind_runtime_resources_map.items():
|
|
1148
|
-
structured_dict.setdefault(project, {})[
|
|
1149
|
-
|
|
1150
|
-
)
|
|
1146
|
+
structured_dict.setdefault(project, {})[
|
|
1147
|
+
kind
|
|
1148
|
+
] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
|
|
1151
1149
|
return structured_dict
|
|
1152
1150
|
else:
|
|
1153
1151
|
raise NotImplementedError(
|
|
@@ -1175,8 +1173,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1175
1173
|
:param force: Force deletion - delete the runtime resource even if it's not in terminal state or if the grace
|
|
1176
1174
|
period didn't pass.
|
|
1177
1175
|
:param grace_period: Grace period given to the runtime resource before they are actually removed, counted from
|
|
1178
|
-
the moment they moved to terminal state
|
|
1179
|
-
(defaults to mlrun.config.config.runtime_resources_deletion_grace_period).
|
|
1176
|
+
the moment they moved to terminal state (defaults to mlrun.mlconf.runtime_resources_deletion_grace_period).
|
|
1180
1177
|
|
|
1181
1178
|
:returns: :py:class:`~mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput` listing the runtime resources
|
|
1182
1179
|
that were removed.
|
|
@@ -1206,9 +1203,9 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1206
1203
|
structured_dict = {}
|
|
1207
1204
|
for project, kind_runtime_resources_map in response.json().items():
|
|
1208
1205
|
for kind, runtime_resources in kind_runtime_resources_map.items():
|
|
1209
|
-
structured_dict.setdefault(project, {})[
|
|
1210
|
-
|
|
1211
|
-
)
|
|
1206
|
+
structured_dict.setdefault(project, {})[
|
|
1207
|
+
kind
|
|
1208
|
+
] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
|
|
1212
1209
|
return structured_dict
|
|
1213
1210
|
|
|
1214
1211
|
def create_schedule(
|
|
@@ -1343,7 +1340,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1343
1340
|
logger.warning(
|
|
1344
1341
|
"Building a function image to ECR and loading an S3 source to the image may require conflicting access "
|
|
1345
1342
|
"keys. Only the permissions granted to the platform's configured secret will take affect "
|
|
1346
|
-
"(see mlrun.
|
|
1343
|
+
"(see mlrun.mlconf.httpdb.builder.docker_registry_secret). "
|
|
1347
1344
|
"In case the permissions are limited to ECR scope, you may use pull_at_runtime=True instead",
|
|
1348
1345
|
source=func.spec.build.source,
|
|
1349
1346
|
load_source_on_run=func.spec.build.load_source_on_run,
|
|
@@ -1498,7 +1495,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1498
1495
|
Retrieve updated information on project background tasks being executed.
|
|
1499
1496
|
If no filter is provided, will return background tasks from the last week.
|
|
1500
1497
|
|
|
1501
|
-
:param project: Project name (defaults to mlrun.
|
|
1498
|
+
:param project: Project name (defaults to mlrun.mlconf.default_project).
|
|
1502
1499
|
:param state: List only background tasks whose state is specified.
|
|
1503
1500
|
:param created_from: Filter by background task created time in ``[created_from, created_to]``.
|
|
1504
1501
|
:param created_to: Filter by background task created time in ``[created_from, created_to]``.
|
|
@@ -3453,8 +3450,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3453
3450
|
source: Optional[str] = None,
|
|
3454
3451
|
run_name: Optional[str] = None,
|
|
3455
3452
|
namespace: Optional[str] = None,
|
|
3456
|
-
notifications:
|
|
3457
|
-
)
|
|
3453
|
+
notifications: typing.List[mlrun.model.Notification] = None,
|
|
3454
|
+
):
|
|
3458
3455
|
"""
|
|
3459
3456
|
Submitting workflow for a remote execution.
|
|
3460
3457
|
|
mlrun/execution.py
CHANGED
|
@@ -559,9 +559,9 @@ class MLClientCtx(object):
|
|
|
559
559
|
for k, v in get_in(task, ["status", "results"], {}).items():
|
|
560
560
|
self._results[k] = v
|
|
561
561
|
for artifact in get_in(task, ["status", run_keys.artifacts], []):
|
|
562
|
-
self._artifacts_manager.artifacts[
|
|
563
|
-
artifact
|
|
564
|
-
|
|
562
|
+
self._artifacts_manager.artifacts[
|
|
563
|
+
artifact["metadata"]["key"]
|
|
564
|
+
] = artifact
|
|
565
565
|
self._artifacts_manager.link_artifact(
|
|
566
566
|
self.project,
|
|
567
567
|
self.name,
|
|
@@ -389,9 +389,9 @@ class LoggingCallback(Callback):
|
|
|
389
389
|
):
|
|
390
390
|
try:
|
|
391
391
|
self._get_hyperparameter(key_chain=learning_rate_key_chain)
|
|
392
|
-
self._dynamic_hyperparameters_keys[
|
|
393
|
-
|
|
394
|
-
|
|
392
|
+
self._dynamic_hyperparameters_keys[
|
|
393
|
+
learning_rate_key
|
|
394
|
+
] = learning_rate_key_chain
|
|
395
395
|
except (KeyError, IndexError, ValueError):
|
|
396
396
|
pass
|
|
397
397
|
|
|
@@ -263,13 +263,13 @@ class TFKerasModelHandler(DLModelHandler):
|
|
|
263
263
|
# Update the paths and log artifacts if context is available:
|
|
264
264
|
if self._weights_file is not None:
|
|
265
265
|
if self._context is not None:
|
|
266
|
-
artifacts[
|
|
267
|
-
self.
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
266
|
+
artifacts[
|
|
267
|
+
self._get_weights_file_artifact_name()
|
|
268
|
+
] = self._context.log_artifact(
|
|
269
|
+
self._weights_file,
|
|
270
|
+
local_path=self._weights_file,
|
|
271
|
+
artifact_path=output_path,
|
|
272
|
+
db_key=False,
|
|
273
273
|
)
|
|
274
274
|
|
|
275
275
|
return artifacts if self._context is not None else None
|
mlrun/k8s_utils.py
CHANGED
|
@@ -134,13 +134,13 @@ def sanitize_label_value(value: str) -> str:
|
|
|
134
134
|
return re.sub(r"([^a-zA-Z0-9_.-]|^[^a-zA-Z0-9]|[^a-zA-Z0-9]$)", "-", value[:63])
|
|
135
135
|
|
|
136
136
|
|
|
137
|
-
def verify_label_key(key
|
|
138
|
-
"""
|
|
139
|
-
Verify that the label key is valid for Kubernetes.
|
|
140
|
-
Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
|
|
141
|
-
"""
|
|
137
|
+
def verify_label_key(key):
|
|
142
138
|
if not key:
|
|
143
139
|
raise mlrun.errors.MLRunInvalidArgumentError("label key cannot be empty")
|
|
140
|
+
if key.startswith("k8s.io") or key.startswith("kubernetes.io"):
|
|
141
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
142
|
+
"Labels cannot start with 'k8s.io' or 'kubernetes.io'"
|
|
143
|
+
)
|
|
144
144
|
|
|
145
145
|
mlrun.utils.helpers.verify_field_regex(
|
|
146
146
|
f"project.metadata.labels.'{key}'",
|
|
@@ -148,11 +148,6 @@ def verify_label_key(key: str):
|
|
|
148
148
|
mlrun.utils.regex.k8s_character_limit,
|
|
149
149
|
)
|
|
150
150
|
|
|
151
|
-
if key.startswith("k8s.io/") or key.startswith("kubernetes.io/"):
|
|
152
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
153
|
-
"Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
|
|
154
|
-
)
|
|
155
|
-
|
|
156
151
|
parts = key.split("/")
|
|
157
152
|
if len(parts) == 1:
|
|
158
153
|
name = parts[0]
|
mlrun/kfpops.py
CHANGED
|
@@ -41,8 +41,8 @@ from .utils import (
|
|
|
41
41
|
|
|
42
42
|
# default KFP artifacts and output (ui metadata, metrics etc.)
|
|
43
43
|
# directories to /tmp to allow running with security context
|
|
44
|
-
KFPMETA_DIR = "/tmp"
|
|
45
|
-
KFP_ARTIFACTS_DIR = "/tmp"
|
|
44
|
+
KFPMETA_DIR = os.environ.get("KFPMETA_OUT_DIR", "/tmp")
|
|
45
|
+
KFP_ARTIFACTS_DIR = os.environ.get("KFP_ARTIFACTS_DIR", "/tmp")
|
|
46
46
|
|
|
47
47
|
project_annotation = "mlrun/project"
|
|
48
48
|
run_annotation = "mlrun/pipeline-step-type"
|
|
@@ -71,7 +71,7 @@ def write_kfpmeta(struct):
|
|
|
71
71
|
{"name": k, "numberValue": v} for k, v in results.items() if is_num(v)
|
|
72
72
|
],
|
|
73
73
|
}
|
|
74
|
-
with open(
|
|
74
|
+
with open(KFPMETA_DIR + "/mlpipeline-metrics.json", "w") as f:
|
|
75
75
|
json.dump(metrics, f)
|
|
76
76
|
|
|
77
77
|
struct = deepcopy(struct)
|
|
@@ -91,14 +91,7 @@ def write_kfpmeta(struct):
|
|
|
91
91
|
elif key in results:
|
|
92
92
|
val = results[key]
|
|
93
93
|
try:
|
|
94
|
-
|
|
95
|
-
path = os.path.join(KFP_ARTIFACTS_DIR, key)
|
|
96
|
-
if not mlrun.utils.helpers.is_safe_path(KFP_ARTIFACTS_DIR, path):
|
|
97
|
-
logger.warning(
|
|
98
|
-
"Path traversal is not allowed ignoring", path=path, key=key
|
|
99
|
-
)
|
|
100
|
-
continue
|
|
101
|
-
path = os.path.abspath(path)
|
|
94
|
+
path = "/".join([KFP_ARTIFACTS_DIR, key])
|
|
102
95
|
logger.info("Writing artifact output", path=path, val=val)
|
|
103
96
|
with open(path, "w") as fp:
|
|
104
97
|
fp.write(str(val))
|
|
@@ -116,7 +109,7 @@ def write_kfpmeta(struct):
|
|
|
116
109
|
"outputs": output_artifacts
|
|
117
110
|
+ [{"type": "markdown", "storage": "inline", "source": text}]
|
|
118
111
|
}
|
|
119
|
-
with open(
|
|
112
|
+
with open(KFPMETA_DIR + "/mlpipeline-ui-metadata.json", "w") as f:
|
|
120
113
|
json.dump(metadata, f)
|
|
121
114
|
|
|
122
115
|
|
|
@@ -408,9 +401,9 @@ def mlrun_op(
|
|
|
408
401
|
cmd += ["--label", f"{label}={val}"]
|
|
409
402
|
for output in outputs:
|
|
410
403
|
cmd += ["-o", str(output)]
|
|
411
|
-
file_outputs[
|
|
412
|
-
|
|
413
|
-
|
|
404
|
+
file_outputs[
|
|
405
|
+
output.replace(".", "_")
|
|
406
|
+
] = f"/tmp/{output}" # not using path.join to avoid windows "\"
|
|
414
407
|
if project:
|
|
415
408
|
cmd += ["--project", project]
|
|
416
409
|
if handler:
|
|
@@ -457,10 +450,8 @@ def mlrun_op(
|
|
|
457
450
|
command=cmd + [command],
|
|
458
451
|
file_outputs=file_outputs,
|
|
459
452
|
output_artifact_paths={
|
|
460
|
-
"mlpipeline-ui-metadata":
|
|
461
|
-
|
|
462
|
-
),
|
|
463
|
-
"mlpipeline-metrics": os.path.join(KFPMETA_DIR, "mlpipeline-metrics.json"),
|
|
453
|
+
"mlpipeline-ui-metadata": KFPMETA_DIR + "/mlpipeline-ui-metadata.json",
|
|
454
|
+
"mlpipeline-metrics": KFPMETA_DIR + "/mlpipeline-metrics.json",
|
|
464
455
|
},
|
|
465
456
|
)
|
|
466
457
|
cop = add_default_function_resources(cop)
|
mlrun/model.py
CHANGED
|
@@ -359,7 +359,6 @@ class ImageBuilder(ModelObj):
|
|
|
359
359
|
requirements: list = None,
|
|
360
360
|
extra_args=None,
|
|
361
361
|
builder_env=None,
|
|
362
|
-
source_code_target_dir=None,
|
|
363
362
|
):
|
|
364
363
|
self.functionSourceCode = functionSourceCode #: functionSourceCode
|
|
365
364
|
self.codeEntryType = "" #: codeEntryType
|
|
@@ -380,7 +379,6 @@ class ImageBuilder(ModelObj):
|
|
|
380
379
|
self.auto_build = auto_build #: auto_build
|
|
381
380
|
self.build_pod = None
|
|
382
381
|
self.requirements = requirements or [] #: pip requirements
|
|
383
|
-
self.source_code_target_dir = source_code_target_dir or None
|
|
384
382
|
|
|
385
383
|
@property
|
|
386
384
|
def source(self):
|
|
@@ -417,7 +415,6 @@ class ImageBuilder(ModelObj):
|
|
|
417
415
|
overwrite=False,
|
|
418
416
|
builder_env=None,
|
|
419
417
|
extra_args=None,
|
|
420
|
-
source_code_target_dir=None,
|
|
421
418
|
):
|
|
422
419
|
if image:
|
|
423
420
|
self.image = image
|
|
@@ -443,8 +440,6 @@ class ImageBuilder(ModelObj):
|
|
|
443
440
|
self.builder_env = builder_env
|
|
444
441
|
if extra_args:
|
|
445
442
|
self.extra_args = extra_args
|
|
446
|
-
if source_code_target_dir:
|
|
447
|
-
self.source_code_target_dir = source_code_target_dir
|
|
448
443
|
|
|
449
444
|
def with_commands(
|
|
450
445
|
self,
|
mlrun/model_monitoring/api.py
CHANGED
|
@@ -436,9 +436,9 @@ def _generate_model_endpoint(
|
|
|
436
436
|
] = possible_drift_threshold
|
|
437
437
|
|
|
438
438
|
model_endpoint.spec.monitoring_mode = monitoring_mode
|
|
439
|
-
model_endpoint.status.first_request =
|
|
440
|
-
|
|
441
|
-
)
|
|
439
|
+
model_endpoint.status.first_request = (
|
|
440
|
+
model_endpoint.status.last_request
|
|
441
|
+
) = datetime_now().isoformat()
|
|
442
442
|
if sample_set_statistics:
|
|
443
443
|
model_endpoint.status.feature_stats = sample_set_statistics
|
|
444
444
|
|
|
@@ -476,11 +476,11 @@ def trigger_drift_batch_job(
|
|
|
476
476
|
db_session = mlrun.get_run_db()
|
|
477
477
|
|
|
478
478
|
# Register the monitoring batch job (do nothing if already exist) and get the job function as a dictionary
|
|
479
|
-
batch_function_dict: typing.Dict[
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
479
|
+
batch_function_dict: typing.Dict[
|
|
480
|
+
str, typing.Any
|
|
481
|
+
] = db_session.deploy_monitoring_batch_job(
|
|
482
|
+
project=project,
|
|
483
|
+
default_batch_image=default_batch_image,
|
|
484
484
|
)
|
|
485
485
|
|
|
486
486
|
# Prepare current run params
|
mlrun/model_monitoring/batch.py
CHANGED
|
@@ -540,24 +540,24 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
540
540
|
and endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.METRICS]
|
|
541
541
|
== "null"
|
|
542
542
|
):
|
|
543
|
-
endpoint[
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
543
|
+
endpoint[
|
|
544
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.METRICS
|
|
545
|
+
] = json.dumps(
|
|
546
|
+
{
|
|
547
|
+
mlrun.common.schemas.model_monitoring.EventKeyMetrics.GENERIC: {
|
|
548
|
+
mlrun.common.schemas.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
|
|
549
|
+
mlrun.common.schemas.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
|
|
550
550
|
}
|
|
551
|
-
|
|
551
|
+
}
|
|
552
552
|
)
|
|
553
553
|
# Validate key `uid` instead of `endpoint_id`
|
|
554
554
|
# For backwards compatibility reasons, we replace the `endpoint_id` with `uid` which is the updated key name
|
|
555
555
|
if mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID in endpoint:
|
|
556
|
-
endpoint[
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
556
|
+
endpoint[
|
|
557
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.UID
|
|
558
|
+
] = endpoint[
|
|
559
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
|
|
560
|
+
]
|
|
561
561
|
|
|
562
562
|
@staticmethod
|
|
563
563
|
def _encode_field(field: typing.Union[str, bytes]) -> bytes:
|
|
@@ -31,6 +31,7 @@ from .models import get_model_endpoints_table
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
class SQLModelEndpointStore(ModelEndpointStore):
|
|
34
|
+
|
|
34
35
|
"""
|
|
35
36
|
Handles the DB operations when the DB target is from type SQL. For the SQL operations, we use SQLAlchemy, a Python
|
|
36
37
|
SQL toolkit that handles the communication with the database. When using SQL for storing the model endpoints
|
|
@@ -838,9 +838,9 @@ class PandasDataFramePackager(DefaultPackager):
|
|
|
838
838
|
"""
|
|
839
839
|
if isinstance(obj, dict):
|
|
840
840
|
for key, value in obj.items():
|
|
841
|
-
obj[
|
|
842
|
-
PandasDataFramePackager._prepare_result(obj=
|
|
843
|
-
)
|
|
841
|
+
obj[
|
|
842
|
+
PandasDataFramePackager._prepare_result(obj=key)
|
|
843
|
+
] = PandasDataFramePackager._prepare_result(obj=value)
|
|
844
844
|
elif isinstance(obj, list):
|
|
845
845
|
for i, value in enumerate(obj):
|
|
846
846
|
obj[i] = PandasDataFramePackager._prepare_result(obj=value)
|
mlrun/package/utils/_archiver.py
CHANGED
|
@@ -179,9 +179,7 @@ class _TarArchiver(_Archiver):
|
|
|
179
179
|
|
|
180
180
|
# Extract:
|
|
181
181
|
with tarfile.open(archive_path, f"r:{cls._MODE_STRING}") as tar_file:
|
|
182
|
-
|
|
183
|
-
# see: https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractall
|
|
184
|
-
tar_file.extractall(directory_path, filter="data")
|
|
182
|
+
tar_file.extractall(directory_path)
|
|
185
183
|
|
|
186
184
|
return str(directory_path)
|
|
187
185
|
|
mlrun/platforms/iguazio.py
CHANGED
|
@@ -16,15 +16,19 @@ import json
|
|
|
16
16
|
import os
|
|
17
17
|
import urllib
|
|
18
18
|
from collections import namedtuple
|
|
19
|
+
from datetime import datetime
|
|
20
|
+
from http import HTTPStatus
|
|
19
21
|
from urllib.parse import urlparse
|
|
20
22
|
|
|
21
23
|
import kfp.dsl
|
|
22
24
|
import requests
|
|
23
25
|
import semver
|
|
26
|
+
import urllib3
|
|
24
27
|
import v3io
|
|
25
28
|
|
|
26
29
|
import mlrun.errors
|
|
27
30
|
from mlrun.config import config as mlconf
|
|
31
|
+
from mlrun.errors import err_to_str
|
|
28
32
|
from mlrun.utils import dict_to_json
|
|
29
33
|
|
|
30
34
|
_cached_control_session = None
|
|
@@ -484,6 +488,25 @@ class V3ioStreamClient:
|
|
|
484
488
|
return response.output.records
|
|
485
489
|
|
|
486
490
|
|
|
491
|
+
def create_control_session(url, username, password):
|
|
492
|
+
# for systems without production cert - silence no cert verification WARN
|
|
493
|
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
494
|
+
if not username or not password:
|
|
495
|
+
raise ValueError("cannot create session key, missing username or password")
|
|
496
|
+
|
|
497
|
+
session = requests.Session()
|
|
498
|
+
session.auth = (username, password)
|
|
499
|
+
try:
|
|
500
|
+
auth = session.post(f"{url}/api/sessions", verify=False)
|
|
501
|
+
except OSError as exc:
|
|
502
|
+
raise OSError(f"error: cannot connect to {url}: {err_to_str(exc)}")
|
|
503
|
+
|
|
504
|
+
if not auth.ok:
|
|
505
|
+
raise OSError(f"failed to create session: {url}, {auth.text}")
|
|
506
|
+
|
|
507
|
+
return auth.json()["data"]["id"]
|
|
508
|
+
|
|
509
|
+
|
|
487
510
|
def is_iguazio_endpoint(endpoint_url: str) -> bool:
|
|
488
511
|
# TODO: find a better heuristic
|
|
489
512
|
return ".default-tenant." in endpoint_url
|
|
@@ -510,6 +533,21 @@ def is_iguazio_session_cookie(session_cookie: str) -> bool:
|
|
|
510
533
|
return False
|
|
511
534
|
|
|
512
535
|
|
|
536
|
+
def is_iguazio_system_2_10_or_above(dashboard_url):
|
|
537
|
+
# for systems without production cert - silence no cert verification WARN
|
|
538
|
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
539
|
+
response = requests.get(f"{dashboard_url}/api/external_versions", verify=False)
|
|
540
|
+
|
|
541
|
+
if not response.ok:
|
|
542
|
+
if response.status_code == HTTPStatus.NOT_FOUND.value:
|
|
543
|
+
# in iguazio systems prior to 2.10 this endpoint didn't exist, so the api returns 404 cause endpoint not
|
|
544
|
+
# found
|
|
545
|
+
return False
|
|
546
|
+
response.raise_for_status()
|
|
547
|
+
|
|
548
|
+
return True
|
|
549
|
+
|
|
550
|
+
|
|
513
551
|
# we assign the control session or access key to the password since this is iguazio auth scheme
|
|
514
552
|
# (requests should be sent with username:control_session/access_key as auth header)
|
|
515
553
|
def add_or_refresh_credentials(
|
|
@@ -539,12 +577,33 @@ def add_or_refresh_credentials(
|
|
|
539
577
|
# (ideally if we could identify we're in enterprise we would have verify here that token and username have value)
|
|
540
578
|
if not is_iguazio_endpoint(api_url):
|
|
541
579
|
return "", "", token
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
580
|
+
iguazio_dashboard_url = "https://dashboard" + api_url[api_url.find(".") :]
|
|
581
|
+
|
|
582
|
+
# in 2.8 mlrun api is protected with control session, from 2.10 it's protected with access key
|
|
583
|
+
is_access_key_auth = is_iguazio_system_2_10_or_above(iguazio_dashboard_url)
|
|
584
|
+
if is_access_key_auth:
|
|
585
|
+
if not username or not token:
|
|
586
|
+
raise ValueError(
|
|
587
|
+
"username and access key required to authenticate against iguazio system"
|
|
588
|
+
)
|
|
589
|
+
return username, token, ""
|
|
590
|
+
|
|
591
|
+
if not username or not password:
|
|
592
|
+
raise ValueError("username and password needed to create session")
|
|
593
|
+
|
|
594
|
+
global _cached_control_session
|
|
595
|
+
now = datetime.now()
|
|
596
|
+
if _cached_control_session:
|
|
597
|
+
if (
|
|
598
|
+
_cached_control_session[2] == username
|
|
599
|
+
and _cached_control_session[3] == password
|
|
600
|
+
and (now - _cached_control_session[1]).seconds < 20 * 60 * 60
|
|
601
|
+
):
|
|
602
|
+
return _cached_control_session[2], _cached_control_session[0], ""
|
|
603
|
+
|
|
604
|
+
control_session = create_control_session(iguazio_dashboard_url, username, password)
|
|
605
|
+
_cached_control_session = (control_session, now, username, password)
|
|
606
|
+
return username, control_session, ""
|
|
548
607
|
|
|
549
608
|
|
|
550
609
|
def parse_path(url, suffix="/"):
|
mlrun/projects/pipelines.py
CHANGED
|
@@ -69,16 +69,16 @@ class WorkflowSpec(mlrun.model.ModelObj):
|
|
|
69
69
|
|
|
70
70
|
def __init__(
|
|
71
71
|
self,
|
|
72
|
-
engine
|
|
73
|
-
code
|
|
74
|
-
path
|
|
75
|
-
args
|
|
76
|
-
name
|
|
77
|
-
handler
|
|
78
|
-
args_schema:
|
|
72
|
+
engine=None,
|
|
73
|
+
code=None,
|
|
74
|
+
path=None,
|
|
75
|
+
args=None,
|
|
76
|
+
name=None,
|
|
77
|
+
handler=None,
|
|
78
|
+
args_schema: dict = None,
|
|
79
79
|
schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
|
|
80
|
-
cleanup_ttl:
|
|
81
|
-
image:
|
|
80
|
+
cleanup_ttl: int = None,
|
|
81
|
+
image: str = None,
|
|
82
82
|
):
|
|
83
83
|
self.engine = engine
|
|
84
84
|
self.code = code
|
|
@@ -401,9 +401,6 @@ def enrich_function_object(
|
|
|
401
401
|
else:
|
|
402
402
|
f.spec.build.source = project.spec.source
|
|
403
403
|
f.spec.build.load_source_on_run = project.spec.load_source_on_run
|
|
404
|
-
f.spec.build.source_code_target_dir = (
|
|
405
|
-
project.spec.build.source_code_target_dir
|
|
406
|
-
)
|
|
407
404
|
f.spec.workdir = project.spec.workdir or project.spec.subpath
|
|
408
405
|
f.prepare_image_for_deploy()
|
|
409
406
|
|
|
@@ -865,11 +862,6 @@ class _RemoteRunner(_PipelineRunner):
|
|
|
865
862
|
)
|
|
866
863
|
return
|
|
867
864
|
|
|
868
|
-
logger.debug(
|
|
869
|
-
"Workflow submitted, waiting for pipeline run to start",
|
|
870
|
-
workflow_name=workflow_response.name,
|
|
871
|
-
)
|
|
872
|
-
|
|
873
865
|
# Getting workflow id from run:
|
|
874
866
|
response = retry_until_successful(
|
|
875
867
|
1,
|
|
@@ -996,7 +988,6 @@ def load_and_run(
|
|
|
996
988
|
cleanup_ttl: int = None,
|
|
997
989
|
load_only: bool = False,
|
|
998
990
|
wait_for_completion: bool = False,
|
|
999
|
-
project_context: str = None,
|
|
1000
991
|
):
|
|
1001
992
|
"""
|
|
1002
993
|
Auxiliary function that the RemoteRunner run once or run every schedule.
|
|
@@ -1027,11 +1018,10 @@ def load_and_run(
|
|
|
1027
1018
|
workflow and all its resources are deleted)
|
|
1028
1019
|
:param load_only: for just loading the project, inner use.
|
|
1029
1020
|
:param wait_for_completion: wait for workflow completion before returning
|
|
1030
|
-
:param project_context: project context path (used for loading the project)
|
|
1031
1021
|
"""
|
|
1032
1022
|
try:
|
|
1033
1023
|
project = mlrun.load_project(
|
|
1034
|
-
context=
|
|
1024
|
+
context=f"./{project_name}",
|
|
1035
1025
|
url=url,
|
|
1036
1026
|
name=project_name,
|
|
1037
1027
|
init_git=init_git,
|
|
@@ -1063,7 +1053,7 @@ def load_and_run(
|
|
|
1063
1053
|
|
|
1064
1054
|
raise error
|
|
1065
1055
|
|
|
1066
|
-
context.logger.info(f"Loaded project {project.name} successfully")
|
|
1056
|
+
context.logger.info(f"Loaded project {project.name} from remote successfully")
|
|
1067
1057
|
|
|
1068
1058
|
if load_only:
|
|
1069
1059
|
return
|