mlrun 1.7.0rc37__py3-none-any.whl → 1.7.0rc39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/alerts/alert.py +34 -30
- mlrun/common/schemas/alert.py +3 -0
- mlrun/common/schemas/model_monitoring/constants.py +4 -0
- mlrun/common/schemas/notification.py +4 -3
- mlrun/datastore/alibaba_oss.py +2 -2
- mlrun/datastore/azure_blob.py +124 -31
- mlrun/datastore/base.py +1 -1
- mlrun/datastore/dbfs_store.py +2 -2
- mlrun/datastore/google_cloud_storage.py +83 -20
- mlrun/datastore/s3.py +2 -2
- mlrun/datastore/sources.py +54 -0
- mlrun/datastore/targets.py +9 -53
- mlrun/db/httpdb.py +6 -1
- mlrun/errors.py +8 -0
- mlrun/execution.py +7 -0
- mlrun/feature_store/api.py +5 -0
- mlrun/feature_store/common.py +6 -11
- mlrun/feature_store/retrieval/job.py +1 -0
- mlrun/model.py +29 -3
- mlrun/model_monitoring/api.py +9 -0
- mlrun/model_monitoring/applications/_application_steps.py +36 -0
- mlrun/model_monitoring/applications/histogram_data_drift.py +15 -13
- mlrun/model_monitoring/controller.py +15 -11
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +14 -11
- mlrun/model_monitoring/db/tsdb/base.py +121 -1
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +85 -47
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +100 -12
- mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +23 -1
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +214 -36
- mlrun/model_monitoring/helpers.py +16 -17
- mlrun/model_monitoring/stream_processing.py +68 -27
- mlrun/projects/operations.py +1 -1
- mlrun/projects/pipelines.py +19 -30
- mlrun/projects/project.py +76 -52
- mlrun/run.py +8 -6
- mlrun/runtimes/__init__.py +19 -8
- mlrun/runtimes/nuclio/api_gateway.py +9 -0
- mlrun/runtimes/nuclio/application/application.py +64 -9
- mlrun/runtimes/nuclio/function.py +1 -1
- mlrun/runtimes/pod.py +2 -2
- mlrun/runtimes/remotesparkjob.py +2 -5
- mlrun/runtimes/sparkjob/spark3job.py +7 -9
- mlrun/serving/v2_serving.py +1 -0
- mlrun/track/trackers/mlflow_tracker.py +5 -0
- mlrun/utils/helpers.py +21 -0
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc37.dist-info → mlrun-1.7.0rc39.dist-info}/METADATA +14 -11
- {mlrun-1.7.0rc37.dist-info → mlrun-1.7.0rc39.dist-info}/RECORD +52 -52
- {mlrun-1.7.0rc37.dist-info → mlrun-1.7.0rc39.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc37.dist-info → mlrun-1.7.0rc39.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc37.dist-info → mlrun-1.7.0rc39.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc37.dist-info → mlrun-1.7.0rc39.dist-info}/top_level.txt +0 -0
mlrun/projects/pipelines.py
CHANGED
|
@@ -447,7 +447,6 @@ class _PipelineRunner(abc.ABC):
|
|
|
447
447
|
namespace=None,
|
|
448
448
|
source=None,
|
|
449
449
|
notifications: list[mlrun.model.Notification] = None,
|
|
450
|
-
send_start_notification: bool = True,
|
|
451
450
|
) -> _PipelineRunStatus:
|
|
452
451
|
pass
|
|
453
452
|
|
|
@@ -567,7 +566,6 @@ class _KFPRunner(_PipelineRunner):
|
|
|
567
566
|
namespace=None,
|
|
568
567
|
source=None,
|
|
569
568
|
notifications: list[mlrun.model.Notification] = None,
|
|
570
|
-
send_start_notification: bool = True,
|
|
571
569
|
) -> _PipelineRunStatus:
|
|
572
570
|
pipeline_context.set(project, workflow_spec)
|
|
573
571
|
workflow_handler = _PipelineRunner._get_handler(
|
|
@@ -585,7 +583,8 @@ class _KFPRunner(_PipelineRunner):
|
|
|
585
583
|
"Notifications will only be sent if you wait for pipeline completion. "
|
|
586
584
|
"To use the new notification behavior, use the remote pipeline runner."
|
|
587
585
|
)
|
|
588
|
-
for notification
|
|
586
|
+
# for start message, fallback to old notification behavior
|
|
587
|
+
for notification in notifications or []:
|
|
589
588
|
project.notifiers.add_notification(
|
|
590
589
|
notification.kind, notification.params
|
|
591
590
|
)
|
|
@@ -616,13 +615,12 @@ class _KFPRunner(_PipelineRunner):
|
|
|
616
615
|
func_name=func.metadata.name,
|
|
617
616
|
exc_info=err_to_str(exc),
|
|
618
617
|
)
|
|
619
|
-
|
|
620
|
-
project.
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
)
|
|
618
|
+
project.notifiers.push_pipeline_start_message(
|
|
619
|
+
project.metadata.name,
|
|
620
|
+
project.get_param("commit_id", None),
|
|
621
|
+
run_id,
|
|
622
|
+
True,
|
|
623
|
+
)
|
|
626
624
|
pipeline_context.clear()
|
|
627
625
|
return _PipelineRunStatus(run_id, cls, project=project, workflow=workflow_spec)
|
|
628
626
|
|
|
@@ -670,7 +668,6 @@ class _LocalRunner(_PipelineRunner):
|
|
|
670
668
|
namespace=None,
|
|
671
669
|
source=None,
|
|
672
670
|
notifications: list[mlrun.model.Notification] = None,
|
|
673
|
-
send_start_notification: bool = True,
|
|
674
671
|
) -> _PipelineRunStatus:
|
|
675
672
|
pipeline_context.set(project, workflow_spec)
|
|
676
673
|
workflow_handler = _PipelineRunner._get_handler(
|
|
@@ -692,10 +689,9 @@ class _LocalRunner(_PipelineRunner):
|
|
|
692
689
|
project.set_source(source=source)
|
|
693
690
|
pipeline_context.workflow_artifact_path = artifact_path
|
|
694
691
|
|
|
695
|
-
|
|
696
|
-
project.
|
|
697
|
-
|
|
698
|
-
)
|
|
692
|
+
project.notifiers.push_pipeline_start_message(
|
|
693
|
+
project.metadata.name, pipeline_id=workflow_id
|
|
694
|
+
)
|
|
699
695
|
err = None
|
|
700
696
|
try:
|
|
701
697
|
workflow_handler(**workflow_spec.args)
|
|
@@ -755,22 +751,10 @@ class _RemoteRunner(_PipelineRunner):
|
|
|
755
751
|
namespace: str = None,
|
|
756
752
|
source: str = None,
|
|
757
753
|
notifications: list[mlrun.model.Notification] = None,
|
|
758
|
-
send_start_notification: bool = True,
|
|
759
754
|
) -> typing.Optional[_PipelineRunStatus]:
|
|
760
755
|
workflow_name = normalize_workflow_name(name=name, project_name=project.name)
|
|
761
756
|
workflow_id = None
|
|
762
757
|
|
|
763
|
-
# for start message, fallback to old notification behavior
|
|
764
|
-
if send_start_notification:
|
|
765
|
-
for notification in notifications or []:
|
|
766
|
-
project.notifiers.add_notification(
|
|
767
|
-
notification.kind, notification.params
|
|
768
|
-
)
|
|
769
|
-
# if a notification with `when=running` is provided, it will be used explicitly and others
|
|
770
|
-
# will be ignored
|
|
771
|
-
if "running" in notification.when:
|
|
772
|
-
break
|
|
773
|
-
|
|
774
758
|
# The returned engine for this runner is the engine of the workflow.
|
|
775
759
|
# In this way wait_for_completion/get_run_status would be executed by the correct pipeline runner.
|
|
776
760
|
inner_engine = get_workflow_engine(workflow_spec.engine)
|
|
@@ -870,9 +854,6 @@ class _RemoteRunner(_PipelineRunner):
|
|
|
870
854
|
state = mlrun_pipelines.common.models.RunStatuses.failed
|
|
871
855
|
else:
|
|
872
856
|
state = mlrun_pipelines.common.models.RunStatuses.running
|
|
873
|
-
project.notifiers.push_pipeline_start_message(
|
|
874
|
-
project.metadata.name,
|
|
875
|
-
)
|
|
876
857
|
pipeline_context.clear()
|
|
877
858
|
return _PipelineRunStatus(
|
|
878
859
|
run_id=workflow_id,
|
|
@@ -1078,6 +1059,13 @@ def load_and_run(
|
|
|
1078
1059
|
if load_only:
|
|
1079
1060
|
return
|
|
1080
1061
|
|
|
1062
|
+
# extract "start" notification if exists
|
|
1063
|
+
start_notifications = [
|
|
1064
|
+
notification
|
|
1065
|
+
for notification in context.get_notifications()
|
|
1066
|
+
if "running" in notification.when
|
|
1067
|
+
]
|
|
1068
|
+
|
|
1081
1069
|
workflow_log_message = workflow_name or workflow_path
|
|
1082
1070
|
context.logger.info(f"Running workflow {workflow_log_message} from remote")
|
|
1083
1071
|
run = project.run(
|
|
@@ -1093,6 +1081,7 @@ def load_and_run(
|
|
|
1093
1081
|
cleanup_ttl=cleanup_ttl,
|
|
1094
1082
|
engine=engine,
|
|
1095
1083
|
local=local,
|
|
1084
|
+
notifications=start_notifications,
|
|
1096
1085
|
)
|
|
1097
1086
|
context.log_result(key="workflow_id", value=run.run_id)
|
|
1098
1087
|
context.log_result(key="engine", value=run._engine.engine, commit=True)
|
mlrun/projects/project.py
CHANGED
|
@@ -517,17 +517,24 @@ def get_or_create_project(
|
|
|
517
517
|
parameters=parameters,
|
|
518
518
|
allow_cross_project=allow_cross_project,
|
|
519
519
|
)
|
|
520
|
-
logger.info("Project loaded successfully", project_name=name)
|
|
520
|
+
logger.info("Project loaded successfully", project_name=project.name)
|
|
521
521
|
return project
|
|
522
522
|
except mlrun.errors.MLRunNotFoundError:
|
|
523
|
-
logger.debug(
|
|
523
|
+
logger.debug(
|
|
524
|
+
"Project not found in db", project_name=name, user_project=user_project
|
|
525
|
+
)
|
|
524
526
|
|
|
525
527
|
spec_path = path.join(context, subpath or "", "project.yaml")
|
|
526
528
|
load_from_path = url or path.isfile(spec_path)
|
|
527
529
|
# do not nest under "try" or else the exceptions raised below will be logged along with the "not found" message
|
|
528
530
|
if load_from_path:
|
|
529
531
|
# loads a project from archive or local project.yaml
|
|
530
|
-
logger.info(
|
|
532
|
+
logger.info(
|
|
533
|
+
"Loading project from path",
|
|
534
|
+
project_name=name,
|
|
535
|
+
user_project=user_project,
|
|
536
|
+
path=url or context,
|
|
537
|
+
)
|
|
531
538
|
project = load_project(
|
|
532
539
|
context,
|
|
533
540
|
url,
|
|
@@ -544,7 +551,7 @@ def get_or_create_project(
|
|
|
544
551
|
|
|
545
552
|
logger.info(
|
|
546
553
|
"Project loaded successfully",
|
|
547
|
-
project_name=name,
|
|
554
|
+
project_name=project.name,
|
|
548
555
|
path=url or context,
|
|
549
556
|
stored_in_db=save,
|
|
550
557
|
)
|
|
@@ -562,7 +569,9 @@ def get_or_create_project(
|
|
|
562
569
|
save=save,
|
|
563
570
|
parameters=parameters,
|
|
564
571
|
)
|
|
565
|
-
logger.info(
|
|
572
|
+
logger.info(
|
|
573
|
+
"Project created successfully", project_name=project.name, stored_in_db=save
|
|
574
|
+
)
|
|
566
575
|
return project
|
|
567
576
|
|
|
568
577
|
|
|
@@ -600,6 +609,10 @@ def _run_project_setup(
|
|
|
600
609
|
if hasattr(mod, "setup"):
|
|
601
610
|
try:
|
|
602
611
|
project = getattr(mod, "setup")(project)
|
|
612
|
+
if not project or not isinstance(project, mlrun.projects.MlrunProject):
|
|
613
|
+
raise ValueError(
|
|
614
|
+
"MLRun project_setup:setup() must return a project object"
|
|
615
|
+
)
|
|
603
616
|
except Exception as exc:
|
|
604
617
|
logger.error(
|
|
605
618
|
"Failed to run project_setup script",
|
|
@@ -610,7 +623,9 @@ def _run_project_setup(
|
|
|
610
623
|
if save:
|
|
611
624
|
project.save()
|
|
612
625
|
else:
|
|
613
|
-
logger.warn(
|
|
626
|
+
logger.warn(
|
|
627
|
+
f"skipping setup, setup() handler was not found in {path.basename(setup_file_path)}"
|
|
628
|
+
)
|
|
614
629
|
return project
|
|
615
630
|
|
|
616
631
|
|
|
@@ -2388,7 +2403,11 @@ class MlrunProject(ModelObj):
|
|
|
2388
2403
|
requirements: typing.Union[str, list[str]] = None,
|
|
2389
2404
|
requirements_file: str = "",
|
|
2390
2405
|
) -> tuple[str, str, mlrun.runtimes.BaseRuntime, dict]:
|
|
2391
|
-
if
|
|
2406
|
+
if (
|
|
2407
|
+
func is None
|
|
2408
|
+
and not _has_module(handler, kind)
|
|
2409
|
+
and mlrun.runtimes.RuntimeKinds.supports_from_notebook(kind)
|
|
2410
|
+
):
|
|
2392
2411
|
# if function path is not provided and it is not a module (no ".")
|
|
2393
2412
|
# use the current notebook as default
|
|
2394
2413
|
if is_ipython:
|
|
@@ -2967,7 +2986,6 @@ class MlrunProject(ModelObj):
|
|
|
2967
2986
|
source: str = None,
|
|
2968
2987
|
cleanup_ttl: int = None,
|
|
2969
2988
|
notifications: list[mlrun.model.Notification] = None,
|
|
2970
|
-
send_start_notification: bool = True,
|
|
2971
2989
|
) -> _PipelineRunStatus:
|
|
2972
2990
|
"""Run a workflow using kubeflow pipelines
|
|
2973
2991
|
|
|
@@ -3004,8 +3022,6 @@ class MlrunProject(ModelObj):
|
|
|
3004
3022
|
workflow and all its resources are deleted)
|
|
3005
3023
|
:param notifications:
|
|
3006
3024
|
List of notifications to send for workflow completion
|
|
3007
|
-
:param send_start_notification:
|
|
3008
|
-
Send a notification when the workflow starts
|
|
3009
3025
|
|
|
3010
3026
|
:returns: ~py:class:`~mlrun.projects.pipelines._PipelineRunStatus` instance
|
|
3011
3027
|
"""
|
|
@@ -3083,7 +3099,6 @@ class MlrunProject(ModelObj):
|
|
|
3083
3099
|
namespace=namespace,
|
|
3084
3100
|
source=source,
|
|
3085
3101
|
notifications=notifications,
|
|
3086
|
-
send_start_notification=send_start_notification,
|
|
3087
3102
|
)
|
|
3088
3103
|
# run is None when scheduling
|
|
3089
3104
|
if run and run.state == mlrun_pipelines.common.models.RunStatuses.failed:
|
|
@@ -3216,30 +3231,30 @@ class MlrunProject(ModelObj):
|
|
|
3216
3231
|
infrastructure functions. Important to note that you have to set the credentials before deploying any
|
|
3217
3232
|
model monitoring or serving function.
|
|
3218
3233
|
|
|
3219
|
-
:param access_key: Model
|
|
3220
|
-
:param endpoint_store_connection: Endpoint store connection string. By default, None.
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
custom topic, for example kafka://<some_kafka_broker>:<port>.
|
|
3234
|
+
:param access_key: Model monitoring access key for managing user permissions.
|
|
3235
|
+
:param endpoint_store_connection: Endpoint store connection string. By default, None. Options:
|
|
3236
|
+
|
|
3237
|
+
* None - will be set from the system configuration.
|
|
3238
|
+
* v3io - for v3io endpoint store, pass `v3io` and the system will generate the
|
|
3239
|
+
exact path.
|
|
3240
|
+
* MySQL/SQLite - for SQL endpoint store, provide the full connection string,
|
|
3241
|
+
for example: mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>
|
|
3242
|
+
:param stream_path: Path to the model monitoring stream. By default, None. Options:
|
|
3243
|
+
|
|
3244
|
+
* None - will be set from the system configuration.
|
|
3245
|
+
* v3io - for v3io stream, pass `v3io` and the system will generate the exact
|
|
3246
|
+
path.
|
|
3247
|
+
* Kafka - for Kafka stream, provide the full connection string without custom
|
|
3248
|
+
topic, for example kafka://<some_kafka_broker>:<port>.
|
|
3235
3249
|
:param tsdb_connection: Connection string to the time series database. By default, None.
|
|
3236
3250
|
Options:
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3251
|
+
|
|
3252
|
+
* None - will be set from the system configuration.
|
|
3253
|
+
* v3io - for v3io stream, pass `v3io` and the system will generate the exact
|
|
3254
|
+
path.
|
|
3255
|
+
* TDEngine - for TDEngine tsdb, provide the full websocket connection URL,
|
|
3256
|
+
for example taosws://<username>:<password>@<host>:<port>.
|
|
3257
|
+
:param replace_creds: If True, will override the existing credentials.
|
|
3243
3258
|
Please keep in mind that if you already enabled model monitoring on
|
|
3244
3259
|
your project this action can cause data loose and will require redeploying
|
|
3245
3260
|
all model monitoring functions & model monitoring infra
|
|
@@ -3345,7 +3360,8 @@ class MlrunProject(ModelObj):
|
|
|
3345
3360
|
* A dictionary of configurations to use when logging. Further info per object type and
|
|
3346
3361
|
artifact type can be given there. The artifact key must appear in the dictionary as
|
|
3347
3362
|
"key": "the_key".
|
|
3348
|
-
:param builder_env:
|
|
3363
|
+
:param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN":
|
|
3364
|
+
token}
|
|
3349
3365
|
:param reset_on_run: When True, function python modules would reload prior to code execution.
|
|
3350
3366
|
This ensures latest code changes are executed. This argument must be used in
|
|
3351
3367
|
conjunction with the local=True argument.
|
|
@@ -4055,7 +4071,7 @@ class MlrunProject(ModelObj):
|
|
|
4055
4071
|
mlrun.db.get_run_db().delete_api_gateway(name=name, project=self.name)
|
|
4056
4072
|
|
|
4057
4073
|
def store_alert_config(
|
|
4058
|
-
self, alert_data: AlertConfig, alert_name=None
|
|
4074
|
+
self, alert_data: AlertConfig, alert_name: typing.Optional[str] = None
|
|
4059
4075
|
) -> AlertConfig:
|
|
4060
4076
|
"""
|
|
4061
4077
|
Create/modify an alert.
|
|
@@ -4064,9 +4080,11 @@ class MlrunProject(ModelObj):
|
|
|
4064
4080
|
:param alert_name: The name of the alert.
|
|
4065
4081
|
:return: the created/modified alert.
|
|
4066
4082
|
"""
|
|
4083
|
+
if not alert_data:
|
|
4084
|
+
raise mlrun.errors.MLRunInvalidArgumentError("Alert data must be provided")
|
|
4085
|
+
|
|
4067
4086
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
4068
|
-
|
|
4069
|
-
alert_name = alert_data.name
|
|
4087
|
+
alert_name = alert_name or alert_data.name
|
|
4070
4088
|
if alert_data.project is not None and alert_data.project != self.metadata.name:
|
|
4071
4089
|
logger.warn(
|
|
4072
4090
|
"Project in alert does not match project in operation",
|
|
@@ -4369,18 +4387,17 @@ def _init_function_from_dict(
|
|
|
4369
4387
|
)
|
|
4370
4388
|
|
|
4371
4389
|
elif url.endswith(".py"):
|
|
4372
|
-
# when load_source_on_run is used we allow not providing image as code will be loaded pre-run. ML-4994
|
|
4373
|
-
if (
|
|
4374
|
-
not image
|
|
4375
|
-
and not project.default_image
|
|
4376
|
-
and kind != "local"
|
|
4377
|
-
and not project.spec.load_source_on_run
|
|
4378
|
-
):
|
|
4379
|
-
raise ValueError(
|
|
4380
|
-
"image must be provided with py code files which do not "
|
|
4381
|
-
"run on 'local' engine kind"
|
|
4382
|
-
)
|
|
4383
4390
|
if in_context and with_repo:
|
|
4391
|
+
# when load_source_on_run is used we allow not providing image as code will be loaded pre-run. ML-4994
|
|
4392
|
+
if (
|
|
4393
|
+
not image
|
|
4394
|
+
and not project.default_image
|
|
4395
|
+
and kind != "local"
|
|
4396
|
+
and not project.spec.load_source_on_run
|
|
4397
|
+
):
|
|
4398
|
+
raise ValueError(
|
|
4399
|
+
"image must be provided with py code files which do not run on 'local' engine kind"
|
|
4400
|
+
)
|
|
4384
4401
|
func = new_function(
|
|
4385
4402
|
name,
|
|
4386
4403
|
command=relative_url,
|
|
@@ -4402,7 +4419,6 @@ def _init_function_from_dict(
|
|
|
4402
4419
|
elif kind in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
|
|
4403
4420
|
func = new_function(
|
|
4404
4421
|
name,
|
|
4405
|
-
command=relative_url,
|
|
4406
4422
|
image=image,
|
|
4407
4423
|
kind=kind,
|
|
4408
4424
|
handler=handler,
|
|
@@ -4456,9 +4472,17 @@ def _init_function_from_obj(
|
|
|
4456
4472
|
def _has_module(handler, kind):
|
|
4457
4473
|
if not handler:
|
|
4458
4474
|
return False
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
|
|
4475
|
+
|
|
4476
|
+
if (
|
|
4477
|
+
kind in mlrun.runtimes.RuntimeKinds.pure_nuclio_deployed_runtimes()
|
|
4478
|
+
and ":" in handler
|
|
4479
|
+
):
|
|
4480
|
+
return True
|
|
4481
|
+
|
|
4482
|
+
if "." in handler:
|
|
4483
|
+
return True
|
|
4484
|
+
|
|
4485
|
+
return False
|
|
4462
4486
|
|
|
4463
4487
|
|
|
4464
4488
|
def _is_imported_artifact(artifact):
|
mlrun/run.py
CHANGED
|
@@ -65,6 +65,7 @@ from .runtimes.nuclio.application import ApplicationRuntime
|
|
|
65
65
|
from .runtimes.utils import add_code_metadata, global_context
|
|
66
66
|
from .utils import (
|
|
67
67
|
RunKeys,
|
|
68
|
+
create_ipython_display,
|
|
68
69
|
extend_hub_uri_if_needed,
|
|
69
70
|
get_in,
|
|
70
71
|
logger,
|
|
@@ -744,11 +745,10 @@ def code_to_function(
|
|
|
744
745
|
raise ValueError("Databricks tasks only support embed_code=True")
|
|
745
746
|
|
|
746
747
|
if kind == RuntimeKinds.application:
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
filename, handler = ApplicationRuntime.get_filename_and_handler()
|
|
748
|
+
raise MLRunInvalidArgumentError(
|
|
749
|
+
"Embedding a code file is not supported for application runtime. "
|
|
750
|
+
"Code files should be specified via project/function source."
|
|
751
|
+
)
|
|
752
752
|
|
|
753
753
|
is_nuclio, sub_kind = RuntimeKinds.resolve_nuclio_sub_kind(kind)
|
|
754
754
|
code_origin = add_name(add_code_metadata(filename), name)
|
|
@@ -942,10 +942,12 @@ def wait_for_pipeline_completion(
|
|
|
942
942
|
if remote:
|
|
943
943
|
mldb = mlrun.db.get_run_db()
|
|
944
944
|
|
|
945
|
+
dag_display_id = create_ipython_display()
|
|
946
|
+
|
|
945
947
|
def _wait_for_pipeline_completion():
|
|
946
948
|
pipeline = mldb.get_pipeline(run_id, namespace=namespace, project=project)
|
|
947
949
|
pipeline_status = pipeline["run"]["status"]
|
|
948
|
-
show_kfp_run(pipeline,
|
|
950
|
+
show_kfp_run(pipeline, dag_display_id=dag_display_id, with_html=False)
|
|
949
951
|
if pipeline_status not in RunStatuses.stable_statuses():
|
|
950
952
|
logger.debug(
|
|
951
953
|
"Waiting for pipeline completion",
|
mlrun/runtimes/__init__.py
CHANGED
|
@@ -30,6 +30,8 @@ __all__ = [
|
|
|
30
30
|
"MpiRuntimeV1",
|
|
31
31
|
]
|
|
32
32
|
|
|
33
|
+
import typing
|
|
34
|
+
|
|
33
35
|
from mlrun.runtimes.utils import resolve_spark_operator_version
|
|
34
36
|
|
|
35
37
|
from ..common.runtimes.constants import MPIJobCRDVersions
|
|
@@ -181,7 +183,7 @@ class RuntimeKinds:
|
|
|
181
183
|
]
|
|
182
184
|
|
|
183
185
|
@staticmethod
|
|
184
|
-
def is_log_collectable_runtime(kind: str):
|
|
186
|
+
def is_log_collectable_runtime(kind: typing.Optional[str]):
|
|
185
187
|
"""
|
|
186
188
|
whether log collector can collect logs for that runtime
|
|
187
189
|
:param kind: kind name
|
|
@@ -192,13 +194,18 @@ class RuntimeKinds:
|
|
|
192
194
|
if RuntimeKinds.is_local_runtime(kind):
|
|
193
195
|
return False
|
|
194
196
|
|
|
195
|
-
if
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
197
|
+
if (
|
|
198
|
+
kind
|
|
199
|
+
not in [
|
|
200
|
+
# dask implementation is different from other runtimes, because few runs can be run against the same
|
|
201
|
+
# runtime resource, so collecting logs on that runtime resource won't be correct, the way we collect
|
|
202
|
+
# logs for dask is by using `log_std` on client side after we execute the code against the cluster,
|
|
203
|
+
# as submitting the run with the dask client will return the run stdout.
|
|
204
|
+
# For more information head to `DaskCluster._run`.
|
|
205
|
+
RuntimeKinds.dask
|
|
206
|
+
]
|
|
207
|
+
+ RuntimeKinds.nuclio_runtimes()
|
|
208
|
+
):
|
|
202
209
|
return True
|
|
203
210
|
|
|
204
211
|
return False
|
|
@@ -235,6 +242,10 @@ class RuntimeKinds:
|
|
|
235
242
|
# both spark and remote spark uses different mechanism for assigning images
|
|
236
243
|
return kind not in [RuntimeKinds.spark, RuntimeKinds.remotespark]
|
|
237
244
|
|
|
245
|
+
@staticmethod
|
|
246
|
+
def supports_from_notebook(kind):
|
|
247
|
+
return kind not in [RuntimeKinds.application]
|
|
248
|
+
|
|
238
249
|
@staticmethod
|
|
239
250
|
def resolve_nuclio_runtime(kind: str, sub_kind: str):
|
|
240
251
|
kind = kind.split(":")[0]
|
|
@@ -386,6 +386,7 @@ class APIGateway(ModelObj):
|
|
|
386
386
|
headers: dict = None,
|
|
387
387
|
credentials: Optional[tuple[str, str]] = None,
|
|
388
388
|
path: Optional[str] = None,
|
|
389
|
+
body: Optional[Union[str, bytes, dict]] = None,
|
|
389
390
|
**kwargs,
|
|
390
391
|
):
|
|
391
392
|
"""
|
|
@@ -396,6 +397,7 @@ class APIGateway(ModelObj):
|
|
|
396
397
|
:param credentials: (Optional[tuple[str, str]], optional) The (username,password) for the invocation if required
|
|
397
398
|
can also be set by the environment variable (_, V3IO_ACCESS_KEY) for access key authentication.
|
|
398
399
|
:param path: (str, optional) The sub-path for the invocation.
|
|
400
|
+
:param body: (Optional[Union[str, bytes, dict]]) The body of the invocation.
|
|
399
401
|
:param kwargs: (dict) Additional keyword arguments.
|
|
400
402
|
|
|
401
403
|
:return: The response from the API gateway invocation.
|
|
@@ -444,6 +446,13 @@ class APIGateway(ModelObj):
|
|
|
444
446
|
"API Gateway invocation requires authentication. Please set V3IO_ACCESS_KEY env var"
|
|
445
447
|
)
|
|
446
448
|
url = urljoin(self.invoke_url, path or "")
|
|
449
|
+
|
|
450
|
+
# Determine the correct keyword argument for the body
|
|
451
|
+
if isinstance(body, dict):
|
|
452
|
+
kwargs["json"] = body
|
|
453
|
+
elif isinstance(body, (str, bytes)):
|
|
454
|
+
kwargs["data"] = body
|
|
455
|
+
|
|
447
456
|
return requests.request(
|
|
448
457
|
method=method,
|
|
449
458
|
url=url,
|
|
@@ -122,6 +122,11 @@ class ApplicationSpec(NuclioSpec):
|
|
|
122
122
|
state_thresholds=state_thresholds,
|
|
123
123
|
disable_default_http_trigger=disable_default_http_trigger,
|
|
124
124
|
)
|
|
125
|
+
|
|
126
|
+
# Override default min/max replicas (don't assume application is stateless)
|
|
127
|
+
self.min_replicas = min_replicas or 1
|
|
128
|
+
self.max_replicas = max_replicas or 1
|
|
129
|
+
|
|
125
130
|
self.internal_application_port = (
|
|
126
131
|
internal_application_port
|
|
127
132
|
or mlrun.mlconf.function.application.default_sidecar_internal_port
|
|
@@ -169,7 +174,7 @@ class ApplicationStatus(NuclioStatus):
|
|
|
169
174
|
self.application_source = application_source or None
|
|
170
175
|
self.sidecar_name = sidecar_name or None
|
|
171
176
|
self.api_gateway_name = api_gateway_name or None
|
|
172
|
-
self.api_gateway = api_gateway or None
|
|
177
|
+
self.api_gateway: typing.Optional[APIGateway] = api_gateway or None
|
|
173
178
|
self.url = url or None
|
|
174
179
|
|
|
175
180
|
|
|
@@ -254,6 +259,15 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
254
259
|
"Application sidecar spec must include a command if args are provided"
|
|
255
260
|
)
|
|
256
261
|
|
|
262
|
+
def prepare_image_for_deploy(self):
|
|
263
|
+
if self.spec.build.source and self.spec.build.load_source_on_run:
|
|
264
|
+
logger.warning(
|
|
265
|
+
"Application runtime requires loading the source into the application image. "
|
|
266
|
+
f"Even though {self.spec.build.load_source_on_run=}, loading on build will be forced."
|
|
267
|
+
)
|
|
268
|
+
self.spec.build.load_source_on_run = False
|
|
269
|
+
super().prepare_image_for_deploy()
|
|
270
|
+
|
|
257
271
|
def deploy(
|
|
258
272
|
self,
|
|
259
273
|
project="",
|
|
@@ -275,6 +289,7 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
275
289
|
"""
|
|
276
290
|
Deploy function, builds the application image if required (self.requires_build()) or force_build is True,
|
|
277
291
|
Once the image is built, the function is deployed.
|
|
292
|
+
|
|
278
293
|
:param project: Project name
|
|
279
294
|
:param tag: Function tag
|
|
280
295
|
:param verbose: Set True for verbose logging
|
|
@@ -349,9 +364,13 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
349
364
|
)
|
|
350
365
|
|
|
351
366
|
def with_source_archive(
|
|
352
|
-
self,
|
|
367
|
+
self,
|
|
368
|
+
source,
|
|
369
|
+
workdir=None,
|
|
370
|
+
pull_at_runtime: bool = False,
|
|
371
|
+
target_dir: str = None,
|
|
353
372
|
):
|
|
354
|
-
"""load the code from git/tar/zip archive at
|
|
373
|
+
"""load the code from git/tar/zip archive at build
|
|
355
374
|
|
|
356
375
|
:param source: valid absolute path or URL to git, zip, or tar file, e.g.
|
|
357
376
|
git://github.com/mlrun/something.git
|
|
@@ -359,13 +378,20 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
359
378
|
note path source must exist on the image or exist locally when run is local
|
|
360
379
|
(it is recommended to use 'workdir' when source is a filepath instead)
|
|
361
380
|
:param workdir: working dir relative to the archive root (e.g. './subdir') or absolute to the image root
|
|
362
|
-
:param pull_at_runtime:
|
|
381
|
+
:param pull_at_runtime: currently not supported, source must be loaded into the image during the build process
|
|
363
382
|
:param target_dir: target dir on runtime pod or repo clone / archive extraction
|
|
364
383
|
"""
|
|
384
|
+
if pull_at_runtime:
|
|
385
|
+
logger.warning(
|
|
386
|
+
f"{pull_at_runtime=} is currently not supported for application runtime "
|
|
387
|
+
"and will be overridden to False",
|
|
388
|
+
pull_at_runtime=pull_at_runtime,
|
|
389
|
+
)
|
|
390
|
+
|
|
365
391
|
self._configure_mlrun_build_with_source(
|
|
366
392
|
source=source,
|
|
367
393
|
workdir=workdir,
|
|
368
|
-
pull_at_runtime=
|
|
394
|
+
pull_at_runtime=False,
|
|
369
395
|
target_dir=target_dir,
|
|
370
396
|
)
|
|
371
397
|
|
|
@@ -455,7 +481,7 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
455
481
|
def invoke(
|
|
456
482
|
self,
|
|
457
483
|
path: str,
|
|
458
|
-
body: typing.Union[str, bytes, dict] = None,
|
|
484
|
+
body: typing.Optional[typing.Union[str, bytes, dict]] = None,
|
|
459
485
|
method: str = None,
|
|
460
486
|
headers: dict = None,
|
|
461
487
|
dashboard: str = "",
|
|
@@ -483,11 +509,13 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
483
509
|
|
|
484
510
|
if not method:
|
|
485
511
|
method = "POST" if body else "GET"
|
|
512
|
+
|
|
486
513
|
return self.status.api_gateway.invoke(
|
|
487
514
|
method=method,
|
|
488
515
|
headers=headers,
|
|
489
516
|
credentials=credentials,
|
|
490
517
|
path=path,
|
|
518
|
+
body=body,
|
|
491
519
|
**http_client_kwargs,
|
|
492
520
|
)
|
|
493
521
|
|
|
@@ -524,6 +552,20 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
524
552
|
reverse_proxy_func.metadata.name, reverse_proxy_func.metadata.project
|
|
525
553
|
)
|
|
526
554
|
|
|
555
|
+
@min_nuclio_versions("1.13.1")
|
|
556
|
+
def disable_default_http_trigger(
|
|
557
|
+
self,
|
|
558
|
+
):
|
|
559
|
+
raise mlrun.runtimes.RunError(
|
|
560
|
+
"Application runtime does not support disabling the default HTTP trigger"
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
@min_nuclio_versions("1.13.1")
|
|
564
|
+
def enable_default_http_trigger(
|
|
565
|
+
self,
|
|
566
|
+
):
|
|
567
|
+
pass
|
|
568
|
+
|
|
527
569
|
def _run(self, runobj: "mlrun.RunObject", execution):
|
|
528
570
|
raise mlrun.runtimes.RunError(
|
|
529
571
|
"Application runtime .run() is not yet supported. Use .invoke() instead."
|
|
@@ -551,6 +593,13 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
551
593
|
args=self.spec.args,
|
|
552
594
|
)
|
|
553
595
|
|
|
596
|
+
if self.spec.build.source in [".", "./"]:
|
|
597
|
+
logger.info(
|
|
598
|
+
"The application is configured to use the project's source. "
|
|
599
|
+
"Application runtime requires loading the source into the application image. "
|
|
600
|
+
"Loading on build will be forced regardless of whether 'pull_at_runtime=True' was configured."
|
|
601
|
+
)
|
|
602
|
+
|
|
554
603
|
with_mlrun = self._resolve_build_with_mlrun(with_mlrun)
|
|
555
604
|
return self._build_image(
|
|
556
605
|
builder_env=builder_env,
|
|
@@ -580,6 +629,13 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
580
629
|
)
|
|
581
630
|
function.spec.nuclio_runtime = mlrun.utils.get_in(spec, "spec.runtime")
|
|
582
631
|
|
|
632
|
+
# default the reverse proxy logger level to info
|
|
633
|
+
logger_sinks_key = "spec.loggerSinks"
|
|
634
|
+
if not function.spec.config.get(logger_sinks_key):
|
|
635
|
+
function.set_config(
|
|
636
|
+
logger_sinks_key, [{"level": "info", "sink": "myStdoutLoggerSink"}]
|
|
637
|
+
)
|
|
638
|
+
|
|
583
639
|
def _configure_application_sidecar(self):
|
|
584
640
|
# Save the application image in the status to allow overriding it with the reverse proxy entry point
|
|
585
641
|
if self.spec.image and (
|
|
@@ -608,9 +664,8 @@ class ApplicationRuntime(RemoteRuntime):
|
|
|
608
664
|
self.set_env("SIDECAR_HOST", "http://localhost")
|
|
609
665
|
|
|
610
666
|
# configure the sidecar container as the default container for logging purposes
|
|
611
|
-
self.
|
|
612
|
-
|
|
613
|
-
{"kubectl.kubernetes.io/default-container": self.status.sidecar_name},
|
|
667
|
+
self.metadata.annotations["kubectl.kubernetes.io/default-container"] = (
|
|
668
|
+
self.status.sidecar_name
|
|
614
669
|
)
|
|
615
670
|
|
|
616
671
|
def _sync_api_gateway(self):
|