mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +2 -35
- mlrun/__main__.py +3 -41
- mlrun/api/api/api.py +6 -0
- mlrun/api/api/endpoints/feature_store.py +0 -4
- mlrun/api/api/endpoints/files.py +14 -2
- mlrun/api/api/endpoints/frontend_spec.py +2 -1
- mlrun/api/api/endpoints/functions.py +95 -59
- mlrun/api/api/endpoints/grafana_proxy.py +9 -9
- mlrun/api/api/endpoints/logs.py +17 -3
- mlrun/api/api/endpoints/model_endpoints.py +3 -2
- mlrun/api/api/endpoints/pipelines.py +1 -5
- mlrun/api/api/endpoints/projects.py +88 -0
- mlrun/api/api/endpoints/runs.py +48 -6
- mlrun/api/api/endpoints/submit.py +2 -1
- mlrun/api/api/endpoints/workflows.py +355 -0
- mlrun/api/api/utils.py +3 -4
- mlrun/api/crud/__init__.py +1 -0
- mlrun/api/crud/client_spec.py +6 -2
- mlrun/api/crud/feature_store.py +5 -0
- mlrun/api/crud/model_monitoring/__init__.py +1 -0
- mlrun/api/crud/model_monitoring/deployment.py +497 -0
- mlrun/api/crud/model_monitoring/grafana.py +96 -42
- mlrun/api/crud/model_monitoring/helpers.py +159 -0
- mlrun/api/crud/model_monitoring/model_endpoints.py +202 -476
- mlrun/api/crud/notifications.py +9 -4
- mlrun/api/crud/pipelines.py +6 -11
- mlrun/api/crud/projects.py +2 -2
- mlrun/api/crud/runtime_resources.py +4 -3
- mlrun/api/crud/runtimes/nuclio/helpers.py +5 -1
- mlrun/api/crud/secrets.py +21 -0
- mlrun/api/crud/workflows.py +352 -0
- mlrun/api/db/base.py +16 -1
- mlrun/api/db/init_db.py +2 -4
- mlrun/api/db/session.py +1 -1
- mlrun/api/db/sqldb/db.py +129 -31
- mlrun/api/db/sqldb/models/models_mysql.py +15 -1
- mlrun/api/db/sqldb/models/models_sqlite.py +16 -2
- mlrun/api/launcher.py +38 -6
- mlrun/api/main.py +3 -2
- mlrun/api/rundb/__init__.py +13 -0
- mlrun/{db → api/rundb}/sqldb.py +36 -84
- mlrun/api/runtime_handlers/__init__.py +56 -0
- mlrun/api/runtime_handlers/base.py +1247 -0
- mlrun/api/runtime_handlers/daskjob.py +209 -0
- mlrun/api/runtime_handlers/kubejob.py +37 -0
- mlrun/api/runtime_handlers/mpijob.py +147 -0
- mlrun/api/runtime_handlers/remotesparkjob.py +29 -0
- mlrun/api/runtime_handlers/sparkjob.py +148 -0
- mlrun/api/schemas/__init__.py +17 -6
- mlrun/api/utils/builder.py +1 -4
- mlrun/api/utils/clients/chief.py +14 -0
- mlrun/api/utils/clients/iguazio.py +33 -33
- mlrun/api/utils/clients/nuclio.py +2 -2
- mlrun/api/utils/periodic.py +9 -2
- mlrun/api/utils/projects/follower.py +14 -7
- mlrun/api/utils/projects/leader.py +2 -1
- mlrun/api/utils/projects/remotes/nop_follower.py +2 -2
- mlrun/api/utils/projects/remotes/nop_leader.py +2 -2
- mlrun/api/utils/runtimes/__init__.py +14 -0
- mlrun/api/utils/runtimes/nuclio.py +43 -0
- mlrun/api/utils/scheduler.py +98 -15
- mlrun/api/utils/singletons/db.py +5 -1
- mlrun/api/utils/singletons/project_member.py +4 -1
- mlrun/api/utils/singletons/scheduler.py +1 -1
- mlrun/artifacts/base.py +6 -6
- mlrun/artifacts/dataset.py +4 -4
- mlrun/artifacts/manager.py +2 -3
- mlrun/artifacts/model.py +2 -2
- mlrun/artifacts/plots.py +8 -8
- mlrun/common/db/__init__.py +14 -0
- mlrun/common/helpers.py +37 -0
- mlrun/{mlutils → common/model_monitoring}/__init__.py +3 -2
- mlrun/common/model_monitoring/helpers.py +69 -0
- mlrun/common/schemas/__init__.py +13 -1
- mlrun/common/schemas/auth.py +4 -1
- mlrun/common/schemas/client_spec.py +1 -1
- mlrun/common/schemas/function.py +17 -0
- mlrun/common/schemas/model_monitoring/__init__.py +48 -0
- mlrun/common/{model_monitoring.py → schemas/model_monitoring/constants.py} +11 -23
- mlrun/common/schemas/model_monitoring/grafana.py +55 -0
- mlrun/common/schemas/{model_endpoints.py → model_monitoring/model_endpoints.py} +32 -65
- mlrun/common/schemas/notification.py +1 -0
- mlrun/common/schemas/object.py +4 -0
- mlrun/common/schemas/project.py +1 -0
- mlrun/common/schemas/regex.py +1 -1
- mlrun/common/schemas/runs.py +1 -8
- mlrun/common/schemas/schedule.py +1 -8
- mlrun/common/schemas/workflow.py +54 -0
- mlrun/config.py +45 -42
- mlrun/datastore/__init__.py +21 -0
- mlrun/datastore/base.py +1 -1
- mlrun/datastore/datastore.py +9 -0
- mlrun/datastore/dbfs_store.py +168 -0
- mlrun/datastore/helpers.py +18 -0
- mlrun/datastore/sources.py +1 -0
- mlrun/datastore/store_resources.py +2 -5
- mlrun/datastore/v3io.py +1 -2
- mlrun/db/__init__.py +4 -68
- mlrun/db/base.py +12 -0
- mlrun/db/factory.py +65 -0
- mlrun/db/httpdb.py +175 -20
- mlrun/db/nopdb.py +4 -2
- mlrun/execution.py +4 -2
- mlrun/feature_store/__init__.py +1 -0
- mlrun/feature_store/api.py +1 -2
- mlrun/feature_store/common.py +2 -1
- mlrun/feature_store/feature_set.py +1 -11
- mlrun/feature_store/feature_vector.py +340 -2
- mlrun/feature_store/ingestion.py +5 -10
- mlrun/feature_store/retrieval/base.py +118 -104
- mlrun/feature_store/retrieval/dask_merger.py +17 -10
- mlrun/feature_store/retrieval/job.py +4 -1
- mlrun/feature_store/retrieval/local_merger.py +18 -18
- mlrun/feature_store/retrieval/spark_merger.py +21 -14
- mlrun/feature_store/retrieval/storey_merger.py +22 -16
- mlrun/kfpops.py +3 -9
- mlrun/launcher/base.py +57 -53
- mlrun/launcher/client.py +5 -4
- mlrun/launcher/factory.py +24 -13
- mlrun/launcher/local.py +6 -6
- mlrun/launcher/remote.py +4 -4
- mlrun/lists.py +0 -11
- mlrun/model.py +11 -17
- mlrun/model_monitoring/__init__.py +2 -22
- mlrun/model_monitoring/features_drift_table.py +1 -1
- mlrun/model_monitoring/helpers.py +22 -210
- mlrun/model_monitoring/model_endpoint.py +1 -1
- mlrun/model_monitoring/model_monitoring_batch.py +127 -50
- mlrun/model_monitoring/prometheus.py +219 -0
- mlrun/model_monitoring/stores/__init__.py +16 -11
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +95 -23
- mlrun/model_monitoring/stores/models/mysql.py +47 -29
- mlrun/model_monitoring/stores/models/sqlite.py +47 -29
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +31 -19
- mlrun/model_monitoring/{stream_processing_fs.py → stream_processing.py} +206 -64
- mlrun/model_monitoring/tracking_policy.py +104 -0
- mlrun/package/packager.py +6 -8
- mlrun/package/packagers/default_packager.py +121 -10
- mlrun/package/packagers/numpy_packagers.py +1 -1
- mlrun/platforms/__init__.py +0 -2
- mlrun/platforms/iguazio.py +0 -56
- mlrun/projects/pipelines.py +53 -159
- mlrun/projects/project.py +10 -37
- mlrun/render.py +1 -1
- mlrun/run.py +8 -124
- mlrun/runtimes/__init__.py +6 -42
- mlrun/runtimes/base.py +29 -1249
- mlrun/runtimes/daskjob.py +2 -198
- mlrun/runtimes/funcdoc.py +0 -9
- mlrun/runtimes/function.py +25 -29
- mlrun/runtimes/kubejob.py +5 -29
- mlrun/runtimes/local.py +1 -1
- mlrun/runtimes/mpijob/__init__.py +2 -2
- mlrun/runtimes/mpijob/abstract.py +10 -1
- mlrun/runtimes/mpijob/v1.py +0 -76
- mlrun/runtimes/mpijob/v1alpha1.py +1 -74
- mlrun/runtimes/nuclio.py +3 -2
- mlrun/runtimes/pod.py +28 -18
- mlrun/runtimes/remotesparkjob.py +1 -15
- mlrun/runtimes/serving.py +14 -6
- mlrun/runtimes/sparkjob/__init__.py +0 -1
- mlrun/runtimes/sparkjob/abstract.py +4 -131
- mlrun/runtimes/utils.py +0 -26
- mlrun/serving/routers.py +7 -7
- mlrun/serving/server.py +11 -8
- mlrun/serving/states.py +7 -1
- mlrun/serving/v2_serving.py +6 -6
- mlrun/utils/helpers.py +23 -42
- mlrun/utils/notifications/notification/__init__.py +4 -0
- mlrun/utils/notifications/notification/webhook.py +61 -0
- mlrun/utils/notifications/notification_pusher.py +5 -25
- mlrun/utils/regex.py +7 -2
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/METADATA +26 -25
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/RECORD +180 -158
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/WHEEL +1 -1
- mlrun/mlutils/data.py +0 -160
- mlrun/mlutils/models.py +0 -78
- mlrun/mlutils/plots.py +0 -902
- mlrun/utils/model_monitoring.py +0 -249
- /mlrun/{api/db/sqldb/session.py → common/db/sql_session.py} +0 -0
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/LICENSE +0 -0
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/entry_points.txt +0 -0
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/top_level.txt +0 -0
mlrun/__init__.py
CHANGED
|
@@ -36,14 +36,7 @@ from .errors import MLRunInvalidArgumentError, MLRunNotFoundError
|
|
|
36
36
|
from .execution import MLClientCtx
|
|
37
37
|
from .model import RunObject, RunTemplate, new_task
|
|
38
38
|
from .package import ArtifactType, DefaultPackager, Packager, handler
|
|
39
|
-
from .platforms import
|
|
40
|
-
VolumeMount,
|
|
41
|
-
auto_mount,
|
|
42
|
-
mount_v3io,
|
|
43
|
-
mount_v3io_extended,
|
|
44
|
-
mount_v3io_legacy,
|
|
45
|
-
v3io_cred,
|
|
46
|
-
)
|
|
39
|
+
from .platforms import VolumeMount, auto_mount, mount_v3io, v3io_cred
|
|
47
40
|
from .projects import (
|
|
48
41
|
ProjectMetadata,
|
|
49
42
|
build_function,
|
|
@@ -66,7 +59,6 @@ from .run import (
|
|
|
66
59
|
import_function,
|
|
67
60
|
new_function,
|
|
68
61
|
run_local,
|
|
69
|
-
run_pipeline,
|
|
70
62
|
wait_for_pipeline_completion,
|
|
71
63
|
)
|
|
72
64
|
from .runtimes import new_model_server
|
|
@@ -91,9 +83,7 @@ if "IGZ_NAMESPACE_DOMAIN" in environ:
|
|
|
91
83
|
def set_environment(
|
|
92
84
|
api_path: str = None,
|
|
93
85
|
artifact_path: str = "",
|
|
94
|
-
project: str = "",
|
|
95
86
|
access_key: str = None,
|
|
96
|
-
user_project=False,
|
|
97
87
|
username: str = None,
|
|
98
88
|
env_file: str = None,
|
|
99
89
|
mock_functions: str = None,
|
|
@@ -109,18 +99,14 @@ def set_environment(
|
|
|
109
99
|
example::
|
|
110
100
|
|
|
111
101
|
from os import path
|
|
112
|
-
project_name, artifact_path = set_environment(
|
|
102
|
+
project_name, artifact_path = set_environment()
|
|
113
103
|
set_environment("http://localhost:8080", artifact_path="./")
|
|
114
104
|
set_environment(env_file="mlrun.env")
|
|
115
105
|
set_environment("<remote-service-url>", access_key="xyz", username="joe")
|
|
116
106
|
|
|
117
107
|
:param api_path: location/url of mlrun api service
|
|
118
108
|
:param artifact_path: path/url for storing experiment artifacts
|
|
119
|
-
:param project: default project name (deprecated in 1.3.0 and will be removed in 1.5.0) - use project
|
|
120
|
-
APIs such as `get_or_create_project`, `load_project` to configure the active project
|
|
121
109
|
:param access_key: set the remote cluster access key (V3IO_ACCESS_KEY)
|
|
122
|
-
:param user_project: add the current user name to the provided project name (making it unique per user)
|
|
123
|
-
(deprecated in 1.3.0 and will be removed in 1.5.0)
|
|
124
110
|
:param username: name of the user to authenticate
|
|
125
111
|
:param env_file: path/url to .env file (holding MLRun config and other env vars), see: set_env_from_file()
|
|
126
112
|
:param mock_functions: set to True to create local/mock functions instead of real containers,
|
|
@@ -129,14 +115,6 @@ def set_environment(
|
|
|
129
115
|
default project name
|
|
130
116
|
actual artifact path/url, can be used to create subpaths per task or group of artifacts
|
|
131
117
|
"""
|
|
132
|
-
if user_project or project:
|
|
133
|
-
warnings.warn(
|
|
134
|
-
"'user_project' and 'project' are deprecated in 1.3.0, and will be removed in 1.5.0, use project "
|
|
135
|
-
"APIs such as 'get_or_create_project', 'load_project' to configure the active project.",
|
|
136
|
-
# TODO: Remove in 1.5.0
|
|
137
|
-
FutureWarning,
|
|
138
|
-
)
|
|
139
|
-
|
|
140
118
|
if env_file:
|
|
141
119
|
set_env_from_file(env_file)
|
|
142
120
|
|
|
@@ -160,17 +138,6 @@ def set_environment(
|
|
|
160
138
|
if api_path:
|
|
161
139
|
environ["MLRUN_DBPATH"] = mlconf.dbpath
|
|
162
140
|
|
|
163
|
-
project = _add_username_to_project_name_if_needed(project, user_project)
|
|
164
|
-
if project:
|
|
165
|
-
ProjectMetadata.validate_project_name(project)
|
|
166
|
-
|
|
167
|
-
mlconf.default_project = project or mlconf.default_project
|
|
168
|
-
# We want to ensure the project exists, and verify we're authorized to work on it
|
|
169
|
-
# if it doesn't exist this will create it (and obviously if we created it, we're authorized to work on it)
|
|
170
|
-
# if it does exist - this will get it, which will fail if we're not authorized to work on it
|
|
171
|
-
if project:
|
|
172
|
-
get_or_create_project(mlconf.default_project, "./")
|
|
173
|
-
|
|
174
141
|
if not mlconf.artifact_path and not artifact_path:
|
|
175
142
|
raise ValueError(
|
|
176
143
|
"default artifact_path was not configured, please specify a valid artifact_path"
|
mlrun/__main__.py
CHANGED
|
@@ -17,7 +17,6 @@ import json
|
|
|
17
17
|
import pathlib
|
|
18
18
|
import socket
|
|
19
19
|
import traceback
|
|
20
|
-
import warnings
|
|
21
20
|
from ast import literal_eval
|
|
22
21
|
from base64 import b64decode, b64encode
|
|
23
22
|
from os import environ, path, remove
|
|
@@ -33,6 +32,7 @@ import yaml
|
|
|
33
32
|
from tabulate import tabulate
|
|
34
33
|
|
|
35
34
|
import mlrun
|
|
35
|
+
from mlrun.common.helpers import parse_versioned_object_uri
|
|
36
36
|
|
|
37
37
|
from .config import config as mlconf
|
|
38
38
|
from .db import get_run_db
|
|
@@ -55,7 +55,6 @@ from .utils import (
|
|
|
55
55
|
is_relative_path,
|
|
56
56
|
list2dict,
|
|
57
57
|
logger,
|
|
58
|
-
parse_versioned_object_uri,
|
|
59
58
|
run_keys,
|
|
60
59
|
update_in,
|
|
61
60
|
)
|
|
@@ -592,12 +591,6 @@ def build(
|
|
|
592
591
|
default="",
|
|
593
592
|
help="path/url of function yaml or function " "yaml or db://<project>/<name>[:tag]",
|
|
594
593
|
)
|
|
595
|
-
@click.option(
|
|
596
|
-
"--dashboard",
|
|
597
|
-
"-d",
|
|
598
|
-
default="",
|
|
599
|
-
help="Deprecated. Keep empty to allow auto-detect by MLRun API",
|
|
600
|
-
)
|
|
601
594
|
@click.option("--project", "-p", default="", help="project name")
|
|
602
595
|
@click.option("--model", "-m", multiple=True, help="model name and path (name=path)")
|
|
603
596
|
@click.option("--kind", "-k", default=None, help="runtime sub kind")
|
|
@@ -616,7 +609,6 @@ def deploy(
|
|
|
616
609
|
spec,
|
|
617
610
|
source,
|
|
618
611
|
func_url,
|
|
619
|
-
dashboard,
|
|
620
612
|
project,
|
|
621
613
|
model,
|
|
622
614
|
tag,
|
|
@@ -677,16 +669,8 @@ def deploy(
|
|
|
677
669
|
function.set_env(k, v)
|
|
678
670
|
function.verbose = verbose
|
|
679
671
|
|
|
680
|
-
if dashboard:
|
|
681
|
-
warnings.warn(
|
|
682
|
-
"'--dashboard' is deprecated in 1.3.0, and will be removed in 1.5.0, "
|
|
683
|
-
"Keep '--dashboard' value empty to allow auto-detection by MLRun API.",
|
|
684
|
-
# TODO: Remove in 1.5.0
|
|
685
|
-
FutureWarning,
|
|
686
|
-
)
|
|
687
|
-
|
|
688
672
|
try:
|
|
689
|
-
addr = function.deploy(
|
|
673
|
+
addr = function.deploy(project=project, tag=tag)
|
|
690
674
|
except Exception as err:
|
|
691
675
|
print(f"deploy error: {err_to_str(err)}")
|
|
692
676
|
exit(1)
|
|
@@ -767,6 +751,7 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
|
|
|
767
751
|
]
|
|
768
752
|
df["tree"] = df["tree"].apply(lambda x: f"..{x[-8:]}")
|
|
769
753
|
df["hash"] = df["hash"].apply(lambda x: f"..{x[-6:]}")
|
|
754
|
+
df["updated"] = df["updated"].apply(time_str)
|
|
770
755
|
print(tabulate(df, headers="keys"))
|
|
771
756
|
|
|
772
757
|
elif kind.startswith("func"):
|
|
@@ -1004,12 +989,6 @@ def logs(uid, project, offset, db, watch):
|
|
|
1004
989
|
@click.option(
|
|
1005
990
|
"--env-file", default="", help="path to .env file to load config/variables from"
|
|
1006
991
|
)
|
|
1007
|
-
# TODO: Remove --ensure-project in 1.5.0
|
|
1008
|
-
@click.option(
|
|
1009
|
-
"--ensure-project",
|
|
1010
|
-
is_flag=True,
|
|
1011
|
-
help="ensure the project exists, if not, create project",
|
|
1012
|
-
)
|
|
1013
992
|
@click.option(
|
|
1014
993
|
"--save/--no-save",
|
|
1015
994
|
default=True,
|
|
@@ -1024,13 +1003,6 @@ def logs(uid, project, offset, db, watch):
|
|
|
1024
1003
|
"https://apscheduler.readthedocs.io/en/3.x/modules/triggers/cron.html#module-apscheduler.triggers.cron."
|
|
1025
1004
|
"For using the pre-defined workflow's schedule, set --schedule 'true'",
|
|
1026
1005
|
)
|
|
1027
|
-
# TODO: Remove in 1.5.0
|
|
1028
|
-
@click.option(
|
|
1029
|
-
"--overwrite-schedule",
|
|
1030
|
-
"-os",
|
|
1031
|
-
is_flag=True,
|
|
1032
|
-
help="Overwrite a schedule when submitting a new one with the same name.",
|
|
1033
|
-
)
|
|
1034
1006
|
@click.option(
|
|
1035
1007
|
"--save-secrets",
|
|
1036
1008
|
is_flag=True,
|
|
@@ -1068,10 +1040,8 @@ def project(
|
|
|
1068
1040
|
local,
|
|
1069
1041
|
env_file,
|
|
1070
1042
|
timeout,
|
|
1071
|
-
ensure_project,
|
|
1072
1043
|
schedule,
|
|
1073
1044
|
notifications,
|
|
1074
|
-
overwrite_schedule,
|
|
1075
1045
|
save_secrets,
|
|
1076
1046
|
save,
|
|
1077
1047
|
):
|
|
@@ -1079,13 +1049,6 @@ def project(
|
|
|
1079
1049
|
if env_file:
|
|
1080
1050
|
mlrun.set_env_from_file(env_file)
|
|
1081
1051
|
|
|
1082
|
-
if ensure_project:
|
|
1083
|
-
warnings.warn(
|
|
1084
|
-
"'ensure_project' is deprecated and will be removed in 1.5.0, use 'save' (True by default) instead. ",
|
|
1085
|
-
# TODO: Remove this in 1.5.0
|
|
1086
|
-
FutureWarning,
|
|
1087
|
-
)
|
|
1088
|
-
|
|
1089
1052
|
if db:
|
|
1090
1053
|
mlconf.dbpath = db
|
|
1091
1054
|
|
|
@@ -1168,7 +1131,6 @@ def project(
|
|
|
1168
1131
|
local=local,
|
|
1169
1132
|
schedule=schedule,
|
|
1170
1133
|
timeout=timeout,
|
|
1171
|
-
overwrite=overwrite_schedule,
|
|
1172
1134
|
)
|
|
1173
1135
|
except Exception as err:
|
|
1174
1136
|
print(traceback.format_exc())
|
mlrun/api/api/api.py
CHANGED
|
@@ -41,6 +41,7 @@ from mlrun.api.api.endpoints import (
|
|
|
41
41
|
secrets,
|
|
42
42
|
submit,
|
|
43
43
|
tags,
|
|
44
|
+
workflows,
|
|
44
45
|
)
|
|
45
46
|
|
|
46
47
|
api_router = APIRouter(dependencies=[Depends(mlrun.api.api.deps.verify_api_state)])
|
|
@@ -145,3 +146,8 @@ api_router.include_router(
|
|
|
145
146
|
tags=["internal"],
|
|
146
147
|
dependencies=[Depends(mlrun.api.api.deps.authenticate_request)],
|
|
147
148
|
)
|
|
149
|
+
api_router.include_router(
|
|
150
|
+
workflows.router,
|
|
151
|
+
tags=["workflows"],
|
|
152
|
+
dependencies=[Depends(mlrun.api.api.deps.authenticate_request)],
|
|
153
|
+
)
|
|
@@ -408,10 +408,6 @@ async def ingest_feature_set(
|
|
|
408
408
|
mlrun.common.schemas.AuthorizationAction.read,
|
|
409
409
|
auth_info,
|
|
410
410
|
)
|
|
411
|
-
# Need to override the default rundb since we're in the server.
|
|
412
|
-
# this is done so further down the flow when running the function created for ingestion we won't access the httpdb
|
|
413
|
-
# but rather "understand" that we are running on server side and call the DB.
|
|
414
|
-
await run_in_threadpool(feature_set._override_run_db, db_session)
|
|
415
411
|
|
|
416
412
|
if ingest_parameters.targets:
|
|
417
413
|
data_targets = [
|
mlrun/api/api/endpoints/files.py
CHANGED
|
@@ -31,7 +31,13 @@ from mlrun.utils import logger
|
|
|
31
31
|
router = fastapi.APIRouter()
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
|
|
34
|
+
# TODO: Remove in 1.7.0
|
|
35
|
+
@router.get(
|
|
36
|
+
"/files",
|
|
37
|
+
deprecated=True,
|
|
38
|
+
description="'/files' and '/filestat' will be removed in 1.7.0, "
|
|
39
|
+
"use /projects/{project}/files instead.",
|
|
40
|
+
)
|
|
35
41
|
def get_files(
|
|
36
42
|
schema: str = "",
|
|
37
43
|
objpath: str = fastapi.Query("", alias="path"),
|
|
@@ -73,7 +79,13 @@ async def get_files_with_project_secrets(
|
|
|
73
79
|
)
|
|
74
80
|
|
|
75
81
|
|
|
76
|
-
|
|
82
|
+
# TODO: Remove in 1.7.0
|
|
83
|
+
@router.get(
|
|
84
|
+
"/filestat",
|
|
85
|
+
deprecated=True,
|
|
86
|
+
description="'/files' and '/filestat' will be removed in 1.7.0, "
|
|
87
|
+
"use /projects/{project}/filestat instead.",
|
|
88
|
+
)
|
|
77
89
|
def get_filestat(
|
|
78
90
|
schema: str = "",
|
|
79
91
|
path: str = "",
|
|
@@ -20,6 +20,7 @@ import semver
|
|
|
20
20
|
import mlrun.api.api.deps
|
|
21
21
|
import mlrun.api.utils.builder
|
|
22
22
|
import mlrun.api.utils.clients.iguazio
|
|
23
|
+
import mlrun.api.utils.runtimes.nuclio
|
|
23
24
|
import mlrun.common.schemas
|
|
24
25
|
import mlrun.runtimes
|
|
25
26
|
import mlrun.runtimes.utils
|
|
@@ -117,7 +118,7 @@ def _resolve_feature_flags() -> mlrun.common.schemas.FeatureFlags:
|
|
|
117
118
|
nuclio_streams = mlrun.common.schemas.NuclioStreamsFeatureFlag.disabled
|
|
118
119
|
|
|
119
120
|
if mlrun.mlconf.get_parsed_igz_version() and semver.VersionInfo.parse(
|
|
120
|
-
mlrun.
|
|
121
|
+
mlrun.api.utils.runtimes.nuclio.resolve_nuclio_version()
|
|
121
122
|
) >= semver.VersionInfo.parse("1.7.8"):
|
|
122
123
|
nuclio_streams = mlrun.common.schemas.NuclioStreamsFeatureFlag.enabled
|
|
123
124
|
|
|
@@ -34,7 +34,7 @@ from fastapi.concurrency import run_in_threadpool
|
|
|
34
34
|
from kubernetes.client.rest import ApiException
|
|
35
35
|
from sqlalchemy.orm import Session
|
|
36
36
|
|
|
37
|
-
import mlrun.api.crud
|
|
37
|
+
import mlrun.api.crud.model_monitoring.deployment
|
|
38
38
|
import mlrun.api.crud.runtimes.nuclio.function
|
|
39
39
|
import mlrun.api.db.session
|
|
40
40
|
import mlrun.api.launcher
|
|
@@ -44,19 +44,22 @@ import mlrun.api.utils.clients.chief
|
|
|
44
44
|
import mlrun.api.utils.singletons.k8s
|
|
45
45
|
import mlrun.api.utils.singletons.project_member
|
|
46
46
|
import mlrun.common.model_monitoring
|
|
47
|
+
import mlrun.common.model_monitoring.helpers
|
|
47
48
|
import mlrun.common.schemas
|
|
48
49
|
from mlrun.api.api import deps
|
|
49
50
|
from mlrun.api.api.utils import get_run_db_instance, log_and_raise, log_path
|
|
50
51
|
from mlrun.api.crud.secrets import Secrets, SecretsClientType
|
|
51
52
|
from mlrun.api.utils.builder import build_runtime
|
|
52
53
|
from mlrun.api.utils.singletons.scheduler import get_scheduler
|
|
54
|
+
from mlrun.common.helpers import parse_versioned_object_uri
|
|
55
|
+
from mlrun.common.model_monitoring.helpers import parse_model_endpoint_store_prefix
|
|
53
56
|
from mlrun.config import config
|
|
54
57
|
from mlrun.errors import MLRunRuntimeError, err_to_str
|
|
58
|
+
from mlrun.model_monitoring.tracking_policy import TrackingPolicy
|
|
55
59
|
from mlrun.run import new_function
|
|
56
60
|
from mlrun.runtimes import RuntimeKinds, ServingRuntime, runtime_resources_map
|
|
57
61
|
from mlrun.runtimes.utils import get_item_name
|
|
58
|
-
from mlrun.utils import get_in, logger,
|
|
59
|
-
from mlrun.utils.model_monitoring import parse_model_endpoint_store_prefix
|
|
62
|
+
from mlrun.utils import get_in, logger, update_in
|
|
60
63
|
|
|
61
64
|
router = APIRouter()
|
|
62
65
|
|
|
@@ -448,21 +451,21 @@ async def build_status(
|
|
|
448
451
|
|
|
449
452
|
|
|
450
453
|
def _handle_job_deploy_status(
|
|
451
|
-
db_session,
|
|
452
|
-
fn,
|
|
453
|
-
name,
|
|
454
|
-
project,
|
|
455
|
-
tag,
|
|
456
|
-
offset,
|
|
457
|
-
logs,
|
|
454
|
+
db_session: Session,
|
|
455
|
+
fn: dict,
|
|
456
|
+
name: str,
|
|
457
|
+
project: str,
|
|
458
|
+
tag: str,
|
|
459
|
+
offset: int,
|
|
460
|
+
logs: bool,
|
|
458
461
|
):
|
|
459
462
|
# job deploy status
|
|
460
|
-
|
|
463
|
+
function_state = get_in(fn, "status.state", "")
|
|
461
464
|
pod = get_in(fn, "status.build_pod", "")
|
|
462
465
|
image = get_in(fn, "spec.build.image", "")
|
|
463
466
|
out = b""
|
|
464
467
|
if not pod:
|
|
465
|
-
if
|
|
468
|
+
if function_state == mlrun.common.schemas.FunctionState.ready:
|
|
466
469
|
# when the function has been built we set the created image into the `spec.image` for reference see at the
|
|
467
470
|
# end of the function where we resolve if the status is ready and then set the spec.build.image to
|
|
468
471
|
# spec.image
|
|
@@ -474,17 +477,19 @@ def _handle_job_deploy_status(
|
|
|
474
477
|
content=out,
|
|
475
478
|
media_type="text/plain",
|
|
476
479
|
headers={
|
|
477
|
-
"function_status":
|
|
480
|
+
"function_status": function_state,
|
|
478
481
|
"function_image": image,
|
|
479
482
|
"builder_pod": pod,
|
|
480
483
|
},
|
|
481
484
|
)
|
|
482
485
|
|
|
483
486
|
# read from log file
|
|
484
|
-
terminal_states = ["failed", "error", "ready"]
|
|
485
487
|
log_file = log_path(project, f"build_{name}__{tag or 'latest'}")
|
|
486
|
-
if
|
|
487
|
-
|
|
488
|
+
if (
|
|
489
|
+
function_state in mlrun.common.schemas.FunctionState.terminal_states()
|
|
490
|
+
and log_file.exists()
|
|
491
|
+
):
|
|
492
|
+
if function_state == mlrun.common.schemas.FunctionState.ready:
|
|
488
493
|
# when the function has been built we set the created image into the `spec.image` for reference see at the
|
|
489
494
|
# end of the function where we resolve if the status is ready and then set the spec.build.image to
|
|
490
495
|
# spec.image
|
|
@@ -500,40 +505,65 @@ def _handle_job_deploy_status(
|
|
|
500
505
|
content=out,
|
|
501
506
|
media_type="text/plain",
|
|
502
507
|
headers={
|
|
503
|
-
"x-mlrun-function-status":
|
|
504
|
-
"function_status":
|
|
508
|
+
"x-mlrun-function-status": function_state,
|
|
509
|
+
"function_status": function_state,
|
|
505
510
|
"function_image": image,
|
|
506
511
|
"builder_pod": pod,
|
|
507
512
|
},
|
|
508
513
|
)
|
|
509
514
|
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
515
|
+
build_pod_state = mlrun.api.utils.singletons.k8s.get_k8s_helper(
|
|
516
|
+
silent=False
|
|
517
|
+
).get_pod_status(pod)
|
|
518
|
+
logger.debug(
|
|
519
|
+
"Resolved pod status",
|
|
520
|
+
function_name=name,
|
|
521
|
+
pod_status=build_pod_state,
|
|
522
|
+
pod_name=pod,
|
|
513
523
|
)
|
|
514
|
-
logger.info("Resolved pod status", pod_status=state, pod_name=pod)
|
|
515
524
|
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
525
|
+
normalized_pod_function_state = (
|
|
526
|
+
mlrun.common.schemas.FunctionState.get_function_state_from_pod_state(
|
|
527
|
+
build_pod_state
|
|
528
|
+
)
|
|
529
|
+
)
|
|
530
|
+
if normalized_pod_function_state == mlrun.common.schemas.FunctionState.ready:
|
|
531
|
+
logger.info(
|
|
532
|
+
"Build completed successfully",
|
|
533
|
+
function_name=name,
|
|
534
|
+
pod=pod,
|
|
535
|
+
pod_state=build_pod_state,
|
|
536
|
+
)
|
|
537
|
+
elif normalized_pod_function_state == mlrun.common.schemas.FunctionState.error:
|
|
538
|
+
logger.error(
|
|
539
|
+
"Build failed", function_name=name, pod_name=pod, pod_status=build_pod_state
|
|
540
|
+
)
|
|
522
541
|
|
|
523
|
-
if (
|
|
542
|
+
if (
|
|
543
|
+
(
|
|
544
|
+
logs
|
|
545
|
+
and normalized_pod_function_state
|
|
546
|
+
!= mlrun.common.schemas.FunctionState.pending
|
|
547
|
+
)
|
|
548
|
+
or normalized_pod_function_state
|
|
549
|
+
in mlrun.common.schemas.FunctionState.terminal_states()
|
|
550
|
+
):
|
|
524
551
|
try:
|
|
525
552
|
resp = mlrun.api.utils.singletons.k8s.get_k8s_helper(silent=False).logs(pod)
|
|
526
553
|
except ApiException as exc:
|
|
527
554
|
logger.warning(
|
|
528
555
|
"Failed to get build logs",
|
|
529
556
|
function_name=name,
|
|
530
|
-
function_state=
|
|
557
|
+
function_state=normalized_pod_function_state,
|
|
531
558
|
pod=pod,
|
|
532
559
|
exc_info=exc,
|
|
533
560
|
)
|
|
534
561
|
resp = ""
|
|
535
562
|
|
|
536
|
-
if
|
|
563
|
+
if (
|
|
564
|
+
normalized_pod_function_state
|
|
565
|
+
in mlrun.common.schemas.FunctionState.terminal_states()
|
|
566
|
+
):
|
|
537
567
|
# TODO: move to log collector
|
|
538
568
|
log_file.parent.mkdir(parents=True, exist_ok=True)
|
|
539
569
|
with log_file.open("wb") as fp:
|
|
@@ -543,28 +573,31 @@ def _handle_job_deploy_status(
|
|
|
543
573
|
# begin from the offset number and then encode
|
|
544
574
|
out = resp[offset:].encode()
|
|
545
575
|
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
576
|
+
# check if the previous function state is different from the current build pod state, if that is the case then
|
|
577
|
+
# update the function and store to the database
|
|
578
|
+
if function_state != normalized_pod_function_state:
|
|
579
|
+
update_in(fn, "status.state", normalized_pod_function_state)
|
|
549
580
|
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
581
|
+
versioned = False
|
|
582
|
+
if normalized_pod_function_state == mlrun.common.schemas.FunctionState.ready:
|
|
583
|
+
update_in(fn, "spec.image", image)
|
|
584
|
+
versioned = True
|
|
585
|
+
|
|
586
|
+
mlrun.api.crud.Functions().store_function(
|
|
587
|
+
db_session,
|
|
588
|
+
fn,
|
|
589
|
+
name,
|
|
590
|
+
project,
|
|
591
|
+
tag,
|
|
592
|
+
versioned=versioned,
|
|
593
|
+
)
|
|
561
594
|
|
|
562
595
|
return Response(
|
|
563
596
|
content=out,
|
|
564
597
|
media_type="text/plain",
|
|
565
598
|
headers={
|
|
566
|
-
"x-mlrun-function-status":
|
|
567
|
-
"function_status":
|
|
599
|
+
"x-mlrun-function-status": normalized_pod_function_state,
|
|
600
|
+
"function_status": normalized_pod_function_state,
|
|
568
601
|
"function_image": image,
|
|
569
602
|
"builder_pod": pod,
|
|
570
603
|
},
|
|
@@ -673,9 +706,14 @@ def _build_function(
|
|
|
673
706
|
reason=f"runtime error: {err_to_str(err)}",
|
|
674
707
|
)
|
|
675
708
|
try:
|
|
709
|
+
# connect to run db
|
|
676
710
|
run_db = get_run_db_instance(db_session)
|
|
677
711
|
fn.set_db_connection(run_db)
|
|
678
|
-
|
|
712
|
+
|
|
713
|
+
# enrich runtime with project defaults
|
|
714
|
+
launcher = mlrun.api.launcher.ServerSideLauncher()
|
|
715
|
+
launcher.enrich_runtime(runtime=fn)
|
|
716
|
+
|
|
679
717
|
fn.save(versioned=False)
|
|
680
718
|
if fn.kind in RuntimeKinds.nuclio_runtimes():
|
|
681
719
|
mlrun.api.api.utils.apply_enrichment_and_validation_on_function(
|
|
@@ -695,11 +733,13 @@ def _build_function(
|
|
|
695
733
|
model_monitoring_access_key = _process_model_monitoring_secret(
|
|
696
734
|
db_session,
|
|
697
735
|
fn.metadata.project,
|
|
698
|
-
mlrun.common.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
|
|
736
|
+
mlrun.common.schemas.model_monitoring.ProjectSecretKeys.ACCESS_KEY,
|
|
699
737
|
)
|
|
700
738
|
|
|
701
|
-
stream_path =
|
|
702
|
-
|
|
739
|
+
stream_path = (
|
|
740
|
+
mlrun.api.crud.model_monitoring.get_stream_path(
|
|
741
|
+
project=fn.metadata.project
|
|
742
|
+
)
|
|
703
743
|
)
|
|
704
744
|
|
|
705
745
|
if stream_path.startswith("v3io://"):
|
|
@@ -712,19 +752,15 @@ def _build_function(
|
|
|
712
752
|
|
|
713
753
|
if fn.spec.tracking_policy:
|
|
714
754
|
# Convert to `TrackingPolicy` object as `fn.spec.tracking_policy` is provided as a dict
|
|
715
|
-
fn.spec.tracking_policy = (
|
|
716
|
-
|
|
717
|
-
fn.spec.tracking_policy
|
|
718
|
-
)
|
|
755
|
+
fn.spec.tracking_policy = TrackingPolicy.from_dict(
|
|
756
|
+
fn.spec.tracking_policy
|
|
719
757
|
)
|
|
720
758
|
else:
|
|
721
759
|
# Initialize tracking policy with default values
|
|
722
|
-
fn.spec.tracking_policy = (
|
|
723
|
-
mlrun.utils.model_monitoring.TrackingPolicy()
|
|
724
|
-
)
|
|
760
|
+
fn.spec.tracking_policy = TrackingPolicy()
|
|
725
761
|
|
|
726
762
|
# deploy both model monitoring stream and model monitoring batch job
|
|
727
|
-
mlrun.api.crud.
|
|
763
|
+
mlrun.api.crud.model_monitoring.deployment.MonitoringDeployment().deploy_monitoring_functions(
|
|
728
764
|
project=fn.metadata.project,
|
|
729
765
|
db_session=db_session,
|
|
730
766
|
auth_info=auth_info,
|
|
@@ -21,11 +21,10 @@ from fastapi import APIRouter, Depends, Request, Response
|
|
|
21
21
|
from fastapi.concurrency import run_in_threadpool
|
|
22
22
|
from sqlalchemy.orm import Session
|
|
23
23
|
|
|
24
|
-
import mlrun.api.crud
|
|
24
|
+
import mlrun.api.crud.model_monitoring.deployment
|
|
25
25
|
import mlrun.api.crud.model_monitoring.grafana
|
|
26
|
-
import mlrun.api.
|
|
27
|
-
import mlrun.common.model_monitoring
|
|
28
|
-
import mlrun.common.schemas
|
|
26
|
+
import mlrun.api.crud.model_monitoring.helpers
|
|
27
|
+
import mlrun.common.schemas.model_monitoring.grafana
|
|
29
28
|
from mlrun.api.api import deps
|
|
30
29
|
|
|
31
30
|
router = APIRouter(prefix="/grafana-proxy/model-endpoints")
|
|
@@ -53,7 +52,7 @@ def grafana_proxy_model_endpoints_check_connection(
|
|
|
53
52
|
connectivity.
|
|
54
53
|
"""
|
|
55
54
|
if not mlrun.mlconf.is_ce_mode():
|
|
56
|
-
mlrun.api.crud.
|
|
55
|
+
mlrun.api.crud.model_monitoring.helpers.get_access_key(auth_info)
|
|
57
56
|
return Response(status_code=HTTPStatus.OK.value)
|
|
58
57
|
|
|
59
58
|
|
|
@@ -77,7 +76,7 @@ async def grafana_proxy_model_endpoints_search(
|
|
|
77
76
|
:return: List of results. e.g. list of available project names.
|
|
78
77
|
"""
|
|
79
78
|
if not mlrun.mlconf.is_ce_mode():
|
|
80
|
-
mlrun.api.crud.
|
|
79
|
+
mlrun.api.crud.model_monitoring.helpers.get_access_key(auth_info)
|
|
81
80
|
body = await request.json()
|
|
82
81
|
query_parameters = mlrun.api.crud.model_monitoring.grafana.parse_search_parameters(
|
|
83
82
|
body
|
|
@@ -104,8 +103,8 @@ async def grafana_proxy_model_endpoints_search(
|
|
|
104
103
|
"/query",
|
|
105
104
|
response_model=List[
|
|
106
105
|
Union[
|
|
107
|
-
mlrun.common.schemas.GrafanaTable,
|
|
108
|
-
mlrun.common.schemas.GrafanaTimeSeriesTarget,
|
|
106
|
+
mlrun.common.schemas.model_monitoring.grafana.GrafanaTable,
|
|
107
|
+
mlrun.common.schemas.model_monitoring.grafana.GrafanaTimeSeriesTarget,
|
|
109
108
|
]
|
|
110
109
|
],
|
|
111
110
|
)
|
|
@@ -114,7 +113,8 @@ async def grafana_proxy_model_endpoints_query(
|
|
|
114
113
|
auth_info: mlrun.common.schemas.AuthInfo = Depends(deps.authenticate_request),
|
|
115
114
|
) -> List[
|
|
116
115
|
Union[
|
|
117
|
-
mlrun.common.schemas.GrafanaTable,
|
|
116
|
+
mlrun.common.schemas.model_monitoring.grafana.GrafanaTable,
|
|
117
|
+
mlrun.common.schemas.model_monitoring.grafana.GrafanaTimeSeriesTarget,
|
|
118
118
|
]
|
|
119
119
|
]:
|
|
120
120
|
"""
|
mlrun/api/api/endpoints/logs.py
CHANGED
|
@@ -21,10 +21,17 @@ import mlrun.api.crud
|
|
|
21
21
|
import mlrun.api.utils.auth.verifier
|
|
22
22
|
import mlrun.common.schemas
|
|
23
23
|
|
|
24
|
-
router = fastapi.APIRouter(
|
|
24
|
+
router = fastapi.APIRouter()
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
# TODO: remove /log/{project}/{uid} in 1.7.0
|
|
28
|
+
@router.post(
|
|
29
|
+
"/log/{project}/{uid}",
|
|
30
|
+
deprecated=True,
|
|
31
|
+
description="/log/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
|
|
32
|
+
"use /projects/{project}/logs/{uid} instead",
|
|
33
|
+
)
|
|
34
|
+
@router.post("/projects/{project}/logs/{uid}")
|
|
28
35
|
async def store_log(
|
|
29
36
|
request: fastapi.Request,
|
|
30
37
|
project: str,
|
|
@@ -52,7 +59,14 @@ async def store_log(
|
|
|
52
59
|
return {}
|
|
53
60
|
|
|
54
61
|
|
|
55
|
-
|
|
62
|
+
# TODO: remove /log/{project}/{uid} in 1.7.0
|
|
63
|
+
@router.get(
|
|
64
|
+
"/log/{project}/{uid}",
|
|
65
|
+
deprecated=True,
|
|
66
|
+
description="/log/{project}/{uid} is deprecated in 1.5.0 and will be removed in 1.7.0, "
|
|
67
|
+
"use /projects/{project}/logs/{uid} instead",
|
|
68
|
+
)
|
|
69
|
+
@router.get("/projects/{project}/logs/{uid}")
|
|
56
70
|
async def get_log(
|
|
57
71
|
project: str,
|
|
58
72
|
uid: str,
|
|
@@ -23,7 +23,8 @@ from fastapi.concurrency import run_in_threadpool
|
|
|
23
23
|
from sqlalchemy.orm import Session
|
|
24
24
|
|
|
25
25
|
import mlrun.api.api.deps
|
|
26
|
-
import mlrun.api.crud
|
|
26
|
+
import mlrun.api.crud.model_monitoring.deployment
|
|
27
|
+
import mlrun.api.crud.model_monitoring.helpers
|
|
27
28
|
import mlrun.api.utils.auth.verifier
|
|
28
29
|
import mlrun.common.schemas
|
|
29
30
|
from mlrun.errors import MLRunConflictError
|
|
@@ -64,7 +65,7 @@ async def create_or_patch(
|
|
|
64
65
|
auth_info,
|
|
65
66
|
)
|
|
66
67
|
# get_access_key will validate the needed auth (which is used later) exists in the request
|
|
67
|
-
mlrun.api.crud.
|
|
68
|
+
mlrun.api.crud.model_monitoring.helpers.get_access_key(auth_info)
|
|
68
69
|
if project != model_endpoint.metadata.project:
|
|
69
70
|
raise MLRunConflictError(
|
|
70
71
|
f"Can't store endpoint of project {model_endpoint.metadata.project} into project {project}"
|