mlrun 1.7.0rc13__py3-none-any.whl → 1.7.0rc15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +0 -105
- mlrun/artifacts/__init__.py +1 -2
- mlrun/artifacts/base.py +8 -250
- mlrun/artifacts/dataset.py +1 -190
- mlrun/artifacts/manager.py +2 -41
- mlrun/artifacts/model.py +1 -140
- mlrun/artifacts/plots.py +1 -375
- mlrun/common/schemas/model_monitoring/__init__.py +4 -0
- mlrun/common/schemas/model_monitoring/constants.py +24 -3
- mlrun/common/schemas/model_monitoring/model_endpoints.py +13 -1
- mlrun/common/schemas/project.py +1 -0
- mlrun/config.py +14 -4
- mlrun/data_types/to_pandas.py +4 -4
- mlrun/datastore/base.py +41 -9
- mlrun/datastore/datastore_profile.py +50 -3
- mlrun/datastore/hdfs.py +5 -0
- mlrun/datastore/inmem.py +2 -2
- mlrun/datastore/sources.py +43 -2
- mlrun/datastore/store_resources.py +2 -6
- mlrun/datastore/targets.py +125 -6
- mlrun/datastore/v3io.py +1 -1
- mlrun/db/auth_utils.py +152 -0
- mlrun/db/base.py +1 -1
- mlrun/db/httpdb.py +69 -33
- mlrun/feature_store/__init__.py +0 -2
- mlrun/feature_store/api.py +12 -47
- mlrun/feature_store/feature_set.py +9 -0
- mlrun/feature_store/retrieval/base.py +9 -4
- mlrun/feature_store/retrieval/conversion.py +4 -4
- mlrun/feature_store/retrieval/dask_merger.py +2 -0
- mlrun/feature_store/retrieval/job.py +2 -0
- mlrun/feature_store/retrieval/local_merger.py +2 -0
- mlrun/feature_store/retrieval/spark_merger.py +5 -0
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +5 -10
- mlrun/kfpops.py +5 -10
- mlrun/launcher/base.py +1 -1
- mlrun/launcher/client.py +1 -1
- mlrun/lists.py +2 -2
- mlrun/model.py +36 -9
- mlrun/model_monitoring/api.py +41 -18
- mlrun/model_monitoring/application.py +5 -305
- mlrun/model_monitoring/applications/__init__.py +11 -0
- mlrun/model_monitoring/applications/_application_steps.py +158 -0
- mlrun/model_monitoring/applications/base.py +282 -0
- mlrun/model_monitoring/applications/context.py +214 -0
- mlrun/model_monitoring/applications/evidently_base.py +211 -0
- mlrun/model_monitoring/applications/histogram_data_drift.py +92 -77
- mlrun/model_monitoring/applications/results.py +99 -0
- mlrun/model_monitoring/controller.py +3 -1
- mlrun/model_monitoring/db/stores/sqldb/models/base.py +7 -6
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +1 -1
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +67 -4
- mlrun/model_monitoring/evidently_application.py +6 -118
- mlrun/model_monitoring/helpers.py +1 -1
- mlrun/model_monitoring/model_endpoint.py +3 -2
- mlrun/model_monitoring/stream_processing.py +2 -3
- mlrun/model_monitoring/writer.py +69 -39
- mlrun/platforms/iguazio.py +2 -2
- mlrun/projects/pipelines.py +24 -7
- mlrun/projects/project.py +130 -65
- mlrun/render.py +2 -10
- mlrun/run.py +1 -4
- mlrun/runtimes/__init__.py +3 -3
- mlrun/runtimes/base.py +3 -3
- mlrun/runtimes/funcdoc.py +0 -28
- mlrun/runtimes/local.py +1 -1
- mlrun/runtimes/mpijob/__init__.py +0 -20
- mlrun/runtimes/mpijob/v1.py +1 -1
- mlrun/runtimes/nuclio/api_gateway.py +275 -153
- mlrun/runtimes/nuclio/function.py +1 -1
- mlrun/runtimes/pod.py +5 -5
- mlrun/runtimes/utils.py +1 -1
- mlrun/serving/states.py +53 -2
- mlrun/utils/helpers.py +27 -40
- mlrun/utils/notifications/notification/slack.py +31 -8
- mlrun/utils/notifications/notification_pusher.py +133 -14
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/METADATA +2 -2
- {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/RECORD +84 -79
- mlrun/runtimes/mpijob/v1alpha1.py +0 -29
- /mlrun/{runtimes → common/runtimes}/constants.py +0 -0
- {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/WHEEL +0 -0
- {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc13.dist-info → mlrun-1.7.0rc15.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py
CHANGED
|
@@ -41,6 +41,7 @@ import mlrun.common.schemas.model_monitoring.constants as mm_constants
|
|
|
41
41
|
import mlrun.db
|
|
42
42
|
import mlrun.errors
|
|
43
43
|
import mlrun.k8s_utils
|
|
44
|
+
import mlrun.model_monitoring.applications as mm_app
|
|
44
45
|
import mlrun.runtimes
|
|
45
46
|
import mlrun.runtimes.nuclio.api_gateway
|
|
46
47
|
import mlrun.runtimes.pod
|
|
@@ -56,14 +57,10 @@ from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifa
|
|
|
56
57
|
from ..datastore import store_manager
|
|
57
58
|
from ..features import Feature
|
|
58
59
|
from ..model import EntrypointParam, ImageBuilder, ModelObj
|
|
59
|
-
from ..model_monitoring.application import (
|
|
60
|
-
ModelMonitoringApplicationBase,
|
|
61
|
-
)
|
|
62
60
|
from ..run import code_to_function, get_object, import_function, new_function
|
|
63
61
|
from ..secrets import SecretsStore
|
|
64
62
|
from ..utils import (
|
|
65
63
|
is_ipython,
|
|
66
|
-
is_legacy_artifact,
|
|
67
64
|
is_relative_path,
|
|
68
65
|
is_yaml_path,
|
|
69
66
|
logger,
|
|
@@ -207,14 +204,16 @@ def new_project(
|
|
|
207
204
|
"Unsupported option, cannot use subpath argument with project templates"
|
|
208
205
|
)
|
|
209
206
|
if from_template.endswith(".yaml"):
|
|
210
|
-
project = _load_project_file(
|
|
207
|
+
project = _load_project_file(
|
|
208
|
+
from_template, name, secrets, allow_cross_project=True
|
|
209
|
+
)
|
|
211
210
|
elif from_template.startswith("git://"):
|
|
212
211
|
clone_git(from_template, context, secrets, clone=True)
|
|
213
212
|
shutil.rmtree(path.join(context, ".git"))
|
|
214
|
-
project = _load_project_dir(context, name)
|
|
213
|
+
project = _load_project_dir(context, name, allow_cross_project=True)
|
|
215
214
|
elif from_template.endswith(".zip"):
|
|
216
215
|
clone_zip(from_template, context, secrets)
|
|
217
|
-
project = _load_project_dir(context, name)
|
|
216
|
+
project = _load_project_dir(context, name, allow_cross_project=True)
|
|
218
217
|
else:
|
|
219
218
|
raise ValueError("template must be a path to .yaml or .zip file")
|
|
220
219
|
project.metadata.name = name
|
|
@@ -296,6 +295,7 @@ def load_project(
|
|
|
296
295
|
save: bool = True,
|
|
297
296
|
sync_functions: bool = False,
|
|
298
297
|
parameters: dict = None,
|
|
298
|
+
allow_cross_project: bool = None,
|
|
299
299
|
) -> "MlrunProject":
|
|
300
300
|
"""Load an MLRun project from git or tar or dir
|
|
301
301
|
|
|
@@ -342,6 +342,8 @@ def load_project(
|
|
|
342
342
|
:param save: whether to save the created project and artifact in the DB
|
|
343
343
|
:param sync_functions: sync the project's functions into the project object (will be saved to the DB if save=True)
|
|
344
344
|
:param parameters: key/value pairs to add to the project.spec.params
|
|
345
|
+
:param allow_cross_project: if True, override the loaded project name. This flag ensures awareness of
|
|
346
|
+
loading an existing project yaml as a baseline for a new project with a different name
|
|
345
347
|
|
|
346
348
|
:returns: project object
|
|
347
349
|
"""
|
|
@@ -357,7 +359,7 @@ def load_project(
|
|
|
357
359
|
if url:
|
|
358
360
|
url = str(url) # to support path objects
|
|
359
361
|
if is_yaml_path(url):
|
|
360
|
-
project = _load_project_file(url, name, secrets)
|
|
362
|
+
project = _load_project_file(url, name, secrets, allow_cross_project)
|
|
361
363
|
project.spec.context = context
|
|
362
364
|
elif url.startswith("git://"):
|
|
363
365
|
url, repo = clone_git(url, context, secrets, clone)
|
|
@@ -384,7 +386,7 @@ def load_project(
|
|
|
384
386
|
repo, url = init_repo(context, url, init_git)
|
|
385
387
|
|
|
386
388
|
if not project:
|
|
387
|
-
project = _load_project_dir(context, name, subpath)
|
|
389
|
+
project = _load_project_dir(context, name, subpath, allow_cross_project)
|
|
388
390
|
|
|
389
391
|
if not project.metadata.name:
|
|
390
392
|
raise ValueError("Project name must be specified")
|
|
@@ -438,6 +440,7 @@ def get_or_create_project(
|
|
|
438
440
|
from_template: str = None,
|
|
439
441
|
save: bool = True,
|
|
440
442
|
parameters: dict = None,
|
|
443
|
+
allow_cross_project: bool = None,
|
|
441
444
|
) -> "MlrunProject":
|
|
442
445
|
"""Load a project from MLRun DB, or create/import if it does not exist
|
|
443
446
|
|
|
@@ -482,12 +485,12 @@ def get_or_create_project(
|
|
|
482
485
|
:param from_template: path to project YAML file that will be used as from_template (for new projects)
|
|
483
486
|
:param save: whether to save the created project in the DB
|
|
484
487
|
:param parameters: key/value pairs to add to the project.spec.params
|
|
488
|
+
:param allow_cross_project: if True, override the loaded project name. This flag ensures awareness of
|
|
489
|
+
loading an existing project yaml as a baseline for a new project with a different name
|
|
485
490
|
|
|
486
491
|
:returns: project object
|
|
487
492
|
"""
|
|
488
493
|
context = context or "./"
|
|
489
|
-
spec_path = path.join(context, subpath or "", "project.yaml")
|
|
490
|
-
load_from_path = url or path.isfile(spec_path)
|
|
491
494
|
try:
|
|
492
495
|
# load project from the DB.
|
|
493
496
|
# use `name` as `url` as we load the project from the DB
|
|
@@ -503,13 +506,15 @@ def get_or_create_project(
|
|
|
503
506
|
# only loading project from db so no need to save it
|
|
504
507
|
save=False,
|
|
505
508
|
parameters=parameters,
|
|
509
|
+
allow_cross_project=allow_cross_project,
|
|
506
510
|
)
|
|
507
511
|
logger.info("Project loaded successfully", project_name=name)
|
|
508
512
|
return project
|
|
509
|
-
|
|
510
513
|
except mlrun.errors.MLRunNotFoundError:
|
|
511
514
|
logger.debug("Project not found in db", project_name=name)
|
|
512
515
|
|
|
516
|
+
spec_path = path.join(context, subpath or "", "project.yaml")
|
|
517
|
+
load_from_path = url or path.isfile(spec_path)
|
|
513
518
|
# do not nest under "try" or else the exceptions raised below will be logged along with the "not found" message
|
|
514
519
|
if load_from_path:
|
|
515
520
|
# loads a project from archive or local project.yaml
|
|
@@ -525,6 +530,7 @@ def get_or_create_project(
|
|
|
525
530
|
user_project=user_project,
|
|
526
531
|
save=save,
|
|
527
532
|
parameters=parameters,
|
|
533
|
+
allow_cross_project=allow_cross_project,
|
|
528
534
|
)
|
|
529
535
|
|
|
530
536
|
logger.info(
|
|
@@ -599,7 +605,7 @@ def _run_project_setup(
|
|
|
599
605
|
return project
|
|
600
606
|
|
|
601
607
|
|
|
602
|
-
def _load_project_dir(context, name="", subpath=""):
|
|
608
|
+
def _load_project_dir(context, name="", subpath="", allow_cross_project=None):
|
|
603
609
|
subpath_str = subpath or ""
|
|
604
610
|
|
|
605
611
|
# support both .yaml and .yml file extensions
|
|
@@ -613,7 +619,7 @@ def _load_project_dir(context, name="", subpath=""):
|
|
|
613
619
|
with open(project_file_path) as fp:
|
|
614
620
|
data = fp.read()
|
|
615
621
|
struct = yaml.load(data, Loader=yaml.FullLoader)
|
|
616
|
-
project = _project_instance_from_struct(struct, name)
|
|
622
|
+
project = _project_instance_from_struct(struct, name, allow_cross_project)
|
|
617
623
|
project.spec.context = context
|
|
618
624
|
elif function_files := glob.glob(function_file_path):
|
|
619
625
|
function_path = function_files[0]
|
|
@@ -686,19 +692,41 @@ def _delete_project_from_db(project_name, secrets, deletion_strategy):
|
|
|
686
692
|
return db.delete_project(project_name, deletion_strategy=deletion_strategy)
|
|
687
693
|
|
|
688
694
|
|
|
689
|
-
def _load_project_file(url, name="", secrets=None):
|
|
695
|
+
def _load_project_file(url, name="", secrets=None, allow_cross_project=None):
|
|
690
696
|
try:
|
|
691
697
|
obj = get_object(url, secrets)
|
|
692
698
|
except FileNotFoundError as exc:
|
|
693
699
|
raise FileNotFoundError(f"cant find project file at {url}") from exc
|
|
694
700
|
struct = yaml.load(obj, Loader=yaml.FullLoader)
|
|
695
|
-
return _project_instance_from_struct(struct, name)
|
|
701
|
+
return _project_instance_from_struct(struct, name, allow_cross_project)
|
|
696
702
|
|
|
697
703
|
|
|
698
|
-
def _project_instance_from_struct(struct, name):
|
|
699
|
-
struct.
|
|
700
|
-
|
|
701
|
-
|
|
704
|
+
def _project_instance_from_struct(struct, name, allow_cross_project):
|
|
705
|
+
name_from_struct = struct.get("metadata", {}).get("name", "")
|
|
706
|
+
if name and name_from_struct and name_from_struct != name:
|
|
707
|
+
error_message = (
|
|
708
|
+
f"project name mismatch, {name_from_struct} != {name}, please do one of the following:\n"
|
|
709
|
+
"1. Set the `allow_cross_project=True` when loading the project.\n"
|
|
710
|
+
f"2. Delete the existing project yaml, or ensure its name is equal to {name}.\n"
|
|
711
|
+
"3. Use different project context dir."
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
if allow_cross_project is None:
|
|
715
|
+
# TODO: Remove this warning in version 1.9.0 and also fix cli to support allow_cross_project
|
|
716
|
+
logger.warn(
|
|
717
|
+
"Project name is different than specified on its project yaml."
|
|
718
|
+
"You should fix it until version 1.9.0",
|
|
719
|
+
description=error_message,
|
|
720
|
+
)
|
|
721
|
+
elif allow_cross_project:
|
|
722
|
+
logger.warn(
|
|
723
|
+
"Project name is different than specified on its project yaml. Overriding.",
|
|
724
|
+
existing_name=name_from_struct,
|
|
725
|
+
overriding_name=name,
|
|
726
|
+
)
|
|
727
|
+
else:
|
|
728
|
+
raise ValueError(error_message)
|
|
729
|
+
struct.setdefault("metadata", {})["name"] = name or name_from_struct
|
|
702
730
|
return MlrunProject.from_dict(struct)
|
|
703
731
|
|
|
704
732
|
|
|
@@ -960,13 +988,9 @@ class ProjectSpec(ModelObj):
|
|
|
960
988
|
if not isinstance(artifact, dict) and not hasattr(artifact, "to_dict"):
|
|
961
989
|
raise ValueError("artifacts must be a dict or class")
|
|
962
990
|
if isinstance(artifact, dict):
|
|
963
|
-
|
|
964
|
-
if is_legacy_artifact(artifact) or _is_imported_artifact(artifact):
|
|
965
|
-
key = artifact.get("key")
|
|
966
|
-
else:
|
|
967
|
-
key = artifact.get("metadata").get("key", "")
|
|
991
|
+
key = artifact.get("metadata", {}).get("key", "")
|
|
968
992
|
if not key:
|
|
969
|
-
raise ValueError('artifacts "key" must be specified')
|
|
993
|
+
raise ValueError('artifacts "metadata.key" must be specified')
|
|
970
994
|
else:
|
|
971
995
|
key = artifact.key
|
|
972
996
|
artifact = artifact.to_dict()
|
|
@@ -1814,10 +1838,18 @@ class MlrunProject(ModelObj):
|
|
|
1814
1838
|
"""
|
|
1815
1839
|
context = context or self.spec.context
|
|
1816
1840
|
if context:
|
|
1817
|
-
project = _load_project_dir(
|
|
1841
|
+
project = _load_project_dir(
|
|
1842
|
+
context,
|
|
1843
|
+
self.metadata.name,
|
|
1844
|
+
self.spec.subpath,
|
|
1845
|
+
allow_cross_project=False,
|
|
1846
|
+
)
|
|
1818
1847
|
else:
|
|
1819
1848
|
project = _load_project_file(
|
|
1820
|
-
self.spec.origin_url,
|
|
1849
|
+
self.spec.origin_url,
|
|
1850
|
+
self.metadata.name,
|
|
1851
|
+
self._secrets,
|
|
1852
|
+
allow_cross_project=None,
|
|
1821
1853
|
)
|
|
1822
1854
|
project.spec.source = self.spec.source
|
|
1823
1855
|
project.spec.repo = self.spec.repo
|
|
@@ -1846,7 +1878,11 @@ class MlrunProject(ModelObj):
|
|
|
1846
1878
|
def set_model_monitoring_function(
|
|
1847
1879
|
self,
|
|
1848
1880
|
func: typing.Union[str, mlrun.runtimes.BaseRuntime, None] = None,
|
|
1849
|
-
application_class: typing.Union[
|
|
1881
|
+
application_class: typing.Union[
|
|
1882
|
+
str,
|
|
1883
|
+
mm_app.ModelMonitoringApplicationBase,
|
|
1884
|
+
mm_app.ModelMonitoringApplicationBaseV2,
|
|
1885
|
+
] = None,
|
|
1850
1886
|
name: str = None,
|
|
1851
1887
|
image: str = None,
|
|
1852
1888
|
handler=None,
|
|
@@ -1884,11 +1920,6 @@ class MlrunProject(ModelObj):
|
|
|
1884
1920
|
monitoring application's constructor.
|
|
1885
1921
|
"""
|
|
1886
1922
|
|
|
1887
|
-
if name in mm_constants.MonitoringFunctionNames.list():
|
|
1888
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
1889
|
-
f"An application cannot have the following names: "
|
|
1890
|
-
f"{mm_constants.MonitoringFunctionNames.list()}"
|
|
1891
|
-
)
|
|
1892
1923
|
function_object: RemoteRuntime = None
|
|
1893
1924
|
(
|
|
1894
1925
|
resolved_function_name,
|
|
@@ -1914,7 +1945,11 @@ class MlrunProject(ModelObj):
|
|
|
1914
1945
|
def create_model_monitoring_function(
|
|
1915
1946
|
self,
|
|
1916
1947
|
func: str = None,
|
|
1917
|
-
application_class: typing.Union[
|
|
1948
|
+
application_class: typing.Union[
|
|
1949
|
+
str,
|
|
1950
|
+
mm_app.ModelMonitoringApplicationBase,
|
|
1951
|
+
mm_app.ModelMonitoringApplicationBaseV2,
|
|
1952
|
+
] = None,
|
|
1918
1953
|
name: str = None,
|
|
1919
1954
|
image: str = None,
|
|
1920
1955
|
handler: str = None,
|
|
@@ -1967,7 +2002,10 @@ class MlrunProject(ModelObj):
|
|
|
1967
2002
|
self,
|
|
1968
2003
|
func: typing.Union[str, mlrun.runtimes.BaseRuntime, None] = None,
|
|
1969
2004
|
application_class: typing.Union[
|
|
1970
|
-
str,
|
|
2005
|
+
str,
|
|
2006
|
+
mm_app.ModelMonitoringApplicationBase,
|
|
2007
|
+
mm_app.ModelMonitoringApplicationBaseV2,
|
|
2008
|
+
None,
|
|
1971
2009
|
] = None,
|
|
1972
2010
|
name: typing.Optional[str] = None,
|
|
1973
2011
|
image: typing.Optional[str] = None,
|
|
@@ -2024,12 +2062,24 @@ class MlrunProject(ModelObj):
|
|
|
2024
2062
|
|
|
2025
2063
|
return resolved_function_name, function_object, func
|
|
2026
2064
|
|
|
2065
|
+
def _wait_for_functions_deployment(self, function_names: list[str]) -> None:
|
|
2066
|
+
"""
|
|
2067
|
+
Wait for the deployment of functions on the backend.
|
|
2068
|
+
|
|
2069
|
+
:param function_names: A list of function names.
|
|
2070
|
+
"""
|
|
2071
|
+
for fn_name in function_names:
|
|
2072
|
+
fn = typing.cast(RemoteRuntime, self.get_function(key=fn_name))
|
|
2073
|
+
fn._wait_for_function_deployment(db=fn._get_db())
|
|
2074
|
+
|
|
2027
2075
|
def enable_model_monitoring(
|
|
2028
2076
|
self,
|
|
2029
2077
|
default_controller_image: str = "mlrun/mlrun",
|
|
2030
2078
|
base_period: int = 10,
|
|
2031
2079
|
image: str = "mlrun/mlrun",
|
|
2080
|
+
*,
|
|
2032
2081
|
deploy_histogram_data_drift_app: bool = True,
|
|
2082
|
+
wait_for_deployment: bool = False,
|
|
2033
2083
|
) -> None:
|
|
2034
2084
|
"""
|
|
2035
2085
|
Deploy model monitoring application controller, writer and stream functions.
|
|
@@ -2039,7 +2089,6 @@ class MlrunProject(ModelObj):
|
|
|
2039
2089
|
The stream function goal is to monitor the log of the data stream. It is triggered when a new log entry
|
|
2040
2090
|
is detected. It processes the new events into statistics that are then written to statistics databases.
|
|
2041
2091
|
|
|
2042
|
-
|
|
2043
2092
|
:param default_controller_image: Deprecated.
|
|
2044
2093
|
:param base_period: The time period in minutes in which the model monitoring controller
|
|
2045
2094
|
function is triggered. By default, the base period is 10 minutes.
|
|
@@ -2047,6 +2096,9 @@ class MlrunProject(ModelObj):
|
|
|
2047
2096
|
stream & histogram data drift functions, which are real time nuclio
|
|
2048
2097
|
functions. By default, the image is mlrun/mlrun.
|
|
2049
2098
|
:param deploy_histogram_data_drift_app: If true, deploy the default histogram-based data drift application.
|
|
2099
|
+
:param wait_for_deployment: If true, return only after the deployment is done on the backend.
|
|
2100
|
+
Otherwise, deploy the model monitoring infrastructure on the
|
|
2101
|
+
background, including the histogram data drift app if selected.
|
|
2050
2102
|
"""
|
|
2051
2103
|
if default_controller_image != "mlrun/mlrun":
|
|
2052
2104
|
# TODO: Remove this in 1.9.0
|
|
@@ -2064,37 +2116,55 @@ class MlrunProject(ModelObj):
|
|
|
2064
2116
|
deploy_histogram_data_drift_app=deploy_histogram_data_drift_app,
|
|
2065
2117
|
)
|
|
2066
2118
|
|
|
2119
|
+
if wait_for_deployment:
|
|
2120
|
+
deployment_functions = mm_constants.MonitoringFunctionNames.list()
|
|
2121
|
+
if deploy_histogram_data_drift_app:
|
|
2122
|
+
deployment_functions.append(
|
|
2123
|
+
mm_constants.HistogramDataDriftApplicationConstants.NAME
|
|
2124
|
+
)
|
|
2125
|
+
self._wait_for_functions_deployment(deployment_functions)
|
|
2126
|
+
|
|
2067
2127
|
def deploy_histogram_data_drift_app(
|
|
2068
2128
|
self,
|
|
2069
2129
|
*,
|
|
2070
2130
|
image: str = "mlrun/mlrun",
|
|
2071
2131
|
db: Optional[mlrun.db.RunDBInterface] = None,
|
|
2132
|
+
wait_for_deployment: bool = False,
|
|
2072
2133
|
) -> None:
|
|
2073
2134
|
"""
|
|
2074
2135
|
Deploy the histogram data drift application.
|
|
2075
2136
|
|
|
2076
|
-
:param image:
|
|
2077
|
-
:param db:
|
|
2137
|
+
:param image: The image on which the application will run.
|
|
2138
|
+
:param db: An optional DB object.
|
|
2139
|
+
:param wait_for_deployment: If true, return only after the deployment is done on the backend.
|
|
2140
|
+
Otherwise, deploy the application on the background.
|
|
2078
2141
|
"""
|
|
2079
2142
|
if db is None:
|
|
2080
2143
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
2081
2144
|
db.deploy_histogram_data_drift_app(project=self.name, image=image)
|
|
2082
2145
|
|
|
2146
|
+
if wait_for_deployment:
|
|
2147
|
+
self._wait_for_functions_deployment(
|
|
2148
|
+
[mm_constants.HistogramDataDriftApplicationConstants.NAME]
|
|
2149
|
+
)
|
|
2150
|
+
|
|
2083
2151
|
def update_model_monitoring_controller(
|
|
2084
2152
|
self,
|
|
2085
2153
|
base_period: int = 10,
|
|
2086
2154
|
image: str = "mlrun/mlrun",
|
|
2155
|
+
*,
|
|
2156
|
+
wait_for_deployment: bool = False,
|
|
2087
2157
|
) -> None:
|
|
2088
2158
|
"""
|
|
2089
2159
|
Redeploy model monitoring application controller functions.
|
|
2090
2160
|
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2161
|
+
:param base_period: The time period in minutes in which the model monitoring controller function
|
|
2162
|
+
is triggered. By default, the base period is 10 minutes.
|
|
2163
|
+
:param image: The image of the model monitoring controller, writer & monitoring
|
|
2164
|
+
stream functions, which are real time nuclio functions.
|
|
2165
|
+
By default, the image is mlrun/mlrun.
|
|
2166
|
+
:param wait_for_deployment: If true, return only after the deployment is done on the backend.
|
|
2167
|
+
Otherwise, deploy the controller on the background.
|
|
2098
2168
|
"""
|
|
2099
2169
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
2100
2170
|
db.update_model_monitoring_controller(
|
|
@@ -2103,6 +2173,11 @@ class MlrunProject(ModelObj):
|
|
|
2103
2173
|
image=image,
|
|
2104
2174
|
)
|
|
2105
2175
|
|
|
2176
|
+
if wait_for_deployment:
|
|
2177
|
+
self._wait_for_functions_deployment(
|
|
2178
|
+
[mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER]
|
|
2179
|
+
)
|
|
2180
|
+
|
|
2106
2181
|
def disable_model_monitoring(
|
|
2107
2182
|
self, *, delete_histogram_data_drift_app: bool = True
|
|
2108
2183
|
) -> None:
|
|
@@ -2824,12 +2899,14 @@ class MlrunProject(ModelObj):
|
|
|
2824
2899
|
"Remote repo is not defined, use .create_remote() + push()"
|
|
2825
2900
|
)
|
|
2826
2901
|
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2902
|
+
if engine not in ["remote"]:
|
|
2903
|
+
# for remote runs we don't require the functions to be synced as they can be loaded dynamically during run
|
|
2904
|
+
self.sync_functions(always=sync)
|
|
2905
|
+
if not self.spec._function_objects:
|
|
2906
|
+
raise ValueError(
|
|
2907
|
+
"There are no functions in the project."
|
|
2908
|
+
" Make sure you've set your functions with project.set_function()."
|
|
2909
|
+
)
|
|
2833
2910
|
|
|
2834
2911
|
if not name and not workflow_path and not workflow_handler:
|
|
2835
2912
|
raise ValueError("Workflow name, path, or handler must be specified")
|
|
@@ -3266,7 +3343,6 @@ class MlrunProject(ModelObj):
|
|
|
3266
3343
|
image: str = None,
|
|
3267
3344
|
set_as_default: bool = True,
|
|
3268
3345
|
with_mlrun: bool = None,
|
|
3269
|
-
skip_deployed: bool = False,
|
|
3270
3346
|
base_image: str = None,
|
|
3271
3347
|
commands: list = None,
|
|
3272
3348
|
secret_name: str = None,
|
|
@@ -3287,7 +3363,6 @@ class MlrunProject(ModelObj):
|
|
|
3287
3363
|
used. If not set, the `mlconf.default_project_image_name` value will be used
|
|
3288
3364
|
:param set_as_default: set `image` to be the project's default image (default False)
|
|
3289
3365
|
:param with_mlrun: add the current mlrun package to the container build
|
|
3290
|
-
:param skip_deployed: *Deprecated* parameter is ignored
|
|
3291
3366
|
:param base_image: base image name/path (commands and source code will be added to it) defaults to
|
|
3292
3367
|
mlrun.mlconf.default_base_image
|
|
3293
3368
|
:param commands: list of docker build (RUN) commands e.g. ['pip install pandas']
|
|
@@ -3312,14 +3387,6 @@ class MlrunProject(ModelObj):
|
|
|
3312
3387
|
base_image=base_image,
|
|
3313
3388
|
)
|
|
3314
3389
|
|
|
3315
|
-
if skip_deployed:
|
|
3316
|
-
warnings.warn(
|
|
3317
|
-
"The 'skip_deployed' parameter is deprecated and will be removed in 1.7.0. "
|
|
3318
|
-
"This parameter is ignored.",
|
|
3319
|
-
# TODO: remove in 1.7.0
|
|
3320
|
-
FutureWarning,
|
|
3321
|
-
)
|
|
3322
|
-
|
|
3323
3390
|
if not overwrite_build_params:
|
|
3324
3391
|
# TODO: change overwrite_build_params default to True in 1.8.0
|
|
3325
3392
|
warnings.warn(
|
|
@@ -3582,9 +3649,7 @@ class MlrunProject(ModelObj):
|
|
|
3582
3649
|
:returns: List of function objects.
|
|
3583
3650
|
"""
|
|
3584
3651
|
|
|
3585
|
-
model_monitoring_labels_list = [
|
|
3586
|
-
f"{mm_constants.ModelMonitoringAppLabel.KEY}={mm_constants.ModelMonitoringAppLabel.VAL}"
|
|
3587
|
-
]
|
|
3652
|
+
model_monitoring_labels_list = [str(mm_constants.ModelMonitoringAppLabel())]
|
|
3588
3653
|
if labels:
|
|
3589
3654
|
model_monitoring_labels_list += labels
|
|
3590
3655
|
return self.list_functions(
|
mlrun/render.py
CHANGED
|
@@ -121,16 +121,8 @@ def artifacts_html(
|
|
|
121
121
|
html = ""
|
|
122
122
|
|
|
123
123
|
for artifact in artifacts:
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
attribute_value = artifact.get(attribute_name)
|
|
127
|
-
else:
|
|
128
|
-
attribute_value = artifact["spec"].get(attribute_name)
|
|
129
|
-
|
|
130
|
-
if mlrun.utils.is_legacy_artifact(artifact):
|
|
131
|
-
key = artifact["key"]
|
|
132
|
-
else:
|
|
133
|
-
key = artifact["metadata"]["key"]
|
|
124
|
+
attribute_value = artifact["spec"].get(attribute_name)
|
|
125
|
+
key = artifact["metadata"]["key"]
|
|
134
126
|
|
|
135
127
|
if not attribute_value:
|
|
136
128
|
mlrun.utils.logger.warning(
|
mlrun/run.py
CHANGED
|
@@ -47,7 +47,6 @@ from .runtimes import (
|
|
|
47
47
|
KubejobRuntime,
|
|
48
48
|
LocalRuntime,
|
|
49
49
|
MpiRuntimeV1,
|
|
50
|
-
MpiRuntimeV1Alpha1,
|
|
51
50
|
RemoteRuntime,
|
|
52
51
|
RemoteSparkRuntime,
|
|
53
52
|
RuntimeKinds,
|
|
@@ -606,7 +605,6 @@ def code_to_function(
|
|
|
606
605
|
ignored_tags: Optional[str] = None,
|
|
607
606
|
requirements_file: Optional[str] = "",
|
|
608
607
|
) -> Union[
|
|
609
|
-
MpiRuntimeV1Alpha1,
|
|
610
608
|
MpiRuntimeV1,
|
|
611
609
|
RemoteRuntime,
|
|
612
610
|
ServingRuntime,
|
|
@@ -661,7 +659,6 @@ def code_to_function(
|
|
|
661
659
|
:param embed_code: indicates whether or not to inject the code directly into the function runtime spec,
|
|
662
660
|
defaults to True
|
|
663
661
|
:param description: short function description, defaults to ''
|
|
664
|
-
:param requirements: list of python packages or pip requirements file path, defaults to None
|
|
665
662
|
:param requirements: a list of python packages
|
|
666
663
|
:param requirements_file: path to a python requirements file
|
|
667
664
|
:param categories: list of categories for mlrun Function Hub, defaults to None
|
|
@@ -1151,7 +1148,7 @@ def wait_for_runs_completion(
|
|
|
1151
1148
|
running = []
|
|
1152
1149
|
for run in runs:
|
|
1153
1150
|
state = run.state()
|
|
1154
|
-
if state in mlrun.runtimes.constants.RunStates.terminal_states():
|
|
1151
|
+
if state in mlrun.common.runtimes.constants.RunStates.terminal_states():
|
|
1155
1152
|
completed.append(run)
|
|
1156
1153
|
else:
|
|
1157
1154
|
running.append(run)
|
mlrun/runtimes/__init__.py
CHANGED
|
@@ -30,13 +30,13 @@ __all__ = [
|
|
|
30
30
|
|
|
31
31
|
from mlrun.runtimes.utils import resolve_spark_operator_version
|
|
32
32
|
|
|
33
|
+
from ..common.runtimes.constants import MPIJobCRDVersions
|
|
33
34
|
from .base import BaseRuntime, RunError, RuntimeClassMode # noqa
|
|
34
|
-
from .constants import MPIJobCRDVersions
|
|
35
35
|
from .daskjob import DaskCluster # noqa
|
|
36
36
|
from .databricks_job.databricks_runtime import DatabricksRuntime
|
|
37
37
|
from .kubejob import KubejobRuntime, KubeResource # noqa
|
|
38
38
|
from .local import HandlerRuntime, LocalRuntime # noqa
|
|
39
|
-
from .mpijob import
|
|
39
|
+
from .mpijob import MpiRuntimeV1 # noqa
|
|
40
40
|
from .nuclio import (
|
|
41
41
|
RemoteRuntime,
|
|
42
42
|
ServingRuntime,
|
|
@@ -264,7 +264,7 @@ class RuntimeKinds:
|
|
|
264
264
|
|
|
265
265
|
def get_runtime_class(kind: str):
|
|
266
266
|
if kind == RuntimeKinds.mpijob:
|
|
267
|
-
return
|
|
267
|
+
return MpiRuntimeV1
|
|
268
268
|
|
|
269
269
|
if kind == RuntimeKinds.spark:
|
|
270
270
|
return Spark3Runtime
|
mlrun/runtimes/base.py
CHANGED
|
@@ -469,7 +469,7 @@ class BaseRuntime(ModelObj):
|
|
|
469
469
|
def _store_function(self, runspec, meta, db):
|
|
470
470
|
meta.labels["kind"] = self.kind
|
|
471
471
|
mlrun.runtimes.utils.enrich_run_labels(
|
|
472
|
-
meta.labels, [mlrun.runtimes.constants.RunLabels.owner]
|
|
472
|
+
meta.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
|
|
473
473
|
)
|
|
474
474
|
if runspec.spec.output_path:
|
|
475
475
|
runspec.spec.output_path = runspec.spec.output_path.replace(
|
|
@@ -580,9 +580,9 @@ class BaseRuntime(ModelObj):
|
|
|
580
580
|
|
|
581
581
|
elif (
|
|
582
582
|
not was_none
|
|
583
|
-
and last_state != mlrun.runtimes.constants.RunStates.completed
|
|
583
|
+
and last_state != mlrun.common.runtimes.constants.RunStates.completed
|
|
584
584
|
and last_state
|
|
585
|
-
not in mlrun.runtimes.constants.RunStates.error_and_abortion_states()
|
|
585
|
+
not in mlrun.common.runtimes.constants.RunStates.error_and_abortion_states()
|
|
586
586
|
):
|
|
587
587
|
try:
|
|
588
588
|
runtime_cls = mlrun.runtimes.get_runtime_class(kind)
|
mlrun/runtimes/funcdoc.py
CHANGED
|
@@ -16,8 +16,6 @@ import ast
|
|
|
16
16
|
import inspect
|
|
17
17
|
import re
|
|
18
18
|
|
|
19
|
-
from deprecated import deprecated
|
|
20
|
-
|
|
21
19
|
from mlrun.model import FunctionEntrypoint
|
|
22
20
|
|
|
23
21
|
|
|
@@ -73,32 +71,6 @@ def func_dict(
|
|
|
73
71
|
}
|
|
74
72
|
|
|
75
73
|
|
|
76
|
-
# TODO: remove in 1.7.0
|
|
77
|
-
@deprecated(
|
|
78
|
-
version="1.5.0",
|
|
79
|
-
reason="'func_info' is deprecated and will be removed in 1.7.0, use 'ast_func_info' instead",
|
|
80
|
-
category=FutureWarning,
|
|
81
|
-
)
|
|
82
|
-
def func_info(fn) -> dict:
|
|
83
|
-
sig = inspect.signature(fn)
|
|
84
|
-
doc = inspect.getdoc(fn) or ""
|
|
85
|
-
|
|
86
|
-
out = func_dict(
|
|
87
|
-
name=fn.__name__,
|
|
88
|
-
doc=doc,
|
|
89
|
-
params=[inspect_param(p) for p in sig.parameters.values()],
|
|
90
|
-
returns=param_dict(
|
|
91
|
-
type=type_name(sig.return_annotation, empty_is_none=True), default=None
|
|
92
|
-
),
|
|
93
|
-
lineno=func_lineno(fn),
|
|
94
|
-
)
|
|
95
|
-
|
|
96
|
-
if not fn.__doc__ or not fn.__doc__.strip():
|
|
97
|
-
return out
|
|
98
|
-
|
|
99
|
-
return merge_doc(out, doc)
|
|
100
|
-
|
|
101
|
-
|
|
102
74
|
def func_lineno(fn):
|
|
103
75
|
try:
|
|
104
76
|
return inspect.getsourcelines(fn)[1]
|
mlrun/runtimes/local.py
CHANGED
|
@@ -493,7 +493,7 @@ def exec_from_params(handler, runobj: RunObject, context: MLClientCtx, cwd=None)
|
|
|
493
493
|
logger.warning("Run was aborted", err=err_to_str(exc))
|
|
494
494
|
# Run was aborted, the state run state is updated by the abort job, no need to commit again
|
|
495
495
|
context.set_state(
|
|
496
|
-
mlrun.runtimes.constants.RunStates.aborted, commit=False
|
|
496
|
+
mlrun.common.runtimes.constants.RunStates.aborted, commit=False
|
|
497
497
|
)
|
|
498
498
|
commit = False
|
|
499
499
|
except Exception as exc:
|
|
@@ -21,28 +21,8 @@ from mlrun.config import config
|
|
|
21
21
|
from .. import MPIJobCRDVersions
|
|
22
22
|
from .abstract import AbstractMPIJobRuntime
|
|
23
23
|
from .v1 import MpiRuntimeV1
|
|
24
|
-
from .v1alpha1 import MpiRuntimeV1Alpha1
|
|
25
24
|
|
|
26
25
|
|
|
27
26
|
def _resolve_mpijob_crd_version():
|
|
28
27
|
# config is expected to get enriched from the API through the client-spec
|
|
29
28
|
return config.mpijob_crd_version or MPIJobCRDVersions.default()
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
class MpiRuntimeContainer(containers.DeclarativeContainer):
|
|
33
|
-
resolver = providers.Callable(
|
|
34
|
-
_resolve_mpijob_crd_version,
|
|
35
|
-
)
|
|
36
|
-
|
|
37
|
-
selector = providers.Selector(
|
|
38
|
-
resolver,
|
|
39
|
-
v1=providers.Object(MpiRuntimeV1),
|
|
40
|
-
v1alpha1=providers.Object(MpiRuntimeV1Alpha1),
|
|
41
|
-
)
|
|
42
|
-
|
|
43
|
-
# An empty selector to be overriden by the API
|
|
44
|
-
handler_selector = providers.Selector(
|
|
45
|
-
resolver,
|
|
46
|
-
v1=providers.Object(None),
|
|
47
|
-
v1alpha1=providers.Object(None),
|
|
48
|
-
)
|
mlrun/runtimes/mpijob/v1.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
from mlrun.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
|
|
14
|
+
from mlrun.common.runtimes.constants import MPIJobCRDVersions, MPIJobV1CleanPodPolicies
|
|
15
15
|
from mlrun.runtimes.mpijob.abstract import AbstractMPIJobRuntime, MPIResourceSpec
|
|
16
16
|
|
|
17
17
|
|