mlrun 1.8.0rc1__py3-none-any.whl → 1.8.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +5 -7
- mlrun/__main__.py +1 -1
- mlrun/common/formatters/project.py +9 -0
- mlrun/common/schemas/__init__.py +3 -0
- mlrun/common/schemas/alert.py +31 -18
- mlrun/common/schemas/api_gateway.py +3 -3
- mlrun/common/schemas/artifact.py +7 -7
- mlrun/common/schemas/auth.py +6 -4
- mlrun/common/schemas/background_task.py +7 -7
- mlrun/common/schemas/client_spec.py +2 -2
- mlrun/common/schemas/clusterization_spec.py +2 -2
- mlrun/common/schemas/common.py +5 -5
- mlrun/common/schemas/datastore_profile.py +1 -1
- mlrun/common/schemas/feature_store.py +9 -9
- mlrun/common/schemas/frontend_spec.py +4 -4
- mlrun/common/schemas/function.py +10 -10
- mlrun/common/schemas/hub.py +1 -1
- mlrun/common/schemas/k8s.py +3 -3
- mlrun/common/schemas/memory_reports.py +3 -3
- mlrun/common/schemas/model_monitoring/grafana.py +1 -1
- mlrun/common/schemas/model_monitoring/model_endpoint_v2.py +1 -1
- mlrun/common/schemas/model_monitoring/model_endpoints.py +1 -1
- mlrun/common/schemas/notification.py +18 -3
- mlrun/common/schemas/object.py +1 -1
- mlrun/common/schemas/pagination.py +4 -4
- mlrun/common/schemas/partition.py +16 -1
- mlrun/common/schemas/pipeline.py +2 -2
- mlrun/common/schemas/project.py +22 -17
- mlrun/common/schemas/runs.py +2 -2
- mlrun/common/schemas/runtime_resource.py +5 -5
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/schemas/secret.py +1 -1
- mlrun/common/schemas/tag.py +3 -3
- mlrun/common/schemas/workflow.py +5 -5
- mlrun/config.py +22 -0
- mlrun/datastore/datastore_profile.py +19 -19
- mlrun/db/base.py +48 -6
- mlrun/db/httpdb.py +221 -9
- mlrun/db/nopdb.py +34 -5
- mlrun/model.py +2 -2
- mlrun/model_monitoring/applications/results.py +2 -2
- mlrun/model_monitoring/db/tsdb/base.py +2 -2
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +37 -13
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +32 -40
- mlrun/model_monitoring/helpers.py +4 -10
- mlrun/model_monitoring/stream_processing.py +14 -11
- mlrun/platforms/__init__.py +0 -13
- mlrun/projects/__init__.py +6 -1
- mlrun/projects/pipelines.py +184 -55
- mlrun/projects/project.py +95 -28
- mlrun/run.py +4 -1
- mlrun/runtimes/base.py +2 -1
- mlrun/runtimes/mounts.py +572 -0
- mlrun/runtimes/nuclio/function.py +1 -2
- mlrun/runtimes/pod.py +82 -18
- mlrun/runtimes/remotesparkjob.py +1 -1
- mlrun/runtimes/sparkjob/spark3job.py +1 -1
- mlrun/utils/helpers.py +12 -2
- mlrun/utils/logger.py +2 -2
- mlrun/utils/notifications/notification/__init__.py +22 -19
- mlrun/utils/notifications/notification/base.py +12 -12
- mlrun/utils/notifications/notification/console.py +6 -6
- mlrun/utils/notifications/notification/git.py +6 -6
- mlrun/utils/notifications/notification/ipython.py +6 -6
- mlrun/utils/notifications/notification/mail.py +149 -0
- mlrun/utils/notifications/notification/slack.py +6 -6
- mlrun/utils/notifications/notification/webhook.py +6 -6
- mlrun/utils/notifications/notification_pusher.py +20 -12
- mlrun/utils/regex.py +2 -0
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc2.dist-info}/METADATA +190 -186
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc2.dist-info}/RECORD +76 -74
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc2.dist-info}/WHEEL +1 -1
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc2.dist-info}/LICENSE +0 -0
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc2.dist-info}/entry_points.txt +0 -0
- {mlrun-1.8.0rc1.dist-info → mlrun-1.8.0rc2.dist-info}/top_level.txt +0 -0
mlrun/projects/project.py
CHANGED
|
@@ -49,6 +49,7 @@ import mlrun.k8s_utils
|
|
|
49
49
|
import mlrun.lists
|
|
50
50
|
import mlrun.model_monitoring.applications as mm_app
|
|
51
51
|
import mlrun.runtimes
|
|
52
|
+
import mlrun.runtimes.mounts
|
|
52
53
|
import mlrun.runtimes.nuclio.api_gateway
|
|
53
54
|
import mlrun.runtimes.pod
|
|
54
55
|
import mlrun.runtimes.utils
|
|
@@ -56,7 +57,6 @@ import mlrun.serving
|
|
|
56
57
|
import mlrun.utils
|
|
57
58
|
import mlrun.utils.regex
|
|
58
59
|
import mlrun_pipelines.common.models
|
|
59
|
-
import mlrun_pipelines.mounts
|
|
60
60
|
from mlrun.alerts.alert import AlertConfig
|
|
61
61
|
from mlrun.common.schemas.alert import AlertTemplate
|
|
62
62
|
from mlrun.datastore.datastore_profile import DatastoreProfile, DatastoreProfile2Json
|
|
@@ -1995,8 +1995,6 @@ class MlrunProject(ModelObj):
|
|
|
1995
1995
|
:param application_kwargs: Additional keyword arguments to be passed to the
|
|
1996
1996
|
monitoring application's constructor.
|
|
1997
1997
|
"""
|
|
1998
|
-
|
|
1999
|
-
function_object: RemoteRuntime = None
|
|
2000
1998
|
(
|
|
2001
1999
|
resolved_function_name,
|
|
2002
2000
|
function_object,
|
|
@@ -2094,7 +2092,6 @@ class MlrunProject(ModelObj):
|
|
|
2094
2092
|
) -> tuple[str, mlrun.runtimes.BaseRuntime, dict]:
|
|
2095
2093
|
import mlrun.model_monitoring.api
|
|
2096
2094
|
|
|
2097
|
-
function_object: RemoteRuntime = None
|
|
2098
2095
|
kind = None
|
|
2099
2096
|
if (isinstance(func, str) or func is None) and application_class is not None:
|
|
2100
2097
|
kind = mlrun.run.RuntimeKinds.serving
|
|
@@ -2133,9 +2130,6 @@ class MlrunProject(ModelObj):
|
|
|
2133
2130
|
mm_constants.ModelMonitoringAppLabel.VAL,
|
|
2134
2131
|
)
|
|
2135
2132
|
|
|
2136
|
-
if not mlrun.mlconf.is_ce_mode():
|
|
2137
|
-
function_object.apply(mlrun.mount_v3io())
|
|
2138
|
-
|
|
2139
2133
|
return resolved_function_name, function_object, func
|
|
2140
2134
|
|
|
2141
2135
|
def _wait_for_functions_deployment(self, function_names: list[str]) -> None:
|
|
@@ -3390,6 +3384,61 @@ class MlrunProject(ModelObj):
|
|
|
3390
3384
|
"and set `rebuild_images=True`"
|
|
3391
3385
|
)
|
|
3392
3386
|
|
|
3387
|
+
def list_model_endpoints(
|
|
3388
|
+
self,
|
|
3389
|
+
model: Optional[str] = None,
|
|
3390
|
+
function: Optional[str] = None,
|
|
3391
|
+
labels: Optional[list[str]] = None,
|
|
3392
|
+
start: str = "now-1h",
|
|
3393
|
+
end: str = "now",
|
|
3394
|
+
top_level: bool = False,
|
|
3395
|
+
uids: Optional[list[str]] = None,
|
|
3396
|
+
) -> list[mlrun.model_monitoring.model_endpoint.ModelEndpoint]:
|
|
3397
|
+
"""
|
|
3398
|
+
Returns a list of `ModelEndpoint` objects. Each `ModelEndpoint` object represents the current state of a
|
|
3399
|
+
model endpoint. This functions supports filtering by the following parameters:
|
|
3400
|
+
1) model
|
|
3401
|
+
2) function
|
|
3402
|
+
3) labels
|
|
3403
|
+
4) top level
|
|
3404
|
+
5) uids
|
|
3405
|
+
By default, when no filters are applied, all available endpoints for the given project will be listed.
|
|
3406
|
+
|
|
3407
|
+
In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
|
|
3408
|
+
and depends on the 'start' and 'end' parameters.
|
|
3409
|
+
|
|
3410
|
+
:param model: The name of the model to filter by
|
|
3411
|
+
:param function: The name of the function to filter by
|
|
3412
|
+
:param labels: Filter model endpoints by label key-value pairs or key existence. This can be provided as:
|
|
3413
|
+
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
3414
|
+
or `{"label": None}` to check for key existence.
|
|
3415
|
+
- A list of strings formatted as `"label=value"` to match specific label key-value pairs,
|
|
3416
|
+
or just `"label"` for key existence.
|
|
3417
|
+
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
3418
|
+
the specified key-value pairs or key existence.
|
|
3419
|
+
:param start: The start time of the metrics. Can be represented by a string containing an RFC 3339 time, a
|
|
3420
|
+
Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
|
|
3421
|
+
`m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
|
|
3422
|
+
:param end: The end time of the metrics. Can be represented by a string containing an RFC 3339 time, a
|
|
3423
|
+
Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
|
|
3424
|
+
`m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
|
|
3425
|
+
:param top_level: if true will return only routers and endpoint that are NOT children of any router
|
|
3426
|
+
:param uids: if passed will return a list `ModelEndpoint` object with uid in uids
|
|
3427
|
+
|
|
3428
|
+
:returns: Returns a list of `ModelEndpoint` objects.
|
|
3429
|
+
"""
|
|
3430
|
+
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
3431
|
+
return db.list_model_endpoints(
|
|
3432
|
+
project=self.name,
|
|
3433
|
+
model=model,
|
|
3434
|
+
function=function,
|
|
3435
|
+
labels=labels,
|
|
3436
|
+
start=start,
|
|
3437
|
+
end=end,
|
|
3438
|
+
top_level=top_level,
|
|
3439
|
+
uids=uids,
|
|
3440
|
+
)
|
|
3441
|
+
|
|
3393
3442
|
def run_function(
|
|
3394
3443
|
self,
|
|
3395
3444
|
function: typing.Union[str, mlrun.runtimes.BaseRuntime],
|
|
@@ -4051,6 +4100,8 @@ class MlrunProject(ModelObj):
|
|
|
4051
4100
|
name: Optional[str] = None,
|
|
4052
4101
|
tag: Optional[str] = None,
|
|
4053
4102
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
4103
|
+
kind: Optional[str] = None,
|
|
4104
|
+
format_: Optional[str] = None,
|
|
4054
4105
|
):
|
|
4055
4106
|
"""Retrieve a list of functions, filtered by specific criteria.
|
|
4056
4107
|
|
|
@@ -4068,10 +4119,19 @@ class MlrunProject(ModelObj):
|
|
|
4068
4119
|
or just `"label"` for key existence.
|
|
4069
4120
|
- A comma-separated string formatted as `"label1=value1,label2"` to match entities with
|
|
4070
4121
|
the specified key-value pairs or key existence.
|
|
4122
|
+
:param kind: Return functions of the specified kind. If not provided, all function kinds will be returned.
|
|
4123
|
+
:param format_: The format in which to return the functions. Default is 'full'.
|
|
4071
4124
|
:returns: List of function objects.
|
|
4072
4125
|
"""
|
|
4073
4126
|
db = mlrun.db.get_run_db(secrets=self._secrets)
|
|
4074
|
-
functions = db.list_functions(
|
|
4127
|
+
functions = db.list_functions(
|
|
4128
|
+
name,
|
|
4129
|
+
project=self.metadata.name,
|
|
4130
|
+
tag=tag,
|
|
4131
|
+
kind=kind,
|
|
4132
|
+
labels=labels,
|
|
4133
|
+
format_=format_,
|
|
4134
|
+
)
|
|
4075
4135
|
if functions:
|
|
4076
4136
|
# convert dict to function objects
|
|
4077
4137
|
return [mlrun.new_function(runtime=func) for func in functions]
|
|
@@ -4133,9 +4193,8 @@ class MlrunProject(ModelObj):
|
|
|
4133
4193
|
page_token=page_token,
|
|
4134
4194
|
**kwargs,
|
|
4135
4195
|
)
|
|
4136
|
-
|
|
4137
|
-
|
|
4138
|
-
return [mlrun.new_function(runtime=func) for func in functions], token
|
|
4196
|
+
# convert dict to function objects
|
|
4197
|
+
return [mlrun.new_function(runtime=func) for func in functions], token
|
|
4139
4198
|
|
|
4140
4199
|
def list_model_monitoring_functions(
|
|
4141
4200
|
self,
|
|
@@ -4241,9 +4300,11 @@ class MlrunProject(ModelObj):
|
|
|
4241
4300
|
uid,
|
|
4242
4301
|
self.metadata.name,
|
|
4243
4302
|
labels=labels,
|
|
4244
|
-
states=
|
|
4245
|
-
|
|
4246
|
-
|
|
4303
|
+
states=(
|
|
4304
|
+
mlrun.utils.helpers.as_list(state)
|
|
4305
|
+
if state is not None
|
|
4306
|
+
else states or None
|
|
4307
|
+
),
|
|
4247
4308
|
sort=sort,
|
|
4248
4309
|
last=last,
|
|
4249
4310
|
iter=iter,
|
|
@@ -4686,23 +4747,29 @@ class MlrunProject(ModelObj):
|
|
|
4686
4747
|
)
|
|
4687
4748
|
|
|
4688
4749
|
if producer_dict.get("kind", "") == "run":
|
|
4689
|
-
return
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4750
|
+
return (
|
|
4751
|
+
ArtifactProducer(
|
|
4752
|
+
name=producer_dict.get("name", ""),
|
|
4753
|
+
kind=producer_dict.get("kind", ""),
|
|
4754
|
+
project=producer_project,
|
|
4755
|
+
tag=producer_tag,
|
|
4756
|
+
owner=producer_dict.get("owner", ""),
|
|
4757
|
+
),
|
|
4758
|
+
True,
|
|
4759
|
+
)
|
|
4696
4760
|
|
|
4697
4761
|
# do not retain the artifact's producer, replace it with the project as the producer
|
|
4698
4762
|
project_producer_tag = project_producer_tag or self._get_project_tag()
|
|
4699
|
-
return
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
|
|
4705
|
-
|
|
4763
|
+
return (
|
|
4764
|
+
ArtifactProducer(
|
|
4765
|
+
kind="project",
|
|
4766
|
+
name=self.metadata.name,
|
|
4767
|
+
project=self.metadata.name,
|
|
4768
|
+
tag=project_producer_tag,
|
|
4769
|
+
owner=self._resolve_artifact_owner(),
|
|
4770
|
+
),
|
|
4771
|
+
False,
|
|
4772
|
+
)
|
|
4706
4773
|
|
|
4707
4774
|
def _resolve_existing_artifact(
|
|
4708
4775
|
self,
|
mlrun/run.py
CHANGED
|
@@ -73,6 +73,9 @@ from .utils import (
|
|
|
73
73
|
update_in,
|
|
74
74
|
)
|
|
75
75
|
|
|
76
|
+
if typing.TYPE_CHECKING:
|
|
77
|
+
from mlrun.datastore import DataItem
|
|
78
|
+
|
|
76
79
|
|
|
77
80
|
def function_to_module(code="", workdir=None, secrets=None, silent=False):
|
|
78
81
|
"""Load code, notebook or mlrun function as .py module
|
|
@@ -1089,7 +1092,7 @@ def get_object(url, secrets=None, size=None, offset=0, db=None):
|
|
|
1089
1092
|
return stores.object(url=url).get(size, offset)
|
|
1090
1093
|
|
|
1091
1094
|
|
|
1092
|
-
def get_dataitem(url, secrets=None, db=None) ->
|
|
1095
|
+
def get_dataitem(url, secrets=None, db=None) -> "DataItem":
|
|
1093
1096
|
"""get mlrun dataitem object (from path/url)"""
|
|
1094
1097
|
stores = store_manager.set(secrets, db=db)
|
|
1095
1098
|
return stores.object(url=url)
|
mlrun/runtimes/base.py
CHANGED
|
@@ -25,9 +25,10 @@ from nuclio.build import mlrun_footer
|
|
|
25
25
|
|
|
26
26
|
import mlrun.common.constants
|
|
27
27
|
import mlrun.common.constants as mlrun_constants
|
|
28
|
+
import mlrun.common.formatters
|
|
29
|
+
import mlrun.common.runtimes
|
|
28
30
|
import mlrun.common.schemas
|
|
29
31
|
import mlrun.common.schemas.model_monitoring.constants as mm_constants
|
|
30
|
-
import mlrun.db
|
|
31
32
|
import mlrun.errors
|
|
32
33
|
import mlrun.launcher.factory
|
|
33
34
|
import mlrun.utils.helpers
|