mlrun 1.7.0rc20__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +10 -8
- mlrun/alerts/alert.py +55 -18
- mlrun/api/schemas/__init__.py +3 -3
- mlrun/artifacts/manager.py +26 -0
- mlrun/common/constants.py +3 -2
- mlrun/common/formatters/__init__.py +1 -0
- mlrun/common/formatters/artifact.py +26 -3
- mlrun/common/formatters/base.py +44 -9
- mlrun/common/formatters/function.py +12 -7
- mlrun/common/formatters/run.py +26 -0
- mlrun/common/helpers.py +11 -0
- mlrun/common/schemas/__init__.py +4 -0
- mlrun/common/schemas/alert.py +5 -9
- mlrun/common/schemas/api_gateway.py +64 -16
- mlrun/common/schemas/artifact.py +11 -0
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/feature_store.py +58 -28
- mlrun/common/schemas/model_monitoring/constants.py +21 -12
- mlrun/common/schemas/model_monitoring/model_endpoints.py +0 -12
- mlrun/common/schemas/pipeline.py +16 -0
- mlrun/common/schemas/project.py +17 -0
- mlrun/common/schemas/runs.py +17 -0
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/types.py +6 -0
- mlrun/config.py +17 -25
- mlrun/datastore/azure_blob.py +2 -1
- mlrun/datastore/datastore.py +3 -3
- mlrun/datastore/google_cloud_storage.py +6 -2
- mlrun/datastore/snowflake_utils.py +3 -1
- mlrun/datastore/sources.py +26 -11
- mlrun/datastore/store_resources.py +2 -0
- mlrun/datastore/targets.py +68 -16
- mlrun/db/base.py +83 -2
- mlrun/db/httpdb.py +280 -63
- mlrun/db/nopdb.py +60 -3
- mlrun/errors.py +5 -3
- mlrun/execution.py +28 -13
- mlrun/feature_store/feature_vector.py +8 -0
- mlrun/feature_store/retrieval/spark_merger.py +13 -2
- mlrun/launcher/local.py +4 -0
- mlrun/launcher/remote.py +1 -0
- mlrun/model.py +32 -3
- mlrun/model_monitoring/api.py +7 -52
- mlrun/model_monitoring/applications/base.py +5 -7
- mlrun/model_monitoring/applications/histogram_data_drift.py +1 -1
- mlrun/model_monitoring/db/stores/__init__.py +37 -24
- mlrun/model_monitoring/db/stores/base/store.py +40 -1
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +42 -87
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +27 -35
- mlrun/model_monitoring/db/tsdb/__init__.py +15 -15
- mlrun/model_monitoring/db/tsdb/base.py +1 -14
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +22 -18
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +86 -56
- mlrun/model_monitoring/helpers.py +34 -9
- mlrun/model_monitoring/stream_processing.py +12 -11
- mlrun/model_monitoring/writer.py +11 -11
- mlrun/projects/operations.py +5 -0
- mlrun/projects/pipelines.py +35 -21
- mlrun/projects/project.py +216 -107
- mlrun/render.py +10 -5
- mlrun/run.py +15 -5
- mlrun/runtimes/__init__.py +2 -0
- mlrun/runtimes/base.py +17 -4
- mlrun/runtimes/daskjob.py +8 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +1 -0
- mlrun/runtimes/local.py +23 -4
- mlrun/runtimes/nuclio/application/application.py +0 -2
- mlrun/runtimes/nuclio/function.py +31 -2
- mlrun/runtimes/nuclio/serving.py +9 -6
- mlrun/runtimes/pod.py +5 -29
- mlrun/runtimes/remotesparkjob.py +8 -2
- mlrun/serving/__init__.py +8 -1
- mlrun/serving/routers.py +75 -59
- mlrun/serving/server.py +11 -0
- mlrun/serving/states.py +80 -8
- mlrun/serving/utils.py +19 -11
- mlrun/serving/v2_serving.py +66 -39
- mlrun/utils/helpers.py +91 -11
- mlrun/utils/logger.py +36 -2
- mlrun/utils/notifications/notification/base.py +43 -7
- mlrun/utils/notifications/notification/git.py +21 -0
- mlrun/utils/notifications/notification/slack.py +9 -14
- mlrun/utils/notifications/notification/webhook.py +41 -1
- mlrun/utils/notifications/notification_pusher.py +3 -9
- mlrun/utils/regex.py +9 -0
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/METADATA +16 -9
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/RECORD +92 -91
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/top_level.txt +0 -0
mlrun/__main__.py
CHANGED
|
@@ -50,12 +50,12 @@ from .run import (
|
|
|
50
50
|
from .runtimes import RemoteRuntime, RunError, RuntimeKinds, ServingRuntime
|
|
51
51
|
from .secrets import SecretsStore
|
|
52
52
|
from .utils import (
|
|
53
|
+
RunKeys,
|
|
53
54
|
dict_to_yaml,
|
|
54
55
|
get_in,
|
|
55
56
|
is_relative_path,
|
|
56
57
|
list2dict,
|
|
57
58
|
logger,
|
|
58
|
-
run_keys,
|
|
59
59
|
update_in,
|
|
60
60
|
)
|
|
61
61
|
from .utils.version import Version
|
|
@@ -102,7 +102,9 @@ def main():
|
|
|
102
102
|
)
|
|
103
103
|
@click.option("--uid", help="unique run ID")
|
|
104
104
|
@click.option("--name", help="run name")
|
|
105
|
-
@click.option(
|
|
105
|
+
@click.option(
|
|
106
|
+
"--workflow", help="sets the run labels to match the given workflow name/id"
|
|
107
|
+
)
|
|
106
108
|
@click.option("--project", help="project name/id")
|
|
107
109
|
@click.option("--db", default="", help="save run results to path or DB url")
|
|
108
110
|
@click.option(
|
|
@@ -378,15 +380,15 @@ def run(
|
|
|
378
380
|
set_item(runobj.spec.hyper_param_options, hyper_param_strategy, "strategy")
|
|
379
381
|
set_item(runobj.spec.hyper_param_options, selector, "selector")
|
|
380
382
|
|
|
381
|
-
set_item(runobj.spec, inputs,
|
|
383
|
+
set_item(runobj.spec, inputs, RunKeys.inputs, list2dict(inputs))
|
|
382
384
|
set_item(
|
|
383
|
-
runobj.spec, returns,
|
|
385
|
+
runobj.spec, returns, RunKeys.returns, [py_eval(value) for value in returns]
|
|
384
386
|
)
|
|
385
|
-
set_item(runobj.spec, in_path,
|
|
386
|
-
set_item(runobj.spec, out_path,
|
|
387
|
-
set_item(runobj.spec, outputs,
|
|
387
|
+
set_item(runobj.spec, in_path, RunKeys.input_path)
|
|
388
|
+
set_item(runobj.spec, out_path, RunKeys.output_path)
|
|
389
|
+
set_item(runobj.spec, outputs, RunKeys.outputs, list(outputs))
|
|
388
390
|
set_item(
|
|
389
|
-
runobj.spec, secrets,
|
|
391
|
+
runobj.spec, secrets, RunKeys.secrets, line2keylist(secrets, "kind", "source")
|
|
390
392
|
)
|
|
391
393
|
set_item(runobj.spec, verbose, "verbose")
|
|
392
394
|
set_item(runobj.spec, scrape_metrics, "scrape_metrics")
|
mlrun/alerts/alert.py
CHANGED
|
@@ -26,10 +26,15 @@ class AlertConfig(ModelObj):
|
|
|
26
26
|
"description",
|
|
27
27
|
"summary",
|
|
28
28
|
"severity",
|
|
29
|
-
"criteria",
|
|
30
29
|
"reset_policy",
|
|
31
30
|
"state",
|
|
32
31
|
]
|
|
32
|
+
_fields_to_serialize = ModelObj._fields_to_serialize + [
|
|
33
|
+
"entities",
|
|
34
|
+
"notifications",
|
|
35
|
+
"trigger",
|
|
36
|
+
"criteria",
|
|
37
|
+
]
|
|
33
38
|
|
|
34
39
|
def __init__(
|
|
35
40
|
self,
|
|
@@ -71,24 +76,52 @@ class AlertConfig(ModelObj):
|
|
|
71
76
|
if not self.project or not self.name:
|
|
72
77
|
raise mlrun.errors.MLRunBadRequestError("Project and name must be provided")
|
|
73
78
|
|
|
79
|
+
def _serialize_field(
|
|
80
|
+
self, struct: dict, field_name: str = None, strip: bool = False
|
|
81
|
+
):
|
|
82
|
+
if field_name == "entities":
|
|
83
|
+
if self.entities:
|
|
84
|
+
return (
|
|
85
|
+
self.entities.dict()
|
|
86
|
+
if not isinstance(self.entities, dict)
|
|
87
|
+
else self.entities
|
|
88
|
+
)
|
|
89
|
+
return None
|
|
90
|
+
if field_name == "notifications":
|
|
91
|
+
if self.notifications:
|
|
92
|
+
return [
|
|
93
|
+
notification_data.dict()
|
|
94
|
+
if not isinstance(notification_data, dict)
|
|
95
|
+
else notification_data
|
|
96
|
+
for notification_data in self.notifications
|
|
97
|
+
]
|
|
98
|
+
return None
|
|
99
|
+
if field_name == "trigger":
|
|
100
|
+
if self.trigger:
|
|
101
|
+
return (
|
|
102
|
+
self.trigger.dict()
|
|
103
|
+
if not isinstance(self.trigger, dict)
|
|
104
|
+
else self.trigger
|
|
105
|
+
)
|
|
106
|
+
return None
|
|
107
|
+
if field_name == "criteria":
|
|
108
|
+
if self.criteria:
|
|
109
|
+
return (
|
|
110
|
+
self.criteria.dict()
|
|
111
|
+
if not isinstance(self.criteria, dict)
|
|
112
|
+
else self.criteria
|
|
113
|
+
)
|
|
114
|
+
return None
|
|
115
|
+
return super()._serialize_field(struct, field_name, strip)
|
|
116
|
+
|
|
74
117
|
def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
)
|
|
82
|
-
data["notifications"] = [
|
|
83
|
-
notification_data.dict()
|
|
84
|
-
if not isinstance(notification_data, dict)
|
|
85
|
-
else notification_data
|
|
86
|
-
for notification_data in self.notifications
|
|
87
|
-
]
|
|
88
|
-
data["trigger"] = (
|
|
89
|
-
self.trigger.dict() if not isinstance(self.trigger, dict) else self.trigger
|
|
90
|
-
)
|
|
91
|
-
return data
|
|
118
|
+
if self.entities is None:
|
|
119
|
+
raise mlrun.errors.MLRunBadRequestError("Alert entity field is missing")
|
|
120
|
+
if not self.notifications:
|
|
121
|
+
raise mlrun.errors.MLRunBadRequestError(
|
|
122
|
+
"Alert must have at least one notification"
|
|
123
|
+
)
|
|
124
|
+
return super().to_dict(self._dict_fields)
|
|
92
125
|
|
|
93
126
|
@classmethod
|
|
94
127
|
def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
|
|
@@ -112,6 +145,10 @@ class AlertConfig(ModelObj):
|
|
|
112
145
|
trigger_obj = alert_objects.AlertTrigger.parse_obj(trigger_data)
|
|
113
146
|
new_obj.trigger = trigger_obj
|
|
114
147
|
|
|
148
|
+
criteria_data = struct.get("criteria")
|
|
149
|
+
if criteria_data:
|
|
150
|
+
criteria_obj = alert_objects.AlertCriteria.parse_obj(criteria_data)
|
|
151
|
+
new_obj.criteria = criteria_obj
|
|
115
152
|
return new_obj
|
|
116
153
|
|
|
117
154
|
def with_notifications(self, notifications: list[alert_objects.AlertNotification]):
|
mlrun/api/schemas/__init__.py
CHANGED
|
@@ -97,7 +97,7 @@ sys.modules["mlrun.api.schemas.tag"] = old_tag
|
|
|
97
97
|
# and return the new schema. This is done for backwards compatibility with mlrun.api.schemas.
|
|
98
98
|
ArtifactCategories = DeprecationHelper(mlrun.common.schemas.ArtifactCategories)
|
|
99
99
|
ArtifactIdentifier = DeprecationHelper(mlrun.common.schemas.ArtifactIdentifier)
|
|
100
|
-
ArtifactsFormat = DeprecationHelper(mlrun.common.formatters.
|
|
100
|
+
ArtifactsFormat = DeprecationHelper(mlrun.common.formatters.ArtifactFormat)
|
|
101
101
|
AuthInfo = DeprecationHelper(mlrun.common.schemas.AuthInfo)
|
|
102
102
|
AuthorizationAction = DeprecationHelper(mlrun.common.schemas.AuthorizationAction)
|
|
103
103
|
AuthorizationResourceTypes = DeprecationHelper(
|
|
@@ -222,7 +222,7 @@ ObjectKind = DeprecationHelper(mlrun.common.schemas.ObjectKind)
|
|
|
222
222
|
ObjectMetadata = DeprecationHelper(mlrun.common.schemas.ObjectMetadata)
|
|
223
223
|
ObjectSpec = DeprecationHelper(mlrun.common.schemas.ObjectSpec)
|
|
224
224
|
ObjectStatus = DeprecationHelper(mlrun.common.schemas.ObjectStatus)
|
|
225
|
-
PipelinesFormat = DeprecationHelper(mlrun.common.formatters.
|
|
225
|
+
PipelinesFormat = DeprecationHelper(mlrun.common.formatters.PipelineFormat)
|
|
226
226
|
PipelinesOutput = DeprecationHelper(mlrun.common.schemas.PipelinesOutput)
|
|
227
227
|
PipelinesPagination = DeprecationHelper(mlrun.common.schemas.PipelinesPagination)
|
|
228
228
|
IguazioProject = DeprecationHelper(mlrun.common.schemas.IguazioProject)
|
|
@@ -230,7 +230,7 @@ Project = DeprecationHelper(mlrun.common.schemas.Project)
|
|
|
230
230
|
ProjectDesiredState = DeprecationHelper(mlrun.common.schemas.ProjectDesiredState)
|
|
231
231
|
ProjectMetadata = DeprecationHelper(mlrun.common.schemas.ProjectMetadata)
|
|
232
232
|
ProjectOwner = DeprecationHelper(mlrun.common.schemas.ProjectOwner)
|
|
233
|
-
ProjectsFormat = DeprecationHelper(mlrun.common.formatters.
|
|
233
|
+
ProjectsFormat = DeprecationHelper(mlrun.common.formatters.ProjectFormat)
|
|
234
234
|
ProjectsOutput = DeprecationHelper(mlrun.common.schemas.ProjectsOutput)
|
|
235
235
|
ProjectSpec = DeprecationHelper(mlrun.common.schemas.ProjectSpec)
|
|
236
236
|
ProjectState = DeprecationHelper(mlrun.common.schemas.ProjectState)
|
mlrun/artifacts/manager.py
CHANGED
|
@@ -12,12 +12,14 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import pathlib
|
|
15
|
+
import re
|
|
15
16
|
import typing
|
|
16
17
|
from os.path import exists, isdir
|
|
17
18
|
from urllib.parse import urlparse
|
|
18
19
|
|
|
19
20
|
import mlrun.common.schemas.artifact
|
|
20
21
|
import mlrun.config
|
|
22
|
+
import mlrun.utils.regex
|
|
21
23
|
from mlrun.utils.helpers import (
|
|
22
24
|
get_local_file_schema,
|
|
23
25
|
template_artifact_path,
|
|
@@ -76,9 +78,33 @@ class ArtifactProducer:
|
|
|
76
78
|
def uid(self):
|
|
77
79
|
return None
|
|
78
80
|
|
|
81
|
+
@staticmethod
|
|
82
|
+
def parse_uri(uri: str) -> tuple[str, str, str]:
|
|
83
|
+
"""Parse artifact producer's uri
|
|
84
|
+
|
|
85
|
+
:param uri: artifact producer's uri in the format <project>/<uid>[-<iteration>]
|
|
86
|
+
:returns: tuple of project, uid, iteration
|
|
87
|
+
"""
|
|
88
|
+
uri_pattern = mlrun.utils.regex.artifact_producer_uri_pattern
|
|
89
|
+
match = re.match(uri_pattern, uri)
|
|
90
|
+
if not match:
|
|
91
|
+
return "", "", ""
|
|
92
|
+
group_dict = match.groupdict()
|
|
93
|
+
|
|
94
|
+
return (
|
|
95
|
+
group_dict["project"] or "",
|
|
96
|
+
group_dict["uid"] or "",
|
|
97
|
+
group_dict["iteration"] or "",
|
|
98
|
+
)
|
|
99
|
+
|
|
79
100
|
|
|
80
101
|
def dict_to_artifact(struct: dict) -> Artifact:
|
|
81
102
|
kind = struct.get("kind", "")
|
|
103
|
+
|
|
104
|
+
# TODO: remove this in 1.8.0
|
|
105
|
+
if mlrun.utils.is_legacy_artifact(struct):
|
|
106
|
+
return mlrun.artifacts.base.convert_legacy_artifact_to_new_format(struct)
|
|
107
|
+
|
|
82
108
|
artifact_class = artifact_types[kind]
|
|
83
109
|
return artifact_class.from_dict(struct)
|
|
84
110
|
|
mlrun/common/constants.py
CHANGED
|
@@ -42,6 +42,7 @@ class MLRunInternalLabels:
|
|
|
42
42
|
|
|
43
43
|
### nuclio
|
|
44
44
|
nuclio_project_name = f"{NUCLIO_LABEL_PREFIX}project-name"
|
|
45
|
+
nuclio_function_name = f"{NUCLIO_LABEL_PREFIX}function-name"
|
|
45
46
|
nuclio_class = f"{NUCLIO_LABEL_PREFIX}class"
|
|
46
47
|
|
|
47
48
|
### mlrun
|
|
@@ -63,12 +64,12 @@ class MLRunInternalLabels:
|
|
|
63
64
|
username = f"{MLRUN_LABEL_PREFIX}username"
|
|
64
65
|
username_domain = f"{MLRUN_LABEL_PREFIX}username_domain"
|
|
65
66
|
task_name = f"{MLRUN_LABEL_PREFIX}task-name"
|
|
67
|
+
resource_name = f"{MLRUN_LABEL_PREFIX}resource_name"
|
|
68
|
+
created = f"{MLRUN_LABEL_PREFIX}created"
|
|
66
69
|
host = "host"
|
|
67
70
|
job_type = "job-type"
|
|
68
71
|
kind = "kind"
|
|
69
72
|
component = "component"
|
|
70
|
-
resource_name = "resource_name"
|
|
71
|
-
created = "mlrun-created"
|
|
72
73
|
|
|
73
74
|
owner = "owner"
|
|
74
75
|
v3io_user = "v3io_user"
|
|
@@ -13,9 +13,32 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
15
|
|
|
16
|
+
import typing
|
|
17
|
+
|
|
16
18
|
import mlrun.common.types
|
|
17
19
|
|
|
20
|
+
from .base import ObjectFormat
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ArtifactFormat(ObjectFormat, mlrun.common.types.StrEnum):
|
|
24
|
+
minimal = "minimal"
|
|
18
25
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
26
|
+
@staticmethod
|
|
27
|
+
def format_method(_format: str) -> typing.Optional[typing.Callable]:
|
|
28
|
+
return {
|
|
29
|
+
ArtifactFormat.full: None,
|
|
30
|
+
ArtifactFormat.minimal: ArtifactFormat.filter_obj_method(
|
|
31
|
+
[
|
|
32
|
+
"kind",
|
|
33
|
+
"metadata",
|
|
34
|
+
"status",
|
|
35
|
+
"project",
|
|
36
|
+
"spec.producer",
|
|
37
|
+
"spec.db_key",
|
|
38
|
+
"spec.size",
|
|
39
|
+
"spec.framework",
|
|
40
|
+
"spec.metrics",
|
|
41
|
+
"spec.target_path",
|
|
42
|
+
]
|
|
43
|
+
),
|
|
44
|
+
}[_format]
|
mlrun/common/formatters/base.py
CHANGED
|
@@ -19,32 +19,51 @@ import mlrun.errors
|
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
class ObjectFormat:
|
|
22
|
+
"""
|
|
23
|
+
MLRun object formatter. Any class that inherits from this class should implement the `format_method` method
|
|
24
|
+
to specify the formatting method for each format.
|
|
25
|
+
A `filter_obj_method` utility method is provided to filter the object based on a list of keys.
|
|
26
|
+
"""
|
|
27
|
+
|
|
22
28
|
full = "full"
|
|
23
29
|
|
|
24
30
|
@staticmethod
|
|
25
|
-
def format_method(
|
|
31
|
+
def format_method(format_: str) -> typing.Optional[typing.Callable]:
|
|
32
|
+
"""
|
|
33
|
+
Get the formatting method for the provided format.
|
|
34
|
+
A `None` value signifies a pass-through formatting method (no formatting).
|
|
35
|
+
:param format_: The format as a string representation.
|
|
36
|
+
:return: The formatting method.
|
|
37
|
+
"""
|
|
26
38
|
return {
|
|
27
39
|
ObjectFormat.full: None,
|
|
28
|
-
}[
|
|
40
|
+
}[format_]
|
|
29
41
|
|
|
30
42
|
@classmethod
|
|
31
43
|
def format_obj(
|
|
32
44
|
cls,
|
|
33
45
|
obj: typing.Any,
|
|
34
|
-
|
|
46
|
+
format_: str,
|
|
35
47
|
exclude_formats: typing.Optional[list[str]] = None,
|
|
36
48
|
) -> typing.Any:
|
|
49
|
+
"""
|
|
50
|
+
Format the provided object based on the provided format.
|
|
51
|
+
:param obj: The object to format.
|
|
52
|
+
:param format_: The format as a string representation.
|
|
53
|
+
:param exclude_formats: A list of formats to exclude from the formatting process. If the provided format is in
|
|
54
|
+
this list, an invalid format exception will be raised.
|
|
55
|
+
"""
|
|
37
56
|
exclude_formats = exclude_formats or []
|
|
38
|
-
|
|
57
|
+
format_ = format_ or cls.full
|
|
39
58
|
invalid_format_exc = mlrun.errors.MLRunBadRequestError(
|
|
40
|
-
f"Provided format is not supported. format={
|
|
59
|
+
f"Provided format is not supported. format={format_}"
|
|
41
60
|
)
|
|
42
61
|
|
|
43
|
-
if
|
|
62
|
+
if format_ in exclude_formats:
|
|
44
63
|
raise invalid_format_exc
|
|
45
64
|
|
|
46
65
|
try:
|
|
47
|
-
format_method = cls.format_method(
|
|
66
|
+
format_method = cls.format_method(format_)
|
|
48
67
|
except KeyError:
|
|
49
68
|
raise invalid_format_exc
|
|
50
69
|
|
|
@@ -54,10 +73,26 @@ class ObjectFormat:
|
|
|
54
73
|
return format_method(obj)
|
|
55
74
|
|
|
56
75
|
@staticmethod
|
|
57
|
-
def filter_obj_method(_filter: list[
|
|
76
|
+
def filter_obj_method(_filter: list[str]) -> typing.Callable:
|
|
77
|
+
"""
|
|
78
|
+
Returns a method that filters the object based on the provided list of keys.
|
|
79
|
+
The keys should be in a dot-separated format, denoting the path within the dictionary to the desired key.
|
|
80
|
+
The object maintains its structure, with the filtered keys and their values, while all other keys are removed.
|
|
81
|
+
:param _filter: The list of keys to filter by.
|
|
82
|
+
Example:
|
|
83
|
+
[
|
|
84
|
+
"kind",
|
|
85
|
+
"metadata.name",
|
|
86
|
+
"spec.something.else",
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
:return: The filtering method.
|
|
90
|
+
"""
|
|
91
|
+
|
|
58
92
|
def _filter_method(obj: dict) -> dict:
|
|
59
93
|
formatted_obj = {}
|
|
60
|
-
for
|
|
94
|
+
for key in _filter:
|
|
95
|
+
key_list = key.split(".")
|
|
61
96
|
obj_recursive_iterator = obj
|
|
62
97
|
formatted_obj_recursive_iterator = formatted_obj
|
|
63
98
|
for idx, key in enumerate(key_list):
|
|
@@ -29,13 +29,18 @@ class FunctionFormat(ObjectFormat, mlrun.common.types.StrEnum):
|
|
|
29
29
|
FunctionFormat.full: None,
|
|
30
30
|
FunctionFormat.minimal: FunctionFormat.filter_obj_method(
|
|
31
31
|
[
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
32
|
+
"kind",
|
|
33
|
+
"metadata",
|
|
34
|
+
"status",
|
|
35
|
+
"spec.description",
|
|
36
|
+
"spec.command",
|
|
37
|
+
"spec.image",
|
|
38
|
+
"spec.default_handler",
|
|
39
|
+
"spec.default_class",
|
|
40
|
+
"spec.graph",
|
|
41
|
+
"spec.preemption_mode",
|
|
42
|
+
"spec.node_selector",
|
|
43
|
+
"spec.priority_class_name",
|
|
39
44
|
]
|
|
40
45
|
),
|
|
41
46
|
}[_format]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Copyright 2024 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
import mlrun.common.types
|
|
18
|
+
from mlrun.common.formatters.base import ObjectFormat
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class RunFormat(ObjectFormat, mlrun.common.types.StrEnum):
|
|
22
|
+
# No enrichment, data is pulled as-is from the database.
|
|
23
|
+
standard = "standard"
|
|
24
|
+
|
|
25
|
+
# Performs run enrichment, including the run's artifacts. Only available for the `get` run API.
|
|
26
|
+
full = "full"
|
mlrun/common/helpers.py
CHANGED
|
@@ -34,3 +34,14 @@ def parse_versioned_object_uri(
|
|
|
34
34
|
uri = uri[:loc]
|
|
35
35
|
|
|
36
36
|
return project, uri, tag, hash_key
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def generate_api_gateway_name(project: str, name: str) -> str:
|
|
40
|
+
"""
|
|
41
|
+
Generate a unique (within project) api gateway name
|
|
42
|
+
:param project: project name
|
|
43
|
+
:param name: api gateway name
|
|
44
|
+
|
|
45
|
+
:return: the resolved api gateway name
|
|
46
|
+
"""
|
|
47
|
+
return f"{project}-{name}" if project else name
|
mlrun/common/schemas/__init__.py
CHANGED
|
@@ -82,6 +82,7 @@ from .events import (
|
|
|
82
82
|
)
|
|
83
83
|
from .feature_store import (
|
|
84
84
|
EntitiesOutput,
|
|
85
|
+
EntitiesOutputV2,
|
|
85
86
|
Entity,
|
|
86
87
|
EntityListOutput,
|
|
87
88
|
EntityRecord,
|
|
@@ -90,7 +91,9 @@ from .feature_store import (
|
|
|
90
91
|
FeatureRecord,
|
|
91
92
|
FeatureSet,
|
|
92
93
|
FeatureSetDigestOutput,
|
|
94
|
+
FeatureSetDigestOutputV2,
|
|
93
95
|
FeatureSetDigestSpec,
|
|
96
|
+
FeatureSetDigestSpecV2,
|
|
94
97
|
FeatureSetIngestInput,
|
|
95
98
|
FeatureSetIngestOutput,
|
|
96
99
|
FeatureSetRecord,
|
|
@@ -98,6 +101,7 @@ from .feature_store import (
|
|
|
98
101
|
FeatureSetSpec,
|
|
99
102
|
FeatureSetsTagsOutput,
|
|
100
103
|
FeaturesOutput,
|
|
104
|
+
FeaturesOutputV2,
|
|
101
105
|
FeatureVector,
|
|
102
106
|
FeatureVectorRecord,
|
|
103
107
|
FeatureVectorsOutput,
|
mlrun/common/schemas/alert.py
CHANGED
|
@@ -39,8 +39,8 @@ class EventKind(StrEnum):
|
|
|
39
39
|
CONCEPT_DRIFT_SUSPECTED = "concept_drift_suspected"
|
|
40
40
|
MODEL_PERFORMANCE_DETECTED = "model_performance_detected"
|
|
41
41
|
MODEL_PERFORMANCE_SUSPECTED = "model_performance_suspected"
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
SYSTEM_PERFORMANCE_DETECTED = "system_performance_detected"
|
|
43
|
+
SYSTEM_PERFORMANCE_SUSPECTED = "system_performance_suspected"
|
|
44
44
|
MM_APP_ANOMALY_DETECTED = "mm_app_anomaly_detected"
|
|
45
45
|
MM_APP_ANOMALY_SUSPECTED = "mm_app_anomaly_suspected"
|
|
46
46
|
FAILED = "failed"
|
|
@@ -53,12 +53,8 @@ _event_kind_entity_map = {
|
|
|
53
53
|
EventKind.CONCEPT_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
|
|
54
54
|
EventKind.MODEL_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
|
|
55
55
|
EventKind.MODEL_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
|
|
56
|
-
EventKind.
|
|
57
|
-
|
|
58
|
-
],
|
|
59
|
-
EventKind.MODEL_SERVING_PERFORMANCE_SUSPECTED: [
|
|
60
|
-
EventEntityKind.MODEL_ENDPOINT_RESULT
|
|
61
|
-
],
|
|
56
|
+
EventKind.SYSTEM_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
|
|
57
|
+
EventKind.SYSTEM_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
|
|
62
58
|
EventKind.MM_APP_ANOMALY_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
|
|
63
59
|
EventKind.MM_APP_ANOMALY_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
|
|
64
60
|
EventKind.FAILED: [EventEntityKind.JOB],
|
|
@@ -104,7 +100,7 @@ class AlertCriteria(pydantic.BaseModel):
|
|
|
104
100
|
pydantic.Field(
|
|
105
101
|
description="Number of events to wait until notification is sent"
|
|
106
102
|
),
|
|
107
|
-
] =
|
|
103
|
+
] = 1
|
|
108
104
|
period: Annotated[
|
|
109
105
|
str,
|
|
110
106
|
pydantic.Field(
|
|
@@ -17,8 +17,10 @@ from typing import Optional
|
|
|
17
17
|
|
|
18
18
|
import pydantic
|
|
19
19
|
|
|
20
|
+
import mlrun.common.constants as mlrun_constants
|
|
20
21
|
import mlrun.common.types
|
|
21
22
|
from mlrun.common.constants import MLRUN_FUNCTIONS_ANNOTATION
|
|
23
|
+
from mlrun.common.helpers import generate_api_gateway_name
|
|
22
24
|
|
|
23
25
|
|
|
24
26
|
class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
|
|
@@ -100,7 +102,58 @@ class APIGateway(_APIGatewayBaseModel):
|
|
|
100
102
|
if upstream.nucliofunction.get("name")
|
|
101
103
|
]
|
|
102
104
|
|
|
103
|
-
def
|
|
105
|
+
def get_invoke_url(self):
|
|
106
|
+
return (
|
|
107
|
+
self.spec.host + self.spec.path
|
|
108
|
+
if self.spec.path and self.spec.host
|
|
109
|
+
else self.spec.host
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
def enrich_mlrun_names(self):
|
|
113
|
+
self._enrich_api_gateway_mlrun_name()
|
|
114
|
+
self._enrich_mlrun_function_names()
|
|
115
|
+
return self
|
|
116
|
+
|
|
117
|
+
def replace_nuclio_names_with_mlrun_names(self):
|
|
118
|
+
self._replace_nuclio_api_gateway_name_with_mlrun_name()
|
|
119
|
+
self._replace_nuclio_function_names_with_mlrun_names()
|
|
120
|
+
return self
|
|
121
|
+
|
|
122
|
+
def _replace_nuclio_function_names_with_mlrun_names(self):
|
|
123
|
+
# replace function names from nuclio names to mlrun names
|
|
124
|
+
# and adds mlrun function URI's to an api gateway annotations
|
|
125
|
+
# so when we then get api gateway entity from nuclio, we are able to get mlrun function names
|
|
126
|
+
mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
|
|
127
|
+
if mlrun_functions:
|
|
128
|
+
mlrun_function_uris = (
|
|
129
|
+
mlrun_functions.split("&")
|
|
130
|
+
if "&" in mlrun_functions
|
|
131
|
+
else [mlrun_functions]
|
|
132
|
+
)
|
|
133
|
+
if len(mlrun_function_uris) != len(self.spec.upstreams):
|
|
134
|
+
raise mlrun.errors.MLRunValueError(
|
|
135
|
+
"Error when translating nuclio names to mlrun names in api gateway:"
|
|
136
|
+
" number of functions doesn't match the mlrun functions in annotation"
|
|
137
|
+
)
|
|
138
|
+
for i in range(len(mlrun_function_uris)):
|
|
139
|
+
self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
|
|
140
|
+
return self
|
|
141
|
+
|
|
142
|
+
def _replace_nuclio_api_gateway_name_with_mlrun_name(self):
|
|
143
|
+
# replace api gateway name
|
|
144
|
+
# in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
|
|
145
|
+
# remove the project prefix from the name if it exists
|
|
146
|
+
project_name = self.metadata.labels.get(
|
|
147
|
+
mlrun_constants.MLRunInternalLabels.nuclio_project_name
|
|
148
|
+
)
|
|
149
|
+
if project_name and self.spec.name.startswith(f"{project_name}-"):
|
|
150
|
+
self.spec.name = self.spec.name[len(project_name) + 1 :]
|
|
151
|
+
self.metadata.name = self.spec.name
|
|
152
|
+
return self
|
|
153
|
+
|
|
154
|
+
def _enrich_mlrun_function_names(self):
|
|
155
|
+
# enrich mlrun names with nuclio prefixes
|
|
156
|
+
# and add mlrun function's URIs to Nuclio function annotations
|
|
104
157
|
upstream_with_nuclio_names = []
|
|
105
158
|
mlrun_function_uris = []
|
|
106
159
|
for upstream in self.spec.upstreams:
|
|
@@ -126,21 +179,16 @@ class APIGateway(_APIGatewayBaseModel):
|
|
|
126
179
|
)
|
|
127
180
|
return self
|
|
128
181
|
|
|
129
|
-
def
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
"Error when translating nuclio names to mlrun names in api gateway:"
|
|
140
|
-
" number of functions doesn't match the mlrun functions in annotation"
|
|
141
|
-
)
|
|
142
|
-
for i in range(len(mlrun_function_uris)):
|
|
143
|
-
self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
|
|
182
|
+
def _enrich_api_gateway_mlrun_name(self):
|
|
183
|
+
# replace api gateway name
|
|
184
|
+
# in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
|
|
185
|
+
# add the project prefix to the name
|
|
186
|
+
project_name = self.metadata.labels.get(
|
|
187
|
+
mlrun_constants.MLRunInternalLabels.nuclio_project_name
|
|
188
|
+
)
|
|
189
|
+
if project_name:
|
|
190
|
+
self.spec.name = generate_api_gateway_name(project_name, self.spec.name)
|
|
191
|
+
self.metadata.name = self.spec.name
|
|
144
192
|
return self
|
|
145
193
|
|
|
146
194
|
|
mlrun/common/schemas/artifact.py
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import typing
|
|
16
16
|
|
|
17
17
|
import pydantic
|
|
18
|
+
from deprecated import deprecated
|
|
18
19
|
|
|
19
20
|
import mlrun.common.types
|
|
20
21
|
|
|
@@ -58,6 +59,16 @@ class ArtifactIdentifier(pydantic.BaseModel):
|
|
|
58
59
|
# hash: typing.Optional[str]
|
|
59
60
|
|
|
60
61
|
|
|
62
|
+
@deprecated(
|
|
63
|
+
version="1.7.0",
|
|
64
|
+
reason="mlrun.common.schemas.ArtifactsFormat is deprecated and will be removed in 1.9.0. "
|
|
65
|
+
"Use mlrun.common.formatters.ArtifactFormat instead.",
|
|
66
|
+
category=FutureWarning,
|
|
67
|
+
)
|
|
68
|
+
class ArtifactsFormat(mlrun.common.types.StrEnum):
|
|
69
|
+
full = "full"
|
|
70
|
+
|
|
71
|
+
|
|
61
72
|
class ArtifactMetadata(pydantic.BaseModel):
|
|
62
73
|
key: str
|
|
63
74
|
project: str
|
|
@@ -120,10 +120,13 @@ class FeatureStorePartitionByField(mlrun.common.types.StrEnum):
|
|
|
120
120
|
|
|
121
121
|
class RunPartitionByField(mlrun.common.types.StrEnum):
|
|
122
122
|
name = "name" # Supported for runs objects
|
|
123
|
+
project_and_name = "project_and_name" # Supported for runs objects
|
|
123
124
|
|
|
124
125
|
def to_partition_by_db_field(self, db_cls):
|
|
125
126
|
if self.value == RunPartitionByField.name:
|
|
126
127
|
return db_cls.name
|
|
128
|
+
elif self.value == RunPartitionByField.project_and_name:
|
|
129
|
+
return db_cls.project, db_cls.name
|
|
127
130
|
else:
|
|
128
131
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
129
132
|
f"Unknown group by field: {self.value}"
|