mlrun 1.7.0rc20__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (92) hide show
  1. mlrun/__main__.py +10 -8
  2. mlrun/alerts/alert.py +55 -18
  3. mlrun/api/schemas/__init__.py +3 -3
  4. mlrun/artifacts/manager.py +26 -0
  5. mlrun/common/constants.py +3 -2
  6. mlrun/common/formatters/__init__.py +1 -0
  7. mlrun/common/formatters/artifact.py +26 -3
  8. mlrun/common/formatters/base.py +44 -9
  9. mlrun/common/formatters/function.py +12 -7
  10. mlrun/common/formatters/run.py +26 -0
  11. mlrun/common/helpers.py +11 -0
  12. mlrun/common/schemas/__init__.py +4 -0
  13. mlrun/common/schemas/alert.py +5 -9
  14. mlrun/common/schemas/api_gateway.py +64 -16
  15. mlrun/common/schemas/artifact.py +11 -0
  16. mlrun/common/schemas/constants.py +3 -0
  17. mlrun/common/schemas/feature_store.py +58 -28
  18. mlrun/common/schemas/model_monitoring/constants.py +21 -12
  19. mlrun/common/schemas/model_monitoring/model_endpoints.py +0 -12
  20. mlrun/common/schemas/pipeline.py +16 -0
  21. mlrun/common/schemas/project.py +17 -0
  22. mlrun/common/schemas/runs.py +17 -0
  23. mlrun/common/schemas/schedule.py +1 -1
  24. mlrun/common/types.py +6 -0
  25. mlrun/config.py +17 -25
  26. mlrun/datastore/azure_blob.py +2 -1
  27. mlrun/datastore/datastore.py +3 -3
  28. mlrun/datastore/google_cloud_storage.py +6 -2
  29. mlrun/datastore/snowflake_utils.py +3 -1
  30. mlrun/datastore/sources.py +26 -11
  31. mlrun/datastore/store_resources.py +2 -0
  32. mlrun/datastore/targets.py +68 -16
  33. mlrun/db/base.py +83 -2
  34. mlrun/db/httpdb.py +280 -63
  35. mlrun/db/nopdb.py +60 -3
  36. mlrun/errors.py +5 -3
  37. mlrun/execution.py +28 -13
  38. mlrun/feature_store/feature_vector.py +8 -0
  39. mlrun/feature_store/retrieval/spark_merger.py +13 -2
  40. mlrun/launcher/local.py +4 -0
  41. mlrun/launcher/remote.py +1 -0
  42. mlrun/model.py +32 -3
  43. mlrun/model_monitoring/api.py +7 -52
  44. mlrun/model_monitoring/applications/base.py +5 -7
  45. mlrun/model_monitoring/applications/histogram_data_drift.py +1 -1
  46. mlrun/model_monitoring/db/stores/__init__.py +37 -24
  47. mlrun/model_monitoring/db/stores/base/store.py +40 -1
  48. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +42 -87
  49. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +27 -35
  50. mlrun/model_monitoring/db/tsdb/__init__.py +15 -15
  51. mlrun/model_monitoring/db/tsdb/base.py +1 -14
  52. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +22 -18
  53. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +86 -56
  54. mlrun/model_monitoring/helpers.py +34 -9
  55. mlrun/model_monitoring/stream_processing.py +12 -11
  56. mlrun/model_monitoring/writer.py +11 -11
  57. mlrun/projects/operations.py +5 -0
  58. mlrun/projects/pipelines.py +35 -21
  59. mlrun/projects/project.py +216 -107
  60. mlrun/render.py +10 -5
  61. mlrun/run.py +15 -5
  62. mlrun/runtimes/__init__.py +2 -0
  63. mlrun/runtimes/base.py +17 -4
  64. mlrun/runtimes/daskjob.py +8 -1
  65. mlrun/runtimes/databricks_job/databricks_runtime.py +1 -0
  66. mlrun/runtimes/local.py +23 -4
  67. mlrun/runtimes/nuclio/application/application.py +0 -2
  68. mlrun/runtimes/nuclio/function.py +31 -2
  69. mlrun/runtimes/nuclio/serving.py +9 -6
  70. mlrun/runtimes/pod.py +5 -29
  71. mlrun/runtimes/remotesparkjob.py +8 -2
  72. mlrun/serving/__init__.py +8 -1
  73. mlrun/serving/routers.py +75 -59
  74. mlrun/serving/server.py +11 -0
  75. mlrun/serving/states.py +80 -8
  76. mlrun/serving/utils.py +19 -11
  77. mlrun/serving/v2_serving.py +66 -39
  78. mlrun/utils/helpers.py +91 -11
  79. mlrun/utils/logger.py +36 -2
  80. mlrun/utils/notifications/notification/base.py +43 -7
  81. mlrun/utils/notifications/notification/git.py +21 -0
  82. mlrun/utils/notifications/notification/slack.py +9 -14
  83. mlrun/utils/notifications/notification/webhook.py +41 -1
  84. mlrun/utils/notifications/notification_pusher.py +3 -9
  85. mlrun/utils/regex.py +9 -0
  86. mlrun/utils/version/version.json +2 -2
  87. {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/METADATA +16 -9
  88. {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/RECORD +92 -91
  89. {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/WHEEL +1 -1
  90. {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/LICENSE +0 -0
  91. {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/entry_points.txt +0 -0
  92. {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/top_level.txt +0 -0
mlrun/__main__.py CHANGED
@@ -50,12 +50,12 @@ from .run import (
50
50
  from .runtimes import RemoteRuntime, RunError, RuntimeKinds, ServingRuntime
51
51
  from .secrets import SecretsStore
52
52
  from .utils import (
53
+ RunKeys,
53
54
  dict_to_yaml,
54
55
  get_in,
55
56
  is_relative_path,
56
57
  list2dict,
57
58
  logger,
58
- run_keys,
59
59
  update_in,
60
60
  )
61
61
  from .utils.version import Version
@@ -102,7 +102,9 @@ def main():
102
102
  )
103
103
  @click.option("--uid", help="unique run ID")
104
104
  @click.option("--name", help="run name")
105
- @click.option("--workflow", help="workflow name/id")
105
+ @click.option(
106
+ "--workflow", help="sets the run labels to match the given workflow name/id"
107
+ )
106
108
  @click.option("--project", help="project name/id")
107
109
  @click.option("--db", default="", help="save run results to path or DB url")
108
110
  @click.option(
@@ -378,15 +380,15 @@ def run(
378
380
  set_item(runobj.spec.hyper_param_options, hyper_param_strategy, "strategy")
379
381
  set_item(runobj.spec.hyper_param_options, selector, "selector")
380
382
 
381
- set_item(runobj.spec, inputs, run_keys.inputs, list2dict(inputs))
383
+ set_item(runobj.spec, inputs, RunKeys.inputs, list2dict(inputs))
382
384
  set_item(
383
- runobj.spec, returns, run_keys.returns, [py_eval(value) for value in returns]
385
+ runobj.spec, returns, RunKeys.returns, [py_eval(value) for value in returns]
384
386
  )
385
- set_item(runobj.spec, in_path, run_keys.input_path)
386
- set_item(runobj.spec, out_path, run_keys.output_path)
387
- set_item(runobj.spec, outputs, run_keys.outputs, list(outputs))
387
+ set_item(runobj.spec, in_path, RunKeys.input_path)
388
+ set_item(runobj.spec, out_path, RunKeys.output_path)
389
+ set_item(runobj.spec, outputs, RunKeys.outputs, list(outputs))
388
390
  set_item(
389
- runobj.spec, secrets, run_keys.secrets, line2keylist(secrets, "kind", "source")
391
+ runobj.spec, secrets, RunKeys.secrets, line2keylist(secrets, "kind", "source")
390
392
  )
391
393
  set_item(runobj.spec, verbose, "verbose")
392
394
  set_item(runobj.spec, scrape_metrics, "scrape_metrics")
mlrun/alerts/alert.py CHANGED
@@ -26,10 +26,15 @@ class AlertConfig(ModelObj):
26
26
  "description",
27
27
  "summary",
28
28
  "severity",
29
- "criteria",
30
29
  "reset_policy",
31
30
  "state",
32
31
  ]
32
+ _fields_to_serialize = ModelObj._fields_to_serialize + [
33
+ "entities",
34
+ "notifications",
35
+ "trigger",
36
+ "criteria",
37
+ ]
33
38
 
34
39
  def __init__(
35
40
  self,
@@ -71,24 +76,52 @@ class AlertConfig(ModelObj):
71
76
  if not self.project or not self.name:
72
77
  raise mlrun.errors.MLRunBadRequestError("Project and name must be provided")
73
78
 
79
+ def _serialize_field(
80
+ self, struct: dict, field_name: str = None, strip: bool = False
81
+ ):
82
+ if field_name == "entities":
83
+ if self.entities:
84
+ return (
85
+ self.entities.dict()
86
+ if not isinstance(self.entities, dict)
87
+ else self.entities
88
+ )
89
+ return None
90
+ if field_name == "notifications":
91
+ if self.notifications:
92
+ return [
93
+ notification_data.dict()
94
+ if not isinstance(notification_data, dict)
95
+ else notification_data
96
+ for notification_data in self.notifications
97
+ ]
98
+ return None
99
+ if field_name == "trigger":
100
+ if self.trigger:
101
+ return (
102
+ self.trigger.dict()
103
+ if not isinstance(self.trigger, dict)
104
+ else self.trigger
105
+ )
106
+ return None
107
+ if field_name == "criteria":
108
+ if self.criteria:
109
+ return (
110
+ self.criteria.dict()
111
+ if not isinstance(self.criteria, dict)
112
+ else self.criteria
113
+ )
114
+ return None
115
+ return super()._serialize_field(struct, field_name, strip)
116
+
74
117
  def to_dict(self, fields: list = None, exclude: list = None, strip: bool = False):
75
- data = super().to_dict(self._dict_fields)
76
-
77
- data["entities"] = (
78
- self.entities.dict()
79
- if not isinstance(self.entities, dict)
80
- else self.entities
81
- )
82
- data["notifications"] = [
83
- notification_data.dict()
84
- if not isinstance(notification_data, dict)
85
- else notification_data
86
- for notification_data in self.notifications
87
- ]
88
- data["trigger"] = (
89
- self.trigger.dict() if not isinstance(self.trigger, dict) else self.trigger
90
- )
91
- return data
118
+ if self.entities is None:
119
+ raise mlrun.errors.MLRunBadRequestError("Alert entity field is missing")
120
+ if not self.notifications:
121
+ raise mlrun.errors.MLRunBadRequestError(
122
+ "Alert must have at least one notification"
123
+ )
124
+ return super().to_dict(self._dict_fields)
92
125
 
93
126
  @classmethod
94
127
  def from_dict(cls, struct=None, fields=None, deprecated_fields: dict = None):
@@ -112,6 +145,10 @@ class AlertConfig(ModelObj):
112
145
  trigger_obj = alert_objects.AlertTrigger.parse_obj(trigger_data)
113
146
  new_obj.trigger = trigger_obj
114
147
 
148
+ criteria_data = struct.get("criteria")
149
+ if criteria_data:
150
+ criteria_obj = alert_objects.AlertCriteria.parse_obj(criteria_data)
151
+ new_obj.criteria = criteria_obj
115
152
  return new_obj
116
153
 
117
154
  def with_notifications(self, notifications: list[alert_objects.AlertNotification]):
@@ -97,7 +97,7 @@ sys.modules["mlrun.api.schemas.tag"] = old_tag
97
97
  # and return the new schema. This is done for backwards compatibility with mlrun.api.schemas.
98
98
  ArtifactCategories = DeprecationHelper(mlrun.common.schemas.ArtifactCategories)
99
99
  ArtifactIdentifier = DeprecationHelper(mlrun.common.schemas.ArtifactIdentifier)
100
- ArtifactsFormat = DeprecationHelper(mlrun.common.formatters.ArtifactsFormat)
100
+ ArtifactsFormat = DeprecationHelper(mlrun.common.formatters.ArtifactFormat)
101
101
  AuthInfo = DeprecationHelper(mlrun.common.schemas.AuthInfo)
102
102
  AuthorizationAction = DeprecationHelper(mlrun.common.schemas.AuthorizationAction)
103
103
  AuthorizationResourceTypes = DeprecationHelper(
@@ -222,7 +222,7 @@ ObjectKind = DeprecationHelper(mlrun.common.schemas.ObjectKind)
222
222
  ObjectMetadata = DeprecationHelper(mlrun.common.schemas.ObjectMetadata)
223
223
  ObjectSpec = DeprecationHelper(mlrun.common.schemas.ObjectSpec)
224
224
  ObjectStatus = DeprecationHelper(mlrun.common.schemas.ObjectStatus)
225
- PipelinesFormat = DeprecationHelper(mlrun.common.formatters.PipelinesFormat)
225
+ PipelinesFormat = DeprecationHelper(mlrun.common.formatters.PipelineFormat)
226
226
  PipelinesOutput = DeprecationHelper(mlrun.common.schemas.PipelinesOutput)
227
227
  PipelinesPagination = DeprecationHelper(mlrun.common.schemas.PipelinesPagination)
228
228
  IguazioProject = DeprecationHelper(mlrun.common.schemas.IguazioProject)
@@ -230,7 +230,7 @@ Project = DeprecationHelper(mlrun.common.schemas.Project)
230
230
  ProjectDesiredState = DeprecationHelper(mlrun.common.schemas.ProjectDesiredState)
231
231
  ProjectMetadata = DeprecationHelper(mlrun.common.schemas.ProjectMetadata)
232
232
  ProjectOwner = DeprecationHelper(mlrun.common.schemas.ProjectOwner)
233
- ProjectsFormat = DeprecationHelper(mlrun.common.formatters.ProjectsFormat)
233
+ ProjectsFormat = DeprecationHelper(mlrun.common.formatters.ProjectFormat)
234
234
  ProjectsOutput = DeprecationHelper(mlrun.common.schemas.ProjectsOutput)
235
235
  ProjectSpec = DeprecationHelper(mlrun.common.schemas.ProjectSpec)
236
236
  ProjectState = DeprecationHelper(mlrun.common.schemas.ProjectState)
@@ -12,12 +12,14 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  import pathlib
15
+ import re
15
16
  import typing
16
17
  from os.path import exists, isdir
17
18
  from urllib.parse import urlparse
18
19
 
19
20
  import mlrun.common.schemas.artifact
20
21
  import mlrun.config
22
+ import mlrun.utils.regex
21
23
  from mlrun.utils.helpers import (
22
24
  get_local_file_schema,
23
25
  template_artifact_path,
@@ -76,9 +78,33 @@ class ArtifactProducer:
76
78
  def uid(self):
77
79
  return None
78
80
 
81
+ @staticmethod
82
+ def parse_uri(uri: str) -> tuple[str, str, str]:
83
+ """Parse artifact producer's uri
84
+
85
+ :param uri: artifact producer's uri in the format <project>/<uid>[-<iteration>]
86
+ :returns: tuple of project, uid, iteration
87
+ """
88
+ uri_pattern = mlrun.utils.regex.artifact_producer_uri_pattern
89
+ match = re.match(uri_pattern, uri)
90
+ if not match:
91
+ return "", "", ""
92
+ group_dict = match.groupdict()
93
+
94
+ return (
95
+ group_dict["project"] or "",
96
+ group_dict["uid"] or "",
97
+ group_dict["iteration"] or "",
98
+ )
99
+
79
100
 
80
101
  def dict_to_artifact(struct: dict) -> Artifact:
81
102
  kind = struct.get("kind", "")
103
+
104
+ # TODO: remove this in 1.8.0
105
+ if mlrun.utils.is_legacy_artifact(struct):
106
+ return mlrun.artifacts.base.convert_legacy_artifact_to_new_format(struct)
107
+
82
108
  artifact_class = artifact_types[kind]
83
109
  return artifact_class.from_dict(struct)
84
110
 
mlrun/common/constants.py CHANGED
@@ -42,6 +42,7 @@ class MLRunInternalLabels:
42
42
 
43
43
  ### nuclio
44
44
  nuclio_project_name = f"{NUCLIO_LABEL_PREFIX}project-name"
45
+ nuclio_function_name = f"{NUCLIO_LABEL_PREFIX}function-name"
45
46
  nuclio_class = f"{NUCLIO_LABEL_PREFIX}class"
46
47
 
47
48
  ### mlrun
@@ -63,12 +64,12 @@ class MLRunInternalLabels:
63
64
  username = f"{MLRUN_LABEL_PREFIX}username"
64
65
  username_domain = f"{MLRUN_LABEL_PREFIX}username_domain"
65
66
  task_name = f"{MLRUN_LABEL_PREFIX}task-name"
67
+ resource_name = f"{MLRUN_LABEL_PREFIX}resource_name"
68
+ created = f"{MLRUN_LABEL_PREFIX}created"
66
69
  host = "host"
67
70
  job_type = "job-type"
68
71
  kind = "kind"
69
72
  component = "component"
70
- resource_name = "resource_name"
71
- created = "mlrun-created"
72
73
 
73
74
  owner = "owner"
74
75
  v3io_user = "v3io_user"
@@ -17,3 +17,4 @@ from .artifact import ArtifactFormat # noqa
17
17
  from .function import FunctionFormat # noqa
18
18
  from .pipeline import PipelineFormat # noqa
19
19
  from .project import ProjectFormat # noqa
20
+ from .run import RunFormat # noqa
@@ -13,9 +13,32 @@
13
13
  # limitations under the License.
14
14
  #
15
15
 
16
+ import typing
17
+
16
18
  import mlrun.common.types
17
19
 
20
+ from .base import ObjectFormat
21
+
22
+
23
+ class ArtifactFormat(ObjectFormat, mlrun.common.types.StrEnum):
24
+ minimal = "minimal"
18
25
 
19
- # TODO: add a format that returns a minimal response with ObjectFormat
20
- class ArtifactFormat(mlrun.common.types.StrEnum):
21
- full = "full"
26
+ @staticmethod
27
+ def format_method(_format: str) -> typing.Optional[typing.Callable]:
28
+ return {
29
+ ArtifactFormat.full: None,
30
+ ArtifactFormat.minimal: ArtifactFormat.filter_obj_method(
31
+ [
32
+ "kind",
33
+ "metadata",
34
+ "status",
35
+ "project",
36
+ "spec.producer",
37
+ "spec.db_key",
38
+ "spec.size",
39
+ "spec.framework",
40
+ "spec.metrics",
41
+ "spec.target_path",
42
+ ]
43
+ ),
44
+ }[_format]
@@ -19,32 +19,51 @@ import mlrun.errors
19
19
 
20
20
 
21
21
  class ObjectFormat:
22
+ """
23
+ MLRun object formatter. Any class that inherits from this class should implement the `format_method` method
24
+ to specify the formatting method for each format.
25
+ A `filter_obj_method` utility method is provided to filter the object based on a list of keys.
26
+ """
27
+
22
28
  full = "full"
23
29
 
24
30
  @staticmethod
25
- def format_method(_format: str) -> typing.Optional[typing.Callable]:
31
+ def format_method(format_: str) -> typing.Optional[typing.Callable]:
32
+ """
33
+ Get the formatting method for the provided format.
34
+ A `None` value signifies a pass-through formatting method (no formatting).
35
+ :param format_: The format as a string representation.
36
+ :return: The formatting method.
37
+ """
26
38
  return {
27
39
  ObjectFormat.full: None,
28
- }[_format]
40
+ }[format_]
29
41
 
30
42
  @classmethod
31
43
  def format_obj(
32
44
  cls,
33
45
  obj: typing.Any,
34
- _format: str,
46
+ format_: str,
35
47
  exclude_formats: typing.Optional[list[str]] = None,
36
48
  ) -> typing.Any:
49
+ """
50
+ Format the provided object based on the provided format.
51
+ :param obj: The object to format.
52
+ :param format_: The format as a string representation.
53
+ :param exclude_formats: A list of formats to exclude from the formatting process. If the provided format is in
54
+ this list, an invalid format exception will be raised.
55
+ """
37
56
  exclude_formats = exclude_formats or []
38
- _format = _format or cls.full
57
+ format_ = format_ or cls.full
39
58
  invalid_format_exc = mlrun.errors.MLRunBadRequestError(
40
- f"Provided format is not supported. format={_format}"
59
+ f"Provided format is not supported. format={format_}"
41
60
  )
42
61
 
43
- if _format in exclude_formats:
62
+ if format_ in exclude_formats:
44
63
  raise invalid_format_exc
45
64
 
46
65
  try:
47
- format_method = cls.format_method(_format)
66
+ format_method = cls.format_method(format_)
48
67
  except KeyError:
49
68
  raise invalid_format_exc
50
69
 
@@ -54,10 +73,26 @@ class ObjectFormat:
54
73
  return format_method(obj)
55
74
 
56
75
  @staticmethod
57
- def filter_obj_method(_filter: list[list[str]]) -> typing.Callable:
76
+ def filter_obj_method(_filter: list[str]) -> typing.Callable:
77
+ """
78
+ Returns a method that filters the object based on the provided list of keys.
79
+ The keys should be in a dot-separated format, denoting the path within the dictionary to the desired key.
80
+ The object maintains its structure, with the filtered keys and their values, while all other keys are removed.
81
+ :param _filter: The list of keys to filter by.
82
+ Example:
83
+ [
84
+ "kind",
85
+ "metadata.name",
86
+ "spec.something.else",
87
+ ]
88
+
89
+ :return: The filtering method.
90
+ """
91
+
58
92
  def _filter_method(obj: dict) -> dict:
59
93
  formatted_obj = {}
60
- for key_list in _filter:
94
+ for key in _filter:
95
+ key_list = key.split(".")
61
96
  obj_recursive_iterator = obj
62
97
  formatted_obj_recursive_iterator = formatted_obj
63
98
  for idx, key in enumerate(key_list):
@@ -29,13 +29,18 @@ class FunctionFormat(ObjectFormat, mlrun.common.types.StrEnum):
29
29
  FunctionFormat.full: None,
30
30
  FunctionFormat.minimal: FunctionFormat.filter_obj_method(
31
31
  [
32
- ["kind"],
33
- ["metadata"],
34
- ["status"],
35
- ["spec", "description"],
36
- ["spec", "image"],
37
- ["spec", "default_handler"],
38
- ["spec", "entry_points"],
32
+ "kind",
33
+ "metadata",
34
+ "status",
35
+ "spec.description",
36
+ "spec.command",
37
+ "spec.image",
38
+ "spec.default_handler",
39
+ "spec.default_class",
40
+ "spec.graph",
41
+ "spec.preemption_mode",
42
+ "spec.node_selector",
43
+ "spec.priority_class_name",
39
44
  ]
40
45
  ),
41
46
  }[_format]
@@ -0,0 +1,26 @@
1
+ # Copyright 2024 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+
17
+ import mlrun.common.types
18
+ from mlrun.common.formatters.base import ObjectFormat
19
+
20
+
21
+ class RunFormat(ObjectFormat, mlrun.common.types.StrEnum):
22
+ # No enrichment, data is pulled as-is from the database.
23
+ standard = "standard"
24
+
25
+ # Performs run enrichment, including the run's artifacts. Only available for the `get` run API.
26
+ full = "full"
mlrun/common/helpers.py CHANGED
@@ -34,3 +34,14 @@ def parse_versioned_object_uri(
34
34
  uri = uri[:loc]
35
35
 
36
36
  return project, uri, tag, hash_key
37
+
38
+
39
+ def generate_api_gateway_name(project: str, name: str) -> str:
40
+ """
41
+ Generate a unique (within project) api gateway name
42
+ :param project: project name
43
+ :param name: api gateway name
44
+
45
+ :return: the resolved api gateway name
46
+ """
47
+ return f"{project}-{name}" if project else name
@@ -82,6 +82,7 @@ from .events import (
82
82
  )
83
83
  from .feature_store import (
84
84
  EntitiesOutput,
85
+ EntitiesOutputV2,
85
86
  Entity,
86
87
  EntityListOutput,
87
88
  EntityRecord,
@@ -90,7 +91,9 @@ from .feature_store import (
90
91
  FeatureRecord,
91
92
  FeatureSet,
92
93
  FeatureSetDigestOutput,
94
+ FeatureSetDigestOutputV2,
93
95
  FeatureSetDigestSpec,
96
+ FeatureSetDigestSpecV2,
94
97
  FeatureSetIngestInput,
95
98
  FeatureSetIngestOutput,
96
99
  FeatureSetRecord,
@@ -98,6 +101,7 @@ from .feature_store import (
98
101
  FeatureSetSpec,
99
102
  FeatureSetsTagsOutput,
100
103
  FeaturesOutput,
104
+ FeaturesOutputV2,
101
105
  FeatureVector,
102
106
  FeatureVectorRecord,
103
107
  FeatureVectorsOutput,
@@ -39,8 +39,8 @@ class EventKind(StrEnum):
39
39
  CONCEPT_DRIFT_SUSPECTED = "concept_drift_suspected"
40
40
  MODEL_PERFORMANCE_DETECTED = "model_performance_detected"
41
41
  MODEL_PERFORMANCE_SUSPECTED = "model_performance_suspected"
42
- MODEL_SERVING_PERFORMANCE_DETECTED = "model_serving_performance_detected"
43
- MODEL_SERVING_PERFORMANCE_SUSPECTED = "model_serving_performance_suspected"
42
+ SYSTEM_PERFORMANCE_DETECTED = "system_performance_detected"
43
+ SYSTEM_PERFORMANCE_SUSPECTED = "system_performance_suspected"
44
44
  MM_APP_ANOMALY_DETECTED = "mm_app_anomaly_detected"
45
45
  MM_APP_ANOMALY_SUSPECTED = "mm_app_anomaly_suspected"
46
46
  FAILED = "failed"
@@ -53,12 +53,8 @@ _event_kind_entity_map = {
53
53
  EventKind.CONCEPT_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
54
54
  EventKind.MODEL_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
55
55
  EventKind.MODEL_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
56
- EventKind.MODEL_SERVING_PERFORMANCE_DETECTED: [
57
- EventEntityKind.MODEL_ENDPOINT_RESULT
58
- ],
59
- EventKind.MODEL_SERVING_PERFORMANCE_SUSPECTED: [
60
- EventEntityKind.MODEL_ENDPOINT_RESULT
61
- ],
56
+ EventKind.SYSTEM_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
57
+ EventKind.SYSTEM_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
62
58
  EventKind.MM_APP_ANOMALY_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
63
59
  EventKind.MM_APP_ANOMALY_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
64
60
  EventKind.FAILED: [EventEntityKind.JOB],
@@ -104,7 +100,7 @@ class AlertCriteria(pydantic.BaseModel):
104
100
  pydantic.Field(
105
101
  description="Number of events to wait until notification is sent"
106
102
  ),
107
- ] = 0
103
+ ] = 1
108
104
  period: Annotated[
109
105
  str,
110
106
  pydantic.Field(
@@ -17,8 +17,10 @@ from typing import Optional
17
17
 
18
18
  import pydantic
19
19
 
20
+ import mlrun.common.constants as mlrun_constants
20
21
  import mlrun.common.types
21
22
  from mlrun.common.constants import MLRUN_FUNCTIONS_ANNOTATION
23
+ from mlrun.common.helpers import generate_api_gateway_name
22
24
 
23
25
 
24
26
  class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
@@ -100,7 +102,58 @@ class APIGateway(_APIGatewayBaseModel):
100
102
  if upstream.nucliofunction.get("name")
101
103
  ]
102
104
 
103
- def enrich_mlrun_function_names(self):
105
+ def get_invoke_url(self):
106
+ return (
107
+ self.spec.host + self.spec.path
108
+ if self.spec.path and self.spec.host
109
+ else self.spec.host
110
+ )
111
+
112
+ def enrich_mlrun_names(self):
113
+ self._enrich_api_gateway_mlrun_name()
114
+ self._enrich_mlrun_function_names()
115
+ return self
116
+
117
+ def replace_nuclio_names_with_mlrun_names(self):
118
+ self._replace_nuclio_api_gateway_name_with_mlrun_name()
119
+ self._replace_nuclio_function_names_with_mlrun_names()
120
+ return self
121
+
122
+ def _replace_nuclio_function_names_with_mlrun_names(self):
123
+ # replace function names from nuclio names to mlrun names
124
+ # and adds mlrun function URI's to an api gateway annotations
125
+ # so when we then get api gateway entity from nuclio, we are able to get mlrun function names
126
+ mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
127
+ if mlrun_functions:
128
+ mlrun_function_uris = (
129
+ mlrun_functions.split("&")
130
+ if "&" in mlrun_functions
131
+ else [mlrun_functions]
132
+ )
133
+ if len(mlrun_function_uris) != len(self.spec.upstreams):
134
+ raise mlrun.errors.MLRunValueError(
135
+ "Error when translating nuclio names to mlrun names in api gateway:"
136
+ " number of functions doesn't match the mlrun functions in annotation"
137
+ )
138
+ for i in range(len(mlrun_function_uris)):
139
+ self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
140
+ return self
141
+
142
+ def _replace_nuclio_api_gateway_name_with_mlrun_name(self):
143
+ # replace api gateway name
144
+ # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
145
+ # remove the project prefix from the name if it exists
146
+ project_name = self.metadata.labels.get(
147
+ mlrun_constants.MLRunInternalLabels.nuclio_project_name
148
+ )
149
+ if project_name and self.spec.name.startswith(f"{project_name}-"):
150
+ self.spec.name = self.spec.name[len(project_name) + 1 :]
151
+ self.metadata.name = self.spec.name
152
+ return self
153
+
154
+ def _enrich_mlrun_function_names(self):
155
+ # enrich mlrun names with nuclio prefixes
156
+ # and add mlrun function's URIs to Nuclio function annotations
104
157
  upstream_with_nuclio_names = []
105
158
  mlrun_function_uris = []
106
159
  for upstream in self.spec.upstreams:
@@ -126,21 +179,16 @@ class APIGateway(_APIGatewayBaseModel):
126
179
  )
127
180
  return self
128
181
 
129
- def replace_nuclio_names_with_mlrun_uri(self):
130
- mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
131
- if mlrun_functions:
132
- mlrun_function_uris = (
133
- mlrun_functions.split("&")
134
- if "&" in mlrun_functions
135
- else [mlrun_functions]
136
- )
137
- if len(mlrun_function_uris) != len(self.spec.upstreams):
138
- raise mlrun.errors.MLRunValueError(
139
- "Error when translating nuclio names to mlrun names in api gateway:"
140
- " number of functions doesn't match the mlrun functions in annotation"
141
- )
142
- for i in range(len(mlrun_function_uris)):
143
- self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
182
+ def _enrich_api_gateway_mlrun_name(self):
183
+ # replace api gateway name
184
+ # in Nuclio, api gateways are named as `<project>-<mlrun-api-gateway-name>`
185
+ # add the project prefix to the name
186
+ project_name = self.metadata.labels.get(
187
+ mlrun_constants.MLRunInternalLabels.nuclio_project_name
188
+ )
189
+ if project_name:
190
+ self.spec.name = generate_api_gateway_name(project_name, self.spec.name)
191
+ self.metadata.name = self.spec.name
144
192
  return self
145
193
 
146
194
 
@@ -15,6 +15,7 @@
15
15
  import typing
16
16
 
17
17
  import pydantic
18
+ from deprecated import deprecated
18
19
 
19
20
  import mlrun.common.types
20
21
 
@@ -58,6 +59,16 @@ class ArtifactIdentifier(pydantic.BaseModel):
58
59
  # hash: typing.Optional[str]
59
60
 
60
61
 
62
+ @deprecated(
63
+ version="1.7.0",
64
+ reason="mlrun.common.schemas.ArtifactsFormat is deprecated and will be removed in 1.9.0. "
65
+ "Use mlrun.common.formatters.ArtifactFormat instead.",
66
+ category=FutureWarning,
67
+ )
68
+ class ArtifactsFormat(mlrun.common.types.StrEnum):
69
+ full = "full"
70
+
71
+
61
72
  class ArtifactMetadata(pydantic.BaseModel):
62
73
  key: str
63
74
  project: str
@@ -120,10 +120,13 @@ class FeatureStorePartitionByField(mlrun.common.types.StrEnum):
120
120
 
121
121
  class RunPartitionByField(mlrun.common.types.StrEnum):
122
122
  name = "name" # Supported for runs objects
123
+ project_and_name = "project_and_name" # Supported for runs objects
123
124
 
124
125
  def to_partition_by_db_field(self, db_cls):
125
126
  if self.value == RunPartitionByField.name:
126
127
  return db_cls.name
128
+ elif self.value == RunPartitionByField.project_and_name:
129
+ return db_cls.project, db_cls.name
127
130
  else:
128
131
  raise mlrun.errors.MLRunInvalidArgumentError(
129
132
  f"Unknown group by field: {self.value}"