mlrun 1.7.0rc14__py3-none-any.whl → 1.7.0rc21__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (152) hide show
  1. mlrun/__init__.py +10 -1
  2. mlrun/__main__.py +23 -111
  3. mlrun/alerts/__init__.py +15 -0
  4. mlrun/alerts/alert.py +144 -0
  5. mlrun/api/schemas/__init__.py +4 -3
  6. mlrun/artifacts/__init__.py +8 -3
  7. mlrun/artifacts/base.py +36 -253
  8. mlrun/artifacts/dataset.py +9 -190
  9. mlrun/artifacts/manager.py +46 -42
  10. mlrun/artifacts/model.py +9 -141
  11. mlrun/artifacts/plots.py +14 -375
  12. mlrun/common/constants.py +65 -3
  13. mlrun/common/formatters/__init__.py +19 -0
  14. mlrun/{runtimes/mpijob/v1alpha1.py → common/formatters/artifact.py} +6 -14
  15. mlrun/common/formatters/base.py +113 -0
  16. mlrun/common/formatters/function.py +46 -0
  17. mlrun/common/formatters/pipeline.py +53 -0
  18. mlrun/common/formatters/project.py +51 -0
  19. mlrun/{runtimes → common/runtimes}/constants.py +32 -4
  20. mlrun/common/schemas/__init__.py +10 -5
  21. mlrun/common/schemas/alert.py +92 -11
  22. mlrun/common/schemas/api_gateway.py +56 -0
  23. mlrun/common/schemas/artifact.py +15 -5
  24. mlrun/common/schemas/auth.py +2 -0
  25. mlrun/common/schemas/client_spec.py +1 -0
  26. mlrun/common/schemas/frontend_spec.py +1 -0
  27. mlrun/common/schemas/function.py +4 -0
  28. mlrun/common/schemas/model_monitoring/__init__.py +15 -3
  29. mlrun/common/schemas/model_monitoring/constants.py +58 -7
  30. mlrun/common/schemas/model_monitoring/grafana.py +9 -5
  31. mlrun/common/schemas/model_monitoring/model_endpoints.py +86 -2
  32. mlrun/common/schemas/pipeline.py +0 -9
  33. mlrun/common/schemas/project.py +5 -11
  34. mlrun/common/types.py +1 -0
  35. mlrun/config.py +27 -9
  36. mlrun/data_types/to_pandas.py +9 -9
  37. mlrun/datastore/base.py +41 -9
  38. mlrun/datastore/datastore.py +6 -2
  39. mlrun/datastore/datastore_profile.py +56 -4
  40. mlrun/datastore/inmem.py +2 -2
  41. mlrun/datastore/redis.py +2 -2
  42. mlrun/datastore/s3.py +5 -0
  43. mlrun/datastore/sources.py +147 -7
  44. mlrun/datastore/store_resources.py +7 -7
  45. mlrun/datastore/targets.py +110 -42
  46. mlrun/datastore/utils.py +42 -0
  47. mlrun/db/base.py +54 -10
  48. mlrun/db/httpdb.py +282 -79
  49. mlrun/db/nopdb.py +52 -10
  50. mlrun/errors.py +11 -0
  51. mlrun/execution.py +24 -9
  52. mlrun/feature_store/__init__.py +0 -2
  53. mlrun/feature_store/api.py +12 -47
  54. mlrun/feature_store/feature_set.py +9 -0
  55. mlrun/feature_store/feature_vector.py +8 -0
  56. mlrun/feature_store/ingestion.py +7 -6
  57. mlrun/feature_store/retrieval/base.py +9 -4
  58. mlrun/feature_store/retrieval/conversion.py +9 -9
  59. mlrun/feature_store/retrieval/dask_merger.py +2 -0
  60. mlrun/feature_store/retrieval/job.py +9 -3
  61. mlrun/feature_store/retrieval/local_merger.py +2 -0
  62. mlrun/feature_store/retrieval/spark_merger.py +16 -0
  63. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +7 -12
  64. mlrun/frameworks/parallel_coordinates.py +2 -1
  65. mlrun/frameworks/tf_keras/__init__.py +4 -1
  66. mlrun/k8s_utils.py +10 -11
  67. mlrun/launcher/base.py +4 -3
  68. mlrun/launcher/client.py +5 -3
  69. mlrun/launcher/local.py +8 -2
  70. mlrun/launcher/remote.py +8 -2
  71. mlrun/lists.py +6 -2
  72. mlrun/model.py +45 -21
  73. mlrun/model_monitoring/__init__.py +1 -1
  74. mlrun/model_monitoring/api.py +41 -18
  75. mlrun/model_monitoring/application.py +5 -305
  76. mlrun/model_monitoring/applications/__init__.py +11 -0
  77. mlrun/model_monitoring/applications/_application_steps.py +157 -0
  78. mlrun/model_monitoring/applications/base.py +280 -0
  79. mlrun/model_monitoring/applications/context.py +214 -0
  80. mlrun/model_monitoring/applications/evidently_base.py +211 -0
  81. mlrun/model_monitoring/applications/histogram_data_drift.py +132 -91
  82. mlrun/model_monitoring/applications/results.py +99 -0
  83. mlrun/model_monitoring/controller.py +3 -1
  84. mlrun/model_monitoring/db/__init__.py +2 -0
  85. mlrun/model_monitoring/db/stores/__init__.py +0 -2
  86. mlrun/model_monitoring/db/stores/base/store.py +22 -37
  87. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +43 -21
  88. mlrun/model_monitoring/db/stores/sqldb/models/base.py +39 -8
  89. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +27 -7
  90. mlrun/model_monitoring/db/stores/sqldb/models/sqlite.py +5 -0
  91. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +246 -224
  92. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +232 -216
  93. mlrun/model_monitoring/db/tsdb/__init__.py +100 -0
  94. mlrun/model_monitoring/db/tsdb/base.py +329 -0
  95. mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
  96. mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
  97. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +240 -0
  98. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +45 -0
  99. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +397 -0
  100. mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
  101. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +117 -0
  102. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +636 -0
  103. mlrun/model_monitoring/evidently_application.py +6 -118
  104. mlrun/model_monitoring/helpers.py +46 -1
  105. mlrun/model_monitoring/model_endpoint.py +3 -2
  106. mlrun/model_monitoring/stream_processing.py +57 -216
  107. mlrun/model_monitoring/writer.py +134 -124
  108. mlrun/package/utils/_formatter.py +2 -2
  109. mlrun/platforms/__init__.py +10 -9
  110. mlrun/platforms/iguazio.py +21 -202
  111. mlrun/projects/operations.py +19 -12
  112. mlrun/projects/pipelines.py +79 -102
  113. mlrun/projects/project.py +265 -103
  114. mlrun/render.py +15 -14
  115. mlrun/run.py +16 -46
  116. mlrun/runtimes/__init__.py +6 -3
  117. mlrun/runtimes/base.py +8 -7
  118. mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
  119. mlrun/runtimes/funcdoc.py +0 -28
  120. mlrun/runtimes/kubejob.py +2 -1
  121. mlrun/runtimes/local.py +5 -2
  122. mlrun/runtimes/mpijob/__init__.py +0 -20
  123. mlrun/runtimes/mpijob/v1.py +1 -1
  124. mlrun/runtimes/nuclio/api_gateway.py +194 -84
  125. mlrun/runtimes/nuclio/application/application.py +170 -8
  126. mlrun/runtimes/nuclio/function.py +39 -49
  127. mlrun/runtimes/pod.py +16 -36
  128. mlrun/runtimes/remotesparkjob.py +9 -3
  129. mlrun/runtimes/sparkjob/spark3job.py +1 -1
  130. mlrun/runtimes/utils.py +6 -45
  131. mlrun/serving/server.py +2 -1
  132. mlrun/serving/v2_serving.py +5 -1
  133. mlrun/track/tracker.py +2 -1
  134. mlrun/utils/async_http.py +25 -5
  135. mlrun/utils/helpers.py +107 -75
  136. mlrun/utils/logger.py +39 -7
  137. mlrun/utils/notifications/notification/__init__.py +14 -9
  138. mlrun/utils/notifications/notification/base.py +1 -1
  139. mlrun/utils/notifications/notification/slack.py +34 -7
  140. mlrun/utils/notifications/notification/webhook.py +1 -1
  141. mlrun/utils/notifications/notification_pusher.py +147 -16
  142. mlrun/utils/regex.py +9 -0
  143. mlrun/utils/v3io_clients.py +0 -1
  144. mlrun/utils/version/version.json +2 -2
  145. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/METADATA +14 -6
  146. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/RECORD +150 -130
  147. mlrun/kfpops.py +0 -865
  148. mlrun/platforms/other.py +0 -305
  149. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/LICENSE +0 -0
  150. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/WHEEL +0 -0
  151. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/entry_points.txt +0 -0
  152. {mlrun-1.7.0rc14.dist-info → mlrun-1.7.0rc21.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,46 @@
1
+ # Copyright 2024 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ import typing
17
+
18
+ import mlrun.common.types
19
+
20
+ from .base import ObjectFormat
21
+
22
+
23
+ class FunctionFormat(ObjectFormat, mlrun.common.types.StrEnum):
24
+ minimal = "minimal"
25
+
26
+ @staticmethod
27
+ def format_method(_format: str) -> typing.Optional[typing.Callable]:
28
+ return {
29
+ FunctionFormat.full: None,
30
+ FunctionFormat.minimal: FunctionFormat.filter_obj_method(
31
+ [
32
+ "kind",
33
+ "metadata",
34
+ "status",
35
+ "spec.description",
36
+ "spec.command",
37
+ "spec.image",
38
+ "spec.default_handler",
39
+ "spec.default_class",
40
+ "spec.graph",
41
+ "spec.preemption_mode",
42
+ "spec.node_selector",
43
+ "spec.priority_class_name",
44
+ ]
45
+ ),
46
+ }[_format]
@@ -0,0 +1,53 @@
1
+ # Copyright 2024 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ import typing
17
+
18
+ import mlrun_pipelines.common.ops
19
+ import mlrun_pipelines.models
20
+
21
+ import mlrun.common.types
22
+
23
+ from .base import ObjectFormat
24
+
25
+
26
+ class PipelineFormat(ObjectFormat, mlrun.common.types.StrEnum):
27
+ full = "full"
28
+ metadata_only = "metadata_only"
29
+ name_only = "name_only"
30
+ summary = "summary"
31
+
32
+ @staticmethod
33
+ def format_method(_format: str) -> typing.Optional[typing.Callable]:
34
+ def _full(run: mlrun_pipelines.models.PipelineRun) -> dict:
35
+ return run.to_dict()
36
+
37
+ def _metadata_only(run: mlrun_pipelines.models.PipelineRun) -> dict:
38
+ return mlrun.utils.helpers.format_run(run, with_project=True)
39
+
40
+ def _name_only(run: mlrun_pipelines.models.PipelineRun) -> str:
41
+ return run.get("name")
42
+
43
+ def _summary(run: mlrun_pipelines.models.PipelineRun) -> dict:
44
+ return mlrun_pipelines.common.ops.format_summary_from_kfp_run(
45
+ run, run["project"]
46
+ )
47
+
48
+ return {
49
+ PipelineFormat.full: _full,
50
+ PipelineFormat.metadata_only: _metadata_only,
51
+ PipelineFormat.name_only: _name_only,
52
+ PipelineFormat.summary: _summary,
53
+ }[_format]
@@ -0,0 +1,51 @@
1
+ # Copyright 2024 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ import typing
17
+
18
+ import mlrun.common.schemas
19
+ import mlrun.common.types
20
+
21
+ from .base import ObjectFormat
22
+
23
+
24
+ class ProjectFormat(ObjectFormat, mlrun.common.types.StrEnum):
25
+ full = "full"
26
+ name_only = "name_only"
27
+ # minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
28
+ # and is used for faster response times (in the UI)
29
+ minimal = "minimal"
30
+ # internal - allowed only in follower mode, only for the leader for upgrade purposes
31
+ leader = "leader"
32
+
33
+ @staticmethod
34
+ def format_method(_format: str) -> typing.Optional[typing.Callable]:
35
+ def _name_only(project: mlrun.common.schemas.Project) -> str:
36
+ return project.metadata.name
37
+
38
+ def _minimal(
39
+ project: mlrun.common.schemas.Project,
40
+ ) -> mlrun.common.schemas.Project:
41
+ project.spec.functions = None
42
+ project.spec.workflows = None
43
+ project.spec.artifacts = None
44
+ return project
45
+
46
+ return {
47
+ ProjectFormat.full: None,
48
+ ProjectFormat.name_only: _name_only,
49
+ ProjectFormat.minimal: _minimal,
50
+ ProjectFormat.leader: None,
51
+ }[_format]
@@ -15,6 +15,10 @@
15
15
  import enum
16
16
  import typing
17
17
 
18
+ import mlrun_pipelines.common.models
19
+
20
+ import mlrun.common.constants as mlrun_constants
21
+
18
22
 
19
23
  class PodPhases:
20
24
  """
@@ -122,8 +126,8 @@ class MPIJobCRDVersions:
122
126
  @staticmethod
123
127
  def role_label_by_version(version):
124
128
  return {
125
- MPIJobCRDVersions.v1alpha1: "mpi_role_type",
126
- MPIJobCRDVersions.v1: "mpi-job-role",
129
+ MPIJobCRDVersions.v1alpha1: mlrun_constants.MLRunInternalLabels.mpi_role_type,
130
+ MPIJobCRDVersions.v1: mlrun_constants.MLRunInternalLabels.mpi_job_role,
127
131
  }[version]
128
132
 
129
133
 
@@ -136,6 +140,7 @@ class RunStates:
136
140
  unknown = "unknown"
137
141
  aborted = "aborted"
138
142
  aborting = "aborting"
143
+ skipped = "skipped"
139
144
 
140
145
  @staticmethod
141
146
  def all():
@@ -148,6 +153,7 @@ class RunStates:
148
153
  RunStates.unknown,
149
154
  RunStates.aborted,
150
155
  RunStates.aborting,
156
+ RunStates.skipped,
151
157
  ]
152
158
 
153
159
  @staticmethod
@@ -156,6 +162,7 @@ class RunStates:
156
162
  RunStates.completed,
157
163
  RunStates.error,
158
164
  RunStates.aborted,
165
+ RunStates.skipped,
159
166
  ]
160
167
 
161
168
  @staticmethod
@@ -188,10 +195,31 @@ class RunStates:
188
195
  # TODO: add aborting state once we have it
189
196
  ]
190
197
 
198
+ @staticmethod
199
+ def run_state_to_pipeline_run_status(run_state: str):
200
+ if not run_state:
201
+ return mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified
191
202
 
203
+ if run_state not in RunStates.all():
204
+ raise ValueError(f"Invalid run state: {run_state}")
205
+
206
+ return {
207
+ RunStates.completed: mlrun_pipelines.common.models.RunStatuses.succeeded,
208
+ RunStates.error: mlrun_pipelines.common.models.RunStatuses.failed,
209
+ RunStates.running: mlrun_pipelines.common.models.RunStatuses.running,
210
+ RunStates.created: mlrun_pipelines.common.models.RunStatuses.pending,
211
+ RunStates.pending: mlrun_pipelines.common.models.RunStatuses.pending,
212
+ RunStates.unknown: mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified,
213
+ RunStates.aborted: mlrun_pipelines.common.models.RunStatuses.canceled,
214
+ RunStates.aborting: mlrun_pipelines.common.models.RunStatuses.canceling,
215
+ RunStates.skipped: mlrun_pipelines.common.models.RunStatuses.skipped,
216
+ }[run_state]
217
+
218
+
219
+ # TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
192
220
  class RunLabels(enum.Enum):
193
- owner = "owner"
194
- v3io_user = "v3io_user"
221
+ owner = mlrun_constants.MLRunInternalLabels.owner
222
+ v3io_user = mlrun_constants.MLRunInternalLabels.v3io_user
195
223
 
196
224
  @staticmethod
197
225
  def all():
@@ -14,7 +14,13 @@
14
14
  #
15
15
  # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
16
 
17
- from .alert import AlertActiveState, AlertConfig, Event
17
+ from .alert import (
18
+ AlertActiveState,
19
+ AlertConfig,
20
+ AlertNotification,
21
+ AlertTemplate,
22
+ Event,
23
+ )
18
24
  from .api_gateway import (
19
25
  APIGateway,
20
26
  APIGatewayAuthenticationMode,
@@ -31,7 +37,6 @@ from .artifact import (
31
37
  ArtifactCategories,
32
38
  ArtifactIdentifier,
33
39
  ArtifactMetadata,
34
- ArtifactsFormat,
35
40
  ArtifactSpec,
36
41
  )
37
42
  from .auth import (
@@ -143,7 +148,8 @@ from .model_monitoring import (
143
148
  ModelMonitoringStoreKinds,
144
149
  MonitoringFunctionNames,
145
150
  PrometheusEndpoints,
146
- TimeSeriesTarget,
151
+ TSDBTarget,
152
+ V3IOTSDBTables,
147
153
  )
148
154
  from .notification import (
149
155
  Notification,
@@ -154,7 +160,7 @@ from .notification import (
154
160
  )
155
161
  from .object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
156
162
  from .pagination import PaginationInfo
157
- from .pipeline import PipelinesFormat, PipelinesOutput, PipelinesPagination
163
+ from .pipeline import PipelinesOutput, PipelinesPagination
158
164
  from .project import (
159
165
  IguazioProject,
160
166
  Project,
@@ -162,7 +168,6 @@ from .project import (
162
168
  ProjectMetadata,
163
169
  ProjectOutput,
164
170
  ProjectOwner,
165
- ProjectsFormat,
166
171
  ProjectsOutput,
167
172
  ProjectSpec,
168
173
  ProjectState,
@@ -22,25 +22,45 @@ from mlrun.common.types import StrEnum
22
22
 
23
23
 
24
24
  class EventEntityKind(StrEnum):
25
- MODEL = "model"
25
+ MODEL_ENDPOINT_RESULT = "model-endpoint-result"
26
26
  JOB = "job"
27
27
 
28
28
 
29
- class EventEntity(pydantic.BaseModel):
29
+ class EventEntities(pydantic.BaseModel):
30
30
  kind: EventEntityKind
31
31
  project: str
32
- id: str
32
+ ids: pydantic.conlist(str, min_items=1, max_items=1)
33
33
 
34
34
 
35
35
  class EventKind(StrEnum):
36
- DRIFT_DETECTED = "drift_detected"
37
- DRIFT_SUSPECTED = "drift_suspected"
36
+ DATA_DRIFT_DETECTED = "data_drift_detected"
37
+ DATA_DRIFT_SUSPECTED = "data_drift_suspected"
38
+ CONCEPT_DRIFT_DETECTED = "concept_drift_detected"
39
+ CONCEPT_DRIFT_SUSPECTED = "concept_drift_suspected"
40
+ MODEL_PERFORMANCE_DETECTED = "model_performance_detected"
41
+ MODEL_PERFORMANCE_SUSPECTED = "model_performance_suspected"
42
+ MODEL_SERVING_PERFORMANCE_DETECTED = "model_serving_performance_detected"
43
+ MODEL_SERVING_PERFORMANCE_SUSPECTED = "model_serving_performance_suspected"
44
+ MM_APP_ANOMALY_DETECTED = "mm_app_anomaly_detected"
45
+ MM_APP_ANOMALY_SUSPECTED = "mm_app_anomaly_suspected"
38
46
  FAILED = "failed"
39
47
 
40
48
 
41
49
  _event_kind_entity_map = {
42
- EventKind.DRIFT_SUSPECTED: [EventEntityKind.MODEL],
43
- EventKind.DRIFT_DETECTED: [EventEntityKind.MODEL],
50
+ EventKind.DATA_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
51
+ EventKind.DATA_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
52
+ EventKind.CONCEPT_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
53
+ EventKind.CONCEPT_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
54
+ EventKind.MODEL_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
55
+ EventKind.MODEL_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
56
+ EventKind.MODEL_SERVING_PERFORMANCE_DETECTED: [
57
+ EventEntityKind.MODEL_ENDPOINT_RESULT
58
+ ],
59
+ EventKind.MODEL_SERVING_PERFORMANCE_SUSPECTED: [
60
+ EventEntityKind.MODEL_ENDPOINT_RESULT
61
+ ],
62
+ EventKind.MM_APP_ANOMALY_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
63
+ EventKind.MM_APP_ANOMALY_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
44
64
  EventKind.FAILED: [EventEntityKind.JOB],
45
65
  }
46
66
 
@@ -48,7 +68,7 @@ _event_kind_entity_map = {
48
68
  class Event(pydantic.BaseModel):
49
69
  kind: EventKind
50
70
  timestamp: Union[str, datetime] = None # occurrence time
51
- entity: EventEntity
71
+ entity: EventEntities
52
72
  value_dict: Optional[dict] = pydantic.Field(default_factory=dict)
53
73
 
54
74
  def is_valid(self):
@@ -71,6 +91,12 @@ class AlertTrigger(pydantic.BaseModel):
71
91
  events: list[EventKind] = []
72
92
  prometheus_alert: str = None
73
93
 
94
+ def __eq__(self, other):
95
+ return (
96
+ self.prometheus_alert == other.prometheus_alert
97
+ and self.events == other.events
98
+ )
99
+
74
100
 
75
101
  class AlertCriteria(pydantic.BaseModel):
76
102
  count: Annotated[
@@ -86,12 +112,27 @@ class AlertCriteria(pydantic.BaseModel):
86
112
  ),
87
113
  ] = None
88
114
 
115
+ def __eq__(self, other):
116
+ return self.count == other.count and self.period == other.period
117
+
89
118
 
90
119
  class ResetPolicy(StrEnum):
91
120
  MANUAL = "manual"
92
121
  AUTO = "auto"
93
122
 
94
123
 
124
+ class AlertNotification(pydantic.BaseModel):
125
+ notification: Notification
126
+ cooldown_period: Annotated[
127
+ str,
128
+ pydantic.Field(
129
+ description="Period during which notifications "
130
+ "will not be sent after initial send. The format of this would be in time."
131
+ " e.g. 1d, 3h, 5m, 15s"
132
+ ),
133
+ ] = None
134
+
135
+
95
136
  class AlertConfig(pydantic.BaseModel):
96
137
  project: str
97
138
  id: int = None
@@ -102,21 +143,61 @@ class AlertConfig(pydantic.BaseModel):
102
143
  pydantic.Field(
103
144
  description=(
104
145
  "String to be sent in the notifications generated."
105
- "e.g. 'Model {{ $project }}/{{ $entity }} is drifting.'"
146
+ "e.g. 'Model {{project}}/{{entity}} is drifting.'"
147
+ "Supported variables: project, entity, name"
106
148
  )
107
149
  ),
108
150
  ]
109
151
  created: Union[str, datetime] = None
110
152
  severity: AlertSeverity
111
- entity: EventEntity
153
+ entities: EventEntities
112
154
  trigger: AlertTrigger
113
155
  criteria: Optional[AlertCriteria]
114
156
  reset_policy: ResetPolicy = ResetPolicy.MANUAL
115
- notifications: pydantic.conlist(Notification, min_items=1)
157
+ notifications: pydantic.conlist(AlertNotification, min_items=1)
116
158
  state: AlertActiveState = AlertActiveState.INACTIVE
117
159
  count: Optional[int] = 0
118
160
 
161
+ def get_raw_notifications(self) -> list[Notification]:
162
+ return [
163
+ alert_notification.notification for alert_notification in self.notifications
164
+ ]
165
+
119
166
 
120
167
  class AlertsModes(StrEnum):
121
168
  enabled = "enabled"
122
169
  disabled = "disabled"
170
+
171
+
172
+ class AlertTemplate(
173
+ pydantic.BaseModel
174
+ ): # Template fields that are not shared with created configs
175
+ template_id: int = None
176
+ template_name: str
177
+ template_description: Optional[str] = (
178
+ "String explaining the purpose of this template"
179
+ )
180
+
181
+ # A property that identifies templates that were created by the system and cannot be modified/deleted by the user
182
+ system_generated: bool = False
183
+
184
+ # AlertConfig fields that are pre-defined
185
+ summary: Optional[str] = (
186
+ "String to be sent in the generated notifications e.g. 'Model {{project}}/{{entity}} is drifting.'"
187
+ "See AlertConfig.summary description"
188
+ )
189
+ severity: AlertSeverity
190
+ trigger: AlertTrigger
191
+ criteria: Optional[AlertCriteria]
192
+ reset_policy: ResetPolicy = ResetPolicy.MANUAL
193
+
194
+ # This is slightly different than __eq__ as it doesn't compare everything
195
+ def templates_differ(self, other):
196
+ return (
197
+ self.template_description != other.template_description
198
+ or self.summary != other.summary
199
+ or self.severity != other.severity
200
+ or self.trigger != other.trigger
201
+ or self.reset_policy != other.reset_policy
202
+ or self.criteria != other.criteria
203
+ )
@@ -18,11 +18,13 @@ from typing import Optional
18
18
  import pydantic
19
19
 
20
20
  import mlrun.common.types
21
+ from mlrun.common.constants import MLRUN_FUNCTIONS_ANNOTATION
21
22
 
22
23
 
23
24
  class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
24
25
  basic = "basicAuth"
25
26
  none = "none"
27
+ access_key = "accessKey"
26
28
 
27
29
  @classmethod
28
30
  def from_str(cls, authentication_mode: str):
@@ -30,6 +32,8 @@ class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
30
32
  return cls.none
31
33
  elif authentication_mode == "basicAuth":
32
34
  return cls.basic
35
+ elif authentication_mode == "accessKey":
36
+ return cls.access_key
33
37
  else:
34
38
  raise mlrun.errors.MLRunInvalidArgumentError(
35
39
  f"Authentication mode `{authentication_mode}` is not supported",
@@ -52,6 +56,7 @@ class APIGatewayMetadata(_APIGatewayBaseModel):
52
56
  name: str
53
57
  namespace: Optional[str]
54
58
  labels: Optional[dict] = {}
59
+ annotations: Optional[dict] = {}
55
60
 
56
61
 
57
62
  class APIGatewayBasicAuth(_APIGatewayBaseModel):
@@ -63,6 +68,7 @@ class APIGatewayUpstream(_APIGatewayBaseModel):
63
68
  kind: Optional[str] = "nucliofunction"
64
69
  nucliofunction: dict[str, str]
65
70
  percentage: Optional[int] = 0
71
+ port: Optional[int] = 0
66
72
 
67
73
 
68
74
  class APIGatewaySpec(_APIGatewayBaseModel):
@@ -87,6 +93,56 @@ class APIGateway(_APIGatewayBaseModel):
87
93
  spec: APIGatewaySpec
88
94
  status: Optional[APIGatewayStatus]
89
95
 
96
+ def get_function_names(self):
97
+ return [
98
+ upstream.nucliofunction.get("name")
99
+ for upstream in self.spec.upstreams
100
+ if upstream.nucliofunction.get("name")
101
+ ]
102
+
103
+ def enrich_mlrun_function_names(self):
104
+ upstream_with_nuclio_names = []
105
+ mlrun_function_uris = []
106
+ for upstream in self.spec.upstreams:
107
+ uri = upstream.nucliofunction.get("name")
108
+ project, function_name, tag, _ = (
109
+ mlrun.common.helpers.parse_versioned_object_uri(uri)
110
+ )
111
+ upstream.nucliofunction["name"] = (
112
+ mlrun.runtimes.nuclio.function.get_fullname(function_name, project, tag)
113
+ )
114
+
115
+ upstream_with_nuclio_names.append(upstream)
116
+ mlrun_function_uris.append(uri)
117
+
118
+ self.spec.upstreams = upstream_with_nuclio_names
119
+ if len(mlrun_function_uris) == 1:
120
+ self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = mlrun_function_uris[
121
+ 0
122
+ ]
123
+ elif len(mlrun_function_uris) == 2:
124
+ self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = "&".join(
125
+ mlrun_function_uris
126
+ )
127
+ return self
128
+
129
+ def replace_nuclio_names_with_mlrun_uri(self):
130
+ mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
131
+ if mlrun_functions:
132
+ mlrun_function_uris = (
133
+ mlrun_functions.split("&")
134
+ if "&" in mlrun_functions
135
+ else [mlrun_functions]
136
+ )
137
+ if len(mlrun_function_uris) != len(self.spec.upstreams):
138
+ raise mlrun.errors.MLRunValueError(
139
+ "Error when translating nuclio names to mlrun names in api gateway:"
140
+ " number of functions doesn't match the mlrun functions in annotation"
141
+ )
142
+ for i in range(len(mlrun_function_uris)):
143
+ self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
144
+ return self
145
+
90
146
 
91
147
  class APIGatewaysOutput(_APIGatewayBaseModel):
92
148
  api_gateways: typing.Optional[dict[str, APIGateway]] = {}
@@ -58,11 +58,6 @@ class ArtifactIdentifier(pydantic.BaseModel):
58
58
  # hash: typing.Optional[str]
59
59
 
60
60
 
61
- class ArtifactsFormat(mlrun.common.types.StrEnum):
62
- # TODO: add a format that returns a minimal response
63
- full = "full"
64
-
65
-
66
61
  class ArtifactMetadata(pydantic.BaseModel):
67
62
  key: str
68
63
  project: str
@@ -93,3 +88,18 @@ class Artifact(pydantic.BaseModel):
93
88
  metadata: ArtifactMetadata
94
89
  spec: ArtifactSpec
95
90
  status: ObjectStatus
91
+
92
+
93
+ class ArtifactsDeletionStrategies(mlrun.common.types.StrEnum):
94
+ """Artifacts deletion strategies types."""
95
+
96
+ metadata_only = "metadata-only"
97
+ """Only removes the artifact db record, leaving all related artifact data in-place"""
98
+
99
+ data_optional = "data-optional"
100
+ """Delete the artifact data of the artifact as a best-effort.
101
+ If artifact data deletion fails still try to delete the artifact db record"""
102
+
103
+ data_force = "data-force"
104
+ """Delete the artifact data, and if cannot delete it fail the deletion
105
+ and don’t delete the artifact db record"""
@@ -59,6 +59,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
59
59
  hub_source = "hub-source"
60
60
  workflow = "workflow"
61
61
  alert = "alert"
62
+ alert_templates = "alert-templates"
62
63
  event = "event"
63
64
  datastore_profile = "datastore-profile"
64
65
  api_gateway = "api-gateway"
@@ -87,6 +88,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
87
88
  AuthorizationResourceTypes.run: "/projects/{project_name}/runs/{resource_name}",
88
89
  AuthorizationResourceTypes.event: "/projects/{project_name}/events/{resource_name}",
89
90
  AuthorizationResourceTypes.alert: "/projects/{project_name}/alerts/{resource_name}",
91
+ AuthorizationResourceTypes.alert_templates: "/alert-templates/{resource_name}",
90
92
  # runtime resource doesn't have an identifier, we don't need any auth granularity behind project level
91
93
  AuthorizationResourceTypes.runtime_resource: "/projects/{project_name}/runtime-resources",
92
94
  AuthorizationResourceTypes.model_endpoint: "/projects/{project_name}/model-endpoints/{resource_name}",
@@ -59,6 +59,7 @@ class ClientSpec(pydantic.BaseModel):
59
59
  sql_url: typing.Optional[str]
60
60
  model_endpoint_monitoring_store_type: typing.Optional[str]
61
61
  model_endpoint_monitoring_endpoint_store_connection: typing.Optional[str]
62
+ model_monitoring_tsdb_connection: typing.Optional[str]
62
63
  ce: typing.Optional[dict]
63
64
  # not passing them as one object as it possible client user would like to override only one of the params
64
65
  calculate_artifact_hash: typing.Optional[str]
@@ -70,3 +70,4 @@ class FrontendSpec(pydantic.BaseModel):
70
70
  feature_store_data_prefixes: typing.Optional[dict[str, str]]
71
71
  allowed_artifact_path_prefixes_list: list[str]
72
72
  ce: typing.Optional[dict]
73
+ internal_labels: list[str] = []
@@ -45,6 +45,9 @@ class FunctionState:
45
45
  # same goes for the build which is not coming from the pod, but is used and we can't just omit it for BC reasons
46
46
  build = "build"
47
47
 
48
+ # for pipeline steps
49
+ skipped = "skipped"
50
+
48
51
  @classmethod
49
52
  def get_function_state_from_pod_state(cls, pod_state: str):
50
53
  if pod_state == "succeeded":
@@ -60,6 +63,7 @@ class FunctionState:
60
63
  return [
61
64
  cls.ready,
62
65
  cls.error,
66
+ cls.skipped,
63
67
  ]
64
68
 
65
69
 
@@ -11,8 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
- # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
14
 
17
15
  from .constants import (
18
16
  ControllerPolicy,
@@ -25,20 +23,29 @@ from .constants import (
25
23
  FeatureSetFeatures,
26
24
  FileTargetKind,
27
25
  FunctionURI,
26
+ MetricData,
28
27
  ModelEndpointTarget,
29
28
  ModelMonitoringMode,
30
29
  ModelMonitoringStoreKinds,
31
30
  MonitoringFunctionNames,
31
+ PredictionsQueryConstants,
32
32
  ProjectSecretKeys,
33
33
  PrometheusEndpoints,
34
34
  PrometheusMetric,
35
+ ResultData,
36
+ ResultKindApp,
35
37
  SchedulingKeys,
36
- TimeSeriesTarget,
38
+ SpecialApps,
39
+ TDEngineSuperTables,
40
+ TSDBTarget,
41
+ V3IOTSDBTables,
37
42
  VersionedModel,
38
43
  WriterEvent,
44
+ WriterEventKind,
39
45
  )
40
46
  from .grafana import (
41
47
  GrafanaColumn,
48
+ GrafanaColumnType,
42
49
  GrafanaDataPoint,
43
50
  GrafanaNumberColumn,
44
51
  GrafanaStringColumn,
@@ -51,6 +58,11 @@ from .model_endpoints import (
51
58
  ModelEndpoint,
52
59
  ModelEndpointList,
53
60
  ModelEndpointMetadata,
61
+ ModelEndpointMonitoringMetric,
62
+ ModelEndpointMonitoringMetricNoData,
63
+ ModelEndpointMonitoringMetricType,
64
+ ModelEndpointMonitoringMetricValues,
65
+ ModelEndpointMonitoringResultValues,
54
66
  ModelEndpointSpec,
55
67
  ModelEndpointStatus,
56
68
  )