mlrun 1.7.0rc4__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (200) hide show
  1. mlrun/__init__.py +11 -1
  2. mlrun/__main__.py +25 -111
  3. mlrun/{datastore/helpers.py → alerts/__init__.py} +2 -5
  4. mlrun/alerts/alert.py +144 -0
  5. mlrun/api/schemas/__init__.py +4 -3
  6. mlrun/artifacts/__init__.py +8 -3
  7. mlrun/artifacts/base.py +38 -254
  8. mlrun/artifacts/dataset.py +9 -190
  9. mlrun/artifacts/manager.py +41 -47
  10. mlrun/artifacts/model.py +30 -158
  11. mlrun/artifacts/plots.py +23 -380
  12. mlrun/common/constants.py +68 -0
  13. mlrun/common/formatters/__init__.py +19 -0
  14. mlrun/{model_monitoring/stores/models/sqlite.py → common/formatters/artifact.py} +6 -8
  15. mlrun/common/formatters/base.py +78 -0
  16. mlrun/common/formatters/function.py +41 -0
  17. mlrun/common/formatters/pipeline.py +53 -0
  18. mlrun/common/formatters/project.py +51 -0
  19. mlrun/{runtimes → common/runtimes}/constants.py +32 -4
  20. mlrun/common/schemas/__init__.py +25 -4
  21. mlrun/common/schemas/alert.py +203 -0
  22. mlrun/common/schemas/api_gateway.py +148 -0
  23. mlrun/common/schemas/artifact.py +15 -5
  24. mlrun/common/schemas/auth.py +8 -2
  25. mlrun/common/schemas/client_spec.py +2 -0
  26. mlrun/common/schemas/frontend_spec.py +1 -0
  27. mlrun/common/schemas/function.py +4 -0
  28. mlrun/common/schemas/hub.py +7 -9
  29. mlrun/common/schemas/model_monitoring/__init__.py +19 -3
  30. mlrun/common/schemas/model_monitoring/constants.py +96 -26
  31. mlrun/common/schemas/model_monitoring/grafana.py +9 -5
  32. mlrun/common/schemas/model_monitoring/model_endpoints.py +86 -2
  33. mlrun/{runtimes/mpijob/v1alpha1.py → common/schemas/pagination.py} +10 -13
  34. mlrun/common/schemas/pipeline.py +0 -9
  35. mlrun/common/schemas/project.py +22 -21
  36. mlrun/common/types.py +7 -1
  37. mlrun/config.py +87 -19
  38. mlrun/data_types/data_types.py +4 -0
  39. mlrun/data_types/to_pandas.py +9 -9
  40. mlrun/datastore/__init__.py +5 -8
  41. mlrun/datastore/alibaba_oss.py +130 -0
  42. mlrun/datastore/azure_blob.py +4 -5
  43. mlrun/datastore/base.py +69 -30
  44. mlrun/datastore/datastore.py +10 -2
  45. mlrun/datastore/datastore_profile.py +90 -6
  46. mlrun/datastore/google_cloud_storage.py +1 -1
  47. mlrun/datastore/hdfs.py +5 -0
  48. mlrun/datastore/inmem.py +2 -2
  49. mlrun/datastore/redis.py +2 -2
  50. mlrun/datastore/s3.py +5 -0
  51. mlrun/datastore/snowflake_utils.py +43 -0
  52. mlrun/datastore/sources.py +172 -44
  53. mlrun/datastore/store_resources.py +7 -7
  54. mlrun/datastore/targets.py +285 -41
  55. mlrun/datastore/utils.py +68 -5
  56. mlrun/datastore/v3io.py +27 -50
  57. mlrun/db/auth_utils.py +152 -0
  58. mlrun/db/base.py +149 -14
  59. mlrun/db/factory.py +1 -1
  60. mlrun/db/httpdb.py +608 -178
  61. mlrun/db/nopdb.py +191 -7
  62. mlrun/errors.py +11 -0
  63. mlrun/execution.py +37 -20
  64. mlrun/feature_store/__init__.py +0 -2
  65. mlrun/feature_store/api.py +21 -52
  66. mlrun/feature_store/feature_set.py +48 -23
  67. mlrun/feature_store/feature_vector.py +2 -1
  68. mlrun/feature_store/ingestion.py +7 -6
  69. mlrun/feature_store/retrieval/base.py +9 -4
  70. mlrun/feature_store/retrieval/conversion.py +9 -9
  71. mlrun/feature_store/retrieval/dask_merger.py +2 -0
  72. mlrun/feature_store/retrieval/job.py +9 -3
  73. mlrun/feature_store/retrieval/local_merger.py +2 -0
  74. mlrun/feature_store/retrieval/spark_merger.py +34 -24
  75. mlrun/feature_store/steps.py +30 -19
  76. mlrun/features.py +4 -13
  77. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +7 -12
  78. mlrun/frameworks/auto_mlrun/auto_mlrun.py +2 -2
  79. mlrun/frameworks/lgbm/__init__.py +1 -1
  80. mlrun/frameworks/lgbm/callbacks/callback.py +2 -4
  81. mlrun/frameworks/lgbm/model_handler.py +1 -1
  82. mlrun/frameworks/parallel_coordinates.py +2 -1
  83. mlrun/frameworks/pytorch/__init__.py +2 -2
  84. mlrun/frameworks/sklearn/__init__.py +1 -1
  85. mlrun/frameworks/tf_keras/__init__.py +5 -2
  86. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  87. mlrun/frameworks/tf_keras/mlrun_interface.py +2 -2
  88. mlrun/frameworks/xgboost/__init__.py +1 -1
  89. mlrun/k8s_utils.py +10 -11
  90. mlrun/launcher/__init__.py +1 -1
  91. mlrun/launcher/base.py +6 -5
  92. mlrun/launcher/client.py +8 -6
  93. mlrun/launcher/factory.py +1 -1
  94. mlrun/launcher/local.py +9 -3
  95. mlrun/launcher/remote.py +9 -3
  96. mlrun/lists.py +6 -2
  97. mlrun/model.py +58 -19
  98. mlrun/model_monitoring/__init__.py +1 -1
  99. mlrun/model_monitoring/api.py +127 -301
  100. mlrun/model_monitoring/application.py +5 -296
  101. mlrun/model_monitoring/applications/__init__.py +11 -0
  102. mlrun/model_monitoring/applications/_application_steps.py +157 -0
  103. mlrun/model_monitoring/applications/base.py +282 -0
  104. mlrun/model_monitoring/applications/context.py +214 -0
  105. mlrun/model_monitoring/applications/evidently_base.py +211 -0
  106. mlrun/model_monitoring/applications/histogram_data_drift.py +224 -93
  107. mlrun/model_monitoring/applications/results.py +99 -0
  108. mlrun/model_monitoring/controller.py +30 -36
  109. mlrun/model_monitoring/db/__init__.py +18 -0
  110. mlrun/model_monitoring/{stores → db/stores}/__init__.py +43 -36
  111. mlrun/model_monitoring/db/stores/base/__init__.py +15 -0
  112. mlrun/model_monitoring/{stores/model_endpoint_store.py → db/stores/base/store.py} +58 -32
  113. mlrun/model_monitoring/db/stores/sqldb/__init__.py +13 -0
  114. mlrun/model_monitoring/db/stores/sqldb/models/__init__.py +71 -0
  115. mlrun/model_monitoring/{stores → db/stores/sqldb}/models/base.py +109 -5
  116. mlrun/model_monitoring/db/stores/sqldb/models/mysql.py +88 -0
  117. mlrun/model_monitoring/{stores/models/mysql.py → db/stores/sqldb/models/sqlite.py} +19 -13
  118. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +684 -0
  119. mlrun/model_monitoring/db/stores/v3io_kv/__init__.py +13 -0
  120. mlrun/model_monitoring/{stores/kv_model_endpoint_store.py → db/stores/v3io_kv/kv_store.py} +302 -155
  121. mlrun/model_monitoring/db/tsdb/__init__.py +100 -0
  122. mlrun/model_monitoring/db/tsdb/base.py +329 -0
  123. mlrun/model_monitoring/db/tsdb/helpers.py +30 -0
  124. mlrun/model_monitoring/db/tsdb/tdengine/__init__.py +15 -0
  125. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +240 -0
  126. mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py +45 -0
  127. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +397 -0
  128. mlrun/model_monitoring/db/tsdb/v3io/__init__.py +15 -0
  129. mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py +117 -0
  130. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +630 -0
  131. mlrun/model_monitoring/evidently_application.py +6 -118
  132. mlrun/model_monitoring/features_drift_table.py +34 -22
  133. mlrun/model_monitoring/helpers.py +100 -7
  134. mlrun/model_monitoring/model_endpoint.py +3 -2
  135. mlrun/model_monitoring/stream_processing.py +93 -228
  136. mlrun/model_monitoring/tracking_policy.py +7 -1
  137. mlrun/model_monitoring/writer.py +152 -124
  138. mlrun/package/packagers_manager.py +1 -0
  139. mlrun/package/utils/_formatter.py +2 -2
  140. mlrun/platforms/__init__.py +11 -10
  141. mlrun/platforms/iguazio.py +21 -202
  142. mlrun/projects/operations.py +30 -16
  143. mlrun/projects/pipelines.py +92 -99
  144. mlrun/projects/project.py +757 -268
  145. mlrun/render.py +15 -14
  146. mlrun/run.py +160 -162
  147. mlrun/runtimes/__init__.py +55 -3
  148. mlrun/runtimes/base.py +33 -19
  149. mlrun/runtimes/databricks_job/databricks_wrapper.py +1 -1
  150. mlrun/runtimes/funcdoc.py +0 -28
  151. mlrun/runtimes/kubejob.py +28 -122
  152. mlrun/runtimes/local.py +5 -2
  153. mlrun/runtimes/mpijob/__init__.py +0 -20
  154. mlrun/runtimes/mpijob/abstract.py +8 -8
  155. mlrun/runtimes/mpijob/v1.py +1 -1
  156. mlrun/runtimes/nuclio/__init__.py +1 -0
  157. mlrun/runtimes/nuclio/api_gateway.py +709 -0
  158. mlrun/runtimes/nuclio/application/__init__.py +15 -0
  159. mlrun/runtimes/nuclio/application/application.py +523 -0
  160. mlrun/runtimes/nuclio/application/reverse_proxy.go +95 -0
  161. mlrun/runtimes/nuclio/function.py +98 -58
  162. mlrun/runtimes/nuclio/serving.py +36 -42
  163. mlrun/runtimes/pod.py +196 -45
  164. mlrun/runtimes/remotesparkjob.py +1 -1
  165. mlrun/runtimes/sparkjob/spark3job.py +1 -1
  166. mlrun/runtimes/utils.py +6 -73
  167. mlrun/secrets.py +6 -2
  168. mlrun/serving/remote.py +2 -3
  169. mlrun/serving/routers.py +7 -4
  170. mlrun/serving/server.py +7 -8
  171. mlrun/serving/states.py +73 -43
  172. mlrun/serving/v2_serving.py +8 -7
  173. mlrun/track/tracker.py +2 -1
  174. mlrun/utils/async_http.py +25 -5
  175. mlrun/utils/helpers.py +141 -75
  176. mlrun/utils/http.py +1 -1
  177. mlrun/utils/logger.py +39 -7
  178. mlrun/utils/notifications/notification/__init__.py +14 -9
  179. mlrun/utils/notifications/notification/base.py +12 -0
  180. mlrun/utils/notifications/notification/console.py +2 -0
  181. mlrun/utils/notifications/notification/git.py +3 -1
  182. mlrun/utils/notifications/notification/ipython.py +2 -0
  183. mlrun/utils/notifications/notification/slack.py +101 -21
  184. mlrun/utils/notifications/notification/webhook.py +11 -1
  185. mlrun/utils/notifications/notification_pusher.py +147 -16
  186. mlrun/utils/retryer.py +3 -2
  187. mlrun/utils/v3io_clients.py +0 -1
  188. mlrun/utils/version/version.json +2 -2
  189. {mlrun-1.7.0rc4.dist-info → mlrun-1.7.0rc20.dist-info}/METADATA +33 -18
  190. mlrun-1.7.0rc20.dist-info/RECORD +353 -0
  191. {mlrun-1.7.0rc4.dist-info → mlrun-1.7.0rc20.dist-info}/WHEEL +1 -1
  192. mlrun/kfpops.py +0 -868
  193. mlrun/model_monitoring/batch.py +0 -974
  194. mlrun/model_monitoring/stores/models/__init__.py +0 -27
  195. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -382
  196. mlrun/platforms/other.py +0 -305
  197. mlrun-1.7.0rc4.dist-info/RECORD +0 -321
  198. {mlrun-1.7.0rc4.dist-info → mlrun-1.7.0rc20.dist-info}/LICENSE +0 -0
  199. {mlrun-1.7.0rc4.dist-info → mlrun-1.7.0rc20.dist-info}/entry_points.txt +0 -0
  200. {mlrun-1.7.0rc4.dist-info → mlrun-1.7.0rc20.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,53 @@
1
+ # Copyright 2024 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ import typing
17
+
18
+ import mlrun_pipelines.common.ops
19
+ import mlrun_pipelines.models
20
+
21
+ import mlrun.common.types
22
+
23
+ from .base import ObjectFormat
24
+
25
+
26
+ class PipelineFormat(ObjectFormat, mlrun.common.types.StrEnum):
27
+ full = "full"
28
+ metadata_only = "metadata_only"
29
+ name_only = "name_only"
30
+ summary = "summary"
31
+
32
+ @staticmethod
33
+ def format_method(_format: str) -> typing.Optional[typing.Callable]:
34
+ def _full(run: mlrun_pipelines.models.PipelineRun) -> dict:
35
+ return run.to_dict()
36
+
37
+ def _metadata_only(run: mlrun_pipelines.models.PipelineRun) -> dict:
38
+ return mlrun.utils.helpers.format_run(run, with_project=True)
39
+
40
+ def _name_only(run: mlrun_pipelines.models.PipelineRun) -> str:
41
+ return run.get("name")
42
+
43
+ def _summary(run: mlrun_pipelines.models.PipelineRun) -> dict:
44
+ return mlrun_pipelines.common.ops.format_summary_from_kfp_run(
45
+ run, run["project"]
46
+ )
47
+
48
+ return {
49
+ PipelineFormat.full: _full,
50
+ PipelineFormat.metadata_only: _metadata_only,
51
+ PipelineFormat.name_only: _name_only,
52
+ PipelineFormat.summary: _summary,
53
+ }[_format]
@@ -0,0 +1,51 @@
1
+ # Copyright 2024 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ import typing
17
+
18
+ import mlrun.common.schemas
19
+ import mlrun.common.types
20
+
21
+ from .base import ObjectFormat
22
+
23
+
24
+ class ProjectFormat(ObjectFormat, mlrun.common.types.StrEnum):
25
+ full = "full"
26
+ name_only = "name_only"
27
+ # minimal format removes large fields from the response (e.g. functions, workflows, artifacts)
28
+ # and is used for faster response times (in the UI)
29
+ minimal = "minimal"
30
+ # internal - allowed only in follower mode, only for the leader for upgrade purposes
31
+ leader = "leader"
32
+
33
+ @staticmethod
34
+ def format_method(_format: str) -> typing.Optional[typing.Callable]:
35
+ def _name_only(project: mlrun.common.schemas.Project) -> str:
36
+ return project.metadata.name
37
+
38
+ def _minimal(
39
+ project: mlrun.common.schemas.Project,
40
+ ) -> mlrun.common.schemas.Project:
41
+ project.spec.functions = None
42
+ project.spec.workflows = None
43
+ project.spec.artifacts = None
44
+ return project
45
+
46
+ return {
47
+ ProjectFormat.full: None,
48
+ ProjectFormat.name_only: _name_only,
49
+ ProjectFormat.minimal: _minimal,
50
+ ProjectFormat.leader: None,
51
+ }[_format]
@@ -15,6 +15,10 @@
15
15
  import enum
16
16
  import typing
17
17
 
18
+ import mlrun_pipelines.common.models
19
+
20
+ import mlrun.common.constants as mlrun_constants
21
+
18
22
 
19
23
  class PodPhases:
20
24
  """
@@ -122,8 +126,8 @@ class MPIJobCRDVersions:
122
126
  @staticmethod
123
127
  def role_label_by_version(version):
124
128
  return {
125
- MPIJobCRDVersions.v1alpha1: "mpi_role_type",
126
- MPIJobCRDVersions.v1: "mpi-job-role",
129
+ MPIJobCRDVersions.v1alpha1: mlrun_constants.MLRunInternalLabels.mpi_role_type,
130
+ MPIJobCRDVersions.v1: mlrun_constants.MLRunInternalLabels.mpi_job_role,
127
131
  }[version]
128
132
 
129
133
 
@@ -136,6 +140,7 @@ class RunStates:
136
140
  unknown = "unknown"
137
141
  aborted = "aborted"
138
142
  aborting = "aborting"
143
+ skipped = "skipped"
139
144
 
140
145
  @staticmethod
141
146
  def all():
@@ -148,6 +153,7 @@ class RunStates:
148
153
  RunStates.unknown,
149
154
  RunStates.aborted,
150
155
  RunStates.aborting,
156
+ RunStates.skipped,
151
157
  ]
152
158
 
153
159
  @staticmethod
@@ -156,6 +162,7 @@ class RunStates:
156
162
  RunStates.completed,
157
163
  RunStates.error,
158
164
  RunStates.aborted,
165
+ RunStates.skipped,
159
166
  ]
160
167
 
161
168
  @staticmethod
@@ -188,10 +195,31 @@ class RunStates:
188
195
  # TODO: add aborting state once we have it
189
196
  ]
190
197
 
198
+ @staticmethod
199
+ def run_state_to_pipeline_run_status(run_state: str):
200
+ if not run_state:
201
+ return mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified
191
202
 
203
+ if run_state not in RunStates.all():
204
+ raise ValueError(f"Invalid run state: {run_state}")
205
+
206
+ return {
207
+ RunStates.completed: mlrun_pipelines.common.models.RunStatuses.succeeded,
208
+ RunStates.error: mlrun_pipelines.common.models.RunStatuses.failed,
209
+ RunStates.running: mlrun_pipelines.common.models.RunStatuses.running,
210
+ RunStates.created: mlrun_pipelines.common.models.RunStatuses.pending,
211
+ RunStates.pending: mlrun_pipelines.common.models.RunStatuses.pending,
212
+ RunStates.unknown: mlrun_pipelines.common.models.RunStatuses.runtime_state_unspecified,
213
+ RunStates.aborted: mlrun_pipelines.common.models.RunStatuses.canceled,
214
+ RunStates.aborting: mlrun_pipelines.common.models.RunStatuses.canceling,
215
+ RunStates.skipped: mlrun_pipelines.common.models.RunStatuses.skipped,
216
+ }[run_state]
217
+
218
+
219
+ # TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
192
220
  class RunLabels(enum.Enum):
193
- owner = "owner"
194
- v3io_user = "v3io_user"
221
+ owner = mlrun_constants.MLRunInternalLabels.owner
222
+ v3io_user = mlrun_constants.MLRunInternalLabels.v3io_user
195
223
 
196
224
  @staticmethod
197
225
  def all():
@@ -14,12 +14,29 @@
14
14
  #
15
15
  # flake8: noqa - this is until we take care of the F401 violations with respect to __all__ & sphinx
16
16
 
17
+ from .alert import (
18
+ AlertActiveState,
19
+ AlertConfig,
20
+ AlertNotification,
21
+ AlertTemplate,
22
+ Event,
23
+ )
24
+ from .api_gateway import (
25
+ APIGateway,
26
+ APIGatewayAuthenticationMode,
27
+ APIGatewayBasicAuth,
28
+ APIGatewayMetadata,
29
+ APIGatewaysOutput,
30
+ APIGatewaySpec,
31
+ APIGatewayState,
32
+ APIGatewayStatus,
33
+ APIGatewayUpstream,
34
+ )
17
35
  from .artifact import (
18
36
  Artifact,
19
37
  ArtifactCategories,
20
38
  ArtifactIdentifier,
21
39
  ArtifactMetadata,
22
- ArtifactsFormat,
23
40
  ArtifactSpec,
24
41
  )
25
42
  from .auth import (
@@ -114,6 +131,7 @@ from .model_monitoring import (
114
131
  EventFieldType,
115
132
  EventKeyMetrics,
116
133
  Features,
134
+ FeatureSetFeatures,
117
135
  FeatureValues,
118
136
  GrafanaColumn,
119
137
  GrafanaDataPoint,
@@ -129,7 +147,9 @@ from .model_monitoring import (
129
147
  ModelMonitoringMode,
130
148
  ModelMonitoringStoreKinds,
131
149
  MonitoringFunctionNames,
132
- TimeSeriesTarget,
150
+ PrometheusEndpoints,
151
+ TSDBTarget,
152
+ V3IOTSDBTables,
133
153
  )
134
154
  from .notification import (
135
155
  Notification,
@@ -139,14 +159,15 @@ from .notification import (
139
159
  SetNotificationRequest,
140
160
  )
141
161
  from .object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus
142
- from .pipeline import PipelinesFormat, PipelinesOutput, PipelinesPagination
162
+ from .pagination import PaginationInfo
163
+ from .pipeline import PipelinesOutput, PipelinesPagination
143
164
  from .project import (
144
165
  IguazioProject,
145
166
  Project,
146
167
  ProjectDesiredState,
147
168
  ProjectMetadata,
169
+ ProjectOutput,
148
170
  ProjectOwner,
149
- ProjectsFormat,
150
171
  ProjectsOutput,
151
172
  ProjectSpec,
152
173
  ProjectState,
@@ -0,0 +1,203 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ from datetime import datetime
16
+ from typing import Annotated, Optional, Union
17
+
18
+ import pydantic
19
+
20
+ from mlrun.common.schemas.notification import Notification
21
+ from mlrun.common.types import StrEnum
22
+
23
+
24
+ class EventEntityKind(StrEnum):
25
+ MODEL_ENDPOINT_RESULT = "model-endpoint-result"
26
+ JOB = "job"
27
+
28
+
29
+ class EventEntities(pydantic.BaseModel):
30
+ kind: EventEntityKind
31
+ project: str
32
+ ids: pydantic.conlist(str, min_items=1, max_items=1)
33
+
34
+
35
+ class EventKind(StrEnum):
36
+ DATA_DRIFT_DETECTED = "data_drift_detected"
37
+ DATA_DRIFT_SUSPECTED = "data_drift_suspected"
38
+ CONCEPT_DRIFT_DETECTED = "concept_drift_detected"
39
+ CONCEPT_DRIFT_SUSPECTED = "concept_drift_suspected"
40
+ MODEL_PERFORMANCE_DETECTED = "model_performance_detected"
41
+ MODEL_PERFORMANCE_SUSPECTED = "model_performance_suspected"
42
+ MODEL_SERVING_PERFORMANCE_DETECTED = "model_serving_performance_detected"
43
+ MODEL_SERVING_PERFORMANCE_SUSPECTED = "model_serving_performance_suspected"
44
+ MM_APP_ANOMALY_DETECTED = "mm_app_anomaly_detected"
45
+ MM_APP_ANOMALY_SUSPECTED = "mm_app_anomaly_suspected"
46
+ FAILED = "failed"
47
+
48
+
49
+ _event_kind_entity_map = {
50
+ EventKind.DATA_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
51
+ EventKind.DATA_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
52
+ EventKind.CONCEPT_DRIFT_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
53
+ EventKind.CONCEPT_DRIFT_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
54
+ EventKind.MODEL_PERFORMANCE_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
55
+ EventKind.MODEL_PERFORMANCE_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
56
+ EventKind.MODEL_SERVING_PERFORMANCE_DETECTED: [
57
+ EventEntityKind.MODEL_ENDPOINT_RESULT
58
+ ],
59
+ EventKind.MODEL_SERVING_PERFORMANCE_SUSPECTED: [
60
+ EventEntityKind.MODEL_ENDPOINT_RESULT
61
+ ],
62
+ EventKind.MM_APP_ANOMALY_DETECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
63
+ EventKind.MM_APP_ANOMALY_SUSPECTED: [EventEntityKind.MODEL_ENDPOINT_RESULT],
64
+ EventKind.FAILED: [EventEntityKind.JOB],
65
+ }
66
+
67
+
68
+ class Event(pydantic.BaseModel):
69
+ kind: EventKind
70
+ timestamp: Union[str, datetime] = None # occurrence time
71
+ entity: EventEntities
72
+ value_dict: Optional[dict] = pydantic.Field(default_factory=dict)
73
+
74
+ def is_valid(self):
75
+ return self.entity.kind in _event_kind_entity_map[self.kind]
76
+
77
+
78
+ class AlertActiveState(StrEnum):
79
+ ACTIVE = "active"
80
+ INACTIVE = "inactive"
81
+
82
+
83
+ class AlertSeverity(StrEnum):
84
+ LOW = "low"
85
+ MEDIUM = "medium"
86
+ HIGH = "high"
87
+
88
+
89
+ # what should trigger the alert. must be either event (at least 1), or prometheus query
90
+ class AlertTrigger(pydantic.BaseModel):
91
+ events: list[EventKind] = []
92
+ prometheus_alert: str = None
93
+
94
+ def __eq__(self, other):
95
+ return (
96
+ self.prometheus_alert == other.prometheus_alert
97
+ and self.events == other.events
98
+ )
99
+
100
+
101
+ class AlertCriteria(pydantic.BaseModel):
102
+ count: Annotated[
103
+ int,
104
+ pydantic.Field(
105
+ description="Number of events to wait until notification is sent"
106
+ ),
107
+ ] = 0
108
+ period: Annotated[
109
+ str,
110
+ pydantic.Field(
111
+ description="Time period during which event occurred. e.g. 1d, 3h, 5m, 15s"
112
+ ),
113
+ ] = None
114
+
115
+ def __eq__(self, other):
116
+ return self.count == other.count and self.period == other.period
117
+
118
+
119
+ class ResetPolicy(StrEnum):
120
+ MANUAL = "manual"
121
+ AUTO = "auto"
122
+
123
+
124
+ class AlertNotification(pydantic.BaseModel):
125
+ notification: Notification
126
+ cooldown_period: Annotated[
127
+ str,
128
+ pydantic.Field(
129
+ description="Period during which notifications "
130
+ "will not be sent after initial send. The format of this would be in time."
131
+ " e.g. 1d, 3h, 5m, 15s"
132
+ ),
133
+ ] = None
134
+
135
+
136
+ class AlertConfig(pydantic.BaseModel):
137
+ project: str
138
+ id: int = None
139
+ name: str
140
+ description: Optional[str] = ""
141
+ summary: Annotated[
142
+ str,
143
+ pydantic.Field(
144
+ description=(
145
+ "String to be sent in the notifications generated."
146
+ "e.g. 'Model {{project}}/{{entity}} is drifting.'"
147
+ "Supported variables: project, entity, name"
148
+ )
149
+ ),
150
+ ]
151
+ created: Union[str, datetime] = None
152
+ severity: AlertSeverity
153
+ entities: EventEntities
154
+ trigger: AlertTrigger
155
+ criteria: Optional[AlertCriteria]
156
+ reset_policy: ResetPolicy = ResetPolicy.MANUAL
157
+ notifications: pydantic.conlist(AlertNotification, min_items=1)
158
+ state: AlertActiveState = AlertActiveState.INACTIVE
159
+ count: Optional[int] = 0
160
+
161
+ def get_raw_notifications(self) -> list[Notification]:
162
+ return [
163
+ alert_notification.notification for alert_notification in self.notifications
164
+ ]
165
+
166
+
167
+ class AlertsModes(StrEnum):
168
+ enabled = "enabled"
169
+ disabled = "disabled"
170
+
171
+
172
+ class AlertTemplate(
173
+ pydantic.BaseModel
174
+ ): # Template fields that are not shared with created configs
175
+ template_id: int = None
176
+ template_name: str
177
+ template_description: Optional[str] = (
178
+ "String explaining the purpose of this template"
179
+ )
180
+
181
+ # A property that identifies templates that were created by the system and cannot be modified/deleted by the user
182
+ system_generated: bool = False
183
+
184
+ # AlertConfig fields that are pre-defined
185
+ summary: Optional[str] = (
186
+ "String to be sent in the generated notifications e.g. 'Model {{project}}/{{entity}} is drifting.'"
187
+ "See AlertConfig.summary description"
188
+ )
189
+ severity: AlertSeverity
190
+ trigger: AlertTrigger
191
+ criteria: Optional[AlertCriteria]
192
+ reset_policy: ResetPolicy = ResetPolicy.MANUAL
193
+
194
+ # This is slightly different than __eq__ as it doesn't compare everything
195
+ def templates_differ(self, other):
196
+ return (
197
+ self.template_description != other.template_description
198
+ or self.summary != other.summary
199
+ or self.severity != other.severity
200
+ or self.trigger != other.trigger
201
+ or self.reset_policy != other.reset_policy
202
+ or self.criteria != other.criteria
203
+ )
@@ -0,0 +1,148 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ import typing
16
+ from typing import Optional
17
+
18
+ import pydantic
19
+
20
+ import mlrun.common.types
21
+ from mlrun.common.constants import MLRUN_FUNCTIONS_ANNOTATION
22
+
23
+
24
+ class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
25
+ basic = "basicAuth"
26
+ none = "none"
27
+ access_key = "accessKey"
28
+
29
+ @classmethod
30
+ def from_str(cls, authentication_mode: str):
31
+ if authentication_mode == "none":
32
+ return cls.none
33
+ elif authentication_mode == "basicAuth":
34
+ return cls.basic
35
+ elif authentication_mode == "accessKey":
36
+ return cls.access_key
37
+ else:
38
+ raise mlrun.errors.MLRunInvalidArgumentError(
39
+ f"Authentication mode `{authentication_mode}` is not supported",
40
+ )
41
+
42
+
43
+ class APIGatewayState(mlrun.common.types.StrEnum):
44
+ none = ""
45
+ ready = "ready"
46
+ error = "error"
47
+ waiting_for_provisioning = "waitingForProvisioning"
48
+
49
+
50
+ class _APIGatewayBaseModel(pydantic.BaseModel):
51
+ class Config:
52
+ extra = pydantic.Extra.allow
53
+
54
+
55
+ class APIGatewayMetadata(_APIGatewayBaseModel):
56
+ name: str
57
+ namespace: Optional[str]
58
+ labels: Optional[dict] = {}
59
+ annotations: Optional[dict] = {}
60
+
61
+
62
+ class APIGatewayBasicAuth(_APIGatewayBaseModel):
63
+ username: str
64
+ password: str
65
+
66
+
67
+ class APIGatewayUpstream(_APIGatewayBaseModel):
68
+ kind: Optional[str] = "nucliofunction"
69
+ nucliofunction: dict[str, str]
70
+ percentage: Optional[int] = 0
71
+ port: Optional[int] = 0
72
+
73
+
74
+ class APIGatewaySpec(_APIGatewayBaseModel):
75
+ name: str
76
+ description: Optional[str]
77
+ path: Optional[str] = "/"
78
+ authenticationMode: Optional[APIGatewayAuthenticationMode] = (
79
+ APIGatewayAuthenticationMode.none
80
+ )
81
+ upstreams: list[APIGatewayUpstream]
82
+ authentication: Optional[dict[str, Optional[APIGatewayBasicAuth]]]
83
+ host: Optional[str]
84
+
85
+
86
+ class APIGatewayStatus(_APIGatewayBaseModel):
87
+ name: Optional[str]
88
+ state: Optional[APIGatewayState]
89
+
90
+
91
+ class APIGateway(_APIGatewayBaseModel):
92
+ metadata: APIGatewayMetadata
93
+ spec: APIGatewaySpec
94
+ status: Optional[APIGatewayStatus]
95
+
96
+ def get_function_names(self):
97
+ return [
98
+ upstream.nucliofunction.get("name")
99
+ for upstream in self.spec.upstreams
100
+ if upstream.nucliofunction.get("name")
101
+ ]
102
+
103
+ def enrich_mlrun_function_names(self):
104
+ upstream_with_nuclio_names = []
105
+ mlrun_function_uris = []
106
+ for upstream in self.spec.upstreams:
107
+ uri = upstream.nucliofunction.get("name")
108
+ project, function_name, tag, _ = (
109
+ mlrun.common.helpers.parse_versioned_object_uri(uri)
110
+ )
111
+ upstream.nucliofunction["name"] = (
112
+ mlrun.runtimes.nuclio.function.get_fullname(function_name, project, tag)
113
+ )
114
+
115
+ upstream_with_nuclio_names.append(upstream)
116
+ mlrun_function_uris.append(uri)
117
+
118
+ self.spec.upstreams = upstream_with_nuclio_names
119
+ if len(mlrun_function_uris) == 1:
120
+ self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = mlrun_function_uris[
121
+ 0
122
+ ]
123
+ elif len(mlrun_function_uris) == 2:
124
+ self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = "&".join(
125
+ mlrun_function_uris
126
+ )
127
+ return self
128
+
129
+ def replace_nuclio_names_with_mlrun_uri(self):
130
+ mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
131
+ if mlrun_functions:
132
+ mlrun_function_uris = (
133
+ mlrun_functions.split("&")
134
+ if "&" in mlrun_functions
135
+ else [mlrun_functions]
136
+ )
137
+ if len(mlrun_function_uris) != len(self.spec.upstreams):
138
+ raise mlrun.errors.MLRunValueError(
139
+ "Error when translating nuclio names to mlrun names in api gateway:"
140
+ " number of functions doesn't match the mlrun functions in annotation"
141
+ )
142
+ for i in range(len(mlrun_function_uris)):
143
+ self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
144
+ return self
145
+
146
+
147
+ class APIGatewaysOutput(_APIGatewayBaseModel):
148
+ api_gateways: typing.Optional[dict[str, APIGateway]] = {}
@@ -58,11 +58,6 @@ class ArtifactIdentifier(pydantic.BaseModel):
58
58
  # hash: typing.Optional[str]
59
59
 
60
60
 
61
- class ArtifactsFormat(mlrun.common.types.StrEnum):
62
- # TODO: add a format that returns a minimal response
63
- full = "full"
64
-
65
-
66
61
  class ArtifactMetadata(pydantic.BaseModel):
67
62
  key: str
68
63
  project: str
@@ -93,3 +88,18 @@ class Artifact(pydantic.BaseModel):
93
88
  metadata: ArtifactMetadata
94
89
  spec: ArtifactSpec
95
90
  status: ObjectStatus
91
+
92
+
93
+ class ArtifactsDeletionStrategies(mlrun.common.types.StrEnum):
94
+ """Artifacts deletion strategies types."""
95
+
96
+ metadata_only = "metadata-only"
97
+ """Only removes the artifact db record, leaving all related artifact data in-place"""
98
+
99
+ data_optional = "data-optional"
100
+ """Delete the artifact data of the artifact as a best-effort.
101
+ If artifact data deletion fails still try to delete the artifact db record"""
102
+
103
+ data_force = "data-force"
104
+ """Delete the artifact data, and if cannot delete it fail the deletion
105
+ and don’t delete the artifact db record"""
@@ -58,8 +58,11 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
58
58
  pipeline = "pipeline"
59
59
  hub_source = "hub-source"
60
60
  workflow = "workflow"
61
+ alert = "alert"
62
+ alert_templates = "alert-templates"
63
+ event = "event"
61
64
  datastore_profile = "datastore-profile"
62
- api_gateways = "api-gateways"
65
+ api_gateway = "api-gateway"
63
66
 
64
67
  def to_resource_string(
65
68
  self,
@@ -83,6 +86,9 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
83
86
  AuthorizationResourceTypes.schedule: "/projects/{project_name}/schedules/{resource_name}",
84
87
  AuthorizationResourceTypes.secret: "/projects/{project_name}/secrets/{resource_name}",
85
88
  AuthorizationResourceTypes.run: "/projects/{project_name}/runs/{resource_name}",
89
+ AuthorizationResourceTypes.event: "/projects/{project_name}/events/{resource_name}",
90
+ AuthorizationResourceTypes.alert: "/projects/{project_name}/alerts/{resource_name}",
91
+ AuthorizationResourceTypes.alert_templates: "/alert-templates/{resource_name}",
86
92
  # runtime resource doesn't have an identifier, we don't need any auth granularity behind project level
87
93
  AuthorizationResourceTypes.runtime_resource: "/projects/{project_name}/runtime-resources",
88
94
  AuthorizationResourceTypes.model_endpoint: "/projects/{project_name}/model-endpoints/{resource_name}",
@@ -94,7 +100,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
94
100
  AuthorizationResourceTypes.hub_source: "/marketplace/sources",
95
101
  # workflow define how to run a pipeline and can be considered as the specification of a pipeline.
96
102
  AuthorizationResourceTypes.workflow: "/projects/{project_name}/workflows/{resource_name}",
97
- AuthorizationResourceTypes.api_gateways: "/projects/{project_name}/api-gateways",
103
+ AuthorizationResourceTypes.api_gateway: "/projects/{project_name}/api-gateways/{resource_name}",
98
104
  }[self].format(project_name=project_name, resource_name=resource_name)
99
105
 
100
106
 
@@ -59,6 +59,7 @@ class ClientSpec(pydantic.BaseModel):
59
59
  sql_url: typing.Optional[str]
60
60
  model_endpoint_monitoring_store_type: typing.Optional[str]
61
61
  model_endpoint_monitoring_endpoint_store_connection: typing.Optional[str]
62
+ model_monitoring_tsdb_connection: typing.Optional[str]
62
63
  ce: typing.Optional[dict]
63
64
  # not passing them as one object as it possible client user would like to override only one of the params
64
65
  calculate_artifact_hash: typing.Optional[str]
@@ -66,3 +67,4 @@ class ClientSpec(pydantic.BaseModel):
66
67
  logs: typing.Optional[dict]
67
68
  packagers: typing.Optional[dict]
68
69
  external_platform_tracking: typing.Optional[dict]
70
+ alerts_mode: typing.Optional[str]
@@ -70,3 +70,4 @@ class FrontendSpec(pydantic.BaseModel):
70
70
  feature_store_data_prefixes: typing.Optional[dict[str, str]]
71
71
  allowed_artifact_path_prefixes_list: list[str]
72
72
  ce: typing.Optional[dict]
73
+ internal_labels: list[str] = []