mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (184) hide show
  1. mlrun/__init__.py +2 -35
  2. mlrun/__main__.py +3 -41
  3. mlrun/api/api/api.py +6 -0
  4. mlrun/api/api/endpoints/feature_store.py +0 -4
  5. mlrun/api/api/endpoints/files.py +14 -2
  6. mlrun/api/api/endpoints/frontend_spec.py +2 -1
  7. mlrun/api/api/endpoints/functions.py +95 -59
  8. mlrun/api/api/endpoints/grafana_proxy.py +9 -9
  9. mlrun/api/api/endpoints/logs.py +17 -3
  10. mlrun/api/api/endpoints/model_endpoints.py +3 -2
  11. mlrun/api/api/endpoints/pipelines.py +1 -5
  12. mlrun/api/api/endpoints/projects.py +88 -0
  13. mlrun/api/api/endpoints/runs.py +48 -6
  14. mlrun/api/api/endpoints/submit.py +2 -1
  15. mlrun/api/api/endpoints/workflows.py +355 -0
  16. mlrun/api/api/utils.py +3 -4
  17. mlrun/api/crud/__init__.py +1 -0
  18. mlrun/api/crud/client_spec.py +6 -2
  19. mlrun/api/crud/feature_store.py +5 -0
  20. mlrun/api/crud/model_monitoring/__init__.py +1 -0
  21. mlrun/api/crud/model_monitoring/deployment.py +497 -0
  22. mlrun/api/crud/model_monitoring/grafana.py +96 -42
  23. mlrun/api/crud/model_monitoring/helpers.py +159 -0
  24. mlrun/api/crud/model_monitoring/model_endpoints.py +202 -476
  25. mlrun/api/crud/notifications.py +9 -4
  26. mlrun/api/crud/pipelines.py +6 -11
  27. mlrun/api/crud/projects.py +2 -2
  28. mlrun/api/crud/runtime_resources.py +4 -3
  29. mlrun/api/crud/runtimes/nuclio/helpers.py +5 -1
  30. mlrun/api/crud/secrets.py +21 -0
  31. mlrun/api/crud/workflows.py +352 -0
  32. mlrun/api/db/base.py +16 -1
  33. mlrun/api/db/init_db.py +2 -4
  34. mlrun/api/db/session.py +1 -1
  35. mlrun/api/db/sqldb/db.py +129 -31
  36. mlrun/api/db/sqldb/models/models_mysql.py +15 -1
  37. mlrun/api/db/sqldb/models/models_sqlite.py +16 -2
  38. mlrun/api/launcher.py +38 -6
  39. mlrun/api/main.py +3 -2
  40. mlrun/api/rundb/__init__.py +13 -0
  41. mlrun/{db → api/rundb}/sqldb.py +36 -84
  42. mlrun/api/runtime_handlers/__init__.py +56 -0
  43. mlrun/api/runtime_handlers/base.py +1247 -0
  44. mlrun/api/runtime_handlers/daskjob.py +209 -0
  45. mlrun/api/runtime_handlers/kubejob.py +37 -0
  46. mlrun/api/runtime_handlers/mpijob.py +147 -0
  47. mlrun/api/runtime_handlers/remotesparkjob.py +29 -0
  48. mlrun/api/runtime_handlers/sparkjob.py +148 -0
  49. mlrun/api/schemas/__init__.py +17 -6
  50. mlrun/api/utils/builder.py +1 -4
  51. mlrun/api/utils/clients/chief.py +14 -0
  52. mlrun/api/utils/clients/iguazio.py +33 -33
  53. mlrun/api/utils/clients/nuclio.py +2 -2
  54. mlrun/api/utils/periodic.py +9 -2
  55. mlrun/api/utils/projects/follower.py +14 -7
  56. mlrun/api/utils/projects/leader.py +2 -1
  57. mlrun/api/utils/projects/remotes/nop_follower.py +2 -2
  58. mlrun/api/utils/projects/remotes/nop_leader.py +2 -2
  59. mlrun/api/utils/runtimes/__init__.py +14 -0
  60. mlrun/api/utils/runtimes/nuclio.py +43 -0
  61. mlrun/api/utils/scheduler.py +98 -15
  62. mlrun/api/utils/singletons/db.py +5 -1
  63. mlrun/api/utils/singletons/project_member.py +4 -1
  64. mlrun/api/utils/singletons/scheduler.py +1 -1
  65. mlrun/artifacts/base.py +6 -6
  66. mlrun/artifacts/dataset.py +4 -4
  67. mlrun/artifacts/manager.py +2 -3
  68. mlrun/artifacts/model.py +2 -2
  69. mlrun/artifacts/plots.py +8 -8
  70. mlrun/common/db/__init__.py +14 -0
  71. mlrun/common/helpers.py +37 -0
  72. mlrun/{mlutils → common/model_monitoring}/__init__.py +3 -2
  73. mlrun/common/model_monitoring/helpers.py +69 -0
  74. mlrun/common/schemas/__init__.py +13 -1
  75. mlrun/common/schemas/auth.py +4 -1
  76. mlrun/common/schemas/client_spec.py +1 -1
  77. mlrun/common/schemas/function.py +17 -0
  78. mlrun/common/schemas/model_monitoring/__init__.py +48 -0
  79. mlrun/common/{model_monitoring.py → schemas/model_monitoring/constants.py} +11 -23
  80. mlrun/common/schemas/model_monitoring/grafana.py +55 -0
  81. mlrun/common/schemas/{model_endpoints.py → model_monitoring/model_endpoints.py} +32 -65
  82. mlrun/common/schemas/notification.py +1 -0
  83. mlrun/common/schemas/object.py +4 -0
  84. mlrun/common/schemas/project.py +1 -0
  85. mlrun/common/schemas/regex.py +1 -1
  86. mlrun/common/schemas/runs.py +1 -8
  87. mlrun/common/schemas/schedule.py +1 -8
  88. mlrun/common/schemas/workflow.py +54 -0
  89. mlrun/config.py +45 -42
  90. mlrun/datastore/__init__.py +21 -0
  91. mlrun/datastore/base.py +1 -1
  92. mlrun/datastore/datastore.py +9 -0
  93. mlrun/datastore/dbfs_store.py +168 -0
  94. mlrun/datastore/helpers.py +18 -0
  95. mlrun/datastore/sources.py +1 -0
  96. mlrun/datastore/store_resources.py +2 -5
  97. mlrun/datastore/v3io.py +1 -2
  98. mlrun/db/__init__.py +4 -68
  99. mlrun/db/base.py +12 -0
  100. mlrun/db/factory.py +65 -0
  101. mlrun/db/httpdb.py +175 -20
  102. mlrun/db/nopdb.py +4 -2
  103. mlrun/execution.py +4 -2
  104. mlrun/feature_store/__init__.py +1 -0
  105. mlrun/feature_store/api.py +1 -2
  106. mlrun/feature_store/common.py +2 -1
  107. mlrun/feature_store/feature_set.py +1 -11
  108. mlrun/feature_store/feature_vector.py +340 -2
  109. mlrun/feature_store/ingestion.py +5 -10
  110. mlrun/feature_store/retrieval/base.py +118 -104
  111. mlrun/feature_store/retrieval/dask_merger.py +17 -10
  112. mlrun/feature_store/retrieval/job.py +4 -1
  113. mlrun/feature_store/retrieval/local_merger.py +18 -18
  114. mlrun/feature_store/retrieval/spark_merger.py +21 -14
  115. mlrun/feature_store/retrieval/storey_merger.py +22 -16
  116. mlrun/kfpops.py +3 -9
  117. mlrun/launcher/base.py +57 -53
  118. mlrun/launcher/client.py +5 -4
  119. mlrun/launcher/factory.py +24 -13
  120. mlrun/launcher/local.py +6 -6
  121. mlrun/launcher/remote.py +4 -4
  122. mlrun/lists.py +0 -11
  123. mlrun/model.py +11 -17
  124. mlrun/model_monitoring/__init__.py +2 -22
  125. mlrun/model_monitoring/features_drift_table.py +1 -1
  126. mlrun/model_monitoring/helpers.py +22 -210
  127. mlrun/model_monitoring/model_endpoint.py +1 -1
  128. mlrun/model_monitoring/model_monitoring_batch.py +127 -50
  129. mlrun/model_monitoring/prometheus.py +219 -0
  130. mlrun/model_monitoring/stores/__init__.py +16 -11
  131. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +95 -23
  132. mlrun/model_monitoring/stores/models/mysql.py +47 -29
  133. mlrun/model_monitoring/stores/models/sqlite.py +47 -29
  134. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +31 -19
  135. mlrun/model_monitoring/{stream_processing_fs.py → stream_processing.py} +206 -64
  136. mlrun/model_monitoring/tracking_policy.py +104 -0
  137. mlrun/package/packager.py +6 -8
  138. mlrun/package/packagers/default_packager.py +121 -10
  139. mlrun/package/packagers/numpy_packagers.py +1 -1
  140. mlrun/platforms/__init__.py +0 -2
  141. mlrun/platforms/iguazio.py +0 -56
  142. mlrun/projects/pipelines.py +53 -159
  143. mlrun/projects/project.py +10 -37
  144. mlrun/render.py +1 -1
  145. mlrun/run.py +8 -124
  146. mlrun/runtimes/__init__.py +6 -42
  147. mlrun/runtimes/base.py +29 -1249
  148. mlrun/runtimes/daskjob.py +2 -198
  149. mlrun/runtimes/funcdoc.py +0 -9
  150. mlrun/runtimes/function.py +25 -29
  151. mlrun/runtimes/kubejob.py +5 -29
  152. mlrun/runtimes/local.py +1 -1
  153. mlrun/runtimes/mpijob/__init__.py +2 -2
  154. mlrun/runtimes/mpijob/abstract.py +10 -1
  155. mlrun/runtimes/mpijob/v1.py +0 -76
  156. mlrun/runtimes/mpijob/v1alpha1.py +1 -74
  157. mlrun/runtimes/nuclio.py +3 -2
  158. mlrun/runtimes/pod.py +28 -18
  159. mlrun/runtimes/remotesparkjob.py +1 -15
  160. mlrun/runtimes/serving.py +14 -6
  161. mlrun/runtimes/sparkjob/__init__.py +0 -1
  162. mlrun/runtimes/sparkjob/abstract.py +4 -131
  163. mlrun/runtimes/utils.py +0 -26
  164. mlrun/serving/routers.py +7 -7
  165. mlrun/serving/server.py +11 -8
  166. mlrun/serving/states.py +7 -1
  167. mlrun/serving/v2_serving.py +6 -6
  168. mlrun/utils/helpers.py +23 -42
  169. mlrun/utils/notifications/notification/__init__.py +4 -0
  170. mlrun/utils/notifications/notification/webhook.py +61 -0
  171. mlrun/utils/notifications/notification_pusher.py +5 -25
  172. mlrun/utils/regex.py +7 -2
  173. mlrun/utils/version/version.json +2 -2
  174. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/METADATA +26 -25
  175. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/RECORD +180 -158
  176. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/WHEEL +1 -1
  177. mlrun/mlutils/data.py +0 -160
  178. mlrun/mlutils/models.py +0 -78
  179. mlrun/mlutils/plots.py +0 -902
  180. mlrun/utils/model_monitoring.py +0 -249
  181. /mlrun/{api/db/sqldb/session.py → common/db/sql_session.py} +0 -0
  182. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/LICENSE +0 -0
  183. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/entry_points.txt +0 -0
  184. {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,219 @@
1
+ # Copyright 2023 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ import typing
16
+
17
+ import prometheus_client
18
+
19
+ from mlrun.common.schemas.model_monitoring import EventFieldType, PrometheusMetric
20
+
21
+ # Memory path for Prometheus registry file
22
+ _registry_path = "/tmp/prom-reg.txt"
23
+
24
+ # Initializing Promethues metric collector registry
25
+ _registry: prometheus_client.CollectorRegistry = prometheus_client.CollectorRegistry()
26
+
27
+ # The following real-time metrics are being updated through the monitoring stream graph steps
28
+ _prediction_counter: prometheus_client.Counter = prometheus_client.Counter(
29
+ name=PrometheusMetric.PREDICTIONS_TOTAL,
30
+ documentation="Counter for total predictions",
31
+ registry=_registry,
32
+ labelnames=[
33
+ EventFieldType.PROJECT,
34
+ EventFieldType.ENDPOINT_ID,
35
+ EventFieldType.MODEL,
36
+ EventFieldType.ENDPOINT_TYPE,
37
+ ],
38
+ )
39
+ _model_latency: prometheus_client.Summary = prometheus_client.Summary(
40
+ name=PrometheusMetric.MODEL_LATENCY_SECONDS,
41
+ documentation="Summary for for model latency",
42
+ registry=_registry,
43
+ labelnames=[
44
+ EventFieldType.PROJECT,
45
+ EventFieldType.ENDPOINT_ID,
46
+ EventFieldType.MODEL,
47
+ EventFieldType.ENDPOINT_TYPE,
48
+ ],
49
+ )
50
+ _income_features: prometheus_client.Gauge = prometheus_client.Gauge(
51
+ name=PrometheusMetric.INCOME_FEATURES,
52
+ documentation="Samples of features and predictions",
53
+ registry=_registry,
54
+ labelnames=[
55
+ EventFieldType.PROJECT,
56
+ EventFieldType.ENDPOINT_ID,
57
+ EventFieldType.METRIC,
58
+ ],
59
+ )
60
+ _error_counter: prometheus_client.Counter = prometheus_client.Counter(
61
+ name=PrometheusMetric.ERRORS_TOTAL,
62
+ documentation="Counter for total errors",
63
+ registry=_registry,
64
+ labelnames=[
65
+ EventFieldType.PROJECT,
66
+ EventFieldType.ENDPOINT_ID,
67
+ EventFieldType.MODEL,
68
+ ],
69
+ )
70
+
71
+ # The following metrics are being updated through the model monitoring batch job
72
+ _batch_metrics: prometheus_client.Gauge = prometheus_client.Gauge(
73
+ name=PrometheusMetric.DRIFT_METRICS,
74
+ documentation="Results from the batch drift analysis",
75
+ registry=_registry,
76
+ labelnames=[
77
+ EventFieldType.PROJECT,
78
+ EventFieldType.ENDPOINT_ID,
79
+ EventFieldType.METRIC,
80
+ ],
81
+ )
82
+ _drift_status: prometheus_client.Enum = prometheus_client.Enum(
83
+ name=PrometheusMetric.DRIFT_STATUS,
84
+ documentation="Drift status of the model endpoint",
85
+ registry=_registry,
86
+ states=["NO_DRIFT", "DRIFT_DETECTED", "POSSIBLE_DRIFT"],
87
+ labelnames=[EventFieldType.PROJECT, EventFieldType.ENDPOINT_ID],
88
+ )
89
+
90
+
91
+ def _write_registry(func):
92
+ def wrapper(*args, **kwargs):
93
+ global _registry
94
+ """A wrapper function to update the registry file each time a metric has been updated"""
95
+ func(*args, **kwargs)
96
+ prometheus_client.write_to_textfile(path=_registry_path, registry=_registry)
97
+
98
+ return wrapper
99
+
100
+
101
+ @_write_registry
102
+ def write_predictions_and_latency_metrics(
103
+ project: str, endpoint_id: str, latency: int, model_name: str, endpoint_type: int
104
+ ):
105
+ """
106
+ Update the prediction counter and the latency value of the provided model endpoint within Prometheus registry.
107
+ Please note that while the prediction counter is ALWAYS increasing by 1,the latency summary metric is being
108
+ increased by the event latency time. Grafana dashboard will query the average latency time by dividing the total
109
+ latency value by the total amount of predictions.
110
+
111
+ :param project: Project name.
112
+ :param endpoint_id: Model endpoint unique id.
113
+ :param latency: Latency time (microsecond) in which the event has been processed through the model server.
114
+ :param model_name: Model name which will be used by Grafana for displaying the results by model.
115
+ :param endpoint_type: Endpoint type that is represented by an int (possible values: 1,2,3) corresponding to the
116
+ Enum class :py:class:`~mlrun.common.schemas.model_monitoring.EndpointType`.
117
+ """
118
+
119
+ # Increase the prediction counter by 1
120
+ _prediction_counter.labels(
121
+ project=project,
122
+ endpoint_id=endpoint_id,
123
+ model=model_name,
124
+ endpoint_type=endpoint_type,
125
+ ).inc(1)
126
+
127
+ # Increase the latency value according to the provided latency of the current event
128
+ _model_latency.labels(
129
+ project=project,
130
+ endpoint_id=endpoint_id,
131
+ model=model_name,
132
+ endpoint_type=endpoint_type,
133
+ ).observe(latency)
134
+
135
+
136
+ @_write_registry
137
+ def write_income_features(
138
+ project: str, endpoint_id: str, features: typing.Dict[str, float]
139
+ ):
140
+ """Update a sample of features.
141
+
142
+ :param project: Project name.
143
+ :param endpoint_id: Model endpoint unique id.
144
+ :param features: Dictionary in which the key is a feature name and the value is a float number.
145
+
146
+
147
+ """
148
+
149
+ for metric in features:
150
+ _income_features.labels(
151
+ project=project, endpoint_id=endpoint_id, metric=metric
152
+ ).set(value=features[metric])
153
+
154
+
155
+ @_write_registry
156
+ def write_drift_metrics(project: str, endpoint_id: str, metric: str, value: float):
157
+ """Update drift metrics that have been calculated through the monitoring batch job
158
+
159
+ :param project: Project name.
160
+ :param endpoint_id: Model endpoint unique id.
161
+ :param metric: Metric name (e.g. TVD, Hellinger).
162
+ :param value: Metric value as a float.
163
+
164
+ """
165
+
166
+ _batch_metrics.labels(project=project, endpoint_id=endpoint_id, metric=metric).set(
167
+ value=value
168
+ )
169
+
170
+
171
+ @_write_registry
172
+ def write_drift_status(project: str, endpoint_id: str, drift_status: str):
173
+ """
174
+ Update the drift status enum for a specific model endpoint.
175
+
176
+ :param project: Project name.
177
+ :param endpoint_id: Model endpoint unique id.
178
+ :param drift_status: Drift status value, can be one of the following: 'NO_DRIFT', 'DRIFT_DETECTED', or
179
+ 'POSSIBLE_DRIFT'.
180
+ """
181
+
182
+ _drift_status.labels(project=project, endpoint_id=endpoint_id).state(drift_status)
183
+
184
+
185
+ @_write_registry
186
+ def write_errors(project: str, endpoint_id: str, model_name: str):
187
+ """
188
+ Update the error counter for a specific model endpoint.
189
+
190
+ :param project: Project name.
191
+ :param endpoint_id: Model endpoint unique id.
192
+ :param model_name: Model name. Will be used by Grafana to show the amount of errors per model by time.
193
+ """
194
+
195
+ _error_counter.labels(
196
+ project=project, endpoint_id=endpoint_id, model=model_name
197
+ ).inc(1)
198
+
199
+
200
+ def get_registry() -> str:
201
+ """Returns the parsed registry file according to the exposition format of Prometheus."""
202
+
203
+ # Read the registry file (note that the text is stored in UTF-8 format)
204
+ f = open(_registry_path)
205
+ lines = f.read()
206
+ f.close()
207
+
208
+ # Reset part of the metrics to avoid a repeating scraping of the same value
209
+ clean_metrics()
210
+
211
+ return lines
212
+
213
+
214
+ @_write_registry
215
+ def clean_metrics():
216
+ """Clean the income features values. As these results are relevant only for a certain timestamp, we will remove
217
+ them from the global registry after they have been scraped by Prometheus."""
218
+
219
+ _income_features.clear()
@@ -17,7 +17,8 @@
17
17
  import enum
18
18
  import typing
19
19
 
20
- import mlrun
20
+ import mlrun.common.schemas.secret
21
+ import mlrun.errors
21
22
 
22
23
  from .model_endpoint_store import ModelEndpointStore
23
24
 
@@ -33,6 +34,7 @@ class ModelEndpointStoreType(enum.Enum):
33
34
  project: str,
34
35
  access_key: str = None,
35
36
  endpoint_store_connection: str = None,
37
+ secret_provider: typing.Callable = None,
36
38
  ) -> ModelEndpointStore:
37
39
  """
38
40
  Return a ModelEndpointStore object based on the provided enum value.
@@ -46,13 +48,13 @@ class ModelEndpointStoreType(enum.Enum):
46
48
  e.g. A root user with password 1234, tries to connect a schema called
47
49
  mlrun within a local MySQL DB instance:
48
50
  'mysql+pymysql://root:1234@localhost:3306/mlrun'.
51
+ :param secret_provider: An optional secret provider to get the connection string secret.
49
52
 
50
53
  :return: `ModelEndpointStore` object.
51
54
 
52
55
  """
53
56
 
54
57
  if self.value == ModelEndpointStoreType.v3io_nosql.value:
55
-
56
58
  from .kv_model_endpoint_store import KVModelEndpointStore
57
59
 
58
60
  # Get V3IO access key from env
@@ -62,15 +64,13 @@ class ModelEndpointStoreType(enum.Enum):
62
64
 
63
65
  # Assuming SQL store target if store type is not KV.
64
66
  # Update these lines once there are more than two store target types.
65
- from mlrun.utils.model_monitoring import get_connection_string
66
67
 
67
- sql_connection_string = endpoint_store_connection or get_connection_string(
68
- project=project
69
- )
70
68
  from .sql_model_endpoint_store import SQLModelEndpointStore
71
69
 
72
70
  return SQLModelEndpointStore(
73
- project=project, sql_connection_string=sql_connection_string
71
+ project=project,
72
+ sql_connection_string=endpoint_store_connection,
73
+ secret_provider=secret_provider,
74
74
  )
75
75
 
76
76
  @classmethod
@@ -85,13 +85,16 @@ class ModelEndpointStoreType(enum.Enum):
85
85
 
86
86
 
87
87
  def get_model_endpoint_store(
88
- project: str, access_key: str = None
88
+ project: str,
89
+ access_key: str = None,
90
+ secret_provider: typing.Callable = None,
89
91
  ) -> ModelEndpointStore:
90
92
  """
91
93
  Getting the DB target type based on mlrun.config.model_endpoint_monitoring.store_type.
92
94
 
93
- :param project: The name of the project.
94
- :param access_key: Access key with permission to the DB table.
95
+ :param project: The name of the project.
96
+ :param access_key: Access key with permission to the DB table.
97
+ :param secret_provider: An optional secret provider to get the connection string secret.
95
98
 
96
99
  :return: `ModelEndpointStore` object. Using this object, the user can apply different operations on the
97
100
  model endpoint record such as write, update, get and delete.
@@ -103,4 +106,6 @@ def get_model_endpoint_store(
103
106
  )
104
107
 
105
108
  # Convert into model endpoint store target object
106
- return model_endpoint_store_type.to_endpoint_store(project, access_key)
109
+ return model_endpoint_store_type.to_endpoint_store(
110
+ project=project, access_key=access_key, secret_provider=secret_provider
111
+ )
@@ -13,15 +13,16 @@
13
13
  # limitations under the License.
14
14
  #
15
15
 
16
+ import json
16
17
  import os
17
18
  import typing
19
+ import warnings
18
20
 
19
21
  import v3io.dataplane
20
22
  import v3io_frames
21
23
 
22
- import mlrun
23
- import mlrun.common.model_monitoring as model_monitoring_constants
24
- import mlrun.utils.model_monitoring
24
+ import mlrun.common.model_monitoring.helpers
25
+ import mlrun.common.schemas.model_monitoring
25
26
  import mlrun.utils.v3io_clients
26
27
  from mlrun.utils import logger
27
28
 
@@ -54,7 +55,7 @@ class KVModelEndpointStore(ModelEndpointStore):
54
55
  self.client.kv.put(
55
56
  container=self.container,
56
57
  table_path=self.path,
57
- key=endpoint[model_monitoring_constants.EventFieldType.UID],
58
+ key=endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.UID],
58
59
  attributes=endpoint,
59
60
  )
60
61
 
@@ -121,7 +122,7 @@ class KVModelEndpointStore(ModelEndpointStore):
121
122
  raise mlrun.errors.MLRunNotFoundError(f"Endpoint {endpoint_id} not found")
122
123
 
123
124
  # For backwards compatability: replace null values for `error_count` and `metrics`
124
- mlrun.utils.model_monitoring.validate_old_schema_fields(endpoint=endpoint)
125
+ self.validate_old_schema_fields(endpoint=endpoint)
125
126
 
126
127
  return endpoint
127
128
 
@@ -129,13 +130,15 @@ class KVModelEndpointStore(ModelEndpointStore):
129
130
  """Getting path and container based on the model monitoring configurations"""
130
131
  path = mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
131
132
  project=self.project,
132
- kind=model_monitoring_constants.ModelMonitoringStoreKinds.ENDPOINTS,
133
+ kind=mlrun.common.schemas.ModelMonitoringStoreKinds.ENDPOINTS,
133
134
  )
134
135
  (
135
136
  _,
136
137
  container,
137
138
  path,
138
- ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(path)
139
+ ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
140
+ path
141
+ )
139
142
  return path, container
140
143
 
141
144
  def list_model_endpoints(
@@ -190,13 +193,17 @@ class KVModelEndpointStore(ModelEndpointStore):
190
193
  if uids is None:
191
194
  uids = []
192
195
  for item in items:
193
- if model_monitoring_constants.EventFieldType.UID not in item:
196
+ if mlrun.common.schemas.model_monitoring.EventFieldType.UID not in item:
194
197
  # This is kept for backwards compatibility - in old versions the key column named endpoint_id
195
198
  uids.append(
196
- item[model_monitoring_constants.EventFieldType.ENDPOINT_ID]
199
+ item[
200
+ mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
201
+ ]
197
202
  )
198
203
  else:
199
- uids.append(item[model_monitoring_constants.EventFieldType.UID])
204
+ uids.append(
205
+ item[mlrun.common.schemas.model_monitoring.EventFieldType.UID]
206
+ )
200
207
 
201
208
  # Add each relevant model endpoint to the model endpoints list
202
209
  for endpoint_id in uids:
@@ -218,14 +225,17 @@ class KVModelEndpointStore(ModelEndpointStore):
218
225
 
219
226
  # Delete model endpoint record from KV table
220
227
  for endpoint_dict in endpoints:
221
- if model_monitoring_constants.EventFieldType.UID not in endpoint_dict:
228
+ if (
229
+ mlrun.common.schemas.model_monitoring.EventFieldType.UID
230
+ not in endpoint_dict
231
+ ):
222
232
  # This is kept for backwards compatibility - in old versions the key column named endpoint_id
223
233
  endpoint_id = endpoint_dict[
224
- model_monitoring_constants.EventFieldType.ENDPOINT_ID
234
+ mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
225
235
  ]
226
236
  else:
227
237
  endpoint_id = endpoint_dict[
228
- model_monitoring_constants.EventFieldType.UID
238
+ mlrun.common.schemas.model_monitoring.EventFieldType.UID
229
239
  ]
230
240
  self.delete_model_endpoint(
231
241
  endpoint_id,
@@ -262,7 +272,7 @@ class KVModelEndpointStore(ModelEndpointStore):
262
272
  # Delete time series DB resources
263
273
  try:
264
274
  frames.delete(
265
- backend=model_monitoring_constants.TimeSeriesTarget.TSDB,
275
+ backend=mlrun.common.schemas.model_monitoring.TimeSeriesTarget.TSDB,
266
276
  table=filtered_path,
267
277
  )
268
278
  except (v3io_frames.errors.DeleteError, v3io_frames.errors.CreateError) as e:
@@ -319,14 +329,16 @@ class KVModelEndpointStore(ModelEndpointStore):
319
329
  events_path = (
320
330
  mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
321
331
  project=self.project,
322
- kind=model_monitoring_constants.ModelMonitoringStoreKinds.EVENTS,
332
+ kind=mlrun.common.schemas.ModelMonitoringStoreKinds.EVENTS,
323
333
  )
324
334
  )
325
335
  (
326
336
  _,
327
337
  container,
328
338
  events_path,
329
- ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(events_path)
339
+ ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
340
+ events_path
341
+ )
330
342
 
331
343
  # Retrieve the raw data from the time series DB based on the provided metrics and time ranges
332
344
  frames_client = mlrun.utils.v3io_clients.get_frames_client(
@@ -337,7 +349,7 @@ class KVModelEndpointStore(ModelEndpointStore):
337
349
 
338
350
  try:
339
351
  data = frames_client.read(
340
- backend=model_monitoring_constants.TimeSeriesTarget.TSDB,
352
+ backend=mlrun.common.schemas.model_monitoring.TimeSeriesTarget.TSDB,
341
353
  table=events_path,
342
354
  columns=["endpoint_id", *metrics],
343
355
  filter=f"endpoint_id=='{endpoint_id}'",
@@ -372,13 +384,15 @@ class KVModelEndpointStore(ModelEndpointStore):
372
384
  full_path = (
373
385
  mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
374
386
  project=self.project,
375
- kind=model_monitoring_constants.ModelMonitoringStoreKinds.EVENTS,
387
+ kind=mlrun.common.schemas.ModelMonitoringStoreKinds.EVENTS,
376
388
  )
377
389
  )
378
390
 
379
391
  # Generate the main directory with the TSDB resources
380
- tsdb_path = mlrun.utils.model_monitoring.parse_model_endpoint_project_prefix(
381
- full_path, self.project
392
+ tsdb_path = (
393
+ mlrun.common.model_monitoring.helpers.parse_model_endpoint_project_prefix(
394
+ full_path, self.project
395
+ )
382
396
  )
383
397
 
384
398
  # Generate filtered path without schema and container as required by the frames object
@@ -386,7 +400,9 @@ class KVModelEndpointStore(ModelEndpointStore):
386
400
  _,
387
401
  _,
388
402
  filtered_path,
389
- ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(full_path)
403
+ ) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
404
+ full_path
405
+ )
390
406
  return tsdb_path, filtered_path
391
407
 
392
408
  @staticmethod
@@ -441,8 +457,64 @@ class KVModelEndpointStore(ModelEndpointStore):
441
457
  # Apply top_level filter (remove endpoints that considered a child of a router)
442
458
  if top_level:
443
459
  filter_expression.append(
444
- f"(endpoint_type=='{str(model_monitoring_constants.EndpointType.NODE_EP.value)}' "
445
- f"OR endpoint_type=='{str(model_monitoring_constants.EndpointType.ROUTER.value)}')"
460
+ f"(endpoint_type=='{str(mlrun.common.schemas.model_monitoring.EndpointType.NODE_EP.value)}' "
461
+ f"OR endpoint_type=='{str(mlrun.common.schemas.model_monitoring.EndpointType.ROUTER.value)}')"
446
462
  )
447
463
 
448
464
  return " AND ".join(filter_expression)
465
+
466
+ @staticmethod
467
+ def validate_old_schema_fields(endpoint: dict):
468
+ """
469
+ Replace default null values for `error_count` and `metrics` for users that logged a model endpoint before 1.3.0.
470
+ In addition, this function also validates that the key name of the endpoint unique id is `uid` and not
471
+ `endpoint_id` that has been used before 1.3.0.
472
+
473
+ Leaving here for backwards compatibility which related to the model endpoint schema.
474
+
475
+ :param endpoint: An endpoint flattened dictionary.
476
+ """
477
+ warnings.warn(
478
+ "This will be deprecated in 1.3.0, and will be removed in 1.5.0",
479
+ # TODO: In 1.3.0 do changes in examples & demos In 1.5.0 remove
480
+ FutureWarning,
481
+ )
482
+
483
+ # Validate default value for `error_count`
484
+ # For backwards compatibility reasons, we validate that the model endpoint includes the `error_count` key
485
+ if (
486
+ mlrun.common.schemas.model_monitoring.EventFieldType.ERROR_COUNT in endpoint
487
+ and endpoint[
488
+ mlrun.common.schemas.model_monitoring.EventFieldType.ERROR_COUNT
489
+ ]
490
+ == "null"
491
+ ):
492
+ endpoint[
493
+ mlrun.common.schemas.model_monitoring.EventFieldType.ERROR_COUNT
494
+ ] = "0"
495
+
496
+ # Validate default value for `metrics`
497
+ # For backwards compatibility reasons, we validate that the model endpoint includes the `metrics` key
498
+ if (
499
+ mlrun.common.schemas.model_monitoring.EventFieldType.METRICS in endpoint
500
+ and endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.METRICS]
501
+ == "null"
502
+ ):
503
+ endpoint[
504
+ mlrun.common.schemas.model_monitoring.EventFieldType.METRICS
505
+ ] = json.dumps(
506
+ {
507
+ mlrun.common.schemas.model_monitoring.EventKeyMetrics.GENERIC: {
508
+ mlrun.common.schemas.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
509
+ mlrun.common.schemas.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
510
+ }
511
+ }
512
+ )
513
+ # Validate key `uid` instead of `endpoint_id`
514
+ # For backwards compatibility reasons, we replace the `endpoint_id` with `uid` which is the updated key name
515
+ if mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID in endpoint:
516
+ endpoint[
517
+ mlrun.common.schemas.model_monitoring.EventFieldType.UID
518
+ ] = endpoint[
519
+ mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
520
+ ]
@@ -17,84 +17,102 @@
17
17
  import sqlalchemy.dialects
18
18
  from sqlalchemy import Boolean, Column, Integer, String, Text
19
19
 
20
- import mlrun.common.model_monitoring as model_monitoring_constants
20
+ import mlrun.common.schemas.model_monitoring
21
21
  from mlrun.utils.db import BaseModel
22
22
 
23
23
  from .base import Base
24
24
 
25
25
 
26
26
  class ModelEndpointsTable(Base, BaseModel):
27
- __tablename__ = model_monitoring_constants.EventFieldType.MODEL_ENDPOINTS
27
+ __tablename__ = mlrun.common.schemas.model_monitoring.EventFieldType.MODEL_ENDPOINTS
28
28
 
29
29
  uid = Column(
30
- model_monitoring_constants.EventFieldType.UID,
30
+ mlrun.common.schemas.model_monitoring.EventFieldType.UID,
31
31
  String(40),
32
32
  primary_key=True,
33
33
  )
34
- state = Column(model_monitoring_constants.EventFieldType.STATE, String(10))
35
- project = Column(model_monitoring_constants.EventFieldType.PROJECT, String(40))
34
+ state = Column(
35
+ mlrun.common.schemas.model_monitoring.EventFieldType.STATE, String(10)
36
+ )
37
+ project = Column(
38
+ mlrun.common.schemas.model_monitoring.EventFieldType.PROJECT, String(40)
39
+ )
36
40
  function_uri = Column(
37
- model_monitoring_constants.EventFieldType.FUNCTION_URI,
41
+ mlrun.common.schemas.model_monitoring.EventFieldType.FUNCTION_URI,
38
42
  String(255),
39
43
  )
40
- model = Column(model_monitoring_constants.EventFieldType.MODEL, String(255))
44
+ model = Column(
45
+ mlrun.common.schemas.model_monitoring.EventFieldType.MODEL, String(255)
46
+ )
41
47
  model_class = Column(
42
- model_monitoring_constants.EventFieldType.MODEL_CLASS,
48
+ mlrun.common.schemas.model_monitoring.EventFieldType.MODEL_CLASS,
43
49
  String(255),
44
50
  )
45
- labels = Column(model_monitoring_constants.EventFieldType.LABELS, Text)
46
- model_uri = Column(model_monitoring_constants.EventFieldType.MODEL_URI, String(255))
47
- stream_path = Column(model_monitoring_constants.EventFieldType.STREAM_PATH, Text)
51
+ labels = Column(mlrun.common.schemas.model_monitoring.EventFieldType.LABELS, Text)
52
+ model_uri = Column(
53
+ mlrun.common.schemas.model_monitoring.EventFieldType.MODEL_URI, String(255)
54
+ )
55
+ stream_path = Column(
56
+ mlrun.common.schemas.model_monitoring.EventFieldType.STREAM_PATH, Text
57
+ )
48
58
  algorithm = Column(
49
- model_monitoring_constants.EventFieldType.ALGORITHM,
59
+ mlrun.common.schemas.model_monitoring.EventFieldType.ALGORITHM,
50
60
  String(255),
51
61
  )
52
- active = Column(model_monitoring_constants.EventFieldType.ACTIVE, Boolean)
62
+ active = Column(
63
+ mlrun.common.schemas.model_monitoring.EventFieldType.ACTIVE, Boolean
64
+ )
53
65
  monitoring_mode = Column(
54
- model_monitoring_constants.EventFieldType.MONITORING_MODE,
66
+ mlrun.common.schemas.model_monitoring.EventFieldType.MONITORING_MODE,
55
67
  String(10),
56
68
  )
57
69
  feature_stats = Column(
58
- model_monitoring_constants.EventFieldType.FEATURE_STATS, Text
70
+ mlrun.common.schemas.model_monitoring.EventFieldType.FEATURE_STATS, Text
59
71
  )
60
72
  current_stats = Column(
61
- model_monitoring_constants.EventFieldType.CURRENT_STATS, Text
73
+ mlrun.common.schemas.model_monitoring.EventFieldType.CURRENT_STATS, Text
62
74
  )
63
75
  feature_names = Column(
64
- model_monitoring_constants.EventFieldType.FEATURE_NAMES, Text
76
+ mlrun.common.schemas.model_monitoring.EventFieldType.FEATURE_NAMES, Text
77
+ )
78
+ children = Column(
79
+ mlrun.common.schemas.model_monitoring.EventFieldType.CHILDREN, Text
80
+ )
81
+ label_names = Column(
82
+ mlrun.common.schemas.model_monitoring.EventFieldType.LABEL_NAMES, Text
65
83
  )
66
- children = Column(model_monitoring_constants.EventFieldType.CHILDREN, Text)
67
- label_names = Column(model_monitoring_constants.EventFieldType.LABEL_NAMES, Text)
68
84
 
69
85
  endpoint_type = Column(
70
- model_monitoring_constants.EventFieldType.ENDPOINT_TYPE,
86
+ mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_TYPE,
71
87
  String(10),
72
88
  )
73
89
  children_uids = Column(
74
- model_monitoring_constants.EventFieldType.CHILDREN_UIDS, Text
90
+ mlrun.common.schemas.model_monitoring.EventFieldType.CHILDREN_UIDS, Text
75
91
  )
76
92
  drift_measures = Column(
77
- model_monitoring_constants.EventFieldType.DRIFT_MEASURES, Text
93
+ mlrun.common.schemas.model_monitoring.EventFieldType.DRIFT_MEASURES, Text
78
94
  )
79
95
  drift_status = Column(
80
- model_monitoring_constants.EventFieldType.DRIFT_STATUS,
96
+ mlrun.common.schemas.model_monitoring.EventFieldType.DRIFT_STATUS,
81
97
  String(40),
82
98
  )
83
99
  monitor_configuration = Column(
84
- model_monitoring_constants.EventFieldType.MONITOR_CONFIGURATION,
100
+ mlrun.common.schemas.model_monitoring.EventFieldType.MONITOR_CONFIGURATION,
85
101
  Text,
86
102
  )
87
103
  monitoring_feature_set_uri = Column(
88
- model_monitoring_constants.EventFieldType.FEATURE_SET_URI,
104
+ mlrun.common.schemas.model_monitoring.EventFieldType.FEATURE_SET_URI,
89
105
  String(255),
90
106
  )
91
107
  first_request = Column(
92
- model_monitoring_constants.EventFieldType.FIRST_REQUEST,
108
+ mlrun.common.schemas.model_monitoring.EventFieldType.FIRST_REQUEST,
93
109
  sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
94
110
  )
95
111
  last_request = Column(
96
- model_monitoring_constants.EventFieldType.LAST_REQUEST,
112
+ mlrun.common.schemas.model_monitoring.EventFieldType.LAST_REQUEST,
97
113
  sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
98
114
  )
99
- error_count = Column(model_monitoring_constants.EventFieldType.ERROR_COUNT, Integer)
100
- metrics = Column(model_monitoring_constants.EventFieldType.METRICS, Text)
115
+ error_count = Column(
116
+ mlrun.common.schemas.model_monitoring.EventFieldType.ERROR_COUNT, Integer
117
+ )
118
+ metrics = Column(mlrun.common.schemas.model_monitoring.EventFieldType.METRICS, Text)