mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +2 -35
- mlrun/__main__.py +3 -41
- mlrun/api/api/api.py +6 -0
- mlrun/api/api/endpoints/feature_store.py +0 -4
- mlrun/api/api/endpoints/files.py +14 -2
- mlrun/api/api/endpoints/frontend_spec.py +2 -1
- mlrun/api/api/endpoints/functions.py +95 -59
- mlrun/api/api/endpoints/grafana_proxy.py +9 -9
- mlrun/api/api/endpoints/logs.py +17 -3
- mlrun/api/api/endpoints/model_endpoints.py +3 -2
- mlrun/api/api/endpoints/pipelines.py +1 -5
- mlrun/api/api/endpoints/projects.py +88 -0
- mlrun/api/api/endpoints/runs.py +48 -6
- mlrun/api/api/endpoints/submit.py +2 -1
- mlrun/api/api/endpoints/workflows.py +355 -0
- mlrun/api/api/utils.py +3 -4
- mlrun/api/crud/__init__.py +1 -0
- mlrun/api/crud/client_spec.py +6 -2
- mlrun/api/crud/feature_store.py +5 -0
- mlrun/api/crud/model_monitoring/__init__.py +1 -0
- mlrun/api/crud/model_monitoring/deployment.py +497 -0
- mlrun/api/crud/model_monitoring/grafana.py +96 -42
- mlrun/api/crud/model_monitoring/helpers.py +159 -0
- mlrun/api/crud/model_monitoring/model_endpoints.py +202 -476
- mlrun/api/crud/notifications.py +9 -4
- mlrun/api/crud/pipelines.py +6 -11
- mlrun/api/crud/projects.py +2 -2
- mlrun/api/crud/runtime_resources.py +4 -3
- mlrun/api/crud/runtimes/nuclio/helpers.py +5 -1
- mlrun/api/crud/secrets.py +21 -0
- mlrun/api/crud/workflows.py +352 -0
- mlrun/api/db/base.py +16 -1
- mlrun/api/db/init_db.py +2 -4
- mlrun/api/db/session.py +1 -1
- mlrun/api/db/sqldb/db.py +129 -31
- mlrun/api/db/sqldb/models/models_mysql.py +15 -1
- mlrun/api/db/sqldb/models/models_sqlite.py +16 -2
- mlrun/api/launcher.py +38 -6
- mlrun/api/main.py +3 -2
- mlrun/api/rundb/__init__.py +13 -0
- mlrun/{db → api/rundb}/sqldb.py +36 -84
- mlrun/api/runtime_handlers/__init__.py +56 -0
- mlrun/api/runtime_handlers/base.py +1247 -0
- mlrun/api/runtime_handlers/daskjob.py +209 -0
- mlrun/api/runtime_handlers/kubejob.py +37 -0
- mlrun/api/runtime_handlers/mpijob.py +147 -0
- mlrun/api/runtime_handlers/remotesparkjob.py +29 -0
- mlrun/api/runtime_handlers/sparkjob.py +148 -0
- mlrun/api/schemas/__init__.py +17 -6
- mlrun/api/utils/builder.py +1 -4
- mlrun/api/utils/clients/chief.py +14 -0
- mlrun/api/utils/clients/iguazio.py +33 -33
- mlrun/api/utils/clients/nuclio.py +2 -2
- mlrun/api/utils/periodic.py +9 -2
- mlrun/api/utils/projects/follower.py +14 -7
- mlrun/api/utils/projects/leader.py +2 -1
- mlrun/api/utils/projects/remotes/nop_follower.py +2 -2
- mlrun/api/utils/projects/remotes/nop_leader.py +2 -2
- mlrun/api/utils/runtimes/__init__.py +14 -0
- mlrun/api/utils/runtimes/nuclio.py +43 -0
- mlrun/api/utils/scheduler.py +98 -15
- mlrun/api/utils/singletons/db.py +5 -1
- mlrun/api/utils/singletons/project_member.py +4 -1
- mlrun/api/utils/singletons/scheduler.py +1 -1
- mlrun/artifacts/base.py +6 -6
- mlrun/artifacts/dataset.py +4 -4
- mlrun/artifacts/manager.py +2 -3
- mlrun/artifacts/model.py +2 -2
- mlrun/artifacts/plots.py +8 -8
- mlrun/common/db/__init__.py +14 -0
- mlrun/common/helpers.py +37 -0
- mlrun/{mlutils → common/model_monitoring}/__init__.py +3 -2
- mlrun/common/model_monitoring/helpers.py +69 -0
- mlrun/common/schemas/__init__.py +13 -1
- mlrun/common/schemas/auth.py +4 -1
- mlrun/common/schemas/client_spec.py +1 -1
- mlrun/common/schemas/function.py +17 -0
- mlrun/common/schemas/model_monitoring/__init__.py +48 -0
- mlrun/common/{model_monitoring.py → schemas/model_monitoring/constants.py} +11 -23
- mlrun/common/schemas/model_monitoring/grafana.py +55 -0
- mlrun/common/schemas/{model_endpoints.py → model_monitoring/model_endpoints.py} +32 -65
- mlrun/common/schemas/notification.py +1 -0
- mlrun/common/schemas/object.py +4 -0
- mlrun/common/schemas/project.py +1 -0
- mlrun/common/schemas/regex.py +1 -1
- mlrun/common/schemas/runs.py +1 -8
- mlrun/common/schemas/schedule.py +1 -8
- mlrun/common/schemas/workflow.py +54 -0
- mlrun/config.py +45 -42
- mlrun/datastore/__init__.py +21 -0
- mlrun/datastore/base.py +1 -1
- mlrun/datastore/datastore.py +9 -0
- mlrun/datastore/dbfs_store.py +168 -0
- mlrun/datastore/helpers.py +18 -0
- mlrun/datastore/sources.py +1 -0
- mlrun/datastore/store_resources.py +2 -5
- mlrun/datastore/v3io.py +1 -2
- mlrun/db/__init__.py +4 -68
- mlrun/db/base.py +12 -0
- mlrun/db/factory.py +65 -0
- mlrun/db/httpdb.py +175 -20
- mlrun/db/nopdb.py +4 -2
- mlrun/execution.py +4 -2
- mlrun/feature_store/__init__.py +1 -0
- mlrun/feature_store/api.py +1 -2
- mlrun/feature_store/common.py +2 -1
- mlrun/feature_store/feature_set.py +1 -11
- mlrun/feature_store/feature_vector.py +340 -2
- mlrun/feature_store/ingestion.py +5 -10
- mlrun/feature_store/retrieval/base.py +118 -104
- mlrun/feature_store/retrieval/dask_merger.py +17 -10
- mlrun/feature_store/retrieval/job.py +4 -1
- mlrun/feature_store/retrieval/local_merger.py +18 -18
- mlrun/feature_store/retrieval/spark_merger.py +21 -14
- mlrun/feature_store/retrieval/storey_merger.py +22 -16
- mlrun/kfpops.py +3 -9
- mlrun/launcher/base.py +57 -53
- mlrun/launcher/client.py +5 -4
- mlrun/launcher/factory.py +24 -13
- mlrun/launcher/local.py +6 -6
- mlrun/launcher/remote.py +4 -4
- mlrun/lists.py +0 -11
- mlrun/model.py +11 -17
- mlrun/model_monitoring/__init__.py +2 -22
- mlrun/model_monitoring/features_drift_table.py +1 -1
- mlrun/model_monitoring/helpers.py +22 -210
- mlrun/model_monitoring/model_endpoint.py +1 -1
- mlrun/model_monitoring/model_monitoring_batch.py +127 -50
- mlrun/model_monitoring/prometheus.py +219 -0
- mlrun/model_monitoring/stores/__init__.py +16 -11
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +95 -23
- mlrun/model_monitoring/stores/models/mysql.py +47 -29
- mlrun/model_monitoring/stores/models/sqlite.py +47 -29
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +31 -19
- mlrun/model_monitoring/{stream_processing_fs.py → stream_processing.py} +206 -64
- mlrun/model_monitoring/tracking_policy.py +104 -0
- mlrun/package/packager.py +6 -8
- mlrun/package/packagers/default_packager.py +121 -10
- mlrun/package/packagers/numpy_packagers.py +1 -1
- mlrun/platforms/__init__.py +0 -2
- mlrun/platforms/iguazio.py +0 -56
- mlrun/projects/pipelines.py +53 -159
- mlrun/projects/project.py +10 -37
- mlrun/render.py +1 -1
- mlrun/run.py +8 -124
- mlrun/runtimes/__init__.py +6 -42
- mlrun/runtimes/base.py +29 -1249
- mlrun/runtimes/daskjob.py +2 -198
- mlrun/runtimes/funcdoc.py +0 -9
- mlrun/runtimes/function.py +25 -29
- mlrun/runtimes/kubejob.py +5 -29
- mlrun/runtimes/local.py +1 -1
- mlrun/runtimes/mpijob/__init__.py +2 -2
- mlrun/runtimes/mpijob/abstract.py +10 -1
- mlrun/runtimes/mpijob/v1.py +0 -76
- mlrun/runtimes/mpijob/v1alpha1.py +1 -74
- mlrun/runtimes/nuclio.py +3 -2
- mlrun/runtimes/pod.py +28 -18
- mlrun/runtimes/remotesparkjob.py +1 -15
- mlrun/runtimes/serving.py +14 -6
- mlrun/runtimes/sparkjob/__init__.py +0 -1
- mlrun/runtimes/sparkjob/abstract.py +4 -131
- mlrun/runtimes/utils.py +0 -26
- mlrun/serving/routers.py +7 -7
- mlrun/serving/server.py +11 -8
- mlrun/serving/states.py +7 -1
- mlrun/serving/v2_serving.py +6 -6
- mlrun/utils/helpers.py +23 -42
- mlrun/utils/notifications/notification/__init__.py +4 -0
- mlrun/utils/notifications/notification/webhook.py +61 -0
- mlrun/utils/notifications/notification_pusher.py +5 -25
- mlrun/utils/regex.py +7 -2
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/METADATA +26 -25
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/RECORD +180 -158
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/WHEEL +1 -1
- mlrun/mlutils/data.py +0 -160
- mlrun/mlutils/models.py +0 -78
- mlrun/mlutils/plots.py +0 -902
- mlrun/utils/model_monitoring.py +0 -249
- /mlrun/{api/db/sqldb/session.py → common/db/sql_session.py} +0 -0
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/LICENSE +0 -0
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/entry_points.txt +0 -0
- {mlrun-1.4.0rc25.dist-info → mlrun-1.5.0rc2.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
# Copyright 2023 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
import typing
|
|
16
|
+
|
|
17
|
+
import prometheus_client
|
|
18
|
+
|
|
19
|
+
from mlrun.common.schemas.model_monitoring import EventFieldType, PrometheusMetric
|
|
20
|
+
|
|
21
|
+
# Memory path for Prometheus registry file
|
|
22
|
+
_registry_path = "/tmp/prom-reg.txt"
|
|
23
|
+
|
|
24
|
+
# Initializing Promethues metric collector registry
|
|
25
|
+
_registry: prometheus_client.CollectorRegistry = prometheus_client.CollectorRegistry()
|
|
26
|
+
|
|
27
|
+
# The following real-time metrics are being updated through the monitoring stream graph steps
|
|
28
|
+
_prediction_counter: prometheus_client.Counter = prometheus_client.Counter(
|
|
29
|
+
name=PrometheusMetric.PREDICTIONS_TOTAL,
|
|
30
|
+
documentation="Counter for total predictions",
|
|
31
|
+
registry=_registry,
|
|
32
|
+
labelnames=[
|
|
33
|
+
EventFieldType.PROJECT,
|
|
34
|
+
EventFieldType.ENDPOINT_ID,
|
|
35
|
+
EventFieldType.MODEL,
|
|
36
|
+
EventFieldType.ENDPOINT_TYPE,
|
|
37
|
+
],
|
|
38
|
+
)
|
|
39
|
+
_model_latency: prometheus_client.Summary = prometheus_client.Summary(
|
|
40
|
+
name=PrometheusMetric.MODEL_LATENCY_SECONDS,
|
|
41
|
+
documentation="Summary for for model latency",
|
|
42
|
+
registry=_registry,
|
|
43
|
+
labelnames=[
|
|
44
|
+
EventFieldType.PROJECT,
|
|
45
|
+
EventFieldType.ENDPOINT_ID,
|
|
46
|
+
EventFieldType.MODEL,
|
|
47
|
+
EventFieldType.ENDPOINT_TYPE,
|
|
48
|
+
],
|
|
49
|
+
)
|
|
50
|
+
_income_features: prometheus_client.Gauge = prometheus_client.Gauge(
|
|
51
|
+
name=PrometheusMetric.INCOME_FEATURES,
|
|
52
|
+
documentation="Samples of features and predictions",
|
|
53
|
+
registry=_registry,
|
|
54
|
+
labelnames=[
|
|
55
|
+
EventFieldType.PROJECT,
|
|
56
|
+
EventFieldType.ENDPOINT_ID,
|
|
57
|
+
EventFieldType.METRIC,
|
|
58
|
+
],
|
|
59
|
+
)
|
|
60
|
+
_error_counter: prometheus_client.Counter = prometheus_client.Counter(
|
|
61
|
+
name=PrometheusMetric.ERRORS_TOTAL,
|
|
62
|
+
documentation="Counter for total errors",
|
|
63
|
+
registry=_registry,
|
|
64
|
+
labelnames=[
|
|
65
|
+
EventFieldType.PROJECT,
|
|
66
|
+
EventFieldType.ENDPOINT_ID,
|
|
67
|
+
EventFieldType.MODEL,
|
|
68
|
+
],
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
# The following metrics are being updated through the model monitoring batch job
|
|
72
|
+
_batch_metrics: prometheus_client.Gauge = prometheus_client.Gauge(
|
|
73
|
+
name=PrometheusMetric.DRIFT_METRICS,
|
|
74
|
+
documentation="Results from the batch drift analysis",
|
|
75
|
+
registry=_registry,
|
|
76
|
+
labelnames=[
|
|
77
|
+
EventFieldType.PROJECT,
|
|
78
|
+
EventFieldType.ENDPOINT_ID,
|
|
79
|
+
EventFieldType.METRIC,
|
|
80
|
+
],
|
|
81
|
+
)
|
|
82
|
+
_drift_status: prometheus_client.Enum = prometheus_client.Enum(
|
|
83
|
+
name=PrometheusMetric.DRIFT_STATUS,
|
|
84
|
+
documentation="Drift status of the model endpoint",
|
|
85
|
+
registry=_registry,
|
|
86
|
+
states=["NO_DRIFT", "DRIFT_DETECTED", "POSSIBLE_DRIFT"],
|
|
87
|
+
labelnames=[EventFieldType.PROJECT, EventFieldType.ENDPOINT_ID],
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _write_registry(func):
|
|
92
|
+
def wrapper(*args, **kwargs):
|
|
93
|
+
global _registry
|
|
94
|
+
"""A wrapper function to update the registry file each time a metric has been updated"""
|
|
95
|
+
func(*args, **kwargs)
|
|
96
|
+
prometheus_client.write_to_textfile(path=_registry_path, registry=_registry)
|
|
97
|
+
|
|
98
|
+
return wrapper
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@_write_registry
|
|
102
|
+
def write_predictions_and_latency_metrics(
|
|
103
|
+
project: str, endpoint_id: str, latency: int, model_name: str, endpoint_type: int
|
|
104
|
+
):
|
|
105
|
+
"""
|
|
106
|
+
Update the prediction counter and the latency value of the provided model endpoint within Prometheus registry.
|
|
107
|
+
Please note that while the prediction counter is ALWAYS increasing by 1,the latency summary metric is being
|
|
108
|
+
increased by the event latency time. Grafana dashboard will query the average latency time by dividing the total
|
|
109
|
+
latency value by the total amount of predictions.
|
|
110
|
+
|
|
111
|
+
:param project: Project name.
|
|
112
|
+
:param endpoint_id: Model endpoint unique id.
|
|
113
|
+
:param latency: Latency time (microsecond) in which the event has been processed through the model server.
|
|
114
|
+
:param model_name: Model name which will be used by Grafana for displaying the results by model.
|
|
115
|
+
:param endpoint_type: Endpoint type that is represented by an int (possible values: 1,2,3) corresponding to the
|
|
116
|
+
Enum class :py:class:`~mlrun.common.schemas.model_monitoring.EndpointType`.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
# Increase the prediction counter by 1
|
|
120
|
+
_prediction_counter.labels(
|
|
121
|
+
project=project,
|
|
122
|
+
endpoint_id=endpoint_id,
|
|
123
|
+
model=model_name,
|
|
124
|
+
endpoint_type=endpoint_type,
|
|
125
|
+
).inc(1)
|
|
126
|
+
|
|
127
|
+
# Increase the latency value according to the provided latency of the current event
|
|
128
|
+
_model_latency.labels(
|
|
129
|
+
project=project,
|
|
130
|
+
endpoint_id=endpoint_id,
|
|
131
|
+
model=model_name,
|
|
132
|
+
endpoint_type=endpoint_type,
|
|
133
|
+
).observe(latency)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@_write_registry
|
|
137
|
+
def write_income_features(
|
|
138
|
+
project: str, endpoint_id: str, features: typing.Dict[str, float]
|
|
139
|
+
):
|
|
140
|
+
"""Update a sample of features.
|
|
141
|
+
|
|
142
|
+
:param project: Project name.
|
|
143
|
+
:param endpoint_id: Model endpoint unique id.
|
|
144
|
+
:param features: Dictionary in which the key is a feature name and the value is a float number.
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
for metric in features:
|
|
150
|
+
_income_features.labels(
|
|
151
|
+
project=project, endpoint_id=endpoint_id, metric=metric
|
|
152
|
+
).set(value=features[metric])
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@_write_registry
|
|
156
|
+
def write_drift_metrics(project: str, endpoint_id: str, metric: str, value: float):
|
|
157
|
+
"""Update drift metrics that have been calculated through the monitoring batch job
|
|
158
|
+
|
|
159
|
+
:param project: Project name.
|
|
160
|
+
:param endpoint_id: Model endpoint unique id.
|
|
161
|
+
:param metric: Metric name (e.g. TVD, Hellinger).
|
|
162
|
+
:param value: Metric value as a float.
|
|
163
|
+
|
|
164
|
+
"""
|
|
165
|
+
|
|
166
|
+
_batch_metrics.labels(project=project, endpoint_id=endpoint_id, metric=metric).set(
|
|
167
|
+
value=value
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@_write_registry
|
|
172
|
+
def write_drift_status(project: str, endpoint_id: str, drift_status: str):
|
|
173
|
+
"""
|
|
174
|
+
Update the drift status enum for a specific model endpoint.
|
|
175
|
+
|
|
176
|
+
:param project: Project name.
|
|
177
|
+
:param endpoint_id: Model endpoint unique id.
|
|
178
|
+
:param drift_status: Drift status value, can be one of the following: 'NO_DRIFT', 'DRIFT_DETECTED', or
|
|
179
|
+
'POSSIBLE_DRIFT'.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
_drift_status.labels(project=project, endpoint_id=endpoint_id).state(drift_status)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
@_write_registry
|
|
186
|
+
def write_errors(project: str, endpoint_id: str, model_name: str):
|
|
187
|
+
"""
|
|
188
|
+
Update the error counter for a specific model endpoint.
|
|
189
|
+
|
|
190
|
+
:param project: Project name.
|
|
191
|
+
:param endpoint_id: Model endpoint unique id.
|
|
192
|
+
:param model_name: Model name. Will be used by Grafana to show the amount of errors per model by time.
|
|
193
|
+
"""
|
|
194
|
+
|
|
195
|
+
_error_counter.labels(
|
|
196
|
+
project=project, endpoint_id=endpoint_id, model=model_name
|
|
197
|
+
).inc(1)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def get_registry() -> str:
|
|
201
|
+
"""Returns the parsed registry file according to the exposition format of Prometheus."""
|
|
202
|
+
|
|
203
|
+
# Read the registry file (note that the text is stored in UTF-8 format)
|
|
204
|
+
f = open(_registry_path)
|
|
205
|
+
lines = f.read()
|
|
206
|
+
f.close()
|
|
207
|
+
|
|
208
|
+
# Reset part of the metrics to avoid a repeating scraping of the same value
|
|
209
|
+
clean_metrics()
|
|
210
|
+
|
|
211
|
+
return lines
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
@_write_registry
|
|
215
|
+
def clean_metrics():
|
|
216
|
+
"""Clean the income features values. As these results are relevant only for a certain timestamp, we will remove
|
|
217
|
+
them from the global registry after they have been scraped by Prometheus."""
|
|
218
|
+
|
|
219
|
+
_income_features.clear()
|
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
import enum
|
|
18
18
|
import typing
|
|
19
19
|
|
|
20
|
-
import mlrun
|
|
20
|
+
import mlrun.common.schemas.secret
|
|
21
|
+
import mlrun.errors
|
|
21
22
|
|
|
22
23
|
from .model_endpoint_store import ModelEndpointStore
|
|
23
24
|
|
|
@@ -33,6 +34,7 @@ class ModelEndpointStoreType(enum.Enum):
|
|
|
33
34
|
project: str,
|
|
34
35
|
access_key: str = None,
|
|
35
36
|
endpoint_store_connection: str = None,
|
|
37
|
+
secret_provider: typing.Callable = None,
|
|
36
38
|
) -> ModelEndpointStore:
|
|
37
39
|
"""
|
|
38
40
|
Return a ModelEndpointStore object based on the provided enum value.
|
|
@@ -46,13 +48,13 @@ class ModelEndpointStoreType(enum.Enum):
|
|
|
46
48
|
e.g. A root user with password 1234, tries to connect a schema called
|
|
47
49
|
mlrun within a local MySQL DB instance:
|
|
48
50
|
'mysql+pymysql://root:1234@localhost:3306/mlrun'.
|
|
51
|
+
:param secret_provider: An optional secret provider to get the connection string secret.
|
|
49
52
|
|
|
50
53
|
:return: `ModelEndpointStore` object.
|
|
51
54
|
|
|
52
55
|
"""
|
|
53
56
|
|
|
54
57
|
if self.value == ModelEndpointStoreType.v3io_nosql.value:
|
|
55
|
-
|
|
56
58
|
from .kv_model_endpoint_store import KVModelEndpointStore
|
|
57
59
|
|
|
58
60
|
# Get V3IO access key from env
|
|
@@ -62,15 +64,13 @@ class ModelEndpointStoreType(enum.Enum):
|
|
|
62
64
|
|
|
63
65
|
# Assuming SQL store target if store type is not KV.
|
|
64
66
|
# Update these lines once there are more than two store target types.
|
|
65
|
-
from mlrun.utils.model_monitoring import get_connection_string
|
|
66
67
|
|
|
67
|
-
sql_connection_string = endpoint_store_connection or get_connection_string(
|
|
68
|
-
project=project
|
|
69
|
-
)
|
|
70
68
|
from .sql_model_endpoint_store import SQLModelEndpointStore
|
|
71
69
|
|
|
72
70
|
return SQLModelEndpointStore(
|
|
73
|
-
project=project,
|
|
71
|
+
project=project,
|
|
72
|
+
sql_connection_string=endpoint_store_connection,
|
|
73
|
+
secret_provider=secret_provider,
|
|
74
74
|
)
|
|
75
75
|
|
|
76
76
|
@classmethod
|
|
@@ -85,13 +85,16 @@ class ModelEndpointStoreType(enum.Enum):
|
|
|
85
85
|
|
|
86
86
|
|
|
87
87
|
def get_model_endpoint_store(
|
|
88
|
-
project: str,
|
|
88
|
+
project: str,
|
|
89
|
+
access_key: str = None,
|
|
90
|
+
secret_provider: typing.Callable = None,
|
|
89
91
|
) -> ModelEndpointStore:
|
|
90
92
|
"""
|
|
91
93
|
Getting the DB target type based on mlrun.config.model_endpoint_monitoring.store_type.
|
|
92
94
|
|
|
93
|
-
:param project:
|
|
94
|
-
:param access_key:
|
|
95
|
+
:param project: The name of the project.
|
|
96
|
+
:param access_key: Access key with permission to the DB table.
|
|
97
|
+
:param secret_provider: An optional secret provider to get the connection string secret.
|
|
95
98
|
|
|
96
99
|
:return: `ModelEndpointStore` object. Using this object, the user can apply different operations on the
|
|
97
100
|
model endpoint record such as write, update, get and delete.
|
|
@@ -103,4 +106,6 @@ def get_model_endpoint_store(
|
|
|
103
106
|
)
|
|
104
107
|
|
|
105
108
|
# Convert into model endpoint store target object
|
|
106
|
-
return model_endpoint_store_type.to_endpoint_store(
|
|
109
|
+
return model_endpoint_store_type.to_endpoint_store(
|
|
110
|
+
project=project, access_key=access_key, secret_provider=secret_provider
|
|
111
|
+
)
|
|
@@ -13,15 +13,16 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
15
|
|
|
16
|
+
import json
|
|
16
17
|
import os
|
|
17
18
|
import typing
|
|
19
|
+
import warnings
|
|
18
20
|
|
|
19
21
|
import v3io.dataplane
|
|
20
22
|
import v3io_frames
|
|
21
23
|
|
|
22
|
-
import mlrun
|
|
23
|
-
import mlrun.common.model_monitoring
|
|
24
|
-
import mlrun.utils.model_monitoring
|
|
24
|
+
import mlrun.common.model_monitoring.helpers
|
|
25
|
+
import mlrun.common.schemas.model_monitoring
|
|
25
26
|
import mlrun.utils.v3io_clients
|
|
26
27
|
from mlrun.utils import logger
|
|
27
28
|
|
|
@@ -54,7 +55,7 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
54
55
|
self.client.kv.put(
|
|
55
56
|
container=self.container,
|
|
56
57
|
table_path=self.path,
|
|
57
|
-
key=endpoint[
|
|
58
|
+
key=endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.UID],
|
|
58
59
|
attributes=endpoint,
|
|
59
60
|
)
|
|
60
61
|
|
|
@@ -121,7 +122,7 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
121
122
|
raise mlrun.errors.MLRunNotFoundError(f"Endpoint {endpoint_id} not found")
|
|
122
123
|
|
|
123
124
|
# For backwards compatability: replace null values for `error_count` and `metrics`
|
|
124
|
-
|
|
125
|
+
self.validate_old_schema_fields(endpoint=endpoint)
|
|
125
126
|
|
|
126
127
|
return endpoint
|
|
127
128
|
|
|
@@ -129,13 +130,15 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
129
130
|
"""Getting path and container based on the model monitoring configurations"""
|
|
130
131
|
path = mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
|
|
131
132
|
project=self.project,
|
|
132
|
-
kind=
|
|
133
|
+
kind=mlrun.common.schemas.ModelMonitoringStoreKinds.ENDPOINTS,
|
|
133
134
|
)
|
|
134
135
|
(
|
|
135
136
|
_,
|
|
136
137
|
container,
|
|
137
138
|
path,
|
|
138
|
-
) = mlrun.
|
|
139
|
+
) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
|
|
140
|
+
path
|
|
141
|
+
)
|
|
139
142
|
return path, container
|
|
140
143
|
|
|
141
144
|
def list_model_endpoints(
|
|
@@ -190,13 +193,17 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
190
193
|
if uids is None:
|
|
191
194
|
uids = []
|
|
192
195
|
for item in items:
|
|
193
|
-
if
|
|
196
|
+
if mlrun.common.schemas.model_monitoring.EventFieldType.UID not in item:
|
|
194
197
|
# This is kept for backwards compatibility - in old versions the key column named endpoint_id
|
|
195
198
|
uids.append(
|
|
196
|
-
item[
|
|
199
|
+
item[
|
|
200
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
|
|
201
|
+
]
|
|
197
202
|
)
|
|
198
203
|
else:
|
|
199
|
-
uids.append(
|
|
204
|
+
uids.append(
|
|
205
|
+
item[mlrun.common.schemas.model_monitoring.EventFieldType.UID]
|
|
206
|
+
)
|
|
200
207
|
|
|
201
208
|
# Add each relevant model endpoint to the model endpoints list
|
|
202
209
|
for endpoint_id in uids:
|
|
@@ -218,14 +225,17 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
218
225
|
|
|
219
226
|
# Delete model endpoint record from KV table
|
|
220
227
|
for endpoint_dict in endpoints:
|
|
221
|
-
if
|
|
228
|
+
if (
|
|
229
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.UID
|
|
230
|
+
not in endpoint_dict
|
|
231
|
+
):
|
|
222
232
|
# This is kept for backwards compatibility - in old versions the key column named endpoint_id
|
|
223
233
|
endpoint_id = endpoint_dict[
|
|
224
|
-
|
|
234
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
|
|
225
235
|
]
|
|
226
236
|
else:
|
|
227
237
|
endpoint_id = endpoint_dict[
|
|
228
|
-
|
|
238
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.UID
|
|
229
239
|
]
|
|
230
240
|
self.delete_model_endpoint(
|
|
231
241
|
endpoint_id,
|
|
@@ -262,7 +272,7 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
262
272
|
# Delete time series DB resources
|
|
263
273
|
try:
|
|
264
274
|
frames.delete(
|
|
265
|
-
backend=
|
|
275
|
+
backend=mlrun.common.schemas.model_monitoring.TimeSeriesTarget.TSDB,
|
|
266
276
|
table=filtered_path,
|
|
267
277
|
)
|
|
268
278
|
except (v3io_frames.errors.DeleteError, v3io_frames.errors.CreateError) as e:
|
|
@@ -319,14 +329,16 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
319
329
|
events_path = (
|
|
320
330
|
mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
|
|
321
331
|
project=self.project,
|
|
322
|
-
kind=
|
|
332
|
+
kind=mlrun.common.schemas.ModelMonitoringStoreKinds.EVENTS,
|
|
323
333
|
)
|
|
324
334
|
)
|
|
325
335
|
(
|
|
326
336
|
_,
|
|
327
337
|
container,
|
|
328
338
|
events_path,
|
|
329
|
-
) = mlrun.
|
|
339
|
+
) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
|
|
340
|
+
events_path
|
|
341
|
+
)
|
|
330
342
|
|
|
331
343
|
# Retrieve the raw data from the time series DB based on the provided metrics and time ranges
|
|
332
344
|
frames_client = mlrun.utils.v3io_clients.get_frames_client(
|
|
@@ -337,7 +349,7 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
337
349
|
|
|
338
350
|
try:
|
|
339
351
|
data = frames_client.read(
|
|
340
|
-
backend=
|
|
352
|
+
backend=mlrun.common.schemas.model_monitoring.TimeSeriesTarget.TSDB,
|
|
341
353
|
table=events_path,
|
|
342
354
|
columns=["endpoint_id", *metrics],
|
|
343
355
|
filter=f"endpoint_id=='{endpoint_id}'",
|
|
@@ -372,13 +384,15 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
372
384
|
full_path = (
|
|
373
385
|
mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
|
|
374
386
|
project=self.project,
|
|
375
|
-
kind=
|
|
387
|
+
kind=mlrun.common.schemas.ModelMonitoringStoreKinds.EVENTS,
|
|
376
388
|
)
|
|
377
389
|
)
|
|
378
390
|
|
|
379
391
|
# Generate the main directory with the TSDB resources
|
|
380
|
-
tsdb_path =
|
|
381
|
-
|
|
392
|
+
tsdb_path = (
|
|
393
|
+
mlrun.common.model_monitoring.helpers.parse_model_endpoint_project_prefix(
|
|
394
|
+
full_path, self.project
|
|
395
|
+
)
|
|
382
396
|
)
|
|
383
397
|
|
|
384
398
|
# Generate filtered path without schema and container as required by the frames object
|
|
@@ -386,7 +400,9 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
386
400
|
_,
|
|
387
401
|
_,
|
|
388
402
|
filtered_path,
|
|
389
|
-
) = mlrun.
|
|
403
|
+
) = mlrun.common.model_monitoring.helpers.parse_model_endpoint_store_prefix(
|
|
404
|
+
full_path
|
|
405
|
+
)
|
|
390
406
|
return tsdb_path, filtered_path
|
|
391
407
|
|
|
392
408
|
@staticmethod
|
|
@@ -441,8 +457,64 @@ class KVModelEndpointStore(ModelEndpointStore):
|
|
|
441
457
|
# Apply top_level filter (remove endpoints that considered a child of a router)
|
|
442
458
|
if top_level:
|
|
443
459
|
filter_expression.append(
|
|
444
|
-
f"(endpoint_type=='{str(
|
|
445
|
-
f"OR endpoint_type=='{str(
|
|
460
|
+
f"(endpoint_type=='{str(mlrun.common.schemas.model_monitoring.EndpointType.NODE_EP.value)}' "
|
|
461
|
+
f"OR endpoint_type=='{str(mlrun.common.schemas.model_monitoring.EndpointType.ROUTER.value)}')"
|
|
446
462
|
)
|
|
447
463
|
|
|
448
464
|
return " AND ".join(filter_expression)
|
|
465
|
+
|
|
466
|
+
@staticmethod
|
|
467
|
+
def validate_old_schema_fields(endpoint: dict):
|
|
468
|
+
"""
|
|
469
|
+
Replace default null values for `error_count` and `metrics` for users that logged a model endpoint before 1.3.0.
|
|
470
|
+
In addition, this function also validates that the key name of the endpoint unique id is `uid` and not
|
|
471
|
+
`endpoint_id` that has been used before 1.3.0.
|
|
472
|
+
|
|
473
|
+
Leaving here for backwards compatibility which related to the model endpoint schema.
|
|
474
|
+
|
|
475
|
+
:param endpoint: An endpoint flattened dictionary.
|
|
476
|
+
"""
|
|
477
|
+
warnings.warn(
|
|
478
|
+
"This will be deprecated in 1.3.0, and will be removed in 1.5.0",
|
|
479
|
+
# TODO: In 1.3.0 do changes in examples & demos In 1.5.0 remove
|
|
480
|
+
FutureWarning,
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
# Validate default value for `error_count`
|
|
484
|
+
# For backwards compatibility reasons, we validate that the model endpoint includes the `error_count` key
|
|
485
|
+
if (
|
|
486
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ERROR_COUNT in endpoint
|
|
487
|
+
and endpoint[
|
|
488
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ERROR_COUNT
|
|
489
|
+
]
|
|
490
|
+
== "null"
|
|
491
|
+
):
|
|
492
|
+
endpoint[
|
|
493
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ERROR_COUNT
|
|
494
|
+
] = "0"
|
|
495
|
+
|
|
496
|
+
# Validate default value for `metrics`
|
|
497
|
+
# For backwards compatibility reasons, we validate that the model endpoint includes the `metrics` key
|
|
498
|
+
if (
|
|
499
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.METRICS in endpoint
|
|
500
|
+
and endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.METRICS]
|
|
501
|
+
== "null"
|
|
502
|
+
):
|
|
503
|
+
endpoint[
|
|
504
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.METRICS
|
|
505
|
+
] = json.dumps(
|
|
506
|
+
{
|
|
507
|
+
mlrun.common.schemas.model_monitoring.EventKeyMetrics.GENERIC: {
|
|
508
|
+
mlrun.common.schemas.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
|
|
509
|
+
mlrun.common.schemas.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
)
|
|
513
|
+
# Validate key `uid` instead of `endpoint_id`
|
|
514
|
+
# For backwards compatibility reasons, we replace the `endpoint_id` with `uid` which is the updated key name
|
|
515
|
+
if mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID in endpoint:
|
|
516
|
+
endpoint[
|
|
517
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.UID
|
|
518
|
+
] = endpoint[
|
|
519
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
|
|
520
|
+
]
|
|
@@ -17,84 +17,102 @@
|
|
|
17
17
|
import sqlalchemy.dialects
|
|
18
18
|
from sqlalchemy import Boolean, Column, Integer, String, Text
|
|
19
19
|
|
|
20
|
-
import mlrun.common.model_monitoring
|
|
20
|
+
import mlrun.common.schemas.model_monitoring
|
|
21
21
|
from mlrun.utils.db import BaseModel
|
|
22
22
|
|
|
23
23
|
from .base import Base
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
class ModelEndpointsTable(Base, BaseModel):
|
|
27
|
-
__tablename__ =
|
|
27
|
+
__tablename__ = mlrun.common.schemas.model_monitoring.EventFieldType.MODEL_ENDPOINTS
|
|
28
28
|
|
|
29
29
|
uid = Column(
|
|
30
|
-
|
|
30
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.UID,
|
|
31
31
|
String(40),
|
|
32
32
|
primary_key=True,
|
|
33
33
|
)
|
|
34
|
-
state = Column(
|
|
35
|
-
|
|
34
|
+
state = Column(
|
|
35
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.STATE, String(10)
|
|
36
|
+
)
|
|
37
|
+
project = Column(
|
|
38
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.PROJECT, String(40)
|
|
39
|
+
)
|
|
36
40
|
function_uri = Column(
|
|
37
|
-
|
|
41
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.FUNCTION_URI,
|
|
38
42
|
String(255),
|
|
39
43
|
)
|
|
40
|
-
model = Column(
|
|
44
|
+
model = Column(
|
|
45
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.MODEL, String(255)
|
|
46
|
+
)
|
|
41
47
|
model_class = Column(
|
|
42
|
-
|
|
48
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.MODEL_CLASS,
|
|
43
49
|
String(255),
|
|
44
50
|
)
|
|
45
|
-
labels = Column(
|
|
46
|
-
model_uri = Column(
|
|
47
|
-
|
|
51
|
+
labels = Column(mlrun.common.schemas.model_monitoring.EventFieldType.LABELS, Text)
|
|
52
|
+
model_uri = Column(
|
|
53
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.MODEL_URI, String(255)
|
|
54
|
+
)
|
|
55
|
+
stream_path = Column(
|
|
56
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.STREAM_PATH, Text
|
|
57
|
+
)
|
|
48
58
|
algorithm = Column(
|
|
49
|
-
|
|
59
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ALGORITHM,
|
|
50
60
|
String(255),
|
|
51
61
|
)
|
|
52
|
-
active = Column(
|
|
62
|
+
active = Column(
|
|
63
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ACTIVE, Boolean
|
|
64
|
+
)
|
|
53
65
|
monitoring_mode = Column(
|
|
54
|
-
|
|
66
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.MONITORING_MODE,
|
|
55
67
|
String(10),
|
|
56
68
|
)
|
|
57
69
|
feature_stats = Column(
|
|
58
|
-
|
|
70
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.FEATURE_STATS, Text
|
|
59
71
|
)
|
|
60
72
|
current_stats = Column(
|
|
61
|
-
|
|
73
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.CURRENT_STATS, Text
|
|
62
74
|
)
|
|
63
75
|
feature_names = Column(
|
|
64
|
-
|
|
76
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.FEATURE_NAMES, Text
|
|
77
|
+
)
|
|
78
|
+
children = Column(
|
|
79
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.CHILDREN, Text
|
|
80
|
+
)
|
|
81
|
+
label_names = Column(
|
|
82
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.LABEL_NAMES, Text
|
|
65
83
|
)
|
|
66
|
-
children = Column(model_monitoring_constants.EventFieldType.CHILDREN, Text)
|
|
67
|
-
label_names = Column(model_monitoring_constants.EventFieldType.LABEL_NAMES, Text)
|
|
68
84
|
|
|
69
85
|
endpoint_type = Column(
|
|
70
|
-
|
|
86
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_TYPE,
|
|
71
87
|
String(10),
|
|
72
88
|
)
|
|
73
89
|
children_uids = Column(
|
|
74
|
-
|
|
90
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.CHILDREN_UIDS, Text
|
|
75
91
|
)
|
|
76
92
|
drift_measures = Column(
|
|
77
|
-
|
|
93
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.DRIFT_MEASURES, Text
|
|
78
94
|
)
|
|
79
95
|
drift_status = Column(
|
|
80
|
-
|
|
96
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.DRIFT_STATUS,
|
|
81
97
|
String(40),
|
|
82
98
|
)
|
|
83
99
|
monitor_configuration = Column(
|
|
84
|
-
|
|
100
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.MONITOR_CONFIGURATION,
|
|
85
101
|
Text,
|
|
86
102
|
)
|
|
87
103
|
monitoring_feature_set_uri = Column(
|
|
88
|
-
|
|
104
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.FEATURE_SET_URI,
|
|
89
105
|
String(255),
|
|
90
106
|
)
|
|
91
107
|
first_request = Column(
|
|
92
|
-
|
|
108
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.FIRST_REQUEST,
|
|
93
109
|
sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
|
|
94
110
|
)
|
|
95
111
|
last_request = Column(
|
|
96
|
-
|
|
112
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.LAST_REQUEST,
|
|
97
113
|
sqlalchemy.dialects.mysql.TIMESTAMP(fsp=3),
|
|
98
114
|
)
|
|
99
|
-
error_count = Column(
|
|
100
|
-
|
|
115
|
+
error_count = Column(
|
|
116
|
+
mlrun.common.schemas.model_monitoring.EventFieldType.ERROR_COUNT, Integer
|
|
117
|
+
)
|
|
118
|
+
metrics = Column(mlrun.common.schemas.model_monitoring.EventFieldType.METRICS, Text)
|