mlrun 1.3.2rc1__py3-none-any.whl → 1.3.2rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/api/api/deps.py +14 -1
- mlrun/api/api/endpoints/frontend_spec.py +0 -2
- mlrun/api/api/endpoints/functions.py +15 -27
- mlrun/api/api/endpoints/grafana_proxy.py +435 -74
- mlrun/api/api/endpoints/healthz.py +5 -18
- mlrun/api/api/endpoints/model_endpoints.py +33 -37
- mlrun/api/api/utils.py +6 -13
- mlrun/api/crud/__init__.py +14 -16
- mlrun/api/crud/logs.py +5 -7
- mlrun/api/crud/model_monitoring/__init__.py +2 -2
- mlrun/api/crud/model_monitoring/model_endpoint_store.py +847 -0
- mlrun/api/crud/model_monitoring/model_endpoints.py +105 -328
- mlrun/api/crud/pipelines.py +2 -3
- mlrun/api/db/sqldb/models/models_mysql.py +52 -19
- mlrun/api/db/sqldb/models/models_sqlite.py +52 -19
- mlrun/api/db/sqldb/session.py +19 -26
- mlrun/api/schemas/__init__.py +2 -0
- mlrun/api/schemas/constants.py +0 -13
- mlrun/api/schemas/frontend_spec.py +0 -1
- mlrun/api/schemas/model_endpoints.py +38 -195
- mlrun/api/schemas/schedule.py +2 -2
- mlrun/api/utils/clients/log_collector.py +5 -0
- mlrun/builder.py +9 -41
- mlrun/config.py +1 -76
- mlrun/data_types/__init__.py +1 -6
- mlrun/data_types/data_types.py +1 -3
- mlrun/datastore/__init__.py +2 -9
- mlrun/datastore/sources.py +20 -25
- mlrun/datastore/store_resources.py +1 -1
- mlrun/datastore/targets.py +34 -67
- mlrun/datastore/utils.py +4 -26
- mlrun/db/base.py +2 -4
- mlrun/db/filedb.py +5 -13
- mlrun/db/httpdb.py +32 -64
- mlrun/db/sqldb.py +2 -4
- mlrun/errors.py +0 -5
- mlrun/execution.py +0 -2
- mlrun/feature_store/api.py +8 -24
- mlrun/feature_store/feature_set.py +6 -28
- mlrun/feature_store/feature_vector.py +0 -2
- mlrun/feature_store/ingestion.py +11 -8
- mlrun/feature_store/retrieval/base.py +43 -271
- mlrun/feature_store/retrieval/dask_merger.py +153 -55
- mlrun/feature_store/retrieval/job.py +3 -12
- mlrun/feature_store/retrieval/local_merger.py +130 -48
- mlrun/feature_store/retrieval/spark_merger.py +125 -126
- mlrun/features.py +2 -7
- mlrun/model_monitoring/constants.py +6 -48
- mlrun/model_monitoring/helpers.py +35 -118
- mlrun/model_monitoring/model_monitoring_batch.py +260 -293
- mlrun/model_monitoring/stream_processing_fs.py +253 -220
- mlrun/platforms/iguazio.py +0 -33
- mlrun/projects/project.py +72 -34
- mlrun/runtimes/base.py +0 -5
- mlrun/runtimes/daskjob.py +0 -2
- mlrun/runtimes/function.py +3 -29
- mlrun/runtimes/kubejob.py +15 -39
- mlrun/runtimes/local.py +45 -7
- mlrun/runtimes/mpijob/abstract.py +0 -2
- mlrun/runtimes/mpijob/v1.py +0 -2
- mlrun/runtimes/pod.py +0 -2
- mlrun/runtimes/remotesparkjob.py +0 -2
- mlrun/runtimes/serving.py +0 -6
- mlrun/runtimes/sparkjob/abstract.py +2 -39
- mlrun/runtimes/sparkjob/spark3job.py +0 -2
- mlrun/serving/__init__.py +1 -2
- mlrun/serving/routers.py +35 -35
- mlrun/serving/server.py +12 -22
- mlrun/serving/states.py +30 -162
- mlrun/serving/v2_serving.py +10 -13
- mlrun/utils/clones.py +1 -1
- mlrun/utils/model_monitoring.py +96 -122
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/METADATA +27 -23
- {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/RECORD +79 -92
- mlrun/api/crud/model_monitoring/grafana.py +0 -427
- mlrun/datastore/spark_udf.py +0 -40
- mlrun/model_monitoring/__init__.py +0 -44
- mlrun/model_monitoring/common.py +0 -112
- mlrun/model_monitoring/model_endpoint.py +0 -141
- mlrun/model_monitoring/stores/__init__.py +0 -106
- mlrun/model_monitoring/stores/kv_model_endpoint_store.py +0 -448
- mlrun/model_monitoring/stores/model_endpoint_store.py +0 -147
- mlrun/model_monitoring/stores/models/__init__.py +0 -23
- mlrun/model_monitoring/stores/models/base.py +0 -18
- mlrun/model_monitoring/stores/models/mysql.py +0 -100
- mlrun/model_monitoring/stores/models/sqlite.py +0 -98
- mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -375
- mlrun/utils/db.py +0 -52
- {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/LICENSE +0 -0
- {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/WHEEL +0 -0
- {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/entry_points.txt +0 -0
- {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/top_level.txt +0 -0
|
@@ -12,10 +12,10 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
#
|
|
15
|
-
|
|
15
|
+
|
|
16
|
+
|
|
16
17
|
import os
|
|
17
18
|
import typing
|
|
18
|
-
import warnings
|
|
19
19
|
|
|
20
20
|
import sqlalchemy.orm
|
|
21
21
|
|
|
@@ -35,9 +35,10 @@ import mlrun.runtimes.function
|
|
|
35
35
|
import mlrun.utils.helpers
|
|
36
36
|
import mlrun.utils.model_monitoring
|
|
37
37
|
import mlrun.utils.v3io_clients
|
|
38
|
-
from mlrun.model_monitoring.stores import get_model_endpoint_store
|
|
39
38
|
from mlrun.utils import logger
|
|
40
39
|
|
|
40
|
+
from .model_endpoint_store import get_model_endpoint_target
|
|
41
|
+
|
|
41
42
|
|
|
42
43
|
class ModelEndpoints:
|
|
43
44
|
"""Provide different methods for handling model endpoints such as listing, writing and deleting"""
|
|
@@ -49,14 +50,9 @@ class ModelEndpoints:
|
|
|
49
50
|
model_endpoint: mlrun.api.schemas.ModelEndpoint,
|
|
50
51
|
auth_info: mlrun.api.schemas.AuthInfo = mlrun.api.schemas.AuthInfo(),
|
|
51
52
|
) -> mlrun.api.schemas.ModelEndpoint:
|
|
52
|
-
# TODO: deprecated
|
|
53
|
-
warnings.warn(
|
|
54
|
-
"This is deprecated in 1.3.0, and will be removed in 1.5.0."
|
|
55
|
-
"Please use create_model_endpoint() for create or patch_model_endpoint() for update",
|
|
56
|
-
FutureWarning,
|
|
57
|
-
)
|
|
53
|
+
# TODO: deprecated, remove in 1.5.0.
|
|
58
54
|
"""
|
|
59
|
-
Either create or updates the record of a given
|
|
55
|
+
Either create or updates the record of a given ModelEndpoint object.
|
|
60
56
|
Leaving here for backwards compatibility, remove in 1.5.0.
|
|
61
57
|
|
|
62
58
|
:param db_session: A session that manages the current dialog with the database
|
|
@@ -64,7 +60,7 @@ class ModelEndpoints:
|
|
|
64
60
|
:param model_endpoint: Model endpoint object to update
|
|
65
61
|
:param auth_info: The auth info of the request
|
|
66
62
|
|
|
67
|
-
:return:
|
|
63
|
+
:return: Model endpoint object.
|
|
68
64
|
"""
|
|
69
65
|
|
|
70
66
|
return self.create_model_endpoint(
|
|
@@ -78,12 +74,12 @@ class ModelEndpoints:
|
|
|
78
74
|
) -> mlrun.api.schemas.ModelEndpoint:
|
|
79
75
|
"""
|
|
80
76
|
Creates model endpoint record in DB. The DB target type is defined under
|
|
81
|
-
|
|
77
|
+
mlrun.config.model_endpoint_monitoring.store_type (KV by default).
|
|
82
78
|
|
|
83
79
|
:param db_session: A session that manages the current dialog with the database.
|
|
84
80
|
:param model_endpoint: Model endpoint object to update.
|
|
85
81
|
|
|
86
|
-
:return:
|
|
82
|
+
:return: Model endpoint object.
|
|
87
83
|
"""
|
|
88
84
|
|
|
89
85
|
if model_endpoint.spec.model_uri or model_endpoint.status.feature_stats:
|
|
@@ -111,22 +107,23 @@ class ModelEndpoints:
|
|
|
111
107
|
if not model_endpoint.status.feature_stats and hasattr(
|
|
112
108
|
model_obj, "feature_stats"
|
|
113
109
|
):
|
|
114
|
-
model_endpoint.status.feature_stats = model_obj.
|
|
110
|
+
model_endpoint.status.feature_stats = model_obj.feature_stats
|
|
111
|
+
|
|
115
112
|
# Get labels from model object if not found in model endpoint object
|
|
116
|
-
if not model_endpoint.spec.label_names and model_obj
|
|
113
|
+
if not model_endpoint.spec.label_names and hasattr(model_obj, "outputs"):
|
|
117
114
|
model_label_names = [
|
|
118
|
-
self._clean_feature_name(f.name) for f in model_obj.
|
|
115
|
+
self._clean_feature_name(f.name) for f in model_obj.outputs
|
|
119
116
|
]
|
|
120
117
|
model_endpoint.spec.label_names = model_label_names
|
|
121
118
|
|
|
122
119
|
# Get algorithm from model object if not found in model endpoint object
|
|
123
|
-
if not model_endpoint.spec.algorithm and model_obj
|
|
124
|
-
model_endpoint.spec.algorithm = model_obj.
|
|
120
|
+
if not model_endpoint.spec.algorithm and hasattr(model_obj, "algorithm"):
|
|
121
|
+
model_endpoint.spec.algorithm = model_obj.algorithm
|
|
125
122
|
|
|
126
123
|
# Create monitoring feature set if monitoring found in model endpoint object
|
|
127
124
|
if (
|
|
128
125
|
model_endpoint.spec.monitoring_mode
|
|
129
|
-
== mlrun.
|
|
126
|
+
== mlrun.api.schemas.ModelMonitoringMode.enabled.value
|
|
130
127
|
):
|
|
131
128
|
monitoring_feature_set = self.create_monitoring_feature_set(
|
|
132
129
|
model_endpoint, model_obj, db_session, run_db
|
|
@@ -161,17 +158,17 @@ class ModelEndpoints:
|
|
|
161
158
|
logger.info("Creating model endpoint", endpoint_id=model_endpoint.metadata.uid)
|
|
162
159
|
|
|
163
160
|
# Write the new model endpoint
|
|
164
|
-
|
|
161
|
+
model_endpoint_target = get_model_endpoint_target(
|
|
165
162
|
project=model_endpoint.metadata.project,
|
|
166
163
|
)
|
|
167
|
-
|
|
164
|
+
model_endpoint_target.write_model_endpoint(endpoint=model_endpoint)
|
|
168
165
|
|
|
169
166
|
logger.info("Model endpoint created", endpoint_id=model_endpoint.metadata.uid)
|
|
170
167
|
|
|
171
168
|
return model_endpoint
|
|
172
169
|
|
|
170
|
+
@staticmethod
|
|
173
171
|
def create_monitoring_feature_set(
|
|
174
|
-
self,
|
|
175
172
|
model_endpoint: mlrun.api.schemas.ModelEndpoint,
|
|
176
173
|
model_obj: mlrun.artifacts.ModelArtifact,
|
|
177
174
|
db_session: sqlalchemy.orm.Session,
|
|
@@ -198,29 +195,29 @@ class ModelEndpoints:
|
|
|
198
195
|
|
|
199
196
|
feature_set = mlrun.feature_store.FeatureSet(
|
|
200
197
|
f"monitoring-{serving_function_name}-{model_name}",
|
|
201
|
-
entities=[
|
|
202
|
-
timestamp_key=
|
|
198
|
+
entities=["endpoint_id"],
|
|
199
|
+
timestamp_key="timestamp",
|
|
203
200
|
description=f"Monitoring feature set for endpoint: {model_endpoint.spec.model}",
|
|
204
201
|
)
|
|
205
202
|
feature_set.metadata.project = model_endpoint.metadata.project
|
|
206
203
|
|
|
207
204
|
feature_set.metadata.labels = {
|
|
208
|
-
|
|
209
|
-
|
|
205
|
+
"endpoint_id": model_endpoint.metadata.uid,
|
|
206
|
+
"model_class": model_endpoint.spec.model_class,
|
|
210
207
|
}
|
|
211
208
|
|
|
212
209
|
# Add features to the feature set according to the model object
|
|
213
|
-
if model_obj.
|
|
214
|
-
for feature in model_obj.
|
|
210
|
+
if model_obj.inputs.values():
|
|
211
|
+
for feature in model_obj.inputs.values():
|
|
215
212
|
feature_set.add_feature(
|
|
216
213
|
mlrun.feature_store.Feature(
|
|
217
214
|
name=feature.name, value_type=feature.value_type
|
|
218
215
|
)
|
|
219
216
|
)
|
|
220
217
|
# Check if features can be found within the feature vector
|
|
221
|
-
elif model_obj.
|
|
218
|
+
elif model_obj.feature_vector:
|
|
222
219
|
_, name, _, tag, _ = mlrun.utils.helpers.parse_artifact_uri(
|
|
223
|
-
model_obj.
|
|
220
|
+
model_obj.feature_vector
|
|
224
221
|
)
|
|
225
222
|
fv = run_db.get_feature_vector(
|
|
226
223
|
name=name, project=model_endpoint.metadata.project, tag=tag
|
|
@@ -239,22 +236,16 @@ class ModelEndpoints:
|
|
|
239
236
|
|
|
240
237
|
# Define parquet target for this feature set
|
|
241
238
|
parquet_path = (
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
)
|
|
245
|
-
+ f"/key={model_endpoint.metadata.uid}"
|
|
246
|
-
)
|
|
247
|
-
|
|
248
|
-
parquet_target = mlrun.datastore.targets.ParquetTarget(
|
|
249
|
-
model_monitoring_constants.FileTargetKind.PARQUET, parquet_path
|
|
239
|
+
f"v3io:///projects/{model_endpoint.metadata.project}"
|
|
240
|
+
f"/model-endpoints/parquet/key={model_endpoint.metadata.uid}"
|
|
250
241
|
)
|
|
242
|
+
parquet_target = mlrun.datastore.targets.ParquetTarget("parquet", parquet_path)
|
|
251
243
|
driver = mlrun.datastore.targets.get_target_driver(parquet_target, feature_set)
|
|
252
|
-
|
|
244
|
+
driver.update_resource_status("created")
|
|
253
245
|
feature_set.set_targets(
|
|
254
246
|
[mlrun.datastore.targets.ParquetTarget(path=parquet_path)],
|
|
255
247
|
with_defaults=False,
|
|
256
248
|
)
|
|
257
|
-
driver.update_resource_status("created")
|
|
258
249
|
|
|
259
250
|
# Save the new feature set
|
|
260
251
|
feature_set._override_run_db(db_session)
|
|
@@ -267,39 +258,10 @@ class ModelEndpoints:
|
|
|
267
258
|
|
|
268
259
|
return feature_set
|
|
269
260
|
|
|
270
|
-
@staticmethod
|
|
271
|
-
def _get_monitoring_parquet_path(
|
|
272
|
-
db_session: sqlalchemy.orm.Session, project: str
|
|
273
|
-
) -> str:
|
|
274
|
-
"""Getting model monitoring parquet target for the current project. The parquet target path is based on the
|
|
275
|
-
project artifact path. If project artifact path is not defined, the parquet target path will be based on MLRun
|
|
276
|
-
artifact path.
|
|
277
|
-
|
|
278
|
-
:param db_session: A session that manages the current dialog with the database. Will be used in this function
|
|
279
|
-
to get the project record from DB.
|
|
280
|
-
:param project: Project name.
|
|
281
|
-
|
|
282
|
-
:return: Monitoring parquet target path.
|
|
283
|
-
"""
|
|
284
|
-
|
|
285
|
-
# Get the artifact path from the project record that was stored in the DB
|
|
286
|
-
project_obj = mlrun.api.crud.projects.Projects().get_project(
|
|
287
|
-
session=db_session, name=project
|
|
288
|
-
)
|
|
289
|
-
artifact_path = project_obj.spec.artifact_path
|
|
290
|
-
# Generate monitoring parquet path value
|
|
291
|
-
parquet_path = mlrun.mlconf.get_model_monitoring_file_target_path(
|
|
292
|
-
project=project,
|
|
293
|
-
kind=model_monitoring_constants.FileTargetKind.PARQUET,
|
|
294
|
-
target="offline",
|
|
295
|
-
artifact_path=artifact_path,
|
|
296
|
-
)
|
|
297
|
-
return parquet_path
|
|
298
|
-
|
|
299
261
|
@staticmethod
|
|
300
262
|
def _validate_length_features_and_labels(model_endpoint):
|
|
301
263
|
"""
|
|
302
|
-
Validate that the length of feature_stats is equal to the length of
|
|
264
|
+
Validate that the length of feature_stats is equal to the length of feature_names and label_names
|
|
303
265
|
|
|
304
266
|
:param model_endpoint: An object representing the model endpoint.
|
|
305
267
|
"""
|
|
@@ -326,8 +288,8 @@ class ModelEndpoints:
|
|
|
326
288
|
self, model_endpoint
|
|
327
289
|
) -> typing.Tuple[typing.Dict, typing.List]:
|
|
328
290
|
"""
|
|
329
|
-
Create a clean matching version of feature names for both
|
|
330
|
-
label names exist only in
|
|
291
|
+
Create a clean matching version of feature names for both feature_stats and feature_names. Please note that
|
|
292
|
+
label names exist only in feature_stats and label_names.
|
|
331
293
|
|
|
332
294
|
:param model_endpoint: An object representing the model endpoint.
|
|
333
295
|
:return: A tuple of:
|
|
@@ -350,8 +312,8 @@ class ModelEndpoints:
|
|
|
350
312
|
clean_feature_names.append(clean_name)
|
|
351
313
|
return clean_feature_stats, clean_feature_names
|
|
352
314
|
|
|
315
|
+
@staticmethod
|
|
353
316
|
def patch_model_endpoint(
|
|
354
|
-
self,
|
|
355
317
|
project: str,
|
|
356
318
|
endpoint_id: str,
|
|
357
319
|
attributes: dict,
|
|
@@ -362,30 +324,24 @@ class ModelEndpoints:
|
|
|
362
324
|
:param project: The name of the project.
|
|
363
325
|
:param endpoint_id: The unique id of the model endpoint.
|
|
364
326
|
:param attributes: Dictionary of attributes that will be used for update the model endpoint. Note that the keys
|
|
365
|
-
of the attributes dictionary should exist in the
|
|
327
|
+
of the attributes dictionary should exist in the KV table. More details about the model
|
|
366
328
|
endpoint available attributes can be found under
|
|
367
329
|
:py:class:`~mlrun.api.schemas.ModelEndpoint`.
|
|
368
330
|
|
|
369
|
-
:return: A patched
|
|
331
|
+
:return: A patched ModelEndpoint object.
|
|
370
332
|
"""
|
|
371
333
|
|
|
372
|
-
|
|
373
|
-
model_endpoint_store = get_model_endpoint_store(
|
|
334
|
+
model_endpoint_target = get_model_endpoint_target(
|
|
374
335
|
project=project,
|
|
375
336
|
)
|
|
376
|
-
|
|
337
|
+
model_endpoint_target.update_model_endpoint(
|
|
377
338
|
endpoint_id=endpoint_id, attributes=attributes
|
|
378
339
|
)
|
|
379
340
|
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
# Get the patched model endpoint record
|
|
383
|
-
model_endpoint_record = model_endpoint_store.get_model_endpoint(
|
|
384
|
-
endpoint_id=endpoint_id,
|
|
341
|
+
return model_endpoint_target.get_model_endpoint(
|
|
342
|
+
endpoint_id=endpoint_id, start="now-1h", end="now"
|
|
385
343
|
)
|
|
386
344
|
|
|
387
|
-
return self._convert_into_model_endpoint_object(endpoint=model_endpoint_record)
|
|
388
|
-
|
|
389
345
|
@staticmethod
|
|
390
346
|
def delete_model_endpoint(
|
|
391
347
|
project: str,
|
|
@@ -397,16 +353,13 @@ class ModelEndpoints:
|
|
|
397
353
|
:param project: The name of the project.
|
|
398
354
|
:param endpoint_id: The id of the endpoint.
|
|
399
355
|
"""
|
|
400
|
-
|
|
356
|
+
model_endpoint_target = get_model_endpoint_target(
|
|
401
357
|
project=project,
|
|
402
358
|
)
|
|
359
|
+
model_endpoint_target.delete_model_endpoint(endpoint_id=endpoint_id)
|
|
403
360
|
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
logger.info("Model endpoint table cleared", endpoint_id=endpoint_id)
|
|
407
|
-
|
|
361
|
+
@staticmethod
|
|
408
362
|
def get_model_endpoint(
|
|
409
|
-
self,
|
|
410
363
|
auth_info: mlrun.api.schemas.AuthInfo,
|
|
411
364
|
project: str,
|
|
412
365
|
endpoint_id: str,
|
|
@@ -418,61 +371,40 @@ class ModelEndpoints:
|
|
|
418
371
|
"""Get a single model endpoint object. You can apply different time series metrics that will be added to the
|
|
419
372
|
result.
|
|
420
373
|
|
|
421
|
-
:param auth_info:
|
|
422
|
-
:param project:
|
|
423
|
-
:param endpoint_id:
|
|
424
|
-
:param metrics:
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
be added to the output of the resulting object.
|
|
439
|
-
|
|
440
|
-
:return: A `ModelEndpoint` object.
|
|
441
|
-
"""
|
|
374
|
+
:param auth_info: The auth info of the request
|
|
375
|
+
:param project: The name of the project
|
|
376
|
+
:param endpoint_id: The unique id of the model endpoint.
|
|
377
|
+
:param metrics: A list of metrics to return for the model endpoint. There are pre-defined metrics for
|
|
378
|
+
model endpoints such as predictions_per_second and latency_avg_5m but also custom
|
|
379
|
+
metrics defined by the user. Please note that these metrics are stored in the time
|
|
380
|
+
series DB and the results will be appeared under model_endpoint.spec.metrics.
|
|
381
|
+
:param start: The start time of the metrics. Can be represented by a string containing an RFC 3339
|
|
382
|
+
time, a Unix timestamp in milliseconds, a relative time (`'now'` or
|
|
383
|
+
`'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` =
|
|
384
|
+
days), or 0 for the earliest time.
|
|
385
|
+
:param end: The end time of the metrics. Can be represented by a string containing an RFC 3339
|
|
386
|
+
time, a Unix timestamp in milliseconds, a relative time (`'now'` or
|
|
387
|
+
`'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` =
|
|
388
|
+
days), or 0 for the earliest time.
|
|
389
|
+
:param feature_analysis: When True, the base feature statistics and current feature statistics will be added to
|
|
390
|
+
the output of the resulting object.
|
|
442
391
|
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
endpoint_id=endpoint_id,
|
|
446
|
-
)
|
|
392
|
+
:return: A ModelEndpoint object.
|
|
393
|
+
"""
|
|
447
394
|
|
|
448
|
-
|
|
449
|
-
model_endpoint_store = get_model_endpoint_store(
|
|
395
|
+
model_endpoint_target = get_model_endpoint_target(
|
|
450
396
|
project=project, access_key=auth_info.data_session
|
|
451
397
|
)
|
|
452
|
-
|
|
453
|
-
model_endpoint_record = model_endpoint_store.get_model_endpoint(
|
|
398
|
+
return model_endpoint_target.get_model_endpoint(
|
|
454
399
|
endpoint_id=endpoint_id,
|
|
400
|
+
metrics=metrics,
|
|
401
|
+
start=start,
|
|
402
|
+
end=end,
|
|
403
|
+
feature_analysis=feature_analysis,
|
|
455
404
|
)
|
|
456
405
|
|
|
457
|
-
|
|
458
|
-
model_endpoint_object = self._convert_into_model_endpoint_object(
|
|
459
|
-
endpoint=model_endpoint_record, feature_analysis=feature_analysis
|
|
460
|
-
)
|
|
461
|
-
|
|
462
|
-
# If time metrics were provided, retrieve the results from the time series DB
|
|
463
|
-
if metrics:
|
|
464
|
-
self._add_real_time_metrics(
|
|
465
|
-
model_endpoint_store=model_endpoint_store,
|
|
466
|
-
model_endpoint_object=model_endpoint_object,
|
|
467
|
-
metrics=metrics,
|
|
468
|
-
start=start,
|
|
469
|
-
end=end,
|
|
470
|
-
)
|
|
471
|
-
|
|
472
|
-
return model_endpoint_object
|
|
473
|
-
|
|
406
|
+
@staticmethod
|
|
474
407
|
def list_model_endpoints(
|
|
475
|
-
self,
|
|
476
408
|
auth_info: mlrun.api.schemas.AuthInfo,
|
|
477
409
|
project: str,
|
|
478
410
|
model: str = None,
|
|
@@ -483,11 +415,10 @@ class ModelEndpoints:
|
|
|
483
415
|
end: str = "now",
|
|
484
416
|
top_level: bool = False,
|
|
485
417
|
uids: typing.List[str] = None,
|
|
486
|
-
) -> mlrun.api.schemas.ModelEndpointList:
|
|
418
|
+
) -> mlrun.api.schemas.model_endpoints.ModelEndpointList:
|
|
487
419
|
"""
|
|
488
|
-
Returns a list of
|
|
489
|
-
|
|
490
|
-
parameters:
|
|
420
|
+
Returns a list of ModelEndpointState objects. Each object represents the current state of a model endpoint.
|
|
421
|
+
This functions supports filtering by the following parameters:
|
|
491
422
|
1) model
|
|
492
423
|
2) function
|
|
493
424
|
3) labels
|
|
@@ -504,22 +435,22 @@ class ModelEndpoints:
|
|
|
504
435
|
:param model: The name of the model to filter by.
|
|
505
436
|
:param function: The name of the function to filter by.
|
|
506
437
|
:param labels: A list of labels to filter by. Label filters work by either filtering a specific value of a
|
|
507
|
-
label (i.e. list("key
|
|
438
|
+
label (i.e. list("key==value")) or by looking for the existence of a given key (i.e. "key").
|
|
508
439
|
:param metrics: A list of metrics to return for each endpoint. There are pre-defined metrics for model
|
|
509
|
-
endpoints such as
|
|
510
|
-
|
|
511
|
-
|
|
440
|
+
endpoints such as predictions_per_second and latency_avg_5m but also custom metrics defined
|
|
441
|
+
by the user. Please note that these metrics are stored in the time series DB and the results
|
|
442
|
+
will be appeared under model_endpoint.spec.metrics of each endpoint.
|
|
512
443
|
:param start: The start time of the metrics. Can be represented by a string containing an RFC 3339 time,
|
|
513
444
|
a Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where `m`
|
|
514
445
|
= minutes, `h` = hours, and `'d'` = days), or 0 for the earliest time.
|
|
515
446
|
:param end: The end time of the metrics. Can be represented by a string containing an RFC 3339 time,
|
|
516
447
|
a Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where `m`
|
|
517
448
|
= minutes, `h` = hours, and `'d'` = days), or 0 for the earliest time.
|
|
518
|
-
:param top_level: If True
|
|
519
|
-
:param uids:
|
|
449
|
+
:param top_level: If True will return only routers and endpoint that are NOT children of any router.
|
|
450
|
+
:param uids: Will return ModelEndpointList of endpoints with uid in uids.
|
|
520
451
|
|
|
521
|
-
:return: An object of
|
|
522
|
-
To get a standard list of model endpoints use
|
|
452
|
+
:return: An object of ModelEndpointList which is literally a list of model endpoints along with some metadata.
|
|
453
|
+
To get a standard list of model endpoints use ModelEndpointList.endpoints.
|
|
523
454
|
"""
|
|
524
455
|
|
|
525
456
|
logger.info(
|
|
@@ -535,174 +466,32 @@ class ModelEndpoints:
|
|
|
535
466
|
uids=uids,
|
|
536
467
|
)
|
|
537
468
|
|
|
538
|
-
|
|
539
|
-
endpoint_list = mlrun.api.schemas.model_endpoints.ModelEndpointList(
|
|
540
|
-
endpoints=[]
|
|
541
|
-
)
|
|
542
|
-
|
|
543
|
-
# Generate a model endpoint store object and get a list of model endpoint dictionaries
|
|
544
|
-
endpoint_store = get_model_endpoint_store(
|
|
469
|
+
endpoint_target = get_model_endpoint_target(
|
|
545
470
|
access_key=auth_info.data_session, project=project
|
|
546
471
|
)
|
|
547
472
|
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
labels=labels,
|
|
552
|
-
top_level=top_level,
|
|
553
|
-
uids=uids,
|
|
554
|
-
)
|
|
555
|
-
|
|
556
|
-
for endpoint_dict in endpoint_dictionary_list:
|
|
557
|
-
|
|
558
|
-
# Convert to `ModelEndpoint` object
|
|
559
|
-
endpoint_obj = self._convert_into_model_endpoint_object(
|
|
560
|
-
endpoint=endpoint_dict
|
|
561
|
-
)
|
|
562
|
-
|
|
563
|
-
# If time metrics were provided, retrieve the results from the time series DB
|
|
564
|
-
if metrics:
|
|
565
|
-
self._add_real_time_metrics(
|
|
566
|
-
model_endpoint_store=endpoint_store,
|
|
567
|
-
model_endpoint_object=endpoint_obj,
|
|
568
|
-
metrics=metrics,
|
|
569
|
-
start=start,
|
|
570
|
-
end=end,
|
|
571
|
-
)
|
|
572
|
-
|
|
573
|
-
# Add the `ModelEndpoint` object into the model endpoints list
|
|
574
|
-
endpoint_list.endpoints.append(endpoint_obj)
|
|
575
|
-
|
|
576
|
-
return endpoint_list
|
|
577
|
-
|
|
578
|
-
@staticmethod
|
|
579
|
-
def _add_real_time_metrics(
|
|
580
|
-
model_endpoint_store: mlrun.model_monitoring.stores.ModelEndpointStore,
|
|
581
|
-
model_endpoint_object: mlrun.api.schemas.ModelEndpoint,
|
|
582
|
-
metrics: typing.List[str] = None,
|
|
583
|
-
start: str = "now-1h",
|
|
584
|
-
end: str = "now",
|
|
585
|
-
) -> mlrun.api.schemas.ModelEndpoint:
|
|
586
|
-
"""Add real time metrics from the time series DB to a provided `ModelEndpoint` object. The real time metrics
|
|
587
|
-
will be stored under `ModelEndpoint.status.metrics.real_time`
|
|
588
|
-
|
|
589
|
-
:param model_endpoint_store: `ModelEndpointStore` object that will be used for communicating with the database
|
|
590
|
-
and querying the required metrics.
|
|
591
|
-
:param model_endpoint_object: `ModelEndpoint` object that will be filled with the relevant
|
|
592
|
-
real time metrics.
|
|
593
|
-
:param metrics: A list of metrics to return for each endpoint. There are pre-defined metrics for
|
|
594
|
-
model endpoints such as `predictions_per_second` and `latency_avg_5m` but also
|
|
595
|
-
custom metrics defined by the user. Please note that these metrics are stored in
|
|
596
|
-
the time series DB and the results will be appeared under
|
|
597
|
-
model_endpoint.spec.metrics of each endpoint.
|
|
598
|
-
:param start: The start time of the metrics. Can be represented by a string containing an RFC
|
|
599
|
-
3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
|
|
600
|
-
`'now-[0-9]+[mhd]'`, where `m`= minutes, `h` = hours, and `'d'` = days), or 0
|
|
601
|
-
for the earliest time.
|
|
602
|
-
:param end: The end time of the metrics. Can be represented by a string containing an RFC
|
|
603
|
-
3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
|
|
604
|
-
`'now-[0-9]+[mhd]'`, where `m`= minutes, `h` = hours, and `'d'` = days), or 0
|
|
605
|
-
for the earliest time.
|
|
606
|
-
|
|
607
|
-
"""
|
|
608
|
-
if model_endpoint_object.status.metrics is None:
|
|
609
|
-
model_endpoint_object.status.metrics = {}
|
|
610
|
-
|
|
611
|
-
endpoint_metrics = model_endpoint_store.get_endpoint_real_time_metrics(
|
|
612
|
-
endpoint_id=model_endpoint_object.metadata.uid,
|
|
613
|
-
start=start,
|
|
614
|
-
end=end,
|
|
615
|
-
metrics=metrics,
|
|
473
|
+
# Initialize an empty model endpoints list
|
|
474
|
+
endpoint_list = mlrun.api.schemas.model_endpoints.ModelEndpointList(
|
|
475
|
+
endpoints=[]
|
|
616
476
|
)
|
|
617
|
-
if endpoint_metrics:
|
|
618
|
-
model_endpoint_object.status.metrics[
|
|
619
|
-
model_monitoring_constants.EventKeyMetrics.REAL_TIME
|
|
620
|
-
] = endpoint_metrics
|
|
621
|
-
return model_endpoint_object
|
|
622
|
-
|
|
623
|
-
def _convert_into_model_endpoint_object(
|
|
624
|
-
self, endpoint: typing.Dict[str, typing.Any], feature_analysis: bool = False
|
|
625
|
-
) -> mlrun.api.schemas.ModelEndpoint:
|
|
626
|
-
"""
|
|
627
|
-
Create a `ModelEndpoint` object according to a provided model endpoint dictionary.
|
|
628
|
-
|
|
629
|
-
:param endpoint: Dictinoary that represents a DB record of a model endpoint which need to be converted
|
|
630
|
-
into a valid `ModelEndpoint` object.
|
|
631
|
-
:param feature_analysis: When True, the base feature statistics and current feature statistics will be added to
|
|
632
|
-
the output of the resulting object.
|
|
633
477
|
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
endpoint_obj = mlrun.api.schemas.ModelEndpoint().from_flat_dict(endpoint)
|
|
639
|
-
|
|
640
|
-
# If feature analysis was applied, add feature stats and current stats to the model endpoint result
|
|
641
|
-
if feature_analysis and endpoint_obj.spec.feature_names:
|
|
642
|
-
|
|
643
|
-
endpoint_features = self.get_endpoint_features(
|
|
644
|
-
feature_names=endpoint_obj.spec.feature_names,
|
|
645
|
-
feature_stats=endpoint_obj.status.feature_stats,
|
|
646
|
-
current_stats=endpoint_obj.status.current_stats,
|
|
478
|
+
# If list of model endpoint ids was not provided, retrieve it from the DB
|
|
479
|
+
if uids is None:
|
|
480
|
+
uids = endpoint_target.list_model_endpoints(
|
|
481
|
+
function=function, model=model, labels=labels, top_level=top_level
|
|
647
482
|
)
|
|
648
|
-
if endpoint_features:
|
|
649
|
-
endpoint_obj.status.features = endpoint_features
|
|
650
|
-
# Add the latest drift measures results (calculated by the model monitoring batch)
|
|
651
|
-
drift_measures = self._json_loads_if_not_none(
|
|
652
|
-
endpoint.get(
|
|
653
|
-
model_monitoring_constants.EventFieldType.DRIFT_MEASURES
|
|
654
|
-
)
|
|
655
|
-
)
|
|
656
|
-
endpoint_obj.status.drift_measures = drift_measures
|
|
657
483
|
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
) -> typing.List[mlrun.api.schemas.Features]:
|
|
666
|
-
"""
|
|
667
|
-
Getting a new list of features that exist in feature_names along with their expected (feature_stats) and
|
|
668
|
-
actual (current_stats) stats. The expected stats were calculated during the creation of the model endpoint,
|
|
669
|
-
usually based on the data from the Model Artifact. The actual stats are based on the results from the latest
|
|
670
|
-
model monitoring batch job.
|
|
671
|
-
|
|
672
|
-
param feature_names: List of feature names.
|
|
673
|
-
param feature_stats: Dictionary of feature stats that were stored during the creation of the model endpoint
|
|
674
|
-
object.
|
|
675
|
-
param current_stats: Dictionary of the latest stats that were stored during the last run of the model monitoring
|
|
676
|
-
batch job.
|
|
677
|
-
|
|
678
|
-
return: List of feature objects. Each feature has a name, weight, expected values, and actual values. More info
|
|
679
|
-
can be found under `mlrun.api.schemas.Features`.
|
|
680
|
-
"""
|
|
681
|
-
|
|
682
|
-
# Initialize feature and current stats dictionaries
|
|
683
|
-
safe_feature_stats = feature_stats or {}
|
|
684
|
-
safe_current_stats = current_stats or {}
|
|
685
|
-
|
|
686
|
-
# Create feature object and add it to a general features list
|
|
687
|
-
features = []
|
|
688
|
-
for name in feature_names:
|
|
689
|
-
if feature_stats is not None and name not in feature_stats:
|
|
690
|
-
logger.warn("Feature missing from 'feature_stats'", name=name)
|
|
691
|
-
if current_stats is not None and name not in current_stats:
|
|
692
|
-
logger.warn("Feature missing from 'current_stats'", name=name)
|
|
693
|
-
f = mlrun.api.schemas.Features.new(
|
|
694
|
-
name, safe_feature_stats.get(name), safe_current_stats.get(name)
|
|
484
|
+
# Add each relevant model endpoint to the model endpoints list
|
|
485
|
+
for endpoint_id in uids:
|
|
486
|
+
endpoint = endpoint_target.get_model_endpoint(
|
|
487
|
+
metrics=metrics,
|
|
488
|
+
endpoint_id=endpoint_id,
|
|
489
|
+
start=start,
|
|
490
|
+
end=end,
|
|
695
491
|
)
|
|
696
|
-
|
|
697
|
-
return features
|
|
492
|
+
endpoint_list.endpoints.append(endpoint)
|
|
698
493
|
|
|
699
|
-
|
|
700
|
-
def _json_loads_if_not_none(field: typing.Any) -> typing.Any:
|
|
701
|
-
return (
|
|
702
|
-
json.loads(field)
|
|
703
|
-
if field and field != "null" and field is not None
|
|
704
|
-
else None
|
|
705
|
-
)
|
|
494
|
+
return endpoint_list
|
|
706
495
|
|
|
707
496
|
def deploy_monitoring_functions(
|
|
708
497
|
self,
|
|
@@ -750,8 +539,7 @@ class ModelEndpoints:
|
|
|
750
539
|
f"Project {project_name} can not be deleted since related resources found: model endpoints"
|
|
751
540
|
)
|
|
752
541
|
|
|
753
|
-
|
|
754
|
-
def delete_model_endpoints_resources(project_name: str):
|
|
542
|
+
def delete_model_endpoints_resources(self, project_name: str):
|
|
755
543
|
"""
|
|
756
544
|
Delete all model endpoints resources.
|
|
757
545
|
|
|
@@ -766,17 +554,15 @@ class ModelEndpoints:
|
|
|
766
554
|
if not mlrun.mlconf.igz_version or not mlrun.mlconf.v3io_api:
|
|
767
555
|
return
|
|
768
556
|
|
|
769
|
-
|
|
770
|
-
|
|
557
|
+
endpoints = self.list_model_endpoints(auth_info, project_name)
|
|
558
|
+
|
|
559
|
+
endpoint_target = get_model_endpoint_target(
|
|
771
560
|
access_key=auth_info.data_session, project=project_name
|
|
772
561
|
)
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
# Delete model endpoints resources from databases using the model endpoint store object
|
|
776
|
-
endpoint_store.delete_model_endpoints_resources(endpoints)
|
|
562
|
+
endpoint_target.delete_model_endpoints_resources(endpoints)
|
|
777
563
|
|
|
564
|
+
@staticmethod
|
|
778
565
|
def deploy_model_monitoring_stream_processing(
|
|
779
|
-
self,
|
|
780
566
|
project: str,
|
|
781
567
|
model_monitoring_access_key: str,
|
|
782
568
|
db_session: sqlalchemy.orm.Session,
|
|
@@ -817,17 +603,8 @@ class ModelEndpoints:
|
|
|
817
603
|
"Deploying model monitoring stream processing function", project=project
|
|
818
604
|
)
|
|
819
605
|
|
|
820
|
-
# Get parquet target value for model monitoring stream function
|
|
821
|
-
parquet_target = self._get_monitoring_parquet_path(
|
|
822
|
-
db_session=db_session, project=project
|
|
823
|
-
)
|
|
824
|
-
|
|
825
606
|
fn = mlrun.model_monitoring.helpers.initial_model_monitoring_stream_processing_function(
|
|
826
|
-
project
|
|
827
|
-
model_monitoring_access_key=model_monitoring_access_key,
|
|
828
|
-
tracking_policy=tracking_policy,
|
|
829
|
-
auth_info=auth_info,
|
|
830
|
-
parquet_target=parquet_target,
|
|
607
|
+
project, model_monitoring_access_key, db_session, tracking_policy
|
|
831
608
|
)
|
|
832
609
|
|
|
833
610
|
mlrun.api.api.endpoints.functions._build_function(
|