mlrun 1.3.2rc1__py3-none-any.whl → 1.3.2rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (93) hide show
  1. mlrun/api/api/deps.py +14 -1
  2. mlrun/api/api/endpoints/frontend_spec.py +0 -2
  3. mlrun/api/api/endpoints/functions.py +15 -27
  4. mlrun/api/api/endpoints/grafana_proxy.py +435 -74
  5. mlrun/api/api/endpoints/healthz.py +5 -18
  6. mlrun/api/api/endpoints/model_endpoints.py +33 -37
  7. mlrun/api/api/utils.py +6 -13
  8. mlrun/api/crud/__init__.py +14 -16
  9. mlrun/api/crud/logs.py +5 -7
  10. mlrun/api/crud/model_monitoring/__init__.py +2 -2
  11. mlrun/api/crud/model_monitoring/model_endpoint_store.py +847 -0
  12. mlrun/api/crud/model_monitoring/model_endpoints.py +105 -328
  13. mlrun/api/crud/pipelines.py +2 -3
  14. mlrun/api/db/sqldb/models/models_mysql.py +52 -19
  15. mlrun/api/db/sqldb/models/models_sqlite.py +52 -19
  16. mlrun/api/db/sqldb/session.py +19 -26
  17. mlrun/api/schemas/__init__.py +2 -0
  18. mlrun/api/schemas/constants.py +0 -13
  19. mlrun/api/schemas/frontend_spec.py +0 -1
  20. mlrun/api/schemas/model_endpoints.py +38 -195
  21. mlrun/api/schemas/schedule.py +2 -2
  22. mlrun/api/utils/clients/log_collector.py +5 -0
  23. mlrun/builder.py +9 -41
  24. mlrun/config.py +1 -76
  25. mlrun/data_types/__init__.py +1 -6
  26. mlrun/data_types/data_types.py +1 -3
  27. mlrun/datastore/__init__.py +2 -9
  28. mlrun/datastore/sources.py +20 -25
  29. mlrun/datastore/store_resources.py +1 -1
  30. mlrun/datastore/targets.py +34 -67
  31. mlrun/datastore/utils.py +4 -26
  32. mlrun/db/base.py +2 -4
  33. mlrun/db/filedb.py +5 -13
  34. mlrun/db/httpdb.py +32 -64
  35. mlrun/db/sqldb.py +2 -4
  36. mlrun/errors.py +0 -5
  37. mlrun/execution.py +0 -2
  38. mlrun/feature_store/api.py +8 -24
  39. mlrun/feature_store/feature_set.py +6 -28
  40. mlrun/feature_store/feature_vector.py +0 -2
  41. mlrun/feature_store/ingestion.py +11 -8
  42. mlrun/feature_store/retrieval/base.py +43 -271
  43. mlrun/feature_store/retrieval/dask_merger.py +153 -55
  44. mlrun/feature_store/retrieval/job.py +3 -12
  45. mlrun/feature_store/retrieval/local_merger.py +130 -48
  46. mlrun/feature_store/retrieval/spark_merger.py +125 -126
  47. mlrun/features.py +2 -7
  48. mlrun/model_monitoring/constants.py +6 -48
  49. mlrun/model_monitoring/helpers.py +35 -118
  50. mlrun/model_monitoring/model_monitoring_batch.py +260 -293
  51. mlrun/model_monitoring/stream_processing_fs.py +253 -220
  52. mlrun/platforms/iguazio.py +0 -33
  53. mlrun/projects/project.py +72 -34
  54. mlrun/runtimes/base.py +0 -5
  55. mlrun/runtimes/daskjob.py +0 -2
  56. mlrun/runtimes/function.py +3 -29
  57. mlrun/runtimes/kubejob.py +15 -39
  58. mlrun/runtimes/local.py +45 -7
  59. mlrun/runtimes/mpijob/abstract.py +0 -2
  60. mlrun/runtimes/mpijob/v1.py +0 -2
  61. mlrun/runtimes/pod.py +0 -2
  62. mlrun/runtimes/remotesparkjob.py +0 -2
  63. mlrun/runtimes/serving.py +0 -6
  64. mlrun/runtimes/sparkjob/abstract.py +2 -39
  65. mlrun/runtimes/sparkjob/spark3job.py +0 -2
  66. mlrun/serving/__init__.py +1 -2
  67. mlrun/serving/routers.py +35 -35
  68. mlrun/serving/server.py +12 -22
  69. mlrun/serving/states.py +30 -162
  70. mlrun/serving/v2_serving.py +10 -13
  71. mlrun/utils/clones.py +1 -1
  72. mlrun/utils/model_monitoring.py +96 -122
  73. mlrun/utils/version/version.json +2 -2
  74. {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/METADATA +27 -23
  75. {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/RECORD +79 -92
  76. mlrun/api/crud/model_monitoring/grafana.py +0 -427
  77. mlrun/datastore/spark_udf.py +0 -40
  78. mlrun/model_monitoring/__init__.py +0 -44
  79. mlrun/model_monitoring/common.py +0 -112
  80. mlrun/model_monitoring/model_endpoint.py +0 -141
  81. mlrun/model_monitoring/stores/__init__.py +0 -106
  82. mlrun/model_monitoring/stores/kv_model_endpoint_store.py +0 -448
  83. mlrun/model_monitoring/stores/model_endpoint_store.py +0 -147
  84. mlrun/model_monitoring/stores/models/__init__.py +0 -23
  85. mlrun/model_monitoring/stores/models/base.py +0 -18
  86. mlrun/model_monitoring/stores/models/mysql.py +0 -100
  87. mlrun/model_monitoring/stores/models/sqlite.py +0 -98
  88. mlrun/model_monitoring/stores/sql_model_endpoint_store.py +0 -375
  89. mlrun/utils/db.py +0 -52
  90. {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/LICENSE +0 -0
  91. {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/WHEEL +0 -0
  92. {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/entry_points.txt +0 -0
  93. {mlrun-1.3.2rc1.dist-info → mlrun-1.3.2rc2.dist-info}/top_level.txt +0 -0
@@ -12,10 +12,10 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  #
15
- import json
15
+
16
+
16
17
  import os
17
18
  import typing
18
- import warnings
19
19
 
20
20
  import sqlalchemy.orm
21
21
 
@@ -35,9 +35,10 @@ import mlrun.runtimes.function
35
35
  import mlrun.utils.helpers
36
36
  import mlrun.utils.model_monitoring
37
37
  import mlrun.utils.v3io_clients
38
- from mlrun.model_monitoring.stores import get_model_endpoint_store
39
38
  from mlrun.utils import logger
40
39
 
40
+ from .model_endpoint_store import get_model_endpoint_target
41
+
41
42
 
42
43
  class ModelEndpoints:
43
44
  """Provide different methods for handling model endpoints such as listing, writing and deleting"""
@@ -49,14 +50,9 @@ class ModelEndpoints:
49
50
  model_endpoint: mlrun.api.schemas.ModelEndpoint,
50
51
  auth_info: mlrun.api.schemas.AuthInfo = mlrun.api.schemas.AuthInfo(),
51
52
  ) -> mlrun.api.schemas.ModelEndpoint:
52
- # TODO: deprecated in 1.3.0, remove in 1.5.0.
53
- warnings.warn(
54
- "This is deprecated in 1.3.0, and will be removed in 1.5.0."
55
- "Please use create_model_endpoint() for create or patch_model_endpoint() for update",
56
- FutureWarning,
57
- )
53
+ # TODO: deprecated, remove in 1.5.0.
58
54
  """
59
- Either create or updates the record of a given `ModelEndpoint` object.
55
+ Either create or updates the record of a given ModelEndpoint object.
60
56
  Leaving here for backwards compatibility, remove in 1.5.0.
61
57
 
62
58
  :param db_session: A session that manages the current dialog with the database
@@ -64,7 +60,7 @@ class ModelEndpoints:
64
60
  :param model_endpoint: Model endpoint object to update
65
61
  :param auth_info: The auth info of the request
66
62
 
67
- :return: `ModelEndpoint` object.
63
+ :return: Model endpoint object.
68
64
  """
69
65
 
70
66
  return self.create_model_endpoint(
@@ -78,12 +74,12 @@ class ModelEndpoints:
78
74
  ) -> mlrun.api.schemas.ModelEndpoint:
79
75
  """
80
76
  Creates model endpoint record in DB. The DB target type is defined under
81
- `mlrun.config.model_endpoint_monitoring.store_type` (V3IO-NOSQL by default).
77
+ mlrun.config.model_endpoint_monitoring.store_type (KV by default).
82
78
 
83
79
  :param db_session: A session that manages the current dialog with the database.
84
80
  :param model_endpoint: Model endpoint object to update.
85
81
 
86
- :return: `ModelEndpoint` object.
82
+ :return: Model endpoint object.
87
83
  """
88
84
 
89
85
  if model_endpoint.spec.model_uri or model_endpoint.status.feature_stats:
@@ -111,22 +107,23 @@ class ModelEndpoints:
111
107
  if not model_endpoint.status.feature_stats and hasattr(
112
108
  model_obj, "feature_stats"
113
109
  ):
114
- model_endpoint.status.feature_stats = model_obj.spec.feature_stats
110
+ model_endpoint.status.feature_stats = model_obj.feature_stats
111
+
115
112
  # Get labels from model object if not found in model endpoint object
116
- if not model_endpoint.spec.label_names and model_obj.spec.outputs:
113
+ if not model_endpoint.spec.label_names and hasattr(model_obj, "outputs"):
117
114
  model_label_names = [
118
- self._clean_feature_name(f.name) for f in model_obj.spec.outputs
115
+ self._clean_feature_name(f.name) for f in model_obj.outputs
119
116
  ]
120
117
  model_endpoint.spec.label_names = model_label_names
121
118
 
122
119
  # Get algorithm from model object if not found in model endpoint object
123
- if not model_endpoint.spec.algorithm and model_obj.spec.algorithm:
124
- model_endpoint.spec.algorithm = model_obj.spec.algorithm
120
+ if not model_endpoint.spec.algorithm and hasattr(model_obj, "algorithm"):
121
+ model_endpoint.spec.algorithm = model_obj.algorithm
125
122
 
126
123
  # Create monitoring feature set if monitoring found in model endpoint object
127
124
  if (
128
125
  model_endpoint.spec.monitoring_mode
129
- == mlrun.model_monitoring.ModelMonitoringMode.enabled.value
126
+ == mlrun.api.schemas.ModelMonitoringMode.enabled.value
130
127
  ):
131
128
  monitoring_feature_set = self.create_monitoring_feature_set(
132
129
  model_endpoint, model_obj, db_session, run_db
@@ -161,17 +158,17 @@ class ModelEndpoints:
161
158
  logger.info("Creating model endpoint", endpoint_id=model_endpoint.metadata.uid)
162
159
 
163
160
  # Write the new model endpoint
164
- model_endpoint_store = get_model_endpoint_store(
161
+ model_endpoint_target = get_model_endpoint_target(
165
162
  project=model_endpoint.metadata.project,
166
163
  )
167
- model_endpoint_store.write_model_endpoint(endpoint=model_endpoint.flat_dict())
164
+ model_endpoint_target.write_model_endpoint(endpoint=model_endpoint)
168
165
 
169
166
  logger.info("Model endpoint created", endpoint_id=model_endpoint.metadata.uid)
170
167
 
171
168
  return model_endpoint
172
169
 
170
+ @staticmethod
173
171
  def create_monitoring_feature_set(
174
- self,
175
172
  model_endpoint: mlrun.api.schemas.ModelEndpoint,
176
173
  model_obj: mlrun.artifacts.ModelArtifact,
177
174
  db_session: sqlalchemy.orm.Session,
@@ -198,29 +195,29 @@ class ModelEndpoints:
198
195
 
199
196
  feature_set = mlrun.feature_store.FeatureSet(
200
197
  f"monitoring-{serving_function_name}-{model_name}",
201
- entities=[model_monitoring_constants.EventFieldType.ENDPOINT_ID],
202
- timestamp_key=model_monitoring_constants.EventFieldType.TIMESTAMP,
198
+ entities=["endpoint_id"],
199
+ timestamp_key="timestamp",
203
200
  description=f"Monitoring feature set for endpoint: {model_endpoint.spec.model}",
204
201
  )
205
202
  feature_set.metadata.project = model_endpoint.metadata.project
206
203
 
207
204
  feature_set.metadata.labels = {
208
- model_monitoring_constants.EventFieldType.ENDPOINT_ID: model_endpoint.metadata.uid,
209
- model_monitoring_constants.EventFieldType.MODEL_CLASS: model_endpoint.spec.model_class,
205
+ "endpoint_id": model_endpoint.metadata.uid,
206
+ "model_class": model_endpoint.spec.model_class,
210
207
  }
211
208
 
212
209
  # Add features to the feature set according to the model object
213
- if model_obj.spec.inputs:
214
- for feature in model_obj.spec.inputs:
210
+ if model_obj.inputs.values():
211
+ for feature in model_obj.inputs.values():
215
212
  feature_set.add_feature(
216
213
  mlrun.feature_store.Feature(
217
214
  name=feature.name, value_type=feature.value_type
218
215
  )
219
216
  )
220
217
  # Check if features can be found within the feature vector
221
- elif model_obj.spec.feature_vector:
218
+ elif model_obj.feature_vector:
222
219
  _, name, _, tag, _ = mlrun.utils.helpers.parse_artifact_uri(
223
- model_obj.spec.feature_vector
220
+ model_obj.feature_vector
224
221
  )
225
222
  fv = run_db.get_feature_vector(
226
223
  name=name, project=model_endpoint.metadata.project, tag=tag
@@ -239,22 +236,16 @@ class ModelEndpoints:
239
236
 
240
237
  # Define parquet target for this feature set
241
238
  parquet_path = (
242
- self._get_monitoring_parquet_path(
243
- db_session=db_session, project=model_endpoint.metadata.project
244
- )
245
- + f"/key={model_endpoint.metadata.uid}"
246
- )
247
-
248
- parquet_target = mlrun.datastore.targets.ParquetTarget(
249
- model_monitoring_constants.FileTargetKind.PARQUET, parquet_path
239
+ f"v3io:///projects/{model_endpoint.metadata.project}"
240
+ f"/model-endpoints/parquet/key={model_endpoint.metadata.uid}"
250
241
  )
242
+ parquet_target = mlrun.datastore.targets.ParquetTarget("parquet", parquet_path)
251
243
  driver = mlrun.datastore.targets.get_target_driver(parquet_target, feature_set)
252
-
244
+ driver.update_resource_status("created")
253
245
  feature_set.set_targets(
254
246
  [mlrun.datastore.targets.ParquetTarget(path=parquet_path)],
255
247
  with_defaults=False,
256
248
  )
257
- driver.update_resource_status("created")
258
249
 
259
250
  # Save the new feature set
260
251
  feature_set._override_run_db(db_session)
@@ -267,39 +258,10 @@ class ModelEndpoints:
267
258
 
268
259
  return feature_set
269
260
 
270
- @staticmethod
271
- def _get_monitoring_parquet_path(
272
- db_session: sqlalchemy.orm.Session, project: str
273
- ) -> str:
274
- """Getting model monitoring parquet target for the current project. The parquet target path is based on the
275
- project artifact path. If project artifact path is not defined, the parquet target path will be based on MLRun
276
- artifact path.
277
-
278
- :param db_session: A session that manages the current dialog with the database. Will be used in this function
279
- to get the project record from DB.
280
- :param project: Project name.
281
-
282
- :return: Monitoring parquet target path.
283
- """
284
-
285
- # Get the artifact path from the project record that was stored in the DB
286
- project_obj = mlrun.api.crud.projects.Projects().get_project(
287
- session=db_session, name=project
288
- )
289
- artifact_path = project_obj.spec.artifact_path
290
- # Generate monitoring parquet path value
291
- parquet_path = mlrun.mlconf.get_model_monitoring_file_target_path(
292
- project=project,
293
- kind=model_monitoring_constants.FileTargetKind.PARQUET,
294
- target="offline",
295
- artifact_path=artifact_path,
296
- )
297
- return parquet_path
298
-
299
261
  @staticmethod
300
262
  def _validate_length_features_and_labels(model_endpoint):
301
263
  """
302
- Validate that the length of feature_stats is equal to the length of `feature_names` and `label_names`
264
+ Validate that the length of feature_stats is equal to the length of feature_names and label_names
303
265
 
304
266
  :param model_endpoint: An object representing the model endpoint.
305
267
  """
@@ -326,8 +288,8 @@ class ModelEndpoints:
326
288
  self, model_endpoint
327
289
  ) -> typing.Tuple[typing.Dict, typing.List]:
328
290
  """
329
- Create a clean matching version of feature names for both `feature_stats` and `feature_names`. Please note that
330
- label names exist only in `feature_stats` and `label_names`.
291
+ Create a clean matching version of feature names for both feature_stats and feature_names. Please note that
292
+ label names exist only in feature_stats and label_names.
331
293
 
332
294
  :param model_endpoint: An object representing the model endpoint.
333
295
  :return: A tuple of:
@@ -350,8 +312,8 @@ class ModelEndpoints:
350
312
  clean_feature_names.append(clean_name)
351
313
  return clean_feature_stats, clean_feature_names
352
314
 
315
+ @staticmethod
353
316
  def patch_model_endpoint(
354
- self,
355
317
  project: str,
356
318
  endpoint_id: str,
357
319
  attributes: dict,
@@ -362,30 +324,24 @@ class ModelEndpoints:
362
324
  :param project: The name of the project.
363
325
  :param endpoint_id: The unique id of the model endpoint.
364
326
  :param attributes: Dictionary of attributes that will be used for update the model endpoint. Note that the keys
365
- of the attributes dictionary should exist in the DB table. More details about the model
327
+ of the attributes dictionary should exist in the KV table. More details about the model
366
328
  endpoint available attributes can be found under
367
329
  :py:class:`~mlrun.api.schemas.ModelEndpoint`.
368
330
 
369
- :return: A patched `ModelEndpoint` object.
331
+ :return: A patched ModelEndpoint object.
370
332
  """
371
333
 
372
- # Generate a model endpoint store object and apply the update process
373
- model_endpoint_store = get_model_endpoint_store(
334
+ model_endpoint_target = get_model_endpoint_target(
374
335
  project=project,
375
336
  )
376
- model_endpoint_store.update_model_endpoint(
337
+ model_endpoint_target.update_model_endpoint(
377
338
  endpoint_id=endpoint_id, attributes=attributes
378
339
  )
379
340
 
380
- logger.info("Model endpoint table updated", endpoint_id=endpoint_id)
381
-
382
- # Get the patched model endpoint record
383
- model_endpoint_record = model_endpoint_store.get_model_endpoint(
384
- endpoint_id=endpoint_id,
341
+ return model_endpoint_target.get_model_endpoint(
342
+ endpoint_id=endpoint_id, start="now-1h", end="now"
385
343
  )
386
344
 
387
- return self._convert_into_model_endpoint_object(endpoint=model_endpoint_record)
388
-
389
345
  @staticmethod
390
346
  def delete_model_endpoint(
391
347
  project: str,
@@ -397,16 +353,13 @@ class ModelEndpoints:
397
353
  :param project: The name of the project.
398
354
  :param endpoint_id: The id of the endpoint.
399
355
  """
400
- model_endpoint_store = get_model_endpoint_store(
356
+ model_endpoint_target = get_model_endpoint_target(
401
357
  project=project,
402
358
  )
359
+ model_endpoint_target.delete_model_endpoint(endpoint_id=endpoint_id)
403
360
 
404
- model_endpoint_store.delete_model_endpoint(endpoint_id=endpoint_id)
405
-
406
- logger.info("Model endpoint table cleared", endpoint_id=endpoint_id)
407
-
361
+ @staticmethod
408
362
  def get_model_endpoint(
409
- self,
410
363
  auth_info: mlrun.api.schemas.AuthInfo,
411
364
  project: str,
412
365
  endpoint_id: str,
@@ -418,61 +371,40 @@ class ModelEndpoints:
418
371
  """Get a single model endpoint object. You can apply different time series metrics that will be added to the
419
372
  result.
420
373
 
421
- :param auth_info: The auth info of the request
422
- :param project: The name of the project
423
- :param endpoint_id: The unique id of the model endpoint.
424
- :param metrics: A list of metrics to return for the model endpoint. There are pre-defined
425
- metrics for model endpoints such as predictions_per_second and
426
- latency_avg_5m but also custom metrics defined by the user. Please note that
427
- these metrics are stored in the time series DB and the results will be
428
- appeared under `model_endpoint.spec.metrics`.
429
- :param start: The start time of the metrics. Can be represented by a string containing an
430
- RFC 3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
431
- `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or
432
- 0 for the earliest time.
433
- :param end: The end time of the metrics. Can be represented by a string containing an
434
- RFC 3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
435
- `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` = days), or
436
- 0 for the earliest time.
437
- :param feature_analysis: When True, the base feature statistics and current feature statistics will
438
- be added to the output of the resulting object.
439
-
440
- :return: A `ModelEndpoint` object.
441
- """
374
+ :param auth_info: The auth info of the request
375
+ :param project: The name of the project
376
+ :param endpoint_id: The unique id of the model endpoint.
377
+ :param metrics: A list of metrics to return for the model endpoint. There are pre-defined metrics for
378
+ model endpoints such as predictions_per_second and latency_avg_5m but also custom
379
+ metrics defined by the user. Please note that these metrics are stored in the time
380
+ series DB and the results will be appeared under model_endpoint.spec.metrics.
381
+ :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339
382
+ time, a Unix timestamp in milliseconds, a relative time (`'now'` or
383
+ `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` =
384
+ days), or 0 for the earliest time.
385
+ :param end: The end time of the metrics. Can be represented by a string containing an RFC 3339
386
+ time, a Unix timestamp in milliseconds, a relative time (`'now'` or
387
+ `'now-[0-9]+[mhd]'`, where `m` = minutes, `h` = hours, and `'d'` =
388
+ days), or 0 for the earliest time.
389
+ :param feature_analysis: When True, the base feature statistics and current feature statistics will be added to
390
+ the output of the resulting object.
442
391
 
443
- logger.info(
444
- "Getting model endpoint record from DB",
445
- endpoint_id=endpoint_id,
446
- )
392
+ :return: A ModelEndpoint object.
393
+ """
447
394
 
448
- # Generate a model endpoint store object and get the model endpoint record as a dictionary
449
- model_endpoint_store = get_model_endpoint_store(
395
+ model_endpoint_target = get_model_endpoint_target(
450
396
  project=project, access_key=auth_info.data_session
451
397
  )
452
-
453
- model_endpoint_record = model_endpoint_store.get_model_endpoint(
398
+ return model_endpoint_target.get_model_endpoint(
454
399
  endpoint_id=endpoint_id,
400
+ metrics=metrics,
401
+ start=start,
402
+ end=end,
403
+ feature_analysis=feature_analysis,
455
404
  )
456
405
 
457
- # Convert to `ModelEndpoint` object
458
- model_endpoint_object = self._convert_into_model_endpoint_object(
459
- endpoint=model_endpoint_record, feature_analysis=feature_analysis
460
- )
461
-
462
- # If time metrics were provided, retrieve the results from the time series DB
463
- if metrics:
464
- self._add_real_time_metrics(
465
- model_endpoint_store=model_endpoint_store,
466
- model_endpoint_object=model_endpoint_object,
467
- metrics=metrics,
468
- start=start,
469
- end=end,
470
- )
471
-
472
- return model_endpoint_object
473
-
406
+ @staticmethod
474
407
  def list_model_endpoints(
475
- self,
476
408
  auth_info: mlrun.api.schemas.AuthInfo,
477
409
  project: str,
478
410
  model: str = None,
@@ -483,11 +415,10 @@ class ModelEndpoints:
483
415
  end: str = "now",
484
416
  top_level: bool = False,
485
417
  uids: typing.List[str] = None,
486
- ) -> mlrun.api.schemas.ModelEndpointList:
418
+ ) -> mlrun.api.schemas.model_endpoints.ModelEndpointList:
487
419
  """
488
- Returns a list of `ModelEndpoint` objects, wrapped in `ModelEndpointList` object. Each `ModelEndpoint`
489
- object represents the current state of a model endpoint. This functions supports filtering by the following
490
- parameters:
420
+ Returns a list of ModelEndpointState objects. Each object represents the current state of a model endpoint.
421
+ This functions supports filtering by the following parameters:
491
422
  1) model
492
423
  2) function
493
424
  3) labels
@@ -504,22 +435,22 @@ class ModelEndpoints:
504
435
  :param model: The name of the model to filter by.
505
436
  :param function: The name of the function to filter by.
506
437
  :param labels: A list of labels to filter by. Label filters work by either filtering a specific value of a
507
- label (i.e. list("key=value")) or by looking for the existence of a given key (i.e. "key").
438
+ label (i.e. list("key==value")) or by looking for the existence of a given key (i.e. "key").
508
439
  :param metrics: A list of metrics to return for each endpoint. There are pre-defined metrics for model
509
- endpoints such as `predictions_per_second` and `latency_avg_5m` but also custom metrics
510
- defined by the user. Please note that these metrics are stored in the time series DB and the
511
- results will be appeared under model_endpoint.spec.metrics of each endpoint.
440
+ endpoints such as predictions_per_second and latency_avg_5m but also custom metrics defined
441
+ by the user. Please note that these metrics are stored in the time series DB and the results
442
+ will be appeared under model_endpoint.spec.metrics of each endpoint.
512
443
  :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339 time,
513
444
  a Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where `m`
514
445
  = minutes, `h` = hours, and `'d'` = days), or 0 for the earliest time.
515
446
  :param end: The end time of the metrics. Can be represented by a string containing an RFC 3339 time,
516
447
  a Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where `m`
517
448
  = minutes, `h` = hours, and `'d'` = days), or 0 for the earliest time.
518
- :param top_level: If True, return only routers and endpoints that are NOT children of any router.
519
- :param uids: List of model endpoint unique ids to include in the result.
449
+ :param top_level: If True will return only routers and endpoint that are NOT children of any router.
450
+ :param uids: Will return ModelEndpointList of endpoints with uid in uids.
520
451
 
521
- :return: An object of `ModelEndpointList` which is literally a list of model endpoints along with some metadata.
522
- To get a standard list of model endpoints use `ModelEndpointList.endpoints`.
452
+ :return: An object of ModelEndpointList which is literally a list of model endpoints along with some metadata.
453
+ To get a standard list of model endpoints use ModelEndpointList.endpoints.
523
454
  """
524
455
 
525
456
  logger.info(
@@ -535,174 +466,32 @@ class ModelEndpoints:
535
466
  uids=uids,
536
467
  )
537
468
 
538
- # Initialize an empty model endpoints list
539
- endpoint_list = mlrun.api.schemas.model_endpoints.ModelEndpointList(
540
- endpoints=[]
541
- )
542
-
543
- # Generate a model endpoint store object and get a list of model endpoint dictionaries
544
- endpoint_store = get_model_endpoint_store(
469
+ endpoint_target = get_model_endpoint_target(
545
470
  access_key=auth_info.data_session, project=project
546
471
  )
547
472
 
548
- endpoint_dictionary_list = endpoint_store.list_model_endpoints(
549
- function=function,
550
- model=model,
551
- labels=labels,
552
- top_level=top_level,
553
- uids=uids,
554
- )
555
-
556
- for endpoint_dict in endpoint_dictionary_list:
557
-
558
- # Convert to `ModelEndpoint` object
559
- endpoint_obj = self._convert_into_model_endpoint_object(
560
- endpoint=endpoint_dict
561
- )
562
-
563
- # If time metrics were provided, retrieve the results from the time series DB
564
- if metrics:
565
- self._add_real_time_metrics(
566
- model_endpoint_store=endpoint_store,
567
- model_endpoint_object=endpoint_obj,
568
- metrics=metrics,
569
- start=start,
570
- end=end,
571
- )
572
-
573
- # Add the `ModelEndpoint` object into the model endpoints list
574
- endpoint_list.endpoints.append(endpoint_obj)
575
-
576
- return endpoint_list
577
-
578
- @staticmethod
579
- def _add_real_time_metrics(
580
- model_endpoint_store: mlrun.model_monitoring.stores.ModelEndpointStore,
581
- model_endpoint_object: mlrun.api.schemas.ModelEndpoint,
582
- metrics: typing.List[str] = None,
583
- start: str = "now-1h",
584
- end: str = "now",
585
- ) -> mlrun.api.schemas.ModelEndpoint:
586
- """Add real time metrics from the time series DB to a provided `ModelEndpoint` object. The real time metrics
587
- will be stored under `ModelEndpoint.status.metrics.real_time`
588
-
589
- :param model_endpoint_store: `ModelEndpointStore` object that will be used for communicating with the database
590
- and querying the required metrics.
591
- :param model_endpoint_object: `ModelEndpoint` object that will be filled with the relevant
592
- real time metrics.
593
- :param metrics: A list of metrics to return for each endpoint. There are pre-defined metrics for
594
- model endpoints such as `predictions_per_second` and `latency_avg_5m` but also
595
- custom metrics defined by the user. Please note that these metrics are stored in
596
- the time series DB and the results will be appeared under
597
- model_endpoint.spec.metrics of each endpoint.
598
- :param start: The start time of the metrics. Can be represented by a string containing an RFC
599
- 3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
600
- `'now-[0-9]+[mhd]'`, where `m`= minutes, `h` = hours, and `'d'` = days), or 0
601
- for the earliest time.
602
- :param end: The end time of the metrics. Can be represented by a string containing an RFC
603
- 3339 time, a Unix timestamp in milliseconds, a relative time (`'now'` or
604
- `'now-[0-9]+[mhd]'`, where `m`= minutes, `h` = hours, and `'d'` = days), or 0
605
- for the earliest time.
606
-
607
- """
608
- if model_endpoint_object.status.metrics is None:
609
- model_endpoint_object.status.metrics = {}
610
-
611
- endpoint_metrics = model_endpoint_store.get_endpoint_real_time_metrics(
612
- endpoint_id=model_endpoint_object.metadata.uid,
613
- start=start,
614
- end=end,
615
- metrics=metrics,
473
+ # Initialize an empty model endpoints list
474
+ endpoint_list = mlrun.api.schemas.model_endpoints.ModelEndpointList(
475
+ endpoints=[]
616
476
  )
617
- if endpoint_metrics:
618
- model_endpoint_object.status.metrics[
619
- model_monitoring_constants.EventKeyMetrics.REAL_TIME
620
- ] = endpoint_metrics
621
- return model_endpoint_object
622
-
623
- def _convert_into_model_endpoint_object(
624
- self, endpoint: typing.Dict[str, typing.Any], feature_analysis: bool = False
625
- ) -> mlrun.api.schemas.ModelEndpoint:
626
- """
627
- Create a `ModelEndpoint` object according to a provided model endpoint dictionary.
628
-
629
- :param endpoint: Dictinoary that represents a DB record of a model endpoint which need to be converted
630
- into a valid `ModelEndpoint` object.
631
- :param feature_analysis: When True, the base feature statistics and current feature statistics will be added to
632
- the output of the resulting object.
633
477
 
634
- :return: A `ModelEndpoint` object.
635
- """
636
-
637
- # Convert into `ModelEndpoint` object
638
- endpoint_obj = mlrun.api.schemas.ModelEndpoint().from_flat_dict(endpoint)
639
-
640
- # If feature analysis was applied, add feature stats and current stats to the model endpoint result
641
- if feature_analysis and endpoint_obj.spec.feature_names:
642
-
643
- endpoint_features = self.get_endpoint_features(
644
- feature_names=endpoint_obj.spec.feature_names,
645
- feature_stats=endpoint_obj.status.feature_stats,
646
- current_stats=endpoint_obj.status.current_stats,
478
+ # If list of model endpoint ids was not provided, retrieve it from the DB
479
+ if uids is None:
480
+ uids = endpoint_target.list_model_endpoints(
481
+ function=function, model=model, labels=labels, top_level=top_level
647
482
  )
648
- if endpoint_features:
649
- endpoint_obj.status.features = endpoint_features
650
- # Add the latest drift measures results (calculated by the model monitoring batch)
651
- drift_measures = self._json_loads_if_not_none(
652
- endpoint.get(
653
- model_monitoring_constants.EventFieldType.DRIFT_MEASURES
654
- )
655
- )
656
- endpoint_obj.status.drift_measures = drift_measures
657
483
 
658
- return endpoint_obj
659
-
660
- @staticmethod
661
- def get_endpoint_features(
662
- feature_names: typing.List[str],
663
- feature_stats: dict = None,
664
- current_stats: dict = None,
665
- ) -> typing.List[mlrun.api.schemas.Features]:
666
- """
667
- Getting a new list of features that exist in feature_names along with their expected (feature_stats) and
668
- actual (current_stats) stats. The expected stats were calculated during the creation of the model endpoint,
669
- usually based on the data from the Model Artifact. The actual stats are based on the results from the latest
670
- model monitoring batch job.
671
-
672
- param feature_names: List of feature names.
673
- param feature_stats: Dictionary of feature stats that were stored during the creation of the model endpoint
674
- object.
675
- param current_stats: Dictionary of the latest stats that were stored during the last run of the model monitoring
676
- batch job.
677
-
678
- return: List of feature objects. Each feature has a name, weight, expected values, and actual values. More info
679
- can be found under `mlrun.api.schemas.Features`.
680
- """
681
-
682
- # Initialize feature and current stats dictionaries
683
- safe_feature_stats = feature_stats or {}
684
- safe_current_stats = current_stats or {}
685
-
686
- # Create feature object and add it to a general features list
687
- features = []
688
- for name in feature_names:
689
- if feature_stats is not None and name not in feature_stats:
690
- logger.warn("Feature missing from 'feature_stats'", name=name)
691
- if current_stats is not None and name not in current_stats:
692
- logger.warn("Feature missing from 'current_stats'", name=name)
693
- f = mlrun.api.schemas.Features.new(
694
- name, safe_feature_stats.get(name), safe_current_stats.get(name)
484
+ # Add each relevant model endpoint to the model endpoints list
485
+ for endpoint_id in uids:
486
+ endpoint = endpoint_target.get_model_endpoint(
487
+ metrics=metrics,
488
+ endpoint_id=endpoint_id,
489
+ start=start,
490
+ end=end,
695
491
  )
696
- features.append(f)
697
- return features
492
+ endpoint_list.endpoints.append(endpoint)
698
493
 
699
- @staticmethod
700
- def _json_loads_if_not_none(field: typing.Any) -> typing.Any:
701
- return (
702
- json.loads(field)
703
- if field and field != "null" and field is not None
704
- else None
705
- )
494
+ return endpoint_list
706
495
 
707
496
  def deploy_monitoring_functions(
708
497
  self,
@@ -750,8 +539,7 @@ class ModelEndpoints:
750
539
  f"Project {project_name} can not be deleted since related resources found: model endpoints"
751
540
  )
752
541
 
753
- @staticmethod
754
- def delete_model_endpoints_resources(project_name: str):
542
+ def delete_model_endpoints_resources(self, project_name: str):
755
543
  """
756
544
  Delete all model endpoints resources.
757
545
 
@@ -766,17 +554,15 @@ class ModelEndpoints:
766
554
  if not mlrun.mlconf.igz_version or not mlrun.mlconf.v3io_api:
767
555
  return
768
556
 
769
- # Generate a model endpoint store object and get a list of model endpoint dictionaries
770
- endpoint_store = get_model_endpoint_store(
557
+ endpoints = self.list_model_endpoints(auth_info, project_name)
558
+
559
+ endpoint_target = get_model_endpoint_target(
771
560
  access_key=auth_info.data_session, project=project_name
772
561
  )
773
- endpoints = endpoint_store.list_model_endpoints()
774
-
775
- # Delete model endpoints resources from databases using the model endpoint store object
776
- endpoint_store.delete_model_endpoints_resources(endpoints)
562
+ endpoint_target.delete_model_endpoints_resources(endpoints)
777
563
 
564
+ @staticmethod
778
565
  def deploy_model_monitoring_stream_processing(
779
- self,
780
566
  project: str,
781
567
  model_monitoring_access_key: str,
782
568
  db_session: sqlalchemy.orm.Session,
@@ -817,17 +603,8 @@ class ModelEndpoints:
817
603
  "Deploying model monitoring stream processing function", project=project
818
604
  )
819
605
 
820
- # Get parquet target value for model monitoring stream function
821
- parquet_target = self._get_monitoring_parquet_path(
822
- db_session=db_session, project=project
823
- )
824
-
825
606
  fn = mlrun.model_monitoring.helpers.initial_model_monitoring_stream_processing_function(
826
- project=project,
827
- model_monitoring_access_key=model_monitoring_access_key,
828
- tracking_policy=tracking_policy,
829
- auth_info=auth_info,
830
- parquet_target=parquet_target,
607
+ project, model_monitoring_access_key, db_session, tracking_policy
831
608
  )
832
609
 
833
610
  mlrun.api.api.endpoints.functions._build_function(