mlrun 1.10.0rc1__py3-none-any.whl → 1.10.0rc3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (59) hide show
  1. mlrun/__init__.py +2 -2
  2. mlrun/__main__.py +15 -4
  3. mlrun/artifacts/base.py +6 -6
  4. mlrun/artifacts/dataset.py +1 -1
  5. mlrun/artifacts/document.py +1 -1
  6. mlrun/artifacts/model.py +1 -1
  7. mlrun/artifacts/plots.py +2 -2
  8. mlrun/common/constants.py +7 -0
  9. mlrun/common/runtimes/constants.py +1 -1
  10. mlrun/common/schemas/__init__.py +1 -0
  11. mlrun/common/schemas/artifact.py +1 -1
  12. mlrun/common/schemas/pipeline.py +1 -1
  13. mlrun/common/schemas/project.py +1 -1
  14. mlrun/common/schemas/runs.py +1 -1
  15. mlrun/common/schemas/serving.py +17 -0
  16. mlrun/config.py +4 -4
  17. mlrun/datastore/datastore_profile.py +7 -57
  18. mlrun/datastore/sources.py +24 -16
  19. mlrun/datastore/store_resources.py +3 -3
  20. mlrun/datastore/targets.py +5 -5
  21. mlrun/datastore/utils.py +21 -6
  22. mlrun/db/base.py +7 -7
  23. mlrun/db/httpdb.py +88 -76
  24. mlrun/db/nopdb.py +1 -1
  25. mlrun/errors.py +29 -1
  26. mlrun/execution.py +9 -0
  27. mlrun/feature_store/common.py +5 -5
  28. mlrun/feature_store/feature_set.py +10 -6
  29. mlrun/feature_store/feature_vector.py +8 -6
  30. mlrun/launcher/base.py +1 -1
  31. mlrun/launcher/client.py +1 -1
  32. mlrun/lists.py +1 -1
  33. mlrun/model_monitoring/__init__.py +0 -1
  34. mlrun/model_monitoring/api.py +0 -44
  35. mlrun/model_monitoring/applications/evidently/base.py +57 -107
  36. mlrun/model_monitoring/controller.py +27 -14
  37. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +13 -5
  38. mlrun/model_monitoring/writer.py +1 -4
  39. mlrun/projects/operations.py +3 -3
  40. mlrun/projects/project.py +114 -52
  41. mlrun/render.py +5 -9
  42. mlrun/run.py +10 -10
  43. mlrun/runtimes/base.py +7 -7
  44. mlrun/runtimes/kubejob.py +2 -2
  45. mlrun/runtimes/nuclio/function.py +3 -3
  46. mlrun/runtimes/nuclio/serving.py +13 -23
  47. mlrun/runtimes/utils.py +25 -8
  48. mlrun/serving/__init__.py +5 -1
  49. mlrun/serving/server.py +39 -3
  50. mlrun/serving/states.py +176 -10
  51. mlrun/utils/helpers.py +10 -4
  52. mlrun/utils/version/version.json +2 -2
  53. {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/METADATA +27 -15
  54. {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/RECORD +58 -59
  55. {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/WHEEL +1 -1
  56. mlrun/model_monitoring/tracking_policy.py +0 -124
  57. {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/entry_points.txt +0 -0
  58. {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/licenses/LICENSE +0 -0
  59. {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/top_level.txt +0 -0
mlrun/errors.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
-
14
+ import copy
15
15
  import typing
16
16
  from http import HTTPStatus
17
17
 
@@ -230,6 +230,13 @@ class MLRunTSDBConnectionFailureError(MLRunHTTPStatusError, ValueError):
230
230
  error_status_code = HTTPStatus.BAD_REQUEST.value
231
231
 
232
232
 
233
+ class MLRunMissingProjectError(MLRunBadRequestError):
234
+ default_message = "Project must be provided"
235
+
236
+ def __init__(self, message=None):
237
+ super().__init__(message or self.default_message)
238
+
239
+
233
240
  class MLRunRetryExhaustedError(Exception):
234
241
  pass
235
242
 
@@ -256,6 +263,27 @@ class MLRunFatalFailureError(Exception):
256
263
  self.original_exception = original_exception
257
264
 
258
265
 
266
+ class ModelRunnerError(MLRunBaseError):
267
+ def __init__(self, models_errors: dict[str:str], *args) -> None:
268
+ self.models_errors = models_errors
269
+ super().__init__(self.__repr__(), *args)
270
+
271
+ def __repr__(self):
272
+ return f"ModelRunnerError: {repr(self.models_errors)}"
273
+
274
+ def __copy__(self):
275
+ return type(self)(models_errors=self.models_errors)
276
+
277
+ def __deepcopy__(self, memo):
278
+ return type(self)(copy.deepcopy(self.models_errors, memo))
279
+
280
+ def get_errors(self):
281
+ return self.models_errors
282
+
283
+ def get_model_error(self, model: str):
284
+ return self.models_errors.get(model)
285
+
286
+
259
287
  STATUS_ERRORS = {
260
288
  HTTPStatus.BAD_REQUEST.value: MLRunBadRequestError,
261
289
  HTTPStatus.UNAUTHORIZED.value: MLRunUnauthorizedError,
mlrun/execution.py CHANGED
@@ -15,6 +15,7 @@
15
15
  import logging
16
16
  import os
17
17
  import uuid
18
+ import warnings
18
19
  from copy import deepcopy
19
20
  from typing import Optional, Union, cast
20
21
 
@@ -991,6 +992,14 @@ class MLClientCtx:
991
992
  self._update_run()
992
993
  return item
993
994
 
995
+ def get_cached_artifact(self, key):
996
+ """Return a logged artifact from cache (for potential updates)"""
997
+ warnings.warn(
998
+ "get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.11.0. Use get_artifact instead.",
999
+ FutureWarning,
1000
+ )
1001
+ return self.get_artifact(key)
1002
+
994
1003
  def get_artifact(
995
1004
  self, key, tag=None, iter=None, tree=None, uid=None
996
1005
  ) -> Optional[Artifact]:
@@ -63,7 +63,7 @@ def parse_project_name_from_feature_string(feature):
63
63
 
64
64
  def parse_feature_set_uri(uri, project=None):
65
65
  """get feature set object from db by uri"""
66
- default_project = project or config.default_project
66
+ active_project = project or config.active_project
67
67
 
68
68
  # parse store://.. uri
69
69
  if mlrun.datastore.is_store_uri(uri):
@@ -74,7 +74,7 @@ def parse_feature_set_uri(uri, project=None):
74
74
  )
75
75
  uri = new_uri
76
76
 
77
- return parse_versioned_object_uri(uri, default_project)
77
+ return parse_versioned_object_uri(uri, active_project)
78
78
 
79
79
 
80
80
  def get_feature_set_by_uri(uri, project=None):
@@ -98,7 +98,7 @@ def get_feature_set_by_uri(uri, project=None):
98
98
  def get_feature_vector_by_uri(uri, project=None, update=True):
99
99
  """get feature vector object from db by uri"""
100
100
  db = mlrun.get_run_db()
101
- default_project = project or config.default_project
101
+ active_project = project or config.active_project
102
102
 
103
103
  # parse store://.. uri
104
104
  if mlrun.datastore.is_store_uri(uri):
@@ -109,7 +109,7 @@ def get_feature_vector_by_uri(uri, project=None, update=True):
109
109
  )
110
110
  uri = new_uri
111
111
 
112
- project, name, tag, uid = parse_versioned_object_uri(uri, default_project)
112
+ project, name, tag, uid = parse_versioned_object_uri(uri, active_project)
113
113
 
114
114
  resource = mlrun.common.schemas.AuthorizationResourceTypes.feature_vector.to_resource_string(
115
115
  project, "feature-vector"
@@ -161,7 +161,7 @@ def verify_feature_set_exists(feature_set):
161
161
  def verify_feature_vector_permissions(
162
162
  feature_vector, action: mlrun.common.schemas.AuthorizationAction
163
163
  ):
164
- project = feature_vector._metadata.project or config.default_project
164
+ project = feature_vector._metadata.project or config.active_project
165
165
 
166
166
  resource = mlrun.common.schemas.AuthorizationResourceTypes.feature_vector.to_resource_string(
167
167
  project, "feature-vector"
@@ -413,11 +413,15 @@ class FeatureSet(ModelObj):
413
413
  @property
414
414
  def fullname(self) -> str:
415
415
  """full name in the form ``{project}/{name}[:{tag}]``"""
416
- fullname = (
417
- f"{self._metadata.project or mlconf.default_project}/{self._metadata.name}"
418
- )
419
- if self._metadata.tag:
420
- fullname += ":" + self._metadata.tag
416
+ project = self._metadata.project or mlconf.active_project
417
+ name = self._metadata.name
418
+ tag = self._metadata.tag
419
+
420
+ fullname = name
421
+ if project:
422
+ fullname = f"{project}/{fullname}"
423
+ if tag:
424
+ fullname += f":{tag}"
421
425
  return fullname
422
426
 
423
427
  def _get_run_db(self):
@@ -971,7 +975,7 @@ class FeatureSet(ModelObj):
971
975
  def save(self, tag="", versioned=False):
972
976
  """save to mlrun db"""
973
977
  db = self._get_run_db()
974
- self.metadata.project = self.metadata.project or mlconf.default_project
978
+ self.metadata.project = self.metadata.project or mlconf.active_project
975
979
  tag = tag or self.metadata.tag or "latest"
976
980
  as_dict = self.to_dict()
977
981
  as_dict["spec"]["features"] = as_dict["spec"].get(
@@ -333,12 +333,14 @@ class FeatureVector(ModelObj):
333
333
  @property
334
334
  def uri(self):
335
335
  """fully qualified feature vector uri"""
336
- uri = (
337
- f"{self._metadata.project or mlconf.default_project}/{self._metadata.name}"
338
- )
339
- uri = get_store_uri(StorePrefix.FeatureVector, uri)
336
+ project = self._metadata.project or mlconf.active_project
337
+ name = self._metadata.name
338
+
339
+ base = name if not project else f"{project}/{name}"
340
+ uri = get_store_uri(StorePrefix.FeatureVector, base)
341
+
340
342
  if self._metadata.tag:
341
- uri += ":" + self._metadata.tag
343
+ uri += f":{self._metadata.tag}"
342
344
  return uri
343
345
 
344
346
  def link_analysis(self, name, uri):
@@ -385,7 +387,7 @@ class FeatureVector(ModelObj):
385
387
  def save(self, tag="", versioned=False):
386
388
  """save to mlrun db"""
387
389
  db = mlrun.get_run_db()
388
- self.metadata.project = self.metadata.project or mlconf.default_project
390
+ self.metadata.project = self.metadata.project or mlconf.active_project
389
391
  tag = tag or self.metadata.tag
390
392
  as_dict = self.to_dict()
391
393
  db.store_feature_vector(as_dict, tag=tag, versioned=versioned)
mlrun/launcher/base.py CHANGED
@@ -273,7 +273,7 @@ class BaseLauncher(abc.ABC):
273
273
  project_name
274
274
  or run.metadata.project
275
275
  or runtime.metadata.project
276
- or mlrun.mlconf.default_project
276
+ or mlrun.mlconf.active_project
277
277
  )
278
278
  run.spec.parameters = params or run.spec.parameters
279
279
  run.spec.inputs = inputs or run.spec.inputs
mlrun/launcher/client.py CHANGED
@@ -72,7 +72,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
72
72
  ):
73
73
  run.metadata.labels[mlrun_constants.MLRunInternalLabels.kind] = runtime.kind
74
74
  mlrun.runtimes.utils.enrich_run_labels(
75
- run.metadata.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
75
+ run.metadata.labels, [mlrun_constants.MLRunInternalLabels.owner]
76
76
  )
77
77
  if run.spec.output_path:
78
78
  run.spec.output_path = run.spec.output_path.replace(
mlrun/lists.py CHANGED
@@ -55,7 +55,7 @@ class RunList(list):
55
55
  for run in self:
56
56
  iterations = get_in(run, "status.iterations", "")
57
57
  row = [
58
- get_in(run, "metadata.project", config.default_project),
58
+ get_in(run, "metadata.project", config.active_project),
59
59
  get_in(run, "metadata.uid", ""),
60
60
  get_in(run, "metadata.iteration", ""),
61
61
  get_in(run, "status.start_time", ""),
@@ -16,4 +16,3 @@ from mlrun.common.schemas import ModelEndpoint, ModelEndpointList
16
16
 
17
17
  from .db import get_tsdb_connector
18
18
  from .helpers import get_stream_path
19
- from .tracking_policy import TrackingPolicy
@@ -14,7 +14,6 @@
14
14
 
15
15
  import hashlib
16
16
  import typing
17
- import warnings
18
17
  from datetime import datetime
19
18
 
20
19
  import numpy as np
@@ -136,12 +135,6 @@ def record_results(
136
135
  infer_results_df: typing.Optional[pd.DataFrame] = None,
137
136
  sample_set_statistics: typing.Optional[dict[str, typing.Any]] = None,
138
137
  monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
139
- # Deprecated arguments:
140
- drift_threshold: typing.Optional[float] = None,
141
- possible_drift_threshold: typing.Optional[float] = None,
142
- trigger_monitoring_job: bool = False,
143
- artifacts_tag: str = "",
144
- default_batch_image: str = "mlrun/mlrun",
145
138
  ) -> ModelEndpoint:
146
139
  """
147
140
  Write a provided inference dataset to model endpoint parquet target. If not exist, generate a new model endpoint
@@ -166,47 +159,10 @@ def record_results(
166
159
  the current model endpoint.
167
160
  :param monitoring_mode: If enabled, apply model monitoring features on the provided endpoint id. Enabled
168
161
  by default.
169
- :param drift_threshold: (deprecated) The threshold of which to mark drifts.
170
- :param possible_drift_threshold: (deprecated) The threshold of which to mark possible drifts.
171
- :param trigger_monitoring_job: (deprecated) If true, run the batch drift job. If not exists, the monitoring
172
- batch function will be registered through MLRun API with the provided image.
173
- :param artifacts_tag: (deprecated) Tag to use for all the artifacts resulted from the function.
174
- Will be relevant only if the monitoring batch job has been triggered.
175
- :param default_batch_image: (deprecated) The image that will be used when registering the model monitoring
176
- batch job.
177
162
 
178
163
  :return: A ModelEndpoint object
179
164
  """
180
165
 
181
- if drift_threshold is not None or possible_drift_threshold is not None:
182
- warnings.warn(
183
- "Custom drift threshold arguments are deprecated since version "
184
- "1.7.0 and have no effect. They will be removed in version 1.9.0.\n"
185
- "To enable the default histogram data drift application, run:\n"
186
- "`project.enable_model_monitoring()`.",
187
- FutureWarning,
188
- )
189
- if trigger_monitoring_job is not False:
190
- warnings.warn(
191
- "`trigger_monitoring_job` argument is deprecated since version "
192
- "1.7.0 and has no effect. It will be removed in version 1.9.0.\n"
193
- "To enable the default histogram data drift application, run:\n"
194
- "`project.enable_model_monitoring()`.",
195
- FutureWarning,
196
- )
197
- if artifacts_tag != "":
198
- warnings.warn(
199
- "`artifacts_tag` argument is deprecated since version "
200
- "1.7.0 and has no effect. It will be removed in version 1.9.0.",
201
- FutureWarning,
202
- )
203
- if default_batch_image != "mlrun/mlrun":
204
- warnings.warn(
205
- "`default_batch_image` argument is deprecated since version "
206
- "1.7.0 and has no effect. It will be removed in version 1.9.0.",
207
- FutureWarning,
208
- )
209
-
210
166
  db = mlrun.get_run_db()
211
167
 
212
168
  model_endpoint = get_or_create_model_endpoint(
@@ -12,21 +12,18 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- import json
16
- import posixpath
17
- import uuid
18
15
  import warnings
19
16
  from abc import ABC
17
+ from tempfile import NamedTemporaryFile
18
+ from typing import Optional
20
19
 
21
- import pandas as pd
22
20
  import semver
23
- from evidently.ui.storage.local.base import METADATA_PATH, FSLocation
24
21
 
25
22
  import mlrun.model_monitoring.applications.base as mm_base
26
23
  import mlrun.model_monitoring.applications.context as mm_context
27
- from mlrun.errors import MLRunIncompatibleVersionError
24
+ from mlrun.errors import MLRunIncompatibleVersionError, MLRunValueError
28
25
 
29
- SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.6.0")
26
+ SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.7.5")
30
27
 
31
28
 
32
29
  def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
@@ -60,75 +57,69 @@ except ModuleNotFoundError:
60
57
 
61
58
 
62
59
  if _HAS_EVIDENTLY:
63
- from evidently.suite.base_suite import Display
64
- from evidently.ui.type_aliases import STR_UUID
65
- from evidently.ui.workspace import Workspace
66
- from evidently.utils.dashboard import TemplateParams, file_html_template
60
+ from evidently.core.report import Snapshot
61
+ from evidently.ui.workspace import (
62
+ STR_UUID,
63
+ CloudWorkspace,
64
+ Project,
65
+ Workspace,
66
+ WorkspaceBase,
67
+ )
67
68
 
68
69
 
69
70
  class EvidentlyModelMonitoringApplicationBase(
70
71
  mm_base.ModelMonitoringApplicationBase, ABC
71
72
  ):
72
73
  def __init__(
73
- self, evidently_workspace_path: str, evidently_project_id: "STR_UUID"
74
+ self,
75
+ evidently_project_id: "STR_UUID",
76
+ evidently_workspace_path: Optional[str] = None,
77
+ cloud_workspace: bool = False,
74
78
  ) -> None:
75
79
  """
76
- A class for integrating Evidently for mlrun model monitoring within a monitoring application.
77
- Note: evidently is not installed by default in the mlrun/mlrun image.
78
- It must be installed separately to use this class.
80
+ A class for integrating Evidently for MLRun model monitoring within a monitoring application.
81
+
82
+ .. note::
83
+
84
+ The ``evidently`` package is not installed by default in the mlrun/mlrun image.
85
+ It must be installed separately to use this class.
79
86
 
80
- :param evidently_workspace_path: (str) The path to the Evidently workspace.
81
87
  :param evidently_project_id: (str) The ID of the Evidently project.
88
+ :param evidently_workspace_path: (str) The path to the Evidently workspace.
89
+ :param cloud_workspace: (bool) Whether the workspace is an Evidently Cloud workspace.
82
90
  """
83
-
84
- # TODO : more then one project (mep -> project)
85
91
  if not _HAS_EVIDENTLY:
86
92
  raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
87
- self._log_location(evidently_workspace_path)
88
- self.evidently_workspace = Workspace.create(evidently_workspace_path)
93
+ self.evidently_workspace_path = evidently_workspace_path
94
+ if cloud_workspace:
95
+ self.get_workspace = self.get_cloud_workspace
96
+ self.evidently_workspace = self.get_workspace()
89
97
  self.evidently_project_id = evidently_project_id
90
- self.evidently_project = self.evidently_workspace.get_project(
91
- evidently_project_id
92
- )
93
-
94
- @staticmethod
95
- def _log_location(evidently_workspace_path):
96
- # TODO remove function + usage after solving issue ML-9530
97
- location = FSLocation(base_path=evidently_workspace_path)
98
- location.invalidate_cache("")
99
- paths = [p for p in location.listdir("") if location.isdir(p)]
100
-
101
- for path in paths:
102
- metadata_path = posixpath.join(path, METADATA_PATH)
103
- full_path = posixpath.join(location.path, metadata_path)
104
- print(f"evidently json issue, working on path: {full_path}")
105
- try:
106
- with location.open(metadata_path) as f:
107
- content = json.load(f)
108
- print(
109
- f"evidently json issue, successful load path: {full_path}, content: {content}"
110
- )
111
- except FileNotFoundError:
112
- print(f"evidently json issue, path not found: {full_path}")
113
- continue
114
- except json.decoder.JSONDecodeError as json_error:
115
- print(
116
- f"evidently json issue, path got json error, path:{full_path}, error: {json_error}"
117
- )
118
- print("evidently json issue, file content:")
119
- with location.open(metadata_path) as f:
120
- print(f.read())
121
- continue
122
- except Exception as error:
123
- print(
124
- f"evidently json issue, path got general error, path:{full_path}, error: {error}"
125
- )
126
- continue
98
+ self.evidently_project = self.load_project()
99
+
100
+ def load_project(self) -> "Project":
101
+ """Load the Evidently project."""
102
+ return self.evidently_workspace.get_project(self.evidently_project_id)
103
+
104
+ def get_workspace(self) -> "WorkspaceBase":
105
+ """Get the Evidently workspace. Override this method for customize access to the workspace."""
106
+ if self.evidently_workspace_path:
107
+ return Workspace.create(self.evidently_workspace_path)
108
+ else:
109
+ raise MLRunValueError(
110
+ "A local workspace could not be created as `evidently_workspace_path` is not set.\n"
111
+ "If you intend to use a cloud workspace, please use `cloud_workspace=True` and set the "
112
+ "`EVIDENTLY_API_KEY` environment variable. In other cases, override this method."
113
+ )
114
+
115
+ def get_cloud_workspace(self) -> "CloudWorkspace":
116
+ """Load the Evidently cloud workspace according to the `EVIDENTLY_API_KEY` environment variable."""
117
+ return CloudWorkspace()
127
118
 
128
119
  @staticmethod
129
120
  def log_evidently_object(
130
121
  monitoring_context: mm_context.MonitoringApplicationContext,
131
- evidently_object: "Display",
122
+ evidently_object: "Snapshot",
132
123
  artifact_name: str,
133
124
  unique_per_endpoint: bool = True,
134
125
  ) -> None:
@@ -141,56 +132,15 @@ class EvidentlyModelMonitoringApplicationBase(
141
132
  This method should be called on special occasions only.
142
133
 
143
134
  :param monitoring_context: (MonitoringApplicationContext) The monitoring context to process.
144
- :param evidently_object: (Display) The Evidently display to log, e.g. a report or a test suite object.
145
- :param artifact_name: (str) The name for the logged artifact.
146
- :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
147
- set to ``False`` without changing item key will cause artifact override.
148
- """
149
- evidently_object_html = evidently_object.get_html()
150
- monitoring_context.log_artifact(
151
- artifact_name,
152
- body=evidently_object_html.encode("utf-8"),
153
- format="html",
154
- unique_per_endpoint=unique_per_endpoint,
155
- )
156
-
157
- def log_project_dashboard(
158
- self,
159
- monitoring_context: mm_context.MonitoringApplicationContext,
160
- timestamp_start: pd.Timestamp,
161
- timestamp_end: pd.Timestamp,
162
- artifact_name: str = "dashboard",
163
- unique_per_endpoint: bool = True,
164
- ) -> None:
165
- """
166
- Logs an Evidently project dashboard.
167
-
168
- .. caution::
169
-
170
- Logging Evidently dashboards in every model monitoring window may cause scale issues.
171
- This method should be called on special occasions only.
172
-
173
- :param monitoring_context: (MonitoringApplicationContext) The monitoring context to process.
174
- :param timestamp_start: (pd.Timestamp) The start timestamp for the dashboard data.
175
- :param timestamp_end: (pd.Timestamp) The end timestamp for the dashboard data.
135
+ :param evidently_object: (Snapshot) The Evidently run to log, e.g. a report run.
176
136
  :param artifact_name: (str) The name for the logged artifact.
177
137
  :param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
178
138
  set to ``False`` without changing item key will cause artifact override.
179
139
  """
180
-
181
- dashboard_info = self.evidently_project.build_dashboard_info(
182
- timestamp_start, timestamp_end
183
- )
184
- template_params = TemplateParams(
185
- dashboard_id="pd_" + str(uuid.uuid4()).replace("-", ""),
186
- dashboard_info=dashboard_info,
187
- additional_graphs={},
188
- )
189
-
190
- dashboard_html = file_html_template(params=template_params)
191
- monitoring_context.log_artifact(
192
- artifact_name,
193
- body=dashboard_html.encode("utf-8"),
194
- format="html",
195
- unique_per_endpoint=unique_per_endpoint,
196
- )
140
+ with NamedTemporaryFile(suffix=".html") as file:
141
+ evidently_object.save_html(filename=file.name)
142
+ monitoring_context.log_artifact(
143
+ artifact_name,
144
+ local_path=file.name,
145
+ unique_per_endpoint=unique_per_endpoint,
146
+ )
@@ -25,6 +25,7 @@ from types import TracebackType
25
25
  from typing import Any, NamedTuple, Optional, Union, cast
26
26
 
27
27
  import nuclio_sdk
28
+ import pandas as pd
28
29
 
29
30
  import mlrun
30
31
  import mlrun.common.schemas.model_monitoring.constants as mm_constants
@@ -250,7 +251,7 @@ class MonitoringApplicationController:
250
251
 
251
252
  def __init__(self) -> None:
252
253
  """Initialize Monitoring Application Controller"""
253
- self.project = cast(str, mlrun.mlconf.default_project)
254
+ self.project = cast(str, mlrun.mlconf.active_project)
254
255
  self.project_obj = mlrun.get_run_db().get_project(name=self.project)
255
256
  logger.debug(f"Initializing {self.__class__.__name__}", project=self.project)
256
257
 
@@ -673,9 +674,15 @@ class MonitoringApplicationController:
673
674
  """
674
675
  logger.info("Starting monitoring controller chief")
675
676
  applications_names = []
676
- endpoints = self.project_obj.list_model_endpoints(
677
- metric_list=["last_request"]
678
- ).endpoints
677
+ endpoints = self.project_obj.list_model_endpoints(tsdb_metrics=False).endpoints
678
+ last_request_dict = self.tsdb_connector.get_last_request(
679
+ endpoint_ids=[mep.metadata.uid for mep in endpoints]
680
+ )
681
+ if isinstance(last_request_dict, pd.DataFrame):
682
+ last_request_dict = last_request_dict.set_index(
683
+ mm_constants.EventFieldType.ENDPOINT_ID
684
+ )[mm_constants.ModelEndpointSchema.LAST_REQUEST].to_dict()
685
+
679
686
  if not endpoints:
680
687
  logger.info("No model endpoints found", project=self.project)
681
688
  return
@@ -721,16 +728,22 @@ class MonitoringApplicationController:
721
728
  with schedules.ModelMonitoringSchedulesFileChief(
722
729
  self.project
723
730
  ) as schedule_file:
724
- futures = {
725
- pool.submit(
726
- self.endpoint_to_regular_event,
727
- endpoint,
728
- policy,
729
- set(applications_names),
730
- schedule_file,
731
- ): endpoint
732
- for endpoint in endpoints
733
- }
731
+ for endpoint in endpoints:
732
+ last_request = last_request_dict.get(endpoint.metadata.uid, None)
733
+ if isinstance(last_request, float):
734
+ last_request = pd.to_datetime(last_request, unit="s", utc=True)
735
+ endpoint.status.last_request = (
736
+ last_request or endpoint.status.last_request
737
+ )
738
+ futures = {
739
+ pool.submit(
740
+ self.endpoint_to_regular_event,
741
+ endpoint,
742
+ policy,
743
+ set(applications_names),
744
+ schedule_file,
745
+ ): endpoint
746
+ }
734
747
  for future in concurrent.futures.as_completed(futures):
735
748
  if future.exception():
736
749
  exception = future.exception()
@@ -455,12 +455,20 @@ class V3IOTSDBConnector(TSDBConnector):
455
455
  # Delete all tables
456
456
  tables = mm_schemas.V3IOTSDBTables.list()
457
457
  for table_to_delete in tables:
458
- try:
459
- self.frames_client.delete(backend=_TSDB_BE, table=table_to_delete)
460
- except v3io_frames.DeleteError as e:
458
+ if table_to_delete in self.tables:
459
+ try:
460
+ self.frames_client.delete(
461
+ backend=_TSDB_BE, table=self.tables[table_to_delete]
462
+ )
463
+ except v3io_frames.DeleteError as e:
464
+ logger.warning(
465
+ f"Failed to delete TSDB table '{table_to_delete}'",
466
+ err=mlrun.errors.err_to_str(e),
467
+ )
468
+ else:
461
469
  logger.warning(
462
- f"Failed to delete TSDB table '{table}'",
463
- err=mlrun.errors.err_to_str(e),
470
+ f"Skipping deletion: table '{table_to_delete}' is not among the initialized tables.",
471
+ initialized_tables=list(self.tables.keys()),
464
472
  )
465
473
 
466
474
  # Final cleanup of tsdb path
@@ -129,10 +129,7 @@ class ModelMonitoringWriter(StepToDict):
129
129
  )
130
130
  kind = event.pop(WriterEvent.EVENT_KIND, WriterEventKind.RESULT)
131
131
  result_event = _AppResultEvent(json.loads(event.pop(WriterEvent.DATA, "{}")))
132
- if not result_event: # BC for < 1.7.0, can be removed in 1.9.0
133
- result_event = _AppResultEvent(event)
134
- else:
135
- result_event.update(_AppResultEvent(event))
132
+ result_event.update(_AppResultEvent(event))
136
133
 
137
134
  expected_keys = list(
138
135
  set(WriterEvent.list()).difference(
@@ -309,9 +309,9 @@ def build_function(
309
309
  :param force_build: Force building the image, even when no changes were made
310
310
  """
311
311
  if not overwrite_build_params:
312
- # TODO: change overwrite_build_params default to True in 1.9.0
312
+ # TODO: change overwrite_build_params default to True in 1.10.0
313
313
  warnings.warn(
314
- "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.9.0.",
314
+ "The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.10.0.",
315
315
  mlrun.utils.OverwriteBuildParamsWarning,
316
316
  )
317
317
 
@@ -340,7 +340,7 @@ def build_function(
340
340
  skip_deployed=skip_deployed,
341
341
  )
342
342
  else:
343
- # TODO: remove filter once overwrite_build_params default is changed to True in 1.9.0
343
+ # TODO: remove filter once overwrite_build_params default is changed to True in 1.10.0
344
344
  with warnings.catch_warnings():
345
345
  warnings.simplefilter(
346
346
  "ignore", category=mlrun.utils.OverwriteBuildParamsWarning