mlrun 1.10.0rc1__py3-none-any.whl → 1.10.0rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__init__.py +2 -2
- mlrun/__main__.py +15 -4
- mlrun/artifacts/base.py +6 -6
- mlrun/artifacts/dataset.py +1 -1
- mlrun/artifacts/document.py +1 -1
- mlrun/artifacts/model.py +1 -1
- mlrun/artifacts/plots.py +2 -2
- mlrun/common/constants.py +7 -0
- mlrun/common/runtimes/constants.py +1 -1
- mlrun/common/schemas/__init__.py +1 -0
- mlrun/common/schemas/artifact.py +1 -1
- mlrun/common/schemas/pipeline.py +1 -1
- mlrun/common/schemas/project.py +1 -1
- mlrun/common/schemas/runs.py +1 -1
- mlrun/common/schemas/serving.py +17 -0
- mlrun/config.py +4 -4
- mlrun/datastore/datastore_profile.py +7 -57
- mlrun/datastore/sources.py +24 -16
- mlrun/datastore/store_resources.py +3 -3
- mlrun/datastore/targets.py +5 -5
- mlrun/datastore/utils.py +21 -6
- mlrun/db/base.py +7 -7
- mlrun/db/httpdb.py +88 -76
- mlrun/db/nopdb.py +1 -1
- mlrun/errors.py +29 -1
- mlrun/execution.py +9 -0
- mlrun/feature_store/common.py +5 -5
- mlrun/feature_store/feature_set.py +10 -6
- mlrun/feature_store/feature_vector.py +8 -6
- mlrun/launcher/base.py +1 -1
- mlrun/launcher/client.py +1 -1
- mlrun/lists.py +1 -1
- mlrun/model_monitoring/__init__.py +0 -1
- mlrun/model_monitoring/api.py +0 -44
- mlrun/model_monitoring/applications/evidently/base.py +57 -107
- mlrun/model_monitoring/controller.py +27 -14
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +13 -5
- mlrun/model_monitoring/writer.py +1 -4
- mlrun/projects/operations.py +3 -3
- mlrun/projects/project.py +114 -52
- mlrun/render.py +5 -9
- mlrun/run.py +10 -10
- mlrun/runtimes/base.py +7 -7
- mlrun/runtimes/kubejob.py +2 -2
- mlrun/runtimes/nuclio/function.py +3 -3
- mlrun/runtimes/nuclio/serving.py +13 -23
- mlrun/runtimes/utils.py +25 -8
- mlrun/serving/__init__.py +5 -1
- mlrun/serving/server.py +39 -3
- mlrun/serving/states.py +176 -10
- mlrun/utils/helpers.py +10 -4
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/METADATA +27 -15
- {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/RECORD +58 -59
- {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/WHEEL +1 -1
- mlrun/model_monitoring/tracking_policy.py +0 -124
- {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/entry_points.txt +0 -0
- {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.10.0rc1.dist-info → mlrun-1.10.0rc3.dist-info}/top_level.txt +0 -0
mlrun/errors.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
import copy
|
|
15
15
|
import typing
|
|
16
16
|
from http import HTTPStatus
|
|
17
17
|
|
|
@@ -230,6 +230,13 @@ class MLRunTSDBConnectionFailureError(MLRunHTTPStatusError, ValueError):
|
|
|
230
230
|
error_status_code = HTTPStatus.BAD_REQUEST.value
|
|
231
231
|
|
|
232
232
|
|
|
233
|
+
class MLRunMissingProjectError(MLRunBadRequestError):
|
|
234
|
+
default_message = "Project must be provided"
|
|
235
|
+
|
|
236
|
+
def __init__(self, message=None):
|
|
237
|
+
super().__init__(message or self.default_message)
|
|
238
|
+
|
|
239
|
+
|
|
233
240
|
class MLRunRetryExhaustedError(Exception):
|
|
234
241
|
pass
|
|
235
242
|
|
|
@@ -256,6 +263,27 @@ class MLRunFatalFailureError(Exception):
|
|
|
256
263
|
self.original_exception = original_exception
|
|
257
264
|
|
|
258
265
|
|
|
266
|
+
class ModelRunnerError(MLRunBaseError):
|
|
267
|
+
def __init__(self, models_errors: dict[str:str], *args) -> None:
|
|
268
|
+
self.models_errors = models_errors
|
|
269
|
+
super().__init__(self.__repr__(), *args)
|
|
270
|
+
|
|
271
|
+
def __repr__(self):
|
|
272
|
+
return f"ModelRunnerError: {repr(self.models_errors)}"
|
|
273
|
+
|
|
274
|
+
def __copy__(self):
|
|
275
|
+
return type(self)(models_errors=self.models_errors)
|
|
276
|
+
|
|
277
|
+
def __deepcopy__(self, memo):
|
|
278
|
+
return type(self)(copy.deepcopy(self.models_errors, memo))
|
|
279
|
+
|
|
280
|
+
def get_errors(self):
|
|
281
|
+
return self.models_errors
|
|
282
|
+
|
|
283
|
+
def get_model_error(self, model: str):
|
|
284
|
+
return self.models_errors.get(model)
|
|
285
|
+
|
|
286
|
+
|
|
259
287
|
STATUS_ERRORS = {
|
|
260
288
|
HTTPStatus.BAD_REQUEST.value: MLRunBadRequestError,
|
|
261
289
|
HTTPStatus.UNAUTHORIZED.value: MLRunUnauthorizedError,
|
mlrun/execution.py
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import logging
|
|
16
16
|
import os
|
|
17
17
|
import uuid
|
|
18
|
+
import warnings
|
|
18
19
|
from copy import deepcopy
|
|
19
20
|
from typing import Optional, Union, cast
|
|
20
21
|
|
|
@@ -991,6 +992,14 @@ class MLClientCtx:
|
|
|
991
992
|
self._update_run()
|
|
992
993
|
return item
|
|
993
994
|
|
|
995
|
+
def get_cached_artifact(self, key):
|
|
996
|
+
"""Return a logged artifact from cache (for potential updates)"""
|
|
997
|
+
warnings.warn(
|
|
998
|
+
"get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.11.0. Use get_artifact instead.",
|
|
999
|
+
FutureWarning,
|
|
1000
|
+
)
|
|
1001
|
+
return self.get_artifact(key)
|
|
1002
|
+
|
|
994
1003
|
def get_artifact(
|
|
995
1004
|
self, key, tag=None, iter=None, tree=None, uid=None
|
|
996
1005
|
) -> Optional[Artifact]:
|
mlrun/feature_store/common.py
CHANGED
|
@@ -63,7 +63,7 @@ def parse_project_name_from_feature_string(feature):
|
|
|
63
63
|
|
|
64
64
|
def parse_feature_set_uri(uri, project=None):
|
|
65
65
|
"""get feature set object from db by uri"""
|
|
66
|
-
|
|
66
|
+
active_project = project or config.active_project
|
|
67
67
|
|
|
68
68
|
# parse store://.. uri
|
|
69
69
|
if mlrun.datastore.is_store_uri(uri):
|
|
@@ -74,7 +74,7 @@ def parse_feature_set_uri(uri, project=None):
|
|
|
74
74
|
)
|
|
75
75
|
uri = new_uri
|
|
76
76
|
|
|
77
|
-
return parse_versioned_object_uri(uri,
|
|
77
|
+
return parse_versioned_object_uri(uri, active_project)
|
|
78
78
|
|
|
79
79
|
|
|
80
80
|
def get_feature_set_by_uri(uri, project=None):
|
|
@@ -98,7 +98,7 @@ def get_feature_set_by_uri(uri, project=None):
|
|
|
98
98
|
def get_feature_vector_by_uri(uri, project=None, update=True):
|
|
99
99
|
"""get feature vector object from db by uri"""
|
|
100
100
|
db = mlrun.get_run_db()
|
|
101
|
-
|
|
101
|
+
active_project = project or config.active_project
|
|
102
102
|
|
|
103
103
|
# parse store://.. uri
|
|
104
104
|
if mlrun.datastore.is_store_uri(uri):
|
|
@@ -109,7 +109,7 @@ def get_feature_vector_by_uri(uri, project=None, update=True):
|
|
|
109
109
|
)
|
|
110
110
|
uri = new_uri
|
|
111
111
|
|
|
112
|
-
project, name, tag, uid = parse_versioned_object_uri(uri,
|
|
112
|
+
project, name, tag, uid = parse_versioned_object_uri(uri, active_project)
|
|
113
113
|
|
|
114
114
|
resource = mlrun.common.schemas.AuthorizationResourceTypes.feature_vector.to_resource_string(
|
|
115
115
|
project, "feature-vector"
|
|
@@ -161,7 +161,7 @@ def verify_feature_set_exists(feature_set):
|
|
|
161
161
|
def verify_feature_vector_permissions(
|
|
162
162
|
feature_vector, action: mlrun.common.schemas.AuthorizationAction
|
|
163
163
|
):
|
|
164
|
-
project = feature_vector._metadata.project or config.
|
|
164
|
+
project = feature_vector._metadata.project or config.active_project
|
|
165
165
|
|
|
166
166
|
resource = mlrun.common.schemas.AuthorizationResourceTypes.feature_vector.to_resource_string(
|
|
167
167
|
project, "feature-vector"
|
|
@@ -413,11 +413,15 @@ class FeatureSet(ModelObj):
|
|
|
413
413
|
@property
|
|
414
414
|
def fullname(self) -> str:
|
|
415
415
|
"""full name in the form ``{project}/{name}[:{tag}]``"""
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
416
|
+
project = self._metadata.project or mlconf.active_project
|
|
417
|
+
name = self._metadata.name
|
|
418
|
+
tag = self._metadata.tag
|
|
419
|
+
|
|
420
|
+
fullname = name
|
|
421
|
+
if project:
|
|
422
|
+
fullname = f"{project}/{fullname}"
|
|
423
|
+
if tag:
|
|
424
|
+
fullname += f":{tag}"
|
|
421
425
|
return fullname
|
|
422
426
|
|
|
423
427
|
def _get_run_db(self):
|
|
@@ -971,7 +975,7 @@ class FeatureSet(ModelObj):
|
|
|
971
975
|
def save(self, tag="", versioned=False):
|
|
972
976
|
"""save to mlrun db"""
|
|
973
977
|
db = self._get_run_db()
|
|
974
|
-
self.metadata.project = self.metadata.project or mlconf.
|
|
978
|
+
self.metadata.project = self.metadata.project or mlconf.active_project
|
|
975
979
|
tag = tag or self.metadata.tag or "latest"
|
|
976
980
|
as_dict = self.to_dict()
|
|
977
981
|
as_dict["spec"]["features"] = as_dict["spec"].get(
|
|
@@ -333,12 +333,14 @@ class FeatureVector(ModelObj):
|
|
|
333
333
|
@property
|
|
334
334
|
def uri(self):
|
|
335
335
|
"""fully qualified feature vector uri"""
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
336
|
+
project = self._metadata.project or mlconf.active_project
|
|
337
|
+
name = self._metadata.name
|
|
338
|
+
|
|
339
|
+
base = name if not project else f"{project}/{name}"
|
|
340
|
+
uri = get_store_uri(StorePrefix.FeatureVector, base)
|
|
341
|
+
|
|
340
342
|
if self._metadata.tag:
|
|
341
|
-
uri += ":
|
|
343
|
+
uri += f":{self._metadata.tag}"
|
|
342
344
|
return uri
|
|
343
345
|
|
|
344
346
|
def link_analysis(self, name, uri):
|
|
@@ -385,7 +387,7 @@ class FeatureVector(ModelObj):
|
|
|
385
387
|
def save(self, tag="", versioned=False):
|
|
386
388
|
"""save to mlrun db"""
|
|
387
389
|
db = mlrun.get_run_db()
|
|
388
|
-
self.metadata.project = self.metadata.project or mlconf.
|
|
390
|
+
self.metadata.project = self.metadata.project or mlconf.active_project
|
|
389
391
|
tag = tag or self.metadata.tag
|
|
390
392
|
as_dict = self.to_dict()
|
|
391
393
|
db.store_feature_vector(as_dict, tag=tag, versioned=versioned)
|
mlrun/launcher/base.py
CHANGED
|
@@ -273,7 +273,7 @@ class BaseLauncher(abc.ABC):
|
|
|
273
273
|
project_name
|
|
274
274
|
or run.metadata.project
|
|
275
275
|
or runtime.metadata.project
|
|
276
|
-
or mlrun.mlconf.
|
|
276
|
+
or mlrun.mlconf.active_project
|
|
277
277
|
)
|
|
278
278
|
run.spec.parameters = params or run.spec.parameters
|
|
279
279
|
run.spec.inputs = inputs or run.spec.inputs
|
mlrun/launcher/client.py
CHANGED
|
@@ -72,7 +72,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
|
|
|
72
72
|
):
|
|
73
73
|
run.metadata.labels[mlrun_constants.MLRunInternalLabels.kind] = runtime.kind
|
|
74
74
|
mlrun.runtimes.utils.enrich_run_labels(
|
|
75
|
-
run.metadata.labels, [
|
|
75
|
+
run.metadata.labels, [mlrun_constants.MLRunInternalLabels.owner]
|
|
76
76
|
)
|
|
77
77
|
if run.spec.output_path:
|
|
78
78
|
run.spec.output_path = run.spec.output_path.replace(
|
mlrun/lists.py
CHANGED
|
@@ -55,7 +55,7 @@ class RunList(list):
|
|
|
55
55
|
for run in self:
|
|
56
56
|
iterations = get_in(run, "status.iterations", "")
|
|
57
57
|
row = [
|
|
58
|
-
get_in(run, "metadata.project", config.
|
|
58
|
+
get_in(run, "metadata.project", config.active_project),
|
|
59
59
|
get_in(run, "metadata.uid", ""),
|
|
60
60
|
get_in(run, "metadata.iteration", ""),
|
|
61
61
|
get_in(run, "status.start_time", ""),
|
mlrun/model_monitoring/api.py
CHANGED
|
@@ -14,7 +14,6 @@
|
|
|
14
14
|
|
|
15
15
|
import hashlib
|
|
16
16
|
import typing
|
|
17
|
-
import warnings
|
|
18
17
|
from datetime import datetime
|
|
19
18
|
|
|
20
19
|
import numpy as np
|
|
@@ -136,12 +135,6 @@ def record_results(
|
|
|
136
135
|
infer_results_df: typing.Optional[pd.DataFrame] = None,
|
|
137
136
|
sample_set_statistics: typing.Optional[dict[str, typing.Any]] = None,
|
|
138
137
|
monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
|
|
139
|
-
# Deprecated arguments:
|
|
140
|
-
drift_threshold: typing.Optional[float] = None,
|
|
141
|
-
possible_drift_threshold: typing.Optional[float] = None,
|
|
142
|
-
trigger_monitoring_job: bool = False,
|
|
143
|
-
artifacts_tag: str = "",
|
|
144
|
-
default_batch_image: str = "mlrun/mlrun",
|
|
145
138
|
) -> ModelEndpoint:
|
|
146
139
|
"""
|
|
147
140
|
Write a provided inference dataset to model endpoint parquet target. If not exist, generate a new model endpoint
|
|
@@ -166,47 +159,10 @@ def record_results(
|
|
|
166
159
|
the current model endpoint.
|
|
167
160
|
:param monitoring_mode: If enabled, apply model monitoring features on the provided endpoint id. Enabled
|
|
168
161
|
by default.
|
|
169
|
-
:param drift_threshold: (deprecated) The threshold of which to mark drifts.
|
|
170
|
-
:param possible_drift_threshold: (deprecated) The threshold of which to mark possible drifts.
|
|
171
|
-
:param trigger_monitoring_job: (deprecated) If true, run the batch drift job. If not exists, the monitoring
|
|
172
|
-
batch function will be registered through MLRun API with the provided image.
|
|
173
|
-
:param artifacts_tag: (deprecated) Tag to use for all the artifacts resulted from the function.
|
|
174
|
-
Will be relevant only if the monitoring batch job has been triggered.
|
|
175
|
-
:param default_batch_image: (deprecated) The image that will be used when registering the model monitoring
|
|
176
|
-
batch job.
|
|
177
162
|
|
|
178
163
|
:return: A ModelEndpoint object
|
|
179
164
|
"""
|
|
180
165
|
|
|
181
|
-
if drift_threshold is not None or possible_drift_threshold is not None:
|
|
182
|
-
warnings.warn(
|
|
183
|
-
"Custom drift threshold arguments are deprecated since version "
|
|
184
|
-
"1.7.0 and have no effect. They will be removed in version 1.9.0.\n"
|
|
185
|
-
"To enable the default histogram data drift application, run:\n"
|
|
186
|
-
"`project.enable_model_monitoring()`.",
|
|
187
|
-
FutureWarning,
|
|
188
|
-
)
|
|
189
|
-
if trigger_monitoring_job is not False:
|
|
190
|
-
warnings.warn(
|
|
191
|
-
"`trigger_monitoring_job` argument is deprecated since version "
|
|
192
|
-
"1.7.0 and has no effect. It will be removed in version 1.9.0.\n"
|
|
193
|
-
"To enable the default histogram data drift application, run:\n"
|
|
194
|
-
"`project.enable_model_monitoring()`.",
|
|
195
|
-
FutureWarning,
|
|
196
|
-
)
|
|
197
|
-
if artifacts_tag != "":
|
|
198
|
-
warnings.warn(
|
|
199
|
-
"`artifacts_tag` argument is deprecated since version "
|
|
200
|
-
"1.7.0 and has no effect. It will be removed in version 1.9.0.",
|
|
201
|
-
FutureWarning,
|
|
202
|
-
)
|
|
203
|
-
if default_batch_image != "mlrun/mlrun":
|
|
204
|
-
warnings.warn(
|
|
205
|
-
"`default_batch_image` argument is deprecated since version "
|
|
206
|
-
"1.7.0 and has no effect. It will be removed in version 1.9.0.",
|
|
207
|
-
FutureWarning,
|
|
208
|
-
)
|
|
209
|
-
|
|
210
166
|
db = mlrun.get_run_db()
|
|
211
167
|
|
|
212
168
|
model_endpoint = get_or_create_model_endpoint(
|
|
@@ -12,21 +12,18 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
import json
|
|
16
|
-
import posixpath
|
|
17
|
-
import uuid
|
|
18
15
|
import warnings
|
|
19
16
|
from abc import ABC
|
|
17
|
+
from tempfile import NamedTemporaryFile
|
|
18
|
+
from typing import Optional
|
|
20
19
|
|
|
21
|
-
import pandas as pd
|
|
22
20
|
import semver
|
|
23
|
-
from evidently.ui.storage.local.base import METADATA_PATH, FSLocation
|
|
24
21
|
|
|
25
22
|
import mlrun.model_monitoring.applications.base as mm_base
|
|
26
23
|
import mlrun.model_monitoring.applications.context as mm_context
|
|
27
|
-
from mlrun.errors import MLRunIncompatibleVersionError
|
|
24
|
+
from mlrun.errors import MLRunIncompatibleVersionError, MLRunValueError
|
|
28
25
|
|
|
29
|
-
SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.
|
|
26
|
+
SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.7.5")
|
|
30
27
|
|
|
31
28
|
|
|
32
29
|
def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
|
|
@@ -60,75 +57,69 @@ except ModuleNotFoundError:
|
|
|
60
57
|
|
|
61
58
|
|
|
62
59
|
if _HAS_EVIDENTLY:
|
|
63
|
-
from evidently.
|
|
64
|
-
from evidently.ui.
|
|
65
|
-
|
|
66
|
-
|
|
60
|
+
from evidently.core.report import Snapshot
|
|
61
|
+
from evidently.ui.workspace import (
|
|
62
|
+
STR_UUID,
|
|
63
|
+
CloudWorkspace,
|
|
64
|
+
Project,
|
|
65
|
+
Workspace,
|
|
66
|
+
WorkspaceBase,
|
|
67
|
+
)
|
|
67
68
|
|
|
68
69
|
|
|
69
70
|
class EvidentlyModelMonitoringApplicationBase(
|
|
70
71
|
mm_base.ModelMonitoringApplicationBase, ABC
|
|
71
72
|
):
|
|
72
73
|
def __init__(
|
|
73
|
-
self,
|
|
74
|
+
self,
|
|
75
|
+
evidently_project_id: "STR_UUID",
|
|
76
|
+
evidently_workspace_path: Optional[str] = None,
|
|
77
|
+
cloud_workspace: bool = False,
|
|
74
78
|
) -> None:
|
|
75
79
|
"""
|
|
76
|
-
A class for integrating Evidently for
|
|
77
|
-
|
|
78
|
-
|
|
80
|
+
A class for integrating Evidently for MLRun model monitoring within a monitoring application.
|
|
81
|
+
|
|
82
|
+
.. note::
|
|
83
|
+
|
|
84
|
+
The ``evidently`` package is not installed by default in the mlrun/mlrun image.
|
|
85
|
+
It must be installed separately to use this class.
|
|
79
86
|
|
|
80
|
-
:param evidently_workspace_path: (str) The path to the Evidently workspace.
|
|
81
87
|
:param evidently_project_id: (str) The ID of the Evidently project.
|
|
88
|
+
:param evidently_workspace_path: (str) The path to the Evidently workspace.
|
|
89
|
+
:param cloud_workspace: (bool) Whether the workspace is an Evidently Cloud workspace.
|
|
82
90
|
"""
|
|
83
|
-
|
|
84
|
-
# TODO : more then one project (mep -> project)
|
|
85
91
|
if not _HAS_EVIDENTLY:
|
|
86
92
|
raise ModuleNotFoundError("Evidently is not installed - the app cannot run")
|
|
87
|
-
self.
|
|
88
|
-
|
|
93
|
+
self.evidently_workspace_path = evidently_workspace_path
|
|
94
|
+
if cloud_workspace:
|
|
95
|
+
self.get_workspace = self.get_cloud_workspace
|
|
96
|
+
self.evidently_workspace = self.get_workspace()
|
|
89
97
|
self.evidently_project_id = evidently_project_id
|
|
90
|
-
self.evidently_project = self.
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
)
|
|
111
|
-
except FileNotFoundError:
|
|
112
|
-
print(f"evidently json issue, path not found: {full_path}")
|
|
113
|
-
continue
|
|
114
|
-
except json.decoder.JSONDecodeError as json_error:
|
|
115
|
-
print(
|
|
116
|
-
f"evidently json issue, path got json error, path:{full_path}, error: {json_error}"
|
|
117
|
-
)
|
|
118
|
-
print("evidently json issue, file content:")
|
|
119
|
-
with location.open(metadata_path) as f:
|
|
120
|
-
print(f.read())
|
|
121
|
-
continue
|
|
122
|
-
except Exception as error:
|
|
123
|
-
print(
|
|
124
|
-
f"evidently json issue, path got general error, path:{full_path}, error: {error}"
|
|
125
|
-
)
|
|
126
|
-
continue
|
|
98
|
+
self.evidently_project = self.load_project()
|
|
99
|
+
|
|
100
|
+
def load_project(self) -> "Project":
|
|
101
|
+
"""Load the Evidently project."""
|
|
102
|
+
return self.evidently_workspace.get_project(self.evidently_project_id)
|
|
103
|
+
|
|
104
|
+
def get_workspace(self) -> "WorkspaceBase":
|
|
105
|
+
"""Get the Evidently workspace. Override this method for customize access to the workspace."""
|
|
106
|
+
if self.evidently_workspace_path:
|
|
107
|
+
return Workspace.create(self.evidently_workspace_path)
|
|
108
|
+
else:
|
|
109
|
+
raise MLRunValueError(
|
|
110
|
+
"A local workspace could not be created as `evidently_workspace_path` is not set.\n"
|
|
111
|
+
"If you intend to use a cloud workspace, please use `cloud_workspace=True` and set the "
|
|
112
|
+
"`EVIDENTLY_API_KEY` environment variable. In other cases, override this method."
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
def get_cloud_workspace(self) -> "CloudWorkspace":
|
|
116
|
+
"""Load the Evidently cloud workspace according to the `EVIDENTLY_API_KEY` environment variable."""
|
|
117
|
+
return CloudWorkspace()
|
|
127
118
|
|
|
128
119
|
@staticmethod
|
|
129
120
|
def log_evidently_object(
|
|
130
121
|
monitoring_context: mm_context.MonitoringApplicationContext,
|
|
131
|
-
evidently_object: "
|
|
122
|
+
evidently_object: "Snapshot",
|
|
132
123
|
artifact_name: str,
|
|
133
124
|
unique_per_endpoint: bool = True,
|
|
134
125
|
) -> None:
|
|
@@ -141,56 +132,15 @@ class EvidentlyModelMonitoringApplicationBase(
|
|
|
141
132
|
This method should be called on special occasions only.
|
|
142
133
|
|
|
143
134
|
:param monitoring_context: (MonitoringApplicationContext) The monitoring context to process.
|
|
144
|
-
:param evidently_object: (
|
|
145
|
-
:param artifact_name: (str) The name for the logged artifact.
|
|
146
|
-
:param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
|
|
147
|
-
set to ``False`` without changing item key will cause artifact override.
|
|
148
|
-
"""
|
|
149
|
-
evidently_object_html = evidently_object.get_html()
|
|
150
|
-
monitoring_context.log_artifact(
|
|
151
|
-
artifact_name,
|
|
152
|
-
body=evidently_object_html.encode("utf-8"),
|
|
153
|
-
format="html",
|
|
154
|
-
unique_per_endpoint=unique_per_endpoint,
|
|
155
|
-
)
|
|
156
|
-
|
|
157
|
-
def log_project_dashboard(
|
|
158
|
-
self,
|
|
159
|
-
monitoring_context: mm_context.MonitoringApplicationContext,
|
|
160
|
-
timestamp_start: pd.Timestamp,
|
|
161
|
-
timestamp_end: pd.Timestamp,
|
|
162
|
-
artifact_name: str = "dashboard",
|
|
163
|
-
unique_per_endpoint: bool = True,
|
|
164
|
-
) -> None:
|
|
165
|
-
"""
|
|
166
|
-
Logs an Evidently project dashboard.
|
|
167
|
-
|
|
168
|
-
.. caution::
|
|
169
|
-
|
|
170
|
-
Logging Evidently dashboards in every model monitoring window may cause scale issues.
|
|
171
|
-
This method should be called on special occasions only.
|
|
172
|
-
|
|
173
|
-
:param monitoring_context: (MonitoringApplicationContext) The monitoring context to process.
|
|
174
|
-
:param timestamp_start: (pd.Timestamp) The start timestamp for the dashboard data.
|
|
175
|
-
:param timestamp_end: (pd.Timestamp) The end timestamp for the dashboard data.
|
|
135
|
+
:param evidently_object: (Snapshot) The Evidently run to log, e.g. a report run.
|
|
176
136
|
:param artifact_name: (str) The name for the logged artifact.
|
|
177
137
|
:param unique_per_endpoint: by default ``True``, we will log different artifact for each model endpoint,
|
|
178
138
|
set to ``False`` without changing item key will cause artifact override.
|
|
179
139
|
"""
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
additional_graphs={},
|
|
188
|
-
)
|
|
189
|
-
|
|
190
|
-
dashboard_html = file_html_template(params=template_params)
|
|
191
|
-
monitoring_context.log_artifact(
|
|
192
|
-
artifact_name,
|
|
193
|
-
body=dashboard_html.encode("utf-8"),
|
|
194
|
-
format="html",
|
|
195
|
-
unique_per_endpoint=unique_per_endpoint,
|
|
196
|
-
)
|
|
140
|
+
with NamedTemporaryFile(suffix=".html") as file:
|
|
141
|
+
evidently_object.save_html(filename=file.name)
|
|
142
|
+
monitoring_context.log_artifact(
|
|
143
|
+
artifact_name,
|
|
144
|
+
local_path=file.name,
|
|
145
|
+
unique_per_endpoint=unique_per_endpoint,
|
|
146
|
+
)
|
|
@@ -25,6 +25,7 @@ from types import TracebackType
|
|
|
25
25
|
from typing import Any, NamedTuple, Optional, Union, cast
|
|
26
26
|
|
|
27
27
|
import nuclio_sdk
|
|
28
|
+
import pandas as pd
|
|
28
29
|
|
|
29
30
|
import mlrun
|
|
30
31
|
import mlrun.common.schemas.model_monitoring.constants as mm_constants
|
|
@@ -250,7 +251,7 @@ class MonitoringApplicationController:
|
|
|
250
251
|
|
|
251
252
|
def __init__(self) -> None:
|
|
252
253
|
"""Initialize Monitoring Application Controller"""
|
|
253
|
-
self.project = cast(str, mlrun.mlconf.
|
|
254
|
+
self.project = cast(str, mlrun.mlconf.active_project)
|
|
254
255
|
self.project_obj = mlrun.get_run_db().get_project(name=self.project)
|
|
255
256
|
logger.debug(f"Initializing {self.__class__.__name__}", project=self.project)
|
|
256
257
|
|
|
@@ -673,9 +674,15 @@ class MonitoringApplicationController:
|
|
|
673
674
|
"""
|
|
674
675
|
logger.info("Starting monitoring controller chief")
|
|
675
676
|
applications_names = []
|
|
676
|
-
endpoints = self.project_obj.list_model_endpoints(
|
|
677
|
-
|
|
678
|
-
|
|
677
|
+
endpoints = self.project_obj.list_model_endpoints(tsdb_metrics=False).endpoints
|
|
678
|
+
last_request_dict = self.tsdb_connector.get_last_request(
|
|
679
|
+
endpoint_ids=[mep.metadata.uid for mep in endpoints]
|
|
680
|
+
)
|
|
681
|
+
if isinstance(last_request_dict, pd.DataFrame):
|
|
682
|
+
last_request_dict = last_request_dict.set_index(
|
|
683
|
+
mm_constants.EventFieldType.ENDPOINT_ID
|
|
684
|
+
)[mm_constants.ModelEndpointSchema.LAST_REQUEST].to_dict()
|
|
685
|
+
|
|
679
686
|
if not endpoints:
|
|
680
687
|
logger.info("No model endpoints found", project=self.project)
|
|
681
688
|
return
|
|
@@ -721,16 +728,22 @@ class MonitoringApplicationController:
|
|
|
721
728
|
with schedules.ModelMonitoringSchedulesFileChief(
|
|
722
729
|
self.project
|
|
723
730
|
) as schedule_file:
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
731
|
+
for endpoint in endpoints:
|
|
732
|
+
last_request = last_request_dict.get(endpoint.metadata.uid, None)
|
|
733
|
+
if isinstance(last_request, float):
|
|
734
|
+
last_request = pd.to_datetime(last_request, unit="s", utc=True)
|
|
735
|
+
endpoint.status.last_request = (
|
|
736
|
+
last_request or endpoint.status.last_request
|
|
737
|
+
)
|
|
738
|
+
futures = {
|
|
739
|
+
pool.submit(
|
|
740
|
+
self.endpoint_to_regular_event,
|
|
741
|
+
endpoint,
|
|
742
|
+
policy,
|
|
743
|
+
set(applications_names),
|
|
744
|
+
schedule_file,
|
|
745
|
+
): endpoint
|
|
746
|
+
}
|
|
734
747
|
for future in concurrent.futures.as_completed(futures):
|
|
735
748
|
if future.exception():
|
|
736
749
|
exception = future.exception()
|
|
@@ -455,12 +455,20 @@ class V3IOTSDBConnector(TSDBConnector):
|
|
|
455
455
|
# Delete all tables
|
|
456
456
|
tables = mm_schemas.V3IOTSDBTables.list()
|
|
457
457
|
for table_to_delete in tables:
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
458
|
+
if table_to_delete in self.tables:
|
|
459
|
+
try:
|
|
460
|
+
self.frames_client.delete(
|
|
461
|
+
backend=_TSDB_BE, table=self.tables[table_to_delete]
|
|
462
|
+
)
|
|
463
|
+
except v3io_frames.DeleteError as e:
|
|
464
|
+
logger.warning(
|
|
465
|
+
f"Failed to delete TSDB table '{table_to_delete}'",
|
|
466
|
+
err=mlrun.errors.err_to_str(e),
|
|
467
|
+
)
|
|
468
|
+
else:
|
|
461
469
|
logger.warning(
|
|
462
|
-
f"
|
|
463
|
-
|
|
470
|
+
f"Skipping deletion: table '{table_to_delete}' is not among the initialized tables.",
|
|
471
|
+
initialized_tables=list(self.tables.keys()),
|
|
464
472
|
)
|
|
465
473
|
|
|
466
474
|
# Final cleanup of tsdb path
|
mlrun/model_monitoring/writer.py
CHANGED
|
@@ -129,10 +129,7 @@ class ModelMonitoringWriter(StepToDict):
|
|
|
129
129
|
)
|
|
130
130
|
kind = event.pop(WriterEvent.EVENT_KIND, WriterEventKind.RESULT)
|
|
131
131
|
result_event = _AppResultEvent(json.loads(event.pop(WriterEvent.DATA, "{}")))
|
|
132
|
-
|
|
133
|
-
result_event = _AppResultEvent(event)
|
|
134
|
-
else:
|
|
135
|
-
result_event.update(_AppResultEvent(event))
|
|
132
|
+
result_event.update(_AppResultEvent(event))
|
|
136
133
|
|
|
137
134
|
expected_keys = list(
|
|
138
135
|
set(WriterEvent.list()).difference(
|
mlrun/projects/operations.py
CHANGED
|
@@ -309,9 +309,9 @@ def build_function(
|
|
|
309
309
|
:param force_build: Force building the image, even when no changes were made
|
|
310
310
|
"""
|
|
311
311
|
if not overwrite_build_params:
|
|
312
|
-
# TODO: change overwrite_build_params default to True in 1.
|
|
312
|
+
# TODO: change overwrite_build_params default to True in 1.10.0
|
|
313
313
|
warnings.warn(
|
|
314
|
-
"The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.
|
|
314
|
+
"The `overwrite_build_params` parameter default will change from 'False' to 'True' in 1.10.0.",
|
|
315
315
|
mlrun.utils.OverwriteBuildParamsWarning,
|
|
316
316
|
)
|
|
317
317
|
|
|
@@ -340,7 +340,7 @@ def build_function(
|
|
|
340
340
|
skip_deployed=skip_deployed,
|
|
341
341
|
)
|
|
342
342
|
else:
|
|
343
|
-
# TODO: remove filter once overwrite_build_params default is changed to True in 1.
|
|
343
|
+
# TODO: remove filter once overwrite_build_params default is changed to True in 1.10.0
|
|
344
344
|
with warnings.catch_warnings():
|
|
345
345
|
warnings.simplefilter(
|
|
346
346
|
"ignore", category=mlrun.utils.OverwriteBuildParamsWarning
|