mlrun 1.7.0rc20__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +10 -8
- mlrun/alerts/alert.py +55 -18
- mlrun/api/schemas/__init__.py +3 -3
- mlrun/artifacts/manager.py +26 -0
- mlrun/common/constants.py +3 -2
- mlrun/common/formatters/__init__.py +1 -0
- mlrun/common/formatters/artifact.py +26 -3
- mlrun/common/formatters/base.py +44 -9
- mlrun/common/formatters/function.py +12 -7
- mlrun/common/formatters/run.py +26 -0
- mlrun/common/helpers.py +11 -0
- mlrun/common/schemas/__init__.py +4 -0
- mlrun/common/schemas/alert.py +5 -9
- mlrun/common/schemas/api_gateway.py +64 -16
- mlrun/common/schemas/artifact.py +11 -0
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/feature_store.py +58 -28
- mlrun/common/schemas/model_monitoring/constants.py +21 -12
- mlrun/common/schemas/model_monitoring/model_endpoints.py +0 -12
- mlrun/common/schemas/pipeline.py +16 -0
- mlrun/common/schemas/project.py +17 -0
- mlrun/common/schemas/runs.py +17 -0
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/types.py +6 -0
- mlrun/config.py +17 -25
- mlrun/datastore/azure_blob.py +2 -1
- mlrun/datastore/datastore.py +3 -3
- mlrun/datastore/google_cloud_storage.py +6 -2
- mlrun/datastore/snowflake_utils.py +3 -1
- mlrun/datastore/sources.py +26 -11
- mlrun/datastore/store_resources.py +2 -0
- mlrun/datastore/targets.py +68 -16
- mlrun/db/base.py +83 -2
- mlrun/db/httpdb.py +280 -63
- mlrun/db/nopdb.py +60 -3
- mlrun/errors.py +5 -3
- mlrun/execution.py +28 -13
- mlrun/feature_store/feature_vector.py +8 -0
- mlrun/feature_store/retrieval/spark_merger.py +13 -2
- mlrun/launcher/local.py +4 -0
- mlrun/launcher/remote.py +1 -0
- mlrun/model.py +32 -3
- mlrun/model_monitoring/api.py +7 -52
- mlrun/model_monitoring/applications/base.py +5 -7
- mlrun/model_monitoring/applications/histogram_data_drift.py +1 -1
- mlrun/model_monitoring/db/stores/__init__.py +37 -24
- mlrun/model_monitoring/db/stores/base/store.py +40 -1
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +42 -87
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +27 -35
- mlrun/model_monitoring/db/tsdb/__init__.py +15 -15
- mlrun/model_monitoring/db/tsdb/base.py +1 -14
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +22 -18
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +86 -56
- mlrun/model_monitoring/helpers.py +34 -9
- mlrun/model_monitoring/stream_processing.py +12 -11
- mlrun/model_monitoring/writer.py +11 -11
- mlrun/projects/operations.py +5 -0
- mlrun/projects/pipelines.py +35 -21
- mlrun/projects/project.py +216 -107
- mlrun/render.py +10 -5
- mlrun/run.py +15 -5
- mlrun/runtimes/__init__.py +2 -0
- mlrun/runtimes/base.py +17 -4
- mlrun/runtimes/daskjob.py +8 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +1 -0
- mlrun/runtimes/local.py +23 -4
- mlrun/runtimes/nuclio/application/application.py +0 -2
- mlrun/runtimes/nuclio/function.py +31 -2
- mlrun/runtimes/nuclio/serving.py +9 -6
- mlrun/runtimes/pod.py +5 -29
- mlrun/runtimes/remotesparkjob.py +8 -2
- mlrun/serving/__init__.py +8 -1
- mlrun/serving/routers.py +75 -59
- mlrun/serving/server.py +11 -0
- mlrun/serving/states.py +80 -8
- mlrun/serving/utils.py +19 -11
- mlrun/serving/v2_serving.py +66 -39
- mlrun/utils/helpers.py +91 -11
- mlrun/utils/logger.py +36 -2
- mlrun/utils/notifications/notification/base.py +43 -7
- mlrun/utils/notifications/notification/git.py +21 -0
- mlrun/utils/notifications/notification/slack.py +9 -14
- mlrun/utils/notifications/notification/webhook.py +41 -1
- mlrun/utils/notifications/notification_pusher.py +3 -9
- mlrun/utils/regex.py +9 -0
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/METADATA +16 -9
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/RECORD +92 -91
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/top_level.txt +0 -0
mlrun/db/nopdb.py
CHANGED
|
@@ -73,7 +73,13 @@ class NopDB(RunDBInterface):
|
|
|
73
73
|
def abort_run(self, uid, project="", iter=0, timeout=45, status_text=""):
|
|
74
74
|
pass
|
|
75
75
|
|
|
76
|
-
def read_run(
|
|
76
|
+
def read_run(
|
|
77
|
+
self,
|
|
78
|
+
uid,
|
|
79
|
+
project="",
|
|
80
|
+
iter=0,
|
|
81
|
+
format_: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
|
|
82
|
+
):
|
|
77
83
|
pass
|
|
78
84
|
|
|
79
85
|
def list_runs(
|
|
@@ -115,7 +121,16 @@ class NopDB(RunDBInterface):
|
|
|
115
121
|
):
|
|
116
122
|
pass
|
|
117
123
|
|
|
118
|
-
def read_artifact(
|
|
124
|
+
def read_artifact(
|
|
125
|
+
self,
|
|
126
|
+
key,
|
|
127
|
+
tag="",
|
|
128
|
+
iter=None,
|
|
129
|
+
project="",
|
|
130
|
+
tree=None,
|
|
131
|
+
uid=None,
|
|
132
|
+
format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
|
|
133
|
+
):
|
|
119
134
|
pass
|
|
120
135
|
|
|
121
136
|
def list_artifacts(
|
|
@@ -131,6 +146,8 @@ class NopDB(RunDBInterface):
|
|
|
131
146
|
kind: str = None,
|
|
132
147
|
category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
|
|
133
148
|
tree: str = None,
|
|
149
|
+
format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
|
|
150
|
+
limit: int = None,
|
|
134
151
|
):
|
|
135
152
|
pass
|
|
136
153
|
|
|
@@ -252,11 +269,26 @@ class NopDB(RunDBInterface):
|
|
|
252
269
|
) -> mlrun.common.schemas.FeaturesOutput:
|
|
253
270
|
pass
|
|
254
271
|
|
|
272
|
+
def list_features_v2(
|
|
273
|
+
self,
|
|
274
|
+
project: str,
|
|
275
|
+
name: str = None,
|
|
276
|
+
tag: str = None,
|
|
277
|
+
entities: list[str] = None,
|
|
278
|
+
labels: list[str] = None,
|
|
279
|
+
) -> mlrun.common.schemas.FeaturesOutputV2:
|
|
280
|
+
pass
|
|
281
|
+
|
|
255
282
|
def list_entities(
|
|
256
283
|
self, project: str, name: str = None, tag: str = None, labels: list[str] = None
|
|
257
284
|
) -> mlrun.common.schemas.EntitiesOutput:
|
|
258
285
|
pass
|
|
259
286
|
|
|
287
|
+
def list_entities_v2(
|
|
288
|
+
self, project: str, name: str = None, tag: str = None, labels: list[str] = None
|
|
289
|
+
) -> mlrun.common.schemas.EntitiesOutputV2:
|
|
290
|
+
pass
|
|
291
|
+
|
|
260
292
|
def list_feature_sets(
|
|
261
293
|
self,
|
|
262
294
|
project: str = "",
|
|
@@ -675,13 +707,38 @@ class NopDB(RunDBInterface):
|
|
|
675
707
|
base_period: int = 10,
|
|
676
708
|
image: str = "mlrun/mlrun",
|
|
677
709
|
deploy_histogram_data_drift_app: bool = True,
|
|
710
|
+
rebuild_images: bool = False,
|
|
711
|
+
fetch_credentials_from_sys_config: bool = False,
|
|
678
712
|
) -> None:
|
|
679
713
|
pass
|
|
680
714
|
|
|
715
|
+
def disable_model_monitoring(
|
|
716
|
+
self,
|
|
717
|
+
project: str,
|
|
718
|
+
delete_resources: bool = True,
|
|
719
|
+
delete_stream_function: bool = False,
|
|
720
|
+
delete_histogram_data_drift_app: bool = True,
|
|
721
|
+
delete_user_applications: bool = False,
|
|
722
|
+
user_application_list: list[str] = None,
|
|
723
|
+
) -> bool:
|
|
724
|
+
pass
|
|
725
|
+
|
|
726
|
+
def delete_model_monitoring_function(
|
|
727
|
+
self, project: str, functions: list[str]
|
|
728
|
+
) -> bool:
|
|
729
|
+
pass
|
|
730
|
+
|
|
681
731
|
def deploy_histogram_data_drift_app(
|
|
682
732
|
self, project: str, image: str = "mlrun/mlrun"
|
|
683
733
|
) -> None:
|
|
684
|
-
|
|
734
|
+
pass
|
|
735
|
+
|
|
736
|
+
def set_model_monitoring_credentials(
|
|
737
|
+
self,
|
|
738
|
+
project: str,
|
|
739
|
+
credentials: dict[str, str],
|
|
740
|
+
) -> None:
|
|
741
|
+
pass
|
|
685
742
|
|
|
686
743
|
def generate_event(
|
|
687
744
|
self, name: str, event_data: Union[dict, mlrun.common.schemas.Event], project=""
|
mlrun/errors.py
CHANGED
|
@@ -92,9 +92,7 @@ def raise_for_status(
|
|
|
92
92
|
try:
|
|
93
93
|
response.raise_for_status()
|
|
94
94
|
except (requests.HTTPError, aiohttp.ClientResponseError) as exc:
|
|
95
|
-
error_message = err_to_str(exc)
|
|
96
|
-
if message:
|
|
97
|
-
error_message = f"{error_message}: {message}"
|
|
95
|
+
error_message = err_to_str(exc) if not message else message
|
|
98
96
|
status_code = (
|
|
99
97
|
response.status_code
|
|
100
98
|
if hasattr(response, "status_code")
|
|
@@ -207,6 +205,10 @@ class MLRunTimeoutError(MLRunHTTPStatusError, TimeoutError):
|
|
|
207
205
|
error_status_code = HTTPStatus.GATEWAY_TIMEOUT.value
|
|
208
206
|
|
|
209
207
|
|
|
208
|
+
class MLRunInvalidMMStoreType(MLRunHTTPStatusError, ValueError):
|
|
209
|
+
error_status_code = HTTPStatus.BAD_REQUEST.value
|
|
210
|
+
|
|
211
|
+
|
|
210
212
|
class MLRunRetryExhaustedError(Exception):
|
|
211
213
|
pass
|
|
212
214
|
|
mlrun/execution.py
CHANGED
|
@@ -34,13 +34,13 @@ from .features import Feature
|
|
|
34
34
|
from .model import HyperParamOptions
|
|
35
35
|
from .secrets import SecretsStore
|
|
36
36
|
from .utils import (
|
|
37
|
+
RunKeys,
|
|
37
38
|
dict_to_json,
|
|
38
39
|
dict_to_yaml,
|
|
39
40
|
get_in,
|
|
40
41
|
is_relative_path,
|
|
41
42
|
logger,
|
|
42
43
|
now_date,
|
|
43
|
-
run_keys,
|
|
44
44
|
to_date_str,
|
|
45
45
|
update_in,
|
|
46
46
|
)
|
|
@@ -85,6 +85,7 @@ class MLClientCtx:
|
|
|
85
85
|
|
|
86
86
|
self._labels = {}
|
|
87
87
|
self._annotations = {}
|
|
88
|
+
self._node_selector = {}
|
|
88
89
|
|
|
89
90
|
self._function = ""
|
|
90
91
|
self._parameters = {}
|
|
@@ -111,6 +112,7 @@ class MLClientCtx:
|
|
|
111
112
|
|
|
112
113
|
self._project_object = None
|
|
113
114
|
self._allow_empty_resources = None
|
|
115
|
+
self._reset_on_run = None
|
|
114
116
|
|
|
115
117
|
def __enter__(self):
|
|
116
118
|
return self
|
|
@@ -206,6 +208,11 @@ class MLClientCtx:
|
|
|
206
208
|
"""Dictionary with labels (read-only)"""
|
|
207
209
|
return deepcopy(self._labels)
|
|
208
210
|
|
|
211
|
+
@property
|
|
212
|
+
def node_selector(self):
|
|
213
|
+
"""Dictionary with node selectors (read-only)"""
|
|
214
|
+
return deepcopy(self._node_selector)
|
|
215
|
+
|
|
209
216
|
@property
|
|
210
217
|
def annotations(self):
|
|
211
218
|
"""Dictionary with annotations (read-only)"""
|
|
@@ -364,7 +371,7 @@ class MLClientCtx:
|
|
|
364
371
|
self._labels = meta.get("labels", self._labels)
|
|
365
372
|
spec = attrs.get("spec")
|
|
366
373
|
if spec:
|
|
367
|
-
self._secrets_manager = SecretsStore.from_list(spec.get(
|
|
374
|
+
self._secrets_manager = SecretsStore.from_list(spec.get(RunKeys.secrets))
|
|
368
375
|
self._log_level = spec.get("log_level", self._log_level)
|
|
369
376
|
self._function = spec.get("function", self._function)
|
|
370
377
|
self._parameters = spec.get("parameters", self._parameters)
|
|
@@ -382,13 +389,15 @@ class MLClientCtx:
|
|
|
382
389
|
self._allow_empty_resources = spec.get(
|
|
383
390
|
"allow_empty_resources", self._allow_empty_resources
|
|
384
391
|
)
|
|
385
|
-
self.artifact_path = spec.get(
|
|
386
|
-
self._in_path = spec.get(
|
|
387
|
-
inputs = spec.get(
|
|
392
|
+
self.artifact_path = spec.get(RunKeys.output_path, self.artifact_path)
|
|
393
|
+
self._in_path = spec.get(RunKeys.input_path, self._in_path)
|
|
394
|
+
inputs = spec.get(RunKeys.inputs)
|
|
388
395
|
self._notifications = spec.get("notifications", self._notifications)
|
|
389
396
|
self._state_thresholds = spec.get(
|
|
390
397
|
"state_thresholds", self._state_thresholds
|
|
391
398
|
)
|
|
399
|
+
self._node_selector = spec.get("node_selector", self._node_selector)
|
|
400
|
+
self._reset_on_run = spec.get("reset_on_run", self._reset_on_run)
|
|
392
401
|
|
|
393
402
|
self._init_dbs(rundb)
|
|
394
403
|
|
|
@@ -565,7 +574,7 @@ class MLClientCtx:
|
|
|
565
574
|
self._results["best_iteration"] = best
|
|
566
575
|
for k, v in get_in(task, ["status", "results"], {}).items():
|
|
567
576
|
self._results[k] = v
|
|
568
|
-
for artifact in get_in(task, ["status",
|
|
577
|
+
for artifact in get_in(task, ["status", RunKeys.artifacts], []):
|
|
569
578
|
self._artifacts_manager.artifacts[artifact["metadata"]["key"]] = (
|
|
570
579
|
artifact
|
|
571
580
|
)
|
|
@@ -937,10 +946,11 @@ class MLClientCtx:
|
|
|
937
946
|
"parameters": self._parameters,
|
|
938
947
|
"handler": self._handler,
|
|
939
948
|
"outputs": self._outputs,
|
|
940
|
-
|
|
941
|
-
|
|
949
|
+
RunKeys.output_path: self.artifact_path,
|
|
950
|
+
RunKeys.inputs: self._inputs,
|
|
942
951
|
"notifications": self._notifications,
|
|
943
952
|
"state_thresholds": self._state_thresholds,
|
|
953
|
+
"node_selector": self._node_selector,
|
|
944
954
|
},
|
|
945
955
|
"status": {
|
|
946
956
|
"results": self._results,
|
|
@@ -962,7 +972,7 @@ class MLClientCtx:
|
|
|
962
972
|
set_if_not_none(struct["status"], "commit", self._commit)
|
|
963
973
|
set_if_not_none(struct["status"], "iterations", self._iteration_results)
|
|
964
974
|
|
|
965
|
-
struct["status"][
|
|
975
|
+
struct["status"][RunKeys.artifacts] = self._artifacts_manager.artifact_list()
|
|
966
976
|
self._data_stores.to_dict(struct["spec"])
|
|
967
977
|
return struct
|
|
968
978
|
|
|
@@ -1039,9 +1049,14 @@ class MLClientCtx:
|
|
|
1039
1049
|
"status.last_update": to_date_str(self._last_update),
|
|
1040
1050
|
}
|
|
1041
1051
|
|
|
1042
|
-
#
|
|
1043
|
-
# multiple executions for a single run (e.g. mpi)
|
|
1044
|
-
|
|
1052
|
+
# Completion of runs is decided by the API runs monitoring as there may be
|
|
1053
|
+
# multiple executions for a single run (e.g. mpi).
|
|
1054
|
+
# For kinds that are not monitored by the API (local) we allow changing the state.
|
|
1055
|
+
run_kind = self.labels.get(mlrun_constants.MLRunInternalLabels.kind, "")
|
|
1056
|
+
if (
|
|
1057
|
+
mlrun.runtimes.RuntimeKinds.is_local_runtime(run_kind)
|
|
1058
|
+
or self._state != "completed"
|
|
1059
|
+
):
|
|
1045
1060
|
struct["status.state"] = self._state
|
|
1046
1061
|
|
|
1047
1062
|
if self.is_logging_worker():
|
|
@@ -1051,7 +1066,7 @@ class MLClientCtx:
|
|
|
1051
1066
|
set_if_not_none(struct, "status.commit", self._commit)
|
|
1052
1067
|
set_if_not_none(struct, "status.iterations", self._iteration_results)
|
|
1053
1068
|
|
|
1054
|
-
struct[f"status.{
|
|
1069
|
+
struct[f"status.{RunKeys.artifacts}"] = self._artifacts_manager.artifact_list()
|
|
1055
1070
|
return struct
|
|
1056
1071
|
|
|
1057
1072
|
def _init_dbs(self, rundb):
|
|
@@ -741,6 +741,7 @@ class FeatureVector(ModelObj):
|
|
|
741
741
|
order_by: Union[str, list[str]] = None,
|
|
742
742
|
spark_service: str = None,
|
|
743
743
|
timestamp_for_filtering: Union[str, dict[str, str]] = None,
|
|
744
|
+
additional_filters: list = None,
|
|
744
745
|
):
|
|
745
746
|
"""retrieve offline feature vector results
|
|
746
747
|
|
|
@@ -797,6 +798,12 @@ class FeatureVector(ModelObj):
|
|
|
797
798
|
By default, the filter executes on the timestamp_key of each feature set.
|
|
798
799
|
Note: the time filtering is performed on each feature set before the
|
|
799
800
|
merge process using start_time and end_time params.
|
|
801
|
+
:param additional_filters: List of additional_filter conditions as tuples.
|
|
802
|
+
Each tuple should be in the format (column_name, operator, value).
|
|
803
|
+
Supported operators: "=", ">=", "<=", ">", "<".
|
|
804
|
+
Example: [("Product", "=", "Computer")]
|
|
805
|
+
For all supported filters, please see:
|
|
806
|
+
https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetDataset.html
|
|
800
807
|
|
|
801
808
|
"""
|
|
802
809
|
|
|
@@ -817,6 +824,7 @@ class FeatureVector(ModelObj):
|
|
|
817
824
|
order_by,
|
|
818
825
|
spark_service,
|
|
819
826
|
timestamp_for_filtering,
|
|
827
|
+
additional_filters,
|
|
820
828
|
)
|
|
821
829
|
|
|
822
830
|
def get_online_feature_service(
|
|
@@ -17,7 +17,9 @@ import pandas as pd
|
|
|
17
17
|
import semver
|
|
18
18
|
|
|
19
19
|
import mlrun
|
|
20
|
+
from mlrun.datastore.sources import ParquetSource
|
|
20
21
|
from mlrun.datastore.targets import get_offline_target
|
|
22
|
+
from mlrun.utils.helpers import additional_filters_warning
|
|
21
23
|
|
|
22
24
|
from ...runtimes import RemoteSparkRuntime
|
|
23
25
|
from ...runtimes.sparkjob import Spark3Runtime
|
|
@@ -43,6 +45,7 @@ def spark_df_to_pandas(spark_df):
|
|
|
43
45
|
),
|
|
44
46
|
)
|
|
45
47
|
type_conversion_dict[field.name] = "datetime64[ns]"
|
|
48
|
+
|
|
46
49
|
df = PandasConversionMixin.toPandas(spark_df)
|
|
47
50
|
if type_conversion_dict:
|
|
48
51
|
df = df.astype(type_conversion_dict)
|
|
@@ -241,6 +244,7 @@ class SparkFeatureMerger(BaseMerger):
|
|
|
241
244
|
source_kind = feature_set.spec.source.kind
|
|
242
245
|
source_path = feature_set.spec.source.path
|
|
243
246
|
source_kwargs.update(feature_set.spec.source.attributes)
|
|
247
|
+
source_kwargs.pop("additional_filters", None)
|
|
244
248
|
else:
|
|
245
249
|
target = get_offline_target(feature_set)
|
|
246
250
|
if not target:
|
|
@@ -249,17 +253,24 @@ class SparkFeatureMerger(BaseMerger):
|
|
|
249
253
|
)
|
|
250
254
|
source_kind = target.kind
|
|
251
255
|
source_path = target.get_target_path()
|
|
252
|
-
|
|
256
|
+
source_kwargs = target.source_spark_attributes
|
|
253
257
|
# handling case where there are multiple feature sets and user creates vector where
|
|
254
258
|
# entity_timestamp_column is from a specific feature set (can't be entity timestamp)
|
|
255
259
|
source_driver = mlrun.datastore.sources.source_kind_to_driver[source_kind]
|
|
260
|
+
|
|
261
|
+
if source_driver != ParquetSource:
|
|
262
|
+
additional_filters_warning(additional_filters, source_driver)
|
|
263
|
+
additional_filters = None
|
|
264
|
+
additional_filters_dict = (
|
|
265
|
+
{"additional_filters": additional_filters} if additional_filters else {}
|
|
266
|
+
)
|
|
256
267
|
source = source_driver(
|
|
257
268
|
name=self.vector.metadata.name,
|
|
258
269
|
path=source_path,
|
|
259
270
|
time_field=time_column,
|
|
260
271
|
start_time=start_time,
|
|
261
272
|
end_time=end_time,
|
|
262
|
-
|
|
273
|
+
**additional_filters_dict,
|
|
263
274
|
**source_kwargs,
|
|
264
275
|
)
|
|
265
276
|
|
mlrun/launcher/local.py
CHANGED
|
@@ -69,6 +69,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
69
69
|
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
70
70
|
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
71
71
|
state_thresholds: Optional[dict[str, int]] = None,
|
|
72
|
+
reset_on_run: Optional[bool] = None,
|
|
72
73
|
) -> "mlrun.run.RunObject":
|
|
73
74
|
# do not allow local function to be scheduled
|
|
74
75
|
if self._is_run_local and schedule is not None:
|
|
@@ -88,6 +89,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
88
89
|
name=name,
|
|
89
90
|
workdir=workdir,
|
|
90
91
|
handler=handler,
|
|
92
|
+
reset_on_run=reset_on_run,
|
|
91
93
|
)
|
|
92
94
|
|
|
93
95
|
# sanity check
|
|
@@ -212,6 +214,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
212
214
|
name: Optional[str] = "",
|
|
213
215
|
workdir: Optional[str] = "",
|
|
214
216
|
handler: Optional[str] = None,
|
|
217
|
+
reset_on_run: Optional[bool] = None,
|
|
215
218
|
):
|
|
216
219
|
project = project or runtime.metadata.project
|
|
217
220
|
function_name = name or runtime.metadata.name
|
|
@@ -250,6 +253,7 @@ class ClientLocalLauncher(launcher.ClientBaseLauncher):
|
|
|
250
253
|
fn.spec.build = runtime.spec.build
|
|
251
254
|
|
|
252
255
|
run.spec.handler = handler
|
|
256
|
+
run.spec.reset_on_run = reset_on_run
|
|
253
257
|
return fn
|
|
254
258
|
|
|
255
259
|
@staticmethod
|
mlrun/launcher/remote.py
CHANGED
|
@@ -59,6 +59,7 @@ class ClientRemoteLauncher(launcher.ClientBaseLauncher):
|
|
|
59
59
|
notifications: Optional[list[mlrun.model.Notification]] = None,
|
|
60
60
|
returns: Optional[list[Union[str, dict[str, str]]]] = None,
|
|
61
61
|
state_thresholds: Optional[dict[str, int]] = None,
|
|
62
|
+
reset_on_run: Optional[bool] = None,
|
|
62
63
|
) -> "mlrun.run.RunObject":
|
|
63
64
|
self.enrich_runtime(runtime, project)
|
|
64
65
|
run = self._create_run_object(task)
|
mlrun/model.py
CHANGED
|
@@ -29,6 +29,7 @@ import pydantic.error_wrappers
|
|
|
29
29
|
import mlrun
|
|
30
30
|
import mlrun.common.constants as mlrun_constants
|
|
31
31
|
import mlrun.common.schemas.notification
|
|
32
|
+
import mlrun.utils.regex
|
|
32
33
|
|
|
33
34
|
from .utils import (
|
|
34
35
|
dict_to_json,
|
|
@@ -731,6 +732,25 @@ class Notification(ModelObj):
|
|
|
731
732
|
"Notification params size exceeds max size of 1 MB"
|
|
732
733
|
)
|
|
733
734
|
|
|
735
|
+
def validate_notification_params(self):
|
|
736
|
+
notification_class = mlrun.utils.notifications.NotificationTypes(
|
|
737
|
+
self.kind
|
|
738
|
+
).get_notification()
|
|
739
|
+
|
|
740
|
+
secret_params = self.secret_params
|
|
741
|
+
params = self.params
|
|
742
|
+
|
|
743
|
+
if not secret_params and not params:
|
|
744
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
745
|
+
"Both 'secret_params' and 'params' are empty, at least one must be defined."
|
|
746
|
+
)
|
|
747
|
+
if secret_params and params and secret_params != params:
|
|
748
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
749
|
+
"Both 'secret_params' and 'params' are defined but they contain different values"
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
notification_class.validate_params(secret_params or params)
|
|
753
|
+
|
|
734
754
|
@staticmethod
|
|
735
755
|
def validate_notification_uniqueness(notifications: list["Notification"]):
|
|
736
756
|
"""Validate that all notifications in the list are unique by name"""
|
|
@@ -871,6 +891,8 @@ class RunSpec(ModelObj):
|
|
|
871
891
|
returns=None,
|
|
872
892
|
notifications=None,
|
|
873
893
|
state_thresholds=None,
|
|
894
|
+
reset_on_run=None,
|
|
895
|
+
node_selector=None,
|
|
874
896
|
):
|
|
875
897
|
# A dictionary of parsing configurations that will be read from the inputs the user set. The keys are the inputs
|
|
876
898
|
# keys (parameter names) and the values are the type hint given in the input keys after the colon.
|
|
@@ -907,6 +929,8 @@ class RunSpec(ModelObj):
|
|
|
907
929
|
self.allow_empty_resources = allow_empty_resources
|
|
908
930
|
self._notifications = notifications or []
|
|
909
931
|
self.state_thresholds = state_thresholds or {}
|
|
932
|
+
self.reset_on_run = reset_on_run
|
|
933
|
+
self.node_selector = node_selector or {}
|
|
910
934
|
|
|
911
935
|
def _serialize_field(
|
|
912
936
|
self, struct: dict, field_name: str = None, strip: bool = False
|
|
@@ -1649,9 +1673,12 @@ class RunObject(RunTemplate):
|
|
|
1649
1673
|
|
|
1650
1674
|
@staticmethod
|
|
1651
1675
|
def parse_uri(uri: str) -> tuple[str, str, str, str]:
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1676
|
+
"""Parse the run's uri
|
|
1677
|
+
|
|
1678
|
+
:param uri: run uri in the format of <project>@<uid>#<iteration>[:tag]
|
|
1679
|
+
:return: project, uid, iteration, tag
|
|
1680
|
+
"""
|
|
1681
|
+
uri_pattern = mlrun.utils.regex.run_uri_pattern
|
|
1655
1682
|
match = re.match(uri_pattern, uri)
|
|
1656
1683
|
if not match:
|
|
1657
1684
|
raise ValueError(
|
|
@@ -1983,6 +2010,7 @@ class DataTarget(DataTargetBase):
|
|
|
1983
2010
|
"name",
|
|
1984
2011
|
"kind",
|
|
1985
2012
|
"path",
|
|
2013
|
+
"attributes",
|
|
1986
2014
|
"start_time",
|
|
1987
2015
|
"online",
|
|
1988
2016
|
"status",
|
|
@@ -2014,6 +2042,7 @@ class DataTarget(DataTargetBase):
|
|
|
2014
2042
|
self.last_written = None
|
|
2015
2043
|
self._producer = None
|
|
2016
2044
|
self.producer = {}
|
|
2045
|
+
self.attributes = {}
|
|
2017
2046
|
|
|
2018
2047
|
@property
|
|
2019
2048
|
def producer(self) -> FeatureSetProducer:
|
mlrun/model_monitoring/api.py
CHANGED
|
@@ -47,8 +47,8 @@ def get_or_create_model_endpoint(
|
|
|
47
47
|
function_name: str = "",
|
|
48
48
|
context: mlrun.MLClientCtx = None,
|
|
49
49
|
sample_set_statistics: dict[str, typing.Any] = None,
|
|
50
|
-
drift_threshold: float = None,
|
|
51
|
-
possible_drift_threshold: float = None,
|
|
50
|
+
drift_threshold: typing.Optional[float] = None,
|
|
51
|
+
possible_drift_threshold: typing.Optional[float] = None,
|
|
52
52
|
monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
|
|
53
53
|
db_session=None,
|
|
54
54
|
) -> ModelEndpoint:
|
|
@@ -69,14 +69,14 @@ def get_or_create_model_endpoint(
|
|
|
69
69
|
full function hash.
|
|
70
70
|
:param sample_set_statistics: Dictionary of sample set statistics that will be used as a reference data for
|
|
71
71
|
the new model endpoint (applicable only to new endpoint_id).
|
|
72
|
-
:param drift_threshold: The threshold of which to mark drifts (applicable only to new
|
|
73
|
-
|
|
72
|
+
:param drift_threshold: (deprecated) The threshold of which to mark drifts (applicable only to new
|
|
73
|
+
endpoint_id).
|
|
74
|
+
:param possible_drift_threshold: (deprecated) The threshold of which to mark possible drifts (applicable only to new
|
|
74
75
|
endpoint_id).
|
|
75
76
|
:param monitoring_mode: If enabled, apply model monitoring features on the provided endpoint id
|
|
76
77
|
(applicable only to new endpoint_id).
|
|
77
78
|
:param db_session: A runtime session that manages the current dialog with the database.
|
|
78
79
|
|
|
79
|
-
|
|
80
80
|
:return: A ModelEndpoint object
|
|
81
81
|
"""
|
|
82
82
|
|
|
@@ -98,8 +98,6 @@ def get_or_create_model_endpoint(
|
|
|
98
98
|
model_endpoint=model_endpoint,
|
|
99
99
|
model_path=model_path,
|
|
100
100
|
sample_set_statistics=sample_set_statistics,
|
|
101
|
-
drift_threshold=drift_threshold,
|
|
102
|
-
possible_drift_threshold=possible_drift_threshold,
|
|
103
101
|
)
|
|
104
102
|
|
|
105
103
|
except mlrun.errors.MLRunNotFoundError:
|
|
@@ -113,8 +111,6 @@ def get_or_create_model_endpoint(
|
|
|
113
111
|
function_name=function_name,
|
|
114
112
|
context=context,
|
|
115
113
|
sample_set_statistics=sample_set_statistics,
|
|
116
|
-
drift_threshold=drift_threshold,
|
|
117
|
-
possible_drift_threshold=possible_drift_threshold,
|
|
118
114
|
monitoring_mode=monitoring_mode,
|
|
119
115
|
)
|
|
120
116
|
return model_endpoint
|
|
@@ -241,9 +237,7 @@ def _model_endpoint_validations(
|
|
|
241
237
|
model_endpoint: ModelEndpoint,
|
|
242
238
|
model_path: str = "",
|
|
243
239
|
sample_set_statistics: dict[str, typing.Any] = None,
|
|
244
|
-
|
|
245
|
-
possible_drift_threshold: float = None,
|
|
246
|
-
):
|
|
240
|
+
) -> None:
|
|
247
241
|
"""
|
|
248
242
|
Validate that provided model endpoint configurations match the stored fields of the provided `ModelEndpoint`
|
|
249
243
|
object. Usually, this method is called by `get_or_create_model_endpoint()` in cases that the model endpoint
|
|
@@ -257,11 +251,6 @@ def _model_endpoint_validations(
|
|
|
257
251
|
is forbidden to provide a different reference data to that model endpoint.
|
|
258
252
|
In case of discrepancy between the provided `sample_set_statistics` and the
|
|
259
253
|
`model_endpoints.spec.feature_stats`, a warning will be presented to the user.
|
|
260
|
-
:param drift_threshold: The threshold of which to mark drifts. Should be similar to the drift threshold
|
|
261
|
-
that has already assigned to the current model endpoint.
|
|
262
|
-
:param possible_drift_threshold: The threshold of which to mark possible drifts. Should be similar to the possible
|
|
263
|
-
drift threshold that has already assigned to the current model endpoint.
|
|
264
|
-
|
|
265
254
|
"""
|
|
266
255
|
# Model path
|
|
267
256
|
if model_path and model_endpoint.spec.model_uri != model_path:
|
|
@@ -280,28 +269,6 @@ def _model_endpoint_validations(
|
|
|
280
269
|
"Provided sample set statistics is different from the registered statistics. "
|
|
281
270
|
"If new sample set statistics is to be used, new model endpoint should be created"
|
|
282
271
|
)
|
|
283
|
-
# drift and possible drift thresholds
|
|
284
|
-
if drift_threshold:
|
|
285
|
-
current_drift_threshold = model_endpoint.spec.monitor_configuration.get(
|
|
286
|
-
mm_constants.EventFieldType.DRIFT_DETECTED_THRESHOLD,
|
|
287
|
-
mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.drift_detected,
|
|
288
|
-
)
|
|
289
|
-
if current_drift_threshold != drift_threshold:
|
|
290
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
291
|
-
f"Cannot change existing drift threshold. Expected {current_drift_threshold}, got {drift_threshold} "
|
|
292
|
-
f"Please update drift threshold or generate a new model endpoint record"
|
|
293
|
-
)
|
|
294
|
-
|
|
295
|
-
if possible_drift_threshold:
|
|
296
|
-
current_possible_drift_threshold = model_endpoint.spec.monitor_configuration.get(
|
|
297
|
-
mm_constants.EventFieldType.POSSIBLE_DRIFT_THRESHOLD,
|
|
298
|
-
mlrun.mlconf.model_endpoint_monitoring.drift_thresholds.default.possible_drift,
|
|
299
|
-
)
|
|
300
|
-
if current_possible_drift_threshold != possible_drift_threshold:
|
|
301
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
302
|
-
f"Cannot change existing possible drift threshold. Expected {current_possible_drift_threshold}, "
|
|
303
|
-
f"got {possible_drift_threshold}. Please update drift threshold or generate a new model endpoint record"
|
|
304
|
-
)
|
|
305
272
|
|
|
306
273
|
|
|
307
274
|
def write_monitoring_df(
|
|
@@ -354,8 +321,6 @@ def _generate_model_endpoint(
|
|
|
354
321
|
function_name: str,
|
|
355
322
|
context: mlrun.MLClientCtx,
|
|
356
323
|
sample_set_statistics: dict[str, typing.Any],
|
|
357
|
-
drift_threshold: float,
|
|
358
|
-
possible_drift_threshold: float,
|
|
359
324
|
monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
|
|
360
325
|
) -> ModelEndpoint:
|
|
361
326
|
"""
|
|
@@ -374,8 +339,6 @@ def _generate_model_endpoint(
|
|
|
374
339
|
:param sample_set_statistics: Dictionary of sample set statistics that will be used as a reference data for
|
|
375
340
|
the current model endpoint. Will be stored under
|
|
376
341
|
`model_endpoint.status.feature_stats`.
|
|
377
|
-
:param drift_threshold: The threshold of which to mark drifts.
|
|
378
|
-
:param possible_drift_threshold: The threshold of which to mark possible drifts.
|
|
379
342
|
|
|
380
343
|
:return `mlrun.model_monitoring.model_endpoint.ModelEndpoint` object.
|
|
381
344
|
"""
|
|
@@ -393,15 +356,6 @@ def _generate_model_endpoint(
|
|
|
393
356
|
model_endpoint.spec.model_uri = model_path
|
|
394
357
|
model_endpoint.spec.model = model_endpoint_name
|
|
395
358
|
model_endpoint.spec.model_class = "drift-analysis"
|
|
396
|
-
if drift_threshold:
|
|
397
|
-
model_endpoint.spec.monitor_configuration[
|
|
398
|
-
mm_constants.EventFieldType.DRIFT_DETECTED_THRESHOLD
|
|
399
|
-
] = drift_threshold
|
|
400
|
-
if possible_drift_threshold:
|
|
401
|
-
model_endpoint.spec.monitor_configuration[
|
|
402
|
-
mm_constants.EventFieldType.POSSIBLE_DRIFT_THRESHOLD
|
|
403
|
-
] = possible_drift_threshold
|
|
404
|
-
|
|
405
359
|
model_endpoint.spec.monitoring_mode = monitoring_mode
|
|
406
360
|
model_endpoint.status.first_request = model_endpoint.status.last_request = (
|
|
407
361
|
datetime_now().isoformat()
|
|
@@ -645,6 +599,7 @@ def _create_model_monitoring_function_base(
|
|
|
645
599
|
app_step = prepare_step.to(class_name=application_class, **application_kwargs)
|
|
646
600
|
else:
|
|
647
601
|
app_step = prepare_step.to(class_name=application_class)
|
|
602
|
+
app_step.__class__ = mlrun.serving.MonitoringApplicationStep
|
|
648
603
|
app_step.to(
|
|
649
604
|
class_name="mlrun.model_monitoring.applications._application_steps._PushToMonitoringWriter",
|
|
650
605
|
name="PushToMonitoringWriter",
|
|
@@ -21,16 +21,16 @@ import pandas as pd
|
|
|
21
21
|
import mlrun
|
|
22
22
|
import mlrun.model_monitoring.applications.context as mm_context
|
|
23
23
|
import mlrun.model_monitoring.applications.results as mm_results
|
|
24
|
-
from mlrun.serving.utils import
|
|
24
|
+
from mlrun.serving.utils import MonitoringApplicationToDict
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
class ModelMonitoringApplicationBaseV2(
|
|
27
|
+
class ModelMonitoringApplicationBaseV2(MonitoringApplicationToDict, ABC):
|
|
28
28
|
"""
|
|
29
29
|
A base class for a model monitoring application.
|
|
30
30
|
Inherit from this class to create a custom model monitoring application.
|
|
31
31
|
|
|
32
32
|
example for very simple custom application::
|
|
33
|
-
|
|
33
|
+
|
|
34
34
|
class MyApp(ApplicationBase):
|
|
35
35
|
def do_tracking(
|
|
36
36
|
self,
|
|
@@ -49,7 +49,6 @@ class ModelMonitoringApplicationBaseV2(StepToDict, ABC):
|
|
|
49
49
|
)
|
|
50
50
|
|
|
51
51
|
|
|
52
|
-
# mlrun: end-code
|
|
53
52
|
"""
|
|
54
53
|
|
|
55
54
|
kind = "monitoring_application"
|
|
@@ -113,13 +112,13 @@ class ModelMonitoringApplicationBaseV2(StepToDict, ABC):
|
|
|
113
112
|
raise NotImplementedError
|
|
114
113
|
|
|
115
114
|
|
|
116
|
-
class ModelMonitoringApplicationBase(
|
|
115
|
+
class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
|
|
117
116
|
"""
|
|
118
117
|
A base class for a model monitoring application.
|
|
119
118
|
Inherit from this class to create a custom model monitoring application.
|
|
120
119
|
|
|
121
120
|
example for very simple custom application::
|
|
122
|
-
|
|
121
|
+
|
|
123
122
|
class MyApp(ApplicationBase):
|
|
124
123
|
def do_tracking(
|
|
125
124
|
self,
|
|
@@ -145,7 +144,6 @@ class ModelMonitoringApplicationBase(StepToDict, ABC):
|
|
|
145
144
|
)
|
|
146
145
|
|
|
147
146
|
|
|
148
|
-
# mlrun: end-code
|
|
149
147
|
"""
|
|
150
148
|
|
|
151
149
|
kind = "monitoring_application"
|
|
@@ -193,7 +193,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBaseV2):
|
|
|
193
193
|
status=status,
|
|
194
194
|
extra_data={
|
|
195
195
|
EventFieldType.CURRENT_STATS: json.dumps(
|
|
196
|
-
monitoring_context.
|
|
196
|
+
monitoring_context.sample_df_stats
|
|
197
197
|
),
|
|
198
198
|
EventFieldType.DRIFT_MEASURES: metrics_per_feature.T.to_json(),
|
|
199
199
|
EventFieldType.DRIFT_STATUS: status.value,
|