mlrun 1.7.0rc20__py3-none-any.whl → 1.7.0rc28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +10 -8
- mlrun/alerts/alert.py +55 -18
- mlrun/api/schemas/__init__.py +3 -3
- mlrun/artifacts/manager.py +26 -0
- mlrun/common/constants.py +3 -2
- mlrun/common/formatters/__init__.py +1 -0
- mlrun/common/formatters/artifact.py +26 -3
- mlrun/common/formatters/base.py +44 -9
- mlrun/common/formatters/function.py +12 -7
- mlrun/common/formatters/run.py +26 -0
- mlrun/common/helpers.py +11 -0
- mlrun/common/schemas/__init__.py +4 -0
- mlrun/common/schemas/alert.py +5 -9
- mlrun/common/schemas/api_gateway.py +64 -16
- mlrun/common/schemas/artifact.py +11 -0
- mlrun/common/schemas/constants.py +3 -0
- mlrun/common/schemas/feature_store.py +58 -28
- mlrun/common/schemas/model_monitoring/constants.py +21 -12
- mlrun/common/schemas/model_monitoring/model_endpoints.py +0 -12
- mlrun/common/schemas/pipeline.py +16 -0
- mlrun/common/schemas/project.py +17 -0
- mlrun/common/schemas/runs.py +17 -0
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/types.py +6 -0
- mlrun/config.py +17 -25
- mlrun/datastore/azure_blob.py +2 -1
- mlrun/datastore/datastore.py +3 -3
- mlrun/datastore/google_cloud_storage.py +6 -2
- mlrun/datastore/snowflake_utils.py +3 -1
- mlrun/datastore/sources.py +26 -11
- mlrun/datastore/store_resources.py +2 -0
- mlrun/datastore/targets.py +68 -16
- mlrun/db/base.py +83 -2
- mlrun/db/httpdb.py +280 -63
- mlrun/db/nopdb.py +60 -3
- mlrun/errors.py +5 -3
- mlrun/execution.py +28 -13
- mlrun/feature_store/feature_vector.py +8 -0
- mlrun/feature_store/retrieval/spark_merger.py +13 -2
- mlrun/launcher/local.py +4 -0
- mlrun/launcher/remote.py +1 -0
- mlrun/model.py +32 -3
- mlrun/model_monitoring/api.py +7 -52
- mlrun/model_monitoring/applications/base.py +5 -7
- mlrun/model_monitoring/applications/histogram_data_drift.py +1 -1
- mlrun/model_monitoring/db/stores/__init__.py +37 -24
- mlrun/model_monitoring/db/stores/base/store.py +40 -1
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +42 -87
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +27 -35
- mlrun/model_monitoring/db/tsdb/__init__.py +15 -15
- mlrun/model_monitoring/db/tsdb/base.py +1 -14
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +22 -18
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +86 -56
- mlrun/model_monitoring/helpers.py +34 -9
- mlrun/model_monitoring/stream_processing.py +12 -11
- mlrun/model_monitoring/writer.py +11 -11
- mlrun/projects/operations.py +5 -0
- mlrun/projects/pipelines.py +35 -21
- mlrun/projects/project.py +216 -107
- mlrun/render.py +10 -5
- mlrun/run.py +15 -5
- mlrun/runtimes/__init__.py +2 -0
- mlrun/runtimes/base.py +17 -4
- mlrun/runtimes/daskjob.py +8 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +1 -0
- mlrun/runtimes/local.py +23 -4
- mlrun/runtimes/nuclio/application/application.py +0 -2
- mlrun/runtimes/nuclio/function.py +31 -2
- mlrun/runtimes/nuclio/serving.py +9 -6
- mlrun/runtimes/pod.py +5 -29
- mlrun/runtimes/remotesparkjob.py +8 -2
- mlrun/serving/__init__.py +8 -1
- mlrun/serving/routers.py +75 -59
- mlrun/serving/server.py +11 -0
- mlrun/serving/states.py +80 -8
- mlrun/serving/utils.py +19 -11
- mlrun/serving/v2_serving.py +66 -39
- mlrun/utils/helpers.py +91 -11
- mlrun/utils/logger.py +36 -2
- mlrun/utils/notifications/notification/base.py +43 -7
- mlrun/utils/notifications/notification/git.py +21 -0
- mlrun/utils/notifications/notification/slack.py +9 -14
- mlrun/utils/notifications/notification/webhook.py +41 -1
- mlrun/utils/notifications/notification_pusher.py +3 -9
- mlrun/utils/regex.py +9 -0
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/METADATA +16 -9
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/RECORD +92 -91
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc20.dist-info → mlrun-1.7.0rc28.dist-info}/top_level.txt +0 -0
mlrun/db/httpdb.py
CHANGED
|
@@ -38,6 +38,7 @@ import mlrun.model_monitoring.model_endpoint
|
|
|
38
38
|
import mlrun.platforms
|
|
39
39
|
import mlrun.projects
|
|
40
40
|
import mlrun.runtimes.nuclio.api_gateway
|
|
41
|
+
import mlrun.runtimes.nuclio.function
|
|
41
42
|
import mlrun.utils
|
|
42
43
|
from mlrun.alerts.alert import AlertConfig
|
|
43
44
|
from mlrun.db.auth_utils import OAuthClientIDTokenProvider, StaticTokenProvider
|
|
@@ -215,7 +216,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
215
216
|
:param version: API version to use, None (the default) will mean to use the default value from config,
|
|
216
217
|
for un-versioned api set an empty string.
|
|
217
218
|
|
|
218
|
-
:
|
|
219
|
+
:returns: `requests.Response` HTTP response object
|
|
219
220
|
"""
|
|
220
221
|
url = self.get_base_api_url(path, version)
|
|
221
222
|
kw = {
|
|
@@ -536,6 +537,10 @@ class HTTPRunDB(RunDBInterface):
|
|
|
536
537
|
server_cfg.get("model_monitoring_tsdb_connection")
|
|
537
538
|
or config.model_endpoint_monitoring.tsdb_connection
|
|
538
539
|
)
|
|
540
|
+
config.model_endpoint_monitoring.stream_connection = (
|
|
541
|
+
server_cfg.get("stream_connection")
|
|
542
|
+
or config.model_endpoint_monitoring.stream_connection
|
|
543
|
+
)
|
|
539
544
|
config.packagers = server_cfg.get("packagers") or config.packagers
|
|
540
545
|
server_data_prefixes = server_cfg.get("feature_store_data_prefixes") or {}
|
|
541
546
|
for prefix in ["default", "nosql", "redisnosql"]:
|
|
@@ -725,16 +730,26 @@ class HTTPRunDB(RunDBInterface):
|
|
|
725
730
|
)
|
|
726
731
|
return None
|
|
727
732
|
|
|
728
|
-
def read_run(
|
|
733
|
+
def read_run(
|
|
734
|
+
self,
|
|
735
|
+
uid,
|
|
736
|
+
project="",
|
|
737
|
+
iter=0,
|
|
738
|
+
format_: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
|
|
739
|
+
):
|
|
729
740
|
"""Read the details of a stored run from the DB.
|
|
730
741
|
|
|
731
|
-
:param uid:
|
|
732
|
-
:param project:
|
|
733
|
-
:param iter:
|
|
742
|
+
:param uid: The run's unique ID.
|
|
743
|
+
:param project: Project name.
|
|
744
|
+
:param iter: Iteration within a specific execution.
|
|
745
|
+
:param format_: The format in which to return the run details.
|
|
734
746
|
"""
|
|
735
747
|
|
|
736
748
|
path = self._path_of("runs", project, uid)
|
|
737
|
-
params = {
|
|
749
|
+
params = {
|
|
750
|
+
"iter": iter,
|
|
751
|
+
"format": format_.value,
|
|
752
|
+
}
|
|
738
753
|
error = f"get run {project}/{uid}"
|
|
739
754
|
resp = self.api_call("GET", path, error, params=params)
|
|
740
755
|
return resp.json()["data"]
|
|
@@ -860,7 +875,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
860
875
|
):
|
|
861
876
|
# default to last week on no filter
|
|
862
877
|
start_time_from = datetime.now() - timedelta(days=7)
|
|
863
|
-
partition_by = mlrun.common.schemas.RunPartitionByField.
|
|
878
|
+
partition_by = mlrun.common.schemas.RunPartitionByField.project_and_name
|
|
864
879
|
partition_sort_by = mlrun.common.schemas.SortField.updated
|
|
865
880
|
|
|
866
881
|
params = {
|
|
@@ -953,7 +968,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
953
968
|
|
|
954
969
|
# we do this because previously the 'uid' name was used for the 'tree' parameter
|
|
955
970
|
tree = tree or uid
|
|
956
|
-
|
|
971
|
+
project = project or mlrun.mlconf.default_project
|
|
957
972
|
endpoint_path = f"projects/{project}/artifacts/{key}"
|
|
958
973
|
|
|
959
974
|
error = f"store artifact {project}/{key}"
|
|
@@ -979,6 +994,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
979
994
|
project="",
|
|
980
995
|
tree=None,
|
|
981
996
|
uid=None,
|
|
997
|
+
format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
|
|
982
998
|
):
|
|
983
999
|
"""Read an artifact, identified by its key, tag, tree and iteration.
|
|
984
1000
|
|
|
@@ -988,20 +1004,20 @@ class HTTPRunDB(RunDBInterface):
|
|
|
988
1004
|
:param project: Project that the artifact belongs to.
|
|
989
1005
|
:param tree: The tree which generated this artifact.
|
|
990
1006
|
:param uid: A unique ID for this specific version of the artifact (the uid that was generated in the backend)
|
|
1007
|
+
:param format_: The format in which to return the artifact. Default is 'full'.
|
|
991
1008
|
"""
|
|
992
1009
|
|
|
993
|
-
project = project or
|
|
1010
|
+
project = project or mlrun.mlconf.default_project
|
|
994
1011
|
tag = tag or "latest"
|
|
995
1012
|
endpoint_path = f"projects/{project}/artifacts/{key}"
|
|
996
1013
|
error = f"read artifact {project}/{key}"
|
|
997
|
-
# explicitly set artifacts format to 'full' since old servers may default to 'legacy'
|
|
998
1014
|
params = {
|
|
999
|
-
"format":
|
|
1015
|
+
"format": format_,
|
|
1000
1016
|
"tag": tag,
|
|
1001
1017
|
"tree": tree,
|
|
1002
1018
|
"uid": uid,
|
|
1003
1019
|
}
|
|
1004
|
-
if iter:
|
|
1020
|
+
if iter is not None:
|
|
1005
1021
|
params["iter"] = str(iter)
|
|
1006
1022
|
resp = self.api_call("GET", endpoint_path, error, params=params, version="v2")
|
|
1007
1023
|
return resp.json()
|
|
@@ -1028,7 +1044,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1028
1044
|
:param deletion_strategy: The artifact deletion strategy types.
|
|
1029
1045
|
:param secrets: Credentials needed to access the artifact data.
|
|
1030
1046
|
"""
|
|
1031
|
-
|
|
1047
|
+
project = project or mlrun.mlconf.default_project
|
|
1032
1048
|
endpoint_path = f"projects/{project}/artifacts/{key}"
|
|
1033
1049
|
params = {
|
|
1034
1050
|
"key": key,
|
|
@@ -1061,6 +1077,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1061
1077
|
category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
|
|
1062
1078
|
tree: str = None,
|
|
1063
1079
|
producer_uri: str = None,
|
|
1080
|
+
format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
|
|
1081
|
+
limit: int = None,
|
|
1064
1082
|
) -> ArtifactList:
|
|
1065
1083
|
"""List artifacts filtered by various parameters.
|
|
1066
1084
|
|
|
@@ -1095,6 +1113,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1095
1113
|
:param producer_uri: Return artifacts produced by the requested producer URI. Producer URI usually
|
|
1096
1114
|
points to a run and is used to filter artifacts by the run that produced them when the artifact producer id
|
|
1097
1115
|
is a workflow id (artifact was created as part of a workflow).
|
|
1116
|
+
:param format_: The format in which to return the artifacts. Default is 'full'.
|
|
1117
|
+
:param limit: Maximum number of artifacts to return.
|
|
1098
1118
|
"""
|
|
1099
1119
|
|
|
1100
1120
|
project = project or config.default_project
|
|
@@ -1112,8 +1132,9 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1112
1132
|
"kind": kind,
|
|
1113
1133
|
"category": category,
|
|
1114
1134
|
"tree": tree,
|
|
1115
|
-
"format":
|
|
1135
|
+
"format": format_,
|
|
1116
1136
|
"producer_uri": producer_uri,
|
|
1137
|
+
"limit": limit,
|
|
1117
1138
|
}
|
|
1118
1139
|
error = "list artifacts"
|
|
1119
1140
|
endpoint_path = f"projects/{project}/artifacts"
|
|
@@ -1224,7 +1245,10 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1224
1245
|
== mlrun.common.schemas.BackgroundTaskState.failed
|
|
1225
1246
|
):
|
|
1226
1247
|
logger.info(
|
|
1227
|
-
"Function deletion failed",
|
|
1248
|
+
"Function deletion failed",
|
|
1249
|
+
reason=background_task.status.error,
|
|
1250
|
+
project_name=project,
|
|
1251
|
+
function_name=name,
|
|
1228
1252
|
)
|
|
1229
1253
|
|
|
1230
1254
|
def list_functions(self, name=None, project=None, tag=None, labels=None):
|
|
@@ -1526,6 +1550,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1526
1550
|
):
|
|
1527
1551
|
"""
|
|
1528
1552
|
Deploy a Nuclio function.
|
|
1553
|
+
|
|
1529
1554
|
:param func: Function to build.
|
|
1530
1555
|
:param builder_env: Kaniko builder pod env vars dict (for config/credentials)
|
|
1531
1556
|
"""
|
|
@@ -1590,20 +1615,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1590
1615
|
raise RunDBError("bad function build response")
|
|
1591
1616
|
|
|
1592
1617
|
if resp.headers:
|
|
1593
|
-
func.status.state = resp.headers.get("x-mlrun-function-status", "")
|
|
1594
1618
|
last_log_timestamp = float(
|
|
1595
1619
|
resp.headers.get("x-mlrun-last-timestamp", "0.0")
|
|
1596
1620
|
)
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
func.status.internal_invocation_urls = resp.headers.get(
|
|
1600
|
-
"x-mlrun-internal-invocation-urls", ""
|
|
1601
|
-
).split(",")
|
|
1602
|
-
func.status.external_invocation_urls = resp.headers.get(
|
|
1603
|
-
"x-mlrun-external-invocation-urls", ""
|
|
1604
|
-
).split(",")
|
|
1605
|
-
func.status.container_image = resp.headers.get(
|
|
1606
|
-
"x-mlrun-container-image", ""
|
|
1621
|
+
mlrun.runtimes.nuclio.function.enrich_nuclio_function_from_headers(
|
|
1622
|
+
func, resp.headers
|
|
1607
1623
|
)
|
|
1608
1624
|
|
|
1609
1625
|
text = ""
|
|
@@ -1661,16 +1677,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1661
1677
|
resp.headers.get("x-mlrun-last-timestamp", "0.0")
|
|
1662
1678
|
)
|
|
1663
1679
|
if func.kind in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
func.status.internal_invocation_urls = resp.headers.get(
|
|
1667
|
-
"x-mlrun-internal-invocation-urls", ""
|
|
1668
|
-
).split(",")
|
|
1669
|
-
func.status.external_invocation_urls = resp.headers.get(
|
|
1670
|
-
"x-mlrun-external-invocation-urls", ""
|
|
1671
|
-
).split(",")
|
|
1672
|
-
func.status.container_image = resp.headers.get(
|
|
1673
|
-
"x-mlrun-container-image", ""
|
|
1680
|
+
mlrun.runtimes.nuclio.function.enrich_nuclio_function_from_headers(
|
|
1681
|
+
func, resp.headers
|
|
1674
1682
|
)
|
|
1675
1683
|
|
|
1676
1684
|
builder_pod = resp.headers.get("builder_pod", "")
|
|
@@ -2106,6 +2114,41 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2106
2114
|
resp = self.api_call("GET", path, error_message, params=params)
|
|
2107
2115
|
return resp.json()["features"]
|
|
2108
2116
|
|
|
2117
|
+
def list_features_v2(
|
|
2118
|
+
self,
|
|
2119
|
+
project: str,
|
|
2120
|
+
name: str = None,
|
|
2121
|
+
tag: str = None,
|
|
2122
|
+
entities: list[str] = None,
|
|
2123
|
+
labels: list[str] = None,
|
|
2124
|
+
) -> dict[str, list[dict]]:
|
|
2125
|
+
"""List feature-sets which contain specific features. This function may return multiple versions of the same
|
|
2126
|
+
feature-set if a specific tag is not requested. Note that the various filters of this function actually
|
|
2127
|
+
refer to the feature-set object containing the features, not to the features themselves.
|
|
2128
|
+
|
|
2129
|
+
:param project: Project which contains these features.
|
|
2130
|
+
:param name: Name of the feature to look for. The name is used in a like query, and is not case-sensitive. For
|
|
2131
|
+
example, looking for ``feat`` will return features which are named ``MyFeature`` as well as ``defeat``.
|
|
2132
|
+
:param tag: Return feature-sets which contain the features looked for, and are tagged with the specific tag.
|
|
2133
|
+
:param entities: Return only feature-sets which contain an entity whose name is contained in this list.
|
|
2134
|
+
:param labels: Return only feature-sets which are labeled as requested.
|
|
2135
|
+
:returns: A list of features, and a list of their corresponding feature sets.
|
|
2136
|
+
"""
|
|
2137
|
+
|
|
2138
|
+
project = project or config.default_project
|
|
2139
|
+
params = {
|
|
2140
|
+
"name": name,
|
|
2141
|
+
"tag": tag,
|
|
2142
|
+
"entity": entities or [],
|
|
2143
|
+
"label": labels or [],
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
path = f"projects/{project}/features"
|
|
2147
|
+
|
|
2148
|
+
error_message = f"Failed listing features, project: {project}, query: {params}"
|
|
2149
|
+
resp = self.api_call("GET", path, error_message, params=params, version="v2")
|
|
2150
|
+
return resp.json()
|
|
2151
|
+
|
|
2109
2152
|
def list_entities(
|
|
2110
2153
|
self,
|
|
2111
2154
|
project: str,
|
|
@@ -2131,6 +2174,31 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2131
2174
|
resp = self.api_call("GET", path, error_message, params=params)
|
|
2132
2175
|
return resp.json()["entities"]
|
|
2133
2176
|
|
|
2177
|
+
def list_entities_v2(
|
|
2178
|
+
self,
|
|
2179
|
+
project: str,
|
|
2180
|
+
name: str = None,
|
|
2181
|
+
tag: str = None,
|
|
2182
|
+
labels: list[str] = None,
|
|
2183
|
+
) -> dict[str, list[dict]]:
|
|
2184
|
+
"""Retrieve a list of entities and their mapping to the containing feature-sets. This function is similar
|
|
2185
|
+
to the :py:func:`~list_features_v2` function, and uses the same logic. However, the entities are matched
|
|
2186
|
+
against the name rather than the features.
|
|
2187
|
+
"""
|
|
2188
|
+
|
|
2189
|
+
project = project or config.default_project
|
|
2190
|
+
params = {
|
|
2191
|
+
"name": name,
|
|
2192
|
+
"tag": tag,
|
|
2193
|
+
"label": labels or [],
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
path = f"projects/{project}/entities"
|
|
2197
|
+
|
|
2198
|
+
error_message = f"Failed listing entities, project: {project}, query: {params}"
|
|
2199
|
+
resp = self.api_call("GET", path, error_message, params=params, version="v2")
|
|
2200
|
+
return resp.json()
|
|
2201
|
+
|
|
2134
2202
|
@staticmethod
|
|
2135
2203
|
def _generate_partition_by_params(
|
|
2136
2204
|
partition_by_cls,
|
|
@@ -3225,7 +3293,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3225
3293
|
:param feature_analysis: When True, the base feature statistics and current feature statistics will
|
|
3226
3294
|
be added to the output of the resulting object.
|
|
3227
3295
|
|
|
3228
|
-
:
|
|
3296
|
+
:returns: A `ModelEndpoint` object.
|
|
3229
3297
|
"""
|
|
3230
3298
|
|
|
3231
3299
|
path = f"projects/{project}/model-endpoints/{endpoint_id}"
|
|
@@ -3316,6 +3384,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3316
3384
|
base_period: int = 10,
|
|
3317
3385
|
image: str = "mlrun/mlrun",
|
|
3318
3386
|
deploy_histogram_data_drift_app: bool = True,
|
|
3387
|
+
rebuild_images: bool = False,
|
|
3388
|
+
fetch_credentials_from_sys_config: bool = False,
|
|
3319
3389
|
) -> None:
|
|
3320
3390
|
"""
|
|
3321
3391
|
Deploy model monitoring application controller, writer and stream functions.
|
|
@@ -3325,13 +3395,16 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3325
3395
|
The stream function goal is to monitor the log of the data stream. It is triggered when a new log entry
|
|
3326
3396
|
is detected. It processes the new events into statistics that are then written to statistics databases.
|
|
3327
3397
|
|
|
3328
|
-
:param project:
|
|
3329
|
-
:param base_period:
|
|
3330
|
-
|
|
3331
|
-
:param image:
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
:param deploy_histogram_data_drift_app:
|
|
3398
|
+
:param project: Project name.
|
|
3399
|
+
:param base_period: The time period in minutes in which the model monitoring controller
|
|
3400
|
+
function triggers. By default, the base period is 10 minutes.
|
|
3401
|
+
:param image: The image of the model monitoring controller, writer & monitoring
|
|
3402
|
+
stream functions, which are real time nuclio functions.
|
|
3403
|
+
By default, the image is mlrun/mlrun.
|
|
3404
|
+
:param deploy_histogram_data_drift_app: If true, deploy the default histogram-based data drift application.
|
|
3405
|
+
:param rebuild_images: If true, force rebuild of model monitoring infrastructure images.
|
|
3406
|
+
:param fetch_credentials_from_sys_config: If true, fetch the credentials from the system configuration.
|
|
3407
|
+
|
|
3335
3408
|
"""
|
|
3336
3409
|
self.api_call(
|
|
3337
3410
|
method=mlrun.common.types.HTTPMethod.POST,
|
|
@@ -3340,9 +3413,118 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3340
3413
|
"base_period": base_period,
|
|
3341
3414
|
"image": image,
|
|
3342
3415
|
"deploy_histogram_data_drift_app": deploy_histogram_data_drift_app,
|
|
3416
|
+
"rebuild_images": rebuild_images,
|
|
3417
|
+
"fetch_credentials_from_sys_config": fetch_credentials_from_sys_config,
|
|
3343
3418
|
},
|
|
3344
3419
|
)
|
|
3345
3420
|
|
|
3421
|
+
def disable_model_monitoring(
|
|
3422
|
+
self,
|
|
3423
|
+
project: str,
|
|
3424
|
+
delete_resources: bool = True,
|
|
3425
|
+
delete_stream_function: bool = False,
|
|
3426
|
+
delete_histogram_data_drift_app: bool = True,
|
|
3427
|
+
delete_user_applications: bool = False,
|
|
3428
|
+
user_application_list: list[str] = None,
|
|
3429
|
+
) -> bool:
|
|
3430
|
+
"""
|
|
3431
|
+
Disable model monitoring application controller, writer, stream, histogram data drift application
|
|
3432
|
+
and the user's applications functions, according to the given params.
|
|
3433
|
+
|
|
3434
|
+
:param project: Project name.
|
|
3435
|
+
:param delete_resources: If True, it would delete the model monitoring controller & writer
|
|
3436
|
+
functions. Default True
|
|
3437
|
+
:param delete_stream_function: If True, it would delete model monitoring stream function,
|
|
3438
|
+
need to use wisely because if you're deleting this function
|
|
3439
|
+
this can cause data loss in case you will want to
|
|
3440
|
+
enable the model monitoring capability to the project.
|
|
3441
|
+
Default False.
|
|
3442
|
+
:param delete_histogram_data_drift_app: If True, it would delete the default histogram-based data drift
|
|
3443
|
+
application. Default False.
|
|
3444
|
+
:param delete_user_applications: If True, it would delete the user's model monitoring
|
|
3445
|
+
application according to user_application_list, Default False.
|
|
3446
|
+
:param user_application_list: List of the user's model monitoring application to disable.
|
|
3447
|
+
Default all the applications.
|
|
3448
|
+
Note: you have to set delete_user_applications to True
|
|
3449
|
+
in order to delete the desired application.
|
|
3450
|
+
|
|
3451
|
+
:returns: True if the deletion was successful, False otherwise.
|
|
3452
|
+
"""
|
|
3453
|
+
response = self.api_call(
|
|
3454
|
+
method=mlrun.common.types.HTTPMethod.DELETE,
|
|
3455
|
+
path=f"projects/{project}/model-monitoring/disable-model-monitoring",
|
|
3456
|
+
params={
|
|
3457
|
+
"delete_resources": delete_resources,
|
|
3458
|
+
"delete_stream_function": delete_stream_function,
|
|
3459
|
+
"delete_histogram_data_drift_app": delete_histogram_data_drift_app,
|
|
3460
|
+
"delete_user_applications": delete_user_applications,
|
|
3461
|
+
"user_application_list": user_application_list,
|
|
3462
|
+
},
|
|
3463
|
+
)
|
|
3464
|
+
deletion_failed = False
|
|
3465
|
+
if response.status_code == http.HTTPStatus.ACCEPTED:
|
|
3466
|
+
if delete_resources:
|
|
3467
|
+
logger.info(
|
|
3468
|
+
"Model Monitoring is being disable",
|
|
3469
|
+
project_name=project,
|
|
3470
|
+
)
|
|
3471
|
+
if delete_user_applications:
|
|
3472
|
+
logger.info("User applications are being deleted", project_name=project)
|
|
3473
|
+
background_tasks = mlrun.common.schemas.BackgroundTaskList(
|
|
3474
|
+
**response.json()
|
|
3475
|
+
).background_tasks
|
|
3476
|
+
for task in background_tasks:
|
|
3477
|
+
task = self._wait_for_background_task_to_reach_terminal_state(
|
|
3478
|
+
task.metadata.name, project=project
|
|
3479
|
+
)
|
|
3480
|
+
if (
|
|
3481
|
+
task.status.state
|
|
3482
|
+
== mlrun.common.schemas.BackgroundTaskState.succeeded
|
|
3483
|
+
):
|
|
3484
|
+
continue
|
|
3485
|
+
elif (
|
|
3486
|
+
task.status.state == mlrun.common.schemas.BackgroundTaskState.failed
|
|
3487
|
+
):
|
|
3488
|
+
deletion_failed = True
|
|
3489
|
+
return not deletion_failed
|
|
3490
|
+
|
|
3491
|
+
def delete_model_monitoring_function(
|
|
3492
|
+
self, project: str, functions: list[str]
|
|
3493
|
+
) -> bool:
|
|
3494
|
+
"""
|
|
3495
|
+
Delete a model monitoring application.
|
|
3496
|
+
|
|
3497
|
+
:param functions: List of the model monitoring function to delete.
|
|
3498
|
+
:param project: Project name.
|
|
3499
|
+
|
|
3500
|
+
:returns: True if the deletion was successful, False otherwise.
|
|
3501
|
+
"""
|
|
3502
|
+
response = self.api_call(
|
|
3503
|
+
method=mlrun.common.types.HTTPMethod.DELETE,
|
|
3504
|
+
path=f"projects/{project}/model-monitoring/functions",
|
|
3505
|
+
params={"functions": functions},
|
|
3506
|
+
)
|
|
3507
|
+
deletion_failed = False
|
|
3508
|
+
if response.status_code == http.HTTPStatus.ACCEPTED:
|
|
3509
|
+
logger.info("User applications are being deleted", project_name=project)
|
|
3510
|
+
background_tasks = mlrun.common.schemas.BackgroundTaskList(
|
|
3511
|
+
**response.json()
|
|
3512
|
+
).background_tasks
|
|
3513
|
+
for task in background_tasks:
|
|
3514
|
+
task = self._wait_for_background_task_to_reach_terminal_state(
|
|
3515
|
+
task.metadata.name, project=project
|
|
3516
|
+
)
|
|
3517
|
+
if (
|
|
3518
|
+
task.status.state
|
|
3519
|
+
== mlrun.common.schemas.BackgroundTaskState.succeeded
|
|
3520
|
+
):
|
|
3521
|
+
continue
|
|
3522
|
+
elif (
|
|
3523
|
+
task.status.state == mlrun.common.schemas.BackgroundTaskState.failed
|
|
3524
|
+
):
|
|
3525
|
+
deletion_failed = True
|
|
3526
|
+
return not deletion_failed
|
|
3527
|
+
|
|
3346
3528
|
def deploy_histogram_data_drift_app(
|
|
3347
3529
|
self, project: str, image: str = "mlrun/mlrun"
|
|
3348
3530
|
) -> None:
|
|
@@ -3358,6 +3540,23 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3358
3540
|
params={"image": image},
|
|
3359
3541
|
)
|
|
3360
3542
|
|
|
3543
|
+
def set_model_monitoring_credentials(
|
|
3544
|
+
self,
|
|
3545
|
+
project: str,
|
|
3546
|
+
credentials: dict[str, str],
|
|
3547
|
+
) -> None:
|
|
3548
|
+
"""
|
|
3549
|
+
Set the credentials for the model monitoring application.
|
|
3550
|
+
|
|
3551
|
+
:param project: Project name.
|
|
3552
|
+
:param credentials: Credentials to set.
|
|
3553
|
+
"""
|
|
3554
|
+
self.api_call(
|
|
3555
|
+
method=mlrun.common.types.HTTPMethod.POST,
|
|
3556
|
+
path=f"projects/{project}/model-monitoring/set-model-monitoring-credentials",
|
|
3557
|
+
params={**credentials},
|
|
3558
|
+
)
|
|
3559
|
+
|
|
3361
3560
|
def create_hub_source(
|
|
3362
3561
|
self, source: Union[dict, mlrun.common.schemas.IndexedHubSource]
|
|
3363
3562
|
):
|
|
@@ -3570,7 +3769,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3570
3769
|
:param version: Get a specific version of the item. Default is ``None``.
|
|
3571
3770
|
:param tag: Get a specific version of the item identified by tag. Default is ``latest``.
|
|
3572
3771
|
|
|
3573
|
-
:
|
|
3772
|
+
:returns: http response with the asset in the content attribute
|
|
3574
3773
|
"""
|
|
3575
3774
|
path = f"hub/sources/{source_name}/items/{item_name}/assets/{asset_name}"
|
|
3576
3775
|
params = {
|
|
@@ -3601,9 +3800,10 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3601
3800
|
def list_api_gateways(self, project=None) -> mlrun.common.schemas.APIGatewaysOutput:
|
|
3602
3801
|
"""
|
|
3603
3802
|
Returns a list of Nuclio api gateways
|
|
3803
|
+
|
|
3604
3804
|
:param project: optional str parameter to filter by project, if not passed, default project value is taken
|
|
3605
3805
|
|
|
3606
|
-
:
|
|
3806
|
+
:returns: :py:class:`~mlrun.common.schemas.APIGateways`.
|
|
3607
3807
|
"""
|
|
3608
3808
|
project = project or config.default_project
|
|
3609
3809
|
error = "list api gateways"
|
|
@@ -3614,10 +3814,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3614
3814
|
def get_api_gateway(self, name, project=None) -> mlrun.common.schemas.APIGateway:
|
|
3615
3815
|
"""
|
|
3616
3816
|
Returns an API gateway
|
|
3817
|
+
|
|
3617
3818
|
:param name: API gateway name
|
|
3618
3819
|
:param project: optional str parameter to filter by project, if not passed, default project value is taken
|
|
3619
3820
|
|
|
3620
|
-
:
|
|
3821
|
+
:returns: :py:class:`~mlrun.common.schemas.APIGateway`.
|
|
3621
3822
|
"""
|
|
3622
3823
|
project = project or config.default_project
|
|
3623
3824
|
error = "get api gateway"
|
|
@@ -3628,6 +3829,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3628
3829
|
def delete_api_gateway(self, name, project=None):
|
|
3629
3830
|
"""
|
|
3630
3831
|
Deletes an API gateway
|
|
3832
|
+
|
|
3631
3833
|
:param name: API gateway name
|
|
3632
3834
|
:param project: Project name
|
|
3633
3835
|
"""
|
|
@@ -3646,11 +3848,12 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3646
3848
|
) -> mlrun.common.schemas.APIGateway:
|
|
3647
3849
|
"""
|
|
3648
3850
|
Stores an API Gateway.
|
|
3649
|
-
|
|
3851
|
+
|
|
3852
|
+
:param api_gateway: :py:class:`~mlrun.runtimes.nuclio.APIGateway`
|
|
3650
3853
|
or :py:class:`~mlrun.common.schemas.APIGateway`: API Gateway entity.
|
|
3651
3854
|
:param project: project name. Mandatory if api_gateway is mlrun.common.schemas.APIGateway.
|
|
3652
3855
|
|
|
3653
|
-
:
|
|
3856
|
+
:returns: :py:class:`~mlrun.common.schemas.APIGateway`.
|
|
3654
3857
|
"""
|
|
3655
3858
|
|
|
3656
3859
|
if isinstance(api_gateway, mlrun.runtimes.nuclio.api_gateway.APIGateway):
|
|
@@ -3668,6 +3871,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3668
3871
|
def trigger_migrations(self) -> Optional[mlrun.common.schemas.BackgroundTask]:
|
|
3669
3872
|
"""Trigger migrations (will do nothing if no migrations are needed) and wait for them to finish if actually
|
|
3670
3873
|
triggered
|
|
3874
|
+
|
|
3671
3875
|
:returns: :py:class:`~mlrun.common.schemas.BackgroundTask`.
|
|
3672
3876
|
"""
|
|
3673
3877
|
response = self.api_call(
|
|
@@ -3690,6 +3894,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3690
3894
|
):
|
|
3691
3895
|
"""
|
|
3692
3896
|
Set notifications on a run. This will override any existing notifications on the run.
|
|
3897
|
+
|
|
3693
3898
|
:param project: Project containing the run.
|
|
3694
3899
|
:param run_uid: UID of the run.
|
|
3695
3900
|
:param notifications: List of notifications to set on the run. Default is an empty list.
|
|
@@ -3715,6 +3920,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3715
3920
|
):
|
|
3716
3921
|
"""
|
|
3717
3922
|
Set notifications on a schedule. This will override any existing notifications on the schedule.
|
|
3923
|
+
|
|
3718
3924
|
:param project: Project containing the schedule.
|
|
3719
3925
|
:param schedule_name: Name of the schedule.
|
|
3720
3926
|
:param notifications: List of notifications to set on the schedule. Default is an empty list.
|
|
@@ -3863,15 +4069,16 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3863
4069
|
) -> str:
|
|
3864
4070
|
"""
|
|
3865
4071
|
Loading a project remotely from the given source.
|
|
4072
|
+
|
|
3866
4073
|
:param name: project name
|
|
3867
4074
|
:param url: git or tar.gz or .zip sources archive path e.g.:
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
4075
|
+
git://github.com/mlrun/demo-xgb-project.git
|
|
4076
|
+
http://mysite/archived-project.zip
|
|
4077
|
+
The git project should include the project yaml file.
|
|
3871
4078
|
:param secrets: Secrets to store in project in order to load it from the provided url. For more
|
|
3872
|
-
|
|
4079
|
+
information see :py:func:`mlrun.load_project` function.
|
|
3873
4080
|
:param save_secrets: Whether to store secrets in the loaded project. Setting to False will cause waiting
|
|
3874
|
-
|
|
4081
|
+
for the process completion.
|
|
3875
4082
|
|
|
3876
4083
|
:returns: The terminal state of load project process.
|
|
3877
4084
|
"""
|
|
@@ -3969,9 +4176,10 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3969
4176
|
):
|
|
3970
4177
|
"""
|
|
3971
4178
|
Generate an event.
|
|
3972
|
-
|
|
4179
|
+
|
|
4180
|
+
:param name: The name of the event.
|
|
3973
4181
|
:param event_data: The data of the event.
|
|
3974
|
-
:param project:
|
|
4182
|
+
:param project: The project that the event belongs to.
|
|
3975
4183
|
"""
|
|
3976
4184
|
project = project or config.default_project
|
|
3977
4185
|
endpoint_path = f"projects/{project}/events/{name}"
|
|
@@ -3990,10 +4198,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3990
4198
|
) -> AlertConfig:
|
|
3991
4199
|
"""
|
|
3992
4200
|
Create/modify an alert.
|
|
4201
|
+
|
|
3993
4202
|
:param alert_name: The name of the alert.
|
|
3994
4203
|
:param alert_data: The data of the alert.
|
|
3995
|
-
:param project:
|
|
3996
|
-
:
|
|
4204
|
+
:param project: The project that the alert belongs to.
|
|
4205
|
+
:returns: The created/modified alert.
|
|
3997
4206
|
"""
|
|
3998
4207
|
project = project or config.default_project
|
|
3999
4208
|
endpoint_path = f"projects/{project}/alerts/{alert_name}"
|
|
@@ -4013,9 +4222,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4013
4222
|
def get_alert_config(self, alert_name: str, project="") -> AlertConfig:
|
|
4014
4223
|
"""
|
|
4015
4224
|
Retrieve an alert.
|
|
4225
|
+
|
|
4016
4226
|
:param alert_name: The name of the alert to retrieve.
|
|
4017
|
-
:param project:
|
|
4018
|
-
|
|
4227
|
+
:param project: The project that the alert belongs to.
|
|
4228
|
+
|
|
4229
|
+
:returns: The alert object.
|
|
4019
4230
|
"""
|
|
4020
4231
|
project = project or config.default_project
|
|
4021
4232
|
endpoint_path = f"projects/{project}/alerts/{alert_name}"
|
|
@@ -4026,8 +4237,10 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4026
4237
|
def list_alerts_configs(self, project="") -> list[AlertConfig]:
|
|
4027
4238
|
"""
|
|
4028
4239
|
Retrieve list of alerts of a project.
|
|
4240
|
+
|
|
4029
4241
|
:param project: The project name.
|
|
4030
|
-
|
|
4242
|
+
|
|
4243
|
+
:returns: All the alerts objects of the project.
|
|
4031
4244
|
"""
|
|
4032
4245
|
project = project or config.default_project
|
|
4033
4246
|
endpoint_path = f"projects/{project}/alerts"
|
|
@@ -4052,6 +4265,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4052
4265
|
def reset_alert_config(self, alert_name: str, project=""):
|
|
4053
4266
|
"""
|
|
4054
4267
|
Reset an alert.
|
|
4268
|
+
|
|
4055
4269
|
:param alert_name: The name of the alert to reset.
|
|
4056
4270
|
:param project: The project that the alert belongs to.
|
|
4057
4271
|
"""
|
|
@@ -4065,8 +4279,10 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4065
4279
|
) -> mlrun.common.schemas.AlertTemplate:
|
|
4066
4280
|
"""
|
|
4067
4281
|
Retrieve a specific alert template.
|
|
4282
|
+
|
|
4068
4283
|
:param template_name: The name of the template to retrieve.
|
|
4069
|
-
|
|
4284
|
+
|
|
4285
|
+
:returns: The template object.
|
|
4070
4286
|
"""
|
|
4071
4287
|
endpoint_path = f"alert-templates/{template_name}"
|
|
4072
4288
|
error_message = f"get template alert-templates/{template_name}"
|
|
@@ -4076,7 +4292,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4076
4292
|
def list_alert_templates(self) -> list[mlrun.common.schemas.AlertTemplate]:
|
|
4077
4293
|
"""
|
|
4078
4294
|
Retrieve list of all alert templates.
|
|
4079
|
-
|
|
4295
|
+
|
|
4296
|
+
:returns: All the alert template objects in the database.
|
|
4080
4297
|
"""
|
|
4081
4298
|
endpoint_path = "alert-templates"
|
|
4082
4299
|
error_message = "get templates /alert-templates"
|