mlrun 1.7.0rc43__py3-none-any.whl → 1.7.0rc46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +4 -2
- mlrun/artifacts/manager.py +3 -1
- mlrun/common/formatters/__init__.py +1 -0
- mlrun/common/formatters/feature_set.py +33 -0
- mlrun/common/schemas/__init__.py +1 -0
- mlrun/common/schemas/alert.py +11 -11
- mlrun/common/schemas/auth.py +2 -0
- mlrun/common/schemas/client_spec.py +0 -1
- mlrun/common/schemas/model_monitoring/__init__.py +1 -0
- mlrun/common/schemas/workflow.py +1 -0
- mlrun/config.py +28 -21
- mlrun/data_types/data_types.py +5 -0
- mlrun/datastore/base.py +4 -4
- mlrun/datastore/s3.py +12 -9
- mlrun/datastore/storeytargets.py +2 -2
- mlrun/db/base.py +3 -0
- mlrun/db/httpdb.py +17 -12
- mlrun/db/nopdb.py +24 -4
- mlrun/execution.py +3 -1
- mlrun/feature_store/api.py +1 -0
- mlrun/feature_store/retrieval/spark_merger.py +7 -3
- mlrun/frameworks/_common/plan.py +3 -3
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/parallel_coordinates.py +2 -3
- mlrun/launcher/client.py +6 -6
- mlrun/model_monitoring/applications/results.py +4 -4
- mlrun/model_monitoring/controller.py +1 -1
- mlrun/model_monitoring/db/stores/sqldb/sql_store.py +15 -1
- mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +12 -0
- mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +7 -7
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +13 -12
- mlrun/model_monitoring/helpers.py +7 -8
- mlrun/model_monitoring/writer.py +3 -1
- mlrun/projects/pipelines.py +2 -0
- mlrun/projects/project.py +43 -19
- mlrun/render.py +3 -3
- mlrun/runtimes/daskjob.py +1 -1
- mlrun/runtimes/kubejob.py +6 -6
- mlrun/runtimes/nuclio/api_gateway.py +6 -0
- mlrun/runtimes/nuclio/application/application.py +3 -3
- mlrun/runtimes/nuclio/function.py +41 -0
- mlrun/runtimes/pod.py +19 -13
- mlrun/serving/server.py +2 -0
- mlrun/utils/helpers.py +22 -16
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/METADATA +22 -22
- {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/RECORD +51 -50
- {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/WHEEL +1 -1
- {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/top_level.txt +0 -0
mlrun/__main__.py
CHANGED
|
@@ -734,9 +734,11 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
|
|
|
734
734
|
if db:
|
|
735
735
|
mlconf.dbpath = db
|
|
736
736
|
if not project:
|
|
737
|
-
|
|
737
|
+
logger.warning(
|
|
738
|
+
"Project parameter was not specified. Defaulting to 'default' project"
|
|
739
|
+
)
|
|
738
740
|
if kind.startswith("po"):
|
|
739
|
-
|
|
741
|
+
logger.warning("Unsupported, use 'get runtimes' instead")
|
|
740
742
|
return
|
|
741
743
|
|
|
742
744
|
elif kind.startswith("runtime"):
|
mlrun/artifacts/manager.py
CHANGED
|
@@ -200,7 +200,9 @@ class ArtifactManager:
|
|
|
200
200
|
:param artifact_path: The path to store the artifact.
|
|
201
201
|
If not provided, the artifact will be stored in the default artifact path.
|
|
202
202
|
:param format: The format of the artifact. (e.g. csv, json, html, etc.)
|
|
203
|
-
:param upload: Whether to upload the artifact
|
|
203
|
+
:param upload: Whether to upload the artifact to the datastore. If not provided, and the
|
|
204
|
+
`local_path` is not a directory, upload occurs by default. Directories are uploaded only when this
|
|
205
|
+
flag is explicitly set to `True`.
|
|
204
206
|
:param labels: Labels to add to the artifact.
|
|
205
207
|
:param db_key: The key to use when logging the artifact to the DB.
|
|
206
208
|
If not provided, will generate a key based on the producer name and the artifact key.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Copyright 2024 Iguazio
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
|
|
16
|
+
import typing
|
|
17
|
+
|
|
18
|
+
import mlrun.common.types
|
|
19
|
+
|
|
20
|
+
from .base import ObjectFormat
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class FeatureSetFormat(ObjectFormat, mlrun.common.types.StrEnum):
|
|
24
|
+
minimal = "minimal"
|
|
25
|
+
|
|
26
|
+
@staticmethod
|
|
27
|
+
def format_method(_format: str) -> typing.Optional[typing.Callable]:
|
|
28
|
+
return {
|
|
29
|
+
FeatureSetFormat.full: None,
|
|
30
|
+
FeatureSetFormat.minimal: FeatureSetFormat.filter_obj_method(
|
|
31
|
+
["kind", "metadata", "spec", "status.state"]
|
|
32
|
+
),
|
|
33
|
+
}[_format]
|
mlrun/common/schemas/__init__.py
CHANGED
mlrun/common/schemas/alert.py
CHANGED
|
@@ -34,17 +34,17 @@ class EventEntities(pydantic.BaseModel):
|
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
class EventKind(StrEnum):
|
|
37
|
-
DATA_DRIFT_DETECTED = "
|
|
38
|
-
DATA_DRIFT_SUSPECTED = "
|
|
39
|
-
CONCEPT_DRIFT_DETECTED = "
|
|
40
|
-
CONCEPT_DRIFT_SUSPECTED = "
|
|
41
|
-
MODEL_PERFORMANCE_DETECTED = "
|
|
42
|
-
MODEL_PERFORMANCE_SUSPECTED = "
|
|
43
|
-
SYSTEM_PERFORMANCE_DETECTED = "
|
|
44
|
-
SYSTEM_PERFORMANCE_SUSPECTED = "
|
|
45
|
-
MM_APP_ANOMALY_DETECTED = "
|
|
46
|
-
MM_APP_ANOMALY_SUSPECTED = "
|
|
47
|
-
MM_APP_FAILED = "
|
|
37
|
+
DATA_DRIFT_DETECTED = "data-drift-detected"
|
|
38
|
+
DATA_DRIFT_SUSPECTED = "data-drift-suspected"
|
|
39
|
+
CONCEPT_DRIFT_DETECTED = "concept-drift-detected"
|
|
40
|
+
CONCEPT_DRIFT_SUSPECTED = "concept-drift-suspected"
|
|
41
|
+
MODEL_PERFORMANCE_DETECTED = "model-performance-detected"
|
|
42
|
+
MODEL_PERFORMANCE_SUSPECTED = "model-performance-suspected"
|
|
43
|
+
SYSTEM_PERFORMANCE_DETECTED = "system-performance-detected"
|
|
44
|
+
SYSTEM_PERFORMANCE_SUSPECTED = "system-performance-suspected"
|
|
45
|
+
MM_APP_ANOMALY_DETECTED = "mm-app-anomaly-detected"
|
|
46
|
+
MM_APP_ANOMALY_SUSPECTED = "mm-app-anomaly-suspected"
|
|
47
|
+
MM_APP_FAILED = "mm-app-failed"
|
|
48
48
|
FAILED = "failed"
|
|
49
49
|
|
|
50
50
|
|
mlrun/common/schemas/auth.py
CHANGED
|
@@ -63,6 +63,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
|
|
|
63
63
|
event = "event"
|
|
64
64
|
datastore_profile = "datastore-profile"
|
|
65
65
|
api_gateway = "api-gateway"
|
|
66
|
+
project_summaries = "project-summaries"
|
|
66
67
|
|
|
67
68
|
def to_resource_string(
|
|
68
69
|
self,
|
|
@@ -72,6 +73,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
|
|
|
72
73
|
return {
|
|
73
74
|
# project is the resource itself, so no need for both resource_name and project_name
|
|
74
75
|
AuthorizationResourceTypes.project: "/projects/{project_name}",
|
|
76
|
+
AuthorizationResourceTypes.project_summaries: "/projects/{project_name}/project-summaries/{resource_name}",
|
|
75
77
|
AuthorizationResourceTypes.function: "/projects/{project_name}/functions/{resource_name}",
|
|
76
78
|
AuthorizationResourceTypes.artifact: "/projects/{project_name}/artifacts/{resource_name}",
|
|
77
79
|
AuthorizationResourceTypes.project_background_task: (
|
|
@@ -57,7 +57,6 @@ class ClientSpec(pydantic.BaseModel):
|
|
|
57
57
|
redis_url: typing.Optional[str]
|
|
58
58
|
redis_type: typing.Optional[str]
|
|
59
59
|
sql_url: typing.Optional[str]
|
|
60
|
-
model_endpoint_monitoring_store_type: typing.Optional[str]
|
|
61
60
|
model_endpoint_monitoring_endpoint_store_connection: typing.Optional[str]
|
|
62
61
|
model_monitoring_tsdb_connection: typing.Optional[str]
|
|
63
62
|
ce: typing.Optional[dict]
|
mlrun/common/schemas/workflow.py
CHANGED
|
@@ -32,6 +32,7 @@ class WorkflowSpec(pydantic.BaseModel):
|
|
|
32
32
|
schedule: typing.Union[str, ScheduleCronTrigger] = None
|
|
33
33
|
run_local: typing.Optional[bool] = None
|
|
34
34
|
image: typing.Optional[str] = None
|
|
35
|
+
workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
|
|
35
36
|
|
|
36
37
|
|
|
37
38
|
class WorkflowRequest(pydantic.BaseModel):
|
mlrun/config.py
CHANGED
|
@@ -539,7 +539,6 @@ default_config = {
|
|
|
539
539
|
"store_prefixes": {
|
|
540
540
|
"default": "v3io:///users/pipelines/{project}/model-endpoints/{kind}",
|
|
541
541
|
"user_space": "v3io:///projects/{project}/model-endpoints/{kind}",
|
|
542
|
-
"stream": "", # TODO: Delete in 1.9.0
|
|
543
542
|
"monitoring_application": "v3io:///users/pipelines/{project}/monitoring-apps/",
|
|
544
543
|
},
|
|
545
544
|
# Offline storage path can be either relative or a full path. This path is used for general offline data
|
|
@@ -552,7 +551,6 @@ default_config = {
|
|
|
552
551
|
"parquet_batching_max_events": 10_000,
|
|
553
552
|
"parquet_batching_timeout_secs": timedelta(minutes=1).total_seconds(),
|
|
554
553
|
# See mlrun.model_monitoring.db.stores.ObjectStoreFactory for available options
|
|
555
|
-
"store_type": "v3io-nosql", # TODO: Delete in 1.9.0
|
|
556
554
|
"endpoint_store_connection": "",
|
|
557
555
|
# See mlrun.model_monitoring.db.tsdb.ObjectTSDBFactory for available options
|
|
558
556
|
"tsdb_connection": "",
|
|
@@ -736,7 +734,7 @@ default_config = {
|
|
|
736
734
|
"grafana_url": "",
|
|
737
735
|
"alerts": {
|
|
738
736
|
# supported modes: "enabled", "disabled".
|
|
739
|
-
"mode": "
|
|
737
|
+
"mode": "disabled",
|
|
740
738
|
# maximum number of alerts we allow to be configured.
|
|
741
739
|
# user will get an error when exceeding this
|
|
742
740
|
"max_allowed": 10000,
|
|
@@ -798,7 +796,21 @@ class Config:
|
|
|
798
796
|
for key, value in cfg.items():
|
|
799
797
|
if hasattr(self, key):
|
|
800
798
|
if isinstance(value, dict):
|
|
801
|
-
|
|
799
|
+
# ignore the `skip_errors` flag here
|
|
800
|
+
# if the key does not align with what mlrun config expects it is a user
|
|
801
|
+
# input error that can lead to unexpected behavior.
|
|
802
|
+
# raise the exception to ensure configuration is loaded correctly and do not
|
|
803
|
+
# ignore any errors.
|
|
804
|
+
config_value = getattr(self, key)
|
|
805
|
+
try:
|
|
806
|
+
config_value.update(value)
|
|
807
|
+
except AttributeError as exc:
|
|
808
|
+
if not isinstance(config_value, (dict, Config)):
|
|
809
|
+
raise ValueError(
|
|
810
|
+
f"Can not update `{key}` config. "
|
|
811
|
+
f"Expected a configuration but received {type(value)}"
|
|
812
|
+
) from exc
|
|
813
|
+
raise exc
|
|
802
814
|
else:
|
|
803
815
|
try:
|
|
804
816
|
setattr(self, key, value)
|
|
@@ -1102,6 +1114,9 @@ class Config:
|
|
|
1102
1114
|
# importing here to avoid circular dependency
|
|
1103
1115
|
import mlrun.db
|
|
1104
1116
|
|
|
1117
|
+
# It ensures that SSL verification is set before establishing a connection
|
|
1118
|
+
_configure_ssl_verification(self.httpdb.http.verify)
|
|
1119
|
+
|
|
1105
1120
|
# when dbpath is set we want to connect to it which will sync configuration from it to the client
|
|
1106
1121
|
mlrun.db.get_run_db(value, force_reconnect=True)
|
|
1107
1122
|
|
|
@@ -1130,10 +1145,10 @@ class Config:
|
|
|
1130
1145
|
project: str = "",
|
|
1131
1146
|
kind: str = "",
|
|
1132
1147
|
target: str = "online",
|
|
1133
|
-
artifact_path: str = None,
|
|
1134
|
-
function_name: str = None,
|
|
1148
|
+
artifact_path: typing.Optional[str] = None,
|
|
1149
|
+
function_name: typing.Optional[str] = None,
|
|
1135
1150
|
**kwargs,
|
|
1136
|
-
) ->
|
|
1151
|
+
) -> str:
|
|
1137
1152
|
"""Get the full path from the configuration based on the provided project and kind.
|
|
1138
1153
|
|
|
1139
1154
|
:param project: Project name.
|
|
@@ -1149,8 +1164,7 @@ class Config:
|
|
|
1149
1164
|
relative artifact path will be taken from the global MLRun artifact path.
|
|
1150
1165
|
:param function_name: Application name, None for model_monitoring_stream.
|
|
1151
1166
|
|
|
1152
|
-
:return: Full configured path for the provided kind.
|
|
1153
|
-
or a list of paths in the case of the online model monitoring stream path.
|
|
1167
|
+
:return: Full configured path for the provided kind.
|
|
1154
1168
|
"""
|
|
1155
1169
|
|
|
1156
1170
|
if target != "offline":
|
|
@@ -1171,18 +1185,11 @@ class Config:
|
|
|
1171
1185
|
if function_name is None
|
|
1172
1186
|
else f"{kind}-{function_name.lower()}",
|
|
1173
1187
|
)
|
|
1174
|
-
elif kind == "stream":
|
|
1175
|
-
return
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
kind=kind,
|
|
1180
|
-
), # old stream uri (pipelines) for BC ML-6043
|
|
1181
|
-
mlrun.mlconf.model_endpoint_monitoring.store_prefixes.user_space.format(
|
|
1182
|
-
project=project,
|
|
1183
|
-
kind=kind,
|
|
1184
|
-
), # new stream uri (projects)
|
|
1185
|
-
]
|
|
1188
|
+
elif kind == "stream":
|
|
1189
|
+
return mlrun.mlconf.model_endpoint_monitoring.store_prefixes.user_space.format(
|
|
1190
|
+
project=project,
|
|
1191
|
+
kind=kind,
|
|
1192
|
+
)
|
|
1186
1193
|
else:
|
|
1187
1194
|
return mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
|
|
1188
1195
|
project=project,
|
mlrun/data_types/data_types.py
CHANGED
|
@@ -70,6 +70,11 @@ def pa_type_to_value_type(type_):
|
|
|
70
70
|
if isinstance(type_, TimestampType):
|
|
71
71
|
return ValueType.DATETIME
|
|
72
72
|
|
|
73
|
+
# pandas category type translates to pyarrow DictionaryType
|
|
74
|
+
# we need to unpack the value type (ML-7868)
|
|
75
|
+
if isinstance(type_, pyarrow.DictionaryType):
|
|
76
|
+
type_ = type_.value_type
|
|
77
|
+
|
|
73
78
|
type_map = {
|
|
74
79
|
pyarrow.bool_(): ValueType.BOOL,
|
|
75
80
|
pyarrow.int64(): ValueType.INT64,
|
mlrun/datastore/base.py
CHANGED
|
@@ -29,7 +29,7 @@ from deprecated import deprecated
|
|
|
29
29
|
import mlrun.config
|
|
30
30
|
import mlrun.errors
|
|
31
31
|
from mlrun.errors import err_to_str
|
|
32
|
-
from mlrun.utils import StorePrefix,
|
|
32
|
+
from mlrun.utils import StorePrefix, is_jupyter, logger
|
|
33
33
|
|
|
34
34
|
from .store_resources import is_store_uri, parse_store_uri
|
|
35
35
|
from .utils import filter_df_start_end_time, select_columns_from_df
|
|
@@ -619,14 +619,14 @@ class DataItem:
|
|
|
619
619
|
)
|
|
620
620
|
return df
|
|
621
621
|
|
|
622
|
-
def show(self, format=None):
|
|
622
|
+
def show(self, format: Optional[str] = None) -> None:
|
|
623
623
|
"""show the data object content in Jupyter
|
|
624
624
|
|
|
625
625
|
:param format: format to use (when there is no/wrong suffix), e.g. 'png'
|
|
626
626
|
"""
|
|
627
|
-
if not
|
|
627
|
+
if not is_jupyter:
|
|
628
628
|
logger.warning(
|
|
629
|
-
"Jupyter
|
|
629
|
+
"Jupyter was not detected. `.show()` displays only inside Jupyter."
|
|
630
630
|
)
|
|
631
631
|
return
|
|
632
632
|
|
mlrun/datastore/s3.py
CHANGED
|
@@ -36,6 +36,7 @@ class S3Store(DataStore):
|
|
|
36
36
|
|
|
37
37
|
access_key_id = self._get_secret_or_env("AWS_ACCESS_KEY_ID")
|
|
38
38
|
secret_key = self._get_secret_or_env("AWS_SECRET_ACCESS_KEY")
|
|
39
|
+
token_file = self._get_secret_or_env("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")
|
|
39
40
|
endpoint_url = self._get_secret_or_env("S3_ENDPOINT_URL")
|
|
40
41
|
force_non_anonymous = self._get_secret_or_env("S3_NON_ANONYMOUS")
|
|
41
42
|
profile_name = self._get_secret_or_env("AWS_PROFILE")
|
|
@@ -94,14 +95,15 @@ class S3Store(DataStore):
|
|
|
94
95
|
self.s3 = boto3.resource(
|
|
95
96
|
"s3", region_name=region, endpoint_url=endpoint_url
|
|
96
97
|
)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
98
|
+
if not token_file:
|
|
99
|
+
# If not using credentials, boto will still attempt to sign the requests, and will fail any operations
|
|
100
|
+
# due to no credentials found. These commands disable signing and allow anonymous mode (same as
|
|
101
|
+
# anon in the storage_options when working with fsspec).
|
|
102
|
+
from botocore.handlers import disable_signing
|
|
103
|
+
|
|
104
|
+
self.s3.meta.client.meta.events.register(
|
|
105
|
+
"choose-signer.s3.*", disable_signing
|
|
106
|
+
)
|
|
105
107
|
|
|
106
108
|
def get_spark_options(self):
|
|
107
109
|
res = {}
|
|
@@ -139,6 +141,7 @@ class S3Store(DataStore):
|
|
|
139
141
|
endpoint_url = self._get_secret_or_env("S3_ENDPOINT_URL")
|
|
140
142
|
access_key_id = self._get_secret_or_env("AWS_ACCESS_KEY_ID")
|
|
141
143
|
secret = self._get_secret_or_env("AWS_SECRET_ACCESS_KEY")
|
|
144
|
+
token_file = self._get_secret_or_env("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")
|
|
142
145
|
|
|
143
146
|
if self._temp_credentials:
|
|
144
147
|
access_key_id = self._temp_credentials["AccessKeyId"]
|
|
@@ -148,7 +151,7 @@ class S3Store(DataStore):
|
|
|
148
151
|
token = None
|
|
149
152
|
|
|
150
153
|
storage_options = dict(
|
|
151
|
-
anon=not (force_non_anonymous or (access_key_id and secret)),
|
|
154
|
+
anon=not (force_non_anonymous or (access_key_id and secret) or token_file),
|
|
152
155
|
key=access_key_id,
|
|
153
156
|
secret=secret,
|
|
154
157
|
token=token,
|
mlrun/datastore/storeytargets.py
CHANGED
|
@@ -89,8 +89,8 @@ class StreamStoreyTarget(storey.StreamTarget):
|
|
|
89
89
|
raise mlrun.errors.MLRunInvalidArgumentError("StreamTarget requires a path")
|
|
90
90
|
|
|
91
91
|
access_key = storage_options.get("v3io_access_key")
|
|
92
|
-
storage = (
|
|
93
|
-
|
|
92
|
+
storage = V3ioDriver(
|
|
93
|
+
webapi=endpoint or mlrun.mlconf.v3io_api, access_key=access_key
|
|
94
94
|
)
|
|
95
95
|
|
|
96
96
|
if storage_options:
|
mlrun/db/base.py
CHANGED
|
@@ -395,6 +395,9 @@ class RunDBInterface(ABC):
|
|
|
395
395
|
partition_order: Union[
|
|
396
396
|
mlrun.common.schemas.OrderType, str
|
|
397
397
|
] = mlrun.common.schemas.OrderType.desc,
|
|
398
|
+
format_: Union[
|
|
399
|
+
str, mlrun.common.formatters.FeatureSetFormat
|
|
400
|
+
] = mlrun.common.formatters.FeatureSetFormat.full,
|
|
398
401
|
) -> list[dict]:
|
|
399
402
|
pass
|
|
400
403
|
|
mlrun/db/httpdb.py
CHANGED
|
@@ -525,10 +525,6 @@ class HTTPRunDB(RunDBInterface):
|
|
|
525
525
|
server_cfg.get("external_platform_tracking")
|
|
526
526
|
or config.external_platform_tracking
|
|
527
527
|
)
|
|
528
|
-
config.model_endpoint_monitoring.store_type = (
|
|
529
|
-
server_cfg.get("model_endpoint_monitoring_store_type")
|
|
530
|
-
or config.model_endpoint_monitoring.store_type
|
|
531
|
-
)
|
|
532
528
|
config.model_endpoint_monitoring.endpoint_store_connection = (
|
|
533
529
|
server_cfg.get("model_endpoint_monitoring_endpoint_store_connection")
|
|
534
530
|
or config.model_endpoint_monitoring.endpoint_store_connection
|
|
@@ -1374,20 +1370,14 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1374
1370
|
:returns: :py:class:`~mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput` listing the runtime resources
|
|
1375
1371
|
that were removed.
|
|
1376
1372
|
"""
|
|
1377
|
-
if grace_period is None:
|
|
1378
|
-
grace_period = config.runtime_resources_deletion_grace_period
|
|
1379
|
-
logger.info(
|
|
1380
|
-
"Using default grace period for runtime resources deletion",
|
|
1381
|
-
grace_period=grace_period,
|
|
1382
|
-
)
|
|
1383
|
-
|
|
1384
1373
|
params = {
|
|
1385
1374
|
"label-selector": label_selector,
|
|
1386
1375
|
"kind": kind,
|
|
1387
1376
|
"object-id": object_id,
|
|
1388
1377
|
"force": force,
|
|
1389
|
-
"grace-period": grace_period,
|
|
1390
1378
|
}
|
|
1379
|
+
if grace_period is not None:
|
|
1380
|
+
params["grace-period"] = grace_period
|
|
1391
1381
|
error = "Failed deleting runtime resources"
|
|
1392
1382
|
project_path = project if project else "*"
|
|
1393
1383
|
response = self.api_call(
|
|
@@ -2245,6 +2235,9 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2245
2235
|
partition_order: Union[
|
|
2246
2236
|
mlrun.common.schemas.OrderType, str
|
|
2247
2237
|
] = mlrun.common.schemas.OrderType.desc,
|
|
2238
|
+
format_: Union[
|
|
2239
|
+
str, mlrun.common.formatters.FeatureSetFormat
|
|
2240
|
+
] = mlrun.common.formatters.FeatureSetFormat.full,
|
|
2248
2241
|
) -> list[FeatureSet]:
|
|
2249
2242
|
"""Retrieve a list of feature-sets matching the criteria provided.
|
|
2250
2243
|
|
|
@@ -2262,6 +2255,9 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2262
2255
|
:param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
|
|
2263
2256
|
Currently the only allowed value are `created` and `updated`.
|
|
2264
2257
|
:param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
|
|
2258
|
+
:param format_: Format of the results. Possible values are:
|
|
2259
|
+
- ``minimal`` - Return minimal feature set objects, not including stats and preview for each feature set.
|
|
2260
|
+
- ``full`` - Return full feature set objects.
|
|
2265
2261
|
:returns: List of matching :py:class:`~mlrun.feature_store.FeatureSet` objects.
|
|
2266
2262
|
"""
|
|
2267
2263
|
|
|
@@ -2274,6 +2270,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2274
2270
|
"entity": entities or [],
|
|
2275
2271
|
"feature": features or [],
|
|
2276
2272
|
"label": labels or [],
|
|
2273
|
+
"format": format_,
|
|
2277
2274
|
}
|
|
2278
2275
|
if partition_by:
|
|
2279
2276
|
params.update(
|
|
@@ -4193,6 +4190,9 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4193
4190
|
:param event_data: The data of the event.
|
|
4194
4191
|
:param project: The project that the event belongs to.
|
|
4195
4192
|
"""
|
|
4193
|
+
if mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.disabled:
|
|
4194
|
+
logger.warning("Alerts are disabled, event will not be generated")
|
|
4195
|
+
|
|
4196
4196
|
project = project or config.default_project
|
|
4197
4197
|
endpoint_path = f"projects/{project}/events/{name}"
|
|
4198
4198
|
error_message = f"post event {project}/events/{name}"
|
|
@@ -4219,6 +4219,11 @@ class HTTPRunDB(RunDBInterface):
|
|
|
4219
4219
|
if not alert_data:
|
|
4220
4220
|
raise mlrun.errors.MLRunInvalidArgumentError("Alert data must be provided")
|
|
4221
4221
|
|
|
4222
|
+
if mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.disabled:
|
|
4223
|
+
logger.warning(
|
|
4224
|
+
"Alerts are disabled, alert will still be stored but will not be triggered"
|
|
4225
|
+
)
|
|
4226
|
+
|
|
4222
4227
|
project = project or config.default_project
|
|
4223
4228
|
endpoint_path = f"projects/{project}/alerts/{alert_name}"
|
|
4224
4229
|
error_message = f"put alert {project}/alerts/{alert_name}"
|
mlrun/db/nopdb.py
CHANGED
|
@@ -21,6 +21,7 @@ import mlrun.common.formatters
|
|
|
21
21
|
import mlrun.common.runtimes.constants
|
|
22
22
|
import mlrun.common.schemas
|
|
23
23
|
import mlrun.errors
|
|
24
|
+
import mlrun.lists
|
|
24
25
|
|
|
25
26
|
from ..config import config
|
|
26
27
|
from ..utils import logger
|
|
@@ -73,6 +74,22 @@ class NopDB(RunDBInterface):
|
|
|
73
74
|
def abort_run(self, uid, project="", iter=0, timeout=45, status_text=""):
|
|
74
75
|
pass
|
|
75
76
|
|
|
77
|
+
def list_runtime_resources(
|
|
78
|
+
self,
|
|
79
|
+
project: Optional[str] = None,
|
|
80
|
+
label_selector: Optional[str] = None,
|
|
81
|
+
kind: Optional[str] = None,
|
|
82
|
+
object_id: Optional[str] = None,
|
|
83
|
+
group_by: Optional[
|
|
84
|
+
mlrun.common.schemas.ListRuntimeResourcesGroupByField
|
|
85
|
+
] = None,
|
|
86
|
+
) -> Union[
|
|
87
|
+
mlrun.common.schemas.RuntimeResourcesOutput,
|
|
88
|
+
mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
|
|
89
|
+
mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
|
|
90
|
+
]:
|
|
91
|
+
return []
|
|
92
|
+
|
|
76
93
|
def read_run(
|
|
77
94
|
self,
|
|
78
95
|
uid,
|
|
@@ -108,7 +125,7 @@ class NopDB(RunDBInterface):
|
|
|
108
125
|
max_partitions: int = 0,
|
|
109
126
|
with_notifications: bool = False,
|
|
110
127
|
):
|
|
111
|
-
|
|
128
|
+
return mlrun.lists.RunList()
|
|
112
129
|
|
|
113
130
|
def del_run(self, uid, project="", iter=0):
|
|
114
131
|
pass
|
|
@@ -149,7 +166,7 @@ class NopDB(RunDBInterface):
|
|
|
149
166
|
format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
|
|
150
167
|
limit: int = None,
|
|
151
168
|
):
|
|
152
|
-
|
|
169
|
+
return mlrun.lists.ArtifactList()
|
|
153
170
|
|
|
154
171
|
def del_artifact(
|
|
155
172
|
self,
|
|
@@ -181,7 +198,7 @@ class NopDB(RunDBInterface):
|
|
|
181
198
|
def list_functions(
|
|
182
199
|
self, name=None, project="", tag="", labels=None, since=None, until=None
|
|
183
200
|
):
|
|
184
|
-
|
|
201
|
+
return []
|
|
185
202
|
|
|
186
203
|
def tag_objects(
|
|
187
204
|
self,
|
|
@@ -309,6 +326,9 @@ class NopDB(RunDBInterface):
|
|
|
309
326
|
partition_order: Union[
|
|
310
327
|
mlrun.common.schemas.OrderType, str
|
|
311
328
|
] = mlrun.common.schemas.OrderType.desc,
|
|
329
|
+
format_: Union[
|
|
330
|
+
str, mlrun.common.formatters.FeatureSetFormat
|
|
331
|
+
] = mlrun.common.formatters.FeatureSetFormat.full,
|
|
312
332
|
) -> list[dict]:
|
|
313
333
|
pass
|
|
314
334
|
|
|
@@ -421,7 +441,7 @@ class NopDB(RunDBInterface):
|
|
|
421
441
|
] = mlrun.common.formatters.PipelineFormat.metadata_only,
|
|
422
442
|
page_size: int = None,
|
|
423
443
|
) -> mlrun.common.schemas.PipelinesOutput:
|
|
424
|
-
|
|
444
|
+
return mlrun.common.schemas.PipelinesOutput(runs=[], total_size=0)
|
|
425
445
|
|
|
426
446
|
def create_project_secrets(
|
|
427
447
|
self,
|
mlrun/execution.py
CHANGED
|
@@ -634,7 +634,9 @@ class MLClientCtx:
|
|
|
634
634
|
:param viewer: Kubeflow viewer type
|
|
635
635
|
:param target_path: Absolute target path (instead of using artifact_path + local_path)
|
|
636
636
|
:param src_path: Deprecated, use local_path
|
|
637
|
-
:param upload:
|
|
637
|
+
:param upload: Whether to upload the artifact to the datastore. If not provided, and the `local_path`
|
|
638
|
+
is not a directory, upload occurs by default. Directories are uploaded only when this
|
|
639
|
+
flag is explicitly set to `True`.
|
|
638
640
|
:param labels: A set of key/value labels to tag the artifact with
|
|
639
641
|
:param format: Optional, format to use (e.g. csv, parquet, ..)
|
|
640
642
|
:param db_key: The key to use in the artifact DB table, by default its run name + '_' + key
|
mlrun/feature_store/api.py
CHANGED
|
@@ -188,9 +188,13 @@ class SparkFeatureMerger(BaseMerger):
|
|
|
188
188
|
|
|
189
189
|
if self.spark is None:
|
|
190
190
|
# create spark context
|
|
191
|
-
self.spark =
|
|
192
|
-
|
|
193
|
-
|
|
191
|
+
self.spark = (
|
|
192
|
+
SparkSession.builder.appName(
|
|
193
|
+
f"vector-merger-{self.vector.metadata.name}"
|
|
194
|
+
)
|
|
195
|
+
.config("spark.driver.memory", "2g")
|
|
196
|
+
.getOrCreate()
|
|
197
|
+
)
|
|
194
198
|
|
|
195
199
|
def _get_engine_df(
|
|
196
200
|
self,
|
mlrun/frameworks/_common/plan.py
CHANGED
|
@@ -11,12 +11,12 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
from abc import ABC, abstractmethod
|
|
16
16
|
|
|
17
17
|
import mlrun
|
|
18
18
|
from mlrun.artifacts import Artifact
|
|
19
|
-
from mlrun.utils.helpers import
|
|
19
|
+
from mlrun.utils.helpers import is_jupyter
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
class Plan(ABC):
|
|
@@ -84,7 +84,7 @@ class Plan(ABC):
|
|
|
84
84
|
return
|
|
85
85
|
|
|
86
86
|
# Call the correct display method according to the kernel:
|
|
87
|
-
if
|
|
87
|
+
if is_jupyter:
|
|
88
88
|
self._gui_display()
|
|
89
89
|
else:
|
|
90
90
|
self._cli_display()
|
|
@@ -18,8 +18,7 @@ from typing import Union
|
|
|
18
18
|
|
|
19
19
|
import numpy as np
|
|
20
20
|
import pandas as pd
|
|
21
|
-
from IPython.
|
|
22
|
-
from IPython.display import display
|
|
21
|
+
from IPython.display import HTML, display
|
|
23
22
|
from pandas.api.types import is_numeric_dtype, is_string_dtype
|
|
24
23
|
|
|
25
24
|
import mlrun
|
|
@@ -216,7 +215,7 @@ def _show_and_export_html(html: str, show=None, filename=None, runs_list=None):
|
|
|
216
215
|
fp.write("</body></html>")
|
|
217
216
|
else:
|
|
218
217
|
fp.write(html)
|
|
219
|
-
if show or (show is None and mlrun.utils.
|
|
218
|
+
if show or (show is None and mlrun.utils.is_jupyter):
|
|
220
219
|
display(HTML(html))
|
|
221
220
|
if runs_list and len(runs_list) <= max_table_rows:
|
|
222
221
|
display(HTML(html_table))
|