mlrun 1.7.0rc41__py3-none-any.whl → 1.7.0rc43__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/artifacts/manager.py +6 -1
- mlrun/common/db/sql_session.py +3 -2
- mlrun/common/schemas/__init__.py +1 -0
- mlrun/common/schemas/api_gateway.py +6 -6
- mlrun/common/schemas/common.py +4 -4
- mlrun/common/schemas/frontend_spec.py +7 -0
- mlrun/common/schemas/notification.py +32 -5
- mlrun/config.py +19 -1
- mlrun/data_types/to_pandas.py +3 -3
- mlrun/datastore/base.py +0 -3
- mlrun/datastore/storeytargets.py +2 -1
- mlrun/datastore/targets.py +17 -4
- mlrun/errors.py +7 -4
- mlrun/execution.py +4 -1
- mlrun/feature_store/feature_vector.py +3 -1
- mlrun/feature_store/retrieval/job.py +3 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +13 -3
- mlrun/k8s_utils.py +48 -2
- mlrun/model.py +3 -2
- mlrun/model_monitoring/db/stores/__init__.py +3 -3
- mlrun/model_monitoring/db/tsdb/__init__.py +3 -3
- mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +7 -7
- mlrun/model_monitoring/helpers.py +0 -7
- mlrun/model_monitoring/writer.py +5 -1
- mlrun/package/packagers/default_packager.py +2 -2
- mlrun/projects/project.py +125 -47
- mlrun/runtimes/funcdoc.py +1 -1
- mlrun/runtimes/local.py +4 -1
- mlrun/runtimes/nuclio/application/application.py +3 -2
- mlrun/runtimes/pod.py +2 -0
- mlrun/runtimes/sparkjob/spark3job.py +5 -1
- mlrun/utils/async_http.py +1 -1
- mlrun/utils/helpers.py +17 -0
- mlrun/utils/notifications/notification/__init__.py +0 -1
- mlrun/utils/v3io_clients.py +2 -2
- mlrun/utils/version/version.json +2 -2
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/METADATA +10 -10
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/RECORD +42 -42
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/LICENSE +0 -0
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/WHEEL +0 -0
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/entry_points.txt +0 -0
- {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/top_level.txt +0 -0
mlrun/artifacts/manager.py
CHANGED
|
@@ -72,7 +72,12 @@ class ArtifactProducer:
|
|
|
72
72
|
self.inputs = {}
|
|
73
73
|
|
|
74
74
|
def get_meta(self) -> dict:
|
|
75
|
-
return {
|
|
75
|
+
return {
|
|
76
|
+
"kind": self.kind,
|
|
77
|
+
"name": self.name,
|
|
78
|
+
"tag": self.tag,
|
|
79
|
+
"owner": self.owner,
|
|
80
|
+
}
|
|
76
81
|
|
|
77
82
|
@property
|
|
78
83
|
def uid(self):
|
mlrun/common/db/sql_session.py
CHANGED
|
@@ -11,13 +11,14 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
#
|
|
15
14
|
|
|
16
15
|
|
|
17
16
|
from sqlalchemy import create_engine
|
|
18
17
|
from sqlalchemy.engine import Engine
|
|
19
18
|
from sqlalchemy.orm import Session
|
|
20
|
-
from sqlalchemy.orm import
|
|
19
|
+
from sqlalchemy.orm import (
|
|
20
|
+
sessionmaker as SessionMaker, # noqa: N812 - `sessionmaker` is a class
|
|
21
|
+
)
|
|
21
22
|
|
|
22
23
|
from mlrun.config import config
|
|
23
24
|
|
mlrun/common/schemas/__init__.py
CHANGED
|
@@ -77,7 +77,7 @@ class APIGatewaySpec(_APIGatewayBaseModel):
|
|
|
77
77
|
name: str
|
|
78
78
|
description: Optional[str]
|
|
79
79
|
path: Optional[str] = "/"
|
|
80
|
-
authenticationMode: Optional[APIGatewayAuthenticationMode] = (
|
|
80
|
+
authenticationMode: Optional[APIGatewayAuthenticationMode] = ( # noqa: N815 - for compatibility with Nuclio https://github.com/nuclio/nuclio/blob/672b8e36f9edd6e42b4685ec1d27cabae3c5f045/pkg/platform/types.go#L476
|
|
81
81
|
APIGatewayAuthenticationMode.none
|
|
82
82
|
)
|
|
83
83
|
upstreams: list[APIGatewayUpstream]
|
|
@@ -103,11 +103,11 @@ class APIGateway(_APIGatewayBaseModel):
|
|
|
103
103
|
]
|
|
104
104
|
|
|
105
105
|
def get_invoke_url(self):
|
|
106
|
-
|
|
107
|
-
self.spec.host
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
106
|
+
if self.spec.host and self.spec.path:
|
|
107
|
+
return f"{self.spec.host.rstrip('/')}/{self.spec.path.lstrip('/')}".rstrip(
|
|
108
|
+
"/"
|
|
109
|
+
)
|
|
110
|
+
return self.spec.host.rstrip("/")
|
|
111
111
|
|
|
112
112
|
def enrich_mlrun_names(self):
|
|
113
113
|
self._enrich_api_gateway_mlrun_name()
|
mlrun/common/schemas/common.py
CHANGED
|
@@ -11,16 +11,16 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
import typing
|
|
16
16
|
|
|
17
17
|
import pydantic
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class ImageBuilder(pydantic.BaseModel):
|
|
21
|
-
functionSourceCode: typing.Optional[str] = None
|
|
22
|
-
codeEntryType: typing.Optional[str] = None
|
|
23
|
-
codeEntryAttributes: typing.Optional[str] = None
|
|
21
|
+
functionSourceCode: typing.Optional[str] = None # noqa: N815
|
|
22
|
+
codeEntryType: typing.Optional[str] = None # noqa: N815
|
|
23
|
+
codeEntryAttributes: typing.Optional[str] = None # noqa: N815
|
|
24
24
|
source: typing.Optional[str] = None
|
|
25
25
|
code_origin: typing.Optional[str] = None
|
|
26
26
|
origin_filename: typing.Optional[str] = None
|
|
@@ -50,6 +50,12 @@ class FeatureFlags(pydantic.BaseModel):
|
|
|
50
50
|
preemption_nodes: PreemptionNodesFeatureFlag
|
|
51
51
|
|
|
52
52
|
|
|
53
|
+
class ArtifactLimits(pydantic.BaseModel):
|
|
54
|
+
max_chunk_size: int
|
|
55
|
+
max_preview_size: int
|
|
56
|
+
max_download_size: int
|
|
57
|
+
|
|
58
|
+
|
|
53
59
|
class FrontendSpec(pydantic.BaseModel):
|
|
54
60
|
jobs_dashboard_url: typing.Optional[str]
|
|
55
61
|
model_monitoring_dashboard_url: typing.Optional[str]
|
|
@@ -71,3 +77,4 @@ class FrontendSpec(pydantic.BaseModel):
|
|
|
71
77
|
allowed_artifact_path_prefixes_list: list[str]
|
|
72
78
|
ce: typing.Optional[dict]
|
|
73
79
|
internal_labels: list[str] = []
|
|
80
|
+
artifact_limits: ArtifactLimits
|
|
@@ -22,11 +22,38 @@ import mlrun.common.types
|
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
class NotificationKind(mlrun.common.types.StrEnum):
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
"""Currently, the supported notification kinds and their params are as follows:"""
|
|
26
|
+
|
|
27
|
+
console: str = "console"
|
|
28
|
+
"""no params, local only"""
|
|
29
|
+
|
|
30
|
+
git: str = "git"
|
|
31
|
+
"""
|
|
32
|
+
**token** - The git token to use for the git notification.\n
|
|
33
|
+
**repo** - The git repo to which to send the notification.\n
|
|
34
|
+
**issue** - The git issue to which to send the notification.\n
|
|
35
|
+
**merge_request** -
|
|
36
|
+
In GitLab (as opposed to GitHub), merge requests and issues are separate entities.
|
|
37
|
+
If using merge request, the issue will be ignored, and vice versa.\n
|
|
38
|
+
**server** - The git server to which to send the notification.\n
|
|
39
|
+
**gitlab** - (bool) Whether the git server is GitLab or not.\n
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
ipython: str = "ipython"
|
|
43
|
+
"""no params, local only"""
|
|
44
|
+
|
|
45
|
+
slack: str = "slack"
|
|
46
|
+
"""**webhook** - The slack webhook to which to send the notification."""
|
|
47
|
+
|
|
48
|
+
webhook: str = "webhook"
|
|
49
|
+
"""
|
|
50
|
+
**url** - The webhook url to which to send the notification.\n
|
|
51
|
+
**method** - The http method to use when sending the notification (GET, POST, PUT, etc…).\n
|
|
52
|
+
**headers** - (dict) The http headers to send with the notification.\n
|
|
53
|
+
**override_body** - (dict) The body to send with the notification.\n
|
|
54
|
+
**verify_ssl** -
|
|
55
|
+
(bool) Whether SSL certificates are validated during HTTP requests or not, The default is set to True.
|
|
56
|
+
"""
|
|
30
57
|
|
|
31
58
|
|
|
32
59
|
class NotificationSeverity(mlrun.common.types.StrEnum):
|
mlrun/config.py
CHANGED
|
@@ -27,6 +27,7 @@ import copy
|
|
|
27
27
|
import json
|
|
28
28
|
import os
|
|
29
29
|
import typing
|
|
30
|
+
import warnings
|
|
30
31
|
from collections.abc import Mapping
|
|
31
32
|
from datetime import timedelta
|
|
32
33
|
from distutils.util import strtobool
|
|
@@ -35,6 +36,7 @@ from threading import Lock
|
|
|
35
36
|
|
|
36
37
|
import dotenv
|
|
37
38
|
import semver
|
|
39
|
+
import urllib3.exceptions
|
|
38
40
|
import yaml
|
|
39
41
|
|
|
40
42
|
import mlrun.common.constants
|
|
@@ -152,6 +154,11 @@ default_config = {
|
|
|
152
154
|
"datasets": {
|
|
153
155
|
"max_preview_columns": 100,
|
|
154
156
|
},
|
|
157
|
+
"limits": {
|
|
158
|
+
"max_chunk_size": 1024 * 1024 * 1, # 1MB
|
|
159
|
+
"max_preview_size": 1024 * 1024 * 10, # 10MB
|
|
160
|
+
"max_download_size": 1024 * 1024 * 100, # 100MB
|
|
161
|
+
},
|
|
155
162
|
},
|
|
156
163
|
# FIXME: Adding these defaults here so we won't need to patch the "installing component" (provazio-controller) to
|
|
157
164
|
# configure this values on field systems, for newer system this will be configured correctly
|
|
@@ -326,7 +333,7 @@ default_config = {
|
|
|
326
333
|
"http": {
|
|
327
334
|
# when True, the client will verify the server's TLS
|
|
328
335
|
# set to False for backwards compatibility.
|
|
329
|
-
"verify":
|
|
336
|
+
"verify": True,
|
|
330
337
|
},
|
|
331
338
|
"db": {
|
|
332
339
|
"commit_retry_timeout": 30,
|
|
@@ -1292,6 +1299,7 @@ def _do_populate(env=None, skip_errors=False):
|
|
|
1292
1299
|
if data:
|
|
1293
1300
|
config.update(data, skip_errors=skip_errors)
|
|
1294
1301
|
|
|
1302
|
+
_configure_ssl_verification(config.httpdb.http.verify)
|
|
1295
1303
|
_validate_config(config)
|
|
1296
1304
|
|
|
1297
1305
|
|
|
@@ -1351,6 +1359,16 @@ def _convert_str(value, typ):
|
|
|
1351
1359
|
return typ(value)
|
|
1352
1360
|
|
|
1353
1361
|
|
|
1362
|
+
def _configure_ssl_verification(verify_ssl: bool) -> None:
|
|
1363
|
+
"""Configure SSL verification warnings based on the setting."""
|
|
1364
|
+
if not verify_ssl:
|
|
1365
|
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
1366
|
+
else:
|
|
1367
|
+
# If the user changes the `verify` setting to `True` at runtime using `mlrun.set_env_from_file` after
|
|
1368
|
+
# importing `mlrun`, we need to reload the `mlrun` configuration and enable this warning.
|
|
1369
|
+
warnings.simplefilter("default", urllib3.exceptions.InsecureRequestWarning)
|
|
1370
|
+
|
|
1371
|
+
|
|
1354
1372
|
def read_env(env=None, prefix=env_prefix):
|
|
1355
1373
|
"""Read configuration from environment"""
|
|
1356
1374
|
env = os.environ if env is None else env
|
mlrun/data_types/to_pandas.py
CHANGED
|
@@ -19,7 +19,7 @@ import pandas as pd
|
|
|
19
19
|
import semver
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
def
|
|
22
|
+
def _to_pandas(spark_df):
|
|
23
23
|
"""
|
|
24
24
|
Modified version of spark DataFrame.toPandas() -
|
|
25
25
|
https://github.com/apache/spark/blob/v3.2.3/python/pyspark/sql/pandas/conversion.py#L35
|
|
@@ -262,9 +262,9 @@ def spark_df_to_pandas(spark_df):
|
|
|
262
262
|
)
|
|
263
263
|
type_conversion_dict[field.name] = "datetime64[ns]"
|
|
264
264
|
|
|
265
|
-
df =
|
|
265
|
+
df = _to_pandas(spark_df)
|
|
266
266
|
if type_conversion_dict:
|
|
267
267
|
df = df.astype(type_conversion_dict)
|
|
268
268
|
return df
|
|
269
269
|
else:
|
|
270
|
-
return
|
|
270
|
+
return _to_pandas(spark_df)
|
mlrun/datastore/base.py
CHANGED
|
@@ -24,7 +24,6 @@ import pandas as pd
|
|
|
24
24
|
import pyarrow
|
|
25
25
|
import pytz
|
|
26
26
|
import requests
|
|
27
|
-
import urllib3
|
|
28
27
|
from deprecated import deprecated
|
|
29
28
|
|
|
30
29
|
import mlrun.config
|
|
@@ -745,8 +744,6 @@ class HttpStore(DataStore):
|
|
|
745
744
|
|
|
746
745
|
verify_ssl = mlconf.httpdb.http.verify
|
|
747
746
|
try:
|
|
748
|
-
if not verify_ssl:
|
|
749
|
-
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
750
747
|
response = requests.get(url, headers=headers, auth=auth, verify=verify_ssl)
|
|
751
748
|
except OSError as exc:
|
|
752
749
|
raise OSError(f"error: cannot connect to {url}: {err_to_str(exc)}")
|
mlrun/datastore/storeytargets.py
CHANGED
|
@@ -137,7 +137,8 @@ class RedisNoSqlStoreyTarget(storey.NoSqlTarget):
|
|
|
137
137
|
def __init__(self, *args, **kwargs):
|
|
138
138
|
path = kwargs.pop("path")
|
|
139
139
|
endpoint, uri = mlrun.datastore.targets.RedisNoSqlTarget.get_server_endpoint(
|
|
140
|
-
path
|
|
140
|
+
path,
|
|
141
|
+
kwargs.pop("credentials_prefix", None),
|
|
141
142
|
)
|
|
142
143
|
kwargs["path"] = endpoint + "/" + uri
|
|
143
144
|
super().__init__(*args, **kwargs)
|
mlrun/datastore/targets.py
CHANGED
|
@@ -439,6 +439,12 @@ class BaseStoreTarget(DataTargetBase):
|
|
|
439
439
|
self.storage_options = storage_options
|
|
440
440
|
self.schema = schema or {}
|
|
441
441
|
self.credentials_prefix = credentials_prefix
|
|
442
|
+
if credentials_prefix:
|
|
443
|
+
warnings.warn(
|
|
444
|
+
"The 'credentials_prefix' parameter is deprecated and will be removed in "
|
|
445
|
+
"1.9.0. Please use datastore profiles instead.",
|
|
446
|
+
FutureWarning,
|
|
447
|
+
)
|
|
442
448
|
|
|
443
449
|
self._target = None
|
|
444
450
|
self._resource = None
|
|
@@ -1479,7 +1485,7 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
|
|
|
1479
1485
|
writer_step_name = "RedisNoSqlTarget"
|
|
1480
1486
|
|
|
1481
1487
|
@staticmethod
|
|
1482
|
-
def get_server_endpoint(path):
|
|
1488
|
+
def get_server_endpoint(path, credentials_prefix=None):
|
|
1483
1489
|
endpoint, uri = parse_path(path)
|
|
1484
1490
|
endpoint = endpoint or mlrun.mlconf.redis.url
|
|
1485
1491
|
if endpoint.startswith("ds://"):
|
|
@@ -1497,7 +1503,9 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
|
|
|
1497
1503
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
1498
1504
|
"Provide Redis username and password only via secrets"
|
|
1499
1505
|
)
|
|
1500
|
-
credentials_prefix = mlrun.get_secret_or_env(
|
|
1506
|
+
credentials_prefix = credentials_prefix or mlrun.get_secret_or_env(
|
|
1507
|
+
key="CREDENTIALS_PREFIX"
|
|
1508
|
+
)
|
|
1501
1509
|
user = mlrun.get_secret_or_env(
|
|
1502
1510
|
"REDIS_USER", default="", prefix=credentials_prefix
|
|
1503
1511
|
)
|
|
@@ -1517,7 +1525,9 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
|
|
|
1517
1525
|
from storey import Table
|
|
1518
1526
|
from storey.redis_driver import RedisDriver
|
|
1519
1527
|
|
|
1520
|
-
endpoint, uri = self.get_server_endpoint(
|
|
1528
|
+
endpoint, uri = self.get_server_endpoint(
|
|
1529
|
+
self.get_target_path(), self.credentials_prefix
|
|
1530
|
+
)
|
|
1521
1531
|
|
|
1522
1532
|
return Table(
|
|
1523
1533
|
uri,
|
|
@@ -1526,7 +1536,9 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
|
|
|
1526
1536
|
)
|
|
1527
1537
|
|
|
1528
1538
|
def get_spark_options(self, key_column=None, timestamp_key=None, overwrite=True):
|
|
1529
|
-
endpoint, uri = self.get_server_endpoint(
|
|
1539
|
+
endpoint, uri = self.get_server_endpoint(
|
|
1540
|
+
self.get_target_path(), self.credentials_prefix
|
|
1541
|
+
)
|
|
1530
1542
|
parsed_endpoint = urlparse(endpoint)
|
|
1531
1543
|
store, path_in_store, path = self._get_store_and_path()
|
|
1532
1544
|
return {
|
|
@@ -1577,6 +1589,7 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
|
|
|
1577
1589
|
class_name="mlrun.datastore.storeytargets.RedisNoSqlStoreyTarget",
|
|
1578
1590
|
columns=column_list,
|
|
1579
1591
|
table=table,
|
|
1592
|
+
credentials_prefix=self.credentials_prefix,
|
|
1580
1593
|
**self.attributes,
|
|
1581
1594
|
)
|
|
1582
1595
|
|
mlrun/errors.py
CHANGED
|
@@ -29,11 +29,14 @@ class MLRunBaseError(Exception):
|
|
|
29
29
|
pass
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
class
|
|
32
|
+
class MLRunTaskNotReadyError(MLRunBaseError):
|
|
33
33
|
"""indicate we are trying to read a value which is not ready
|
|
34
34
|
or need to come from a job which is in progress"""
|
|
35
35
|
|
|
36
36
|
|
|
37
|
+
MLRunTaskNotReady = MLRunTaskNotReadyError # kept for BC only
|
|
38
|
+
|
|
39
|
+
|
|
37
40
|
class MLRunHTTPError(MLRunBaseError, requests.HTTPError):
|
|
38
41
|
def __init__(
|
|
39
42
|
self,
|
|
@@ -205,15 +208,15 @@ class MLRunTimeoutError(MLRunHTTPStatusError, TimeoutError):
|
|
|
205
208
|
error_status_code = HTTPStatus.GATEWAY_TIMEOUT.value
|
|
206
209
|
|
|
207
210
|
|
|
208
|
-
class
|
|
211
|
+
class MLRunInvalidMMStoreTypeError(MLRunHTTPStatusError, ValueError):
|
|
209
212
|
error_status_code = HTTPStatus.BAD_REQUEST.value
|
|
210
213
|
|
|
211
214
|
|
|
212
|
-
class
|
|
215
|
+
class MLRunStreamConnectionFailureError(MLRunHTTPStatusError, ValueError):
|
|
213
216
|
error_status_code = HTTPStatus.BAD_REQUEST.value
|
|
214
217
|
|
|
215
218
|
|
|
216
|
-
class
|
|
219
|
+
class MLRunTSDBConnectionFailureError(MLRunHTTPStatusError, ValueError):
|
|
217
220
|
error_status_code = HTTPStatus.BAD_REQUEST.value
|
|
218
221
|
|
|
219
222
|
|
mlrun/execution.py
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
import logging
|
|
15
16
|
import os
|
|
16
17
|
import uuid
|
|
17
18
|
from copy import deepcopy
|
|
@@ -168,6 +169,8 @@ class MLClientCtx:
|
|
|
168
169
|
@log_level.setter
|
|
169
170
|
def log_level(self, value: str):
|
|
170
171
|
"""Set the logging level, e.g. 'debug', 'info', 'error'"""
|
|
172
|
+
level = logging.getLevelName(value.upper())
|
|
173
|
+
self._logger.set_logger_level(level)
|
|
171
174
|
self._log_level = value
|
|
172
175
|
|
|
173
176
|
@property
|
|
@@ -335,7 +338,7 @@ class MLClientCtx:
|
|
|
335
338
|
"name": self.name,
|
|
336
339
|
"kind": "run",
|
|
337
340
|
"uri": uri,
|
|
338
|
-
"owner": get_in(self._labels,
|
|
341
|
+
"owner": get_in(self._labels, mlrun_constants.MLRunInternalLabels.owner),
|
|
339
342
|
}
|
|
340
343
|
if mlrun_constants.MLRunInternalLabels.workflow in self._labels:
|
|
341
344
|
resp[mlrun_constants.MLRunInternalLabels.workflow] = self._labels[
|
|
@@ -1086,7 +1086,9 @@ class OfflineVectorResponse:
|
|
|
1086
1086
|
def to_dataframe(self, to_pandas=True):
|
|
1087
1087
|
"""return result as dataframe"""
|
|
1088
1088
|
if self.status != "completed":
|
|
1089
|
-
raise mlrun.errors.
|
|
1089
|
+
raise mlrun.errors.MLRunTaskNotReadyError(
|
|
1090
|
+
"feature vector dataset is not ready"
|
|
1091
|
+
)
|
|
1090
1092
|
return self._merger.get_df(to_pandas=to_pandas)
|
|
1091
1093
|
|
|
1092
1094
|
def to_parquet(self, target_path, **kw):
|
|
@@ -156,7 +156,9 @@ class RemoteVectorResponse:
|
|
|
156
156
|
|
|
157
157
|
def _is_ready(self):
|
|
158
158
|
if self.status != "completed":
|
|
159
|
-
raise mlrun.errors.
|
|
159
|
+
raise mlrun.errors.MLRunTaskNotReadyError(
|
|
160
|
+
"feature vector dataset is not ready"
|
|
161
|
+
)
|
|
160
162
|
self.vector.reload()
|
|
161
163
|
|
|
162
164
|
def to_dataframe(self, columns=None, df_module=None, **kwargs):
|
|
@@ -97,7 +97,7 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
|
|
|
97
97
|
|
|
98
98
|
def wrapper(
|
|
99
99
|
self: SKLearnTypes.ModelType,
|
|
100
|
-
X: SKLearnTypes.DatasetType,
|
|
100
|
+
X: SKLearnTypes.DatasetType, # noqa: N803 - should be lowercase "x", kept for BC
|
|
101
101
|
y: SKLearnTypes.DatasetType = None,
|
|
102
102
|
*args,
|
|
103
103
|
**kwargs,
|
|
@@ -124,7 +124,12 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
|
|
|
124
124
|
|
|
125
125
|
return wrapper
|
|
126
126
|
|
|
127
|
-
def mlrun_predict(
|
|
127
|
+
def mlrun_predict(
|
|
128
|
+
self,
|
|
129
|
+
X: SKLearnTypes.DatasetType, # noqa: N803 - should be lowercase "x", kept for BC
|
|
130
|
+
*args,
|
|
131
|
+
**kwargs,
|
|
132
|
+
):
|
|
128
133
|
"""
|
|
129
134
|
MLRun's wrapper for the common ML API predict method.
|
|
130
135
|
"""
|
|
@@ -136,7 +141,12 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
|
|
|
136
141
|
|
|
137
142
|
return y_pred
|
|
138
143
|
|
|
139
|
-
def mlrun_predict_proba(
|
|
144
|
+
def mlrun_predict_proba(
|
|
145
|
+
self,
|
|
146
|
+
X: SKLearnTypes.DatasetType, # noqa: N803 - should be lowercase "x", kept for BC
|
|
147
|
+
*args,
|
|
148
|
+
**kwargs,
|
|
149
|
+
):
|
|
140
150
|
"""
|
|
141
151
|
MLRun's wrapper for the common ML API predict_proba method.
|
|
142
152
|
"""
|
mlrun/k8s_utils.py
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import re
|
|
15
|
+
import warnings
|
|
15
16
|
|
|
16
17
|
import kubernetes.client
|
|
17
18
|
|
|
@@ -133,7 +134,7 @@ def sanitize_label_value(value: str) -> str:
|
|
|
133
134
|
return re.sub(r"([^a-zA-Z0-9_.-]|^[^a-zA-Z0-9]|[^a-zA-Z0-9]$)", "-", value[:63])
|
|
134
135
|
|
|
135
136
|
|
|
136
|
-
def verify_label_key(key: str):
|
|
137
|
+
def verify_label_key(key: str, allow_k8s_prefix: bool = False):
|
|
137
138
|
"""
|
|
138
139
|
Verify that the label key is valid for Kubernetes.
|
|
139
140
|
Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
|
|
@@ -146,6 +147,10 @@ def verify_label_key(key: str):
|
|
|
146
147
|
name = parts[0]
|
|
147
148
|
elif len(parts) == 2:
|
|
148
149
|
prefix, name = parts
|
|
150
|
+
if len(name) == 0:
|
|
151
|
+
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
152
|
+
"Label key name cannot be empty when a prefix is set"
|
|
153
|
+
)
|
|
149
154
|
if len(prefix) == 0:
|
|
150
155
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
151
156
|
"Label key prefix cannot be empty"
|
|
@@ -173,7 +178,13 @@ def verify_label_key(key: str):
|
|
|
173
178
|
mlrun.utils.regex.qualified_name,
|
|
174
179
|
)
|
|
175
180
|
|
|
176
|
-
|
|
181
|
+
# Allow the use of Kubernetes reserved prefixes ('k8s.io/' or 'kubernetes.io/')
|
|
182
|
+
# only when setting node selectors, not when adding new labels.
|
|
183
|
+
if (
|
|
184
|
+
key.startswith("k8s.io/")
|
|
185
|
+
or key.startswith("kubernetes.io/")
|
|
186
|
+
and not allow_k8s_prefix
|
|
187
|
+
):
|
|
177
188
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
178
189
|
"Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
|
|
179
190
|
)
|
|
@@ -185,3 +196,38 @@ def verify_label_value(value, label_key):
|
|
|
185
196
|
value,
|
|
186
197
|
mlrun.utils.regex.label_value,
|
|
187
198
|
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def validate_node_selectors(
|
|
202
|
+
node_selectors: dict[str, str], raise_on_error: bool = True
|
|
203
|
+
) -> bool:
|
|
204
|
+
"""
|
|
205
|
+
Ensures that user-defined node selectors adhere to Kubernetes label standards:
|
|
206
|
+
- Validates that each key conforms to Kubernetes naming conventions, with specific rules for name and prefix.
|
|
207
|
+
- Ensures values comply with Kubernetes label value rules.
|
|
208
|
+
- If raise_on_error is True, raises errors for invalid selectors.
|
|
209
|
+
- If raise_on_error is False, logs warnings for invalid selectors.
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
# Helper function for handling errors or warnings
|
|
213
|
+
def handle_invalid(message):
|
|
214
|
+
if raise_on_error:
|
|
215
|
+
raise
|
|
216
|
+
else:
|
|
217
|
+
warnings.warn(
|
|
218
|
+
f"{message}\n"
|
|
219
|
+
f"The node selector you’ve set does not meet the validation rules for the current Kubernetes version. "
|
|
220
|
+
f"Please note that invalid node selectors may cause issues with function scheduling."
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
node_selectors = node_selectors or {}
|
|
224
|
+
for key, value in node_selectors.items():
|
|
225
|
+
try:
|
|
226
|
+
verify_label_key(key, allow_k8s_prefix=True)
|
|
227
|
+
verify_label_value(value, label_key=key)
|
|
228
|
+
except mlrun.errors.MLRunInvalidArgumentError as err:
|
|
229
|
+
# An error or warning is raised by handle_invalid due to validation failure.
|
|
230
|
+
# Returning False indicates validation failed, allowing us to exit the function.
|
|
231
|
+
handle_invalid(str(err))
|
|
232
|
+
return False
|
|
233
|
+
return True
|
mlrun/model.py
CHANGED
|
@@ -487,7 +487,7 @@ class ImageBuilder(ModelObj):
|
|
|
487
487
|
|
|
488
488
|
def __init__(
|
|
489
489
|
self,
|
|
490
|
-
functionSourceCode=None,
|
|
490
|
+
functionSourceCode=None, # noqa: N803 - should be "snake_case", kept for BC
|
|
491
491
|
source=None,
|
|
492
492
|
image=None,
|
|
493
493
|
base_image=None,
|
|
@@ -681,7 +681,8 @@ class ImageBuilder(ModelObj):
|
|
|
681
681
|
class Notification(ModelObj):
|
|
682
682
|
"""Notification object
|
|
683
683
|
|
|
684
|
-
:param kind: notification implementation kind - slack, webhook, etc.
|
|
684
|
+
:param kind: notification implementation kind - slack, webhook, etc. See
|
|
685
|
+
:py:class:`mlrun.common.schemas.notification.NotificationKind`
|
|
685
686
|
:param name: for logging and identification
|
|
686
687
|
:param message: message content in the notification
|
|
687
688
|
:param severity: severity to display in the notification
|
|
@@ -63,7 +63,7 @@ class ObjectStoreFactory(enum.Enum):
|
|
|
63
63
|
:param value: Provided enum (invalid) value.
|
|
64
64
|
"""
|
|
65
65
|
valid_values = list(cls.__members__.keys())
|
|
66
|
-
raise mlrun.errors.
|
|
66
|
+
raise mlrun.errors.MLRunInvalidMMStoreTypeError(
|
|
67
67
|
f"{value} is not a valid endpoint store, please choose a valid value: %{valid_values}."
|
|
68
68
|
)
|
|
69
69
|
|
|
@@ -101,7 +101,7 @@ def get_store_object(
|
|
|
101
101
|
|
|
102
102
|
:return: `StoreBase` object. Using this object, the user can apply different operations such as write, update, get
|
|
103
103
|
and delete a model endpoint record.
|
|
104
|
-
:raise: `
|
|
104
|
+
:raise: `MLRunInvalidMMStoreTypeError` if the user didn't provide store connection
|
|
105
105
|
or the provided store connection is invalid.
|
|
106
106
|
"""
|
|
107
107
|
|
|
@@ -123,7 +123,7 @@ def get_store_object(
|
|
|
123
123
|
mlrun.common.schemas.model_monitoring.ModelEndpointTarget.V3IO_NOSQL
|
|
124
124
|
)
|
|
125
125
|
else:
|
|
126
|
-
raise mlrun.errors.
|
|
126
|
+
raise mlrun.errors.MLRunInvalidMMStoreTypeError(
|
|
127
127
|
"You must provide a valid store connection by using "
|
|
128
128
|
"set_model_monitoring_credentials API."
|
|
129
129
|
)
|
|
@@ -57,7 +57,7 @@ class ObjectTSDBFactory(enum.Enum):
|
|
|
57
57
|
:param value: Provided enum (invalid) value.
|
|
58
58
|
"""
|
|
59
59
|
valid_values = list(cls.__members__.keys())
|
|
60
|
-
raise mlrun.errors.
|
|
60
|
+
raise mlrun.errors.MLRunInvalidMMStoreTypeError(
|
|
61
61
|
f"{value} is not a valid tsdb, please choose a valid value: %{valid_values}."
|
|
62
62
|
)
|
|
63
63
|
|
|
@@ -76,7 +76,7 @@ def get_tsdb_connector(
|
|
|
76
76
|
|
|
77
77
|
:return: `TSDBConnector` object. The main goal of this object is to handle different operations on the
|
|
78
78
|
TSDB connector such as updating drift metrics or write application record result.
|
|
79
|
-
:raise: `
|
|
79
|
+
:raise: `MLRunInvalidMMStoreTypeError` if the user didn't provide TSDB connection
|
|
80
80
|
or the provided TSDB connection is invalid.
|
|
81
81
|
"""
|
|
82
82
|
|
|
@@ -93,7 +93,7 @@ def get_tsdb_connector(
|
|
|
93
93
|
elif tsdb_connection_string and tsdb_connection_string == "v3io":
|
|
94
94
|
tsdb_connector_type = mlrun.common.schemas.model_monitoring.TSDBTarget.V3IO_TSDB
|
|
95
95
|
else:
|
|
96
|
-
raise mlrun.errors.
|
|
96
|
+
raise mlrun.errors.MLRunInvalidMMStoreTypeError(
|
|
97
97
|
"You must provide a valid tsdb store connection by using "
|
|
98
98
|
"set_model_monitoring_credentials API."
|
|
99
99
|
)
|
|
@@ -68,7 +68,7 @@ class TDEngineConnector(TSDBConnector):
|
|
|
68
68
|
try:
|
|
69
69
|
conn.execute(f"USE {self.database}")
|
|
70
70
|
except taosws.QueryError as e:
|
|
71
|
-
raise mlrun.errors.
|
|
71
|
+
raise mlrun.errors.MLRunTSDBConnectionFailureError(
|
|
72
72
|
f"Failed to use TDEngine database {self.database}, {mlrun.errors.err_to_str(e)}"
|
|
73
73
|
)
|
|
74
74
|
return conn
|
|
@@ -91,7 +91,7 @@ class TDEngineConnector(TSDBConnector):
|
|
|
91
91
|
"""Create TDEngine supertables."""
|
|
92
92
|
for table in self.tables:
|
|
93
93
|
create_table_query = self.tables[table]._create_super_table_query()
|
|
94
|
-
self.
|
|
94
|
+
self.connection.execute(create_table_query)
|
|
95
95
|
|
|
96
96
|
def write_application_event(
|
|
97
97
|
self,
|
|
@@ -135,10 +135,10 @@ class TDEngineConnector(TSDBConnector):
|
|
|
135
135
|
create_table_query = table._create_subtable_query(
|
|
136
136
|
subtable=table_name, values=event
|
|
137
137
|
)
|
|
138
|
-
self.
|
|
138
|
+
self.connection.execute(create_table_query)
|
|
139
139
|
|
|
140
140
|
insert_statement = table._insert_subtable_query(
|
|
141
|
-
self.
|
|
141
|
+
self.connection,
|
|
142
142
|
subtable=table_name,
|
|
143
143
|
values=event,
|
|
144
144
|
)
|
|
@@ -204,12 +204,12 @@ class TDEngineConnector(TSDBConnector):
|
|
|
204
204
|
get_subtable_names_query = self.tables[table]._get_subtables_query(
|
|
205
205
|
values={mm_schemas.EventFieldType.PROJECT: self.project}
|
|
206
206
|
)
|
|
207
|
-
subtables = self.
|
|
207
|
+
subtables = self.connection.query(get_subtable_names_query)
|
|
208
208
|
for subtable in subtables:
|
|
209
209
|
drop_query = self.tables[table]._drop_subtable_query(
|
|
210
210
|
subtable=subtable[0]
|
|
211
211
|
)
|
|
212
|
-
self.
|
|
212
|
+
self.connection.execute(drop_query)
|
|
213
213
|
logger.info(
|
|
214
214
|
f"Deleted all project resources in the TSDB connector for project {self.project}"
|
|
215
215
|
)
|
|
@@ -281,7 +281,7 @@ class TDEngineConnector(TSDBConnector):
|
|
|
281
281
|
database=self.database,
|
|
282
282
|
)
|
|
283
283
|
try:
|
|
284
|
-
query_result = self.
|
|
284
|
+
query_result = self.connection.query(full_query)
|
|
285
285
|
except taosws.QueryError as e:
|
|
286
286
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
287
287
|
f"Failed to query table {table} in database {self.database}, {str(e)}"
|