mlrun 1.7.0rc41__py3-none-any.whl → 1.7.0rc43__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (42) hide show
  1. mlrun/artifacts/manager.py +6 -1
  2. mlrun/common/db/sql_session.py +3 -2
  3. mlrun/common/schemas/__init__.py +1 -0
  4. mlrun/common/schemas/api_gateway.py +6 -6
  5. mlrun/common/schemas/common.py +4 -4
  6. mlrun/common/schemas/frontend_spec.py +7 -0
  7. mlrun/common/schemas/notification.py +32 -5
  8. mlrun/config.py +19 -1
  9. mlrun/data_types/to_pandas.py +3 -3
  10. mlrun/datastore/base.py +0 -3
  11. mlrun/datastore/storeytargets.py +2 -1
  12. mlrun/datastore/targets.py +17 -4
  13. mlrun/errors.py +7 -4
  14. mlrun/execution.py +4 -1
  15. mlrun/feature_store/feature_vector.py +3 -1
  16. mlrun/feature_store/retrieval/job.py +3 -1
  17. mlrun/frameworks/sklearn/mlrun_interface.py +13 -3
  18. mlrun/k8s_utils.py +48 -2
  19. mlrun/model.py +3 -2
  20. mlrun/model_monitoring/db/stores/__init__.py +3 -3
  21. mlrun/model_monitoring/db/tsdb/__init__.py +3 -3
  22. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +7 -7
  23. mlrun/model_monitoring/helpers.py +0 -7
  24. mlrun/model_monitoring/writer.py +5 -1
  25. mlrun/package/packagers/default_packager.py +2 -2
  26. mlrun/projects/project.py +125 -47
  27. mlrun/runtimes/funcdoc.py +1 -1
  28. mlrun/runtimes/local.py +4 -1
  29. mlrun/runtimes/nuclio/application/application.py +3 -2
  30. mlrun/runtimes/pod.py +2 -0
  31. mlrun/runtimes/sparkjob/spark3job.py +5 -1
  32. mlrun/utils/async_http.py +1 -1
  33. mlrun/utils/helpers.py +17 -0
  34. mlrun/utils/notifications/notification/__init__.py +0 -1
  35. mlrun/utils/v3io_clients.py +2 -2
  36. mlrun/utils/version/version.json +2 -2
  37. {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/METADATA +10 -10
  38. {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/RECORD +42 -42
  39. {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/LICENSE +0 -0
  40. {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/WHEEL +0 -0
  41. {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/entry_points.txt +0 -0
  42. {mlrun-1.7.0rc41.dist-info → mlrun-1.7.0rc43.dist-info}/top_level.txt +0 -0
@@ -72,7 +72,12 @@ class ArtifactProducer:
72
72
  self.inputs = {}
73
73
 
74
74
  def get_meta(self) -> dict:
75
- return {"kind": self.kind, "name": self.name, "tag": self.tag}
75
+ return {
76
+ "kind": self.kind,
77
+ "name": self.name,
78
+ "tag": self.tag,
79
+ "owner": self.owner,
80
+ }
76
81
 
77
82
  @property
78
83
  def uid(self):
@@ -11,13 +11,14 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
14
 
16
15
 
17
16
  from sqlalchemy import create_engine
18
17
  from sqlalchemy.engine import Engine
19
18
  from sqlalchemy.orm import Session
20
- from sqlalchemy.orm import sessionmaker as SessionMaker
19
+ from sqlalchemy.orm import (
20
+ sessionmaker as SessionMaker, # noqa: N812 - `sessionmaker` is a class
21
+ )
21
22
 
22
23
  from mlrun.config import config
23
24
 
@@ -108,6 +108,7 @@ from .feature_store import (
108
108
  FeatureVectorsTagsOutput,
109
109
  )
110
110
  from .frontend_spec import (
111
+ ArtifactLimits,
111
112
  AuthenticationFeatureFlag,
112
113
  FeatureFlags,
113
114
  FrontendSpec,
@@ -77,7 +77,7 @@ class APIGatewaySpec(_APIGatewayBaseModel):
77
77
  name: str
78
78
  description: Optional[str]
79
79
  path: Optional[str] = "/"
80
- authenticationMode: Optional[APIGatewayAuthenticationMode] = (
80
+ authenticationMode: Optional[APIGatewayAuthenticationMode] = ( # noqa: N815 - for compatibility with Nuclio https://github.com/nuclio/nuclio/blob/672b8e36f9edd6e42b4685ec1d27cabae3c5f045/pkg/platform/types.go#L476
81
81
  APIGatewayAuthenticationMode.none
82
82
  )
83
83
  upstreams: list[APIGatewayUpstream]
@@ -103,11 +103,11 @@ class APIGateway(_APIGatewayBaseModel):
103
103
  ]
104
104
 
105
105
  def get_invoke_url(self):
106
- return (
107
- self.spec.host + self.spec.path
108
- if self.spec.path and self.spec.host
109
- else self.spec.host
110
- ).rstrip("/")
106
+ if self.spec.host and self.spec.path:
107
+ return f"{self.spec.host.rstrip('/')}/{self.spec.path.lstrip('/')}".rstrip(
108
+ "/"
109
+ )
110
+ return self.spec.host.rstrip("/")
111
111
 
112
112
  def enrich_mlrun_names(self):
113
113
  self._enrich_api_gateway_mlrun_name()
@@ -11,16 +11,16 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import typing
16
16
 
17
17
  import pydantic
18
18
 
19
19
 
20
20
  class ImageBuilder(pydantic.BaseModel):
21
- functionSourceCode: typing.Optional[str] = None
22
- codeEntryType: typing.Optional[str] = None
23
- codeEntryAttributes: typing.Optional[str] = None
21
+ functionSourceCode: typing.Optional[str] = None # noqa: N815
22
+ codeEntryType: typing.Optional[str] = None # noqa: N815
23
+ codeEntryAttributes: typing.Optional[str] = None # noqa: N815
24
24
  source: typing.Optional[str] = None
25
25
  code_origin: typing.Optional[str] = None
26
26
  origin_filename: typing.Optional[str] = None
@@ -50,6 +50,12 @@ class FeatureFlags(pydantic.BaseModel):
50
50
  preemption_nodes: PreemptionNodesFeatureFlag
51
51
 
52
52
 
53
+ class ArtifactLimits(pydantic.BaseModel):
54
+ max_chunk_size: int
55
+ max_preview_size: int
56
+ max_download_size: int
57
+
58
+
53
59
  class FrontendSpec(pydantic.BaseModel):
54
60
  jobs_dashboard_url: typing.Optional[str]
55
61
  model_monitoring_dashboard_url: typing.Optional[str]
@@ -71,3 +77,4 @@ class FrontendSpec(pydantic.BaseModel):
71
77
  allowed_artifact_path_prefixes_list: list[str]
72
78
  ce: typing.Optional[dict]
73
79
  internal_labels: list[str] = []
80
+ artifact_limits: ArtifactLimits
@@ -22,11 +22,38 @@ import mlrun.common.types
22
22
 
23
23
 
24
24
  class NotificationKind(mlrun.common.types.StrEnum):
25
- console = "console"
26
- git = "git"
27
- ipython = "ipython"
28
- slack = "slack"
29
- webhook = "webhook"
25
+ """Currently, the supported notification kinds and their params are as follows:"""
26
+
27
+ console: str = "console"
28
+ """no params, local only"""
29
+
30
+ git: str = "git"
31
+ """
32
+ **token** - The git token to use for the git notification.\n
33
+ **repo** - The git repo to which to send the notification.\n
34
+ **issue** - The git issue to which to send the notification.\n
35
+ **merge_request** -
36
+ In GitLab (as opposed to GitHub), merge requests and issues are separate entities.
37
+ If using merge request, the issue will be ignored, and vice versa.\n
38
+ **server** - The git server to which to send the notification.\n
39
+ **gitlab** - (bool) Whether the git server is GitLab or not.\n
40
+ """
41
+
42
+ ipython: str = "ipython"
43
+ """no params, local only"""
44
+
45
+ slack: str = "slack"
46
+ """**webhook** - The slack webhook to which to send the notification."""
47
+
48
+ webhook: str = "webhook"
49
+ """
50
+ **url** - The webhook url to which to send the notification.\n
51
+ **method** - The http method to use when sending the notification (GET, POST, PUT, etc…).\n
52
+ **headers** - (dict) The http headers to send with the notification.\n
53
+ **override_body** - (dict) The body to send with the notification.\n
54
+ **verify_ssl** -
55
+ (bool) Whether SSL certificates are validated during HTTP requests or not, The default is set to True.
56
+ """
30
57
 
31
58
 
32
59
  class NotificationSeverity(mlrun.common.types.StrEnum):
mlrun/config.py CHANGED
@@ -27,6 +27,7 @@ import copy
27
27
  import json
28
28
  import os
29
29
  import typing
30
+ import warnings
30
31
  from collections.abc import Mapping
31
32
  from datetime import timedelta
32
33
  from distutils.util import strtobool
@@ -35,6 +36,7 @@ from threading import Lock
35
36
 
36
37
  import dotenv
37
38
  import semver
39
+ import urllib3.exceptions
38
40
  import yaml
39
41
 
40
42
  import mlrun.common.constants
@@ -152,6 +154,11 @@ default_config = {
152
154
  "datasets": {
153
155
  "max_preview_columns": 100,
154
156
  },
157
+ "limits": {
158
+ "max_chunk_size": 1024 * 1024 * 1, # 1MB
159
+ "max_preview_size": 1024 * 1024 * 10, # 10MB
160
+ "max_download_size": 1024 * 1024 * 100, # 100MB
161
+ },
155
162
  },
156
163
  # FIXME: Adding these defaults here so we won't need to patch the "installing component" (provazio-controller) to
157
164
  # configure this values on field systems, for newer system this will be configured correctly
@@ -326,7 +333,7 @@ default_config = {
326
333
  "http": {
327
334
  # when True, the client will verify the server's TLS
328
335
  # set to False for backwards compatibility.
329
- "verify": False,
336
+ "verify": True,
330
337
  },
331
338
  "db": {
332
339
  "commit_retry_timeout": 30,
@@ -1292,6 +1299,7 @@ def _do_populate(env=None, skip_errors=False):
1292
1299
  if data:
1293
1300
  config.update(data, skip_errors=skip_errors)
1294
1301
 
1302
+ _configure_ssl_verification(config.httpdb.http.verify)
1295
1303
  _validate_config(config)
1296
1304
 
1297
1305
 
@@ -1351,6 +1359,16 @@ def _convert_str(value, typ):
1351
1359
  return typ(value)
1352
1360
 
1353
1361
 
1362
+ def _configure_ssl_verification(verify_ssl: bool) -> None:
1363
+ """Configure SSL verification warnings based on the setting."""
1364
+ if not verify_ssl:
1365
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
1366
+ else:
1367
+ # If the user changes the `verify` setting to `True` at runtime using `mlrun.set_env_from_file` after
1368
+ # importing `mlrun`, we need to reload the `mlrun` configuration and enable this warning.
1369
+ warnings.simplefilter("default", urllib3.exceptions.InsecureRequestWarning)
1370
+
1371
+
1354
1372
  def read_env(env=None, prefix=env_prefix):
1355
1373
  """Read configuration from environment"""
1356
1374
  env = os.environ if env is None else env
@@ -19,7 +19,7 @@ import pandas as pd
19
19
  import semver
20
20
 
21
21
 
22
- def _toPandas(spark_df):
22
+ def _to_pandas(spark_df):
23
23
  """
24
24
  Modified version of spark DataFrame.toPandas() -
25
25
  https://github.com/apache/spark/blob/v3.2.3/python/pyspark/sql/pandas/conversion.py#L35
@@ -262,9 +262,9 @@ def spark_df_to_pandas(spark_df):
262
262
  )
263
263
  type_conversion_dict[field.name] = "datetime64[ns]"
264
264
 
265
- df = _toPandas(spark_df)
265
+ df = _to_pandas(spark_df)
266
266
  if type_conversion_dict:
267
267
  df = df.astype(type_conversion_dict)
268
268
  return df
269
269
  else:
270
- return _toPandas(spark_df)
270
+ return _to_pandas(spark_df)
mlrun/datastore/base.py CHANGED
@@ -24,7 +24,6 @@ import pandas as pd
24
24
  import pyarrow
25
25
  import pytz
26
26
  import requests
27
- import urllib3
28
27
  from deprecated import deprecated
29
28
 
30
29
  import mlrun.config
@@ -745,8 +744,6 @@ class HttpStore(DataStore):
745
744
 
746
745
  verify_ssl = mlconf.httpdb.http.verify
747
746
  try:
748
- if not verify_ssl:
749
- urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
750
747
  response = requests.get(url, headers=headers, auth=auth, verify=verify_ssl)
751
748
  except OSError as exc:
752
749
  raise OSError(f"error: cannot connect to {url}: {err_to_str(exc)}")
@@ -137,7 +137,8 @@ class RedisNoSqlStoreyTarget(storey.NoSqlTarget):
137
137
  def __init__(self, *args, **kwargs):
138
138
  path = kwargs.pop("path")
139
139
  endpoint, uri = mlrun.datastore.targets.RedisNoSqlTarget.get_server_endpoint(
140
- path
140
+ path,
141
+ kwargs.pop("credentials_prefix", None),
141
142
  )
142
143
  kwargs["path"] = endpoint + "/" + uri
143
144
  super().__init__(*args, **kwargs)
@@ -439,6 +439,12 @@ class BaseStoreTarget(DataTargetBase):
439
439
  self.storage_options = storage_options
440
440
  self.schema = schema or {}
441
441
  self.credentials_prefix = credentials_prefix
442
+ if credentials_prefix:
443
+ warnings.warn(
444
+ "The 'credentials_prefix' parameter is deprecated and will be removed in "
445
+ "1.9.0. Please use datastore profiles instead.",
446
+ FutureWarning,
447
+ )
442
448
 
443
449
  self._target = None
444
450
  self._resource = None
@@ -1479,7 +1485,7 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
1479
1485
  writer_step_name = "RedisNoSqlTarget"
1480
1486
 
1481
1487
  @staticmethod
1482
- def get_server_endpoint(path):
1488
+ def get_server_endpoint(path, credentials_prefix=None):
1483
1489
  endpoint, uri = parse_path(path)
1484
1490
  endpoint = endpoint or mlrun.mlconf.redis.url
1485
1491
  if endpoint.startswith("ds://"):
@@ -1497,7 +1503,9 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
1497
1503
  raise mlrun.errors.MLRunInvalidArgumentError(
1498
1504
  "Provide Redis username and password only via secrets"
1499
1505
  )
1500
- credentials_prefix = mlrun.get_secret_or_env(key="CREDENTIALS_PREFIX")
1506
+ credentials_prefix = credentials_prefix or mlrun.get_secret_or_env(
1507
+ key="CREDENTIALS_PREFIX"
1508
+ )
1501
1509
  user = mlrun.get_secret_or_env(
1502
1510
  "REDIS_USER", default="", prefix=credentials_prefix
1503
1511
  )
@@ -1517,7 +1525,9 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
1517
1525
  from storey import Table
1518
1526
  from storey.redis_driver import RedisDriver
1519
1527
 
1520
- endpoint, uri = self.get_server_endpoint(self.get_target_path())
1528
+ endpoint, uri = self.get_server_endpoint(
1529
+ self.get_target_path(), self.credentials_prefix
1530
+ )
1521
1531
 
1522
1532
  return Table(
1523
1533
  uri,
@@ -1526,7 +1536,9 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
1526
1536
  )
1527
1537
 
1528
1538
  def get_spark_options(self, key_column=None, timestamp_key=None, overwrite=True):
1529
- endpoint, uri = self.get_server_endpoint(self.get_target_path())
1539
+ endpoint, uri = self.get_server_endpoint(
1540
+ self.get_target_path(), self.credentials_prefix
1541
+ )
1530
1542
  parsed_endpoint = urlparse(endpoint)
1531
1543
  store, path_in_store, path = self._get_store_and_path()
1532
1544
  return {
@@ -1577,6 +1589,7 @@ class RedisNoSqlTarget(NoSqlBaseTarget):
1577
1589
  class_name="mlrun.datastore.storeytargets.RedisNoSqlStoreyTarget",
1578
1590
  columns=column_list,
1579
1591
  table=table,
1592
+ credentials_prefix=self.credentials_prefix,
1580
1593
  **self.attributes,
1581
1594
  )
1582
1595
 
mlrun/errors.py CHANGED
@@ -29,11 +29,14 @@ class MLRunBaseError(Exception):
29
29
  pass
30
30
 
31
31
 
32
- class MLRunTaskNotReady(MLRunBaseError):
32
+ class MLRunTaskNotReadyError(MLRunBaseError):
33
33
  """indicate we are trying to read a value which is not ready
34
34
  or need to come from a job which is in progress"""
35
35
 
36
36
 
37
+ MLRunTaskNotReady = MLRunTaskNotReadyError # kept for BC only
38
+
39
+
37
40
  class MLRunHTTPError(MLRunBaseError, requests.HTTPError):
38
41
  def __init__(
39
42
  self,
@@ -205,15 +208,15 @@ class MLRunTimeoutError(MLRunHTTPStatusError, TimeoutError):
205
208
  error_status_code = HTTPStatus.GATEWAY_TIMEOUT.value
206
209
 
207
210
 
208
- class MLRunInvalidMMStoreType(MLRunHTTPStatusError, ValueError):
211
+ class MLRunInvalidMMStoreTypeError(MLRunHTTPStatusError, ValueError):
209
212
  error_status_code = HTTPStatus.BAD_REQUEST.value
210
213
 
211
214
 
212
- class MLRunStreamConnectionFailure(MLRunHTTPStatusError, ValueError):
215
+ class MLRunStreamConnectionFailureError(MLRunHTTPStatusError, ValueError):
213
216
  error_status_code = HTTPStatus.BAD_REQUEST.value
214
217
 
215
218
 
216
- class MLRunTSDBConnectionFailure(MLRunHTTPStatusError, ValueError):
219
+ class MLRunTSDBConnectionFailureError(MLRunHTTPStatusError, ValueError):
217
220
  error_status_code = HTTPStatus.BAD_REQUEST.value
218
221
 
219
222
 
mlrun/execution.py CHANGED
@@ -12,6 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
+ import logging
15
16
  import os
16
17
  import uuid
17
18
  from copy import deepcopy
@@ -168,6 +169,8 @@ class MLClientCtx:
168
169
  @log_level.setter
169
170
  def log_level(self, value: str):
170
171
  """Set the logging level, e.g. 'debug', 'info', 'error'"""
172
+ level = logging.getLevelName(value.upper())
173
+ self._logger.set_logger_level(level)
171
174
  self._log_level = value
172
175
 
173
176
  @property
@@ -335,7 +338,7 @@ class MLClientCtx:
335
338
  "name": self.name,
336
339
  "kind": "run",
337
340
  "uri": uri,
338
- "owner": get_in(self._labels, "owner"),
341
+ "owner": get_in(self._labels, mlrun_constants.MLRunInternalLabels.owner),
339
342
  }
340
343
  if mlrun_constants.MLRunInternalLabels.workflow in self._labels:
341
344
  resp[mlrun_constants.MLRunInternalLabels.workflow] = self._labels[
@@ -1086,7 +1086,9 @@ class OfflineVectorResponse:
1086
1086
  def to_dataframe(self, to_pandas=True):
1087
1087
  """return result as dataframe"""
1088
1088
  if self.status != "completed":
1089
- raise mlrun.errors.MLRunTaskNotReady("feature vector dataset is not ready")
1089
+ raise mlrun.errors.MLRunTaskNotReadyError(
1090
+ "feature vector dataset is not ready"
1091
+ )
1090
1092
  return self._merger.get_df(to_pandas=to_pandas)
1091
1093
 
1092
1094
  def to_parquet(self, target_path, **kw):
@@ -156,7 +156,9 @@ class RemoteVectorResponse:
156
156
 
157
157
  def _is_ready(self):
158
158
  if self.status != "completed":
159
- raise mlrun.errors.MLRunTaskNotReady("feature vector dataset is not ready")
159
+ raise mlrun.errors.MLRunTaskNotReadyError(
160
+ "feature vector dataset is not ready"
161
+ )
160
162
  self.vector.reload()
161
163
 
162
164
  def to_dataframe(self, columns=None, df_module=None, **kwargs):
@@ -97,7 +97,7 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
97
97
 
98
98
  def wrapper(
99
99
  self: SKLearnTypes.ModelType,
100
- X: SKLearnTypes.DatasetType,
100
+ X: SKLearnTypes.DatasetType, # noqa: N803 - should be lowercase "x", kept for BC
101
101
  y: SKLearnTypes.DatasetType = None,
102
102
  *args,
103
103
  **kwargs,
@@ -124,7 +124,12 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
124
124
 
125
125
  return wrapper
126
126
 
127
- def mlrun_predict(self, X: SKLearnTypes.DatasetType, *args, **kwargs):
127
+ def mlrun_predict(
128
+ self,
129
+ X: SKLearnTypes.DatasetType, # noqa: N803 - should be lowercase "x", kept for BC
130
+ *args,
131
+ **kwargs,
132
+ ):
128
133
  """
129
134
  MLRun's wrapper for the common ML API predict method.
130
135
  """
@@ -136,7 +141,12 @@ class SKLearnMLRunInterface(MLRunInterface, ABC):
136
141
 
137
142
  return y_pred
138
143
 
139
- def mlrun_predict_proba(self, X: SKLearnTypes.DatasetType, *args, **kwargs):
144
+ def mlrun_predict_proba(
145
+ self,
146
+ X: SKLearnTypes.DatasetType, # noqa: N803 - should be lowercase "x", kept for BC
147
+ *args,
148
+ **kwargs,
149
+ ):
140
150
  """
141
151
  MLRun's wrapper for the common ML API predict_proba method.
142
152
  """
mlrun/k8s_utils.py CHANGED
@@ -12,6 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  import re
15
+ import warnings
15
16
 
16
17
  import kubernetes.client
17
18
 
@@ -133,7 +134,7 @@ def sanitize_label_value(value: str) -> str:
133
134
  return re.sub(r"([^a-zA-Z0-9_.-]|^[^a-zA-Z0-9]|[^a-zA-Z0-9]$)", "-", value[:63])
134
135
 
135
136
 
136
- def verify_label_key(key: str):
137
+ def verify_label_key(key: str, allow_k8s_prefix: bool = False):
137
138
  """
138
139
  Verify that the label key is valid for Kubernetes.
139
140
  Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
@@ -146,6 +147,10 @@ def verify_label_key(key: str):
146
147
  name = parts[0]
147
148
  elif len(parts) == 2:
148
149
  prefix, name = parts
150
+ if len(name) == 0:
151
+ raise mlrun.errors.MLRunInvalidArgumentError(
152
+ "Label key name cannot be empty when a prefix is set"
153
+ )
149
154
  if len(prefix) == 0:
150
155
  raise mlrun.errors.MLRunInvalidArgumentError(
151
156
  "Label key prefix cannot be empty"
@@ -173,7 +178,13 @@ def verify_label_key(key: str):
173
178
  mlrun.utils.regex.qualified_name,
174
179
  )
175
180
 
176
- if key.startswith("k8s.io/") or key.startswith("kubernetes.io/"):
181
+ # Allow the use of Kubernetes reserved prefixes ('k8s.io/' or 'kubernetes.io/')
182
+ # only when setting node selectors, not when adding new labels.
183
+ if (
184
+ key.startswith("k8s.io/")
185
+ or key.startswith("kubernetes.io/")
186
+ and not allow_k8s_prefix
187
+ ):
177
188
  raise mlrun.errors.MLRunInvalidArgumentError(
178
189
  "Labels cannot start with 'k8s.io/' or 'kubernetes.io/'"
179
190
  )
@@ -185,3 +196,38 @@ def verify_label_value(value, label_key):
185
196
  value,
186
197
  mlrun.utils.regex.label_value,
187
198
  )
199
+
200
+
201
+ def validate_node_selectors(
202
+ node_selectors: dict[str, str], raise_on_error: bool = True
203
+ ) -> bool:
204
+ """
205
+ Ensures that user-defined node selectors adhere to Kubernetes label standards:
206
+ - Validates that each key conforms to Kubernetes naming conventions, with specific rules for name and prefix.
207
+ - Ensures values comply with Kubernetes label value rules.
208
+ - If raise_on_error is True, raises errors for invalid selectors.
209
+ - If raise_on_error is False, logs warnings for invalid selectors.
210
+ """
211
+
212
+ # Helper function for handling errors or warnings
213
+ def handle_invalid(message):
214
+ if raise_on_error:
215
+ raise
216
+ else:
217
+ warnings.warn(
218
+ f"{message}\n"
219
+ f"The node selector you’ve set does not meet the validation rules for the current Kubernetes version. "
220
+ f"Please note that invalid node selectors may cause issues with function scheduling."
221
+ )
222
+
223
+ node_selectors = node_selectors or {}
224
+ for key, value in node_selectors.items():
225
+ try:
226
+ verify_label_key(key, allow_k8s_prefix=True)
227
+ verify_label_value(value, label_key=key)
228
+ except mlrun.errors.MLRunInvalidArgumentError as err:
229
+ # An error or warning is raised by handle_invalid due to validation failure.
230
+ # Returning False indicates validation failed, allowing us to exit the function.
231
+ handle_invalid(str(err))
232
+ return False
233
+ return True
mlrun/model.py CHANGED
@@ -487,7 +487,7 @@ class ImageBuilder(ModelObj):
487
487
 
488
488
  def __init__(
489
489
  self,
490
- functionSourceCode=None,
490
+ functionSourceCode=None, # noqa: N803 - should be "snake_case", kept for BC
491
491
  source=None,
492
492
  image=None,
493
493
  base_image=None,
@@ -681,7 +681,8 @@ class ImageBuilder(ModelObj):
681
681
  class Notification(ModelObj):
682
682
  """Notification object
683
683
 
684
- :param kind: notification implementation kind - slack, webhook, etc.
684
+ :param kind: notification implementation kind - slack, webhook, etc. See
685
+ :py:class:`mlrun.common.schemas.notification.NotificationKind`
685
686
  :param name: for logging and identification
686
687
  :param message: message content in the notification
687
688
  :param severity: severity to display in the notification
@@ -63,7 +63,7 @@ class ObjectStoreFactory(enum.Enum):
63
63
  :param value: Provided enum (invalid) value.
64
64
  """
65
65
  valid_values = list(cls.__members__.keys())
66
- raise mlrun.errors.MLRunInvalidMMStoreType(
66
+ raise mlrun.errors.MLRunInvalidMMStoreTypeError(
67
67
  f"{value} is not a valid endpoint store, please choose a valid value: %{valid_values}."
68
68
  )
69
69
 
@@ -101,7 +101,7 @@ def get_store_object(
101
101
 
102
102
  :return: `StoreBase` object. Using this object, the user can apply different operations such as write, update, get
103
103
  and delete a model endpoint record.
104
- :raise: `MLRunInvalidMMStoreType` if the user didn't provide store connection
104
+ :raise: `MLRunInvalidMMStoreTypeError` if the user didn't provide store connection
105
105
  or the provided store connection is invalid.
106
106
  """
107
107
 
@@ -123,7 +123,7 @@ def get_store_object(
123
123
  mlrun.common.schemas.model_monitoring.ModelEndpointTarget.V3IO_NOSQL
124
124
  )
125
125
  else:
126
- raise mlrun.errors.MLRunInvalidMMStoreType(
126
+ raise mlrun.errors.MLRunInvalidMMStoreTypeError(
127
127
  "You must provide a valid store connection by using "
128
128
  "set_model_monitoring_credentials API."
129
129
  )
@@ -57,7 +57,7 @@ class ObjectTSDBFactory(enum.Enum):
57
57
  :param value: Provided enum (invalid) value.
58
58
  """
59
59
  valid_values = list(cls.__members__.keys())
60
- raise mlrun.errors.MLRunInvalidMMStoreType(
60
+ raise mlrun.errors.MLRunInvalidMMStoreTypeError(
61
61
  f"{value} is not a valid tsdb, please choose a valid value: %{valid_values}."
62
62
  )
63
63
 
@@ -76,7 +76,7 @@ def get_tsdb_connector(
76
76
 
77
77
  :return: `TSDBConnector` object. The main goal of this object is to handle different operations on the
78
78
  TSDB connector such as updating drift metrics or write application record result.
79
- :raise: `MLRunInvalidMMStoreType` if the user didn't provide TSDB connection
79
+ :raise: `MLRunInvalidMMStoreTypeError` if the user didn't provide TSDB connection
80
80
  or the provided TSDB connection is invalid.
81
81
  """
82
82
 
@@ -93,7 +93,7 @@ def get_tsdb_connector(
93
93
  elif tsdb_connection_string and tsdb_connection_string == "v3io":
94
94
  tsdb_connector_type = mlrun.common.schemas.model_monitoring.TSDBTarget.V3IO_TSDB
95
95
  else:
96
- raise mlrun.errors.MLRunInvalidMMStoreType(
96
+ raise mlrun.errors.MLRunInvalidMMStoreTypeError(
97
97
  "You must provide a valid tsdb store connection by using "
98
98
  "set_model_monitoring_credentials API."
99
99
  )
@@ -68,7 +68,7 @@ class TDEngineConnector(TSDBConnector):
68
68
  try:
69
69
  conn.execute(f"USE {self.database}")
70
70
  except taosws.QueryError as e:
71
- raise mlrun.errors.MLRunTSDBConnectionFailure(
71
+ raise mlrun.errors.MLRunTSDBConnectionFailureError(
72
72
  f"Failed to use TDEngine database {self.database}, {mlrun.errors.err_to_str(e)}"
73
73
  )
74
74
  return conn
@@ -91,7 +91,7 @@ class TDEngineConnector(TSDBConnector):
91
91
  """Create TDEngine supertables."""
92
92
  for table in self.tables:
93
93
  create_table_query = self.tables[table]._create_super_table_query()
94
- self._connection.execute(create_table_query)
94
+ self.connection.execute(create_table_query)
95
95
 
96
96
  def write_application_event(
97
97
  self,
@@ -135,10 +135,10 @@ class TDEngineConnector(TSDBConnector):
135
135
  create_table_query = table._create_subtable_query(
136
136
  subtable=table_name, values=event
137
137
  )
138
- self._connection.execute(create_table_query)
138
+ self.connection.execute(create_table_query)
139
139
 
140
140
  insert_statement = table._insert_subtable_query(
141
- self._connection,
141
+ self.connection,
142
142
  subtable=table_name,
143
143
  values=event,
144
144
  )
@@ -204,12 +204,12 @@ class TDEngineConnector(TSDBConnector):
204
204
  get_subtable_names_query = self.tables[table]._get_subtables_query(
205
205
  values={mm_schemas.EventFieldType.PROJECT: self.project}
206
206
  )
207
- subtables = self._connection.query(get_subtable_names_query)
207
+ subtables = self.connection.query(get_subtable_names_query)
208
208
  for subtable in subtables:
209
209
  drop_query = self.tables[table]._drop_subtable_query(
210
210
  subtable=subtable[0]
211
211
  )
212
- self._connection.execute(drop_query)
212
+ self.connection.execute(drop_query)
213
213
  logger.info(
214
214
  f"Deleted all project resources in the TSDB connector for project {self.project}"
215
215
  )
@@ -281,7 +281,7 @@ class TDEngineConnector(TSDBConnector):
281
281
  database=self.database,
282
282
  )
283
283
  try:
284
- query_result = self._connection.query(full_query)
284
+ query_result = self.connection.query(full_query)
285
285
  except taosws.QueryError as e:
286
286
  raise mlrun.errors.MLRunInvalidArgumentError(
287
287
  f"Failed to query table {table} in database {self.database}, {str(e)}"