mlrun 1.7.0rc43__py3-none-any.whl → 1.7.0rc46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (51) hide show
  1. mlrun/__main__.py +4 -2
  2. mlrun/artifacts/manager.py +3 -1
  3. mlrun/common/formatters/__init__.py +1 -0
  4. mlrun/common/formatters/feature_set.py +33 -0
  5. mlrun/common/schemas/__init__.py +1 -0
  6. mlrun/common/schemas/alert.py +11 -11
  7. mlrun/common/schemas/auth.py +2 -0
  8. mlrun/common/schemas/client_spec.py +0 -1
  9. mlrun/common/schemas/model_monitoring/__init__.py +1 -0
  10. mlrun/common/schemas/workflow.py +1 -0
  11. mlrun/config.py +28 -21
  12. mlrun/data_types/data_types.py +5 -0
  13. mlrun/datastore/base.py +4 -4
  14. mlrun/datastore/s3.py +12 -9
  15. mlrun/datastore/storeytargets.py +2 -2
  16. mlrun/db/base.py +3 -0
  17. mlrun/db/httpdb.py +17 -12
  18. mlrun/db/nopdb.py +24 -4
  19. mlrun/execution.py +3 -1
  20. mlrun/feature_store/api.py +1 -0
  21. mlrun/feature_store/retrieval/spark_merger.py +7 -3
  22. mlrun/frameworks/_common/plan.py +3 -3
  23. mlrun/frameworks/_ml_common/plan.py +1 -1
  24. mlrun/frameworks/parallel_coordinates.py +2 -3
  25. mlrun/launcher/client.py +6 -6
  26. mlrun/model_monitoring/applications/results.py +4 -4
  27. mlrun/model_monitoring/controller.py +1 -1
  28. mlrun/model_monitoring/db/stores/sqldb/sql_store.py +15 -1
  29. mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py +12 -0
  30. mlrun/model_monitoring/db/tsdb/tdengine/schemas.py +7 -7
  31. mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py +13 -12
  32. mlrun/model_monitoring/helpers.py +7 -8
  33. mlrun/model_monitoring/writer.py +3 -1
  34. mlrun/projects/pipelines.py +2 -0
  35. mlrun/projects/project.py +43 -19
  36. mlrun/render.py +3 -3
  37. mlrun/runtimes/daskjob.py +1 -1
  38. mlrun/runtimes/kubejob.py +6 -6
  39. mlrun/runtimes/nuclio/api_gateway.py +6 -0
  40. mlrun/runtimes/nuclio/application/application.py +3 -3
  41. mlrun/runtimes/nuclio/function.py +41 -0
  42. mlrun/runtimes/pod.py +19 -13
  43. mlrun/serving/server.py +2 -0
  44. mlrun/utils/helpers.py +22 -16
  45. mlrun/utils/version/version.json +2 -2
  46. {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/METADATA +22 -22
  47. {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/RECORD +51 -50
  48. {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/WHEEL +1 -1
  49. {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/LICENSE +0 -0
  50. {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/entry_points.txt +0 -0
  51. {mlrun-1.7.0rc43.dist-info → mlrun-1.7.0rc46.dist-info}/top_level.txt +0 -0
mlrun/__main__.py CHANGED
@@ -734,9 +734,11 @@ def get(kind, name, selector, namespace, uid, project, tag, db, extra_args):
734
734
  if db:
735
735
  mlconf.dbpath = db
736
736
  if not project:
737
- print("Warning, project parameter was not specified using default !")
737
+ logger.warning(
738
+ "Project parameter was not specified. Defaulting to 'default' project"
739
+ )
738
740
  if kind.startswith("po"):
739
- print("Unsupported, use 'get runtimes' instead")
741
+ logger.warning("Unsupported, use 'get runtimes' instead")
740
742
  return
741
743
 
742
744
  elif kind.startswith("runtime"):
@@ -200,7 +200,9 @@ class ArtifactManager:
200
200
  :param artifact_path: The path to store the artifact.
201
201
  If not provided, the artifact will be stored in the default artifact path.
202
202
  :param format: The format of the artifact. (e.g. csv, json, html, etc.)
203
- :param upload: Whether to upload the artifact or not.
203
+ :param upload: Whether to upload the artifact to the datastore. If not provided, and the
204
+ `local_path` is not a directory, upload occurs by default. Directories are uploaded only when this
205
+ flag is explicitly set to `True`.
204
206
  :param labels: Labels to add to the artifact.
205
207
  :param db_key: The key to use when logging the artifact to the DB.
206
208
  If not provided, will generate a key based on the producer name and the artifact key.
@@ -18,3 +18,4 @@ from .function import FunctionFormat # noqa
18
18
  from .pipeline import PipelineFormat # noqa
19
19
  from .project import ProjectFormat # noqa
20
20
  from .run import RunFormat # noqa
21
+ from .feature_set import FeatureSetFormat # noqa
@@ -0,0 +1,33 @@
1
+ # Copyright 2024 Iguazio
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+
16
+ import typing
17
+
18
+ import mlrun.common.types
19
+
20
+ from .base import ObjectFormat
21
+
22
+
23
+ class FeatureSetFormat(ObjectFormat, mlrun.common.types.StrEnum):
24
+ minimal = "minimal"
25
+
26
+ @staticmethod
27
+ def format_method(_format: str) -> typing.Optional[typing.Callable]:
28
+ return {
29
+ FeatureSetFormat.full: None,
30
+ FeatureSetFormat.minimal: FeatureSetFormat.filter_obj_method(
31
+ ["kind", "metadata", "spec", "status.state"]
32
+ ),
33
+ }[_format]
@@ -175,6 +175,7 @@ from .project import (
175
175
  ProjectOwner,
176
176
  ProjectsOutput,
177
177
  ProjectSpec,
178
+ ProjectSpecOut,
178
179
  ProjectState,
179
180
  ProjectStatus,
180
181
  ProjectSummariesOutput,
@@ -34,17 +34,17 @@ class EventEntities(pydantic.BaseModel):
34
34
 
35
35
 
36
36
  class EventKind(StrEnum):
37
- DATA_DRIFT_DETECTED = "data_drift_detected"
38
- DATA_DRIFT_SUSPECTED = "data_drift_suspected"
39
- CONCEPT_DRIFT_DETECTED = "concept_drift_detected"
40
- CONCEPT_DRIFT_SUSPECTED = "concept_drift_suspected"
41
- MODEL_PERFORMANCE_DETECTED = "model_performance_detected"
42
- MODEL_PERFORMANCE_SUSPECTED = "model_performance_suspected"
43
- SYSTEM_PERFORMANCE_DETECTED = "system_performance_detected"
44
- SYSTEM_PERFORMANCE_SUSPECTED = "system_performance_suspected"
45
- MM_APP_ANOMALY_DETECTED = "mm_app_anomaly_detected"
46
- MM_APP_ANOMALY_SUSPECTED = "mm_app_anomaly_suspected"
47
- MM_APP_FAILED = "mm_app_failed"
37
+ DATA_DRIFT_DETECTED = "data-drift-detected"
38
+ DATA_DRIFT_SUSPECTED = "data-drift-suspected"
39
+ CONCEPT_DRIFT_DETECTED = "concept-drift-detected"
40
+ CONCEPT_DRIFT_SUSPECTED = "concept-drift-suspected"
41
+ MODEL_PERFORMANCE_DETECTED = "model-performance-detected"
42
+ MODEL_PERFORMANCE_SUSPECTED = "model-performance-suspected"
43
+ SYSTEM_PERFORMANCE_DETECTED = "system-performance-detected"
44
+ SYSTEM_PERFORMANCE_SUSPECTED = "system-performance-suspected"
45
+ MM_APP_ANOMALY_DETECTED = "mm-app-anomaly-detected"
46
+ MM_APP_ANOMALY_SUSPECTED = "mm-app-anomaly-suspected"
47
+ MM_APP_FAILED = "mm-app-failed"
48
48
  FAILED = "failed"
49
49
 
50
50
 
@@ -63,6 +63,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
63
63
  event = "event"
64
64
  datastore_profile = "datastore-profile"
65
65
  api_gateway = "api-gateway"
66
+ project_summaries = "project-summaries"
66
67
 
67
68
  def to_resource_string(
68
69
  self,
@@ -72,6 +73,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
72
73
  return {
73
74
  # project is the resource itself, so no need for both resource_name and project_name
74
75
  AuthorizationResourceTypes.project: "/projects/{project_name}",
76
+ AuthorizationResourceTypes.project_summaries: "/projects/{project_name}/project-summaries/{resource_name}",
75
77
  AuthorizationResourceTypes.function: "/projects/{project_name}/functions/{resource_name}",
76
78
  AuthorizationResourceTypes.artifact: "/projects/{project_name}/artifacts/{resource_name}",
77
79
  AuthorizationResourceTypes.project_background_task: (
@@ -57,7 +57,6 @@ class ClientSpec(pydantic.BaseModel):
57
57
  redis_url: typing.Optional[str]
58
58
  redis_type: typing.Optional[str]
59
59
  sql_url: typing.Optional[str]
60
- model_endpoint_monitoring_store_type: typing.Optional[str]
61
60
  model_endpoint_monitoring_endpoint_store_connection: typing.Optional[str]
62
61
  model_monitoring_tsdb_connection: typing.Optional[str]
63
62
  ce: typing.Optional[dict]
@@ -34,6 +34,7 @@ from .constants import (
34
34
  ProjectSecretKeys,
35
35
  ResultData,
36
36
  ResultKindApp,
37
+ ResultStatusApp,
37
38
  SchedulingKeys,
38
39
  SpecialApps,
39
40
  TDEngineSuperTables,
@@ -32,6 +32,7 @@ class WorkflowSpec(pydantic.BaseModel):
32
32
  schedule: typing.Union[str, ScheduleCronTrigger] = None
33
33
  run_local: typing.Optional[bool] = None
34
34
  image: typing.Optional[str] = None
35
+ workflow_runner_node_selector: typing.Optional[dict[str, str]] = None
35
36
 
36
37
 
37
38
  class WorkflowRequest(pydantic.BaseModel):
mlrun/config.py CHANGED
@@ -539,7 +539,6 @@ default_config = {
539
539
  "store_prefixes": {
540
540
  "default": "v3io:///users/pipelines/{project}/model-endpoints/{kind}",
541
541
  "user_space": "v3io:///projects/{project}/model-endpoints/{kind}",
542
- "stream": "", # TODO: Delete in 1.9.0
543
542
  "monitoring_application": "v3io:///users/pipelines/{project}/monitoring-apps/",
544
543
  },
545
544
  # Offline storage path can be either relative or a full path. This path is used for general offline data
@@ -552,7 +551,6 @@ default_config = {
552
551
  "parquet_batching_max_events": 10_000,
553
552
  "parquet_batching_timeout_secs": timedelta(minutes=1).total_seconds(),
554
553
  # See mlrun.model_monitoring.db.stores.ObjectStoreFactory for available options
555
- "store_type": "v3io-nosql", # TODO: Delete in 1.9.0
556
554
  "endpoint_store_connection": "",
557
555
  # See mlrun.model_monitoring.db.tsdb.ObjectTSDBFactory for available options
558
556
  "tsdb_connection": "",
@@ -736,7 +734,7 @@ default_config = {
736
734
  "grafana_url": "",
737
735
  "alerts": {
738
736
  # supported modes: "enabled", "disabled".
739
- "mode": "enabled",
737
+ "mode": "disabled",
740
738
  # maximum number of alerts we allow to be configured.
741
739
  # user will get an error when exceeding this
742
740
  "max_allowed": 10000,
@@ -798,7 +796,21 @@ class Config:
798
796
  for key, value in cfg.items():
799
797
  if hasattr(self, key):
800
798
  if isinstance(value, dict):
801
- getattr(self, key).update(value)
799
+ # ignore the `skip_errors` flag here
800
+ # if the key does not align with what mlrun config expects it is a user
801
+ # input error that can lead to unexpected behavior.
802
+ # raise the exception to ensure configuration is loaded correctly and do not
803
+ # ignore any errors.
804
+ config_value = getattr(self, key)
805
+ try:
806
+ config_value.update(value)
807
+ except AttributeError as exc:
808
+ if not isinstance(config_value, (dict, Config)):
809
+ raise ValueError(
810
+ f"Can not update `{key}` config. "
811
+ f"Expected a configuration but received {type(value)}"
812
+ ) from exc
813
+ raise exc
802
814
  else:
803
815
  try:
804
816
  setattr(self, key, value)
@@ -1102,6 +1114,9 @@ class Config:
1102
1114
  # importing here to avoid circular dependency
1103
1115
  import mlrun.db
1104
1116
 
1117
+ # It ensures that SSL verification is set before establishing a connection
1118
+ _configure_ssl_verification(self.httpdb.http.verify)
1119
+
1105
1120
  # when dbpath is set we want to connect to it which will sync configuration from it to the client
1106
1121
  mlrun.db.get_run_db(value, force_reconnect=True)
1107
1122
 
@@ -1130,10 +1145,10 @@ class Config:
1130
1145
  project: str = "",
1131
1146
  kind: str = "",
1132
1147
  target: str = "online",
1133
- artifact_path: str = None,
1134
- function_name: str = None,
1148
+ artifact_path: typing.Optional[str] = None,
1149
+ function_name: typing.Optional[str] = None,
1135
1150
  **kwargs,
1136
- ) -> typing.Union[str, list[str]]:
1151
+ ) -> str:
1137
1152
  """Get the full path from the configuration based on the provided project and kind.
1138
1153
 
1139
1154
  :param project: Project name.
@@ -1149,8 +1164,7 @@ class Config:
1149
1164
  relative artifact path will be taken from the global MLRun artifact path.
1150
1165
  :param function_name: Application name, None for model_monitoring_stream.
1151
1166
 
1152
- :return: Full configured path for the provided kind. Can be either a single path
1153
- or a list of paths in the case of the online model monitoring stream path.
1167
+ :return: Full configured path for the provided kind.
1154
1168
  """
1155
1169
 
1156
1170
  if target != "offline":
@@ -1171,18 +1185,11 @@ class Config:
1171
1185
  if function_name is None
1172
1186
  else f"{kind}-{function_name.lower()}",
1173
1187
  )
1174
- elif kind == "stream": # return list for mlrun<1.6.3 BC
1175
- return [
1176
- # TODO: remove the first stream in 1.9.0
1177
- mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
1178
- project=project,
1179
- kind=kind,
1180
- ), # old stream uri (pipelines) for BC ML-6043
1181
- mlrun.mlconf.model_endpoint_monitoring.store_prefixes.user_space.format(
1182
- project=project,
1183
- kind=kind,
1184
- ), # new stream uri (projects)
1185
- ]
1188
+ elif kind == "stream":
1189
+ return mlrun.mlconf.model_endpoint_monitoring.store_prefixes.user_space.format(
1190
+ project=project,
1191
+ kind=kind,
1192
+ )
1186
1193
  else:
1187
1194
  return mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
1188
1195
  project=project,
@@ -70,6 +70,11 @@ def pa_type_to_value_type(type_):
70
70
  if isinstance(type_, TimestampType):
71
71
  return ValueType.DATETIME
72
72
 
73
+ # pandas category type translates to pyarrow DictionaryType
74
+ # we need to unpack the value type (ML-7868)
75
+ if isinstance(type_, pyarrow.DictionaryType):
76
+ type_ = type_.value_type
77
+
73
78
  type_map = {
74
79
  pyarrow.bool_(): ValueType.BOOL,
75
80
  pyarrow.int64(): ValueType.INT64,
mlrun/datastore/base.py CHANGED
@@ -29,7 +29,7 @@ from deprecated import deprecated
29
29
  import mlrun.config
30
30
  import mlrun.errors
31
31
  from mlrun.errors import err_to_str
32
- from mlrun.utils import StorePrefix, is_ipython, logger
32
+ from mlrun.utils import StorePrefix, is_jupyter, logger
33
33
 
34
34
  from .store_resources import is_store_uri, parse_store_uri
35
35
  from .utils import filter_df_start_end_time, select_columns_from_df
@@ -619,14 +619,14 @@ class DataItem:
619
619
  )
620
620
  return df
621
621
 
622
- def show(self, format=None):
622
+ def show(self, format: Optional[str] = None) -> None:
623
623
  """show the data object content in Jupyter
624
624
 
625
625
  :param format: format to use (when there is no/wrong suffix), e.g. 'png'
626
626
  """
627
- if not is_ipython:
627
+ if not is_jupyter:
628
628
  logger.warning(
629
- "Jupyter/IPython was not detected, .show() will only display inside Jupyter"
629
+ "Jupyter was not detected. `.show()` displays only inside Jupyter."
630
630
  )
631
631
  return
632
632
 
mlrun/datastore/s3.py CHANGED
@@ -36,6 +36,7 @@ class S3Store(DataStore):
36
36
 
37
37
  access_key_id = self._get_secret_or_env("AWS_ACCESS_KEY_ID")
38
38
  secret_key = self._get_secret_or_env("AWS_SECRET_ACCESS_KEY")
39
+ token_file = self._get_secret_or_env("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")
39
40
  endpoint_url = self._get_secret_or_env("S3_ENDPOINT_URL")
40
41
  force_non_anonymous = self._get_secret_or_env("S3_NON_ANONYMOUS")
41
42
  profile_name = self._get_secret_or_env("AWS_PROFILE")
@@ -94,14 +95,15 @@ class S3Store(DataStore):
94
95
  self.s3 = boto3.resource(
95
96
  "s3", region_name=region, endpoint_url=endpoint_url
96
97
  )
97
- # If not using credentials, boto will still attempt to sign the requests, and will fail any operations
98
- # due to no credentials found. These commands disable signing and allow anonymous mode (same as
99
- # anon in the storage_options when working with fsspec).
100
- from botocore.handlers import disable_signing
101
-
102
- self.s3.meta.client.meta.events.register(
103
- "choose-signer.s3.*", disable_signing
104
- )
98
+ if not token_file:
99
+ # If not using credentials, boto will still attempt to sign the requests, and will fail any operations
100
+ # due to no credentials found. These commands disable signing and allow anonymous mode (same as
101
+ # anon in the storage_options when working with fsspec).
102
+ from botocore.handlers import disable_signing
103
+
104
+ self.s3.meta.client.meta.events.register(
105
+ "choose-signer.s3.*", disable_signing
106
+ )
105
107
 
106
108
  def get_spark_options(self):
107
109
  res = {}
@@ -139,6 +141,7 @@ class S3Store(DataStore):
139
141
  endpoint_url = self._get_secret_or_env("S3_ENDPOINT_URL")
140
142
  access_key_id = self._get_secret_or_env("AWS_ACCESS_KEY_ID")
141
143
  secret = self._get_secret_or_env("AWS_SECRET_ACCESS_KEY")
144
+ token_file = self._get_secret_or_env("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")
142
145
 
143
146
  if self._temp_credentials:
144
147
  access_key_id = self._temp_credentials["AccessKeyId"]
@@ -148,7 +151,7 @@ class S3Store(DataStore):
148
151
  token = None
149
152
 
150
153
  storage_options = dict(
151
- anon=not (force_non_anonymous or (access_key_id and secret)),
154
+ anon=not (force_non_anonymous or (access_key_id and secret) or token_file),
152
155
  key=access_key_id,
153
156
  secret=secret,
154
157
  token=token,
@@ -89,8 +89,8 @@ class StreamStoreyTarget(storey.StreamTarget):
89
89
  raise mlrun.errors.MLRunInvalidArgumentError("StreamTarget requires a path")
90
90
 
91
91
  access_key = storage_options.get("v3io_access_key")
92
- storage = (
93
- V3ioDriver(webapi=endpoint or mlrun.mlconf.v3io_api, access_key=access_key),
92
+ storage = V3ioDriver(
93
+ webapi=endpoint or mlrun.mlconf.v3io_api, access_key=access_key
94
94
  )
95
95
 
96
96
  if storage_options:
mlrun/db/base.py CHANGED
@@ -395,6 +395,9 @@ class RunDBInterface(ABC):
395
395
  partition_order: Union[
396
396
  mlrun.common.schemas.OrderType, str
397
397
  ] = mlrun.common.schemas.OrderType.desc,
398
+ format_: Union[
399
+ str, mlrun.common.formatters.FeatureSetFormat
400
+ ] = mlrun.common.formatters.FeatureSetFormat.full,
398
401
  ) -> list[dict]:
399
402
  pass
400
403
 
mlrun/db/httpdb.py CHANGED
@@ -525,10 +525,6 @@ class HTTPRunDB(RunDBInterface):
525
525
  server_cfg.get("external_platform_tracking")
526
526
  or config.external_platform_tracking
527
527
  )
528
- config.model_endpoint_monitoring.store_type = (
529
- server_cfg.get("model_endpoint_monitoring_store_type")
530
- or config.model_endpoint_monitoring.store_type
531
- )
532
528
  config.model_endpoint_monitoring.endpoint_store_connection = (
533
529
  server_cfg.get("model_endpoint_monitoring_endpoint_store_connection")
534
530
  or config.model_endpoint_monitoring.endpoint_store_connection
@@ -1374,20 +1370,14 @@ class HTTPRunDB(RunDBInterface):
1374
1370
  :returns: :py:class:`~mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput` listing the runtime resources
1375
1371
  that were removed.
1376
1372
  """
1377
- if grace_period is None:
1378
- grace_period = config.runtime_resources_deletion_grace_period
1379
- logger.info(
1380
- "Using default grace period for runtime resources deletion",
1381
- grace_period=grace_period,
1382
- )
1383
-
1384
1373
  params = {
1385
1374
  "label-selector": label_selector,
1386
1375
  "kind": kind,
1387
1376
  "object-id": object_id,
1388
1377
  "force": force,
1389
- "grace-period": grace_period,
1390
1378
  }
1379
+ if grace_period is not None:
1380
+ params["grace-period"] = grace_period
1391
1381
  error = "Failed deleting runtime resources"
1392
1382
  project_path = project if project else "*"
1393
1383
  response = self.api_call(
@@ -2245,6 +2235,9 @@ class HTTPRunDB(RunDBInterface):
2245
2235
  partition_order: Union[
2246
2236
  mlrun.common.schemas.OrderType, str
2247
2237
  ] = mlrun.common.schemas.OrderType.desc,
2238
+ format_: Union[
2239
+ str, mlrun.common.formatters.FeatureSetFormat
2240
+ ] = mlrun.common.formatters.FeatureSetFormat.full,
2248
2241
  ) -> list[FeatureSet]:
2249
2242
  """Retrieve a list of feature-sets matching the criteria provided.
2250
2243
 
@@ -2262,6 +2255,9 @@ class HTTPRunDB(RunDBInterface):
2262
2255
  :param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
2263
2256
  Currently the only allowed value are `created` and `updated`.
2264
2257
  :param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
2258
+ :param format_: Format of the results. Possible values are:
2259
+ - ``minimal`` - Return minimal feature set objects, not including stats and preview for each feature set.
2260
+ - ``full`` - Return full feature set objects.
2265
2261
  :returns: List of matching :py:class:`~mlrun.feature_store.FeatureSet` objects.
2266
2262
  """
2267
2263
 
@@ -2274,6 +2270,7 @@ class HTTPRunDB(RunDBInterface):
2274
2270
  "entity": entities or [],
2275
2271
  "feature": features or [],
2276
2272
  "label": labels or [],
2273
+ "format": format_,
2277
2274
  }
2278
2275
  if partition_by:
2279
2276
  params.update(
@@ -4193,6 +4190,9 @@ class HTTPRunDB(RunDBInterface):
4193
4190
  :param event_data: The data of the event.
4194
4191
  :param project: The project that the event belongs to.
4195
4192
  """
4193
+ if mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.disabled:
4194
+ logger.warning("Alerts are disabled, event will not be generated")
4195
+
4196
4196
  project = project or config.default_project
4197
4197
  endpoint_path = f"projects/{project}/events/{name}"
4198
4198
  error_message = f"post event {project}/events/{name}"
@@ -4219,6 +4219,11 @@ class HTTPRunDB(RunDBInterface):
4219
4219
  if not alert_data:
4220
4220
  raise mlrun.errors.MLRunInvalidArgumentError("Alert data must be provided")
4221
4221
 
4222
+ if mlrun.mlconf.alerts.mode == mlrun.common.schemas.alert.AlertsModes.disabled:
4223
+ logger.warning(
4224
+ "Alerts are disabled, alert will still be stored but will not be triggered"
4225
+ )
4226
+
4222
4227
  project = project or config.default_project
4223
4228
  endpoint_path = f"projects/{project}/alerts/{alert_name}"
4224
4229
  error_message = f"put alert {project}/alerts/{alert_name}"
mlrun/db/nopdb.py CHANGED
@@ -21,6 +21,7 @@ import mlrun.common.formatters
21
21
  import mlrun.common.runtimes.constants
22
22
  import mlrun.common.schemas
23
23
  import mlrun.errors
24
+ import mlrun.lists
24
25
 
25
26
  from ..config import config
26
27
  from ..utils import logger
@@ -73,6 +74,22 @@ class NopDB(RunDBInterface):
73
74
  def abort_run(self, uid, project="", iter=0, timeout=45, status_text=""):
74
75
  pass
75
76
 
77
+ def list_runtime_resources(
78
+ self,
79
+ project: Optional[str] = None,
80
+ label_selector: Optional[str] = None,
81
+ kind: Optional[str] = None,
82
+ object_id: Optional[str] = None,
83
+ group_by: Optional[
84
+ mlrun.common.schemas.ListRuntimeResourcesGroupByField
85
+ ] = None,
86
+ ) -> Union[
87
+ mlrun.common.schemas.RuntimeResourcesOutput,
88
+ mlrun.common.schemas.GroupedByJobRuntimeResourcesOutput,
89
+ mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput,
90
+ ]:
91
+ return []
92
+
76
93
  def read_run(
77
94
  self,
78
95
  uid,
@@ -108,7 +125,7 @@ class NopDB(RunDBInterface):
108
125
  max_partitions: int = 0,
109
126
  with_notifications: bool = False,
110
127
  ):
111
- pass
128
+ return mlrun.lists.RunList()
112
129
 
113
130
  def del_run(self, uid, project="", iter=0):
114
131
  pass
@@ -149,7 +166,7 @@ class NopDB(RunDBInterface):
149
166
  format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
150
167
  limit: int = None,
151
168
  ):
152
- pass
169
+ return mlrun.lists.ArtifactList()
153
170
 
154
171
  def del_artifact(
155
172
  self,
@@ -181,7 +198,7 @@ class NopDB(RunDBInterface):
181
198
  def list_functions(
182
199
  self, name=None, project="", tag="", labels=None, since=None, until=None
183
200
  ):
184
- pass
201
+ return []
185
202
 
186
203
  def tag_objects(
187
204
  self,
@@ -309,6 +326,9 @@ class NopDB(RunDBInterface):
309
326
  partition_order: Union[
310
327
  mlrun.common.schemas.OrderType, str
311
328
  ] = mlrun.common.schemas.OrderType.desc,
329
+ format_: Union[
330
+ str, mlrun.common.formatters.FeatureSetFormat
331
+ ] = mlrun.common.formatters.FeatureSetFormat.full,
312
332
  ) -> list[dict]:
313
333
  pass
314
334
 
@@ -421,7 +441,7 @@ class NopDB(RunDBInterface):
421
441
  ] = mlrun.common.formatters.PipelineFormat.metadata_only,
422
442
  page_size: int = None,
423
443
  ) -> mlrun.common.schemas.PipelinesOutput:
424
- pass
444
+ return mlrun.common.schemas.PipelinesOutput(runs=[], total_size=0)
425
445
 
426
446
  def create_project_secrets(
427
447
  self,
mlrun/execution.py CHANGED
@@ -634,7 +634,9 @@ class MLClientCtx:
634
634
  :param viewer: Kubeflow viewer type
635
635
  :param target_path: Absolute target path (instead of using artifact_path + local_path)
636
636
  :param src_path: Deprecated, use local_path
637
- :param upload: Upload to datastore (default is True)
637
+ :param upload: Whether to upload the artifact to the datastore. If not provided, and the `local_path`
638
+ is not a directory, upload occurs by default. Directories are uploaded only when this
639
+ flag is explicitly set to `True`.
638
640
  :param labels: A set of key/value labels to tag the artifact with
639
641
  :param format: Optional, format to use (e.g. csv, parquet, ..)
640
642
  :param db_key: The key to use in the artifact DB table, by default its run name + '_' + key
@@ -1051,6 +1051,7 @@ def _ingest_with_spark(
1051
1051
 
1052
1052
  spark = (
1053
1053
  pyspark.sql.SparkSession.builder.appName(session_name)
1054
+ .config("spark.driver.memory", "2g")
1054
1055
  .config("spark.sql.session.timeZone", "UTC")
1055
1056
  .getOrCreate()
1056
1057
  )
@@ -188,9 +188,13 @@ class SparkFeatureMerger(BaseMerger):
188
188
 
189
189
  if self.spark is None:
190
190
  # create spark context
191
- self.spark = SparkSession.builder.appName(
192
- f"vector-merger-{self.vector.metadata.name}"
193
- ).getOrCreate()
191
+ self.spark = (
192
+ SparkSession.builder.appName(
193
+ f"vector-merger-{self.vector.metadata.name}"
194
+ )
195
+ .config("spark.driver.memory", "2g")
196
+ .getOrCreate()
197
+ )
194
198
 
195
199
  def _get_engine_df(
196
200
  self,
@@ -11,12 +11,12 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  from abc import ABC, abstractmethod
16
16
 
17
17
  import mlrun
18
18
  from mlrun.artifacts import Artifact
19
- from mlrun.utils.helpers import is_ipython
19
+ from mlrun.utils.helpers import is_jupyter
20
20
 
21
21
 
22
22
  class Plan(ABC):
@@ -84,7 +84,7 @@ class Plan(ABC):
84
84
  return
85
85
 
86
86
  # Call the correct display method according to the kernel:
87
- if is_ipython:
87
+ if is_jupyter:
88
88
  self._gui_display()
89
89
  else:
90
90
  self._cli_display()
@@ -16,7 +16,7 @@ import json
16
16
  from abc import ABC, abstractmethod
17
17
  from enum import Enum
18
18
 
19
- from IPython.core.display import HTML, display
19
+ from IPython.display import HTML, display
20
20
 
21
21
  import mlrun
22
22
 
@@ -18,8 +18,7 @@ from typing import Union
18
18
 
19
19
  import numpy as np
20
20
  import pandas as pd
21
- from IPython.core.display import HTML
22
- from IPython.display import display
21
+ from IPython.display import HTML, display
23
22
  from pandas.api.types import is_numeric_dtype, is_string_dtype
24
23
 
25
24
  import mlrun
@@ -216,7 +215,7 @@ def _show_and_export_html(html: str, show=None, filename=None, runs_list=None):
216
215
  fp.write("</body></html>")
217
216
  else:
218
217
  fp.write(html)
219
- if show or (show is None and mlrun.utils.is_ipython):
218
+ if show or (show is None and mlrun.utils.is_jupyter):
220
219
  display(HTML(html))
221
220
  if runs_list and len(runs_list) <= max_table_rows:
222
221
  display(HTML(html_table))