mlrun 1.9.0rc4__py3-none-any.whl → 1.10.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mlrun might be problematic. Click here for more details.

Files changed (193) hide show
  1. mlrun/__main__.py +13 -2
  2. mlrun/api/schemas/__init__.py +0 -1
  3. mlrun/common/__init__.py +0 -1
  4. mlrun/common/constants.py +7 -0
  5. mlrun/common/db/__init__.py +0 -1
  6. mlrun/common/db/sql_session.py +0 -1
  7. mlrun/common/formatters/__init__.py +0 -1
  8. mlrun/common/formatters/artifact.py +0 -1
  9. mlrun/common/formatters/base.py +0 -1
  10. mlrun/common/formatters/feature_set.py +0 -1
  11. mlrun/common/formatters/function.py +0 -1
  12. mlrun/common/formatters/model_endpoint.py +0 -1
  13. mlrun/common/formatters/pipeline.py +0 -1
  14. mlrun/common/formatters/project.py +0 -1
  15. mlrun/common/formatters/run.py +0 -2
  16. mlrun/common/runtimes/constants.py +9 -2
  17. mlrun/common/schemas/__init__.py +2 -0
  18. mlrun/common/schemas/alert.py +1 -1
  19. mlrun/common/schemas/api_gateway.py +1 -1
  20. mlrun/common/schemas/artifact.py +1 -1
  21. mlrun/common/schemas/auth.py +1 -1
  22. mlrun/common/schemas/background_task.py +1 -1
  23. mlrun/common/schemas/client_spec.py +1 -1
  24. mlrun/common/schemas/clusterization_spec.py +1 -1
  25. mlrun/common/schemas/constants.py +1 -1
  26. mlrun/common/schemas/datastore_profile.py +0 -1
  27. mlrun/common/schemas/events.py +1 -1
  28. mlrun/common/schemas/feature_store.py +1 -1
  29. mlrun/common/schemas/frontend_spec.py +1 -1
  30. mlrun/common/schemas/function.py +1 -1
  31. mlrun/common/schemas/http.py +1 -1
  32. mlrun/common/schemas/hub.py +1 -1
  33. mlrun/common/schemas/k8s.py +1 -1
  34. mlrun/common/schemas/memory_reports.py +0 -1
  35. mlrun/common/schemas/notification.py +4 -0
  36. mlrun/common/schemas/object.py +1 -1
  37. mlrun/common/schemas/partition.py +1 -1
  38. mlrun/common/schemas/pipeline.py +1 -1
  39. mlrun/common/schemas/project.py +1 -1
  40. mlrun/common/schemas/regex.py +1 -1
  41. mlrun/common/schemas/runtime_resource.py +1 -1
  42. mlrun/common/schemas/schedule.py +1 -1
  43. mlrun/common/schemas/secret.py +1 -1
  44. mlrun/common/schemas/serving.py +17 -0
  45. mlrun/common/schemas/tag.py +0 -1
  46. mlrun/common/schemas/workflow.py +1 -1
  47. mlrun/common/secrets.py +0 -1
  48. mlrun/config.py +9 -11
  49. mlrun/data_types/infer.py +1 -1
  50. mlrun/data_types/spark.py +1 -1
  51. mlrun/datastore/datastore.py +1 -1
  52. mlrun/datastore/datastore_profile.py +5 -55
  53. mlrun/datastore/snowflake_utils.py +0 -1
  54. mlrun/datastore/sources.py +21 -13
  55. mlrun/datastore/spark_utils.py +0 -1
  56. mlrun/datastore/utils.py +20 -5
  57. mlrun/db/base.py +1 -1
  58. mlrun/db/httpdb.py +17 -12
  59. mlrun/db/nopdb.py +1 -2
  60. mlrun/errors.py +8 -1
  61. mlrun/execution.py +22 -1
  62. mlrun/feature_store/feature_set.py +0 -12
  63. mlrun/feature_store/retrieval/base.py +1 -1
  64. mlrun/feature_store/retrieval/dask_merger.py +1 -1
  65. mlrun/feature_store/retrieval/job.py +1 -1
  66. mlrun/feature_store/retrieval/spark_merger.py +0 -2
  67. mlrun/feature_store/steps.py +1 -1
  68. mlrun/features.py +1 -1
  69. mlrun/frameworks/_common/artifacts_library.py +1 -1
  70. mlrun/frameworks/_common/mlrun_interface.py +1 -1
  71. mlrun/frameworks/_common/model_handler.py +3 -3
  72. mlrun/frameworks/_common/producer.py +0 -1
  73. mlrun/frameworks/_common/utils.py +1 -1
  74. mlrun/frameworks/_dl_common/loggers/logger.py +0 -1
  75. mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
  76. mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
  77. mlrun/frameworks/_dl_common/model_handler.py +1 -1
  78. mlrun/frameworks/_dl_common/utils.py +1 -1
  79. mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
  80. mlrun/frameworks/_ml_common/loggers/logger.py +0 -1
  81. mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
  82. mlrun/frameworks/_ml_common/model_handler.py +1 -1
  83. mlrun/frameworks/_ml_common/pkl_model_server.py +1 -1
  84. mlrun/frameworks/_ml_common/plan.py +1 -1
  85. mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +0 -1
  86. mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +0 -1
  87. mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
  88. mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
  89. mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
  90. mlrun/frameworks/_ml_common/producer.py +1 -1
  91. mlrun/frameworks/_ml_common/utils.py +1 -1
  92. mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
  93. mlrun/frameworks/lgbm/callbacks/logging_callback.py +0 -1
  94. mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +0 -1
  95. mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
  96. mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
  97. mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
  98. mlrun/frameworks/lgbm/model_handler.py +1 -1
  99. mlrun/frameworks/lgbm/model_server.py +1 -1
  100. mlrun/frameworks/lgbm/utils.py +1 -1
  101. mlrun/frameworks/onnx/dataset.py +1 -1
  102. mlrun/frameworks/onnx/mlrun_interface.py +1 -1
  103. mlrun/frameworks/onnx/model_handler.py +1 -1
  104. mlrun/frameworks/onnx/model_server.py +1 -1
  105. mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
  106. mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
  107. mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
  108. mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
  109. mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
  110. mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
  111. mlrun/frameworks/pytorch/model_handler.py +1 -1
  112. mlrun/frameworks/pytorch/model_server.py +1 -1
  113. mlrun/frameworks/pytorch/utils.py +1 -1
  114. mlrun/frameworks/sklearn/__init__.py +0 -14
  115. mlrun/frameworks/sklearn/estimator.py +1 -1
  116. mlrun/frameworks/sklearn/metric.py +1 -1
  117. mlrun/frameworks/sklearn/metrics_library.py +1 -1
  118. mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
  119. mlrun/frameworks/sklearn/model_handler.py +1 -1
  120. mlrun/frameworks/sklearn/utils.py +1 -1
  121. mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
  122. mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
  123. mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
  124. mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
  125. mlrun/frameworks/tf_keras/model_handler.py +1 -1
  126. mlrun/frameworks/tf_keras/model_server.py +1 -1
  127. mlrun/frameworks/tf_keras/utils.py +1 -1
  128. mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
  129. mlrun/frameworks/xgboost/model_handler.py +1 -1
  130. mlrun/frameworks/xgboost/utils.py +1 -1
  131. mlrun/k8s_utils.py +340 -0
  132. mlrun/launcher/base.py +3 -3
  133. mlrun/launcher/client.py +1 -1
  134. mlrun/launcher/local.py +2 -2
  135. mlrun/launcher/remote.py +2 -2
  136. mlrun/model.py +14 -0
  137. mlrun/model_monitoring/applications/__init__.py +0 -1
  138. mlrun/model_monitoring/applications/_application_steps.py +3 -1
  139. mlrun/model_monitoring/applications/evidently/base.py +59 -71
  140. mlrun/model_monitoring/controller.py +26 -13
  141. mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +13 -5
  142. mlrun/package/context_handler.py +1 -1
  143. mlrun/package/errors.py +1 -1
  144. mlrun/package/packager.py +1 -1
  145. mlrun/package/packagers/default_packager.py +1 -1
  146. mlrun/package/packagers/numpy_packagers.py +1 -1
  147. mlrun/package/packagers/pandas_packagers.py +1 -1
  148. mlrun/package/packagers/python_standard_library_packagers.py +1 -1
  149. mlrun/package/packagers_manager.py +1 -1
  150. mlrun/package/utils/_archiver.py +1 -1
  151. mlrun/package/utils/_formatter.py +1 -1
  152. mlrun/package/utils/_pickler.py +1 -1
  153. mlrun/package/utils/_supported_format.py +1 -1
  154. mlrun/package/utils/log_hint_utils.py +1 -1
  155. mlrun/package/utils/type_hint_utils.py +1 -1
  156. mlrun/projects/operations.py +36 -21
  157. mlrun/projects/project.py +147 -92
  158. mlrun/render.py +5 -9
  159. mlrun/run.py +1 -1
  160. mlrun/runtimes/base.py +17 -7
  161. mlrun/runtimes/daskjob.py +2 -1
  162. mlrun/runtimes/databricks_job/databricks_cancel_task.py +0 -1
  163. mlrun/runtimes/databricks_job/databricks_runtime.py +2 -1
  164. mlrun/runtimes/databricks_job/databricks_wrapper.py +0 -1
  165. mlrun/runtimes/mounts.py +2 -0
  166. mlrun/runtimes/nuclio/function.py +5 -0
  167. mlrun/runtimes/nuclio/serving.py +1 -1
  168. mlrun/runtimes/pod.py +4 -349
  169. mlrun/runtimes/sparkjob/spark3job.py +0 -12
  170. mlrun/runtimes/utils.py +25 -8
  171. mlrun/serving/merger.py +0 -1
  172. mlrun/serving/remote.py +1 -1
  173. mlrun/serving/serving_wrapper.py +1 -1
  174. mlrun/serving/states.py +142 -9
  175. mlrun/serving/utils.py +1 -1
  176. mlrun/utils/async_http.py +0 -1
  177. mlrun/utils/clones.py +1 -1
  178. mlrun/utils/db.py +1 -1
  179. mlrun/utils/helpers.py +3 -1
  180. mlrun/utils/http.py +0 -1
  181. mlrun/utils/regex.py +0 -1
  182. mlrun/utils/singleton.py +1 -1
  183. mlrun/utils/vault.py +1 -1
  184. mlrun/utils/version/__init__.py +1 -1
  185. mlrun/utils/version/version.json +2 -2
  186. mlrun/utils/version/version.py +1 -1
  187. {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/METADATA +12 -12
  188. mlrun-1.10.0rc2.dist-info/RECORD +351 -0
  189. {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/WHEEL +1 -1
  190. mlrun-1.9.0rc4.dist-info/RECORD +0 -351
  191. {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/entry_points.txt +0 -0
  192. {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/licenses/LICENSE +0 -0
  193. {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/top_level.txt +0 -0
@@ -16,7 +16,6 @@ import ast
16
16
  import base64
17
17
  import json
18
18
  import typing
19
- import warnings
20
19
  from urllib.parse import ParseResult, urlparse
21
20
 
22
21
  import pydantic.v1
@@ -142,7 +141,6 @@ class ConfigProfile(DatastoreProfile):
142
141
  class DatastoreProfileKafkaTarget(DatastoreProfile):
143
142
  type: str = pydantic.v1.Field("kafka_target")
144
143
  _private_attributes = "kwargs_private"
145
- bootstrap_servers: typing.Optional[str] = None
146
144
  brokers: typing.Optional[str] = None
147
145
  topic: str
148
146
  kwargs_public: typing.Optional[dict]
@@ -151,31 +149,16 @@ class DatastoreProfileKafkaTarget(DatastoreProfile):
151
149
  def __init__(self, **kwargs):
152
150
  super().__init__(**kwargs)
153
151
 
154
- if not self.brokers and not self.bootstrap_servers:
152
+ if not self.brokers:
155
153
  raise mlrun.errors.MLRunInvalidArgumentError(
156
154
  "DatastoreProfileKafkaTarget requires the 'brokers' field to be set"
157
155
  )
158
156
 
159
- if self.bootstrap_servers:
160
- if self.brokers:
161
- raise mlrun.errors.MLRunInvalidArgumentError(
162
- "DatastoreProfileKafkaTarget cannot be created with both 'brokers' and 'bootstrap_servers'"
163
- )
164
- else:
165
- self.brokers = self.bootstrap_servers
166
- self.bootstrap_servers = None
167
- warnings.warn(
168
- "'bootstrap_servers' parameter is deprecated in 1.7.0 and will be removed in 1.9.0, "
169
- "use 'brokers' instead.",
170
- # TODO: Remove this in 1.9.0
171
- FutureWarning,
172
- )
173
-
174
157
  def get_topic(self) -> typing.Optional[str]:
175
158
  return self.topic
176
159
 
177
160
  def attributes(self):
178
- attributes = {"brokers": self.brokers or self.bootstrap_servers}
161
+ attributes = {"brokers": self.brokers}
179
162
  if self.kwargs_public:
180
163
  attributes = merge(attributes, self.kwargs_public)
181
164
  if self.kwargs_private:
@@ -248,18 +231,7 @@ class DatastoreProfileS3(DatastoreProfile):
248
231
  assume_role_arn: typing.Optional[str] = None
249
232
  access_key_id: typing.Optional[str] = None
250
233
  secret_key: typing.Optional[str] = None
251
- bucket: typing.Optional[str] = None
252
-
253
- @pydantic.v1.validator("bucket")
254
- @classmethod
255
- def check_bucket(cls, v):
256
- if not v:
257
- warnings.warn(
258
- "The 'bucket' attribute will be mandatory starting from version 1.9",
259
- FutureWarning,
260
- stacklevel=2,
261
- )
262
- return v
234
+ bucket: str
263
235
 
264
236
  def secrets(self) -> dict:
265
237
  res = {}
@@ -353,18 +325,7 @@ class DatastoreProfileGCS(DatastoreProfile):
353
325
  _private_attributes = ("gcp_credentials",)
354
326
  credentials_path: typing.Optional[str] = None # path to file.
355
327
  gcp_credentials: typing.Optional[typing.Union[str, dict]] = None
356
- bucket: typing.Optional[str] = None
357
-
358
- @pydantic.v1.validator("bucket")
359
- @classmethod
360
- def check_bucket(cls, v):
361
- if not v:
362
- warnings.warn(
363
- "The 'bucket' attribute will be mandatory starting from version 1.9",
364
- FutureWarning,
365
- stacklevel=2,
366
- )
367
- return v
328
+ bucket: str
368
329
 
369
330
  @pydantic.v1.validator("gcp_credentials", pre=True, always=True)
370
331
  @classmethod
@@ -410,18 +371,7 @@ class DatastoreProfileAzureBlob(DatastoreProfile):
410
371
  client_secret: typing.Optional[str] = None
411
372
  sas_token: typing.Optional[str] = None
412
373
  credential: typing.Optional[str] = None
413
- container: typing.Optional[str] = None
414
-
415
- @pydantic.v1.validator("container")
416
- @classmethod
417
- def check_container(cls, v):
418
- if not v:
419
- warnings.warn(
420
- "The 'container' attribute will be mandatory starting from version 1.9",
421
- FutureWarning,
422
- stacklevel=2,
423
- )
424
- return v
374
+ container: str
425
375
 
426
376
  def url(self, subpath) -> str:
427
377
  if subpath.startswith("/"):
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
14
 
16
15
  import mlrun
17
16
 
@@ -11,6 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
+
14
15
  import json
15
16
  import operator
16
17
  import os
@@ -18,7 +19,7 @@ import warnings
18
19
  from base64 import b64encode
19
20
  from copy import copy
20
21
  from datetime import datetime
21
- from typing import Any, Optional, Union
22
+ from typing import Any, Literal, Optional, Union
22
23
 
23
24
  import pandas as pd
24
25
  import semver
@@ -1063,16 +1064,17 @@ class KafkaSource(OnlineSource):
1063
1064
 
1064
1065
  def __init__(
1065
1066
  self,
1066
- brokers=None,
1067
- topics=None,
1068
- group="serving",
1069
- initial_offset="earliest",
1070
- partitions=None,
1071
- sasl_user=None,
1072
- sasl_pass=None,
1073
- attributes=None,
1067
+ brokers: Optional[list[str]] = None,
1068
+ topics: Optional[list[str]] = None,
1069
+ group: str = "serving",
1070
+ initial_offset: Literal["earliest", "latest"] = "earliest",
1071
+ partitions: Optional[list[int]] = None,
1072
+ sasl_user: Optional[str] = None,
1073
+ sasl_pass: Optional[str] = None,
1074
+ tls_enable: Optional[bool] = None,
1075
+ attributes: Optional[dict] = None,
1074
1076
  **kwargs,
1075
- ):
1077
+ ) -> None:
1076
1078
  """Sets kafka source for the flow
1077
1079
 
1078
1080
  :param brokers: list of broker IP addresses
@@ -1082,6 +1084,7 @@ class KafkaSource(OnlineSource):
1082
1084
  :param partitions: Optional, A list of partitions numbers for which the function receives events.
1083
1085
  :param sasl_user: Optional, user name to use for sasl authentications
1084
1086
  :param sasl_pass: Optional, password to use for sasl authentications
1087
+ :param tls_enable: Optional, if set - whether to enable TLS or not.
1085
1088
  :param attributes: Optional, extra attributes to be passed to kafka trigger
1086
1089
  """
1087
1090
  if isinstance(topics, str):
@@ -1095,10 +1098,15 @@ class KafkaSource(OnlineSource):
1095
1098
  attributes["initial_offset"] = initial_offset
1096
1099
  if partitions is not None:
1097
1100
  attributes["partitions"] = partitions
1098
- if sasl := mlrun.datastore.utils.KafkaParameters(attributes).sasl(
1099
- usr=sasl_user, pwd=sasl_pass
1100
- ):
1101
+
1102
+ kafka_params = mlrun.datastore.utils.KafkaParameters(attributes)
1103
+
1104
+ if sasl := kafka_params.sasl(usr=sasl_user, pwd=sasl_pass):
1101
1105
  attributes["sasl"] = sasl
1106
+
1107
+ if tls := kafka_params.tls(tls_enable=tls_enable):
1108
+ attributes["tls"] = tls
1109
+
1102
1110
  super().__init__(attributes=attributes, **kwargs)
1103
1111
 
1104
1112
  def to_dataframe(
@@ -12,7 +12,6 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
-
16
15
  from typing import Union
17
16
 
18
17
  import mlrun
mlrun/datastore/utils.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import math
16
16
  import tarfile
17
17
  import tempfile
@@ -246,6 +246,9 @@ class KafkaParameters:
246
246
  "partitions": "",
247
247
  "sasl": "",
248
248
  "worker_allocation_mode": "",
249
+ "tls_enable": "", # for Nuclio with Confluent Kafka (Sarama client)
250
+ "tls": "",
251
+ "new_topic": "",
249
252
  }
250
253
  self._reference_dicts = (
251
254
  self._custom_attributes,
@@ -270,7 +273,9 @@ class KafkaParameters:
270
273
  }
271
274
  if sasl := self._kwargs.get("sasl"):
272
275
  res |= {
273
- "security_protocol": "SASL_PLAINTEXT",
276
+ "security_protocol": self._kwargs.get(
277
+ "security_protocol", "SASL_PLAINTEXT"
278
+ ),
274
279
  "sasl_mechanism": sasl["mechanism"],
275
280
  "sasl_plain_username": sasl["user"],
276
281
  "sasl_plain_password": sasl["password"],
@@ -288,15 +293,25 @@ class KafkaParameters:
288
293
 
289
294
  def sasl(
290
295
  self, *, usr: typing.Optional[str] = None, pwd: typing.Optional[str] = None
291
- ) -> dict:
292
- usr = usr or self._kwargs.get("sasl_plain_username", None)
293
- pwd = pwd or self._kwargs.get("sasl_plain_password", None)
296
+ ) -> dict[str, typing.Union[str, bool]]:
294
297
  res = self._kwargs.get("sasl", {})
298
+ usr = usr or self._kwargs.get("sasl_plain_username")
299
+ pwd = pwd or self._kwargs.get("sasl_plain_password")
295
300
  if usr and pwd:
296
301
  res["enable"] = True
297
302
  res["user"] = usr
298
303
  res["password"] = pwd
299
304
  res["mechanism"] = self._kwargs.get("sasl_mechanism", "PLAIN")
305
+ res["handshake"] = self._kwargs.get("sasl_handshake", True)
306
+ return res
307
+
308
+ def tls(self, *, tls_enable: typing.Optional[bool] = None) -> dict[str, bool]:
309
+ res = self._kwargs.get("tls", {})
310
+ tls_enable = (
311
+ tls_enable if tls_enable is not None else self._kwargs.get("tls_enable")
312
+ )
313
+ if tls_enable:
314
+ res["enable"] = tls_enable
300
315
  return res
301
316
 
302
317
  def valid_entries_only(self, input_dict: dict) -> dict:
mlrun/db/base.py CHANGED
@@ -734,7 +734,7 @@ class RunDBInterface(ABC):
734
734
  labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
735
735
  start: Optional[datetime.datetime] = None,
736
736
  end: Optional[datetime.datetime] = None,
737
- tsdb_metrics: bool = True,
737
+ tsdb_metrics: bool = False,
738
738
  metric_list: Optional[list[str]] = None,
739
739
  top_level: bool = False,
740
740
  uids: Optional[list[str]] = None,
mlrun/db/httpdb.py CHANGED
@@ -21,7 +21,7 @@ import typing
21
21
  import warnings
22
22
  from copy import deepcopy
23
23
  from datetime import datetime, timedelta
24
- from os import path, remove
24
+ from os import environ, path, remove
25
25
  from typing import Literal, Optional, Union
26
26
  from urllib.parse import urlparse
27
27
 
@@ -129,7 +129,9 @@ class HTTPRunDB(RunDBInterface):
129
129
  self._wait_for_background_task_terminal_state_retry_interval = 3
130
130
  self._wait_for_project_deletion_interval = 3
131
131
  self.client_version = version.Version().get()["version"]
132
- self.python_version = str(version.Version().get_python_version())
132
+ self.python_version = environ.get("MLRUN_PYTHON_VERSION") or str(
133
+ version.Version().get_python_version()
134
+ )
133
135
 
134
136
  self._enrich_and_validate(url)
135
137
 
@@ -937,7 +939,7 @@ class HTTPRunDB(RunDBInterface):
937
939
 
938
940
  :param name: Name of the run to retrieve.
939
941
  :param uid: Unique ID of the run, or a list of run UIDs.
940
- :param project: Project that the runs belongs to.
942
+ :param project: Project that the runs belongs to. If not specified, the default project will be used.
941
943
  :param labels: Filter runs by label key-value pairs or key existence. This can be provided as:
942
944
  - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
943
945
  or `{"label": None}` to check for key existence.
@@ -1277,7 +1279,7 @@ class HTTPRunDB(RunDBInterface):
1277
1279
  points to a run and is used to filter artifacts by the run that produced them when the artifact producer id
1278
1280
  is a workflow id (artifact was created as part of a workflow).
1279
1281
  :param format_: The format in which to return the artifacts. Default is 'full'.
1280
- :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.10.0).
1282
+ :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
1281
1283
  :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
1282
1284
  parameter must be provided as well.
1283
1285
  :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
@@ -2221,18 +2223,20 @@ class HTTPRunDB(RunDBInterface):
2221
2223
  elif pipe_file.endswith(".zip"):
2222
2224
  headers = {"content-type": "application/zip"}
2223
2225
  else:
2224
- raise ValueError("pipeline file must be .yaml or .zip")
2226
+ raise ValueError("'pipeline' file must be .yaml or .zip")
2225
2227
  if arguments:
2226
2228
  if not isinstance(arguments, dict):
2227
- raise ValueError("arguments must be dict type")
2229
+ raise ValueError("'arguments' must be dict type")
2228
2230
  headers[mlrun.common.schemas.HeaderNames.pipeline_arguments] = str(
2229
2231
  arguments
2230
2232
  )
2231
2233
 
2232
2234
  if not path.isfile(pipe_file):
2233
- raise OSError(f"file {pipe_file} doesnt exist")
2235
+ raise OSError(f"File {pipe_file} doesnt exist")
2234
2236
  with open(pipe_file, "rb") as fp:
2235
2237
  data = fp.read()
2238
+ if not data:
2239
+ raise ValueError("The compiled pipe file is empty")
2236
2240
  if not isinstance(pipeline, str):
2237
2241
  remove(pipe_file)
2238
2242
 
@@ -3767,7 +3771,7 @@ class HTTPRunDB(RunDBInterface):
3767
3771
  labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
3768
3772
  start: Optional[datetime] = None,
3769
3773
  end: Optional[datetime] = None,
3770
- tsdb_metrics: bool = True,
3774
+ tsdb_metrics: bool = False,
3771
3775
  metric_list: Optional[list[str]] = None,
3772
3776
  top_level: bool = False,
3773
3777
  uids: Optional[list[str]] = None,
@@ -3889,8 +3893,8 @@ class HTTPRunDB(RunDBInterface):
3889
3893
  attributes_keys = list(attributes.keys())
3890
3894
  attributes["name"] = name
3891
3895
  attributes["project"] = project
3892
- attributes["function-name"] = function_name or None
3893
- attributes["function-tag"] = function_tag or None
3896
+ attributes["function_name"] = function_name or None
3897
+ attributes["function_tag"] = function_tag or None
3894
3898
  attributes["uid"] = endpoint_id or None
3895
3899
  model_endpoint = mlrun.common.schemas.ModelEndpoint.from_flat_dict(attributes)
3896
3900
  path = f"projects/{project}/model-endpoints"
@@ -3981,6 +3985,7 @@ class HTTPRunDB(RunDBInterface):
3981
3985
  "deploy_histogram_data_drift_app": deploy_histogram_data_drift_app,
3982
3986
  "fetch_credentials_from_sys_config": fetch_credentials_from_sys_config,
3983
3987
  },
3988
+ timeout=300, # 5 minutes
3984
3989
  )
3985
3990
 
3986
3991
  def disable_model_monitoring(
@@ -5100,9 +5105,9 @@ class HTTPRunDB(RunDBInterface):
5100
5105
  labels = self._parse_labels(labels)
5101
5106
 
5102
5107
  if limit:
5103
- # TODO: Remove this in 1.10.0
5108
+ # TODO: Remove this in 1.11.0
5104
5109
  warnings.warn(
5105
- "'limit' is deprecated and will be removed in 1.10.0. Use 'page' and 'page_size' instead.",
5110
+ "'limit' is deprecated and will be removed in 1.11.0. Use 'page' and 'page_size' instead.",
5106
5111
  FutureWarning,
5107
5112
  )
5108
5113
 
mlrun/db/nopdb.py CHANGED
@@ -12,7 +12,6 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
-
16
15
  import datetime
17
16
  from typing import Optional, Union
18
17
 
@@ -631,7 +630,7 @@ class NopDB(RunDBInterface):
631
630
  labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
632
631
  start: Optional[datetime.datetime] = None,
633
632
  end: Optional[datetime.datetime] = None,
634
- tsdb_metrics: bool = True,
633
+ tsdb_metrics: bool = False,
635
634
  metric_list: Optional[list[str]] = None,
636
635
  top_level: bool = False,
637
636
  uids: Optional[list[str]] = None,
mlrun/errors.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import typing
16
16
  from http import HTTPStatus
17
17
 
@@ -230,6 +230,13 @@ class MLRunTSDBConnectionFailureError(MLRunHTTPStatusError, ValueError):
230
230
  error_status_code = HTTPStatus.BAD_REQUEST.value
231
231
 
232
232
 
233
+ class MLRunMissingProjectError(MLRunBadRequestError):
234
+ default_message = "Project must be provided"
235
+
236
+ def __init__(self, message=None):
237
+ super().__init__(message or self.default_message)
238
+
239
+
233
240
  class MLRunRetryExhaustedError(Exception):
234
241
  pass
235
242
 
mlrun/execution.py CHANGED
@@ -94,6 +94,8 @@ class MLClientCtx:
94
94
  self._labels = {}
95
95
  self._annotations = {}
96
96
  self._node_selector = {}
97
+ self._tolerations = {}
98
+ self._affinity = {}
97
99
 
98
100
  self._function = ""
99
101
  self._parameters = {}
@@ -227,11 +229,26 @@ class MLClientCtx:
227
229
  """Dictionary with node selectors (read-only)"""
228
230
  return deepcopy(self._node_selector)
229
231
 
232
+ @property
233
+ def tolerations(self):
234
+ """Dictionary with tolerations (read-only)"""
235
+ return deepcopy(self._tolerations)
236
+
237
+ @property
238
+ def affinity(self):
239
+ """Dictionary with affinities (read-only)"""
240
+ return deepcopy(self._affinity)
241
+
230
242
  @property
231
243
  def annotations(self):
232
244
  """Dictionary with annotations (read-only)"""
233
245
  return deepcopy(self._annotations)
234
246
 
247
+ @property
248
+ def host(self):
249
+ """Execution host"""
250
+ return self._host
251
+
235
252
  def get_child_context(self, with_parent_params=False, **params):
236
253
  """Get child context (iteration)
237
254
 
@@ -411,6 +428,8 @@ class MLClientCtx:
411
428
  "state_thresholds", self._state_thresholds
412
429
  )
413
430
  self._node_selector = spec.get("node_selector", self._node_selector)
431
+ self._tolerations = spec.get("tolerations", self._tolerations)
432
+ self._affinity = spec.get("affinity", self._affinity)
414
433
  self._reset_on_run = spec.get("reset_on_run", self._reset_on_run)
415
434
 
416
435
  self._init_dbs(rundb)
@@ -976,7 +995,7 @@ class MLClientCtx:
976
995
  def get_cached_artifact(self, key):
977
996
  """Return a logged artifact from cache (for potential updates)"""
978
997
  warnings.warn(
979
- "get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.10.0. Use get_artifact instead.",
998
+ "get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.11.0. Use get_artifact instead.",
980
999
  FutureWarning,
981
1000
  )
982
1001
  return self.get_artifact(key)
@@ -1134,6 +1153,8 @@ class MLClientCtx:
1134
1153
  "notifications": self._notifications,
1135
1154
  "state_thresholds": self._state_thresholds,
1136
1155
  "node_selector": self._node_selector,
1156
+ "tolerations": self._tolerations,
1157
+ "affinity": self._affinity,
1137
1158
  },
1138
1159
  "status": {
1139
1160
  "results": self._results,
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- import warnings
15
14
  from datetime import datetime
16
15
  from typing import Optional, Union
17
16
 
@@ -447,7 +446,6 @@ class FeatureSet(ModelObj):
447
446
  targets=None,
448
447
  with_defaults=True,
449
448
  default_final_step=None,
450
- default_final_state=None,
451
449
  ):
452
450
  """set the desired target list or defaults
453
451
 
@@ -457,17 +455,7 @@ class FeatureSet(ModelObj):
457
455
  :param default_final_step: the final graph step after which we add the
458
456
  target writers, used when the graph branches and
459
457
  the end cant be determined automatically
460
- :param default_final_state: *Deprecated* - use default_final_step instead
461
458
  """
462
- if default_final_state:
463
- warnings.warn(
464
- "The 'default_final_state' parameter is deprecated in 1.3.0 and will be remove in 1.5.0. "
465
- "Use 'default_final_step' instead.",
466
- # TODO: remove in 1.5.0
467
- FutureWarning,
468
- )
469
- default_final_step = default_final_step or default_final_state
470
-
471
459
  if targets is not None and not isinstance(targets, list):
472
460
  raise mlrun.errors.MLRunInvalidArgumentError(
473
461
  "targets can only be None or a list of kinds or DataTargetBase derivatives"
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import abc
16
16
  import typing
17
17
  from datetime import datetime
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import re
16
16
 
17
17
  import mlrun
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import uuid
16
16
  from typing import Optional
17
17
 
@@ -11,8 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
-
16
14
 
17
15
  import mlrun
18
16
  from mlrun.data_types.to_pandas import spark_df_to_pandas
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import math
16
16
  import re
17
17
  import uuid
mlrun/features.py CHANGED
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import math
16
16
  import re
17
17
  from typing import Optional, Union
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  from abc import ABC, abstractmethod
16
16
  from typing import Optional, Union
17
17
 
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import copy
16
16
  import functools
17
17
  import inspect
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import importlib.util
16
16
  import json
17
17
  import os
@@ -690,10 +690,10 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
690
690
  }
691
691
  self._registered_artifacts = {}
692
692
 
693
- # Get the model artifact. If the model was logged during this run, use the cached artifact, otherwise use the
693
+ # Get the model artifact. If the model was logged during this run, use the artifact, otherwise use the
694
694
  # user's given model path:
695
695
  model_artifact = (
696
- self._context.get_cached_artifact(self._model_name)
696
+ self._context.get_artifact(self._model_name)
697
697
  if self._is_logged
698
698
  else self._model_path
699
699
  )
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
14
 
16
15
  from typing import Optional
17
16
 
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import re
16
16
  from abc import ABC
17
17
  from enum import Enum
@@ -11,7 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
15
14
 
16
15
  import mlrun
17
16
 
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import itertools
16
16
  from typing import Optional, Union
17
17
 
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  import os
16
16
  from abc import abstractmethod
17
17
  from datetime import datetime
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- #
14
+
15
15
  from abc import ABC
16
16
 
17
17
  from .._common import ModelHandler