mlrun 1.9.0rc4__py3-none-any.whl → 1.10.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mlrun might be problematic. Click here for more details.
- mlrun/__main__.py +13 -2
- mlrun/api/schemas/__init__.py +0 -1
- mlrun/common/__init__.py +0 -1
- mlrun/common/constants.py +7 -0
- mlrun/common/db/__init__.py +0 -1
- mlrun/common/db/sql_session.py +0 -1
- mlrun/common/formatters/__init__.py +0 -1
- mlrun/common/formatters/artifact.py +0 -1
- mlrun/common/formatters/base.py +0 -1
- mlrun/common/formatters/feature_set.py +0 -1
- mlrun/common/formatters/function.py +0 -1
- mlrun/common/formatters/model_endpoint.py +0 -1
- mlrun/common/formatters/pipeline.py +0 -1
- mlrun/common/formatters/project.py +0 -1
- mlrun/common/formatters/run.py +0 -2
- mlrun/common/runtimes/constants.py +9 -2
- mlrun/common/schemas/__init__.py +2 -0
- mlrun/common/schemas/alert.py +1 -1
- mlrun/common/schemas/api_gateway.py +1 -1
- mlrun/common/schemas/artifact.py +1 -1
- mlrun/common/schemas/auth.py +1 -1
- mlrun/common/schemas/background_task.py +1 -1
- mlrun/common/schemas/client_spec.py +1 -1
- mlrun/common/schemas/clusterization_spec.py +1 -1
- mlrun/common/schemas/constants.py +1 -1
- mlrun/common/schemas/datastore_profile.py +0 -1
- mlrun/common/schemas/events.py +1 -1
- mlrun/common/schemas/feature_store.py +1 -1
- mlrun/common/schemas/frontend_spec.py +1 -1
- mlrun/common/schemas/function.py +1 -1
- mlrun/common/schemas/http.py +1 -1
- mlrun/common/schemas/hub.py +1 -1
- mlrun/common/schemas/k8s.py +1 -1
- mlrun/common/schemas/memory_reports.py +0 -1
- mlrun/common/schemas/notification.py +4 -0
- mlrun/common/schemas/object.py +1 -1
- mlrun/common/schemas/partition.py +1 -1
- mlrun/common/schemas/pipeline.py +1 -1
- mlrun/common/schemas/project.py +1 -1
- mlrun/common/schemas/regex.py +1 -1
- mlrun/common/schemas/runtime_resource.py +1 -1
- mlrun/common/schemas/schedule.py +1 -1
- mlrun/common/schemas/secret.py +1 -1
- mlrun/common/schemas/serving.py +17 -0
- mlrun/common/schemas/tag.py +0 -1
- mlrun/common/schemas/workflow.py +1 -1
- mlrun/common/secrets.py +0 -1
- mlrun/config.py +9 -11
- mlrun/data_types/infer.py +1 -1
- mlrun/data_types/spark.py +1 -1
- mlrun/datastore/datastore.py +1 -1
- mlrun/datastore/datastore_profile.py +5 -55
- mlrun/datastore/snowflake_utils.py +0 -1
- mlrun/datastore/sources.py +21 -13
- mlrun/datastore/spark_utils.py +0 -1
- mlrun/datastore/utils.py +20 -5
- mlrun/db/base.py +1 -1
- mlrun/db/httpdb.py +17 -12
- mlrun/db/nopdb.py +1 -2
- mlrun/errors.py +8 -1
- mlrun/execution.py +22 -1
- mlrun/feature_store/feature_set.py +0 -12
- mlrun/feature_store/retrieval/base.py +1 -1
- mlrun/feature_store/retrieval/dask_merger.py +1 -1
- mlrun/feature_store/retrieval/job.py +1 -1
- mlrun/feature_store/retrieval/spark_merger.py +0 -2
- mlrun/feature_store/steps.py +1 -1
- mlrun/features.py +1 -1
- mlrun/frameworks/_common/artifacts_library.py +1 -1
- mlrun/frameworks/_common/mlrun_interface.py +1 -1
- mlrun/frameworks/_common/model_handler.py +3 -3
- mlrun/frameworks/_common/producer.py +0 -1
- mlrun/frameworks/_common/utils.py +1 -1
- mlrun/frameworks/_dl_common/loggers/logger.py +0 -1
- mlrun/frameworks/_dl_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_dl_common/loggers/tensorboard_logger.py +1 -1
- mlrun/frameworks/_dl_common/model_handler.py +1 -1
- mlrun/frameworks/_dl_common/utils.py +1 -1
- mlrun/frameworks/_ml_common/artifacts_library.py +1 -1
- mlrun/frameworks/_ml_common/loggers/logger.py +0 -1
- mlrun/frameworks/_ml_common/loggers/mlrun_logger.py +1 -1
- mlrun/frameworks/_ml_common/model_handler.py +1 -1
- mlrun/frameworks/_ml_common/pkl_model_server.py +1 -1
- mlrun/frameworks/_ml_common/plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/calibration_curve_plan.py +0 -1
- mlrun/frameworks/_ml_common/plans/confusion_matrix_plan.py +0 -1
- mlrun/frameworks/_ml_common/plans/dataset_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/feature_importance_plan.py +1 -1
- mlrun/frameworks/_ml_common/plans/roc_curve_plan.py +1 -1
- mlrun/frameworks/_ml_common/producer.py +1 -1
- mlrun/frameworks/_ml_common/utils.py +1 -1
- mlrun/frameworks/lgbm/callbacks/callback.py +1 -1
- mlrun/frameworks/lgbm/callbacks/logging_callback.py +0 -1
- mlrun/frameworks/lgbm/callbacks/mlrun_logging_callback.py +0 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/booster_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/mlrun_interfaces/model_mlrun_interface.py +1 -1
- mlrun/frameworks/lgbm/model_handler.py +1 -1
- mlrun/frameworks/lgbm/model_server.py +1 -1
- mlrun/frameworks/lgbm/utils.py +1 -1
- mlrun/frameworks/onnx/dataset.py +1 -1
- mlrun/frameworks/onnx/mlrun_interface.py +1 -1
- mlrun/frameworks/onnx/model_handler.py +1 -1
- mlrun/frameworks/onnx/model_server.py +1 -1
- mlrun/frameworks/pytorch/callbacks/callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/pytorch/callbacks_handler.py +1 -1
- mlrun/frameworks/pytorch/mlrun_interface.py +1 -1
- mlrun/frameworks/pytorch/model_handler.py +1 -1
- mlrun/frameworks/pytorch/model_server.py +1 -1
- mlrun/frameworks/pytorch/utils.py +1 -1
- mlrun/frameworks/sklearn/__init__.py +0 -14
- mlrun/frameworks/sklearn/estimator.py +1 -1
- mlrun/frameworks/sklearn/metric.py +1 -1
- mlrun/frameworks/sklearn/metrics_library.py +1 -1
- mlrun/frameworks/sklearn/mlrun_interface.py +1 -1
- mlrun/frameworks/sklearn/model_handler.py +1 -1
- mlrun/frameworks/sklearn/utils.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/mlrun_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/callbacks/tensorboard_logging_callback.py +1 -1
- mlrun/frameworks/tf_keras/mlrun_interface.py +1 -1
- mlrun/frameworks/tf_keras/model_handler.py +1 -1
- mlrun/frameworks/tf_keras/model_server.py +1 -1
- mlrun/frameworks/tf_keras/utils.py +1 -1
- mlrun/frameworks/xgboost/mlrun_interface.py +1 -1
- mlrun/frameworks/xgboost/model_handler.py +1 -1
- mlrun/frameworks/xgboost/utils.py +1 -1
- mlrun/k8s_utils.py +340 -0
- mlrun/launcher/base.py +3 -3
- mlrun/launcher/client.py +1 -1
- mlrun/launcher/local.py +2 -2
- mlrun/launcher/remote.py +2 -2
- mlrun/model.py +14 -0
- mlrun/model_monitoring/applications/__init__.py +0 -1
- mlrun/model_monitoring/applications/_application_steps.py +3 -1
- mlrun/model_monitoring/applications/evidently/base.py +59 -71
- mlrun/model_monitoring/controller.py +26 -13
- mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py +13 -5
- mlrun/package/context_handler.py +1 -1
- mlrun/package/errors.py +1 -1
- mlrun/package/packager.py +1 -1
- mlrun/package/packagers/default_packager.py +1 -1
- mlrun/package/packagers/numpy_packagers.py +1 -1
- mlrun/package/packagers/pandas_packagers.py +1 -1
- mlrun/package/packagers/python_standard_library_packagers.py +1 -1
- mlrun/package/packagers_manager.py +1 -1
- mlrun/package/utils/_archiver.py +1 -1
- mlrun/package/utils/_formatter.py +1 -1
- mlrun/package/utils/_pickler.py +1 -1
- mlrun/package/utils/_supported_format.py +1 -1
- mlrun/package/utils/log_hint_utils.py +1 -1
- mlrun/package/utils/type_hint_utils.py +1 -1
- mlrun/projects/operations.py +36 -21
- mlrun/projects/project.py +147 -92
- mlrun/render.py +5 -9
- mlrun/run.py +1 -1
- mlrun/runtimes/base.py +17 -7
- mlrun/runtimes/daskjob.py +2 -1
- mlrun/runtimes/databricks_job/databricks_cancel_task.py +0 -1
- mlrun/runtimes/databricks_job/databricks_runtime.py +2 -1
- mlrun/runtimes/databricks_job/databricks_wrapper.py +0 -1
- mlrun/runtimes/mounts.py +2 -0
- mlrun/runtimes/nuclio/function.py +5 -0
- mlrun/runtimes/nuclio/serving.py +1 -1
- mlrun/runtimes/pod.py +4 -349
- mlrun/runtimes/sparkjob/spark3job.py +0 -12
- mlrun/runtimes/utils.py +25 -8
- mlrun/serving/merger.py +0 -1
- mlrun/serving/remote.py +1 -1
- mlrun/serving/serving_wrapper.py +1 -1
- mlrun/serving/states.py +142 -9
- mlrun/serving/utils.py +1 -1
- mlrun/utils/async_http.py +0 -1
- mlrun/utils/clones.py +1 -1
- mlrun/utils/db.py +1 -1
- mlrun/utils/helpers.py +3 -1
- mlrun/utils/http.py +0 -1
- mlrun/utils/regex.py +0 -1
- mlrun/utils/singleton.py +1 -1
- mlrun/utils/vault.py +1 -1
- mlrun/utils/version/__init__.py +1 -1
- mlrun/utils/version/version.json +2 -2
- mlrun/utils/version/version.py +1 -1
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/METADATA +12 -12
- mlrun-1.10.0rc2.dist-info/RECORD +351 -0
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/WHEEL +1 -1
- mlrun-1.9.0rc4.dist-info/RECORD +0 -351
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/entry_points.txt +0 -0
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/licenses/LICENSE +0 -0
- {mlrun-1.9.0rc4.dist-info → mlrun-1.10.0rc2.dist-info}/top_level.txt +0 -0
|
@@ -16,7 +16,6 @@ import ast
|
|
|
16
16
|
import base64
|
|
17
17
|
import json
|
|
18
18
|
import typing
|
|
19
|
-
import warnings
|
|
20
19
|
from urllib.parse import ParseResult, urlparse
|
|
21
20
|
|
|
22
21
|
import pydantic.v1
|
|
@@ -142,7 +141,6 @@ class ConfigProfile(DatastoreProfile):
|
|
|
142
141
|
class DatastoreProfileKafkaTarget(DatastoreProfile):
|
|
143
142
|
type: str = pydantic.v1.Field("kafka_target")
|
|
144
143
|
_private_attributes = "kwargs_private"
|
|
145
|
-
bootstrap_servers: typing.Optional[str] = None
|
|
146
144
|
brokers: typing.Optional[str] = None
|
|
147
145
|
topic: str
|
|
148
146
|
kwargs_public: typing.Optional[dict]
|
|
@@ -151,31 +149,16 @@ class DatastoreProfileKafkaTarget(DatastoreProfile):
|
|
|
151
149
|
def __init__(self, **kwargs):
|
|
152
150
|
super().__init__(**kwargs)
|
|
153
151
|
|
|
154
|
-
if not self.brokers
|
|
152
|
+
if not self.brokers:
|
|
155
153
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
156
154
|
"DatastoreProfileKafkaTarget requires the 'brokers' field to be set"
|
|
157
155
|
)
|
|
158
156
|
|
|
159
|
-
if self.bootstrap_servers:
|
|
160
|
-
if self.brokers:
|
|
161
|
-
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
162
|
-
"DatastoreProfileKafkaTarget cannot be created with both 'brokers' and 'bootstrap_servers'"
|
|
163
|
-
)
|
|
164
|
-
else:
|
|
165
|
-
self.brokers = self.bootstrap_servers
|
|
166
|
-
self.bootstrap_servers = None
|
|
167
|
-
warnings.warn(
|
|
168
|
-
"'bootstrap_servers' parameter is deprecated in 1.7.0 and will be removed in 1.9.0, "
|
|
169
|
-
"use 'brokers' instead.",
|
|
170
|
-
# TODO: Remove this in 1.9.0
|
|
171
|
-
FutureWarning,
|
|
172
|
-
)
|
|
173
|
-
|
|
174
157
|
def get_topic(self) -> typing.Optional[str]:
|
|
175
158
|
return self.topic
|
|
176
159
|
|
|
177
160
|
def attributes(self):
|
|
178
|
-
attributes = {"brokers": self.brokers
|
|
161
|
+
attributes = {"brokers": self.brokers}
|
|
179
162
|
if self.kwargs_public:
|
|
180
163
|
attributes = merge(attributes, self.kwargs_public)
|
|
181
164
|
if self.kwargs_private:
|
|
@@ -248,18 +231,7 @@ class DatastoreProfileS3(DatastoreProfile):
|
|
|
248
231
|
assume_role_arn: typing.Optional[str] = None
|
|
249
232
|
access_key_id: typing.Optional[str] = None
|
|
250
233
|
secret_key: typing.Optional[str] = None
|
|
251
|
-
bucket:
|
|
252
|
-
|
|
253
|
-
@pydantic.v1.validator("bucket")
|
|
254
|
-
@classmethod
|
|
255
|
-
def check_bucket(cls, v):
|
|
256
|
-
if not v:
|
|
257
|
-
warnings.warn(
|
|
258
|
-
"The 'bucket' attribute will be mandatory starting from version 1.9",
|
|
259
|
-
FutureWarning,
|
|
260
|
-
stacklevel=2,
|
|
261
|
-
)
|
|
262
|
-
return v
|
|
234
|
+
bucket: str
|
|
263
235
|
|
|
264
236
|
def secrets(self) -> dict:
|
|
265
237
|
res = {}
|
|
@@ -353,18 +325,7 @@ class DatastoreProfileGCS(DatastoreProfile):
|
|
|
353
325
|
_private_attributes = ("gcp_credentials",)
|
|
354
326
|
credentials_path: typing.Optional[str] = None # path to file.
|
|
355
327
|
gcp_credentials: typing.Optional[typing.Union[str, dict]] = None
|
|
356
|
-
bucket:
|
|
357
|
-
|
|
358
|
-
@pydantic.v1.validator("bucket")
|
|
359
|
-
@classmethod
|
|
360
|
-
def check_bucket(cls, v):
|
|
361
|
-
if not v:
|
|
362
|
-
warnings.warn(
|
|
363
|
-
"The 'bucket' attribute will be mandatory starting from version 1.9",
|
|
364
|
-
FutureWarning,
|
|
365
|
-
stacklevel=2,
|
|
366
|
-
)
|
|
367
|
-
return v
|
|
328
|
+
bucket: str
|
|
368
329
|
|
|
369
330
|
@pydantic.v1.validator("gcp_credentials", pre=True, always=True)
|
|
370
331
|
@classmethod
|
|
@@ -410,18 +371,7 @@ class DatastoreProfileAzureBlob(DatastoreProfile):
|
|
|
410
371
|
client_secret: typing.Optional[str] = None
|
|
411
372
|
sas_token: typing.Optional[str] = None
|
|
412
373
|
credential: typing.Optional[str] = None
|
|
413
|
-
container:
|
|
414
|
-
|
|
415
|
-
@pydantic.v1.validator("container")
|
|
416
|
-
@classmethod
|
|
417
|
-
def check_container(cls, v):
|
|
418
|
-
if not v:
|
|
419
|
-
warnings.warn(
|
|
420
|
-
"The 'container' attribute will be mandatory starting from version 1.9",
|
|
421
|
-
FutureWarning,
|
|
422
|
-
stacklevel=2,
|
|
423
|
-
)
|
|
424
|
-
return v
|
|
374
|
+
container: str
|
|
425
375
|
|
|
426
376
|
def url(self, subpath) -> str:
|
|
427
377
|
if subpath.startswith("/"):
|
mlrun/datastore/sources.py
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
+
|
|
14
15
|
import json
|
|
15
16
|
import operator
|
|
16
17
|
import os
|
|
@@ -18,7 +19,7 @@ import warnings
|
|
|
18
19
|
from base64 import b64encode
|
|
19
20
|
from copy import copy
|
|
20
21
|
from datetime import datetime
|
|
21
|
-
from typing import Any, Optional, Union
|
|
22
|
+
from typing import Any, Literal, Optional, Union
|
|
22
23
|
|
|
23
24
|
import pandas as pd
|
|
24
25
|
import semver
|
|
@@ -1063,16 +1064,17 @@ class KafkaSource(OnlineSource):
|
|
|
1063
1064
|
|
|
1064
1065
|
def __init__(
|
|
1065
1066
|
self,
|
|
1066
|
-
brokers=None,
|
|
1067
|
-
topics=None,
|
|
1068
|
-
group="serving",
|
|
1069
|
-
initial_offset="earliest",
|
|
1070
|
-
partitions=None,
|
|
1071
|
-
sasl_user=None,
|
|
1072
|
-
sasl_pass=None,
|
|
1073
|
-
|
|
1067
|
+
brokers: Optional[list[str]] = None,
|
|
1068
|
+
topics: Optional[list[str]] = None,
|
|
1069
|
+
group: str = "serving",
|
|
1070
|
+
initial_offset: Literal["earliest", "latest"] = "earliest",
|
|
1071
|
+
partitions: Optional[list[int]] = None,
|
|
1072
|
+
sasl_user: Optional[str] = None,
|
|
1073
|
+
sasl_pass: Optional[str] = None,
|
|
1074
|
+
tls_enable: Optional[bool] = None,
|
|
1075
|
+
attributes: Optional[dict] = None,
|
|
1074
1076
|
**kwargs,
|
|
1075
|
-
):
|
|
1077
|
+
) -> None:
|
|
1076
1078
|
"""Sets kafka source for the flow
|
|
1077
1079
|
|
|
1078
1080
|
:param brokers: list of broker IP addresses
|
|
@@ -1082,6 +1084,7 @@ class KafkaSource(OnlineSource):
|
|
|
1082
1084
|
:param partitions: Optional, A list of partitions numbers for which the function receives events.
|
|
1083
1085
|
:param sasl_user: Optional, user name to use for sasl authentications
|
|
1084
1086
|
:param sasl_pass: Optional, password to use for sasl authentications
|
|
1087
|
+
:param tls_enable: Optional, if set - whether to enable TLS or not.
|
|
1085
1088
|
:param attributes: Optional, extra attributes to be passed to kafka trigger
|
|
1086
1089
|
"""
|
|
1087
1090
|
if isinstance(topics, str):
|
|
@@ -1095,10 +1098,15 @@ class KafkaSource(OnlineSource):
|
|
|
1095
1098
|
attributes["initial_offset"] = initial_offset
|
|
1096
1099
|
if partitions is not None:
|
|
1097
1100
|
attributes["partitions"] = partitions
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
+
|
|
1102
|
+
kafka_params = mlrun.datastore.utils.KafkaParameters(attributes)
|
|
1103
|
+
|
|
1104
|
+
if sasl := kafka_params.sasl(usr=sasl_user, pwd=sasl_pass):
|
|
1101
1105
|
attributes["sasl"] = sasl
|
|
1106
|
+
|
|
1107
|
+
if tls := kafka_params.tls(tls_enable=tls_enable):
|
|
1108
|
+
attributes["tls"] = tls
|
|
1109
|
+
|
|
1102
1110
|
super().__init__(attributes=attributes, **kwargs)
|
|
1103
1111
|
|
|
1104
1112
|
def to_dataframe(
|
mlrun/datastore/spark_utils.py
CHANGED
mlrun/datastore/utils.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
import math
|
|
16
16
|
import tarfile
|
|
17
17
|
import tempfile
|
|
@@ -246,6 +246,9 @@ class KafkaParameters:
|
|
|
246
246
|
"partitions": "",
|
|
247
247
|
"sasl": "",
|
|
248
248
|
"worker_allocation_mode": "",
|
|
249
|
+
"tls_enable": "", # for Nuclio with Confluent Kafka (Sarama client)
|
|
250
|
+
"tls": "",
|
|
251
|
+
"new_topic": "",
|
|
249
252
|
}
|
|
250
253
|
self._reference_dicts = (
|
|
251
254
|
self._custom_attributes,
|
|
@@ -270,7 +273,9 @@ class KafkaParameters:
|
|
|
270
273
|
}
|
|
271
274
|
if sasl := self._kwargs.get("sasl"):
|
|
272
275
|
res |= {
|
|
273
|
-
"security_protocol":
|
|
276
|
+
"security_protocol": self._kwargs.get(
|
|
277
|
+
"security_protocol", "SASL_PLAINTEXT"
|
|
278
|
+
),
|
|
274
279
|
"sasl_mechanism": sasl["mechanism"],
|
|
275
280
|
"sasl_plain_username": sasl["user"],
|
|
276
281
|
"sasl_plain_password": sasl["password"],
|
|
@@ -288,15 +293,25 @@ class KafkaParameters:
|
|
|
288
293
|
|
|
289
294
|
def sasl(
|
|
290
295
|
self, *, usr: typing.Optional[str] = None, pwd: typing.Optional[str] = None
|
|
291
|
-
) -> dict:
|
|
292
|
-
usr = usr or self._kwargs.get("sasl_plain_username", None)
|
|
293
|
-
pwd = pwd or self._kwargs.get("sasl_plain_password", None)
|
|
296
|
+
) -> dict[str, typing.Union[str, bool]]:
|
|
294
297
|
res = self._kwargs.get("sasl", {})
|
|
298
|
+
usr = usr or self._kwargs.get("sasl_plain_username")
|
|
299
|
+
pwd = pwd or self._kwargs.get("sasl_plain_password")
|
|
295
300
|
if usr and pwd:
|
|
296
301
|
res["enable"] = True
|
|
297
302
|
res["user"] = usr
|
|
298
303
|
res["password"] = pwd
|
|
299
304
|
res["mechanism"] = self._kwargs.get("sasl_mechanism", "PLAIN")
|
|
305
|
+
res["handshake"] = self._kwargs.get("sasl_handshake", True)
|
|
306
|
+
return res
|
|
307
|
+
|
|
308
|
+
def tls(self, *, tls_enable: typing.Optional[bool] = None) -> dict[str, bool]:
|
|
309
|
+
res = self._kwargs.get("tls", {})
|
|
310
|
+
tls_enable = (
|
|
311
|
+
tls_enable if tls_enable is not None else self._kwargs.get("tls_enable")
|
|
312
|
+
)
|
|
313
|
+
if tls_enable:
|
|
314
|
+
res["enable"] = tls_enable
|
|
300
315
|
return res
|
|
301
316
|
|
|
302
317
|
def valid_entries_only(self, input_dict: dict) -> dict:
|
mlrun/db/base.py
CHANGED
|
@@ -734,7 +734,7 @@ class RunDBInterface(ABC):
|
|
|
734
734
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
735
735
|
start: Optional[datetime.datetime] = None,
|
|
736
736
|
end: Optional[datetime.datetime] = None,
|
|
737
|
-
tsdb_metrics: bool =
|
|
737
|
+
tsdb_metrics: bool = False,
|
|
738
738
|
metric_list: Optional[list[str]] = None,
|
|
739
739
|
top_level: bool = False,
|
|
740
740
|
uids: Optional[list[str]] = None,
|
mlrun/db/httpdb.py
CHANGED
|
@@ -21,7 +21,7 @@ import typing
|
|
|
21
21
|
import warnings
|
|
22
22
|
from copy import deepcopy
|
|
23
23
|
from datetime import datetime, timedelta
|
|
24
|
-
from os import path, remove
|
|
24
|
+
from os import environ, path, remove
|
|
25
25
|
from typing import Literal, Optional, Union
|
|
26
26
|
from urllib.parse import urlparse
|
|
27
27
|
|
|
@@ -129,7 +129,9 @@ class HTTPRunDB(RunDBInterface):
|
|
|
129
129
|
self._wait_for_background_task_terminal_state_retry_interval = 3
|
|
130
130
|
self._wait_for_project_deletion_interval = 3
|
|
131
131
|
self.client_version = version.Version().get()["version"]
|
|
132
|
-
self.python_version =
|
|
132
|
+
self.python_version = environ.get("MLRUN_PYTHON_VERSION") or str(
|
|
133
|
+
version.Version().get_python_version()
|
|
134
|
+
)
|
|
133
135
|
|
|
134
136
|
self._enrich_and_validate(url)
|
|
135
137
|
|
|
@@ -937,7 +939,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
937
939
|
|
|
938
940
|
:param name: Name of the run to retrieve.
|
|
939
941
|
:param uid: Unique ID of the run, or a list of run UIDs.
|
|
940
|
-
:param project: Project that the runs belongs to.
|
|
942
|
+
:param project: Project that the runs belongs to. If not specified, the default project will be used.
|
|
941
943
|
:param labels: Filter runs by label key-value pairs or key existence. This can be provided as:
|
|
942
944
|
- A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
|
|
943
945
|
or `{"label": None}` to check for key existence.
|
|
@@ -1277,7 +1279,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
1277
1279
|
points to a run and is used to filter artifacts by the run that produced them when the artifact producer id
|
|
1278
1280
|
is a workflow id (artifact was created as part of a workflow).
|
|
1279
1281
|
:param format_: The format in which to return the artifacts. Default is 'full'.
|
|
1280
|
-
:param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.
|
|
1282
|
+
:param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
|
|
1281
1283
|
:param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
|
|
1282
1284
|
parameter must be provided as well.
|
|
1283
1285
|
:param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
|
|
@@ -2221,18 +2223,20 @@ class HTTPRunDB(RunDBInterface):
|
|
|
2221
2223
|
elif pipe_file.endswith(".zip"):
|
|
2222
2224
|
headers = {"content-type": "application/zip"}
|
|
2223
2225
|
else:
|
|
2224
|
-
raise ValueError("pipeline file must be .yaml or .zip")
|
|
2226
|
+
raise ValueError("'pipeline' file must be .yaml or .zip")
|
|
2225
2227
|
if arguments:
|
|
2226
2228
|
if not isinstance(arguments, dict):
|
|
2227
|
-
raise ValueError("arguments must be dict type")
|
|
2229
|
+
raise ValueError("'arguments' must be dict type")
|
|
2228
2230
|
headers[mlrun.common.schemas.HeaderNames.pipeline_arguments] = str(
|
|
2229
2231
|
arguments
|
|
2230
2232
|
)
|
|
2231
2233
|
|
|
2232
2234
|
if not path.isfile(pipe_file):
|
|
2233
|
-
raise OSError(f"
|
|
2235
|
+
raise OSError(f"File {pipe_file} doesnt exist")
|
|
2234
2236
|
with open(pipe_file, "rb") as fp:
|
|
2235
2237
|
data = fp.read()
|
|
2238
|
+
if not data:
|
|
2239
|
+
raise ValueError("The compiled pipe file is empty")
|
|
2236
2240
|
if not isinstance(pipeline, str):
|
|
2237
2241
|
remove(pipe_file)
|
|
2238
2242
|
|
|
@@ -3767,7 +3771,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3767
3771
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
3768
3772
|
start: Optional[datetime] = None,
|
|
3769
3773
|
end: Optional[datetime] = None,
|
|
3770
|
-
tsdb_metrics: bool =
|
|
3774
|
+
tsdb_metrics: bool = False,
|
|
3771
3775
|
metric_list: Optional[list[str]] = None,
|
|
3772
3776
|
top_level: bool = False,
|
|
3773
3777
|
uids: Optional[list[str]] = None,
|
|
@@ -3889,8 +3893,8 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3889
3893
|
attributes_keys = list(attributes.keys())
|
|
3890
3894
|
attributes["name"] = name
|
|
3891
3895
|
attributes["project"] = project
|
|
3892
|
-
attributes["
|
|
3893
|
-
attributes["
|
|
3896
|
+
attributes["function_name"] = function_name or None
|
|
3897
|
+
attributes["function_tag"] = function_tag or None
|
|
3894
3898
|
attributes["uid"] = endpoint_id or None
|
|
3895
3899
|
model_endpoint = mlrun.common.schemas.ModelEndpoint.from_flat_dict(attributes)
|
|
3896
3900
|
path = f"projects/{project}/model-endpoints"
|
|
@@ -3981,6 +3985,7 @@ class HTTPRunDB(RunDBInterface):
|
|
|
3981
3985
|
"deploy_histogram_data_drift_app": deploy_histogram_data_drift_app,
|
|
3982
3986
|
"fetch_credentials_from_sys_config": fetch_credentials_from_sys_config,
|
|
3983
3987
|
},
|
|
3988
|
+
timeout=300, # 5 minutes
|
|
3984
3989
|
)
|
|
3985
3990
|
|
|
3986
3991
|
def disable_model_monitoring(
|
|
@@ -5100,9 +5105,9 @@ class HTTPRunDB(RunDBInterface):
|
|
|
5100
5105
|
labels = self._parse_labels(labels)
|
|
5101
5106
|
|
|
5102
5107
|
if limit:
|
|
5103
|
-
# TODO: Remove this in 1.
|
|
5108
|
+
# TODO: Remove this in 1.11.0
|
|
5104
5109
|
warnings.warn(
|
|
5105
|
-
"'limit' is deprecated and will be removed in 1.
|
|
5110
|
+
"'limit' is deprecated and will be removed in 1.11.0. Use 'page' and 'page_size' instead.",
|
|
5106
5111
|
FutureWarning,
|
|
5107
5112
|
)
|
|
5108
5113
|
|
mlrun/db/nopdb.py
CHANGED
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
|
|
16
15
|
import datetime
|
|
17
16
|
from typing import Optional, Union
|
|
18
17
|
|
|
@@ -631,7 +630,7 @@ class NopDB(RunDBInterface):
|
|
|
631
630
|
labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
|
|
632
631
|
start: Optional[datetime.datetime] = None,
|
|
633
632
|
end: Optional[datetime.datetime] = None,
|
|
634
|
-
tsdb_metrics: bool =
|
|
633
|
+
tsdb_metrics: bool = False,
|
|
635
634
|
metric_list: Optional[list[str]] = None,
|
|
636
635
|
top_level: bool = False,
|
|
637
636
|
uids: Optional[list[str]] = None,
|
mlrun/errors.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
import typing
|
|
16
16
|
from http import HTTPStatus
|
|
17
17
|
|
|
@@ -230,6 +230,13 @@ class MLRunTSDBConnectionFailureError(MLRunHTTPStatusError, ValueError):
|
|
|
230
230
|
error_status_code = HTTPStatus.BAD_REQUEST.value
|
|
231
231
|
|
|
232
232
|
|
|
233
|
+
class MLRunMissingProjectError(MLRunBadRequestError):
|
|
234
|
+
default_message = "Project must be provided"
|
|
235
|
+
|
|
236
|
+
def __init__(self, message=None):
|
|
237
|
+
super().__init__(message or self.default_message)
|
|
238
|
+
|
|
239
|
+
|
|
233
240
|
class MLRunRetryExhaustedError(Exception):
|
|
234
241
|
pass
|
|
235
242
|
|
mlrun/execution.py
CHANGED
|
@@ -94,6 +94,8 @@ class MLClientCtx:
|
|
|
94
94
|
self._labels = {}
|
|
95
95
|
self._annotations = {}
|
|
96
96
|
self._node_selector = {}
|
|
97
|
+
self._tolerations = {}
|
|
98
|
+
self._affinity = {}
|
|
97
99
|
|
|
98
100
|
self._function = ""
|
|
99
101
|
self._parameters = {}
|
|
@@ -227,11 +229,26 @@ class MLClientCtx:
|
|
|
227
229
|
"""Dictionary with node selectors (read-only)"""
|
|
228
230
|
return deepcopy(self._node_selector)
|
|
229
231
|
|
|
232
|
+
@property
|
|
233
|
+
def tolerations(self):
|
|
234
|
+
"""Dictionary with tolerations (read-only)"""
|
|
235
|
+
return deepcopy(self._tolerations)
|
|
236
|
+
|
|
237
|
+
@property
|
|
238
|
+
def affinity(self):
|
|
239
|
+
"""Dictionary with affinities (read-only)"""
|
|
240
|
+
return deepcopy(self._affinity)
|
|
241
|
+
|
|
230
242
|
@property
|
|
231
243
|
def annotations(self):
|
|
232
244
|
"""Dictionary with annotations (read-only)"""
|
|
233
245
|
return deepcopy(self._annotations)
|
|
234
246
|
|
|
247
|
+
@property
|
|
248
|
+
def host(self):
|
|
249
|
+
"""Execution host"""
|
|
250
|
+
return self._host
|
|
251
|
+
|
|
235
252
|
def get_child_context(self, with_parent_params=False, **params):
|
|
236
253
|
"""Get child context (iteration)
|
|
237
254
|
|
|
@@ -411,6 +428,8 @@ class MLClientCtx:
|
|
|
411
428
|
"state_thresholds", self._state_thresholds
|
|
412
429
|
)
|
|
413
430
|
self._node_selector = spec.get("node_selector", self._node_selector)
|
|
431
|
+
self._tolerations = spec.get("tolerations", self._tolerations)
|
|
432
|
+
self._affinity = spec.get("affinity", self._affinity)
|
|
414
433
|
self._reset_on_run = spec.get("reset_on_run", self._reset_on_run)
|
|
415
434
|
|
|
416
435
|
self._init_dbs(rundb)
|
|
@@ -976,7 +995,7 @@ class MLClientCtx:
|
|
|
976
995
|
def get_cached_artifact(self, key):
|
|
977
996
|
"""Return a logged artifact from cache (for potential updates)"""
|
|
978
997
|
warnings.warn(
|
|
979
|
-
"get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.
|
|
998
|
+
"get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.11.0. Use get_artifact instead.",
|
|
980
999
|
FutureWarning,
|
|
981
1000
|
)
|
|
982
1001
|
return self.get_artifact(key)
|
|
@@ -1134,6 +1153,8 @@ class MLClientCtx:
|
|
|
1134
1153
|
"notifications": self._notifications,
|
|
1135
1154
|
"state_thresholds": self._state_thresholds,
|
|
1136
1155
|
"node_selector": self._node_selector,
|
|
1156
|
+
"tolerations": self._tolerations,
|
|
1157
|
+
"affinity": self._affinity,
|
|
1137
1158
|
},
|
|
1138
1159
|
"status": {
|
|
1139
1160
|
"results": self._results,
|
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
import warnings
|
|
15
14
|
from datetime import datetime
|
|
16
15
|
from typing import Optional, Union
|
|
17
16
|
|
|
@@ -447,7 +446,6 @@ class FeatureSet(ModelObj):
|
|
|
447
446
|
targets=None,
|
|
448
447
|
with_defaults=True,
|
|
449
448
|
default_final_step=None,
|
|
450
|
-
default_final_state=None,
|
|
451
449
|
):
|
|
452
450
|
"""set the desired target list or defaults
|
|
453
451
|
|
|
@@ -457,17 +455,7 @@ class FeatureSet(ModelObj):
|
|
|
457
455
|
:param default_final_step: the final graph step after which we add the
|
|
458
456
|
target writers, used when the graph branches and
|
|
459
457
|
the end cant be determined automatically
|
|
460
|
-
:param default_final_state: *Deprecated* - use default_final_step instead
|
|
461
458
|
"""
|
|
462
|
-
if default_final_state:
|
|
463
|
-
warnings.warn(
|
|
464
|
-
"The 'default_final_state' parameter is deprecated in 1.3.0 and will be remove in 1.5.0. "
|
|
465
|
-
"Use 'default_final_step' instead.",
|
|
466
|
-
# TODO: remove in 1.5.0
|
|
467
|
-
FutureWarning,
|
|
468
|
-
)
|
|
469
|
-
default_final_step = default_final_step or default_final_state
|
|
470
|
-
|
|
471
459
|
if targets is not None and not isinstance(targets, list):
|
|
472
460
|
raise mlrun.errors.MLRunInvalidArgumentError(
|
|
473
461
|
"targets can only be None or a list of kinds or DataTargetBase derivatives"
|
|
@@ -11,8 +11,6 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
#
|
|
15
|
-
|
|
16
14
|
|
|
17
15
|
import mlrun
|
|
18
16
|
from mlrun.data_types.to_pandas import spark_df_to_pandas
|
mlrun/feature_store/steps.py
CHANGED
mlrun/features.py
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
import math
|
|
16
16
|
import re
|
|
17
17
|
from typing import Optional, Union
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
from abc import ABC, abstractmethod
|
|
16
16
|
from typing import Optional, Union
|
|
17
17
|
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
import importlib.util
|
|
16
16
|
import json
|
|
17
17
|
import os
|
|
@@ -690,10 +690,10 @@ class ModelHandler(ABC, Generic[CommonTypes.ModelType, CommonTypes.IOSampleType]
|
|
|
690
690
|
}
|
|
691
691
|
self._registered_artifacts = {}
|
|
692
692
|
|
|
693
|
-
# Get the model artifact. If the model was logged during this run, use the
|
|
693
|
+
# Get the model artifact. If the model was logged during this run, use the artifact, otherwise use the
|
|
694
694
|
# user's given model path:
|
|
695
695
|
model_artifact = (
|
|
696
|
-
self._context.
|
|
696
|
+
self._context.get_artifact(self._model_name)
|
|
697
697
|
if self._is_logged
|
|
698
698
|
else self._model_path
|
|
699
699
|
)
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
import os
|
|
16
16
|
from abc import abstractmethod
|
|
17
17
|
from datetime import datetime
|