mlrun 1.10.0rc1__py3-none-any.whl → 1.10.0rc2__py3-none-any.whl

mlrun/__main__.py

@@ -13,6 +13,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import functools
+import importlib.metadata
 import json
 import pathlib
 import socket
@@ -25,12 +27,14 @@ from pprint import pprint
 import click
 import dotenv
 import pandas as pd
+import semver
 import yaml
 from tabulate import tabulate
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
+import mlrun.platforms
 import mlrun.utils.helpers
 from mlrun.common.helpers import parse_versioned_object_uri
 from mlrun.runtimes.mounts import auto_mount as auto_mount_modifier
@@ -63,12 +67,19 @@ from .utils.version import Version
 pd.set_option("mode.chained_assignment", None)
 
 
-def validate_base_argument(ctx, param, value):
+def validate_base_argument(ctx: click.Context, param: click.Parameter, value: str):
+    # click 8.2 expects the context to be passed to make_metavar
+    if semver.VersionInfo.parse(
+        importlib.metadata.version("click")
+    ) < semver.VersionInfo.parse("8.2.0"):
+        metavar_func = functools.partial(param.make_metavar)
+    else:
+        metavar_func = functools.partial(param.make_metavar, ctx)
     if value and value.startswith("-"):
         raise click.BadParameter(
             f"{param.human_readable_name} ({value}) cannot start with '-', ensure the command options are typed "
             f"correctly. Preferably use '--' to separate options and arguments "
-            f"e.g. 'mlrun run --option1 --option2 -- {param.make_metavar()} [--arg1|arg1] [--arg2|arg2]'",
+            f"e.g. 'mlrun run --option1 --option2 -- {metavar_func()} [--arg1|arg1] [--arg2|arg2]'",
             ctx=ctx,
             param=param,
         )

mlrun/common/constants.py

@@ -90,6 +90,13 @@ class MLRunInternalLabels:
             if not key.startswith("__") and isinstance(value, str)
         ]
 
+    @staticmethod
+    def default_run_labels_to_enrich():
+        return [
+            MLRunInternalLabels.owner,
+            MLRunInternalLabels.v3io_user,
+        ]
+
 
 class DeployStatusTextKind(mlrun.common.types.StrEnum):
     logs = "logs"

mlrun/common/runtimes/constants.py

@@ -15,6 +15,8 @@
 import enum
 import typing
 
+from deprecated import deprecated
+
 import mlrun.common.constants as mlrun_constants
 import mlrun_pipelines.common.models
 
@@ -237,7 +239,12 @@ class RunStates:
         }[pipeline_run_status]
 
 
-# TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
+# TODO: remove this class in 1.11.0 - use only MLRunInternalLabels
+@deprecated(
+    version="1.9.0",
+    reason="This class is deprecated and will be removed in 1.11.0. Use MLRunInternalLabels instead.",
+    category=FutureWarning,
+)
 class RunLabels(enum.Enum):
     owner = mlrun_constants.MLRunInternalLabels.owner
     v3io_user = mlrun_constants.MLRunInternalLabels.v3io_user

mlrun/common/schemas/__init__.py

@@ -214,6 +214,7 @@ from .secret import (
     SecretsData,
     UserSecretCreationRequest,
 )
+from .serving import ModelRunnerStepData, MonitoringData
 from .tag import Tag, TagObjects
 from .workflow import (
     GetWorkflowResponse,

mlrun/common/schemas/serving.py

@@ -14,9 +14,26 @@
 
 from pydantic.v1 import BaseModel
 
+from mlrun.common.types import StrEnum
+
 from .background_task import BackgroundTaskList
 
 
 class DeployResponse(BaseModel):
     data: dict
     background_tasks: BackgroundTaskList
+
+
+class ModelRunnerStepData(StrEnum):
+    MODELS = "models"
+    MONITORING_DATA = "monitoring_data"
+
+
+class MonitoringData(StrEnum):
+    INPUTS = "inputs"
+    OUTPUTS = "outputs"
+    INPUT_PATH = "input_path"
+    CREATION_STRATEGY = "creation_strategy"
+    LABELS = "labels"
+    MODEL_PATH = "model_path"
+    MODEL_ENDPOINT_UID = "model_endpoint_uid"

mlrun/datastore/datastore_profile.py

@@ -16,7 +16,6 @@ import ast
 import base64
 import json
 import typing
-import warnings
 from urllib.parse import ParseResult, urlparse
 
 import pydantic.v1
@@ -142,7 +141,6 @@ class ConfigProfile(DatastoreProfile):
 class DatastoreProfileKafkaTarget(DatastoreProfile):
     type: str = pydantic.v1.Field("kafka_target")
     _private_attributes = "kwargs_private"
-    bootstrap_servers: typing.Optional[str] = None
     brokers: typing.Optional[str] = None
     topic: str
     kwargs_public: typing.Optional[dict]
@@ -151,31 +149,16 @@ class DatastoreProfileKafkaTarget(DatastoreProfile):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        if not self.brokers and not self.bootstrap_servers:
+        if not self.brokers:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "DatastoreProfileKafkaTarget requires the 'brokers' field to be set"
             )
 
-        if self.bootstrap_servers:
-            if self.brokers:
-                raise mlrun.errors.MLRunInvalidArgumentError(
-                    "DatastoreProfileKafkaTarget cannot be created with both 'brokers' and 'bootstrap_servers'"
-                )
-            else:
-                self.brokers = self.bootstrap_servers
-                self.bootstrap_servers = None
-            warnings.warn(
-                "'bootstrap_servers' parameter is deprecated in 1.7.0 and will be removed in 1.9.0, "
-                "use 'brokers' instead.",
-                # TODO: Remove this in 1.9.0
-                FutureWarning,
-            )
-
     def get_topic(self) -> typing.Optional[str]:
         return self.topic
 
     def attributes(self):
-        attributes = {"brokers": self.brokers or self.bootstrap_servers}
+        attributes = {"brokers": self.brokers}
         if self.kwargs_public:
             attributes = merge(attributes, self.kwargs_public)
         if self.kwargs_private:
@@ -248,18 +231,7 @@ class DatastoreProfileS3(DatastoreProfile):
     assume_role_arn: typing.Optional[str] = None
     access_key_id: typing.Optional[str] = None
     secret_key: typing.Optional[str] = None
-    bucket: typing.Optional[str] = None
-
-    @pydantic.v1.validator("bucket")
-    @classmethod
-    def check_bucket(cls, v):
-        if not v:
-            warnings.warn(
-                "The 'bucket' attribute will be mandatory starting from version 1.9",
-                FutureWarning,
-                stacklevel=2,
-            )
-        return v
+    bucket: str
 
     def secrets(self) -> dict:
         res = {}
@@ -353,18 +325,7 @@ class DatastoreProfileGCS(DatastoreProfile):
     _private_attributes = ("gcp_credentials",)
     credentials_path: typing.Optional[str] = None  # path to file.
     gcp_credentials: typing.Optional[typing.Union[str, dict]] = None
-    bucket: typing.Optional[str] = None
-
-    @pydantic.v1.validator("bucket")
-    @classmethod
-    def check_bucket(cls, v):
-        if not v:
-            warnings.warn(
-                "The 'bucket' attribute will be mandatory starting from version 1.9",
-                FutureWarning,
-                stacklevel=2,
-            )
-        return v
+    bucket: str
 
     @pydantic.v1.validator("gcp_credentials", pre=True, always=True)
     @classmethod
@@ -410,18 +371,7 @@ class DatastoreProfileAzureBlob(DatastoreProfile):
     client_secret: typing.Optional[str] = None
     sas_token: typing.Optional[str] = None
     credential: typing.Optional[str] = None
-    container: typing.Optional[str] = None
-
-    @pydantic.v1.validator("container")
-    @classmethod
-    def check_container(cls, v):
-        if not v:
-            warnings.warn(
-                "The 'container' attribute will be mandatory starting from version 1.9",
-                FutureWarning,
-                stacklevel=2,
-            )
-        return v
+    container: str
 
     def url(self, subpath) -> str:
         if subpath.startswith("/"):

mlrun/datastore/sources.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import json
 import operator
 import os
@@ -18,7 +19,7 @@ import warnings
 from base64 import b64encode
 from copy import copy
 from datetime import datetime
-from typing import Any, Optional, Union
+from typing import Any, Literal, Optional, Union
 
 import pandas as pd
 import semver
@@ -1063,16 +1064,17 @@ class KafkaSource(OnlineSource):
 
     def __init__(
         self,
-        brokers=None,
-        topics=None,
-        group="serving",
-        initial_offset="earliest",
-        partitions=None,
-        sasl_user=None,
-        sasl_pass=None,
-        attributes=None,
+        brokers: Optional[list[str]] = None,
+        topics: Optional[list[str]] = None,
+        group: str = "serving",
+        initial_offset: Literal["earliest", "latest"] = "earliest",
+        partitions: Optional[list[int]] = None,
+        sasl_user: Optional[str] = None,
+        sasl_pass: Optional[str] = None,
+        tls_enable: Optional[bool] = None,
+        attributes: Optional[dict] = None,
         **kwargs,
-    ):
+    ) -> None:
         """Sets kafka source for the flow
 
         :param brokers: list of broker IP addresses
@@ -1082,6 +1084,7 @@ class KafkaSource(OnlineSource):
         :param partitions: Optional, A list of partitions numbers for which the function receives events.
         :param sasl_user: Optional, user name to use for sasl authentications
         :param sasl_pass: Optional, password to use for sasl authentications
+        :param tls_enable: Optional, if set - whether to enable TLS or not.
         :param attributes: Optional, extra attributes to be passed to kafka trigger
         """
         if isinstance(topics, str):
@@ -1095,10 +1098,15 @@ class KafkaSource(OnlineSource):
         attributes["initial_offset"] = initial_offset
         if partitions is not None:
             attributes["partitions"] = partitions
-        if sasl := mlrun.datastore.utils.KafkaParameters(attributes).sasl(
-            usr=sasl_user, pwd=sasl_pass
-        ):
+
+        kafka_params = mlrun.datastore.utils.KafkaParameters(attributes)
+
+        if sasl := kafka_params.sasl(usr=sasl_user, pwd=sasl_pass):
             attributes["sasl"] = sasl
+
+        if tls := kafka_params.tls(tls_enable=tls_enable):
+            attributes["tls"] = tls
+
         super().__init__(attributes=attributes, **kwargs)
 
     def to_dataframe(

mlrun/datastore/utils.py

@@ -246,6 +246,9 @@ class KafkaParameters:
             "partitions": "",
             "sasl": "",
             "worker_allocation_mode": "",
+            "tls_enable": "",  # for Nuclio with Confluent Kafka (Sarama client)
+            "tls": "",
+            "new_topic": "",
         }
         self._reference_dicts = (
             self._custom_attributes,
@@ -270,7 +273,9 @@ class KafkaParameters:
         }
         if sasl := self._kwargs.get("sasl"):
             res |= {
-                "security_protocol": "SASL_PLAINTEXT",
+                "security_protocol": self._kwargs.get(
+                    "security_protocol", "SASL_PLAINTEXT"
+                ),
                 "sasl_mechanism": sasl["mechanism"],
                 "sasl_plain_username": sasl["user"],
                 "sasl_plain_password": sasl["password"],
@@ -288,15 +293,25 @@ class KafkaParameters:
 
     def sasl(
         self, *, usr: typing.Optional[str] = None, pwd: typing.Optional[str] = None
-    ) -> dict:
-        usr = usr or self._kwargs.get("sasl_plain_username", None)
-        pwd = pwd or self._kwargs.get("sasl_plain_password", None)
+    ) -> dict[str, typing.Union[str, bool]]:
         res = self._kwargs.get("sasl", {})
+        usr = usr or self._kwargs.get("sasl_plain_username")
+        pwd = pwd or self._kwargs.get("sasl_plain_password")
         if usr and pwd:
             res["enable"] = True
             res["user"] = usr
             res["password"] = pwd
             res["mechanism"] = self._kwargs.get("sasl_mechanism", "PLAIN")
+            res["handshake"] = self._kwargs.get("sasl_handshake", True)
+        return res
+
+    def tls(self, *, tls_enable: typing.Optional[bool] = None) -> dict[str, bool]:
+        res = self._kwargs.get("tls", {})
+        tls_enable = (
+            tls_enable if tls_enable is not None else self._kwargs.get("tls_enable")
+        )
+        if tls_enable:
+            res["enable"] = tls_enable
         return res
 
     def valid_entries_only(self, input_dict: dict) -> dict:

mlrun/db/base.py

@@ -734,7 +734,7 @@ class RunDBInterface(ABC):
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
         start: Optional[datetime.datetime] = None,
         end: Optional[datetime.datetime] = None,
-        tsdb_metrics: bool = True,
+        tsdb_metrics: bool = False,
         metric_list: Optional[list[str]] = None,
         top_level: bool = False,
         uids: Optional[list[str]] = None,

mlrun/db/httpdb.py

@@ -21,7 +21,7 @@ import typing
 import warnings
 from copy import deepcopy
 from datetime import datetime, timedelta
-from os import path, remove
+from os import environ, path, remove
 from typing import Literal, Optional, Union
 from urllib.parse import urlparse
 
@@ -129,7 +129,9 @@ class HTTPRunDB(RunDBInterface):
         self._wait_for_background_task_terminal_state_retry_interval = 3
         self._wait_for_project_deletion_interval = 3
         self.client_version = version.Version().get()["version"]
-        self.python_version = str(version.Version().get_python_version())
+        self.python_version = environ.get("MLRUN_PYTHON_VERSION") or str(
+            version.Version().get_python_version()
+        )
 
         self._enrich_and_validate(url)
 
@@ -1276,8 +1278,8 @@ class HTTPRunDB(RunDBInterface):
         :param producer_uri:    Return artifacts produced by the requested producer URI. Producer URI usually
             points to a run and is used to filter artifacts by the run that produced them when the artifact producer id
             is a workflow id (artifact was created as part of a workflow).
-        :param format_:         The format in which to return the artifacts. Default is 'full'.
-        :param limit:           Maximum number of artifacts to return.
+        :param format_: The format in which to return the artifacts. Default is 'full'.
+        :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
         :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
             parameter must be provided as well.
         :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
@@ -2221,18 +2223,20 @@ class HTTPRunDB(RunDBInterface):
         elif pipe_file.endswith(".zip"):
             headers = {"content-type": "application/zip"}
         else:
-            raise ValueError("pipeline file must be .yaml or .zip")
+            raise ValueError("'pipeline' file must be .yaml or .zip")
         if arguments:
             if not isinstance(arguments, dict):
-                raise ValueError("arguments must be dict type")
+                raise ValueError("'arguments' must be dict type")
             headers[mlrun.common.schemas.HeaderNames.pipeline_arguments] = str(
                 arguments
             )
 
         if not path.isfile(pipe_file):
-            raise OSError(f"file {pipe_file} doesnt exist")
+            raise OSError(f"File {pipe_file} doesnt exist")
         with open(pipe_file, "rb") as fp:
             data = fp.read()
+            if not data:
+                raise ValueError("The compiled pipe file is empty")
         if not isinstance(pipeline, str):
             remove(pipe_file)
 
@@ -3767,7 +3771,7 @@ class HTTPRunDB(RunDBInterface):
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
-        tsdb_metrics: bool = True,
+        tsdb_metrics: bool = False,
         metric_list: Optional[list[str]] = None,
         top_level: bool = False,
         uids: Optional[list[str]] = None,
@@ -3889,8 +3893,8 @@ class HTTPRunDB(RunDBInterface):
         attributes_keys = list(attributes.keys())
         attributes["name"] = name
         attributes["project"] = project
-        attributes["function-name"] = function_name or None
-        attributes["function-tag"] = function_tag or None
+        attributes["function_name"] = function_name or None
+        attributes["function_tag"] = function_tag or None
         attributes["uid"] = endpoint_id or None
         model_endpoint = mlrun.common.schemas.ModelEndpoint.from_flat_dict(attributes)
         path = f"projects/{project}/model-endpoints"
@@ -3981,6 +3985,7 @@ class HTTPRunDB(RunDBInterface):
                 "deploy_histogram_data_drift_app": deploy_histogram_data_drift_app,
                 "fetch_credentials_from_sys_config": fetch_credentials_from_sys_config,
             },
+            timeout=300,  # 5 minutes
         )
 
     def disable_model_monitoring(
@@ -5099,6 +5104,13 @@ class HTTPRunDB(RunDBInterface):
         project = project or config.default_project
         labels = self._parse_labels(labels)
 
+        if limit:
+            # TODO: Remove this in 1.11.0
+            warnings.warn(
+                "'limit' is deprecated and will be removed in 1.11.0. Use 'page' and 'page_size' instead.",
+                FutureWarning,
+            )
+
         params = {
             "name": name,
             "tag": tag,

mlrun/db/nopdb.py

@@ -630,7 +630,7 @@ class NopDB(RunDBInterface):
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
         start: Optional[datetime.datetime] = None,
         end: Optional[datetime.datetime] = None,
-        tsdb_metrics: bool = True,
+        tsdb_metrics: bool = False,
         metric_list: Optional[list[str]] = None,
         top_level: bool = False,
         uids: Optional[list[str]] = None,

mlrun/errors.py

@@ -230,6 +230,13 @@ class MLRunTSDBConnectionFailureError(MLRunHTTPStatusError, ValueError):
     error_status_code = HTTPStatus.BAD_REQUEST.value
 
 
+class MLRunMissingProjectError(MLRunBadRequestError):
+    default_message = "Project must be provided"
+
+    def __init__(self, message=None):
+        super().__init__(message or self.default_message)
+
+
 class MLRunRetryExhaustedError(Exception):
     pass
 

mlrun/execution.py

@@ -15,6 +15,7 @@
 import logging
 import os
 import uuid
+import warnings
 from copy import deepcopy
 from typing import Optional, Union, cast
 
@@ -991,6 +992,14 @@ class MLClientCtx:
         self._update_run()
         return item
 
+    def get_cached_artifact(self, key):
+        """Return a logged artifact from cache (for potential updates)"""
+        warnings.warn(
+            "get_cached_artifact is deprecated in 1.8.0 and will be removed in 1.11.0. Use get_artifact instead.",
+            FutureWarning,
+        )
+        return self.get_artifact(key)
+
     def get_artifact(
         self, key, tag=None, iter=None, tree=None, uid=None
     ) -> Optional[Artifact]:

mlrun/launcher/client.py

@@ -72,7 +72,7 @@ class ClientBaseLauncher(launcher.BaseLauncher, abc.ABC):
     ):
         run.metadata.labels[mlrun_constants.MLRunInternalLabels.kind] = runtime.kind
         mlrun.runtimes.utils.enrich_run_labels(
-            run.metadata.labels, [mlrun.common.runtimes.constants.RunLabels.owner]
+            run.metadata.labels, [mlrun_constants.MLRunInternalLabels.owner]
         )
         if run.spec.output_path:
             run.spec.output_path = run.spec.output_path.replace(