mlrun 1.7.0rc43__py3-none-any.whl → 1.7.0rc56__py3-none-any.whl

mlrun/projects/project.py

@@ -67,13 +67,7 @@ from ..features import Feature
 from ..model import EntrypointParam, ImageBuilder, ModelObj
 from ..run import code_to_function, get_object, import_function, new_function
 from ..secrets import SecretsStore
-from ..utils import (
-    is_ipython,
-    is_relative_path,
-    is_yaml_path,
-    logger,
-    update_in,
-)
+from ..utils import is_jupyter, is_relative_path, is_yaml_path, logger, update_in
 from ..utils.clones import (
     add_credentials_git_remote_url,
     clone_git,
@@ -714,7 +708,7 @@ def _load_project_from_db(url, secrets, user_project=False):
 
 def _delete_project_from_db(project_name, secrets, deletion_strategy):
     db = mlrun.db.get_run_db(secrets=secrets)
-    return db.delete_project(project_name, deletion_strategy=deletion_strategy)
+    db.delete_project(project_name, deletion_strategy=deletion_strategy)
 
 
 def _load_project_file(url, name="", secrets=None, allow_cross_project=None):
@@ -1558,7 +1552,7 @@ class MlrunProject(ModelObj):
             url = path.normpath(path.join(self.spec.get_code_path(), url))
 
         if (not in_context or check_path_in_context) and not path.isfile(url):
-            raise mlrun.errors.MLRunNotFoundError(f"{url} not found")
+            raise FileNotFoundError(f"{url} not found")
 
         return url, in_context
 
@@ -1599,7 +1593,9 @@ class MlrunProject(ModelObj):
         :param format:        artifact file format: csv, png, ..
         :param tag:           version tag
         :param target_path:   absolute target path (instead of using artifact_path + local_path)
-        :param upload:        upload to datastore (default is True)
+        :param upload:        Whether to upload the artifact to the datastore. If not provided, and the `local_path`
+                              is not a directory, upload occurs by default. Directories are uploaded only when this
+                              flag is explicitly set to `True`.
         :param labels:        a set of key/value labels to tag the artifact with
 
         :returns: artifact object
@@ -1954,7 +1950,6 @@ class MlrunProject(ModelObj):
         application_class: typing.Union[
             str,
             mm_app.ModelMonitoringApplicationBase,
-            mm_app.ModelMonitoringApplicationBaseV2,
         ] = None,
         name: str = None,
         image: str = None,
@@ -2022,7 +2017,6 @@ class MlrunProject(ModelObj):
         application_class: typing.Union[
             str,
             mm_app.ModelMonitoringApplicationBase,
-            mm_app.ModelMonitoringApplicationBaseV2,
         ] = None,
         name: str = None,
         image: str = None,
@@ -2080,7 +2074,6 @@ class MlrunProject(ModelObj):
         application_class: typing.Union[
             str,
             mm_app.ModelMonitoringApplicationBase,
-            mm_app.ModelMonitoringApplicationBaseV2,
             None,
         ] = None,
         name: typing.Optional[str] = None,
@@ -2169,7 +2162,8 @@ class MlrunProject(ModelObj):
 
         :param default_controller_image:          Deprecated.
         :param base_period:                       The time period in minutes in which the model monitoring controller
-                                                  function is triggered. By default, the base period is 10 minutes.
+                                                  function is triggered. By default, the base period is 10 minutes
+                                                  (which is also the minimum value for production environments).
         :param image:                             The image of the model monitoring controller, writer, monitoring
                                                   stream & histogram data drift functions, which are real time nuclio
                                                   functions. By default, the image is mlrun/mlrun.
@@ -2188,6 +2182,12 @@ class MlrunProject(ModelObj):
                 FutureWarning,
             )
             image = default_controller_image
+        if base_period < 10:
+            logger.warn(
+                "enable_model_monitoring: 'base_period' < 10 minutes is not supported in production environments",
+                project=self.name,
+            )
+
         db = mlrun.db.get_run_db(secrets=self._secrets)
         db.enable_model_monitoring(
             project=self.name,
@@ -2439,7 +2439,7 @@ class MlrunProject(ModelObj):
         ):
             # if function path is not provided and it is not a module (no ".")
             # use the current notebook as default
-            if is_ipython:
+            if is_jupyter:
                 from IPython import get_ipython
 
                 kernel = get_ipython()
@@ -2842,11 +2842,13 @@ class MlrunProject(ModelObj):
         The function objects are synced against the definitions spec in `self.spec._function_definitions`.
         Referenced files/URLs in the function spec will be reloaded.
         Function definitions are parsed by the following precedence:
-            1. Contains runtime spec.
-            2. Contains module in the project's context.
-            3. Contains path to function definition (yaml, DB, Hub).
-            4. Contains path to .ipynb or .py files.
-            5. Contains a Nuclio/Serving function image / an 'Application' kind definition.
+
+        1. Contains runtime spec.
+        2. Contains module in the project's context.
+        3. Contains path to function definition (yaml, DB, Hub).
+        4. Contains path to .ipynb or .py files.
+        5. Contains a Nuclio/Serving function image / an 'Application' kind definition.
+
         If function definition is already an object, some project metadata updates will apply however,
         it will not be reloaded.
 
@@ -2902,6 +2904,16 @@ class MlrunProject(ModelObj):
                         continue
 
                     raise mlrun.errors.MLRunMissingDependencyError(message) from exc
+
+                except Exception as exc:
+                    if silent:
+                        logger.warn(
+                            "Failed to instantiate function",
+                            name=name,
+                            error=mlrun.utils.err_to_str(exc),
+                        )
+                        continue
+                    raise exc
             else:
                 message = f"Function {name} must be an object or dict."
                 if silent:
@@ -3060,6 +3072,7 @@ class MlrunProject(ModelObj):
         source: str = None,
         cleanup_ttl: int = None,
         notifications: list[mlrun.model.Notification] = None,
+        workflow_runner_node_selector: typing.Optional[dict[str, str]] = None,
     ) -> _PipelineRunStatus:
         """Run a workflow using kubeflow pipelines
 
@@ -3088,15 +3101,20 @@ class MlrunProject(ModelObj):
 
                           * Remote URL which is loaded dynamically to the workflow runner.
                           * A path to the project's context on the workflow runner's image.
-                          Path can be absolute or relative to `project.spec.build.source_code_target_dir` if defined
-                          (enriched when building a project image with source, see `MlrunProject.build_image`).
-                          For other engines the source is used to validate that the code is up-to-date.
+                            Path can be absolute or relative to `project.spec.build.source_code_target_dir` if defined
+                            (enriched when building a project image with source, see `MlrunProject.build_image`).
+                            For other engines the source is used to validate that the code is up-to-date.
+
         :param cleanup_ttl:
                           Pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
                           workflow and all its resources are deleted)
         :param notifications:
                           List of notifications to send for workflow completion
-
+        :param workflow_runner_node_selector:
+                          Defines the node selector for the workflow runner pod when using a remote engine.
+                          This allows you to control and specify where the workflow runner pod will be scheduled.
+                          This setting is only relevant when the engine is set to 'remote' or for scheduled workflows,
+                          and it will be ignored if the workflow is not run on a remote engine.
         :returns: ~py:class:`~mlrun.projects.pipelines._PipelineRunStatus` instance
         """
 
@@ -3162,6 +3180,16 @@ class MlrunProject(ModelObj):
             )
             inner_engine = get_workflow_engine(engine_kind, local).engine
         workflow_spec.engine = inner_engine or workflow_engine.engine
+        if workflow_runner_node_selector:
+            if workflow_engine.engine == "remote":
+                workflow_spec.workflow_runner_node_selector = (
+                    workflow_runner_node_selector
+                )
+            else:
+                logger.warn(
+                    "'workflow_runner_node_selector' applies only to remote engines"
+                    " and is ignored for non-remote runs."
+                )
 
         run = workflow_engine.run(
             self,

mlrun/render.py

@@ -22,7 +22,7 @@ import mlrun.utils
 
 from .config import config
 from .datastore import uri_to_ipython
-from .utils import dict_to_list, get_in, is_ipython
+from .utils import dict_to_list, get_in, is_jupyter
 
 JUPYTER_SERVER_ROOT = environ.get("HOME", "/User")
 supported_viewers = [
@@ -181,8 +181,8 @@ def run_to_html(results, display=True):
 
 
 def ipython_display(html, display=True, alt_text=None):
-    if display and html and is_ipython:
-        import IPython
+    if display and html and is_jupyter:
+        import IPython.display
 
         IPython.display.display(IPython.display.HTML(html))
     elif alt_text:

mlrun/runtimes/daskjob.py

@@ -379,7 +379,7 @@ class DaskCluster(KubejobRuntime):
         :param show_on_failure:         show logs only in case of build failure
         :param force_build:             force building the image, even when no changes were made
 
-        :return True if the function is ready (deployed)
+        :return:                        True if the function is ready (deployed)
         """
         return super().deploy(
             watch,

mlrun/runtimes/kubejob.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import typing
 import warnings
 
 from mlrun_pipelines.common.ops import build_op
@@ -143,11 +143,11 @@ class KubejobRuntime(KubeResource):
 
     def deploy(
         self,
-        watch=True,
-        with_mlrun=None,
-        skip_deployed=False,
-        is_kfp=False,
-        mlrun_version_specifier=None,
+        watch: bool = True,
+        with_mlrun: typing.Optional[bool] = None,
+        skip_deployed: bool = False,
+        is_kfp: bool = False,
+        mlrun_version_specifier: typing.Optional[bool] = None,
         builder_env: dict = None,
         show_on_failure: bool = False,
         force_build: bool = False,

mlrun/runtimes/nuclio/api_gateway.py

@@ -22,6 +22,7 @@ from nuclio.auth import AuthKinds as NuclioAuthKinds
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
+import mlrun.common.helpers
 import mlrun.common.schemas as schemas
 import mlrun.common.types
 from mlrun.model import ModelObj
@@ -202,8 +203,13 @@ class APIGatewaySpec(ModelObj):
         self.project = project
         self.ports = ports
 
+        self.enrich()
         self.validate(project=project, functions=functions, canary=canary, ports=ports)
 
+    def enrich(self):
+        if self.path and not self.path.startswith("/"):
+            self.path = f"/{self.path}"
+
     def validate(
         self,
         project: str,
@@ -587,6 +593,12 @@ class APIGateway(ModelObj):
             self.metadata.annotations, gateway_timeout
         )
 
+    def with_annotations(self, annotations: dict):
+        """set a key/value annotations in the metadata of the api gateway"""
+        for key, value in annotations.items():
+            self.metadata.annotations[key] = str(value)
+        return self
+
     @classmethod
     def from_scheme(cls, api_gateway: schemas.APIGateway):
         project = api_gateway.metadata.labels.get(

mlrun/runtimes/nuclio/application/application.py

@@ -438,9 +438,10 @@ class ApplicationRuntime(RemoteRuntime):
         """
         Create the application API gateway. Once the application is deployed, the API gateway can be created.
         An application without an API gateway is not accessible.
+
         :param name:                    The name of the API gateway
         :param path:                    Optional path of the API gateway, default value is "/".
-            The given path should be supported by the deployed application
+                                        The given path should be supported by the deployed application
         :param direct_port_access:      Set True to allow direct port access to the application sidecar
         :param authentication_mode:     API Gateway authentication mode
         :param authentication_creds:    API Gateway basic authentication credentials as a tuple (username, password)
@@ -449,8 +450,7 @@ class ApplicationRuntime(RemoteRuntime):
         :param set_as_default:          Set the API gateway as the default for the application (`status.api_gateway`)
         :param gateway_timeout:         nginx ingress timeout in sec (request timeout, when will the gateway return an
                                         error)
-
-        :return:    The API gateway URL
+        :return:                        The API gateway URL
         """
         if not name:
             raise mlrun.errors.MLRunInvalidArgumentError(

mlrun/runtimes/nuclio/function.py

@@ -23,6 +23,7 @@ import inflection
 import nuclio
 import nuclio.utils
 import requests
+import semver
 from aiohttp.client import ClientSession
 from kubernetes import client
 from mlrun_pipelines.common.mounts import VolumeMount
@@ -296,10 +297,37 @@ class RemoteRuntime(KubeResource):
         """
         if hasattr(spec, "to_dict"):
             spec = spec.to_dict()
+
+        self._validate_triggers(spec)
+
         spec["name"] = name
         self.spec.config[f"spec.triggers.{name}"] = spec
         return self
 
+    def _validate_triggers(self, spec):
+        # ML-7763 / NUC-233
+        min_nuclio_version = "1.13.12"
+        if mlconf.nuclio_version and semver.VersionInfo.parse(
+            mlconf.nuclio_version
+        ) < semver.VersionInfo.parse(min_nuclio_version):
+            explicit_ack_enabled = False
+            num_triggers = 0
+            trigger_name = spec.get("name", "UNKNOWN")
+            for key, config in [(f"spec.triggers.{trigger_name}", spec)] + list(
+                self.spec.config.items()
+            ):
+                if key.startswith("spec.triggers."):
+                    num_triggers += 1
+                    explicit_ack_enabled = (
+                        config.get("explicitAckMode", "disable") != "disable"
+                    )
+
+            if num_triggers > 1 and explicit_ack_enabled:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "Multiple triggers cannot be used in conjunction with explicit ack. "
+                    f"Please upgrade to nuclio {min_nuclio_version} or newer."
+                )
+
     def with_source_archive(
         self,
         source,
@@ -495,6 +523,11 @@ class RemoteRuntime(KubeResource):
         extra_attributes = extra_attributes or {}
         if ack_window_size:
             extra_attributes["ackWindowSize"] = ack_window_size
+
+        access_key = kwargs.pop("access_key", None)
+        if not access_key:
+            access_key = self._resolve_v3io_access_key()
+
         self.add_trigger(
             name,
             V3IOStreamTrigger(
@@ -506,6 +539,7 @@ class RemoteRuntime(KubeResource):
                 webapi=endpoint or "http://v3io-webapi:8081",
                 extra_attributes=extra_attributes,
                 read_batch_size=256,
+                access_key=access_key,
                 **kwargs,
             ),
         )
@@ -1241,6 +1275,13 @@ class RemoteRuntime(KubeResource):
 
         return self._resolve_invocation_url("", force_external_address)
 
+    @staticmethod
+    def _resolve_v3io_access_key():
+        # Nuclio supports generating access key for v3io stream trigger only from version 1.13.11
+        if validate_nuclio_version_compatibility("1.13.11"):
+            return mlrun.model.Credentials.generate_access_key
+        return None
+
 
 def parse_logs(logs):
     logs = json.loads(logs)

mlrun/runtimes/nuclio/serving.py

@@ -314,8 +314,8 @@ class ServingRuntime(RemoteRuntime):
         tracking_policy: Optional[Union["TrackingPolicy", dict]] = None,
         enable_tracking: bool = True,
     ) -> None:
-        """apply on your serving function to monitor a deployed model, including real-time dashboards to detect drift
-           and analyze performance.
+        """Apply on your serving function to monitor a deployed model, including real-time dashboards to detect drift
+        and analyze performance.
 
         :param stream_path:         Path/url of the tracking stream e.g. v3io:///users/mike/mystream
                                     you can use the "dummy://" path for test/simulation.

mlrun/runtimes/pod.py

@@ -1107,12 +1107,12 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
 
         :param state_thresholds: A dictionary of state to threshold. The supported states are:
 
-            * pending_scheduled - The pod/crd is scheduled on a node but not yet running
-            * pending_not_scheduled - The pod/crd is not yet scheduled on a node
-            * executing - The pod/crd started and is running
-            * image_pull_backoff - The pod/crd is in image pull backoff
-            See mlrun.mlconf.function.spec.state_thresholds for the default thresholds.
+                                 * pending_scheduled - The pod/crd is scheduled on a node but not yet running
+                                 * pending_not_scheduled - The pod/crd is not yet scheduled on a node
+                                 * executing - The pod/crd started and is running
+                                 * image_pull_backoff - The pod/crd is in image pull backoff
 
+                                See :code:`mlrun.mlconf.function.spec.state_thresholds` for the default thresholds.
         :param patch: Whether to merge the given thresholds with the existing thresholds (True, default)
                       or override them (False)
         """
@@ -1347,20 +1347,26 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
 
     def _build_image(
         self,
-        builder_env,
-        force_build,
-        mlrun_version_specifier,
-        show_on_failure,
-        skip_deployed,
-        watch,
-        is_kfp,
-        with_mlrun,
+        builder_env: dict,
+        force_build: bool,
+        mlrun_version_specifier: typing.Optional[bool],
+        show_on_failure: bool,
+        skip_deployed: bool,
+        watch: bool,
+        is_kfp: bool,
+        with_mlrun: typing.Optional[bool],
     ):
         # When we're in pipelines context we must watch otherwise the pipelines pod will exit before the operation
         # is actually done. (when a pipelines pod exits, the pipeline step marked as done)
         if is_kfp:
             watch = True
 
+        if skip_deployed and self.requires_build() and not self.is_deployed():
+            logger.warning(
+                f"Even though {skip_deployed=}, the build might be triggered due to the function's configuration. "
+                "See requires_build() and is_deployed() for reasoning."
+            )
+
         db = self._get_db()
         data = db.remote_builder(
             self,

mlrun/serving/server.py

@@ -401,6 +401,8 @@ def v2_serving_handler(context, event, get_body=False):
         "kafka-cluster",
         "v3ioStream",
         "v3io-stream",
+        "rabbit-mq",
+        "rabbitMq",
     ):
         event.path = "/"
 

mlrun/utils/helpers.py

@@ -41,7 +41,7 @@ import semver
 import yaml
 from dateutil import parser
 from mlrun_pipelines.models import PipelineRun
-from pandas._libs.tslibs.timestamps import Timedelta, Timestamp
+from pandas import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
 import mlrun
@@ -111,9 +111,12 @@ def get_artifact_target(item: dict, project=None):
     project_str = project or item["metadata"].get("project")
     tree = item["metadata"].get("tree")
     tag = item["metadata"].get("tag")
+    kind = item.get("kind")
 
-    if item.get("kind") in {"dataset", "model", "artifact"} and db_key:
-        target = f"{DB_SCHEMA}://{StorePrefix.Artifact}/{project_str}/{db_key}"
+    if kind in {"dataset", "model", "artifact"} and db_key:
+        target = (
+            f"{DB_SCHEMA}://{StorePrefix.kind_to_prefix(kind)}/{project_str}/{db_key}"
+        )
         target += f":{tag}" if tag else ":latest"
         if tree:
             target += f"@{tree}"
@@ -133,18 +136,25 @@ def is_legacy_artifact(artifact):
 logger = create_logger(config.log_level, config.log_formatter, "mlrun", sys.stdout)
 missing = object()
 
-is_ipython = False
+is_ipython = False  # is IPython terminal, including Jupyter
+is_jupyter = False  # is Jupyter notebook/lab terminal
 try:
-    import IPython
+    import IPython.core.getipython
+
+    ipy = IPython.core.getipython.get_ipython()
+
+    is_ipython = ipy is not None
+    is_jupyter = (
+        is_ipython
+        # not IPython
+        and "Terminal" not in str(type(ipy))
+    )
 
-    ipy = IPython.get_ipython()
-    # if its IPython terminal ignore (cant show html)
-    if ipy and "Terminal" not in str(type(ipy)):
-        is_ipython = True
-except ImportError:
+    del ipy
+except ModuleNotFoundError:
     pass
 
-if is_ipython and config.nest_asyncio_enabled in ["1", "True"]:
+if is_jupyter and config.nest_asyncio_enabled in ["1", "True"]:
     # bypass Jupyter asyncio bug
     import nest_asyncio
 
@@ -1421,11 +1431,7 @@ def is_running_in_jupyter_notebook() -> bool:
     Check if the code is running inside a Jupyter Notebook.
     :return: True if running inside a Jupyter Notebook, False otherwise.
     """
-    import IPython
-
-    ipy = IPython.get_ipython()
-    # if its IPython terminal, it isn't a Jupyter ipython
-    return ipy and "Terminal" not in str(type(ipy))
+    return is_jupyter
 
 
 def create_ipython_display():
@@ -1776,3 +1782,43 @@ def _reload(module, max_recursion_depth):
         attribute = getattr(module, attribute_name)
         if type(attribute) is ModuleType:
             _reload(attribute, max_recursion_depth - 1)
+
+
+def run_with_retry(
+    retry_count: int,
+    func: typing.Callable,
+    retry_on_exceptions: typing.Union[
+        type[Exception],
+        tuple[type[Exception]],
+    ] = None,
+    *args,
+    **kwargs,
+):
+    """
+    Executes a function with retry logic upon encountering specified exceptions.
+
+    :param retry_count: The number of times to retry the function execution.
+    :param func: The function to execute.
+    :param retry_on_exceptions: Exception(s) that trigger a retry. Can be a single exception or a tuple of exceptions.
+    :param args: Positional arguments to pass to the function.
+    :param kwargs: Keyword arguments to pass to the function.
+    :return: The result of the function execution if successful.
+    :raises Exception: Re-raises the last exception encountered after all retries are exhausted.
+    """
+    if retry_on_exceptions is None:
+        retry_on_exceptions = (Exception,)
+    elif isinstance(retry_on_exceptions, list):
+        retry_on_exceptions = tuple(retry_on_exceptions)
+
+    last_exception = None
+    for attempt in range(retry_count + 1):
+        try:
+            return func(*args, **kwargs)
+        except retry_on_exceptions as exc:
+            last_exception = exc
+            logger.warning(
+                f"Attempt {{{attempt}/ {retry_count}}} failed with exception: {exc}",
+            )
+            if attempt == retry_count:
+                raise
+    raise last_exception

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "13896619ed9e935aab2e08c075dc12ae6f3e449f",
-  "version": "1.7.0-rc43"
+  "git_commit": "77b699660fb5165d378227e4260714d68aee9fc2",
+  "version": "1.7.0-rc56"
 }