mlrun 1.7.0rc38__py3-none-any.whl → 1.7.0rc40__py3-none-any.whl

mlrun/projects/pipelines.py

@@ -447,7 +447,6 @@ class _PipelineRunner(abc.ABC):
         namespace=None,
         source=None,
         notifications: list[mlrun.model.Notification] = None,
-        send_start_notification: bool = True,
     ) -> _PipelineRunStatus:
         pass
 
@@ -567,7 +566,6 @@ class _KFPRunner(_PipelineRunner):
         namespace=None,
         source=None,
         notifications: list[mlrun.model.Notification] = None,
-        send_start_notification: bool = True,
     ) -> _PipelineRunStatus:
         pipeline_context.set(project, workflow_spec)
         workflow_handler = _PipelineRunner._get_handler(
@@ -585,7 +583,8 @@ class _KFPRunner(_PipelineRunner):
                 "Notifications will only be sent if you wait for pipeline completion. "
                 "To use the new notification behavior, use the remote pipeline runner."
             )
-            for notification in notifications:
+            # for start message, fallback to old notification behavior
+            for notification in notifications or []:
                 project.notifiers.add_notification(
                     notification.kind, notification.params
                 )
@@ -616,13 +615,12 @@ class _KFPRunner(_PipelineRunner):
                     func_name=func.metadata.name,
                     exc_info=err_to_str(exc),
                 )
-        if send_start_notification:
-            project.notifiers.push_pipeline_start_message(
-                project.metadata.name,
-                project.get_param("commit_id", None),
-                run_id,
-                True,
-            )
+        project.notifiers.push_pipeline_start_message(
+            project.metadata.name,
+            project.get_param("commit_id", None),
+            run_id,
+            True,
+        )
         pipeline_context.clear()
         return _PipelineRunStatus(run_id, cls, project=project, workflow=workflow_spec)
 
@@ -670,7 +668,6 @@ class _LocalRunner(_PipelineRunner):
         namespace=None,
         source=None,
         notifications: list[mlrun.model.Notification] = None,
-        send_start_notification: bool = True,
     ) -> _PipelineRunStatus:
         pipeline_context.set(project, workflow_spec)
         workflow_handler = _PipelineRunner._get_handler(
@@ -692,10 +689,9 @@ class _LocalRunner(_PipelineRunner):
             project.set_source(source=source)
         pipeline_context.workflow_artifact_path = artifact_path
 
-        if send_start_notification:
-            project.notifiers.push_pipeline_start_message(
-                project.metadata.name, pipeline_id=workflow_id
-            )
+        project.notifiers.push_pipeline_start_message(
+            project.metadata.name, pipeline_id=workflow_id
+        )
         err = None
         try:
             workflow_handler(**workflow_spec.args)
@@ -755,22 +751,10 @@ class _RemoteRunner(_PipelineRunner):
         namespace: str = None,
         source: str = None,
         notifications: list[mlrun.model.Notification] = None,
-        send_start_notification: bool = True,
     ) -> typing.Optional[_PipelineRunStatus]:
         workflow_name = normalize_workflow_name(name=name, project_name=project.name)
         workflow_id = None
 
-        # for start message, fallback to old notification behavior
-        if send_start_notification:
-            for notification in notifications or []:
-                project.notifiers.add_notification(
-                    notification.kind, notification.params
-                )
-                # if a notification with `when=running` is provided, it will be used explicitly and others
-                # will be ignored
-                if "running" in notification.when:
-                    break
-
         # The returned engine for this runner is the engine of the workflow.
         # In this way wait_for_completion/get_run_status would be executed by the correct pipeline runner.
         inner_engine = get_workflow_engine(workflow_spec.engine)
@@ -870,9 +854,6 @@ class _RemoteRunner(_PipelineRunner):
             state = mlrun_pipelines.common.models.RunStatuses.failed
         else:
             state = mlrun_pipelines.common.models.RunStatuses.running
-            project.notifiers.push_pipeline_start_message(
-                project.metadata.name,
-            )
             pipeline_context.clear()
         return _PipelineRunStatus(
             run_id=workflow_id,
@@ -1078,6 +1059,13 @@ def load_and_run(
     if load_only:
         return
 
+    # extract "start" notification if exists
+    start_notifications = [
+        notification
+        for notification in context.get_notifications()
+        if "running" in notification.when
+    ]
+
     workflow_log_message = workflow_name or workflow_path
     context.logger.info(f"Running workflow {workflow_log_message} from remote")
     run = project.run(
@@ -1093,6 +1081,7 @@ def load_and_run(
         cleanup_ttl=cleanup_ttl,
         engine=engine,
         local=local,
+        notifications=start_notifications,
     )
     context.log_result(key="workflow_id", value=run.run_id)
     context.log_result(key="engine", value=run._engine.engine, commit=True)

mlrun/projects/project.py

@@ -517,17 +517,24 @@ def get_or_create_project(
             parameters=parameters,
             allow_cross_project=allow_cross_project,
         )
-        logger.info("Project loaded successfully", project_name=name)
+        logger.info("Project loaded successfully", project_name=project.name)
         return project
     except mlrun.errors.MLRunNotFoundError:
-        logger.debug("Project not found in db", project_name=name)
+        logger.debug(
+            "Project not found in db", project_name=name, user_project=user_project
+        )
 
     spec_path = path.join(context, subpath or "", "project.yaml")
     load_from_path = url or path.isfile(spec_path)
     # do not nest under "try" or else the exceptions raised below will be logged along with the "not found" message
     if load_from_path:
         # loads a project from archive or local project.yaml
-        logger.info("Loading project from path", project_name=name, path=url or context)
+        logger.info(
+            "Loading project from path",
+            project_name=name,
+            user_project=user_project,
+            path=url or context,
+        )
         project = load_project(
             context,
             url,
@@ -544,7 +551,7 @@ def get_or_create_project(
 
         logger.info(
             "Project loaded successfully",
-            project_name=name,
+            project_name=project.name,
             path=url or context,
             stored_in_db=save,
         )
@@ -562,7 +569,9 @@ def get_or_create_project(
         save=save,
         parameters=parameters,
     )
-    logger.info("Project created successfully", project_name=name, stored_in_db=save)
+    logger.info(
+        "Project created successfully", project_name=project.name, stored_in_db=save
+    )
     return project
 
 
@@ -2394,7 +2403,11 @@ class MlrunProject(ModelObj):
         requirements: typing.Union[str, list[str]] = None,
         requirements_file: str = "",
     ) -> tuple[str, str, mlrun.runtimes.BaseRuntime, dict]:
-        if func is None and not _has_module(handler, kind):
+        if (
+            func is None
+            and not _has_module(handler, kind)
+            and mlrun.runtimes.RuntimeKinds.supports_from_notebook(kind)
+        ):
             # if function path is not provided and it is not a module (no ".")
             # use the current notebook as default
             if is_ipython:
@@ -2973,7 +2986,6 @@ class MlrunProject(ModelObj):
         source: str = None,
         cleanup_ttl: int = None,
         notifications: list[mlrun.model.Notification] = None,
-        send_start_notification: bool = True,
     ) -> _PipelineRunStatus:
         """Run a workflow using kubeflow pipelines
 
@@ -3010,8 +3022,6 @@ class MlrunProject(ModelObj):
                           workflow and all its resources are deleted)
         :param notifications:
                           List of notifications to send for workflow completion
-        :param send_start_notification:
-                          Send a notification when the workflow starts
 
         :returns: ~py:class:`~mlrun.projects.pipelines._PipelineRunStatus` instance
         """
@@ -3089,7 +3099,6 @@ class MlrunProject(ModelObj):
             namespace=namespace,
             source=source,
             notifications=notifications,
-            send_start_notification=send_start_notification,
         )
         # run is None when scheduling
         if run and run.state == mlrun_pipelines.common.models.RunStatuses.failed:
@@ -3222,30 +3231,30 @@ class MlrunProject(ModelObj):
         infrastructure functions. Important to note that you have to set the credentials before deploying any
         model monitoring or serving function.
 
-        :param access_key:                Model Monitoring access key for managing user permissions.
-        :param endpoint_store_connection: Endpoint store connection string. By default, None.
-                                          Options:
-                                          1. None, will be set from the system configuration.
-                                          2. v3io - for v3io endpoint store,
-                                             pass `v3io` and the system will generate the exact path.
-                                          3. MySQL/SQLite - for SQL endpoint store, please provide full
-                                             connection string, for example
-                                             mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>
-        :param stream_path:               Path to the model monitoring stream. By default, None.
-                                          Options:
-                                          1. None, will be set from the system configuration.
-                                          2. v3io - for v3io stream,
-                                             pass `v3io` and the system will generate the exact path.
-                                          3. Kafka - for Kafka stream, please provide full connection string without
-                                             custom topic, for example kafka://<some_kafka_broker>:<port>.
+        :param access_key:                Model monitoring access key for managing user permissions.
+        :param endpoint_store_connection: Endpoint store connection string. By default, None. Options:
+
+                                          * None - will be set from the system configuration.
+                                          * v3io - for v3io endpoint store, pass `v3io` and the system will generate the
+                                            exact path.
+                                          * MySQL/SQLite - for SQL endpoint store, provide the full connection string,
+                                            for example: mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>
+        :param stream_path:               Path to the model monitoring stream. By default, None. Options:
+
+                                          * None - will be set from the system configuration.
+                                          * v3io - for v3io stream, pass `v3io` and the system will generate the exact
+                                            path.
+                                          * Kafka - for Kafka stream, provide the full connection string without custom
+                                            topic, for example kafka://<some_kafka_broker>:<port>.
         :param tsdb_connection:           Connection string to the time series database. By default, None.
                                           Options:
-                                          1. None, will be set from the system configuration.
-                                          2. v3io - for v3io stream,
-                                             pass `v3io` and the system will generate the exact path.
-                                          3. TDEngine - for TDEngine tsdb, please provide full websocket connection URL,
-                                             for example taosws://<username>:<password>@<host>:<port>.
-        :param replace_creds:                     If True, will override the existing credentials.
+
+                                          * None - will be set from the system configuration.
+                                          * v3io - for v3io stream, pass `v3io` and the system will generate the exact
+                                            path.
+                                          * TDEngine - for TDEngine tsdb, provide the full websocket connection URL,
+                                            for example taosws://<username>:<password>@<host>:<port>.
+        :param replace_creds:             If True, will override the existing credentials.
                                           Please keep in mind that if you already enabled model monitoring on
                                           your project this action can cause data loose and will require redeploying
                                           all model monitoring functions & model monitoring infra
@@ -3351,7 +3360,8 @@ class MlrunProject(ModelObj):
                                 * A dictionary of configurations to use when logging. Further info per object type and
                                   artifact type can be given there. The artifact key must appear in the dictionary as
                                   "key": "the_key".
-        :param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
+        :param builder_env:     env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN":
+                                token}
         :param reset_on_run:    When True, function python modules would reload prior to code execution.
                                 This ensures latest code changes are executed. This argument must be used in
                                 conjunction with the local=True argument.
@@ -4061,7 +4071,7 @@ class MlrunProject(ModelObj):
         mlrun.db.get_run_db().delete_api_gateway(name=name, project=self.name)
 
     def store_alert_config(
-        self, alert_data: AlertConfig, alert_name=None
+        self, alert_data: AlertConfig, alert_name: typing.Optional[str] = None
     ) -> AlertConfig:
         """
         Create/modify an alert.
@@ -4070,9 +4080,11 @@ class MlrunProject(ModelObj):
         :param alert_name: The name of the alert.
         :return: the created/modified alert.
         """
+        if not alert_data:
+            raise mlrun.errors.MLRunInvalidArgumentError("Alert data must be provided")
+
         db = mlrun.db.get_run_db(secrets=self._secrets)
-        if alert_name is None:
-            alert_name = alert_data.name
+        alert_name = alert_name or alert_data.name
         if alert_data.project is not None and alert_data.project != self.metadata.name:
             logger.warn(
                 "Project in alert does not match project in operation",
@@ -4375,18 +4387,17 @@ def _init_function_from_dict(
         )
 
     elif url.endswith(".py"):
-        # when load_source_on_run is used we allow not providing image as code will be loaded pre-run. ML-4994
-        if (
-            not image
-            and not project.default_image
-            and kind != "local"
-            and not project.spec.load_source_on_run
-        ):
-            raise ValueError(
-                "image must be provided with py code files which do not "
-                "run on 'local' engine kind"
-            )
         if in_context and with_repo:
+            # when load_source_on_run is used we allow not providing image as code will be loaded pre-run. ML-4994
+            if (
+                not image
+                and not project.default_image
+                and kind != "local"
+                and not project.spec.load_source_on_run
+            ):
+                raise ValueError(
+                    "image must be provided with py code files which do not run on 'local' engine kind"
+                )
             func = new_function(
                 name,
                 command=relative_url,
@@ -4408,7 +4419,6 @@ def _init_function_from_dict(
     elif kind in mlrun.runtimes.RuntimeKinds.nuclio_runtimes():
         func = new_function(
             name,
-            command=relative_url,
             image=image,
             kind=kind,
             handler=handler,
@@ -4462,9 +4472,17 @@ def _init_function_from_obj(
 def _has_module(handler, kind):
     if not handler:
         return False
-    return (
-        kind in mlrun.runtimes.RuntimeKinds.nuclio_runtimes() and ":" in handler
-    ) or "." in handler
+
+    if (
+        kind in mlrun.runtimes.RuntimeKinds.pure_nuclio_deployed_runtimes()
+        and ":" in handler
+    ):
+        return True
+
+    if "." in handler:
+        return True
+
+    return False
 
 
 def _is_imported_artifact(artifact):

mlrun/run.py

@@ -65,6 +65,7 @@ from .runtimes.nuclio.application import ApplicationRuntime
 from .runtimes.utils import add_code_metadata, global_context
 from .utils import (
     RunKeys,
+    create_ipython_display,
     extend_hub_uri_if_needed,
     get_in,
     logger,
@@ -744,11 +745,10 @@ def code_to_function(
         raise ValueError("Databricks tasks only support embed_code=True")
 
     if kind == RuntimeKinds.application:
-        if handler:
-            raise MLRunInvalidArgumentError(
-                "Handler is not supported for application runtime"
-            )
-        filename, handler = ApplicationRuntime.get_filename_and_handler()
+        raise MLRunInvalidArgumentError(
+            "Embedding a code file is not supported for application runtime. "
+            "Code files should be specified via project/function source."
+        )
 
     is_nuclio, sub_kind = RuntimeKinds.resolve_nuclio_sub_kind(kind)
     code_origin = add_name(add_code_metadata(filename), name)
@@ -942,10 +942,12 @@ def wait_for_pipeline_completion(
     if remote:
         mldb = mlrun.db.get_run_db()
 
+        dag_display_id = create_ipython_display()
+
         def _wait_for_pipeline_completion():
             pipeline = mldb.get_pipeline(run_id, namespace=namespace, project=project)
             pipeline_status = pipeline["run"]["status"]
-            show_kfp_run(pipeline, clear_output=True)
+            show_kfp_run(pipeline, dag_display_id=dag_display_id, with_html=False)
             if pipeline_status not in RunStatuses.stable_statuses():
                 logger.debug(
                     "Waiting for pipeline completion",

mlrun/runtimes/__init__.py

@@ -242,6 +242,10 @@ class RuntimeKinds:
         # both spark and remote spark uses different mechanism for assigning images
         return kind not in [RuntimeKinds.spark, RuntimeKinds.remotespark]
 
+    @staticmethod
+    def supports_from_notebook(kind):
+        return kind not in [RuntimeKinds.application]
+
     @staticmethod
     def resolve_nuclio_runtime(kind: str, sub_kind: str):
         kind = kind.split(":")[0]

mlrun/runtimes/nuclio/api_gateway.py

@@ -386,6 +386,7 @@ class APIGateway(ModelObj):
         headers: dict = None,
         credentials: Optional[tuple[str, str]] = None,
         path: Optional[str] = None,
+        body: Optional[Union[str, bytes, dict]] = None,
         **kwargs,
     ):
         """
@@ -396,6 +397,7 @@ class APIGateway(ModelObj):
         :param credentials: (Optional[tuple[str, str]], optional) The (username,password) for the invocation if required
             can also be set by the environment variable (_, V3IO_ACCESS_KEY) for access key authentication.
         :param path: (str, optional) The sub-path for the invocation.
+        :param body: (Optional[Union[str, bytes, dict]]) The body of the invocation.
         :param kwargs: (dict) Additional keyword arguments.
 
         :return: The response from the API gateway invocation.
@@ -444,6 +446,13 @@ class APIGateway(ModelObj):
                     "API Gateway invocation requires authentication. Please set V3IO_ACCESS_KEY env var"
                 )
         url = urljoin(self.invoke_url, path or "")
+
+        # Determine the correct keyword argument for the body
+        if isinstance(body, dict):
+            kwargs["json"] = body
+        elif isinstance(body, (str, bytes)):
+            kwargs["data"] = body
+
         return requests.request(
             method=method,
             url=url,