mlrun 1.7.0rc28__py3-none-any.whl → 1.7.0rc55__py3-none-any.whl

mlrun/render.py

@@ -22,7 +22,7 @@ import mlrun.utils
 
 from .config import config
 from .datastore import uri_to_ipython
-from .utils import dict_to_list, get_in, is_ipython
+from .utils import dict_to_list, get_in, is_jupyter
 
 JUPYTER_SERVER_ROOT = environ.get("HOME", "/User")
 supported_viewers = [
@@ -181,8 +181,8 @@ def run_to_html(results, display=True):
 
 
 def ipython_display(html, display=True, alt_text=None):
-    if display and html and is_ipython:
-        import IPython
+    if display and html and is_jupyter:
+        import IPython.display
 
         IPython.display.display(IPython.display.HTML(html))
     elif alt_text:

mlrun/run.py

@@ -21,6 +21,7 @@ import tempfile
 import time
 import typing
 import uuid
+import warnings
 from base64 import b64decode
 from copy import deepcopy
 from os import environ, makedirs, path
@@ -64,6 +65,7 @@ from .runtimes.nuclio.application import ApplicationRuntime
 from .runtimes.utils import add_code_metadata, global_context
 from .utils import (
     RunKeys,
+    create_ipython_display,
     extend_hub_uri_if_needed,
     get_in,
     logger,
@@ -196,18 +198,19 @@ def load_func_code(command="", workdir=None, secrets=None, name="name"):
 def get_or_create_ctx(
     name: str,
     event=None,
-    spec=None,
+    spec: Optional[dict] = None,
     with_env: bool = True,
     rundb: str = "",
     project: str = "",
-    upload_artifacts=False,
-    labels: dict = None,
-):
-    """called from within the user program to obtain a run context
+    upload_artifacts: bool = False,
+    labels: Optional[dict] = None,
+) -> MLClientCtx:
+    """
+    Called from within the user program to obtain a run context.
 
-    the run context is an interface for receiving parameters, data and logging
+    The run context is an interface for receiving parameters, data and logging
     run results, the run context is read from the event, spec, or environment
-    (in that order), user can also work without a context (local defaults mode)
+    (in that order), user can also work without a context (local defaults mode).
 
     all results are automatically stored in the "rundb" or artifact store,
     the path to the rundb can be specified in the call or obtained from env.
@@ -217,10 +220,10 @@ def get_or_create_ctx(
     :param spec:     dictionary holding run spec
     :param with_env: look for context in environment vars, default True
     :param rundb:    path/url to the metadata and artifact database
-    :param project:  project to initiate the context in (by default mlrun.mlctx.default_project)
+    :param project:  project to initiate the context in (by default `mlrun.mlconf.default_project`)
     :param upload_artifacts:  when using local context (not as part of a job/run), upload artifacts to the
                               system default artifact path location
-    :param labels:      dict of the context labels
+    :param labels: (deprecated - use spec instead) dict of the context labels.
     :return: execution context
 
     Examples::
@@ -253,6 +256,20 @@ def get_or_create_ctx(
         context.log_artifact("results.html", body=b"<b> Some HTML <b>", viewer="web-app")
 
     """
+    if labels:
+        warnings.warn(
+            "The `labels` argument is deprecated and will be removed in 1.9.0. "
+            "Please use `spec` instead, e.g.:\n"
+            "spec={'metadata': {'labels': {'key': 'value'}}}",
+            FutureWarning,
+        )
+        if spec is None:
+            spec = {}
+        if "metadata" not in spec:
+            spec["metadata"] = {}
+        if "labels" not in spec["metadata"]:
+            spec["metadata"]["labels"] = {}
+        spec["metadata"]["labels"].update(labels)
 
     if global_context.get() and not spec and not event:
         return global_context.get()
@@ -306,9 +323,6 @@ def get_or_create_ctx(
     ctx = MLClientCtx.from_dict(
         newspec, rundb=out, autocommit=autocommit, tmp=tmp, host=socket.gethostname()
     )
-    labels = labels or {}
-    for key, val in labels.items():
-        ctx.set_label(key=key, value=val)
     global_context.set(ctx)
     return ctx
 
@@ -639,7 +653,7 @@ def code_to_function(
     :param requirements: a list of python packages
     :param requirements_file: path to a python requirements file
     :param categories:   list of categories for mlrun Function Hub, defaults to None
-    :param labels:       immutable name/value pairs to tag the function with useful metadata, defaults to None
+    :param labels:       name/value pairs dict to tag the function with useful metadata, defaults to None
     :param with_doc:     indicates whether to document the function parameters, defaults to True
     :param ignored_tags: notebook cells to ignore when converting notebooks to py code (separated by ';')
 
@@ -731,11 +745,10 @@ def code_to_function(
         raise ValueError("Databricks tasks only support embed_code=True")
 
     if kind == RuntimeKinds.application:
-        if handler:
-            raise MLRunInvalidArgumentError(
-                "Handler is not supported for application runtime"
-            )
-        filename, handler = ApplicationRuntime.get_filename_and_handler()
+        raise MLRunInvalidArgumentError(
+            "Embedding a code file is not supported for application runtime. "
+            "Code files should be specified via project/function source."
+        )
 
     is_nuclio, sub_kind = RuntimeKinds.resolve_nuclio_sub_kind(kind)
     code_origin = add_name(add_code_metadata(filename), name)
@@ -778,6 +791,10 @@ def code_to_function(
             raise ValueError("code_output option is only used with notebooks")
 
     if is_nuclio:
+        mlrun.utils.helpers.validate_single_def_handler(
+            function_kind=sub_kind, code=code
+        )
+
         runtime = RuntimeKinds.resolve_nuclio_runtime(kind, sub_kind)
         # default_handler is only used in :mlrun sub kind, determine the handler to invoke in function.run()
         runtime.spec.default_handler = handler if sub_kind == "mlrun" else ""
@@ -925,10 +942,12 @@ def wait_for_pipeline_completion(
     if remote:
         mldb = mlrun.db.get_run_db()
 
+        dag_display_id = create_ipython_display()
+
         def _wait_for_pipeline_completion():
             pipeline = mldb.get_pipeline(run_id, namespace=namespace, project=project)
             pipeline_status = pipeline["run"]["status"]
-            show_kfp_run(pipeline, clear_output=True)
+            show_kfp_run(pipeline, dag_display_id=dag_display_id, with_html=False)
             if pipeline_status not in RunStatuses.stable_statuses():
                 logger.debug(
                     "Waiting for pipeline completion",

mlrun/runtimes/__init__.py

@@ -30,6 +30,8 @@ __all__ = [
     "MpiRuntimeV1",
 ]
 
+import typing
+
 from mlrun.runtimes.utils import resolve_spark_operator_version
 
 from ..common.runtimes.constants import MPIJobCRDVersions
@@ -181,7 +183,7 @@ class RuntimeKinds:
         ]
 
     @staticmethod
-    def is_log_collectable_runtime(kind: str):
+    def is_log_collectable_runtime(kind: typing.Optional[str]):
         """
         whether log collector can collect logs for that runtime
         :param kind: kind name
@@ -192,13 +194,18 @@ class RuntimeKinds:
         if RuntimeKinds.is_local_runtime(kind):
             return False
 
-        if kind not in [
-            # dask implementation is different than other runtimes, because few runs can be run against the same runtime
-            # resource, so collecting logs on that runtime resource won't be correct, the way we collect logs for dask
-            # is by using `log_std` on client side after we execute the code against the cluster, as submitting the
-            # run with the dask client will return the run stdout. for more information head to `DaskCluster._run`
-            RuntimeKinds.dask
-        ]:
+        if (
+            kind
+            not in [
+                # dask implementation is different from other runtimes, because few runs can be run against the same
+                # runtime resource, so collecting logs on that runtime resource won't be correct, the way we collect
+                # logs for dask is by using `log_std` on client side after we execute the code against the cluster,
+                # as submitting the run with the dask client will return the run stdout.
+                # For more information head to `DaskCluster._run`.
+                RuntimeKinds.dask
+            ]
+            + RuntimeKinds.nuclio_runtimes()
+        ):
             return True
 
         return False
@@ -235,6 +242,10 @@ class RuntimeKinds:
         # both spark and remote spark uses different mechanism for assigning images
         return kind not in [RuntimeKinds.spark, RuntimeKinds.remotespark]
 
+    @staticmethod
+    def supports_from_notebook(kind):
+        return kind not in [RuntimeKinds.application]
+
     @staticmethod
     def resolve_nuclio_runtime(kind: str, sub_kind: str):
         kind = kind.split(":")[0]

mlrun/runtimes/base.py

@@ -674,7 +674,7 @@ class BaseRuntime(ModelObj):
         selector="",
         hyper_param_options: HyperParamOptions = None,
         inputs: dict = None,
-        outputs: dict = None,
+        outputs: list = None,
         workdir: str = "",
         artifact_path: str = "",
         image: str = "",
@@ -929,3 +929,6 @@ class BaseRuntime(ModelObj):
                         if "default" in p:
                             line += f", default={p['default']}"
                         print("    " + line)
+
+    def skip_image_enrichment(self):
+        return False

mlrun/runtimes/daskjob.py

@@ -379,7 +379,7 @@ class DaskCluster(KubejobRuntime):
         :param show_on_failure:         show logs only in case of build failure
         :param force_build:             force building the image, even when no changes were made
 
-        :return True if the function is ready (deployed)
+        :return:                        True if the function is ready (deployed)
         """
         return super().deploy(
             watch,

mlrun/runtimes/funcdoc.py

@@ -247,7 +247,7 @@ class ASTVisitor(ast.NodeVisitor):
         self.exprs.append(node)
         super().generic_visit(node)
 
-    def visit_FunctionDef(self, node):
+    def visit_FunctionDef(self, node):  # noqa: N802
         self.funcs.append(node)
         self.generic_visit(node)
 

mlrun/runtimes/kubejob.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import typing
 import warnings
 
 from mlrun_pipelines.common.ops import build_op
@@ -143,11 +143,11 @@ class KubejobRuntime(KubeResource):
 
     def deploy(
         self,
-        watch=True,
-        with_mlrun=None,
-        skip_deployed=False,
-        is_kfp=False,
-        mlrun_version_specifier=None,
+        watch: bool = True,
+        with_mlrun: typing.Optional[bool] = None,
+        skip_deployed: bool = False,
+        is_kfp: bool = False,
+        mlrun_version_specifier: typing.Optional[bool] = None,
         builder_env: dict = None,
         show_on_failure: bool = False,
         force_build: bool = False,

mlrun/runtimes/local.py

@@ -58,7 +58,9 @@ class ParallelRunner:
 
         return TrackerManager()
 
-    def _get_handler(self, handler, context):
+    def _get_handler(
+        self, handler: str, context: MLClientCtx, embed_in_sys: bool = True
+    ):
         return handler
 
     def _get_dask_client(self, options):
@@ -86,7 +88,7 @@ class ParallelRunner:
         handler = runobj.spec.handler
         self._force_handler(handler)
         set_paths(self.spec.pythonpath)
-        handler = self._get_handler(handler, execution)
+        handler = self._get_handler(handler, execution, embed_in_sys=False)
 
         client, function_name = self._get_dask_client(generator.options)
         parallel_runs = generator.options.parallel_runs or 4
@@ -143,7 +145,10 @@ class ParallelRunner:
         if function_name and generator.options.teardown_dask:
             logger.info("Tearing down the dask cluster..")
             mlrun.get_run_db().delete_runtime_resources(
-                kind="dask", object_id=function_name, force=True
+                project=self.metadata.project,
+                kind=mlrun.runtimes.RuntimeKinds.dask,
+                object_id=function_name,
+                force=True,
             )
 
         return results
@@ -224,12 +229,14 @@ class LocalRuntime(BaseRuntime, ParallelRunner):
     def is_deployed(self):
         return True
 
-    def _get_handler(self, handler, context):
+    def _get_handler(
+        self, handler: str, context: MLClientCtx, embed_in_sys: bool = True
+    ):
         command = self.spec.command
         if not command and self.spec.build.functionSourceCode:
             # if the code is embedded in the function object extract or find it
             command, _ = mlrun.run.load_func_code(self)
-        return load_module(command, handler, context)
+        return load_module(command, handler, context, embed_in_sys=embed_in_sys)
 
     def _pre_run(self, runobj: RunObject, execution: MLClientCtx):
         workdir = self.spec.workdir

mlrun/runtimes/nuclio/api_gateway.py

@@ -22,6 +22,7 @@ from nuclio.auth import AuthKinds as NuclioAuthKinds
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
+import mlrun.common.helpers
 import mlrun.common.schemas as schemas
 import mlrun.common.types
 from mlrun.model import ModelObj
@@ -202,8 +203,13 @@ class APIGatewaySpec(ModelObj):
         self.project = project
         self.ports = ports
 
+        self.enrich()
         self.validate(project=project, functions=functions, canary=canary, ports=ports)
 
+    def enrich(self):
+        if self.path and not self.path.startswith("/"):
+            self.path = f"/{self.path}"
+
     def validate(
         self,
         project: str,
@@ -326,6 +332,11 @@ class APIGatewaySpec(ModelObj):
         return function_names
 
 
+class APIGatewayStatus(ModelObj):
+    def __init__(self, state: Optional[schemas.APIGatewayState] = None):
+        self.state = state or schemas.APIGatewayState.none
+
+
 class APIGateway(ModelObj):
     _dict_fields = [
         "metadata",
@@ -338,16 +349,18 @@ class APIGateway(ModelObj):
         self,
         metadata: APIGatewayMetadata,
         spec: APIGatewaySpec,
+        status: Optional[APIGatewayStatus] = None,
     ):
         """
         Initialize the APIGateway instance.
 
         :param metadata: (APIGatewayMetadata) The metadata of the API gateway.
         :param spec: (APIGatewaySpec) The spec of the API gateway.
+        :param status: (APIGatewayStatus) The status of the API gateway.
         """
         self.metadata = metadata
         self.spec = spec
-        self.state = ""
+        self.status = status
 
     @property
     def metadata(self) -> APIGatewayMetadata:
@@ -365,12 +378,21 @@ class APIGateway(ModelObj):
     def spec(self, spec):
         self._spec = self._verify_dict(spec, "spec", APIGatewaySpec)
 
+    @property
+    def status(self) -> APIGatewayStatus:
+        return self._status
+
+    @status.setter
+    def status(self, status):
+        self._status = self._verify_dict(status, "status", APIGatewayStatus)
+
     def invoke(
         self,
         method="POST",
         headers: dict = None,
         credentials: Optional[tuple[str, str]] = None,
         path: Optional[str] = None,
+        body: Optional[Union[str, bytes, dict]] = None,
         **kwargs,
     ):
         """
@@ -381,6 +403,7 @@ class APIGateway(ModelObj):
         :param credentials: (Optional[tuple[str, str]], optional) The (username,password) for the invocation if required
             can also be set by the environment variable (_, V3IO_ACCESS_KEY) for access key authentication.
         :param path: (str, optional) The sub-path for the invocation.
+        :param body: (Optional[Union[str, bytes, dict]]) The body of the invocation.
         :param kwargs: (dict) Additional keyword arguments.
 
         :return: The response from the API gateway invocation.
@@ -394,7 +417,7 @@ class APIGateway(ModelObj):
                 )
         if not self.is_ready():
             raise mlrun.errors.MLRunPreconditionFailedError(
-                f"API gateway is not ready. " f"Current state: {self.state}"
+                f"API gateway is not ready. " f"Current state: {self.status.state}"
             )
 
         auth = None
@@ -429,6 +452,13 @@ class APIGateway(ModelObj):
                     "API Gateway invocation requires authentication. Please set V3IO_ACCESS_KEY env var"
                 )
         url = urljoin(self.invoke_url, path or "")
+
+        # Determine the correct keyword argument for the body
+        if isinstance(body, dict):
+            kwargs["json"] = body
+        elif isinstance(body, (str, bytes)):
+            kwargs["data"] = body
+
         return requests.request(
             method=method,
             url=url,
@@ -459,10 +489,10 @@ class APIGateway(ModelObj):
         )
 
     def is_ready(self):
-        if self.state is not schemas.api_gateway.APIGatewayState.ready:
+        if self.status.state is not schemas.api_gateway.APIGatewayState.ready:
             # try to sync the state
             self.sync()
-        return self.state == schemas.api_gateway.APIGatewayState.ready
+        return self.status.state == schemas.api_gateway.APIGatewayState.ready
 
     def sync(self):
         """
@@ -479,7 +509,7 @@ class APIGateway(ModelObj):
         self.spec.functions = synced_gateway.spec.functions
         self.spec.canary = synced_gateway.spec.canary
         self.spec.description = synced_gateway.spec.description
-        self.state = synced_gateway.state
+        self.status.state = synced_gateway.status.state
 
     def with_basic_auth(self, username: str, password: str):
         """
@@ -546,6 +576,29 @@ class APIGateway(ModelObj):
             project=self.spec.project, functions=self.spec.functions, ports=ports
         )
 
+    def with_force_ssl_redirect(self):
+        """
+        Set SSL redirect annotation for the API gateway.
+        """
+        self.metadata.annotations["nginx.ingress.kubernetes.io/force-ssl-redirect"] = (
+            "true"
+        )
+
+    def with_gateway_timeout(self, gateway_timeout: int):
+        """
+        Set gateway proxy connect/read/send timeout annotations
+        :param gateway_timeout: The timeout in seconds
+        """
+        mlrun.runtimes.utils.enrich_gateway_timeout_annotations(
+            self.metadata.annotations, gateway_timeout
+        )
+
+    def with_annotations(self, annotations: dict):
+        """set a key/value annotations in the metadata of the api gateway"""
+        for key, value in annotations.items():
+            self.metadata.annotations[key] = str(value)
+        return self
+
     @classmethod
     def from_scheme(cls, api_gateway: schemas.APIGateway):
         project = api_gateway.metadata.labels.get(
@@ -560,6 +613,8 @@ class APIGateway(ModelObj):
         new_api_gateway = cls(
             metadata=APIGatewayMetadata(
                 name=api_gateway.spec.name,
+                annotations=api_gateway.metadata.annotations,
+                labels=api_gateway.metadata.labels,
             ),
             spec=APIGatewaySpec(
                 project=project,
@@ -570,8 +625,8 @@ class APIGateway(ModelObj):
                 functions=functions,
                 canary=canary,
             ),
+            status=APIGatewayStatus(state=state),
         )
-        new_api_gateway.state = state
         return new_api_gateway
 
     def to_scheme(self) -> schemas.APIGateway:
@@ -600,7 +655,11 @@ class APIGateway(ModelObj):
                 upstreams[i].port = port
 
         api_gateway = schemas.APIGateway(
-            metadata=schemas.APIGatewayMetadata(name=self.metadata.name, labels={}),
+            metadata=schemas.APIGatewayMetadata(
+                name=self.metadata.name,
+                labels=self.metadata.labels,
+                annotations=self.metadata.annotations,
+            ),
             spec=schemas.APIGatewaySpec(
                 name=self.metadata.name,
                 description=self.spec.description,
@@ -611,6 +670,7 @@ class APIGateway(ModelObj):
                 ),
                 upstreams=upstreams,
             ),
+            status=schemas.APIGatewayStatus(state=self.status.state),
         )
         api_gateway.spec.authentication = self.spec.authentication.to_scheme()
         return api_gateway
@@ -627,7 +687,7 @@ class APIGateway(ModelObj):
         host = self.spec.host
         if not self.spec.host.startswith("http"):
             host = f"https://{self.spec.host}"
-        return urljoin(host, self.spec.path)
+        return urljoin(host, self.spec.path).rstrip("/")
 
     @staticmethod
     def _generate_basic_auth(username: str, password: str):