mlrun 1.7.2__py3-none-any.whl → 1.8.0rc1__py3-none-any.whl

mlrun/run.py

@@ -30,15 +30,15 @@ from typing import Optional, Union
 
 import nuclio
 import yaml
-from mlrun_pipelines.common.models import RunStatuses
-from mlrun_pipelines.common.ops import format_summary_from_kfp_run, show_kfp_run
-from mlrun_pipelines.utils import get_client
 
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.formatters
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.helpers
+from mlrun_pipelines.common.models import RunStatuses
+from mlrun_pipelines.common.ops import format_summary_from_kfp_run, show_kfp_run
+from mlrun_pipelines.utils import get_client
 
 from .common.helpers import parse_versioned_object_uri
 from .config import config as mlconf
@@ -200,7 +200,7 @@ def get_or_create_ctx(
     event=None,
     spec: Optional[dict] = None,
     with_env: bool = True,
-    rundb: str = "",
+    rundb: Union[str, "mlrun.db.RunDBInterface"] = "",
     project: str = "",
     upload_artifacts: bool = False,
     labels: Optional[dict] = None,
@@ -306,7 +306,7 @@ def get_or_create_ctx(
     out = rundb or mlconf.dbpath or environ.get("MLRUN_DBPATH")
     if out:
         autocommit = True
-        logger.info(f"logging run results to: {out}")
+        logger.info(f"Logging run results to: {out}")
 
     newspec["metadata"]["project"] = (
         newspec["metadata"].get("project") or project or mlconf.default_project
@@ -434,7 +434,7 @@ def new_function(
     mode: Optional[str] = None,
     handler: Optional[str] = None,
     source: Optional[str] = None,
-    requirements: Union[str, list[str]] = None,
+    requirements: Optional[Union[str, list[str]]] = None,
     kfp: Optional[bool] = None,
     requirements_file: str = "",
 ):
@@ -909,10 +909,10 @@ def _run_pipeline(
 def wait_for_pipeline_completion(
     run_id,
     timeout=60 * 60,
-    expected_statuses: list[str] = None,
+    expected_statuses: Optional[list[str]] = None,
     namespace=None,
     remote=True,
-    project: str = None,
+    project: Optional[str] = None,
 ):
     """Wait for Pipeline status, timeout in sec
 
@@ -1004,7 +1004,7 @@ def get_pipeline(
     format_: Union[
         str, mlrun.common.formatters.PipelineFormat
     ] = mlrun.common.formatters.PipelineFormat.summary,
-    project: str = None,
+    project: Optional[str] = None,
     remote: bool = True,
 ):
     """Get Pipeline status

mlrun/runtimes/__init__.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 __all__ = [
     "BaseRuntime",
     "KubejobRuntime",
@@ -60,7 +58,7 @@ from ..serving import MLModelServer, new_v1_model_server  # noqa isort: skip
 def new_model_server(
     name,
     model_class: str,
-    models: dict = None,
+    models: typing.Optional[dict] = None,
     filename="",
     protocol="",
     image="",

mlrun/runtimes/base.py

@@ -21,7 +21,6 @@ from os import environ
 from typing import Callable, Optional, Union
 
 import requests.exceptions
-from mlrun_pipelines.common.ops import mlrun_op
 from nuclio.build import mlrun_footer
 
 import mlrun.common.constants
@@ -35,6 +34,7 @@ import mlrun.utils.helpers
 import mlrun.utils.notifications
 import mlrun.utils.regex
 from mlrun.utils.helpers import generate_object_uri, verify_field_regex
+from mlrun_pipelines.common.ops import mlrun_op
 
 from ..config import config
 from ..datastore import store_manager
@@ -517,7 +517,7 @@ class BaseRuntime(ModelObj):
                 run_results = resp["status"].get("results", {})
                 if generator.eval_stop_condition(run_results):
                     logger.info(
-                        f"reached early stop condition ({generator.options.stop_condition}), stopping iterations!"
+                        f"Reached early stop condition ({generator.options.stop_condition}), stopping iterations!"
                     )
                     results.append(resp)
                     break
@@ -529,7 +529,7 @@ class BaseRuntime(ModelObj):
                 resp = self._update_run_state(task=task, err=error_string)
                 num_errors += 1
                 if num_errors > generator.max_errors:
-                    logger.error("too many errors, stopping iterations!")
+                    logger.error("Too many errors, stopping iterations!")
                     results.append(resp)
                     break
 
@@ -553,9 +553,9 @@ class BaseRuntime(ModelObj):
 
     def _update_run_state(
         self,
-        resp: dict = None,
+        resp: Optional[dict] = None,
         task: RunObject = None,
-        err: Union[Exception, str] = None,
+        err: Optional[Union[Exception, str]] = None,
         run_format: mlrun.common.formatters.RunFormat = mlrun.common.formatters.RunFormat.full,
     ) -> typing.Optional[dict]:
         """update the task state in the DB"""
@@ -644,7 +644,10 @@ class BaseRuntime(ModelObj):
         return updates
 
     def full_image_path(
-        self, image=None, client_version: str = None, client_python_version: str = None
+        self,
+        image=None,
+        client_version: Optional[str] = None,
+        client_python_version: Optional[str] = None,
     ):
         image = image or self.spec.image or ""
 
@@ -669,16 +672,16 @@ class BaseRuntime(ModelObj):
         handler=None,
         name: str = "",
         project: str = "",
-        params: dict = None,
+        params: Optional[dict] = None,
         hyperparams=None,
         selector="",
         hyper_param_options: HyperParamOptions = None,
-        inputs: dict = None,
-        outputs: list = None,
+        inputs: Optional[dict] = None,
+        outputs: Optional[list] = None,
         workdir: str = "",
         artifact_path: str = "",
         image: str = "",
-        labels: dict = None,
+        labels: Optional[dict] = None,
         use_db=True,
         verbose=None,
         scrape_metrics=False,

mlrun/runtimes/daskjob.py

@@ -363,7 +363,7 @@ class DaskCluster(KubejobRuntime):
         skip_deployed=False,
         is_kfp=False,
         mlrun_version_specifier=None,
-        builder_env: dict = None,
+        builder_env: Optional[dict] = None,
         show_on_failure: bool = False,
         force_build: bool = False,
     ):
@@ -406,9 +406,9 @@ class DaskCluster(KubejobRuntime):
 
     def with_scheduler_limits(
         self,
-        mem: str = None,
-        cpu: str = None,
-        gpus: int = None,
+        mem: Optional[str] = None,
+        cpu: Optional[str] = None,
+        gpus: Optional[int] = None,
         gpu_type: str = "nvidia.com/gpu",
         patch: bool = False,
     ):
@@ -422,9 +422,9 @@ class DaskCluster(KubejobRuntime):
 
     def with_worker_limits(
         self,
-        mem: str = None,
-        cpu: str = None,
-        gpus: int = None,
+        mem: Optional[str] = None,
+        cpu: Optional[str] = None,
+        gpus: Optional[int] = None,
         gpu_type: str = "nvidia.com/gpu",
         patch: bool = False,
     ):
@@ -442,7 +442,7 @@ class DaskCluster(KubejobRuntime):
         )
 
     def with_scheduler_requests(
-        self, mem: str = None, cpu: str = None, patch: bool = False
+        self, mem: Optional[str] = None, cpu: Optional[str] = None, patch: bool = False
     ):
         """
         set scheduler pod resources requests
@@ -451,7 +451,7 @@ class DaskCluster(KubejobRuntime):
         self.spec._verify_and_set_requests("scheduler_resources", mem, cpu, patch=patch)
 
     def with_worker_requests(
-        self, mem: str = None, cpu: str = None, patch: bool = False
+        self, mem: Optional[str] = None, cpu: Optional[str] = None, patch: bool = False
     ):
         """
         set worker pod resources requests

mlrun/runtimes/generators.py

@@ -15,6 +15,7 @@ import json
 import random
 import sys
 from copy import deepcopy
+from typing import Optional
 
 import pandas as pd
 
@@ -26,7 +27,7 @@ default_max_iterations = 10
 default_max_errors = 3
 
 
-def get_generator(spec: RunSpec, execution, param_file_secrets: dict = None):
+def get_generator(spec: RunSpec, execution, param_file_secrets: Optional[dict] = None):
     options = spec.hyper_param_options
     strategy = spec.strategy or options.strategy
     if not spec.is_hyper_job() or strategy == "custom":

mlrun/runtimes/kubejob.py

@@ -14,11 +14,10 @@
 import typing
 import warnings
 
-from mlrun_pipelines.common.ops import build_op
-
 import mlrun.common.schemas
 import mlrun.db
 import mlrun.errors
+from mlrun_pipelines.common.ops import build_op
 
 from ..model import RunObject
 from .pod import KubeResource
@@ -76,7 +75,7 @@ class KubejobRuntime(KubeResource):
         self,
         image="",
         base_image=None,
-        commands: list = None,
+        commands: typing.Optional[list] = None,
         secret=None,
         source=None,
         extra=None,
@@ -148,7 +147,7 @@ class KubejobRuntime(KubeResource):
         skip_deployed: bool = False,
         is_kfp: bool = False,
         mlrun_version_specifier: typing.Optional[bool] = None,
-        builder_env: dict = None,
+        builder_env: typing.Optional[dict] = None,
         show_on_failure: bool = False,
         force_build: bool = False,
     ) -> bool:
@@ -190,7 +189,7 @@ class KubejobRuntime(KubeResource):
         self,
         image=None,
         base_image=None,
-        commands: list = None,
+        commands: typing.Optional[list] = None,
         secret_name="",
         with_mlrun=True,
         skip_deployed=False,

mlrun/runtimes/mpijob/__init__.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 from dependency_injector import containers, providers
 
 from mlrun.config import config

mlrun/runtimes/mpijob/abstract.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 import abc
 import os
+from typing import Optional
 
 from mlrun.config import config
 from mlrun.runtimes.kubejob import KubejobRuntime
@@ -117,7 +118,7 @@ class AbstractMPIJobRuntime(KubejobRuntime, abc.ABC):
         return {}
 
     def with_tracing(
-        self, log_file_path: str = None, enable_cycle_markers: bool = False
+        self, log_file_path: Optional[str] = None, enable_cycle_markers: bool = False
     ):
         """Add Horovod Timeline activity tracking to the job to analyse
         its performance.
@@ -149,11 +150,11 @@ class AbstractMPIJobRuntime(KubejobRuntime, abc.ABC):
 
     def with_autotune(
         self,
-        log_file_path: str = None,
-        warmup_samples: int = None,
-        steps_per_sample: int = None,
-        bayes_opt_max_samples: int = None,
-        gaussian_process_noise: float = None,
+        log_file_path: Optional[str] = None,
+        warmup_samples: Optional[int] = None,
+        steps_per_sample: Optional[int] = None,
+        bayes_opt_max_samples: Optional[int] = None,
+        gaussian_process_noise: Optional[float] = None,
     ):
         """Adds an Autotuner to help optimize Horovod's Parameters for better performance.
 

mlrun/runtimes/nuclio/api_gateway.py

@@ -118,10 +118,10 @@ class APIGatewayMetadata(ModelObj):
     def __init__(
         self,
         name: str,
-        namespace: str = None,
-        labels: dict = None,
-        annotations: dict = None,
-        creation_timestamp: str = None,
+        namespace: Optional[str] = None,
+        labels: Optional[dict] = None,
+        annotations: Optional[dict] = None,
+        creation_timestamp: Optional[str] = None,
     ):
         """
         :param name: The name of the API gateway
@@ -169,9 +169,9 @@ class APIGatewaySpec(ModelObj):
             "mlrun.runtimes.nuclio.serving.ServingRuntime",
             "mlrun.runtimes.nuclio.application.ApplicationRuntime",
         ],
-        project: str = None,
+        project: Optional[str] = None,
         description: str = "",
-        host: str = None,
+        host: Optional[str] = None,
         path: str = "/",
         authentication: Optional[APIGatewayAuthenticator] = NoneAuth(),
         canary: Optional[list[int]] = None,
@@ -389,7 +389,7 @@ class APIGateway(ModelObj):
     def invoke(
         self,
         method="POST",
-        headers: dict = None,
+        headers: Optional[dict] = None,
         credentials: Optional[tuple[str, str]] = None,
         path: Optional[str] = None,
         body: Optional[Union[str, bytes, dict]] = None,

mlrun/runtimes/nuclio/application/application.py

@@ -275,7 +275,7 @@ class ApplicationRuntime(RemoteRuntime):
         tag="",
         verbose=False,
         auth_info: schemas.AuthInfo = None,
-        builder_env: dict = None,
+        builder_env: typing.Optional[dict] = None,
         force_build: bool = False,
         with_mlrun=None,
         skip_deployed=False,
@@ -367,7 +367,7 @@ class ApplicationRuntime(RemoteRuntime):
         source,
         workdir=None,
         pull_at_runtime: bool = False,
-        target_dir: str = None,
+        target_dir: typing.Optional[str] = None,
     ):
         """load the code from git/tar/zip archive at build
 
@@ -426,12 +426,12 @@ class ApplicationRuntime(RemoteRuntime):
 
     def create_api_gateway(
         self,
-        name: str = None,
-        path: str = None,
+        name: typing.Optional[str] = None,
+        path: typing.Optional[str] = None,
         direct_port_access: bool = False,
         authentication_mode: schemas.APIGatewayAuthenticationMode = None,
-        authentication_creds: tuple[str, str] = None,
-        ssl_redirect: bool = None,
+        authentication_creds: typing.Optional[tuple[str, str]] = None,
+        ssl_redirect: typing.Optional[bool] = None,
         set_as_default: bool = False,
         gateway_timeout: typing.Optional[int] = None,
     ):
@@ -540,13 +540,13 @@ class ApplicationRuntime(RemoteRuntime):
         self,
         path: str = "",
         body: typing.Optional[typing.Union[str, bytes, dict]] = None,
-        method: str = None,
-        headers: dict = None,
+        method: typing.Optional[str] = None,
+        headers: typing.Optional[dict] = None,
         dashboard: str = "",
         force_external_address: bool = False,
         auth_info: schemas.AuthInfo = None,
-        mock: bool = None,
-        credentials: tuple[str, str] = None,
+        mock: typing.Optional[bool] = None,
+        credentials: typing.Optional[tuple[str, str]] = None,
         **http_client_kwargs,
     ):
         self._sync_api_gateway()
@@ -653,7 +653,7 @@ class ApplicationRuntime(RemoteRuntime):
 
     def _build_application_image(
         self,
-        builder_env: dict = None,
+        builder_env: typing.Optional[dict] = None,
         force_build: bool = False,
         watch=True,
         with_mlrun=None,

mlrun/runtimes/nuclio/function.py

@@ -26,9 +26,6 @@ import requests
 import semver
 from aiohttp.client import ClientSession
 from kubernetes import client
-from mlrun_pipelines.common.mounts import VolumeMount
-from mlrun_pipelines.common.ops import deploy_op
-from mlrun_pipelines.mounts import mount_v3io, v3io_cred
 from nuclio.deploy import find_dashboard_url, get_deploy_status
 from nuclio.triggers import V3IOStreamTrigger
 
@@ -50,6 +47,9 @@ from mlrun.runtimes.base import FunctionStatus, RunError
 from mlrun.runtimes.pod import KubeResource, KubeResourceSpec
 from mlrun.runtimes.utils import get_item_name, log_std
 from mlrun.utils import get_in, logger, update_in
+from mlrun_pipelines.common.mounts import VolumeMount
+from mlrun_pipelines.common.ops import deploy_op
+from mlrun_pipelines.mounts import mount_v3io, v3io_cred
 
 
 def validate_nuclio_version_compatibility(*min_versions):
@@ -554,7 +554,7 @@ class RemoteRuntime(KubeResource):
         tag="",
         verbose=False,
         auth_info: AuthInfo = None,
-        builder_env: dict = None,
+        builder_env: typing.Optional[dict] = None,
         force_build: bool = False,
     ):
         """Deploy the nuclio function to the cluster
@@ -693,7 +693,9 @@ class RemoteRuntime(KubeResource):
         super().with_priority_class(name)
 
     def with_service_type(
-        self, service_type: str, add_templated_ingress_host_mode: str = None
+        self,
+        service_type: str,
+        add_templated_ingress_host_mode: typing.Optional[str] = None,
     ):
         """
         Enables to control the service type of the pod and the addition of templated ingress host
@@ -887,13 +889,13 @@ class RemoteRuntime(KubeResource):
     def invoke(
         self,
         path: str,
-        body: typing.Union[str, bytes, dict] = None,
-        method: str = None,
-        headers: dict = None,
+        body: typing.Optional[typing.Union[str, bytes, dict]] = None,
+        method: typing.Optional[str] = None,
+        headers: typing.Optional[dict] = None,
         dashboard: str = "",
         force_external_address: bool = False,
         auth_info: AuthInfo = None,
-        mock: bool = None,
+        mock: typing.Optional[bool] = None,
         **http_client_kwargs,
     ):
         """Invoke the remote (live) function and return the results
@@ -995,15 +997,14 @@ class RemoteRuntime(KubeResource):
 
     def with_sidecar(
         self,
-        name: str = None,
-        image: str = None,
+        name: typing.Optional[str] = None,
+        image: typing.Optional[str] = None,
         ports: typing.Optional[typing.Union[int, list[int]]] = None,
         command: typing.Optional[str] = None,
         args: typing.Optional[list[str]] = None,
     ):
         """
         Add a sidecar container to the function pod
-
         :param name:    Sidecar container name.
         :param image:   Sidecar container image.
         :param ports:   Sidecar container ports to expose. Can be a single port or a list of ports.
@@ -1036,10 +1037,9 @@ class RemoteRuntime(KubeResource):
         if args and sidecar.get("command"):
             sidecar["args"] = mlrun.utils.helpers.as_list(args)
 
-        # put the configured resources on the sidecar container instead of the reverse proxy container
+        # populate the sidecar resources from the function spec
         if self.spec.resources:
             sidecar["resources"] = self.spec.resources
-            self.spec.resources = None
 
     def _set_sidecar(self, name: str) -> dict:
         self.spec.config.setdefault("spec.sidecars", [])

mlrun/runtimes/nuclio/serving.py

@@ -51,7 +51,7 @@ if TYPE_CHECKING:
 def new_v2_model_server(
     name,
     model_class: str,
-    models: dict = None,
+    models: Optional[dict] = None,
     filename="",
     protocol="",
     image="",
@@ -356,12 +356,12 @@ class ServingRuntime(RemoteRuntime):
     def add_model(
         self,
         key: str,
-        model_path: str = None,
-        class_name: str = None,
-        model_url: str = None,
-        handler: str = None,
-        router_step: str = None,
-        child_function: str = None,
+        model_path: Optional[str] = None,
+        class_name: Optional[str] = None,
+        model_url: Optional[str] = None,
+        handler: Optional[str] = None,
+        router_step: Optional[str] = None,
+        child_function: Optional[str] = None,
         **class_args,
     ):
         """add ml model and/or route to the function.
@@ -509,7 +509,7 @@ class ServingRuntime(RemoteRuntime):
                         stream.path, group=group, shards=stream.shards, **trigger_args
                     )
 
-    def _deploy_function_refs(self, builder_env: dict = None):
+    def _deploy_function_refs(self, builder_env: Optional[dict] = None):
         """set metadata and deploy child functions"""
         for function_ref in self._spec.function_refs.values():
             logger.info(f"deploy child function {function_ref.name} ...")
@@ -583,7 +583,7 @@ class ServingRuntime(RemoteRuntime):
         tag="",
         verbose=False,
         auth_info: mlrun.common.schemas.AuthInfo = None,
-        builder_env: dict = None,
+        builder_env: Optional[dict] = None,
         force_build: bool = False,
     ):
         """deploy model serving function to a local/remote cluster