mlrun 1.7.2rc3__py3-none-any.whl → 1.8.0rc1__py3-none-any.whl

mlrun/runtimes/pod.py

@@ -21,16 +21,17 @@ from enum import Enum
 
 import dotenv
 import kubernetes.client as k8s_client
-import mlrun_pipelines.mounts
-from mlrun_pipelines.mixins import KfpAdapterMixin
 
+import mlrun.common.constants
 import mlrun.errors
 import mlrun.utils.regex
+import mlrun_pipelines.mounts
 from mlrun.common.schemas import (
     NodeSelectorOperator,
     PreemptionModes,
     SecurityContextEnrichmentModes,
 )
+from mlrun_pipelines.mixins import KfpAdapterMixin
 
 from ..config import config as mlconf
 from ..k8s_utils import (
@@ -309,7 +310,7 @@ class KubeResourceSpec(FunctionSpec):
         return self._termination_grace_period_seconds
 
     def _serialize_field(
-        self, struct: dict, field_name: str = None, strip: bool = False
+        self, struct: dict, field_name: typing.Optional[str] = None, strip: bool = False
     ) -> typing.Any:
         """
         Serialize a field to a dict, list, or primitive type.
@@ -321,7 +322,7 @@ class KubeResourceSpec(FunctionSpec):
         return super()._serialize_field(struct, field_name, strip)
 
     def _enrich_field(
-        self, struct: dict, field_name: str = None, strip: bool = False
+        self, struct: dict, field_name: typing.Optional[str] = None, strip: bool = False
     ) -> typing.Any:
         k8s_api = k8s_client.ApiClient()
         if strip:
@@ -380,9 +381,9 @@ class KubeResourceSpec(FunctionSpec):
     def _verify_and_set_limits(
         self,
         resources_field_name,
-        mem: str = None,
-        cpu: str = None,
-        gpus: int = None,
+        mem: typing.Optional[str] = None,
+        cpu: typing.Optional[str] = None,
+        gpus: typing.Optional[int] = None,
         gpu_type: str = "nvidia.com/gpu",
         patch: bool = False,
     ):
@@ -430,8 +431,8 @@ class KubeResourceSpec(FunctionSpec):
     def _verify_and_set_requests(
         self,
         resources_field_name,
-        mem: str = None,
-        cpu: str = None,
+        mem: typing.Optional[str] = None,
+        cpu: typing.Optional[str] = None,
         patch: bool = False,
     ):
         resources = verify_requests(resources_field_name, mem=mem, cpu=cpu)
@@ -456,9 +457,9 @@ class KubeResourceSpec(FunctionSpec):
 
     def with_limits(
         self,
-        mem: str = None,
-        cpu: str = None,
-        gpus: int = None,
+        mem: typing.Optional[str] = None,
+        cpu: typing.Optional[str] = None,
+        gpus: typing.Optional[int] = None,
         gpu_type: str = "nvidia.com/gpu",
         patch: bool = False,
     ):
@@ -474,7 +475,12 @@ class KubeResourceSpec(FunctionSpec):
         """
         self._verify_and_set_limits("resources", mem, cpu, gpus, gpu_type, patch=patch)
 
-    def with_requests(self, mem: str = None, cpu: str = None, patch: bool = False):
+    def with_requests(
+        self,
+        mem: typing.Optional[str] = None,
+        cpu: typing.Optional[str] = None,
+        patch: bool = False,
+    ):
         """
         Set requested (desired) pod cpu/memory resources
 
@@ -1050,7 +1056,11 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
         self.spec.env.append(new_var)
         return self
 
-    def set_envs(self, env_vars: dict = None, file_path: str = None):
+    def set_envs(
+        self,
+        env_vars: typing.Optional[dict] = None,
+        file_path: typing.Optional[str] = None,
+    ):
         """set pod environment var from key/value dict or .env file
 
         :param env_vars:  dict with env key/values
@@ -1074,7 +1084,9 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
         return self
 
     def set_image_pull_configuration(
-        self, image_pull_policy: str = None, image_pull_secret_name: str = None
+        self,
+        image_pull_policy: typing.Optional[str] = None,
+        image_pull_secret_name: typing.Optional[str] = None,
     ):
         """
         Configure the image pull parameters for the runtime.
@@ -1123,9 +1135,9 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
 
     def with_limits(
         self,
-        mem: str = None,
-        cpu: str = None,
-        gpus: int = None,
+        mem: typing.Optional[str] = None,
+        cpu: typing.Optional[str] = None,
+        gpus: typing.Optional[int] = None,
         gpu_type: str = "nvidia.com/gpu",
         patch: bool = False,
     ):
@@ -1141,7 +1153,12 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
         """
         self.spec.with_limits(mem, cpu, gpus, gpu_type, patch=patch)
 
-    def with_requests(self, mem: str = None, cpu: str = None, patch: bool = False):
+    def with_requests(
+        self,
+        mem: typing.Optional[str] = None,
+        cpu: typing.Optional[str] = None,
+        patch: bool = False,
+    ):
         """
         Set requested (desired) pod cpu/memory resources
 
@@ -1412,20 +1429,32 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
     ):
         db = self._get_db()
         offset = 0
+        events_offset = 0
         try:
-            text, _ = db.get_builder_status(self, 0, logs=logs)
+            text, _, deploy_status_text_kind = db.get_builder_status(
+                self,
+                offset=0,
+                logs=logs,
+                events_offset=0,
+            )
         except mlrun.db.RunDBError:
-            raise ValueError("function or build process not found")
+            raise ValueError("Function or build process not found")
 
-        def print_log(text):
-            if text and (
+        def print_log(_text):
+            if _text and (
                 not show_on_failure
                 or self.status.state == mlrun.common.schemas.FunctionState.error
             ):
-                print(text, end="")
+                print(_text, end="")
 
         print_log(text)
-        offset += len(text)
+        if (
+            deploy_status_text_kind
+            == mlrun.common.constants.DeployStatusTextKind.events
+        ):
+            events_offset += len(text)
+        else:
+            offset += len(text)
         if watch:
             while self.status.state in [
                 mlrun.common.schemas.FunctionState.pending,
@@ -1434,14 +1463,30 @@ class KubeResource(BaseRuntime, KfpAdapterMixin):
                 time.sleep(2)
                 if show_on_failure:
                     text = ""
-                    db.get_builder_status(self, 0, logs=False)
+                    db.get_builder_status(self, offset=0, logs=False, events_offset=0)
                     if self.status.state == mlrun.common.schemas.FunctionState.error:
                         # re-read the full log on failure
-                        text, _ = db.get_builder_status(self, offset, logs=logs)
+                        text, _, deploy_status_text_kind = db.get_builder_status(
+                            self,
+                            offset=offset,
+                            logs=logs,
+                            events_offset=events_offset,
+                        )
                 else:
-                    text, _ = db.get_builder_status(self, offset, logs=logs)
+                    text, _, deploy_status_text_kind = db.get_builder_status(
+                        self,
+                        offset=offset,
+                        logs=logs,
+                        events_offset=events_offset,
+                    )
                 print_log(text)
-                offset += len(text)
+                if (
+                    deploy_status_text_kind
+                    == mlrun.common.constants.DeployStatusTextKind.events
+                ):
+                    events_offset += len(text)
+                else:
+                    offset += len(text)
 
         return self.status.state
 

mlrun/runtimes/remotesparkjob.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 import re
 from subprocess import run
+from typing import Optional
 
 import kubernetes.client
-from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
 
 import mlrun.errors
 from mlrun.config import config
+from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
 
 from .kubejob import KubejobRuntime
 from .pod import KubeResourceSpec
@@ -179,7 +180,7 @@ class RemoteSparkRuntime(KubejobRuntime):
         skip_deployed=False,
         is_kfp=False,
         mlrun_version_specifier=None,
-        builder_env: dict = None,
+        builder_env: Optional[dict] = None,
         show_on_failure: bool = False,
         force_build: bool = False,
     ):

mlrun/runtimes/sparkjob/__init__.py

@@ -12,6 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-
 from .spark3job import Spark3Runtime

mlrun/runtimes/sparkjob/spark3job.py

@@ -14,13 +14,13 @@
 import typing
 
 import kubernetes.client
-from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
 
 import mlrun.common.schemas.function
 import mlrun.errors
 import mlrun.k8s_utils
 import mlrun.runtimes.pod
 from mlrun.config import config
+from mlrun_pipelines.mounts import mount_v3io, mount_v3iod
 
 from ...execution import MLClientCtx
 from ...model import RunObject
@@ -405,8 +405,8 @@ class Spark3JobSpec(KubeResourceSpec):
     def _verify_and_set_requests(
         self,
         resources_field_name,
-        mem: str = None,
-        cpu: str = None,
+        mem: typing.Optional[str] = None,
+        cpu: typing.Optional[str] = None,
         patch: bool = False,
     ):
         # Spark operator uses JVM notation for memory, so we must verify it separately
@@ -774,7 +774,11 @@ class Spark3Runtime(KubejobRuntime):
                 exporter_jar="/spark/jars/jmx_prometheus_javaagent-0.16.1.jar",
             )
 
-    def with_cores(self, executor_cores: int = None, driver_cores: int = None):
+    def with_cores(
+        self,
+        executor_cores: typing.Optional[int] = None,
+        driver_cores: typing.Optional[int] = None,
+    ):
         """
         Allows to configure spark.executor.cores and spark.driver.cores parameters. The values must be integers
         greater than or equal to 1. If a parameter is not specified, it defaults to 1.
@@ -853,7 +857,7 @@ class Spark3Runtime(KubejobRuntime):
         skip_deployed=False,
         is_kfp=False,
         mlrun_version_specifier=None,
-        builder_env: dict = None,
+        builder_env: typing.Optional[dict] = None,
         show_on_failure: bool = False,
         force_build: bool = False,
     ):
@@ -944,7 +948,10 @@ class Spark3Runtime(KubejobRuntime):
         )
 
     def with_executor_requests(
-        self, mem: str = None, cpu: str = None, patch: bool = False
+        self,
+        mem: typing.Optional[str] = None,
+        cpu: typing.Optional[str] = None,
+        patch: bool = False,
     ):
         """
         set executor pod required cpu/memory/gpu resources
@@ -954,8 +961,8 @@ class Spark3Runtime(KubejobRuntime):
 
     def with_executor_limits(
         self,
-        cpu: str = None,
-        gpus: int = None,
+        cpu: typing.Optional[str] = None,
+        gpus: typing.Optional[int] = None,
         gpu_type: str = "nvidia.com/gpu",
         patch: bool = False,
     ):
@@ -970,7 +977,10 @@ class Spark3Runtime(KubejobRuntime):
         )
 
     def with_driver_requests(
-        self, mem: str = None, cpu: str = None, patch: bool = False
+        self,
+        mem: typing.Optional[str] = None,
+        cpu: typing.Optional[str] = None,
+        patch: bool = False,
     ):
         """
         set driver pod required cpu/memory/gpu resources
@@ -980,8 +990,8 @@ class Spark3Runtime(KubejobRuntime):
 
     def with_driver_limits(
         self,
-        cpu: str = None,
-        gpus: int = None,
+        cpu: typing.Optional[str] = None,
+        gpus: typing.Optional[int] = None,
         gpu_type: str = "nvidia.com/gpu",
         patch: bool = False,
     ):

mlrun/runtimes/utils.py

@@ -18,6 +18,7 @@ import os
 import re
 from io import StringIO
 from sys import stderr
+from typing import Optional
 
 import pandas as pd
 
@@ -147,7 +148,7 @@ def add_code_metadata(path=""):
 
 def results_to_iter(results, runspec, execution):
     if not results:
-        logger.error("got an empty results list in to_iter")
+        logger.error("Got an empty results list in to_iter")
         return
 
     iter = []
@@ -174,7 +175,7 @@ def results_to_iter(results, runspec, execution):
 
     if not iter:
         execution.set_state("completed", commit=True)
-        logger.warning("warning!, zero iteration results")
+        logger.warning("Warning!, zero iteration results")
         return
     if hasattr(pd, "json_normalize"):
         df = pd.json_normalize(iter).sort_values("iter")
@@ -189,10 +190,10 @@ def results_to_iter(results, runspec, execution):
     item, id = selector(results, criteria)
     if runspec.spec.selector and not id:
         logger.warning(
-            f"no best result selected, check selector ({criteria}) or results"
+            f"No best result selected, check selector ({criteria}) or results"
         )
     if id:
-        logger.info(f"best iteration={id}, used criteria {criteria}")
+        logger.info(f"Best iteration={id}, used criteria {criteria}")
     task = results[item] if id and results else None
     execution.log_iteration_results(id, summary, task)
 
@@ -432,7 +433,7 @@ def enrich_function_from_dict(function, function_dict):
 
 def enrich_run_labels(
     labels: dict,
-    labels_to_enrich: list[RunLabels] = None,
+    labels_to_enrich: Optional[list[RunLabels]] = None,
 ):
     labels_enrichment = {
         RunLabels.owner: os.environ.get("V3IO_USERNAME") or getpass.getuser(),

mlrun/serving/merger.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 #
 
+from typing import Optional
+
 import storey
 
 
@@ -37,10 +39,10 @@ class CacheEntry:
 class Merge(storey.Flow):
     def __init__(
         self,
-        full_event: bool = None,
-        key_path: str = None,
-        max_behind: int = None,
-        expected_num_events: int = None,
+        full_event: Optional[bool] = None,
+        key_path: Optional[str] = None,
+        max_behind: Optional[int] = None,
+        expected_num_events: Optional[int] = None,
         **kwargs,
     ):
         """Merge multiple events based on event id or provided key path

mlrun/serving/remote.py

@@ -14,6 +14,7 @@
 #
 import asyncio
 import json
+from typing import Optional
 
 import aiohttp
 import requests
@@ -40,14 +41,14 @@ class RemoteStep(storey.SendToHttp):
     def __init__(
         self,
         url: str,
-        subpath: str = None,
-        method: str = None,
-        headers: dict = None,
-        url_expression: str = None,
-        body_expression: str = None,
+        subpath: Optional[str] = None,
+        method: Optional[str] = None,
+        headers: Optional[dict] = None,
+        url_expression: Optional[str] = None,
+        body_expression: Optional[str] = None,
         return_json: bool = True,
-        input_path: str = None,
-        result_path: str = None,
+        input_path: Optional[str] = None,
+        result_path: Optional[str] = None,
         max_in_flight=None,
         retries=None,
         backoff_factor=None,
@@ -57,7 +58,7 @@ class RemoteStep(storey.SendToHttp):
         """class for calling remote endpoints
 
         sync and async graph step implementation for request/resp to remote service (class shortcut = "$remote")
-        url can be an http(s) url (e.g. "https://myservice/path") or an mlrun function uri ([project/]name).
+        url can be an http(s) url (e.g. `https://myservice/path`) or an mlrun function uri ([project/]name).
         alternatively the url_expression can be specified to build the url from the event (e.g. "event['url']").
 
         example pipeline::
@@ -241,15 +242,15 @@ class RemoteStep(storey.SendToHttp):
 class BatchHttpRequests(_ConcurrentJobExecution):
     def __init__(
         self,
-        url: str = None,
-        subpath: str = None,
-        method: str = None,
-        headers: dict = None,
-        url_expression: str = None,
-        body_expression: str = None,
+        url: Optional[str] = None,
+        subpath: Optional[str] = None,
+        method: Optional[str] = None,
+        headers: Optional[dict] = None,
+        url_expression: Optional[str] = None,
+        body_expression: Optional[str] = None,
         return_json: bool = True,
-        input_path: str = None,
-        result_path: str = None,
+        input_path: Optional[str] = None,
+        result_path: Optional[str] = None,
         retries=None,
         backoff_factor=None,
         timeout=None,
@@ -258,7 +259,7 @@ class BatchHttpRequests(_ConcurrentJobExecution):
         """class for calling remote endpoints in parallel
 
         sync and async graph step implementation for request/resp to remote service (class shortcut = "$remote")
-        url can be an http(s) url (e.g. "https://myservice/path") or an mlrun function uri ([project/]name).
+        url can be an http(s) url (e.g. `https://myservice/path`) or an mlrun function uri ([project/]name).
         alternatively the url_expression can be specified to build the url from the event (e.g. "event['url']").
 
         example pipeline::

mlrun/serving/routers.py

@@ -46,13 +46,13 @@ class BaseModelRouter(RouterToDict):
     def __init__(
         self,
         context=None,
-        name: str = None,
+        name: typing.Optional[str] = None,
         routes=None,
-        protocol: str = None,
-        url_prefix: str = None,
-        health_prefix: str = None,
-        input_path: str = None,
-        result_path: str = None,
+        protocol: typing.Optional[str] = None,
+        url_prefix: typing.Optional[str] = None,
+        health_prefix: typing.Optional[str] = None,
+        input_path: typing.Optional[str] = None,
+        result_path: typing.Optional[str] = None,
         **kwargs,
     ):
         """Model Serving Router, route between child models
@@ -249,11 +249,11 @@ class ParallelRun(BaseModelRouter):
     def __init__(
         self,
         context=None,
-        name: str = None,
+        name: typing.Optional[str] = None,
         routes=None,
-        protocol: str = None,
-        url_prefix: str = None,
-        health_prefix: str = None,
+        protocol: typing.Optional[str] = None,
+        url_prefix: typing.Optional[str] = None,
+        health_prefix: typing.Optional[str] = None,
         extend_event=None,
         executor_type: Union[ParallelRunnerModes, str] = ParallelRunnerModes.thread,
         **kwargs,
@@ -481,13 +481,13 @@ class VotingEnsemble(ParallelRun):
     def __init__(
         self,
         context=None,
-        name: str = None,
+        name: typing.Optional[str] = None,
         routes=None,
-        protocol: str = None,
-        url_prefix: str = None,
-        health_prefix: str = None,
-        vote_type: str = None,
-        weights: dict[str, float] = None,
+        protocol: typing.Optional[str] = None,
+        url_prefix: typing.Optional[str] = None,
+        health_prefix: typing.Optional[str] = None,
+        vote_type: typing.Optional[str] = None,
+        weights: typing.Optional[dict[str, float]] = None,
         executor_type: Union[ParallelRunnerModes, str] = ParallelRunnerModes.thread,
         format_response_with_col_name_flag: bool = False,
         prediction_col_name: str = "prediction",
@@ -1130,13 +1130,13 @@ class EnrichmentModelRouter(ModelRouter):
     def __init__(
         self,
         context=None,
-        name: str = None,
+        name: typing.Optional[str] = None,
         routes=None,
-        protocol: str = None,
-        url_prefix: str = None,
-        health_prefix: str = None,
+        protocol: typing.Optional[str] = None,
+        url_prefix: typing.Optional[str] = None,
+        health_prefix: typing.Optional[str] = None,
         feature_vector_uri: str = "",
-        impute_policy: dict = None,
+        impute_policy: typing.Optional[dict] = None,
         **kwargs,
     ):
         """Model router with feature enrichment (from the feature store)
@@ -1211,16 +1211,16 @@ class EnrichmentVotingEnsemble(VotingEnsemble):
     def __init__(
         self,
         context=None,
-        name: str = None,
+        name: typing.Optional[str] = None,
         routes=None,
         protocol=None,
-        url_prefix: str = None,
-        health_prefix: str = None,
-        vote_type: str = None,
+        url_prefix: typing.Optional[str] = None,
+        health_prefix: typing.Optional[str] = None,
+        vote_type: typing.Optional[str] = None,
         executor_type: Union[ParallelRunnerModes, str] = ParallelRunnerModes.thread,
-        prediction_col_name: str = None,
+        prediction_col_name: typing.Optional[str] = None,
         feature_vector_uri: str = "",
-        impute_policy: dict = None,
+        impute_policy: typing.Optional[dict] = None,
         **kwargs,
     ):
         """Voting Ensemble with feature enrichment (from the feature store)

mlrun/serving/server.py

@@ -196,7 +196,7 @@ class GraphServer(ModelObj):
     def test(
         self,
         path: str = "/",
-        body: Union[str, bytes, dict] = None,
+        body: Optional[Union[str, bytes, dict]] = None,
         method: str = "",
         headers: Optional[str] = None,
         content_type: Optional[str] = None,