mlrun 1.7.0rc7__py3-none-any.whl → 1.7.0rc9__py3-none-any.whl

mlrun/runtimes/nuclio/function.py

@@ -543,11 +543,16 @@ class RemoteRuntime(KubeResource):
         :param project:    project name
         :param tag:        function tag
         :param verbose:    set True for verbose logging
-        :param auth_info:  service AuthInfo
+        :param auth_info:  service AuthInfo (deprecated and ignored)
         :param builder_env: env vars dict for source archive config/credentials e.g. builder_env={"GIT_TOKEN": token}
         :param force_build: set True for force building the image
         """
-        # todo: verify that the function name is normalized
+        if auth_info:
+            # TODO: remove in 1.9.0
+            warnings.warn(
+                "'auth_info' is deprecated for nuclio runtimes in 1.7.0 and will be removed in 1.9.0",
+                FutureWarning,
+            )
 
         old_http_session = getattr(self, "_http_session", None)
         if old_http_session:
@@ -570,9 +575,7 @@ class RemoteRuntime(KubeResource):
         self._fill_credentials()
         db = self._get_db()
         logger.info("Starting remote function deploy")
-        data = db.remote_builder(
-            self, False, builder_env=builder_env, force_build=force_build
-        )
+        data = db.deploy_nuclio_function(func=self, builder_env=builder_env)
         self.status = data["data"].get("status")
         self._update_credentials_from_remote_build(data["data"])
 
@@ -613,7 +616,7 @@ class RemoteRuntime(KubeResource):
                 int(mlrun.mlconf.httpdb.logs.nuclio.pull_deploy_status_default_interval)
             )
             try:
-                text, last_log_timestamp = db.get_builder_status(
+                text, last_log_timestamp = db.get_nuclio_deploy_status(
                     self, last_log_timestamp=last_log_timestamp, verbose=verbose
                 )
             except mlrun.db.RunDBError:
@@ -995,10 +998,10 @@ class RemoteRuntime(KubeResource):
         ]
 
         if command:
-            sidecar["command"] = command
+            sidecar["command"] = mlrun.utils.helpers.as_list(command)
 
         if args:
-            sidecar["args"] = args
+            sidecar["args"] = mlrun.utils.helpers.as_list(args)
 
     def _set_sidecar(self, name: str) -> dict:
         self.spec.config.setdefault("spec.sidecars", [])

mlrun/runtimes/nuclio/serving.py

@@ -491,9 +491,9 @@ class ServingRuntime(RemoteRuntime):
 
                 if (
                     stream.path.startswith("kafka://")
-                    or "kafka_bootstrap_servers" in stream.options
+                    or "kafka_brokers" in stream.options
                 ):
-                    brokers = stream.options.get("kafka_bootstrap_servers")
+                    brokers = stream.options.get("kafka_brokers")
                     if brokers:
                         brokers = brokers.split(",")
                     topic, brokers = parse_kafka_url(stream.path, brokers)

mlrun/runtimes/pod.py

@@ -15,6 +15,7 @@ import copy
 import inspect
 import os
 import re
+import time
 import typing
 from enum import Enum
 
@@ -1338,6 +1339,150 @@ class KubeResource(BaseRuntime):
 
         self.spec.validate_service_account(allowed_service_accounts)
 
+    def _configure_mlrun_build_with_source(
+        self, source, workdir=None, handler=None, pull_at_runtime=True, target_dir=None
+    ):
+        mlrun.utils.helpers.validate_builder_source(source, pull_at_runtime, workdir)
+
+        self.spec.build.source = source
+        if handler:
+            self.spec.default_handler = handler
+        if workdir:
+            self.spec.workdir = workdir
+        if target_dir:
+            self.spec.build.source_code_target_dir = target_dir
+
+        self.spec.build.load_source_on_run = pull_at_runtime
+        if (
+            self.spec.build.base_image
+            and not self.spec.build.commands
+            and pull_at_runtime
+            and not self.spec.image
+        ):
+            # if we load source from repo and don't need a full build use the base_image as the image
+            self.spec.image = self.spec.build.base_image
+        elif not pull_at_runtime:
+            # clear the image so build will not be skipped
+            self.spec.build.base_image = self.spec.build.base_image or self.spec.image
+            self.spec.image = ""
+
+    def _resolve_build_with_mlrun(self, with_mlrun: typing.Optional[bool] = None):
+        build = self.spec.build
+        if with_mlrun is None:
+            if build.with_mlrun is not None:
+                with_mlrun = build.with_mlrun
+            else:
+                with_mlrun = build.base_image and not (
+                    build.base_image.startswith("mlrun/")
+                    or "/mlrun/" in build.base_image
+                )
+        if (
+            not build.source
+            and not build.commands
+            and not build.requirements
+            and not build.extra
+            and with_mlrun
+        ):
+            logger.info(
+                "Running build to add mlrun package, set "
+                "with_mlrun=False to skip if its already in the image"
+            )
+        return with_mlrun
+
+    def _build_image(
+        self,
+        builder_env,
+        force_build,
+        mlrun_version_specifier,
+        show_on_failure,
+        skip_deployed,
+        watch,
+        is_kfp,
+        with_mlrun,
+    ):
+        # When we're in pipelines context we must watch otherwise the pipelines pod will exit before the operation
+        # is actually done. (when a pipelines pod exits, the pipeline step marked as done)
+        if is_kfp:
+            watch = True
+
+        db = self._get_db()
+        data = db.remote_builder(
+            self,
+            with_mlrun,
+            mlrun_version_specifier,
+            skip_deployed,
+            builder_env=builder_env,
+            force_build=force_build,
+        )
+        self.status = data["data"].get("status", None)
+        self.spec.image = mlrun.utils.get_in(
+            data, "data.spec.image"
+        ) or mlrun.utils.get_in(data, "data.spec.build.image")
+        self.spec.build.base_image = self.spec.build.base_image or mlrun.utils.get_in(
+            data, "data.spec.build.base_image"
+        )
+        # Get the source target dir in case it was enriched due to loading source
+        self.spec.build.source_code_target_dir = mlrun.utils.get_in(
+            data, "data.spec.build.source_code_target_dir"
+        ) or mlrun.utils.get_in(data, "data.spec.clone_target_dir")
+        ready = data.get("ready", False)
+        if not ready:
+            logger.info(
+                f"Started building image: {data.get('data', {}).get('spec', {}).get('build', {}).get('image')}"
+            )
+        if watch and not ready:
+            state = self._build_watch(
+                watch=watch,
+                show_on_failure=show_on_failure,
+            )
+            ready = state == "ready"
+            self.status.state = state
+
+        if watch and not ready:
+            raise mlrun.errors.MLRunRuntimeError("Deploy failed")
+        return ready
+
+    def _build_watch(
+        self,
+        watch: bool = True,
+        logs: bool = True,
+        show_on_failure: bool = False,
+    ):
+        db = self._get_db()
+        offset = 0
+        try:
+            text, _ = db.get_builder_status(self, 0, logs=logs)
+        except mlrun.db.RunDBError:
+            raise ValueError("function or build process not found")
+
+        def print_log(text):
+            if text and (
+                not show_on_failure
+                or self.status.state == mlrun.common.schemas.FunctionState.error
+            ):
+                print(text, end="")
+
+        print_log(text)
+        offset += len(text)
+        if watch:
+            while self.status.state in [
+                mlrun.common.schemas.FunctionState.pending,
+                mlrun.common.schemas.FunctionState.running,
+            ]:
+                time.sleep(2)
+                if show_on_failure:
+                    text = ""
+                    db.get_builder_status(self, 0, logs=False)
+                    if self.status.state == mlrun.common.schemas.FunctionState.error:
+                        # re-read the full log on failure
+                        text, _ = db.get_builder_status(self, offset, logs=logs)
+                else:
+                    text, _ = db.get_builder_status(self, offset, logs=logs)
+                print_log(text)
+                offset += len(text)
+
+        return self.status.state
+
 
 def _resolve_if_type_sanitized(attribute_name, attribute):
     attribute_config = sanitized_attributes[attribute_name]

mlrun/runtimes/utils.py

@@ -417,34 +417,6 @@ def get_func_selector(project, name=None, tag=None):
     return s
 
 
-class k8s_resource:
-    kind = ""
-    per_run = False
-    per_function = False
-    k8client = None
-
-    def deploy_function(self, function):
-        pass
-
-    def release_function(self, function):
-        pass
-
-    def submit_run(self, function, runobj):
-        pass
-
-    def get_object(self, name, namespace=None):
-        return None
-
-    def get_status(self, name, namespace=None):
-        return None
-
-    def del_object(self, name, namespace=None):
-        pass
-
-    def get_pods(self, name, namespace=None, master=False):
-        return {}
-
-
 def enrich_function_from_dict(function, function_dict):
     override_function = mlrun.new_function(runtime=function_dict, kind=function.kind)
     for attribute in [

mlrun/serving/remote.py

@@ -172,8 +172,7 @@ class RemoteStep(storey.SendToHttp):
         if not self._session:
             self._session = mlrun.utils.HTTPSessionWithRetry(
                 self.retries,
-                self.backoff_factor
-                or mlrun.config.config.http_retry_defaults.backoff_factor,
+                self.backoff_factor or mlrun.mlconf.http_retry_defaults.backoff_factor,
                 retry_on_exception=False,
                 retry_on_status=self.retries > 0,
                 retry_on_post=True,
@@ -185,7 +184,7 @@ class RemoteStep(storey.SendToHttp):
             resp = self._session.request(
                 method,
                 url,
-                verify=mlrun.config.config.httpdb.http.verify,
+                verify=mlrun.mlconf.httpdb.http.verify,
                 headers=headers,
                 data=body,
                 timeout=self.timeout,

mlrun/serving/routers.py

@@ -28,6 +28,7 @@ import numpy as np
 import mlrun
 import mlrun.common.model_monitoring
 import mlrun.common.schemas.model_monitoring
+from mlrun.errors import err_to_str
 from mlrun.utils import logger, now_date
 
 from ..common.helpers import parse_versioned_object_uri
@@ -1013,7 +1014,7 @@ def _init_endpoint_record(
             graph_server.function_uri
         )
     except Exception as e:
-        logger.error("Failed to parse function URI", exc=e)
+        logger.error("Failed to parse function URI", exc=err_to_str(e))
         return None
 
     # Generating version model value based on the model name and model version
@@ -1089,12 +1090,12 @@ def _init_endpoint_record(
         except Exception as exc:
             logger.warning(
                 "Failed creating model endpoint record",
-                exc=exc,
+                exc=err_to_str(exc),
                 traceback=traceback.format_exc(),
             )
 
     except Exception as e:
-        logger.error("Failed to retrieve model endpoint object", exc=e)
+        logger.error("Failed to retrieve model endpoint object", exc=err_to_str(e))
 
     return endpoint_uid
 

mlrun/serving/server.py

@@ -53,7 +53,7 @@ class _StreamContext:
         Initialize _StreamContext object.
         :param enabled:      A boolean indication for applying the stream context
         :param parameters:   Dictionary of optional parameters, such as `log_stream` and `stream_args`. Note that these
-                             parameters might be relevant to the output source such as `kafka_bootstrap_servers` if
+                             parameters might be relevant to the output source such as `kafka_brokers` if
                              the output source is from type Kafka.
         :param function_uri: Full value of the function uri, usually it's <project-name>/<function-name>
         """

mlrun/serving/states.py

@@ -1524,21 +1524,18 @@ def _init_async_objects(context, steps):
                     endpoint = None
                     options = {}
                     options.update(step.options)
-                    kafka_bootstrap_servers = options.pop(
-                        "kafka_bootstrap_servers", None
-                    )
-                    if stream_path.startswith("kafka://") or kafka_bootstrap_servers:
-                        topic, bootstrap_servers = parse_kafka_url(
-                            stream_path, kafka_bootstrap_servers
-                        )
+                    kafka_brokers = options.pop("kafka_brokers", None)
+                    if stream_path.startswith("kafka://") or kafka_brokers:
+                        topic, brokers = parse_kafka_url(stream_path, kafka_brokers)
 
                         kafka_producer_options = options.pop(
-                            "kafka_producer_options", None
+                            "kafka_producer_options",
+                            options.pop("kafka_bootstrap_servers", None),
                         )
 
                         step._async_object = storey.KafkaTarget(
                             topic=topic,
-                            bootstrap_servers=bootstrap_servers,
+                            brokers=brokers,
                             producer_options=kafka_producer_options,
                             context=context,
                             **options,

mlrun/serving/v2_serving.py

@@ -21,6 +21,7 @@ import mlrun.common.model_monitoring
 import mlrun.common.schemas.model_monitoring
 from mlrun.artifacts import ModelArtifact  # noqa: F401
 from mlrun.config import config
+from mlrun.errors import err_to_str
 from mlrun.utils import logger, now_date
 
 from ..common.helpers import parse_versioned_object_uri
@@ -523,7 +524,7 @@ def _init_endpoint_record(
             graph_server.function_uri
         )
     except Exception as e:
-        logger.error("Failed to parse function URI", exc=e)
+        logger.error("Failed to parse function URI", exc=err_to_str(e))
         return None
 
     # Generating version model value based on the model name and model version
@@ -576,9 +577,9 @@ def _init_endpoint_record(
             )
 
         except Exception as e:
-            logger.error("Failed to create endpoint record", exc=e)
+            logger.error("Failed to create endpoint record", exc=err_to_str(e))
 
     except Exception as e:
-        logger.error("Failed to retrieve model endpoint object", exc=e)
+        logger.error("Failed to retrieve model endpoint object", exc=err_to_str(e))
 
     return uid

mlrun/utils/http.py

@@ -122,7 +122,7 @@ class HTTPSessionWithRetry(requests.Session):
 
                 self._logger.warning(
                     "Error during request handling, retrying",
-                    exc=str(exc),
+                    exc=err_to_str(exc),
                     retry_count=retry_count,
                     url=url,
                     method=method,

mlrun/utils/retryer.py

@@ -138,6 +138,7 @@ class Retryer:
         except mlrun.errors.MLRunFatalFailureError as exc:
             raise exc.original_exception
         except Exception as exc:
+            self.last_exception = exc
             return (
                 None,
                 self.last_exception,

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "06b1879c4a1857b20f07e805c46f51aa4ac74cef",
-  "version": "1.7.0-rc7"
+  "git_commit": "3396f9ef4c278411ca6227b1792ff7df20005a42",
+  "version": "1.7.0-rc9"
 }

{mlrun-1.7.0rc7.dist-info → mlrun-1.7.0rc9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc7
+Version: 1.7.0rc9
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -37,20 +37,20 @@ Requires-Dist: pyyaml ~=5.1
 Requires-Dist: requests ~=2.31
 Requires-Dist: tabulate ~=0.8.6
 Requires-Dist: v3io ~=0.6.4
-Requires-Dist: pydantic >=1.10.8,~=1.10
+Requires-Dist: pydantic <1.10.15,>=1.10.8
 Requires-Dist: mergedeep ~=1.3
 Requires-Dist: v3io-frames ~=0.10.12
 Requires-Dist: semver ~=3.0
 Requires-Dist: dependency-injector ~=4.41
-Requires-Dist: fsspec ==2023.9.2
+Requires-Dist: fsspec <2024.4,>=2023.9.2
 Requires-Dist: v3iofs ~=0.1.17
-Requires-Dist: storey ~=1.7.6
+Requires-Dist: storey ~=1.7.7
 Requires-Dist: inflection ~=0.5.0
 Requires-Dist: python-dotenv ~=0.17.0
 Requires-Dist: setuptools ~=69.1
 Requires-Dist: deprecated ~=1.2
 Requires-Dist: jinja2 >=3.1.3,~=3.1
-Requires-Dist: orjson ~=3.9
+Requires-Dist: orjson <4,>=3.9.15
 Provides-Extra: alibaba-oss
 Requires-Dist: ossfs ==2023.12.0 ; extra == 'alibaba-oss'
 Requires-Dist: oss2 ==2.18.1 ; extra == 'alibaba-oss'
@@ -66,7 +66,7 @@ Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 'all'
 Requires-Dist: dask ~=2023.9.0 ; extra == 'all'
 Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'all'
 Requires-Dist: distributed ~=2023.9.0 ; extra == 'all'
-Requires-Dist: gcsfs ==2023.9.2 ; extra == 'all'
+Requires-Dist: gcsfs <2024.4,>=2023.9.2 ; extra == 'all'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'all'
 Requires-Dist: google-cloud-storage ==2.14.0 ; extra == 'all'
 Requires-Dist: google-cloud ==0.34 ; extra == 'all'
@@ -79,14 +79,14 @@ Requires-Dist: ossfs ==2023.12.0 ; extra == 'all'
 Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'all'
 Requires-Dist: pyopenssl >=23 ; extra == 'all'
 Requires-Dist: redis ~=4.3 ; extra == 'all'
-Requires-Dist: s3fs ==2023.9.2 ; extra == 'all'
+Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'all'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'all'
 Provides-Extra: api
 Requires-Dist: uvicorn ~=0.27.1 ; extra == 'api'
 Requires-Dist: dask-kubernetes ~=0.11.0 ; extra == 'api'
 Requires-Dist: apscheduler <4,>=3.10.3 ; extra == 'api'
 Requires-Dist: objgraph ~=3.6 ; extra == 'api'
-Requires-Dist: igz-mgmt ~=0.1.1 ; extra == 'api'
+Requires-Dist: igz-mgmt ~=0.1.3 ; extra == 'api'
 Requires-Dist: humanfriendly ~=10.0 ; extra == 'api'
 Requires-Dist: fastapi ~=0.110.0 ; extra == 'api'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'api'
@@ -115,7 +115,7 @@ Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 'complete'
 Requires-Dist: dask ~=2023.9.0 ; extra == 'complete'
 Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'complete'
 Requires-Dist: distributed ~=2023.9.0 ; extra == 'complete'
-Requires-Dist: gcsfs ==2023.9.2 ; extra == 'complete'
+Requires-Dist: gcsfs <2024.4,>=2023.9.2 ; extra == 'complete'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'complete'
 Requires-Dist: graphviz ~=0.20.0 ; extra == 'complete'
 Requires-Dist: kafka-python ~=2.0 ; extra == 'complete'
@@ -126,7 +126,7 @@ Requires-Dist: ossfs ==2023.12.0 ; extra == 'complete'
 Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete'
 Requires-Dist: pyopenssl >=23 ; extra == 'complete'
 Requires-Dist: redis ~=4.3 ; extra == 'complete'
-Requires-Dist: s3fs ==2023.9.2 ; extra == 'complete'
+Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete'
 Provides-Extra: complete-api
 Requires-Dist: adlfs ==2023.9.0 ; extra == 'complete-api'
@@ -143,11 +143,11 @@ Requires-Dist: dask ~=2023.9.0 ; extra == 'complete-api'
 Requires-Dist: databricks-sdk ~=0.13.0 ; extra == 'complete-api'
 Requires-Dist: distributed ~=2023.9.0 ; extra == 'complete-api'
 Requires-Dist: fastapi ~=0.110.0 ; extra == 'complete-api'
-Requires-Dist: gcsfs ==2023.9.2 ; extra == 'complete-api'
+Requires-Dist: gcsfs <2024.4,>=2023.9.2 ; extra == 'complete-api'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'complete-api'
 Requires-Dist: graphviz ~=0.20.0 ; extra == 'complete-api'
 Requires-Dist: humanfriendly ~=10.0 ; extra == 'complete-api'
-Requires-Dist: igz-mgmt ~=0.1.1 ; extra == 'complete-api'
+Requires-Dist: igz-mgmt ~=0.1.3 ; extra == 'complete-api'
 Requires-Dist: kafka-python ~=2.0 ; extra == 'complete-api'
 Requires-Dist: mlflow ~=2.8 ; extra == 'complete-api'
 Requires-Dist: msrest ~=0.6.21 ; extra == 'complete-api'
@@ -158,7 +158,7 @@ Requires-Dist: plotly <5.12.0,~=5.4 ; extra == 'complete-api'
 Requires-Dist: pymysql ~=1.0 ; extra == 'complete-api'
 Requires-Dist: pyopenssl >=23 ; extra == 'complete-api'
 Requires-Dist: redis ~=4.3 ; extra == 'complete-api'
-Requires-Dist: s3fs ==2023.9.2 ; extra == 'complete-api'
+Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 'complete-api'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
 Requires-Dist: timelength ~=1.1 ; extra == 'complete-api'
 Requires-Dist: uvicorn ~=0.27.1 ; extra == 'complete-api'
@@ -174,7 +174,7 @@ Requires-Dist: google-cloud ==0.34 ; extra == 'google-cloud'
 Provides-Extra: google-cloud-bigquery
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] ==3.14.1 ; extra == 'google-cloud-bigquery'
 Provides-Extra: google-cloud-storage
-Requires-Dist: gcsfs ==2023.9.2 ; extra == 'google-cloud-storage'
+Requires-Dist: gcsfs <2024.4,>=2023.9.2 ; extra == 'google-cloud-storage'
 Provides-Extra: graphviz
 Requires-Dist: graphviz ~=0.20.0 ; extra == 'graphviz'
 Provides-Extra: kafka
@@ -189,7 +189,7 @@ Requires-Dist: redis ~=4.3 ; extra == 'redis'
 Provides-Extra: s3
 Requires-Dist: boto3 <1.29.0,>=1.28.0 ; extra == 's3'
 Requires-Dist: aiobotocore <2.8,>=2.5.0 ; extra == 's3'
-Requires-Dist: s3fs ==2023.9.2 ; extra == 's3'
+Requires-Dist: s3fs <2024.4,>=2023.9.2 ; extra == 's3'
 Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'sqlalchemy'