mlrun 1.6.0rc26__py3-none-any.whl → 1.6.3rc1__py3-none-any.whl

mlrun/datastore/datastore_profile.py

@@ -367,7 +367,7 @@ class DatastoreProfile2Json(pydantic.BaseModel):
             )
 
 
-def datastore_profile_read(url):
+def datastore_profile_read(url, project_name="", secrets: dict = None):
     parsed_url = urlparse(url)
     if parsed_url.scheme.lower() != "ds":
         raise mlrun.errors.MLRunInvalidArgumentError(
@@ -375,17 +375,29 @@ def datastore_profile_read(url):
         )
 
     profile_name = parsed_url.hostname
-    project_name = parsed_url.username or mlrun.mlconf.default_project
+    project_name = project_name or mlrun.mlconf.default_project
     datastore = TemporaryClientDatastoreProfiles().get(profile_name)
     if datastore:
         return datastore
     public_profile = mlrun.db.get_run_db().get_datastore_profile(
         profile_name, project_name
     )
+    # The mlrun.db.get_run_db().get_datastore_profile() function is capable of returning
+    # two distinct types of objects based on its execution context.
+    # If it operates from the client or within the pod (which is the common scenario),
+    # it yields an instance of `mlrun.datastore.DatastoreProfile`. Conversely,
+    # when executed on the server with a direct call to `sqldb`, it produces an instance of
+    # mlrun.common.schemas.DatastoreProfile.
+    # In the latter scenario, an extra conversion step is required to transform the object
+    # into mlrun.datastore.DatastoreProfile.
+    if isinstance(public_profile, mlrun.common.schemas.DatastoreProfile):
+        public_profile = DatastoreProfile2Json.create_from_json(
+            public_json=public_profile.object
+        )
     project_ds_name_private = DatastoreProfile.generate_secret_key(
         profile_name, project_name
     )
-    private_body = get_secret_or_env(project_ds_name_private)
+    private_body = get_secret_or_env(project_ds_name_private, secret_provider=secrets)
     if not public_profile or not private_body:
         raise mlrun.errors.MLRunInvalidArgumentError(
             f"Unable to retrieve the datastore profile '{url}' from either the server or local environment. "

mlrun/datastore/google_cloud_storage.py

@@ -60,7 +60,7 @@ class GoogleCloudStorageStore(DataStore):
             except json.JSONDecodeError:
                 # If it's not json, handle it as a filename
                 token = credentials
-                return self._sanitize_storage_options(dict(token=token))
+            return self._sanitize_storage_options(dict(token=token))
         else:
             logger.info(
                 "No GCS credentials available - auth will rely on auto-discovery of credentials"
@@ -147,13 +147,13 @@ class GoogleCloudStorageStore(DataStore):
             if "project_id" in credentials:
                 res["spark.hadoop.fs.gs.project.id"] = credentials["project_id"]
             if "private_key_id" in credentials:
-                res[
-                    "spark.hadoop.fs.gs.auth.service.account.private.key.id"
-                ] = credentials["private_key_id"]
+                res["spark.hadoop.fs.gs.auth.service.account.private.key.id"] = (
+                    credentials["private_key_id"]
+                )
             if "private_key" in credentials:
-                res[
-                    "spark.hadoop.fs.gs.auth.service.account.private.key"
-                ] = credentials["private_key"]
+                res["spark.hadoop.fs.gs.auth.service.account.private.key"] = (
+                    credentials["private_key"]
+                )
             if "client_email" in credentials:
                 res["spark.hadoop.fs.gs.auth.service.account.email"] = credentials[
                     "client_email"

mlrun/datastore/sources.py

@@ -118,7 +118,10 @@ class BaseSourceDriver(DataSource):
             if named_view:
                 df.createOrReplaceTempView(self.name)
             return self._filter_spark_df(df, time_field, columns)
-        raise NotImplementedError()
+        raise NotImplementedError(
+            f"Conversion of a source of type '{type(self).__name__}' "
+            "to a Spark dataframe is not possible, as this operation is not supported"
+        )
 
     def _filter_spark_df(self, df, time_field=None, columns=None):
         if not (columns or time_field):
@@ -897,7 +900,7 @@ class StreamSource(OnlineSource):
             engine = function.spec.graph.engine
         if mlrun.mlconf.is_explicit_ack() and engine == "async":
             kwargs["explicit_ack_mode"] = "explicitOnly"
-            kwargs["workerAllocationMode"] = "static"
+            kwargs["worker_allocation_mode"] = "static"
 
         function.add_v3io_stream_trigger(
             self.path,
@@ -986,9 +989,13 @@ class KafkaSource(OnlineSource):
         if mlrun.mlconf.is_explicit_ack() and engine == "async":
             explicit_ack_mode = "explicitOnly"
             extra_attributes["workerAllocationMode"] = extra_attributes.get(
-                "workerAllocationMode", "static"
+                "worker_allocation_mode", "static"
             )
 
+        trigger_kwargs = {}
+        if "max_workers" in extra_attributes:
+            trigger_kwargs = {"max_workers": extra_attributes.pop("max_workers")}
+
         trigger = KafkaTrigger(
             brokers=extra_attributes.pop("brokers"),
             topics=extra_attributes.pop("topics"),
@@ -997,7 +1004,7 @@ class KafkaSource(OnlineSource):
             initial_offset=extra_attributes.pop("initial_offset"),
             explicit_ack_mode=explicit_ack_mode,
             extra_attributes=extra_attributes,
-            max_workers=extra_attributes.pop("max_workers", 4),
+            **trigger_kwargs,
         )
         function = function.add_trigger("kafka", trigger)
 
@@ -1016,6 +1023,12 @@ class KafkaSource(OnlineSource):
 
         return function
 
+    def to_spark_df(self, session, named_view=False, time_field=None, columns=None):
+        raise NotImplementedError(
+            "Conversion of a source of type 'KafkaSource' "
+            "to a Spark dataframe is not possible, as this operation is not supported by Spark"
+        )
+
 
 class SQLSource(BaseSourceDriver):
     kind = "sqldb"

mlrun/datastore/targets.py

@@ -456,7 +456,7 @@ class BaseStoreTarget(DataTargetBase):
             self.get_target_path(),
             credentials_prefix_secrets,
         )
-        if self.get_target_path().startswith("ds://"):
+        if self.get_target_path() and self.get_target_path().startswith("ds://"):
             return store, store.url + resolved_store_path
         else:
             return store, self.get_target_path()
@@ -1984,6 +1984,8 @@ def _get_target_path(driver, resource, run_id_mode=False):
 
 
 def generate_path_with_chunk(target, chunk_id, path):
+    if path is None:
+        return ""
     prefix, suffix = os.path.splitext(path)
     if chunk_id and not target.partitioned and not target.time_partitioning_granularity:
         return f"{prefix}/{chunk_id:0>4}{suffix}"

mlrun/datastore/utils.py

@@ -18,6 +18,7 @@ import typing
 from urllib.parse import parse_qs, urlparse, urlunparse
 
 import pandas as pd
+import semver
 
 import mlrun.datastore
 
@@ -137,7 +138,16 @@ def filter_df_generator(
 def _execute_time_filter(
     df: pd.DataFrame, time_column: str, start_time: pd.Timestamp, end_time: pd.Timestamp
 ):
-    df[time_column] = pd.to_datetime(df[time_column], format="mixed", yearfirst=True)
+    if semver.parse(pd.__version__)["major"] >= 2:
+        # pandas 2 is too strict by default (ML-5629)
+        kwargs = {
+            "format": "mixed",
+            "yearfirst": True,
+        }
+    else:
+        # pandas 1 may fail on format "mixed" (ML-5661)
+        kwargs = {}
+    df[time_column] = pd.to_datetime(df[time_column], **kwargs)
     if start_time:
         df = df[df[time_column] > start_time]
     if end_time:

mlrun/datastore/v3io.py

@@ -15,12 +15,11 @@
 import mmap
 import os
 import time
-from copy import deepcopy
 from datetime import datetime
 
 import fsspec
-import requests
-import v3io.dataplane
+import v3io
+from v3io.dataplane.response import HttpResponseError
 
 import mlrun
 from mlrun.datastore.helpers import ONE_GB, ONE_MB
@@ -30,11 +29,6 @@ from .base import (
     DataStore,
     FileStats,
     basic_auth_header,
-    get_range,
-    http_get,
-    http_head,
-    http_put,
-    http_upload,
 )
 
 V3IO_LOCAL_ROOT = "v3io"
@@ -47,17 +41,18 @@ class V3ioStore(DataStore):
 
         self.headers = None
         self.secure = self.kind == "v3ios"
+
+        token = self._get_secret_or_env("V3IO_ACCESS_KEY")
+        username = self._get_secret_or_env("V3IO_USERNAME")
+        password = self._get_secret_or_env("V3IO_PASSWORD")
         if self.endpoint.startswith("https://"):
             self.endpoint = self.endpoint[len("https://") :]
             self.secure = True
         elif self.endpoint.startswith("http://"):
             self.endpoint = self.endpoint[len("http://") :]
             self.secure = False
-
-        token = self._get_secret_or_env("V3IO_ACCESS_KEY")
-        username = self._get_secret_or_env("V3IO_USERNAME")
-        password = self._get_secret_or_env("V3IO_PASSWORD")
-
+        self.client = v3io.dataplane.Client(access_key=token, endpoint=self.url)
+        self.object = self.client.object
         self.auth = None
         self.token = token
         if token:
@@ -65,6 +60,16 @@ class V3ioStore(DataStore):
         elif username and password:
             self.headers = basic_auth_header(username, password)
 
+    @staticmethod
+    def _do_object_request(function: callable, *args, **kwargs):
+        try:
+            return function(*args, **kwargs)
+        except HttpResponseError as http_response_error:
+            raise mlrun.errors.err_for_status_code(
+                status_code=http_response_error.status_code,
+                message=mlrun.errors.err_to_str(http_response_error),
+            )
+
     @staticmethod
     def uri_to_ipython(endpoint, subpath):
         return V3IO_LOCAL_ROOT + subpath
@@ -91,13 +96,19 @@ class V3ioStore(DataStore):
 
     def _upload(self, key: str, src_path: str, max_chunk_size: int = ONE_GB):
         """helper function for upload method, allows for controlling max_chunk_size in testing"""
+        container, path = split_path(self._join(key))
         file_size = os.path.getsize(src_path)  # in bytes
         if file_size <= ONE_MB:
-            http_upload(self.url + self._join(key), src_path, self.headers, None)
+            with open(src_path, "rb") as source_file:
+                data = source_file.read()
+            self._do_object_request(
+                self.object.put,
+                container=container,
+                path=path,
+                body=data,
+                append=False,
+            )
             return
-        append_header = deepcopy(self.headers)
-        append_header["Range"] = "-1"
-
         # chunk must be a multiple of the ALLOCATIONGRANULARITY
         # https://docs.python.org/3/library/mmap.html
         if residue := max_chunk_size % mmap.ALLOCATIONGRANULARITY:
@@ -114,11 +125,13 @@ class V3ioStore(DataStore):
                     access=mmap.ACCESS_READ,
                     offset=file_offset,
                 ) as mmap_obj:
-                    http_put(
-                        self.url + self._join(key),
-                        mmap_obj,
-                        append_header if file_offset else self.headers,
-                        None,
+                    append = file_offset != 0
+                    self._do_object_request(
+                        self.object.put,
+                        container=container,
+                        path=path,
+                        body=mmap_obj,
+                        append=append,
                     )
                     file_offset += chunk_size
 
@@ -126,43 +139,55 @@ class V3ioStore(DataStore):
         return self._upload(key, src_path)
 
     def get(self, key, size=None, offset=0):
-        headers = self.headers
-        if size or offset:
-            headers = deepcopy(headers)
-            headers["Range"] = get_range(size, offset)
-        return http_get(self.url + self._join(key), headers)
+        container, path = split_path(self._join(key))
+        return self._do_object_request(
+            function=self.object.get,
+            container=container,
+            path=path,
+            offset=offset,
+            num_bytes=size,
+        ).body
 
-    def _put(self, key, data, max_chunk_size: int = ONE_GB):
+    def _put(self, key, data, append=False, max_chunk_size: int = ONE_GB):
         """helper function for put method, allows for controlling max_chunk_size in testing"""
+        container, path = split_path(self._join(key))
         buffer_size = len(data)  # in bytes
         if buffer_size <= ONE_MB:
-            http_put(self.url + self._join(key), data, self.headers, None)
+            self._do_object_request(
+                self.object.put,
+                container=container,
+                path=path,
+                body=data,
+                append=append,
+            )
             return
-        append_header = deepcopy(self.headers)
-        append_header["Range"] = "-1"
         buffer_offset = 0
         try:
             data = memoryview(data)
         except TypeError:
             pass
 
-        with requests.Session() as requests_session:
-            while buffer_offset < buffer_size:
-                chunk_size = min(buffer_size - buffer_offset, max_chunk_size)
-                http_put(
-                    self.url + self._join(key),
-                    data[buffer_offset : buffer_offset + chunk_size],
-                    append_header if buffer_offset else self.headers,
-                    None,
-                    requests_session,
-                )
-                buffer_offset += chunk_size
+        while buffer_offset < buffer_size:
+            chunk_size = min(buffer_size - buffer_offset, max_chunk_size)
+            append = True if buffer_offset or append else False
+            self._do_object_request(
+                self.object.put,
+                container=container,
+                path=path,
+                body=data[buffer_offset : buffer_offset + chunk_size],
+                append=append,
+            )
+            buffer_offset += chunk_size
 
     def put(self, key, data, append=False):
-        return self._put(key, data)
+        return self._put(key, data, append)
 
     def stat(self, key):
-        head = http_head(self.url + self._join(key), self.headers)
+        container, path = split_path(self._join(key))
+        response = self._do_object_request(
+            function=self.object.head, container=container, path=path
+        )
+        head = dict(response.headers)
         size = int(head.get("Content-Length", "0"))
         datestr = head.get("Last-Modified", "0")
         modified = time.mktime(
@@ -171,7 +196,6 @@ class V3ioStore(DataStore):
         return FileStats(size, modified)
 
     def listdir(self, key):
-        v3io_client = v3io.dataplane.Client(endpoint=self.url, access_key=self.token)
         container, subpath = split_path(self._join(key))
         if not subpath.endswith("/"):
             subpath += "/"
@@ -180,7 +204,7 @@ class V3ioStore(DataStore):
         subpath_length = len(subpath) - 1
 
         try:
-            response = v3io_client.container.list(
+            response = self.client.container.list(
                 container=container,
                 path=subpath,
                 get_all_attributes=False,

mlrun/db/base.py

@@ -677,3 +677,21 @@ class RunDBInterface(ABC):
         self, func_url: str = None, function: "mlrun.runtimes.BaseRuntime" = None
     ):
         pass
+
+    def submit_workflow(
+        self,
+        project: str,
+        name: str,
+        workflow_spec: Union[
+            "mlrun.projects.pipelines.WorkflowSpec",
+            "mlrun.common.schemas.WorkflowSpec",
+            dict,
+        ],
+        arguments: Optional[dict] = None,
+        artifact_path: Optional[str] = None,
+        source: Optional[str] = None,
+        run_name: Optional[str] = None,
+        namespace: Optional[str] = None,
+        notifications: list["mlrun.model.Notification"] = None,
+    ) -> "mlrun.common.schemas.WorkflowResponse":
+        pass

mlrun/db/httpdb.py

@@ -152,7 +152,7 @@ class HTTPRunDB(RunDBInterface):
     @staticmethod
     def get_api_path_prefix(version: str = None) -> str:
         """
-        :param version: API version to use, None (the default) will mean to use the default value from mlconf,
+        :param version: API version to use, None (the default) will mean to use the default value from mlrun.config,
          for un-versioned api set an empty string.
         """
         if version is not None:
@@ -250,7 +250,11 @@ class HTTPRunDB(RunDBInterface):
 
         try:
             response = self.session.request(
-                method, url, timeout=timeout, verify=False, **kw
+                method,
+                url,
+                timeout=timeout,
+                verify=config.httpdb.http.verify,
+                **kw,
             )
         except requests.RequestException as exc:
             error = f"{err_to_str(exc)}: {error}" if error else err_to_str(exc)
@@ -302,11 +306,11 @@ class HTTPRunDB(RunDBInterface):
 
     def connect(self, secrets=None):
         """Connect to the MLRun API server. Must be called prior to executing any other method.
-        The code utilizes the URL for the API server from the configuration - ``mlconf.dbpath``.
+        The code utilizes the URL for the API server from the configuration - ``config.dbpath``.
 
         For example::
 
-            mlconf.dbpath = mlconf.dbpath or 'http://mlrun-api:8080'
+            config.dbpath = config.dbpath or 'http://mlrun-api:8080'
             db = get_run_db().connect()
         """
         # hack to allow unit tests to instantiate HTTPRunDB without a real server behind
@@ -500,7 +504,7 @@ class HTTPRunDB(RunDBInterface):
         if offset < 0:
             raise MLRunInvalidArgumentError("Offset cannot be negative")
         if size is None:
-            size = int(mlrun.mlconf.httpdb.logs.pull_logs_default_size_limit)
+            size = int(config.httpdb.logs.pull_logs_default_size_limit)
         elif size == -1:
             logger.warning(
                 "Retrieving all logs. This may be inefficient and can result in a large log."
@@ -546,25 +550,23 @@ class HTTPRunDB(RunDBInterface):
 
         state, text = self.get_log(uid, project, offset=offset)
         if text:
-            print(text.decode(errors=mlrun.mlconf.httpdb.logs.decode.errors))
+            print(text.decode(errors=config.httpdb.logs.decode.errors))
         nil_resp = 0
         while True:
             offset += len(text)
             # if we get 3 nil responses in a row, increase the sleep time to 10 seconds
             # TODO: refactor this to use a conditional backoff mechanism
             if nil_resp < 3:
-                time.sleep(int(mlrun.mlconf.httpdb.logs.pull_logs_default_interval))
+                time.sleep(int(config.httpdb.logs.pull_logs_default_interval))
             else:
                 time.sleep(
-                    int(
-                        mlrun.mlconf.httpdb.logs.pull_logs_backoff_no_logs_default_interval
-                    )
+                    int(config.httpdb.logs.pull_logs_backoff_no_logs_default_interval)
                 )
             state, text = self.get_log(uid, project, offset=offset)
             if text:
                 nil_resp = 0
                 print(
-                    text.decode(errors=mlrun.mlconf.httpdb.logs.decode.errors),
+                    text.decode(errors=config.httpdb.logs.decode.errors),
                     end="",
                 )
             else:
@@ -1135,17 +1137,17 @@ class HTTPRunDB(RunDBInterface):
             structured_dict = {}
             for project, job_runtime_resources_map in response.json().items():
                 for job_id, runtime_resources in job_runtime_resources_map.items():
-                    structured_dict.setdefault(project, {})[
-                        job_id
-                    ] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                    structured_dict.setdefault(project, {})[job_id] = (
+                        mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                    )
             return structured_dict
         elif group_by == mlrun.common.schemas.ListRuntimeResourcesGroupByField.project:
             structured_dict = {}
             for project, kind_runtime_resources_map in response.json().items():
                 for kind, runtime_resources in kind_runtime_resources_map.items():
-                    structured_dict.setdefault(project, {})[
-                        kind
-                    ] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                    structured_dict.setdefault(project, {})[kind] = (
+                        mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                    )
             return structured_dict
         else:
             raise NotImplementedError(
@@ -1173,7 +1175,8 @@ class HTTPRunDB(RunDBInterface):
         :param force: Force deletion - delete the runtime resource even if it's not in terminal state or if the grace
             period didn't pass.
         :param grace_period: Grace period given to the runtime resource before they are actually removed, counted from
-            the moment they moved to terminal state (defaults to mlrun.mlconf.runtime_resources_deletion_grace_period).
+            the moment they moved to terminal state
+            (defaults to mlrun.config.config.runtime_resources_deletion_grace_period).
 
         :returns: :py:class:`~mlrun.common.schemas.GroupedByProjectRuntimeResourcesOutput` listing the runtime resources
             that were removed.
@@ -1203,9 +1206,9 @@ class HTTPRunDB(RunDBInterface):
         structured_dict = {}
         for project, kind_runtime_resources_map in response.json().items():
             for kind, runtime_resources in kind_runtime_resources_map.items():
-                structured_dict.setdefault(project, {})[
-                    kind
-                ] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                structured_dict.setdefault(project, {})[kind] = (
+                    mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                )
         return structured_dict
 
     def create_schedule(
@@ -1340,7 +1343,7 @@ class HTTPRunDB(RunDBInterface):
             logger.warning(
                 "Building a function image to ECR and loading an S3 source to the image may require conflicting access "
                 "keys. Only the permissions granted to the platform's configured secret will take affect "
-                "(see mlrun.mlconf.httpdb.builder.docker_registry_secret). "
+                "(see mlrun.config.config.httpdb.builder.docker_registry_secret). "
                 "In case the permissions are limited to ECR scope, you may use pull_at_runtime=True instead",
                 source=func.spec.build.source,
                 load_source_on_run=func.spec.build.load_source_on_run,
@@ -1495,7 +1498,7 @@ class HTTPRunDB(RunDBInterface):
         Retrieve updated information on project background tasks being executed.
         If no filter is provided, will return background tasks from the last week.
 
-        :param project: Project name (defaults to mlrun.mlconf.default_project).
+        :param project: Project name (defaults to mlrun.config.config.default_project).
         :param state:   List only background tasks whose state is specified.
         :param created_from: Filter by background task created time in ``[created_from, created_to]``.
         :param created_to:  Filter by background task created time in ``[created_from, created_to]``.
@@ -1608,19 +1611,21 @@ class HTTPRunDB(RunDBInterface):
         artifact_path=None,
         ops=None,
         cleanup_ttl=None,
+        timeout=60,
     ):
         """Submit a KFP pipeline for execution.
 
-        :param project: The project of the pipeline
-        :param pipeline: Pipeline function or path to .yaml/.zip pipeline file.
-        :param arguments: A dictionary of arguments to pass to the pipeline.
-        :param experiment: A name to assign for the specific experiment.
-        :param run: A name for this specific run.
-        :param namespace: Kubernetes namespace to execute the pipeline in.
-        :param artifact_path: A path to artifacts used by this pipeline.
-        :param ops: Transformers to apply on all ops in the pipeline.
-        :param cleanup_ttl: pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
-                            workflow and all its resources are deleted)
+        :param project:         The project of the pipeline
+        :param pipeline:        Pipeline function or path to .yaml/.zip pipeline file.
+        :param arguments:       A dictionary of arguments to pass to the pipeline.
+        :param experiment:      A name to assign for the specific experiment.
+        :param run:             A name for this specific run.
+        :param namespace:       Kubernetes namespace to execute the pipeline in.
+        :param artifact_path:   A path to artifacts used by this pipeline.
+        :param ops:             Transformers to apply on all ops in the pipeline.
+        :param cleanup_ttl:     Pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
+                                workflow and all its resources are deleted)
+        :param timeout:         Timeout for the API call.
         """
 
         if isinstance(pipeline, str):
@@ -1662,7 +1667,7 @@ class HTTPRunDB(RunDBInterface):
                 "POST",
                 f"projects/{project}/pipelines",
                 params=params,
-                timeout=20,
+                timeout=timeout,
                 body=data,
                 headers=headers,
             )
@@ -3450,8 +3455,8 @@ class HTTPRunDB(RunDBInterface):
         source: Optional[str] = None,
         run_name: Optional[str] = None,
         namespace: Optional[str] = None,
-        notifications: typing.List[mlrun.model.Notification] = None,
-    ):
+        notifications: list[mlrun.model.Notification] = None,
+    ) -> mlrun.common.schemas.WorkflowResponse:
         """
         Submitting workflow for a remote execution.
 

mlrun/execution.py

@@ -559,9 +559,9 @@ class MLClientCtx(object):
             for k, v in get_in(task, ["status", "results"], {}).items():
                 self._results[k] = v
             for artifact in get_in(task, ["status", run_keys.artifacts], []):
-                self._artifacts_manager.artifacts[
-                    artifact["metadata"]["key"]
-                ] = artifact
+                self._artifacts_manager.artifacts[artifact["metadata"]["key"]] = (
+                    artifact
+                )
                 self._artifacts_manager.link_artifact(
                     self.project,
                     self.name,