mlrun 1.6.0rc21__py3-none-any.whl → 1.6.0rc22__py3-none-any.whl

mlrun/artifacts/base.py

@@ -714,10 +714,10 @@ class LinkArtifact(Artifact):
         self._spec = self._verify_dict(spec, "spec", LinkArtifactSpec)
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyArtifact' will be removed in 1.6.0, use 'Artifact' instead",
+    reason="'LegacyArtifact' will be removed in 1.7.0, use 'Artifact' instead",
     category=FutureWarning,
 )
 class LegacyArtifact(ModelObj):
@@ -880,10 +880,10 @@ class LegacyArtifact(ModelObj):
         return generate_target_path(self, artifact_path, producer)
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyDirArtifact' will be removed in 1.6.0, use 'DirArtifact' instead",
+    reason="'LegacyDirArtifact' will be removed in 1.7.0, use 'DirArtifact' instead",
     category=FutureWarning,
 )
 class LegacyDirArtifact(LegacyArtifact):
@@ -916,10 +916,10 @@ class LegacyDirArtifact(LegacyArtifact):
             mlrun.datastore.store_manager.object(url=target).upload(file_path)
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyLinkArtifact' will be removed in 1.6.0, use 'LinkArtifact' instead",
+    reason="'LegacyLinkArtifact' will be removed in 1.7.0, use 'LinkArtifact' instead",
     category=FutureWarning,
 )
 class LegacyLinkArtifact(LegacyArtifact):

mlrun/artifacts/dataset.py

@@ -283,14 +283,16 @@ class DatasetArtifact(Artifact):
         if artifact.spec.length > preview_rows_length and not ignore_preview_limits:
             preview_df = df.head(preview_rows_length)
 
-        # reset index while dropping existing index
-        # that way it wont create another index if one already there
-        preview_df = preview_df.reset_index(drop=True)
+        preview_df = preview_df.reset_index()
         artifact.status.header_original_length = len(preview_df.columns)
         if len(preview_df.columns) > max_preview_columns and not ignore_preview_limits:
             preview_df = preview_df.iloc[:, :max_preview_columns]
         artifact.spec.header = preview_df.columns.values.tolist()
         artifact.status.preview = preview_df.values.tolist()
+        # Table schema parsing doesn't require a column named "index"
+        # to align its output with previously generated header and preview data
+        if "index" in preview_df.columns:
+            preview_df.drop("index", axis=1, inplace=True)
         artifact.spec.schema = build_table_schema(preview_df)
 
         # set artifact stats if stats is explicitly set to true, or if stats is None and the dataframe is small
@@ -344,10 +346,10 @@ class DatasetArtifact(Artifact):
         self.status.stats = stats
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyTableArtifact' will be removed in 1.6.0, use 'TableArtifact' instead",
+    reason="'LegacyTableArtifact' will be removed in 1.7.0, use 'TableArtifact' instead",
     category=FutureWarning,
 )
 class LegacyTableArtifact(LegacyArtifact):
@@ -400,10 +402,10 @@ class LegacyTableArtifact(LegacyArtifact):
         return csv_buffer.getvalue()
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyDatasetArtifact' will be removed in 1.6.0, use 'DatasetArtifact' instead",
+    reason="'LegacyDatasetArtifact' will be removed in 1.7.0, use 'DatasetArtifact' instead",
     category=FutureWarning,
 )
 class LegacyDatasetArtifact(LegacyArtifact):
@@ -513,11 +515,16 @@ class LegacyDatasetArtifact(LegacyArtifact):
 
         if artifact.length > preview_rows_length and not ignore_preview_limits:
             preview_df = df.head(preview_rows_length)
-        preview_df = preview_df.reset_index(drop=True)
+
+        preview_df = preview_df.reset_index()
         if len(preview_df.columns) > max_preview_columns and not ignore_preview_limits:
             preview_df = preview_df.iloc[:, :max_preview_columns]
         artifact.header = preview_df.columns.values.tolist()
         artifact.preview = preview_df.values.tolist()
+        # Table schema parsing doesn't require a column named "index"
+        # to align its output with previously generated header and preview data
+        if "index" in preview_df.columns:
+            preview_df.drop("index", axis=1, inplace=True)
         artifact.schema = build_table_schema(preview_df)
         if (
             stats

mlrun/artifacts/manager.py

@@ -66,7 +66,7 @@ artifact_types = {
     "bokeh": BokehArtifact,
 }
 
-# TODO - Remove this when legacy types are deleted in 1.6.0
+# TODO - Remove this when legacy types are deleted in 1.7.0
 legacy_artifact_types = {
     "": LegacyArtifact,
     "dir": LegacyDirArtifact,

mlrun/artifacts/model.py

@@ -390,10 +390,10 @@ class ModelArtifact(Artifact):
         return mlrun.get_dataitem(target_model_path).get()
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyModelArtifact' will be removed in 1.6.0, use 'ModelArtifact' instead",
+    reason="'LegacyModelArtifact' will be removed in 1.7.0, use 'ModelArtifact' instead",
     category=FutureWarning,
 )
 class LegacyModelArtifact(LegacyArtifact):

mlrun/artifacts/plots.py

@@ -256,10 +256,10 @@ class PlotlyArtifact(Artifact):
         return self._figure.to_html()
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyPlotArtifact' will be removed in 1.6.0, use 'PlotArtifact' instead",
+    reason="'LegacyPlotArtifact' will be removed in 1.7.0, use 'PlotArtifact' instead",
     category=FutureWarning,
 )
 class LegacyPlotArtifact(LegacyArtifact):
@@ -303,10 +303,10 @@ class LegacyPlotArtifact(LegacyArtifact):
         return self._TEMPLATE.format(self.description or self.key, self.key, data_uri)
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyChartArtifact' will be removed in 1.6.0, use 'ChartArtifact' instead",
+    reason="'LegacyChartArtifact' will be removed in 1.7.0, use 'ChartArtifact' instead",
     category=FutureWarning,
 )
 class LegacyChartArtifact(LegacyArtifact):
@@ -377,10 +377,10 @@ class LegacyChartArtifact(LegacyArtifact):
         )
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyBokehArtifact' will be removed in 1.6.0, use 'BokehArtifact' instead",
+    reason="'LegacyBokehArtifact' will be removed in 1.7.0, use 'BokehArtifact' instead",
     category=FutureWarning,
 )
 class LegacyBokehArtifact(LegacyArtifact):
@@ -433,10 +433,10 @@ class LegacyBokehArtifact(LegacyArtifact):
         return file_html(self._figure, CDN, self.key)
 
 
-# TODO: remove in 1.6.0
+# TODO: remove in 1.7.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyPlotlyArtifact' will be removed in 1.6.0, use 'PlotlyArtifact' instead",
+    reason="'LegacyPlotlyArtifact' will be removed in 1.7.0, use 'PlotlyArtifact' instead",
     category=FutureWarning,
 )
 class LegacyPlotlyArtifact(LegacyArtifact):

mlrun/datastore/azure_blob.py

@@ -20,7 +20,6 @@ from azure.storage.blob._shared.base_client import parse_connection_str
 from fsspec.registry import get_filesystem_class
 
 import mlrun.errors
-from mlrun.errors import err_to_str
 
 from .base import DataStore, FileStats, makeDatastoreSchemaSanitizer
 
@@ -33,20 +32,16 @@ class AzureBlobStore(DataStore):
 
     def __init__(self, parent, schema, name, endpoint="", secrets: dict = None):
         super().__init__(parent, name, schema, endpoint, secrets=secrets)
-        self.get_filesystem()
 
-    def get_filesystem(self, silent=True):
+    @property
+    def filesystem(self):
         """return fsspec file system object, if supported"""
         if self._filesystem:
             return self._filesystem
         try:
             import adlfs  # noqa
         except ImportError as exc:
-            if not silent:
-                raise ImportError(
-                    f"Azure adlfs not installed, run pip install adlfs, {err_to_str(exc)}"
-                )
-            return None
+            raise ImportError("Azure adlfs not installed") from exc
         # in order to support az and wasbs kinds.
         filesystem_class = get_filesystem_class(protocol=self.kind)
         self._filesystem = makeDatastoreSchemaSanitizer(
@@ -87,12 +82,12 @@ class AzureBlobStore(DataStore):
 
     def upload(self, key, src_path):
         remote_path = self._convert_key_to_remote_path(key)
-        self._filesystem.put_file(src_path, remote_path, overwrite=True)
+        self.filesystem.put_file(src_path, remote_path, overwrite=True)
 
     def get(self, key, size=None, offset=0):
         remote_path = self._convert_key_to_remote_path(key)
         end = offset + size if size else None
-        blob = self._filesystem.cat_file(remote_path, start=offset, end=end)
+        blob = self.filesystem.cat_file(remote_path, start=offset, end=end)
         return blob
 
     def put(self, key, data, append=False):
@@ -107,12 +102,12 @@ class AzureBlobStore(DataStore):
             mode = "w"
         else:
             raise TypeError("Data type unknown.  Unable to put in Azure!")
-        with self._filesystem.open(remote_path, mode) as f:
+        with self.filesystem.open(remote_path, mode) as f:
             f.write(data)
 
     def stat(self, key):
         remote_path = self._convert_key_to_remote_path(key)
-        files = self._filesystem.ls(remote_path, detail=True)
+        files = self.filesystem.ls(remote_path, detail=True)
         if len(files) == 1 and files[0]["type"] == "file":
             size = files[0]["size"]
             modified = files[0]["last_modified"]
@@ -124,10 +119,10 @@ class AzureBlobStore(DataStore):
 
     def listdir(self, key):
         remote_path = self._convert_key_to_remote_path(key)
-        if self._filesystem.isfile(remote_path):
+        if self.filesystem.isfile(remote_path):
             return key
         remote_path = f"{remote_path}/**"
-        files = self._filesystem.glob(remote_path)
+        files = self.filesystem.glob(remote_path)
         key_length = len(key)
         files = [
             f.split("/", 1)[1][key_length:] for f in files if len(f.split("/")) > 1

mlrun/datastore/base.py

@@ -25,6 +25,7 @@ import pyarrow
 import pytz
 import requests
 import urllib3
+from deprecated import deprecated
 
 import mlrun.errors
 from mlrun.errors import err_to_str
@@ -98,7 +99,18 @@ class DataStore:
     def uri_to_ipython(endpoint, subpath):
         return ""
 
-    def get_filesystem(self, silent=True) -> Optional[fsspec.AbstractFileSystem]:
+    # TODO: remove in 1.8.0
+    @deprecated(
+        version="1.8.0",
+        reason="'get_filesystem()' will be removed in 1.8.0, use "
+        "'filesystem' property instead",
+        category=FutureWarning,
+    )
+    def get_filesystem(self):
+        return self.filesystem
+
+    @property
+    def filesystem(self) -> Optional[fsspec.AbstractFileSystem]:
         """return fsspec file system object, if supported"""
         return None
 
@@ -117,7 +129,7 @@ class DataStore:
         return self._sanitize_storage_options(None)
 
     def open(self, filepath, mode):
-        file_system = self.get_filesystem(False)
+        file_system = self.filesystem
         return file_system.open(filepath, mode)
 
     def _join(self, key):
@@ -238,7 +250,7 @@ class DataStore:
         df_module = df_module or pd
         file_url = self._sanitize_url(url)
         is_csv, is_json, drop_time_column = False, False, False
-        file_system = self.get_filesystem()
+        file_system = self.filesystem
         if file_url.endswith(".csv") or format == "csv":
             is_csv = True
             drop_time_column = False
@@ -363,7 +375,7 @@ class DataStore:
         }
 
     def rm(self, path, recursive=False, maxdepth=None):
-        self.get_filesystem().rm(path=path, recursive=recursive, maxdepth=maxdepth)
+        self.filesystem.rm(path=path, recursive=recursive, maxdepth=maxdepth)
 
     @staticmethod
     def _is_dd(df_module):
@@ -653,9 +665,10 @@ def http_head(url, headers=None, auth=None):
     return response.headers
 
 
-def http_put(url, data, headers=None, auth=None):
+def http_put(url, data, headers=None, auth=None, session=None):
     try:
-        response = requests.put(
+        put_api = session.put if session else requests.put
+        response = put_api(
             url, data=data, headers=headers, auth=auth, verify=verify_ssl
         )
     except OSError as exc:
@@ -679,7 +692,8 @@ class HttpStore(DataStore):
         self._enrich_https_token()
         self._validate_https_token()
 
-    def get_filesystem(self, silent=True):
+    @property
+    def filesystem(self):
         """return fsspec file system object, if supported"""
         if not self._filesystem:
             self._filesystem = fsspec.filesystem("http")

mlrun/datastore/dbfs_store.py

@@ -83,9 +83,9 @@ class DatabricksFileSystemDisableCache(DatabricksFileSystem):
 class DBFSStore(DataStore):
     def __init__(self, parent, schema, name, endpoint="", secrets: dict = None):
         super().__init__(parent, name, schema, endpoint, secrets=secrets)
-        self.get_filesystem(silent=False)
 
-    def get_filesystem(self, silent=True):
+    @property
+    def filesystem(self):
         """return fsspec file system object, if supported"""
         filesystem_class = get_filesystem_class(protocol=self.kind)
         if not self._filesystem:
@@ -104,7 +104,7 @@ class DBFSStore(DataStore):
         return self._sanitize_storage_options(res)
 
     def _verify_filesystem_and_key(self, key: str):
-        if not self._filesystem:
+        if not self.filesystem:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Performing actions on data-item without a valid filesystem"
             )
@@ -121,7 +121,7 @@ class DBFSStore(DataStore):
             raise mlrun.errors.MLRunInvalidArgumentError("offset cannot be None")
         start = offset or None
         end = offset + size if size else None
-        return self._filesystem.cat_file(key, start=start, end=end)
+        return self.filesystem.cat_file(key, start=start, end=end)
 
     def put(self, key, data, append=False):
         self._verify_filesystem_and_key(key)
@@ -135,16 +135,16 @@ class DBFSStore(DataStore):
             mode += "b"
         elif not isinstance(data, str):
             raise TypeError(f"Unknown data type {type(data)}")
-        with self._filesystem.open(key, mode) as f:
+        with self.filesystem.open(key, mode) as f:
             f.write(data)
 
     def upload(self, key: str, src_path: str):
         self._verify_filesystem_and_key(key)
-        self._filesystem.put_file(src_path, key, overwrite=True)
+        self.filesystem.put_file(src_path, key, overwrite=True)
 
     def stat(self, key: str):
         self._verify_filesystem_and_key(key)
-        file = self._filesystem.stat(key)
+        file = self.filesystem.stat(key)
         if file["type"] == "file":
             size = file["size"]
         elif file["type"] == "directory":
@@ -156,10 +156,10 @@ class DBFSStore(DataStore):
         Basic ls of file/dir - without recursion.
         """
         self._verify_filesystem_and_key(key)
-        if self._filesystem.isfile(key):
+        if self.filesystem.isfile(key):
             return key
         remote_path = f"{key}/*"
-        files = self._filesystem.glob(remote_path)
+        files = self.filesystem.glob(remote_path)
         # Get only the files and directories under key path, without the key path itself.
         # for example in a filesystem that has this path: /test_mlrun_dbfs_objects/test.txt
         # listdir with the input /test_mlrun_dbfs_objects as a key will return ['test.txt'].
@@ -171,4 +171,4 @@ class DBFSStore(DataStore):
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "dbfs file system does not support maxdepth option in rm function"
             )
-        self.get_filesystem().rm(path=path, recursive=recursive)
+        self.filesystem.rm(path=path, recursive=recursive)

mlrun/datastore/filestore.py

@@ -47,7 +47,8 @@ class FileStore(DataStore):
                 key = path.join(self._real_path, suffix)
         return path.join(self.subpath, key)
 
-    def get_filesystem(self, silent=True):
+    @property
+    def filesystem(self):
         """return fsspec file system object, if supported"""
         if not self._filesystem:
             self._filesystem = fsspec.filesystem("file")

mlrun/datastore/google_cloud_storage.py

@@ -30,7 +30,8 @@ class GoogleCloudStorageStore(DataStore):
     def __init__(self, parent, schema, name, endpoint="", secrets: dict = None):
         super().__init__(parent, name, schema, endpoint, secrets=secrets)
 
-    def get_filesystem(self):
+    @property
+    def filesystem(self):
         """return fsspec file system object, if supported"""
         if self._filesystem:
             return self._filesystem
@@ -75,7 +76,7 @@ class GoogleCloudStorageStore(DataStore):
         path = self._make_path(key)
 
         end = offset + size if size else None
-        blob = self.get_filesystem().cat_file(path, start=offset, end=end)
+        blob = self.filesystem.cat_file(path, start=offset, end=end)
         return blob
 
     def put(self, key, data, append=False):
@@ -94,17 +95,17 @@ class GoogleCloudStorageStore(DataStore):
             raise TypeError(
                 "Data type unknown.  Unable to put in Google cloud storage!"
             )
-        with self.get_filesystem().open(path, mode) as f:
+        with self.filesystem.open(path, mode) as f:
             f.write(data)
 
     def upload(self, key, src_path):
         path = self._make_path(key)
-        self.get_filesystem().put_file(src_path, path, overwrite=True)
+        self.filesystem.put_file(src_path, path, overwrite=True)
 
     def stat(self, key):
         path = self._make_path(key)
 
-        files = self.get_filesystem().ls(path, detail=True)
+        files = self.filesystem.ls(path, detail=True)
         if len(files) == 1 and files[0]["type"] == "file":
             size = files[0]["size"]
             modified = files[0]["updated"]
@@ -116,10 +117,10 @@ class GoogleCloudStorageStore(DataStore):
 
     def listdir(self, key):
         path = self._make_path(key)
-        if self.get_filesystem().isfile(path):
+        if self.filesystem.isfile(path):
             return key
         remote_path = f"{path}/**"
-        files = self.get_filesystem().glob(remote_path)
+        files = self.filesystem.glob(remote_path)
         key_length = len(key)
         files = [
             f.split("/", 1)[1][key_length:] for f in files if len(f.split("/")) > 1
@@ -128,7 +129,7 @@ class GoogleCloudStorageStore(DataStore):
 
     def rm(self, path, recursive=False, maxdepth=None):
         path = self._make_path(path)
-        self.get_filesystem().rm(path=path, recursive=recursive, maxdepth=maxdepth)
+        self.filesystem.rm(path=path, recursive=recursive, maxdepth=maxdepth)
 
     def get_spark_options(self):
         res = None

mlrun/datastore/redis.py

@@ -73,7 +73,8 @@ class RedisStore(DataStore):
 
         return self._redis
 
-    def get_filesystem(self, silent):
+    @property
+    def filesystem(self):
         return None  # no support for fsspec
 
     def supports_isdir(self):

mlrun/datastore/s3.py

@@ -109,18 +109,15 @@ class S3Store(DataStore):
             res["spark.hadoop.fs.s3a.aws.profile"] = st.get("profile")
         return res
 
-    def get_filesystem(self, silent=False):
+    @property
+    def filesystem(self):
         """return fsspec file system object, if supported"""
         if self._filesystem:
             return self._filesystem
         try:
             import s3fs  # noqa
         except ImportError as exc:
-            if not silent:
-                raise ImportError(
-                    "AWS s3fs not installed, run pip install s3fs"
-                ) from exc
-            return None
+            raise ImportError("AWS s3fs not installed") from exc
         filesystem_class = get_filesystem_class(protocol=self.kind)
         self._filesystem = makeDatastoreSchemaSanitizer(
             filesystem_class,

mlrun/datastore/sources.py

@@ -1027,7 +1027,7 @@ class KafkaSource(OnlineSource):
         ) < semver.VersionInfo.parse(bug_fix_version):
             warnings.warn(
                 f"Detected nuclio version {config.nuclio_version}, which is older "
-                f"than {bug_fix_version}. Forcing number of replicas of 1 in function '{function.metdata.name}'. "
+                f"than {bug_fix_version}. Forcing number of replicas of 1 in function '{function.metadata.name}'. "
                 f"To resolve this, please upgrade Nuclio."
             )
             function.spec.min_replicas = 1
@@ -1053,7 +1053,6 @@ class SQLSource(BaseSourceDriver):
         db_url: str = None,
         table_name: str = None,
         spark_options: dict = None,
-        time_fields: List[str] = None,
         parse_dates: List[str] = None,
         **kwargs,
     ):
@@ -1078,17 +1077,8 @@ class SQLSource(BaseSourceDriver):
         :param table_name:      the name of the collection to access,
                                 from the current database
         :param spark_options:   additional spark read options
-        :param time_fields :    all the field to be parsed as timestamp.
         :param parse_dates :    all the field to be parsed as timestamp.
         """
-        if time_fields:
-            warnings.warn(
-                "'time_fields' is deprecated, use 'parse_dates' instead. "
-                "This will be removed in 1.6.0",
-                # TODO: Remove this in 1.6.0
-                FutureWarning,
-            )
-            parse_dates = time_fields
         db_url = db_url or mlrun.mlconf.sql.url
         if db_url is None:
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -1096,7 +1086,7 @@ class SQLSource(BaseSourceDriver):
             )
         if time_field:
             if parse_dates:
-                time_fields.append(time_field)
+                parse_dates.append(time_field)
             else:
                 parse_dates = [time_field]
         attrs = {

mlrun/datastore/targets.py

@@ -17,7 +17,6 @@ import os
 import random
 import sys
 import time
-import warnings
 from collections import Counter
 from copy import copy
 from typing import Any, Dict, List, Optional, Union
@@ -553,7 +552,7 @@ class BaseStoreTarget(DataTargetBase):
         else:
             store, target_path = self._get_store_and_path()
             target_path = generate_path_with_chunk(self, chunk_id, target_path)
-            file_system = store.get_filesystem(False)
+            file_system = store.filesystem
             if file_system.protocol == "file":
                 dir = os.path.dirname(target_path)
                 if dir:
@@ -1407,7 +1406,7 @@ class StreamTarget(BaseStoreTarget):
         from storey import V3ioDriver
 
         key_columns = list(key_columns.keys())
-        endpoint, uri = parse_path(self.get_target_path())
+        endpoint, uri = parse_path(self.path)
         column_list = self._get_column_list(
             features=features, timestamp_key=timestamp_key, key_columns=key_columns
         )
@@ -1680,7 +1679,6 @@ class SQLTarget(BaseStoreTarget):
         if_exists: str = "append",
         create_table: bool = False,
         # create_according_to_data: bool = False,
-        time_fields: List[str] = None,
         varchar_len: int = 50,
         parse_dates: List[str] = None,
     ):
@@ -1718,20 +1716,11 @@ class SQLTarget(BaseStoreTarget):
         :param create_table:                pass True if you want to create new table named by
                                             table_name with schema on current database.
         :param create_according_to_data:    (not valid)
-        :param time_fields :    all the field to be parsed as timestamp.
         :param varchar_len :    the defalut len of the all the varchar column (using if needed to create the table).
         :param parse_dates :    all the field to be parsed as timestamp.
         """
 
         create_according_to_data = False  # TODO: open for user
-        if time_fields:
-            warnings.warn(
-                "'time_fields' is deprecated, use 'parse_dates' instead. "
-                "This will be removed in 1.6.0",
-                # TODO: Remove this in 1.6.0
-                FutureWarning,
-            )
-            parse_dates = time_fields
         db_url = db_url or mlrun.mlconf.sql.url
         if db_url is None or table_name is None:
             attr = {}

mlrun/datastore/v3io.py

@@ -19,6 +19,7 @@ from copy import deepcopy
 from datetime import datetime
 
 import fsspec
+import requests
 import v3io.dataplane
 
 import mlrun
@@ -73,18 +74,11 @@ class V3ioStore(DataStore):
         schema = "https" if self.secure else "http"
         return f"{schema}://{self.endpoint}"
 
-    def get_filesystem(self, silent=True):
+    @property
+    def filesystem(self):
         """return fsspec file system object, if supported"""
         if self._filesystem:
             return self._filesystem
-        try:
-            import v3iofs  # noqa
-        except ImportError as exc:
-            if not silent:
-                raise ImportError(
-                    "v3iofs or storey not installed, run pip install storey"
-                ) from exc
-            return None
         self._filesystem = fsspec.filesystem("v3io", **self.get_storage_options())
         return self._filesystem
 
@@ -151,15 +145,18 @@ class V3ioStore(DataStore):
             data = memoryview(data)
         except TypeError:
             pass
-        while buffer_offset < buffer_size:
-            chunk_size = min(buffer_size - buffer_offset, max_chunk_size)
-            http_put(
-                self.url + self._join(key),
-                data[buffer_offset : buffer_offset + chunk_size],
-                append_header if buffer_offset else self.headers,
-                None,
-            )
-            buffer_offset += chunk_size
+
+        with requests.Session() as requests_session:
+            while buffer_offset < buffer_size:
+                chunk_size = min(buffer_size - buffer_offset, max_chunk_size)
+                http_put(
+                    self.url + self._join(key),
+                    data[buffer_offset : buffer_offset + chunk_size],
+                    append_header if buffer_offset else self.headers,
+                    None,
+                    requests_session,
+                )
+                buffer_offset += chunk_size
 
     def put(self, key, data, append=False):
         return self._put(key, data)
@@ -207,7 +204,7 @@ class V3ioStore(DataStore):
         """Recursive rm file/folder
         Workaround for v3io-fs not supporting recursive directory removal"""
 
-        file_system = self.get_filesystem()
+        file_system = self.filesystem
         if isinstance(path, str):
             path = [path]
         maxdepth = maxdepth if not maxdepth else maxdepth - 1