fsspec 2023.12.2__py3-none-any.whl → 2024.2.0__py3-none-any.whl

fsspec/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-12-11T16:18:48-0500",
+ "date": "2024-02-04T20:21:42-0500",
  "dirty": false,
  "error": null,
- "full-revisionid": "dd8cb9bf620be4d9153e854dd1431c23a2be6db0",
- "version": "2023.12.2"
+ "full-revisionid": "5dc364e13b63609717d77b7361e80cfa64e3b8fd",
+ "version": "2024.2.0"
 }
 '''  # END VERSION_JSON
 

fsspec/asyn.py

@@ -11,7 +11,7 @@ from contextlib import contextmanager
 from glob import has_magic
 from typing import TYPE_CHECKING, Iterable
 
-from .callbacks import _DEFAULT_CALLBACK
+from .callbacks import DEFAULT_CALLBACK
 from .exceptions import FSTimeoutError
 from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep
 from .spec import AbstractBufferedFile, AbstractFileSystem
@@ -205,7 +205,7 @@ def running_async() -> bool:
 async def _run_coros_in_chunks(
     coros,
     batch_size=None,
-    callback=_DEFAULT_CALLBACK,
+    callback=DEFAULT_CALLBACK,
     timeout=None,
     return_exceptions=False,
     nofiles=False,
@@ -245,7 +245,7 @@ async def _run_coros_in_chunks(
             asyncio.Task(asyncio.wait_for(c, timeout=timeout))
             for c in coros[start : start + batch_size]
         ]
-        if callback is not _DEFAULT_CALLBACK:
+        if callback is not DEFAULT_CALLBACK:
             [
                 t.add_done_callback(lambda *_, **__: callback.relative_update(1))
                 for t in chunk
@@ -506,7 +506,7 @@ class AsyncFileSystem(AbstractFileSystem):
         lpath,
         rpath,
         recursive=False,
-        callback=_DEFAULT_CALLBACK,
+        callback=DEFAULT_CALLBACK,
         batch_size=None,
         maxdepth=None,
         **kwargs,
@@ -568,8 +568,8 @@ class AsyncFileSystem(AbstractFileSystem):
         coros = []
         callback.set_size(len(file_pairs))
         for lfile, rfile in file_pairs:
-            callback.branch(lfile, rfile, kwargs)
-            coros.append(self._put_file(lfile, rfile, **kwargs))
+            put_file = callback.branch_coro(self._put_file)
+            coros.append(put_file(lfile, rfile, **kwargs))
 
         return await _run_coros_in_chunks(
             coros, batch_size=batch_size, callback=callback
@@ -583,7 +583,7 @@ class AsyncFileSystem(AbstractFileSystem):
         rpath,
         lpath,
         recursive=False,
-        callback=_DEFAULT_CALLBACK,
+        callback=DEFAULT_CALLBACK,
         maxdepth=None,
         **kwargs,
     ):
@@ -645,8 +645,8 @@ class AsyncFileSystem(AbstractFileSystem):
         coros = []
         callback.set_size(len(lpaths))
         for lpath, rpath in zip(lpaths, rpaths):
-            callback.branch(rpath, lpath, kwargs)
-            coros.append(self._get_file(rpath, lpath, **kwargs))
+            get_file = callback.branch_coro(self._get_file)
+            coros.append(get_file(rpath, lpath, **kwargs))
         return await _run_coros_in_chunks(
             coros, batch_size=batch_size, callback=callback
         )

fsspec/callbacks.py

@@ -1,3 +1,6 @@
+from functools import wraps
+
+
 class Callback:
     """
     Base class and interface for callback mechanism
@@ -25,6 +28,60 @@ class Callback:
         self.hooks = hooks or {}
         self.kw = kwargs
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_args):
+        self.close()
+
+    def close(self):
+        """Close callback."""
+
+    def branched(self, path_1, path_2, **kwargs):
+        """
+        Return callback for child transfers
+
+        If this callback is operating at a higher level, e.g., put, which may
+        trigger transfers that can also be monitored. The function returns a callback
+        that has to be passed to the child method, e.g., put_file,
+        as `callback=` argument.
+
+        The implementation uses `callback.branch` for compatibility.
+        When implementing callbacks, it is recommended to override this function instead
+        of `branch` and avoid calling `super().branched(...)`.
+
+        Prefer using this function over `branch`.
+
+        Parameters
+        ----------
+        path_1: str
+            Child's source path
+        path_2: str
+            Child's destination path
+        **kwargs:
+            Arbitrary keyword arguments
+
+        Returns
+        -------
+        callback: Callback
+            A callback instance to be passed to the child method
+        """
+        self.branch(path_1, path_2, kwargs)
+        # mutate kwargs so that we can force the caller to pass "callback=" explicitly
+        return kwargs.pop("callback", DEFAULT_CALLBACK)
+
+    def branch_coro(self, fn):
+        """
+        Wraps a coroutine, and pass a new child callback to it.
+        """
+
+        @wraps(fn)
+        async def func(path1, path2: str, **kwargs):
+            with self.branched(path1, path2, **kwargs) as child:
+                return await fn(path1, path2, callback=child, **kwargs)
+
+        return func
+
     def set_size(self, size):
         """
         Set the internal maximum size attribute
@@ -140,10 +197,10 @@ class Callback:
 
         For the special value of ``None``, return the global instance of
         ``NoOpCallback``. This is an alternative to including
-        ``callback=_DEFAULT_CALLBACK`` directly in a method signature.
+        ``callback=DEFAULT_CALLBACK`` directly in a method signature.
         """
         if maybe_callback is None:
-            return _DEFAULT_CALLBACK
+            return DEFAULT_CALLBACK
         return maybe_callback
 
 
@@ -186,7 +243,9 @@ class TqdmCallback(Callback):
     tqdm_kwargs : dict, (optional)
         Any argument accepted by the tqdm constructor.
         See the `tqdm doc <https://tqdm.github.io/docs/tqdm/#__init__>`_.
-        Will be forwarded to tqdm.
+        Will be forwarded to `tqdm_cls`.
+    tqdm_cls: (optional)
+        subclass of `tqdm.tqdm`. If not passed, it will default to `tqdm.tqdm`.
 
     Examples
     --------
@@ -209,30 +268,57 @@ class TqdmCallback(Callback):
             recursive=True,
             callback=TqdmCallback(tqdm_kwargs={"desc": "Your tqdm description"}),
         )
+
+    You can also customize the progress bar by passing a subclass of `tqdm`.
+
+    .. code-block:: python
+
+        class TqdmFormat(tqdm):
+            '''Provides a `total_time` format parameter'''
+            @property
+            def format_dict(self):
+                d = super().format_dict
+                total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1)
+                d.update(total_time=self.format_interval(total_time) + " in total")
+                return d
+
+    >>> with TqdmCallback(
+            tqdm_kwargs={
+                "desc": "desc",
+                "bar_format": "{total_time}: {percentage:.0f}%|{bar}{r_bar}",
+            },
+            tqdm_cls=TqdmFormat,
+        ) as callback:
+            fs.upload(".", path2distant_data, recursive=True, callback=callback)
     """
 
     def __init__(self, tqdm_kwargs=None, *args, **kwargs):
         try:
-            import tqdm
+            from tqdm import tqdm
 
-            self._tqdm = tqdm
         except ImportError as exce:
             raise ImportError(
                 "Using TqdmCallback requires tqdm to be installed"
             ) from exce
 
+        self._tqdm_cls = kwargs.pop("tqdm_cls", tqdm)
         self._tqdm_kwargs = tqdm_kwargs or {}
+        self.tqdm = None
         super().__init__(*args, **kwargs)
 
-    def set_size(self, size):
-        self.tqdm = self._tqdm.tqdm(total=size, **self._tqdm_kwargs)
+    def call(self, *args, **kwargs):
+        if self.tqdm is None:
+            self.tqdm = self._tqdm_cls(total=self.size, **self._tqdm_kwargs)
+        self.tqdm.total = self.size
+        self.tqdm.update(self.value - self.tqdm.n)
 
-    def relative_update(self, inc=1):
-        self.tqdm.update(inc)
+    def close(self):
+        if self.tqdm is not None:
+            self.tqdm.close()
+            self.tqdm = None
 
     def __del__(self):
-        self.tqdm.close()
-        self.tqdm = None
+        return self.close()
 
 
-_DEFAULT_CALLBACK = NoOpCallback()
+DEFAULT_CALLBACK = _DEFAULT_CALLBACK = NoOpCallback()

fsspec/compression.py

@@ -90,15 +90,15 @@ except ImportError:
 try:
     from lzma import LZMAFile
 
-    register_compression("lzma", LZMAFile, "xz")
-    register_compression("xz", LZMAFile, "xz", force=True)
+    register_compression("lzma", LZMAFile, "lzma")
+    register_compression("xz", LZMAFile, "xz")
 except ImportError:
     pass
 
 try:
     import lzmaffi
 
-    register_compression("lzma", lzmaffi.LZMAFile, "xz", force=True)
+    register_compression("lzma", lzmaffi.LZMAFile, "lzma", force=True)
     register_compression("xz", lzmaffi.LZMAFile, "xz", force=True)
 except ImportError:
     pass

fsspec/exceptions.py

@@ -10,12 +10,8 @@ class BlocksizeMismatchError(ValueError):
     written with
     """
 
-    ...
-
 
 class FSTimeoutError(asyncio.TimeoutError):
     """
     Raised when a fsspec function timed out occurs
     """
-
-    ...

fsspec/generic.py

@@ -8,7 +8,7 @@ import uuid
 from typing import Optional
 
 from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper
-from .callbacks import _DEFAULT_CALLBACK
+from .callbacks import DEFAULT_CALLBACK
 from .core import filesystem, get_filesystem_class, split_protocol, url_to_fs
 
 _generic_fs = {}
@@ -279,7 +279,7 @@ class GenericFileSystem(AsyncFileSystem):
         url,
         url2,
         blocksize=2**20,
-        callback=_DEFAULT_CALLBACK,
+        callback=DEFAULT_CALLBACK,
         **kwargs,
     ):
         fs = _resolve_fs(url, self.method)

fsspec/gui.py

@@ -153,8 +153,9 @@ class SigSlot:
                         break
                 except Exception as e:
                     logger.exception(
-                        "Exception (%s) while executing callback for signal: %s"
-                        "" % (e, sig)
+                        "Exception (%s) while executing callback for signal: %s",
+                        e,
+                        sig,
                     )
 
     def show(self, threads=False):

fsspec/implementations/arrow.py

@@ -5,6 +5,7 @@ import secrets
 import shutil
 from contextlib import suppress
 from functools import cached_property, wraps
+from urllib.parse import parse_qs
 
 from fsspec.spec import AbstractFileSystem
 from fsspec.utils import (
@@ -255,6 +256,7 @@ class HadoopFileSystem(ArrowFSWrapper):
         port=0,
         user=None,
         kerb_ticket=None,
+        replication=3,
         extra_conf=None,
         **kwargs,
     ):
@@ -270,6 +272,8 @@ class HadoopFileSystem(ArrowFSWrapper):
             If given, connect as this username
         kerb_ticket: str or None
             If given, use this ticket for authentication
+        replication: int
+            set replication factor of file for write operations. default value is 3.
         extra_conf: None or dict
             Passed on to HadoopFileSystem
         """
@@ -280,6 +284,7 @@ class HadoopFileSystem(ArrowFSWrapper):
             port=port,
             user=user,
             kerb_ticket=kerb_ticket,
+            replication=replication,
             extra_conf=extra_conf,
         )
         super().__init__(fs=fs, **kwargs)
@@ -294,4 +299,8 @@ class HadoopFileSystem(ArrowFSWrapper):
             out["user"] = ops["username"]
         if ops.get("port", None):
             out["port"] = ops["port"]
+        if ops.get("url_query", None):
+            queries = parse_qs(ops["url_query"])
+            if queries.get("replication", None):
+                out["replication"] = int(queries["replication"][0])
         return out

fsspec/implementations/cache_mapper.py

@@ -2,13 +2,9 @@ from __future__ import annotations
 
 import abc
 import hashlib
-from typing import TYPE_CHECKING
 
 from fsspec.implementations.local import make_path_posix
 
-if TYPE_CHECKING:
-    from typing import Any
-
 
 class AbstractCacheMapper(abc.ABC):
     """Abstract super-class for mappers from remote URLs to local cached
@@ -19,7 +15,7 @@ class AbstractCacheMapper(abc.ABC):
     def __call__(self, path: str) -> str:
         ...
 
-    def __eq__(self, other: Any) -> bool:
+    def __eq__(self, other: object) -> bool:
         # Identity only depends on class. When derived classes have attributes
         # they will need to be included.
         return isinstance(other, type(self))
@@ -56,7 +52,7 @@ class BasenameCacheMapper(AbstractCacheMapper):
         else:
             return prefix  # No separator found, simple filename
 
-    def __eq__(self, other: Any) -> bool:
+    def __eq__(self, other: object) -> bool:
         return super().__eq__(other) and self.directory_levels == other.directory_levels
 
     def __hash__(self) -> int:

fsspec/implementations/cached.py

@@ -10,7 +10,7 @@ from shutil import rmtree
 from typing import TYPE_CHECKING, Any, Callable, ClassVar
 
 from fsspec import AbstractFileSystem, filesystem
-from fsspec.callbacks import _DEFAULT_CALLBACK
+from fsspec.callbacks import DEFAULT_CALLBACK
 from fsspec.compression import compr
 from fsspec.core import BaseCache, MMapCache
 from fsspec.exceptions import BlocksizeMismatchError
@@ -524,7 +524,7 @@ class WholeFileCacheFileSystem(CachingFileSystem):
     protocol = "filecache"
     local_file = True
 
-    def open_many(self, open_files):
+    def open_many(self, open_files, **kwargs):
         paths = [of.path for of in open_files]
         if "r" in open_files.mode:
             self._mkcache()
@@ -535,6 +535,7 @@ class WholeFileCacheFileSystem(CachingFileSystem):
                     path,
                     mode=open_files.mode,
                     fn=os.path.join(self.storage[-1], self._mapper(path)),
+                    **kwargs,
                 )
                 for path in paths
             ]
@@ -606,7 +607,7 @@ class WholeFileCacheFileSystem(CachingFileSystem):
         path,
         recursive=False,
         on_error="raise",
-        callback=_DEFAULT_CALLBACK,
+        callback=DEFAULT_CALLBACK,
         **kwargs,
     ):
         paths = self.expand_path(
@@ -650,7 +651,13 @@ class WholeFileCacheFileSystem(CachingFileSystem):
         path = self._strip_protocol(path)
         if "r" not in mode:
             fn = self._make_local_details(path)
-            return LocalTempFile(self, path, mode=mode, fn=fn)
+            user_specified_kwargs = {
+                k: v
+                for k, v in kwargs.items()
+                # those kwargs were added by open(), we don't want them
+                if k not in ["autocommit", "block_size", "cache_options"]
+            }
+            return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs)
         detail = self._check_file(path)
         if detail:
             detail, fn = detail
@@ -775,8 +782,18 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
 
         if "r" not in mode:
             fn = os.path.join(self.storage[-1], sha)
+            user_specified_kwargs = {
+                k: v
+                for k, v in kwargs.items()
+                if k not in ["autocommit", "block_size", "cache_options"]
+            }  # those were added by open()
             return LocalTempFile(
-                self, path, mode=mode, autocommit=not self._intrans, fn=fn
+                self,
+                path,
+                mode=mode,
+                autocommit=not self._intrans,
+                fn=fn,
+                **user_specified_kwargs,
             )
         fn = self._check_file(path)
         if fn:
@@ -812,7 +829,7 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
 class LocalTempFile:
     """A temporary local file, which will be uploaded on commit"""
 
-    def __init__(self, fs, path, fn, mode="wb", autocommit=True, seek=0):
+    def __init__(self, fs, path, fn, mode="wb", autocommit=True, seek=0, **kwargs):
         self.fn = fn
         self.fh = open(fn, mode)
         self.mode = mode
@@ -822,6 +839,7 @@ class LocalTempFile:
         self.fs = fs
         self.closed = False
         self.autocommit = autocommit
+        self.kwargs = kwargs
 
     def __reduce__(self):
         # always open in r+b to allow continuing writing at a location
@@ -849,7 +867,7 @@ class LocalTempFile:
         os.remove(self.fn)
 
     def commit(self):
-        self.fs.put(self.fn, self.path)
+        self.fs.put(self.fn, self.path, **self.kwargs)
         try:
             os.remove(self.fn)
         except (PermissionError, FileNotFoundError):

fsspec/implementations/dbfs.py

@@ -2,6 +2,8 @@ import base64
 import urllib
 
 import requests
+import requests.exceptions
+from requests.adapters import HTTPAdapter, Retry
 
 from fsspec import AbstractFileSystem
 from fsspec.spec import AbstractBufferedFile
@@ -42,13 +44,19 @@ class DatabricksFileSystem(AbstractFileSystem):
         """
         self.instance = instance
         self.token = token
-
         self.session = requests.Session()
+        self.retries = Retry(
+            total=10,
+            backoff_factor=0.05,
+            status_forcelist=[408, 429, 500, 502, 503, 504],
+        )
+
+        self.session.mount("https://", HTTPAdapter(max_retries=self.retries))
         self.session.headers.update({"Authorization": f"Bearer {self.token}"})
 
         super().__init__(**kwargs)
 
-    def ls(self, path, detail=True):
+    def ls(self, path, detail=True, **kwargs):
         """
         List the contents of the given path.
 
@@ -137,7 +145,7 @@ class DatabricksFileSystem(AbstractFileSystem):
 
         self.mkdirs(path, **kwargs)
 
-    def rm(self, path, recursive=False):
+    def rm(self, path, recursive=False, **kwargs):
         """
         Remove the file or folder at the given absolute path.
 
@@ -166,7 +174,9 @@ class DatabricksFileSystem(AbstractFileSystem):
             raise e
         self.invalidate_cache(self._parent(path))
 
-    def mv(self, source_path, destination_path, recursive=False, maxdepth=None):
+    def mv(
+        self, source_path, destination_path, recursive=False, maxdepth=None, **kwargs
+    ):
         """
         Move a source to a destination path.
 

fsspec/implementations/dirfs.py

@@ -124,6 +124,12 @@ class DirFileSystem(AsyncFileSystem):
     def pipe(self, path, *args, **kwargs):
         return self.fs.pipe(self._join(path), *args, **kwargs)
 
+    async def _pipe_file(self, path, *args, **kwargs):
+        return await self.fs._pipe_file(self._join(path), *args, **kwargs)
+
+    def pipe_file(self, path, *args, **kwargs):
+        return self.fs.pipe_file(self._join(path), *args, **kwargs)
+
     async def _cat_file(self, path, *args, **kwargs):
         return await self.fs._cat_file(self._join(path), *args, **kwargs)
 

fsspec/implementations/ftp.py

@@ -171,12 +171,15 @@ class FTPFileSystem(AbstractFileSystem):
         def cb(x):
             out.append(x)
 
-        self.ftp.retrbinary(
-            f"RETR {path}",
-            blocksize=self.blocksize,
-            rest=start,
-            callback=cb,
-        )
+        try:
+            self.ftp.retrbinary(
+                f"RETR {path}",
+                blocksize=self.blocksize,
+                rest=start,
+                callback=cb,
+            )
+        except (Error, error_perm) as orig_exc:
+            raise FileNotFoundError(path) from orig_exc
         return b"".join(out)
 
     def _open(
@@ -361,15 +364,17 @@ def _mlsd2(ftp, path="."):
     minfo = []
     ftp.dir(path, lines.append)
     for line in lines:
-        line = line.split()
+        split_line = line.split()
+        if len(split_line) < 9:
+            continue
         this = (
-            line[-1],
+            split_line[-1],
             {
-                "modify": " ".join(line[5:8]),
-                "unix.owner": line[2],
-                "unix.group": line[3],
-                "unix.mode": line[0],
-                "size": line[4],
+                "modify": " ".join(split_line[5:8]),
+                "unix.owner": split_line[2],
+                "unix.group": split_line[3],
+                "unix.mode": split_line[0],
+                "size": split_line[4],
             },
         )
         if "d" == this[1]["unix.mode"][0]:

fsspec/implementations/github.py

@@ -36,8 +36,11 @@ class GithubFileSystem(AbstractFileSystem):
     url = "https://api.github.com/repos/{org}/{repo}/git/trees/{sha}"
     rurl = "https://raw.githubusercontent.com/{org}/{repo}/{sha}/{path}"
     protocol = "github"
+    timeout = (60, 60)  # connect, read timeouts
 
-    def __init__(self, org, repo, sha=None, username=None, token=None, **kwargs):
+    def __init__(
+        self, org, repo, sha=None, username=None, token=None, timeout=None, **kwargs
+    ):
         super().__init__(**kwargs)
         self.org = org
         self.repo = repo
@@ -45,10 +48,14 @@ class GithubFileSystem(AbstractFileSystem):
             raise ValueError("Auth required both username and token")
         self.username = username
         self.token = token
+        if timeout is not None:
+            self.timeout = timeout
         if sha is None:
             # look up default branch (not necessarily "master")
             u = "https://api.github.com/repos/{org}/{repo}"
-            r = requests.get(u.format(org=org, repo=repo), **self.kw)
+            r = requests.get(
+                u.format(org=org, repo=repo), timeout=self.timeout, **self.kw
+            )
             r.raise_for_status()
             sha = r.json()["default_branch"]
 
@@ -79,7 +86,8 @@ class GithubFileSystem(AbstractFileSystem):
         List of string
         """
         r = requests.get(
-            f"https://api.github.com/{['users', 'orgs'][is_org]}/{org_or_user}/repos"
+            f"https://api.github.com/{['users', 'orgs'][is_org]}/{org_or_user}/repos",
+            timeout=cls.timeout,
         )
         r.raise_for_status()
         return [repo["name"] for repo in r.json()]
@@ -89,6 +97,7 @@ class GithubFileSystem(AbstractFileSystem):
         """Names of tags in the repo"""
         r = requests.get(
             f"https://api.github.com/repos/{self.org}/{self.repo}/tags",
+            timeout=self.timeout,
             **self.kw,
         )
         r.raise_for_status()
@@ -99,6 +108,7 @@ class GithubFileSystem(AbstractFileSystem):
         """Names of branches in the repo"""
         r = requests.get(
             f"https://api.github.com/repos/{self.org}/{self.repo}/branches",
+            timeout=self.timeout,
             **self.kw,
         )
         r.raise_for_status()
@@ -147,7 +157,9 @@ class GithubFileSystem(AbstractFileSystem):
                 _sha = out["sha"]
         if path not in self.dircache or sha not in [self.root, None]:
             r = requests.get(
-                self.url.format(org=self.org, repo=self.repo, sha=_sha), **self.kw
+                self.url.format(org=self.org, repo=self.repo, sha=_sha),
+                timeout=self.timeout,
+                **self.kw,
             )
             if r.status_code == 404:
                 raise FileNotFoundError(path)
@@ -208,7 +220,7 @@ class GithubFileSystem(AbstractFileSystem):
         url = self.rurl.format(
             org=self.org, repo=self.repo, path=path, sha=sha or self.root
         )
-        r = requests.get(url, **self.kw)
+        r = requests.get(url, timeout=self.timeout, **self.kw)
         if r.status_code == 404:
             raise FileNotFoundError(path)
         r.raise_for_status()