fsspec 2024.5.0__py3-none-any.whl → 2024.6.1__py3-none-any.whl

fsspec/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2024.5.0'
-__version_tuple__ = version_tuple = (2024, 5, 0)
+__version__ = version = '2024.6.1'
+__version_tuple__ = version_tuple = (2024, 6, 1)

fsspec/caching.py

@@ -15,6 +15,7 @@ from typing import (
     ClassVar,
     Generic,
     NamedTuple,
+    Optional,
     OrderedDict,
     TypeVar,
 )
@@ -574,7 +575,7 @@ class KnownPartsOfAFile(BaseCache):
         blocksize: int,
         fetcher: Fetcher,
         size: int,
-        data: dict[tuple[int, int], bytes] = {},
+        data: Optional[dict[tuple[int, int], bytes]] = None,
         strict: bool = True,
         **_: Any,
     ):
@@ -597,7 +598,7 @@ class KnownPartsOfAFile(BaseCache):
 
             self.data = dict(zip(offsets, blocks))
         else:
-            self.data = data
+            self.data = {}
 
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
         if start is None:

fsspec/compression.py

@@ -139,7 +139,7 @@ class SnappyFile(AbstractBufferedFile):
 try:
     import snappy
 
-    snappy.compress
+    snappy.compress(b"")
     # Snappy may use the .sz file extension, but this is not part of the
     # standard implementation.
     register_compression("snappy", SnappyFile, [])

fsspec/generic.py

@@ -197,6 +197,7 @@ class GenericFileSystem(AsyncFileSystem):
             )
         result = {}
         for k, v in out.items():
+            v = v.copy()  # don't corrupt target FS dircache
             name = fs.unstrip_protocol(k)
             v["name"] = name
             result[name] = v
@@ -210,6 +211,7 @@ class GenericFileSystem(AsyncFileSystem):
             out = await fs._info(url, **kwargs)
         else:
             out = fs.info(url, **kwargs)
+        out = out.copy()  # don't edit originals
         out["name"] = fs.unstrip_protocol(out["name"])
         return out
 
@@ -224,6 +226,7 @@ class GenericFileSystem(AsyncFileSystem):
             out = await fs._ls(url, detail=True, **kwargs)
         else:
             out = fs.ls(url, detail=True, **kwargs)
+        out = [o.copy() for o in out]  # don't edit originals
         for o in out:
             o["name"] = fs.unstrip_protocol(o["name"])
         if detail:

fsspec/implementations/cached.py

@@ -425,7 +425,6 @@ class CachingFileSystem(AbstractFileSystem):
             "clear_cache",
             "clear_expired_cache",
             "pop_from_cache",
-            "_mkcache",
             "local_file",
             "_paths_from_path",
             "get_mapper",
@@ -435,12 +434,10 @@ class CachingFileSystem(AbstractFileSystem):
             "__hash__",
             "__eq__",
             "to_json",
+            "to_dict",
             "cache_size",
             "pipe_file",
             "pipe",
-            "isdir",
-            "isfile",
-            "exists",
             "start_transaction",
             "end_transaction",
         }:
@@ -510,15 +507,6 @@ class CachingFileSystem(AbstractFileSystem):
             ^ hash(self.target_protocol)
         )
 
-    def to_json(self):
-        """Calculate JSON representation.
-
-        Not implemented yet for CachingFileSystem.
-        """
-        raise NotImplementedError(
-            "CachingFileSystem JSON representation not implemented"
-        )
-
 
 class WholeFileCacheFileSystem(CachingFileSystem):
     """Caches whole remote files on first access
@@ -663,7 +651,8 @@ class WholeFileCacheFileSystem(CachingFileSystem):
     def _open(self, path, mode="rb", **kwargs):
         path = self._strip_protocol(path)
         if "r" not in mode:
-            fn = self._make_local_details(path)
+            hash = self._mapper(path)
+            fn = os.path.join(self.storage[-1], hash)
             user_specified_kwargs = {
                 k: v
                 for k, v in kwargs.items()
@@ -806,7 +795,8 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         if self._intrans:
             f = [_ for _ in self.transaction.files if _.path == path]
             if f:
-                return {"name": path, "size": f[0].size or f[0].tell(), "type": "file"}
+                size = os.path.getsize(f[0].fn) if f[0].closed else f[0].tell()
+                return {"name": path, "size": size, "type": "file"}
             f = any(_.path.startswith(path + "/") for _ in self.transaction.files)
             if f:
                 return {"name": path, "size": 0, "type": "directory"}
@@ -912,7 +902,7 @@ class LocalTempFile:
         self.close()
 
     def close(self):
-        self.size = self.fh.tell()
+        # self.size = self.fh.tell()
         if self.closed:
             return
         self.fh.close()

fsspec/implementations/dirfs.py

@@ -56,6 +56,8 @@ class DirFileSystem(AsyncFileSystem):
             if not path:
                 return self.path
             return self.fs.sep.join((self.path, self._strip_protocol(path)))
+        if isinstance(path, dict):
+            return {self._join(_path): value for _path, value in path.items()}
         return [self._join(_path) for _path in path]
 
     def _relpath(self, path):

fsspec/implementations/github.py

@@ -1,5 +1,7 @@
 import requests
 
+import fsspec
+
 from ..spec import AbstractFileSystem
 from ..utils import infer_storage_options
 from .memory import MemoryFile
@@ -225,3 +227,13 @@ class GithubFileSystem(AbstractFileSystem):
             raise FileNotFoundError(path)
         r.raise_for_status()
         return MemoryFile(None, None, r.content)
+
+    def cat(self, path, recursive=False, on_error="raise", **kwargs):
+        paths = self.expand_path(path, recursive=recursive)
+        urls = [
+            self.rurl.format(org=self.org, repo=self.repo, path=u, sha=self.root)
+            for u, sh in paths
+        ]
+        fs = fsspec.filesystem("http")
+        data = fs.cat(urls, on_error="return")
+        return {u: v for ((k, v), u) in zip(data.items(), urls)}

fsspec/implementations/http.py

@@ -560,6 +560,7 @@ class HTTPFile(AbstractBufferedFile):
         if mode != "rb":
             raise NotImplementedError("File mode not supported")
         self.asynchronous = asynchronous
+        self.loop = loop
         self.url = url
         self.session = session
         self.details = {"name": url, "size": size, "type": "file"}
@@ -572,7 +573,6 @@ class HTTPFile(AbstractBufferedFile):
             cache_options=cache_options,
             **kwargs,
         )
-        self.loop = loop
 
     def read(self, length=-1):
         """Read bytes from file
@@ -736,6 +736,7 @@ class HTTPStreamFile(AbstractBufferedFile):
             return r
 
         self.r = sync(self.loop, cor)
+        self.loop = fs.loop
 
     def seek(self, loc, whence=0):
         if loc == 0 and whence == 1:

fsspec/implementations/reference.py

@@ -935,6 +935,13 @@ class ReferenceFileSystem(AsyncFileSystem):
 
     def _process_references0(self, references):
         """Make reference dict for Spec Version 0"""
+        if isinstance(references, dict):
+            # do not do this for lazy/parquet backend, which will not make dicts,
+            # but must remain writable in the original object
+            references = {
+                key: json.dumps(val) if isinstance(val, dict) else val
+                for key, val in references.items()
+            }
         self.references = references
 
     def _process_references1(self, references, template_overrides=None):
@@ -952,6 +959,8 @@ class ReferenceFileSystem(AsyncFileSystem):
                 if v.startswith("base64:"):
                     self.references[k] = base64.b64decode(v[7:])
                 self.references[k] = v
+            elif isinstance(v, dict):
+                self.references[k] = json.dumps(v)
             elif self.templates:
                 u = v[0]
                 if "{{" in u:

fsspec/implementations/smb.py

@@ -68,6 +68,7 @@ class SMBFileSystem(AbstractFileSystem):
         encrypt=None,
         share_access=None,
         register_session_retries=5,
+        auto_mkdir=False,
         **kwargs,
     ):
         """
@@ -102,6 +103,10 @@ class SMBFileSystem(AbstractFileSystem):
             - 'r': Allow other handles to be opened with read access.
             - 'w': Allow other handles to be opened with write access.
             - 'd': Allow other handles to be opened with delete access.
+        auto_mkdir: bool
+            Whether, when opening a file, the directory containing it should
+            be created (if it doesn't already exist). This is assumed by pyarrow
+            and zarr-python code.
         """
         super().__init__(**kwargs)
         self.host = host
@@ -113,6 +118,7 @@ class SMBFileSystem(AbstractFileSystem):
         self.temppath = kwargs.pop("temppath", "")
         self.share_access = share_access
         self.register_session_retries = register_session_retries
+        self.auto_mkdir = auto_mkdir
         self._connect()
 
     @property
@@ -224,6 +230,8 @@ class SMBFileSystem(AbstractFileSystem):
         By specifying 'share_access' in 'kwargs' it is possible to override the
         default shared access setting applied in the constructor of this object.
         """
+        if self.auto_mkdir and "w" in mode:
+            self.makedirs(self._parent(path), exist_ok=True)
         bls = block_size if block_size is not None and block_size >= 0 else -1
         wpath = _as_unc_path(self.host, path)
         share_access = kwargs.pop("share_access", self.share_access)
@@ -245,6 +253,8 @@ class SMBFileSystem(AbstractFileSystem):
         """Copy within two locations in the same filesystem"""
         wpath1 = _as_unc_path(self.host, path1)
         wpath2 = _as_unc_path(self.host, path2)
+        if self.auto_mkdir:
+            self.makedirs(self._parent(path2), exist_ok=True)
         smbclient.copyfile(wpath1, wpath2, port=self._port, **kwargs)
 
     def _rm(self, path):

fsspec/json.py

@@ -0,0 +1,121 @@
+import json
+from contextlib import suppress
+from pathlib import PurePath
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+)
+
+from .registry import _import_class, get_filesystem_class
+from .spec import AbstractFileSystem
+
+
+class FilesystemJSONEncoder(json.JSONEncoder):
+    include_password: ClassVar[bool] = True
+
+    def default(self, o: Any) -> Any:
+        if isinstance(o, AbstractFileSystem):
+            return o.to_dict(include_password=self.include_password)
+        if isinstance(o, PurePath):
+            cls = type(o)
+            return {"cls": f"{cls.__module__}.{cls.__name__}", "str": str(o)}
+
+        return super().default(o)
+
+    def make_serializable(self, obj: Any) -> Any:
+        """
+        Recursively converts an object so that it can be JSON serialized via
+        :func:`json.dumps` and :func:`json.dump`, without actually calling
+        said functions.
+        """
+        if isinstance(obj, (str, int, float, bool)):
+            return obj
+        if isinstance(obj, Mapping):
+            return {k: self.make_serializable(v) for k, v in obj.items()}
+        if isinstance(obj, Sequence):
+            return [self.make_serializable(v) for v in obj]
+
+        return self.default(obj)
+
+
+class FilesystemJSONDecoder(json.JSONDecoder):
+    def __init__(
+        self,
+        *,
+        object_hook: Optional[Callable[[Dict[str, Any]], Any]] = None,
+        parse_float: Optional[Callable[[str], Any]] = None,
+        parse_int: Optional[Callable[[str], Any]] = None,
+        parse_constant: Optional[Callable[[str], Any]] = None,
+        strict: bool = True,
+        object_pairs_hook: Optional[Callable[[List[Tuple[str, Any]]], Any]] = None,
+    ) -> None:
+        self.original_object_hook = object_hook
+
+        super().__init__(
+            object_hook=self.custom_object_hook,
+            parse_float=parse_float,
+            parse_int=parse_int,
+            parse_constant=parse_constant,
+            strict=strict,
+            object_pairs_hook=object_pairs_hook,
+        )
+
+    @classmethod
+    def try_resolve_path_cls(cls, dct: Dict[str, Any]):
+        with suppress(Exception):
+            fqp = dct["cls"]
+
+            path_cls = _import_class(fqp)
+
+            if issubclass(path_cls, PurePath):
+                return path_cls
+
+        return None
+
+    @classmethod
+    def try_resolve_fs_cls(cls, dct: Dict[str, Any]):
+        with suppress(Exception):
+            if "cls" in dct:
+                try:
+                    fs_cls = _import_class(dct["cls"])
+                    if issubclass(fs_cls, AbstractFileSystem):
+                        return fs_cls
+                except Exception:
+                    if "protocol" in dct:  # Fallback if cls cannot be imported
+                        return get_filesystem_class(dct["protocol"])
+
+                    raise
+
+        return None
+
+    def custom_object_hook(self, dct: Dict[str, Any]):
+        if "cls" in dct:
+            if (obj_cls := self.try_resolve_fs_cls(dct)) is not None:
+                return AbstractFileSystem.from_dict(dct)
+            if (obj_cls := self.try_resolve_path_cls(dct)) is not None:
+                return obj_cls(dct["str"])
+
+        if self.original_object_hook is not None:
+            return self.original_object_hook(dct)
+
+        return dct
+
+    def unmake_serializable(self, obj: Any) -> Any:
+        """
+        Inverse function of :meth:`FilesystemJSONEncoder.make_serializable`.
+        """
+        if isinstance(obj, dict):
+            obj = self.custom_object_hook(obj)
+        if isinstance(obj, dict):
+            return {k: self.unmake_serializable(v) for k, v in obj.items()}
+        if isinstance(obj, (list, tuple)):
+            return [self.unmake_serializable(v) for v in obj]
+
+        return obj

fsspec/registry.py

@@ -257,27 +257,33 @@ update the current installation.
 """
 
 
-def _import_class(cls, minv=None):
-    """Take a string FQP and return the imported class or identifier
+def _import_class(fqp: str):
+    """Take a fully-qualified path and return the imported class or identifier.
 
-    cls is of the form "package.module.klass" or "package.module:subobject.klass"
+    ``fqp`` is of the form "package.module.klass" or
+    "package.module:subobject.klass".
+
+    Warnings
+    --------
+    This can import arbitrary modules. Make sure you haven't installed any modules
+    that may execute malicious code at import time.
     """
-    if ":" in cls:
-        mod, name = cls.rsplit(":", 1)
-        s3 = mod == "s3fs"
-        mod = importlib.import_module(mod)
-        if s3 and mod.__version__.split(".") < ["0", "5"]:
-            warnings.warn(s3_msg)
-        for part in name.split("."):
-            mod = getattr(mod, part)
-        return mod
+    if ":" in fqp:
+        mod, name = fqp.rsplit(":", 1)
     else:
-        mod, name = cls.rsplit(".", 1)
-        s3 = mod == "s3fs"
-        mod = importlib.import_module(mod)
-        if s3 and mod.__version__.split(".") < ["0", "5"]:
-            warnings.warn(s3_msg)
-        return getattr(mod, name)
+        mod, name = fqp.rsplit(".", 1)
+
+    is_s3 = mod == "s3fs"
+    mod = importlib.import_module(mod)
+    if is_s3 and mod.__version__.split(".") < ["0", "5"]:
+        warnings.warn(s3_msg)
+    for part in name.split("."):
+        mod = getattr(mod, part)
+
+    if not isinstance(mod, type):
+        raise TypeError(f"{fqp} is not a class")
+
+    return mod
 
 
 def filesystem(protocol, **storage_options):

fsspec/spec.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import io
+import json
 import logging
 import os
 import threading
@@ -9,7 +10,7 @@ import weakref
 from errno import ESPIPE
 from glob import has_magic
 from hashlib import sha256
-from typing import ClassVar
+from typing import Any, ClassVar, Dict, Tuple
 
 from .callbacks import DEFAULT_CALLBACK
 from .config import apply_config, conf
@@ -115,6 +116,10 @@ class AbstractFileSystem(metaclass=_Cached):
     #: Extra *class attributes* that should be considered when hashing.
     _extra_tokenize_attributes = ()
 
+    # Set by _Cached metaclass
+    storage_args: Tuple[Any, ...]
+    storage_options: Dict[str, Any]
+
     def __init__(self, *args, **storage_options):
         """Create and configure file-system instance
 
@@ -1381,41 +1386,45 @@ class AbstractFileSystem(metaclass=_Cached):
                 length = size - offset
             return read_block(f, offset, length, delimiter)
 
-    def to_json(self):
+    def to_json(self, *, include_password: bool = True) -> str:
         """
-        JSON representation of this filesystem instance
+        JSON representation of this filesystem instance.
+
+        Parameters
+        ----------
+        include_password: bool, default True
+            Whether to include the password (if any) in the output.
 
         Returns
         -------
-        str: JSON structure with keys cls (the python location of this class),
-            protocol (text name of this class's protocol, first one in case of
-            multiple), args (positional args, usually empty), and all other
-            kwargs as their own keys.
+        JSON string with keys ``cls`` (the python location of this class),
+        protocol (text name of this class's protocol, first one in case of
+        multiple), ``args`` (positional args, usually empty), and all other
+        keyword arguments as their own keys.
+
+        Warnings
+        --------
+        Serialized filesystems may contain sensitive information which have been
+        passed to the constructor, such as passwords and tokens. Make sure you
+        store and send them in a secure environment!
         """
-        import json
+        from .json import FilesystemJSONEncoder
 
-        cls = type(self)
-        cls = ".".join((cls.__module__, cls.__name__))
-        proto = (
-            self.protocol[0]
-            if isinstance(self.protocol, (tuple, list))
-            else self.protocol
-        )
         return json.dumps(
-            dict(
-                cls=cls,
-                protocol=proto,
-                args=self.storage_args,
-                **self.storage_options,
-            )
+            self,
+            cls=type(
+                "_FilesystemJSONEncoder",
+                (FilesystemJSONEncoder,),
+                {"include_password": include_password},
+            ),
         )
 
     @staticmethod
-    def from_json(blob):
+    def from_json(blob: str) -> AbstractFileSystem:
         """
-        Recreate a filesystem instance from JSON representation
+        Recreate a filesystem instance from JSON representation.
 
-        See ``.to_json()`` for the expected structure of the input
+        See ``.to_json()`` for the expected structure of the input.
 
         Parameters
         ----------
@@ -1424,18 +1433,95 @@ class AbstractFileSystem(metaclass=_Cached):
         Returns
         -------
         file system instance, not necessarily of this particular class.
+
+        Warnings
+        --------
+        This can import arbitrary modules (as determined by the ``cls`` key).
+        Make sure you haven't installed any modules that may execute malicious code
+        at import time.
         """
-        import json
+        from .json import FilesystemJSONDecoder
 
-        from .registry import _import_class, get_filesystem_class
+        return json.loads(blob, cls=FilesystemJSONDecoder)
 
-        dic = json.loads(blob)
-        protocol = dic.pop("protocol")
-        try:
-            cls = _import_class(dic.pop("cls"))
-        except (ImportError, ValueError, RuntimeError, KeyError):
-            cls = get_filesystem_class(protocol)
-        return cls(*dic.pop("args", ()), **dic)
+    def to_dict(self, *, include_password: bool = True) -> Dict[str, Any]:
+        """
+        JSON-serializable dictionary representation of this filesystem instance.
+
+        Parameters
+        ----------
+        include_password: bool, default True
+            Whether to include the password (if any) in the output.
+
+        Returns
+        -------
+        Dictionary with keys ``cls`` (the python location of this class),
+        protocol (text name of this class's protocol, first one in case of
+        multiple), ``args`` (positional args, usually empty), and all other
+        keyword arguments as their own keys.
+
+        Warnings
+        --------
+        Serialized filesystems may contain sensitive information which have been
+        passed to the constructor, such as passwords and tokens. Make sure you
+        store and send them in a secure environment!
+        """
+        from .json import FilesystemJSONEncoder
+
+        json_encoder = FilesystemJSONEncoder()
+
+        cls = type(self)
+        proto = self.protocol
+
+        storage_options = dict(self.storage_options)
+        if not include_password:
+            storage_options.pop("password", None)
+
+        return dict(
+            cls=f"{cls.__module__}:{cls.__name__}",
+            protocol=proto[0] if isinstance(proto, (tuple, list)) else proto,
+            args=json_encoder.make_serializable(self.storage_args),
+            **json_encoder.make_serializable(storage_options),
+        )
+
+    @staticmethod
+    def from_dict(dct: Dict[str, Any]) -> AbstractFileSystem:
+        """
+        Recreate a filesystem instance from dictionary representation.
+
+        See ``.to_dict()`` for the expected structure of the input.
+
+        Parameters
+        ----------
+        dct: Dict[str, Any]
+
+        Returns
+        -------
+        file system instance, not necessarily of this particular class.
+
+        Warnings
+        --------
+        This can import arbitrary modules (as determined by the ``cls`` key).
+        Make sure you haven't installed any modules that may execute malicious code
+        at import time.
+        """
+        from .json import FilesystemJSONDecoder
+
+        json_decoder = FilesystemJSONDecoder()
+
+        dct = dict(dct)  # Defensive copy
+
+        cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct)
+        if cls is None:
+            raise ValueError("Not a serialized AbstractFileSystem")
+
+        dct.pop("cls", None)
+        dct.pop("protocol", None)
+
+        return cls(
+            *json_decoder.unmake_serializable(dct.pop("args", ())),
+            **json_decoder.unmake_serializable(dct),
+        )
 
     def _get_pyarrow_filesystem(self):
         """

fsspec/utils.py

@@ -435,7 +435,7 @@ def isfilelike(f: Any) -> TypeGuard[IO[bytes]]:
 
 def get_protocol(url: str) -> str:
     url = stringify_path(url)
-    parts = re.split(r"(\:\:|\://)", url, 1)
+    parts = re.split(r"(\:\:|\://)", url, maxsplit=1)
     if len(parts) > 1:
         return parts[0]
     return "file"