fsspec 2024.6.0__tar.gz → 2024.6.1__tar.gz

{fsspec-2024.6.0 → fsspec-2024.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fsspec
-Version: 2024.6.0
+Version: 2024.6.1
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
@@ -184,7 +184,6 @@ Description-Content-Type: text/markdown
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
-[![fsspec Downloads Last Month](https://assets.piptrends.com/get-last-month-downloads-badge/fsspec.svg 'fsspec Downloads Last Month by pip Trends')](https://piptrends.com/package/fsspec)
 
 A specification for pythonic filesystems.
 

{fsspec-2024.6.0 → fsspec-2024.6.1}/README.md

@@ -4,7 +4,6 @@
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
-[![fsspec Downloads Last Month](https://assets.piptrends.com/get-last-month-downloads-badge/fsspec.svg 'fsspec Downloads Last Month by pip Trends')](https://piptrends.com/package/fsspec)
 
 A specification for pythonic filesystems.
 

{fsspec-2024.6.0 → fsspec-2024.6.1}/docs/source/changelog.rst

@@ -1,6 +1,20 @@
 Changelog
 =========
 
+2024.6.1
+--------
+
+Fixes
+
+- fix appending to non-dict reference sets (#1634)
+- don't let generic edit info dicts (#1633)
+- set https's loop before calling super (#1633)
+- cached write file doesn't need to update it's size on close (#1633)
+- fix JSON serialize for FSs with interior FSs (#1628, 1627)
+- option to strip "password" when pickling (#1625)
+- fix filecache write (#1622)
+
+
 2024.6.0
 --------
 

{fsspec-2024.6.0 → fsspec-2024.6.1}/fsspec/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2024.6.0'
-__version_tuple__ = version_tuple = (2024, 6, 0)
+__version__ = version = '2024.6.1'
+__version_tuple__ = version_tuple = (2024, 6, 1)

{fsspec-2024.6.0 → fsspec-2024.6.1}/fsspec/generic.py

@@ -197,6 +197,7 @@ class GenericFileSystem(AsyncFileSystem):
             )
         result = {}
         for k, v in out.items():
+            v = v.copy()  # don't corrupt target FS dircache
             name = fs.unstrip_protocol(k)
             v["name"] = name
             result[name] = v
@@ -210,6 +211,7 @@ class GenericFileSystem(AsyncFileSystem):
             out = await fs._info(url, **kwargs)
         else:
             out = fs.info(url, **kwargs)
+        out = out.copy()  # don't edit originals
         out["name"] = fs.unstrip_protocol(out["name"])
         return out
 
@@ -224,6 +226,7 @@ class GenericFileSystem(AsyncFileSystem):
             out = await fs._ls(url, detail=True, **kwargs)
         else:
             out = fs.ls(url, detail=True, **kwargs)
+        out = [o.copy() for o in out]  # don't edit originals
         for o in out:
             o["name"] = fs.unstrip_protocol(o["name"])
         if detail:

{fsspec-2024.6.0 → fsspec-2024.6.1}/fsspec/implementations/cached.py

@@ -651,7 +651,8 @@ class WholeFileCacheFileSystem(CachingFileSystem):
     def _open(self, path, mode="rb", **kwargs):
         path = self._strip_protocol(path)
         if "r" not in mode:
-            fn = self._make_local_details(path)
+            hash = self._mapper(path)
+            fn = os.path.join(self.storage[-1], hash)
             user_specified_kwargs = {
                 k: v
                 for k, v in kwargs.items()
@@ -794,7 +795,8 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         if self._intrans:
             f = [_ for _ in self.transaction.files if _.path == path]
             if f:
-                return {"name": path, "size": f[0].size or f[0].tell(), "type": "file"}
+                size = os.path.getsize(f[0].fn) if f[0].closed else f[0].tell()
+                return {"name": path, "size": size, "type": "file"}
             f = any(_.path.startswith(path + "/") for _ in self.transaction.files)
             if f:
                 return {"name": path, "size": 0, "type": "directory"}
@@ -900,7 +902,7 @@ class LocalTempFile:
         self.close()
 
     def close(self):
-        self.size = self.fh.tell()
+        # self.size = self.fh.tell()
         if self.closed:
             return
         self.fh.close()

{fsspec-2024.6.0 → fsspec-2024.6.1}/fsspec/implementations/dirfs.py

@@ -56,6 +56,8 @@ class DirFileSystem(AsyncFileSystem):
             if not path:
                 return self.path
             return self.fs.sep.join((self.path, self._strip_protocol(path)))
+        if isinstance(path, dict):
+            return {self._join(_path): value for _path, value in path.items()}
         return [self._join(_path) for _path in path]
 
     def _relpath(self, path):

{fsspec-2024.6.0 → fsspec-2024.6.1}/fsspec/implementations/http.py

@@ -560,6 +560,7 @@ class HTTPFile(AbstractBufferedFile):
         if mode != "rb":
             raise NotImplementedError("File mode not supported")
         self.asynchronous = asynchronous
+        self.loop = loop
         self.url = url
         self.session = session
         self.details = {"name": url, "size": size, "type": "file"}
@@ -572,7 +573,6 @@ class HTTPFile(AbstractBufferedFile):
             cache_options=cache_options,
             **kwargs,
         )
-        self.loop = loop
 
     def read(self, length=-1):
         """Read bytes from file
@@ -736,6 +736,7 @@ class HTTPStreamFile(AbstractBufferedFile):
             return r
 
         self.r = sync(self.loop, cor)
+        self.loop = fs.loop
 
     def seek(self, loc, whence=0):
         if loc == 0 and whence == 1:

{fsspec-2024.6.0 → fsspec-2024.6.1}/fsspec/implementations/reference.py

@@ -935,10 +935,13 @@ class ReferenceFileSystem(AsyncFileSystem):
 
     def _process_references0(self, references):
         """Make reference dict for Spec Version 0"""
-        references = {
-            key: json.dumps(val) if isinstance(val, dict) else val
-            for key, val in references.items()
-        }
+        if isinstance(references, dict):
+            # do not do this for lazy/parquet backend, which will not make dicts,
+            # but must remain writable in the original object
+            references = {
+                key: json.dumps(val) if isinstance(val, dict) else val
+                for key, val in references.items()
+            }
         self.references = references
 
     def _process_references1(self, references, template_overrides=None):

{fsspec-2024.6.0 → fsspec-2024.6.1}/fsspec/json.py

@@ -1,22 +1,49 @@
 import json
 from contextlib import suppress
 from pathlib import PurePath
-from typing import Any, Callable, Dict, List, Optional, Tuple
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+)
 
 from .registry import _import_class, get_filesystem_class
 from .spec import AbstractFileSystem
 
 
 class FilesystemJSONEncoder(json.JSONEncoder):
+    include_password: ClassVar[bool] = True
+
     def default(self, o: Any) -> Any:
         if isinstance(o, AbstractFileSystem):
-            return o.to_dict()
+            return o.to_dict(include_password=self.include_password)
         if isinstance(o, PurePath):
             cls = type(o)
             return {"cls": f"{cls.__module__}.{cls.__name__}", "str": str(o)}
 
         return super().default(o)
 
+    def make_serializable(self, obj: Any) -> Any:
+        """
+        Recursively converts an object so that it can be JSON serialized via
+        :func:`json.dumps` and :func:`json.dump`, without actually calling
+        said functions.
+        """
+        if isinstance(obj, (str, int, float, bool)):
+            return obj
+        if isinstance(obj, Mapping):
+            return {k: self.make_serializable(v) for k, v in obj.items()}
+        if isinstance(obj, Sequence):
+            return [self.make_serializable(v) for v in obj]
+
+        return self.default(obj)
+
 
 class FilesystemJSONDecoder(json.JSONDecoder):
     def __init__(
@@ -79,3 +106,16 @@ class FilesystemJSONDecoder(json.JSONDecoder):
             return self.original_object_hook(dct)
 
         return dct
+
+    def unmake_serializable(self, obj: Any) -> Any:
+        """
+        Inverse function of :meth:`FilesystemJSONEncoder.make_serializable`.
+        """
+        if isinstance(obj, dict):
+            obj = self.custom_object_hook(obj)
+        if isinstance(obj, dict):
+            return {k: self.unmake_serializable(v) for k, v in obj.items()}
+        if isinstance(obj, (list, tuple)):
+            return [self.unmake_serializable(v) for v in obj]
+
+        return obj

{fsspec-2024.6.0 → fsspec-2024.6.1}/fsspec/spec.py

@@ -1386,20 +1386,38 @@ class AbstractFileSystem(metaclass=_Cached):
                 length = size - offset
             return read_block(f, offset, length, delimiter)
 
-    def to_json(self) -> str:
+    def to_json(self, *, include_password: bool = True) -> str:
         """
         JSON representation of this filesystem instance.
 
+        Parameters
+        ----------
+        include_password: bool, default True
+            Whether to include the password (if any) in the output.
+
         Returns
         -------
         JSON string with keys ``cls`` (the python location of this class),
         protocol (text name of this class's protocol, first one in case of
         multiple), ``args`` (positional args, usually empty), and all other
         keyword arguments as their own keys.
+
+        Warnings
+        --------
+        Serialized filesystems may contain sensitive information which have been
+        passed to the constructor, such as passwords and tokens. Make sure you
+        store and send them in a secure environment!
         """
         from .json import FilesystemJSONEncoder
 
-        return json.dumps(self, cls=FilesystemJSONEncoder)
+        return json.dumps(
+            self,
+            cls=type(
+                "_FilesystemJSONEncoder",
+                (FilesystemJSONEncoder,),
+                {"include_password": include_password},
+            ),
+        )
 
     @staticmethod
     def from_json(blob: str) -> AbstractFileSystem:
@@ -1426,25 +1444,44 @@ class AbstractFileSystem(metaclass=_Cached):
 
         return json.loads(blob, cls=FilesystemJSONDecoder)
 
-    def to_dict(self) -> Dict[str, Any]:
+    def to_dict(self, *, include_password: bool = True) -> Dict[str, Any]:
         """
         JSON-serializable dictionary representation of this filesystem instance.
 
+        Parameters
+        ----------
+        include_password: bool, default True
+            Whether to include the password (if any) in the output.
+
         Returns
         -------
         Dictionary with keys ``cls`` (the python location of this class),
         protocol (text name of this class's protocol, first one in case of
         multiple), ``args`` (positional args, usually empty), and all other
         keyword arguments as their own keys.
+
+        Warnings
+        --------
+        Serialized filesystems may contain sensitive information which have been
+        passed to the constructor, such as passwords and tokens. Make sure you
+        store and send them in a secure environment!
         """
+        from .json import FilesystemJSONEncoder
+
+        json_encoder = FilesystemJSONEncoder()
+
         cls = type(self)
         proto = self.protocol
 
+        storage_options = dict(self.storage_options)
+        if not include_password:
+            storage_options.pop("password", None)
+
         return dict(
             cls=f"{cls.__module__}:{cls.__name__}",
             protocol=proto[0] if isinstance(proto, (tuple, list)) else proto,
-            args=self.storage_args,
-            **self.storage_options,
+            args=json_encoder.make_serializable(self.storage_args),
+            **json_encoder.make_serializable(storage_options),
         )
 
     @staticmethod
@@ -1470,6 +1507,10 @@ class AbstractFileSystem(metaclass=_Cached):
         """
         from .json import FilesystemJSONDecoder
 
+        json_decoder = FilesystemJSONDecoder()
+
+        dct = dict(dct)  # Defensive copy
+
         cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct)
         if cls is None:
             raise ValueError("Not a serialized AbstractFileSystem")
@@ -1477,7 +1518,10 @@ class AbstractFileSystem(metaclass=_Cached):
         dct.pop("cls", None)
         dct.pop("protocol", None)
 
-        return cls(*dct.pop("args", ()), **dct)
+        return cls(
+            *json_decoder.unmake_serializable(dct.pop("args", ())),
+            **json_decoder.unmake_serializable(dct),
+        )
 
     def _get_pyarrow_filesystem(self):
         """